Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 0b6e961

Browse files
Top level error handling (#56)
* add a goroutine stack trace when exiting in debug mode * added error logging to the reconfigure daemon * added a couple of additional debug log entries to significant events * improved the top-level error handling so any point of failure ends the application
1 parent 1c4462a commit 0b6e961

File tree

4 files changed

+47
-24
lines changed

4 files changed

+47
-24
lines changed

‎main.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,10 @@ package main
22

33
import (
44
"context"
5+
"log"
56
"os"
67
"os/signal"
8+
"runtime"
79
"time"
810

911
_ "github.com/joho/godotenv/autoload"
@@ -114,6 +116,19 @@ this repository has new commits, Pico will automatically reconfigure.`,
114116
},
115117
}
116118

119+
if os.Getenv("DEBUG") != "" {
120+
go func() {
121+
sigs := make(chan os.Signal, 1)
122+
signal.Notify(sigs, os.Interrupt)
123+
buf := make([]byte, 1<<20)
124+
for {
125+
<-sigs
126+
stacklen := runtime.Stack(buf, true)
127+
log.Printf("\nPrinting goroutine stack trace because `DEBUG` was set.\n%s\n", buf[:stacklen])
128+
}
129+
}()
130+
}
131+
117132
err := app.Run(os.Args)
118133
if err != nil {
119134
zap.L().Fatal("exit", zap.Error(err))

‎reconfigurer/git.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,9 @@ func (p *GitProvider) reconfigure(w watcher.Watcher) (err error) {
8888
state.Env["HOSTNAME"] = p.hostname
8989
}
9090

91+
zap.L().Debug("setting state for watcher",
92+
zap.Any("new_state", state))
93+
9194
return w.SetState(state)
9295
}
9396

‎service/service.go‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@ import (
1010
"github.com/eapache/go-resiliency/retrier"
1111
"github.com/pkg/errors"
1212
"go.uber.org/zap"
13-
"golang.org/x/sync/errgroup"
1413
"gopkg.in/src-d/go-git.v4/plumbing/transport"
1514
"gopkg.in/src-d/go-git.v4/plumbing/transport/http"
1615
"gopkg.in/src-d/go-git.v4/plumbing/transport/ssh"
@@ -110,39 +109,43 @@ func Initialise(c Config) (app *App, err error) {
110109

111110
// Start launches the app and blocks until fatal error
112111
func (app *App) Start(ctx context.Context) error {
113-
g, ctx := errgroup.WithContext(ctx)
114-
115-
zap.L().Debug("starting service daemon")
116-
117-
// TODO: Replace this errgroup with a more resilient solution.
118-
// Not all of these tasks fail in the same way. Some don't fail at all.
119-
// This needs to be rewritten to be more considerate of different failure
120-
// states and potentially retry in some circumstances. Pico should be the
121-
// kind of service that barely goes down, only when absolutely necessary.
112+
errs := make(chan error)
122113

123114
ce := executor.NewCommandExecutor(app.secrets, app.config.PassEnvironment, app.config.VaultConfig, "GLOBAL_")
124-
g.Go(func()error {
115+
gofunc() {
125116
ce.Subscribe(app.bus)
126-
return nil
127-
})
117+
}()
128118

129-
// TODO: gw can fail when setting up the gitwatch instance, it should retry.
130119
gw := app.watcher.(*watcher.GitWatcher)
131-
g.Go(gw.Start)
120+
go func() {
121+
errs <- errors.Wrap(gw.Start(), "git watcher terminated fatally")
122+
}()
132123

133-
// TODO: reconfigurer can also fail when setting up gitwatch.
134-
g.Go(func() error {
135-
return app.reconfigurer.Configure(app.watcher)
136-
})
124+
go func() {
125+
errs <- errors.Wrap(app.reconfigurer.Configure(app.watcher), "git watcher terminated fatally")
126+
}()
137127

138128
if s, ok := app.secrets.(*vault.VaultSecrets); ok {
139-
g.Go(func() error {
140-
return retrier.New(retrier.ConstantBackoff(3, 100*time.Millisecond), nil).
141-
RunCtx(ctx, s.Renew)
142-
})
129+
go func() {
130+
errs <- errors.Wrap(retrier.New(retrier.ConstantBackoff(3, 100*time.Millisecond), nil).RunCtx(ctx, s.Renew), "git watcher terminated fatally")
131+
}()
143132
}
144133

145-
return g.Wait()
134+
handle := func() error {
135+
select {
136+
case err := <-errs:
137+
return err
138+
case <-ctx.Done():
139+
return context.Canceled
140+
}
141+
}
142+
143+
zap.L().Debug("starting service main loop")
144+
for {
145+
if err := handle(); err != nil {
146+
return err
147+
}
148+
}
146149
}
147150

148151
func getAuthMethod(c Config, secretConfig map[string]string) (transport.AuthMethod, error) {

‎watcher/git.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@ func NewGitWatcher(
6262

6363
// Start runs the watcher loop and blocks until a fatal error occurs
6464
func (w *GitWatcher) Start() error {
65+
zap.L().Debug("git watcher initialising, waiting for first state to be set")
66+
6567
// wait for the first config event to set the initial state
6668
<-w.initialise
6769

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /