Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit d32f0a9

Browse files
committed
feat: mask sensitive data in the DLE logs output
1 parent d410d09 commit d32f0a9

File tree

6 files changed

+91
-43
lines changed

6 files changed

+91
-43
lines changed

‎engine/cmd/database-lab/main.go‎

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ func main() {
5656
log.Fatal(errors.WithMessage(err, "failed to parse config"))
5757
}
5858

59+
logFilter := log.GetFilter()
60+
logFilter.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
61+
5962
config.ApplyGlobals(cfg)
6063

6164
docker, err := client.NewClientWithOpts(client.FromEnv)
@@ -188,14 +191,16 @@ func main() {
188191
embeddedUI,
189192
server,
190193
logCleaner,
194+
logFilter,
191195
)
192196
}
193197

194198
server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
195-
obs, est, pm, tm, tokenHolder, embeddedUI, reloadConfigFn)
199+
obs, est, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
196200
shutdownCh := setShutdownListener()
197201

198-
go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, logCleaner)
202+
go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
203+
logCleaner, logFilter)
199204

200205
server.InitHandlers()
201206

@@ -276,12 +281,14 @@ func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *con
276281

277282
func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
278283
retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
279-
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner) error {
284+
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
285+
filtering *log.Filtering) error {
280286
cfg, err := config.LoadConfiguration()
281287
if err != nil {
282288
return err
283289
}
284290

291+
filtering.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
285292
config.ApplyGlobals(cfg)
286293

287294
if err := provision.IsValidConfig(cfg.Provision); err != nil {
@@ -328,14 +335,16 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
328335

329336
func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
330337
retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
331-
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner) {
338+
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
339+
logFilter *log.Filtering) {
332340
reloadCh := make(chan os.Signal, 1)
333341
signal.Notify(reloadCh, syscall.SIGHUP)
334342

335343
for range reloadCh {
336344
log.Msg("Reloading configuration")
337345

338-
if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, cleaner); err != nil {
346+
if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
347+
cleaner, logFilter); err != nil {
339348
log.Err("Failed to reload configuration", err)
340349
}
341350

‎engine/internal/srv/server.go‎

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,8 @@ import (
99
"context"
1010
"fmt"
1111
"net/http"
12-
"regexp"
1312
"strings"
1413
"time"
15-
"unicode"
1614

1715
"github.com/AlekSi/pointer"
1816
"github.com/docker/docker/client"
@@ -41,8 +39,6 @@ import (
4139
"gitlab.com/postgres-ai/database-lab/v3/version"
4240
)
4341

44-
const minTokenLength = 8
45-
4642
// Server defines an HTTP server of the Database Lab.
4743
type Server struct {
4844
validator validator.Service
@@ -61,7 +57,7 @@ type Server struct {
6157
pm *pool.Manager
6258
tm *telemetry.Agent
6359
startedAt *models.LocalTime
64-
re *regexp.Regexp
60+
filtering *log.Filtering
6561
reloadFn func(server *Server) error
6662
}
6763

@@ -77,7 +73,7 @@ func NewServer(cfg *srvCfg.Config, globalCfg *global.Config, engineProps global.
7773
dockerClient *client.Client, cloning *cloning.Base, provisioner *provision.Provisioner,
7874
retrievalSvc *retrieval.Retrieval, platform *platform.Service, observer *observer.Observer,
7975
estimator *estimator.Estimator, pm *pool.Manager, tm *telemetry.Agent, tokenKeeper *ws.TokenKeeper,
80-
uiManager *embeddedui.UIManager, reloadConfigFn func(server *Server) error) *Server {
76+
filtering*log.Filtering, uiManager *embeddedui.UIManager, reloadConfigFn func(server *Server) error) *Server {
8177
server := &Server{
8278
Config: cfg,
8379
Global: globalCfg,
@@ -96,10 +92,10 @@ func NewServer(cfg *srvCfg.Config, globalCfg *global.Config, engineProps global.
9692
docker: dockerClient,
9793
pm: pm,
9894
tm: tm,
95+
filtering: filtering,
9996
startedAt: &models.LocalTime{Time: time.Now().Truncate(time.Second)},
10097
reloadFn: reloadConfigFn,
10198
}
102-
server.initLogRegExp()
10399

104100
return server
105101
}
@@ -185,7 +181,6 @@ func attachAPI(r *mux.Router) error {
185181
// Reload reloads server configuration.
186182
func (s *Server) Reload(cfg srvCfg.Config) {
187183
*s.Config = cfg
188-
s.initLogRegExp()
189184
}
190185

191186
// InitHandlers initializes handler functions of the HTTP server.
@@ -266,31 +261,5 @@ func reportLaunching(cfg *srvCfg.Config) {
266261
}
267262

268263
func (s *Server) initLogRegExp() {
269-
secretPatterns := []string{
270-
"password:\\s?(\\S+)",
271-
"POSTGRES_PASSWORD=(\\S+)",
272-
"PGPASSWORD=(\\S+)",
273-
"accessToken:\\s?(\\S+)",
274-
"ACCESS_KEY(_ID)?:\\s?(\\S+)",
275-
}
276-
277-
if len(s.Config.VerificationToken) >= minTokenLength && !containsSpace(s.Config.VerificationToken) {
278-
secretPatterns = append(secretPatterns, s.Config.VerificationToken)
279-
}
280-
281-
if accessToken := s.Platform.AccessToken(); len(accessToken) >= minTokenLength && !containsSpace(accessToken) {
282-
secretPatterns = append(secretPatterns, accessToken)
283-
}
284-
285-
s.re = regexp.MustCompile("(?i)" + strings.Join(secretPatterns, "|"))
286-
}
287-
288-
func containsSpace(s string) bool {
289-
for _, v := range s {
290-
if unicode.IsSpace(v) {
291-
return true
292-
}
293-
}
294-
295-
return false
264+
s.filtering.ReloadLogRegExp([]string{s.Config.VerificationToken, s.Platform.AccessToken()})
296265
}

‎engine/internal/srv/ws.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,5 +109,5 @@ func (s *Server) instanceLogs(w http.ResponseWriter, r *http.Request) {
109109
}
110110

111111
func (s *Server) filterLogLine(inputLine []byte) []byte {
112-
return s.re.ReplaceAll(inputLine, []byte("********"))
112+
return s.filtering.ReplaceAll(inputLine)
113113
}

‎engine/internal/srv/ws_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,18 @@ import (
99

1010
"gitlab.com/postgres-ai/database-lab/v3/internal/platform"
1111
"gitlab.com/postgres-ai/database-lab/v3/internal/srv/config"
12+
"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
1213
)
1314

1415
func TestLogLineFiltering(t *testing.T) {
1516
pl, err := platform.New(context.Background(), platform.Config{AccessToken: "platformAccessToken"})
1617
require.NoError(t, err)
1718

18-
s := Server{Config: &config.Config{VerificationToken: "secretToken"}, Platform: pl}
19+
s := Server{
20+
Config: &config.Config{VerificationToken: "secretToken"},
21+
Platform: pl,
22+
filtering: log.GetFilter(),
23+
}
1924
s.initLogRegExp()
2025

2126
testCases := []struct {

‎engine/pkg/log/filtering.go‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package log
2+
3+
import (
4+
"regexp"
5+
"strings"
6+
"unicode"
7+
)
8+
9+
const (
10+
minTokenLength = 8
11+
replacingMask = "********"
12+
)
13+
14+
var filter = newFiltering([]string{})
15+
16+
// Filtering represents a struct to filter secrets in logs output.
17+
type Filtering struct {
18+
re *regexp.Regexp
19+
}
20+
21+
// GetFilter gets an instance of Log Filtering.
22+
func GetFilter() *Filtering {
23+
return filter
24+
}
25+
26+
func newFiltering(tokens []string) *Filtering {
27+
f := &Filtering{}
28+
f.ReloadLogRegExp(tokens)
29+
30+
return f
31+
}
32+
33+
// ReloadLogRegExp updates secrets configuration.
34+
func (f *Filtering) ReloadLogRegExp(secretStings []string) {
35+
secretPatterns := []string{
36+
"password:\\s?(\\S+)",
37+
"POSTGRES_PASSWORD=(\\S+)",
38+
"PGPASSWORD=(\\S+)",
39+
"accessToken:\\s?(\\S+)",
40+
"ACCESS_KEY(_ID)?:\\s?(\\S+)",
41+
}
42+
43+
for _, secret := range secretStings {
44+
if len(secret) >= minTokenLength && !containsSpace(secret) {
45+
secretPatterns = append(secretPatterns, secret)
46+
}
47+
}
48+
49+
f.re = regexp.MustCompile("(?i)" + strings.Join(secretPatterns, "|"))
50+
}
51+
52+
// ReplaceAll replaces all secrets in the input line.
53+
func (f *Filtering) ReplaceAll(input []byte) []byte {
54+
return f.re.ReplaceAll(input, []byte(replacingMask))
55+
}
56+
57+
func containsSpace(s string) bool {
58+
for _, v := range s {
59+
if unicode.IsSpace(v) {
60+
return true
61+
}
62+
}
63+
64+
return false
65+
}

‎engine/pkg/log/log.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func prepareMessage(v ...interface{}) string {
7070
builder := strings.Builder{}
7171

7272
for _, value := range v {
73-
builder.WriteString(" " + toString(value))
73+
builder.WriteString(" " + filter.re.ReplaceAllString(toString(value), replacingMask))
7474
}
7575

7676
return builder.String()

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /