Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 0ad2cdd

Browse files
committed
Merge branch '486-se-platform-token' into 'master'
feat(engine): enable SE billing (#486) Closes #486 See merge request postgres-ai/database-lab!693
2 parents dc7004e + 0695c66 commit 0ad2cdd

File tree

29 files changed

+694
-193
lines changed

29 files changed

+694
-193
lines changed

‎engine/cmd/database-lab/main.go‎

Lines changed: 42 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,14 @@ import (
1313
"fmt"
1414
"os"
1515
"os/signal"
16-
"runtime"
1716
"strings"
1817
"syscall"
1918
"time"
2019

2120
"github.com/docker/docker/client"
22-
"github.com/pbnjay/memory"
2321
"github.com/pkg/errors"
2422

23+
"gitlab.com/postgres-ai/database-lab/v3/internal/billing"
2524
"gitlab.com/postgres-ai/database-lab/v3/internal/cloning"
2625
"gitlab.com/postgres-ai/database-lab/v3/internal/diagnostic"
2726
"gitlab.com/postgres-ai/database-lab/v3/internal/embeddedui"
@@ -56,7 +55,7 @@ func main() {
5655
}
5756

5857
logFilter := log.GetFilter()
59-
logFilter.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
58+
logFilter.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken, cfg.Platform.OrgKey})
6059

6160
config.ApplyGlobals(cfg)
6261

@@ -83,6 +82,13 @@ func main() {
8382
log.Msg("Database Lab Instance ID:", engProps.InstanceID)
8483
log.Msg("Database Lab Engine version:", version.GetVersion())
8584

85+
// Create a platform service to make requests to Platform.
86+
platformSvc, err := platform.New(ctx, cfg.Platform, engProps.InstanceID)
87+
if err != nil {
88+
log.Errf(err.Error())
89+
return
90+
}
91+
8692
if cfg.Server.VerificationToken == "" {
8793
log.Warn("Verification Token is empty. Database Lab Engine is insecure")
8894
}
@@ -97,33 +103,22 @@ func main() {
97103

98104
defer networks.Stop(docker, internalNetworkID, engProps.ContainerName)
99105

100-
// Create a platform service to make requests to Platform.
101-
platformSvc, err := platform.New(ctx, cfg.Platform)
102-
if err != nil {
103-
log.Errf(errors.WithMessage(err, "failed to create a new platform service").Error())
104-
return
105-
}
106-
107106
dbCfg := &resources.DB{
108107
Username: cfg.Global.Database.User(),
109108
DBName: cfg.Global.Database.Name(),
110109
}
111110

112-
tm, err := telemetry.New(cfg.Global, engProps)
113-
if err != nil {
114-
log.Errf(errors.WithMessage(err, "failed to initialize a telemetry service").Error())
115-
return
116-
}
111+
tm := telemetry.New(platformSvc, engProps.InstanceID)
117112

118113
pm := pool.NewPoolManager(&cfg.PoolManager, runner)
119114
if err = pm.ReloadPools(); err != nil {
120115
log.Err(err.Error())
121116
}
122117

123118
// Create a new retrieval service to prepare a data directory and start snapshotting.
124-
retrievalSvc, err := retrieval.New(cfg, engProps, docker, pm, tm, runner)
119+
retrievalSvc, err := retrieval.New(cfg, &engProps, docker, pm, tm, runner)
125120
if err != nil {
126-
log.Errf(errors.WithMessage(err, `error in the "retrieval" section of the config`).Error())
121+
log.Errf(errors.WithMessage(err, `error in the "retrieval" section of config`).Error())
127122
return
128123
}
129124

@@ -160,27 +155,34 @@ func main() {
160155

161156
go removeObservingClones(observingChan, obs)
162157

158+
systemMetrics := billing.GetSystemMetrics(pm)
159+
163160
tm.SendEvent(ctx, telemetry.EngineStartedEvent, telemetry.EngineStarted{
164161
EngineVersion: version.GetVersion(),
165162
DBEngine: cfg.Global.Engine,
166163
DBVersion: provisioner.DetectDBVersion(),
167164
Pools: pm.CollectPoolStat(),
168165
Restore: retrievalSvc.ReportState(),
169-
System: telemetry.System{
170-
CPU: runtime.NumCPU(),
171-
TotalMemory: memory.TotalMemory(),
172-
},
166+
System: systemMetrics,
173167
})
174168

169+
billingSvc := billing.New(platformSvc.Client, &engProps, pm)
170+
171+
if err := billingSvc.RegisterInstance(ctx, systemMetrics); err != nil {
172+
log.Msg("Skip registering instance:", err)
173+
}
174+
175+
log.Msg("DLE Edition:", engProps.GetEdition())
176+
175177
embeddedUI := embeddedui.New(cfg.EmbeddedUI, engProps, runner, docker)
176178

177179
logCleaner := diagnostic.NewLogCleaner()
178180

179181
reloadConfigFn := func(server *srv.Server) error {
180182
return reloadConfig(
181183
ctx,
184+
engProps,
182185
provisioner,
183-
tm,
184186
retrievalSvc,
185187
pm,
186188
cloningSvc,
@@ -192,11 +194,13 @@ func main() {
192194
)
193195
}
194196

195-
server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
196-
obs, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
197+
server := srv.NewServer(&cfg.Server, &cfg.Global, &engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
198+
billingSvc, obs, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
197199
shutdownCh := setShutdownListener()
198200

199-
go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, embeddedUI, server,
201+
go setReloadListener(ctx, engProps, provisioner,
202+
retrievalSvc, pm, cloningSvc, platformSvc,
203+
embeddedUI, server,
200204
logCleaner, logFilter)
201205

202206
server.InitHandlers()
@@ -207,6 +211,8 @@ func main() {
207211
}
208212
}()
209213

214+
go billingSvc.CollectUsage(ctx, systemMetrics)
215+
210216
if cfg.EmbeddedUI.Enabled {
211217
go func() {
212218
if err := embeddedUI.Run(ctx); err != nil {
@@ -245,13 +251,13 @@ func main() {
245251
tm.SendEvent(ctxBackground, telemetry.EngineStoppedEvent, telemetry.EngineStopped{Uptime: server.Uptime()})
246252
}
247253

248-
func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *config.Config) (global.EngineProps, error) {
254+
func getEngineProperties(ctx context.Context, docker *client.Client, cfg *config.Config) (global.EngineProps, error) {
249255
hostname := os.Getenv("HOSTNAME")
250256
if hostname == "" {
251257
return global.EngineProps{}, errors.New("hostname is empty")
252258
}
253259

254-
dleContainer, err := dockerCLI.ContainerInspect(ctx, hostname)
260+
dleContainer, err := docker.ContainerInspect(ctx, hostname)
255261
if err != nil {
256262
return global.EngineProps{}, fmt.Errorf("failed to inspect DLE container: %w", err)
257263
}
@@ -276,15 +282,15 @@ func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *con
276282
return engProps, nil
277283
}
278284

279-
func reloadConfig(ctx context.Context, provisionSvc*provision.Provisioner, tm*telemetry.Agent,
285+
func reloadConfig(ctx context.Context, engProp global.EngineProps, provisionSvc*provision.Provisioner,
280286
retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
281287
embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner, filtering *log.Filtering) error {
282288
cfg, err := config.LoadConfiguration()
283289
if err != nil {
284290
return err
285291
}
286292

287-
filtering.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
293+
filtering.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken, cfg.Platform.OrgKey})
288294
config.ApplyGlobals(cfg)
289295

290296
if err := provision.IsValidConfig(cfg.Provision); err != nil {
@@ -296,7 +302,7 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
296302
return err
297303
}
298304

299-
newPlatformSvc, err := platform.New(ctx, cfg.Platform)
305+
newPlatformSvc, err := platform.New(ctx, cfg.Platform, engProp.InstanceID)
300306
if err != nil {
301307
return err
302308
}
@@ -319,7 +325,6 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
319325
}
320326

321327
provisionSvc.Reload(cfg.Provision, dbCfg)
322-
tm.Reload(cfg.Global)
323328
retrievalSvc.Reload(ctx, newRetrievalConfig)
324329
cloningSvc.Reload(cfg.Cloning)
325330
platformSvc.Reload(newPlatformSvc)
@@ -328,7 +333,7 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
328333
return nil
329334
}
330335

331-
func setReloadListener(ctx context.Context, provisionSvc*provision.Provisioner, tm*telemetry.Agent,
336+
func setReloadListener(ctx context.Context, engProp global.EngineProps, provisionSvc*provision.Provisioner,
332337
retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
333338
embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner, logFilter *log.Filtering) {
334339
reloadCh := make(chan os.Signal, 1)
@@ -337,7 +342,11 @@ func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner,
337342
for range reloadCh {
338343
log.Msg("Reloading configuration")
339344

340-
if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, embeddedUI, server,
345+
if err := reloadConfig(ctx, engProp,
346+
provisionSvc, retrievalSvc,
347+
pm, cloningSvc,
348+
platformSvc,
349+
embeddedUI, server,
341350
cleaner, logFilter); err != nil {
342351
log.Err("Failed to reload configuration", err)
343352
}

‎engine/cmd/runci/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,8 @@ func main() {
6565
}
6666

6767
// Create a platform service to make requests to Platform.
68-
platformSvc, err := platform.New(ctx, cfg.Platform)
68+
// Instance ID is not defined for the Run CI service.
69+
platformSvc, err := platform.New(ctx, cfg.Platform, "")
6970
if err != nil {
7071
log.Errf(errors.WithMessage(err, "failed to create a new platform service").Error())
7172
return

‎engine/configs/config.example.logical_generic.yml‎

Lines changed: 16 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ server:
2020
# In this case, the DLE API and the UI application will not require any credentials.
2121
verificationToken: "secret_token"
2222

23+
2324
# HTTP server port. Default: 2345.
2425
port: 2345
2526

@@ -58,15 +59,6 @@ global:
5859
# Default database name.
5960
dbname: postgres
6061

61-
# Telemetry: anonymous statistics sent to Postgres.ai.
62-
# Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
63-
# Please leave it enabled if possible – this will contribute to DLE development.
64-
# The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
65-
telemetry:
66-
enabled: true
67-
# Telemetry API URL. To send anonymous telemetry data, keep it default ("https://postgres.ai/api/general").
68-
url: "https://postgres.ai/api/general"
69-
7062
# Manages filesystem pools (in the case of ZFS) or volume groups.
7163
poolManager:
7264
# The full path which contains the pool mount directories. mountDir can contain multiple pool directories.
@@ -367,10 +359,21 @@ diagnostic:
367359
# Postgres.ai Platform integration (provides GUI) – extends the open source offering.
368360
# Uncomment the following lines if you need GUI, personal tokens, audit logs, more.
369361
#
370-
#platform:
371-
# # Platform API URL. To work with Postgres.ai SaaS, keep it default
372-
# # ("https://postgres.ai/api/general").
373-
# url: "https://postgres.ai/api/general"
362+
platform:
363+
# Platform API URL. To work with Postgres.ai SaaS, keep it default
364+
# ("https://postgres.ai/api/general").
365+
url: "https://postgres.ai/api/general"
366+
# Telemetry: anonymous statistics sent to Postgres.ai.
367+
# Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
368+
# Please leave it enabled if possible – this will contribute to DLE development.
369+
# The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
370+
enableTelemetry: true
371+
#
372+
# # Project name
373+
# projectName: "project_name"
374+
#
375+
# # Organization key
376+
# orgKey: "org_key"
374377
#
375378
# # Token for authorization in Platform API. This token can be obtained on
376379
# # the Postgres.ai Console: https://postgres.ai/console/YOUR_ORG_NAME/tokens

‎engine/configs/config.example.logical_rds_iam.yml‎

Lines changed: 17 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -58,15 +58,6 @@ global:
5858
# Default database name.
5959
dbname: postgres
6060

61-
# Telemetry: anonymous statistics sent to Postgres.ai.
62-
# Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
63-
# Please leave it enabled if possible – this will contribute to DLE development.
64-
# The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
65-
telemetry:
66-
enabled: true
67-
# Telemetry API URL. To send anonymous telemetry data, keep it default ("https://postgres.ai/api/general").
68-
url: "https://postgres.ai/api/general"
69-
7061
# Manages filesystem pools (in the case of ZFS) or volume groups.
7162
poolManager:
7263
# The full path which contains the pool mount directories. mountDir can contain multiple pool directories.
@@ -368,19 +359,30 @@ diagnostic:
368359
# Postgres.ai Platform integration (provides GUI) – extends the open source offering.
369360
# Uncomment the following lines if you need GUI, personal tokens, audit logs, more.
370361
#
371-
#platform:
372-
# # Platform API URL. To work with Postgres.ai SaaS, keep it default
373-
# # ("https://postgres.ai/api/general").
374-
# url: "https://postgres.ai/api/general"
362+
platform:
363+
# Platform API URL. To work with Postgres.ai SaaS, keep it default
364+
# ("https://postgres.ai/api/general").
365+
url: "https://postgres.ai/api/general"
366+
# Telemetry: anonymous statistics sent to Postgres.ai.
367+
# Used to analyze DLE usage, it helps the DLE maintainers make decisions on product development.
368+
# Please leave it enabled if possible – this will contribute to DLE development.
369+
# The full list of data points being collected: https://postgres.ai/docs/database-lab/telemetry
370+
enableTelemetry: true
371+
#
372+
# # Project name
373+
# projectName: "project_name"
374+
#
375+
# # Organization key
376+
# orgKey: "org_key"
375377
#
376378
# # Token for authorization in Platform API. This token can be obtained on
377379
# # the Postgres.ai Console: https://postgres.ai/console/YOUR_ORG_NAME/tokens
378380
# # This token needs to be kept in secret, known only to the administrator.
379381
# accessToken: "platform_access_token"
380382
#
381383
# # Enable authorization with personal tokens of the organization's members.
382-
# # If false: all users must use "accessToken" value for any API request
383-
# # If true: "accessToken" is known only to admin, users use their own tokens,
384+
# # If false: all users must use "verificationToken" value for any API request
385+
# # If true: "verificationToken" is known only to admin, users use their own tokens,
384386
# # and any token can be revoked not affecting others
385387
# enablePersonalTokens: true
386388
#

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /