Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 340e5f8

Browse files
committed
Merge branch '383-logical-dump-container-startup' into 'master'
fix(engine): update logical dump flow to start existing dump container (#383) Closes #383 See merge request postgres-ai/database-lab!544
2 parents 5818bd8 + 2950844 commit 340e5f8

File tree

8 files changed

+195
-124
lines changed

8 files changed

+195
-124
lines changed

‎engine/internal/embeddedui/embedded_ui.go‎

Lines changed: 26 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import (
1313

1414
"github.com/docker/docker/api/types"
1515
"github.com/docker/docker/api/types/container"
16-
"github.com/docker/docker/api/types/network"
1716
"github.com/docker/docker/client"
1817
"github.com/docker/go-connections/nat"
1918

@@ -95,54 +94,37 @@ func (ui *UIManager) Run(ctx context.Context) error {
9594
return fmt.Errorf("failed to prepare Docker image: %w", err)
9695
}
9796

98-
var containerID = ""
99-
100-
// try to fetch an existing UI container
101-
containerData, err := ui.docker.ContainerInspect(ctx, getEmbeddedUIName(ui.engProps.InstanceID))
102-
103-
if err == nil {
104-
containerID = containerData.ID
105-
}
106-
107-
if containerID == "" {
108-
embeddedUI, err := ui.docker.ContainerCreate(ctx,
109-
&container.Config{
110-
Labels: map[string]string{
111-
cont.DBLabSatelliteLabel: cont.DBLabEmbeddedUILabel,
112-
cont.DBLabInstanceIDLabel: ui.engProps.InstanceID,
113-
cont.DBLabEngineNameLabel: ui.engProps.ContainerName,
114-
},
115-
Image: ui.cfg.DockerImage,
116-
Env: []string{
117-
EnvEngineName + "=" + ui.engProps.ContainerName,
118-
EnvEnginePort + "=" + strconv.FormatUint(uint64(ui.engProps.EnginePort), 10),
119-
},
120-
Healthcheck: &container.HealthConfig{
121-
Interval: healthCheckInterval,
122-
Timeout: healthCheckTimeout,
123-
Retries: healthCheckRetries,
124-
},
97+
containerID, err := tools.CreateContainerIfMissing(ctx, ui.docker, getEmbeddedUIName(ui.engProps.InstanceID),
98+
&container.Config{
99+
Labels: map[string]string{
100+
cont.DBLabSatelliteLabel: cont.DBLabEmbeddedUILabel,
101+
cont.DBLabInstanceIDLabel: ui.engProps.InstanceID,
102+
cont.DBLabEngineNameLabel: ui.engProps.ContainerName,
103+
},
104+
Image: ui.cfg.DockerImage,
105+
Env: []string{
106+
EnvEngineName + "=" + ui.engProps.ContainerName,
107+
EnvEnginePort + "=" + strconv.FormatUint(uint64(ui.engProps.EnginePort), 10),
125108
},
126-
&container.HostConfig{
127-
PortBindings: map[nat.Port][]nat.PortBinding{
128-
"80/tcp": {
129-
{
130-
HostIP: ui.cfg.Host,
131-
HostPort: strconv.Itoa(ui.cfg.Port),
132-
},
109+
Healthcheck: &container.HealthConfig{
110+
Interval: healthCheckInterval,
111+
Timeout: healthCheckTimeout,
112+
Retries: healthCheckRetries,
113+
},
114+
},
115+
&container.HostConfig{
116+
PortBindings: map[nat.Port][]nat.PortBinding{
117+
"80/tcp": {
118+
{
119+
HostIP: ui.cfg.Host,
120+
HostPort: strconv.Itoa(ui.cfg.Port),
133121
},
134122
},
135123
},
136-
&network.NetworkingConfig{},
137-
nil,
138-
getEmbeddedUIName(ui.engProps.InstanceID),
139-
)
140-
141-
if err != nil {
142-
return fmt.Errorf("failed to prepare Docker image for embedded UI: %w", err)
143-
}
124+
})
144125

145-
containerID = embeddedUI.ID
126+
if err != nil {
127+
return fmt.Errorf("failed to prepare Docker image for embedded UI: %w", err)
146128
}
147129

148130
if err := networks.Connect(ctx, ui.docker, ui.engProps.InstanceID, containerID); err != nil {

‎engine/internal/retrieval/engine/postgres/logical/dump.go‎

Lines changed: 14 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ import (
1515

1616
"github.com/docker/docker/api/types"
1717
"github.com/docker/docker/api/types/container"
18-
"github.com/docker/docker/api/types/network"
1918
"github.com/docker/docker/client"
2019
"github.com/jackc/pgx/v4"
2120
"github.com/pkg/errors"
@@ -30,7 +29,6 @@ import (
3029
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/defaults"
3130
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools/health"
3231
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/options"
33-
3432
"gitlab.com/postgres-ai/database-lab/v3/pkg/config/global"
3533
"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
3634
)
@@ -270,26 +268,23 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
270268
return errors.Wrap(err, "failed to generate PostgreSQL password")
271269
}
272270

273-
dumpCont, err := d.dockerClient.ContainerCreate(ctx, d.buildContainerConfig(pwd), hostConfig, &network.NetworkingConfig{},
274-
nil, d.dumpContainerName(),
275-
)
276-
if err != nil {
277-
log.Err(err)
271+
containerID, err := tools.CreateContainerIfMissing(ctx, d.dockerClient, d.dumpContainerName(), d.buildContainerConfig(pwd), hostConfig)
278272

279-
return errors.Wrapf(err, "failed to create container %q", d.dumpContainerName())
273+
if err != nil {
274+
return fmt.Errorf("failed to create container %q %w", d.dumpContainerName(), err)
280275
}
281276

282-
defer tools.RemoveContainer(ctx, d.dockerClient, dumpCont.ID, cont.StopTimeout)
277+
defer tools.RemoveContainer(ctx, d.dockerClient, containerID, cont.StopTimeout)
283278

284279
defer func() {
285280
if err != nil {
286281
tools.PrintContainerLogs(ctx, d.dockerClient, d.dumpContainerName())
287282
}
288283
}()
289284

290-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), dumpCont.ID))
285+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", d.dumpContainerName(), containerID))
291286

292-
if err := d.dockerClient.ContainerStart(ctx, dumpCont.ID, types.ContainerStartOptions{}); err != nil {
287+
if err := d.dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{}); err != nil {
293288
return errors.Wrapf(err, "failed to start container %q", d.dumpContainerName())
294289
}
295290

@@ -299,13 +294,13 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
299294

300295
log.Msg("Waiting for container readiness")
301296

302-
if err := tools.MakeDir(ctx, d.dockerClient, dumpCont.ID, tmpDBLabPGDataDir); err != nil {
297+
if err := tools.MakeDir(ctx, d.dockerClient, containerID, tmpDBLabPGDataDir); err != nil {
303298
return err
304299
}
305300

306301
dataDir := d.fsPool.DataDir()
307302

308-
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, dumpCont.ID); err != nil {
303+
if err := tools.CheckContainerReadiness(ctx, d.dockerClient, containerID); err != nil {
309304
var errHealthCheck *tools.ErrHealthCheck
310305
if !errors.As(err, &errHealthCheck) {
311306
return errors.Wrap(err, "failed to readiness check")
@@ -316,13 +311,13 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
316311
pgDataDir = dataDir
317312
}
318313

319-
if err := setupPGData(ctx, d.dockerClient, pgDataDir, dumpCont.ID); err != nil {
314+
if err := setupPGData(ctx, d.dockerClient, pgDataDir, containerID); err != nil {
320315
return errors.Wrap(err, "failed to set up Postgres data")
321316
}
322317
}
323318

324319
if d.DumpOptions.Restore.Enabled && len(d.DumpOptions.Restore.Configs) > 0 {
325-
if err := updateConfigs(ctx, d.dockerClient, dataDir, dumpCont.ID, d.DumpOptions.Restore.Configs); err != nil {
320+
if err := updateConfigs(ctx, d.dockerClient, dataDir, containerID, d.DumpOptions.Restore.Configs); err != nil {
326321
return errors.Wrap(err, "failed to update configs")
327322
}
328323
}
@@ -336,12 +331,12 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
336331
}
337332
}
338333

339-
if err := d.cleanupDumpLocation(ctx, dumpCont.ID, dbList); err != nil {
334+
if err := d.cleanupDumpLocation(ctx, containerID, dbList); err != nil {
340335
return err
341336
}
342337

343338
for dbName, dbDetails := range dbList {
344-
if err := d.dumpDatabase(ctx, dumpCont.ID, dbName, dbDetails); err != nil {
339+
if err := d.dumpDatabase(ctx, containerID, dbName, dbDetails); err != nil {
345340
return errors.Wrapf(err, "failed to dump the database %s", dbName)
346341
}
347342
}
@@ -358,11 +353,11 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
358353

359354
log.Msg("Running analyze command: ", analyzeCmd)
360355

361-
if err := tools.ExecCommand(ctx, d.dockerClient, dumpCont.ID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
356+
if err := tools.ExecCommand(ctx, d.dockerClient, containerID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
362357
return errors.Wrap(err, "failed to recalculate statistics after restore")
363358
}
364359

365-
if err := tools.StopPostgres(ctx, d.dockerClient, dumpCont.ID, dataDir, tools.DefaultStopTimeout); err != nil {
360+
if err := tools.StopPostgres(ctx, d.dockerClient, containerID, dataDir, tools.DefaultStopTimeout); err != nil {
366361
return errors.Wrap(err, "failed to stop Postgres instance")
367362
}
368363
}
Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
//go:build integration
2+
// +build integration
3+
4+
/*
5+
2021 © Postgres.ai
6+
*/
7+
8+
package logical
9+
10+
import (
11+
"context"
12+
"fmt"
13+
"math/rand"
14+
"testing"
15+
"time"
16+
17+
"github.com/docker/docker/api/types"
18+
"github.com/docker/docker/api/types/filters"
19+
"github.com/docker/docker/api/types/network"
20+
"github.com/docker/docker/client"
21+
"github.com/stretchr/testify/assert"
22+
"github.com/stretchr/testify/require"
23+
24+
dockerutils "gitlab.com/postgres-ai/database-lab/v3/internal/provision/docker"
25+
"gitlab.com/postgres-ai/database-lab/v3/internal/provision/resources"
26+
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/config"
27+
"gitlab.com/postgres-ai/database-lab/v3/internal/retrieval/engine/postgres/tools"
28+
"gitlab.com/postgres-ai/database-lab/v3/pkg/config/global"
29+
)
30+
31+
func TestStartExisingDumpContainer(t *testing.T) {
32+
t.Parallel()
33+
ctx := context.Background()
34+
35+
docker, err := client.NewClientWithOpts(client.FromEnv)
36+
require.NoError(t, err)
37+
38+
// create dump job
39+
40+
source := rand.NewSource(time.Now().UnixNano())
41+
random := rand.New(source)
42+
43+
engProps := global.EngineProps{
44+
InstanceID: fmt.Sprintf("dumpjob-%d", random.Intn(10000)),
45+
}
46+
47+
job, err := NewDumpJob(
48+
config.JobConfig{
49+
Spec: config.JobSpec{Name: "test"},
50+
FSPool: &resources.Pool{
51+
DataSubDir: t.TempDir(),
52+
},
53+
Docker: docker,
54+
},
55+
&global.Config{},
56+
engProps,
57+
)
58+
assert.NoError(t, err)
59+
job.DockerImage = "postgresai/extended-postgres:14"
60+
job.DumpOptions.DumpLocation = t.TempDir()
61+
62+
err = dockerutils.PrepareImage(ctx, docker, job.DockerImage)
63+
assert.NoError(t, err)
64+
65+
// create dump container and stop it
66+
container, err := docker.ContainerCreate(ctx, job.buildContainerConfig(""), nil, &network.NetworkingConfig{},
67+
nil, job.dumpContainerName(),
68+
)
69+
assert.NoError(t, err)
70+
71+
// clean container in case of any error
72+
defer tools.RemoveContainer(ctx, docker, container.ID, 10*time.Second)
73+
74+
job.Run(ctx)
75+
76+
// list containers and check that container job container got processed
77+
filterArgs := filters.NewArgs()
78+
filterArgs.Add("name", job.dumpContainerName())
79+
80+
list, err := docker.ContainerList(
81+
ctx,
82+
types.ContainerListOptions{
83+
All: false,
84+
Filters: filterArgs,
85+
},
86+
)
87+
88+
require.NoError(t, err)
89+
assert.Empty(t, list)
90+
91+
}

‎engine/internal/retrieval/engine/postgres/logical/restore.go‎

Lines changed: 13 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020

2121
"github.com/docker/docker/api/types"
2222
"github.com/docker/docker/api/types/container"
23-
"github.com/docker/docker/api/types/network"
2423
"github.com/docker/docker/client"
2524
"github.com/docker/docker/pkg/archive"
2625
"github.com/pkg/errors"
@@ -195,61 +194,56 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
195194
return errors.Wrap(err, "failed to generate PostgreSQL password")
196195
}
197196

198-
restoreCont, err := r.dockerClient.ContainerCreate(ctx,
199-
r.buildContainerConfig(pwd),
200-
hostConfig,
201-
&network.NetworkingConfig{},
202-
nil,
203-
r.restoreContainerName(),
204-
)
197+
containerID, err := tools.CreateContainerIfMissing(ctx, r.dockerClient, r.restoreContainerName(), r.buildContainerConfig(pwd), hostConfig)
198+
205199
if err != nil {
206-
return errors.Wrapf(err, "failed to create container %q", r.restoreContainerName())
200+
return fmt.Errorf("failed to create container %q %w", r.restoreContainerName(), err)
207201
}
208202

209-
defer tools.RemoveContainer(ctx, r.dockerClient, restoreCont.ID, cont.StopTimeout)
203+
defer tools.RemoveContainer(ctx, r.dockerClient, containerID, cont.StopTimeout)
210204

211205
defer func() {
212206
if err != nil {
213207
tools.PrintContainerLogs(ctx, r.dockerClient, r.restoreContainerName())
214208
}
215209
}()
216210

217-
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), restoreCont.ID))
211+
log.Msg(fmt.Sprintf("Running container: %s. ID: %v", r.restoreContainerName(), containerID))
218212

219-
if err := r.dockerClient.ContainerStart(ctx, restoreCont.ID, types.ContainerStartOptions{}); err != nil {
213+
if err := r.dockerClient.ContainerStart(ctx, containerID, types.ContainerStartOptions{}); err != nil {
220214
return errors.Wrapf(err, "failed to start container %q", r.restoreContainerName())
221215
}
222216

223217
dataDir := r.fsPool.DataDir()
224218

225219
log.Msg("Waiting for container readiness")
226220

227-
if err := tools.CheckContainerReadiness(ctx, r.dockerClient, restoreCont.ID); err != nil {
221+
if err := tools.CheckContainerReadiness(ctx, r.dockerClient, containerID); err != nil {
228222
var errHealthCheck *tools.ErrHealthCheck
229223
if !errors.As(err, &errHealthCheck) {
230224
return errors.Wrap(err, "failed to readiness check")
231225
}
232226

233-
if err := setupPGData(ctx, r.dockerClient, dataDir, restoreCont.ID); err != nil {
227+
if err := setupPGData(ctx, r.dockerClient, dataDir, containerID); err != nil {
234228
return errors.Wrap(err, "failed to set up Postgres data")
235229
}
236230
}
237231

238232
if len(r.RestoreOptions.Configs) > 0 {
239-
if err := updateConfigs(ctx, r.dockerClient, dataDir, restoreCont.ID, r.RestoreOptions.Configs); err != nil {
233+
if err := updateConfigs(ctx, r.dockerClient, dataDir, containerID, r.RestoreOptions.Configs); err != nil {
240234
return errors.Wrap(err, "failed to update configs")
241235
}
242236
}
243237

244-
dbList, err := r.getDBList(ctx, restoreCont.ID)
238+
dbList, err := r.getDBList(ctx, containerID)
245239
if err != nil {
246240
return err
247241
}
248242

249243
log.Dbg("Database List to restore: ", dbList)
250244

251245
for dbName, dbDefinition := range dbList {
252-
if err := r.restoreDB(ctx, restoreCont.ID, dbName, dbDefinition); err != nil {
246+
if err := r.restoreDB(ctx, containerID, dbName, dbDefinition); err != nil {
253247
return errors.Wrap(err, "failed to restore a database")
254248
}
255249
}
@@ -261,11 +255,11 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
261255

262256
log.Msg("Running analyze command: ", analyzeCmd)
263257

264-
if err := tools.ExecCommand(ctx, r.dockerClient, restoreCont.ID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
258+
if err := tools.ExecCommand(ctx, r.dockerClient, containerID, types.ExecConfig{Cmd: analyzeCmd}); err != nil {
265259
return errors.Wrap(err, "failed to recalculate statistics after restore")
266260
}
267261

268-
if err := tools.StopPostgres(ctx, r.dockerClient, restoreCont.ID, dataDir, tools.DefaultStopTimeout); err != nil {
262+
if err := tools.StopPostgres(ctx, r.dockerClient, containerID, dataDir, tools.DefaultStopTimeout); err != nil {
269263
return errors.Wrap(err, "failed to stop Postgres instance")
270264
}
271265

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /