Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit 28c2e6d

Browse files
committed
fix: improve performance of the dblab clone list command (#647)
1 parent 6541194 commit 28c2e6d

File tree

19 files changed

+559
-79
lines changed

19 files changed

+559
-79
lines changed

‎engine/.gitlab-ci.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
default:
22
image:
3-
name: golang:1.23
3+
name: golang:1.24
44
pull_policy: if-not-present
55

66
stages:
@@ -58,7 +58,7 @@ lint:
5858
build-binary-alpine:
5959
<<: *only_engine
6060
image:
61-
name: golang:1.23-alpine
61+
name: golang:1.24-alpine
6262
pull_policy: if-not-present
6363
stage: build-binary
6464
artifacts:

‎engine/Dockerfile.dblab-server-debug‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# How to start a container: https://postgres.ai/docs/how-to-guides/administration/engine-manage
22

33
# Compile stage
4-
FROM golang:1.23 AS build-env
4+
FROM golang:1.24 AS build-env
55

66
# Build Delve
77
RUN go install github.com/go-delve/delve/cmd/dlv@latest

‎engine/Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ help: ## Display the help message
3434
all: clean build ## Build all binary components of the project
3535

3636
install-lint: ## Install the linter to $GOPATH/bin which is expected to be in $PATH
37-
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.61.0
37+
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.64.8
3838

3939
run-lint: ## Run linters
4040
golangci-lint run

‎engine/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module gitlab.com/postgres-ai/database-lab/v3
22

3-
go 1.23.12
3+
go 1.24.7
44

55
require (
66
github.com/AlekSi/pointer v1.2.0

‎engine/go.sum‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -310,8 +310,6 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
310310
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
311311
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
312312
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
313-
github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
314-
github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
315313
github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI=
316314
github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
317315
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=

‎engine/internal/cloning/base.go‎

Lines changed: 23 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -435,8 +435,10 @@ func (c *Base) refreshCloneMetadata(w *CloneWrapper) {
435435
return
436436
}
437437

438+
c.cloneMutex.Lock()
438439
w.Clone.Metadata.CloneDiffSize = sessionState.CloneDiffSize
439440
w.Clone.Metadata.LogicalSize = sessionState.LogicalReferenced
441+
c.cloneMutex.Unlock()
440442
}
441443

442444
// UpdateClone updates clone.
@@ -527,11 +529,6 @@ func (c *Base) ResetClone(cloneID string, resetOptions types.ResetCloneRequest)
527529
log.Warn("clone has dependent snapshots", cloneID)
528530
c.cloneMutex.Lock()
529531
w.Clone.Revision++
530-
w.Clone.HasDependent = true
531-
c.cloneMutex.Unlock()
532-
} else {
533-
c.cloneMutex.Lock()
534-
w.Clone.HasDependent = false
535532
c.cloneMutex.Unlock()
536533
}
537534

@@ -630,6 +627,8 @@ func (c *Base) GetClones() []*models.Clone {
630627
clones := make([]*models.Clone, 0, c.lenClones())
631628

632629
c.cloneMutex.RLock()
630+
requestsByPool := make(map[string][]resources.SessionStateRequest)
631+
633632
for _, cloneWrapper := range c.clones {
634633
if cloneWrapper.Clone.Snapshot != nil {
635634
snapshot, err := c.getSnapshotByID(cloneWrapper.Clone.Snapshot.ID)
@@ -642,12 +641,30 @@ func (c *Base) GetClones() []*models.Clone {
642641
}
643642
}
644643

645-
c.refreshCloneMetadata(cloneWrapper)
644+
if cloneWrapper.Session != nil && cloneWrapper.Clone != nil {
645+
pool := cloneWrapper.Session.Pool
646+
requestsByPool[pool] = append(requestsByPool[pool], resources.SessionStateRequest{
647+
CloneID: cloneWrapper.Clone.ID,
648+
Branch: cloneWrapper.Clone.Branch,
649+
})
650+
}
646651

647652
clones = append(clones, cloneWrapper.Clone)
648653
}
649654
c.cloneMutex.RUnlock()
650655

656+
sessionStates, err := c.provision.GetBatchSessionState(requestsByPool)
657+
if err != nil {
658+
log.Err("failed to get batch session states: ", err)
659+
}
660+
661+
for _, clone := range clones {
662+
if state, ok := sessionStates[clone.ID]; ok {
663+
clone.Metadata.CloneDiffSize = state.CloneDiffSize
664+
clone.Metadata.LogicalSize = state.LogicalReferenced
665+
}
666+
}
667+
651668
sort.Slice(clones, func(i, j int) bool {
652669
return clones[i].CreatedAt.After(clones[j].CreatedAt.Time)
653670
})

‎engine/internal/provision/mode_local.go‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -274,10 +274,8 @@ func (p *Provisioner) ResetSession(session *resources.Session, clone *models.Clo
274274
return nil, errors.Wrap(err, "failed to stop container")
275275
}
276276

277-
if clone.Revision == branching.DefaultRevision || !clone.HasDependent {
278-
if err = fsm.DestroyClone(clone.Branch, name, clone.Revision); err != nil {
279-
return nil, errors.Wrap(err, "failed to destroy clone")
280-
}
277+
if err = fsm.DestroyClone(clone.Branch, name, clone.Revision); err != nil {
278+
return nil, errors.Wrap(err, "failed to destroy clone")
281279
}
282280

283281
if err = newFSManager.CreateClone(clone.Branch, name, snapshot.ID, clone.Revision); err != nil {
@@ -300,9 +298,14 @@ func (p *Provisioner) ResetSession(session *resources.Session, clone *models.Clo
300298
}
301299

302300
snapshotModel := &models.Snapshot{
303-
ID: snapshot.ID,
304-
CreatedAt: models.NewLocalTime(snapshot.CreatedAt),
305-
DataStateAt: models.NewLocalTime(snapshot.DataStateAt),
301+
ID: snapshot.ID,
302+
CreatedAt: models.NewLocalTime(snapshot.CreatedAt),
303+
DataStateAt: models.NewLocalTime(snapshot.DataStateAt),
304+
PhysicalSize: snapshot.Used,
305+
LogicalSize: snapshot.LogicalReferenced,
306+
Pool: snapshot.Pool,
307+
Branch: snapshot.Branch,
308+
Message: snapshot.Message,
306309
}
307310

308311
return snapshotModel, nil
@@ -335,6 +338,31 @@ func (p *Provisioner) GetSessionState(s *resources.Session, branch, cloneID stri
335338
return fsm.GetSessionState(branch, cloneID)
336339
}
337340

341+
// GetBatchSessionState retrieves session states for multiple clones efficiently.
342+
func (p *Provisioner) GetBatchSessionState(batch map[string][]resources.SessionStateRequest) (map[string]resources.SessionState, error) {
343+
batchResults := make(map[string]resources.SessionState)
344+
345+
for poolName, reqs := range batch {
346+
fsm, err := p.pm.GetFSManager(poolName)
347+
if err != nil {
348+
log.Err(fmt.Sprintf("failed to find filesystem manager for pool %s: %v", poolName, err))
349+
continue
350+
}
351+
352+
results, err := fsm.GetBatchSessionState(reqs)
353+
if err != nil {
354+
log.Err(fmt.Sprintf("failed to get batch session state for pool %s: %v", poolName, err))
355+
continue
356+
}
357+
358+
for cloneID, state := range results {
359+
batchResults[cloneID] = state
360+
}
361+
}
362+
363+
return batchResults, nil
364+
}
365+
338366
// GetPoolEntryList provides an ordered list of available pools.
339367
func (p *Provisioner) GetPoolEntryList() []models.PoolEntry {
340368
fsmList := p.pm.GetFSManagerOrderedList()
@@ -604,10 +632,8 @@ func (p *Provisioner) CleanupCloneDataset(clone *models.Clone, pool string) erro
604632
return nil
605633
}
606634

607-
if clone.Revision == branching.DefaultRevision && !clone.HasDependent {
608-
if err := fsm.DestroyDataset(branching.CloneDataset(pool, clone.Branch, clone.ID)); err != nil {
609-
return fmt.Errorf("failed to destroy clone dataset: %w", err)
610-
}
635+
if err = fsm.DestroyClone(clone.Branch, clone.ID, clone.Revision); err != nil {
636+
return fmt.Errorf("failed to destroy clone: %w", err)
611637
}
612638

613639
return nil

‎engine/internal/provision/mode_local_test.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,10 @@ func (m mockFSManager) GetSessionState(_, _ string) (*resources.SessionState, er
102102
return nil, nil
103103
}
104104

105+
func (m mockFSManager) GetBatchSessionState(_ []resources.SessionStateRequest) (map[string]resources.SessionState, error) {
106+
return make(map[string]resources.SessionState), nil
107+
}
108+
105109
func (m mockFSManager) GetFilesystemState() (models.FileSystem, error) {
106110
return models.FileSystem{Mode: "zfs"}, nil
107111
}
@@ -214,6 +218,14 @@ func (m mockFSManager) KeepRelation(_ string) error {
214218
return nil
215219
}
216220

221+
func (m mockFSManager) GetDatasetOrigins(_ string) []string {
222+
return nil
223+
}
224+
225+
func (m mockFSManager) GetActiveDatasets(_ string) ([]string, error) {
226+
return nil, nil
227+
}
228+
217229
func TestBuildPoolEntry(t *testing.T) {
218230
testCases := []struct {
219231
pool *resources.Pool

‎engine/internal/provision/pool/manager.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ type Cloner interface {
3939
// StateReporter describes methods of state reporting.
4040
type StateReporter interface {
4141
GetSessionState(branch, name string) (*resources.SessionState, error)
42+
GetBatchSessionState(requests []resources.SessionStateRequest) (map[string]resources.SessionState, error)
4243
GetFilesystemState() (models.FileSystem, error)
4344
}
4445

@@ -57,7 +58,7 @@ type Branching interface {
5758
VerifyBranchMetadata() error
5859
CreateDataset(datasetName string) error
5960
CreateBranch(branchName, snapshotID string) error
60-
DestroyDataset(branchName string) (err error)
61+
DestroyDataset(dataset string) (err error)
6162
ListBranches() (map[string]string, error)
6263
ListAllBranches(filterPools []string) ([]models.BranchEntity, error)
6364
GetRepo() (*models.Repo, error)
@@ -78,6 +79,8 @@ type Branching interface {
7879
Reset(snapshotID string, options thinclones.ResetOptions) error
7980
HasDependentEntity(snapshotName string) ([]string, error)
8081
KeepRelation(snapshotName string) error
82+
GetDatasetOrigins(snapshotName string) []string
83+
GetActiveDatasets(dataset string) ([]string, error)
8184
}
8285

8386
// Pooler describes methods for Pool providing.

‎engine/internal/provision/resources/resources.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,9 @@ type SessionState struct {
4848
CloneDiffSize uint64
4949
LogicalReferenced uint64
5050
}
51+
52+
// SessionStateRequest defines a request for batch session state retrieval.
53+
type SessionStateRequest struct {
54+
CloneID string
55+
Branch string
56+
}

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /