Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Sign up
Appearance settings

Commit e7e2023

Browse files
committed
Merge branch 'collect-source-db-checks' into 'master'
feat: collect and report all source databases checks at once See merge request postgres-ai/database-lab!798
2 parents 9472180 + bf4e058 commit e7e2023

File tree

2 files changed

+270
-72
lines changed
  • engine
    • internal/retrieval/engine/postgres/tools/db
    • pkg/models

2 files changed

+270
-72
lines changed

‎engine/internal/retrieval/engine/postgres/tools/db/pg.go

Lines changed: 232 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -122,55 +122,75 @@ func CheckSource(ctx context.Context, conf *models.ConnectionTest, imageContent
122122

123123
dbSource.TuningParams = tuningParameters
124124

125+
dbCheck, err := checkDatabases(ctx, conn, conf, imageContent)
126+
if err != nil {
127+
dbSource.Status = models.TCStatusError
128+
dbSource.Result = models.TCResultQueryError
129+
dbSource.Message = err.Error()
130+
131+
return dbSource, err
132+
}
133+
134+
dbSource.TestConnection = dbCheck
135+
136+
return dbSource, nil
137+
}
138+
139+
func checkDatabases(ctx context.Context, conn *pgx.Conn, conf *models.ConnectionTest,
140+
imageContent *ImageContent) (*models.TestConnection, error) {
141+
var tcResponse = &models.TestConnection{
142+
Status: models.TCStatusOK,
143+
Result: models.TCResultOK,
144+
Message: models.TCMessageOK,
145+
}
146+
125147
dbList := conf.DBList
126148

127149
if len(dbList) == 0 {
128150
dbSourceList, err := getDBList(ctx, conn, conf.Username)
129151
if err != nil {
130-
dbSource.Status = models.TCStatusError
131-
dbSource.Result = models.TCResultQueryError
132-
dbSource.Message = err.Error()
133-
134-
return dbSource, err
152+
return nil, err
135153
}
136154

137155
dbList = dbSourceList
138156
}
139157

158+
dbReport := make(map[string]*checkContentResp, 0)
159+
140160
if len(dbList) > maxNumberVerifiedDBs {
141161
dbList = dbList[:maxNumberVerifiedDBs]
142-
tcResponse = &models.TestConnection{
143-
Status: models.TCStatusNotice,
144-
Result: models.TCResultUnverifiedDB,
145-
Message: "Too many databases were requested to be checked. Only the following databases have been verified: " +
146-
strings.Join(dbList, ", "),
162+
dbReport[""] = &checkContentResp{
163+
status: models.TCStatusNotice,
164+
result: models.TCResultUnverifiedDB,
165+
message: "Too many databases. Only checked these databases: " +
166+
strings.Join(dbList, ", ")+". ",
147167
}
148-
dbSource.TestConnection = tcResponse
149168
}
150169

151170
for _, dbName := range dbList {
152171
dbConn, listTC := checkConnection(ctx, conf, dbName)
153172
if listTC != nil {
154-
dbSource.TestConnection = listTC
155-
return dbSource, nil
173+
dbReport[dbName] = &checkContentResp{
174+
status: listTC.Status,
175+
result: listTC.Result,
176+
message: listTC.Message,
177+
}
178+
179+
continue
156180
}
157181

158-
listTC, err := checkContent(ctx, dbConn, dbName, imageContent)
159-
if err != nil {
160-
dbSource.Status = models.TCStatusError
161-
dbSource.Result = models.TCResultQueryError
162-
dbSource.Message = err.Error()
182+
contentChecks := checkDBContent(ctx, dbConn, imageContent)
163183

164-
return dbSource, err
184+
if contentChecks != nil {
185+
dbReport[dbName] = contentChecks
165186
}
187+
}
166188

167-
if listTC != nil {
168-
dbSource.TestConnection = listTC
169-
return dbSource, nil
170-
}
189+
if len(dbReport) > 0 {
190+
tcResponse = aggregate(tcResponse, dbReport)
171191
}
172192

173-
return dbSource, nil
193+
return tcResponse, nil
174194
}
175195

176196
func getDBList(ctx context.Context, conn *pgx.Conn, dbUsername string) ([]string, error) {
@@ -193,6 +213,91 @@ func getDBList(ctx context.Context, conn *pgx.Conn, dbUsername string) ([]string
193213
return dbList, nil
194214
}
195215

216+
type aggregateState struct {
217+
general string
218+
errors map[string]string
219+
missingExt map[string][]extension
220+
unsupportedExt map[string][]extension
221+
missingLocales map[string][]locale
222+
unexploredDBs []string
223+
}
224+
225+
func newAggregateState() aggregateState {
226+
return aggregateState{
227+
general: "",
228+
errors: make(map[string]string, 0),
229+
missingExt: make(map[string][]extension, 0),
230+
unsupportedExt: make(map[string][]extension, 0),
231+
missingLocales: make(map[string][]locale, 0),
232+
unexploredDBs: make([]string, 0),
233+
}
234+
}
235+
236+
func aggregate(tcResponse *models.TestConnection, collection map[string]*checkContentResp) *models.TestConnection {
237+
agg := newAggregateState()
238+
sb := strings.Builder{}
239+
240+
for dbName, contentResponse := range collection {
241+
if contentResponse.status > tcResponse.Status {
242+
tcResponse.Status = contentResponse.status
243+
tcResponse.Result = contentResponse.result
244+
}
245+
246+
switch contentResponse.result {
247+
case models.TCResultUnverifiedDB:
248+
agg.general += contentResponse.message
249+
250+
case models.TCResultQueryError, models.TCResultConnectionError:
251+
agg.errors[dbName] = contentResponse.message
252+
253+
case models.TCResultMissingExtension:
254+
if len(contentResponse.missingExt) > 0 {
255+
agg.missingExt[dbName] = append(agg.missingExt[dbName], contentResponse.missingExt...)
256+
}
257+
258+
if len(contentResponse.unsupportedExt) > 0 {
259+
agg.unsupportedExt[dbName] = append(agg.unsupportedExt[dbName], contentResponse.unsupportedExt...)
260+
}
261+
262+
case models.TCResultMissingLocale:
263+
agg.missingLocales[dbName] = append(agg.missingLocales[dbName], contentResponse.missingLocales...)
264+
265+
case models.TCResultUnexploredImage:
266+
agg.unexploredDBs = append(agg.unexploredDBs, dbName)
267+
268+
case models.TCResultOK:
269+
default:
270+
}
271+
}
272+
273+
sb.WriteString(agg.general)
274+
sb.WriteString(buildErrorMessage(agg.errors))
275+
sb.WriteString(buildExtensionsWarningMessage(agg.missingExt, agg.unsupportedExt))
276+
sb.WriteString(buildLocalesWarningMessage(agg.missingLocales))
277+
sb.WriteString(unexploredDBsNoticeMessage(agg.unexploredDBs))
278+
279+
tcResponse.Message = sb.String()
280+
281+
return tcResponse
282+
}
283+
284+
func buildErrorMessage(errors map[string]string) string {
285+
if len(errors) == 0 {
286+
return ""
287+
}
288+
289+
sb := strings.Builder{}
290+
sb.WriteString("Issues detected in databases:\n")
291+
292+
for dbName, message := range errors {
293+
sb.WriteString(fmt.Sprintf(" %q - %s;\n", dbName, message))
294+
}
295+
296+
sb.WriteString(" \n")
297+
298+
return sb.String()
299+
}
300+
196301
func checkConnection(ctx context.Context, conf *models.ConnectionTest, dbName string) (*pgx.Conn, *models.TestConnection) {
197302
connStr := ConnectionString(conf.Host, conf.Port, conf.Username, dbName, conf.Password)
198303

@@ -220,41 +325,59 @@ func checkConnection(ctx context.Context, conf *models.ConnectionTest, dbName st
220325
return conn, nil
221326
}
222327

223-
func checkContent(ctx context.Context, conn *pgx.Conn, dbName string, imageContent *ImageContent) (*models.TestConnection, error) {
328+
type checkContentResp struct {
329+
status models.StatusType
330+
result string
331+
message string
332+
missingExt []extension
333+
unsupportedExt []extension
334+
missingLocales []locale
335+
}
336+
337+
func checkDBContent(ctx context.Context, conn *pgx.Conn, imageContent *ImageContent) *checkContentResp {
224338
if !imageContent.IsReady() {
225-
return &models.TestConnection{
226-
Status: models.TCStatusNotice,
227-
Result: models.TCResultUnexploredImage,
228-
Message: "The connection to the database was successful. " +
229-
"Details about the extensions and locales of the Docker image have not yet been collected. Please try again later",
230-
}, nil
339+
return &checkContentResp{
340+
status: models.TCStatusNotice,
341+
result: models.TCResultUnexploredImage,
342+
message: "Connected to database. " +
343+
"Docker image extensions and locales not yet analyzed. Retry later. ",
344+
}
231345
}
232346

233347
if missing, unsupported, err := checkExtensions(ctx, conn, imageContent.Extensions()); err != nil {
234348
if err != errExtensionWarning {
235-
return nil, fmt.Errorf("failed to check database extensions: %w", err)
349+
return &checkContentResp{
350+
status: models.TCStatusError,
351+
result: models.TCResultQueryError,
352+
message: fmt.Sprintf("failed to check database extensions: %s", err),
353+
}
236354
}
237355

238-
return &models.TestConnection{
239-
Status: models.TCStatusWarning,
240-
Result: models.TCResultMissingExtension,
241-
Message: buildExtensionsWarningMessage(dbName, missing, unsupported),
242-
}, nil
356+
return &checkContentResp{
357+
status: models.TCStatusWarning,
358+
result: models.TCResultMissingExtension,
359+
missingExt: missing,
360+
unsupportedExt: unsupported,
361+
}
243362
}
244363

245364
if missing, err := checkLocales(ctx, conn, imageContent.Locales(), imageContent.Databases()); err != nil {
246365
if err != errLocaleWarning {
247-
return nil, fmt.Errorf("failed to check database locales: %w", err)
366+
return &checkContentResp{
367+
status: models.TCStatusError,
368+
result: models.TCResultQueryError,
369+
message: fmt.Sprintf("failed to check database locales: %s", err),
370+
}
248371
}
249372

250-
return &models.TestConnection{
251-
Status: models.TCStatusWarning,
252-
Result: models.TCResultMissingLocale,
253-
Message: buildLocalesWarningMessage(dbName, missing),
254-
}, nil
373+
return &checkContentResp{
374+
status: models.TCStatusWarning,
375+
result: models.TCResultMissingLocale,
376+
missingLocales: missing,
377+
}
255378
}
256379

257-
return nil, nil
380+
return nil
258381
}
259382

260383
func checkExtensions(ctx context.Context, conn *pgx.Conn, imageExtensions map[string]string) ([]extension, []extension, error) {
@@ -311,38 +434,64 @@ func toCanonicalSemver(v string) string {
311434
return v
312435
}
313436

314-
func buildExtensionsWarningMessage(dbNamestring, missingExtensions, unsupportedVersions []extension) string {
437+
func buildExtensionsWarningMessage(missingExtensions, unsupportedVersions map[string][]extension) string {
315438
sb := &strings.Builder{}
316439

317440
if len(missingExtensions) > 0 {
318-
sb.WriteString("The image specified in section \"databaseContainer\" lacks the following " +
319-
"extensions used in the source database (\""+dbName+"\"):")
441+
sb.WriteString("Image configured in \"databaseContainer\" missing " +
442+
"extensions installed in source databases: ")
320443

321444
formatExtensionList(sb, missingExtensions)
322-
323-
sb.WriteString(".\n")
324445
}
325446

326447
if len(unsupportedVersions) > 0 {
327-
sb.WriteString("The source database (\""+dbName+"\") uses extensions that are present " +
328-
"in image specified in section \"databaseContainer\" but their versions are not supported by the image:")
448+
sb.WriteString("Source databases have extensions with different versions " +
449+
"than image configured in \"databaseContainer\":")
329450

330451
formatExtensionList(sb, unsupportedVersions)
331452
}
332453

333454
return sb.String()
334455
}
335456

336-
func formatExtensionList(sb *strings.Builder, extensions []extension) {
337-
length := len(extensions)
457+
func formatExtensionList(sb *strings.Builder, extensionMap map[string][]extension) {
458+
var j int
459+
460+
lengthDBs := len(extensionMap)
461+
462+
for dbName, extensions := range extensionMap {
463+
lengthExt := len(extensions)
464+
465+
sb.WriteString(" " + dbName + " (")
466+
467+
for i, missing := range extensions {
468+
sb.WriteString(missing.name + " " + missing.defaultVersion)
469+
470+
if i != lengthExt-1 {
471+
sb.WriteString(", ")
472+
}
473+
}
338474

339-
for i, missing := range extensions {
340-
sb.WriteString(" " + missing.name + " " + missing.defaultVersion)
475+
sb.WriteString(")")
341476

342-
if i != length-1 {
343-
sb.WriteRune(',')
477+
if j != lengthDBs-1 {
478+
sb.WriteRune(';')
344479
}
480+
481+
j++
345482
}
483+
484+
sb.WriteString(". \n")
485+
}
486+
487+
func unexploredDBsNoticeMessage(dbs []string) string {
488+
if len(dbs) == 0 {
489+
return ""
490+
}
491+
492+
return fmt.Sprintf("Connected to databases: %s. "+
493+
"Docker image extensions and locales not analyzed. Retry later.\n",
494+
strings.Join(dbs, ","))
346495
}
347496

348497
func checkLocales(ctx context.Context, conn *pgx.Conn, imageLocales, databases map[string]struct{}) ([]locale, error) {
@@ -386,20 +535,38 @@ func checkLocales(ctx context.Context, conn *pgx.Conn, imageLocales, databases m
386535
return nil, nil
387536
}
388537

389-
func buildLocalesWarningMessage(dbName string, missingLocales []locale) string {
538+
func buildLocalesWarningMessage(localeMap map[string][]locale) string {
539+
var j int
540+
390541
sb := &strings.Builder{}
391542

392-
if length := len(missingLocales); length > 0 {
393-
sb.WriteString("The image specified in section \"databaseContainer\" lacks the following " +
394-
"locales used in the source database (\"" + dbName + "\"):")
543+
if lengthDBs := len(localeMap); lengthDBs > 0 {
544+
sb.WriteString("Image configured in \"databaseContainer\" missing " +
545+
"locales from source databases: ")
546+
547+
for dbName, missingLocales := range localeMap {
548+
lengthLoc := len(missingLocales)
549+
550+
sb.WriteString(" " + dbName + " (")
395551

396-
for i, missing := range missingLocales {
397-
sb.WriteString(fmt.Sprintf(" '%s' (collate: %s, ctype: %s)", missing.name, missing.collate, missing.ctype))
552+
for i, missing := range missingLocales {
553+
sb.WriteString(fmt.Sprintf(" '%s' (collate: %s, ctype: %s)", missing.name, missing.collate, missing.ctype))
398554

399-
if i != length-1 {
400-
sb.WriteRune(',')
555+
if i != lengthLoc-1 {
556+
sb.WriteRune(',')
557+
}
401558
}
559+
560+
sb.WriteString(")")
561+
562+
if j != lengthDBs-1 {
563+
sb.WriteRune(';')
564+
}
565+
566+
j++
402567
}
568+
569+
sb.WriteString(". \n")
403570
}
404571

405572
return sb.String()

0 commit comments

Comments
(0)

AltStyle によって変換されたページ (->オリジナル) /