From 197145acf4245b785ddff2131c7e3d0f22dc7499 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: Thu, 3 Dec 2020 15:15:57 +0500 Subject: [PATCH 01/20] Prevent distributed deadlocks. Disable AQO for FDW queries. Tags: shardman. --- preprocessing.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/preprocessing.c b/preprocessing.c index 79097a92..bb81d31e 100644 --- a/preprocessing.c +++ b/preprocessing.c @@ -134,6 +134,8 @@ aqo_planner(Query *parse, */ if ((parse->commandType != CMD_SELECT && parse->commandType != CMD_INSERT && parse->commandType != CMD_UPDATE && parse->commandType != CMD_DELETE) || + strstr(application_name, "postgres_fdw") != NULL || /* Prevent distributed deadlocks */ + strstr(application_name, "pgfdw:") != NULL || /* caused by fdw */ get_extension_oid("aqo", true) == InvalidOid || creating_extension || IsParallelWorker() || From 626d5ee314b8577cb7005ff9766c775cfff9af01 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: Fri, 4 Dec 2020 08:42:56 +0500 Subject: [PATCH 02/20] Allow learning (and predicting for) on a ForeignScan, an Append, a MergeAppend, a SubqueryScan nodes. Tags: shardman. --- path_utils.c | 28 +++++++++++++++++++++++++++- postprocessing.c | 4 +++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/path_utils.c b/path_utils.c index 6e809818..3a8a2a12 100644 --- a/path_utils.c +++ b/path_utils.c @@ -79,7 +79,7 @@ get_path_clauses(Path *path, PlannerInfo *root, List **selectivities) List *inner_sel = NIL; List *outer; List *outer_sel = NIL; - List *cur; + List *cur = NIL; List *cur_sel = NIL; Assert(selectivities != NULL); @@ -160,6 +160,32 @@ get_path_clauses(Path *path, PlannerInfo *root, List **selectivities) return get_path_clauses(((LimitPath *) path)->subpath, root, selectivities); break; + case T_SubqueryScanPath: + return get_path_clauses(((SubqueryScanPath *) path)->subpath, root, + selectivities); + break; + case T_AppendPath: + { + ListCell *lc; + + foreach (lc, ((AppendPath *) path)->subpaths) + { + Path *subpath = lfirst(lc); + + cur = list_concat(cur, list_copy( + get_path_clauses(subpath, root, selectivities))); + cur_sel = list_concat(cur_sel, *selectivities); + } + cur = list_concat(cur, list_copy(path->parent->baserestrictinfo)); + *selectivities = list_concat(cur_sel, + get_selectivities(root, + path->parent->baserestrictinfo, + 0, JOIN_INNER, NULL)); + return cur; + } + break; + case T_ForeignPath: + /* The same as in the default case */ default: cur = list_concat(list_copy(path->parent->baserestrictinfo), path->param_info ? diff --git a/postprocessing.c b/postprocessing.c index db38b2d4..2ca4d6a3 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -252,7 +252,9 @@ learnOnPlanState(PlanState *p, void *context) ctx->relidslist = list_copy(p->plan->path_relids); if (p->instrument && (p->righttree != NULL || p->lefttree == NULL || - p->plan->path_clauses != NIL)) + p->plan->path_clauses != NIL || + IsA(p, ForeignScanState) || + IsA(p, AppendState) || IsA(p, MergeAppendState))) { double learn_rows = 0.; double predicted = 0.; From 386cd6d5ea5c05406c7cab6223385a6420694c43 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: Fri, 4 Dec 2020 17:28:43 +0500 Subject: [PATCH 03/20] Add support of postgres_fdw push-down. Tags: shardman. --- Makefile | 2 ++ postprocessing.c | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 30d5967a..1b1575c0 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,8 @@ REGRESS = aqo_disabled \ schema \ aqo_CVE-2020-14350 +fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw +PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir) EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql diff --git a/postprocessing.c b/postprocessing.c index 2ca4d6a3..8b385a75 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -19,6 +19,7 @@ #include "aqo.h" #include "access/parallel.h" #include "optimizer/optimizer.h" +#include "postgres_fdw.h" #include "utils/queryenvironment.h" typedef struct @@ -544,7 +545,7 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) void aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src) { - bool is_join_path; + bool is_join_path; if (prev_copy_generic_path_info_hook) prev_copy_generic_path_info_hook(root, dest, src); @@ -569,6 +570,23 @@ aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src) dest->path_clauses = ((JoinPath *) src)->joinrestrictinfo; dest->path_jointype = ((JoinPath *) src)->jointype; } + else if (src->type == T_ForeignPath) + { + ForeignPath *fpath = (ForeignPath *) src; + PgFdwRelationInfo *fpinfo = (PgFdwRelationInfo *) fpath->path.parent->fdw_private; + + /* + * Pushed down foreign join keeps clauses in special fdw_private + * structure. + * I'm not sure what fpinfo structure keeps clauses for sufficient time. + * So, copy clauses. + */ + dest->path_clauses = list_concat(list_copy(fpinfo->joinclauses), + list_copy(fpinfo->remote_conds)); + dest->path_clauses = list_concat(dest->path_clauses, + list_copy(fpinfo->local_conds)); + dest->path_jointype = ((JoinPath *) src)->jointype; + } else { dest->path_clauses = list_concat( From 0cd3ce3407a4817635d22bc3bdb58df7fc1ad4a4 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2020年12月15日 10:37:12 +0500 Subject: [PATCH 04/20] Process GatherMergePath in get_path_clauses routine. Replace find_ok assert with elog panic message. --- path_utils.c | 1 + storage.c | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/path_utils.c b/path_utils.c index 3a8a2a12..59edc35c 100644 --- a/path_utils.c +++ b/path_utils.c @@ -113,6 +113,7 @@ get_path_clauses(Path *path, PlannerInfo *root, List **selectivities) selectivities); break; case T_GatherPath: + case T_GatherMergePath: return get_path_clauses(((GatherPath *) path)->subpath, root, selectivities); break; diff --git a/storage.c b/storage.c index 936a37c2..cd733a97 100644 --- a/storage.c +++ b/storage.c @@ -234,7 +234,9 @@ update_query(int query_hash, bool learn_aqo, bool use_aqo, slot = MakeSingleTupleTableSlot(query_index_scan->heapRelation->rd_att, &TTSOpsBufferHeapTuple); find_ok = index_getnext_slot(query_index_scan, ForwardScanDirection, slot); - Assert(find_ok); + if (!find_ok) + elog(PANIC, "Query isn't found in AQO learning database."); + tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); From e8dcb5bfe0e958ad63394830125a3b607090a3af Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: 2021年1月15日 11:35:16 +0500 Subject: [PATCH 05/20] Add a foreign relation estimation hook into the core patch and AQO. Improve the elog panic message introduced in previous commit. Fix ForeignScan estimation logic. --- aqo.c | 2 + aqo.h | 2 + aqo_pg13.patch | 176 +++++++++++++++++++++++++++-------------------- postprocessing.c | 27 +++++++- storage.c | 3 +- 5 files changed, 132 insertions(+), 78 deletions(-) diff --git a/aqo.c b/aqo.c index 4f0eac87..95ef4c12 100644 --- a/aqo.c +++ b/aqo.c @@ -76,6 +76,7 @@ post_parse_analyze_hook_type prev_post_parse_analyze_hook; planner_hook_type prev_planner_hook; ExecutorStart_hook_type prev_ExecutorStart_hook; ExecutorEnd_hook_type prev_ExecutorEnd_hook; +set_baserel_rows_estimate_hook_type prev_set_foreign_rows_estimate_hook; set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook; get_parameterized_baserel_size_hook_type prev_get_parameterized_baserel_size_hook; set_joinrel_size_estimates_hook_type prev_set_joinrel_size_estimates_hook; @@ -126,6 +127,7 @@ _PG_init(void) prev_ExecutorEnd_hook = ExecutorEnd_hook; ExecutorEnd_hook = aqo_ExecutorEnd; prev_set_baserel_rows_estimate_hook = set_baserel_rows_estimate_hook; + set_foreign_rows_estimate_hook = aqo_set_baserel_rows_estimate; set_baserel_rows_estimate_hook = aqo_set_baserel_rows_estimate; prev_get_parameterized_baserel_size_hook = get_parameterized_baserel_size_hook; get_parameterized_baserel_size_hook = aqo_get_parameterized_baserel_size; diff --git a/aqo.h b/aqo.h index 080d076b..b6e934f1 100644 --- a/aqo.h +++ b/aqo.h @@ -253,6 +253,8 @@ extern post_parse_analyze_hook_type prev_post_parse_analyze_hook; extern planner_hook_type prev_planner_hook; extern ExecutorStart_hook_type prev_ExecutorStart_hook; extern ExecutorEnd_hook_type prev_ExecutorEnd_hook; +extern set_baserel_rows_estimate_hook_type + prev_set_foreign_rows_estimate_hook; extern set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook; extern get_parameterized_baserel_size_hook_type diff --git a/aqo_pg13.patch b/aqo_pg13.patch index b933ca49..6925b773 100644 --- a/aqo_pg13.patch +++ b/aqo_pg13.patch @@ -11,7 +11,7 @@ index 7a4866e338..47a18b9698 100644 auto_explain \ bloom \ diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c -index 43f9b01e83..707211308c 100644 +index 5d7eb3574c..87402b6859 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -24,6 +24,7 @@ @@ -32,7 +32,7 @@ index 43f9b01e83..707211308c 100644 /* OR-able flags for ExplainXMLTag() */ #define X_OPENING 0 -@@ -638,6 +642,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, +@@ -654,6 +658,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, ExplainPropertyFloat("Execution Time", "ms", 1000.0 * totaltime, 3, es); @@ -43,7 +43,7 @@ index 43f9b01e83..707211308c 100644 ExplainCloseGroup("Query", NULL, true, es); } -@@ -1579,6 +1587,38 @@ ExplainNode(PlanState *planstate, List *ancestors, +@@ -1595,6 +1603,38 @@ ExplainNode(PlanState *planstate, List *ancestors, appendStringInfo(es->str, " (actual rows=%.0f loops=%.0f)", rows, nloops); @@ -83,7 +83,7 @@ index 43f9b01e83..707211308c 100644 else { diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c -index 530aac68a7..1d94feadb9 100644 +index ba3ccc712c..74a090e6f9 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c @@ -126,6 +126,12 @@ CopyPlanFields(const Plan *from, Plan *newnode) @@ -100,21 +100,22 @@ index 530aac68a7..1d94feadb9 100644 COPY_BITMAPSET_FIELD(allParam); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c -index f1dfdc1a4a..359cafa531 100644 +index 380336518f..ecf0c45629 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c -@@ -97,6 +97,10 @@ +@@ -97,6 +97,11 @@ #include "utils/spccache.h" #include "utils/tuplesort.h" +set_baserel_rows_estimate_hook_type set_baserel_rows_estimate_hook = NULL; ++set_foreign_rows_estimate_hook_type set_foreign_rows_estimate_hook = NULL; +get_parameterized_baserel_size_hook_type get_parameterized_baserel_size_hook = NULL; +get_parameterized_joinrel_size_hook_type get_parameterized_joinrel_size_hook = NULL; +set_joinrel_size_estimates_hook_type set_joinrel_size_estimates_hook = NULL; #define LOG2(x) (log(x) / 0.693147180559945) -@@ -185,7 +189,6 @@ static Cost append_nonpartial_cost(List *subpaths, int numpaths, +@@ -185,7 +190,6 @@ static Cost append_nonpartial_cost(List *subpaths, int numpaths, static void set_rel_width(PlannerInfo *root, RelOptInfo *rel); static double relation_byte_size(double tuples, int width); static double page_size(double tuples, int width); @@ -122,7 +123,7 @@ index f1dfdc1a4a..359cafa531 100644 /* -@@ -266,7 +269,7 @@ cost_seqscan(Path *path, PlannerInfo *root, +@@ -266,7 +270,7 @@ cost_seqscan(Path *path, PlannerInfo *root, /* Adjust costing for parallelism, if used. */ if (path->parallel_workers> 0) { @@ -131,7 +132,7 @@ index f1dfdc1a4a..359cafa531 100644 /* The CPU cost is divided among all the workers. */ cpu_run_cost /= parallel_divisor; -@@ -745,7 +748,7 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, +@@ -745,7 +749,7 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, /* Adjust costing for parallelism, if used. */ if (path->path.parallel_workers> 0) { @@ -140,7 +141,7 @@ index f1dfdc1a4a..359cafa531 100644 path->path.rows = clamp_row_est(path->path.rows / parallel_divisor); -@@ -1026,7 +1029,7 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, +@@ -1026,7 +1030,7 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, /* Adjust costing for parallelism, if used. */ if (path->parallel_workers> 0) { @@ -149,7 +150,7 @@ index f1dfdc1a4a..359cafa531 100644 /* The CPU cost is divided among all the workers. */ cpu_run_cost /= parallel_divisor; -@@ -2129,7 +2132,7 @@ cost_append(AppendPath *apath) +@@ -2129,7 +2133,7 @@ cost_append(AppendPath *apath) else /* parallel-aware */ { int i = 0; @@ -158,7 +159,7 @@ index f1dfdc1a4a..359cafa531 100644 /* Parallel-aware Append never produces ordered output. */ Assert(apath->path.pathkeys == NIL); -@@ -2163,7 +2166,7 @@ cost_append(AppendPath *apath) +@@ -2163,7 +2167,7 @@ cost_append(AppendPath *apath) { double subpath_parallel_divisor; @@ -167,7 +168,7 @@ index f1dfdc1a4a..359cafa531 100644 apath->path.rows += subpath->rows * (subpath_parallel_divisor / parallel_divisor); apath->path.total_cost += subpath->total_cost; -@@ -2761,7 +2764,7 @@ final_cost_nestloop(PlannerInfo *root, NestPath *path, +@@ -2762,7 +2766,7 @@ final_cost_nestloop(PlannerInfo *root, NestPath *path, /* For partial paths, scale row estimate. */ if (path->path.parallel_workers> 0) { @@ -176,7 +177,7 @@ index f1dfdc1a4a..359cafa531 100644 path->path.rows = clamp_row_est(path->path.rows / parallel_divisor); -@@ -3207,7 +3210,7 @@ final_cost_mergejoin(PlannerInfo *root, MergePath *path, +@@ -3208,7 +3212,7 @@ final_cost_mergejoin(PlannerInfo *root, MergePath *path, /* For partial paths, scale row estimate. */ if (path->jpath.path.parallel_workers> 0) { @@ -185,7 +186,7 @@ index f1dfdc1a4a..359cafa531 100644 path->jpath.path.rows = clamp_row_est(path->jpath.path.rows / parallel_divisor); -@@ -3541,7 +3544,7 @@ initial_cost_hashjoin(PlannerInfo *root, JoinCostWorkspace *workspace, +@@ -3542,7 +3546,7 @@ initial_cost_hashjoin(PlannerInfo *root, JoinCostWorkspace *workspace, * number, so we need to undo the division. */ if (parallel_hash) @@ -194,7 +195,7 @@ index f1dfdc1a4a..359cafa531 100644 /* * Get hash table size that executor would use for inner relation. -@@ -3638,7 +3641,7 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path, +@@ -3639,7 +3643,7 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path, /* For partial paths, scale row estimate. */ if (path->jpath.path.parallel_workers> 0) { @@ -203,10 +204,19 @@ index f1dfdc1a4a..359cafa531 100644 path->jpath.path.rows = clamp_row_est(path->jpath.path.rows / parallel_divisor); -@@ -4633,6 +4636,49 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) +@@ -4634,6 +4638,58 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) } ++void ++set_foreign_rows_estimate(PlannerInfo *root, RelOptInfo *rel) ++{ ++ if (set_foreign_rows_estimate_hook) ++ (*set_foreign_rows_estimate_hook) (root, rel); ++ else ++ rel->rows = 1000; /* entirely bogus default estimate */ ++} ++ +/* + * set_baserel_rows_estimate + * Set the rows estimate for the given base relation. @@ -253,7 +263,7 @@ index f1dfdc1a4a..359cafa531 100644 /* * set_baserel_size_estimates * Set the size estimates for the given base relation. -@@ -4649,19 +4695,10 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) +@@ -4650,19 +4706,10 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) void set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) { @@ -274,7 +284,7 @@ index f1dfdc1a4a..359cafa531 100644 cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); -@@ -4672,13 +4709,33 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) +@@ -4673,13 +4720,33 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) * get_parameterized_baserel_size * Make a size estimate for a parameterized scan of a base relation. * @@ -310,7 +320,7 @@ index f1dfdc1a4a..359cafa531 100644 { List *allclauses; double nrows; -@@ -4707,6 +4764,36 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, +@@ -4708,6 +4775,36 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, * set_joinrel_size_estimates * Set the size estimates for the given join relation. * @@ -347,7 +357,7 @@ index f1dfdc1a4a..359cafa531 100644 * The rel's targetlist must have been constructed already, and a * restriction clause list that matches the given component rels must * be provided. -@@ -4726,11 +4813,11 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, +@@ -4727,11 +4824,11 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, * build_joinrel_tlist, and baserestrictcost is not used for join rels. */ void @@ -364,7 +374,7 @@ index f1dfdc1a4a..359cafa531 100644 { rel->rows = calc_joinrel_size_estimate(root, rel, -@@ -4746,6 +4833,35 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, +@@ -4747,6 +4844,35 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, * get_parameterized_joinrel_size * Make a size estimate for a parameterized scan of a join relation. * @@ -400,7 +410,7 @@ index f1dfdc1a4a..359cafa531 100644 * 'rel' is the joinrel under consideration. * 'outer_path', 'inner_path' are (probably also parameterized) Paths that * produce the relations being joined. -@@ -4758,11 +4874,11 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, +@@ -4759,11 +4885,11 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, * set_joinrel_size_estimates must have been applied already. */ double @@ -417,7 +427,16 @@ index f1dfdc1a4a..359cafa531 100644 { double nrows; -@@ -5760,14 +5876,25 @@ page_size(double tuples, int width) +@@ -5479,7 +5605,7 @@ set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel) + /* Should only be applied to base relations */ + Assert(rel->relid> 0); + +- rel->rows = 1000; /* entirely bogus default estimate */ ++ set_foreign_rows_estimate(root, rel); + + cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); + +@@ -5761,14 +5887,25 @@ page_size(double tuples, int width) return ceil(relation_byte_size(tuples, width) / BLCKSZ); } @@ -446,7 +465,7 @@ index f1dfdc1a4a..359cafa531 100644 /* * Early experience with parallel query suggests that when there is only -@@ -5784,7 +5911,7 @@ get_parallel_divisor(Path *path) +@@ -5785,7 +5922,7 @@ get_parallel_divisor(Path *path) { double leader_contribution; @@ -456,7 +475,7 @@ index f1dfdc1a4a..359cafa531 100644 parallel_divisor += leader_contribution; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c -index 40abe6f9f6..9edd6daeff 100644 +index 25d4750ca6..d0ea7bd2ff 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -70,6 +70,8 @@ @@ -558,7 +577,7 @@ index 40abe6f9f6..9edd6daeff 100644 /* Assign the rescan Param. */ gm_plan->rescan_param = assign_special_exec_param(root); -@@ -1901,7 +1903,7 @@ create_projection_plan(PlannerInfo *root, ProjectionPath *best_path, int flags) +@@ -1903,7 +1905,7 @@ create_projection_plan(PlannerInfo *root, ProjectionPath *best_path, int flags) /* We need a Result node */ plan = (Plan *) make_result(tlist, NULL, subplan); @@ -567,7 +586,7 @@ index 40abe6f9f6..9edd6daeff 100644 } return plan; -@@ -2002,7 +2004,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) +@@ -2004,7 +2006,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -576,7 +595,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2028,7 +2030,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, +@@ -2030,7 +2032,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, best_path->spath.path.parent->relids : NULL, best_path->nPresortedCols); @@ -585,7 +604,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2067,7 +2069,7 @@ create_group_plan(PlannerInfo *root, GroupPath *best_path) +@@ -2069,7 +2071,7 @@ create_group_plan(PlannerInfo *root, GroupPath *best_path) subplan->targetlist), subplan); @@ -594,7 +613,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2095,7 +2097,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag +@@ -2097,7 +2099,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag best_path->path.pathkeys, best_path->numkeys); @@ -603,7 +622,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2139,7 +2141,7 @@ create_agg_plan(PlannerInfo *root, AggPath *best_path) +@@ -2141,7 +2143,7 @@ create_agg_plan(PlannerInfo *root, AggPath *best_path) best_path->transitionSpace, subplan); @@ -612,7 +631,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2341,7 +2343,7 @@ create_groupingsets_plan(PlannerInfo *root, GroupingSetsPath *best_path) +@@ -2343,7 +2345,7 @@ create_groupingsets_plan(PlannerInfo *root, GroupingSetsPath *best_path) subplan); /* Copy cost data from Path to Plan */ @@ -621,7 +640,7 @@ index 40abe6f9f6..9edd6daeff 100644 } return (Plan *) plan; -@@ -2399,7 +2401,7 @@ create_minmaxagg_plan(PlannerInfo *root, MinMaxAggPath *best_path) +@@ -2401,7 +2403,7 @@ create_minmaxagg_plan(PlannerInfo *root, MinMaxAggPath *best_path) plan = make_result(tlist, (Node *) best_path->quals, NULL); @@ -630,7 +649,7 @@ index 40abe6f9f6..9edd6daeff 100644 /* * During setrefs.c, we'll need to replace references to the Agg nodes -@@ -2518,7 +2520,7 @@ create_windowagg_plan(PlannerInfo *root, WindowAggPath *best_path) +@@ -2520,7 +2522,7 @@ create_windowagg_plan(PlannerInfo *root, WindowAggPath *best_path) wc->inRangeNullsFirst, subplan); @@ -639,7 +658,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2554,7 +2556,7 @@ create_setop_plan(PlannerInfo *root, SetOpPath *best_path, int flags) +@@ -2556,7 +2558,7 @@ create_setop_plan(PlannerInfo *root, SetOpPath *best_path, int flags) best_path->firstFlag, numGroups); @@ -648,7 +667,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2590,7 +2592,7 @@ create_recursiveunion_plan(PlannerInfo *root, RecursiveUnionPath *best_path) +@@ -2592,7 +2594,7 @@ create_recursiveunion_plan(PlannerInfo *root, RecursiveUnionPath *best_path) best_path->distinctList, numGroups); @@ -657,7 +676,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2613,7 +2615,7 @@ create_lockrows_plan(PlannerInfo *root, LockRowsPath *best_path, +@@ -2615,7 +2617,7 @@ create_lockrows_plan(PlannerInfo *root, LockRowsPath *best_path, plan = make_lockrows(subplan, best_path->rowMarks, best_path->epqParam); @@ -666,7 +685,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2674,7 +2676,7 @@ create_modifytable_plan(PlannerInfo *root, ModifyTablePath *best_path) +@@ -2676,7 +2678,7 @@ create_modifytable_plan(PlannerInfo *root, ModifyTablePath *best_path) best_path->onconflict, best_path->epqParam); @@ -675,7 +694,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2728,7 +2730,7 @@ create_limit_plan(PlannerInfo *root, LimitPath *best_path, int flags) +@@ -2730,7 +2732,7 @@ create_limit_plan(PlannerInfo *root, LimitPath *best_path, int flags) best_path->limitOption, numUniqkeys, uniqColIdx, uniqOperators, uniqCollations); @@ -684,7 +703,7 @@ index 40abe6f9f6..9edd6daeff 100644 return plan; } -@@ -2774,7 +2776,7 @@ create_seqscan_plan(PlannerInfo *root, Path *best_path, +@@ -2776,7 +2778,7 @@ create_seqscan_plan(PlannerInfo *root, Path *best_path, scan_clauses, scan_relid); @@ -693,7 +712,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -2820,7 +2822,7 @@ create_samplescan_plan(PlannerInfo *root, Path *best_path, +@@ -2822,7 +2824,7 @@ create_samplescan_plan(PlannerInfo *root, Path *best_path, scan_relid, tsc); @@ -702,7 +721,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -2998,7 +3000,7 @@ create_indexscan_plan(PlannerInfo *root, +@@ -3000,7 +3002,7 @@ create_indexscan_plan(PlannerInfo *root, indexorderbyops, best_path->indexscandir); @@ -711,7 +730,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3113,7 +3115,7 @@ create_bitmap_scan_plan(PlannerInfo *root, +@@ -3115,7 +3117,7 @@ create_bitmap_scan_plan(PlannerInfo *root, bitmapqualorig, baserelid); @@ -720,7 +739,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3433,7 +3435,7 @@ create_tidscan_plan(PlannerInfo *root, TidPath *best_path, +@@ -3435,7 +3437,7 @@ create_tidscan_plan(PlannerInfo *root, TidPath *best_path, scan_relid, tidquals); @@ -729,7 +748,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3483,7 +3485,7 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, +@@ -3485,7 +3487,7 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, scan_relid, subplan); @@ -738,7 +757,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3526,7 +3528,7 @@ create_functionscan_plan(PlannerInfo *root, Path *best_path, +@@ -3528,7 +3530,7 @@ create_functionscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_functionscan(tlist, scan_clauses, scan_relid, functions, rte->funcordinality); @@ -747,7 +766,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3569,7 +3571,7 @@ create_tablefuncscan_plan(PlannerInfo *root, Path *best_path, +@@ -3571,7 +3573,7 @@ create_tablefuncscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_tablefuncscan(tlist, scan_clauses, scan_relid, tablefunc); @@ -756,7 +775,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3613,7 +3615,7 @@ create_valuesscan_plan(PlannerInfo *root, Path *best_path, +@@ -3615,7 +3617,7 @@ create_valuesscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_valuesscan(tlist, scan_clauses, scan_relid, values_lists); @@ -765,7 +784,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3706,7 +3708,7 @@ create_ctescan_plan(PlannerInfo *root, Path *best_path, +@@ -3708,7 +3710,7 @@ create_ctescan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_ctescan(tlist, scan_clauses, scan_relid, plan_id, cte_param_id); @@ -774,7 +793,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3745,7 +3747,7 @@ create_namedtuplestorescan_plan(PlannerInfo *root, Path *best_path, +@@ -3747,7 +3749,7 @@ create_namedtuplestorescan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_namedtuplestorescan(tlist, scan_clauses, scan_relid, rte->enrname); @@ -783,7 +802,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3783,7 +3785,7 @@ create_resultscan_plan(PlannerInfo *root, Path *best_path, +@@ -3785,7 +3787,7 @@ create_resultscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_result(tlist, (Node *) scan_clauses, NULL); @@ -792,7 +811,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3843,7 +3845,7 @@ create_worktablescan_plan(PlannerInfo *root, Path *best_path, +@@ -3845,7 +3847,7 @@ create_worktablescan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_worktablescan(tlist, scan_clauses, scan_relid, cteroot->wt_param_id); @@ -801,7 +820,7 @@ index 40abe6f9f6..9edd6daeff 100644 return scan_plan; } -@@ -3903,7 +3905,7 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, +@@ -3905,7 +3907,7 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, outer_plan); /* Copy cost data from Path to Plan; no need to make FDW do this */ @@ -810,7 +829,7 @@ index 40abe6f9f6..9edd6daeff 100644 /* Copy foreign server OID; likewise, no need to make FDW do this */ scan_plan->fs_server = rel->serverid; -@@ -4037,7 +4039,7 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path, +@@ -4039,7 +4041,7 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path, * Copy cost data from Path to Plan; no need to make custom-plan providers * do this */ @@ -819,7 +838,7 @@ index 40abe6f9f6..9edd6daeff 100644 /* Likewise, copy the relids that are represented by this custom scan */ cplan->custom_relids = best_path->path.parent->relids; -@@ -4139,7 +4141,7 @@ create_nestloop_plan(PlannerInfo *root, +@@ -4141,7 +4143,7 @@ create_nestloop_plan(PlannerInfo *root, best_path->jointype, best_path->inner_unique); @@ -828,7 +847,7 @@ index 40abe6f9f6..9edd6daeff 100644 return join_plan; } -@@ -4446,7 +4448,7 @@ create_mergejoin_plan(PlannerInfo *root, +@@ -4448,7 +4450,7 @@ create_mergejoin_plan(PlannerInfo *root, best_path->skip_mark_restore); /* Costs of sort and material steps are included in path cost already */ @@ -837,7 +856,7 @@ index 40abe6f9f6..9edd6daeff 100644 return join_plan; } -@@ -4619,7 +4621,7 @@ create_hashjoin_plan(PlannerInfo *root, +@@ -4621,7 +4623,7 @@ create_hashjoin_plan(PlannerInfo *root, best_path->jpath.jointype, best_path->jpath.inner_unique); @@ -846,7 +865,7 @@ index 40abe6f9f6..9edd6daeff 100644 return join_plan; } -@@ -5119,7 +5121,7 @@ order_qual_clauses(PlannerInfo *root, List *clauses) +@@ -5121,7 +5123,7 @@ order_qual_clauses(PlannerInfo *root, List *clauses) * Also copy the parallel-related flags, which the executor *will* use. */ static void @@ -855,7 +874,7 @@ index 40abe6f9f6..9edd6daeff 100644 { dest->startup_cost = src->startup_cost; dest->total_cost = src->total_cost; -@@ -5127,6 +5129,9 @@ copy_generic_path_info(Plan *dest, Path *src) +@@ -5129,6 +5131,9 @@ copy_generic_path_info(Plan *dest, Path *src) dest->plan_width = src->pathtarget->width; dest->parallel_aware = src->parallel_aware; dest->parallel_safe = src->parallel_safe; @@ -866,10 +885,10 @@ index 40abe6f9f6..9edd6daeff 100644 /* diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c -index 76245c1ff3..cac6adf35e 100644 +index 731ff708b9..e862e2a974 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c -@@ -1261,6 +1261,7 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) +@@ -1260,6 +1260,7 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } @@ -877,7 +896,7 @@ index 76245c1ff3..cac6adf35e 100644 /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, -@@ -1329,6 +1330,10 @@ get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, +@@ -1328,6 +1329,10 @@ get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, ppi->ppi_req_outer = required_outer; ppi->ppi_rows = rows; ppi->ppi_clauses = pclauses; @@ -888,7 +907,7 @@ index 76245c1ff3..cac6adf35e 100644 baserel->ppilist = lappend(baserel->ppilist, ppi); return ppi; -@@ -1554,6 +1559,10 @@ get_joinrel_parampathinfo(PlannerInfo *root, RelOptInfo *joinrel, +@@ -1553,6 +1558,10 @@ get_joinrel_parampathinfo(PlannerInfo *root, RelOptInfo *joinrel, ppi->ppi_req_outer = required_outer; ppi->ppi_rows = rows; ppi->ppi_clauses = NIL; @@ -900,7 +919,7 @@ index 76245c1ff3..cac6adf35e 100644 return ppi; diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h -index ba661d32a6..3c2595d639 100644 +index e94d9e49cf..4404155fbd 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h @@ -75,6 +75,12 @@ extern PGDLLIMPORT ExplainOneQuery_hook_type ExplainOneQuery_hook; @@ -917,10 +936,10 @@ index ba661d32a6..3c2595d639 100644 extern void ExplainQuery(ParseState *pstate, ExplainStmt *stmt, ParamListInfo params, DestReceiver *dest); diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h -index 8f62d61702..cfcd2c249d 100644 +index cde2637798..74ffaa9c8a 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h -@@ -734,6 +734,10 @@ typedef struct RelOptInfo +@@ -739,6 +739,10 @@ typedef struct RelOptInfo Relids top_parent_relids; /* Relids of topmost parents (if "other" * rel) */ @@ -931,7 +950,7 @@ index 8f62d61702..cfcd2c249d 100644 /* used for partitioned relations: */ PartitionScheme part_scheme; /* Partitioning scheme */ int nparts; /* Number of partitions; -1 if not yet set; in -@@ -1101,6 +1105,10 @@ typedef struct ParamPathInfo +@@ -1107,6 +1111,10 @@ typedef struct ParamPathInfo Relids ppi_req_outer; /* rels supplying parameters used by path */ double ppi_rows; /* estimated number of result tuples */ List *ppi_clauses; /* join clauses available from outer rels */ @@ -943,7 +962,7 @@ index 8f62d61702..cfcd2c249d 100644 diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h -index 7e6b10f86b..148720a566 100644 +index 43160439f0..86988ca32d 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h @@ -140,6 +140,19 @@ typedef struct Plan @@ -967,18 +986,22 @@ index 7e6b10f86b..148720a566 100644 * Information for management of parameter-change-driven rescanning * diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h -index 6141654e47..0915da8618 100644 +index ed2e4af4be..7e3cbcca14 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h -@@ -39,6 +39,33 @@ typedef enum +@@ -39,6 +39,37 @@ typedef enum } ConstraintExclusionType; +/* Hook for plugins to get control of cardinality estimation */ +typedef void (*set_baserel_rows_estimate_hook_type) (PlannerInfo *root, + RelOptInfo *rel); ++typedef void (*set_foreign_rows_estimate_hook_type) (PlannerInfo *root, ++ RelOptInfo *rel); +extern PGDLLIMPORT set_baserel_rows_estimate_hook_type + set_baserel_rows_estimate_hook; ++extern PGDLLIMPORT set_foreign_rows_estimate_hook_type ++ set_foreign_rows_estimate_hook; +typedef double (*get_parameterized_baserel_size_hook_type) (PlannerInfo *root, + RelOptInfo *rel, + List *param_clauses); @@ -1004,10 +1027,11 @@ index 6141654e47..0915da8618 100644 /* * prototypes for costsize.c * routines to compute costs and sizes -@@ -175,10 +202,21 @@ extern void compute_semi_anti_join_factors(PlannerInfo *root, +@@ -175,10 +206,22 @@ extern void compute_semi_anti_join_factors(PlannerInfo *root, SpecialJoinInfo *sjinfo, List *restrictlist, SemiAntiJoinFactors *semifactors); ++extern void set_foreign_rows_estimate(PlannerInfo *root, RelOptInfo *rel); +extern void set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel); +extern void set_baserel_rows_estimate_standard(PlannerInfo *root, RelOptInfo *rel); extern void set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel); @@ -1026,7 +1050,7 @@ index 6141654e47..0915da8618 100644 extern double get_parameterized_joinrel_size(PlannerInfo *root, RelOptInfo *rel, Path *outer_path, -@@ -190,6 +228,11 @@ extern void set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, +@@ -190,6 +233,11 @@ extern void set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, RelOptInfo *inner_rel, SpecialJoinInfo *sjinfo, List *restrictlist); @@ -1038,7 +1062,7 @@ index 6141654e47..0915da8618 100644 extern void set_subquery_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_function_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern void set_values_size_estimates(PlannerInfo *root, RelOptInfo *rel); -@@ -202,5 +245,7 @@ extern void set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel); +@@ -202,5 +250,7 @@ extern void set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel); extern PathTarget *set_pathtarget_cost_width(PlannerInfo *root, PathTarget *target); extern double compute_bitmap_pages(PlannerInfo *root, RelOptInfo *baserel, Path *bitmapqual, int loop_count, Cost *cost, double *tuple); @@ -1047,7 +1071,7 @@ index 6141654e47..0915da8618 100644 #endif /* COST_H */ diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h -index 715a24ad29..7311ba92f4 100644 +index 23dec14cbd..58489cb620 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -18,6 +18,10 @@ @@ -1062,7 +1086,7 @@ index 715a24ad29..7311ba92f4 100644 * prototypes for pathnode.c */ diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h -index 81c4a7e560..59daf7fb81 100644 +index 777655210b..dac8231291 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -24,6 +24,12 @@ extern double cursor_tuple_fraction; diff --git a/postprocessing.c b/postprocessing.c index 8b385a75..b74701e7 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -581,11 +581,35 @@ aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src) * I'm not sure what fpinfo structure keeps clauses for sufficient time. * So, copy clauses. */ + dest->path_clauses = list_concat(list_copy(fpinfo->joinclauses), list_copy(fpinfo->remote_conds)); dest->path_clauses = list_concat(dest->path_clauses, list_copy(fpinfo->local_conds)); + dest->path_jointype = ((JoinPath *) src)->jointype; + + dest->path_relids = get_list_of_relids(root, fpinfo->lower_subquery_rels); + + if (fpinfo->outerrel) + { + dest->path_clauses = list_concat(dest->path_clauses, + list_copy(fpinfo->outerrel->baserestrictinfo)); + dest->path_clauses = list_concat(dest->path_clauses, + list_copy(fpinfo->outerrel->joininfo)); + dest->path_relids = list_concat(dest->path_relids, + get_list_of_relids(root, fpinfo->outerrel->relids)); + } + + if (fpinfo->innerrel) + { + dest->path_clauses = list_concat(dest->path_clauses, + list_copy(fpinfo->innerrel->baserestrictinfo)); + dest->path_clauses = list_concat(dest->path_clauses, + list_copy(fpinfo->innerrel->joininfo)); + dest->path_relids = list_concat(dest->path_relids, + get_list_of_relids(root, fpinfo->innerrel->relids)); + } } else { @@ -595,7 +619,8 @@ aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src) dest->path_jointype = JOIN_INNER; } - dest->path_relids = get_list_of_relids(root, src->parent->relids); + dest->path_relids = list_concat(dest->path_relids, + get_list_of_relids(root, src->parent->relids)); dest->path_parallel_workers = src->parallel_workers; dest->was_parametrized = (src->param_info != NULL); diff --git a/storage.c b/storage.c index cd733a97..af468f30 100644 --- a/storage.c +++ b/storage.c @@ -235,7 +235,8 @@ update_query(int query_hash, bool learn_aqo, bool use_aqo, &TTSOpsBufferHeapTuple); find_ok = index_getnext_slot(query_index_scan, ForwardScanDirection, slot); if (!find_ok) - elog(PANIC, "Query isn't found in AQO learning database."); + elog(PANIC, "[AQO]: Update of non-existed query: query hash: %d, fss hash: %d, use aqo: %s", + query_hash, fspace_hash, use_aqo ? "true" : "false"); tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); From 38d4f77c945e98f45a659b90035993a3b16c81c4 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: 2021年1月15日 17:45:17 +0500 Subject: [PATCH 06/20] Add basic regression test on FDW support. Switch from AQO_EXPLAIN define to the two user GUCS: aqo.details and aqo.show_hash. New ExplainOneNode hook allows to add info into explain of each node. --- Makefile | 2 + aqo.c | 33 ++++++- aqo.h | 6 ++ aqo_pg13.patch | 209 ++++++++++++++++++++----------------------- conf.add | 4 +- expected/aqo_fdw.out | 74 +++++++++++++++ postprocessing.c | 126 +++++++++++++++++--------- 7 files changed, 295 insertions(+), 159 deletions(-) create mode 100644 expected/aqo_fdw.out diff --git a/Makefile b/Makefile index 1b1575c0..54aa96a5 100644 --- a/Makefile +++ b/Makefile @@ -14,11 +14,13 @@ REGRESS = aqo_disabled \ aqo_forced \ aqo_learn \ schema \ + aqo_fdw \ aqo_CVE-2020-14350 fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir) EXTRA_REGRESS_OPTS=--temp-config=$(top_srcdir)/$(subdir)/conf.add +EXTRA_INSTALL = contrib/postgres_fdw DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql diff --git a/aqo.c b/aqo.c index 95ef4c12..84c5d39f 100644 --- a/aqo.c +++ b/aqo.c @@ -18,6 +18,8 @@ void _PG_init(void); /* Strategy of determining feature space for new queries. */ int aqo_mode; bool force_collect_stat; +bool aqo_show_hash; +bool aqo_details; /* GUC variables */ static const struct config_enum_entry format_options[] = { @@ -83,6 +85,7 @@ set_joinrel_size_estimates_hook_type prev_set_joinrel_size_estimates_hook; get_parameterized_joinrel_size_hook_type prev_get_parameterized_joinrel_size_hook; copy_generic_path_info_hook_type prev_copy_generic_path_info_hook; ExplainOnePlan_hook_type prev_ExplainOnePlan_hook; +ExplainOneNode_hook_type prev_ExplainOneNode_hook; /***************************************************************************** * @@ -116,7 +119,33 @@ _PG_init(void) NULL, NULL, NULL - ); + ); + + DefineCustomBoolVariable( + "aqo.show_hash", + "Show query and node hash on explain.", + "Hash value depend on each instance and is not good to enable it in regression or TAP tests.", + &aqo_show_hash, + false, + PGC_USERSET, + 0, + NULL, + NULL, + NULL + ); + + DefineCustomBoolVariable( + "aqo.details", + "Show AQO state on a query.", + NULL, + &aqo_details, + false, + PGC_USERSET, + 0, + NULL, + NULL, + NULL + ); prev_planner_hook = planner_hook; planner_hook = aqo_planner; @@ -139,6 +168,8 @@ _PG_init(void) copy_generic_path_info_hook = aqo_copy_generic_path_info; prev_ExplainOnePlan_hook = ExplainOnePlan_hook; ExplainOnePlan_hook = print_into_explain; + prev_ExplainOneNode_hook = ExplainOneNode_hook; + ExplainOneNode_hook = print_node_explain; parampathinfo_postinit_hook = ppi_hook; init_deactivated_queries_storage(); diff --git a/aqo.h b/aqo.h index b6e934f1..0381ba3c 100644 --- a/aqo.h +++ b/aqo.h @@ -174,6 +174,8 @@ typedef enum extern int aqo_mode; extern bool force_collect_stat; +extern bool aqo_show_hash; +extern bool aqo_details; /* * It is mostly needed for auto tuning of query. with auto tuning mode aqo @@ -312,6 +314,10 @@ void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, const char *queryString, ParamListInfo params, const instr_time *planduration, QueryEnvironment *queryEnv); +extern void print_node_explain(ExplainState *es, + PlanState *ps, + Plan *plan, + double rows); void disable_aqo_for_query(void); /* Cardinality estimation hooks */ diff --git a/aqo_pg13.patch b/aqo_pg13.patch index 6925b773..1c30cadc 100644 --- a/aqo_pg13.patch +++ b/aqo_pg13.patch @@ -1,5 +1,5 @@ diff --git a/contrib/Makefile b/contrib/Makefile -index 7a4866e338..47a18b9698 100644 +index 1846d415b6..95519ac11d 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -7,6 +7,7 @@ include $(top_builddir)/src/Makefile.global @@ -11,7 +11,7 @@ index 7a4866e338..47a18b9698 100644 auto_explain \ bloom \ diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c -index 5d7eb3574c..87402b6859 100644 +index 0ad49612d2..7c0b82bde7 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -24,6 +24,7 @@ @@ -22,17 +22,20 @@ index 5d7eb3574c..87402b6859 100644 #include "parser/parsetree.h" #include "rewrite/rewriteHandler.h" #include "storage/bufmgr.h" -@@ -46,6 +47,9 @@ ExplainOneQuery_hook_type ExplainOneQuery_hook = NULL; +@@ -46,6 +47,12 @@ ExplainOneQuery_hook_type ExplainOneQuery_hook = NULL; /* Hook for plugins to get control in explain_get_index_name() */ explain_get_index_name_hook_type explain_get_index_name_hook = NULL; +/* Hook for plugins to get control in ExplainOnePlan() */ +ExplainOnePlan_hook_type ExplainOnePlan_hook = NULL; ++ ++/* Hook for plugins to get control in ExplainOnePlan() */ ++ExplainOneNode_hook_type ExplainOneNode_hook = NULL; + /* OR-able flags for ExplainXMLTag() */ #define X_OPENING 0 -@@ -654,6 +658,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, +@@ -638,6 +645,10 @@ ExplainOnePlan(PlannedStmt *plannedstmt, IntoClause *into, ExplainState *es, ExplainPropertyFloat("Execution Time", "ms", 1000.0 * totaltime, 3, es); @@ -43,50 +46,21 @@ index 5d7eb3574c..87402b6859 100644 ExplainCloseGroup("Query", NULL, true, es); } -@@ -1595,6 +1603,38 @@ ExplainNode(PlanState *planstate, List *ancestors, +@@ -1582,6 +1593,9 @@ ExplainNode(PlanState *planstate, List *ancestors, appendStringInfo(es->str, " (actual rows=%.0f loops=%.0f)", rows, nloops); + -+#ifdef AQO_EXPLAIN -+ if (es->verbose && plan && planstate->instrument) -+ { -+ int wrkrs = 1; -+ double error = -1.; -+ -+ if (planstate->worker_instrument && IsParallelTuplesProcessing(plan)) -+ { -+ int i; -+ for (i = 0; i < planstate->worker_instrument->num_workers; i++) -+ { -+ Instrumentation *instrument = &planstate->worker_instrument->instrument[i]; -+ if (instrument->nloops <= 0) -+ continue; -+ wrkrs++; -+ } -+ } -+ -+ if (plan->predicted_cardinality> 0.) -+ { -+ error = 100. * (plan->predicted_cardinality - (rows*wrkrs)) -+ / plan->predicted_cardinality; -+ appendStringInfo(es->str, -+ " (AQO: cardinality=%.0lf, error=%.0lf%%, fsspace_hash=%d)", -+ plan->predicted_cardinality, error, plan->fss_hash); -+ } -+ else -+ appendStringInfo(es->str, " (AQO not used, fsspace_hash=%d)", -+ plan->fss_hash); -+ } -+#endif ++ if (ExplainOneNode_hook) ++ ExplainOneNode_hook(es, planstate, plan, rows); } else { diff --git a/src/backend/nodes/copyfuncs.c b/src/backend/nodes/copyfuncs.c -index ba3ccc712c..74a090e6f9 100644 +index 256ab54003..cfdc0247ec 100644 --- a/src/backend/nodes/copyfuncs.c +++ b/src/backend/nodes/copyfuncs.c -@@ -126,6 +126,12 @@ CopyPlanFields(const Plan *from, Plan *newnode) +@@ -127,6 +127,12 @@ CopyPlanFields(const Plan *from, Plan *newnode) COPY_NODE_FIELD(lefttree); COPY_NODE_FIELD(righttree); COPY_NODE_FIELD(initPlan); @@ -100,7 +74,7 @@ index ba3ccc712c..74a090e6f9 100644 COPY_BITMAPSET_FIELD(allParam); } diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c -index 380336518f..ecf0c45629 100644 +index ef7e8281cc..93d24b905a 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -97,6 +97,11 @@ @@ -115,7 +89,7 @@ index 380336518f..ecf0c45629 100644 #define LOG2(x) (log(x) / 0.693147180559945) -@@ -185,7 +190,6 @@ static Cost append_nonpartial_cost(List *subpaths, int numpaths, +@@ -178,7 +183,6 @@ static Cost append_nonpartial_cost(List *subpaths, int numpaths, static void set_rel_width(PlannerInfo *root, RelOptInfo *rel); static double relation_byte_size(double tuples, int width); static double page_size(double tuples, int width); @@ -123,7 +97,7 @@ index 380336518f..ecf0c45629 100644 /* -@@ -266,7 +270,7 @@ cost_seqscan(Path *path, PlannerInfo *root, +@@ -256,7 +260,7 @@ cost_seqscan(Path *path, PlannerInfo *root, /* Adjust costing for parallelism, if used. */ if (path->parallel_workers> 0) { @@ -132,7 +106,7 @@ index 380336518f..ecf0c45629 100644 /* The CPU cost is divided among all the workers. */ cpu_run_cost /= parallel_divisor; -@@ -745,7 +749,7 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, +@@ -735,7 +739,7 @@ cost_index(IndexPath *path, PlannerInfo *root, double loop_count, /* Adjust costing for parallelism, if used. */ if (path->path.parallel_workers> 0) { @@ -141,7 +115,7 @@ index 380336518f..ecf0c45629 100644 path->path.rows = clamp_row_est(path->path.rows / parallel_divisor); -@@ -1026,7 +1030,7 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, +@@ -1016,7 +1020,7 @@ cost_bitmap_heap_scan(Path *path, PlannerInfo *root, RelOptInfo *baserel, /* Adjust costing for parallelism, if used. */ if (path->parallel_workers> 0) { @@ -150,7 +124,7 @@ index 380336518f..ecf0c45629 100644 /* The CPU cost is divided among all the workers. */ cpu_run_cost /= parallel_divisor; -@@ -2129,7 +2133,7 @@ cost_append(AppendPath *apath) +@@ -2119,7 +2123,7 @@ cost_append(AppendPath *apath) else /* parallel-aware */ { int i = 0; @@ -159,7 +133,7 @@ index 380336518f..ecf0c45629 100644 /* Parallel-aware Append never produces ordered output. */ Assert(apath->path.pathkeys == NIL); -@@ -2163,7 +2167,7 @@ cost_append(AppendPath *apath) +@@ -2153,7 +2157,7 @@ cost_append(AppendPath *apath) { double subpath_parallel_divisor; @@ -168,7 +142,7 @@ index 380336518f..ecf0c45629 100644 apath->path.rows += subpath->rows * (subpath_parallel_divisor / parallel_divisor); apath->path.total_cost += subpath->total_cost; -@@ -2762,7 +2766,7 @@ final_cost_nestloop(PlannerInfo *root, NestPath *path, +@@ -2752,7 +2756,7 @@ final_cost_nestloop(PlannerInfo *root, NestPath *path, /* For partial paths, scale row estimate. */ if (path->path.parallel_workers> 0) { @@ -177,7 +151,7 @@ index 380336518f..ecf0c45629 100644 path->path.rows = clamp_row_est(path->path.rows / parallel_divisor); -@@ -3208,7 +3212,7 @@ final_cost_mergejoin(PlannerInfo *root, MergePath *path, +@@ -3200,7 +3204,7 @@ final_cost_mergejoin(PlannerInfo *root, MergePath *path, /* For partial paths, scale row estimate. */ if (path->jpath.path.parallel_workers> 0) { @@ -186,7 +160,7 @@ index 380336518f..ecf0c45629 100644 path->jpath.path.rows = clamp_row_est(path->jpath.path.rows / parallel_divisor); -@@ -3542,7 +3546,7 @@ initial_cost_hashjoin(PlannerInfo *root, JoinCostWorkspace *workspace, +@@ -3534,7 +3538,7 @@ initial_cost_hashjoin(PlannerInfo *root, JoinCostWorkspace *workspace, * number, so we need to undo the division. */ if (parallel_hash) @@ -195,7 +169,7 @@ index 380336518f..ecf0c45629 100644 /* * Get hash table size that executor would use for inner relation. -@@ -3639,7 +3643,7 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path, +@@ -3631,7 +3635,7 @@ final_cost_hashjoin(PlannerInfo *root, HashPath *path, /* For partial paths, scale row estimate. */ if (path->jpath.path.parallel_workers> 0) { @@ -204,7 +178,7 @@ index 380336518f..ecf0c45629 100644 path->jpath.path.rows = clamp_row_est(path->jpath.path.rows / parallel_divisor); -@@ -4634,6 +4638,58 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) +@@ -4626,6 +4630,58 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) } @@ -263,7 +237,7 @@ index 380336518f..ecf0c45629 100644 /* * set_baserel_size_estimates * Set the size estimates for the given base relation. -@@ -4650,19 +4706,10 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) +@@ -4642,19 +4698,10 @@ approx_tuple_count(PlannerInfo *root, JoinPath *path, List *quals) void set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) { @@ -284,7 +258,7 @@ index 380336518f..ecf0c45629 100644 cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); -@@ -4673,13 +4720,33 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) +@@ -4665,13 +4712,33 @@ set_baserel_size_estimates(PlannerInfo *root, RelOptInfo *rel) * get_parameterized_baserel_size * Make a size estimate for a parameterized scan of a base relation. * @@ -320,7 +294,7 @@ index 380336518f..ecf0c45629 100644 { List *allclauses; double nrows; -@@ -4708,6 +4775,36 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, +@@ -4700,6 +4767,36 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, * set_joinrel_size_estimates * Set the size estimates for the given join relation. * @@ -357,7 +331,7 @@ index 380336518f..ecf0c45629 100644 * The rel's targetlist must have been constructed already, and a * restriction clause list that matches the given component rels must * be provided. -@@ -4727,11 +4824,11 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, +@@ -4719,11 +4816,11 @@ get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, * build_joinrel_tlist, and baserestrictcost is not used for join rels. */ void @@ -374,7 +348,7 @@ index 380336518f..ecf0c45629 100644 { rel->rows = calc_joinrel_size_estimate(root, rel, -@@ -4747,6 +4844,35 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, +@@ -4739,6 +4836,35 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, * get_parameterized_joinrel_size * Make a size estimate for a parameterized scan of a join relation. * @@ -410,7 +384,7 @@ index 380336518f..ecf0c45629 100644 * 'rel' is the joinrel under consideration. * 'outer_path', 'inner_path' are (probably also parameterized) Paths that * produce the relations being joined. -@@ -4759,11 +4885,11 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, +@@ -4751,11 +4877,11 @@ set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, * set_joinrel_size_estimates must have been applied already. */ double @@ -427,7 +401,7 @@ index 380336518f..ecf0c45629 100644 { double nrows; -@@ -5479,7 +5605,7 @@ set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel) +@@ -5424,7 +5550,7 @@ set_foreign_size_estimates(PlannerInfo *root, RelOptInfo *rel) /* Should only be applied to base relations */ Assert(rel->relid> 0); @@ -436,7 +410,7 @@ index 380336518f..ecf0c45629 100644 cost_qual_eval(&rel->baserestrictcost, rel->baserestrictinfo, root); -@@ -5761,14 +5887,25 @@ page_size(double tuples, int width) +@@ -5706,14 +5832,25 @@ page_size(double tuples, int width) return ceil(relation_byte_size(tuples, width) / BLCKSZ); } @@ -465,7 +439,7 @@ index 380336518f..ecf0c45629 100644 /* * Early experience with parallel query suggests that when there is only -@@ -5785,7 +5922,7 @@ get_parallel_divisor(Path *path) +@@ -5730,7 +5867,7 @@ get_parallel_divisor(Path *path) { double leader_contribution; @@ -475,7 +449,7 @@ index 380336518f..ecf0c45629 100644 parallel_divisor += leader_contribution; } diff --git a/src/backend/optimizer/plan/createplan.c b/src/backend/optimizer/plan/createplan.c -index 25d4750ca6..d0ea7bd2ff 100644 +index 84f2d186d9..a35d8ec9ee 100644 --- a/src/backend/optimizer/plan/createplan.c +++ b/src/backend/optimizer/plan/createplan.c @@ -70,6 +70,8 @@ @@ -505,7 +479,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -1257,7 +1259,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) +@@ -1258,7 +1260,7 @@ create_append_plan(PlannerInfo *root, AppendPath *best_path, int flags) plan->first_partial_plan = best_path->first_partial_path; plan->part_prune_info = partpruneinfo; @@ -514,7 +488,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 /* * If prepare_sort_from_pathkeys added sort columns, but we were told to -@@ -1303,7 +1305,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, +@@ -1304,7 +1306,7 @@ create_merge_append_plan(PlannerInfo *root, MergeAppendPath *best_path, * prepare_sort_from_pathkeys on it before we do so on the individual * child plans, to make cross-checking the sort info easier. */ @@ -523,7 +497,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 plan->targetlist = tlist; plan->qual = NIL; plan->lefttree = NULL; -@@ -1456,7 +1458,7 @@ create_group_result_plan(PlannerInfo *root, GroupResultPath *best_path) +@@ -1458,7 +1460,7 @@ create_group_result_plan(PlannerInfo *root, GroupResultPath *best_path) plan = make_result(tlist, (Node *) quals, NULL); @@ -532,7 +506,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -1481,7 +1483,7 @@ create_project_set_plan(PlannerInfo *root, ProjectSetPath *best_path) +@@ -1483,7 +1485,7 @@ create_project_set_plan(PlannerInfo *root, ProjectSetPath *best_path) plan = make_project_set(tlist, subplan); @@ -541,7 +515,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -1509,7 +1511,7 @@ create_material_plan(PlannerInfo *root, MaterialPath *best_path, int flags) +@@ -1511,7 +1513,7 @@ create_material_plan(PlannerInfo *root, MaterialPath *best_path, int flags) plan = make_material(subplan); @@ -550,7 +524,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -1709,7 +1711,7 @@ create_unique_plan(PlannerInfo *root, UniquePath *best_path, int flags) +@@ -1711,7 +1713,7 @@ create_unique_plan(PlannerInfo *root, UniquePath *best_path, int flags) } /* Copy cost data from Path to Plan */ @@ -577,7 +551,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 /* Assign the rescan Param. */ gm_plan->rescan_param = assign_special_exec_param(root); -@@ -1903,7 +1905,7 @@ create_projection_plan(PlannerInfo *root, ProjectionPath *best_path, int flags) +@@ -1901,7 +1903,7 @@ create_projection_plan(PlannerInfo *root, ProjectionPath *best_path, int flags) /* We need a Result node */ plan = (Plan *) make_result(tlist, NULL, subplan); @@ -586,7 +560,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 } return plan; -@@ -2004,7 +2006,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) +@@ -2002,7 +2004,7 @@ create_sort_plan(PlannerInfo *root, SortPath *best_path, int flags) IS_OTHER_REL(best_path->subpath->parent) ? best_path->path.parent->relids : NULL); @@ -595,7 +569,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2030,7 +2032,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, +@@ -2028,7 +2030,7 @@ create_incrementalsort_plan(PlannerInfo *root, IncrementalSortPath *best_path, best_path->spath.path.parent->relids : NULL, best_path->nPresortedCols); @@ -604,7 +578,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2069,7 +2071,7 @@ create_group_plan(PlannerInfo *root, GroupPath *best_path) +@@ -2067,7 +2069,7 @@ create_group_plan(PlannerInfo *root, GroupPath *best_path) subplan->targetlist), subplan); @@ -613,7 +587,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2097,7 +2099,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag +@@ -2095,7 +2097,7 @@ create_upper_unique_plan(PlannerInfo *root, UpperUniquePath *best_path, int flag best_path->path.pathkeys, best_path->numkeys); @@ -622,7 +596,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2141,7 +2143,7 @@ create_agg_plan(PlannerInfo *root, AggPath *best_path) +@@ -2139,7 +2141,7 @@ create_agg_plan(PlannerInfo *root, AggPath *best_path) best_path->transitionSpace, subplan); @@ -631,7 +605,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2343,7 +2345,7 @@ create_groupingsets_plan(PlannerInfo *root, GroupingSetsPath *best_path) +@@ -2341,7 +2343,7 @@ create_groupingsets_plan(PlannerInfo *root, GroupingSetsPath *best_path) subplan); /* Copy cost data from Path to Plan */ @@ -640,7 +614,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 } return (Plan *) plan; -@@ -2401,7 +2403,7 @@ create_minmaxagg_plan(PlannerInfo *root, MinMaxAggPath *best_path) +@@ -2399,7 +2401,7 @@ create_minmaxagg_plan(PlannerInfo *root, MinMaxAggPath *best_path) plan = make_result(tlist, (Node *) best_path->quals, NULL); @@ -649,7 +623,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 /* * During setrefs.c, we'll need to replace references to the Agg nodes -@@ -2520,7 +2522,7 @@ create_windowagg_plan(PlannerInfo *root, WindowAggPath *best_path) +@@ -2518,7 +2520,7 @@ create_windowagg_plan(PlannerInfo *root, WindowAggPath *best_path) wc->inRangeNullsFirst, subplan); @@ -658,7 +632,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2556,7 +2558,7 @@ create_setop_plan(PlannerInfo *root, SetOpPath *best_path, int flags) +@@ -2554,7 +2556,7 @@ create_setop_plan(PlannerInfo *root, SetOpPath *best_path, int flags) best_path->firstFlag, numGroups); @@ -667,7 +641,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2592,7 +2594,7 @@ create_recursiveunion_plan(PlannerInfo *root, RecursiveUnionPath *best_path) +@@ -2590,7 +2592,7 @@ create_recursiveunion_plan(PlannerInfo *root, RecursiveUnionPath *best_path) best_path->distinctList, numGroups); @@ -676,7 +650,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2615,7 +2617,7 @@ create_lockrows_plan(PlannerInfo *root, LockRowsPath *best_path, +@@ -2613,7 +2615,7 @@ create_lockrows_plan(PlannerInfo *root, LockRowsPath *best_path, plan = make_lockrows(subplan, best_path->rowMarks, best_path->epqParam); @@ -685,7 +659,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2676,7 +2678,7 @@ create_modifytable_plan(PlannerInfo *root, ModifyTablePath *best_path) +@@ -2674,7 +2676,7 @@ create_modifytable_plan(PlannerInfo *root, ModifyTablePath *best_path) best_path->onconflict, best_path->epqParam); @@ -694,7 +668,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2730,7 +2732,7 @@ create_limit_plan(PlannerInfo *root, LimitPath *best_path, int flags) +@@ -2728,7 +2730,7 @@ create_limit_plan(PlannerInfo *root, LimitPath *best_path, int flags) best_path->limitOption, numUniqkeys, uniqColIdx, uniqOperators, uniqCollations); @@ -703,7 +677,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return plan; } -@@ -2776,7 +2778,7 @@ create_seqscan_plan(PlannerInfo *root, Path *best_path, +@@ -2774,7 +2776,7 @@ create_seqscan_plan(PlannerInfo *root, Path *best_path, scan_clauses, scan_relid); @@ -712,7 +686,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -2822,7 +2824,7 @@ create_samplescan_plan(PlannerInfo *root, Path *best_path, +@@ -2820,7 +2822,7 @@ create_samplescan_plan(PlannerInfo *root, Path *best_path, scan_relid, tsc); @@ -721,7 +695,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3000,7 +3002,7 @@ create_indexscan_plan(PlannerInfo *root, +@@ -2998,7 +3000,7 @@ create_indexscan_plan(PlannerInfo *root, indexorderbyops, best_path->indexscandir); @@ -730,7 +704,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3115,7 +3117,7 @@ create_bitmap_scan_plan(PlannerInfo *root, +@@ -3113,7 +3115,7 @@ create_bitmap_scan_plan(PlannerInfo *root, bitmapqualorig, baserelid); @@ -739,7 +713,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3435,7 +3437,7 @@ create_tidscan_plan(PlannerInfo *root, TidPath *best_path, +@@ -3433,7 +3435,7 @@ create_tidscan_plan(PlannerInfo *root, TidPath *best_path, scan_relid, tidquals); @@ -748,7 +722,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3485,7 +3487,7 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, +@@ -3483,7 +3485,7 @@ create_subqueryscan_plan(PlannerInfo *root, SubqueryScanPath *best_path, scan_relid, subplan); @@ -757,7 +731,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3528,7 +3530,7 @@ create_functionscan_plan(PlannerInfo *root, Path *best_path, +@@ -3526,7 +3528,7 @@ create_functionscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_functionscan(tlist, scan_clauses, scan_relid, functions, rte->funcordinality); @@ -766,7 +740,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3571,7 +3573,7 @@ create_tablefuncscan_plan(PlannerInfo *root, Path *best_path, +@@ -3569,7 +3571,7 @@ create_tablefuncscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_tablefuncscan(tlist, scan_clauses, scan_relid, tablefunc); @@ -775,7 +749,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3615,7 +3617,7 @@ create_valuesscan_plan(PlannerInfo *root, Path *best_path, +@@ -3613,7 +3615,7 @@ create_valuesscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_valuesscan(tlist, scan_clauses, scan_relid, values_lists); @@ -784,7 +758,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3708,7 +3710,7 @@ create_ctescan_plan(PlannerInfo *root, Path *best_path, +@@ -3706,7 +3708,7 @@ create_ctescan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_ctescan(tlist, scan_clauses, scan_relid, plan_id, cte_param_id); @@ -793,7 +767,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3747,7 +3749,7 @@ create_namedtuplestorescan_plan(PlannerInfo *root, Path *best_path, +@@ -3745,7 +3747,7 @@ create_namedtuplestorescan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_namedtuplestorescan(tlist, scan_clauses, scan_relid, rte->enrname); @@ -802,7 +776,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3785,7 +3787,7 @@ create_resultscan_plan(PlannerInfo *root, Path *best_path, +@@ -3783,7 +3785,7 @@ create_resultscan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_result(tlist, (Node *) scan_clauses, NULL); @@ -811,7 +785,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3845,7 +3847,7 @@ create_worktablescan_plan(PlannerInfo *root, Path *best_path, +@@ -3843,7 +3845,7 @@ create_worktablescan_plan(PlannerInfo *root, Path *best_path, scan_plan = make_worktablescan(tlist, scan_clauses, scan_relid, cteroot->wt_param_id); @@ -820,7 +794,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return scan_plan; } -@@ -3905,7 +3907,7 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, +@@ -3903,7 +3905,7 @@ create_foreignscan_plan(PlannerInfo *root, ForeignPath *best_path, outer_plan); /* Copy cost data from Path to Plan; no need to make FDW do this */ @@ -829,7 +803,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 /* Copy foreign server OID; likewise, no need to make FDW do this */ scan_plan->fs_server = rel->serverid; -@@ -4039,7 +4041,7 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path, +@@ -4037,7 +4039,7 @@ create_customscan_plan(PlannerInfo *root, CustomPath *best_path, * Copy cost data from Path to Plan; no need to make custom-plan providers * do this */ @@ -838,7 +812,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 /* Likewise, copy the relids that are represented by this custom scan */ cplan->custom_relids = best_path->path.parent->relids; -@@ -4141,7 +4143,7 @@ create_nestloop_plan(PlannerInfo *root, +@@ -4139,7 +4141,7 @@ create_nestloop_plan(PlannerInfo *root, best_path->jointype, best_path->inner_unique); @@ -847,7 +821,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return join_plan; } -@@ -4448,7 +4450,7 @@ create_mergejoin_plan(PlannerInfo *root, +@@ -4446,7 +4448,7 @@ create_mergejoin_plan(PlannerInfo *root, best_path->skip_mark_restore); /* Costs of sort and material steps are included in path cost already */ @@ -856,7 +830,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return join_plan; } -@@ -4621,7 +4623,7 @@ create_hashjoin_plan(PlannerInfo *root, +@@ -4619,7 +4621,7 @@ create_hashjoin_plan(PlannerInfo *root, best_path->jpath.jointype, best_path->jpath.inner_unique); @@ -865,7 +839,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 return join_plan; } -@@ -5121,7 +5123,7 @@ order_qual_clauses(PlannerInfo *root, List *clauses) +@@ -5119,7 +5121,7 @@ order_qual_clauses(PlannerInfo *root, List *clauses) * Also copy the parallel-related flags, which the executor *will* use. */ static void @@ -874,7 +848,7 @@ index 25d4750ca6..d0ea7bd2ff 100644 { dest->startup_cost = src->startup_cost; dest->total_cost = src->total_cost; -@@ -5129,6 +5131,9 @@ copy_generic_path_info(Plan *dest, Path *src) +@@ -5127,6 +5129,9 @@ copy_generic_path_info(Plan *dest, Path *src) dest->plan_width = src->pathtarget->width; dest->parallel_aware = src->parallel_aware; dest->parallel_safe = src->parallel_safe; @@ -885,10 +859,10 @@ index 25d4750ca6..d0ea7bd2ff 100644 /* diff --git a/src/backend/optimizer/util/relnode.c b/src/backend/optimizer/util/relnode.c -index 731ff708b9..e862e2a974 100644 +index a203e6f1ff..a335ede976 100644 --- a/src/backend/optimizer/util/relnode.c +++ b/src/backend/optimizer/util/relnode.c -@@ -1260,6 +1260,7 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) +@@ -1264,6 +1264,7 @@ find_childrel_parents(PlannerInfo *root, RelOptInfo *rel) } @@ -896,7 +870,7 @@ index 731ff708b9..e862e2a974 100644 /* * get_baserel_parampathinfo * Get the ParamPathInfo for a parameterized path for a base relation, -@@ -1328,6 +1329,10 @@ get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, +@@ -1332,6 +1333,10 @@ get_baserel_parampathinfo(PlannerInfo *root, RelOptInfo *baserel, ppi->ppi_req_outer = required_outer; ppi->ppi_rows = rows; ppi->ppi_clauses = pclauses; @@ -907,7 +881,7 @@ index 731ff708b9..e862e2a974 100644 baserel->ppilist = lappend(baserel->ppilist, ppi); return ppi; -@@ -1553,6 +1558,10 @@ get_joinrel_parampathinfo(PlannerInfo *root, RelOptInfo *joinrel, +@@ -1557,6 +1562,10 @@ get_joinrel_parampathinfo(PlannerInfo *root, RelOptInfo *joinrel, ppi->ppi_req_outer = required_outer; ppi->ppi_rows = rows; ppi->ppi_clauses = NIL; @@ -919,10 +893,10 @@ index 731ff708b9..e862e2a974 100644 return ppi; diff --git a/src/include/commands/explain.h b/src/include/commands/explain.h -index e94d9e49cf..4404155fbd 100644 +index ba661d32a6..74e4f7592c 100644 --- a/src/include/commands/explain.h +++ b/src/include/commands/explain.h -@@ -75,6 +75,12 @@ extern PGDLLIMPORT ExplainOneQuery_hook_type ExplainOneQuery_hook; +@@ -75,6 +75,19 @@ extern PGDLLIMPORT ExplainOneQuery_hook_type ExplainOneQuery_hook; typedef const char *(*explain_get_index_name_hook_type) (Oid indexId); extern PGDLLIMPORT explain_get_index_name_hook_type explain_get_index_name_hook; @@ -932,14 +906,21 @@ index e94d9e49cf..4404155fbd 100644 + ParamListInfo params, const instr_time *planduration, + QueryEnvironment *queryEnv); +extern PGDLLIMPORT ExplainOnePlan_hook_type ExplainOnePlan_hook; ++ ++/* Explain a node info */ ++typedef void (*ExplainOneNode_hook_type) (ExplainState *es, ++ PlanState *ps, ++ Plan *plan, ++ double rows); ++extern PGDLLIMPORT ExplainOneNode_hook_type ExplainOneNode_hook; extern void ExplainQuery(ParseState *pstate, ExplainStmt *stmt, ParamListInfo params, DestReceiver *dest); diff --git a/src/include/nodes/pathnodes.h b/src/include/nodes/pathnodes.h -index cde2637798..74ffaa9c8a 100644 +index 10f0a149e9..fecf543f44 100644 --- a/src/include/nodes/pathnodes.h +++ b/src/include/nodes/pathnodes.h -@@ -739,6 +739,10 @@ typedef struct RelOptInfo +@@ -738,6 +738,10 @@ typedef struct RelOptInfo Relids top_parent_relids; /* Relids of topmost parents (if "other" * rel) */ @@ -950,7 +931,7 @@ index cde2637798..74ffaa9c8a 100644 /* used for partitioned relations: */ PartitionScheme part_scheme; /* Partitioning scheme */ int nparts; /* Number of partitions; -1 if not yet set; in -@@ -1107,6 +1111,10 @@ typedef struct ParamPathInfo +@@ -1104,6 +1108,10 @@ typedef struct ParamPathInfo Relids ppi_req_outer; /* rels supplying parameters used by path */ double ppi_rows; /* estimated number of result tuples */ List *ppi_clauses; /* join clauses available from outer rels */ @@ -962,10 +943,10 @@ index cde2637798..74ffaa9c8a 100644 diff --git a/src/include/nodes/plannodes.h b/src/include/nodes/plannodes.h -index 43160439f0..86988ca32d 100644 +index 83e01074ed..5f1de775ca 100644 --- a/src/include/nodes/plannodes.h +++ b/src/include/nodes/plannodes.h -@@ -140,6 +140,19 @@ typedef struct Plan +@@ -146,6 +146,19 @@ typedef struct Plan List *initPlan; /* Init Plan nodes (un-correlated expr * subselects) */ @@ -986,7 +967,7 @@ index 43160439f0..86988ca32d 100644 * Information for management of parameter-change-driven rescanning * diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h -index ed2e4af4be..7e3cbcca14 100644 +index 6141654e47..3288548af6 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -39,6 +39,37 @@ typedef enum @@ -1071,7 +1052,7 @@ index ed2e4af4be..7e3cbcca14 100644 #endif /* COST_H */ diff --git a/src/include/optimizer/pathnode.h b/src/include/optimizer/pathnode.h -index 23dec14cbd..58489cb620 100644 +index 3bd7072ae8..21bbaba11c 100644 --- a/src/include/optimizer/pathnode.h +++ b/src/include/optimizer/pathnode.h @@ -18,6 +18,10 @@ @@ -1086,7 +1067,7 @@ index 23dec14cbd..58489cb620 100644 * prototypes for pathnode.c */ diff --git a/src/include/optimizer/planmain.h b/src/include/optimizer/planmain.h -index 777655210b..dac8231291 100644 +index f3cefe67b8..6d77f6e871 100644 --- a/src/include/optimizer/planmain.h +++ b/src/include/optimizer/planmain.h @@ -24,6 +24,12 @@ extern double cursor_tuple_fraction; diff --git a/conf.add b/conf.add index 21843d00..3556e4d6 100644 --- a/conf.add +++ b/conf.add @@ -1 +1,3 @@ -shared_preload_libraries = 'aqo' +autovacuum = off +shared_preload_libraries = 'postgres_fdw, aqo' +max_parallel_workers = 0 # switch off parallel workers because of unsteadiness diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out new file mode 100644 index 00000000..25acc0c8 --- /dev/null +++ b/expected/aqo_fdw.out @@ -0,0 +1,74 @@ +-- Tests on cardinality estimation of FDW-queries: +-- simple ForeignScan. +-- JOIN push-down (check push of baserestrictinfo and joininfo) +-- Aggregate push-down +-- Push-down of groupings with HAVING clause. +CREATE EXTENSION aqo; +CREATE EXTENSION postgres_fdw; +SET aqo.mode = 'learn'; +SET aqo.details = 'true'; -- show AQO info for each node and entire query. +SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it. +DO $d$ + BEGIN + EXECUTE $$CREATE SERVER loopback FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (dbname '$$||current_database()||$$', + port '$$||current_setting('port')||$$' + )$$; + END; +$d$; +CREATE USER MAPPING FOR PUBLIC SERVER loopback; +CREATE TABLE local (x int); +CREATE FOREIGN TABLE frgn(x int) SERVER loopback OPTIONS (table_name 'local'); +INSERT INTO frgn (x) VALUES (1); +ANALYZE local; +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; + QUERY PLAN +------------------------------------------------------------- + Foreign Scan on frgn (actual rows=1 loops=1) (AQO not used) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(4 rows) + +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; + QUERY PLAN +----------------------------------------------------------------------------- + Foreign Scan on frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(4 rows) + +-- Push down base filters. +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; + QUERY PLAN +-------------------------------------------------------------------- + Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO not used) + Output: x + Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(6 rows) + +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; + QUERY PLAN +------------------------------------------------------------------------------------ + Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) + Output: x + Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(6 rows) + +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants + QUERY PLAN +------------------------------------------------------------------------------- + Foreign Scan on frgn (actual rows=0 loops=1) (AQO: cardinality=1, error=100%) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(4 rows) + +DROP EXTENSION aqo; diff --git a/postprocessing.c b/postprocessing.c index b74701e7..1c299326 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -749,59 +749,99 @@ RemoveFromQueryContext(QueryDesc *queryDesc) pfree(enr); } +void +print_node_explain(ExplainState *es, PlanState *ps, Plan *plan, double rows) +{ + int wrkrs = 1; + double error = -1.; + + if (!aqo_details || !plan || !ps->instrument) + return; + + if (ps->worker_instrument && IsParallelTuplesProcessing(plan)) + { + int i; + + for (i = 0; i < ps->worker_instrument->num_workers; i++) + { + Instrumentation *instrument = &ps->worker_instrument->instrument[i]; + + if (instrument->nloops <= 0) + continue; + + wrkrs++; + } + } + + if (plan->predicted_cardinality> 0.) + { + error = 100. * (plan->predicted_cardinality - (rows*wrkrs)) + / plan->predicted_cardinality; + appendStringInfo(es->str, + " (AQO: cardinality=%.0lf, error=%.0lf%%", + plan->predicted_cardinality, error); + } + else + appendStringInfo(es->str, " (AQO not used"); + + if (aqo_show_hash) + appendStringInfo(es->str, ", fss hash = %d", plan->fss_hash); + appendStringInfoChar(es->str, ')'); +} + /* * Prints if the plan was constructed with AQO. */ -void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, - ParamListInfo params, const instr_time *planduration, - QueryEnvironment *queryEnv) +void +print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, + ExplainState *es, const char *queryString, + ParamListInfo params, const instr_time *planduration, + QueryEnvironment *queryEnv) { if (prev_ExplainOnePlan_hook) prev_ExplainOnePlan_hook(plannedstmt, into, es, queryString, - params, planduration, queryEnv); + params, planduration, queryEnv); + + if (!aqo_details) + return; -#ifdef AQO_EXPLAIN /* Report to user about aqo state only in verbose mode */ - if (es->verbose) - { - ExplainPropertyBool("Using aqo", query_context.use_aqo, es); + ExplainPropertyBool("Using aqo", query_context.use_aqo, es); - switch (aqo_mode) - { - case AQO_MODE_INTELLIGENT: - ExplainPropertyText("AQO mode", "INTELLIGENT", es); - break; - case AQO_MODE_FORCED: - ExplainPropertyText("AQO mode", "FORCED", es); - break; - case AQO_MODE_CONTROLLED: - ExplainPropertyText("AQO mode", "CONTROLLED", es); - break; - case AQO_MODE_LEARN: - ExplainPropertyText("AQO mode", "LEARN", es); - break; - case AQO_MODE_FROZEN: - ExplainPropertyText("AQO mode", "FROZEN", es); - break; - case AQO_MODE_DISABLED: - ExplainPropertyText("AQO mode", "DISABLED", es); - break; - default: - elog(ERROR, "Bad AQO state"); - break; - } + switch (aqo_mode) + { + case AQO_MODE_INTELLIGENT: + ExplainPropertyText("AQO mode", "INTELLIGENT", es); + break; + case AQO_MODE_FORCED: + ExplainPropertyText("AQO mode", "FORCED", es); + break; + case AQO_MODE_CONTROLLED: + ExplainPropertyText("AQO mode", "CONTROLLED", es); + break; + case AQO_MODE_LEARN: + ExplainPropertyText("AQO mode", "LEARN", es); + break; + case AQO_MODE_FROZEN: + ExplainPropertyText("AQO mode", "FROZEN", es); + break; + case AQO_MODE_DISABLED: + ExplainPropertyText("AQO mode", "DISABLED", es); + break; + default: + elog(ERROR, "Bad AQO state"); + break; + } - /* - * Query hash provides an user the conveniently use of the AQO - * auxiliary functions. - */ - if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) - { + /* + * Query hash provides an user the conveniently use of the AQO + * auxiliary functions. + */ + if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) + { + if (aqo_show_hash) ExplainPropertyInteger("Query hash", NULL, - query_context.query_hash, es); - ExplainPropertyInteger("JOINS", NULL, njoins, es); - } + query_context.query_hash, es); + ExplainPropertyInteger("JOINS", NULL, njoins, es); } -#endif } From 829122005bfae902e0c84b6a6108f994f3e0d809 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年1月19日 08:31:04 +0500 Subject: [PATCH 07/20] Add missed aqo_fdw.sql --- sql/aqo_fdw.sql | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 sql/aqo_fdw.sql diff --git a/sql/aqo_fdw.sql b/sql/aqo_fdw.sql new file mode 100644 index 00000000..76e528d0 --- /dev/null +++ b/sql/aqo_fdw.sql @@ -0,0 +1,38 @@ +-- Tests on cardinality estimation of FDW-queries: +-- simple ForeignScan. +-- JOIN push-down (check push of baserestrictinfo and joininfo) +-- Aggregate push-down +-- Push-down of groupings with HAVING clause. + +CREATE EXTENSION aqo; +CREATE EXTENSION postgres_fdw; +SET aqo.mode = 'learn'; +SET aqo.details = 'true'; -- show AQO info for each node and entire query. +SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it. + +DO $d$ + BEGIN + EXECUTE $$CREATE SERVER loopback FOREIGN DATA WRAPPER postgres_fdw + OPTIONS (dbname '$$||current_database()||$$', + port '$$||current_setting('port')||$$' + )$$; + END; +$d$; + +CREATE USER MAPPING FOR PUBLIC SERVER loopback; + +CREATE TABLE local (x int); +CREATE FOREIGN TABLE frgn(x int) SERVER loopback OPTIONS (table_name 'local'); +INSERT INTO frgn (x) VALUES (1); +ANALYZE local; + +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; + +-- Push down base filters. +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants + +DROP EXTENSION aqo; + From 9aff6212d48e689fe4099f26e655f4d12a474d8b Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年1月18日 20:00:55 +0500 Subject: [PATCH 08/20] Prepare the insert_index routine for PGv14 signature --- storage.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/storage.c b/storage.c index af468f30..735ceda2 100644 --- a/storage.c +++ b/storage.c @@ -968,10 +968,14 @@ my_index_insert(Relation indexRelation, #if PG_VERSION_NUM < 100000 return index_insert(indexRelation, values, isnull, heap_t_ctid, heapRelation, checkUnique); -#else +#elif PG_VERSION_NUM < 140000 return index_insert(indexRelation, values, isnull, heap_t_ctid, heapRelation, checkUnique, BuildIndexInfo(indexRelation)); +#else + return index_insert(indexRelation, values, isnull, heap_t_ctid, + heapRelation, checkUnique, false, + BuildIndexInfo(indexRelation)); #endif } From 36c97ac685aa387428ec3cd632801f56faf4bfea Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年1月19日 09:15:39 +0500 Subject: [PATCH 09/20] Change the license sentence in accordance with 2021. --- aqo.c | 2 +- aqo.h | 2 +- auto_tuning.c | 2 +- cardinality_estimation.c | 2 +- cardinality_hooks.c | 2 +- hash.c | 2 +- machine_learning.c | 2 +- path_utils.c | 2 +- postprocessing.c | 2 +- preprocessing.c | 2 +- selectivity_cache.c | 2 +- storage.c | 2 +- utils.c | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/aqo.c b/aqo.c index 84c5d39f..98c15a43 100644 --- a/aqo.c +++ b/aqo.c @@ -2,7 +2,7 @@ * aqo.c * Adaptive query optimization extension * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/aqo.c diff --git a/aqo.h b/aqo.h index 0381ba3c..18c12f56 100644 --- a/aqo.h +++ b/aqo.h @@ -105,7 +105,7 @@ * Module storage.c is responsible for storage query settings and models * (i. e. all information which is used in extension). * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/aqo.h diff --git a/auto_tuning.c b/auto_tuning.c index a19f42d0..dae92e48 100644 --- a/auto_tuning.c +++ b/auto_tuning.c @@ -8,7 +8,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/auto_tuning.c diff --git a/cardinality_estimation.c b/cardinality_estimation.c index 89ddf1ee..6483a3ec 100644 --- a/cardinality_estimation.c +++ b/cardinality_estimation.c @@ -8,7 +8,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/cardinality_estimation.c diff --git a/cardinality_hooks.c b/cardinality_hooks.c index 76f54d68..8eb9745b 100644 --- a/cardinality_hooks.c +++ b/cardinality_hooks.c @@ -18,7 +18,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/cardinality_hooks.c diff --git a/hash.c b/hash.c index b039be9e..f853c359 100644 --- a/hash.c +++ b/hash.c @@ -12,7 +12,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/hash.c diff --git a/machine_learning.c b/machine_learning.c index 7b4612cd..9ebbae6a 100644 --- a/machine_learning.c +++ b/machine_learning.c @@ -12,7 +12,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/machine_learning.c diff --git a/path_utils.c b/path_utils.c index 59edc35c..f91d8be8 100644 --- a/path_utils.c +++ b/path_utils.c @@ -5,7 +5,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/path_utils.c diff --git a/postprocessing.c b/postprocessing.c index 1c299326..c8b42f7a 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -9,7 +9,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/postprocessing.c diff --git a/preprocessing.c b/preprocessing.c index bb81d31e..ecfaf1a7 100644 --- a/preprocessing.c +++ b/preprocessing.c @@ -49,7 +49,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/preprocessing.c diff --git a/selectivity_cache.c b/selectivity_cache.c index 455d13b1..12ecd699 100644 --- a/selectivity_cache.c +++ b/selectivity_cache.c @@ -9,7 +9,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/selectivity_cache.c diff --git a/storage.c b/storage.c index 735ceda2..acd421e5 100644 --- a/storage.c +++ b/storage.c @@ -8,7 +8,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/storage.c diff --git a/utils.c b/utils.c index 1ae45abe..62e6d122 100644 --- a/utils.c +++ b/utils.c @@ -5,7 +5,7 @@ * ******************************************************************************* * - * Copyright (c) 2016-2020, Postgres Professional + * Copyright (c) 2016-2021, Postgres Professional * * IDENTIFICATION * aqo/utils.c From 34c640d40f511f236f2186a57e1ff8de1f85f360 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年1月19日 22:04:29 +0500 Subject: [PATCH 10/20] Fix the state: AQO supports push-down of trivial JOIN and JOIN with mergejoinable clauses. --- expected/aqo_fdw.out | 54 +++++++++++++++++++++++++++++++++++++++----- sql/aqo_fdw.sql | 32 ++++++++++++++++++++------ 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index 25acc0c8..7479faad 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -21,7 +21,9 @@ CREATE TABLE local (x int); CREATE FOREIGN TABLE frgn(x int) SERVER loopback OPTIONS (table_name 'local'); INSERT INTO frgn (x) VALUES (1); ANALYZE local; -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; +-- Trivial foreign scan.s +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +SELECT x FROM frgn; QUERY PLAN ------------------------------------------------------------- Foreign Scan on frgn (actual rows=1 loops=1) (AQO not used) @@ -30,7 +32,8 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; JOINS: 0 (4 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +SELECT x FROM frgn; QUERY PLAN ----------------------------------------------------------------------------- Foreign Scan on frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) @@ -39,8 +42,9 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; JOINS: 0 (4 rows) --- Push down base filters. -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; +-- Push down base filters. Use verbose mode to see filters. +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) +SELECT x FROM frgn WHERE x < 10; QUERY PLAN -------------------------------------------------------------------- Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO not used) @@ -51,7 +55,8 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frg JOINS: 0 (6 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) +SELECT x FROM frgn WHERE x < 10; QUERY PLAN ------------------------------------------------------------------------------------ Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) @@ -62,7 +67,8 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frg JOINS: 0 (6 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants QUERY PLAN ------------------------------------------------------------------------------- Foreign Scan on frgn (actual rows=0 loops=1) (AQO: cardinality=1, error=100%) @@ -71,4 +77,40 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x JOINS: 0 (4 rows) +-- Trivial JOIN push-down. +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; + QUERY PLAN +--------------------------------------------------------------------------- + Merge Join (actual rows=1 loops=1) (AQO not used) + Merge Cond: (a.x = b.x) + -> Sort (actual rows=1 loops=1) (AQO not used) + Sort Key: a.x + Sort Method: quicksort Memory: 25kB + -> Foreign Scan on frgn a (actual rows=1 loops=1) (AQO not used) + -> Sort (actual rows=1 loops=1) (AQO not used) + Sort Key: b.x + Sort Method: quicksort Memory: 25kB + -> Foreign Scan on frgn b (actual rows=1 loops=1) (AQO not used) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(13 rows) + +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; + QUERY PLAN +--------------------------------------------------------------------- + Foreign Scan (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) + Relations: (frgn a) INNER JOIN (frgn b) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(5 rows) + +-- Non-mergejoinable join condition +--EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) +--SELECT * FROM frgn AS a, frgn AS b WHERE a.x Date: 2021年1月20日 22:37:43 +0500 Subject: [PATCH 11/20] Rename aqo.details to aqo.show_details. --- aqo.c | 2 +- expected/aqo_fdw.out | 2 +- sql/aqo_fdw.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aqo.c b/aqo.c index 98c15a43..c5eba992 100644 --- a/aqo.c +++ b/aqo.c @@ -135,7 +135,7 @@ _PG_init(void) ); DefineCustomBoolVariable( - "aqo.details", + "aqo.show_details", "Show AQO state on a query.", NULL, &aqo_details, diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index 7479faad..ed0969e5 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -6,7 +6,7 @@ CREATE EXTENSION aqo; CREATE EXTENSION postgres_fdw; SET aqo.mode = 'learn'; -SET aqo.details = 'true'; -- show AQO info for each node and entire query. +SET aqo.show_details = 'true'; -- show AQO info for each node and entire query. SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it. DO $d$ BEGIN diff --git a/sql/aqo_fdw.sql b/sql/aqo_fdw.sql index 06984d4f..f6cdb14a 100644 --- a/sql/aqo_fdw.sql +++ b/sql/aqo_fdw.sql @@ -7,7 +7,7 @@ CREATE EXTENSION aqo; CREATE EXTENSION postgres_fdw; SET aqo.mode = 'learn'; -SET aqo.details = 'true'; -- show AQO info for each node and entire query. +SET aqo.show_details = 'true'; -- show AQO info for each node and entire query. SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it. DO $d$ From d8e18cedaab28e84971a44bddebbd34c8f0dae11 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年1月20日 22:35:07 +0500 Subject: [PATCH 12/20] Update the readme file in accordance with the new AQO version. --- README.md | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 1b5284dd..123e5f55 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,11 @@ complicated queries. ## Installation The module works with PostgreSQL 9.6 and above. +To avoid compatibility issues, the following branches in the git-repository are allocated: +* `stable9_6`. +* `stable11` - for PG v10 and v11. +* `stable12` - for PG v12. +* the `master` branch of the AQO repository correctly works with PGv13 and the PostgreSQL `master` branch. The module contains a patch and an extension. Patch has to be applied to the sources of PostgresSQL. Patch affects header files, that is why PostgreSQL @@ -28,7 +33,7 @@ make check # check whether it works ``` Tag `version` at the patch name corresponds to suitable PostgreSQL release. -For PostgreSQL 10 use aqo_pg10.patch; for PostgreSQL 11 use aqo_pg11.patch and so on. +For PostgreSQL 9.6 use the 'aqo_pg9_6.patch' file; PostgreSQL 10 use aqo_pg10.patch; for PostgreSQL 11 use aqo_pg11.patch and so on. Also, you can see git tags at the master branch for more accurate definition of suitable PostgreSQL version. @@ -50,7 +55,7 @@ of per-database. The typical case is follows: you have complicated query, which executes too long. `EXPLAIN ANALYZE` shows, that the possible reason is bad cardinality -estimnation. +estimation. Example: ``` @@ -127,16 +132,16 @@ When the plan stops changing, you can often observe performance improvement: (23 rows) ``` -The settings system in AQO works with normalized queries, i. e. queries with -removed constants. For example, the normalized version of +The settings system in AQO works with normalised queries, i. e. queries with +removed constants. For example, the normalised version of `SELECT * FROM tbl WHERE a < 25 AND b = 'str';` is `SELECT * FROM tbl WHERE a < CONST and b = CONST;` -So the queries have equal normalization if and only if they differ only +So the queries have equal normalisation if and only if they differ only in their constants. -Each normalized query has its own hash. The correspondence between normalized +Each normalised query has its own hash. The correspondence between normalised query hash and query text is stored in aqo_query_texts table: ``` SELECT * FROM aqo_query_texts; @@ -174,6 +179,10 @@ if the data tends to change significantly), you can do `UPDATE SET aqo_learn=false WHERE query_hash = ;` before commit. +The extension includes two GUC's to display the executed cardinality predictions for a query. +The `aqo.show_details = 'on'` (default - off) allows to see the aqo cardinality prediction results for each node of a query plan and an AQO summary. +The `aqo.show_hash = 'on'` (default - off) will print hash signature for each plan node and overall query. It is system-specific information and should be used for situational analysis. + The more detailed reference of AQO settings mechanism is available further. ## Advanced tuning From 418c7d498c8a4294bbaa6a4fbe722b1e8d4a9860 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年1月20日 22:53:14 +0500 Subject: [PATCH 13/20] Bugfix of the aqo_fdw regress test --- expected/aqo_fdw.out | 55 ++++++++++++++++++++++++++++---------------- sql/aqo_fdw.sql | 2 +- 2 files changed, 36 insertions(+), 21 deletions(-) diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index ed0969e5..040c004a 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -43,6 +43,11 @@ SELECT x FROM frgn; (4 rows) -- Push down base filters. Use verbose mode to see filters. +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)) +SELECT x FROM frgn WHERE x < 10; +ERROR: syntax error at or near ")" +LINE 1: ...LAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)) + ^ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; QUERY PLAN @@ -55,18 +60,6 @@ SELECT x FROM frgn WHERE x < 10; JOINS: 0 (6 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) -SELECT x FROM frgn WHERE x < 10; - QUERY PLAN ------------------------------------------------------------------------------------- - Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) - Output: x - Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) - Using aqo: true - AQO mode: LEARN - JOINS: 0 -(6 rows) - EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants QUERY PLAN @@ -97,20 +90,42 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; JOINS: 0 (13 rows) -EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; - QUERY PLAN ---------------------------------------------------------------------- + QUERY PLAN +-------------------------------------------------------------------------------------------------------- Foreign Scan (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) + Output: a.x, b.x + Relations: (public.frgn a) INNER JOIN (public.frgn b) + Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x = r2.x)))) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(7 rows) + +-- TODO: Non-mergejoinable join condition. +EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) +SELECT * FROM frgn AS a, frgn AS b WHERE a.x Date: Sat, 6 Feb 2021 22:10:45 +0500 Subject: [PATCH 14/20] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 123e5f55..45ea1072 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ installed with `make install`. ``` cd postgresql-9.6 # enter postgresql source directory -git clone https://github.com/tigvarts/aqo.git contrib/aqo # clone aqo into contrib +git clone https://github.com/postgrespro/aqo.git contrib/aqo # clone aqo into contrib patch -p1 --no-backup-if-mismatch < contrib/aqo/aqo_pg.patch # patch postgresql make clean && make && make install # recompile postgresql cd contrib/aqo # enter aqo directory From 453bef5923308d42dd330df76d2aefbb8000553e Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: 2021年2月18日 08:22:19 +0500 Subject: [PATCH 15/20] Rename the aqo_details to aqo_show_details --- aqo.c | 4 ++-- aqo.h | 2 +- postprocessing.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aqo.c b/aqo.c index c5eba992..10bc23ef 100644 --- a/aqo.c +++ b/aqo.c @@ -19,7 +19,7 @@ void _PG_init(void); int aqo_mode; bool force_collect_stat; bool aqo_show_hash; -bool aqo_details; +bool aqo_show_details; /* GUC variables */ static const struct config_enum_entry format_options[] = { @@ -138,7 +138,7 @@ _PG_init(void) "aqo.show_details", "Show AQO state on a query.", NULL, - &aqo_details, + &aqo_show_details, false, PGC_USERSET, 0, diff --git a/aqo.h b/aqo.h index 18c12f56..0a5c209b 100644 --- a/aqo.h +++ b/aqo.h @@ -175,7 +175,7 @@ typedef enum extern int aqo_mode; extern bool force_collect_stat; extern bool aqo_show_hash; -extern bool aqo_details; +extern bool aqo_show_details; /* * It is mostly needed for auto tuning of query. with auto tuning mode aqo diff --git a/postprocessing.c b/postprocessing.c index c8b42f7a..68e1d5a4 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -755,7 +755,7 @@ print_node_explain(ExplainState *es, PlanState *ps, Plan *plan, double rows) int wrkrs = 1; double error = -1.; - if (!aqo_details || !plan || !ps->instrument) + if (!aqo_show_details || !plan || !ps->instrument) return; if (ps->worker_instrument && IsParallelTuplesProcessing(plan)) @@ -802,7 +802,7 @@ print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, prev_ExplainOnePlan_hook(plannedstmt, into, es, queryString, params, planduration, queryEnv); - if (!aqo_details) + if (!aqo_show_details) return; /* Report to user about aqo state only in verbose mode */ From d705b5d2736aab639d5cf285f9a50455882aaed0 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: 2021年2月18日 08:38:44 +0500 Subject: [PATCH 16/20] Store selectivity cache data in the cache memory context instead of short-living planner memory context. Some indentations fixed. --- cardinality_hooks.c | 6 ++++++ postprocessing.c | 18 +++++++++--------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/cardinality_hooks.c b/cardinality_hooks.c index 8eb9745b..977e894c 100644 --- a/cardinality_hooks.c +++ b/cardinality_hooks.c @@ -208,12 +208,16 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root, if (query_context.use_aqo || query_context.learn_aqo) { + MemoryContext mcxt; + allclauses = list_concat(list_copy(param_clauses), list_copy(rel->baserestrictinfo)); selectivities = get_selectivities(root, allclauses, rel->relid, JOIN_INNER, NULL); relid = planner_rt_fetch(rel->relid, root)->relid; get_eclasses(allclauses, &nargs, &args_hash, &eclass_hash); + + mcxt = MemoryContextSwitchTo(CacheMemoryContext); forboth(l, allclauses, l2, selectivities) { current_hash = get_clause_hash( @@ -222,6 +226,8 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root, cache_selectivity(current_hash, rel->relid, relid, *((double *) lfirst(l2))); } + + MemoryContextSwitchTo(mcxt); pfree(args_hash); pfree(eclass_hash); } diff --git a/postprocessing.c b/postprocessing.c index 68e1d5a4..475c791f 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -94,9 +94,9 @@ learn_sample(List *clauselist, List *selectivities, List *relidslist, { int fss_hash; int nfeatures; - double *matrix[aqo_K]; - double targets[aqo_K]; - double *features; + double *matrix[aqo_K]; + double targets[aqo_K]; + double *features; double target; int i; @@ -138,14 +138,14 @@ restore_selectivities(List *clauselist, JoinType join_type, bool was_parametrized) { - List *lst = NIL; - ListCell *l; + List *lst = NIL; + ListCell *l; int i = 0; bool parametrized_sel; int nargs; - int *args_hash; - int *eclass_hash; - double *cur_sel; + int *args_hash; + int *eclass_hash; + double *cur_sel; int cur_hash; int cur_relid; @@ -290,7 +290,7 @@ learnOnPlanState(PlanState *p, void *context) } else /* This node does not required to sum tuples of each worker - * to calculate produced rows. */ + * to calculate produced rows. */ learn_rows = p->instrument->ntuples / p->instrument->nloops; if (p->plan->predicted_cardinality> 0.) From 2b20ff01325af14567a3d76c5c85b865a385cae7 Mon Sep 17 00:00:00 2001 From: Andrey Lepikhov Date: 2021年2月18日 09:33:57 +0500 Subject: [PATCH 17/20] Change insert/update scheme for the ML-knowledge base. Big commit. To avoiding UPDATE/UPDATE/DELETE/INSERT conflicts we use user locks on a pair (query_hash, fspace_hash) to guarantee that only one backend can do INSERT/UPDATE/DELETE of a aqo table row. Also we use dirty snapshot to see concurrently inserted/updated tuples. Such conflict may cause deadlocks on end-of-transaction waiting. Currently, if we detected such conflict, we refuse from we ML-base changing. --- Makefile | 7 +- aqo.c | 94 ++++- aqo.h | 71 ++-- auto_tuning.c | 9 +- cardinality_estimation.c | 3 +- cardinality_hooks.c | 3 +- expected/aqo_fdw.out | 8 +- expected/gucs.out | 93 +++++ expected/schema.out | 11 +- ignorance.c | 182 ++++++++++ ignorance.h | 12 + postprocessing.c | 172 +++++---- preprocessing.c | 44 ++- sql/aqo_fdw.sql | 4 +- sql/gucs.sql | 40 ++ storage.c | 761 +++++++++++++++------------------------ t/001_pgbench.pl | 48 +++ 17 files changed, 970 insertions(+), 592 deletions(-) create mode 100644 expected/gucs.out create mode 100644 ignorance.c create mode 100644 ignorance.h create mode 100644 sql/gucs.sql create mode 100644 t/001_pgbench.pl diff --git a/Makefile b/Makefile index 54aa96a5..50aaea9a 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,9 @@ PGFILEDESC = "AQO - adaptive query optimization" MODULES = aqo OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \ hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \ -selectivity_cache.o storage.o utils.o $(WIN32RES) +selectivity_cache.o storage.o utils.o ignorance.o $(WIN32RES) + +TAP_TESTS = 1 REGRESS = aqo_disabled \ aqo_controlled \ @@ -15,7 +17,8 @@ REGRESS = aqo_disabled \ aqo_learn \ schema \ aqo_fdw \ - aqo_CVE-2020-14350 + aqo_CVE-2020-14350 \ + gucs fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir) diff --git a/aqo.c b/aqo.c index 10bc23ef..b35bc2fc 100644 --- a/aqo.c +++ b/aqo.c @@ -9,15 +9,33 @@ */ #include "aqo.h" +#include "ignorance.h" + +#include "access/relation.h" +#include "access/table.h" +#include "catalog/pg_extension.h" +#include "commands/extension.h" PG_MODULE_MAGIC; void _PG_init(void); +#define AQO_MODULE_MAGIC (1234) /* Strategy of determining feature space for new queries. */ int aqo_mode; bool force_collect_stat; + +/* + * Show special info in EXPLAIN mode. + * + * aqo_show_hash - show query class (hash) and a feature space value (hash) + * of each plan node. This is instance-dependent value and can't be used + * in regression and TAP tests. + * + * aqo_show_details - show AQO settings for this class and prediction + * for each plan node. + */ bool aqo_show_hash; bool aqo_show_details; @@ -106,7 +124,8 @@ _PG_init(void) 0, NULL, NULL, - NULL); + NULL + ); DefineCustomBoolVariable( "aqo.force_collect_stat", @@ -147,6 +166,19 @@ _PG_init(void) NULL ); + DefineCustomBoolVariable( + "aqo.log_ignorance", + "Log in a special table all feature spaces for which the AQO prediction was not successful.", + NULL, + &aqo_log_ignorance, + false, + PGC_SUSET, + 0, + NULL, + set_ignorance, + NULL + ); + prev_planner_hook = planner_hook; planner_hook = aqo_planner; prev_post_parse_analyze_hook = post_parse_analyze_hook; @@ -191,3 +223,63 @@ invalidate_deactivated_queries_cache(PG_FUNCTION_ARGS) init_deactivated_queries_storage(); PG_RETURN_POINTER(NULL); } + +/* + * Return AQO schema's Oid or InvalidOid if that's not possible. + */ +Oid +get_aqo_schema(void) +{ + Oid result; + Relation rel; + SysScanDesc scandesc; + HeapTuple tuple; + ScanKeyData entry[1]; + Oid ext_oid; + + /* It's impossible to fetch pg_aqo's schema now */ + if (!IsTransactionState()) + return InvalidOid; + + ext_oid = get_extension_oid("aqo", true); + if (ext_oid == InvalidOid) + return InvalidOid; /* exit if pg_aqo does not exist */ + + ScanKeyInit(&entry[0], +#if PG_VERSION_NUM>= 120000 + Anum_pg_extension_oid, +#else + ObjectIdAttributeNumber, +#endif + BTEqualStrategyNumber, F_OIDEQ, + ObjectIdGetDatum(ext_oid)); + + rel = relation_open(ExtensionRelationId, AccessShareLock); + scandesc = systable_beginscan(rel, ExtensionOidIndexId, true, + NULL, 1, entry); + tuple = systable_getnext(scandesc); + + /* We assume that there can be at most one matching tuple */ + if (HeapTupleIsValid(tuple)) + result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace; + else + result = InvalidOid; + + systable_endscan(scandesc); + relation_close(rel, AccessShareLock); + return result; +} + +/* + * Init userlock + */ +void +init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2) +{ + tag->locktag_field1 = AQO_MODULE_MAGIC; + tag->locktag_field2 = key1; + tag->locktag_field3 = key2; + tag->locktag_field4 = 0; + tag->locktag_type = LOCKTAG_USERLOCK; + tag->locktag_lockmethodid = USER_LOCKMETHOD; +} diff --git a/aqo.h b/aqo.h index 0a5c209b..1b37a3a7 100644 --- a/aqo.h +++ b/aqo.h @@ -267,6 +267,7 @@ extern get_parameterized_joinrel_size_hook_type prev_get_parameterized_joinrel_size_hook; extern copy_generic_path_info_hook_type prev_copy_generic_path_info_hook; extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook; +extern ExplainOneNode_hook_type prev_ExplainOneNode_hook; extern void ppi_hook(ParamPathInfo *ppi); @@ -281,44 +282,42 @@ int get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash); /* Storage interaction */ -bool find_query(int query_hash, - Datum *search_values, - bool *search_nulls); -bool add_query(int query_hash, bool learn_aqo, bool use_aqo, - int fspace_hash, bool auto_tuning); -bool update_query(int query_hash, bool learn_aqo, bool use_aqo, - int fspace_hash, bool auto_tuning); -bool add_query_text(int query_hash, const char *query_text); -bool load_fss(int fss_hash, int ncols, - double **matrix, double *targets, int *rows); -extern bool update_fss(int fss_hash, int nrows, int ncols, +extern bool find_query(int qhash, Datum *search_values, bool *search_nulls); +extern bool update_query(int qhash, int fhash, + bool learn_aqo, bool use_aqo, bool auto_tuning); +extern bool add_query_text(int query_hash, const char *query_text); +extern bool load_fss(int fhash, int fss_hash, + int ncols, double **matrix, double *targets, int *rows); +extern bool update_fss(int fhash, int fss_hash, int nrows, int ncols, double **matrix, double *targets); QueryStat *get_aqo_stat(int query_hash); void update_aqo_stat(int query_hash, QueryStat * stat); +extern bool my_index_insert(Relation indexRelation, Datum *values, bool *isnull, + ItemPointer heap_t_ctid, Relation heapRelation, + IndexUniqueCheck checkUnique); void init_deactivated_queries_storage(void); void fini_deactivated_queries_storage(void); bool query_is_deactivated(int query_hash); void add_deactivated_query(int query_hash); /* Query preprocessing hooks */ -void get_query_text(ParseState *pstate, Query *query); -PlannedStmt *call_default_planner(Query *parse, - const char *query_string, - int cursorOptions, - ParamListInfo boundParams); -PlannedStmt *aqo_planner(Query *parse, - const char *query_string, - int cursorOptions, - ParamListInfo boundParams); -void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, - ParamListInfo params, const instr_time *planduration, - QueryEnvironment *queryEnv); -extern void print_node_explain(ExplainState *es, - PlanState *ps, - Plan *plan, +extern void get_query_text(ParseState *pstate, Query *query); +extern PlannedStmt *call_default_planner(Query *parse, + const char *query_string, + int cursorOptions, + ParamListInfo boundParams); +extern PlannedStmt *aqo_planner(Query *parse, + const char *query_string, + int cursorOptions, + ParamListInfo boundParams); +extern void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, + ExplainState *es, const char *queryString, + ParamListInfo params, + const instr_time *planduration, + QueryEnvironment *queryEnv); +extern void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan, double rows); -void disable_aqo_for_query(void); +extern void disable_aqo_for_query(void); /* Cardinality estimation hooks */ extern void aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel); @@ -364,7 +363,7 @@ extern int OkNNr_learn(int matrix_rows, int matrix_cols, double *features, double target); /* Automatic query tuning */ -void automatical_query_tuning(int query_hash, QueryStat * stat); +extern void automatical_query_tuning(int query_hash, QueryStat * stat); /* Utilities */ int int_cmp(const void *a, const void *b); @@ -376,11 +375,13 @@ QueryStat *palloc_query_stat(void); void pfree_query_stat(QueryStat *stat); /* Selectivity cache for parametrized baserels */ -void cache_selectivity(int clause_hash, - int relid, - int global_relid, - double selectivity); -double *selectivity_cache_find_global_relid(int clause_hash, int global_relid); -void selectivity_cache_clear(void); +extern void cache_selectivity(int clause_hash, int relid, int global_relid, + double selectivity); +extern double *selectivity_cache_find_global_relid(int clause_hash, + int global_relid); +extern void selectivity_cache_clear(void); + +extern Oid get_aqo_schema(void); +extern void init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2); #endif diff --git a/auto_tuning.c b/auto_tuning.c index dae92e48..b82b415b 100644 --- a/auto_tuning.c +++ b/auto_tuning.c @@ -187,8 +187,11 @@ automatical_query_tuning(int query_hash, QueryStat * stat) } if (num_iterations <= auto_tuning_max_iterations || p_use> 0.5) - update_query(query_hash, query_context.learn_aqo, query_context.use_aqo, - query_context.fspace_hash, true); + update_query(query_hash, + query_context.fspace_hash, + query_context.learn_aqo, + query_context.use_aqo, + true); else - update_query(query_hash, false, false, query_context.fspace_hash, false); + update_query(query_hash, query_context.fspace_hash, false, false, false); } diff --git a/cardinality_estimation.c b/cardinality_estimation.c index 6483a3ec..3b4dda09 100644 --- a/cardinality_estimation.c +++ b/cardinality_estimation.c @@ -40,7 +40,8 @@ predict_for_relation(List *restrict_clauses, List *selectivities, for (i = 0; i < aqo_K; ++i) matrix[i] = palloc0(sizeof(**matrix) * nfeatures); - if (load_fss(*fss_hash, nfeatures, matrix, targets, &rows)) + if (load_fss(query_context.fspace_hash, *fss_hash, nfeatures, matrix, + targets, &rows)) result = OkNNr_predict(rows, nfeatures, matrix, targets, features); else { diff --git a/cardinality_hooks.c b/cardinality_hooks.c index 977e894c..dd631161 100644 --- a/cardinality_hooks.c +++ b/cardinality_hooks.c @@ -156,7 +156,8 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) relids = list_make1_int(relid); restrict_clauses = list_copy(rel->baserestrictinfo); - predicted = predict_for_relation(restrict_clauses, selectivities, relids, &fss); + predicted = predict_for_relation(restrict_clauses, selectivities, + relids, &fss); rel->fss_hash = fss; if (predicted>= 0) diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index 040c004a..c7cb734d 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -128,4 +128,10 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.xpredicted_cardinality < 0.) + { + char nodestr[1024]; + char *qplan = nodeToString(plan); + + memset(nodestr, 0, 1024); + strncpy(nodestr, qplan, 1023); + pfree(qplan); + + /* + * AQO failed to predict cardinality for this node. + */ + values[0] = Int32GetDatum(qhash); + values[1] = Int32GetDatum(fhash); + values[2] = Int32GetDatum(fss_hash); + values[3] = Int32GetDatum(nodeTag(plan)); + values[4] = CStringGetTextDatum(nodestr); + tuple = heap_form_tuple(tupDesc, values, isnull); + + simple_heap_insert(hrel, tuple); + my_index_insert(irel, values, isnull, &(tuple->t_self), + hrel, UNIQUE_CHECK_YES); + } + else + { + /* AQO works as expected. */ + } + } + else if (!TransactionIdIsValid(snap.xmin) && + !TransactionIdIsValid(snap.xmax)) + { + /* + * AQO made prediction for this node. Delete it from the ignorance + * table. + */ + tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); + Assert(shouldFree != true); + simple_heap_delete(hrel, &(tuple->t_self)); + } + else + { + /* + * The data exists. We can't do anything for now. + */ + } + + ExecDropSingleTupleTableSlot(slot); + index_endscan(scan); + index_close(irel, RowExclusiveLock); + table_close(hrel, RowExclusiveLock); + + CommandCounterIncrement(); + LockRelease(&tag, ExclusiveLock, false); +} diff --git a/ignorance.h b/ignorance.h new file mode 100644 index 00000000..bceb855b --- /dev/null +++ b/ignorance.h @@ -0,0 +1,12 @@ +#ifndef IGNORANCE_H +#define IGNORANCE_H + +#include "postgres.h" + +extern bool aqo_log_ignorance; + +extern void set_ignorance(bool newval, void *extra); +extern bool create_ignorance_table(bool fail_ok); +extern void update_ignorance(int qhash, int fhash, int fss_hash, Plan *plan); + +#endif /* IGNORANCE_H */ diff --git a/postprocessing.c b/postprocessing.c index 475c791f..c9fc3280 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -17,11 +17,14 @@ */ #include "aqo.h" +#include "ignorance.h" + #include "access/parallel.h" #include "optimizer/optimizer.h" #include "postgres_fdw.h" #include "utils/queryenvironment.h" + typedef struct { List *clauselist; @@ -41,29 +44,30 @@ static char *PlanStateInfo = "PlanStateInfo"; /* Query execution statistics collecting utilities */ -static void atomic_fss_learn_step(int fss_hash, int ncols, - double **matrix, double *targets, - double *features, double target); +static void atomic_fss_learn_step(int fhash, int fss_hash, int ncols, + double **matrix, double *targets, + double *features, double target); static void learn_sample(List *clauselist, - List *selectivities, - List *relidslist, - double true_cardinality, - double predicted_cardinality); + List *selectivities, + List *relidslist, + double true_cardinality, + Plan *plan); static List *restore_selectivities(List *clauselist, - List *relidslist, - JoinType join_type, - bool was_parametrized); + List *relidslist, + JoinType join_type, + bool was_parametrized); static void update_query_stat_row(double *et, int *et_size, - double *pt, int *pt_size, - double *ce, int *ce_size, - double planning_time, - double execution_time, - double cardinality_error, - int64 *n_exec); -static void StoreToQueryContext(QueryDesc *queryDesc); + double *pt, int *pt_size, + double *ce, int *ce_size, + double planning_time, + double execution_time, + double cardinality_error, + int64 *n_exec); +static void StoreToQueryEnv(QueryDesc *queryDesc); static void StorePlanInternals(QueryDesc *queryDesc); -static bool ExtractFromQueryContext(QueryDesc *queryDesc); -static void RemoveFromQueryContext(QueryDesc *queryDesc); +static bool ExtractFromQueryEnv(QueryDesc *queryDesc); +static void RemoveFromQueryEnv(QueryDesc *queryDesc); + /* * This is the critical section: only one runner is allowed to be inside this @@ -71,17 +75,23 @@ static void RemoveFromQueryContext(QueryDesc *queryDesc); * matrix and targets are just preallocated memory for computations. */ static void -atomic_fss_learn_step(int fss_hash, int ncols, - double **matrix, double *targets, - double *features, double target) +atomic_fss_learn_step(int fhash, int fss_hash, int ncols, + double **matrix, double *targets, + double *features, double target) { - int nrows; + LOCKTAG tag; + int nrows; - if (!load_fss(fss_hash, ncols, matrix, targets, &nrows)) + init_lock_tag(&tag, (uint32) fhash, (uint32) fss_hash); + LockAcquire(&tag, ExclusiveLock, false, false); + + if (!load_fss(fhash, fss_hash, ncols, matrix, targets, &nrows)) nrows = 0; nrows = OkNNr_learn(nrows, ncols, matrix, targets, features, target); - update_fss(fss_hash, nrows, ncols, matrix, targets); + update_fss(fhash, fss_hash, nrows, ncols, matrix, targets); + + LockRelease(&tag, ExclusiveLock, false); } /* @@ -90,36 +100,38 @@ atomic_fss_learn_step(int fss_hash, int ncols, */ static void learn_sample(List *clauselist, List *selectivities, List *relidslist, - double true_cardinality, double predicted_cardinality) + double true_cardinality, Plan *plan) { - int fss_hash; - int nfeatures; - double *matrix[aqo_K]; - double targets[aqo_K]; - double *features; - double target; - int i; + int fhash = query_context.fspace_hash; + int fss_hash; + int nfeatures; + double *matrix[aqo_K]; + double targets[aqo_K]; + double *features; + double target; + int i; -/* - * Suppress the optimization for debug purposes. - if (fabs(log(predicted_cardinality) - log(true_cardinality)) < - object_selection_prediction_threshold) - { - return; - } -*/ target = log(true_cardinality); - fss_hash = get_fss_for_object(clauselist, selectivities, relidslist, - &nfeatures, &features); + &nfeatures, &features); + + if (aqo_log_ignorance /* && load_fss(fhash, fss_hash, 0, NULL, NULL, NULL) */) + { + /* + * If ignorance logging is enabled and the feature space was existed in + * the ML knowledge base, log this issue. + */ + update_ignorance(query_context.query_hash, fhash, fss_hash, plan); + } if (nfeatures> 0) for (i = 0; i < aqo_K; ++i) matrix[i] = palloc(sizeof(double) * nfeatures); - /* Here should be critical section */ - atomic_fss_learn_step(fss_hash, nfeatures, matrix, targets, features, target); - /* Here should be the end of critical section */ + /* Critical section */ + atomic_fss_learn_step(fhash, fss_hash, + nfeatures, matrix, targets, features, target); + /* End of critical section */ if (nfeatures> 0) for (i = 0; i < aqo_K; ++i) @@ -332,13 +344,14 @@ learnOnPlanState(PlanState *p, void *context) if (ctx->learn) learn_sample(SubplanCtx.clauselist, SubplanCtx.selectivities, - p->plan->path_relids, learn_rows, predicted); + p->plan->path_relids, learn_rows, + p->plan); } } ctx->clauselist = list_concat(ctx->clauselist, SubplanCtx.clauselist); ctx->selectivities = list_concat(ctx->selectivities, - SubplanCtx.selectivities); + SubplanCtx.selectivities); return false; } @@ -414,7 +427,7 @@ aqo_ExecutorStart(QueryDesc *queryDesc, int eflags) queryDesc->instrument_options |= INSTRUMENT_ROWS; /* Save all query-related parameters into the query context. */ - StoreToQueryContext(queryDesc); + StoreToQueryEnv(queryDesc); } if (prev_ExecutorStart_hook) @@ -440,11 +453,12 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) QueryStat *stat = NULL; instr_time endtime; EphemeralNamedRelation enr = get_ENR(queryDesc->queryEnv, PlanStateInfo); + LOCKTAG tag; cardinality_sum_errors = 0.; cardinality_num_objects = 0; - if (!ExtractFromQueryContext(queryDesc)) + if (!ExtractFromQueryEnv(queryDesc)) /* AQO keep all query-related preferences at the query context. * It is needed to prevent from possible recursive changes, at * preprocessing stage of subqueries. @@ -474,6 +488,11 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) list_free(ctx.selectivities); } + /* Prevent concurrent updates. */ + init_lock_tag(&tag, (uint32) query_context.query_hash, + (uint32) query_context.fspace_hash); + LockAcquire(&tag, ExclusiveLock, false, false); + if (query_context.collect_stat) { INSTR_TIME_SET_CURRENT(endtime); @@ -490,26 +509,26 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) { if (query_context.use_aqo) update_query_stat_row(stat->execution_time_with_aqo, - &stat->execution_time_with_aqo_size, - stat->planning_time_with_aqo, - &stat->planning_time_with_aqo_size, - stat->cardinality_error_with_aqo, - &stat->cardinality_error_with_aqo_size, - query_context.query_planning_time, - totaltime - query_context.query_planning_time, - cardinality_error, - &stat->executions_with_aqo); + &stat->execution_time_with_aqo_size, + stat->planning_time_with_aqo, + &stat->planning_time_with_aqo_size, + stat->cardinality_error_with_aqo, + &stat->cardinality_error_with_aqo_size, + query_context.query_planning_time, + totaltime - query_context.query_planning_time, + cardinality_error, + &stat->executions_with_aqo); else update_query_stat_row(stat->execution_time_without_aqo, - &stat->execution_time_without_aqo_size, - stat->planning_time_without_aqo, - &stat->planning_time_without_aqo_size, - stat->cardinality_error_without_aqo, - &stat->cardinality_error_without_aqo_size, - query_context.query_planning_time, - totaltime - query_context.query_planning_time, - cardinality_error, - &stat->executions_without_aqo); + &stat->execution_time_without_aqo_size, + stat->planning_time_without_aqo, + &stat->planning_time_without_aqo_size, + stat->cardinality_error_without_aqo, + &stat->cardinality_error_without_aqo_size, + query_context.query_planning_time, + totaltime - query_context.query_planning_time, + cardinality_error, + &stat->executions_without_aqo); } } selectivity_cache_clear(); @@ -525,7 +544,9 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) update_aqo_stat(query_context.fspace_hash, stat); pfree_query_stat(stat); } - RemoveFromQueryContext(queryDesc); + + LockRelease(&tag, ExclusiveLock, false); + RemoveFromQueryEnv(queryDesc); end: if (prev_ExecutorEnd_hook) @@ -561,7 +582,7 @@ aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src) * path_parallel_workers, and was_parameterized. */ Assert(dest->path_clauses && dest->path_jointype && - dest->path_relids && dest->path_parallel_workers); + dest->path_relids && dest->path_parallel_workers); return; } @@ -644,7 +665,7 @@ aqo_copy_generic_path_info(PlannerInfo *root, Plan *dest, Path *src) * top-level query. */ static void -StoreToQueryContext(QueryDesc *queryDesc) +StoreToQueryEnv(QueryDesc *queryDesc) { EphemeralNamedRelation enr; int qcsize = sizeof(QueryContextData); @@ -712,7 +733,7 @@ StorePlanInternals(QueryDesc *queryDesc) * Restore AQO data, related to the query. */ static bool -ExtractFromQueryContext(QueryDesc *queryDesc) +ExtractFromQueryEnv(QueryDesc *queryDesc) { EphemeralNamedRelation enr; @@ -735,7 +756,7 @@ ExtractFromQueryContext(QueryDesc *queryDesc) } static void -RemoveFromQueryContext(QueryDesc *queryDesc) +RemoveFromQueryEnv(QueryDesc *queryDesc) { EphemeralNamedRelation enr = get_ENR(queryDesc->queryEnv, AQOPrivateData); unregister_ENR(queryDesc->queryEnv, AQOPrivateData); @@ -787,6 +808,9 @@ print_node_explain(ExplainState *es, PlanState *ps, Plan *plan, double rows) if (aqo_show_hash) appendStringInfo(es->str, ", fss hash = %d", plan->fss_hash); appendStringInfoChar(es->str, ')'); + + if (prev_ExplainOneNode_hook) + prev_ExplainOneNode_hook(es, ps, plan, rows); } /* @@ -834,7 +858,7 @@ print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, } /* - * Query hash provides an user the conveniently use of the AQO + * Query class provides an user the conveniently use of the AQO * auxiliary functions. */ if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) diff --git a/preprocessing.c b/preprocessing.c index ecfaf1a7..3ef0ac20 100644 --- a/preprocessing.c +++ b/preprocessing.c @@ -124,6 +124,7 @@ aqo_planner(Query *parse, bool query_is_stored; Datum query_params[5]; bool query_nulls[5] = {false, false, false, false, false}; + LOCKTAG tag; selectivity_cache_clear(); @@ -163,6 +164,13 @@ aqo_planner(Query *parse, boundParams); } + /* + * find-add query and query text must be atomic operation to prevent + * concurrent insertions. + */ + init_lock_tag(&tag, (uint32) query_context.query_hash, (uint32) 0); + LockAcquire(&tag, ExclusiveLock, false, false); + query_is_stored = find_query(query_context.query_hash, &query_params[0], &query_nulls[0]); @@ -217,9 +225,18 @@ aqo_planner(Query *parse, if (query_context.adding_query || force_collect_stat) { - add_query(query_context.query_hash, query_context.learn_aqo, - query_context.use_aqo, query_context.fspace_hash, - query_context.auto_tuning); + /* + * Add query into the AQO knowledge base. To process an error with + * concurrent addition from another backend we will try to restart + * preprocessing routine. + */ + update_query(query_context.query_hash, + query_context.fspace_hash, + query_context.learn_aqo, + query_context.use_aqo, + query_context.auto_tuning); + + add_query_text(query_context.query_hash, query_text); } } @@ -273,6 +290,8 @@ aqo_planner(Query *parse, } } + LockRelease(&tag, ExclusiveLock, false); + /* * This mode is possible here, because force collect statistics uses AQO * machinery. @@ -319,6 +338,22 @@ isQueryUsingSystemRelation(Query *query) return isQueryUsingSystemRelation_walker((Node *) query, NULL); } +static bool +IsAQORelation(Relation rel) +{ + char *relname = NameStr(rel->rd_rel->relname); + + if (strcmp(relname, "aqo_data") == 0 || + strcmp(relname, "aqo_query_texts") == 0 || + strcmp(relname, "aqo_query_stat") == 0 || + strcmp(relname, "aqo_queries") == 0 || + strcmp(relname, "aqo_ignorance") == 0 + ) + return true; + + return false; +} + bool isQueryUsingSystemRelation_walker(Node *node, void *context) { @@ -338,9 +373,10 @@ isQueryUsingSystemRelation_walker(Node *node, void *context) { Relation rel = table_open(rte->relid, AccessShareLock); bool is_catalog = IsCatalogRelation(rel); + bool is_aqo_rel = IsAQORelation(rel); table_close(rel, AccessShareLock); - if (is_catalog) + if (is_catalog || is_aqo_rel) return true; } } diff --git a/sql/aqo_fdw.sql b/sql/aqo_fdw.sql index fc3d9115..38e31ea1 100644 --- a/sql/aqo_fdw.sql +++ b/sql/aqo_fdw.sql @@ -52,5 +52,7 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.xheapRelation->rd_att, - &TTSOpsBufferHeapTuple); - find_ok = index_getnext_slot(query_index_scan, ForwardScanDirection, slot); + index_rescan(scan, &key, 1, NULL, 0); + slot = MakeSingleTupleTableSlot(hrel->rd_att, &TTSOpsBufferHeapTuple); + find_ok = index_getnext_slot(scan, ForwardScanDirection, slot); if (find_ok) { tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); - heap_deform_tuple(tuple, aqo_queries_heap->rd_att, - search_values, search_nulls); + heap_deform_tuple(tuple, hrel->rd_att, search_values, search_nulls); } ExecDropSingleTupleTableSlot(slot); - index_endscan(query_index_scan); - index_close(query_index_rel, lockmode); - table_close(aqo_queries_heap, lockmode); + index_endscan(scan); + index_close(irel, AccessShareLock); + table_close(hrel, AccessShareLock); return find_ok; } /* - * Creates entry for new query in aqo_queries table with given fields. - * Returns false if the operation failed, true otherwise. + * Update query status in intelligent mode. + * + * Do it gently: to prevent possible deadlocks, revert this update if any + * concurrent transaction is doing it. + * + * Such logic is possible, because this update is performed by AQO itself. It is + * not break any learning logic besides possible additional learning iterations. */ bool -add_query(int query_hash, bool learn_aqo, bool use_aqo, - int fspace_hash, bool auto_tuning) +update_query(int qhash, int fhash, + bool learn_aqo, bool use_aqo, bool auto_tuning) { - RangeVar *aqo_queries_table_rv; - Relation aqo_queries_heap; - HeapTuple tuple; - - LOCKMODE lockmode = RowExclusiveLock; - - Datum values[5]; - bool nulls[5] = {false, false, false, false, false}; - - Relation query_index_rel; - Oid query_index_rel_oid; - - values[0] = Int32GetDatum(query_hash); - values[1] = BoolGetDatum(learn_aqo); - values[2] = BoolGetDatum(use_aqo); - values[3] = Int32GetDatum(fspace_hash); - values[4] = BoolGetDatum(auto_tuning); - - query_index_rel_oid = RelnameGetRelid("aqo_queries_query_hash_idx"); - if (!OidIsValid(query_index_rel_oid)) - { - disable_aqo_for_query(); - return false; - } - query_index_rel = index_open(query_index_rel_oid, lockmode); - - aqo_queries_table_rv = makeRangeVar("public", "aqo_queries", -1); - aqo_queries_heap = table_openrv(aqo_queries_table_rv, lockmode); - - tuple = heap_form_tuple(RelationGetDescr(aqo_queries_heap), - values, nulls); - PG_TRY(); - { - simple_heap_insert(aqo_queries_heap, tuple); - my_index_insert(query_index_rel, - values, nulls, - &(tuple->t_self), - aqo_queries_heap, - UNIQUE_CHECK_YES); - } - PG_CATCH(); - { - /* - * Main goal is to catch deadlock errors during the index insertion. - */ - CommandCounterIncrement(); - simple_heap_delete(aqo_queries_heap, &(tuple->t_self)); - PG_RE_THROW(); - } - PG_END_TRY(); - - index_close(query_index_rel, lockmode); - table_close(aqo_queries_heap, lockmode); - - CommandCounterIncrement(); - - return true; -} - -bool -update_query(int query_hash, bool learn_aqo, bool use_aqo, - int fspace_hash, bool auto_tuning) -{ - RangeVar *aqo_queries_table_rv; - Relation aqo_queries_heap; + RangeVar *rv; + Relation hrel; + Relation irel; + TupleTableSlot *slot; HeapTuple tuple, nw_tuple; - - TupleTableSlot *slot; - bool shouldFree; - bool find_ok = false; - bool update_indexes; - - LOCKMODE lockmode = RowExclusiveLock; - - Relation query_index_rel; - Oid query_index_rel_oid; - IndexScanDesc query_index_scan; - ScanKeyData key; - Datum values[5]; bool isnull[5] = { false, false, false, false, false }; bool replace[5] = { false, true, true, true, true }; + bool shouldFree; + bool result = true; + bool update_indexes; + Oid reloid; + IndexScanDesc scan; + ScanKeyData key; + SnapshotData snap; - query_index_rel_oid = RelnameGetRelid("aqo_queries_query_hash_idx"); - if (!OidIsValid(query_index_rel_oid)) + reloid = RelnameGetRelid("aqo_queries_query_hash_idx"); + if (!OidIsValid(reloid)) { disable_aqo_for_query(); return false; } - aqo_queries_table_rv = makeRangeVar("public", "aqo_queries", -1); - aqo_queries_heap = table_openrv(aqo_queries_table_rv, lockmode); - - query_index_rel = index_open(query_index_rel_oid, lockmode); - query_index_scan = index_beginscan(aqo_queries_heap, - query_index_rel, - SnapshotSelf, - 1, - 0); - - ScanKeyInit(&key, - 1, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(query_hash)); - - index_rescan(query_index_scan, &key, 1, NULL, 0); - slot = MakeSingleTupleTableSlot(query_index_scan->heapRelation->rd_att, - &TTSOpsBufferHeapTuple); - find_ok = index_getnext_slot(query_index_scan, ForwardScanDirection, slot); - if (!find_ok) - elog(PANIC, "[AQO]: Update of non-existed query: query hash: %d, fss hash: %d, use aqo: %s", - query_hash, fspace_hash, use_aqo ? "true" : "false"); + rv = makeRangeVar("public", "aqo_queries", -1); + hrel = table_openrv(rv, RowExclusiveLock); + irel = index_open(reloid, RowExclusiveLock); - tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); - Assert(shouldFree != true); + /* + * Start an index scan. Use dirty snapshot to check concurrent updates that + * can be made before, but still not visible. + */ + InitDirtySnapshot(snap); + scan = index_beginscan(hrel, irel, &snap, 1, 0); + ScanKeyInit(&key, 1, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(qhash)); - heap_deform_tuple(tuple, aqo_queries_heap->rd_att, - values, isnull); + index_rescan(scan, &key, 1, NULL, 0); + slot = MakeSingleTupleTableSlot(hrel->rd_att, &TTSOpsBufferHeapTuple); + values[0] = Int32GetDatum(qhash); values[1] = BoolGetDatum(learn_aqo); values[2] = BoolGetDatum(use_aqo); - values[3] = Int32GetDatum(fspace_hash); + values[3] = Int32GetDatum(fhash); values[4] = BoolGetDatum(auto_tuning); - nw_tuple = heap_modify_tuple(tuple, aqo_queries_heap->rd_att, - values, isnull, replace); - if (my_simple_heap_update(aqo_queries_heap, &(nw_tuple->t_self), nw_tuple, - &update_indexes)) + if (!index_getnext_slot(scan, ForwardScanDirection, slot)) { - if (update_indexes) - my_index_insert(query_index_rel, values, isnull, - &(nw_tuple->t_self), - aqo_queries_heap, UNIQUE_CHECK_YES); + /* New tuple for the ML knowledge base */ + tuple = heap_form_tuple(RelationGetDescr(hrel), values, isnull); + simple_heap_insert(hrel, tuple); + my_index_insert(irel, values, isnull, &(tuple->t_self), + hrel, UNIQUE_CHECK_YES); + } + else if (!TransactionIdIsValid(snap.xmin) && + !TransactionIdIsValid(snap.xmax)) + { + /* + * Update existed data. No one concurrent transaction doesn't update this + * right now. + */ + tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); + Assert(shouldFree != true); + nw_tuple = heap_modify_tuple(tuple, hrel->rd_att, values, isnull, replace); + + if (my_simple_heap_update(hrel, &(nw_tuple->t_self), nw_tuple, + &update_indexes)) + { + if (update_indexes) + my_index_insert(irel, values, isnull, + &(nw_tuple->t_self), + hrel, UNIQUE_CHECK_YES); + result = true; + } + else + { + /* + * Ooops, somebody concurrently updated the tuple. It is possible + * only in the case of changes made by third-party code. + */ + elog(ERROR, "AQO feature space data for signature (%d, %d) concurrently" + " updated by a stranger backend.", + qhash, fhash); + result = false; + } } else { /* - * Ooops, somebody concurrently updated the tuple. We have to merge - * our changes somehow, but now we just discard ours. We don't believe - * in high probability of simultaneously finishing of two long, - * complex, and important queries, so we don't loss important data. + * Concurrent update was made. To prevent deadlocks refuse to update. */ + result = false; } ExecDropSingleTupleTableSlot(slot); - index_endscan(query_index_scan); - index_close(query_index_rel, lockmode); - table_close(aqo_queries_heap, lockmode); + index_endscan(scan); + index_close(irel, RowExclusiveLock); + table_close(hrel, RowExclusiveLock); CommandCounterIncrement(); - - return true; + return result; } /* @@ -284,64 +215,39 @@ update_query(int query_hash, bool learn_aqo, bool use_aqo, * Returns false if the operation failed, true otherwise. */ bool -add_query_text(int query_hash, const char *query_text) +add_query_text(int qhash, const char *query_text) { - RangeVar *aqo_query_texts_table_rv; - Relation aqo_query_texts_heap; + RangeVar *rv; + Relation hrel; + Relation irel; HeapTuple tuple; - - LOCKMODE lockmode = RowExclusiveLock; - Datum values[2]; bool isnull[2] = {false, false}; + Oid reloid; - Relation query_index_rel; - Oid query_index_rel_oid; - - values[0] = Int32GetDatum(query_hash); + values[0] = Int32GetDatum(qhash); values[1] = CStringGetTextDatum(query_text); - query_index_rel_oid = RelnameGetRelid("aqo_query_texts_query_hash_idx"); - if (!OidIsValid(query_index_rel_oid)) + reloid = RelnameGetRelid("aqo_query_texts_query_hash_idx"); + if (!OidIsValid(reloid)) { disable_aqo_for_query(); return false; } - query_index_rel = index_open(query_index_rel_oid, lockmode); - - aqo_query_texts_table_rv = makeRangeVar("public", - "aqo_query_texts", - -1); - aqo_query_texts_heap = table_openrv(aqo_query_texts_table_rv, - lockmode); - tuple = heap_form_tuple(RelationGetDescr(aqo_query_texts_heap), - values, isnull); + rv = makeRangeVar("public", "aqo_query_texts", -1); + hrel = table_openrv(rv, RowExclusiveLock); + irel = index_open(reloid, RowExclusiveLock); + tuple = heap_form_tuple(RelationGetDescr(hrel), values, isnull); - PG_TRY(); - { - simple_heap_insert(aqo_query_texts_heap, tuple); - my_index_insert(query_index_rel, - values, isnull, - &(tuple->t_self), - aqo_query_texts_heap, - UNIQUE_CHECK_YES); - } - PG_CATCH(); - { - CommandCounterIncrement(); - simple_heap_delete(aqo_query_texts_heap, &(tuple->t_self)); - index_close(query_index_rel, lockmode); - table_close(aqo_query_texts_heap, lockmode); - PG_RE_THROW(); - } - PG_END_TRY(); + simple_heap_insert(hrel, tuple); + my_index_insert(irel, values, isnull, &(tuple->t_self), hrel, + UNIQUE_CHECK_YES); - index_close(query_index_rel, lockmode); - table_close(aqo_query_texts_heap, lockmode); + index_close(irel, RowExclusiveLock); + table_close(hrel, RowExclusiveLock); CommandCounterIncrement(); - return true; } @@ -360,67 +266,52 @@ add_query_text(int query_hash, const char *query_text) * objects in the given feature space */ bool -load_fss(int fss_hash, int ncols, double **matrix, double *targets, int *rows) +load_fss(int fhash, int fss_hash, + int ncols, double **matrix, double *targets, int *rows) { - RangeVar *aqo_data_table_rv; - Relation aqo_data_heap; + RangeVar *rv; + Relation hrel; + Relation irel; HeapTuple tuple; TupleTableSlot *slot; bool shouldFree; bool find_ok = false; - - Relation data_index_rel; - Oid data_index_rel_oid; - IndexScanDesc data_index_scan; + Oid reloid; + IndexScanDesc scan; ScanKeyData key[2]; - - LOCKMODE lockmode = AccessShareLock; - Datum values[5]; bool isnull[5]; - bool success = true; - data_index_rel_oid = RelnameGetRelid("aqo_fss_access_idx"); - if (!OidIsValid(data_index_rel_oid)) + reloid = RelnameGetRelid("aqo_fss_access_idx"); + if (!OidIsValid(reloid)) { disable_aqo_for_query(); return false; } - aqo_data_table_rv = makeRangeVar("public", "aqo_data", -1); - aqo_data_heap = table_openrv(aqo_data_table_rv, lockmode); - - data_index_rel = index_open(data_index_rel_oid, lockmode); - data_index_scan = index_beginscan(aqo_data_heap, - data_index_rel, - SnapshotSelf, - 2, - 0); - - ScanKeyInit(&key[0], - 1, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(query_context.fspace_hash)); + rv = makeRangeVar("public", "aqo_data", -1); + hrel = table_openrv(rv, AccessShareLock); + irel = index_open(reloid, AccessShareLock); + scan = index_beginscan(hrel, irel, SnapshotSelf, 2, 0); - ScanKeyInit(&key[1], - 2, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(fss_hash)); + ScanKeyInit(&key[0], 1, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(fhash)); + ScanKeyInit(&key[1], 2, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(fss_hash)); + index_rescan(scan, key, 2, NULL, 0); - index_rescan(data_index_scan, key, 2, NULL, 0); + slot = MakeSingleTupleTableSlot(hrel->rd_att, &TTSOpsBufferHeapTuple); + find_ok = index_getnext_slot(scan, ForwardScanDirection, slot); - slot = MakeSingleTupleTableSlot(data_index_scan->heapRelation->rd_att, - &TTSOpsBufferHeapTuple); - find_ok = index_getnext_slot(data_index_scan, ForwardScanDirection, slot); - - if (find_ok) + if (matrix == NULL && targets == NULL && rows == NULL) + { + /* Just check availability */ + success = find_ok; + } + else if (find_ok) { tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); - heap_deform_tuple(tuple, aqo_data_heap->rd_att, values, isnull); + heap_deform_tuple(tuple, hrel->rd_att, values, isnull); if (DatumGetInt32(values[2]) == ncols) { @@ -433,21 +324,17 @@ load_fss(int fss_hash, int ncols, double **matrix, double *targets, int *rows) deform_vector(values[4], targets, rows); } else - { - elog(WARNING, "unexpected number of features for hash (%d, %d):\ + elog(ERROR, "unexpected number of features for hash (%d, %d):\ expected %d features, obtained %d", - query_context.fspace_hash, - fss_hash, ncols, DatumGetInt32(values[2])); - success = false; - } + fhash, fss_hash, ncols, DatumGetInt32(values[2])); } else success = false; ExecDropSingleTupleTableSlot(slot); - index_endscan(data_index_scan); - index_close(data_index_rel, lockmode); - table_close(aqo_data_heap, lockmode); + index_endscan(scan); + index_close(irel, AccessShareLock); + table_close(hrel, AccessShareLock); return success; } @@ -456,76 +343,64 @@ load_fss(int fss_hash, int ncols, double **matrix, double *targets, int *rows) * Updates the specified line in the specified feature subspace. * Returns false if the operation failed, true otherwise. * - * 'fss_hash' specifies the feature subspace - * 'nrows' x 'ncols' is the shape of 'matrix' - * 'targets' is vector of size 'nrows' + * 'fss_hash' specifies the feature subspace 'nrows' x 'ncols' is the shape + * of 'matrix' 'targets' is vector of size 'nrows' + * + * Necessary to prevent waiting for another transaction to commit in index + * insertion or heap update. + * + * Caller guaranteed that no one AQO process insert or update this data row. */ bool -update_fss(int fss_hash, int nrows, int ncols, double **matrix, double *targets) +update_fss(int fhash, int fsshash, int nrows, int ncols, + double **matrix, double *targets) { - RangeVar *aqo_data_table_rv; - Relation aqo_data_heap; - TupleDesc tuple_desc; + RangeVar *rv; + Relation hrel; + Relation irel; + SnapshotData snap; + TupleTableSlot *slot; + TupleDesc tupDesc; HeapTuple tuple, nw_tuple; - - TupleTableSlot *slot; + Datum values[5]; + bool isnull[5] = { false, false, false, false, false }; + bool replace[5] = { false, false, false, true, true }; bool shouldFree; bool find_ok = false; bool update_indexes; - - LOCKMODE lockmode = RowExclusiveLock; - - Relation data_index_rel; - Oid data_index_rel_oid; - IndexScanDesc data_index_scan; + Oid reloid; + IndexScanDesc scan; ScanKeyData key[2]; + bool result = true; - Datum values[5]; - bool isnull[5] = { false, false, false, false, false }; - bool replace[5] = { false, false, false, true, true }; - - data_index_rel_oid = RelnameGetRelid("aqo_fss_access_idx"); - if (!OidIsValid(data_index_rel_oid)) + reloid = RelnameGetRelid("aqo_fss_access_idx"); + if (!OidIsValid(reloid)) { disable_aqo_for_query(); return false; } - aqo_data_table_rv = makeRangeVar("public", "aqo_data", -1); - aqo_data_heap = table_openrv(aqo_data_table_rv, lockmode); - - tuple_desc = RelationGetDescr(aqo_data_heap); - - data_index_rel = index_open(data_index_rel_oid, lockmode); - data_index_scan = index_beginscan(aqo_data_heap, - data_index_rel, - SnapshotSelf, - 2, - 0); + rv = makeRangeVar("public", "aqo_data", -1); + hrel = table_openrv(rv, RowExclusiveLock); + irel = index_open(reloid, RowExclusiveLock); + tupDesc = RelationGetDescr(hrel); - ScanKeyInit(&key[0], - 1, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(query_context.fspace_hash)); + InitDirtySnapshot(snap); + scan = index_beginscan(hrel, irel, &snap, 2, 0); - ScanKeyInit(&key[1], - 2, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(fss_hash)); + ScanKeyInit(&key[0], 1, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(fhash)); + ScanKeyInit(&key[1], 2, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(fsshash)); - index_rescan(data_index_scan, key, 2, NULL, 0); + index_rescan(scan, key, 2, NULL, 0); - slot = MakeSingleTupleTableSlot(data_index_scan->heapRelation->rd_att, - &TTSOpsBufferHeapTuple); - find_ok = index_getnext_slot(data_index_scan, ForwardScanDirection, slot); + slot = MakeSingleTupleTableSlot(tupDesc, &TTSOpsBufferHeapTuple); + find_ok = index_getnext_slot(scan, ForwardScanDirection, slot); if (!find_ok) { - values[0] = Int32GetDatum(query_context.fspace_hash); - values[1] = Int32GetDatum(fss_hash); + values[0] = Int32GetDatum(fhash); + values[1] = Int32GetDatum(fsshash); values[2] = Int32GetDatum(ncols); if (ncols> 0) @@ -534,26 +409,22 @@ update_fss(int fss_hash, int nrows, int ncols, double **matrix, double *targets) isnull[3] = true; values[4] = PointerGetDatum(form_vector(targets, nrows)); - tuple = heap_form_tuple(tuple_desc, values, isnull); - PG_TRY(); - { - simple_heap_insert(aqo_data_heap, tuple); - my_index_insert(data_index_rel, values, isnull, &(tuple->t_self), - aqo_data_heap, UNIQUE_CHECK_YES); - } - PG_CATCH(); - { - CommandCounterIncrement(); - simple_heap_delete(aqo_data_heap, &(tuple->t_self)); - PG_RE_THROW(); - } - PG_END_TRY(); + tuple = heap_form_tuple(tupDesc, values, isnull); + + /* + * Don't use PG_TRY() section because of dirty snapshot and caller atomic + * prerequisities guarantees to us that no one concurrent insertion can + * exists. + */ + simple_heap_insert(hrel, tuple); + my_index_insert(irel, values, isnull, &(tuple->t_self), + hrel, UNIQUE_CHECK_YES); } - else + else if (!TransactionIdIsValid(snap.xmin) && !TransactionIdIsValid(snap.xmax)) { tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); - heap_deform_tuple(tuple, aqo_data_heap->rd_att, values, isnull); + heap_deform_tuple(tuple, hrel->rd_att, values, isnull); if (ncols> 0) values[3] = PointerGetDatum(form_matrix(matrix, nrows, ncols)); @@ -561,36 +432,44 @@ update_fss(int fss_hash, int nrows, int ncols, double **matrix, double *targets) isnull[3] = true; values[4] = PointerGetDatum(form_vector(targets, nrows)); - nw_tuple = heap_modify_tuple(tuple, tuple_desc, + nw_tuple = heap_modify_tuple(tuple, tupDesc, values, isnull, replace); - if (my_simple_heap_update(aqo_data_heap, &(nw_tuple->t_self), nw_tuple, + if (my_simple_heap_update(hrel, &(nw_tuple->t_self), nw_tuple, &update_indexes)) { if (update_indexes) - my_index_insert(data_index_rel, values, isnull, + my_index_insert(irel, values, isnull, &(nw_tuple->t_self), - aqo_data_heap, UNIQUE_CHECK_YES); + hrel, UNIQUE_CHECK_YES); + result = true; } else { /* - * Ooops, somebody concurrently updated the tuple. We have to - * merge our changes somehow, but now we just discard ours. We - * don't believe in high probability of simultaneously finishing - * of two long, complex, and important queries, so we don't loss - * important data. + * Ooops, somebody concurrently updated the tuple. It is possible + * only in the case of changes made by third-party code. */ + elog(ERROR, "AQO data piece (%d %d) concurrently updated" + " by a stranger backend.", + fhash, fsshash); + result = false; } } + else + { + /* + * Concurrent update was made. To prevent deadlocks refuse to update. + */ + result = false; + } ExecDropSingleTupleTableSlot(slot); - index_endscan(data_index_scan); - index_close(data_index_rel, lockmode); - table_close(aqo_data_heap, lockmode); + index_endscan(scan); + index_close(irel, RowExclusiveLock); + table_close(hrel, RowExclusiveLock); CommandCounterIncrement(); - - return true; + return result; } /* @@ -600,62 +479,43 @@ update_fss(int fss_hash, int nrows, int ncols, double **matrix, double *targets) * is not found. */ QueryStat * -get_aqo_stat(int query_hash) +get_aqo_stat(int qhash) { - RangeVar *aqo_stat_table_rv; - Relation aqo_stat_heap; - HeapTuple tuple; - LOCKMODE heap_lock = AccessShareLock; - - Relation stat_index_rel; - Oid stat_index_rel_oid; - IndexScanDesc stat_index_scan; + RangeVar *rv; + Relation hrel; + Relation irel; + TupleTableSlot *slot; + Oid reloid; + IndexScanDesc scan; ScanKeyData key; - LOCKMODE index_lock = AccessShareLock; - - Datum values[9]; - bool nulls[9]; - QueryStat *stat = palloc_query_stat(); - - TupleTableSlot *slot; bool shouldFree; - bool find_ok = false; - stat_index_rel_oid = RelnameGetRelid("aqo_query_stat_idx"); - if (!OidIsValid(stat_index_rel_oid)) + reloid = RelnameGetRelid("aqo_query_stat_idx"); + if (!OidIsValid(reloid)) { disable_aqo_for_query(); return NULL; } - aqo_stat_table_rv = makeRangeVar("public", "aqo_query_stat", -1); - aqo_stat_heap = table_openrv(aqo_stat_table_rv, heap_lock); + rv = makeRangeVar("public", "aqo_query_stat", -1); + hrel = table_openrv(rv, AccessShareLock); + irel = index_open(reloid, AccessShareLock); - stat_index_rel = index_open(stat_index_rel_oid, index_lock); - stat_index_scan = index_beginscan(aqo_stat_heap, - stat_index_rel, - SnapshotSelf, - 1, - 0); + scan = index_beginscan(hrel, irel, SnapshotSelf, 1, 0); + ScanKeyInit(&key, 1, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(qhash)); + index_rescan(scan, &key, 1, NULL, 0); + slot = MakeSingleTupleTableSlot(hrel->rd_att, &TTSOpsBufferHeapTuple); - ScanKeyInit(&key, - 1, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(query_hash)); - - index_rescan(stat_index_scan, &key, 1, NULL, 0); - - slot = MakeSingleTupleTableSlot(stat_index_scan->heapRelation->rd_att, - &TTSOpsBufferHeapTuple); - find_ok = index_getnext_slot(stat_index_scan, ForwardScanDirection, slot); - - if (find_ok) + if (index_getnext_slot(scan, ForwardScanDirection, slot)) { + HeapTuple tuple; + Datum values[9]; + bool nulls[9]; + tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); - heap_deform_tuple(tuple, aqo_stat_heap->rd_att, values, nulls); + heap_deform_tuple(tuple, hrel->rd_att, values, nulls); DeformVectorSz(values[1], stat->execution_time_with_aqo); DeformVectorSz(values[2], stat->execution_time_without_aqo); @@ -669,10 +529,9 @@ get_aqo_stat(int query_hash) } ExecDropSingleTupleTableSlot(slot); - index_endscan(stat_index_scan); - index_close(stat_index_rel, index_lock); - table_close(aqo_stat_heap, heap_lock); - + index_endscan(scan); + index_close(irel, AccessShareLock); + table_close(hrel, AccessShareLock); return stat; } @@ -681,26 +540,16 @@ get_aqo_stat(int query_hash) * Executes disable_aqo_for_query if aqo_query_stat is not found. */ void -update_aqo_stat(int query_hash, QueryStat *stat) +update_aqo_stat(int qhash, QueryStat *stat) { - RangeVar *aqo_stat_table_rv; - Relation aqo_stat_heap; + RangeVar *rv; + Relation hrel; + Relation irel; + SnapshotData snap; + TupleTableSlot *slot; + TupleDesc tupDesc; HeapTuple tuple, nw_tuple; - TupleDesc tuple_desc; - - TupleTableSlot *slot; - bool shouldFree; - bool find_ok = false; - bool update_indexes; - - LOCKMODE lockmode = RowExclusiveLock; - - Relation stat_index_rel; - Oid stat_index_rel_oid; - IndexScanDesc stat_index_scan; - ScanKeyData key; - Datum values[9]; bool isnull[9] = { false, false, false, false, false, false, @@ -708,37 +557,29 @@ update_aqo_stat(int query_hash, QueryStat *stat) bool replace[9] = { false, true, true, true, true, true, true, true, true }; + bool shouldFree; + bool update_indexes; + Oid reloid; + IndexScanDesc scan; + ScanKeyData key; - stat_index_rel_oid = RelnameGetRelid("aqo_query_stat_idx"); - if (!OidIsValid(stat_index_rel_oid)) + reloid = RelnameGetRelid("aqo_query_stat_idx"); + if (!OidIsValid(reloid)) { disable_aqo_for_query(); return; } - aqo_stat_table_rv = makeRangeVar("public", "aqo_query_stat", -1); - aqo_stat_heap = table_openrv(aqo_stat_table_rv, lockmode); - - tuple_desc = RelationGetDescr(aqo_stat_heap); - - stat_index_rel = index_open(stat_index_rel_oid, lockmode); - stat_index_scan = index_beginscan(aqo_stat_heap, - stat_index_rel, - SnapshotSelf, - 1, - 0); + rv = makeRangeVar("public", "aqo_query_stat", -1); + hrel = table_openrv(rv, RowExclusiveLock); + irel = index_open(reloid, RowExclusiveLock); + tupDesc = RelationGetDescr(hrel); - ScanKeyInit(&key, - 1, - BTEqualStrategyNumber, - F_INT4EQ, - Int32GetDatum(query_hash)); - - index_rescan(stat_index_scan, &key, 1, NULL, 0); - - slot = MakeSingleTupleTableSlot(stat_index_scan->heapRelation->rd_att, - &TTSOpsBufferHeapTuple); - find_ok = index_getnext_slot(stat_index_scan, ForwardScanDirection, slot); + InitDirtySnapshot(snap); + scan = index_beginscan(hrel, irel, &snap, 1, 0); + ScanKeyInit(&key, 1, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(qhash)); + index_rescan(scan, &key, 1, NULL, 0); + slot = MakeSingleTupleTableSlot(hrel->rd_att, &TTSOpsBufferHeapTuple); /*values[0] will be initialized later */ values[1] = PointerGetDatum(FormVectorSz(stat->execution_time_with_aqo)); @@ -751,57 +592,53 @@ update_aqo_stat(int query_hash, QueryStat *stat) values[7] = Int64GetDatum(stat->executions_with_aqo); values[8] = Int64GetDatum(stat->executions_without_aqo); - if (!find_ok) + if (!index_getnext_slot(scan, ForwardScanDirection, slot)) { - values[0] = Int32GetDatum(query_hash); - tuple = heap_form_tuple(tuple_desc, values, isnull); - PG_TRY(); - { - simple_heap_insert(aqo_stat_heap, tuple); - my_index_insert(stat_index_rel, values, isnull, &(tuple->t_self), - aqo_stat_heap, UNIQUE_CHECK_YES); - } - PG_CATCH(); - { - CommandCounterIncrement(); - simple_heap_delete(aqo_stat_heap, &(tuple->t_self)); - PG_RE_THROW(); - } - PG_END_TRY(); + /* Such signature (hash) doesn't yet exist in the ML knowledge base. */ + values[0] = Int32GetDatum(qhash); + tuple = heap_form_tuple(tupDesc, values, isnull); + simple_heap_insert(hrel, tuple); + my_index_insert(irel, values, isnull, &(tuple->t_self), + hrel, UNIQUE_CHECK_YES); } - else + else if (!TransactionIdIsValid(snap.xmin) && !TransactionIdIsValid(snap.xmax)) { + /* Need to update ML data row and no one backend concurrently doing it. */ tuple = ExecFetchSlotHeapTuple(slot, true, &shouldFree); Assert(shouldFree != true); - values[0] = heap_getattr(tuple, 1, - RelationGetDescr(aqo_stat_heap), &isnull[0]); - nw_tuple = heap_modify_tuple(tuple, tuple_desc, - values, isnull, replace); - if (my_simple_heap_update(aqo_stat_heap, &(nw_tuple->t_self), nw_tuple, + values[0] = heap_getattr(tuple, 1, tupDesc, &isnull[0]); + nw_tuple = heap_modify_tuple(tuple, tupDesc, values, isnull, replace); + if (my_simple_heap_update(hrel, &(nw_tuple->t_self), nw_tuple, &update_indexes)) { /* NOTE: insert index tuple iff heap update succeeded! */ if (update_indexes) - my_index_insert(stat_index_rel, values, isnull, + my_index_insert(irel, values, isnull, &(nw_tuple->t_self), - aqo_stat_heap, UNIQUE_CHECK_YES); + hrel, UNIQUE_CHECK_YES); } else { /* - * Ooops, somebody concurrently updated the tuple. We have to - * merge our changes somehow, but now we just discard ours. We - * don't believe in high probability of simultaneously finishing - * of two long, complex, and important queries, so we don't loss - * important data. + * Ooops, somebody concurrently updated the tuple. It is possible + * only in the case of changes made by third-party code. */ + elog(ERROR, "AQO statistic data for query signature %d concurrently" + " updated by a stranger backend.", + qhash); } } + else + { + /* + * Concurrent update was made. To prevent deadlocks refuse to update. + */ + } ExecDropSingleTupleTableSlot(slot); - index_endscan(stat_index_scan); - index_close(stat_index_rel, lockmode); - table_close(aqo_stat_heap, lockmode); + index_endscan(scan); + index_close(irel, RowExclusiveLock); + table_close(hrel, RowExclusiveLock); CommandCounterIncrement(); } @@ -954,7 +791,7 @@ my_simple_heap_update(Relation relation, ItemPointer otid, HeapTuple tup, /* Provides correct insert in both PostgreQL 9.6.X and 10.X.X */ -static bool +bool my_index_insert(Relation indexRelation, Datum *values, bool *isnull, ItemPointer heap_t_ctid, diff --git a/t/001_pgbench.pl b/t/001_pgbench.pl new file mode 100644 index 00000000..fcb7f3fd --- /dev/null +++ b/t/001_pgbench.pl @@ -0,0 +1,48 @@ +use strict; +use warnings; +use TestLib; +use Test::More tests => 6; +use PostgresNode; + +my $node = get_new_node('aqotest'); +$node->init; +$node->append_conf('postgresql.conf', qq{ + shared_preload_libraries = 'aqo' + aqo.mode = 'intelligent' + aqo.log_ignorance = 'on' + }); + +#my $result1; + +$node->start(); + +# Check conflicts of accessing to the ML knowledge base +# intelligent mode +$node->safe_psql('postgres', "CREATE EXTENSION aqo"); +$node->safe_psql('postgres', "ALTER SYSTEM SET aqo.mode = 'intelligent'"); +$node->command_ok([ 'pgbench', '-i', '-s', '1' ], 'init pgbench tables'); +$node->command_ok([ 'pgbench', '-t', "1000", '-c', "20", '-j', "20" ], + 'pgbench in intelligent mode'); + +$node->safe_psql('postgres', "ALTER SYSTEM SET aqo.mode = 'controlled'"); +$node->command_ok([ 'pgbench', '-t', "1000", '-c', "20", '-j', "20" ], + 'pgbench in controlled mode'); + +$node->safe_psql('postgres', "ALTER SYSTEM SET aqo.mode = 'disabled'"); +$node->command_ok([ 'pgbench', '-t', "1000", '-c', "20", '-j', "20" ], + 'pgbench in disabled mode'); + +$node->safe_psql('postgres', "DROP EXTENSION aqo"); +$node->safe_psql('postgres', "CREATE EXTENSION aqo"); + +$node->safe_psql('postgres', "ALTER SYSTEM SET aqo.mode = 'learn'"); +$node->command_ok([ 'pgbench', '-t', "1000", '-c', "20", '-j', "20" ], + 'pgbench in learn mode'); + +$node->safe_psql('postgres', "ALTER SYSTEM SET aqo.mode = 'frozen'"); +$node->command_ok([ 'pgbench', '-t', "1000", '-c', "20", '-j', "20" ], + 'pgbench in frozen mode'); + +$node->safe_psql('postgres', "DROP EXTENSION aqo"); + +$node->stop(); From 436b3e6f474829a7ee1f330d7e1d011d3bf92a39 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年2月18日 16:23:06 +0500 Subject: [PATCH 18/20] Improve AQO makefile --- Makefile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 50aaea9a..ff9d7af8 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,8 @@ EXTENSION = aqo EXTVERSION = 1.2 -PGFILEDESC = "AQO - adaptive query optimization" -MODULES = aqo +PGFILEDESC = "AQO - Adaptive Query Optimization" +MODULE_big = aqo OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \ hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \ selectivity_cache.o storage.o utils.o ignorance.o $(WIN32RES) @@ -27,7 +27,6 @@ EXTRA_INSTALL = contrib/postgres_fdw DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql -MODULE_big = aqo ifdef USE_PGXS PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) From 8ec65724961b80d551633ad423f566c9a2e406ac Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: 2021年2月18日 16:27:40 +0500 Subject: [PATCH 19/20] Bugfix: don't create ignorance table in parallel worker. --- ignorance.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ignorance.c b/ignorance.c index 84c3f5d9..88bb97d1 100644 --- a/ignorance.c +++ b/ignorance.c @@ -2,6 +2,7 @@ #include "ignorance.h" #include "access/heapam.h" +#include "access/parallel.h" #include "executor/spi.h" #include "utils/lsyscache.h" #include "miscadmin.h" @@ -16,9 +17,10 @@ set_ignorance(bool newval, void *extra) * It is not problem. We will check existence at each update and create this * table in dynamic mode, if needed. */ - if (IsUnderPostmaster && newval && (aqo_log_ignorance != newval)) + if (IsUnderPostmaster && !IsParallelWorker() && newval && + (aqo_log_ignorance != newval)) /* Create storage and no error, if it exists already. */ - (bool) create_ignorance_table(true); + create_ignorance_table(true); aqo_log_ignorance = newval; } @@ -101,7 +103,7 @@ update_ignorance(int qhash, int fhash, int fss_hash, Plan *plan) if (!OidIsValid(reloid)) { /* This table doesn't created on instance startup. Create now. */ - (bool) create_ignorance_table(false); + create_ignorance_table(false); reloid = RangeVarGetRelid(rv, NoLock, true); if (!OidIsValid(reloid)) elog(PANIC, "Ignorance table does not exists!"); From d339ad019ef187fe78ecf8a9ca83c77b2d465e55 Mon Sep 17 00:00:00 2001 From: "Andrey V. Lepikhov" Date: Mon, 1 Mar 2021 10:22:41 +0300 Subject: [PATCH 20/20] Make the explain of AQO more readable. --- expected/aqo_fdw.out | 76 +++++++++++++++++++++++++------------------- expected/gucs.out | 43 ++++++++++++------------- postprocessing.c | 12 ++++--- sql/gucs.sql | 2 +- 4 files changed, 73 insertions(+), 60 deletions(-) diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index c7cb734d..23cd2f3f 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -24,23 +24,25 @@ ANALYZE local; -- Trivial foreign scan.s EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; - QUERY PLAN -------------------------------------------------------------- - Foreign Scan on frgn (actual rows=1 loops=1) (AQO not used) + QUERY PLAN +---------------------------------------------- + Foreign Scan on frgn (actual rows=1 loops=1) + AQO not used Using aqo: true AQO mode: LEARN JOINS: 0 -(4 rows) +(5 rows) EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn; - QUERY PLAN ------------------------------------------------------------------------------ - Foreign Scan on frgn (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) + QUERY PLAN +---------------------------------------------- + Foreign Scan on frgn (actual rows=1 loops=1) + AQO: rows=1, error=0% Using aqo: true AQO mode: LEARN JOINS: 0 -(4 rows) +(5 rows) -- Push down base filters. Use verbose mode to see filters. EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)) @@ -50,83 +52,93 @@ LINE 1: ...LAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE)) ^ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT x FROM frgn WHERE x < 10; - QUERY PLAN --------------------------------------------------------------------- - Foreign Scan on public.frgn (actual rows=1 loops=1) (AQO not used) + QUERY PLAN +----------------------------------------------------------- + Foreign Scan on public.frgn (actual rows=1 loops=1) + AQO not used Output: x Remote SQL: SELECT x FROM public.local WHERE ((x < 10)) Using aqo: true AQO mode: LEARN JOINS: 0 -(6 rows) +(7 rows) EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT x FROM frgn WHERE x < -10; -- AQO ignores constants - QUERY PLAN -------------------------------------------------------------------------------- - Foreign Scan on frgn (actual rows=0 loops=1) (AQO: cardinality=1, error=100%) + QUERY PLAN +---------------------------------------------- + Foreign Scan on frgn (actual rows=0 loops=1) + AQO: rows=1, error=100% Using aqo: true AQO mode: LEARN JOINS: 0 -(4 rows) +(5 rows) -- Trivial JOIN push-down. EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; - QUERY PLAN ---------------------------------------------------------------------------- - Merge Join (actual rows=1 loops=1) (AQO not used) + QUERY PLAN +------------------------------------------------------------ + Merge Join (actual rows=1 loops=1) + AQO not used Merge Cond: (a.x = b.x) - -> Sort (actual rows=1 loops=1) (AQO not used) + -> Sort (actual rows=1 loops=1) + AQO not used Sort Key: a.x Sort Method: quicksort Memory: 25kB - -> Foreign Scan on frgn a (actual rows=1 loops=1) (AQO not used) - -> Sort (actual rows=1 loops=1) (AQO not used) + -> Foreign Scan on frgn a (actual rows=1 loops=1) + AQO not used + -> Sort (actual rows=1 loops=1) + AQO not used Sort Key: b.x Sort Method: quicksort Memory: 25kB - -> Foreign Scan on frgn b (actual rows=1 loops=1) (AQO not used) + -> Foreign Scan on frgn b (actual rows=1 loops=1) + AQO not used Using aqo: true AQO mode: LEARN JOINS: 0 -(13 rows) +(18 rows) EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF, VERBOSE) SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; QUERY PLAN -------------------------------------------------------------------------------------------------------- - Foreign Scan (actual rows=1 loops=1) (AQO: cardinality=1, error=0%) + Foreign Scan (actual rows=1 loops=1) + AQO: rows=1, error=0% Output: a.x, b.x Relations: (public.frgn a) INNER JOIN (public.frgn b) Remote SQL: SELECT r1.x, r2.x FROM (public.local r1 INNER JOIN public.local r2 ON (((r1.x = r2.x)))) Using aqo: true AQO mode: LEARN JOINS: 0 -(7 rows) +(8 rows) -- TODO: Non-mergejoinable join condition. EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM frgn AS a, frgn AS b WHERE a.xstr, '\n'); + Assert(es->format == EXPLAIN_FORMAT_TEXT); + if (es->str->len == 0 || es->str->data[es->str->len - 1] == '\n') + appendStringInfoSpaces(es->str, es->indent * 2); + if (plan->predicted_cardinality> 0.) { error = 100. * (plan->predicted_cardinality - (rows*wrkrs)) / plan->predicted_cardinality; appendStringInfo(es->str, - " (AQO: cardinality=%.0lf, error=%.0lf%%", + "AQO: rows=%.0lf, error=%.0lf%%", plan->predicted_cardinality, error); } else - appendStringInfo(es->str, " (AQO not used"); + appendStringInfo(es->str, "AQO not used"); if (aqo_show_hash) - appendStringInfo(es->str, ", fss hash = %d", plan->fss_hash); - appendStringInfoChar(es->str, ')'); + appendStringInfo(es->str, ", fss=%d", plan->fss_hash); if (prev_ExplainOneNode_hook) prev_ExplainOneNode_hook(es, ps, plan, rows); diff --git a/sql/gucs.sql b/sql/gucs.sql index 5121c928..7c04d98f 100644 --- a/sql/gucs.sql +++ b/sql/gucs.sql @@ -17,7 +17,7 @@ SET aqo.log_ignorance = 'on'; SET aqo.log_ignorance = 'off'; SET aqo.log_ignorance = 'off'; SET aqo.log_ignorance = 'on'; -\d + CREATE EXTENSION aqo; SET aqo.log_ignorance = 'off'; SET aqo.log_ignorance = 'on';

AltStyle によって変換されたページ (->オリジナル) /