Jun 13, 2021 · Dec 3, 2020 · Dec 4, 2020 · Dec 4, 2020 · Dec 15, 2020 · Jan 15, 2021
diff --git a/Makefile b/Makefile
diff --git a/aqo.c b/aqo.c
diff --git a/aqo.h b/aqo.h
diff --git a/auto_tuning.c b/auto_tuning.c
diff --git a/cardinality_estimation.c b/cardinality_estimation.c
diff --git a/cardinality_hooks.c b/cardinality_hooks.c
diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out
diff --git a/expected/gucs.out b/expected/gucs.out
Original file line number	Diff line number	Diff line change
Expand Up		@@ -6,7 +6,9 @@ PGFILEDESC = "AQO - adaptive query optimization"
		MODULES = aqo
		OBJS = aqo.o auto_tuning.o cardinality_estimation.o cardinality_hooks.o \
		hash.o machine_learning.o path_utils.o postprocessing.o preprocessing.o \
	selectivity_cache.o storage.o utils.o $(WIN32RES)
	selectivity_cache.o storage.o utils.o ignorance.o $(WIN32RES)

	TAP_TESTS = 1

		REGRESS = aqo_disabled \
		aqo_controlled \
Expand All		@@ -15,7 +17,8 @@ REGRESS = aqo_disabled \
		aqo_learn \
		schema \
		aqo_fdw \
	aqo_CVE-2020-14350
	aqo_CVE-2020-14350 \
	gucs

		fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw
		PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -9,15 +9,33 @@
		*/

		#include "aqo.h"
	#include "ignorance.h"

	#include "access/relation.h"
	#include "access/table.h"
	#include "catalog/pg_extension.h"
	#include "commands/extension.h"

		PG_MODULE_MAGIC;

		void _PG_init(void);

	#define AQO_MODULE_MAGIC (1234)

		/* Strategy of determining feature space for new queries. */
		int aqo_mode;
		bool force_collect_stat;

	/*
	* Show special info in EXPLAIN mode.
	*
	* aqo_show_hash - show query class (hash) and a feature space value (hash)
	* of each plan node. This is instance-dependent value and can't be used
	* in regression and TAP tests.
	*
	* aqo_show_details - show AQO settings for this class and prediction
	* for each plan node.
	*/
		bool aqo_show_hash;
		bool aqo_show_details;

Expand Down Expand Up		@@ -106,7 +124,8 @@ _PG_init(void)
		0,
		NULL,
		NULL,
	NULL);
	NULL
	);

		DefineCustomBoolVariable(
		"aqo.force_collect_stat",
Expand Down Expand Up		@@ -147,6 +166,19 @@ _PG_init(void)
		NULL
		);

	DefineCustomBoolVariable(
	"aqo.log_ignorance",
	"Log in a special table all feature spaces for which the AQO prediction was not successful.",
	NULL,
	&aqo_log_ignorance,
	false,
	PGC_SUSET,
	0,
	NULL,
	set_ignorance,
	NULL
	);

		prev_planner_hook = planner_hook;
		planner_hook = aqo_planner;
		prev_post_parse_analyze_hook = post_parse_analyze_hook;
Expand Down Expand Up		@@ -191,3 +223,63 @@ invalidate_deactivated_queries_cache(PG_FUNCTION_ARGS)
		init_deactivated_queries_storage();
		PG_RETURN_POINTER(NULL);
		}

	/*
	* Return AQO schema's Oid or InvalidOid if that's not possible.
	*/
	Oid
	get_aqo_schema(void)
	{
	Oid result;
	Relation rel;
	SysScanDesc scandesc;
	HeapTuple tuple;
	ScanKeyData entry[1];
	Oid ext_oid;

	/* It's impossible to fetch pg_aqo's schema now */
	if (!IsTransactionState())
	return InvalidOid;

	ext_oid = get_extension_oid("aqo", true);
	if (ext_oid == InvalidOid)
	return InvalidOid; /* exit if pg_aqo does not exist */

	ScanKeyInit(&entry[0],
	#if PG_VERSION_NUM >= 120000
	Anum_pg_extension_oid,
	#else
	ObjectIdAttributeNumber,
	#endif
	BTEqualStrategyNumber, F_OIDEQ,
	ObjectIdGetDatum(ext_oid));

	rel = relation_open(ExtensionRelationId, AccessShareLock);
	scandesc = systable_beginscan(rel, ExtensionOidIndexId, true,
	NULL, 1, entry);
	tuple = systable_getnext(scandesc);

	/* We assume that there can be at most one matching tuple */
	if (HeapTupleIsValid(tuple))
	result = ((Form_pg_extension) GETSTRUCT(tuple))->extnamespace;
	else
	result = InvalidOid;

	systable_endscan(scandesc);
	relation_close(rel, AccessShareLock);
	return result;
	}

	/*
	* Init userlock
	*/
	void
	init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2)
	{
	tag->locktag_field1 = AQO_MODULE_MAGIC;
	tag->locktag_field2 = key1;
	tag->locktag_field3 = key2;
	tag->locktag_field4 = 0;
	tag->locktag_type = LOCKTAG_USERLOCK;
	tag->locktag_lockmethodid = USER_LOCKMETHOD;
	}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -267,6 +267,7 @@ extern get_parameterized_joinrel_size_hook_type
		prev_get_parameterized_joinrel_size_hook;
		extern copy_generic_path_info_hook_type prev_copy_generic_path_info_hook;
		extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
	extern ExplainOneNode_hook_type prev_ExplainOneNode_hook;

		extern void ppi_hook(ParamPathInfo *ppi);

Expand All		@@ -281,44 +282,42 @@ int get_clause_hash(Expr clause, int nargs, int args_hash, int *eclass_hash);


		/* Storage interaction */
	bool find_query(int query_hash,
	Datum *search_values,
	bool *search_nulls);
	bool add_query(int query_hash, bool learn_aqo, bool use_aqo,
	int fspace_hash, bool auto_tuning);
	bool update_query(int query_hash, bool learn_aqo, bool use_aqo,
	int fspace_hash, bool auto_tuning);
	bool add_query_text(int query_hash, const char *query_text);
	bool load_fss(int fss_hash, int ncols,
	double *matrix, double targets, int *rows);
	extern bool update_fss(int fss_hash, int nrows, int ncols,
	extern bool find_query(int qhash, Datum search_values, bool search_nulls);
	extern bool update_query(int qhash, int fhash,
	bool learn_aqo, bool use_aqo, bool auto_tuning);
	extern bool add_query_text(int query_hash, const char *query_text);
	extern bool load_fss(int fhash, int fss_hash,
	int ncols, double *matrix, double targets, int *rows);
	extern bool update_fss(int fhash, int fss_hash, int nrows, int ncols,
		double *matrix, double targets);
		QueryStat *get_aqo_stat(int query_hash);
		void update_aqo_stat(int query_hash, QueryStat * stat);
	extern bool my_index_insert(Relation indexRelation, Datum values, bool isnull,
	ItemPointer heap_t_ctid, Relation heapRelation,
	IndexUniqueCheck checkUnique);
		void init_deactivated_queries_storage(void);
		void fini_deactivated_queries_storage(void);
		bool query_is_deactivated(int query_hash);
		void add_deactivated_query(int query_hash);

		/* Query preprocessing hooks */
	void get_query_text(ParseState pstate, Query query);
	PlannedStmt call_default_planner(Query parse,
	const char *query_string,
	int cursorOptions,
	ParamListInfo boundParams);
	PlannedStmt aqo_planner(Query parse,
	const char *query_string,
	int cursorOptions,
	ParamListInfo boundParams);
	void print_into_explain(PlannedStmt plannedstmt, IntoClause into,
	ExplainState es, const char queryString,
	ParamListInfo params, const instr_time *planduration,
	QueryEnvironment *queryEnv);
	extern void print_node_explain(ExplainState *es,
	PlanState *ps,
	Plan *plan,
	extern void get_query_text(ParseState pstate, Query query);
	extern PlannedStmt call_default_planner(Query parse,
	const char *query_string,
	int cursorOptions,
	ParamListInfo boundParams);
	extern PlannedStmt aqo_planner(Query parse,
	const char *query_string,
	int cursorOptions,
	ParamListInfo boundParams);
	extern void print_into_explain(PlannedStmt plannedstmt, IntoClause into,
	ExplainState es, const char queryString,
	ParamListInfo params,
	const instr_time *planduration,
	QueryEnvironment *queryEnv);
	extern void print_node_explain(ExplainState es, PlanState ps, Plan *plan,
		double rows);
	void disable_aqo_for_query(void);
	extern void disable_aqo_for_query(void);

		/* Cardinality estimation hooks */
		extern void aqo_set_baserel_rows_estimate(PlannerInfo root, RelOptInfo rel);
Expand Down Expand Up		@@ -364,7 +363,7 @@ extern int OkNNr_learn(int matrix_rows, int matrix_cols,
		double *features, double target);

		/* Automatic query tuning */
	voidautomatical_query_tuning(int query_hash, QueryStat * stat);
	extern voidautomatical_query_tuning(int query_hash, QueryStat * stat);

		/* Utilities */
		int int_cmp(const void a, const void b);
Expand All		@@ -376,11 +375,13 @@ QueryStat *palloc_query_stat(void);
		void pfree_query_stat(QueryStat *stat);

		/* Selectivity cache for parametrized baserels */
	void cache_selectivity(int clause_hash,
	int relid,
	int global_relid,
	double selectivity);
	double *selectivity_cache_find_global_relid(int clause_hash, int global_relid);
	void selectivity_cache_clear(void);
	extern void cache_selectivity(int clause_hash, int relid, int global_relid,
	double selectivity);
	extern double *selectivity_cache_find_global_relid(int clause_hash,
	int global_relid);
	extern void selectivity_cache_clear(void);

	extern Oid get_aqo_schema(void);
	extern void init_lock_tag(LOCKTAG *tag, uint32 key1, uint32 key2);

		#endif
Original file line number	Diff line number	Diff line change
Expand Up		@@ -187,8 +187,11 @@ automatical_query_tuning(int query_hash, QueryStat * stat)
		}

		if (num_iterations <= auto_tuning_max_iterations \|\| p_use > 0.5)
	update_query(query_hash, query_context.learn_aqo, query_context.use_aqo,
	query_context.fspace_hash, true);
	update_query(query_hash,
	query_context.fspace_hash,
	query_context.learn_aqo,
	query_context.use_aqo,
	true);
		else
	update_query(query_hash, false, false, query_context.fspace_hash, false);
	update_query(query_hash, query_context.fspace_hash, false, false, false);
		}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -40,7 +40,8 @@ predict_for_relation(List restrict_clauses, List selectivities,
		for (i = 0; i < aqo_K; ++i)
		matrix[i] = palloc0(sizeof(*matrix) nfeatures);

	if (load_fss(*fss_hash, nfeatures, matrix, targets, &rows))
	if (load_fss(query_context.fspace_hash, *fss_hash, nfeatures, matrix,
	targets, &rows))
		result = OkNNr_predict(rows, nfeatures, matrix, targets, features);
		else
		{
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -156,7 +156,8 @@ aqo_set_baserel_rows_estimate(PlannerInfo root, RelOptInfo rel)
		relids = list_make1_int(relid);

		restrict_clauses = list_copy(rel->baserestrictinfo);
	predicted = predict_for_relation(restrict_clauses, selectivities, relids, &fss);
	predicted = predict_for_relation(restrict_clauses, selectivities,
	relids, &fss);
		rel->fss_hash = fss;

		if (predicted >= 0)
Expand Down
Original file line number	Diff line number	Diff line change
Expand Up		@@ -128,4 +128,10 @@ SELECT * FROM frgn AS a, frgn AS b WHERE a.x<b.x;
		JOINS: 0
		(7 rows)

	DROP EXTENSION aqo;
	DROP EXTENSION aqo CASCADE;
	DROP EXTENSION postgres_fdw CASCADE;
	NOTICE: drop cascades to 3 other objects
	DETAIL: drop cascades to server loopback
	drop cascades to user mapping for public on server loopback
	drop cascades to foreign table frgn
	DROP TABLE local;
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,93 @@
	CREATE EXTENSION aqo;
	SET aqo.mode = 'learn';
	SET aqo.show_details = true;
	CREATE TABLE t(x int);
	INSERT INTO t (x) (SELECT * FROM generate_series(1, 100) AS gs);
	ANALYZE t;
	EXPLAIN (ANALYZE, VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
	SELECT x FROM t;
	QUERY PLAN
	---------------------------------------------------------------
	Seq Scan on public.t (actual rows=100 loops=1) (AQO not used)
	Output: x
	Using aqo: true
	AQO mode: LEARN
	JOINS: 0
	(5 rows)

	EXPLAIN (ANALYZE, VERBOSE, COSTS OFF, TIMING OFF, SUMMARY OFF)
	SELECT x FROM t;
	QUERY PLAN
	---------------------------------------------------------------------------------
	Seq Scan on public.t (actual rows=100 loops=1) (AQO: cardinality=100, error=0%)
	Output: x
	Using aqo: true
	AQO mode: LEARN
	JOINS: 0
	(5 rows)

	DROP EXTENSION aqo;
	SET aqo.log_ignorance = 'on';
	SET aqo.log_ignorance = 'off';
	SET aqo.log_ignorance = 'off';
	SET aqo.log_ignorance = 'on';
	\d
	List of relations
	Schema \| Name \| Type \| Owner
	--------+------+-------+--------
	public \| t \| table \| andrey
	(1 row)

	CREATE EXTENSION aqo;
	SET aqo.log_ignorance = 'off';
	SET aqo.log_ignorance = 'on';
	SET aqo.log_ignorance = 'on';
	\d aqo_ignorance
	Table "public.aqo_ignorance"
	Column \| Type \| Collation \| Nullable \| Default
	-----------+---------+-----------+----------+---------
	qhash \| integer \| \| \|
	fhash \| integer \| \| \|
	fss_hash \| integer \| \| \|
	node_type \| integer \| \| \|
	node \| text \| \| \|
	Indexes:
	"aqo_ignorance_idx" UNIQUE, btree (qhash, fhash, fss_hash)

	EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF)
	SELECT * FROM t;
	QUERY PLAN
	--------------------------------------------------------
	Seq Scan on t (actual rows=100 loops=1) (AQO not used)
	Using aqo: true
	AQO mode: LEARN
	JOINS: 0
	(4 rows)

	SELECT node_type FROM aqo_ignorance; -- SeqScan on t must be existed in ignorance table
	node_type
	-----------
	19
	(1 row)

	EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT * FROM t;
	QUERY PLAN
	--------------------------------------------------------------------------
	Seq Scan on t (actual rows=100 loops=1) (AQO: cardinality=100, error=0%)
	Using aqo: true
	AQO mode: LEARN
	JOINS: 0
	(4 rows)

	SELECT node_type FROM aqo_ignorance; -- SeqScan on t must be excluded from ignorance table
	node_type
	-----------
	(0 rows)

	-- This GUC can be changed by an admin only.
	CREATE ROLE noadmin;
	SET ROLE noadmin;
	SET aqo.log_ignorance = 'off';
	ERROR: permission denied to set parameter "aqo.log_ignorance"
	RESET ROLE;
	DROP EXTENSION aqo;