diff --git a/Makefile b/Makefile index ce9d00ba..1da2994c 100755 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ EXTRA_INSTALL = contrib/postgres_fdw contrib/pg_stat_statements DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql \ aqo--1.2--1.3.sql aqo--1.3--1.4.sql aqo--1.4--1.5.sql \ - aqo--1.5--1.6.sql + aqo--1.5--1.6.sql aqo--1.6.sql ifdef USE_PGXS PG_CONFIG ?= pg_config diff --git a/aqo--1.6.sql b/aqo--1.6.sql new file mode 100644 index 00000000..90d4fb06 --- /dev/null +++ b/aqo--1.6.sql @@ -0,0 +1,208 @@ +/* contrib/aqo/aqo--1.6.sql */ + +-- complain if script is sourced in psql, rather than via CREATE EXTENSION +\echo Use "CREATE EXTENSION aqo" to load this file. \quit + +-- +-- Get cardinality error of queries the last time they were executed. +-- IN: +-- controlled - show queries executed under a control of AQO (true); +-- executed without an AQO control, but AQO has a stat on the query (false). +-- +-- OUT: +-- num - sequental number. Smaller number corresponds to higher error. +-- id - ID of a query. +-- fshash - feature space. Usually equal to zero or ID. +-- error - AQO error that calculated on plan nodes of the query. +-- nexecs - number of executions of queries associated with this ID. +-- +CREATE FUNCTION aqo_cardinality_error(controlled boolean) +RETURNS TABLE(num integer, id bigint, fshash bigint, error double precision, nexecs bigint) +AS 'MODULE_PATHNAME', 'aqo_cardinality_error' +LANGUAGE C STRICT VOLATILE; +COMMENT ON FUNCTION aqo_cardinality_error(boolean) IS +'Get cardinality error of queries the last time they were executed. Order queries according to an error value.'; + +-- +-- Remove unneeded rows from the AQO ML storage. +-- For common feature space, remove rows from aqo_data only. +-- For custom feature space - remove all rows related to the space from all AQO +-- tables even if only one oid for one feature subspace of the space is illegal. +-- Returns number of deleted rows from aqo_queries and aqo_data tables. +-- +CREATE FUNCTION aqo_cleanup(OUT nfs integer, OUT nfss integer) +RETURNS record +AS 'MODULE_PATHNAME', 'aqo_cleanup' +LANGUAGE C STRICT VOLATILE; +COMMENT ON FUNCTION aqo_cleanup() IS +'Remove unneeded rows from the AQO ML storage'; + +CREATE FUNCTION aqo_disable_class(queryid bigint) +RETURNS void +AS 'MODULE_PATHNAME', 'aqo_disable_query' +LANGUAGE C STRICT VOLATILE; +COMMENT ON FUNCTION aqo_disable_class(bigint) IS +'Set learn_aqo, use_aqo and auto_tuning into false for a class of queries with specific queryid.'; + +-- +-- Remove query class settings, text, statistics and ML data from AQO storage. +-- Return number of FSS records, removed from the storage. +-- +CREATE FUNCTION aqo_drop_class(queryid bigint) +RETURNS integer +AS 'MODULE_PATHNAME', 'aqo_drop_class' +LANGUAGE C STRICT VOLATILE; +COMMENT ON FUNCTION aqo_drop_class(bigint) IS +'Remove info about an query class from AQO ML knowledge base.'; + +CREATE FUNCTION aqo_enable_class(queryid bigint) +RETURNS void +AS 'MODULE_PATHNAME', 'aqo_enable_query' +LANGUAGE C STRICT VOLATILE; +COMMENT ON FUNCTION aqo_enable_class(bigint) IS +'Set learn_aqo, use_aqo and auto_tuning (in intelligent mode) into true for a class of queries with specific queryid.'; + +-- +-- Show execution time of queries, for which AQO has statistics. +-- controlled - show stat on executions where AQO was used for cardinality +-- estimations, or not used (controlled = false). +-- Last case is possible in disabled mode with aqo.force_collect_stat = 'on'. +-- +CREATE FUNCTION aqo_execution_time(controlled boolean) +RETURNS TABLE(num integer, id bigint, fshash bigint, exec_time double precision, nexecs bigint) +AS 'MODULE_PATHNAME', 'aqo_execution_time' +LANGUAGE C STRICT VOLATILE; +COMMENT ON FUNCTION aqo_execution_time(boolean) IS +'Get execution time of queries. If controlled = true (AQO could advise cardinality estimations), show time of last execution attempt. Another case (AQO not used), return an average value of execution time across all known executions.'; + +-- Show how much shared memory AQO are using at the moment +CREATE FUNCTION aqo_memory_usage( + OUT name text, + OUT allocated_size int, + OUT used_size int +) +RETURNS SETOF record +AS $$ + SELECT name, allocated_size, size FROM pg_shmem_allocations + WHERE name LIKE 'AQO%'; +$$ LANGUAGE SQL; +COMMENT ON FUNCTION aqo_memory_usage() IS +'Show how much shared memory AQO are using at the moment'; + +-- +-- Update or insert an aqo_data +-- table record for given 'fs' & 'fss'. +-- + +CREATE FUNCTION aqo_data_update( + fs bigint, + fss integer, + nfeatures integer, + features double precision[][], + targets double precision[], + reliability double precision[], + oids Oid[]) +RETURNS bool +AS 'MODULE_PATHNAME', 'aqo_data_update' +LANGUAGE C VOLATILE; + +CREATE FUNCTION aqo_queries_update( + queryid bigint, fs bigint, learn_aqo bool, use_aqo bool, auto_tuning bool) +RETURNS bool +AS 'MODULE_PATHNAME', 'aqo_queries_update' +LANGUAGE C VOLATILE; + +-- +-- Update or insert an aqo_query_stat +-- table record for given 'queryid'. +-- +CREATE FUNCTION aqo_query_stat_update( + queryid bigint, + execution_time_with_aqo double precision[], + execution_time_without_aqo double precision[], + planning_time_with_aqo double precision[], + planning_time_without_aqo double precision[], + cardinality_error_with_aqo double precision[], + cardinality_error_without_aqo double precision[], + executions_with_aqo bigint, + executions_without_aqo bigint) +RETURNS bool +AS 'MODULE_PATHNAME', 'aqo_query_stat_update' +LANGUAGE C VOLATILE; + +-- +-- Update or insert an aqo_query_texts +-- table record for given 'queryid'. +-- +CREATE FUNCTION aqo_query_texts_update( + queryid bigint, query_text text) +RETURNS bool +AS 'MODULE_PATHNAME', 'aqo_query_texts_update' +LANGUAGE C VOLATILE; + +-- +-- Remove all records in the AQO storage. +-- Return number of rows removed. +-- +CREATE FUNCTION aqo_reset() RETURNS bigint +AS 'MODULE_PATHNAME', 'aqo_reset' +LANGUAGE C PARALLEL SAFE; +COMMENT ON FUNCTION aqo_reset() IS +'Reset all data gathered by AQO'; + +-- ----------------------------------------------------------------------------- +-- +-- VIEWs +-- +-- ----------------------------------------------------------------------------- + +CREATE FUNCTION aqo_data ( + OUT fs bigint, + OUT fss integer, + OUT nfeatures integer, + OUT features double precision[][], + OUT targets double precision[], + OUT reliability double precision[], + OUT oids Oid[] +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'aqo_data' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE FUNCTION aqo_queries ( + OUT queryid bigint, + OUT fs bigint, + OUT learn_aqo boolean, + OUT use_aqo boolean, + OUT auto_tuning boolean, + OUT smart_timeout bigint, + OUT count_increase_timeout bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'aqo_queries' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE FUNCTION aqo_query_stat ( + OUT queryid bigint, + OUT execution_time_with_aqo double precision[], + OUT execution_time_without_aqo double precision[], + OUT planning_time_with_aqo double precision[], + OUT planning_time_without_aqo double precision[], + OUT cardinality_error_with_aqo double precision[], + OUT cardinality_error_without_aqo double precision[], + OUT executions_with_aqo bigint, + OUT executions_without_aqo bigint +) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'aqo_query_stat' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE FUNCTION aqo_query_texts(OUT queryid bigint, OUT query_text text) +RETURNS SETOF record +AS 'MODULE_PATHNAME', 'aqo_query_texts' +LANGUAGE C STRICT VOLATILE PARALLEL SAFE; + +CREATE VIEW aqo_data AS SELECT * FROM aqo_data(); +CREATE VIEW aqo_queries AS SELECT * FROM aqo_queries(); +CREATE VIEW aqo_query_stat AS SELECT * FROM aqo_query_stat(); +CREATE VIEW aqo_query_texts AS SELECT * FROM aqo_query_texts(); diff --git a/aqo.c b/aqo.c index 743f9ef6..33b56737 100644 --- a/aqo.c +++ b/aqo.c @@ -19,9 +19,7 @@ #include "aqo.h" #include "aqo_shared.h" -#include "cardinality_hooks.h" #include "path_utils.h" -#include "preprocessing.h" #include "storage.h" @@ -98,19 +96,6 @@ MemoryContext AQOLearnMemCtx = NULL; /* Additional plan info */ int njoins; -/* Saved hook values */ -post_parse_analyze_hook_type prev_post_parse_analyze_hook; -planner_hook_type prev_planner_hook; -ExecutorStart_hook_type prev_ExecutorStart_hook; -ExecutorRun_hook_type prev_ExecutorRun; -ExecutorEnd_hook_type prev_ExecutorEnd_hook; -set_baserel_rows_estimate_hook_type prev_set_foreign_rows_estimate_hook; -set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook; -get_parameterized_baserel_size_hook_type prev_get_parameterized_baserel_size_hook; -set_joinrel_size_estimates_hook_type prev_set_joinrel_size_estimates_hook; -get_parameterized_joinrel_size_hook_type prev_get_parameterized_joinrel_size_hook; -ExplainOnePlan_hook_type prev_ExplainOnePlan_hook; -ExplainOneNode_hook_type prev_ExplainOneNode_hook; /***************************************************************************** * @@ -324,42 +309,11 @@ _PG_init(void) NULL, NULL); - prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = aqo_init_shmem; - prev_planner_hook = planner_hook; - planner_hook = aqo_planner; - prev_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = aqo_ExecutorStart; - prev_ExecutorRun = ExecutorRun_hook; - ExecutorRun_hook = aqo_ExecutorRun; - prev_ExecutorEnd_hook = ExecutorEnd_hook; - ExecutorEnd_hook = aqo_ExecutorEnd; - - /* Cardinality prediction hooks. */ - prev_set_baserel_rows_estimate_hook = set_baserel_rows_estimate_hook; - set_foreign_rows_estimate_hook = aqo_set_baserel_rows_estimate; - set_baserel_rows_estimate_hook = aqo_set_baserel_rows_estimate; - prev_get_parameterized_baserel_size_hook = get_parameterized_baserel_size_hook; - get_parameterized_baserel_size_hook = aqo_get_parameterized_baserel_size; - prev_set_joinrel_size_estimates_hook = set_joinrel_size_estimates_hook; - set_joinrel_size_estimates_hook = aqo_set_joinrel_size_estimates; - prev_get_parameterized_joinrel_size_hook = get_parameterized_joinrel_size_hook; - get_parameterized_joinrel_size_hook = aqo_get_parameterized_joinrel_size; - prev_estimate_num_groups_hook = estimate_num_groups_hook; - estimate_num_groups_hook = aqo_estimate_num_groups_hook; - parampathinfo_postinit_hook = ppi_hook; - - prev_create_plan_hook = create_plan_hook; - create_plan_hook = aqo_create_plan_hook; - - /* Service hooks. */ - prev_ExplainOnePlan_hook = ExplainOnePlan_hook; - ExplainOnePlan_hook = print_into_explain; - prev_ExplainOneNode_hook = ExplainOneNode_hook; - ExplainOneNode_hook = print_node_explain; - - prev_create_upper_paths_hook = create_upper_paths_hook; - create_upper_paths_hook = aqo_store_upper_signature_hook; + aqo_shmem_init(); + aqo_preprocessing_init(); + aqo_postprocessing_init(); + aqo_cardinality_hooks_init(); + aqo_path_utils_init(); init_deactivated_queries_storage(); @@ -394,7 +348,6 @@ _PG_init(void) RegisterAQOPlanNodeMethods(); EmitWarningsOnPlaceholders("aqo"); - RequestAddinShmemSpace(aqo_memsize()); } /* diff --git a/aqo.h b/aqo.h index 9600b136..6f57a4d1 100644 --- a/aqo.h +++ b/aqo.h @@ -132,7 +132,6 @@ #include "nodes/nodeFuncs.h" #include "optimizer/pathnode.h" #include "optimizer/planner.h" -#include "optimizer/cost.h" #include "parser/analyze.h" #include "parser/parsetree.h" #include "utils/builtins.h" @@ -140,11 +139,9 @@ #include "utils/hsearch.h" #include "utils/memutils.h" #include "utils/rel.h" -#include "utils/fmgroids.h" #include "utils/snapmgr.h" #include "machine_learning.h" -//#include "storage.h" /* Check PostgreSQL version (9.6.0 contains important changes in planner) */ #if PG_VERSION_NUM < 90600 @@ -237,58 +234,15 @@ extern MemoryContext AQOCacheMemCtx; extern MemoryContext AQOPredictMemCtx; extern MemoryContext AQOLearnMemCtx; -/* Saved hook values in case of unload */ -extern post_parse_analyze_hook_type prev_post_parse_analyze_hook; -extern planner_hook_type prev_planner_hook; -extern ExecutorStart_hook_type prev_ExecutorStart_hook; -extern ExecutorRun_hook_type prev_ExecutorRun; -extern ExecutorEnd_hook_type prev_ExecutorEnd_hook; -extern set_baserel_rows_estimate_hook_type - prev_set_foreign_rows_estimate_hook; -extern set_baserel_rows_estimate_hook_type - prev_set_baserel_rows_estimate_hook; -extern get_parameterized_baserel_size_hook_type - prev_get_parameterized_baserel_size_hook; -extern set_joinrel_size_estimates_hook_type - prev_set_joinrel_size_estimates_hook; -extern get_parameterized_joinrel_size_hook_type - prev_get_parameterized_joinrel_size_hook; -extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook; -extern ExplainOneNode_hook_type prev_ExplainOneNode_hook; - -extern void ppi_hook(ParamPathInfo *ppi); extern int aqo_statement_timeout; -/* Hash functions */ -void get_eclasses(List *clauselist, int *nargs, int **args_hash, - int **eclass_hash); -int get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash); - - -/* Storage interaction */ -extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids); -extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids); - -/* Query preprocessing hooks */ -extern void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, - ParamListInfo params, - const instr_time *planduration, - QueryEnvironment *queryEnv); -extern void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan); - /* Cardinality estimation */ extern double predict_for_relation(List *restrict_clauses, List *selectivities, List *relsigns, int *fss); -/* Query execution statistics collecting hooks */ -void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags); -void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, - uint64 count, bool execute_once); -void aqo_ExecutorEnd(QueryDesc *queryDesc); - /* Automatic query tuning */ extern void automatical_query_tuning(uint64 query_hash, struct StatEntry *stat); +extern double get_mean(double *elems, int nelems); /* Utilities */ extern int int_cmp(const void *a, const void *b); @@ -306,8 +260,10 @@ extern void selectivity_cache_clear(void); extern bool IsQueryDisabled(void); -extern bool update_query_timeout(uint64 queryid, int64 smart_timeout); -extern double get_mean(double *elems, int nelems); - extern List *cur_classes; + +extern void aqo_cardinality_hooks_init(void); +extern void aqo_preprocessing_init(void); +extern void aqo_postprocessing_init(void); + #endif diff --git a/aqo_shared.c b/aqo_shared.c index 0a6a8db6..b7cfced8 100644 --- a/aqo_shared.c +++ b/aqo_shared.c @@ -6,27 +6,29 @@ #include "lib/dshash.h" #include "miscadmin.h" +#include "storage/ipc.h" #include "storage/shmem.h" #include "aqo_shared.h" #include "storage.h" -shmem_startup_hook_type prev_shmem_startup_hook = NULL; AQOSharedState *aqo_state = NULL; int fs_max_items = 10000; /* Max number of different feature spaces in ML model */ int fss_max_items = 100000; /* Max number of different feature subspaces in ML model */ +static shmem_startup_hook_type aqo_shmem_startup_next = NULL; + static void on_shmem_shutdown(int code, Datum arg); -void +static void aqo_init_shmem(void) { bool found; HASHCTL info; - if (prev_shmem_startup_hook) - prev_shmem_startup_hook(); + if (aqo_shmem_startup_next) + (*aqo_shmem_startup_next)(); aqo_state = NULL; stat_htab = NULL; @@ -116,10 +118,14 @@ on_shmem_shutdown(int code, Datum arg) return; } -Size -aqo_memsize(void) + +/* + * Requests any additional shared memory required for aqo. + */ +static void +aqo_shmem_request(void) { - Size size; + Size size; size = MAXALIGN(sizeof(AQOSharedState)); size = add_size(size, hash_estimate_size(fs_max_items, sizeof(AQOSharedState))); @@ -128,5 +134,14 @@ aqo_memsize(void) size = add_size(size, hash_estimate_size(fss_max_items, sizeof(DataEntry))); size = add_size(size, hash_estimate_size(fs_max_items, sizeof(QueriesEntry))); - return size; + RequestAddinShmemSpace(size); +} + +void +aqo_shmem_init(void) +{ + aqo_shmem_startup_next = shmem_startup_hook; + shmem_startup_hook = aqo_init_shmem; + + aqo_shmem_request(); } diff --git a/aqo_shared.h b/aqo_shared.h index e922fb1c..ee9e3087 100644 --- a/aqo_shared.h +++ b/aqo_shared.h @@ -1,9 +1,6 @@ #ifndef AQO_SHARED_H #define AQO_SHARED_H -#include "lib/dshash.h" -#include "storage/dsm.h" -#include "storage/ipc.h" #include "storage/lwlock.h" #include "utils/dsa.h" @@ -31,13 +28,11 @@ typedef struct AQOSharedState } AQOSharedState; -extern shmem_startup_hook_type prev_shmem_startup_hook; extern AQOSharedState *aqo_state; extern int fs_max_items; /* Max number of feature spaces that AQO can operate */ extern int fss_max_items; -extern Size aqo_memsize(void); -extern void aqo_init_shmem(void); +extern void aqo_shmem_init(void); #endif /* AQO_SHARED_H */ diff --git a/cardinality_hooks.c b/cardinality_hooks.c index 049f674f..1520b4e7 100644 --- a/cardinality_hooks.c +++ b/cardinality_hooks.c @@ -27,115 +27,37 @@ #include "postgres.h" +#include "optimizer/cost.h" +#include "utils/selfuncs.h" + #include "aqo.h" -#include "cardinality_hooks.h" #include "hash.h" #include "machine_learning.h" #include "path_utils.h" - -estimate_num_groups_hook_type prev_estimate_num_groups_hook = NULL; +#include "storage.h" double predicted_ppi_rows; double fss_ppi_hash; - -/* - * Calls standard set_baserel_rows_estimate or its previous hook. - */ -static void -default_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) -{ - if (prev_set_baserel_rows_estimate_hook) - prev_set_baserel_rows_estimate_hook(root, rel); - else - set_baserel_rows_estimate_standard(root, rel); -} - /* - * Calls standard get_parameterized_baserel_size or its previous hook. + * Cardinality prediction hooks. + * It isn't clear what to do if someone else tries to live in this chain. + * Of course, someone may want to just report some stat or something like that. + * So, it can be legal, sometimees. So far, we only report this fact. */ -static double -default_get_parameterized_baserel_size(PlannerInfo *root, - RelOptInfo *rel, - List *param_clauses) -{ - if (prev_get_parameterized_baserel_size_hook) - return prev_get_parameterized_baserel_size_hook(root, rel, param_clauses); - else - return get_parameterized_baserel_size_standard(root, rel, param_clauses); -} - -/* - * Calls standard get_parameterized_joinrel_size or its previous hook. - */ -static double -default_get_parameterized_joinrel_size(PlannerInfo *root, - RelOptInfo *rel, - Path *outer_path, - Path *inner_path, - SpecialJoinInfo *sjinfo, - List *restrict_clauses) -{ - if (prev_get_parameterized_joinrel_size_hook) - return prev_get_parameterized_joinrel_size_hook(root, rel, - outer_path, - inner_path, - sjinfo, - restrict_clauses); - else - return get_parameterized_joinrel_size_standard(root, rel, - outer_path, - inner_path, - sjinfo, - restrict_clauses); -} - -/* - * Calls standard set_joinrel_size_estimates or its previous hook. - */ -static void -default_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, - RelOptInfo *outer_rel, - RelOptInfo *inner_rel, - SpecialJoinInfo *sjinfo, - List *restrictlist) -{ - if (prev_set_joinrel_size_estimates_hook) - prev_set_joinrel_size_estimates_hook(root, rel, - outer_rel, - inner_rel, - sjinfo, - restrictlist); - else - set_joinrel_size_estimates_standard(root, rel, - outer_rel, - inner_rel, - sjinfo, - restrictlist); -} - -static double -default_estimate_num_groups(PlannerInfo *root, List *groupExprs, - Path *subpath, RelOptInfo *grouped_rel, - List **pgset) -{ - double input_rows = subpath->rows; - - if (prev_estimate_num_groups_hook != NULL) - return (*prev_estimate_num_groups_hook)(root, groupExprs, - subpath, - grouped_rel, - pgset); - else - return estimate_num_groups(root, groupExprs, input_rows, pgset); -} +static set_baserel_rows_estimate_hook_type aqo_set_baserel_rows_estimate_next = NULL; +static get_parameterized_baserel_size_hook_type aqo_get_parameterized_baserel_size_next = NULL; +static set_joinrel_size_estimates_hook_type aqo_set_joinrel_size_estimates_next = NULL; +static get_parameterized_joinrel_size_hook_type aqo_get_parameterized_joinrel_size_next = NULL; +static set_parampathinfo_postinit_hook_type aqo_set_parampathinfo_postinit_next = NULL; +static estimate_num_groups_hook_type aqo_estimate_num_groups_next = NULL; /* * Our hook for setting baserel rows estimate. * Extracts clauses, their selectivities and list of relation relids and * passes them to predict_for_relation. */ -void +static void aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) { double predicted; @@ -178,25 +100,37 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) /* Return to the caller's memory context. */ MemoryContextSwitchTo(old_ctx_m); - if (predicted >= 0) - { - rel->rows = predicted; - rel->predicted_cardinality = predicted; - return; - } + if (predicted < 0) + goto default_estimator; + + if ((aqo_set_baserel_rows_estimate_next != set_baserel_rows_estimate_standard || + set_baserel_rows_estimate_hook != aqo_set_baserel_rows_estimate)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the set_baserel_rows_estimate_hook chain"); + + rel->rows = predicted; + rel->predicted_cardinality = predicted; + return; default_estimator: rel->predicted_cardinality = -1.; - default_set_baserel_rows_estimate(root, rel); + (*aqo_set_baserel_rows_estimate_next)(root, rel); } - -void -ppi_hook(ParamPathInfo *ppi) +static void +aqo_parampathinfo_postinit(ParamPathInfo *ppi) { + if (aqo_set_parampathinfo_postinit_next) + (*aqo_set_parampathinfo_postinit_next)(ppi); + if (IsQueryDisabled()) return; + if ((aqo_set_parampathinfo_postinit_next != NULL || + parampathinfo_postinit_hook != aqo_parampathinfo_postinit)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the parampathinfo_postinit_hook chain"); + ppi->predicted_ppi_rows = predicted_ppi_rows; ppi->fss_ppi_hash = fss_ppi_hash; } @@ -206,7 +140,7 @@ ppi_hook(ParamPathInfo *ppi) * Extracts clauses (including parametrization ones), their selectivities * and list of relation relids and passes them to predict_for_relation. */ -double +static double aqo_get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, List *param_clauses) @@ -280,11 +214,18 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root, predicted_ppi_rows = predicted; fss_ppi_hash = fss; - if (predicted >= 0) - return predicted; + if (predicted < 0) + goto default_estimator; + + if ((aqo_get_parameterized_baserel_size_next != get_parameterized_baserel_size_standard || + get_parameterized_baserel_size_hook != aqo_get_parameterized_baserel_size)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the aqo_get_parameterized_baserel_size_next chain"); + + return predicted; default_estimator: - return default_get_parameterized_baserel_size(root, rel, param_clauses); + return (*aqo_get_parameterized_baserel_size_next)(root, rel, param_clauses); } /* @@ -292,7 +233,7 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root, * Extracts clauses, their selectivities and list of relation relids and * passes them to predict_for_relation. */ -void +static void aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, RelOptInfo *outer_rel, RelOptInfo *inner_rel, @@ -345,18 +286,22 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, rel->fss_hash = fss; - if (predicted >= 0) - { - rel->predicted_cardinality = predicted; - rel->rows = predicted; - return; - } + if (predicted < 0) + goto default_estimator; + + if ((aqo_set_joinrel_size_estimates_next != set_joinrel_size_estimates_standard || + set_joinrel_size_estimates_hook != aqo_set_joinrel_size_estimates)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the set_joinrel_size_estimates_hook chain"); + + rel->predicted_cardinality = predicted; + rel->rows = predicted; + return; default_estimator: rel->predicted_cardinality = -1; - default_set_joinrel_size_estimates(root, rel, - outer_rel, inner_rel, - sjinfo, restrictlist); + (*aqo_set_joinrel_size_estimates_next)(root, rel, outer_rel, inner_rel, + sjinfo, restrictlist); } /* @@ -364,7 +309,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, * Extracts clauses (including parametrization ones), their selectivities * and list of relation relids and passes them to predict_for_relation. */ -double +static double aqo_get_parameterized_joinrel_size(PlannerInfo *root, RelOptInfo *rel, Path *outer_path, @@ -417,11 +362,18 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root, predicted_ppi_rows = predicted; fss_ppi_hash = fss; - if (predicted >= 0) - return predicted; + if (predicted < 0) + goto default_estimator; + + if ((aqo_get_parameterized_joinrel_size_next != get_parameterized_joinrel_size_standard || + get_parameterized_joinrel_size_hook != aqo_get_parameterized_joinrel_size)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the get_parameterized_joinrel_size_hook chain"); + + return predicted; default_estimator: - return default_get_parameterized_joinrel_size(root, rel, + return (*aqo_get_parameterized_joinrel_size_next)(root, rel, outer_path, inner_path, sjinfo, clauses); } @@ -460,10 +412,10 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs, return (prediction <= 0) ? -1 : prediction; } -double -aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, - Path *subpath, RelOptInfo *grouped_rel, - List **pgset) +static double +aqo_estimate_num_groups(PlannerInfo *root, List *groupExprs, + Path *subpath, RelOptInfo *grouped_rel, + List **pgset) { int fss; double predicted; @@ -476,8 +428,10 @@ aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, /* XXX: Don't support some GROUPING options */ goto default_estimator; - if (prev_estimate_num_groups_hook != NULL) - elog(WARNING, "AQO replaced another estimator of a groups number"); + if (aqo_estimate_num_groups_next != NULL || + estimate_num_groups_hook != aqo_estimate_num_groups) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the estimate_num_groups_hook chain"); if (groupExprs == NIL) return 1.0; @@ -503,6 +457,44 @@ aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, MemoryContextSwitchTo(old_ctx_m); default_estimator: - return default_estimate_num_groups(root, groupExprs, subpath, grouped_rel, - pgset); + if (aqo_estimate_num_groups_next) + return (*aqo_estimate_num_groups_next)(root, groupExprs, subpath, + grouped_rel, pgset); + else + return estimate_num_groups(root, groupExprs, subpath->rows, + pgset); +} + +void +aqo_cardinality_hooks_init(void) +{ + if (set_baserel_rows_estimate_hook || + set_foreign_rows_estimate_hook || + get_parameterized_baserel_size_hook || + set_joinrel_size_estimates_hook || + get_parameterized_joinrel_size_hook || + parampathinfo_postinit_hook || + estimate_num_groups_hook) + elog(ERROR, "AQO estimation hooks shouldn't be intercepted"); + + aqo_set_baserel_rows_estimate_next = set_baserel_rows_estimate_standard; + set_baserel_rows_estimate_hook = aqo_set_baserel_rows_estimate; + + /* XXX: we have a problem here. Should be redesigned later */ + set_foreign_rows_estimate_hook = aqo_set_baserel_rows_estimate; + + aqo_get_parameterized_baserel_size_next = get_parameterized_baserel_size_standard; + get_parameterized_baserel_size_hook = aqo_get_parameterized_baserel_size; + + aqo_set_joinrel_size_estimates_next = set_joinrel_size_estimates_standard; + set_joinrel_size_estimates_hook = aqo_set_joinrel_size_estimates; + + aqo_get_parameterized_joinrel_size_next = get_parameterized_joinrel_size_standard; + get_parameterized_joinrel_size_hook = aqo_get_parameterized_joinrel_size; + + aqo_set_parampathinfo_postinit_next = parampathinfo_postinit_hook; + parampathinfo_postinit_hook = aqo_parampathinfo_postinit; + + aqo_estimate_num_groups_next = estimate_num_groups_hook; + estimate_num_groups_hook = aqo_estimate_num_groups; } diff --git a/cardinality_hooks.h b/cardinality_hooks.h deleted file mode 100644 index 0e8c65c0..00000000 --- a/cardinality_hooks.h +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef CARDINALITY_HOOKS_H -#define CARDINALITY_HOOKS_H - -#include "optimizer/planner.h" -#include "utils/selfuncs.h" - -extern estimate_num_groups_hook_type prev_estimate_num_groups_hook; - - -/* Cardinality estimation hooks */ -extern void aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel); -extern double aqo_get_parameterized_baserel_size(PlannerInfo *root, - RelOptInfo *rel, - List *param_clauses); -extern void aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, - RelOptInfo *outer_rel, - RelOptInfo *inner_rel, - SpecialJoinInfo *sjinfo, - List *restrictlist); -extern double aqo_get_parameterized_joinrel_size(PlannerInfo *root, - RelOptInfo *rel, - Path *outer_path, - Path *inner_path, - SpecialJoinInfo *sjinfo, - List *restrict_clauses); -extern double aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, - Path *subpath, - RelOptInfo *grouped_rel, - List **pgset); - -#endif /* CARDINALITY_HOOKS_H */ diff --git a/expected/aqo_CVE-2020-14350.out b/expected/aqo_CVE-2020-14350.out index 8685b935..5deb45ae 100644 --- a/expected/aqo_CVE-2020-14350.out +++ b/expected/aqo_CVE-2020-14350.out @@ -49,51 +49,32 @@ SHOW is_superuser; off (1 row) -CREATE FUNCTION aqo_status(hash bigint) -RETURNS TABLE ( - "learn" BOOL, - "use aqo" BOOL, - "auto tune" BOOL, - "fspace hash" bigINT, - "t_naqo" TEXT, - "err_naqo" TEXT, - "iters" BIGINT, - "t_aqo" TEXT, - "err_aqo" TEXT, - "iters_aqo" BIGINT -) +CREATE FUNCTION aqo_reset() +RETURNS bigint AS $$ BEGIN + RETURN 1; END $$ LANGUAGE plpgsql; RESET ROLE; -- Test result (error expected) CREATE EXTENSION aqo; -ERROR: function "aqo_status" already exists with same argument types +ERROR: function "aqo_reset" already exists with same argument types SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_status(hash bigint) -RETURNS TABLE ( - "learn" BOOL, - "use aqo" BOOL, - "auto tune" BOOL, - "fspace hash" bigINT, - "t_naqo" TEXT, - "err_naqo" TEXT, - "iters" BIGINT, - "t_aqo" TEXT, - "err_aqo" TEXT, - "iters_aqo" BIGINT -) +CREATE OR REPLACE FUNCTION aqo_reset() +RETURNS bigint AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; + RETURN 2; END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_status(42); - aqo_status ------------- -(0 rows) +SELECT aqo_reset(); + aqo_reset +----------- + 2 +(1 row) SET ROLE regress_hacker; SHOW is_superuser; @@ -103,7 +84,7 @@ SHOW is_superuser; (1 row) RESET ROLE; -DROP FUNCTION aqo_status(bigint); +DROP FUNCTION aqo_reset(); DROP EXTENSION IF EXISTS aqo; NOTICE: extension "aqo" does not exist, skipping -- Test 3 @@ -208,29 +189,31 @@ SHOW is_superuser; off (1 row) -CREATE FUNCTION aqo_clear_hist(hash bigint) -RETURNS VOID +CREATE FUNCTION aqo_drop_class(queryid bigint) +RETURNS integer AS $$ BEGIN + RETURN 1; END $$ LANGUAGE plpgsql; RESET ROLE; -- Test result (error expected) CREATE EXTENSION aqo; -ERROR: function "aqo_clear_hist" already exists with same argument types +ERROR: function "aqo_drop_class" already exists with same argument types SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_clear_hist(hash bigint) -RETURNS VOID +CREATE OR REPLACE FUNCTION aqo_drop_class(queryid bigint) +RETURNS integer AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; + RETURN 2; END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_clear_hist(42); - aqo_clear_hist +SELECT aqo_drop_class(42); + aqo_drop_class ---------------- - + 2 (1 row) SET ROLE regress_hacker; @@ -241,7 +224,7 @@ SHOW is_superuser; (1 row) RESET ROLE; -DROP FUNCTION aqo_clear_hist(bigint); +DROP FUNCTION aqo_drop_class(bigint); DROP EXTENSION IF EXISTS aqo; NOTICE: extension "aqo" does not exist, skipping -- Test 6 @@ -254,8 +237,8 @@ SHOW is_superuser; off (1 row) -CREATE FUNCTION aqo_drop(hash bigint) -RETURNS VOID +CREATE FUNCTION aqo_execution_time(controlled boolean) +RETURNS TABLE(num integer, id bigint, fshash bigint, exec_time double precision, nexecs bigint) AS $$ BEGIN END @@ -263,21 +246,20 @@ $$ LANGUAGE plpgsql; RESET ROLE; -- Test result (error expected) CREATE EXTENSION aqo; -ERROR: function "aqo_drop" already exists with same argument types +ERROR: function "aqo_execution_time" already exists with same argument types SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_drop(hash bigint) -RETURNS VOID +CREATE OR REPLACE FUNCTION aqo_execution_time(controlled boolean) +RETURNS TABLE(num integer, id bigint, fshash bigint, exec_time double precision, nexecs bigint) AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_drop(42); - aqo_drop ----------- - -(1 row) +SELECT aqo_execution_time(true); + aqo_execution_time +-------------------- +(0 rows) SET ROLE regress_hacker; SHOW is_superuser; @@ -287,7 +269,7 @@ SHOW is_superuser; (1 row) RESET ROLE; -DROP FUNCTION aqo_drop(bigint); +DROP FUNCTION aqo_execution_time(boolean); DROP EXTENSION IF EXISTS aqo; NOTICE: extension "aqo" does not exist, skipping -- Test 7 @@ -300,8 +282,8 @@ SHOW is_superuser; off (1 row) -CREATE FUNCTION aqo_ne_queries() -RETURNS SETOF int +CREATE FUNCTION aqo_memory_usage(OUT name text, OUT allocated_size int, OUT used_size int) +RETURNS SETOF record AS $$ BEGIN END @@ -309,19 +291,19 @@ $$ LANGUAGE plpgsql; RESET ROLE; -- Test result (error expected) CREATE EXTENSION aqo; -ERROR: function "aqo_ne_queries" already exists with same argument types +ERROR: function "aqo_memory_usage" already exists with same argument types SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_ne_queries() -RETURNS SETOF int +CREATE OR REPLACE FUNCTION aqo_memory_usage(OUT name text, OUT allocated_size int, OUT used_size int) +RETURNS SETOF record AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_ne_queries(); - aqo_ne_queries ----------------- +SELECT aqo_memory_usage(); + aqo_memory_usage +------------------ (0 rows) SET ROLE regress_hacker; @@ -332,43 +314,9 @@ SHOW is_superuser; (1 row) RESET ROLE; -DROP FUNCTION aqo_ne_queries(); +DROP FUNCTION aqo_memory_usage(); DROP EXTENSION IF EXISTS aqo; NOTICE: extension "aqo" does not exist, skipping --- Test 8 -RESET ROLE; -ALTER ROLE regress_hacker NOSUPERUSER; -SET ROLE regress_hacker; -SHOW is_superuser; - is_superuser --------------- - off -(1 row) - -CREATE FUNCTION aqo_migrate_to_1_1_get_pk(rel text) -RETURNS regclass -AS $$ -DECLARE - ret regclass; -BEGIN - ALTER ROLE regress_hacker SUPERUSER; - SELECT * FROM aqo_migrate_to_1_1_get_pk(rel::regclass) INTO ret; - RETURN ret; -END -$$ LANGUAGE plpgsql; -RESET ROLE; -CREATE EXTENSION aqo; --- Test result (must be 'off') -SET ROLE regress_hacker; -SHOW is_superuser; - is_superuser --------------- - off -(1 row) - -RESET ROLE; -DROP FUNCTION aqo_migrate_to_1_1_get_pk(text); -DROP EXTENSION IF EXISTS aqo; -- Cleanup RESET ROLE; DROP OWNED BY regress_hacker CASCADE; diff --git a/hash.h b/hash.h index 01c90bed..419941f6 100644 --- a/hash.h +++ b/hash.h @@ -14,4 +14,9 @@ extern int get_fss_for_object(List *relsigns, List *clauselist, extern int get_int_array_hash(int *arr, int len); extern int get_grouped_exprs_hash(int fss, List *group_exprs); +/* Hash functions */ +void get_eclasses(List *clauselist, int *nargs, int **args_hash, + int **eclass_hash); +int get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash); + #endif /* AQO_HASH_H */ \ No newline at end of file diff --git a/path_utils.c b/path_utils.c index 3ad90065..93b42d98 100644 --- a/path_utils.c +++ b/path_utils.c @@ -15,8 +15,11 @@ #include "access/relation.h" #include "nodes/readfuncs.h" +#include "optimizer/cost.h" #include "optimizer/optimizer.h" +#include "optimizer/planmain.h" #include "path_utils.h" +#include "storage/lmgr.h" #include "utils/syscache.h" #include "utils/lsyscache.h" @@ -30,13 +33,6 @@ expression_tree_mutator(node, mutator, context, 0) #endif -/* - * Hook on creation of a plan node. We need to store AQO-specific data to - * support learning stage. - */ -create_plan_hook_type prev_create_plan_hook = NULL; - -create_upper_paths_hook_type prev_create_upper_paths_hook = NULL; static AQOPlanNode DefaultAQOPlanNode = { @@ -54,6 +50,15 @@ static AQOPlanNode DefaultAQOPlanNode = .prediction = -1 }; +/* + * Hook on creation of a plan node. We need to store AQO-specific data to + * support learning stage. + */ +static create_plan_hook_type aqo_create_plan_next = NULL; + +static create_upper_paths_hook_type aqo_create_upper_paths_next = NULL; + + static AQOPlanNode * create_aqo_plan_node() { @@ -180,8 +185,6 @@ hashTempTupleDesc(TupleDesc desc) return s; } -#include "storage/lmgr.h" - /* * Get list of relation indexes and prepare list of permanent table reloids, * list of temporary table reloids (can be changed between query launches) and @@ -531,15 +534,15 @@ is_appropriate_path(Path *path) * store AQO prediction in the same context, as the plan. So, explicitly free * all unneeded data. */ -void -aqo_create_plan_hook(PlannerInfo *root, Path *src, Plan **dest) +static void +aqo_create_plan(PlannerInfo *root, Path *src, Plan **dest) { bool is_join_path; Plan *plan = *dest; AQOPlanNode *node; - if (prev_create_plan_hook) - prev_create_plan_hook(root, src, dest); + if (aqo_create_plan_next) + (*aqo_create_plan_next)(root, src, dest); if (!query_context.use_aqo && !query_context.learn_aqo && !query_context.collect_stat) @@ -784,20 +787,20 @@ RegisterAQOPlanNodeMethods(void) * * Assume, that we are last in the chain of path creators. */ -void -aqo_store_upper_signature_hook(PlannerInfo *root, - UpperRelationKind stage, - RelOptInfo *input_rel, - RelOptInfo *output_rel, - void *extra) +static void +aqo_store_upper_signature(PlannerInfo *root, + UpperRelationKind stage, + RelOptInfo *input_rel, + RelOptInfo *output_rel, + void *extra) { A_Const *fss_node = makeNode(A_Const); RelSortOut rels = {NIL, NIL}; List *clauses; List *selectivities; - if (prev_create_upper_paths_hook) - (*prev_create_upper_paths_hook)(root, stage, input_rel, output_rel, extra); + if (aqo_create_upper_paths_next) + (*aqo_create_upper_paths_next)(root, stage, input_rel, output_rel, extra); if (!query_context.use_aqo && !query_context.learn_aqo && !force_collect_stat) /* Includes 'disabled query' state. */ @@ -816,3 +819,13 @@ aqo_store_upper_signature_hook(PlannerInfo *root, NULL, NULL); output_rel->ext_nodes = lappend(output_rel->ext_nodes, (void *) fss_node); } + +void +aqo_path_utils_init(void) +{ + aqo_create_plan_next = create_plan_hook; + create_plan_hook = aqo_create_plan; + + aqo_create_upper_paths_next = create_upper_paths_hook; + create_upper_paths_hook = aqo_store_upper_signature; +} diff --git a/path_utils.h b/path_utils.h index 1803e08d..cbe83da0 100644 --- a/path_utils.h +++ b/path_utils.h @@ -3,7 +3,6 @@ #include "nodes/extensible.h" #include "nodes/pathnodes.h" -#include "optimizer/planmain.h" #include "optimizer/planner.h" #define AQO_PLAN_NODE "AQOPlanNode" @@ -52,8 +51,6 @@ typedef struct AQOPlanNode #define booltostr(x) ((x) ? "true" : "false") -extern create_plan_hook_type prev_create_plan_hook; - /* Extracting path information utilities */ extern List *get_selectivities(PlannerInfo *root, List *clauses, @@ -67,16 +64,11 @@ extern List *get_path_clauses(Path *path, PlannerInfo *root, List **selectivities); -extern void aqo_create_plan_hook(PlannerInfo *root, Path *src, Plan **dest); extern AQOPlanNode *get_aqo_plan_node(Plan *plan, bool create); extern void RegisterAQOPlanNodeMethods(void); -extern create_upper_paths_hook_type prev_create_upper_paths_hook; -extern void aqo_store_upper_signature_hook(PlannerInfo *root, - UpperRelationKind stage, - RelOptInfo *input_rel, - RelOptInfo *output_rel, - void *extra); extern List *aqo_get_clauses(PlannerInfo *root, List *restrictlist); +void aqo_path_utils_init(void); + #endif /* PATH_UTILS_H */ diff --git a/postprocessing.c b/postprocessing.c index aa82a534..ba2e19e0 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -27,7 +27,6 @@ #include "hash.h" #include "path_utils.h" #include "machine_learning.h" -#include "preprocessing.h" #include "storage.h" @@ -58,6 +57,13 @@ static int64 growth_rate = 3; static char *AQOPrivateData = "AQOPrivateData"; static char *PlanStateInfo = "PlanStateInfo"; +/* Saved hooks */ +static ExecutorStart_hook_type aqo_ExecutorStart_next = NULL; +static ExecutorRun_hook_type aqo_ExecutorRun_next = NULL; +static ExecutorEnd_hook_type aqo_ExecutorEnd_next = NULL; +static ExplainOnePlan_hook_type aqo_ExplainOnePlan_next = NULL; +static ExplainOneNode_hook_type aqo_ExplainOneNode_next = NULL; + /* Query execution statistics collecting utilities */ static void atomic_fss_learn_step(uint64 fhash, int fss, OkNNrdata *data, @@ -542,7 +548,7 @@ learnOnPlanState(PlanState *p, void *context) /* * Set up flags to store cardinality statistics. */ -void +static void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags) { instr_time now; @@ -594,10 +600,7 @@ aqo_ExecutorStart(QueryDesc *queryDesc, int eflags) StoreToQueryEnv(queryDesc); } - if (prev_ExecutorStart_hook) - prev_ExecutorStart_hook(queryDesc, eflags); - else - standard_ExecutorStart(queryDesc, eflags); + (*aqo_ExecutorStart_next)(queryDesc, eflags); if (use_aqo) StorePlanInternals(queryDesc); @@ -706,7 +709,7 @@ set_timeout_if_need(QueryDesc *queryDesc) /* * ExecutorRun hook. */ -void +static void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once) { @@ -722,10 +725,7 @@ aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, PG_TRY(); { - if (prev_ExecutorRun) - prev_ExecutorRun(queryDesc, direction, count, execute_once); - else - standard_ExecutorRun(queryDesc, direction, count, execute_once); + (*aqo_ExecutorRun_next)(queryDesc, direction, count, execute_once); } PG_FINALLY(); { @@ -743,7 +743,7 @@ aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, * cardinality statistics. * Also it updates query execution statistics in aqo_query_stat. */ -void +static void aqo_ExecutorEnd(QueryDesc *queryDesc) { double execution_time; @@ -841,10 +841,7 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) MemoryContextSwitchTo(oldctx); MemoryContextReset(AQOLearnMemCtx); - if (prev_ExecutorEnd_hook) - prev_ExecutorEnd_hook(queryDesc); - else - standard_ExecutorEnd(queryDesc); + (*aqo_ExecutorEnd_next)(queryDesc); /* * standard_ExecutorEnd clears the queryDesc->planstate. After this point no @@ -975,7 +972,64 @@ ExtractFromQueryEnv(QueryDesc *queryDesc) return true; } -void +/* + * Prints if the plan was constructed with AQO. + */ +static void +print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, + ExplainState *es, const char *queryString, + ParamListInfo params, const instr_time *planduration, + QueryEnvironment *queryEnv) +{ + if (aqo_ExplainOnePlan_next) + (*aqo_ExplainOnePlan_next)(plannedstmt, into, es, queryString, + params, planduration, queryEnv); + + if (IsQueryDisabled() || !aqo_show_details) + return; + + /* Report to user about aqo state only in verbose mode */ + ExplainPropertyBool("Using aqo", query_context.use_aqo, es); + + switch (aqo_mode) + { + case AQO_MODE_INTELLIGENT: + ExplainPropertyText("AQO mode", "INTELLIGENT", es); + break; + case AQO_MODE_FORCED: + ExplainPropertyText("AQO mode", "FORCED", es); + break; + case AQO_MODE_CONTROLLED: + ExplainPropertyText("AQO mode", "CONTROLLED", es); + break; + case AQO_MODE_LEARN: + ExplainPropertyText("AQO mode", "LEARN", es); + break; + case AQO_MODE_FROZEN: + ExplainPropertyText("AQO mode", "FROZEN", es); + break; + case AQO_MODE_DISABLED: + ExplainPropertyText("AQO mode", "DISABLED", es); + break; + default: + elog(ERROR, "Bad AQO state"); + break; + } + + /* + * Query class provides an user the conveniently use of the AQO + * auxiliary functions. + */ + if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) + { + if (aqo_show_hash) + ExplainPropertyInteger("Query hash", NULL, + query_context.query_hash, es); + ExplainPropertyInteger("JOINS", NULL, njoins, es); + } +} + +static void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan) { int wrkrs = 1; @@ -983,8 +1037,8 @@ print_node_explain(ExplainState *es, PlanState *ps, Plan *plan) AQOPlanNode *aqo_node; /* Extension, which took a hook early can be executed early too. */ - if (prev_ExplainOneNode_hook) - prev_ExplainOneNode_hook(es, ps, plan); + if (aqo_ExplainOneNode_next) + (*aqo_ExplainOneNode_next)(es, ps, plan); if (IsQueryDisabled() || !plan || es->format != EXPLAIN_FORMAT_TEXT) return; @@ -1042,59 +1096,20 @@ print_node_explain(ExplainState *es, PlanState *ps, Plan *plan) appendStringInfo(es->str, ", fss=%d", aqo_node->fss); } -/* - * Prints if the plan was constructed with AQO. - */ void -print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, - ParamListInfo params, const instr_time *planduration, - QueryEnvironment *queryEnv) +aqo_postprocessing_init(void) { - if (prev_ExplainOnePlan_hook) - prev_ExplainOnePlan_hook(plannedstmt, into, es, queryString, - params, planduration, queryEnv); - - if (IsQueryDisabled() || !aqo_show_details) - return; - - /* Report to user about aqo state only in verbose mode */ - ExplainPropertyBool("Using aqo", query_context.use_aqo, es); - - switch (aqo_mode) - { - case AQO_MODE_INTELLIGENT: - ExplainPropertyText("AQO mode", "INTELLIGENT", es); - break; - case AQO_MODE_FORCED: - ExplainPropertyText("AQO mode", "FORCED", es); - break; - case AQO_MODE_CONTROLLED: - ExplainPropertyText("AQO mode", "CONTROLLED", es); - break; - case AQO_MODE_LEARN: - ExplainPropertyText("AQO mode", "LEARN", es); - break; - case AQO_MODE_FROZEN: - ExplainPropertyText("AQO mode", "FROZEN", es); - break; - case AQO_MODE_DISABLED: - ExplainPropertyText("AQO mode", "DISABLED", es); - break; - default: - elog(ERROR, "Bad AQO state"); - break; - } - - /* - * Query class provides an user the conveniently use of the AQO - * auxiliary functions. - */ - if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) - { - if (aqo_show_hash) - ExplainPropertyInteger("Query hash", NULL, - query_context.query_hash, es); - ExplainPropertyInteger("JOINS", NULL, njoins, es); - } + /* Executor hooks */ + aqo_ExecutorStart_next = ExecutorStart_hook ? ExecutorStart_hook : standard_ExecutorStart; + ExecutorStart_hook = aqo_ExecutorStart; + aqo_ExecutorRun_next = ExecutorRun_hook ? ExecutorRun_hook : standard_ExecutorRun; + ExecutorRun_hook = aqo_ExecutorRun; + aqo_ExecutorEnd_next = ExecutorEnd_hook ? ExecutorEnd_hook : standard_ExecutorEnd; + ExecutorEnd_hook = aqo_ExecutorEnd; + + /* Service hooks. */ + aqo_ExplainOnePlan_next = ExplainOnePlan_hook; + ExplainOnePlan_hook = print_into_explain; + aqo_ExplainOneNode_next = ExplainOneNode_hook; + ExplainOneNode_hook = print_node_explain; } diff --git a/preprocessing.c b/preprocessing.c index 60e599ee..01b28a32 100644 --- a/preprocessing.c +++ b/preprocessing.c @@ -64,7 +64,6 @@ #include "parser/scansup.h" #include "aqo.h" #include "hash.h" -#include "preprocessing.h" #include "storage.h" /* List of feature spaces, that are processing in this backend. */ @@ -72,30 +71,12 @@ List *cur_classes = NIL; int aqo_join_threshold = 0; +static planner_hook_type aqo_planner_next = NULL; + +static void disable_aqo_for_query(void); static bool isQueryUsingSystemRelation(Query *query); static bool isQueryUsingSystemRelation_walker(Node *node, void *context); -/* - * Calls standard query planner or its previous hook. - */ -static PlannedStmt * -call_default_planner(Query *parse, - const char *query_string, - int cursorOptions, - ParamListInfo boundParams) -{ - if (prev_planner_hook) - return prev_planner_hook(parse, - query_string, - cursorOptions, - boundParams); - else - return standard_planner(parse, - query_string, - cursorOptions, - boundParams); -} - /* * Can AQO be used for the query? */ @@ -119,10 +100,8 @@ aqoIsEnabled(Query *parse) * Creates an entry in aqo_queries for new type of query if it is * necessary, i. e. AQO mode is "intelligent". */ -PlannedStmt * -aqo_planner(Query *parse, - const char *query_string, - int cursorOptions, +static PlannedStmt * +aqo_planner(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) { bool query_is_stored = false; @@ -149,10 +128,7 @@ aqo_planner(Query *parse, MemoryContextSwitchTo(oldctx); disable_aqo_for_query(); - return call_default_planner(parse, - query_string, - cursorOptions, - boundParams); + return (*aqo_planner_next)(parse, query_string, cursorOptions, boundParams); } selectivity_cache_clear(); @@ -173,10 +149,7 @@ aqo_planner(Query *parse, MemoryContextSwitchTo(oldctx); disable_aqo_for_query(); - return call_default_planner(parse, - query_string, - cursorOptions, - boundParams); + return (*aqo_planner_next)(parse, query_string, cursorOptions, boundParams); } elog(DEBUG1, "AQO will be used for query '%s', class "UINT64_FORMAT, @@ -345,9 +318,9 @@ aqo_planner(Query *parse, INSTR_TIME_SET_CURRENT(query_context.start_planning_time); { PlannedStmt *stmt; + MemoryContextSwitchTo(oldctx); - stmt = call_default_planner(parse, query_string, - cursorOptions, boundParams); + stmt = (*aqo_planner_next)(parse, query_string, cursorOptions, boundParams); /* Release the memory, allocated for AQO predictions */ MemoryContextReset(AQOPredictMemCtx); @@ -358,7 +331,7 @@ aqo_planner(Query *parse, /* * Turn off all AQO functionality for the current query. */ -void +static void disable_aqo_for_query(void) { query_context.learn_aqo = false; @@ -507,3 +480,10 @@ isQueryUsingSystemRelation_walker(Node *node, void *context) isQueryUsingSystemRelation_walker, context); } + +void +aqo_preprocessing_init(void) +{ + aqo_planner_next = planner_hook ? planner_hook : standard_planner; + planner_hook = aqo_planner; +} \ No newline at end of file diff --git a/preprocessing.h b/preprocessing.h deleted file mode 100644 index f27deb91..00000000 --- a/preprocessing.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __PREPROCESSING_H__ -#define __PREPROCESSING_H__ - -#include "nodes/pathnodes.h" -#include "nodes/plannodes.h" -extern PlannedStmt *aqo_planner(Query *parse, - const char *query_string, - int cursorOptions, - ParamListInfo boundParams); -extern void disable_aqo_for_query(void); - -#endif /* __PREPROCESSING_H__ */ diff --git a/sql/aqo_CVE-2020-14350.sql b/sql/aqo_CVE-2020-14350.sql index 75833223..c4979344 100644 --- a/sql/aqo_CVE-2020-14350.sql +++ b/sql/aqo_CVE-2020-14350.sql @@ -44,21 +44,11 @@ ALTER ROLE regress_hacker NOSUPERUSER; SET ROLE regress_hacker; SHOW is_superuser; -CREATE FUNCTION aqo_status(hash bigint) -RETURNS TABLE ( - "learn" BOOL, - "use aqo" BOOL, - "auto tune" BOOL, - "fspace hash" bigINT, - "t_naqo" TEXT, - "err_naqo" TEXT, - "iters" BIGINT, - "t_aqo" TEXT, - "err_aqo" TEXT, - "iters_aqo" BIGINT -) +CREATE FUNCTION aqo_reset() +RETURNS bigint AS $$ BEGIN + RETURN 1; END $$ LANGUAGE plpgsql; @@ -67,33 +57,23 @@ RESET ROLE; CREATE EXTENSION aqo; SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_status(hash bigint) -RETURNS TABLE ( - "learn" BOOL, - "use aqo" BOOL, - "auto tune" BOOL, - "fspace hash" bigINT, - "t_naqo" TEXT, - "err_naqo" TEXT, - "iters" BIGINT, - "t_aqo" TEXT, - "err_aqo" TEXT, - "iters_aqo" BIGINT -) +CREATE OR REPLACE FUNCTION aqo_reset() +RETURNS bigint AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; + RETURN 2; END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_status(42); +SELECT aqo_reset(); SET ROLE regress_hacker; SHOW is_superuser; RESET ROLE; -DROP FUNCTION aqo_status(bigint); +DROP FUNCTION aqo_reset(); DROP EXTENSION IF EXISTS aqo; -- Test 3 @@ -177,10 +157,11 @@ ALTER ROLE regress_hacker NOSUPERUSER; SET ROLE regress_hacker; SHOW is_superuser; -CREATE FUNCTION aqo_clear_hist(hash bigint) -RETURNS VOID +CREATE FUNCTION aqo_drop_class(queryid bigint) +RETURNS integer AS $$ BEGIN + RETURN 1; END $$ LANGUAGE plpgsql; @@ -189,22 +170,23 @@ RESET ROLE; CREATE EXTENSION aqo; SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_clear_hist(hash bigint) -RETURNS VOID +CREATE OR REPLACE FUNCTION aqo_drop_class(queryid bigint) +RETURNS integer AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; + RETURN 2; END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_clear_hist(42); +SELECT aqo_drop_class(42); SET ROLE regress_hacker; SHOW is_superuser; RESET ROLE; -DROP FUNCTION aqo_clear_hist(bigint); +DROP FUNCTION aqo_drop_class(bigint); DROP EXTENSION IF EXISTS aqo; -- Test 6 @@ -214,8 +196,8 @@ ALTER ROLE regress_hacker NOSUPERUSER; SET ROLE regress_hacker; SHOW is_superuser; -CREATE FUNCTION aqo_drop(hash bigint) -RETURNS VOID +CREATE FUNCTION aqo_execution_time(controlled boolean) +RETURNS TABLE(num integer, id bigint, fshash bigint, exec_time double precision, nexecs bigint) AS $$ BEGIN END @@ -226,8 +208,8 @@ RESET ROLE; CREATE EXTENSION aqo; SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_drop(hash bigint) -RETURNS VOID +CREATE OR REPLACE FUNCTION aqo_execution_time(controlled boolean) +RETURNS TABLE(num integer, id bigint, fshash bigint, exec_time double precision, nexecs bigint) AS $$ BEGIN ALTER ROLE regress_hacker SUPERUSER; @@ -235,13 +217,13 @@ END $$ LANGUAGE plpgsql; RESET ROLE; -SELECT aqo_drop(42); +SELECT aqo_execution_time(true); SET ROLE regress_hacker; SHOW is_superuser; RESET ROLE; -DROP FUNCTION aqo_drop(bigint); +DROP FUNCTION aqo_execution_time(boolean); DROP EXTENSION IF EXISTS aqo; -- Test 7 @@ -251,8 +233,8 @@ ALTER ROLE regress_hacker NOSUPERUSER; SET ROLE regress_hacker; SHOW is_superuser; -CREATE FUNCTION aqo_ne_queries() -RETURNS SETOF int +CREATE FUNCTION aqo_memory_usage(OUT name text, OUT allocated_size int, OUT used_size int) +RETURNS SETOF record AS $$ BEGIN END @@ -263,52 +245,22 @@ RESET ROLE; CREATE EXTENSION aqo; SET ROLE regress_hacker; -CREATE OR REPLACE FUNCTION aqo_ne_queries() -RETURNS SETOF int -AS $$ -BEGIN - ALTER ROLE regress_hacker SUPERUSER; -END -$$ LANGUAGE plpgsql; - -RESET ROLE; -SELECT aqo_ne_queries(); - -SET ROLE regress_hacker; -SHOW is_superuser; - -RESET ROLE; -DROP FUNCTION aqo_ne_queries(); -DROP EXTENSION IF EXISTS aqo; - --- Test 8 -RESET ROLE; -ALTER ROLE regress_hacker NOSUPERUSER; - -SET ROLE regress_hacker; -SHOW is_superuser; - -CREATE FUNCTION aqo_migrate_to_1_1_get_pk(rel text) -RETURNS regclass +CREATE OR REPLACE FUNCTION aqo_memory_usage(OUT name text, OUT allocated_size int, OUT used_size int) +RETURNS SETOF record AS $$ -DECLARE - ret regclass; BEGIN ALTER ROLE regress_hacker SUPERUSER; - SELECT * FROM aqo_migrate_to_1_1_get_pk(rel::regclass) INTO ret; - RETURN ret; END $$ LANGUAGE plpgsql; RESET ROLE; -CREATE EXTENSION aqo; +SELECT aqo_memory_usage(); --- Test result (must be 'off') SET ROLE regress_hacker; SHOW is_superuser; RESET ROLE; -DROP FUNCTION aqo_migrate_to_1_1_get_pk(text); +DROP FUNCTION aqo_memory_usage(); DROP EXTENSION IF EXISTS aqo; -- Cleanup diff --git a/storage.c b/storage.c index 64bc4373..9e6cf306 100644 --- a/storage.c +++ b/storage.c @@ -22,11 +22,11 @@ #include "funcapi.h" #include "miscadmin.h" #include "pgstat.h" +#include "storage/ipc.h" #include "aqo.h" #include "aqo_shared.h" #include "machine_learning.h" -#include "preprocessing.h" #include "storage.h" diff --git a/storage.h b/storage.h index 35d94336..2b4e4cdd 100644 --- a/storage.h +++ b/storage.h @@ -164,4 +164,10 @@ extern void init_deactivated_queries_storage(void); extern bool query_is_deactivated(uint64 query_hash); extern void add_deactivated_query(uint64 query_hash); +/* Storage interaction */ +extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids); +extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids); + +extern bool update_query_timeout(uint64 queryid, int64 smart_timeout); + #endif /* STORAGE_H */