diff --git a/.github/workflows/installchecks.yml b/.github/workflows/installchecks.yml index 075034a0..4a4d478b 100644 --- a/.github/workflows/installchecks.yml +++ b/.github/workflows/installchecks.yml @@ -132,12 +132,24 @@ jobs: psql -c "ALTER SYSTEM SET aqo.mode = 'learn'" psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" psql -c "SELECT pg_reload_conf()" - learn_result=$(make -k installcheck-world) + make -k installcheck-world + + # Should work like a total off for all the AQO features + - name: installcheck_learn_queryid_off + continue-on-error: true + run: | + cd $PG_DIR + aqo_instance_launch.sh + psql -c "ALTER SYSTEM SET compute_query_id = 'off'" + psql -c "SELECT pg_reload_conf()" + # The AQO tests itself wouldn't pass + make -k installcheck-world - name: installcheck_intelligent continue-on-error: true run: | cd $PG_DIR + psql -c "ALTER SYSTEM SET compute_query_id = 'regress'" psql -c "ALTER SYSTEM SET aqo.mode = 'intelligent'" psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" psql -c "SELECT pg_reload_conf()" diff --git a/aqo.c b/aqo.c index 11b26b14..72d6f5fc 100644 --- a/aqo.c +++ b/aqo.c @@ -19,9 +19,7 @@ #include "aqo.h" #include "aqo_shared.h" -#include "cardinality_hooks.h" #include "path_utils.h" -#include "preprocessing.h" #include "storage.h" @@ -98,21 +96,6 @@ MemoryContext AQOLearnMemCtx = NULL; /* Additional plan info */ int njoins; -/* Saved hook values */ -post_parse_analyze_hook_type prev_post_parse_analyze_hook; -planner_hook_type prev_planner_hook; -ExecutorStart_hook_type prev_ExecutorStart_hook; -ExecutorRun_hook_type prev_ExecutorRun; -ExecutorEnd_hook_type prev_ExecutorEnd_hook; -set_baserel_rows_estimate_hook_type prev_set_foreign_rows_estimate_hook; -set_baserel_rows_estimate_hook_type prev_set_baserel_rows_estimate_hook; -get_parameterized_baserel_size_hook_type prev_get_parameterized_baserel_size_hook; -set_joinrel_size_estimates_hook_type prev_set_joinrel_size_estimates_hook; -get_parameterized_joinrel_size_hook_type prev_get_parameterized_joinrel_size_hook; -ExplainOnePlan_hook_type prev_ExplainOnePlan_hook; -ExplainOneNode_hook_type prev_ExplainOneNode_hook; -static shmem_request_hook_type prev_shmem_request_hook = NULL; - /***************************************************************************** * * CREATE/DROP EXTENSION FUNCTIONS @@ -135,18 +118,6 @@ aqo_free_callback(ResourceReleasePhase phase, } } -/* - * Requests any additional shared memory required for aqo. - */ -static void -aqo_shmem_request(void) -{ - if (prev_shmem_request_hook) - prev_shmem_request_hook(); - - RequestAddinShmemSpace(aqo_memsize()); -} - void _PG_init(void) { @@ -343,45 +314,11 @@ _PG_init(void) NULL, NULL); - prev_shmem_startup_hook = shmem_startup_hook; - shmem_startup_hook = aqo_init_shmem; - prev_planner_hook = planner_hook; - planner_hook = aqo_planner; - prev_ExecutorStart_hook = ExecutorStart_hook; - ExecutorStart_hook = aqo_ExecutorStart; - prev_ExecutorRun = ExecutorRun_hook; - ExecutorRun_hook = aqo_ExecutorRun; - prev_ExecutorEnd_hook = ExecutorEnd_hook; - ExecutorEnd_hook = aqo_ExecutorEnd; - - /* Cardinality prediction hooks. */ - prev_set_baserel_rows_estimate_hook = set_baserel_rows_estimate_hook; - set_foreign_rows_estimate_hook = aqo_set_baserel_rows_estimate; - set_baserel_rows_estimate_hook = aqo_set_baserel_rows_estimate; - prev_get_parameterized_baserel_size_hook = get_parameterized_baserel_size_hook; - get_parameterized_baserel_size_hook = aqo_get_parameterized_baserel_size; - prev_set_joinrel_size_estimates_hook = set_joinrel_size_estimates_hook; - set_joinrel_size_estimates_hook = aqo_set_joinrel_size_estimates; - prev_get_parameterized_joinrel_size_hook = get_parameterized_joinrel_size_hook; - get_parameterized_joinrel_size_hook = aqo_get_parameterized_joinrel_size; - prev_estimate_num_groups_hook = estimate_num_groups_hook; - estimate_num_groups_hook = aqo_estimate_num_groups_hook; - parampathinfo_postinit_hook = ppi_hook; - - prev_create_plan_hook = create_plan_hook; - create_plan_hook = aqo_create_plan_hook; - - /* Service hooks. */ - prev_ExplainOnePlan_hook = ExplainOnePlan_hook; - ExplainOnePlan_hook = print_into_explain; - prev_ExplainOneNode_hook = ExplainOneNode_hook; - ExplainOneNode_hook = print_node_explain; - - prev_create_upper_paths_hook = create_upper_paths_hook; - create_upper_paths_hook = aqo_store_upper_signature_hook; - - prev_shmem_request_hook = shmem_request_hook; - shmem_request_hook = aqo_shmem_request; + aqo_shmem_init(); + aqo_preprocessing_init(); + aqo_postprocessing_init(); + aqo_cardinality_hooks_init(); + aqo_path_utils_init(); init_deactivated_queries_storage(); diff --git a/aqo.h b/aqo.h index 9600b136..6f57a4d1 100644 --- a/aqo.h +++ b/aqo.h @@ -132,7 +132,6 @@ #include "nodes/nodeFuncs.h" #include "optimizer/pathnode.h" #include "optimizer/planner.h" -#include "optimizer/cost.h" #include "parser/analyze.h" #include "parser/parsetree.h" #include "utils/builtins.h" @@ -140,11 +139,9 @@ #include "utils/hsearch.h" #include "utils/memutils.h" #include "utils/rel.h" -#include "utils/fmgroids.h" #include "utils/snapmgr.h" #include "machine_learning.h" -//#include "storage.h" /* Check PostgreSQL version (9.6.0 contains important changes in planner) */ #if PG_VERSION_NUM < 90600 @@ -237,58 +234,15 @@ extern MemoryContext AQOCacheMemCtx; extern MemoryContext AQOPredictMemCtx; extern MemoryContext AQOLearnMemCtx; -/* Saved hook values in case of unload */ -extern post_parse_analyze_hook_type prev_post_parse_analyze_hook; -extern planner_hook_type prev_planner_hook; -extern ExecutorStart_hook_type prev_ExecutorStart_hook; -extern ExecutorRun_hook_type prev_ExecutorRun; -extern ExecutorEnd_hook_type prev_ExecutorEnd_hook; -extern set_baserel_rows_estimate_hook_type - prev_set_foreign_rows_estimate_hook; -extern set_baserel_rows_estimate_hook_type - prev_set_baserel_rows_estimate_hook; -extern get_parameterized_baserel_size_hook_type - prev_get_parameterized_baserel_size_hook; -extern set_joinrel_size_estimates_hook_type - prev_set_joinrel_size_estimates_hook; -extern get_parameterized_joinrel_size_hook_type - prev_get_parameterized_joinrel_size_hook; -extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook; -extern ExplainOneNode_hook_type prev_ExplainOneNode_hook; - -extern void ppi_hook(ParamPathInfo *ppi); extern int aqo_statement_timeout; -/* Hash functions */ -void get_eclasses(List *clauselist, int *nargs, int **args_hash, - int **eclass_hash); -int get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash); - - -/* Storage interaction */ -extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids); -extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids); - -/* Query preprocessing hooks */ -extern void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, - ParamListInfo params, - const instr_time *planduration, - QueryEnvironment *queryEnv); -extern void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan); - /* Cardinality estimation */ extern double predict_for_relation(List *restrict_clauses, List *selectivities, List *relsigns, int *fss); -/* Query execution statistics collecting hooks */ -void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags); -void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, - uint64 count, bool execute_once); -void aqo_ExecutorEnd(QueryDesc *queryDesc); - /* Automatic query tuning */ extern void automatical_query_tuning(uint64 query_hash, struct StatEntry *stat); +extern double get_mean(double *elems, int nelems); /* Utilities */ extern int int_cmp(const void *a, const void *b); @@ -306,8 +260,10 @@ extern void selectivity_cache_clear(void); extern bool IsQueryDisabled(void); -extern bool update_query_timeout(uint64 queryid, int64 smart_timeout); -extern double get_mean(double *elems, int nelems); - extern List *cur_classes; + +extern void aqo_cardinality_hooks_init(void); +extern void aqo_preprocessing_init(void); +extern void aqo_postprocessing_init(void); + #endif diff --git a/aqo_shared.c b/aqo_shared.c index 0a6a8db6..0a86ea09 100644 --- a/aqo_shared.c +++ b/aqo_shared.c @@ -6,27 +6,30 @@ #include "lib/dshash.h" #include "miscadmin.h" +#include "storage/ipc.h" #include "storage/shmem.h" #include "aqo_shared.h" #include "storage.h" -shmem_startup_hook_type prev_shmem_startup_hook = NULL; AQOSharedState *aqo_state = NULL; int fs_max_items = 10000; /* Max number of different feature spaces in ML model */ int fss_max_items = 100000; /* Max number of different feature subspaces in ML model */ +static shmem_startup_hook_type aqo_shmem_startup_next = NULL; +static shmem_request_hook_type aqo_shmem_request_next = NULL; + static void on_shmem_shutdown(int code, Datum arg); -void +static void aqo_init_shmem(void) { bool found; HASHCTL info; - if (prev_shmem_startup_hook) - prev_shmem_startup_hook(); + if (aqo_shmem_startup_next) + (*aqo_shmem_startup_next)(); aqo_state = NULL; stat_htab = NULL; @@ -116,10 +119,17 @@ on_shmem_shutdown(int code, Datum arg) return; } -Size -aqo_memsize(void) + +/* + * Requests any additional shared memory required for aqo. + */ +static void +aqo_shmem_request(void) { - Size size; + Size size; + + if (aqo_shmem_request_next) + (*aqo_shmem_request_next)(); size = MAXALIGN(sizeof(AQOSharedState)); size = add_size(size, hash_estimate_size(fs_max_items, sizeof(AQOSharedState))); @@ -128,5 +138,14 @@ aqo_memsize(void) size = add_size(size, hash_estimate_size(fss_max_items, sizeof(DataEntry))); size = add_size(size, hash_estimate_size(fs_max_items, sizeof(QueriesEntry))); - return size; + RequestAddinShmemSpace(size); +} + +void +aqo_shmem_init(void) +{ + aqo_shmem_startup_next = shmem_startup_hook; + shmem_startup_hook = aqo_init_shmem; + aqo_shmem_request_next = shmem_request_hook; + shmem_request_hook = aqo_shmem_request; } diff --git a/aqo_shared.h b/aqo_shared.h index e922fb1c..ee9e3087 100644 --- a/aqo_shared.h +++ b/aqo_shared.h @@ -1,9 +1,6 @@ #ifndef AQO_SHARED_H #define AQO_SHARED_H -#include "lib/dshash.h" -#include "storage/dsm.h" -#include "storage/ipc.h" #include "storage/lwlock.h" #include "utils/dsa.h" @@ -31,13 +28,11 @@ typedef struct AQOSharedState } AQOSharedState; -extern shmem_startup_hook_type prev_shmem_startup_hook; extern AQOSharedState *aqo_state; extern int fs_max_items; /* Max number of feature spaces that AQO can operate */ extern int fss_max_items; -extern Size aqo_memsize(void); -extern void aqo_init_shmem(void); +extern void aqo_shmem_init(void); #endif /* AQO_SHARED_H */ diff --git a/cardinality_hooks.c b/cardinality_hooks.c index c26fcccb..f0d745bb 100644 --- a/cardinality_hooks.c +++ b/cardinality_hooks.c @@ -27,115 +27,38 @@ #include "postgres.h" +#include "optimizer/cost.h" +#include "utils/selfuncs.h" + #include "aqo.h" -#include "cardinality_hooks.h" #include "hash.h" #include "machine_learning.h" #include "path_utils.h" - -estimate_num_groups_hook_type prev_estimate_num_groups_hook = NULL; +#include "storage.h" double predicted_ppi_rows; double fss_ppi_hash; /* - * Calls standard set_baserel_rows_estimate or its previous hook. - */ -static void -default_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) -{ - if (prev_set_baserel_rows_estimate_hook) - prev_set_baserel_rows_estimate_hook(root, rel); - else - set_baserel_rows_estimate_standard(root, rel); -} - -/* - * Calls standard get_parameterized_baserel_size or its previous hook. - */ -static double -default_get_parameterized_baserel_size(PlannerInfo *root, - RelOptInfo *rel, - List *param_clauses) -{ - if (prev_get_parameterized_baserel_size_hook) - return prev_get_parameterized_baserel_size_hook(root, rel, param_clauses); - else - return get_parameterized_baserel_size_standard(root, rel, param_clauses); -} - -/* - * Calls standard get_parameterized_joinrel_size or its previous hook. - */ -static double -default_get_parameterized_joinrel_size(PlannerInfo *root, - RelOptInfo *rel, - Path *outer_path, - Path *inner_path, - SpecialJoinInfo *sjinfo, - List *restrict_clauses) -{ - if (prev_get_parameterized_joinrel_size_hook) - return prev_get_parameterized_joinrel_size_hook(root, rel, - outer_path, - inner_path, - sjinfo, - restrict_clauses); - else - return get_parameterized_joinrel_size_standard(root, rel, - outer_path, - inner_path, - sjinfo, - restrict_clauses); -} - -/* - * Calls standard set_joinrel_size_estimates or its previous hook. + * Cardinality prediction hooks. + * It isn't clear what to do if someone else tries to live in this chain. + * Of course, someone may want to just report some stat or something like that. + * So, it can be legal, sometimees. So far, we only report this fact. */ -static void -default_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, - RelOptInfo *outer_rel, - RelOptInfo *inner_rel, - SpecialJoinInfo *sjinfo, - List *restrictlist) -{ - if (prev_set_joinrel_size_estimates_hook) - prev_set_joinrel_size_estimates_hook(root, rel, - outer_rel, - inner_rel, - sjinfo, - restrictlist); - else - set_joinrel_size_estimates_standard(root, rel, - outer_rel, - inner_rel, - sjinfo, - restrictlist); -} - -static double -default_estimate_num_groups(PlannerInfo *root, List *groupExprs, - Path *subpath, RelOptInfo *grouped_rel, - List **pgset, EstimationInfo *estinfo) -{ - double input_rows = subpath->rows; - - if (prev_estimate_num_groups_hook != NULL) - return (*prev_estimate_num_groups_hook)(root, groupExprs, - subpath, - grouped_rel, - pgset, estinfo); - else - return estimate_num_groups(root, groupExprs, input_rows, pgset, estinfo); -} +static set_baserel_rows_estimate_hook_type aqo_set_baserel_rows_estimate_next = NULL; +static get_parameterized_baserel_size_hook_type aqo_get_parameterized_baserel_size_next = NULL; +static set_joinrel_size_estimates_hook_type aqo_set_joinrel_size_estimates_next = NULL; +static get_parameterized_joinrel_size_hook_type aqo_get_parameterized_joinrel_size_next = NULL; +static set_parampathinfo_postinit_hook_type aqo_set_parampathinfo_postinit_next = NULL; +static estimate_num_groups_hook_type aqo_estimate_num_groups_next = NULL; /* * Our hook for setting baserel rows estimate. * Extracts clauses, their selectivities and list of relation relids and * passes them to predict_for_relation. */ -void +static void aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) { double predicted; @@ -178,25 +101,37 @@ aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel) /* Return to the caller's memory context. */ MemoryContextSwitchTo(old_ctx_m); - if (predicted >= 0) - { - rel->rows = predicted; - rel->predicted_cardinality = predicted; - return; - } + if (predicted < 0) + goto default_estimator; + + if ((aqo_set_baserel_rows_estimate_next != set_baserel_rows_estimate_standard || + set_baserel_rows_estimate_hook != aqo_set_baserel_rows_estimate)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the set_baserel_rows_estimate_hook chain"); + + rel->rows = predicted; + rel->predicted_cardinality = predicted; + return; default_estimator: rel->predicted_cardinality = -1.; - default_set_baserel_rows_estimate(root, rel); + (*aqo_set_baserel_rows_estimate_next)(root, rel); } - -void -ppi_hook(ParamPathInfo *ppi) +static void +aqo_parampathinfo_postinit(ParamPathInfo *ppi) { + if (aqo_set_parampathinfo_postinit_next) + (*aqo_set_parampathinfo_postinit_next)(ppi); + if (IsQueryDisabled()) return; + if ((aqo_set_parampathinfo_postinit_next != NULL || + parampathinfo_postinit_hook != aqo_parampathinfo_postinit)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the parampathinfo_postinit_hook chain"); + ppi->predicted_ppi_rows = predicted_ppi_rows; ppi->fss_ppi_hash = fss_ppi_hash; } @@ -206,7 +141,7 @@ ppi_hook(ParamPathInfo *ppi) * Extracts clauses (including parametrization ones), their selectivities * and list of relation relids and passes them to predict_for_relation. */ -double +static double aqo_get_parameterized_baserel_size(PlannerInfo *root, RelOptInfo *rel, List *param_clauses) @@ -280,11 +215,18 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root, predicted_ppi_rows = predicted; fss_ppi_hash = fss; - if (predicted >= 0) - return predicted; + if (predicted < 0) + goto default_estimator; + + if ((aqo_get_parameterized_baserel_size_next != get_parameterized_baserel_size_standard || + get_parameterized_baserel_size_hook != aqo_get_parameterized_baserel_size)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the aqo_get_parameterized_baserel_size_next chain"); + + return predicted; default_estimator: - return default_get_parameterized_baserel_size(root, rel, param_clauses); + return (*aqo_get_parameterized_baserel_size_next)(root, rel, param_clauses); } /* @@ -292,7 +234,7 @@ aqo_get_parameterized_baserel_size(PlannerInfo *root, * Extracts clauses, their selectivities and list of relation relids and * passes them to predict_for_relation. */ -void +static void aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, RelOptInfo *outer_rel, RelOptInfo *inner_rel, @@ -345,18 +287,22 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, rel->fss_hash = fss; - if (predicted >= 0) - { - rel->predicted_cardinality = predicted; - rel->rows = predicted; - return; - } + if (predicted < 0) + goto default_estimator; + + if ((aqo_set_joinrel_size_estimates_next != set_joinrel_size_estimates_standard || + set_joinrel_size_estimates_hook != aqo_set_joinrel_size_estimates)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the set_joinrel_size_estimates_hook chain"); + + rel->predicted_cardinality = predicted; + rel->rows = predicted; + return; default_estimator: rel->predicted_cardinality = -1; - default_set_joinrel_size_estimates(root, rel, - outer_rel, inner_rel, - sjinfo, restrictlist); + (*aqo_set_joinrel_size_estimates_next)(root, rel, outer_rel, inner_rel, + sjinfo, restrictlist); } /* @@ -364,7 +310,7 @@ aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, * Extracts clauses (including parametrization ones), their selectivities * and list of relation relids and passes them to predict_for_relation. */ -double +static double aqo_get_parameterized_joinrel_size(PlannerInfo *root, RelOptInfo *rel, Path *outer_path, @@ -417,11 +363,18 @@ aqo_get_parameterized_joinrel_size(PlannerInfo *root, predicted_ppi_rows = predicted; fss_ppi_hash = fss; - if (predicted >= 0) - return predicted; + if (predicted < 0) + goto default_estimator; + + if ((aqo_get_parameterized_joinrel_size_next != get_parameterized_joinrel_size_standard || + get_parameterized_joinrel_size_hook != aqo_get_parameterized_joinrel_size)) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the get_parameterized_joinrel_size_hook chain"); + + return predicted; default_estimator: - return default_get_parameterized_joinrel_size(root, rel, + return (*aqo_get_parameterized_joinrel_size_next)(root, rel, outer_path, inner_path, sjinfo, clauses); } @@ -460,10 +413,10 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs, return (prediction <= 0) ? -1 : prediction; } -double -aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, - Path *subpath, RelOptInfo *grouped_rel, - List **pgset, EstimationInfo *estinfo) +static double +aqo_estimate_num_groups(PlannerInfo *root, List *groupExprs, + Path *subpath, RelOptInfo *grouped_rel, + List **pgset, EstimationInfo *estinfo) { int fss; double predicted; @@ -476,13 +429,15 @@ aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, /* XXX: Don't support some GROUPING options */ goto default_estimator; - if (prev_estimate_num_groups_hook != NULL) - elog(WARNING, "AQO replaced another estimator of a groups number"); - /* Zero the estinfo output parameter, if non-NULL */ if (estinfo != NULL) memset(estinfo, 0, sizeof(EstimationInfo)); + if (aqo_estimate_num_groups_next != NULL || + estimate_num_groups_hook != aqo_estimate_num_groups) + /* It is unclear that to do in situation of such kind. Just report it */ + elog(WARNING, "AQO is in the middle of the estimate_num_groups_hook chain"); + if (groupExprs == NIL) return 1.0; @@ -507,6 +462,44 @@ aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, MemoryContextSwitchTo(old_ctx_m); default_estimator: - return default_estimate_num_groups(root, groupExprs, subpath, grouped_rel, - pgset, estinfo); + if (aqo_estimate_num_groups_next) + return (*aqo_estimate_num_groups_next)(root, groupExprs, subpath, + grouped_rel, pgset, estinfo); + else + return estimate_num_groups(root, groupExprs, subpath->rows, + pgset, estinfo); +} + +void +aqo_cardinality_hooks_init(void) +{ + if (set_baserel_rows_estimate_hook || + set_foreign_rows_estimate_hook || + get_parameterized_baserel_size_hook || + set_joinrel_size_estimates_hook || + get_parameterized_joinrel_size_hook || + parampathinfo_postinit_hook || + estimate_num_groups_hook) + elog(ERROR, "AQO estimation hooks shouldn't be intercepted"); + + aqo_set_baserel_rows_estimate_next = set_baserel_rows_estimate_standard; + set_baserel_rows_estimate_hook = aqo_set_baserel_rows_estimate; + + /* XXX: we have a problem here. Should be redesigned later */ + set_foreign_rows_estimate_hook = aqo_set_baserel_rows_estimate; + + aqo_get_parameterized_baserel_size_next = get_parameterized_baserel_size_standard; + get_parameterized_baserel_size_hook = aqo_get_parameterized_baserel_size; + + aqo_set_joinrel_size_estimates_next = set_joinrel_size_estimates_standard; + set_joinrel_size_estimates_hook = aqo_set_joinrel_size_estimates; + + aqo_get_parameterized_joinrel_size_next = get_parameterized_joinrel_size_standard; + get_parameterized_joinrel_size_hook = aqo_get_parameterized_joinrel_size; + + aqo_set_parampathinfo_postinit_next = parampathinfo_postinit_hook; + parampathinfo_postinit_hook = aqo_parampathinfo_postinit; + + aqo_estimate_num_groups_next = estimate_num_groups_hook; + estimate_num_groups_hook = aqo_estimate_num_groups; } diff --git a/cardinality_hooks.h b/cardinality_hooks.h deleted file mode 100644 index c34f9315..00000000 --- a/cardinality_hooks.h +++ /dev/null @@ -1,32 +0,0 @@ -#ifndef CARDINALITY_HOOKS_H -#define CARDINALITY_HOOKS_H - -#include "optimizer/planner.h" -#include "utils/selfuncs.h" - -extern estimate_num_groups_hook_type prev_estimate_num_groups_hook; - - -/* Cardinality estimation hooks */ -extern void aqo_set_baserel_rows_estimate(PlannerInfo *root, RelOptInfo *rel); -extern double aqo_get_parameterized_baserel_size(PlannerInfo *root, - RelOptInfo *rel, - List *param_clauses); -extern void aqo_set_joinrel_size_estimates(PlannerInfo *root, RelOptInfo *rel, - RelOptInfo *outer_rel, - RelOptInfo *inner_rel, - SpecialJoinInfo *sjinfo, - List *restrictlist); -extern double aqo_get_parameterized_joinrel_size(PlannerInfo *root, - RelOptInfo *rel, - Path *outer_path, - Path *inner_path, - SpecialJoinInfo *sjinfo, - List *restrict_clauses); -extern double aqo_estimate_num_groups_hook(PlannerInfo *root, List *groupExprs, - Path *subpath, - RelOptInfo *grouped_rel, - List **pgset, - EstimationInfo *estinfo); - -#endif /* CARDINALITY_HOOKS_H */ diff --git a/hash.h b/hash.h index eb4b2b97..a1738ac4 100644 --- a/hash.h +++ b/hash.h @@ -13,4 +13,9 @@ extern int get_fss_for_object(List *relsigns, List *clauselist, extern int get_int_array_hash(int *arr, int len); extern int get_grouped_exprs_hash(int fss, List *group_exprs); +/* Hash functions */ +void get_eclasses(List *clauselist, int *nargs, int **args_hash, + int **eclass_hash); +int get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash); + #endif /* AQO_HASH_H */ \ No newline at end of file diff --git a/path_utils.c b/path_utils.c index 5becfcc1..5a34b645 100644 --- a/path_utils.c +++ b/path_utils.c @@ -15,8 +15,11 @@ #include "access/relation.h" #include "nodes/readfuncs.h" +#include "optimizer/cost.h" #include "optimizer/optimizer.h" +#include "optimizer/planmain.h" #include "path_utils.h" +#include "storage/lmgr.h" #include "utils/syscache.h" #include "utils/lsyscache.h" @@ -25,13 +28,6 @@ #include "postgres_fdw.h" -/* - * Hook on creation of a plan node. We need to store AQO-specific data to - * support learning stage. - */ -create_plan_hook_type prev_create_plan_hook = NULL; - -create_upper_paths_hook_type prev_create_upper_paths_hook = NULL; static AQOPlanNode DefaultAQOPlanNode = { @@ -49,6 +45,15 @@ static AQOPlanNode DefaultAQOPlanNode = .prediction = -1 }; +/* + * Hook on creation of a plan node. We need to store AQO-specific data to + * support learning stage. + */ +static create_plan_hook_type aqo_create_plan_next = NULL; + +static create_upper_paths_hook_type aqo_create_upper_paths_next = NULL; + + static AQOPlanNode * create_aqo_plan_node() { @@ -175,8 +180,6 @@ hashTempTupleDesc(TupleDesc desc) return s; } -#include "storage/lmgr.h" - /* * Get list of relation indexes and prepare list of permanent table reloids, * list of temporary table reloids (can be changed between query launches) and @@ -514,15 +517,15 @@ is_appropriate_path(Path *path) * store AQO prediction in the same context, as the plan. So, explicitly free * all unneeded data. */ -void -aqo_create_plan_hook(PlannerInfo *root, Path *src, Plan **dest) +static void +aqo_create_plan(PlannerInfo *root, Path *src, Plan **dest) { bool is_join_path; Plan *plan = *dest; AQOPlanNode *node; - if (prev_create_plan_hook) - prev_create_plan_hook(root, src, dest); + if (aqo_create_plan_next) + (*aqo_create_plan_next)(root, src, dest); if (!query_context.use_aqo && !query_context.learn_aqo && !query_context.collect_stat) @@ -767,20 +770,20 @@ RegisterAQOPlanNodeMethods(void) * * Assume, that we are last in the chain of path creators. */ -void -aqo_store_upper_signature_hook(PlannerInfo *root, - UpperRelationKind stage, - RelOptInfo *input_rel, - RelOptInfo *output_rel, - void *extra) +static void +aqo_store_upper_signature(PlannerInfo *root, + UpperRelationKind stage, + RelOptInfo *input_rel, + RelOptInfo *output_rel, + void *extra) { A_Const *fss_node = makeNode(A_Const); RelSortOut rels = {NIL, NIL}; List *clauses; List *selectivities; - if (prev_create_upper_paths_hook) - (*prev_create_upper_paths_hook)(root, stage, input_rel, output_rel, extra); + if (aqo_create_upper_paths_next) + (*aqo_create_upper_paths_next)(root, stage, input_rel, output_rel, extra); if (!query_context.use_aqo && !query_context.learn_aqo && !force_collect_stat) /* Includes 'disabled query' state. */ @@ -799,3 +802,13 @@ aqo_store_upper_signature_hook(PlannerInfo *root, NULL, NULL); output_rel->ext_nodes = lappend(output_rel->ext_nodes, (void *) fss_node); } + +void +aqo_path_utils_init(void) +{ + aqo_create_plan_next = create_plan_hook; + create_plan_hook = aqo_create_plan; + + aqo_create_upper_paths_next = create_upper_paths_hook; + create_upper_paths_hook = aqo_store_upper_signature; +} diff --git a/path_utils.h b/path_utils.h index 1803e08d..cbe83da0 100644 --- a/path_utils.h +++ b/path_utils.h @@ -3,7 +3,6 @@ #include "nodes/extensible.h" #include "nodes/pathnodes.h" -#include "optimizer/planmain.h" #include "optimizer/planner.h" #define AQO_PLAN_NODE "AQOPlanNode" @@ -52,8 +51,6 @@ typedef struct AQOPlanNode #define booltostr(x) ((x) ? "true" : "false") -extern create_plan_hook_type prev_create_plan_hook; - /* Extracting path information utilities */ extern List *get_selectivities(PlannerInfo *root, List *clauses, @@ -67,16 +64,11 @@ extern List *get_path_clauses(Path *path, PlannerInfo *root, List **selectivities); -extern void aqo_create_plan_hook(PlannerInfo *root, Path *src, Plan **dest); extern AQOPlanNode *get_aqo_plan_node(Plan *plan, bool create); extern void RegisterAQOPlanNodeMethods(void); -extern create_upper_paths_hook_type prev_create_upper_paths_hook; -extern void aqo_store_upper_signature_hook(PlannerInfo *root, - UpperRelationKind stage, - RelOptInfo *input_rel, - RelOptInfo *output_rel, - void *extra); extern List *aqo_get_clauses(PlannerInfo *root, List *restrictlist); +void aqo_path_utils_init(void); + #endif /* PATH_UTILS_H */ diff --git a/postprocessing.c b/postprocessing.c index aa82a534..ba2e19e0 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -27,7 +27,6 @@ #include "hash.h" #include "path_utils.h" #include "machine_learning.h" -#include "preprocessing.h" #include "storage.h" @@ -58,6 +57,13 @@ static int64 growth_rate = 3; static char *AQOPrivateData = "AQOPrivateData"; static char *PlanStateInfo = "PlanStateInfo"; +/* Saved hooks */ +static ExecutorStart_hook_type aqo_ExecutorStart_next = NULL; +static ExecutorRun_hook_type aqo_ExecutorRun_next = NULL; +static ExecutorEnd_hook_type aqo_ExecutorEnd_next = NULL; +static ExplainOnePlan_hook_type aqo_ExplainOnePlan_next = NULL; +static ExplainOneNode_hook_type aqo_ExplainOneNode_next = NULL; + /* Query execution statistics collecting utilities */ static void atomic_fss_learn_step(uint64 fhash, int fss, OkNNrdata *data, @@ -542,7 +548,7 @@ learnOnPlanState(PlanState *p, void *context) /* * Set up flags to store cardinality statistics. */ -void +static void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags) { instr_time now; @@ -594,10 +600,7 @@ aqo_ExecutorStart(QueryDesc *queryDesc, int eflags) StoreToQueryEnv(queryDesc); } - if (prev_ExecutorStart_hook) - prev_ExecutorStart_hook(queryDesc, eflags); - else - standard_ExecutorStart(queryDesc, eflags); + (*aqo_ExecutorStart_next)(queryDesc, eflags); if (use_aqo) StorePlanInternals(queryDesc); @@ -706,7 +709,7 @@ set_timeout_if_need(QueryDesc *queryDesc) /* * ExecutorRun hook. */ -void +static void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, bool execute_once) { @@ -722,10 +725,7 @@ aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, PG_TRY(); { - if (prev_ExecutorRun) - prev_ExecutorRun(queryDesc, direction, count, execute_once); - else - standard_ExecutorRun(queryDesc, direction, count, execute_once); + (*aqo_ExecutorRun_next)(queryDesc, direction, count, execute_once); } PG_FINALLY(); { @@ -743,7 +743,7 @@ aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction, uint64 count, * cardinality statistics. * Also it updates query execution statistics in aqo_query_stat. */ -void +static void aqo_ExecutorEnd(QueryDesc *queryDesc) { double execution_time; @@ -841,10 +841,7 @@ aqo_ExecutorEnd(QueryDesc *queryDesc) MemoryContextSwitchTo(oldctx); MemoryContextReset(AQOLearnMemCtx); - if (prev_ExecutorEnd_hook) - prev_ExecutorEnd_hook(queryDesc); - else - standard_ExecutorEnd(queryDesc); + (*aqo_ExecutorEnd_next)(queryDesc); /* * standard_ExecutorEnd clears the queryDesc->planstate. After this point no @@ -975,7 +972,64 @@ ExtractFromQueryEnv(QueryDesc *queryDesc) return true; } -void +/* + * Prints if the plan was constructed with AQO. + */ +static void +print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, + ExplainState *es, const char *queryString, + ParamListInfo params, const instr_time *planduration, + QueryEnvironment *queryEnv) +{ + if (aqo_ExplainOnePlan_next) + (*aqo_ExplainOnePlan_next)(plannedstmt, into, es, queryString, + params, planduration, queryEnv); + + if (IsQueryDisabled() || !aqo_show_details) + return; + + /* Report to user about aqo state only in verbose mode */ + ExplainPropertyBool("Using aqo", query_context.use_aqo, es); + + switch (aqo_mode) + { + case AQO_MODE_INTELLIGENT: + ExplainPropertyText("AQO mode", "INTELLIGENT", es); + break; + case AQO_MODE_FORCED: + ExplainPropertyText("AQO mode", "FORCED", es); + break; + case AQO_MODE_CONTROLLED: + ExplainPropertyText("AQO mode", "CONTROLLED", es); + break; + case AQO_MODE_LEARN: + ExplainPropertyText("AQO mode", "LEARN", es); + break; + case AQO_MODE_FROZEN: + ExplainPropertyText("AQO mode", "FROZEN", es); + break; + case AQO_MODE_DISABLED: + ExplainPropertyText("AQO mode", "DISABLED", es); + break; + default: + elog(ERROR, "Bad AQO state"); + break; + } + + /* + * Query class provides an user the conveniently use of the AQO + * auxiliary functions. + */ + if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) + { + if (aqo_show_hash) + ExplainPropertyInteger("Query hash", NULL, + query_context.query_hash, es); + ExplainPropertyInteger("JOINS", NULL, njoins, es); + } +} + +static void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan) { int wrkrs = 1; @@ -983,8 +1037,8 @@ print_node_explain(ExplainState *es, PlanState *ps, Plan *plan) AQOPlanNode *aqo_node; /* Extension, which took a hook early can be executed early too. */ - if (prev_ExplainOneNode_hook) - prev_ExplainOneNode_hook(es, ps, plan); + if (aqo_ExplainOneNode_next) + (*aqo_ExplainOneNode_next)(es, ps, plan); if (IsQueryDisabled() || !plan || es->format != EXPLAIN_FORMAT_TEXT) return; @@ -1042,59 +1096,20 @@ print_node_explain(ExplainState *es, PlanState *ps, Plan *plan) appendStringInfo(es->str, ", fss=%d", aqo_node->fss); } -/* - * Prints if the plan was constructed with AQO. - */ void -print_into_explain(PlannedStmt *plannedstmt, IntoClause *into, - ExplainState *es, const char *queryString, - ParamListInfo params, const instr_time *planduration, - QueryEnvironment *queryEnv) +aqo_postprocessing_init(void) { - if (prev_ExplainOnePlan_hook) - prev_ExplainOnePlan_hook(plannedstmt, into, es, queryString, - params, planduration, queryEnv); - - if (IsQueryDisabled() || !aqo_show_details) - return; - - /* Report to user about aqo state only in verbose mode */ - ExplainPropertyBool("Using aqo", query_context.use_aqo, es); - - switch (aqo_mode) - { - case AQO_MODE_INTELLIGENT: - ExplainPropertyText("AQO mode", "INTELLIGENT", es); - break; - case AQO_MODE_FORCED: - ExplainPropertyText("AQO mode", "FORCED", es); - break; - case AQO_MODE_CONTROLLED: - ExplainPropertyText("AQO mode", "CONTROLLED", es); - break; - case AQO_MODE_LEARN: - ExplainPropertyText("AQO mode", "LEARN", es); - break; - case AQO_MODE_FROZEN: - ExplainPropertyText("AQO mode", "FROZEN", es); - break; - case AQO_MODE_DISABLED: - ExplainPropertyText("AQO mode", "DISABLED", es); - break; - default: - elog(ERROR, "Bad AQO state"); - break; - } - - /* - * Query class provides an user the conveniently use of the AQO - * auxiliary functions. - */ - if (aqo_mode != AQO_MODE_DISABLED || force_collect_stat) - { - if (aqo_show_hash) - ExplainPropertyInteger("Query hash", NULL, - query_context.query_hash, es); - ExplainPropertyInteger("JOINS", NULL, njoins, es); - } + /* Executor hooks */ + aqo_ExecutorStart_next = ExecutorStart_hook ? ExecutorStart_hook : standard_ExecutorStart; + ExecutorStart_hook = aqo_ExecutorStart; + aqo_ExecutorRun_next = ExecutorRun_hook ? ExecutorRun_hook : standard_ExecutorRun; + ExecutorRun_hook = aqo_ExecutorRun; + aqo_ExecutorEnd_next = ExecutorEnd_hook ? ExecutorEnd_hook : standard_ExecutorEnd; + ExecutorEnd_hook = aqo_ExecutorEnd; + + /* Service hooks. */ + aqo_ExplainOnePlan_next = ExplainOnePlan_hook; + ExplainOnePlan_hook = print_into_explain; + aqo_ExplainOneNode_next = ExplainOneNode_hook; + ExplainOneNode_hook = print_node_explain; } diff --git a/preprocessing.c b/preprocessing.c index aadc959e..03c3432a 100644 --- a/preprocessing.c +++ b/preprocessing.c @@ -64,7 +64,6 @@ #include "parser/scansup.h" #include "aqo.h" #include "hash.h" -#include "preprocessing.h" #include "storage.h" /* List of feature spaces, that are processing in this backend. */ @@ -72,37 +71,19 @@ List *cur_classes = NIL; int aqo_join_threshold = 0; +static planner_hook_type aqo_planner_next = NULL; + +static void disable_aqo_for_query(void); static bool isQueryUsingSystemRelation(Query *query); static bool isQueryUsingSystemRelation_walker(Node *node, void *context); -/* - * Calls standard query planner or its previous hook. - */ -static PlannedStmt * -call_default_planner(Query *parse, - const char *query_string, - int cursorOptions, - ParamListInfo boundParams) -{ - if (prev_planner_hook) - return prev_planner_hook(parse, - query_string, - cursorOptions, - boundParams); - else - return standard_planner(parse, - query_string, - cursorOptions, - boundParams); -} - /* * Can AQO be used for the query? */ static bool aqoIsEnabled(Query *parse) { - if (creating_extension || + if (creating_extension || !IsQueryIdEnabled() || (aqo_mode == AQO_MODE_DISABLED && !force_collect_stat) || (parse->commandType != CMD_SELECT && parse->commandType != CMD_INSERT && parse->commandType != CMD_UPDATE && parse->commandType != CMD_DELETE)) @@ -119,10 +100,8 @@ aqoIsEnabled(Query *parse) * Creates an entry in aqo_queries for new type of query if it is * necessary, i. e. AQO mode is "intelligent". */ -PlannedStmt * -aqo_planner(Query *parse, - const char *query_string, - int cursorOptions, +static PlannedStmt * +aqo_planner(Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams) { bool query_is_stored = false; @@ -146,10 +125,7 @@ aqo_planner(Query *parse, */ disable_aqo_for_query(); - return call_default_planner(parse, - query_string, - cursorOptions, - boundParams); + return (*aqo_planner_next)(parse, query_string, cursorOptions, boundParams); } selectivity_cache_clear(); @@ -175,10 +151,7 @@ aqo_planner(Query *parse, */ disable_aqo_for_query(); - return call_default_planner(parse, - query_string, - cursorOptions, - boundParams); + return (*aqo_planner_next)(parse, query_string, cursorOptions, boundParams); } elog(DEBUG1, "AQO will be used for query '%s', class "UINT64_FORMAT, @@ -346,8 +319,7 @@ aqo_planner(Query *parse, { PlannedStmt *stmt; - stmt = call_default_planner(parse, query_string, - cursorOptions, boundParams); + stmt = (*aqo_planner_next)(parse, query_string, cursorOptions, boundParams); /* Release the memory, allocated for AQO predictions */ MemoryContextReset(AQOPredictMemCtx); @@ -358,7 +330,7 @@ aqo_planner(Query *parse, /* * Turn off all AQO functionality for the current query. */ -void +static void disable_aqo_for_query(void) { query_context.learn_aqo = false; @@ -505,3 +477,10 @@ isQueryUsingSystemRelation_walker(Node *node, void *context) isQueryUsingSystemRelation_walker, context); } + +void +aqo_preprocessing_init(void) +{ + aqo_planner_next = planner_hook ? planner_hook : standard_planner; + planner_hook = aqo_planner; +} \ No newline at end of file diff --git a/preprocessing.h b/preprocessing.h deleted file mode 100644 index f27deb91..00000000 --- a/preprocessing.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __PREPROCESSING_H__ -#define __PREPROCESSING_H__ - -#include "nodes/pathnodes.h" -#include "nodes/plannodes.h" -extern PlannedStmt *aqo_planner(Query *parse, - const char *query_string, - int cursorOptions, - ParamListInfo boundParams); -extern void disable_aqo_for_query(void); - -#endif /* __PREPROCESSING_H__ */ diff --git a/storage.c b/storage.c index be14f3e9..0bdee72d 100644 --- a/storage.c +++ b/storage.c @@ -22,11 +22,11 @@ #include "funcapi.h" #include "miscadmin.h" #include "pgstat.h" +#include "storage/ipc.h" #include "aqo.h" #include "aqo_shared.h" #include "machine_learning.h" -#include "preprocessing.h" #include "storage.h" diff --git a/storage.h b/storage.h index 35d94336..2b4e4cdd 100644 --- a/storage.h +++ b/storage.h @@ -164,4 +164,10 @@ extern void init_deactivated_queries_storage(void); extern bool query_is_deactivated(uint64 query_hash); extern void add_deactivated_query(uint64 query_hash); +/* Storage interaction */ +extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids); +extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids); + +extern bool update_query_timeout(uint64 queryid, int64 smart_timeout); + #endif /* STORAGE_H */