Catch up stable14

Makefile

@@ -30,7 +30,7 @@ EXTRA_INSTALL = contrib/postgres_fdw contrib/pg_stat_statements
 
 DATA = aqo--1.0.sql aqo--1.0--1.1.sql aqo--1.1--1.2.sql aqo--1.2.sql \
 		aqo--1.2--1.3.sql aqo--1.3--1.4.sql aqo--1.4--1.5.sql \
-		aqo--1.5--1.6.sql
+		aqo--1.5--1.6.sql aqo--1.6.sql
 
 ifdef USE_PGXS
 PG_CONFIG ?= pg_config

aqo--1.6.sql

@@ -0,0 +1,208 @@
+/* contrib/aqo/aqo--1.6.sql */
+
+-- complain if script is sourced in psql, rather than via CREATE EXTENSION
+\echo Use "CREATE EXTENSION aqo" to load this file. \quit
+
+--
+-- Get cardinality error of queries the last time they were executed.
+-- IN:
+-- controlled - show queries executed under a control of AQO (true);
+-- executed without an AQO control, but AQO has a stat on the query (false).
+--
+-- OUT:
+-- num - sequental number. Smaller number corresponds to higher error.
+-- id - ID of a query.
+-- fshash - feature space. Usually equal to zero or ID.
+-- error - AQO error that calculated on plan nodes of the query.
+-- nexecs - number of executions of queries associated with this ID.
+--
+CREATE FUNCTION aqo_cardinality_error(controlled boolean)
+RETURNS TABLE(num integer, id bigint, fshash bigint, error double precision, nexecs bigint)
+AS 'MODULE_PATHNAME', 'aqo_cardinality_error'
+LANGUAGE C STRICT VOLATILE;
+COMMENT ON FUNCTION aqo_cardinality_error(boolean) IS
+'Get cardinality error of queries the last time they were executed. Order queries according to an error value.';
+
+--
+-- Remove unneeded rows from the AQO ML storage.
+-- For common feature space, remove rows from aqo_data only.
+-- For custom feature space - remove all rows related to the space from all AQO
+-- tables even if only one oid for one feature subspace of the space is illegal.
+-- Returns number of deleted rows from aqo_queries and aqo_data tables.
+--
+CREATE FUNCTION aqo_cleanup(OUT nfs integer, OUT nfss integer)
+RETURNS record
+AS 'MODULE_PATHNAME', 'aqo_cleanup'
+LANGUAGE C STRICT VOLATILE;
+COMMENT ON FUNCTION aqo_cleanup() IS
+'Remove unneeded rows from the AQO ML storage';
+
+CREATE FUNCTION aqo_disable_class(queryid bigint)
+RETURNS void
+AS 'MODULE_PATHNAME', 'aqo_disable_query'
+LANGUAGE C STRICT VOLATILE;
+COMMENT ON FUNCTION aqo_disable_class(bigint) IS
+'Set learn_aqo, use_aqo and auto_tuning into false for a class of queries with specific queryid.';
+
+--
+-- Remove query class settings, text, statistics and ML data from AQO storage.
+-- Return number of FSS records, removed from the storage.
+--
+CREATE FUNCTION aqo_drop_class(queryid bigint)
+RETURNS integer
+AS 'MODULE_PATHNAME', 'aqo_drop_class'
+LANGUAGE C STRICT VOLATILE;
+COMMENT ON FUNCTION aqo_drop_class(bigint) IS
+'Remove info about an query class from AQO ML knowledge base.';
+
+CREATE FUNCTION aqo_enable_class(queryid bigint)
+RETURNS void
+AS 'MODULE_PATHNAME', 'aqo_enable_query'
+LANGUAGE C STRICT VOLATILE;
+COMMENT ON FUNCTION aqo_enable_class(bigint) IS
+'Set learn_aqo, use_aqo and auto_tuning (in intelligent mode) into true for a class of queries with specific queryid.';
+
+--
+-- Show execution time of queries, for which AQO has statistics.
+-- controlled - show stat on executions where AQO was used for cardinality
+-- estimations, or not used (controlled = false).
+-- Last case is possible in disabled mode with aqo.force_collect_stat = 'on'.
+--
+CREATE FUNCTION aqo_execution_time(controlled boolean)
+RETURNS TABLE(num integer, id bigint, fshash bigint, exec_time double precision, nexecs bigint)
+AS 'MODULE_PATHNAME', 'aqo_execution_time'
+LANGUAGE C STRICT VOLATILE;
+COMMENT ON FUNCTION aqo_execution_time(boolean) IS
+'Get execution time of queries. If controlled = true (AQO could advise cardinality estimations), show time of last execution attempt. Another case (AQO not used), return an average value of execution time across all known executions.';
+
+-- Show how much shared memory AQO are using at the moment
+CREATE FUNCTION aqo_memory_usage(
+  OUT name text,
+  OUT allocated_size int,
+  OUT used_size int
+)
+RETURNS SETOF record
+AS $$
+  SELECT name, allocated_size, size FROM pg_shmem_allocations
+  WHERE name LIKE 'AQO%';
+$$ LANGUAGE SQL;
+COMMENT ON FUNCTION aqo_memory_usage() IS
+'Show how much shared memory AQO are using at the moment';
+
+--
+-- Update or insert an aqo_data
+-- table record for given 'fs' & 'fss'.
+--
+
+CREATE FUNCTION aqo_data_update(
+  fs			bigint,
+  fss			integer,
+  nfeatures		integer,
+  features		double precision[][],
+  targets		double precision[],
+  reliability	double precision[],
+  oids			Oid[])
+RETURNS bool
+AS 'MODULE_PATHNAME', 'aqo_data_update'
+LANGUAGE C VOLATILE;
+
+CREATE FUNCTION aqo_queries_update(
+  queryid bigint, fs bigint, learn_aqo bool, use_aqo bool, auto_tuning bool)
+RETURNS bool
+AS 'MODULE_PATHNAME', 'aqo_queries_update'
+LANGUAGE C VOLATILE;
+
+--
+-- Update or insert an aqo_query_stat
+-- table record for given 'queryid'.
+--
+CREATE FUNCTION aqo_query_stat_update(
+  queryid						bigint,
+  execution_time_with_aqo		double precision[],
+  execution_time_without_aqo	double precision[],
+  planning_time_with_aqo		double precision[],
+  planning_time_without_aqo		double precision[],
+  cardinality_error_with_aqo	double precision[],
+  cardinality_error_without_aqo	double precision[],
+  executions_with_aqo			bigint,
+  executions_without_aqo		bigint)
+RETURNS bool
+AS 'MODULE_PATHNAME', 'aqo_query_stat_update'
+LANGUAGE C VOLATILE;
+
+--
+-- Update or insert an aqo_query_texts
+-- table record for given 'queryid'.
+--
+CREATE FUNCTION aqo_query_texts_update(
+  queryid bigint, query_text text)
+RETURNS bool
+AS 'MODULE_PATHNAME', 'aqo_query_texts_update'
+LANGUAGE C VOLATILE;
+
+--
+-- Remove all records in the AQO storage.
+-- Return number of rows removed.
+--
+CREATE FUNCTION aqo_reset() RETURNS bigint
+AS 'MODULE_PATHNAME', 'aqo_reset'
+LANGUAGE C PARALLEL SAFE;
+COMMENT ON FUNCTION aqo_reset() IS
+'Reset all data gathered by AQO';
+
+-- -----------------------------------------------------------------------------
+--
+-- VIEWs
+--
+-- -----------------------------------------------------------------------------
+
+CREATE FUNCTION aqo_data (
+  OUT fs			bigint,
+  OUT fss			integer,
+  OUT nfeatures		integer,
+  OUT features		double precision[][],
+  OUT targets		double precision[],
+  OUT reliability	double precision[],
+  OUT oids			Oid[]
+)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'aqo_data'
+LANGUAGE C STRICT VOLATILE PARALLEL SAFE;
+
+CREATE FUNCTION aqo_queries (
+  OUT queryid                bigint,
+  OUT fs                     bigint,
+  OUT learn_aqo              boolean,
+  OUT use_aqo                boolean,
+  OUT auto_tuning            boolean,
+  OUT smart_timeout          bigint,
+  OUT count_increase_timeout bigint
+)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'aqo_queries'
+LANGUAGE C STRICT VOLATILE PARALLEL SAFE;
+
+CREATE FUNCTION aqo_query_stat (
+  OUT queryid						bigint,
+  OUT execution_time_with_aqo		double precision[],
+  OUT execution_time_without_aqo	double precision[],
+  OUT planning_time_with_aqo		double precision[],
+  OUT planning_time_without_aqo		double precision[],
+  OUT cardinality_error_with_aqo	double precision[],
+  OUT cardinality_error_without_aqo	double precision[],
+  OUT executions_with_aqo bigint,
+  OUT executions_without_aqo		bigint
+)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'aqo_query_stat'
+LANGUAGE C STRICT VOLATILE PARALLEL SAFE;
+
+CREATE FUNCTION aqo_query_texts(OUT queryid bigint, OUT query_text text)
+RETURNS SETOF record
+AS 'MODULE_PATHNAME', 'aqo_query_texts'
+LANGUAGE C STRICT VOLATILE PARALLEL SAFE;
+
+CREATE VIEW aqo_data AS SELECT * FROM aqo_data();
+CREATE VIEW aqo_queries AS SELECT * FROM aqo_queries();
+CREATE VIEW aqo_query_stat AS SELECT * FROM aqo_query_stat();
+CREATE VIEW aqo_query_texts AS SELECT * FROM aqo_query_texts();

aqo.c

@@ -19,9 +19,7 @@
 
 #include "aqo.h"
 #include "aqo_shared.h"
-#include "cardinality_hooks.h"
 #include "path_utils.h"
-#include "preprocessing.h"
 #include "storage.h"
 
 
@@ -98,19 +96,6 @@ MemoryContext 		AQOLearnMemCtx = NULL;
 /* Additional plan info */
 int njoins;
 
-/* Saved hook values */
-post_parse_analyze_hook_type				prev_post_parse_analyze_hook;
-planner_hook_type							prev_planner_hook;
-ExecutorStart_hook_type						prev_ExecutorStart_hook;
-ExecutorRun_hook_type						prev_ExecutorRun;
-ExecutorEnd_hook_type						prev_ExecutorEnd_hook;
-set_baserel_rows_estimate_hook_type			prev_set_foreign_rows_estimate_hook;
-set_baserel_rows_estimate_hook_type			prev_set_baserel_rows_estimate_hook;
-get_parameterized_baserel_size_hook_type	prev_get_parameterized_baserel_size_hook;
-set_joinrel_size_estimates_hook_type		prev_set_joinrel_size_estimates_hook;
-get_parameterized_joinrel_size_hook_type	prev_get_parameterized_joinrel_size_hook;
-ExplainOnePlan_hook_type					prev_ExplainOnePlan_hook;
-ExplainOneNode_hook_type					prev_ExplainOneNode_hook;
 
 /*****************************************************************************
  *
@@ -324,42 +309,11 @@ _PG_init(void)
 							 NULL,
 							 NULL);
 
-	prev_shmem_startup_hook						= shmem_startup_hook;
-	shmem_startup_hook							= aqo_init_shmem;
-	prev_planner_hook							= planner_hook;
-	planner_hook								= aqo_planner;
-	prev_ExecutorStart_hook						= ExecutorStart_hook;
-	ExecutorStart_hook							= aqo_ExecutorStart;
-	prev_ExecutorRun							= ExecutorRun_hook;
-	ExecutorRun_hook							= aqo_ExecutorRun;
-	prev_ExecutorEnd_hook						= ExecutorEnd_hook;
-	ExecutorEnd_hook							= aqo_ExecutorEnd;
-
-	/* Cardinality prediction hooks. */
-	prev_set_baserel_rows_estimate_hook			= set_baserel_rows_estimate_hook;
-	set_foreign_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
-	set_baserel_rows_estimate_hook				= aqo_set_baserel_rows_estimate;
-	prev_get_parameterized_baserel_size_hook	= get_parameterized_baserel_size_hook;
-	get_parameterized_baserel_size_hook			= aqo_get_parameterized_baserel_size;
-	prev_set_joinrel_size_estimates_hook		= set_joinrel_size_estimates_hook;
-	set_joinrel_size_estimates_hook				= aqo_set_joinrel_size_estimates;
-	prev_get_parameterized_joinrel_size_hook	= get_parameterized_joinrel_size_hook;
-	get_parameterized_joinrel_size_hook			= aqo_get_parameterized_joinrel_size;
-	prev_estimate_num_groups_hook				= estimate_num_groups_hook;
-	estimate_num_groups_hook					= aqo_estimate_num_groups_hook;
-	parampathinfo_postinit_hook					= ppi_hook;
-
-	prev_create_plan_hook						= create_plan_hook;
-	create_plan_hook							= aqo_create_plan_hook;
-
-	/* Service hooks. */
-	prev_ExplainOnePlan_hook					= ExplainOnePlan_hook;
-	ExplainOnePlan_hook							= print_into_explain;
-	prev_ExplainOneNode_hook					= ExplainOneNode_hook;
-	ExplainOneNode_hook							= print_node_explain;
-
-	prev_create_upper_paths_hook				= create_upper_paths_hook;
-	create_upper_paths_hook						= aqo_store_upper_signature_hook;
+	aqo_shmem_init();
+	aqo_preprocessing_init();
+	aqo_postprocessing_init();
+	aqo_cardinality_hooks_init();
+	aqo_path_utils_init();
 
 	init_deactivated_queries_storage();
 
@@ -394,7 +348,6 @@ _PG_init(void)
 	RegisterAQOPlanNodeMethods();
 
 	EmitWarningsOnPlaceholders("aqo");
-	RequestAddinShmemSpace(aqo_memsize());
 }
 
 /*

aqo.h

@@ -132,19 +132,16 @@
 #include "nodes/nodeFuncs.h"
 #include "optimizer/pathnode.h"
 #include "optimizer/planner.h"
-#include "optimizer/cost.h"
 #include "parser/analyze.h"
 #include "parser/parsetree.h"
 #include "utils/builtins.h"
 #include "utils/guc.h"
 #include "utils/hsearch.h"
 #include "utils/memutils.h"
 #include "utils/rel.h"
-#include "utils/fmgroids.h"
 #include "utils/snapmgr.h"
 
 #include "machine_learning.h"
-//#include "storage.h"
 
 /* Check PostgreSQL version (9.6.0 contains important changes in planner) */
 #if PG_VERSION_NUM < 90600
@@ -237,58 +234,15 @@ extern MemoryContext AQOCacheMemCtx;
 extern MemoryContext AQOPredictMemCtx;
 extern MemoryContext AQOLearnMemCtx;
 
-/* Saved hook values in case of unload */
-extern post_parse_analyze_hook_type prev_post_parse_analyze_hook;
-extern planner_hook_type prev_planner_hook;
-extern ExecutorStart_hook_type prev_ExecutorStart_hook;
-extern ExecutorRun_hook_type prev_ExecutorRun;
-extern ExecutorEnd_hook_type prev_ExecutorEnd_hook;
-extern set_baserel_rows_estimate_hook_type
-										prev_set_foreign_rows_estimate_hook;
-extern set_baserel_rows_estimate_hook_type
-										prev_set_baserel_rows_estimate_hook;
-extern get_parameterized_baserel_size_hook_type
-									prev_get_parameterized_baserel_size_hook;
-extern set_joinrel_size_estimates_hook_type
-										prev_set_joinrel_size_estimates_hook;
-extern get_parameterized_joinrel_size_hook_type
-									prev_get_parameterized_joinrel_size_hook;
-extern ExplainOnePlan_hook_type prev_ExplainOnePlan_hook;
-extern ExplainOneNode_hook_type prev_ExplainOneNode_hook;
-
-extern void ppi_hook(ParamPathInfo *ppi);
 extern int aqo_statement_timeout;
 
-/* Hash functions */
-void get_eclasses(List *clauselist, int *nargs, int **args_hash,
-				  int **eclass_hash);
-int get_clause_hash(Expr *clause, int nargs, int *args_hash, int *eclass_hash);
-
-
-/* Storage interaction */
-extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids);
-extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids);
-
-/* Query preprocessing hooks */
-extern void print_into_explain(PlannedStmt *plannedstmt, IntoClause *into,
-							   ExplainState *es, const char *queryString,
-							   ParamListInfo params,
-							   const instr_time *planduration,
-							   QueryEnvironment *queryEnv);
-extern void print_node_explain(ExplainState *es, PlanState *ps, Plan *plan);
-
 /* Cardinality estimation */
 extern double predict_for_relation(List *restrict_clauses, List *selectivities,
 								   List *relsigns, int *fss);
 
-/* Query execution statistics collecting hooks */
-void aqo_ExecutorStart(QueryDesc *queryDesc, int eflags);
-void aqo_ExecutorRun(QueryDesc *queryDesc, ScanDirection direction,
-					 uint64 count, bool execute_once);
-void aqo_ExecutorEnd(QueryDesc *queryDesc);
-
 /* Automatic query tuning */
 extern void automatical_query_tuning(uint64 query_hash, struct StatEntry *stat);
+extern double get_mean(double *elems, int nelems);
 
 /* Utilities */
 extern int int_cmp(const void *a, const void *b);
@@ -306,8 +260,10 @@ extern void selectivity_cache_clear(void);
 
 extern bool IsQueryDisabled(void);
 
-extern bool update_query_timeout(uint64 queryid, int64 smart_timeout);
-extern double get_mean(double *elems, int nelems);
-
 extern List *cur_classes;
+
+extern void aqo_cardinality_hooks_init(void);
+extern void aqo_preprocessing_init(void);
+extern void aqo_postprocessing_init(void);
+
 #endif