diff --git a/.github/scripts/job/aqo_instance_launch.sh b/.github/scripts/job/aqo_instance_launch.sh new file mode 100755 index 00000000..f43d6b8e --- /dev/null +++ b/.github/scripts/job/aqo_instance_launch.sh @@ -0,0 +1,47 @@ +#!/bin/bash +ulimit -c unlimited + +# Kill all orphan processes +pkill -U `whoami` -9 -e postgres +pkill -U `whoami` -9 -e pgbench +pkill -U `whoami` -9 -e psql + +sleep 1 + +M=`pwd`/PGDATA +U=`whoami` + +rm -rf $M || true +mkdir $M +rm -rf logfile.log || true + +export LC_ALL=C +export LANGUAGE="en_US:en" +initdb -D $M --locale=C + +# PG Version-specific settings +ver=$(pg_ctl -V | egrep -o "[0-9]." | head -1) +echo "PostgreSQL version: $ver" +if [ $ver -gt 13 ] +then + echo "compute_query_id = 'regress'" >> $M/postgresql.conf +fi + +# Speed up the 'Join Order Benchmark' test +echo "shared_buffers = 1GB" >> $M/postgresql.conf +echo "work_mem = 128MB" >> $M/postgresql.conf +echo "fsync = off" >> $M/postgresql.conf +echo "autovacuum = 'off'" >> $M/postgresql.conf + +# AQO preferences +echo "shared_preload_libraries = 'aqo, pg_stat_statements'" >> $M/postgresql.conf +echo "aqo.mode = 'disabled'" >> $M/postgresql.conf +echo "aqo.join_threshold = 0" >> $M/postgresql.conf +echo "aqo.force_collect_stat = 'off'" >> $M/postgresql.conf +echo "aqo.fs_max_items = 10000" >> $M/postgresql.conf +echo "aqo.fss_max_items = 20000" >> $M/postgresql.conf + +pg_ctl -w -D $M -l logfile.log start +createdb $U +psql -c "CREATE EXTENSION aqo;" +psql -c "CREATE EXTENSION pg_stat_statements" diff --git a/.github/scripts/job/check_result.sh b/.github/scripts/job/check_result.sh new file mode 100755 index 00000000..ab194cfc --- /dev/null +++ b/.github/scripts/job/check_result.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# ############################################################################## +# +# +# ############################################################################## + +# Show error delta (Negative result is a signal of possible issue) +result=$(psql -t -c "SELECT count(*) FROM aqo_cardinality_error(true) c JOIN aqo_cardinality_error(false) o USING (id) WHERE (o.error - c.error) < 0") + +if [ $result -gt 0 ]; then + exit 1; +fi + +exit 0; diff --git a/.github/scripts/job/dump_knowledge.sh b/.github/scripts/job/dump_knowledge.sh new file mode 100755 index 00000000..c5cb9736 --- /dev/null +++ b/.github/scripts/job/dump_knowledge.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# ############################################################################## +# +# Make dump of a knowledge base +# +# ############################################################################## + +psql -c "CREATE TABLE aqo_data_dump AS SELECT * FROM aqo_data;" +psql -c "CREATE TABLE aqo_queries_dump AS SELECT * FROM aqo_queries;" +psql -c "CREATE TABLE aqo_query_texts_dump AS SELECT * FROM aqo_query_texts;" +psql -c "CREATE TABLE aqo_query_stat_dump AS SELECT * FROM aqo_query_stat;" + +pg_dump --table='aqo*' -f knowledge_base.dump $PGDATABASE + +psql -c "DROP TABLE aqo_data_dump, aqo_queries_dump, aqo_query_texts_dump, aqo_query_stat_dump" + diff --git a/.github/scripts/job/job_pass.sh b/.github/scripts/job/job_pass.sh new file mode 100755 index 00000000..1ad62fbd --- /dev/null +++ b/.github/scripts/job/job_pass.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +# ############################################################################## +# +# Pass each JOB query over the DBMS instance. Use $1 to specify a number of +# iterations, if needed. +# +# Results: +# - explains.txt - explain of each query +# - job_onepass_aqo_stat.dat - short report on execution time +# - knowledge_base.dump - dump of the AQO knowledge base +# +# ############################################################################## + +echo "The Join Order Benchmark 1Pass" +echo -e "Query Number\tITER\tQuery Name\tExecution Time, ms" > report.txt +echo -e "Clear a file with explains" > explains.txt + +if [ $# -eq 0 ] +then + ITERS=1 +else + ITERS=$1 +fi + +echo "Execute JOB with the $ITERS iterations" + +filenum=1 +for file in $JOB_DIR/queries/*.sql +do + # Get filename + short_file=$(basename "$file") + + echo -n "EXPLAIN (ANALYZE, VERBOSE, FORMAT JSON) " > test.sql + cat $file >> test.sql + + for (( i=1; i<=$ITERS; i++ )) + do + result=$(psql -f test.sql) + echo -e $result >> explains.txt + exec_time=$(echo $result | sed -n 's/.*"Execution Time": \([0-9]*\.[0-9]*\).*/\1/p') + echo -e "$filenum\t$short_file\t$i\t$exec_time" >> report.txt + echo -e "$filenum\t$i\t$short_file\t$exec_time" + done +filenum=$((filenum+1)) +done + +# Show total optimizer error in the test +psql -c "SELECT sum(error) AS total_error FROM aqo_cardinality_error(false)" +psql -c "SELECT sum(error) AS total_error_aqo FROM aqo_cardinality_error(true)" + +# Show error delta (Negative result is a signal of possible issue) +psql -c " +SELECT id, (o.error - c.error) AS errdelta + FROM aqo_cardinality_error(true) c JOIN aqo_cardinality_error(false) o + USING (id) +" + diff --git a/.github/scripts/job/load_imdb.sh b/.github/scripts/job/load_imdb.sh new file mode 100755 index 00000000..3cb44fb2 --- /dev/null +++ b/.github/scripts/job/load_imdb.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +psql -f $JOB_DIR/schema.sql +psql -vdatadir="'$JOB_DIR'" -f $JOB_DIR/copy.sql + diff --git a/.github/scripts/job/set_test_conditions_1.sh b/.github/scripts/job/set_test_conditions_1.sh new file mode 100755 index 00000000..2140893d --- /dev/null +++ b/.github/scripts/job/set_test_conditions_1.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +# ############################################################################## +# +# Test conditions No.1: Quick pass in 'disabled' mode with statistics and +# forced usage of a bunch of parallel workers. +# +# - Disabled mode with a stat gathering and AQO details in explain +# - Force usage of parallel workers aggressively +# - Enable pg_stat_statements statistics +# +# ############################################################################## + +# AQO specific settings +psql -c "ALTER SYSTEM SET aqo.mode = 'disabled'" +psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" +psql -c "ALTER SYSTEM SET aqo.show_details = 'on'" +psql -c "ALTER SYSTEM SET aqo.show_hash = 'on'" + +# Core settings: force parallel workers +psql -c "ALTER SYSTEM SET max_parallel_workers_per_gather = 16" +psql -c "ALTER SYSTEM SET force_parallel_mode = 'on'" +psql -c "ALTER SYSTEM SET from_collapse_limit = 20" +psql -c "ALTER SYSTEM SET join_collapse_limit = 20" +psql -c "ALTER SYSTEM SET parallel_setup_cost = 1.0" +psql -c "ALTER SYSTEM SET parallel_tuple_cost = 0.00001" +psql -c "ALTER SYSTEM SET min_parallel_table_scan_size = 0" +psql -c "ALTER SYSTEM SET min_parallel_index_scan_size = 0" + +# pg_stat_statements +psql -c "ALTER SYSTEM SET pg_stat_statements.track = 'all'" +psql -c "ALTER SYSTEM SET pg_stat_statements.track_planning = 'on'" + +psql -c "SELECT pg_reload_conf();" + +# Enable all previously executed queries which could be disabled +psql -c " + SELECT count(*) FROM aqo_queries, LATERAL aqo_disable_class(queryid) + WHERE queryid <> 0 +" + diff --git a/.github/scripts/job/set_test_conditions_2.sh b/.github/scripts/job/set_test_conditions_2.sh new file mode 100755 index 00000000..609b9624 --- /dev/null +++ b/.github/scripts/job/set_test_conditions_2.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# ############################################################################## +# +# Test conditions No.2: Learn mode with forced parallel workers +# +# - Disabled mode with a stat gathering and AQO details in explain +# - Force usage of parallel workers aggressively +# - Enable pg_stat_statements statistics +# +# ############################################################################## + +# AQO specific settings +psql -c "ALTER SYSTEM SET aqo.mode = 'learn'" +psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'off'" +psql -c "ALTER SYSTEM SET aqo.show_details = 'on'" +psql -c "ALTER SYSTEM SET aqo.show_hash = 'on'" +psql -c "ALTER SYSTEM SET aqo.join_threshold = 0" +psql -c "ALTER SYSTEM SET aqo.wide_search = 'off'" + +# Core settings: force parallel workers +psql -c "ALTER SYSTEM SET max_parallel_workers_per_gather = 16" +psql -c "ALTER SYSTEM SET force_parallel_mode = 'on'" +psql -c "ALTER SYSTEM SET from_collapse_limit = 20" +psql -c "ALTER SYSTEM SET join_collapse_limit = 20" +psql -c "ALTER SYSTEM SET parallel_setup_cost = 1.0" +psql -c "ALTER SYSTEM SET parallel_tuple_cost = 0.00001" +psql -c "ALTER SYSTEM SET min_parallel_table_scan_size = 0" +psql -c "ALTER SYSTEM SET min_parallel_index_scan_size = 0" + +# pg_stat_statements +psql -c "ALTER SYSTEM SET pg_stat_statements.track = 'all'" +psql -c "ALTER SYSTEM SET pg_stat_statements.track_planning = 'on'" + +psql -c "SELECT pg_reload_conf();" + +# Enable all previously executed queries which could be disabled +psql -c " + SELECT count(*) FROM aqo_queries, LATERAL aqo_enable_class(queryid) + WHERE queryid <> 0 +" + diff --git a/.github/scripts/job/set_test_conditions_3.sh b/.github/scripts/job/set_test_conditions_3.sh new file mode 100755 index 00000000..00f4dbf3 --- /dev/null +++ b/.github/scripts/job/set_test_conditions_3.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# ############################################################################## +# +# Test conditions No.3: Freeze ML base and forced parallel workers +# +# - Disabled mode with a stat gathering and AQO details in explain +# - Force usage of parallel workers aggressively +# - Enable pg_stat_statements statistics +# +# ############################################################################## + +# AQO specific settings +psql -c "ALTER SYSTEM SET aqo.mode = 'frozen'" +psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'off'" +psql -c "ALTER SYSTEM SET aqo.show_details = 'on'" +psql -c "ALTER SYSTEM SET aqo.show_hash = 'on'" +psql -c "ALTER SYSTEM SET aqo.join_threshold = 0" +psql -c "ALTER SYSTEM SET aqo.wide_search = 'off'" + +# Core settings: force parallel workers +psql -c "ALTER SYSTEM SET max_parallel_workers_per_gather = 16" +psql -c "ALTER SYSTEM SET force_parallel_mode = 'on'" +psql -c "ALTER SYSTEM SET from_collapse_limit = 20" +psql -c "ALTER SYSTEM SET join_collapse_limit = 20" +psql -c "ALTER SYSTEM SET parallel_setup_cost = 1.0" +psql -c "ALTER SYSTEM SET parallel_tuple_cost = 0.00001" +psql -c "ALTER SYSTEM SET min_parallel_table_scan_size = 0" +psql -c "ALTER SYSTEM SET min_parallel_index_scan_size = 0" + +# pg_stat_statements +psql -c "ALTER SYSTEM SET pg_stat_statements.track = 'all'" +psql -c "ALTER SYSTEM SET pg_stat_statements.track_planning = 'on'" + +psql -c "SELECT pg_reload_conf();" + +# Enable all previously executed queries which could be disabled +psql -c " + SELECT count(*) FROM aqo_queries, LATERAL aqo_enable_class(queryid) + WHERE queryid <> 0 +" + diff --git a/.github/workflows/installchecks.yml b/.github/workflows/installchecks.yml new file mode 100644 index 00000000..aeb976e4 --- /dev/null +++ b/.github/workflows/installchecks.yml @@ -0,0 +1,153 @@ +name: "InstallChecks" + +on: + push: + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + + # Set major PostgreSQL version for all underlying steps + - name: "Extract Postgres major version number" + run: | + PG_MAJOR_VERSION=$(echo "$BRANCH_NAME" | grep --only-matching 'stable[0-9].' | grep --only-matching '[0-9].') + + # Declare PG_MAJOR_VERSION as a environment variable + echo "PG_MAJOR_VERSION=$PG_MAJOR_VERSION" >> $GITHUB_ENV + echo "CORE_BRANCH_NAME=REL_${PG_MAJOR_VERSION}_STABLE" >> $GITHUB_ENV + echo "AQO_PATCH_NAME=aqo_pg$PG_MAJOR_VERSION.patch" >> $GITHUB_ENV + - name: "Set proper names for the master case" + if: env.PG_MAJOR_VERSION == '' + run: | + echo "PG_MAJOR_VERSION=master" >> $GITHUB_ENV + echo "CORE_BRANCH_NAME=master" >> $GITHUB_ENV + echo "AQO_PATCH_NAME=aqo_master.patch" >> $GITHUB_ENV + + - name: "Preparations" + run: | + sudo apt install libipc-run-perl libxml2-utils libxml2-dev xsltproc libxslt1-dev + + echo "Deploying to production server on branch" $BRANCH_NAME + git config --global user.email "ci@postgrespro.ru" + git config --global user.name "CI PgPro admin" + git clone https://github.com/postgres/postgres.git pg + cd pg + git checkout $CORE_BRANCH_NAME + git clone https://github.com/postgrespro/aqo.git contrib/aqo + git -C contrib/aqo checkout $BRANCH_NAME + patch -p1 --no-backup-if-mismatch < contrib/aqo/$AQO_PATCH_NAME + COPT="-Werror" + CONFIGURE_OPTS="--prefix=`pwd`/tmp_install --enable-tap-tests --enable-cassert" + echo "CONFIGURE_OPTS=$CONFIGURE_OPTS" >> $GITHUB_ENV + echo "COPT=$COPT" >> $GITHUB_ENV + + - name: "Paths" + run: | + echo "$GITHUB_WORKSPACE/pg/contrib/aqo/.github/scripts/job" >> $GITHUB_PATH + ls -la pg/contrib/aqo/.github/scripts/job + echo "$GITHUB_WORKSPACE/pg/tmp_install/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/pg/tmp_install/lib" >> $GITHUB_ENV + echo "PGDATABASE=`whoami`" >> $GITHUB_ENV + echo "PGHOST=localhost" >> $GITHUB_ENV + echo "PGDATA=PGDATA" >> $GITHUB_ENV + echo "PGUSER=`whoami`" >> $GITHUB_ENV + echo "PGPORT=5432" >> $GITHUB_ENV + + - name: "Debug" + run: | + echo "paths: $PATH" + echo "PG_MAJOR_VERSION: $PG_MAJOR_VERSION, CORE_BRANCH_NAME: $CORE_BRANCH_NAME, AQO_PATCH_NAME: $AQO_PATCH_NAME, CONFIGURE_OPTS: $CONFIGURE_OPTS" + + - name: "Compilation" + run: | + cd pg + ./configure $CONFIGURE_OPTS CFLAGS="-O2" + make -j4 > /dev/null && make -j4 -C contrib > /dev/null + make install >> make.log && make -C contrib install > /dev/null + + - name: "Launch AQO instance" + run: | + cd pg + + # Launch an instance with AQO extension + aqo_instance_launch.sh + AQO_VERSION=$(psql -t -c "SELECT extversion FROM pg_extension WHERE extname='aqo'") + echo "AQO_VERSION=$AQO_VERSION" >> $GITHUB_ENV + echo "Use AQO v.$AQO_VERSION" + + # Pass installcheck in disabled mode + - name: installcheck_disabled + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'off'" + psql -c "SELECT pg_reload_conf()" + make installcheck-world + + - name: installcheck_disabled_forced_stat + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" + psql -c "SELECT pg_reload_conf()" + make installcheck-world + + - name: installcheck_frozen + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.mode = 'frozen'" + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" + psql -c "SELECT pg_reload_conf()" + make installcheck-world + + - name: installcheck_controlled + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.mode = 'controlled'" + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" + psql -c "SELECT pg_reload_conf()" + make installcheck-world + + - name: installcheck_learn + continue-on-error: true + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.mode = 'learn'" + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" + psql -c "SELECT pg_reload_conf()" + learn_result=$(make -k installcheck-world) + + - name: installcheck_intelligent + continue-on-error: true + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.mode = 'intelligent'" + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" + psql -c "SELECT pg_reload_conf()" + make -k installcheck-world + + - name: installcheck_forced + continue-on-error: true + run: | + cd pg + psql -c "ALTER SYSTEM SET aqo.mode = 'forced'" + psql -c "ALTER SYSTEM SET aqo.force_collect_stat = 'on'" + psql -c "SELECT pg_reload_conf()" + make -k installcheck-world + + # Save Artifacts + - name: Archive artifacts + if: ${{ failure() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.AQO_VERSION }}-${{ env.CORE_BRANCH_NAME }}-${{ env.BRANCH_NAME }}-artifacts + path: | + pg/src/test/regress/regression.diffs + pg/logfile.log + pg/contrib/aqo/tmp_check/log + retention-days: 2 + diff --git a/.github/workflows/job.yml b/.github/workflows/job.yml new file mode 100644 index 00000000..682f4b42 --- /dev/null +++ b/.github/workflows/job.yml @@ -0,0 +1,157 @@ +name: 'Join Order Benchmark' + +env: + BRANCH_NAME: ${{ github.head_ref || github.ref_name }} + +# Trigger the workflow on each push +on: push + +jobs: + AQO_Tests: + + runs-on: self-hosted + + steps: + - name: "Set common paths" + run: | + echo "$HOME/aqo/.github/scripts/job" >> $GITHUB_PATH + echo "JOB_DIR=$HOME/jo-bench" >> $GITHUB_ENV + + # PostgreSQL-related environment variables + echo "$GITHUB_WORKSPACE/pg/tmp_install/bin" >> $GITHUB_PATH + echo "LD_LIBRARY_PATH=$GITHUB_WORKSPACE/pg/tmp_install/lib" >> $GITHUB_ENV + echo "PGDATABASE=`whoami`" >> $GITHUB_ENV + echo "PGHOST=localhost" >> $GITHUB_ENV + echo "PGDATA=PGDATA" >> $GITHUB_ENV + echo "PGUSER=`whoami`" >> $GITHUB_ENV + echo "PGPORT=5432" >> $GITHUB_ENV + + # Set major PostgreSQL version for all underlying steps + - name: "Extract Postgres major version number" + run: | + PG_MAJOR_VERSION=$(echo "$BRANCH_NAME" | grep --only-matching 'stable[0-9].' | grep --only-matching '[0-9].') + + # Declare PG_MAJOR_VERSION as a environment variable + echo "PG_MAJOR_VERSION=$PG_MAJOR_VERSION" >> $GITHUB_ENV + echo "CORE_BRANCH_NAME=REL_${PG_MAJOR_VERSION}_STABLE" >> $GITHUB_ENV + echo "AQO_PATCH_NAME=aqo_pg$PG_MAJOR_VERSION.patch" >> $GITHUB_ENV + - name: "Set proper names for the master case" + if: env.PG_MAJOR_VERSION == '' + run: | + echo "PG_MAJOR_VERSION=master" >> $GITHUB_ENV + echo "CORE_BRANCH_NAME=master" >> $GITHUB_ENV + echo "AQO_PATCH_NAME=aqo_master.patch" >> $GITHUB_ENV + + # Just for debug + - name: "Print environment variables" + run: | + echo "Test data: $PG_MAJOR_VERSION; Core branch: $CORE_BRANCH_NAME, AQO patch: $AQO_PATCH_NAME" + echo "Paths: $PATH, JOB path: $JOB_DIR" + echo "PG Libs: $LD_LIBRARY_PATH" + echo "PG Environment: dbname: $PGDATABASE, host: $PGHOST, pgdata: $PGDATA, pguser: $PGUSER, pgport: $PGPORT" + + # Runner contains clone of postgres and AQO repositories. We must refresh them + - name: "Code pre-cleanup" + run: | + rm -rf pg + git -C ~/pg clean -fdx + git -C ~/pg pull + git -C ~/pg checkout $CORE_BRANCH_NAME + git -C ~/pg pull + + git -C ~/aqo clean -fdx + git -C ~/aqo pull + git -C ~/aqo checkout $BRANCH_NAME + git -C ~/aqo pull + + # Copy the codes into test folder, arrange code versions and do the patching + - name: "Prepare code directory" + run: | + cp -r ~/pg pg + cd pg + cp -r ~/aqo contrib/aqo + patch -p1 --no-backup-if-mismatch < contrib/aqo/$AQO_PATCH_NAME + + - name: "Compilation" + run: | + cd pg + export COPT=-Werror + export CONFIGURE_OPTS="--prefix=`pwd`/tmp_install --enable-tap-tests --enable-cassert" + ./configure $CONFIGURE_OPTS CFLAGS="-O0" + make clean > /dev/null + make -C contrib clean > /dev/null + make -j2 > /dev/null && make -j2 -C contrib > /dev/null + make install >> make.log + make -C contrib install >> make.log + make -C doc install > /dev/null + + - name: "Launch AQO instance" + run: | + cd pg + make -j2 > /dev/null && make -j2 -C contrib > /dev/null + make install > /dev/null && make -C contrib install > /dev/null + + # Launch an instance with AQO extension + aqo_instance_launch.sh + AQO_VERSION=$(psql -t -c "SELECT extversion FROM pg_extension WHERE extname='aqo'") + echo "AQO_VERSION=$AQO_VERSION" >> $GITHUB_ENV + + - name: "Load a dump of the test database" + run: | + cd pg + echo "AQO_VERSION: $AQO_VERSION" + load_imdb.sh + + # Quick pass in parallel mode with statistics + - name: "Test No.1: Gather statistics in disabled mode" + run: | + cd pg + set_test_conditions_1.sh + job_pass.sh + dump_knowledge.sh + + - name: "Archive JOB test results" + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.AQO_VERSION }}-${{ env.CORE_BRANCH_NAME }}-${{ env.BRANCH_NAME }}-result_base_stat + path: | + pg/explains.txt + pg/report.txt + pg/knowledge_base.dump + pg/logfile.log + retention-days: 1 + + # Test No.2: Learn on all incoming queries + - name: "Test No.2: Learning stage" + run: | + cd pg + set_test_conditions_2.sh + job_pass.sh 10 + check_result.sh + + # One pass on frozen AQO data, dump knowledge base, check total error + - name: "Test No.3: Frozen execution" + run: | + cd pg + set_test_conditions_3.sh + job_pass.sh + dump_knowledge.sh + + - name: "Archive JOB test results - frozen" + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: ${{ env.AQO_VERSION }}-${{ env.CORE_BRANCH_NAME }}-${{ env.BRANCH_NAME }}-result_frozen + path: | + pg/explains.txt + pg/report.txt + pg/knowledge_base.dump + pg/logfile.log + retention-days: 7 + + - name: "Cleanup" + run: | + cd pg + pg_ctl -D PGDATA stop + diff --git a/Makefile b/Makefile index d3aec440..ce9d00ba 100755 --- a/Makefile +++ b/Makefile @@ -16,6 +16,12 @@ TAP_TESTS = 1 REGRESS = aqo_dummy_test REGRESS_OPTS = --schedule=$(srcdir)/regress_schedule +# Set default values of some gucs to be stable on custom settings during +# a kind of installcheck +PGOPTIONS = --aqo.force_collect_stat=off --max_parallel_maintenance_workers=1 \ + --aqo.join_threshold=0 --max_parallel_workers_per_gather=1 +export PGOPTIONS + fdw_srcdir = $(top_srcdir)/contrib/postgres_fdw stat_srcdir = $(top_srcdir)/contrib/pg_stat_statements PG_CPPFLAGS += -I$(libpq_srcdir) -I$(fdw_srcdir) -I$(stat_srcdir) diff --git a/aqo.conf b/aqo.conf index 03de79ee..069c7dd7 100644 --- a/aqo.conf +++ b/aqo.conf @@ -1,5 +1,3 @@ autovacuum = off shared_preload_libraries = 'postgres_fdw, aqo' -max_parallel_maintenance_workers = 1 # switch off parallel workers because of unsteadiness -aqo.wide_search = 'on' compute_query_id = 'regress' diff --git a/expected/aqo_controlled.out b/expected/aqo_controlled.out index cf88bf42..43d27d74 100644 --- a/expected/aqo_controlled.out +++ b/expected/aqo_controlled.out @@ -1,3 +1,10 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -25,8 +32,6 @@ AS ( ) INSERT INTO aqo_test2 (SELECT * FROM t); CREATE INDEX aqo_test2_idx_a ON aqo_test2 (a); ANALYZE aqo_test2; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'controlled'; EXPLAIN (COSTS FALSE) SELECT * FROM aqo_test0 @@ -199,11 +204,12 @@ WHERE t1.a = t2.b AND t2.a = t3.b; SELECT count(*) FROM (SELECT queryid AS id FROM aqo_queries) AS q1, - LATERAL aqo_queries_update(q1.id, NULL, NULL, true, NULL) + LATERAL aqo_queries_update(q1.id, NULL, NULL, true, NULL) AS ret +WHERE NOT ret ; -- set use = true count ------- - 12 + 1 (1 row) EXPLAIN (COSTS FALSE) @@ -311,11 +317,4 @@ DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; DROP INDEX aqo_test2_idx_a; DROP TABLE aqo_test2; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/aqo_disabled.out b/expected/aqo_disabled.out index 606d258e..cf12e2fb 100644 --- a/expected/aqo_disabled.out +++ b/expected/aqo_disabled.out @@ -1,3 +1,12 @@ +-- Create the extension. Drop all lumps which could survive from +-- previous pass (repeated installcheck as an example). +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -16,8 +25,6 @@ AS ( ) INSERT INTO aqo_test1 (SELECT * FROM t); CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'controlled'; CREATE TABLE tmp1 AS SELECT * FROM aqo_test0 WHERE a < 3 AND b < 3 AND c < 3 AND d < 3; @@ -151,11 +158,12 @@ SELECT count(*) FROM aqo_queries WHERE queryid <> fs; -- Should be zero SET aqo.mode = 'controlled'; SELECT count(*) FROM (SELECT queryid AS id FROM aqo_queries) AS q1, - LATERAL aqo_queries_update(q1.id, NULL, true, true, false) + LATERAL aqo_queries_update(q1.id, NULL, true, true, false) AS ret +WHERE NOT ret ; -- Enable all disabled query classes count ------- - 5 + 1 (1 row) EXPLAIN SELECT * FROM aqo_test0 @@ -223,15 +231,8 @@ SELECT count(*) FROM aqo_queries WHERE queryid <> fs; -- Should be zero 0 (1 row) --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - -DROP EXTENSION aqo; DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; +DROP EXTENSION aqo; diff --git a/expected/aqo_fdw.out b/expected/aqo_fdw.out index e568e993..69c1b132 100644 --- a/expected/aqo_fdw.out +++ b/expected/aqo_fdw.out @@ -3,12 +3,17 @@ -- JOIN push-down (check push of baserestrictinfo and joininfo) -- Aggregate push-down -- Push-down of groupings with HAVING clause. -CREATE EXTENSION aqo; -CREATE EXTENSION postgres_fdw; +CREATE EXTENSION IF NOT EXISTS aqo; +CREATE EXTENSION IF NOT EXISTS postgres_fdw; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'learn'; SET aqo.show_details = 'true'; -- show AQO info for each node and entire query. SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it. -SET aqo.join_threshold = 0; DO $d$ BEGIN EXECUTE $$CREATE SERVER loopback FOREIGN DATA WRAPPER postgres_fdw @@ -100,15 +105,23 @@ SELECT str FROM expln(' EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) SELECT * FROM frgn AS a, frgn AS b WHERE a.x=b.x; ') AS str WHERE str NOT LIKE '%Sort Method%'; - str -------------------------------------------- - Foreign Scan (actual rows=1 loops=1) + str +------------------------------------------------------------ + Merge Join (actual rows=1 loops=1) AQO not used - Relations: (frgn a) INNER JOIN (frgn b) + Merge Cond: (a.x = b.x) + -> Sort (actual rows=1 loops=1) + Sort Key: a.x + -> Foreign Scan on frgn a (actual rows=1 loops=1) + AQO not used + -> Sort (actual rows=1 loops=1) + Sort Key: b.x + -> Foreign Scan on frgn b (actual rows=1 loops=1) + AQO not used Using aqo: true AQO mode: LEARN JOINS: 0 -(6 rows) +(14 rows) -- Should learn on postgres_fdw nodes SELECT str FROM expln(' diff --git a/expected/aqo_forced.out b/expected/aqo_forced.out index 091ead32..6d5d14a9 100644 --- a/expected/aqo_forced.out +++ b/expected/aqo_forced.out @@ -1,3 +1,11 @@ +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -16,8 +24,6 @@ AS ( ) INSERT INTO aqo_test1 (SELECT * FROM t); CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'controlled'; EXPLAIN (COSTS FALSE) SELECT * FROM aqo_test0 @@ -82,11 +88,4 @@ DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/aqo_intelligent.out b/expected/aqo_intelligent.out index 7ec943f5..1d407ea7 100644 --- a/expected/aqo_intelligent.out +++ b/expected/aqo_intelligent.out @@ -1,3 +1,10 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -16,8 +23,6 @@ AS ( ) INSERT INTO aqo_test1 (SELECT * FROM t); CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'intelligent'; EXPLAIN SELECT * FROM aqo_test0 WHERE a < 3 AND b < 3 AND c < 3 AND d < 3; @@ -519,11 +524,4 @@ DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/aqo_learn.out b/expected/aqo_learn.out index db117a0c..9a5ca8dd 100644 --- a/expected/aqo_learn.out +++ b/expected/aqo_learn.out @@ -1,3 +1,10 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + -- The function just copied from stats_ext.sql create function check_estimated_rows(text) returns table (estimated int, actual int) language plpgsql as @@ -36,8 +43,6 @@ AS ( ) INSERT INTO aqo_test1 (SELECT * FROM t); CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'intelligent'; EXPLAIN SELECT * FROM aqo_test0 WHERE a < 3 AND b < 3 AND c < 3 AND d < 3; @@ -236,10 +241,10 @@ SELECT count(*) FROM tmp1; (1 row) -- Remove data on some unneeded instances of tmp1 table. -SELECT * FROM aqo_cleanup(); - nfs | nfss ------+------ - 9 | 18 +SELECT true AS success FROM aqo_cleanup(); + success +--------- + t (1 row) -- Result of the query below should be empty @@ -563,7 +568,7 @@ SELECT * FROM check_estimated_rows( 'SELECT * FROM aqo_test1 AS t1, aqo_test1 AS t2 WHERE t1.a = t2.b'); estimated | actual -----------+-------- - 19 | 19 + 20 | 19 (1 row) SELECT count(*) FROM @@ -716,11 +721,4 @@ DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/clean_aqo_data.out b/expected/clean_aqo_data.out index e66f274b..49b64832 100644 --- a/expected/clean_aqo_data.out +++ b/expected/clean_aqo_data.out @@ -1,5 +1,10 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'learn'; DROP TABLE IF EXISTS a; NOTICE: table "a" does not exist, skipping @@ -11,9 +16,9 @@ SELECT * FROM a; (0 rows) SELECT 'a'::regclass::oid AS a_oid \gset -SELECT true FROM aqo_cleanup(); - bool ------- +SELECT true AS success FROM aqo_cleanup(); + success +--------- t (1 row) @@ -54,9 +59,9 @@ SELECT count(*) FROM aqo_query_stat WHERE (1 row) DROP TABLE a; -SELECT true FROM aqo_cleanup(); - bool ------- +SELECT true AS success FROM aqo_cleanup(); + success +--------- t (1 row) @@ -119,7 +124,7 @@ SELECT 'b'::regclass::oid AS b_oid \gset SELECT count(*) FROM aqo_data WHERE :a_oid=ANY(oids); count ------- - 2 + 3 (1 row) SELECT count(*) FROM aqo_queries WHERE @@ -175,9 +180,9 @@ SELECT count(*) FROM aqo_query_stat WHERE (1 row) DROP TABLE a; -SELECT true FROM aqo_cleanup(); - bool ------- +SELECT true AS success FROM aqo_cleanup(); + success +--------- t (1 row) @@ -253,9 +258,9 @@ SELECT count(*) FROM aqo_query_stat WHERE (1 row) DROP TABLE b; -SELECT true FROM aqo_cleanup(); - bool ------- +SELECT true AS success FROM aqo_cleanup(); + success +--------- t (1 row) diff --git a/expected/feature_subspace.out b/expected/feature_subspace.out index a49be254..a53b57e7 100644 --- a/expected/feature_subspace.out +++ b/expected/feature_subspace.out @@ -1,7 +1,12 @@ -- This test related to some issues on feature subspace calculation -CREATE EXTENSION aqo; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'learn'; -SET aqo.join_threshold = 0; SET aqo.show_details = 'on'; CREATE TABLE a AS (SELECT gs AS x FROM generate_series(1,10) AS gs); CREATE TABLE b AS (SELECT gs AS x FROM generate_series(1,100) AS gs); @@ -46,20 +51,23 @@ SELECT str AS result FROM expln(' SELECT * FROM b LEFT JOIN a USING (x);') AS str WHERE str NOT LIKE '%Memory%'; - result ----------------------------------------------------- - Hash Left Join (actual rows=100 loops=1) - AQO: rows=10, error=-900% - Hash Cond: (b.x = a.x) - -> Seq Scan on b (actual rows=100 loops=1) - AQO: rows=100, error=0% - -> Hash (actual rows=10 loops=1) + result +----------------------------------------------------- + Merge Left Join (actual rows=100 loops=1) + AQO not used + Merge Cond: (b.x = a.x) + -> Sort (actual rows=100 loops=1) + Sort Key: b.x + -> Seq Scan on b (actual rows=100 loops=1) + AQO not used + -> Sort (actual rows=10 loops=1) + Sort Key: a.x -> Seq Scan on a (actual rows=10 loops=1) - AQO: rows=10, error=0% + AQO not used Using aqo: true AQO mode: LEARN JOINS: 0 -(11 rows) +(14 rows) -- Look into the reason: two JOINs from different classes have the same FSS. SELECT to_char(d1.targets[1], 'FM999.00') AS target FROM aqo_data d1 @@ -72,10 +80,4 @@ WHERE 'a'::regclass = ANY (d1.oids) AND 'b'::regclass = ANY (d1.oids) order by t (2 rows) DROP TABLE a,b CASCADE; -SELECT true FROM aqo_reset(); - bool ------- - t -(1 row) - DROP EXTENSION aqo; diff --git a/expected/forced_stat_collection.out b/expected/forced_stat_collection.out index f635fbcc..c5a6ac0e 100644 --- a/expected/forced_stat_collection.out +++ b/expected/forced_stat_collection.out @@ -1,5 +1,11 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + \set citizens 1000 -SET aqo.join_threshold = 0; SET aqo.mode = 'disabled'; SET aqo.force_collect_stat = 'off'; CREATE TABLE person ( @@ -19,7 +25,6 @@ INSERT INTO person (id,age,gender,passport) END FROM (SELECT *, 14+(id % 60) AS age FROM generate_series(1, :citizens) id) AS q1 ); -CREATE EXTENSION aqo; SET aqo.force_collect_stat = 'on'; SELECT count(*) FROM person WHERE age<18; count @@ -64,10 +69,4 @@ SELECT query_text FROM aqo_query_texts ORDER BY (md5(query_text)); (3 rows) DROP TABLE person; -SELECT 1 FROM aqo_reset(); -- Full remove of ML data before the end - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/gucs.out b/expected/gucs.out index d7ef6eeb..f33aa6b2 100644 --- a/expected/gucs.out +++ b/expected/gucs.out @@ -1,4 +1,11 @@ -CREATE EXTENSION aqo; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + -- Utility tool. Allow to filter system-dependent strings from an explain output. CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$ BEGIN @@ -7,16 +14,15 @@ BEGIN RETURN; END; $$ LANGUAGE PLPGSQL; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = true; SET compute_query_id = 'auto'; CREATE TABLE t(x int); INSERT INTO t (x) (SELECT * FROM generate_series(1, 100) AS gs); ANALYZE t; -SELECT true FROM aqo_reset(); -- Remember! DROP EXTENSION doesn't remove any AQO data gathered. - bool ------- +SELECT true AS success FROM aqo_reset(); + success +--------- t (1 row) @@ -127,9 +133,9 @@ SELECT count(*) FROM aqo_query_stat; 1 (1 row) -SELECT true FROM aqo_reset(); -- Remove one record from all tables - bool ------- +SELECT true AS success FROM aqo_reset(); + success +--------- t (1 row) diff --git a/expected/look_a_like.out b/expected/look_a_like.out index 899ef271..fb76fdd6 100644 --- a/expected/look_a_like.out +++ b/expected/look_a_like.out @@ -1,11 +1,12 @@ -CREATE EXTENSION aqo; -SELECT true FROM aqo_reset(); - bool ------- +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- t (1 row) -SET aqo.join_threshold = 0; +SET aqo.wide_search = 'on'; SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; set aqo.show_hash = 'off'; @@ -550,14 +551,9 @@ WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT L JOINS: 1 (24 rows) -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - +RESET aqo.wide_search; +DROP EXTENSION aqo CASCADE; DROP TABLE a; DROP TABLE b; DROP TABLE c; DROP FUNCTION expln; -DROP EXTENSION aqo CASCADE; diff --git a/expected/parallel_workers.out b/expected/parallel_workers.out index fca67006..3e408f49 100644 --- a/expected/parallel_workers.out +++ b/expected/parallel_workers.out @@ -1,6 +1,12 @@ -- Specifically test AQO machinery for queries uses partial paths and executed -- with parallel workers. -CREATE EXTENSION aqo; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + -- Utility tool. Allow to filter system-dependent strings from explain output. CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$ BEGIN @@ -9,7 +15,6 @@ BEGIN RETURN; END; $$ LANGUAGE PLPGSQL; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = true; -- Be generous with a number parallel workers to test the machinery diff --git a/expected/plancache.out b/expected/plancache.out index 6874468a..88698463 100644 --- a/expected/plancache.out +++ b/expected/plancache.out @@ -1,6 +1,11 @@ -- Tests on interaction of AQO with cached plans. -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'intelligent'; SET aqo.show_details = 'on'; SET aqo.show_hash = 'off'; @@ -44,10 +49,4 @@ SELECT * FROM f1(); DROP FUNCTION f1; DROP TABLE test CASCADE; -SELECT true FROM aqo_reset(); - bool ------- - t -(1 row) - DROP EXTENSION aqo; diff --git a/expected/relocatable.out b/expected/relocatable.out index 949896f6..3d7f386f 100644 --- a/expected/relocatable.out +++ b/expected/relocatable.out @@ -1,5 +1,10 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'learn'; -- use this mode for unconditional learning CREATE TABLE test AS (SELECT id, 'payload' || id FROM generate_series(1,100) id); ANALYZE test; diff --git a/expected/schema.out b/expected/schema.out index 0b5a5c07..e712f407 100644 --- a/expected/schema.out +++ b/expected/schema.out @@ -1,5 +1,3 @@ -DROP EXTENSION IF EXISTS aqo CASCADE; -NOTICE: extension "aqo" does not exist, skipping DROP SCHEMA IF EXISTS test CASCADE; NOTICE: schema "test" does not exist, skipping -- Check Zero-schema path behaviour @@ -12,7 +10,12 @@ ERROR: no schema has been selected to create in CREATE SCHEMA IF NOT EXISTS test1; SET search_path TO test1, public; CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'intelligent'; CREATE TABLE test (id SERIAL, data TEXT); INSERT INTO test (data) VALUES ('string'); diff --git a/expected/statement_timeout.out b/expected/statement_timeout.out index a12fe9dd..39796549 100644 --- a/expected/statement_timeout.out +++ b/expected/statement_timeout.out @@ -17,37 +17,43 @@ BEGIN END IF; END LOOP; END; $$; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + CREATE TABLE t AS SELECT * FROM generate_series(1,50) AS x; ANALYZE t; DELETE FROM t WHERE x > 5; -- Force optimizer to make overestimated prediction. -CREATE EXTENSION IF NOT EXISTS aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = 'off'; SET aqo.learn_statement_timeout = 'on'; -SET statement_timeout = 100; -- [0.1s] +SET statement_timeout = 80; -- [0.1s] SELECT *, pg_sleep(0.1) FROM t; NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. ERROR: canceling statement due to statement timeout -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); -- haven't any partial data +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -- haven't any partial data check_estimated_rows ---------------------- 50 (1 row) -- Don't learn because running node has smaller cardinality than an optimizer prediction -SET statement_timeout = 400; +SET statement_timeout = 350; SELECT *, pg_sleep(0.1) FROM t; NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. ERROR: canceling statement due to statement timeout -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); check_estimated_rows ---------------------- 50 (1 row) -- We have a real learning data. -SET statement_timeout = 8000; +SET statement_timeout = 800; SELECT *, pg_sleep(0.1) FROM t; x | pg_sleep ---+---------- @@ -58,7 +64,7 @@ SELECT *, pg_sleep(0.1) FROM t; 5 | (5 rows) -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); check_estimated_rows ---------------------- 5 @@ -68,33 +74,33 @@ SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); DELETE FROM t WHERE x > 2; ANALYZE t; INSERT INTO t (x) (SELECT * FROM generate_series(3,5) AS x); -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 +SELECT true AS success FROM aqo_reset(); + success +--------- + t (1 row) -SET statement_timeout = 100; +SET statement_timeout = 80; SELECT *, pg_sleep(0.1) FROM t; -- Not learned NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. ERROR: canceling statement due to statement timeout -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); check_estimated_rows ---------------------- 2 (1 row) -SET statement_timeout = 500; +SET statement_timeout = 350; SELECT *, pg_sleep(0.1) FROM t; -- Learn! NOTICE: [AQO] Time limit for execution of the statement was expired. AQO tried to learn on partial data. ERROR: canceling statement due to statement timeout -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); check_estimated_rows ---------------------- - 4 + 3 (1 row) -SET statement_timeout = 800; +SET statement_timeout = 550; SELECT *, pg_sleep(0.1) FROM t; -- Get reliable data x | pg_sleep ---+---------- @@ -105,17 +111,17 @@ SELECT *, pg_sleep(0.1) FROM t; -- Get reliable data 5 | (5 rows) -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); check_estimated_rows ---------------------- 5 (1 row) -- Interrupted query should immediately appear in aqo_data -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 +SELECT true AS success FROM aqo_reset(); + success +--------- + t (1 row) SET statement_timeout = 500; @@ -134,10 +140,10 @@ SELECT count(*) FROM aqo_data; -- Must be one 1 (1 row) -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 +SELECT true AS success FROM aqo_reset(); + success +--------- + t (1 row) DROP TABLE t; diff --git a/expected/temp_tables.out b/expected/temp_tables.out index cb1da23f..9fa20e7c 100644 --- a/expected/temp_tables.out +++ b/expected/temp_tables.out @@ -1,5 +1,12 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + +SET aqo.wide_search = 'on'; SET aqo.mode = 'learn'; CREATE TEMP TABLE tt(); CREATE TABLE pt(); @@ -48,10 +55,10 @@ SELECT count(*) FROM aqo_data; -- Don't bother about false negatives because of (1 row) DROP TABLE tt; -SELECT * FROM aqo_cleanup(); - nfs | nfss ------+------ - 0 | 0 +SELECT true AS success FROM aqo_cleanup(); + success +--------- + t (1 row) SELECT count(*) FROM aqo_data; -- Should return the same as previous call above @@ -61,10 +68,10 @@ SELECT count(*) FROM aqo_data; -- Should return the same as previous call above (1 row) DROP TABLE pt; -SELECT * FROM aqo_cleanup(); - nfs | nfss ------+------ - 3 | 10 +SELECT true AS success FROM aqo_cleanup(); + success +--------- + t (1 row) SELECT count(*) FROM aqo_data; -- Should be 0 @@ -133,10 +140,10 @@ SELECT * FROM check_estimated_rows(' SET aqo.mode = 'forced'; -- Now we use all fss records for each query DROP TABLE pt; -SELECT * FROM aqo_cleanup(); - nfs | nfss ------+------ - 2 | 5 +SELECT true AS success FROM aqo_cleanup(); + success +--------- + t (1 row) CREATE TABLE pt AS SELECT x AS x, (x % 10) AS y FROM generate_series(1,100) AS x; @@ -184,12 +191,8 @@ SELECT * FROM check_estimated_rows(' 100 | 0 (1 row) +-- Clear common parts of AQO state +RESET aqo.wide_search; +DROP EXTENSION aqo CASCADE; DROP TABLE pt CASCADE; -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - -DROP EXTENSION aqo; DROP FUNCTION check_estimated_rows; diff --git a/expected/top_queries.out b/expected/top_queries.out index ba72d7c8..62186efc 100644 --- a/expected/top_queries.out +++ b/expected/top_queries.out @@ -1,5 +1,11 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + SET aqo.mode = 'disabled'; SET aqo.force_collect_stat = 'on'; -- @@ -95,10 +101,4 @@ ORDER BY (md5(query_text)); SELECT count(*) FROM (SELECT x, y FROM t1 GROUP BY GROUPING SETS ((x,y), (x), (y), ())) AS q1; | 1 (3 rows) -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/unsupported.out b/expected/unsupported.out index c42a3be5..a1a6f4ae 100644 --- a/expected/unsupported.out +++ b/expected/unsupported.out @@ -1,4 +1,10 @@ -CREATE EXTENSION aqo; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + -- Utility tool. Allow to filter system-dependent strings from an explain output. CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$ BEGIN @@ -7,7 +13,6 @@ BEGIN RETURN; END; $$ LANGUAGE PLPGSQL; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; DROP TABLE IF EXISTS t; @@ -52,7 +57,7 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) AQO not used Group Key: x -> Seq Scan on t (actual rows=801 loops=1) - AQO: rows=801, error=0% + AQO not used Filter: (x > 3) Rows Removed by Filter: 199 Using aqo: true @@ -406,7 +411,7 @@ EXPLAIN (ANALYZE, COSTS OFF, SUMMARY OFF, TIMING OFF) -> Aggregate (actual rows=1 loops=1000) AQO not used -> Seq Scan on t t0 (actual rows=50 loops=1000) - AQO: rows=50, error=0% + AQO not used Filter: (x = t.x) Rows Removed by Filter: 950 SubPlan 2 @@ -616,10 +621,10 @@ SELECT count(*) FROM aqo_data; -- Just to detect some changes in the logic. May 44 (1 row) -SELECT * FROM aqo_cleanup(); - nfs | nfss ------+------ - 13 | 44 +SELECT true AS success FROM aqo_cleanup(); + success +--------- + t (1 row) SELECT count(*) FROM aqo_data; -- No one row should be returned @@ -637,10 +642,4 @@ ORDER BY (md5(query_text),error) DESC; -------+------------ (0 rows) -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - DROP EXTENSION aqo; diff --git a/expected/update_functions.out b/expected/update_functions.out index cf9cee8e..74428a35 100644 --- a/expected/update_functions.out +++ b/expected/update_functions.out @@ -1,3 +1,11 @@ +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + CREATE TABLE aqo_test1(a int, b int); WITH RECURSIVE t(a, b) AS ( @@ -16,8 +24,6 @@ AS ( ) INSERT INTO aqo_test2 (SELECT * FROM t); CREATE INDEX aqo_test2_idx_a ON aqo_test2 (a); ANALYZE aqo_test2; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode='intelligent'; SELECT count(*) FROM aqo_test1 a, aqo_test2 b WHERE a.a=b.a; count @@ -134,10 +140,10 @@ CREATE TABLE aqo_query_texts_dump AS SELECT * FROM aqo_query_texts; CREATE TABLE aqo_queries_dump AS SELECT * FROM aqo_queries; CREATE TABLE aqo_query_stat_dump AS SELECT * FROM aqo_query_stat; CREATE TABLE aqo_data_dump AS SELECT * FROM aqo_data; -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 +SELECT true AS success FROM aqo_reset(); + success +--------- + t (1 row) -- @@ -411,12 +417,6 @@ SELECT aqo_data_update(1, 1, 1, '{{1}, {2}}', '{1}', '{1}', '{1, 2, 3}'); (1 row) SET aqo.mode='disabled'; -SELECT 1 FROM aqo_reset(); - ?column? ----------- - 1 -(1 row) - -DROP EXTENSION aqo; +DROP EXTENSION aqo CASCADE; DROP TABLE aqo_test1, aqo_test2; DROP TABLE aqo_query_texts_dump, aqo_queries_dump, aqo_query_stat_dump, aqo_data_dump; diff --git a/sql/aqo_controlled.sql b/sql/aqo_controlled.sql index 0ba88e56..8c8e5fb8 100644 --- a/sql/aqo_controlled.sql +++ b/sql/aqo_controlled.sql @@ -1,3 +1,6 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -28,9 +31,6 @@ AS ( CREATE INDEX aqo_test2_idx_a ON aqo_test2 (a); ANALYZE aqo_test2; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; - SET aqo.mode = 'controlled'; EXPLAIN (COSTS FALSE) @@ -111,7 +111,8 @@ WHERE t1.a = t2.b AND t2.a = t3.b; SELECT count(*) FROM (SELECT queryid AS id FROM aqo_queries) AS q1, - LATERAL aqo_queries_update(q1.id, NULL, NULL, true, NULL) + LATERAL aqo_queries_update(q1.id, NULL, NULL, true, NULL) AS ret +WHERE NOT ret ; -- set use = true EXPLAIN (COSTS FALSE) @@ -147,14 +148,9 @@ WHERE t1.a = t2.b AND t2.a = t3.b; DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; - DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; - DROP INDEX aqo_test2_idx_a; DROP TABLE aqo_test2; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - DROP EXTENSION aqo; diff --git a/sql/aqo_disabled.sql b/sql/aqo_disabled.sql index fd709cf3..8397f847 100644 --- a/sql/aqo_disabled.sql +++ b/sql/aqo_disabled.sql @@ -1,3 +1,8 @@ +-- Create the extension. Drop all lumps which could survive from +-- previous pass (repeated installcheck as an example). +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -17,8 +22,6 @@ AS ( ) INSERT INTO aqo_test1 (SELECT * FROM t); CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'controlled'; @@ -77,7 +80,8 @@ SET aqo.mode = 'controlled'; SELECT count(*) FROM (SELECT queryid AS id FROM aqo_queries) AS q1, - LATERAL aqo_queries_update(q1.id, NULL, true, true, false) + LATERAL aqo_queries_update(q1.id, NULL, true, true, false) AS ret +WHERE NOT ret ; -- Enable all disabled query classes EXPLAIN SELECT * FROM aqo_test0 @@ -98,13 +102,9 @@ FROM aqo_test1 AS t1, aqo_test0 AS t2, aqo_test0 AS t3 WHERE t1.a < 1 AND t3.b < 1 AND t2.c < 1 AND t3.d < 0 AND t1.a = t2.a AND t1.b = t3.b; SELECT count(*) FROM aqo_queries WHERE queryid <> fs; -- Should be zero --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - -DROP EXTENSION aqo; - DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; - DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; + +DROP EXTENSION aqo; diff --git a/sql/aqo_fdw.sql b/sql/aqo_fdw.sql index bd211326..5425dcf4 100644 --- a/sql/aqo_fdw.sql +++ b/sql/aqo_fdw.sql @@ -4,13 +4,13 @@ -- Aggregate push-down -- Push-down of groupings with HAVING clause. -CREATE EXTENSION aqo; -CREATE EXTENSION postgres_fdw; +CREATE EXTENSION IF NOT EXISTS aqo; +CREATE EXTENSION IF NOT EXISTS postgres_fdw; +SELECT true AS success FROM aqo_reset(); SET aqo.mode = 'learn'; SET aqo.show_details = 'true'; -- show AQO info for each node and entire query. SET aqo.show_hash = 'false'; -- a hash value is system-depended. Ignore it. -SET aqo.join_threshold = 0; DO $d$ BEGIN diff --git a/sql/aqo_forced.sql b/sql/aqo_forced.sql index 92a26564..34f97359 100644 --- a/sql/aqo_forced.sql +++ b/sql/aqo_forced.sql @@ -1,3 +1,7 @@ +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -18,9 +22,6 @@ AS ( CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; - SET aqo.mode = 'controlled'; EXPLAIN (COSTS FALSE) @@ -53,11 +54,7 @@ WHERE a < 5 AND b < 5 AND c < 5 AND d < 5; DROP INDEX aqo_test0_idx_a; DROP TABLE aqo_test0; - DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - DROP EXTENSION aqo; diff --git a/sql/aqo_intelligent.sql b/sql/aqo_intelligent.sql index 545325c1..45ecaecc 100644 --- a/sql/aqo_intelligent.sql +++ b/sql/aqo_intelligent.sql @@ -1,3 +1,6 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + CREATE TABLE aqo_test0(a int, b int, c int, d int); WITH RECURSIVE t(a, b, c, d) AS ( @@ -18,9 +21,6 @@ AS ( CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; - SET aqo.mode = 'intelligent'; EXPLAIN SELECT * FROM aqo_test0 @@ -215,7 +215,4 @@ DROP TABLE aqo_test0; DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - DROP EXTENSION aqo; diff --git a/sql/aqo_learn.sql b/sql/aqo_learn.sql index 8b57972e..8acd2db7 100644 --- a/sql/aqo_learn.sql +++ b/sql/aqo_learn.sql @@ -1,3 +1,6 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + -- The function just copied from stats_ext.sql create function check_estimated_rows(text) returns table (estimated int, actual int) language plpgsql as @@ -39,9 +42,6 @@ AS ( CREATE INDEX aqo_test1_idx_a ON aqo_test1 (a); ANALYZE aqo_test1; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; - SET aqo.mode = 'intelligent'; EXPLAIN SELECT * FROM aqo_test0 @@ -124,7 +124,7 @@ WHERE t1.a = t2.b AND t2.a = t3.b AND t3.a = t4.b; SELECT count(*) FROM tmp1; -- Remove data on some unneeded instances of tmp1 table. -SELECT * FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); -- Result of the query below should be empty SELECT * FROM aqo_query_texts aqt1, aqo_query_texts aqt2 @@ -314,7 +314,4 @@ DROP TABLE aqo_test0; DROP INDEX aqo_test1_idx_a; DROP TABLE aqo_test1; --- XXX: extension dropping doesn't clear file storage. Do it manually. -SELECT 1 FROM aqo_reset(); - DROP EXTENSION aqo; diff --git a/sql/clean_aqo_data.sql b/sql/clean_aqo_data.sql index d2abeb93..3c504bdb 100644 --- a/sql/clean_aqo_data.sql +++ b/sql/clean_aqo_data.sql @@ -1,5 +1,6 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + SET aqo.mode = 'learn'; DROP TABLE IF EXISTS a; @@ -7,7 +8,7 @@ DROP TABLE IF EXISTS b; CREATE TABLE a(); SELECT * FROM a; SELECT 'a'::regclass::oid AS a_oid \gset -SELECT true FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); /* * lines with a_oid in aqo_data, @@ -27,7 +28,7 @@ SELECT count(*) FROM aqo_query_stat WHERE aqo_queries.fs = ANY(SELECT aqo_data.fs FROM aqo_data WHERE :a_oid=ANY(oids))); DROP TABLE a; -SELECT true FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); /* * lines with a_oid in aqo_data, @@ -79,7 +80,7 @@ SELECT count(*) FROM aqo_query_stat WHERE aqo_queries.fs = ANY(SELECT aqo_data.fs FROM aqo_data WHERE :b_oid=ANY(oids))); DROP TABLE a; -SELECT true FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); /* * lines corresponding to a_oid and both a_oid's fs deleted in aqo_data, @@ -115,7 +116,7 @@ SELECT count(*) FROM aqo_query_stat WHERE aqo_queries.fs = aqo_queries.queryid); DROP TABLE b; -SELECT true FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); -- lines corresponding to b_oid in theese tables deleted SELECT count(*) FROM aqo_data WHERE :b_oid=ANY(oids); @@ -131,4 +132,4 @@ SELECT count(*) FROM aqo_query_stat WHERE aqo_queries.fs = ANY(SELECT aqo_data.fs FROM aqo_data WHERE :b_oid=ANY(oids)) AND aqo_queries.fs = aqo_queries.queryid); -DROP EXTENSION aqo; \ No newline at end of file +DROP EXTENSION aqo; diff --git a/sql/feature_subspace.sql b/sql/feature_subspace.sql index 0176a700..c9463d55 100644 --- a/sql/feature_subspace.sql +++ b/sql/feature_subspace.sql @@ -1,9 +1,9 @@ -- This test related to some issues on feature subspace calculation -CREATE EXTENSION aqo; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); SET aqo.mode = 'learn'; -SET aqo.join_threshold = 0; SET aqo.show_details = 'on'; CREATE TABLE a AS (SELECT gs AS x FROM generate_series(1,10) AS gs); @@ -41,5 +41,5 @@ JOIN aqo_data d2 ON (d1.fs <> d2.fs AND d1.fss = d2.fss) WHERE 'a'::regclass = ANY (d1.oids) AND 'b'::regclass = ANY (d1.oids) order by target; DROP TABLE a,b CASCADE; -SELECT true FROM aqo_reset(); + DROP EXTENSION aqo; diff --git a/sql/forced_stat_collection.sql b/sql/forced_stat_collection.sql index d9fac51a..cf3990fc 100644 --- a/sql/forced_stat_collection.sql +++ b/sql/forced_stat_collection.sql @@ -1,6 +1,8 @@ +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + \set citizens 1000 -SET aqo.join_threshold = 0; SET aqo.mode = 'disabled'; SET aqo.force_collect_stat = 'off'; @@ -23,7 +25,6 @@ INSERT INTO person (id,age,gender,passport) FROM (SELECT *, 14+(id % 60) AS age FROM generate_series(1, :citizens) id) AS q1 ); -CREATE EXTENSION aqo; SET aqo.force_collect_stat = 'on'; SELECT count(*) FROM person WHERE age<18; @@ -46,5 +47,5 @@ ORDER BY (cardinality_error_without_aqo); SELECT query_text FROM aqo_query_texts ORDER BY (md5(query_text)); DROP TABLE person; -SELECT 1 FROM aqo_reset(); -- Full remove of ML data before the end + DROP EXTENSION aqo; diff --git a/sql/gucs.sql b/sql/gucs.sql index 9b1bf9b8..0e948cf1 100644 --- a/sql/gucs.sql +++ b/sql/gucs.sql @@ -1,4 +1,6 @@ -CREATE EXTENSION aqo; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); -- Utility tool. Allow to filter system-dependent strings from an explain output. CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$ @@ -9,7 +11,6 @@ BEGIN END; $$ LANGUAGE PLPGSQL; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = true; SET compute_query_id = 'auto'; @@ -18,7 +19,7 @@ CREATE TABLE t(x int); INSERT INTO t (x) (SELECT * FROM generate_series(1, 100) AS gs); ANALYZE t; -SELECT true FROM aqo_reset(); -- Remember! DROP EXTENSION doesn't remove any AQO data gathered. +SELECT true AS success FROM aqo_reset(); -- Check AQO addons to explain (the only stable data) SELECT regexp_replace( str,'Query Identifier: -?\m\d+\M','Query Identifier: N','g') as str FROM expln(' @@ -47,7 +48,7 @@ SELECT obj_description('aqo_reset'::regproc::oid); -- Check stat reset SELECT count(*) FROM aqo_query_stat; -SELECT true FROM aqo_reset(); -- Remove one record from all tables +SELECT true AS success FROM aqo_reset(); SELECT count(*) FROM aqo_query_stat; DROP EXTENSION aqo; diff --git a/sql/look_a_like.sql b/sql/look_a_like.sql index b5e1f671..c9e59249 100644 --- a/sql/look_a_like.sql +++ b/sql/look_a_like.sql @@ -1,6 +1,9 @@ -CREATE EXTENSION aqo; -SELECT true FROM aqo_reset(); -SET aqo.join_threshold = 0; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + +SET aqo.wide_search = 'on'; + SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; set aqo.show_hash = 'off'; @@ -136,9 +139,10 @@ FROM expln(' SELECT * FROM (A LEFT JOIN B ON A.x1 = B.y1) sc left join C on sc.x1=C.z1;') AS str WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; -SELECT 1 FROM aqo_reset(); +RESET aqo.wide_search; +DROP EXTENSION aqo CASCADE; + DROP TABLE a; DROP TABLE b; DROP TABLE c; DROP FUNCTION expln; -DROP EXTENSION aqo CASCADE; diff --git a/sql/parallel_workers.sql b/sql/parallel_workers.sql index b544cf19..2cd04bc2 100644 --- a/sql/parallel_workers.sql +++ b/sql/parallel_workers.sql @@ -1,7 +1,8 @@ -- Specifically test AQO machinery for queries uses partial paths and executed -- with parallel workers. -CREATE EXTENSION aqo; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); -- Utility tool. Allow to filter system-dependent strings from explain output. CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$ @@ -12,7 +13,6 @@ BEGIN END; $$ LANGUAGE PLPGSQL; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = true; @@ -52,7 +52,6 @@ WHERE q1.id = q2.id;') AS str WHERE str NOT LIKE '%Workers%' AND str NOT LIKE '%Sort Method%' AND str NOT LIKE '%Gather Merge%'; - RESET parallel_tuple_cost; RESET parallel_setup_cost; RESET max_parallel_workers; diff --git a/sql/plancache.sql b/sql/plancache.sql index c9aabae7..b2d1c6d6 100644 --- a/sql/plancache.sql +++ b/sql/plancache.sql @@ -1,7 +1,8 @@ -- Tests on interaction of AQO with cached plans. -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + SET aqo.mode = 'intelligent'; SET aqo.show_details = 'on'; SET aqo.show_hash = 'off'; @@ -44,5 +45,5 @@ SELECT * FROM f1(); DROP FUNCTION f1; DROP TABLE test CASCADE; -SELECT true FROM aqo_reset(); + DROP EXTENSION aqo; diff --git a/sql/relocatable.sql b/sql/relocatable.sql index 780c385e..adf20983 100644 --- a/sql/relocatable.sql +++ b/sql/relocatable.sql @@ -1,5 +1,6 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + SET aqo.mode = 'learn'; -- use this mode for unconditional learning CREATE TABLE test AS (SELECT id, 'payload' || id FROM generate_series(1,100) id); diff --git a/sql/schema.sql b/sql/schema.sql index 6f5f4454..28185710 100644 --- a/sql/schema.sql +++ b/sql/schema.sql @@ -1,4 +1,3 @@ -DROP EXTENSION IF EXISTS aqo CASCADE; DROP SCHEMA IF EXISTS test CASCADE; -- Check Zero-schema path behaviour @@ -11,7 +10,7 @@ CREATE EXTENSION aqo; -- fail CREATE SCHEMA IF NOT EXISTS test1; SET search_path TO test1, public; CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +SELECT true AS success FROM aqo_reset(); SET aqo.mode = 'intelligent'; CREATE TABLE test (id SERIAL, data TEXT); diff --git a/sql/statement_timeout.sql b/sql/statement_timeout.sql index b0ebb6ba..43dab39e 100644 --- a/sql/statement_timeout.sql +++ b/sql/statement_timeout.sql @@ -18,56 +18,58 @@ BEGIN END LOOP; END; $$; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + CREATE TABLE t AS SELECT * FROM generate_series(1,50) AS x; ANALYZE t; DELETE FROM t WHERE x > 5; -- Force optimizer to make overestimated prediction. -CREATE EXTENSION IF NOT EXISTS aqo; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = 'off'; SET aqo.learn_statement_timeout = 'on'; -SET statement_timeout = 100; -- [0.1s] +SET statement_timeout = 80; -- [0.1s] SELECT *, pg_sleep(0.1) FROM t; -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); -- haven't any partial data +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -- haven't any partial data -- Don't learn because running node has smaller cardinality than an optimizer prediction -SET statement_timeout = 400; +SET statement_timeout = 350; SELECT *, pg_sleep(0.1) FROM t; -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -- We have a real learning data. -SET statement_timeout = 8000; +SET statement_timeout = 800; SELECT *, pg_sleep(0.1) FROM t; -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -- Force to make an underestimated prediction DELETE FROM t WHERE x > 2; ANALYZE t; INSERT INTO t (x) (SELECT * FROM generate_series(3,5) AS x); -SELECT 1 FROM aqo_reset(); +SELECT true AS success FROM aqo_reset(); -SET statement_timeout = 100; +SET statement_timeout = 80; SELECT *, pg_sleep(0.1) FROM t; -- Not learned -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -SET statement_timeout = 500; +SET statement_timeout = 350; SELECT *, pg_sleep(0.1) FROM t; -- Learn! -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -SET statement_timeout = 800; +SET statement_timeout = 550; SELECT *, pg_sleep(0.1) FROM t; -- Get reliable data -SELECT check_estimated_rows('SELECT *, pg_sleep(1) FROM t;'); +SELECT check_estimated_rows('SELECT *, pg_sleep(0.1) FROM t;'); -- Interrupted query should immediately appear in aqo_data -SELECT 1 FROM aqo_reset(); +SELECT true AS success FROM aqo_reset(); SET statement_timeout = 500; SELECT count(*) FROM aqo_data; -- Must be zero SELECT x, pg_sleep(0.1) FROM t WHERE x > 0; SELECT count(*) FROM aqo_data; -- Must be one -SELECT 1 FROM aqo_reset(); +SELECT true AS success FROM aqo_reset(); DROP TABLE t; DROP EXTENSION aqo; DROP FUNCTION check_estimated_rows; diff --git a/sql/temp_tables.sql b/sql/temp_tables.sql index aba78aba..e7bc8fe5 100644 --- a/sql/temp_tables.sql +++ b/sql/temp_tables.sql @@ -1,5 +1,8 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + +SET aqo.wide_search = 'on'; SET aqo.mode = 'learn'; CREATE TEMP TABLE tt(); @@ -17,10 +20,10 @@ SELECT count(*) FROM pt AS pt1, tt AS tt1, tt AS tt2, pt AS pt2; SELECT count(*) FROM aqo_data; -- Don't bother about false negatives because of trivial query plans DROP TABLE tt; -SELECT * FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); SELECT count(*) FROM aqo_data; -- Should return the same as previous call above DROP TABLE pt; -SELECT * FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); SELECT count(*) FROM aqo_data; -- Should be 0 SELECT query_text FROM aqo_queries aq LEFT JOIN aqo_query_texts aqt ON aq.queryid = aqt.queryid @@ -67,7 +70,7 @@ SELECT * FROM check_estimated_rows(' SET aqo.mode = 'forced'; -- Now we use all fss records for each query DROP TABLE pt; -SELECT * FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); CREATE TABLE pt AS SELECT x AS x, (x % 10) AS y FROM generate_series(1,100) AS x; CREATE TEMP TABLE ttd1 AS SELECT -(x*3) AS x, (x % 9) AS y1 FROM generate_series(1,100) AS x; @@ -91,7 +94,9 @@ SELECT * FROM check_estimated_rows(' SELECT pt.x, avg(pt.y) FROM pt,ttd1 WHERE pt.x = ttd1.x GROUP BY (pt.x); '); -- Don't use AQO for temp table because of different attname +-- Clear common parts of AQO state +RESET aqo.wide_search; +DROP EXTENSION aqo CASCADE; + DROP TABLE pt CASCADE; -SELECT 1 FROM aqo_reset(); -DROP EXTENSION aqo; DROP FUNCTION check_estimated_rows; diff --git a/sql/top_queries.sql b/sql/top_queries.sql index da3817a0..76000ac4 100755 --- a/sql/top_queries.sql +++ b/sql/top_queries.sql @@ -1,5 +1,7 @@ -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + SET aqo.mode = 'disabled'; SET aqo.force_collect_stat = 'on'; @@ -51,5 +53,4 @@ FROM aqo_cardinality_error(false) ce, aqo_query_texts aqt WHERE ce.id = aqt.queryid ORDER BY (md5(query_text)); -SELECT 1 FROM aqo_reset(); DROP EXTENSION aqo; diff --git a/sql/unsupported.sql b/sql/unsupported.sql index 808a19e1..8b36d721 100644 --- a/sql/unsupported.sql +++ b/sql/unsupported.sql @@ -1,4 +1,5 @@ -CREATE EXTENSION aqo; +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); -- Utility tool. Allow to filter system-dependent strings from an explain output. CREATE OR REPLACE FUNCTION expln(query_string text) RETURNS SETOF text AS $$ @@ -9,7 +10,6 @@ BEGIN END; $$ LANGUAGE PLPGSQL; -SET aqo.join_threshold = 0; SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; @@ -182,7 +182,7 @@ ORDER BY (md5(query_text),error) DESC; DROP TABLE t,t1 CASCADE; -- delete all tables used in the test SELECT count(*) FROM aqo_data; -- Just to detect some changes in the logic. May some false positives really bother us here? -SELECT * FROM aqo_cleanup(); +SELECT true AS success FROM aqo_cleanup(); SELECT count(*) FROM aqo_data; -- No one row should be returned -- Look for any remaining queries in the ML storage. @@ -191,5 +191,4 @@ FROM aqo_cardinality_error(true) cef, aqo_query_texts aqt WHERE aqt.queryid = cef.id ORDER BY (md5(query_text),error) DESC; -SELECT 1 FROM aqo_reset(); DROP EXTENSION aqo; diff --git a/sql/update_functions.sql b/sql/update_functions.sql index 84add94a..e2773978 100644 --- a/sql/update_functions.sql +++ b/sql/update_functions.sql @@ -1,3 +1,7 @@ +-- Preliminaries +CREATE EXTENSION IF NOT EXISTS aqo; +SELECT true AS success FROM aqo_reset(); + CREATE TABLE aqo_test1(a int, b int); WITH RECURSIVE t(a, b) AS ( @@ -18,9 +22,6 @@ AS ( CREATE INDEX aqo_test2_idx_a ON aqo_test2 (a); ANALYZE aqo_test2; -CREATE EXTENSION aqo; -SET aqo.join_threshold = 0; - SET aqo.mode='intelligent'; SELECT count(*) FROM aqo_test1 a, aqo_test2 b WHERE a.a=b.a; @@ -61,7 +62,7 @@ CREATE TABLE aqo_queries_dump AS SELECT * FROM aqo_queries; CREATE TABLE aqo_query_stat_dump AS SELECT * FROM aqo_query_stat; CREATE TABLE aqo_data_dump AS SELECT * FROM aqo_data; -SELECT 1 FROM aqo_reset(); +SELECT true AS success FROM aqo_reset(); -- -- aqo_query_texts_update() testing. @@ -202,8 +203,8 @@ SELECT aqo_data_update(1, 1, 1, '{{1}}', '{1}', '{1, 1}', '{1, 2, 3}'); SELECT aqo_data_update(1, 1, 1, '{{1}, {2}}', '{1}', '{1}', '{1, 2, 3}'); SET aqo.mode='disabled'; -SELECT 1 FROM aqo_reset(); -DROP EXTENSION aqo; + +DROP EXTENSION aqo CASCADE; DROP TABLE aqo_test1, aqo_test2; DROP TABLE aqo_query_texts_dump, aqo_queries_dump, aqo_query_stat_dump, aqo_data_dump; diff --git a/t/001_pgbench.pl b/t/001_pgbench.pl index 2374d83d..cb6b76de 100644 --- a/t/001_pgbench.pl +++ b/t/001_pgbench.pl @@ -20,6 +20,9 @@ my $CLIENTS = 10; my $THREADS = 10; +# Disable connection default settings, forced by PGOPTIONS in AQO Makefile +$ENV{PGOPTIONS}=""; + # Change pgbench parameters according to the environment variable. if (defined $ENV{TRANSACTIONS}) { diff --git a/t/002_pg_stat_statements_aqo.pl b/t/002_pg_stat_statements_aqo.pl index 4d8b04d7..edd20a4a 100644 --- a/t/002_pg_stat_statements_aqo.pl +++ b/t/002_pg_stat_statements_aqo.pl @@ -16,7 +16,13 @@ pg_stat_statements.track = 'none' }); my $query_id; -my ($res, $aqo_res); + +# Disable connection default settings, forced by PGOPTIONS in AQO Makefile +$ENV{PGOPTIONS}=""; + +# General purpose variables. +my $res; +my $aqo_res; my $total_classes; $node->start();