From 2be1c9aa37396a77f912d861218db9919a8817df Mon Sep 17 00:00:00 2001 From: David Bitner Date: Mon, 21 Aug 2023 13:45:19 -0500 Subject: [PATCH 01/13] Update Docker with Rust tooling, split into database and pypgstac as separate images --- Dockerfile | 45 --- docker-compose.yml | 23 +- docker/pgstac/Dockerfile | 71 ++++ docker/pgstac/dbinit/docker-entrypoint.sh | 354 ++++++++++++++++++++ docker/pgstac/dbinit/pgstac-rust-preinit.sh | 3 + docker/pgstac/dbinit/pgstac-rust.sh | 4 + docker/pgstac/dbinit/pgstac.sh | 16 + docker/pypgstac/Dockerfile | 30 ++ docker/pypgstac/bin/format | 24 ++ docker/pypgstac/bin/initpgstac | 11 + docker/pypgstac/bin/loadsampledata | 9 + docker/pypgstac/bin/makemigration | 143 ++++++++ docker/pypgstac/bin/resetpgstac | 11 + docker/pypgstac/bin/stageversion | 57 ++++ docker/pypgstac/bin/test | 246 ++++++++++++++ docker/pypgstac/bin/tmpdb | 54 +++ scripts/console | 6 +- scripts/format | 19 +- scripts/migrate | 26 +- scripts/pgstacenv | 19 ++ scripts/runinpypgstac | 40 +++ scripts/server | 4 +- scripts/stageversion | 31 +- scripts/test | 21 +- scripts/update | 2 +- 25 files changed, 1125 insertions(+), 144 deletions(-) delete mode 100644 Dockerfile create mode 100644 docker/pgstac/Dockerfile create mode 100755 docker/pgstac/dbinit/docker-entrypoint.sh create mode 100755 docker/pgstac/dbinit/pgstac-rust-preinit.sh create mode 100755 docker/pgstac/dbinit/pgstac-rust.sh create mode 100755 docker/pgstac/dbinit/pgstac.sh create mode 100644 docker/pypgstac/Dockerfile create mode 100755 docker/pypgstac/bin/format create mode 100755 docker/pypgstac/bin/initpgstac create mode 100755 docker/pypgstac/bin/loadsampledata create mode 100755 docker/pypgstac/bin/makemigration create mode 100755 docker/pypgstac/bin/resetpgstac create mode 100755 docker/pypgstac/bin/stageversion create mode 100755 docker/pypgstac/bin/test create mode 100755 docker/pypgstac/bin/tmpdb create mode 100644 scripts/pgstacenv create mode 100755 scripts/runinpypgstac diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 8e5b5599..00000000 --- a/Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -FROM postgres:15-bullseye as pg -ENV PGSTACDOCKER=1 -ENV POSTGIS_MAJOR 3 -ENV POSTGIS_VERSION 3.3.3+dfsg-1~exp1.pgdg110+1 -ENV PYTHONPATH=/opt/src/pypgstac:/opt/python:${PYTHONPATH} -ENV PATH=/opt/bin:${PATH} -ENV PYTHONWRITEBYTECODE=1 -ENV PYTHONBUFFERED=1 - -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-cache showpkg postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR - -RUN set -ex \ - && apt-get install -y --no-install-recommends \ - ca-certificates \ - python3 python-is-python3 python3-pip \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR=$POSTGIS_VERSION \ - postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ - postgresql-$PG_MAJOR-pgtap \ - postgresql-$PG_MAJOR-partman \ - postgresql-$PG_MAJOR-plpgsql-check \ - && apt-get remove -y apt-transport-https \ - && apt-get clean && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* \ - && mkdir -p /opt/src/pypgstac/pypgstac \ - && touch /opt/src/pypgstac/pypgstac/__init__.py \ - && touch /opt/src/pypgstac/README.md \ - && echo '__version__ = "0.0.0"' > /opt/src/pypgstac/pypgstac/version.py - -COPY ./src/pypgstac/pyproject.toml /opt/src/pypgstac/pyproject.toml - -RUN \ - pip3 install --upgrade pip \ - && pip3 install /opt/src/pypgstac[dev,test,psycopg] - -COPY ./src /opt/src -COPY ./scripts/bin /opt/bin - -RUN \ - echo "initpgstac" > /docker-entrypoint-initdb.d/999_initpgstac.sh \ - && chmod +x /docker-entrypoint-initdb.d/999_initpgstac.sh \ - && chmod +x /opt/bin/* - -WORKDIR /opt/src diff --git a/docker-compose.yml b/docker-compose.yml index 8e27e0fa..75eb5bf9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,7 +5,8 @@ services: image: pgstac build: context: . - dockerfile: Dockerfile + dockerfile: docker/pgstac/Dockerfile + target: pgstac platform: linux/amd64 environment: - POSTGRES_USER=username @@ -18,7 +19,23 @@ services: - "5439:5432" volumes: - pgstac-pgdata:/var/lib/postgresql/data - - ./src:/opt/src - - ./scripts/bin:/opt/bin + command: postgres + pypgstac: + container_name: pypgstac + image: pypgstac + build: + context: . + dockerfile: docker/pypgstac/Dockerfile + target: pypgstac + platform: linux/amd64 + environment: + - PGHOST=pgstac + - PGUSER=username + - PGPASSWORD=password + - PGDATABASE=postgis + volumes: + - .:/opt + depends_on: + - pgstac volumes: pgstac-pgdata: diff --git a/docker/pgstac/Dockerfile b/docker/pgstac/Dockerfile new file mode 100644 index 00000000..6199563a --- /dev/null +++ b/docker/pgstac/Dockerfile @@ -0,0 +1,71 @@ +ARG PG_MAJOR=15 +ARG POSTGIS_MAJOR=3 + +FROM postgres:${PG_MAJOR}-bullseye as pgstacbase +ARG POSTGIS_MAJOR +RUN \ + apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ + postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ + postgresql-$PG_MAJOR-pgtap \ + postgresql-$PG_MAJOR-plpgsql-check \ + postgresql-$PG_MAJOR-partman \ + && apt-get remove -y apt-transport-https \ + && apt-get clean && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* +COPY docker/pgstac/dbinit/docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh + +FROM pgstacbase as pgstacbase-plrust +ENV PLRUSTVERSION=1.2.3 +ENV RUSTVERSION=1.70.0 +ENV PLRUSTDOWNLOADURL=https://github.com/tcdi/plrust/releases/download/ +ENV PLRUSTFILE=plrust-trusted-${PLRUSTVERSION}_${RUSTVERSION}-debian-pg${PG_MAJOR}-amd64.deb +ENV PLRUSTURL=${PLRUSTDOWNLOADURL}v${PLRUSTVERSION}/${PLRUSTFILE} +ADD $PLRUSTURL . +ENV PATH=/home/postgres/.cargo/bin:$PATH +RUN \ + apt-get update \ + && apt-get upgrade -y \ + && apt-get install -y --no-install-recommends \ + postgresql-server-dev-$PG_MAJOR \ + build-essential \ + ca-certificates \ + clang \ + clang-11 \ + gcc \ + git \ + gnupg \ + libssl-dev \ + llvm-11 \ + lsb-release \ + make \ + pkg-config \ + wget \ + && apt-get remove -y apt-transport-https \ + && apt-get clean && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* +USER postgres +RUN \ + wget -qO- https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain=${RUSTVERSION} \ + && $HOME/.cargo/bin/rustup toolchain install ${RUSTVERSION} \ + && $HOME/.cargo/bin/rustup default ${RUSTVERSION} \ + && $HOME/.cargo/bin/rustup component add rustc-dev +WORKDIR /docker-entrypoint-preinitdb.d +COPY docker/pgstac/dbinit/pgstac-rust-preinit.sh preloadplrust.sh +WORKDIR /docker-entrypoint-initdb.d +COPY docker/pgstac/dbinit/pgstac-rust.sh 991_plrust.sh + +USER root +RUN apt-get install -y /${PLRUSTFILE} + +FROM pgstacbase as pgstac +WORKDIR /docker-entrypoint-initdb.d +COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh +COPY src/pgstac/pgstac.sql 999_pgstac.sql + +FROM pgstacbase-plrust as pgstac-plrust +WORKDIR /docker-entrypoint-initdb.d +COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh +COPY src/pgstac/pgstac.sql 999_pgstac.sql diff --git a/docker/pgstac/dbinit/docker-entrypoint.sh b/docker/pgstac/dbinit/docker-entrypoint.sh new file mode 100755 index 00000000..b6c55481 --- /dev/null +++ b/docker/pgstac/dbinit/docker-entrypoint.sh @@ -0,0 +1,354 @@ +#!/usr/bin/env bash +set -Eeo pipefail +# TODO swap to -Eeuo pipefail above (after handling all potentially-unset variables) + +# usage: file_env VAR [DEFAULT] +# ie: file_env 'XYZ_DB_PASSWORD' 'example' +# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of +# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature) +file_env() { + local var="$1" + local fileVar="${var}_FILE" + local def="${2:-}" + if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then + printf >&2 'error: both %s and %s are set (but are exclusive)\n' "$var" "$fileVar" + exit 1 + fi + local val="$def" + if [ "${!var:-}" ]; then + val="${!var}" + elif [ "${!fileVar:-}" ]; then + val="$(< "${!fileVar}")" + fi + export "$var"="$val" + unset "$fileVar" +} + +# check to see if this file is being run or sourced from another script +_is_sourced() { + # https://unix.stackexchange.com/a/215279 + [ "${#FUNCNAME[@]}" -ge 2 ] \ + && [ "${FUNCNAME[0]}" = '_is_sourced' ] \ + && [ "${FUNCNAME[1]}" = 'source' ] +} + +# used to create initial postgres directories and if run as root, ensure ownership to the "postgres" user +docker_create_db_directories() { + local user; user="$(id -u)" + + mkdir -p "$PGDATA" + # ignore failure since there are cases where we can't chmod (and PostgreSQL might fail later anyhow - it's picky about permissions of this directory) + chmod 00700 "$PGDATA" || : + + # ignore failure since it will be fine when using the image provided directory; see also https://github.com/docker-library/postgres/pull/289 + mkdir -p /var/run/postgresql || : + chmod 03775 /var/run/postgresql || : + + # Create the transaction log directory before initdb is run so the directory is owned by the correct user + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + mkdir -p "$POSTGRES_INITDB_WALDIR" + if [ "$user" = '0' ]; then + find "$POSTGRES_INITDB_WALDIR" \! -user postgres -exec chown postgres '{}' + + fi + chmod 700 "$POSTGRES_INITDB_WALDIR" + fi + + # allow the container to be started with `--user` + if [ "$user" = '0' ]; then + find "$PGDATA" \! -user postgres -exec chown postgres '{}' + + find /var/run/postgresql \! -user postgres -exec chown postgres '{}' + + fi +} + +# initialize empty PGDATA directory with new database via 'initdb' +# arguments to `initdb` can be passed via POSTGRES_INITDB_ARGS or as arguments to this function +# `initdb` automatically creates the "postgres", "template0", and "template1" dbnames +# this is also where the database user is created, specified by `POSTGRES_USER` env +docker_init_database_dir() { + # "initdb" is particular about the current user existing in "/etc/passwd", so we use "nss_wrapper" to fake that if necessary + # see https://github.com/docker-library/postgres/pull/253, https://github.com/docker-library/postgres/issues/359, https://cwrap.org/nss_wrapper.html + local uid; uid="$(id -u)" + if ! getent passwd "$uid" &> /dev/null; then + # see if we can find a suitable "libnss_wrapper.so" (https://salsa.debian.org/sssd-team/nss-wrapper/-/commit/b9925a653a54e24d09d9b498a2d913729f7abb15) + local wrapper + for wrapper in {/usr,}/lib{/*,}/libnss_wrapper.so; do + if [ -s "$wrapper" ]; then + NSS_WRAPPER_PASSWD="$(mktemp)" + NSS_WRAPPER_GROUP="$(mktemp)" + export LD_PRELOAD="$wrapper" NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + local gid; gid="$(id -g)" + printf 'postgres:x:%s:%s:PostgreSQL:%s:/bin/false\n' "$uid" "$gid" "$PGDATA" > "$NSS_WRAPPER_PASSWD" + printf 'postgres:x:%s:\n' "$gid" > "$NSS_WRAPPER_GROUP" + break + fi + done + fi + + if [ -n "${POSTGRES_INITDB_WALDIR:-}" ]; then + set -- --waldir "$POSTGRES_INITDB_WALDIR" "$@" + fi + + # --pwfile refuses to handle a properly-empty file (hence the "\n"): https://github.com/docker-library/postgres/issues/1025 + eval 'initdb --username="$POSTGRES_USER" --pwfile=<(printf "%s\n" "$POSTGRES_PASSWORD") '"$POSTGRES_INITDB_ARGS"' "$@"' + + # unset/cleanup "nss_wrapper" bits + if [[ "${LD_PRELOAD:-}" == */libnss_wrapper.so ]]; then + rm -f "$NSS_WRAPPER_PASSWD" "$NSS_WRAPPER_GROUP" + unset LD_PRELOAD NSS_WRAPPER_PASSWD NSS_WRAPPER_GROUP + fi +} + +# print large warning if POSTGRES_PASSWORD is long +# error if both POSTGRES_PASSWORD is empty and POSTGRES_HOST_AUTH_METHOD is not 'trust' +# print large warning if POSTGRES_HOST_AUTH_METHOD is set to 'trust' +# assumes database is not set up, ie: [ -z "$DATABASE_ALREADY_EXISTS" ] +docker_verify_minimum_env() { + # check password first so we can output the warning before postgres + # messes it up + if [ "${#POSTGRES_PASSWORD}" -ge 100 ]; then + cat >&2 <<-'EOWARN' + + WARNING: The supplied POSTGRES_PASSWORD is 100+ characters. + + This will not work if used via PGPASSWORD with "psql". + + https://www.postgresql.org/message-id/flat/E1Rqxp2-0004Qt-PL%40wrigleys.postgresql.org (BUG #6412) + https://github.com/docker-library/postgres/issues/507 + + EOWARN + fi + if [ -z "$POSTGRES_PASSWORD" ] && [ 'trust' != "$POSTGRES_HOST_AUTH_METHOD" ]; then + # The - option suppresses leading tabs but *not* spaces. :) + cat >&2 <<-'EOE' + Error: Database is uninitialized and superuser password is not specified. + You must specify POSTGRES_PASSWORD to a non-empty value for the + superuser. For example, "-e POSTGRES_PASSWORD=password" on "docker run". + + You may also use "POSTGRES_HOST_AUTH_METHOD=trust" to allow all + connections without a password. This is *not* recommended. + + See PostgreSQL documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + EOE + exit 1 + fi + if [ 'trust' = "$POSTGRES_HOST_AUTH_METHOD" ]; then + cat >&2 <<-'EOWARN' + ******************************************************************************** + WARNING: POSTGRES_HOST_AUTH_METHOD has been set to "trust". This will allow + anyone with access to the Postgres port to access your database without + a password, even if POSTGRES_PASSWORD is set. See PostgreSQL + documentation about "trust": + https://www.postgresql.org/docs/current/auth-trust.html + In Docker's default configuration, this is effectively any other + container on the same system. + + It is not recommended to use POSTGRES_HOST_AUTH_METHOD=trust. Replace + it with "-e POSTGRES_PASSWORD=password" instead to set a password in + "docker run". + ******************************************************************************** + EOWARN + fi +} + +# usage: docker_process_init_files [file [file [...]]] +# ie: docker_process_init_files /always-initdb.d/* +# process initializer files, based on file extensions and permissions +docker_process_init_files() { + # psql here for backwards compatibility "${psql[@]}" + psql=( docker_process_sql ) + + printf '\n' + local f + for f; do + case "$f" in + *.sh) + # https://github.com/docker-library/postgres/issues/450#issuecomment-393167936 + # https://github.com/docker-library/postgres/pull/452 + if [ -x "$f" ]; then + printf '%s: running %s\n' "$0" "$f" + "$f" + else + printf '%s: sourcing %s\n' "$0" "$f" + . "$f" + fi + ;; + *.sql) printf '%s: running %s\n' "$0" "$f"; docker_process_sql -f "$f"; printf '\n' ;; + *.sql.gz) printf '%s: running %s\n' "$0" "$f"; gunzip -c "$f" | docker_process_sql; printf '\n' ;; + *.sql.xz) printf '%s: running %s\n' "$0" "$f"; xzcat "$f" | docker_process_sql; printf '\n' ;; + *.sql.zst) printf '%s: running %s\n' "$0" "$f"; zstd -dc "$f" | docker_process_sql; printf '\n' ;; + *) printf '%s: ignoring %s\n' "$0" "$f" ;; + esac + printf '\n' + done +} + +# Execute sql script, passed via stdin (or -f flag of pqsl) +# usage: docker_process_sql [psql-cli-args] +# ie: docker_process_sql --dbname=mydb <<<'INSERT ...' +# ie: docker_process_sql -f my-file.sql +# ie: docker_process_sql > "$PGDATA/pg_hba.conf" +} + +# start socket-only postgresql server for setting up or running scripts +# all arguments will be passed along as arguments to `postgres` (via pg_ctl) +docker_temp_server_start() { + if [ "$1" = 'postgres' ]; then + shift + fi + + # internal start of server in order to allow setup using psql client + # does not listen on external TCP/IP and waits until start finishes + set -- "$@" -c listen_addresses='' -p "${PGPORT:-5432}" + + PGUSER="${PGUSER:-$POSTGRES_USER}" \ + pg_ctl -D "$PGDATA" \ + -o "$(printf '%q ' "$@")" \ + -w start +} + +# stop postgresql server after done setting up user and running scripts +docker_temp_server_stop() { + PGUSER="${PGUSER:-postgres}" \ + pg_ctl -D "$PGDATA" -m fast -w stop +} + +# check arguments for an option that would cause postgres to stop +# return true if there is one +_pg_want_help() { + local arg + for arg; do + case "$arg" in + # postgres --help | grep 'then exit' + # leaving out -C on purpose since it always fails and is unhelpful: + # postgres: could not access the server configuration file "/var/lib/postgresql/data/postgresql.conf": No such file or directory + -'?'|--help|--describe-config|-V|--version) + return 0 + ;; + esac + done + return 1 +} + +_main() { + # if first arg looks like a flag, assume we want to run postgres server + if [ "${1:0:1}" = '-' ]; then + set -- postgres "$@" + fi + + if [ "$1" = 'postgres' ] && ! _pg_want_help "$@"; then + docker_setup_env + # setup data directories and permissions (when run as root) + docker_create_db_directories + if [ "$(id -u)" = '0' ]; then + # then restart script as postgres user + exec gosu postgres "$BASH_SOURCE" "$@" + fi + + # only run initialization on an empty data directory + if [ -z "$DATABASE_ALREADY_EXISTS" ]; then + docker_verify_minimum_env + + # check dir permissions to reduce likelihood of half-initialized database + ls /docker-entrypoint-initdb.d/ > /dev/null + + docker_init_database_dir + pg_setup_hba_conf "$@" + + # PGPASSWORD is required for psql when authentication is required for 'local' connections via pg_hba.conf and is otherwise harmless + # e.g. when '--auth=md5' or '--auth-local=md5' is used in POSTGRES_INITDB_ARGS + export PGPASSWORD="${PGPASSWORD:-$POSTGRES_PASSWORD}" + docker_temp_server_start "$@" + + docker_setup_db + docker_process_init_files /docker-entrypoint-preinitdb.d/* + docker_temp_server_stop + docker_temp_server_start "$@" + docker_process_init_files /docker-entrypoint-initdb.d/* + + docker_temp_server_stop + unset PGPASSWORD + + cat <<-'EOM' + + PostgreSQL init process complete; ready for start up. + + EOM + else + cat <<-'EOM' + + PostgreSQL Database directory appears to contain a database; Skipping initialization + + EOM + fi + fi + + exec "$@" +} + +if ! _is_sourced; then + _main "$@" +fi diff --git a/docker/pgstac/dbinit/pgstac-rust-preinit.sh b/docker/pgstac/dbinit/pgstac-rust-preinit.sh new file mode 100755 index 00000000..91b734c2 --- /dev/null +++ b/docker/pgstac/dbinit/pgstac-rust-preinit.sh @@ -0,0 +1,3 @@ +psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) +cd $SCRIPT_DIR/../../../src + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Format code. + +This scripts is meant to be run inside the dev container. + +" +} + +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + echo "Formatting pypgstac..." + ruff --fix pypgstac/pypgstac + ruff --fix pypgstac/tests +fi diff --git a/docker/pypgstac/bin/initpgstac b/docker/pypgstac/bin/initpgstac new file mode 100755 index 00000000..b205b7cf --- /dev/null +++ b/docker/pypgstac/bin/initpgstac @@ -0,0 +1,11 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/../../../src/pgstac + +psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) +cd $SCRIPT_DIR/../../../src/pgstac +psql -f pgstac.sql +psql -v ON_ERROR_STOP=1 <<-EOSQL + \copy collections (content) FROM 'tests/testdata/collections.ndjson' + \copy items_staging (content) FROM 'tests/testdata/items.ndjson' +EOSQL diff --git a/docker/pypgstac/bin/makemigration b/docker/pypgstac/bin/makemigration new file mode 100755 index 00000000..6837ba2c --- /dev/null +++ b/docker/pypgstac/bin/makemigration @@ -0,0 +1,143 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SRCDIR=$SCRIPT_DIR/../../../src +cd $SRCDIR + +SHORT=f:,t:,o,d,h +LONG=from:,to:,overwrite,debug,help +OPTS=$(getopt --alternative --name $0 --options $SHORT --longoptions $LONG -- "$@") + +eval set -- "$OPTS" + +while : +do + case "$1" in + -f | --from ) + FROM="$2" + shift 2 + ;; + -t | --to ) + TO="$2" + shift 2 + ;; + -o | --overwrite ) + OVERWRITE=1 + shift 1 + ;; + -d | --debug ) + DEBUG=1 + shift 1 + ;; + -h | --help) + "Help" + exit 2 + ;; + --) + shift; + break + ;; + *) + echo "Unexpected option: $1" + ;; + esac +done + +# make sure that from and to exist + + + +BASEDIR=$SRCDIR +PYPGSTACDIR=$BASEDIR/pypgstac +MIGRATIONSDIR=$BASEDIR/pgstac/migrations +SQLDIR=$BASEDIR/pgstac/sql + +# Check if from SQL file exists +FROMSQL=$MIGRATIONSDIR/pgstac.$FROM.sql +if [ -f $FROMSQL ]; then + echo "Migrating From: $FROMSQL" +else + echo "From SQL $FROMSQL does not exist" + exit 1 +fi + +# Check if to SQL file exists +TOSQL=$MIGRATIONSDIR/pgstac.$TO.sql +if [ -f $TOSQL ]; then + echo "Migrating To: $TOSQL" +else + echo "To SQL $TOSQL does not exist" + exit 1 +fi + +MIGRATIONSQL=$MIGRATIONSDIR/pgstac.$FROM-$TO.sql +if [[ -f "$MIGRATIONSQL" ]]; then + if [[ "$OVERWRITE" != 1 ]]; then + echo "$MIGRATIONSQL Already exists." + select yn in "Yes" "No"; do + case $yn in + Yes ) break;; + No ) exit 1;; + esac + done + else + echo "Removing existing $MIGRATIONSQL" + rm $MIGRATIONSQL + fi +else + echo "Creating $MIGRATIONSQL" +fi + +pg_isready -t 10 +# Create Databases to inspect to create migration +psql -q >/dev/null 2>&1 <<-'EOSQL' + DROP DATABASE IF EXISTS migra_from; + CREATE DATABASE migra_from; + DROP DATABASE IF EXISTS migra_to; + CREATE DATABASE migra_to; +EOSQL + +TODBURL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST:-localhost}:${PGPORT:-5432}/migra_to" +FROMDBURL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST:-localhost}:${PGPORT:-5432}/migra_from" + +# Make sure to clean up migra databases +function drop_migra_dbs(){ +psql -q >/dev/null 2>&1 <<-'EOSQL' + DROP DATABASE IF EXISTS migra_from; + DROP DATABASE IF EXISTS migra_to; +EOSQL +} + +trap drop_migra_dbs 0 2 3 15 + +echo "Creating Migration from $FROM to $TO" + +# Install From into Database +psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $FROMSQL $FROMDBURL >/dev/null || exit 1; + +# Install To into Database +psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $TOSQL $TODBURL >/dev/null || exit 1; + + +# Calculate the migration +MIGRATION=$(mktemp) +trap "rm $MIGRATION" 0 2 3 15 + +migra --schema pgstac --unsafe $FROMDBURL $TODBURL >$MIGRATION +if [[ $DEBUG == 1 ]]; then + echo "*************" + cat $MIGRATION + echo "*************" +fi + +# Append wrapper around created migration with idempotent and transaction statements + +echo "SET client_min_messages TO WARNING;" >$MIGRATIONSQL +echo "SET SEARCH_PATH to pgstac, public;" >>$MIGRATIONSQL +cat $SQLDIR/000_idempotent_pre.sql >>$MIGRATIONSQL +echo "-- BEGIN migra calculated SQL" >>$MIGRATIONSQL +cat $MIGRATION >>$MIGRATIONSQL +echo "-- END migra calculated SQL" >>$MIGRATIONSQL +cat $SQLDIR/998_idempotent_post.sql $SQLDIR/999_version.sql >>$MIGRATIONSQL + +echo "Migration created at $MIGRATIONSQL." +exit 0 diff --git a/docker/pypgstac/bin/resetpgstac b/docker/pypgstac/bin/resetpgstac new file mode 100755 index 00000000..097c3d5b --- /dev/null +++ b/docker/pypgstac/bin/resetpgstac @@ -0,0 +1,11 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/../../../src/pgstac +set -e +psql -v ON_ERROR_STOP=1 <<-EOSQL + DROP SCHEMA IF EXISTS pgstac CASCADE; + \i pgstac.sql + SET SEARCH_PATH TO pgstac, public; + \copy collections (content) FROM 'tests/testdata/collections.ndjson' + \copy items_staging (content) FROM 'tests/testdata/items.ndjson' +EOSQL diff --git a/docker/pypgstac/bin/stageversion b/docker/pypgstac/bin/stageversion new file mode 100755 index 00000000..7fa33b55 --- /dev/null +++ b/docker/pypgstac/bin/stageversion @@ -0,0 +1,57 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +SRCDIR=$SCRIPT_DIR/../../../src +cd $SRCDIR +BASEDIR=$SRCDIR +SQLDIR=$BASEDIR/pgstac/sql +PYPGSTACDIR=$BASEDIR/pypgstac +MIGRATIONSDIR=$BASEDIR/pgstac/migrations + + +# Remove any existing unreleased migrations +find $MIGRATIONSDIR -name "*unreleased*" -exec rm {} \; + +# Get Version +if [[ ! -z "$1" ]]; then + VERSION=$1 + if echo "$VERSION" | grep -E "^[0-9]+[.][0-9]+[.][0-9]+$"; then + echo "STAGING VERSION: $VERSION" + else + echo "Version must be in the format 0.1.2" + exit 1 + fi + git tag -f "v$VERSION" +else + VERSION="unreleased" +fi + + +OLDVERSION=$(find $MIGRATIONSDIR -name "pgstac.*.sql" | sed -En 's/^.*pgstac\.([0-9]+\.[0-9]+\.[0-9]+)\.sql$/\1/p' | grep -v "$VERSION" | sort -Vr | head -1) + + + +echo "Bumping version from $OLDVERSION to $VERSION" + +# Assemble a base migration for the version and put it in the migrations directory. +cd $SQLDIR +echo "SELECT set_version('${VERSION}');" >999_version.sql +cat *.sql >$MIGRATIONSDIR/pgstac.${VERSION}.sql +cd $BASEDIR/pgstac + +# make the base pgstac.sql a symbolic link to the most recent version +rm pgstac.sql +ln -s migrations/pgstac.${VERSION}.sql pgstac.sql + +# Update the version number in the appropriate places +[[ $VERSION == 'unreleased' ]] && PYVERSION="${OLDVERSION}-dev" || PYVERSION="$VERSION" +echo "Setting pypgstac version to $PYVERSION" +cat < $PYPGSTACDIR/python/pypgstac/version.py +"""Version.""" +__version__ = "${PYVERSION}" +EOD +sed -i "s/^version[ ]*=[ ]*.*/version = \"${PYVERSION}\"/" $PYPGSTACDIR/pyproject.toml +sed -i "s/^version[ ]*=[ ]*.*/version = \"${PYVERSION}\"/" $PYPGSTACDIR/Cargo.toml + +makemigration -f $OLDVERSION -t $VERSION + +# git add $PYPGSTACDIR/pypgstac/version.py $PYPGSTACDIR/pyproject.toml $PYPGSTACDIR/Cargo.toml $MIGRATIONSDIR/pgstac.${VERSION}.sql $MIGRATIONSDIR/pgstac.${OLDVERSION}-${VERSION}.sql $BASEDIR/pgstac/pgstac.sql diff --git a/docker/pypgstac/bin/test b/docker/pypgstac/bin/test new file mode 100755 index 00000000..3dc653b4 --- /dev/null +++ b/docker/pypgstac/bin/test @@ -0,0 +1,246 @@ +#!/bin/bash +set -e +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +export SRCDIR=$SCRIPT_DIR/../../../src +export PGSTACDIR=$SRCDIR/pgstac + +if [[ "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Run PgSTAC tests. +This scripts is meant to be run inside the dev container. + +" +} + +function setuptestdb(){ + cd $PGSTACDIR + psql -X -q -v ON_ERROR_STOP=1 <"$TMPFILE" + +diff -Z -b -w -B --strip-trailing-cr --suppress-blank-empty -C 1 "$TMPFILE" $SQLOUTFILE && echo "TEST $SQLFILE PASSED" || { echo "***TEST FOR $SQLFILE FAILED***"; exit 1; } + +done +psql -X -q -c "DROP DATABASE IF EXISTS pgstac_test_basicsql WITH (force);"; +} + +function test_pypgstac(){ +[[ $MESSAGELOG == 1 ]] && VERBOSE="-vvv" +TEMPLATEDB=${1:-pgstac_test_db_template} + cd $SRCDIR/pypgstac + psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) + +cd /opt/src/pgstac +# Create template database with pgstac installed +psql -X -q -v ON_ERROR_STOP=1 < 0 ]]; do case $1 in if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - docker-compose up -d + docker compose up -d if [[ "${DB_CONSOLE}" ]]; then - docker-compose exec pgstac psql + docker compose exec pgstac psql exit 0 fi - docker-compose exec pgstac /bin/bash + docker compose exec pgstac /bin/bash fi diff --git a/scripts/format b/scripts/format index 5c687331..0a47433a 100755 --- a/scripts/format +++ b/scripts/format @@ -1,21 +1,4 @@ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. -set -e - -if [[ "${CI}" ]]; then - set -x -fi - -function usage() { - echo -n \ - "Usage: $(basename "$0") -Format code in this project - -" -} - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - echo "Formatting pypgstac..." - docker-compose run --rm pgstac sh -c "ruff --fix pypgstac/pypgstac; ruff --fix pypgstac/tests" -fi +$SCRIPT_DIR/runinpypgstac format "$@" diff --git a/scripts/migrate b/scripts/migrate index f166d7d6..445a5179 100755 --- a/scripts/migrate +++ b/scripts/migrate @@ -1,24 +1,4 @@ #!/bin/bash - -set -e - -if [[ "${CI}" ]]; then - set -x -fi - -function usage() { - echo -n \ - "Usage: $(basename "$0") -Run migrations against the development database. -" -} - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - - # Run database migrations - docker-compose up -d pgstac - docker-compose \ - exec pgstac \ - bash -c "pypgstac pgready && pypgstac migrate --debug" - -fi +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. +$SCRIPT_DIR/runinpypgstac pypgstac migrate "$@" diff --git a/scripts/pgstacenv b/scripts/pgstacenv new file mode 100644 index 00000000..0dc93390 --- /dev/null +++ b/scripts/pgstacenv @@ -0,0 +1,19 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. + +export PATH=$SCRIPT_DIR:$PATH + +set -e + +if [[ "${CI}" ]]; then + set -x +fi + +function usage() { + echo -n \ + "Usage: $(basename "$0") +Format code in this project + +" +} diff --git a/scripts/runinpypgstac b/scripts/runinpypgstac new file mode 100755 index 00000000..b9869f3c --- /dev/null +++ b/scripts/runinpypgstac @@ -0,0 +1,40 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/.. +set -e + +if [[ "${CI}" ]]; then + set -x +fi +function usage() { + echo -n \ + "Usage: $(basename "$0") <--build> <--no-cache>