diff --git a/.github/actions/k3d/action.yaml b/.github/actions/k3d/action.yaml new file mode 100644 index 000000000..538233238 --- /dev/null +++ b/.github/actions/k3d/action.yaml @@ -0,0 +1,85 @@ +name: k3d +description: Start k3s using k3d +inputs: + k3d-tag: + default: latest + required: true + description: > + Git tag from https://github.com/rancher/k3d/releases or "latest" + k3s-channel: + default: latest + required: true + description: > + https://rancher.com/docs/k3s/latest/en/upgrades/basic/#release-channels + prefetch-images: + required: true + description: > + Each line is the name of an image to fetch onto all Kubernetes nodes + prefetch-timeout: + default: 90s + required: true + description: > + Amount of time to wait for images to be fetched + +outputs: + kubernetes-version: + value: ${{ steps.k3s.outputs.server }} + description: > + Kubernetes server version, as reported by the Kubernetes API + +runs: + using: composite + steps: + - id: k3d + name: Install k3d + shell: bash + env: + K3D_TAG: ${{ inputs.k3d-tag }} + run: | + curl --fail --silent https://raw.githubusercontent.com/rancher/k3d/main/install.sh | + TAG="${K3D_TAG#latest}" bash + k3d version | awk '{ print "::set-output name=" tolower($1) "::" $3 }' + + - id: k3s + name: Start k3s + shell: bash + run: | + k3d cluster create --image '+${{ inputs.k3s-channel }}' --no-lb --timeout=2m --wait + kubectl version --short | awk '{ print "::set-output name=" tolower($1) "::" $3 }' + + docker exec $(k3d node list --output json | jq --raw-output 'first.name') \ + k3s agent --help | awk '$1 == "--pause-image" { + match($0, /default: "[^"]*"/) + print "::set-output name=pause-image::" substr($0, RSTART+10, RLENGTH-11) + }' + + - name: Prefetch container images + shell: bash + env: + INPUT_IMAGES: ${{ inputs.prefetch-images }} + INPUT_TIMEOUT: ${{ inputs.prefetch-timeout }} + run: | + jq <<< "$INPUT_IMAGES" --raw-input 'select(. != "")' | + jq --slurp \ + --arg pause '${{ steps.k3s.outputs.pause-image }}' \ + --argjson labels '{"name":"image-prefetch"}' \ + --argjson name '"image-prefetch"' \ + '{ + apiVersion: "apps/v1", kind: "DaemonSet", + metadata: { name: $name, labels: $labels }, + spec: { + selector: { matchLabels: $labels }, + template: { + metadata: { labels: $labels }, + spec: { + initContainers: to_entries | map({ + name: "c\(.key)", image: .value, command: ["true"], + }), + containers: [{ name: "pause", image: $pause }] + } + } + } + }' | + kubectl create --filename=- + kubectl rollout status daemonset.apps/image-prefetch --timeout "$INPUT_TIMEOUT" || + kubectl describe daemonset.apps/image-prefetch diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1bec5415e..cf9b6c472 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -40,12 +40,11 @@ jobs: - run: gzip envtest.coverage - uses: actions/upload-artifact@v3 with: - name: "kubernetes-api=${{ matrix.kubernetes }}" + name: "~coverage~kubernetes-api=${{ matrix.kubernetes }}" path: envtest.coverage.gz retention-days: 1 kubernetes-k3d: - if: "${{ github.repository == 'CrunchyData/postgres-operator' }}" runs-on: ubuntu-latest needs: [go-test] strategy: @@ -57,43 +56,15 @@ jobs: - uses: actions/setup-go@v3 with: { go-version: 1.x } - - name: Install k3d - # Git tag from https://github.com/rancher/k3d/releases or "latest" - env: { K3D_TAG: latest } - run: | - curl --fail --silent https://raw.githubusercontent.com/rancher/k3d/main/install.sh | - TAG="${K3D_TAG#latest}" bash && k3d version | head -n1 - - name: Start k3s - # https://rancher.com/docs/k3s/latest/en/upgrades/basic/#release-channels - env: { K3S_CHANNEL: "${{ matrix.kubernetes }}" } - run: k3d cluster create --image="+${K3S_CHANNEL}" --no-lb --timeout=2m --wait + uses: ./.github/actions/k3d + with: + k3s-channel: "${{ matrix.kubernetes }}" + prefetch-images: | + registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.6-1 + registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-0 + registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-2 - - name: Prefetch container images - run: | - { - echo '"registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-13.6-1"' - echo '"registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-0"' - echo '"registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-2"' - } | - jq --slurp --arg name 'image-prefetch' --argjson labels '{"name":"image-prefetch"}' '{ - apiVersion: "apps/v1", kind: "DaemonSet", - metadata: { name: $name, labels: $labels }, - spec: { - selector: { matchLabels: $labels }, - template: { - metadata: { labels: $labels }, - spec: { - initContainers: to_entries | map({ name: "c\(.key)", command: ["true"], image: .value }), - containers: [{ name: "pause", image: "k8s.gcr.io/pause:3.5" }] - } - } - } - }' | - kubectl create --filename=- && { - kubectl rollout status daemonset.apps/image-prefetch --timeout=90s || - kubectl describe daemonset.apps/image-prefetch - } - run: make createnamespaces check-envtest-existing env: PGO_TEST_TIMEOUT_SCALE: 1.2 @@ -103,16 +74,90 @@ jobs: - run: gzip envtest-existing.coverage - uses: actions/upload-artifact@v3 with: - name: "kubernetes-k3d=${{ matrix.kubernetes }}" + name: "~coverage~kubernetes-k3d=${{ matrix.kubernetes }}" path: envtest-existing.coverage.gz retention-days: 1 + kuttl-k3d: + runs-on: ubuntu-latest + needs: [go-test] + strategy: + fail-fast: false + matrix: + kubernetes: [latest] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: { go-version: 1.x } + + - name: Start k3s + uses: ./.github/actions/k3d + with: + k3s-channel: "${{ matrix.kubernetes }}" + prefetch-images: | + registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-2 + registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-2 + registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-4 + registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.4-0 + registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.4-3.1-0 + + - run: go mod download + - name: Build coverage executable + run: | + make build-postgres-operator \ + GO_BUILD='go test -c --tags main_with_coverage --trimpath --coverpkg ./internal/...' \ + PGO_VERSION='${{ github.sha }}-coverage' + + # Start a Docker container with the working directory mounted. + - name: Start PGO + run: | + kubectl apply --server-side -k ./config/namespace + kubectl apply --server-side -k ./config/dev + hack/create-kubeconfig.sh postgres-operator pgo + + docker run --detach --network host --read-only \ + --volume "$(pwd):/mnt" --workdir '/mnt' --env 'PATH=/mnt/bin' \ + --env 'KUBECONFIG=hack/.kube/postgres-operator/pgo' \ + --env 'RELATED_IMAGE_PGADMIN=registry.developers.crunchydata.com/crunchydata/crunchy-pgadmin4:ubi8-4.30-2' \ + --env 'RELATED_IMAGE_PGBACKREST=registry.developers.crunchydata.com/crunchydata/crunchy-pgbackrest:ubi8-2.38-2' \ + --env 'RELATED_IMAGE_PGBOUNCER=registry.developers.crunchydata.com/crunchydata/crunchy-pgbouncer:ubi8-1.16-4' \ + --env 'RELATED_IMAGE_POSTGRES_14=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.4-0' \ + --env 'RELATED_IMAGE_POSTGRES_14_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.4-3.1-0' \ + --name 'postgres-operator' ubuntu \ + postgres-operator --test.coverprofile 'kuttl.coverage' + + - run: make tools/kuttl + - run: make generate-kuttl + env: + KUTTL_PG_VERSION: '14' + KUTTL_POSTGIS_VERSION: '3.1' + KUTTL_PSQL_IMAGE: 'registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.4-0' + - run: | + make check-kuttl && exit + failed=$? + echo '::group::PGO logs'; docker logs 'postgres-operator'; echo '::endgroup::' + exit $failed + env: + KUTTL_TEST: hack/tools/kuttl test --test='(cluster-start|pgbackrest-restore|pgbouncer)' --timeout=180 --parallel=1 + + - name: Stop PGO + run: docker stop 'postgres-operator' || true + + # Upload coverage to GitHub + - run: gzip kuttl.coverage + - uses: actions/upload-artifact@v3 + with: + name: "~coverage~kuttl-k3d=${{ matrix.kubernetes }}" + path: kuttl.coverage.gz + retention-days: 1 + coverage-report: if: ${{ success() || contains(needs.*.result, 'success') }} runs-on: ubuntu-latest needs: - kubernetes-api - kubernetes-k3d + - kuttl-k3d steps: - uses: actions/checkout@v3 - uses: actions/setup-go@v3 @@ -143,6 +188,6 @@ jobs: - run: gzip total-coverage.html - uses: actions/upload-artifact@v3 with: - name: coverage-report + name: coverage-report=html path: total-coverage.html.gz retention-days: 15 diff --git a/Makefile b/Makefile index ccfde0303..71c54a0dd 100644 --- a/Makefile +++ b/Makefile @@ -47,12 +47,17 @@ ifeq ("$(PGO_BASEOS)", "ubi8") PACKAGER=microdnf endif +# Setting SHELL to bash allows bash commands to be executed by recipes. +# Options are set to exit when a recipe line exits non-zero or a piped command fails. +SHELL = /usr/bin/env bash -O globstar -o pipefail +.SHELLFLAGS = -ec + DEBUG_BUILD ?= false GO ?= go GO_BUILD = $(GO_CMD) build -trimpath GO_CMD = $(GO_ENV) $(GO) GO_TEST ?= $(GO) test -KUTTL_TEST ?= kuttl test +KUTTL_TEST ?= $(KUTTL) test # Disable optimizations if creating a debug build ifeq ("$(DEBUG_BUILD)", "true") @@ -210,7 +215,7 @@ check-envtest-existing: createnamespaces # Expects operator to be running .PHONY: check-kuttl check-kuttl: - ${PGO_KUBE_CLIENT} ${KUTTL_TEST} \ + ${KUTTL_TEST} \ --config testing/kuttl/kuttl-test.yaml .PHONY: generate-kuttl @@ -234,14 +239,12 @@ check-generate: generate-crd generate-deepcopy generate-rbac git diff --exit-code -- config/rbac git diff --exit-code -- pkg/apis -clean: clean-deprecated +clean: clean-deprecated clean-tools rm -f bin/postgres-operator rm -f config/rbac/role.yaml [ ! -d testing/kuttl/e2e-generated ] || rm -r testing/kuttl/e2e-generated [ ! -d testing/kuttl/e2e-generated-other ] || rm -r testing/kuttl/e2e-generated-other [ ! -d build/crd/generated ] || rm -r build/crd/generated - [ ! -d hack/tools/envtest ] || rm -r hack/tools/envtest - [ ! -n "$$(ls hack/tools)" ] || rm hack/tools/* [ ! -d hack/.kube ] || rm -r hack/.kube clean-deprecated: @@ -306,3 +309,32 @@ hack/tools/envtest: license: licenses licenses: ./bin/license_aggregator.sh ./cmd/... + +## Tools + +.PHONY: clean-tools tools +clean-tools: + rm -rf hack/krew hack/tools/envtest + [ ! -n "$$(ls hack/tools)" ] || rm hack/tools/* + +KREW ?= hack/tools/krew +tools: tools/krew +tools/krew: + $(call go-get-tool,$(KREW),sigs.k8s.io/krew/cmd/krew@v0.4.3) + +KUTTL ?= hack/tools/kuttl +tools: tools/kuttl +tools/kuttl: tools/krew + $(call krew-get-tool,$(KUTTL),kuttl) + +# go-get-tool will 'go install' any package $2 and install it to $1. +define go-get-tool +@[ -f '$(1)' ] || { echo Downloading '$(2)'; GOBIN='$(abspath $(dir $(1)))' $(GO) install '$(2)'; } +endef + +# krew-get-tool will 'krew install' any plugin $2 and link it to $1. +define krew-get-tool +@[ -f '$(1)' ] || { KREW_ROOT='hack/krew' $(KREW) install '$(2)'; } +@[ -f '$(1)' ] || { T="$$(readlink 'hack/krew/bin/kubectl-$(notdir $(1))')" && \ + [[ '$(1)' == hack/* ]] && ln -fs "..$${T#$(realpath hack)}" '$(1)'; } +endef diff --git a/cmd/postgres-operator/main_coverage_test.go b/cmd/postgres-operator/main_coverage_test.go new file mode 100644 index 000000000..160f1f015 --- /dev/null +++ b/cmd/postgres-operator/main_coverage_test.go @@ -0,0 +1,28 @@ +//go:build main_with_coverage + +/* + Copyright 2022 Crunchy Data Solutions, Inc. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import "testing" + +// TestWithCoverage is a simple way to run the entire application when built by +// "go test -c" with code coverage enabled. This file has a build constraint, +// and should *not* be run with regular "go test". +// +// - https://www.elastic.co/blog/code-coverage-for-your-golang-system-tests +// +func TestWithCoverage(t *testing.T) { main() }