ci: Unify more of hack/ and tests/

A key thing for me is that the `Justfile` should be a one-stop
shop for development of the project. It can't have everything but
it should answer the basic questions of "how do I build and test
this project".

This aligns the recently added tmt-on-GHA flow a *bit* more closely
with some of that. Biggest is to use the `just build-integration-test-image` as the canonical
way to build a container image with our testing stuff in it;
which uses our main Dockerfile

Other cleanups:
- Change the scripts to accept data via argv[1] and not environment
- Drop the hardcoded testing directory and use `target/` as
  a generic build artifact dir

Signed-off-by: Colin Walters <[email protected]>

Author: cgwalters

.github/workflows/integration.yml

@@ -1,7 +1,8 @@
 name: bootc integration test
 on:
   pull_request:
-    branches: [main]
+    branches: [main]  
+  workflow_dispatch:
 
 jobs:
   build:
@@ -18,14 +19,12 @@ jobs:
           set -eux
           echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
           sudo apt update
-          sudo apt install -y crun/testing podman/testing
+          sudo apt install -y crun/testing podman/testing just
 
       - uses: actions/checkout@v4
 
       - name: Build bootc and bootc image
-        env:
-          TEST_OS: ${{ matrix.test_os }}
-        run: sudo -E TEST_OS=$TEST_OS tests/build.sh
+        run: sudo tests/build.sh ${{ matrix.test_os }}
 
       - name: Grant sudo user permission to archive files
         run: |
@@ -88,9 +87,7 @@ jobs:
           ls -l /dev/kvm
 
       - name: Run test
-        env:
-          TMT_PLAN_NAME: ${{ matrix.tmt_plan }}
-        run: chmod 600 /tmp/tmp-bootc-build/id_rsa && tests/test.sh
+        run: tests/test.sh ${{ matrix.tmt_plan }}
 
       - name: Archive TMT logs
         if: always()

hack/Containerfile

@@ -1,4 +1,7 @@
-# This injects some extra testing stuff into our image
+# Build a container image that has extra testing stuff in it, such
+# as nushell, some preset logically bound images, etc. This expects
+# to create an image derived FROM localhost/bootc which was created
+# by the Dockerfile at top.
 
 FROM scratch as context
 # We only need this stuff in the initial context
@@ -11,7 +14,15 @@ ARG variant=
 # And this layer has additional stuff for testing, such as nushell etc.
 RUN --mount=type=bind,from=context,target=/run/context <<EORUN
 set -xeuo pipefail
-/run/context/hack/provision-derived.sh "$variant"
+cd /run/context/hack
+./provision-derived.sh "$variant"
+
+# For test-22-logically-bound-install
+cp -a lbi/usr/. /usr
+for x in curl.container curl-base.image podman.image; do
+    ln -s /usr/share/containers/systemd/$x /usr/lib/bootc/bound-images.d/$x
+done
+
 # Add some testing kargs into our dev builds
 install -D -t /usr/lib/bootc/kargs.d /run/context/hack/test-kargs/*
 # Also copy in some default install configs we use for testing

hack/lbi/usr/share/containers/systemd/curl-base.image

@@ -0,0 +1,2 @@
+[Image]
+Image=quay.io/curl/curl-base:latest

hack/lbi/usr/share/containers/systemd/curl.container

@@ -0,0 +1,3 @@
+[Container]
+Image=quay.io/curl/curl:latest
+GlobalArgs=--storage-opt=additionalimagestore=/usr/lib/bootc/storage

hack/lbi/usr/share/containers/systemd/jboss-webserver-5.image

@@ -0,0 +1,6 @@
+# This is not symlinked to bound-images.d so it should not be pulled.
+# It's here to represent an app image that exists
+# in a bootc image but is not logically bound.
+[Image]
+Image=registry.redhat.io/jboss-webserver-5/jws5-rhel8-operator:latest
+AuthFile=/root/auth.json

hack/lbi/usr/share/containers/systemd/podman.image

@@ -0,0 +1,2 @@
+[Image]
+Image=registry.access.redhat.com/ubi9/podman:latest

hack/packages.txt

@@ -0,0 +1,4 @@
+# Needed by tmt
+rsync
+/usr/bin/flock
+/usr/bin/awk

hack/provision-derived.sh

@@ -15,13 +15,10 @@ case "${ID}-${VERSION_ID}" in
     "centos-9")
         dnf config-manager --set-enabled crb
         dnf -y install epel-release epel-next-release
-        dnf -y install nu
-        dnf clean all
         ;;
     "rhel-9."*)
         dnf -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm
         dnf -y install nu
-        dnf clean all
         ;;
     "centos-10"|"rhel-10."*)
         # nu is not available in CS10
@@ -32,10 +29,13 @@ case "${ID}-${VERSION_ID}" in
         ;;
     "fedora-"*)
         dnf -y install nu
-        dnf clean all
         ;;
 esac
 
+# Extra packages we install
+grep -Ev -e '^#' packages.txt | xargs dnf -y install
+dnf clean all
+
 # Stock extra cleaning of logs and caches in general (mostly dnf)
 rm /var/log/* /var/cache /var/lib/{dnf,rpm-state,rhsm} -rf
 # And clean root's homedir

tests/build.sh

@@ -4,115 +4,65 @@ set -exuo pipefail
 # This script basically builds bootc from source using the provided base image,
 # then runs the target tests.
 
-mkdir -p /tmp/tmp-bootc-build
-BOOTC_TEMPDIR="/tmp/tmp-bootc-build"
-
-# Get OS info from TEST_OS env
-OS_ID=$(echo "$TEST_OS" | cut -d '-' -f 1)
-OS_VERSION_ID=$(echo "$TEST_OS" | cut -d '-' -f 2)
-
-# Base image
-case "$OS_ID" in
-    "centos")
-        TIER1_IMAGE_URL="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
+# If provided should be of the form fedora-42 or centos-10
+target=${1:-}
+
+build_args=()
+if test -n "${target:-}"; then
+    shift
+    # Get OS info from TEST_OS env
+    OS_ID=$(echo "$target" | cut -d '-' -f 1)
+    OS_VERSION_ID=$(echo "$target" | cut -d '-' -f 2)
+
+    # Base image
+    case "$OS_ID" in
+        "centos")
+            BASE="quay.io/centos-bootc/centos-bootc:stream${OS_VERSION_ID}"
         ;;
-    "fedora")
-        TIER1_IMAGE_URL="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
+        "fedora")
+            BASE="quay.io/fedora/fedora-bootc:${OS_VERSION_ID}"
         ;;
-esac
-
-CONTAINERFILE="${BOOTC_TEMPDIR}/Containerfile"
-tee "$CONTAINERFILE" > /dev/null << CONTAINERFILEOF
-FROM $TIER1_IMAGE_URL as build
-
-WORKDIR /code
-
-RUN <<EORUN
-set -xeuo pipefail
-. /usr/lib/os-release
-case \$ID in
-    centos|rhel) dnf config-manager --set-enabled crb;;
-    fedora) dnf -y install dnf-utils 'dnf5-command(builddep)';;
-esac
-dnf -y builddep contrib/packaging/bootc.spec
-dnf -y install git-core
-EORUN
-
-RUN mkdir -p /build/target/dev-rootfs
-# git config --global --add safe.directory /code to fix "fatal: detected dubious ownership in repository at '/code'" error
-RUN --mount=type=cache,target=/build/target --mount=type=cache,target=/var/roothome git config --global --add safe.directory /code && make test-bin-archive && mkdir -p /out && cp target/bootc.tar.zst /out
-
-FROM $TIER1_IMAGE_URL
-
-# Inject our built code
-COPY --from=build /out/bootc.tar.zst /tmp
-RUN tar -C / --zstd -xvf /tmp/bootc.tar.zst && rm -vrf /tmp/*
-
-RUN <<EORUN
-set -xeuo pipefail
-
-# Provision test requirement
-/code/hack/provision-derived.sh
-# Also copy in some default install configs we use for testing
-cp -a /code/hack/install-test-configs/* /usr/lib/bootc/install/
-# And some test kargs
-cp -a /code/hack/test-kargs/* /usr/lib/bootc/kargs.d/
-
-# For testing farm
-mkdir -p -m 0700 /var/roothome
-
-# Enable ttyS0 console
-mkdir -p /usr/lib/bootc/kargs.d/
-cat <<KARGEOF >> /usr/lib/bootc/kargs.d/20-console.toml
-kargs = ["console=ttyS0,115200n8"]
-KARGEOF
-
-# For test-22-logically-bound-install
-cp -a /code/tmt/tests/lbi/usr/. /usr
-ln -s /usr/share/containers/systemd/curl.container /usr/lib/bootc/bound-images.d/curl.container
-ln -s /usr/share/containers/systemd/curl-base.image /usr/lib/bootc/bound-images.d/curl-base.image
-ln -s /usr/share/containers/systemd/podman.image /usr/lib/bootc/bound-images.d/podman.image
-
-# Install rsync which is required by tmt
-dnf -y install cloud-init rsync
-dnf -y clean all
+        *) echo "Unknown OS: ${OS_ID}" 1>&2; exit 1
+        ;;
+    esac
+    build_args+=("--build-arg=base=$BASE")
+fi
 
-rm -rf /var/cache /var/lib/dnf
-EORUN
-CONTAINERFILEOF
+just build ${build_args[@]}
+just build-integration-test-image
 
-LOCAL_IMAGE="localhost/bootc:test"
-podman build \
-    --retry 5 \
-    --retry-delay 5s \
-    -v "$(pwd)":/code:z \
-    -t "$LOCAL_IMAGE" \
-    -f "$CONTAINERFILE" \
-    "$BOOTC_TEMPDIR"
+# Host builds will have this already, but we use it as a general dumping space
+# for output artifacts
+mkdir -p target
 
-SSH_KEY=${BOOTC_TEMPDIR}/id_rsa
+SSH_KEY=$(pwd)/target/id_rsa
+rm -vf "${SSH_KEY}"*
 ssh-keygen -f "${SSH_KEY}" -N "" -q -t rsa-sha2-256 -b 2048
+chmod 600 "${SSH_KEY}"
 
-truncate -s 10G "${BOOTC_TEMPDIR}/disk.raw"
+rm -vf target/disk.raw
+truncate -s 10G "target/disk.raw"
 
 # For test-22-logically-bound-install
 podman pull --retry 5 --retry-delay 5s quay.io/curl/curl:latest
 podman pull --retry 5 --retry-delay 5s quay.io/curl/curl-base:latest
 podman pull --retry 5 --retry-delay 5s registry.access.redhat.com/ubi9/podman:latest
 
+mkdir -p target/disks
+
 podman run \
   --rm \
   --privileged \
   --pid=host \
   --security-opt label=type:unconfined_t \
   -v /var/lib/containers:/var/lib/containers \
   -v /dev:/dev \
-  -v "$BOOTC_TEMPDIR":/output \
-  "$LOCAL_IMAGE" \
+  -v $(pwd)/target:/target \
+  localhost/bootc-integration \
   bootc install to-disk \
   --filesystem "xfs" \
-  --root-ssh-authorized-keys "/output/id_rsa.pub" \
+  --root-ssh-authorized-keys "/target/id_rsa.pub" \
   --karg=console=ttyS0,115200n8 \
   --generic-image \
   --via-loopback \
-  /output/disk.raw
+  /target/disk.raw

tests/test.sh

@@ -1,45 +1,50 @@
 #!/bin/bash
 set -exuo pipefail
 
-# This script runs disk image with qemu-system and run tmt against this vm.
+# You must have invoked test/build.sh before running this.
+
+# Put ourself in a user+mount+pid namespace to close leaks
+if test -z "${test_unshared:-}"; then
+  exec unshare -m -- env test_unshared=1 "$0" "$@"
+fi
+
+TMT_PLAN_NAME=$1
+shift
 
-BOOTC_TEMPDIR="/tmp/tmp-bootc-build"
 SSH_OPTIONS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5)
-SSH_KEY=${BOOTC_TEMPDIR}/id_rsa
+SSH_KEY=$(pwd)/target/id_rsa
+test -f $SSH_KEY
 
+# TODO replace with tmt's virt provisioner
 ARCH=$(uname -m)
+qemu_args=()
 case "$ARCH" in
 "aarch64")
-  qemu-system-aarch64 \
-    -name bootc-vm \
-    -enable-kvm \
-    -machine virt \
-    -cpu host \
-    -m 2G \
-    -bios /usr/share/AAVMF/AAVMF_CODE.fd \
-    -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \
-    -net nic,model=virtio \
-    -net user,hostfwd=tcp::2222-:22 \
-    -display none \
-    -daemonize
+  qemu_args+=(qemu-system-aarch64
+    -machine virt
+    -bios /usr/share/AAVMF/AAVMF_CODE.fd)
   ;;
 "x86_64")
-  qemu-system-x86_64 \
-    -name bootc-vm \
-    -enable-kvm \
-    -cpu host \
-    -m 2G \
-    -drive file="${BOOTC_TEMPDIR}/disk.raw",if=virtio,format=raw \
-    -net nic,model=virtio \
-    -net user,hostfwd=tcp::2222-:22 \
-    -display none \
-    -daemonize
+  qemu_args+=(qemu-system-x86_64)
   ;;
 *)
   echo "Only support x86_64 and aarch64" >&2
   exit 1
   ;;
 esac
+qemu_args+=(
+    -name bootc-vm \
+    -enable-kvm \
+    -cpu host \
+    -m 2G \
+    -drive file="target/disk.raw",if=virtio,format=raw 
+    -net nic,model=virtio
+    -net user,hostfwd=tcp::2222-:22
+    -display none
+)
+
+# Kill qemu when the test exits by default
+setpriv --pdeathsig SIGTERM -- ${qemu_args[@]} &>/dev/null &
 
 wait_for_ssh_up() {
   SSH_STATUS=$(ssh "${SSH_OPTIONS[@]}" -i "$SSH_KEY" -p 2222 root@"${1}" '/bin/bash -c "echo -n READY"')
@@ -66,5 +71,15 @@ ssh "${SSH_OPTIONS[@]}" \
   root@localhost \
   "bootc status"
 
+# First a tremendous hackaround for tmt blindly rsync'ing all of .
+# including the target/ directory
+rm target/stub -rf
+mkdir -p target/stub
+ls -al $SSH_KEY
+touch -m 0600 target/stub/$(basename $SSH_KEY)
+mount --bind $SSH_KEY target/stub/$(basename $SSH_KEY)
+mount --rbind target/stub target
+ls -al "$SSH_KEY"
+
 # TMT will rsync tmt-* scripts to TMT_SCRIPTS_DIR=/var/lib/tmt/scripts
 tmt run --all --verbose -e TMT_SCRIPTS_DIR=/var/lib/tmt/scripts provision --how connect --guest localhost --port 2222 --user root --key "$SSH_KEY" plan --name "/tmt/plans/bootc-integration/${TMT_PLAN_NAME}"