diff --git a/.github/workflows/smoke-build.yaml b/.github/workflows/smoke-build.yaml index 2c8b7f0..b5987f1 100644 --- a/.github/workflows/smoke-build.yaml +++ b/.github/workflows/smoke-build.yaml @@ -28,7 +28,7 @@ jobs: uses: ./shard with: product-name: smoke - config-file: smoke/conf.py + boil-version: 0.1.0 outputs: versions: ${{ steps.shard.outputs.versions }} @@ -41,8 +41,10 @@ jobs: strategy: matrix: runner: + # Github's ubuntu-latest runners use 24.04. By default Ubicloud's runners use 22.04. + # We therefore explicitly request 24.04 for them as well. - {name: "ubuntu-latest", arch: "amd64"} - - {name: "ubicloud-standard-8-arm", arch: "arm64"} + - {name: "ubicloud-standard-8-arm-ubuntu-2404", arch: "arm64"} versions: ${{ fromJson(needs.generate_matrix.outputs.versions) }} steps: - name: Checkout Repository @@ -57,10 +59,12 @@ jobs: id: build uses: ./build-product-image with: - product-name: smoke product-version: ${{ matrix.versions }} - bake-config-file: smoke/conf.py + boil-config-file: smoke/boil.toml + boil-version: 0.1.0 + registry-namespace: stackable extra-tag-data: pr-321 + product-name: smoke - name: Publish Container Image on oci.stackable.tech uses: ./publish-image @@ -70,7 +74,7 @@ jobs: image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} image-repository: stackable/smoke image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} - source-image-uri: localhost/smoke:${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: localhost/stackable/smoke:${{ steps.build.outputs.image-manifest-tag }} publish_manifests: name: Build/Publish ${{ matrix.versions }} Index Manifest diff --git a/.scripts/actions/shard.py b/.scripts/actions/shard.py deleted file mode 100644 index 88cfe0e..0000000 --- a/.scripts/actions/shard.py +++ /dev/null @@ -1,31 +0,0 @@ -import importlib.util -import sys -import os - -sys.path.append(str(os.getcwd())) - -def import_from_path(module_name, file_path): - spec = importlib.util.spec_from_file_location(module_name, file_path) - module = importlib.util.module_from_spec(spec) - sys.modules[module_name] = module - spec.loader.exec_module(module) - return module - -# import smoke.conf as conf -file = os.environ['CONFIG_FILE'] if os.environ['CONFIG_FILE'] else 'conf' -conf = import_from_path('conf', file) - -product=os.environ['PRODUCT_NAME'] -print(f"Generating version list for {product}") - -# get the product config -product_conf = list(filter(lambda x: x["name"] == product, conf.products))[0] -# list the versions, eg: [1.0, 1.1, 2.0] -versions = [v["product"] for k,v in enumerate(product_conf["versions"])] -output_versions = f"VERSIONS={versions}\n" - -github_outputs_file = os.environ['GITHUB_OUTPUT'] -f = open(github_outputs_file, "w") -print(f"Writing to $GITHUB_OUTPUT: {output_versions}") -f.write(output_versions) -f.close() diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 0000000..76364d7 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,9 @@ +# Release + +To release a new set of actions follow these steps: + +- Choose an appropriate new semantic version based on the latest changes +- Update any references in actions to other local actions to the chosen version +- Raise a PR with the appropriate changes +- Once merged, tag on `main` using `git tag -s v` +- Manually create a GitHub release diff --git a/build-product-image/README.md b/build-product-image/README.md index 1b1098e..bd43527 100644 --- a/build-product-image/README.md +++ b/build-product-image/README.md @@ -8,27 +8,17 @@ > a name. If the name does not contain a registry, `hub.docker.com` (?) is implied. Therefore, > `localhost` will be used as the registry so as to avoid accidental interactions with an unintended > registry. -> -> Ideally, bake should be refactored to use `localhost` as the registry for the previously mentioned -> reason (whether or not that is behind some option). -This action builds a *single* container image using `bake`. It does the following work: +This action builds a *single* container image using `boil`. It does the following work: 1. Free disk space to avoid running out of disk space during larger builds. -2. Build the image using `bake` which internally uses `docker buildx`. -3. Temporarily retag the image to use `localhost` instead of `oci.stackable.tech/sdp`. -4. Produce output values to be used in later steps. +2. Build the image using `boil` which internally uses `docker buildx`. +3. Produce output values to be used in later steps. This action is considered to be the **single** source of truth regarding image index tag and image manifest tag. All subsequent tasks must use these values to ensure consistency. -Currently, bake provides the following ouput in the `bake-target-tags` file: - -```plain -oci.stackable.tech/sdp/kafka:3.4.1-stackable0.0.0-dev-amd64 -``` - -Until bake supports the ability to specify the registry, this action will retag the image as: +boil provides the following output in the `boil-target-tags` file: ```plain localhost/kafka:3.4.1-stackable0.0.0-dev-amd64 @@ -41,16 +31,21 @@ localhost/kafka:3.4.1-stackable0.0.0-dev-amd64 ### Inputs -- `product-name` (eg: `kafka`) -- `product-version` (eg: `3.4.1`) -- `image-tools-version` (eg: `0.0.14`) -- `bake-config-file` (defaults to `./conf.py`) -- `sdp-version` (defaults to: `0.0.0-dev`) -- `extra-tag-data` (optional, eg. `pr321`) +| Input | Required (Default) | Description | +| -------------------- | ------------------ | ------------------------------------------------------------ | +| `product-name` | Yes | The name of the product image to build (eg: `kafka`) | +| `product-version` | Yes | The version of the product image to build (eg: `3.4.1`) | +| `registry-namespace` | Yes | The path of the registry namespace, eg. `sdp` or `stackable` | +| `boil-version` | No (`latest`) | The version of boil used to build the image | +| `boil-config-file` | No (`./boil.toml`) | The path to the boil config file | +| `sdp-version` | No (`0.0.0-dev`) | The SDP version of the image | +| `extra-tag-data` | No | Extra data to be included in the tag, eg. `pr321` | ### Outputs -- `image-manifest-tag` (eg: `3.4.1-stackable0.0.0-dev-amd64`) -- `suggested-image-index-manifest-tag` (eg: `3.4.1-stackable0.0.0-dev`) +| Output | Example | Description | +| ------------------------------------ | -------------------------------- | ----------------------------------------------- | +| `image-manifest-tag` | `3.4.1-stackable0.0.0-dev-amd64` | The image manifest tag (including architecture) | +| `suggested-image-index-manifest-tag` | `3.4.1-stackable0.0.0-dev` | The suggested image index manifest tag | [build-product-image]: ./action.yaml diff --git a/build-product-image/action.yaml b/build-product-image/action.yaml index 336cf67..1b59654 100644 --- a/build-product-image/action.yaml +++ b/build-product-image/action.yaml @@ -1,19 +1,19 @@ --- name: Build Product Image -description: This action builds a product Docker image with a specific version +description: This action builds a product container image with a specific version inputs: product-name: - description: The name of the product to build via bake (directory name) + description: The name of the product to build via boil (directory name) required: true product-version: - description: The version of the product to build via bake + description: The version of the product to build via boil required: true - image-tools-version: - description: The Stackable image-tools version - default: 0.0.17 - bake-config-file: - description: Path to the bake config file, defaults to `./conf.py` - default: ./conf.py + boil-version: + description: The boil version + default: latest + boil-config-file: + description: Path to the boil config file, defaults to `./boil.toml` + default: ./boil.toml sdp-version: description: | Stackable Data Platform version (eg: `24.7.0`) @@ -21,6 +21,9 @@ inputs: extra-tag-data: description: | Extra data to include in the final image manifest tag (eg: `pr321`) + registry-namespace: + description: Path of the registry namespace, eg. `sdp` or `stackable` + required: true outputs: image-manifest-tag: description: | @@ -38,22 +41,18 @@ runs: - name: Setup Docker Buildx uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1 - # NOTE (@Techassi): Why do we install python via apt and not the setup-python action? - - name: Setup Python - shell: bash - run: | - set -euo pipefail - sudo apt update - sudo apt install -y python3 + - name: Setup boil (${{ inputs.boil-version }}) + # Using other "local" actions is super painful. As such, a follow-up PR will make this way + # more comfortable again. + uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version + with: + boil-version: ${{ inputs.boil-version }} - - name: Install image-tools-stackabletech - shell: bash - run: pip install image-tools-stackabletech==${{ inputs.image-tools-version }} - - - name: Building ${{ inputs.product-name }} Container Image using bake + - name: Building ${{ inputs.product-name }} Container Image using boil env: - BAKE_PRODUCT_VERSION: ${{ inputs.product-version }} - BAKE_CONFIG_FILE: ${{ inputs.bake-config-file }} + REGISTRY_NAMESPACE: ${{ inputs.registry-namespace }} + BOIL_PRODUCT_VERSION: ${{ inputs.product-version }} + BOIL_CONFIG_FILE: ${{ inputs.boil-config-file }} IMAGE_REPOSITORY: ${{ inputs.product-name }} EXTRA_TAG_DATA: ${{ inputs.extra-tag-data }} SDP_VERSION: ${{ inputs.sdp-version }} @@ -63,9 +62,9 @@ runs: IMAGE_ARCH=$("$GITHUB_ACTION_PATH/../.scripts/actions/get_architecture.sh") # Will be either: - # - 3.9.2-stackable0.0.0-dev-arm64 or - # - 3.9.2-stackable0.0.0-dev-pr321-arm64 - IMAGE_INDEX_MANIFEST_TAG="${SDP_VERSION}${EXTRA_TAG_DATA:+-$EXTRA_TAG_DATA}-${IMAGE_ARCH}" + # - 3.9.2-stackable0.0.0-dev or + # - 3.9.2-stackable0.0.0-dev-pr321 + IMAGE_INDEX_MANIFEST_TAG="${SDP_VERSION}${EXTRA_TAG_DATA:+-$EXTRA_TAG_DATA}" echo "IMAGE_INDEX_MANIFEST_TAG=$IMAGE_INDEX_MANIFEST_TAG" | tee -a "$GITHUB_ENV" # Validate that final tag is valid according to @@ -75,40 +74,27 @@ runs: exit 1 fi - echo "::group::bake" - bake \ - --product "$IMAGE_REPOSITORY=$BAKE_PRODUCT_VERSION" \ + echo "::group::boil" + boil build \ --image-version "$IMAGE_INDEX_MANIFEST_TAG" \ - --release "$SDP_VERSION" \ - --architecture "linux/${IMAGE_ARCH}" \ - --export-tags-file bake-target-tags \ - --configuration "$BAKE_CONFIG_FILE" + --registry-namespace "$REGISTRY_NAMESPACE" \ + --target-platform "linux/${IMAGE_ARCH}" \ + --configuration "$BOIL_CONFIG_FILE" \ + --write-image-manifest-uris \ + --use-localhost-registry \ + --load \ + "$IMAGE_REPOSITORY=$BOIL_PRODUCT_VERSION" echo "::endgroup::" - - name: Re-tag Image (Temporary) - shell: bash - run: | - set -euo pipefail - - # Extract the image uri and replace 'oci.stackable.tech/sdp' - # with 'localhost' until bake does the right thing - OLD_IMAGE_URI="$(< bake-target-tags)" - - # Replace the image uri in the bake file - sed -i -e 's/oci\.stackable\.tech\/sdp/localhost/' bake-target-tags - - # Finally, re-tag image - docker tag "$OLD_IMAGE_URI" "$(< bake-target-tags)" - - name: Extract Environment Variables id: image_info shell: bash run: | set -euo pipefail - echo "bake-target-tags: "$(< bake-target-tags) + echo "boil-target-tags: "$(< boil-target-tags) - # Extract the image manifest tag from the bake-target-tags file - IMAGE_MANIFEST_TAG=$(cut -d : -f 2 < bake-target-tags) + # Extract the image manifest tag from the boil-target-tags file + IMAGE_MANIFEST_TAG=$(cut -d : -f 2 < boil-target-tags) [[ -n "$IMAGE_MANIFEST_TAG" ]] [[ -n "$IMAGE_INDEX_MANIFEST_TAG" ]] diff --git a/publish-helm-chart/action.yaml b/publish-helm-chart/action.yaml index bebf051..5f61ffe 100644 --- a/publish-helm-chart/action.yaml +++ b/publish-helm-chart/action.yaml @@ -36,7 +36,9 @@ runs: - name: Set up Helm # We cannot use a path to point to the local action # See https://github.com/orgs/community/discussions/74934 - uses: stackabletech/actions/setup-k8s-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag + # Using other "local" actions is super painful. As such, a follow-up PR will make this way + # more comfortable again. + uses: stackabletech/actions/setup-k8s-tools@main # TODO: Replace with latest released version with: helm-version: ${{ inputs.helm-version }} diff --git a/run-integration-test/action.yaml b/run-integration-test/action.yaml index a886fb9..71219dc 100644 --- a/run-integration-test/action.yaml +++ b/run-integration-test/action.yaml @@ -60,7 +60,9 @@ runs: using: composite steps: - name: Install interu - uses: stackabletech/actions/setup-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag + # Using other "local" actions is super painful. As such, a follow-up PR will make this way + # more comfortable again. + uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version with: interu-version: ${{ inputs.interu-version }} - name: Extract Test and Instance Configuration @@ -97,7 +99,9 @@ runs: # and are therefore not available, there is no need to create the cluster or run the tests, # because the tests can never run in the first place. - name: Install kubectl, kubectl-kuttl, and helm - uses: stackablectl/actions/setup-k8s-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag + # Using other "local" actions is super painful. As such, a follow-up PR will make this way + # more comfortable again. + uses: stackablectl/actions/setup-k8s-tools@main # TODO: Replace with latest released version with: kubectl-version: ${{ inputs.kubectl-version }} kuttl-version: ${{ inputs.kuttl-version }} @@ -110,7 +114,9 @@ runs: # mikefarah/yq is already installed on the runner # See https://github.com/actions/runner-images/blob/main/images/ubuntu/scripts/build/install-yq.sh - name: Install stackablectl and beku - uses: stackabletech/actions/setup-tools@7279ffd852a8013df52028de0d99eef68700f803 # TODO: Pin this to the latest tag + # Using other "local" actions is super painful. As such, a follow-up PR will make this way + # more comfortable again. + uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version with: stackablectl-version: ${{ inputs.stackablectl-version }} beku-version: ${{ inputs.beku-version }} diff --git a/setup-tools/action.yaml b/setup-tools/action.yaml index 284db5d..5077b8c 100644 --- a/setup-tools/action.yaml +++ b/setup-tools/action.yaml @@ -25,7 +25,12 @@ runs: ARCH=$(uname -m) - curl -fsSL -o /tmp/boil "https://github.com/stackabletech/docker-images/releases/download/boil-${BOIL_VERSION}/boil-${ARCH}-unknown-linux-gnu" + if [ "$BOIL_VERSION" == "latest" ]; then + curl -fsSL -o /tmp/boil "https://github.com/stackabletech/docker-images/releases/latest/download/boil-${ARCH}-unknown-linux-gnu" + else + curl -fsSL -o /tmp/boil "https://github.com/stackabletech/docker-images/releases/download/boil-${BOIL_VERSION}/boil-${ARCH}-unknown-linux-gnu" + fi + sudo install -m 755 -t /usr/local/bin /tmp/boil - name: Install stackablectl @@ -40,7 +45,12 @@ runs: ARCH=$(uname -m) - curl -fsSL -o /tmp/stackablectl "https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-${STACKABLECTL_VERSION}/stackablectl-${ARCH}-unknown-linux-gnu" + if [ "$STACKABLECTL_VERSION" == "latest" ]; then + curl -fsSL -o /tmp/stackablectl "https://github.com/stackabletech/stackable-cockpit/releases/latest/download/stackablectl-${ARCH}-unknown-linux-gnu" + else + curl -fsSL -o /tmp/stackablectl "https://github.com/stackabletech/stackable-cockpit/releases/download/stackablectl-${STACKABLECTL_VERSION}/stackablectl-${ARCH}-unknown-linux-gnu" + fi + sudo install -m 755 -t /usr/local/bin /tmp/stackablectl - name: Install interu @@ -55,7 +65,12 @@ runs: ARCH=$(uname -m) - curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/download/interu-${INTERU_VERSION}/interu-${ARCH}-unknown-linux-gnu" + if [ "$INTERU_VERSION" == "latest" ]; then + curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/latest/download/interu-${ARCH}-unknown-linux-gnu" + else + curl -fsSL -o /tmp/interu "https://github.com/stackabletech/actions/releases/download/interu-${INTERU_VERSION}/interu-${ARCH}-unknown-linux-gnu" + fi + sudo install -m 755 -t /usr/local/bin /tmp/interu - name: Install beku diff --git a/shard/README.md b/shard/README.md index 77ef992..a9c83bb 100644 --- a/shard/README.md +++ b/shard/README.md @@ -2,11 +2,11 @@ > Manifest: [shard/action.yml][shard] -This action produces a list of versions for a product. This is to be used as a matrix dimension to +This action produces a list of versions for an image. This is to be used as a matrix dimension to parallelize builds. It does the following work: -1. Reads the `conf.py`, filtering versions for the product -2. Write the JSON array of version to `$GITHUB_OUTPUT` for use in a matrix. +1. Reads the `/boil-config.toml` file, extracting the versions for the image +2. Write the JSON array of versions to `$GITHUB_OUTPUT` for use in a matrix. Example usage: @@ -20,7 +20,7 @@ jobs: - id: shard uses: stackabletech/actions/shard with: - product-name: ${{ env.PRODUCT_NAME }} + image-name: ${{ env.IMAGE_NAME }} outputs: versions: ${{ steps.shard.outputs.versions }} @@ -39,10 +39,15 @@ jobs: ### Inputs -- `product-name` (eg: `kafka`) +| Input | Required (Default) | Description | +| -------------- | ------------------ | ------------------------------------------------------- | +| `image-name` | Yes | The name of the image, eg: `kafka` | +| `boil-version` | No (`latest`) | The version of boil used to create the list of versions | ### Outputs -- `versions` (eg: `["3.7.1", "3.8.0"]`) +| Output | Example | Description | +| ---------- | -------------------- | ------------------------------------------ | +| `versions` | `["3.7.1", "3.8.0"]` | A JSON array containing the image versions | [shard]: ./action.yaml diff --git a/shard/action.yaml b/shard/action.yaml index b129516..c2314cd 100644 --- a/shard/action.yaml +++ b/shard/action.yaml @@ -5,9 +5,9 @@ inputs: product-name: description: The name of the product to build via bake (directory name) required: true - config-file: - description: Path the the config file used to generate the shard indices - default: ./conf.py + boil-version: + description: The boil version + default: latest outputs: versions: description: A list of product versions @@ -15,30 +15,31 @@ outputs: runs: using: composite steps: - - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + # Using other "local" actions is super painful. As such, a follow-up PR will make this way + # more comfortable again. + - uses: stackabletech/actions/setup-tools@main # TODO: Replace with latest released version with: - python-version: '3.12' - - - name: Extract Action Path - shell: bash - env: - GITHUB_ACTION_PATH: ${{ github.action_path }} - run: | - set -euo pipefail - echo "GITHUB_ACTION_PATH=$GITHUB_ACTION_PATH" | tee -a "$GITHUB_ENV" + boil-version: ${{ inputs.boil-version }} - name: Generate Shards id: generate_shards - shell: bash env: PRODUCT_NAME: ${{ inputs.product-name }} - CONFIG_FILE: ${{ inputs.config-file }} + GITHUB_DEBUG: ${{ runner.debug }} + shell: bash run: | set -euo pipefail - python "$GITHUB_ACTION_PATH/../.scripts/actions/shard.py" + [ -n "$GITHUB_DEBUG" ] && set -x + + VERSIONS=$(boil show images "$PRODUCT_NAME" | jq --compact-output ".$PRODUCT_NAME") + echo "VERSIONS=$VERSIONS" | tee -a "$GITHUB_OUTPUT" - name: Print Shards + env: + GITHUB_DEBUG: ${{ runner.debug }} shell: bash run: | set -euo pipefail + [ -n "$GITHUB_DEBUG" ] && set -x + echo versions=${{ steps.generate_shards.outputs.VERSIONS }} diff --git a/smoke/boil-config.toml b/smoke/boil-config.toml new file mode 100644 index 0000000..c3bdc6b --- /dev/null +++ b/smoke/boil-config.toml @@ -0,0 +1,2 @@ +[versions."0.0.1"] +[versions."0.0.2"] diff --git a/smoke/boil.toml b/smoke/boil.toml new file mode 100644 index 0000000..5a74845 --- /dev/null +++ b/smoke/boil.toml @@ -0,0 +1,8 @@ +[metadata] +documentation = "https://docs.stackable.tech/home/stable/" +source = "https://github.com/stackabletech/actions/" +authors = "Stackable GmbH " +vendor = "Stackable GmbH" +licenses = "Apache-2.0" + +[build-arguments] diff --git a/smoke/conf.py b/smoke/conf.py deleted file mode 100644 index 572b999..0000000 --- a/smoke/conf.py +++ /dev/null @@ -1,18 +0,0 @@ -import smoke.versions as smoke - -products = [ - { - "name": "smoke", - "versions": smoke.versions, - }, -] - -cache = [ - { - "type": "registry", - "ref_prefix": "build-repo.stackable.tech:8083/sandbox/cache", - "mode": "max", - "compression": "zstd", - "ignore-error": "true", - }, -] diff --git a/smoke/versions.py b/smoke/versions.py deleted file mode 100644 index 4ef767e..0000000 --- a/smoke/versions.py +++ /dev/null @@ -1,8 +0,0 @@ -versions = [ - { - "product": "0.0.1", - }, - { - "product": "0.0.2", - }, -]