Skip to content

ci: Fix demo image build workflows #212

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
May 8, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
69 changes: 11 additions & 58 deletions .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml
Original file line number Diff line number Diff line change
@@ -1,14 +1,6 @@
---
name: Build and publish jupyter-pyspark-with-alibi-detect

env:
IMAGE_NAME: jupyter-pyspark-with-alibi-detect
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
IMAGE_VERSION: python-3.9
REGISTRY_PATH: stackable
DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-jupyter"

on:
workflow_dispatch:
push:
Expand All @@ -22,53 +14,14 @@ on:
- .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml

jobs:
build:
name: Build/Publish ${{ matrix.runner.arch }} Image
permissions:
id-token: write
runs-on: ${{ matrix.runner.name }}
strategy:
matrix:
runner:
- {name: "ubuntu-latest", arch: "amd64"}
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
steps:
- name: Checkout Repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Build image
id: build
uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
with:
image-name: ${{ env.IMAGE_NAME }}
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
container-file: ${{ env.DOCKERFILE_PATH }}

- name: Publish Container Image on oci.stackable.tech
uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
with:
image-registry-uri: oci.stackable.tech
image-registry-username: robot$stackable+github-action-build
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}

publish_manifests:
name: Build/Publish Manifest
needs: [build]
permissions:
id-token: write
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Publish and Sign Image Index Manifest to oci.stackable.tech
uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
with:
image-registry-uri: oci.stackable.tech
image-registry-username: robot$stackable+github-action-build
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
build_image:
name: Reusable Workflow
uses: ./.github/workflows/reusable_build_image.yaml
secrets:
harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_DEMOS_GITHUB_ACTION_BUILD_SECRET }}
with:
image-name: jupyter-pyspark-with-alibi-detect
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
image-version: python-3.9
containerfile-path: demos/signal-processing/Dockerfile-jupyter
69 changes: 11 additions & 58 deletions .github/workflows/dev_nifi.yaml
Original file line number Diff line number Diff line change
@@ -1,14 +1,6 @@
---
name: Build and publish NiFi for signal-processing demo

env:
IMAGE_NAME: nifi
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
IMAGE_VERSION: 2.2.0-postgresql
REGISTRY_PATH: stackable
DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi"

on:
workflow_dispatch:
push:
Expand All @@ -21,53 +13,14 @@ on:
- .github/workflows/dev_nifi.yaml

jobs:
build:
name: Build/Publish ${{ matrix.runner.arch }} Image
permissions:
id-token: write
runs-on: ${{ matrix.runner.name }}
strategy:
matrix:
runner:
- {name: "ubuntu-latest", arch: "amd64"}
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
steps:
- name: Checkout Repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Build image
id: build
uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
with:
image-name: ${{ env.IMAGE_NAME }}
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
container-file: ${{ env.DOCKERFILE_PATH }}

- name: Publish Container Image on oci.stackable.tech
uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
with:
image-registry-uri: oci.stackable.tech
image-registry-username: robot$stackable+github-action-build
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}

publish_manifests:
name: Build/Publish Manifest
needs: [build]
permissions:
id-token: write
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7

- name: Publish and Sign Image Index Manifest to oci.stackable.tech
uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
with:
image-registry-uri: oci.stackable.tech
image-registry-username: robot$stackable+github-action-build
image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
build_image:
name: Reusable Workflow
uses: ./.github/workflows/reusable_build_image.yaml
secrets:
harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_DEMOS_GITHUB_ACTION_BUILD_SECRET }}
with:
image-name: nifi
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
image-version: 2.2.0-postgresql
containerfile-path: demos/signal-processing/Dockerfile-nifi
79 changes: 0 additions & 79 deletions .github/workflows/dev_spark-k8s-with-scikit-learn.yaml

This file was deleted.

12 changes: 5 additions & 7 deletions .github/workflows/pr_pre-commit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,14 @@ name: pre-commit
on:
pull_request:

permissions: {}

jobs:
pre-commit:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
fetch-depth: 0
- uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
with:
python-version: '3.12'
- uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
with:
extra_args: "--from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}"
- uses: stackabletech/actions/run-pre-commit@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0
81 changes: 81 additions & 0 deletions .github/workflows/reusable_build_image.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
on:
workflow_call:
inputs:
image-name:
required: true
type: string
image-version:
required: true
type: string
containerfile-path:
required: true
type: string
secrets:
harbor-robot-secret:
description: The secret for the Harbor robot user used to push images and manifest
required: true

jobs:
build:
name: Build/Publish ${{ inputs.image-version }}-${{ matrix.runner.arch }} Image
permissions:
id-token: write
runs-on: ${{ matrix.runner.name }}
strategy:
matrix:
runner:
- {name: "ubuntu-latest", arch: "amd64"}
- {name: "ubicloud-standard-8-arm", arch: "arm64"}
steps:
- name: Checkout Repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false

- name: Build image
id: build
env:
CONTAINERFILE_PATH: ${{ inputs.containerfile-path }}
IMAGE_VERSION: ${{ inputs.image-version }}
IMAGE_NAME: ${{ inputs.image-name }}
uses: stackabletech/actions/build-container-image@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0
with:
image-name: ${{ env.IMAGE_NAME }}
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
container-file: ${{ env.CONTAINERFILE_PATH }}

- name: Publish Container Image on oci.stackable.tech
env:
IMAGE_NAME: ${{ inputs.image-name }}
uses: stackabletech/actions/publish-image@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0
with:
image-registry-uri: oci.stackable.tech
image-registry-username: robot$demos+github-action-build
image-registry-password: ${{ secrets.harbor-robot-secret }}
image-repository: demos/${{ env.IMAGE_NAME }}
image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}

publish_manifests:
name: Build/Publish Manifest
needs: [build]
permissions:
id-token: write
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false

- name: Publish and Sign Image Index Manifest to oci.stackable.tech
env:
IMAGE_VERSION: ${{ inputs.image-version }}
IMAGE_NAME: ${{ inputs.image-name }}
uses: stackabletech/actions/publish-index-manifest@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0
with:
image-registry-uri: oci.stackable.tech
image-registry-username: robot$demos+github-action-build
image-registry-password: ${{ secrets.harbor-robot-secret }}
image-repository: demos/${{ env.IMAGE_NAME }}
image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
2 changes: 1 addition & 1 deletion stacks/signal-processing/jupyterhub.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ options:
image:
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
name: oci.stackable.tech/stackable/jupyter-pyspark-with-alibi-detect
name: oci.stackable.tech/demos/jupyter-pyspark-with-alibi-detect
tag: python-3.9
serviceAccountName: spark
networkPolicy:
Expand Down
2 changes: 1 addition & 1 deletion stacks/signal-processing/nifi.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ spec:
productVersion: 2.2.0
# TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
# the demo is reproducable for the release and it will be automatically replaced for the release branch.
custom: oci.stackable.tech/stackable/nifi:2.2.0-postgresql
custom: oci.stackable.tech/demos/nifi:2.2.0-postgresql
# pullPolicy: IfNotPresent
clusterConfig:
listenerClass: external-unstable
Expand Down