diff --git a/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml b/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml index e89a8bb8..acca0cde 100644 --- a/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml +++ b/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml @@ -1,14 +1,6 @@ --- name: Build and publish jupyter-pyspark-with-alibi-detect -env: - IMAGE_NAME: jupyter-pyspark-with-alibi-detect - # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that - # the demo is reproducable for the release and it will be automatically replaced for the release branch. - IMAGE_VERSION: python-3.9 - REGISTRY_PATH: stackable - DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-jupyter" - on: workflow_dispatch: push: @@ -22,53 +14,14 @@ on: - .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml jobs: - build: - name: Build/Publish ${{ matrix.runner.arch }} Image - permissions: - id-token: write - runs-on: ${{ matrix.runner.name }} - strategy: - matrix: - runner: - - {name: "ubuntu-latest", arch: "amd64"} - - {name: "ubicloud-standard-8-arm", arch: "arm64"} - steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Build image - id: build - uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-name: ${{ env.IMAGE_NAME }} - image-index-manifest-tag: ${{ env.IMAGE_VERSION }} - container-file: ${{ env.DOCKERFILE_PATH }} - - - name: Publish Container Image on oci.stackable.tech - uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$stackable+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} - image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} - image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} - source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} - - publish_manifests: - name: Build/Publish Manifest - needs: [build] - permissions: - id-token: write - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Publish and Sign Image Index Manifest to oci.stackable.tech - uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$stackable+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} - image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} - image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + build_image: + name: Reusable Workflow + uses: ./.github/workflows/reusable_build_image.yaml + secrets: + harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_DEMOS_GITHUB_ACTION_BUILD_SECRET }} + with: + image-name: jupyter-pyspark-with-alibi-detect + # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that + # the demo is reproducable for the release and it will be automatically replaced for the release branch. + image-version: python-3.9 + containerfile-path: demos/signal-processing/Dockerfile-jupyter diff --git a/.github/workflows/dev_nifi.yaml b/.github/workflows/dev_nifi.yaml index efe956a4..bd2462c2 100644 --- a/.github/workflows/dev_nifi.yaml +++ b/.github/workflows/dev_nifi.yaml @@ -1,14 +1,6 @@ --- name: Build and publish NiFi for signal-processing demo -env: - IMAGE_NAME: nifi - # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that - # the demo is reproducable for the release and it will be automatically replaced for the release branch. - IMAGE_VERSION: 2.2.0-postgresql - REGISTRY_PATH: stackable - DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi" - on: workflow_dispatch: push: @@ -21,53 +13,14 @@ on: - .github/workflows/dev_nifi.yaml jobs: - build: - name: Build/Publish ${{ matrix.runner.arch }} Image - permissions: - id-token: write - runs-on: ${{ matrix.runner.name }} - strategy: - matrix: - runner: - - {name: "ubuntu-latest", arch: "amd64"} - - {name: "ubicloud-standard-8-arm", arch: "arm64"} - steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Build image - id: build - uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-name: ${{ env.IMAGE_NAME }} - image-index-manifest-tag: ${{ env.IMAGE_VERSION }} - container-file: ${{ env.DOCKERFILE_PATH }} - - - name: Publish Container Image on oci.stackable.tech - uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$stackable+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} - image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} - image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} - source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} - - publish_manifests: - name: Build/Publish Manifest - needs: [build] - permissions: - id-token: write - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Publish and Sign Image Index Manifest to oci.stackable.tech - uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$stackable+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} - image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} - image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + build_image: + name: Reusable Workflow + uses: ./.github/workflows/reusable_build_image.yaml + secrets: + harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_DEMOS_GITHUB_ACTION_BUILD_SECRET }} + with: + image-name: nifi + # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that + # the demo is reproducable for the release and it will be automatically replaced for the release branch. + image-version: 2.2.0-postgresql + containerfile-path: demos/signal-processing/Dockerfile-nifi diff --git a/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml b/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml deleted file mode 100644 index 8eb8ddaa..00000000 --- a/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml +++ /dev/null @@ -1,79 +0,0 @@ ---- -name: Build and publish spark-k8s-with-scikit-learn - -env: - IMAGE_NAME: spark-k8s-with-scikit-learn - # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that - # the demo is reproducable for the release and it will be automatically replaced for the release branch. - IMAGE_VERSION: 3.5.0-stackable24.3.0 - REGISTRY_PATH: stackable - DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile" - -on: - workflow_dispatch: - push: - branches: - - main - # TODO (@NickLarsenNZ): Also build on release branches, but with a stackable0.0.0-dev or stackableXX.X.X tag. - # - release-* - paths: - - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile - - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt - - .github/workflows/dev_spark-k8s-with-scikit-learn.yaml - -jobs: - build: - name: Build/Publish ${{ matrix.runner.arch }} Image - permissions: - id-token: write - runs-on: ${{ matrix.runner.name }} - strategy: - matrix: - runner: - - {name: "ubuntu-latest", arch: "amd64"} - # TODO: the image 3.5.0-stackable24.3.0 does not have an arm64 build. - # Re-activate the arm runner when the image is updated to one that does. - # Also adjust publish_manifest step to include arm architecture - # - {name: "ubicloud-standard-8-arm", arch: "arm64"} - steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - - name: Build image - id: build - uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-name: ${{ env.IMAGE_NAME }} - image-index-manifest-tag: ${{ env.IMAGE_VERSION }} - container-file: ${{ env.DOCKERFILE_PATH }} - - - name: Publish Container Image on oci.stackable.tech - uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$stackable+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} - image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} - image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} - source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} - - publish_manifests: - name: Build/Publish Manifest - needs: [build] - permissions: - id-token: write - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - # TODO: remove image-architecture key once arm image is also built - - name: Publish and Sign Image Index Manifest to oci.stackable.tech - uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6 - with: - image-registry-uri: oci.stackable.tech - image-registry-username: robot$stackable+github-action-build - image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }} - image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }} - image-index-manifest-tag: ${{ env.IMAGE_VERSION }} - image-architectures: '["amd64"]' diff --git a/.github/workflows/pr_pre-commit.yml b/.github/workflows/pr_pre-commit.yml index aeff8806..1e9325d4 100644 --- a/.github/workflows/pr_pre-commit.yml +++ b/.github/workflows/pr_pre-commit.yml @@ -4,16 +4,14 @@ name: pre-commit on: pull_request: +permissions: {} + jobs: pre-commit: runs-on: ubuntu-latest steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: + persist-credentials: false fetch-depth: 0 - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - with: - python-version: '3.12' - - uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1 - with: - extra_args: "--from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}" + - uses: stackabletech/actions/run-pre-commit@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0 diff --git a/.github/workflows/reusable_build_image.yaml b/.github/workflows/reusable_build_image.yaml new file mode 100644 index 00000000..3c82f7e6 --- /dev/null +++ b/.github/workflows/reusable_build_image.yaml @@ -0,0 +1,81 @@ +on: + workflow_call: + inputs: + image-name: + required: true + type: string + image-version: + required: true + type: string + containerfile-path: + required: true + type: string + secrets: + harbor-robot-secret: + description: The secret for the Harbor robot user used to push images and manifest + required: true + +jobs: + build: + name: Build/Publish ${{ inputs.image-version }}-${{ matrix.runner.arch }} Image + permissions: + id-token: write + runs-on: ${{ matrix.runner.name }} + strategy: + matrix: + runner: + - {name: "ubuntu-latest", arch: "amd64"} + - {name: "ubicloud-standard-8-arm", arch: "arm64"} + steps: + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Build image + id: build + env: + CONTAINERFILE_PATH: ${{ inputs.containerfile-path }} + IMAGE_VERSION: ${{ inputs.image-version }} + IMAGE_NAME: ${{ inputs.image-name }} + uses: stackabletech/actions/build-container-image@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0 + with: + image-name: ${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} + container-file: ${{ env.CONTAINERFILE_PATH }} + + - name: Publish Container Image on oci.stackable.tech + env: + IMAGE_NAME: ${{ inputs.image-name }} + uses: stackabletech/actions/publish-image@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$demos+github-action-build + image-registry-password: ${{ secrets.harbor-robot-secret }} + image-repository: demos/${{ env.IMAGE_NAME }} + image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }} + source-image-uri: ${{ steps.build.outputs.image-manifest-uri }} + + publish_manifests: + name: Build/Publish Manifest + needs: [build] + permissions: + id-token: write + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + + - name: Publish and Sign Image Index Manifest to oci.stackable.tech + env: + IMAGE_VERSION: ${{ inputs.image-version }} + IMAGE_NAME: ${{ inputs.image-name }} + uses: stackabletech/actions/publish-index-manifest@4bfd3b65f22af597fe784599c077dc34bf5894a7 # v0.8.0 + with: + image-registry-uri: oci.stackable.tech + image-registry-username: robot$demos+github-action-build + image-registry-password: ${{ secrets.harbor-robot-secret }} + image-repository: demos/${{ env.IMAGE_NAME }} + image-index-manifest-tag: ${{ env.IMAGE_VERSION }} diff --git a/stacks/signal-processing/jupyterhub.yaml b/stacks/signal-processing/jupyterhub.yaml index 7bec554e..9ead4039 100644 --- a/stacks/signal-processing/jupyterhub.yaml +++ b/stacks/signal-processing/jupyterhub.yaml @@ -32,7 +32,7 @@ options: image: # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that # the demo is reproducable for the release and it will be automatically replaced for the release branch. - name: oci.stackable.tech/stackable/jupyter-pyspark-with-alibi-detect + name: oci.stackable.tech/demos/jupyter-pyspark-with-alibi-detect tag: python-3.9 serviceAccountName: spark networkPolicy: diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml index 62231374..6ceda2b0 100644 --- a/stacks/signal-processing/nifi.yaml +++ b/stacks/signal-processing/nifi.yaml @@ -8,7 +8,7 @@ spec: productVersion: 2.2.0 # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that # the demo is reproducable for the release and it will be automatically replaced for the release branch. - custom: oci.stackable.tech/stackable/nifi:2.2.0-postgresql + custom: oci.stackable.tech/demos/nifi:2.2.0-postgresql # pullPolicy: IfNotPresent clusterConfig: listenerClass: external-unstable