diff --git a/.github/workflows/aarch64-darwin.yml b/.github/workflows/aarch64-darwin.yml new file mode 100644 index 0000000..a48d9f4 --- /dev/null +++ b/.github/workflows/aarch64-darwin.yml @@ -0,0 +1,11 @@ +name: DevX closures for aarch64-darwin + +on: + push: + +jobs: + build: + name: Prebuild and Upload + uses: ./.github/workflows/main.yml + with: + platform: aarch64-darwin diff --git a/.github/workflows/aarch64-linux.yml b/.github/workflows/aarch64-linux.yml new file mode 100644 index 0000000..1db5c8d --- /dev/null +++ b/.github/workflows/aarch64-linux.yml @@ -0,0 +1,11 @@ +name: DevX closures for aarch64-linux + +on: + push: + +jobs: + build: + name: Prebuild and Upload + uses: ./.github/workflows/main.yml + with: + platform: aarch64-linux diff --git a/.github/workflows/hello.yml b/.github/workflows/hello.yml index a1f7368..d20cee1 100644 --- a/.github/workflows/hello.yml +++ b/.github/workflows/hello.yml @@ -1,4 +1,5 @@ name: Test that we can build the hello world example form hackage + on: push: workflow_dispatch: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3d12f48..36bd19c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,10 +1,11 @@ name: Prebuild DevX closures -on: - push: - branches: - - main - workflow_dispatch: +on: + workflow_call: + inputs: + platform: + required: true + type: string env: REGISTRY: ghcr.io @@ -36,49 +37,28 @@ jobs: # Group the output by platform. RUNS=$(gh api "repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs" --paginate) echo "checks..." - FILTERED=$(jq -c -r '.check_runs[] | select(.name | endswith("-env")) | { "config": .name, "build_path": .output.text }' <<< "$RUNS") - echo "filtered..." - MATRIX=$(jq --slurp -c -r '. as $jobs | map(.config | split(".") | first) | unique | map(. as $group | {"platform": $group, "jobs": $jobs | map(select(.config |startswith($group)))})' <<< "$FILTERED") - echo "groups..." - if jq -e . <<< "$MATRIX" > /dev/null 2>&1; then # JSON validation - echo "Valid JSON" - else - echo "Invalid JSON: $MATRIX" - exit 1 # or handle the error appropriately - fi + FILTERED=$(jq -c -r '.check_runs[] | select(.name | endswith("-env")) | select(.name | startswith("${{ inputs.platform }}")) | { "config": .name, "build_path": .output.text, "short_name": .name | sub("${{ inputs.platform }}\\.";"") }' <<< "$RUNS") + jq . <<< "$FILTERED" + MATRIX=$(jq --slurp -c -r '.' <<< "$FILTERED") + jq . <<< "$MATRIX" echo "creating result matrix." echo "matrix=$MATRIX" >> $GITHUB_OUTPUT # We need this process step in here, because we have in excess of 256 jobs. process: needs: discover - runs-on: ubuntu-latest # You still need runs-on to define a job, but it can be minimal - strategy: - fail-fast: false - matrix: - group: ${{ fromJson(needs.discover.outputs.matrix) }} - name: Process Jobs for ${{ matrix.group.platform }} - outputs: - jobs: ${{ toJson(matrix.group.jobs) }} # Direct output definition - steps: - - name: No-op step (required to have at least one step) - run: echo "No-op" # Minimal step to satisfy step requirement - - ghcr-upload: - needs: process - name: Container Upload + runs-on: ubuntu-latest strategy: fail-fast: false matrix: - job: ${{ fromJson(needs.process.outputs.matrix) }} - runs-on: ubuntu-latest + job: ${{ fromJson(needs.discover.outputs.matrix) }} + name: GHCR Upload for ${{ matrix.job.short_name }} permissions: contents: read packages: write steps: - name: Free Disk Space (Ubuntu) uses: jlumbroso/free-disk-space@main - - name: Install Nix with good defaults uses: cachix/install-nix-action@v20 with: @@ -101,6 +81,8 @@ jobs: NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }} run: ./extra/ghcr-upload.sh + + # codespace-upload: # env: # IMAGE_NAME: input-output-hk/devx-devcontainer diff --git a/.github/workflows/wait-and-upload.yml b/.github/workflows/wait-and-upload.yml deleted file mode 100644 index c95f669..0000000 --- a/.github/workflows/wait-and-upload.yml +++ /dev/null @@ -1,147 +0,0 @@ -name: Wait and Upload - -on: - workflow_call: - inputs: - platform: - required: true - description: 'build platform' - type: string - target-platform: - required: true - description: 'target platform' - type: string - compiler-nix-name: - required: true - description: 'compiler name in nix format. e.g. ghc96' - type: string - variant: - description: 'if set to `minimal` without hls, hlint, ...' - type: string - default: '' - iog: - description: 'if set to `-iog` with libs `libsodium`, `libsecp256k1`, `libblst`, ... and if set to `-iog-full` adds `postgresql` and `R`' - type: string - default: '' - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - DEV_SHELL: ${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}${{ inputs.variant }}${{ inputs.iog }}-env - DEFAULT_TAG: latest - GH_TOKEN: ${{ github.token }} - -jobs: - wait-for-hydra: - name: "Wait for hydra status" - runs-on: ubuntu-latest - steps: - - name: Get specific check run status - run: | - # start with a random sleep to prevent hitting the api too hard. - while true; do - # For GitHub Apps - # conclusion=$(gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/status --jq '.check_runs[] | select(.name == "ci/hydra-build:$DEV_SHELL") | .conclusion') - # For GitHub Statuses; we need --paginate because there are so many statuses - echo "gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs --paginate --jq '.check_runs[] | select(.name == \"ci/hydra-build:$DEV_SHELL\") | .conclusion'" - conclusion=$(gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs --paginate --jq ".check_runs[] | select(.name == \"ci/hydra-build:$DEV_SHELL\") | .conclusion") - case "$conclusion" in - success) - echo "ci/hydra-build:$DEV_SHELL succeeded" - exit 0;; - failure) - echo "ci/hydra-build:$DEV_SHELL failed" - exit 1;; - *) - echo "conclusion is: '$conclusion'" - gh api repos/$GITHUB_REPOSITORY/commits/$GITHUB_SHA/check-runs --paginate --jq '.check_runs[] | .conclusion+"\t"+.name'|sort - WAIT=$((180 + RANDOM % 180)) - echo "ci/hydra-build:$DEV_SHELL pending. Waiting ${WAIT}s..." - sleep $WAIT;; - esac - done - - ghcr-upload: - needs: wait-for-hydra - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - - - name: Install Nix with good defaults - uses: cachix/install-nix-action@v20 - with: - extra_nix_config: | - trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk= - substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/ - nix_path: nixpkgs=channel:nixos-unstable - - name: Checkout repository - uses: actions/checkout@v4 - - name: Log in to the Container registry - uses: docker/login-action@v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Work around issue fetching cabal revision files - run: | - # This will cause the revised `.cabal` files used whan building alex - # for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`. - # Later when we want the non `x86_64-linux` version later it will find the - # fixed output of the derivation is already in the `/nix/store` and will - # not try to fetch it using the platform we do not have a builder for. - if [[ "${{ inputs.platform }}" != "x86_64-linux" && "${{ inputs.target-platform }}" = "-js" ]]; then - nix build ".#hydraJobs.x86_64-linux.${{ inputs.compiler-nix-name }}-js-minimal" --show-trace - fi - - name: Compute and upload closure and developer environment to ghcr.io - env: - NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }} - run: ./extra/ghcr-upload.sh - - codespace-upload: - env: - IMAGE_NAME: input-output-hk/devx-devcontainer - needs: ghcr-upload - permissions: - packages: write - runs-on: ubuntu-latest - # We want a GitHub Codespace image for each combination of devx developer shell option. - # But, since the purpose of GitHub Codespace is to serve a complete development environment, - # the user is likely to always expect HLS (I don't see the point otherwise). - # Therefore, it doesn't seem useful to build an image on the `-minimal` flavor (without HLS), - # or the `-static` one (especially since the latter currently requires `-minimal` to work). - # Likely, we consider using `-iog` as the default and do not generate other images. - # Then the user choices left would be between native, `-windows` or `-js` target platforms, - # and the GHC version (currently `ghc810` and `ghc96`). - if: ${{ contains(fromJSON('["x86_64-linux", "aarch64-linux"]'), inputs.platform) && contains(fromJson('["","-windows","-js"]'), inputs.target-platform) && contains(fromJson('["ghc810","ghc96"]'), inputs.compiler-nix-name) && inputs.variant == '' && inputs.iog == '-iog' }} - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v2.1.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push Docker image - uses: docker/build-push-action@v4 - with: - context: . - push: true - tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}${{ inputs.variant }}${{ inputs.iog }} - build-args: | - PLATFORM=${{ inputs.platform }} - TARGET_PLATFORM=${{ inputs.target-platform }} - COMPILER_NIX_NAME=${{ inputs.compiler-nix-name }} - VARIANT=${{ inputs.variant }} - IOG=${{ inputs.iog }} - - - name: Run test command inside the Dev Container - run: | - docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ inputs.platform }}.${{ inputs.compiler-nix-name }}${{ inputs.target-platform }}${{ inputs.variant }}${{ inputs.iog }} \ - bash -ic "cabal update && cabal unpack hello && cd hello-* && cabal build" diff --git a/.github/workflows/x86_64-darwin.yml b/.github/workflows/x86_64-darwin.yml new file mode 100644 index 0000000..ef6ace9 --- /dev/null +++ b/.github/workflows/x86_64-darwin.yml @@ -0,0 +1,11 @@ +name: DevX closures for x86_64-darwin + +on: + push: + +jobs: + build: + name: Prebuild and Upload + uses: ./.github/workflows/main.yml + with: + platform: x86_64-darwin diff --git a/.github/workflows/x86_64-linux.yml b/.github/workflows/x86_64-linux.yml new file mode 100644 index 0000000..9ffdaf8 --- /dev/null +++ b/.github/workflows/x86_64-linux.yml @@ -0,0 +1,11 @@ +name: DevX closures for x86_64-linux + +on: + push: + +jobs: + build: + name: Prebuild and Upload + uses: ./.github/workflows/main.yml + with: + platform: x86_64-linux diff --git a/extra/ghcr-upload.sh b/extra/ghcr-upload.sh index 9570eac..f942fbb 100755 --- a/extra/ghcr-upload.sh +++ b/extra/ghcr-upload.sh @@ -5,7 +5,8 @@ set -euox pipefail : "${DEV_SHELL:?'DEV_SHELL is not set'}" : "${SHELL_NIX_PATH:?'SHELL_NIX_PATH is not set'}" -nix-store -r "$SHELL_NIX_PATH" +# retry three times in case of intermittent network errors. +nix-store -r "$SHELL_NIX_PATH" || nix-store -r "$SHELL_NIX_PATH" || nix-store -r "$SHELL_NIX_PATH" #nix build ".#hydraJobs.${DEV_SHELL}" --show-trace --accept-flake-config nix-store --export $(nix-store -qR "$SHELL_NIX_PATH") | tee store-paths.txt | zstd -z8T8 >${DEV_SHELL} if [[ ! $(tail -n 1 store-paths.txt) =~ "devx" ]]; then exit 1; fi