Update (ci): github action's artifact upgrade v4 (#20500)

* Update (ci): github action's artifact upgrade due to EOL for versions less than 4

* fix (ci:  linkcheck): ignore 403 status code from habana.ai since it redirects intel.com documentation

* fix with pattern + merge

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* use dist

* Revert "use dist"

This reverts commit ed44fec.

* retention

* ls -lh pypi/

* strategy.job-index

* stupid missing needs: build-packages

* tree pypi

* sudo

* Apply suggestions from code review

---------

Co-authored-by: Jirka B <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Jirka Borovec <[email protected]>

Author: 3 people

.github/workflows/_build-packages.yml

@@ -19,38 +19,16 @@ defaults:
     shell: bash
 
 jobs:
-  init:
-    runs-on: ubuntu-22.04
-    steps:
-      - uses: actions/checkout@v4
-      - run: |
-          mkdir dist && touch dist/.placeholder
-      - name: Keep artifact
-        id: keep-artifact
-        run: python -c "print('DAYS=' + str(5 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_OUTPUT
-      - uses: actions/upload-artifact@v3
-        with:
-          name: ${{ inputs.artifact-name }}
-          path: dist
-          retention-days: ${{ steps.keep-artifact.outputs.DAYS }}
-          include-hidden-files: true
-
   build-packages:
-    needs: init
     runs-on: ubuntu-22.04
     strategy:
-      max-parallel: 1 # run sequential to prevent download/upload collisions
       matrix:
         pkg-name: ${{ fromJSON(inputs.pkg-names) }}
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
-        with:
-          name: ${{ inputs.artifact-name }}
-          path: pypi
       - uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: "3.x"
 
       - run: python -c "print('NB_DIRS=' + str(2 if '${{ matrix.pkg-name }}' == 'pytorch' else 1))" >> $GITHUB_ENV
       - name: Build & check package
@@ -60,11 +38,33 @@ jobs:
           nb-dirs: ${{ env.NB_DIRS }}
 
       - run: |
-          mkdir pypi/${{ matrix.pkg-name }}
+          mkdir -p pypi/${{ matrix.pkg-name }}
           cp dist/* pypi/${{ matrix.pkg-name }}/
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
+        with:
+          name: ${{ inputs.artifact-name }}-${{ matrix.pkg-name }}
+          path: pypi
+          retention-days: 1
+
+  merge-artifacts:
+    needs: build-packages
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/download-artifact@v4
+        with: # download all build artifacts
+          pattern: ${{ inputs.artifact-name }}-*
+          merge-multiple: true
+          path: pypi
+      - run: |
+          sudo apt-get install -y tree
+          tree pypi
+
+      - name: Keep artifact
+        run: python -c "print('DAYS=' + str(5 if '${{ github.event_name }}'.startswith('pull_request') else 0))" >> $GITHUB_ENV
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ inputs.artifact-name }}
           path: pypi
-          include-hidden-files: true
+          retention-days: ${{ env.DAYS }}
+          if-no-files-found: error

.github/workflows/_legacy-checkpoints.yml

@@ -104,7 +104,7 @@ jobs:
           python -c "print('AWS_RUN=' + str('' if '${{inputs.push_to_s3}}' == 'true' else '--dryrun'))" >> $GITHUB_ENV
 
       - name: Upload checkpoints to GitHub Actions artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: checkpoints-${{ github.sha }}
           path: ${{ env.LEGACY_FOLDER }}/checkpoints/

.github/workflows/ci-pkg-install.yml

@@ -50,7 +50,7 @@ jobs:
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist-packages-${{ github.sha }}
           path: dist

.github/workflows/docs-build.yml

@@ -129,7 +129,7 @@ jobs:
         run: echo "ARTIFACT_DAYS=7" >> $GITHUB_ENV
       - name: Upload built docs
         if: ${{ matrix.target == 'html' }}
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: docs-${{ matrix.pkg-name }}-${{ github.sha }}
           path: docs/build/html/
@@ -158,7 +158,7 @@ jobs:
       # use input if dispatch or git tag
       VERSION: ${{ inputs.version || github.ref_name }}
     steps:
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: docs-${{ matrix.pkg-name }}-${{ github.sha }}
           path: docs/build/html/

.github/workflows/release-pkg.yml

@@ -38,7 +38,7 @@ jobs:
     if: github.event_name == 'release'
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist-packages-${{ github.sha }}
           path: dist
@@ -140,7 +140,7 @@ jobs:
         name: ["FABRIC", "PYTORCH", "LIGHTNING"]
     steps:
       - uses: actions/checkout@v4 # needed for local action below
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist-packages-${{ github.sha }}
           path: dist
@@ -165,7 +165,7 @@ jobs:
         name: ["FABRIC", "PYTORCH", "LIGHTNING"]
     steps:
       - uses: actions/checkout@v4 # needed for local action below
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: dist-packages-${{ github.sha }}
           path: dist

docs/source-pytorch/conf.py

@@ -643,6 +643,7 @@ def package_list_from_file(file):
     r"starter/installation.html$",
     r"^../common/trainer.html#trainer-flags$",
     "https://deepgenerativemodels.github.io/assets/slides/cs236_lecture11.pdf",
+    "https://developer.habana.ai", # returns 403 error but redirects to intel.com documentation
     "https://www.intel.com/content/www/us/en/products/docs/processors/what-is-a-gpu.html",
     "https://www.microsoft.com/en-us/research/blog/zero-infinity-and-deepspeed-unlocking-unprecedented-model-scale-for-deep-learning-training/",  # noqa: E501
     "https://stackoverflow.com/questions/66640705/how-can-i-install-grpcio-on-an-apple-m1-silicon-laptop",