Skip to content

Commit eccda24

Browse files
johanna-reiml-hpizynaa
johanna-reiml-hpi
and
zynaa
authored
Add pytorch-notebook image variants with cuda 11 and 12 (x86_64 versions only) (#2091)
* feat: build cuda variants of pytorch * feat: build with variant tag * style: remove unused import * refactor: rename get_prefix params (cherry picked from commit 12b50af) * revert: drop ROOT_CONTAINER addition from Makefile (cherry picked from commit f423145) * style: use consistent three empty lines in Makefile (cherry picked from commit 446b45a) * refactor: add default value for parent-image (cherry picked from commit 32955ce) * revert: use original workflow structure (cherry picked from commit 68c6744) * refactor: use single build image step (cherry picked from commit 5f1ac0a) * fix: run merge tags regardless of repository owner (cherry picked from commit 3fce366) * refactor: build cuda12 instead of cuda tag (cherry picked from commit 217144e) * docs: add note about CUDA tags to documentation * refactor: add default value for variant in build-test-upload * refactor: swap ordering of cuda11/cuda12 variants * refactor: remove optional str type in arg parser * fix: add proper env variables to CUDA Dockerfiles * fix: remove CUDA build for aarch64 * fix: use latest NVIDIA documentation link * fix: skip aarch64 tags file for CUDA variants --------- Co-authored-by: zynaa <[email protected]>
1 parent fe21954 commit eccda24

File tree

15 files changed

+307
-84
lines changed

15 files changed

+307
-84
lines changed

‎.github/actions/load-image/action.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,17 +8,20 @@ inputs:
88
platform:
99
description: Image platform
1010
required: true
11+
variant:
12+
description: Variant tag prefix
13+
required: true
1114

1215
runs:
1316
using: composite
1417
steps:
1518
- name: Download built image 📥
1619
uses: actions/download-artifact@v4
1720
with:
18-
name: ${{ inputs.image }}-${{ inputs.platform }}
21+
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}
1922
path: /tmp/jupyter/images/
2023
- name: Load downloaded image to docker 📥
2124
run: |
22-
zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst | docker load
25+
zstd --uncompress --stdout --rm /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst | docker load
2326
docker image ls --all
2427
shell: bash

‎.github/workflows/docker-build-test-upload.yml‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,20 @@ on:
1111
description: Parent image name
1212
required: true
1313
type: string
14+
parent-variant:
15+
description: Parent variant tag prefix
16+
required: false
17+
type: string
18+
default: default
1419
image:
1520
description: Image name
1621
required: true
1722
type: string
23+
variant:
24+
description: Variant tag prefix
25+
required: false
26+
type: string
27+
default: default
1828
platform:
1929
description: Image platform
2030
required: true
@@ -29,6 +39,19 @@ jobs:
2939
runs-on: ${{ inputs.runs-on }}
3040

3141
steps:
42+
# Image with CUDA needs extra disk space
43+
- name: Free disk space 🧹
44+
if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64'
45+
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
46+
with:
47+
tool-cache: false
48+
android: true
49+
dotnet: true
50+
haskell: true
51+
large-packages: false
52+
docker-images: false
53+
swap-storage: false
54+
3255
- name: Checkout Repo ⚡️
3356
uses: actions/checkout@v4
3457
- name: Create dev environment 📦
@@ -52,14 +75,15 @@ jobs:
5275
with:
5376
image: ${{ inputs.parent-image }}
5477
platform: ${{ inputs.platform }}
78+
variant: ${{ inputs.parent-variant }}
5579

5680
- name: Pull ubuntu:22.04 image 📥
5781
if: inputs.parent-image == ''
5882
run: docker pull ubuntu:22.04
5983
shell: bash
6084

6185
- name: Build image 🛠
62-
run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
86+
run: docker build --rm --force-rm --tag ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} images/${{ inputs.image }}/${{ inputs.variant != 'default' && inputs.variant || '.' }}/ --build-arg REGISTRY=${{ env.REGISTRY }} --build-arg OWNER=${{ env.OWNER }}
6387
env:
6488
DOCKER_BUILDKIT: 1
6589
# Full logs for CI build
@@ -72,39 +96,39 @@ jobs:
7296

7397
- name: Write tags file 🏷
7498
run: |
75-
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
99+
python3 -m tagging.write_tags_file --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
76100
shell: bash
77101
- name: Upload tags file 💾
78102
uses: actions/upload-artifact@v4
79103
with:
80-
name: ${{ inputs.image }}-${{ inputs.platform }}-tags
81-
path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.image }}.txt
104+
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags
105+
path: /tmp/jupyter/tags/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}.txt
82106
retention-days: 3
83107

84108
- name: Write manifest and build history file 🏷
85-
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
109+
run: python3 -m tagging.write_manifest --short-image-name ${{ inputs.image }} --hist-lines-dir /tmp/jupyter/hist_lines/ --manifests-dir /tmp/jupyter/manifests/ --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }} --variant ${{ inputs.variant }}
86110
shell: bash
87111
- name: Upload manifest file 💾
88112
uses: actions/upload-artifact@v4
89113
with:
90-
name: ${{ inputs.image }}-${{ inputs.platform }}-manifest
91-
path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.image }}-*.md
114+
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-manifest
115+
path: /tmp/jupyter/manifests/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.md
92116
retention-days: 3
93117
- name: Upload build history line 💾
94118
uses: actions/upload-artifact@v4
95119
with:
96-
name: ${{ inputs.image }}-${{ inputs.platform }}-history_line
97-
path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.image }}-*.txt
120+
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-history_line
121+
path: /tmp/jupyter/hist_lines/${{ inputs.platform }}-${{ inputs.variant }}-${{ inputs.image }}-*.txt
98122
retention-days: 3
99123

100124
- name: Save image as a tar for later use 💾
101125
run: |
102126
mkdir -p /tmp/jupyter/images/
103-
docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst
127+
docker save ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }} | zstd > /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst
104128
shell: bash
105129
- name: Upload image as artifact 💾
106130
uses: actions/upload-artifact@v4
107131
with:
108-
name: ${{ inputs.image }}-${{ inputs.platform }}
109-
path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}.tar.zst
132+
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}
133+
path: /tmp/jupyter/images/${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}.tar.zst
110134
retention-days: 3

‎.github/workflows/docker-merge-tags.yml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ env:
77
on:
88
workflow_call:
99
inputs:
10+
variant:
11+
description: Variant tag prefix
12+
required: true
13+
type: string
1014
image:
1115
description: Image name
1216
required: true
@@ -30,13 +34,14 @@ jobs:
3034
- name: Download x86_64 tags file 📥
3135
uses: actions/download-artifact@v4
3236
with:
33-
name: ${{ inputs.image }}-x86_64-tags
37+
name: ${{ inputs.image }}-x86_64-${{ inputs.variant }}-tags
3438
path: /tmp/jupyter/tags/
3539
- name: Download aarch64 tags file 📥
3640
uses: actions/download-artifact@v4
3741
with:
38-
name: ${{ inputs.image }}-aarch64-tags
42+
name: ${{ inputs.image }}-aarch64-${{ inputs.variant }}-tags
3943
path: /tmp/jupyter/tags/
44+
if: github.repository_owner == 'jupyter' && !contains(inputs.variant, 'cuda')
4045

4146
# Docker might be stuck when pulling images
4247
# https://github.com/docker/for-mac/issues/2083
@@ -57,5 +62,5 @@ jobs:
5762

5863
- name: Merge tags for the images 🔀
5964
if: env.PUSH_TO_REGISTRY == 'true'
60-
run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/
65+
run: python3 -m tagging.merge_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --variant ${{ inputs.variant }}
6166
shell: bash

‎.github/workflows/docker-tag-push.yml‎

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@ on:
1616
description: Image platform
1717
required: true
1818
type: string
19+
variant:
20+
description: Variant tag prefix
21+
required: true
22+
type: string
1923
secrets:
2024
REGISTRY_USERNAME:
2125
required: true
@@ -27,6 +31,19 @@ jobs:
2731
runs-on: ubuntu-latest
2832

2933
steps:
34+
# Image with CUDA needs extra disk space
35+
- name: Free disk space 🧹
36+
if: contains(inputs.variant, 'cuda') && inputs.platform == 'x86_64'
37+
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be
38+
with:
39+
tool-cache: false
40+
android: true
41+
dotnet: true
42+
haskell: true
43+
large-packages: false
44+
docker-images: false
45+
swap-storage: false
46+
3047
- name: Checkout Repo ⚡️
3148
uses: actions/checkout@v4
3249
- name: Create dev environment 📦
@@ -36,6 +53,7 @@ jobs:
3653
with:
3754
image: ${{ inputs.image }}
3855
platform: ${{ inputs.platform }}
56+
variant: ${{ inputs.variant }}
3957

4058
- name: Login to Registry 🔐
4159
if: env.PUSH_TO_REGISTRY == 'true'
@@ -48,10 +66,10 @@ jobs:
4866
- name: Download tags file 📥
4967
uses: actions/download-artifact@v4
5068
with:
51-
name: ${{ inputs.image }}-${{ inputs.platform }}-tags
69+
name: ${{ inputs.image }}-${{ inputs.platform }}-${{ inputs.variant }}-tags
5270
path: /tmp/jupyter/tags/
5371
- name: Apply tags to the loaded image 🏷
54-
run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
72+
run: python3 -m tagging.apply_tags --short-image-name ${{ inputs.image }} --tags-dir /tmp/jupyter/tags/ --platform ${{ inputs.platform }} --variant ${{ inputs.variant }} --registry ${{ env.REGISTRY }} --owner ${{ env.OWNER }}
5573
# This step is needed to prevent pushing non-multi-arch "latest" tag
5674
- name: Remove the "latest" tag from the image 🗑️
5775
run: docker image rmi ${{ env.REGISTRY }}/${{ env.OWNER }}/${{ inputs.image }}:latest

0 commit comments

Comments
 (0)