Skip to content

Commit b4b4472

Browse files
consideRatioconsideRatio
committed
Makefile / GH Workflow: handle multi arch images
1 parent d759477 commit b4b4472

File tree

4 files changed

+184
-38
lines changed

4 files changed

+184
-38
lines changed

‎.github/workflows/docker.yml‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,28 +22,19 @@ on:
2222
- ".pre-commit-config.yaml"
2323

2424
jobs:
25-
build-images:
26-
name: Build Docker Images
25+
build-test-publish-images:
26+
name: Build, test, and publish Docker Images
2727
runs-on: ubuntu-latest
2828
permissions:
2929
contents: write
3030
if: >
3131
!contains(github.event.head_commit.message, 'ci skip') &&
3232
!contains(github.event.pull_request.title, 'ci skip')
33-
steps:
34-
- name: Should we push this image to a public registry?
35-
run: |
36-
if [ "${{ (github.ref == 'refs/heads/master') || (github.ref == 'refs/heads/main') }}" = "true" ]; then
37-
# Empty => Docker Hub
38-
echo "REGISTRY=" >> $GITHUB_ENV
39-
else
40-
echo "REGISTRY=localhost:5000/" >> $GITHUB_ENV
41-
fi
4233
34+
steps:
4335
# Setup docker to build for multiple platforms, see:
4436
# https://github.com/docker/build-push-action/tree/master#usage
4537
# https://github.com/docker/build-push-action/blob/master/docs/advanced/multi-platform.md
46-
4738
- name: Set up QEMU (for docker buildx)
4839
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # dependabot updates to latest release
4940

@@ -57,37 +48,50 @@ jobs:
5748
uses: actions/checkout@v2
5849
with:
5950
path: main
51+
6052
- name: Set Up Python
6153
uses: actions/setup-python@v2
6254
with:
6355
python-version: 3.x
56+
6457
- name: Install Dev Dependencies
6558
run: |
6659
python -m pip install --upgrade pip
6760
make -C main dev-env
61+
6862
- name: Build Docker Images
69-
run: make -C main build-all
63+
run: make -C main build-all-multi
7064
env:
7165
# Full logs for CI build
7266
BUILDKIT_PROGRESS: plain
67+
7368
- name: Test Docker Images
7469
run: make -C main test-all
70+
7571
- name: Clone Wiki
7672
uses: actions/checkout@v2
7773
with:
7874
repository: ${{github.repository}}.wiki
7975
path: wiki
76+
8077
- name: Run Post-Build Hooks
78+
id: hook-all
8179
run: make -C main hook-all
80+
8281
- name: Push Wiki to GitHub
8382
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
8483
uses: stefanzweifel/git-auto-commit-action@5dd17c3b53a58c1cb5eaab903826abe94765ccd6 # dependabot updates to latest release
8584
with:
8685
commit_message: "[ci skip] Automated publish for ${{github.sha}}"
8786
repository: wiki/
87+
8888
- name: Login to Docker Hub
8989
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
9090
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # dependabot updates to latest release
9191
with:
9292
username: ${{secrets.DOCKERHUB_USERNAME}}
9393
password: ${{secrets.DOCKERHUB_TOKEN}}
94+
95+
- name: Push Images to DockerHub
96+
if: github.ref == 'refs/heads/master' || github.ref == 'refs/heads/main'
97+
run: make -C main push-all-multi

‎Makefile‎

Lines changed: 97 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,23 @@ OWNER?=jupyter
88

99
# Need to list the images in build dependency order
1010

11-
# These are images we can cross-build
12-
CROSS_IMAGES:= base-notebook \
11+
# Images supporting the following architectures:
12+
# - linux/amd64
13+
# - linux/arm64
14+
MULTI_IMAGES:= \
15+
base-notebook \
1316
minimal-notebook
14-
# These images that aren't currently supported for cross-building, your help is welcome.
15-
X86_IMAGES:= r-notebook \
17+
# Images that can only be built on the amd64 architecture (aka. x86_64)
18+
AMD64_ONLY_IMAGES:= \
19+
r-notebook \
1620
scipy-notebook \
1721
tensorflow-notebook \
1822
datascience-notebook \
1923
pyspark-notebook \
2024
all-spark-notebook
2125
# All of the images
22-
ALL_IMAGES:=base-notebook \
26+
ALL_IMAGES:= \
27+
base-notebook \
2328
minimal-notebook \
2429
r-notebook \
2530
scipy-notebook \
@@ -31,42 +36,92 @@ ALL_IMAGES:=base-notebook \
3136
# Enable BuildKit for Docker build
3237
export DOCKER_BUILDKIT:=1
3338

39+
40+
3441
# https://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
3542
help:
3643
@echo "jupyter/docker-stacks"
3744
@echo "====================="
38-
@echo "Replace % with a stack directory name (e.g., make build-cross/minimal-notebook)"
45+
@echo "Replace % with a stack directory name (e.g., make build-multi/minimal-notebook)"
3946
@echo
4047
@grep -E '^[a-zA-Z0-9_%/-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
4148

42-
build-x86/%: DARGS?=
43-
build-x86/%: ## build the latest image for a stack on x86 only
44-
docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64" --push
49+
50+
51+
build/%: DARGS?=
52+
build/%: ## build the latest image for a stack on amd64 only
53+
@echo "::group::Build $(OWNER)/$(notdir $@) (amd64)"
54+
docker build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER)
4555
@echo -n "Built image size: "
4656
@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
47-
48-
build-cross/%: DARGS?=
49-
build-cross/%: ## build the latest image for a stack on x86 and ARM
50-
docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64" --push
57+
@echo "::endgroup::Build $(OWNER)/$(notdir $@) (amd64)"
58+
build-all: $(foreach I,$(ALL_IMAGES), build/$(I) ) ## build all stacks
59+
60+
# Limitations on docker buildx build (using docker/buildx 0.5.1):
61+
#
62+
# 1. Can't --load and --push at the same time
63+
#
64+
# 2. Can't --load multiple platforms
65+
#
66+
# What does it mean to --load?
67+
#
68+
# - It means that the built image can be referenced by `docker` CLI, for example
69+
# when using the `docker tag` or `docker push` commands.
70+
#
71+
# Workarounds due to limitations:
72+
#
73+
# 1. We always make a dedicated amd64 build named as OWNER/<stack>-notebook so
74+
# we always can reference that image no matter what.
75+
#
76+
# 2. We always also build a multi-platform image during build-multi that will be
77+
# inaccessible with `docker tag` and `docker push` etc, but this will help us
78+
# test the build on the different platform and provide cached layers for
79+
# later.
80+
#
81+
# 3. We let push-multi refer to rebuilding a multi image with `--push`.
82+
#
83+
# We now rely on the cached layer.
84+
#
85+
# Outcomes of the workaround:
86+
#
87+
# 1. We can keep using the previously defined Makefile commands that doesn't
88+
# include `-multi` suffix as before.
89+
#
90+
# 2. Assuming we have setup docker/dockerx properly to build in arm64
91+
# architectures as well, then we can build and publish such images via the
92+
# `-multi` suffix without needing a local registry.
93+
#
94+
build-multi/%: DARGS?=
95+
build-multi/%: ## build the latest image for a stack on amd64 and arm64
96+
@echo "::group::Build $(OWNER)/$(notdir $@) (amd64)"
97+
docker buildx build $(DARGS) --rm --force-rm -t $(OWNER)$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64" --load
5198
@echo -n "Built image size: "
5299
@docker images $(OWNER)/$(notdir $@):latest --format "{{.Size}}"
100+
@echo "::endgroup::Build $(OWNER)/$(notdir $@) (amd64)"
101+
102+
@echo "::group::Build $(OWNER)/$(notdir $@) (amd64,arm64)"
103+
docker buildx build $(DARGS) --rm --force-rm -t build-multi-tmp-cache/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
104+
@echo "::endgroup::Build $(OWNER)/$(notdir $@) (amd64,arm64)"
105+
build-all-multi: $(foreach I,$(MULTI_IMAGES), build-multi/$(I)) $(foreach I,$(AMD64_ONLY_IMAGES), build/$(I)) ## build all stacks
106+
53107

54-
build-all: $(foreach I,$(CROSS_IMAGES), build-cross/$(I) ) $(foreach I,$(X86_IMAGES), build-x86/$(I) ) ## build all stacks
55108

56109
check-outdated/%: ## check the outdated conda packages in a stack and produce a report (experimental)
57110
@TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest test/test_outdated.py
58111
check-outdated-all: $(foreach I,$(ALL_IMAGES), check-outdated/$(I) ) ## check all the stacks for outdated conda packages
59112

60-
cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)
61113

114+
115+
cont-clean-all: cont-stop-all cont-rm-all ## clean all containers (stop + rm)
62116
cont-stop-all: ## stop all containers
63117
@echo "Stopping all containers ..."
64118
-docker stop -t0 $(shell docker ps -a -q) 2> /dev/null
65-
66119
cont-rm-all: ## remove all containers
67120
@echo "Removing all containers ..."
68121
-docker rm --force $(shell docker ps -a -q) 2> /dev/null
69122

123+
124+
70125
dev/%: ARGS?=
71126
dev/%: DARGS?=-e JUPYTER_ENABLE_LAB=yes
72127
dev/%: PORT?=8888
@@ -76,49 +131,63 @@ dev/%: ## run a foreground container for a stack
76131
dev-env: ## install libraries required to build docs and run tests
77132
@pip install -r requirements-dev.txt
78133

134+
135+
79136
docs: ## build HTML documentation
80137
sphinx-build docs/ docs/_build/
81138

139+
140+
82141
hook/%: WIKI_PATH?=../wiki
83142
hook/%: ## run post-build hooks for an image
84143
python3 -m tagging.tag_image --short-image-name "$(notdir $@)" --owner "$(OWNER)" && \
85144
python3 -m tagging.create_manifests --short-image-name "$(notdir $@)" --owner "$(OWNER)" --wiki-path "$(WIKI_PATH)"
86-
87145
hook-all: $(foreach I,$(ALL_IMAGES),hook/$(I) ) ## run post-build hooks for all images
88146

89-
img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
90147

148+
149+
img-clean: img-rm-dang img-rm ## clean dangling and jupyter images
91150
img-list: ## list jupyter images
92151
@echo "Listing $(OWNER) images ..."
93152
docker images "$(OWNER)/*"
94-
95153
img-rm: ## remove jupyter images
96154
@echo "Removing $(OWNER) images ..."
97155
-docker rmi --force $(shell docker images --quiet "$(OWNER)/*") 2> /dev/null
98-
99156
img-rm-dang: ## remove dangling images (tagged None)
100157
@echo "Removing dangling images ..."
101158
-docker rmi --force $(shell docker images -f "dangling=true" -q) 2> /dev/null
102159

160+
161+
103162
pre-commit-all: ## run pre-commit hook on all files
104163
@pre-commit run --all-files || (printf "\n\n\n" && git --no-pager diff --color=always)
105-
106164
pre-commit-install: ## set up the git hook scripts
107165
@pre-commit --version
108166
@pre-commit install
109167

168+
169+
110170
pull/%: DARGS?=
111171
pull/%: ## pull a jupyter image
112172
docker pull $(DARGS) $(OWNER)/$(notdir $@)
113-
114-
pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I) ) ## pull all images
173+
pull-all: $(foreach I,$(ALL_IMAGES),pull/$(I)) ## pull all images
115174

116175
push/%: DARGS?=
117176
push/%: ## push all tags for a jupyter image
177+
@echo "::group::Push $(OWNER)/$(notdir $@) (amd64)"
118178
docker push --all-tags $(DARGS) $(OWNER)/$(notdir $@)
119-
179+
@echo "::endgroup::Push $(OWNER)/$(notdir $@) (amd64)"
120180
push-all: $(foreach I,$(ALL_IMAGES),push/$(I) ) ## push all tagged images
121181

182+
push-multi/%: DARGS?=
183+
push-multi/%: ## push all tags for a jupyter image that support multiple architectures
184+
@echo "::group::Push $(OWNER)/$(notdir $@) (amd64,arm64)"
185+
docker buildx build $(DARGS) --rm --force-rm $($(subst -,_,$(notdir $@))_EXTRA_TAG_ARGS) -t $(OWNER)/$(notdir $@):latest ./$(notdir $@) --build-arg OWNER=$(OWNER) --platform "linux/amd64,linux/arm64"
186+
@echo "::endgroup::Push $(OWNER)/$(notdir $@) (amd64,arm64)"
187+
push-all-multi: $(foreach I,$(MULTI_IMAGES),push-multi/$(I)) $(foreach I,$(AMD64_ONLY_IMAGES),push/$(I)) ## push all tagged images
188+
189+
190+
122191
run/%: DARGS?=
123192
run/%: ## run a bash in interactive mode in a stack
124193
docker run -it --rm $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
@@ -127,8 +196,11 @@ run-sudo/%: DARGS?=
127196
run-sudo/%: ## run a bash in interactive mode as root in a stack
128197
docker run -it --rm -u root $(DARGS) $(OWNER)/$(notdir $@) $(SHELL)
129198

199+
200+
130201
test/%: ## run tests against a stack (only common tests or common tests + specific tests)
202+
@echo "::group::test/$(OWNER)/$(notdir $@)"
131203
@if [ ! -d "$(notdir $@)/test" ]; then TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test; \
132204
else TEST_IMAGE="$(OWNER)/$(notdir $@)" pytest -m "not info" test $(notdir $@)/test; fi
133-
205+
@echo "::endgroup::test/$(OWNER)/$(notdir $@)"
134206
test-all: $(foreach I,$(ALL_IMAGES),test/$(I)) ## test all stacks

‎tagging/github_workflow_commands.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
"""
2+
GitHub Workflow Commands (gwc) for GitHub Actions can help us pass information
3+
from a Workflow's Job's various build steps to others via "output" and improve
4+
the presented logs when viewed via the GitHub web based UI.
5+
6+
Reference: https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions
7+
8+
Workflow commands relies on emitting messages:
9+
10+
print("::{command name} parameter1={data},parameter2={data}::{command value}")
11+
12+
The functions defined in this file will only emit such messages if found to be
13+
in a GitHub CI environment.
14+
"""
15+
16+
import json
17+
import os
18+
19+
from contextlib import contextmanager
20+
21+
22+
def _gwc(command_name, command_value="", **params):
23+
if not os.environ.get("GITHUB_ACTIONS"):
24+
return
25+
26+
# Assume non-string values are meant to be dumped as JSON
27+
if not isinstance(command_value, str):
28+
command_value = json.dumps(command_value)
29+
print(f"dumped json: {command_value}")
30+
31+
if params:
32+
comma_sep_params = ",".join([f"{k}={v}" for k, v in params.items()])
33+
print(f"::{command_name} {comma_sep_params}::{command_value}")
34+
else:
35+
print(f"::{command_name}::{command_value}")
36+
37+
38+
@contextmanager
39+
def _gwc_group(group_name):
40+
"""
41+
Entering the context prints the group command, and exiting the context
42+
prints the endgroup command.<<
43+
"""
44+
try:
45+
yield _gwc("group", group_name)
46+
finally:
47+
_gwc("endgroup", group_name)
48+
49+
50+
def _gwc_set_env(env_name, env_value):
51+
if not os.environ.get("GITHUB_ACTIONS") or not os.environ.get("GITHUB_ENV"):
52+
return
53+
54+
with open(os.environ["GITHUB_ENV"], "a") as f:
55+
f.write(f"{env_name}={env_value}\n")

‎tagging/tag_image.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,26 +6,41 @@
66
from plumbum.cmd import docker
77
from .docker_runner import DockerRunner
88
from .get_taggers_and_manifests import get_taggers_and_manifests
9+
from .github_workflow_commands import _gwc_set_env
910

1011

1112
logger = logging.getLogger(__name__)
1213

1314

1415
def tag_image(short_image_name: str, owner: str) -> None:
16+
"""
17+
Tags <owner>/<short_image_name>:latest with the tags reported by all taggers
18+
for the given image.
19+
20+
Tags are in a GitHub Actions environment also saved to environment variables
21+
in a format making it easy to append them.
22+
"""
1523
logger.info(f"Tagging image: {short_image_name}")
1624
taggers, _ = get_taggers_and_manifests(short_image_name)
1725

1826
image = f"{owner}/{short_image_name}:latest"
1927

2028
with DockerRunner(image) as container:
29+
tags = []
2130
for tagger in taggers:
2231
tagger_name = tagger.__name__
2332
tag_value = tagger.tag_value(container)
33+
tags.append(tag_value)
2434
logger.info(
2535
f"Applying tag tagger_name: {tagger_name} tag_value: {tag_value}"
2636
)
2737
docker["tag", image, f"{owner}/{short_image_name}:{tag_value}"]()
2838

39+
if tags:
40+
env_name = f'{short_image_name.replace("-", "_")}_EXTRA_TAG_ARGS'
41+
docker_build_tag_args = "-t " + " -t ".join(tags)
42+
_gwc_set_env(env_name, docker_build_tag_args)
43+
2944

3045
if __name__ == "__main__":
3146
logging.basicConfig(level=logging.INFO)

0 commit comments

Comments
 (0)