Skip to content

Commit 153005d

Browse files
adamnovakadamnovak
authored
Drop external Docker builder (#4523)
* Use a vg-like system toa void needing a Kubernetes docker builder * Specify inline caching correctly * Drop extra import * Fix the wandering typedef two ways * Fix it only one way MyPy can't handle a Pythonic try-except import: python/mypy#1393 We could put # type: ignore[attr-defined] except then whichever branch actually passed on the currently installed version of the module would fail due to having an apparently-unneeded ignore. * Manually force out busted stubs package
1 parent 0efed3f commit 153005d

File tree

7 files changed

+77
-20
lines changed

7 files changed

+77
-20
lines changed

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,4 @@ tmp/
2222
.coverage*
2323
.mypy_cache
2424
/contrib/admin/.issue_cache/
25+
.docker_cache/

‎.gitlab-ci.yml

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@ before_script:
1717
echo "{\"registry-mirrors\": [\"${DOCKER_HUB_MIRROR}\"], \"insecure-registries\": [\"${DOCKER_HUB_MIRROR##*://}\"]}" | sudo tee /etc/docker/daemon.json
1818
export SINGULARITY_DOCKER_HUB_MIRROR="${DOCKER_HUB_MIRROR}"
1919
fi
20+
# Restart or start the Docker daemon
21+
- stopdocker || true
2022
- startdocker || true
2123
- docker info
2224
- cat /etc/hosts
@@ -26,9 +28,9 @@ before_script:
2628
# a service account bearer token for auth and triggers https://github.com/docker/buildx/issues/267
2729
# where buildx can't use a bearer token from a kube config and falls back to anonymous instead
2830
# of using the system's service account.
29-
- KUBECONFIG=/dev/null docker buildx create --use --name toilbuilder --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64"
30-
# Dump the builder info, and make sure it exists.
31-
- docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found in our Kubernetes namespace! Are we on the right Gitlab runner?" && exit 1)
31+
- if [[ "${CI_BUILDKIT_DRIVER}" == "kubernetes" ]] ; then KUBECONFIG=/dev/null docker buildx create --use --name=buildkit --platform=linux/amd64,linux/arm64 --node=buildkit-amd64 --driver=kubernetes --driver-opt="nodeselector=kubernetes.io/arch=amd64" ; else docker buildx create --use --name=container-builder --driver=docker-container ; fi
32+
# Report on the builders, and make sure they exist.
33+
- docker buildx inspect --bootstrap || (echo "Docker builder deployment can't be found! Are we on the right Gitlab runner?" && exit 1)
3234
# This will hang if we can't talk to the builder
3335
- (echo "y" | docker buildx prune --keep-storage 80G) || true
3436

‎Makefile

Lines changed: 42 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -166,24 +166,28 @@ pre_pull_docker:
166166
for i in $$(seq 1 11); do if [[ $$i == "11" ]] ; then exit 1 ; fi ; docker pull sscaling/mtail && break || sleep 60; done
167167

168168
toil_docker: pre_pull_docker docker/Dockerfile
169+
mkdir -p .docker_cache
169170
@set -ex \
170171
; cd docker \
171-
; docker buildx build --platform=$(arch) --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
172+
; docker buildx build --platform=$(arch) --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(docker_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(docker_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../.docker-cache/toil --cache-to type=local,dest=../.docker-cache/toil -f Dockerfile .
172173

173174
prometheus_docker: pre_pull_docker
175+
mkdir -p .docker_cache
174176
@set -ex \
175177
; cd dashboard/prometheus \
176-
; docker buildx build --platform=$(arch) --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
178+
; docker buildx build --platform=$(arch) --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(prometheus_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(prometheus_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/prometheus --cache-to type=local,dest=../../.docker-cache/prometheus -f Dockerfile .
177179

178180
grafana_docker: pre_pull_docker
181+
mkdir -p .docker_cache
179182
@set -ex \
180183
; cd dashboard/grafana \
181-
; docker buildx build --platform=$(arch) --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
184+
; docker buildx build --platform=$(arch) --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(grafana_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(grafana_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/grafana --cache-to type=local,dest=../../.docker-cache/grafana -f Dockerfile .
182185

183186
mtail_docker: pre_pull_docker
187+
mkdir -p .docker_cache
184188
@set -ex \
185189
; cd dashboard/mtail \
186-
; docker buildx build --platform=$(arch) --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
190+
; docker buildx build --platform=$(arch) --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=registry,ref=$(mtail_image):$(TOIL_DOCKER_MAIN_CACHE_TAG) --cache-from type=registry,ref=$(mtail_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/mtail --cache-to type=local,dest=../../.docker-cache/mtail -f Dockerfile .
187191

188192
docker/$(sdist_name): dist/$(sdist_name)
189193
cp $< $@
@@ -196,17 +200,42 @@ clean_docker:
196200
-docker rmi $(docker_image):$(TOIL_DOCKER_TAG)
197201

198202
push_docker: docker
199-
# Weird if logic is so we fail if all the pushes fail
200-
cd docker ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
201-
cd dashboard/prometheus ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
202-
cd dashboard/grafana ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
203-
cd dashboard/mtail ; for i in $$(seq 1 6); do if [[ $$i == "6" ]] ; then exit 1 ; fi ; docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile . && break || sleep 60; done
203+
# Weird if logic is so we fail if all the pushes fail.
204+
# We need to build from the local cache to the cache tag and again from the local cache to the real tag.
205+
cd docker ; \
206+
for i in $$(seq 1 6); do \
207+
if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
208+
docker buildx build --platform $(arch) --push --tag=$(docker_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../.docker-cache/toil --cache-to type=inline -f Dockerfile . && \
209+
docker buildx build --platform $(arch) --push --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../.docker-cache/toil -f Dockerfile . && \
210+
break || sleep 60; \
211+
done
212+
cd dashboard/prometheus ; \
213+
for i in $$(seq 1 6); do \
214+
if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
215+
docker buildx build --platform $(arch) --push --tag=$(prometheus_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/prometheus --cache-to type=inline -f Dockerfile . && \
216+
docker buildx build --platform $(arch) --push --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile . && \
217+
break || sleep 60; \
218+
done
219+
cd dashboard/grafana ; \
220+
for i in $$(seq 1 6); do \
221+
if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
222+
docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/grafana --cache-to type=inline -f Dockerfile . && \
223+
docker buildx build --platform $(arch) --push --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/grafana -f Dockerfile . && \
224+
break || sleep 60; \
225+
done
226+
cd dashboard/mtail ; \
227+
for i in $$(seq 1 6); do \
228+
if [[ $$i == "6" ]] ; then exit 1 ; fi ; \
229+
docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_CACHE_TAG) --cache-from type=local,src=../../.docker-cache/mtail --cache-to type=inline -f Dockerfile . && \
230+
docker buildx build --platform $(arch) --push --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/mtail -f Dockerfile . && \
231+
break || sleep 60; \
232+
done
204233

205234
load_docker: docker
206-
cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
207-
cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
208-
cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
209-
cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) -f Dockerfile .
235+
cd docker ; docker buildx build --platform $(arch) --load --tag=$(docker_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../.docker-cache/toil -f Dockerfile .
236+
cd dashboard/prometheus ; docker buildx build --platform $(arch) --load --tag=$(prometheus_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/prometheus -f Dockerfile .
237+
cd dashboard/grafana ; docker buildx build --platform $(arch) --load --tag=$(grafana_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/grafana -f Dockerfile .
238+
cd dashboard/mtail ; docker buildx build --platform $(arch) --load --tag=$(mtail_image):$(TOIL_DOCKER_TAG) --cache-from type=local,src=../../.docker-cache/mtail -f Dockerfile .
210239

211240
else
212241

‎common.mk

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,8 @@ SHELL=bash
1515
export TOIL_DOCKER_REGISTRY?=quay.io/ucsc_cgl
1616
export TOIL_DOCKER_NAME?=toil
1717
export TOIL_DOCKER_TAG?=$(shell python version_template.py dockerTag)
18+
export TOIL_DOCKER_CACHE_TAG?=$(shell python version_template.py cacheTag)
19+
export TOIL_DOCKER_MAIN_CACHE_TAG?=$(shell python version_template.py mainCacheTag)
1820
export TOIL_APPLIANCE_SELF?=$(TOIL_DOCKER_REGISTRY)/$(TOIL_DOCKER_NAME):$(TOIL_DOCKER_TAG)
1921

2022
# TOIL_CHECK_ENV='' # Determines whether toil refers to the same virtualenv paths it spawned from (across machines)

‎requirements-aws.txt

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
boto>=2.48.0, <3
2-
boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3, <2
3-
moto>=4.1.11, <5
2+
boto3-stubs[s3,sdb,iam,sts,boto3]>=1.28.3.post2, <2
3+
mypy-boto3-iam>=1.28.3.post2, <2 # Need to force .post1 to be replaced
4+
moto>=4.1.11, <5

‎src/toil/lib/aws/iam.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
import boto3
99
from mypy_boto3_iam import IAMClient
10-
from mypy_boto3_iam.type_defs import AttachedPolicyOutputTypeDef
10+
from mypy_boto3_iam.type_defs import AttachedPolicyTypeDef
1111
from mypy_boto3_sts import STSClient
1212

1313
from toil.lib.aws import zone_to_region
@@ -145,7 +145,7 @@ def get_actions_from_policy_document(policy_doc: Dict[str, Any]) -> AllowedActio
145145
allowed_actions[resource][key].append(statement[key])
146146

147147
return allowed_actions
148-
def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPolicyOutputTypeDef]) -> AllowedActionCollection:
148+
def allowed_actions_attached(iam: IAMClient, attached_policies: List[AttachedPolicyTypeDef]) -> AllowedActionCollection:
149149
"""
150150
Go through all attached policy documents and create an AllowedActionCollection representing granted permissions.
151151

‎version_template.py

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,28 @@ def version():
3939
"""
4040
return '-'.join(filter(None, [distVersion(), currentCommit(), ('dirty' if dirty() else None)]))
4141

42+
def cacheTag():
43+
"""
44+
A Docker tag that we should use to cache Docker image build layers for this commit.
45+
"""
46+
47+
import os
48+
return ''.join([
49+
"cache-",
50+
# Pick up branch or tag from Gitlagb CI, or just use "local" for everyone.
51+
((os.getenv('CI_COMMIT_BRANCH', '') + os.getenv('CI_COMMIT_TAG', '')) or 'local').replace('/', '-'),
52+
_pythonVersionSuffix()
53+
])
54+
55+
def mainCacheTag():
56+
"""
57+
A Docker tag where the Toil mainline builds cache their layers.
58+
"""
59+
60+
return ''.join([
61+
"cache-master",
62+
_pythonVersionSuffix()
63+
])
4264

4365
def distVersion():
4466
"""The distribution version identifying a published release on PyPI."""

0 commit comments

Comments
 (0)