diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 562fbacd7..5806189b5 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -102,3 +102,15 @@ steps: command: - make -C examples integ-test TEST_SS=devmapper TEST_POOL=build_${BUILDKITE_BUILD_NUMBER}_example timeout_in_minutes: 10 + + - label: ":rotating_light: :running_shirt_with_sash: runtime isolated tests" + agents: + queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}" + env: + DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER" + EXTRAGOARGS: "-v -count=1 -race" + artifact_paths: + - "runtime/logs/*" + command: + - make -C runtime integ-test FICD_SNAPSHOTTER=devmapper FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime_al2 FIRECRACKER_CONTAINERD_TEST_IMAGE=localhost/firecracker-containerd-test-al2 + timeout_in_minutes: 10 diff --git a/Makefile b/Makefile index a2162b34b..4e8db7260 100644 --- a/Makefile +++ b/Makefile @@ -143,7 +143,7 @@ $(INTEG_TEST_SUBDIRS): test-images test-images: test-images-stamp -test-images-stamp: | image firecracker-containerd-test-image +test-images-stamp: | image firecracker-containerd-test-image firecracker-containerd-test-image-al2 touch $@ firecracker-containerd-test-image: all-in-docker firecracker runc test-cni-bins cni-bins default-vmlinux @@ -153,6 +153,13 @@ firecracker-containerd-test-image: all-in-docker firecracker runc test-cni-bins --build-arg FIRECRACKER_TARGET=$(FIRECRACKER_TARGET) \ --tag $(FIRECRACKER_CONTAINERD_TEST_IMAGE):${DOCKER_IMAGE_TAG} . +firecracker-containerd-test-image-al2: all-in-docker firecracker runc test-cni-bins cni-bins default-vmlinux + DOCKER_BUILDKIT=1 docker build \ + --progress=plain \ + --file tools/docker/Dockerfile.integ-test-al2 \ + --build-arg FIRECRACKER_TARGET=$(FIRECRACKER_TARGET) \ + --tag $(FIRECRACKER_CONTAINERD_TEST_IMAGE)-al2:${DOCKER_IMAGE_TAG} . + .PHONY: all $(SUBDIRS) clean proto deps lint install image test-images firecracker-container-test-image firecracker-containerd-integ-test-image test test-in-docker $(TEST_SUBDIRS) integ-test $(INTEG_TEST_SUBDIRS) ########################## diff --git a/runtime/Makefile b/runtime/Makefile index c3e1467a5..824d29a9c 100644 --- a/runtime/Makefile +++ b/runtime/Makefile @@ -44,9 +44,24 @@ install: containerd-shim-aws-firecracker test: go test ./... $(EXTRAGOARGS) -integ-test: +integ-test: integ-test-deps $(MAKE) $(addprefix integ-test-,$(INTEG_TESTNAMES)) +# Install dependencies in the Go cache volume +integ-test-deps: + docker run --rm -it \ + --volume $(CURDIR)/..:/src \ + --volume $(GO_CACHE_VOLUME_NAME):/go \ + --env GOPATH=/go \ + --env GO111MODULES=on \ + --env GOPROXY=direct \ + --env GOSUMDB=off \ + --env GODEBUG=netdns=cgo+2 \ + --dns-opt=use-vc \ + --workdir="/src/runtime" \ + $(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \ + "go env; curl --verbose 'https://go.etcd.io/bbolt?go-get=1'; go test -c" + integ-test-%: logs $(CURDIR)/../tools/thinpool.sh reset "$(FICD_DM_POOL)" docker run --rm -it \ @@ -61,12 +76,14 @@ integ-test-%: logs --env ENABLE_ISOLATED_TESTS=1 \ --env FICD_SNAPSHOTTER=$(FICD_SNAPSHOTTER) \ --env FICD_DM_POOL=$(FICD_DM_POOL) \ + --env GOPATH=/go \ + --env GO111MODULES=on \ --env GOPROXY=direct \ --env GOSUMDB=off \ --workdir="/src/runtime" \ --init \ $(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \ - "go test $(EXTRAGOARGS) -run '^$(subst integ-test-,,$@)$$'" + "go env; go test $(EXTRAGOARGS) -run '^$(subst integ-test-,,$@)$$'" logs: mkdir logs @@ -120,4 +137,4 @@ clean: distclean: clean - rm -rf logs -.PHONY: all runtime clean distclean install test integ-test +.PHONY: all runtime clean distclean install test integ-test integ-test-deps diff --git a/tools/docker/Dockerfile.integ-test-al2 b/tools/docker/Dockerfile.integ-test-al2 new file mode 100644 index 000000000..84240e20d --- /dev/null +++ b/tools/docker/Dockerfile.integ-test-al2 @@ -0,0 +1,49 @@ +# syntax=docker/dockerfile:experimental +# Test image that starts up containerd and the naive snapshotter. The default CMD will drop to a bash shell. Overrides +# to CMD will be provided appended to /bin/bash -c +FROM amazonlinux:2 +ENV PATH="/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/go/bin" +ENV INSTALLROOT="/usr/local" +ENV DEBIAN_FRONTEND="noninteractive" + +ARG FIRECRACKER_TARGET=x86_64-unknown-linux-musl +ENV FICD_LOG_DIR="/var/log/firecracker-containerd-test" +ENV FICD_SNAPSHOTTER="naive" +ENV FICD_SNAPSHOTTER_OUTFILE="${FICD_LOG_DIR}/snapshotter.out" +ENV FICD_CONTAINERD_OUTFILE="${FICD_LOG_DIR}/containerd.out" + +RUN yum install -y make e2fsprogs tar gzip gcc util-linux git +RUN curl -L https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz | tar zxvf - -C /usr/local +RUN mkdir -p \ + /var/run/firecracker-containerd \ + /src \ + /var/lib/firecracker-containerd/naive \ + ${FICD_LOG_DIR} + +# Pull the images the tests need into the content store so we don't need internet +# access during the tests themselves. This runs as a seperate step before the other +# installs so we can minimize re-runs of the time-expensive downloading of images. +COPY tools/docker/config.toml /etc/containerd/config.toml +RUN --mount=type=bind,source=firecracker-control/cmd/containerd,target=/src \ + make -C /src install && \ + ln -sv /usr/local/bin/firecracker-containerd /usr/local/bin/containerd && \ + ln -sv /usr/local/bin/firecracker-ctr /usr/local/bin/ctr +RUN containerd 2>/dev/null & \ + ctr content fetch docker.io/library/alpine:3.10.1 >/dev/null && \ + ctr content fetch docker.io/mlabbe/iperf3:3.6-r0 >/dev/null + +# Install everything we need in this image. Due to the bind-mount, if the host has already +# up-to-date versions of everything built, this step will be a very quick copy +RUN --mount=type=bind,target=/src make -C /src \ + install \ + install-firecracker \ + install-runc \ + install-default-vmlinux \ + install-default-rootfs \ + install-default-runc-jailer-config \ + install-test-cni-bins \ + demo-network + +COPY tools/docker/entrypoint.sh /entrypoint +ENTRYPOINT ["/entrypoint"] +CMD ["exec /bin/bash"]