Run integration tests on Amazon Linux 2

.buildkite/pipeline.yml

@@ -102,3 +102,15 @@ steps:
     command:
       - make -C examples integ-test TEST_SS=devmapper TEST_POOL=build_${BUILDKITE_BUILD_NUMBER}_example
     timeout_in_minutes: 10
+
+  - label: ":rotating_light: :running_shirt_with_sash: runtime isolated tests"
+    agents:
+      queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
+    env:
+      DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
+      EXTRAGOARGS: "-v -count=1 -race"
+    artifact_paths:
+      - "runtime/logs/*"
+    command:
+      - make -C runtime integ-test FICD_SNAPSHOTTER=devmapper FICD_DM_POOL=build_${BUILDKITE_BUILD_NUMBER}_runtime_al2 FIRECRACKER_CONTAINERD_TEST_IMAGE=localhost/firecracker-containerd-test-al2
+    timeout_in_minutes: 10

Makefile

@@ -143,7 +143,7 @@ $(INTEG_TEST_SUBDIRS): test-images
 
 test-images: test-images-stamp
 
-test-images-stamp: | image firecracker-containerd-test-image
+test-images-stamp: | image firecracker-containerd-test-image firecracker-containerd-test-image-al2
 	touch $@
 
 firecracker-containerd-test-image: all-in-docker firecracker runc test-cni-bins cni-bins default-vmlinux
@@ -153,6 +153,13 @@ firecracker-containerd-test-image: all-in-docker firecracker runc test-cni-bins
 		--build-arg FIRECRACKER_TARGET=$(FIRECRACKER_TARGET) \
 		--tag $(FIRECRACKER_CONTAINERD_TEST_IMAGE):${DOCKER_IMAGE_TAG} .
 
+firecracker-containerd-test-image-al2: all-in-docker firecracker runc test-cni-bins cni-bins default-vmlinux
+	DOCKER_BUILDKIT=1 docker build \
+		--progress=plain \
+		--file tools/docker/Dockerfile.integ-test-al2 \
+		--build-arg FIRECRACKER_TARGET=$(FIRECRACKER_TARGET) \
+		--tag $(FIRECRACKER_CONTAINERD_TEST_IMAGE)-al2:${DOCKER_IMAGE_TAG} .
+
 .PHONY: all $(SUBDIRS) clean proto deps lint install image test-images firecracker-container-test-image firecracker-containerd-integ-test-image test test-in-docker $(TEST_SUBDIRS) integ-test $(INTEG_TEST_SUBDIRS)
 
 ##########################

runtime/Makefile

@@ -44,9 +44,24 @@ install: containerd-shim-aws-firecracker
 test:
 	go test ./... $(EXTRAGOARGS)
 
-integ-test:
+integ-test: integ-test-deps
 	$(MAKE) $(addprefix integ-test-,$(INTEG_TESTNAMES))
 
+# Install dependencies in the Go cache volume
+integ-test-deps:
+	docker run --rm -it \
+		--volume $(CURDIR)/..:/src \
+		--volume $(GO_CACHE_VOLUME_NAME):/go \
+		--env GOPATH=/go \
+		--env GO111MODULES=on \
+		--env GOPROXY=direct \
+		--env GOSUMDB=off \
+		--env GODEBUG=netdns=cgo+2 \
+		--dns-opt=use-vc \
+		--workdir="/src/runtime" \
+		$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \
+		"go env; curl --verbose 'https://go.etcd.io/bbolt?go-get=1'; go test -c"
+
 integ-test-%: logs
 	$(CURDIR)/../tools/thinpool.sh reset "$(FICD_DM_POOL)"
 	docker run --rm -it \
@@ -61,12 +76,14 @@ integ-test-%: logs
 		--env ENABLE_ISOLATED_TESTS=1 \
 		--env FICD_SNAPSHOTTER=$(FICD_SNAPSHOTTER) \
 		--env FICD_DM_POOL=$(FICD_DM_POOL) \
+		--env GOPATH=/go \
+		--env GO111MODULES=on \
 		--env GOPROXY=direct \
 		--env GOSUMDB=off \
 		--workdir="/src/runtime" \
 		--init \
 		$(FIRECRACKER_CONTAINERD_TEST_IMAGE):$(DOCKER_IMAGE_TAG) \
-		"go test $(EXTRAGOARGS) -run '^$(subst integ-test-,,$@)$$'"
+		"go env; go test $(EXTRAGOARGS) -run '^$(subst integ-test-,,$@)$$'"
 
 logs:
 	mkdir logs
@@ -120,4 +137,4 @@ clean:
 distclean: clean
 	- rm -rf logs
 
-.PHONY: all runtime clean distclean install test integ-test
+.PHONY: all runtime clean distclean install test integ-test integ-test-deps

tools/docker/Dockerfile.integ-test-al2

@@ -0,0 +1,49 @@
+# syntax=docker/dockerfile:experimental
+# Test image that starts up containerd and the naive snapshotter. The default CMD will drop to a bash shell. Overrides
+# to CMD will be provided appended to /bin/bash -c
+FROM amazonlinux:2
+ENV PATH="/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/go/bin"
+ENV INSTALLROOT="/usr/local"
+ENV DEBIAN_FRONTEND="noninteractive"
+
+ARG FIRECRACKER_TARGET=x86_64-unknown-linux-musl
+ENV FICD_LOG_DIR="/var/log/firecracker-containerd-test"
+ENV FICD_SNAPSHOTTER="naive"
+ENV FICD_SNAPSHOTTER_OUTFILE="${FICD_LOG_DIR}/snapshotter.out"
+ENV FICD_CONTAINERD_OUTFILE="${FICD_LOG_DIR}/containerd.out"
+
+RUN yum install -y make e2fsprogs tar gzip gcc util-linux git
+RUN curl -L https://dl.google.com/go/go1.13.4.linux-amd64.tar.gz | tar zxvf - -C /usr/local
+RUN mkdir -p \
+    /var/run/firecracker-containerd \
+    /src \
+    /var/lib/firecracker-containerd/naive \
+    ${FICD_LOG_DIR}
+
+# Pull the images the tests need into the content store so we don't need internet
+# access during the tests themselves. This runs as a seperate step before the other
+# installs so we can minimize re-runs of the time-expensive downloading of images.
+COPY tools/docker/config.toml /etc/containerd/config.toml
+RUN --mount=type=bind,source=firecracker-control/cmd/containerd,target=/src \
+  make -C /src install && \
+  ln -sv /usr/local/bin/firecracker-containerd /usr/local/bin/containerd && \
+  ln -sv /usr/local/bin/firecracker-ctr /usr/local/bin/ctr
+RUN containerd 2>/dev/null & \
+  ctr content fetch docker.io/library/alpine:3.10.1 >/dev/null && \
+  ctr content fetch docker.io/mlabbe/iperf3:3.6-r0 >/dev/null
+
+# Install everything we need in this image. Due to the bind-mount, if the host has already
+# up-to-date versions of everything built, this step will be a very quick copy
+RUN --mount=type=bind,target=/src make -C /src \
+  install \
+  install-firecracker \
+  install-runc \
+  install-default-vmlinux \
+  install-default-rootfs \
+  install-default-runc-jailer-config \
+  install-test-cni-bins \
+  demo-network
+
+COPY tools/docker/entrypoint.sh /entrypoint
+ENTRYPOINT ["/entrypoint"]
+CMD ["exec /bin/bash"]