Merge pull request #260 from sipsma/fcsubmod

Build firecracker from a submodule.

Author: sipsma

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "runc"]
 	path = _submodules/runc
 	url = https://github.com/opencontainers/runc
+[submodule "firecracker"]
+	path = _submodules/firecracker
+	url = https://github.com/firecracker-microvm/firecracker.git

Makefile

@@ -23,9 +23,18 @@ export DOCKER_IMAGE_TAG?=latest
 GOPATH:=$(shell go env GOPATH)
 BINPATH:=$(abspath ./bin)
 SUBMODULES=_submodules
+UID:=$(shell id -u)
+
+FIRECRACKER_DIR=$(SUBMODULES)/firecracker
+FIRECRACKER_TARGET?=x86_64-unknown-linux-musl
+FIRECRACKER_BIN=$(FIRECRACKER_DIR)/target/$(FIRECRACKER_TARGET)/release/firecracker
+JAILER_BIN=$(FIRECRACKER_DIR)/target/$(FIRECRACKER_TARGET)/release/jailer
+FIRECRACKER_BUILDER_NAME?=firecracker-builder
+CARGO_CACHE_VOLUME_NAME?=cargocache
+
 RUNC_DIR=$(SUBMODULES)/runc
 RUNC_BIN=$(RUNC_DIR)/runc
-UID:=$(shell id -u)
+RUNC_BUILDER_NAME?=runc-builder
 
 # Set this to pass additional commandline flags to the go compiler, e.g. "make test EXTRAGOARGS=-v"
 export EXTRAGOARGS?=
@@ -42,11 +51,15 @@ clean:
 	for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done
 	- rm -rf $(BINPATH)/
 	$(MAKE) -C $(RUNC_DIR) clean
-	rm -f *stamp
+	$(MAKE) firecracker-clean
+	rm -f tools/*stamp
 	$(MAKE) -C tools/image-builder clean-in-docker
 
+rmi-if-exists = $(if $(shell docker images -q $(1)),docker rmi $(1),true)
 distclean: clean
-	docker rmi localhost/runc-builder:latest
+	$(call rmi-if-exists,localhost/$(RUNC_BUILDER_NAME):$(DOCKER_IMAGE_TAG))
+	$(call rmi-if-exists,localhost/$(FIRECRACKER_BUILDER_NAME):$(DOCKER_IMAGE_TAG))
+	docker volume rm -f $(CARGO_CACHE_VOLUME_NAME)
 	$(MAKE) -C tools/image-builder distclean
 
 lint:
@@ -62,6 +75,9 @@ deps:
 	GOBIN=$(BINPATH) GO111MODULE=off go get -u github.com/containerd/ttrpc/cmd/protoc-gen-gogottrpc
 	GOBIN=$(BINPATH) GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gogo
 
+install:
+	for d in $(SUBDIRS); do $(MAKE) -C $$d install; done
+
 test: $(TEST_SUBDIRS)
 
 test-in-docker: firecracker-containerd-test-image
@@ -81,51 +97,89 @@ integ-test: $(INTEG_TEST_SUBDIRS)
 $(INTEG_TEST_SUBDIRS): test-images
 	$(MAKE) -C $(patsubst integ-test-%,%,$@) integ-test
 
-runc-builder: runc-builder-stamp
-
-runc-builder-stamp: tools/docker/Dockerfile.runc-builder
-	cd tools/docker && docker build -t localhost/runc-builder:latest -f Dockerfile.runc-builder .
-	touch $@
-
-$(RUNC_DIR)/VERSION:
-	git submodule update --init --recursive $(RUNC_DIR)
-
-runc: $(RUNC_BIN)
-
-$(RUNC_BIN): $(RUNC_DIR)/VERSION runc-builder-stamp
-	docker run --rm -it --user $(UID) \
-		--volume $(PWD)/$(RUNC_DIR):/gopath/src/github.com/opencontainers/runc \
-		--volume $(PWD)/deps:/target \
-		-e HOME=/tmp \
-		-e GOPATH=/gopath \
-		--workdir /gopath/src/github.com/opencontainers/runc \
-		localhost/runc-builder:latest \
-		make static
-
 image: $(RUNC_BIN) agent
 	mkdir -p tools/image-builder/files_ephemeral/usr/local/bin
 	cp $(RUNC_BIN) tools/image-builder/files_ephemeral/usr/local/bin
 	cp agent/agent tools/image-builder/files_ephemeral/usr/local/bin
 	touch tools/image-builder/files_ephemeral
 	$(MAKE) -C tools/image-builder all-in-docker
 
-install:
-	for d in $(SUBDIRS); do $(MAKE) -C $$d install; done
-
 test-images: | firecracker-containerd-naive-integ-test-image firecracker-containerd-test-image
 
-firecracker-containerd-test-image: $(RUNC_BIN)
+firecracker-containerd-test-image:
 	DOCKER_BUILDKIT=1 docker build \
 		--progress=plain \
 		--file tools/docker/Dockerfile \
 		--target firecracker-containerd-test \
 		--tag localhost/firecracker-containerd-test:${DOCKER_IMAGE_TAG} .
 
-firecracker-containerd-naive-integ-test-image: $(RUNC_BIN)
+firecracker-containerd-naive-integ-test-image: $(RUNC_BIN) $(FIRECRACKER_BIN) $(JAILER_BIN)
 	DOCKER_BUILDKIT=1 docker build \
 		--progress=plain \
 		--file tools/docker/Dockerfile \
 		--target firecracker-containerd-naive-integ-test \
+		--build-arg FIRECRACKER_TARGET=$(FIRECRACKER_TARGET) \
 		--tag localhost/firecracker-containerd-naive-integ-test:${DOCKER_IMAGE_TAG} .
 
-.PHONY: all $(SUBDIRS) clean proto deps lint install test-images firecracker-container-test-image firecracker-containerd-naive-integ-test-image runc-builder runc test test-in-docker $(TEST_SUBDIRS) integ-test $(INTEG_TEST_SUBDIRS)
+.PHONY: all $(SUBDIRS) clean proto deps lint install test-images firecracker-container-test-image firecracker-containerd-naive-integ-test-image test test-in-docker $(TEST_SUBDIRS) integ-test $(INTEG_TEST_SUBDIRS)
+
+##########################
+# Firecracker submodule
+##########################
+.PHONY: firecracker
+firecracker: $(FIRECRACKER_BIN) $(JAILER_BIN)
+
+$(FIRECRACKER_DIR)/Cargo.toml:
+	git submodule update --init --recursive $(FIRECRACKER_DIR)
+
+tools/firecracker-builder-stamp: tools/docker/Dockerfile.firecracker-builder
+	docker build \
+		-t localhost/$(FIRECRACKER_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+		-f tools/docker/Dockerfile.firecracker-builder \
+		tools/docker
+	touch $@
+
+$(FIRECRACKER_BIN) $(JAILER_BIN): $(FIRECRACKER_DIR)/Cargo.toml tools/firecracker-builder-stamp
+	docker run --rm -it --user $(UID) \
+		--volume $(PWD)/$(FIRECRACKER_DIR):/src \
+		--volume $(CARGO_CACHE_VOLUME_NAME):/usr/local/cargo/registry \
+		-e HOME=/tmp \
+		--workdir /src \
+		localhost/$(FIRECRACKER_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+		cargo build --release --features vsock --target $(FIRECRACKER_TARGET)
+
+.PHONY: firecracker-clean
+firecracker-clean:
+	rm -f $(FIRECRACKER_BIN) $(JAILER_BIN)
+	- docker run --rm -it --user $(UID) \
+		--volume $(PWD)/$(FIRECRACKER_DIR):/src \
+		-e HOME=/tmp \
+		--workdir /src \
+		localhost/$(FIRECRACKER_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+		cargo clean
+
+##########################
+# RunC submodule
+##########################
+.PHONY: runc
+runc: $(RUNC_BIN)
+
+$(RUNC_DIR)/VERSION:
+	git submodule update --init --recursive $(RUNC_DIR)
+
+tools/runc-builder-stamp: tools/docker/Dockerfile.runc-builder
+	docker build \
+		-t localhost/$(RUNC_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+		-f tools/docker/Dockerfile.runc-builder \
+		tools/
+	touch $@
+
+$(RUNC_BIN): $(RUNC_DIR)/VERSION tools/runc-builder-stamp
+	docker run --rm -it --user $(UID) \
+		--volume $(PWD)/$(RUNC_DIR):/gopath/src/github.com/opencontainers/runc \
+		--volume $(PWD)/deps:/target \
+		-e HOME=/tmp \
+		-e GOPATH=/gopath \
+		--workdir /gopath/src/github.com/opencontainers/runc \
+		localhost/$(RUNC_BUILDER_NAME):$(DOCKER_IMAGE_TAG) \
+		make static

_submodules/firecracker

@@ -63,33 +63,6 @@ SHELL ["/bin/bash", "-c"]
 
 
 
-# Build firecracker itself
-FROM build-base as firecracker-build
-ENV RUSTUP_HOME="/home/builder/rustup" \
-	CARGO_HOME="/home/builder/cargo" \
-	PATH="/home/builder/cargo/bin:$PATH" \
-	RUST_VERSION="1.32.0"
-
-RUN curl --silent --show-error --retry 3 --max-time 30 --output rustup-init \
-	"https://static.rust-lang.org/rustup/archive/1.16.0/x86_64-unknown-linux-gnu/rustup-init" \
-	&& echo "2d4ddf4e53915a23dda722608ed24e5c3f29ea1688da55aa4e98765fc6223f71 rustup-init" | sha256sum -c - \
-	&& chmod +x rustup-init \
-	&& ./rustup-init -y --no-modify-path --default-toolchain $RUST_VERSION \
-	&& source ${CARGO_HOME}/env \
-	&& rustup target add x86_64-unknown-linux-musl
-
-RUN --mount=type=cache,from=build-base,source=/home/builder/cargo/registry,target=/home/builder/cargo/registry \
-	source ${CARGO_HOME}/env \
-	&& git clone https://github.com/firecracker-microvm/firecracker.git \
-	&& cd firecracker \
-	&& git checkout v0.17.0 \
-	&& cargo build --release --features vsock --target x86_64-unknown-linux-musl \
-	&& cp target/x86_64-unknown-linux-musl/release/firecracker /output \
-	&& cp target/x86_64-unknown-linux-musl/release/jailer /output
-
-
-
-
 # Build firecracker-containerd
 FROM build-base as firecracker-containerd-build
 ENV STATIC_AGENT='true'
@@ -160,6 +133,7 @@ RUN mkdir -p /output \
 # Derived images should include containerd/config.toml, other configuration needed to start a full
 # firecracker-containerd stack and an entrypoint that starts containerd plus one of our snapshotters.
 FROM base as firecracker-containerd-test
+
 ENV PATH="/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin:/usr/local/sbin:/usr/local/go/bin" \
 	DEBIAN_FRONTEND="noninteractive" \
 	FICD_LOG_DIR="/var/log/firecracker-containerd-test"
@@ -180,11 +154,7 @@ RUN mkdir -p /var/lib/firecracker-containerd/runtime \
 	&& mv default-vmlinux.bin /var/lib/firecracker-containerd/runtime/default-vmlinux.bin
 
 COPY --from=firecracker-containerd-build /home/builder/firecracker-containerd /firecracker-containerd
-COPY --from=firecracker-build /output/* /usr/local/bin/
-COPY --from=firecracker-vm-root-builder /output/vm.ext4 /var/lib/firecracker-containerd/runtime/default-rootfs.img
 COPY --from=firecracker-containerd-build /output/* /usr/local/bin/
-COPY _submodules/runc/runc /usr/local/bin
-COPY tools/docker/firecracker-runtime.json /etc/containerd/firecracker-runtime.json
 
 RUN --mount=type=cache,from=build-base,source=/home/builder/go/pkg/mod,target=/tmp/go/pkg/mod,readonly \
   mkdir -p ${GOPATH}/pkg/mod \
@@ -205,6 +175,14 @@ ENTRYPOINT ["/bin/bash", "-c"]
 # Test image that starts up containerd and the naive snapshotter. The default CMD will drop to a bash shell. Overrides
 # to CMD will be provided appended to /bin/bash -c
 FROM firecracker-containerd-test as firecracker-containerd-naive-integ-test
+ARG FIRECRACKER_TARGET=x86_64-unknown-linux-musl
+
+COPY _submodules/firecracker/target/$FIRECRACKER_TARGET/release/firecracker /usr/local/bin/
+COPY _submodules/firecracker/target/$FIRECRACKER_TARGET/release/jailer /usr/local/bin/
+COPY _submodules/runc/runc /usr/local/bin
+COPY --from=firecracker-vm-root-builder /output/vm.ext4 /var/lib/firecracker-containerd/runtime/default-rootfs.img
+COPY tools/docker/firecracker-runtime.json /etc/containerd/firecracker-runtime.json
+
 COPY tools/docker/naive-snapshotter/config.toml /etc/containerd/config.toml
 COPY tools/docker/naive-snapshotter/entrypoint.sh /entrypoint
 RUN mkdir -p /var/lib/firecracker-containerd/naive

tools/docker/Dockerfile

@@ -0,0 +1,25 @@
+# Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You may
+# not use this file except in compliance with the License. A copy of the
+# License is located at
+#
+# 	http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+
+FROM rust:1.32-stretch
+
+ENV DEBIAN_FRONTEND="noninteractive"
+RUN apt-get update && apt-get install --yes --no-install-recommends \
+  musl-tools
+
+RUN rustup target add x86_64-unknown-linux-musl
+
+VOLUME /src
+
+RUN mkdir --mode=0777 --parents /usr/local/cargo/registry
+VOLUME /usr/local/cargo/registry