Skip to content

Fix CI build by using pre-built rootfs image #488

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Mar 23, 2023
Merged

Fix CI build by using pre-built rootfs image #488

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 5 additions & 6 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,11 +61,9 @@ steps:
# Copy vmlinux and root-drive.img.
- ln -s /var/lib/fc-ci/vmlinux.bin ${FC_TEST_DATA_PATH}/vmlinux
- ln -s /var/lib/fc-ci/rootfs.ext4 ${FC_TEST_DATA_PATH}/root-drive.img
# Download Firecracker and its jailer.
- make deps
# Build a rootfs with SSH enabled.
- sudo -E FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH} make ${FC_TEST_DATA_PATH}/root-drive-ssh-key
Copy link
Contributor Author

@fangn2 fangn2 Mar 21, 2023 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With pre-built rootfs, we no longer need to build them with root, now they are part of deps build(check Makrfile changes below)

- sudo chown $USER ${FC_TEST_DATA_PATH}/root-drive-ssh-key ${FC_TEST_DATA_PATH}/root-drive-with-ssh.img
# Download Firecracker and its jailer
# Download pre-built rootfs with SSH enabled and its ssh key.
- make deps
agents:
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
Expand All @@ -90,6 +88,7 @@ steps:

- label: ':book: examples'
commands:
# Copy those deps from install step to save time re-downloading
- cp ${FC_TEST_DATA_PATH}/root-drive-ssh-key ${FC_TEST_DATA_PATH}/root-drive-with-ssh.img ${FC_TEST_DATA_PATH}/vmlinux examples/cmd/snapshotting
- cd examples/cmd/snapshotting
- make all
Expand All @@ -112,7 +111,7 @@ steps:
commands:
- export FC_TEST_TAP=fc-root-tap${BUILDKITE_BUILD_NUMBER}
- export FC_TEST_DATA_PATH=${FC_TEST_DATA_PATH}
- make test EXTRAGOARGS="-exec 'sudo -E' -count=1 -race" DISABLE_ROOT_TESTS=
- make test EXTRAGOARGS="-exec 'sudo -E' -v -count=1 -race" DISABLE_ROOT_TESTS=
agents:
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
Expand Down
45 changes: 20 additions & 25 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,25 +34,21 @@ firecracker_version=v1.0.0
release_url=https://github.com/firecracker-microvm/firecracker/releases/download/$(firecracker_version)/firecracker-$(firecracker_version)-$(arch).tgz

testdata_objects = \
$(FC_TEST_DATA_PATH)/vmlinux \
$(FC_TEST_DATA_PATH)/root-drive.img \
$(FC_TEST_DATA_PATH)/jailer \
$(FC_TEST_DATA_PATH)/firecracker \
$(FC_TEST_BIN_PATH)/host-local \
$(FC_TEST_DATA_PATH)/jailer \
$(FC_TEST_DATA_PATH)/ltag \
$(FC_TEST_BIN_PATH)/ptp \
$(FC_TEST_BIN_PATH)/host-local \
$(FC_TEST_DATA_PATH)/root-drive.img \
$(FC_TEST_DATA_PATH)/root-drive-with-ssh.img \
$(FC_TEST_DATA_PATH)/root-drive-ssh-key \
$(FC_TEST_BIN_PATH)/static \
$(FC_TEST_BIN_PATH)/tc-redirect-tap

# Enable pulling of artifacts from S3 instead of building
# TODO: https://github.com/firecracker-microvm/firecracker-go-sdk/issues/418
ifeq ($(GID), 0)
testdata_objects += $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img $(FC_TEST_DATA_PATH)/root-drive-ssh-key
endif
$(FC_TEST_BIN_PATH)/tc-redirect-tap \
$(FC_TEST_DATA_PATH)/vmlinux

testdata_dir = testdata/firecracker.tgz testdata/firecracker_spec-$(firecracker_version).yaml testdata/LICENSE testdata/NOTICE testdata/THIRD-PARTY

# --location is needed to follow redirects on github.com
# --location is needed to follow redirects
curl = curl --location

GO_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1,2)
Expand Down Expand Up @@ -96,26 +92,23 @@ $(FC_TEST_DATA_PATH)/vmlinux:

$(FC_TEST_DATA_PATH)/firecracker $(FC_TEST_DATA_PATH)/jailer: $(FC_TEST_DATA_PATH)/fc.stamp

# Download the pinned release version of firecracker and jailer version from github
$(FC_TEST_DATA_PATH)/fc.stamp:
$(curl) ${release_url} | tar -xvzf - -C $(FC_TEST_DATA_PATH)
mv $(FC_TEST_DATA_PATH)/release-$(firecracker_version)-$(arch)/firecracker-$(firecracker_version)-$(arch) $(FC_TEST_DATA_PATH)/firecracker
mv $(FC_TEST_DATA_PATH)/release-$(firecracker_version)-$(arch)/jailer-$(firecracker_version)-$(arch) $(FC_TEST_DATA_PATH)/jailer
rm -rf $(FC_TEST_DATA_PATH)/release-$(firecracker_version)-$(arch)
touch $@

$(FC_TEST_DATA_PATH)/root-drive.img:
$(curl) -o $@ https://s3.amazonaws.com/spec.ccfc.min/img/hello/fsfiles/hello-rootfs.ext4

# Download pre-built rootfs image and its ssh key from S3
# TODO: Change the pre-built rootfs to ubuntu 22.04 once firecracker team has it in the public S3 bucket
# Currently the S3 bucket only has ubuntu 18.04 image
$(FC_TEST_DATA_PATH)/root-drive-ssh-key $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img:
# Need root to move ssh key to testdata location
ifeq ($(GID), 0)
$(MAKE) $(FIRECRACKER_DIR)
$(FIRECRACKER_DIR)/tools/devtool build_rootfs -m $(FC_TEST_DATA_PATH)/mnt
cp $(FIRECRACKER_DIR)/build/rootfs/bionic.rootfs.ext4 $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img
cp $(FIRECRACKER_DIR)/build/rootfs/ssh/id_rsa $(FC_TEST_DATA_PATH)/root-drive-ssh-key
rm -rf $(FIRECRACKER_DIR)
else
$(error unable to place ssh key without root permissions)
endif
$(curl) -o $(FC_TEST_DATA_PATH)/root-drive-with-ssh.img https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/$(arch)/ubuntu-18.04.ext4
$(curl) -o $(FC_TEST_DATA_PATH)/root-drive-ssh-key https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/$(arch)/ubuntu-18.04.id_rsa

$(FC_TEST_BIN_PATH)/ptp:
$(call install_go,github.com/containernetworking/plugins/plugins/main/ptp,v1.1.1)
Expand All @@ -132,12 +125,14 @@ $(FC_TEST_BIN_PATH)/tc-redirect-tap:
$(FC_TEST_DATA_PATH)/ltag:
$(call install_go,github.com/kunalkushwaha/ltag,v0.2.3)

$(FIRECRACKER_DIR):
- git clone https://github.com/firecracker-microvm/firecracker.git $(FIRECRACKER_DIR)

# test-images builds firecracker and jailer from main branch of firecracker
# to test against HEAD of firecracker
.PHONY: test-images
test-images: $(FIRECRACKER_BIN) $(JAILER_BIN)

$(FIRECRACKER_DIR):
- git clone https://github.com/firecracker-microvm/firecracker.git $(FIRECRACKER_DIR)

$(FIRECRACKER_BIN) $(JAILER_BIN): $(FIRECRACKER_DIR)
$(FIRECRACKER_DIR)/tools/devtool -y build --release
cp $(FIRECRACKER_DIR)/build/cargo_target/$(FIRECRACKER_TARGET)/release/firecracker $(FIRECRACKER_BIN)
Expand Down
24 changes: 14 additions & 10 deletions examples/cmd/snapshotting/Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,9 @@ PWD=$(shell pwd)
GOBIN=$(PWD)/bin
FC_TEST_DATA_PATH?=$(PWD)

# --location is needed to follow redirects on github.com
curl = curl --location

GO_VERSION = $(shell go version | cut -c 14- | cut -d' ' -f1 | cut -d'.' -f1,2)
ifeq ($(GO_VERSION), $(filter $(GO_VERSION),1.14 1.15))
define install_go
Expand Down Expand Up @@ -50,28 +53,29 @@ bin/host-local: bin
$(call install_go,github.com/containernetworking/plugins/plugins/ipam/host-local,v1.1.1)

vmlinux:
curl --location -o vmlinux https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/${ARCH}/kernels/vmlinux.bin
$(curl) -o vmlinux https://s3.amazonaws.com/spec.ccfc.min/img/quickstart_guide/${ARCH}/kernels/vmlinux.bin

firecracker:
curl -L ${RELEASE_URL}/download/${VER}/firecracker-${VER}-${ARCH}.tgz | tar -xz
$(curl) ${RELEASE_URL}/download/${VER}/firecracker-${VER}-${ARCH}.tgz | tar -xz
mv release-${VER}-${ARCH}/firecracker-${VER}-${ARCH} firecracker
rm -rf release-${VER}-${ARCH}

# Use rootfs and ssh key pre-built from S3 following firecracker getting-started guide
# https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md
# TODO: Change the pre-built rootfs to ubuntu 22.04 once firecracker team has that in the public S3 bucket
# Currently the S3 bucket only has ubuntu 18.04 image
root-drive-with-ssh.img root-drive-ssh-key:
- mkdir temp
- git clone https://github.com/firecracker-microvm/firecracker temp
temp/tools/devtool build_rootfs
cp temp/build/rootfs/bionic.rootfs.ext4 root-drive-with-ssh.img
cp temp/build/rootfs/ssh/id_rsa root-drive-ssh-key
rm -rf temp

$(curl) -o root-drive-with-ssh.img https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/${ARCH}/ubuntu-18.04.ext4
$(curl) -o root-drive-ssh-key https://s3.amazonaws.com/spec.ccfc.min/ci-artifacts/disks/${ARCH}/ubuntu-18.04.id_rsa


snapshot-example:
go build -o $@

run: snapshot-example
./snapshot-example

clean:
rm -rf bin firecracker root-drive-ssh-key root-drive-with-ssh.img vmlinux
rm -rf bin firecracker root-drive-ssh-key root-drive-with-ssh.img snapshot-example vmlinux

.PHONY: all clean plugins run
21 changes: 3 additions & 18 deletions examples/cmd/snapshotting/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ This test requires both KVM and root access.
Run this test by first running

```
sudo -E env PATH=$PATH make all
make all
```

followed by
Expand All @@ -18,7 +18,7 @@ followed by
sudo -E env PATH=$PATH go run example_demo.go
```

Alternatively, to do both of the above,
or,
```
sudo -E env PATH=$PATH make run
```
Expand Down Expand Up @@ -62,20 +62,5 @@ Pressing enter resumes execution of the program.
You can remove dependencies via a simple `make clean`.

```
sudo make clean
```

## Issues

You may encounter an issue where the image does not build properly. This is often indicated via the following near the end of terminal output:

```
umount: /firecracker/build/rootfs/mnt: not mounted.
```

This is due to an issue in Firecracker's devtool command used to dynamically create an image. Fixing this is often as simple as rerunning the command.

```
sudo rm -rf root-drive-with-ssh.img root-drive-ssh-key
sudo make image
make clean
```
14 changes: 8 additions & 6 deletions examples/cmd/snapshotting/example_demo.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ func createNewConfig(socketPath string, opts ...configOpt) sdk.Config {
SocketPath: socketPath,
KernelImagePath: kernelImagePath,
MachineCfg: models.MachineConfiguration{
VcpuCount: &vcpuCount,
MemSizeMib: &memSizeMib,
Smt: &smt,
VcpuCount: &vcpuCount,
MemSizeMib: &memSizeMib,
Smt: &smt,
},
Drives: []models.Drive{
{
Expand Down Expand Up @@ -412,13 +412,15 @@ func main() {
defer os.Remove(tempdir)
socketPath := filepath.Join(tempdir, "snapshotssh")

err = os.Mkdir("snapshotssh", 0777)
snapshotsshPath := filepath.Join(dir, "snapshotssh")
err = os.Mkdir(snapshotsshPath, 0777)
if err != nil && !errors.Is(err, os.ErrExist) {
log.Fatal(err)
}
defer os.RemoveAll(snapshotsshPath)

snapPath := filepath.Join(dir, "snapshotssh/SnapFile")
memPath := filepath.Join(dir, "snapshotssh/MemFile")
snapPath := filepath.Join(snapshotsshPath, "SnapFile")
memPath := filepath.Join(snapshotsshPath, "MemFile")

ctx := context.Background()

Expand Down