Skip to content

Commit 017fe90

Browse files
xibzxibz
authored
Merge pull request #381 from xibz/instance-test
Adding AL2 pipeline.yml
2 parents 9870928 + 410695d commit 017fe90

File tree

14 files changed

+254
-42
lines changed

14 files changed

+254
-42
lines changed

‎.buildkite/al2_cleanup.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
source .buildkite/al2env.sh
3+
4+
sudo rm -rf $dir
5+
./tools/thinpool.sh remove $unique_id

‎.buildkite/al2_pipeline.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# Copyright 2018-2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License"). You may
4+
# not use this file except in compliance with the License. A copy of the
5+
# License is located at
6+
#
7+
# http://aws.amazon.com/apache2.0/
8+
#
9+
# or in the "license" file accompanying this file. This file is distributed
10+
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
11+
# express or implied. See the License for the specific language governing
12+
# permissions and limitations under the License.
13+
14+
steps:
15+
16+
- label: ":docker: Build"
17+
agents:
18+
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
19+
distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
20+
hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
21+
env:
22+
DOCKER_IMAGE_TAG: "$BUILDKITE_BUILD_NUMBER"
23+
EXTRAGOARGS: "-race"
24+
command:
25+
- ./.buildkite/setup_al2.sh
26+
- docker run --rm -v $PWD:/mnt debian:stretch-slim rm -rf /mnt/tools/image-builder/rootfs
27+
28+
- wait
29+
30+
- label: ":onion: al2 tests"
31+
agents:
32+
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
33+
distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
34+
hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
35+
env:
36+
NUMBER_OF_VMS: "100"
37+
command:
38+
- ./.buildkite/al2_test.sh
39+
timeout_in_minutes: 10
40+
41+
- wait: ~
42+
continue_on_failure: true
43+
44+
- label: ":onion: cleanup"
45+
agents:
46+
queue: "${BUILDKITE_AGENT_META_DATA_QUEUE:-default}"
47+
distro: "${BUILDKITE_AGENT_META_DATA_DISTRO}"
48+
hostname: "${BUILDKITE_AGENT_META_DATA_HOSTNAME}"
49+
command:
50+
- ./.buildkite/al2_cleanup.sh
51+
timeout_in_minutes: 10

‎.buildkite/al2_test.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
set -eu
3+
4+
source ./.buildkite/al2env.sh
5+
6+
export PATH=$bin_path:$PATH
7+
export FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH=$runtime_config_path
8+
export ENABLE_ISOLATED_TESTS=true
9+
export CONTAINERD_SOCKET=$dir/containerd.sock
10+
export SHIM_BASE_DIR=$dir
11+
12+
sudo -E PATH=$PATH $bin_path/firecracker-containerd --config $dir/config.toml &
13+
containerd_pid=$!
14+
sudo $bin_path/firecracker-ctr --address $dir/containerd.sock content fetch docker.io/library/alpine:3.10.1
15+
sudo -E PATH=$bin_path:$PATH /usr/local/bin/go test -count=1 -run TestMultipleVMs_Isolated ./... -v
16+
17+
sudo kill $containerd_pid

‎.buildkite/al2env.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
3+
shim_base=/tmp/shim-base
4+
unique_id=$BUILDKITE_BUILD_NUMBER
5+
dir=$shim_base/$unique_id
6+
bin_path=$dir/bin
7+
devmapper_path=$dir/devmapper
8+
state_path=$dir/state
9+
runtime_config_path=$dir/firecracker-runtime.json
10+
firecracker_bin=firecracker-v0.19.0

‎.buildkite/hooks/pre-exit‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
#!/bin/bash
2-
set -eu
2+
sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean
33

4-
make -C "$BUILDKITE_BUILD_CHECKOUT_PATH" clean
4+
# clean up ephemeral files since this will cause an error to build kite due to
5+
# these files being owned by root
6+
sudo -E PATH=$PATH make -C "$BUILDKITE_BUILD_CHECKOUT_PATH/tools/image-builder" distclean

‎.buildkite/setup_al2.sh‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
#!/bin/bash
2+
3+
set -eux
4+
5+
source ./.buildkite/al2env.sh
6+
7+
mkdir -p $dir
8+
mkdir -p $dir/rootfs
9+
mkdir -p $bin_path
10+
mkdir -p $devmapper_path
11+
mkdir -p $state_path
12+
13+
./tools/thinpool.sh reset $unique_id
14+
15+
export INSTALLROOT=$dir
16+
export FIRECRACKER_CONTAINERD_RUNTIME_DIR=$dir
17+
make
18+
sudo -E INSTALLROOT=$INSTALLROOT PATH=$PATH make install
19+
cp /var/lib/fc-ci/vmlinux.bin $dir/default-vmlinux.bin
20+
make image
21+
sudo -E PATH=$PATH make install-default-rootfs
22+
23+
cat << EOF > $dir/config.toml
24+
disabled_plugins = ["cri"]
25+
root = "$dir"
26+
state = "$state_path"
27+
[grpc]
28+
address = "$dir/containerd.sock"
29+
[plugins]
30+
[plugins.devmapper]
31+
pool_name = "fcci--vg-$unique_id"
32+
base_image_size = "10GB"
33+
root_path = "$devmapper_path"
34+
[debug]
35+
level = "debug"
36+
EOF
37+
38+
cat << EOF > $runtime_config_path
39+
{
40+
"cpu_template": "T2",
41+
"debug": true,
42+
"firecracker_binary_path": "/usr/local/bin/$firecracker_bin",
43+
"shim_base_dir": "$dir",
44+
"kernel_image_path": "$dir/default-vmlinux.bin",
45+
"kernel_args": "ro console=ttyS0 noapic reboot=k panic=1 pci=off nomodules systemd.journald.forward_to_console systemd.log_color=false systemd.unit=firecracker.target init=/sbin/overlay-init",
46+
"log_level": "DEBUG",
47+
"root_drive": "$dir/default-rootfs.img",
48+
"jailer": {
49+
"runc_binary_path": "$bin_path/runc",
50+
"runc_config_path": "$dir/config.json"
51+
}
52+
}
53+
EOF
54+
55+
cp ./runtime/firecracker-runc-config.json.example $dir/config.json
56+
cp ./_submodules/runc/runc $bin_path/runc

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ $(SUBDIRS):
5454
$(MAKE) -C $@ EXTRAGOARGS=$(EXTRAGOARGS)
5555

5656
%-in-docker:
57-
docker run --rm -it \
57+
docker run --rm \
5858
--user $(UID):$(GID) \
5959
--volume $(CURDIR):/src \
6060
--volume $(GO_CACHE_VOLUME_NAME):/go \
@@ -325,7 +325,7 @@ tools/runc-builder-stamp: tools/docker/Dockerfile.runc-builder
325325
touch $@
326326

327327
$(RUNC_BIN): $(RUNC_DIR)/VERSION tools/runc-builder-stamp
328-
docker run --rm -it --user $(UID) \
328+
docker run --rm --user $(UID) \
329329
--volume $(CURDIR)/$(RUNC_DIR):/gopath/src/github.com/opencontainers/runc \
330330
--volume $(CURDIR)/deps:/target \
331331
-e HOME=/tmp \

‎config/config.go‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,14 +25,17 @@ import (
2525
)
2626

2727
const (
28-
configPathEnvName = "FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH"
28+
// ConfigPathEnvName is the name of the environment variable used to
29+
// overwrite the default runtime config path
30+
ConfigPathEnvName = "FIRECRACKER_CONTAINERD_RUNTIME_CONFIG_PATH"
2931
defaultConfigPath = "/etc/containerd/firecracker-runtime.json"
3032
defaultKernelArgs = "console=ttyS0 noapic reboot=k panic=1 pci=off nomodules rw"
3133
defaultFilesPath = "/var/lib/firecracker-containerd/runtime/"
3234
defaultKernelPath = defaultFilesPath + "default-vmlinux.bin"
3335
defaultRootfsPath = defaultFilesPath + "default-rootfs.img"
3436
defaultCPUTemplate = models.CPUTemplateT2
3537
defaultShimBaseDir = "/var/lib/firecracker-containerd/shim-base"
38+
runcConfigPath = "/etc/containerd/firecracker-runc-config.json"
3639
)
3740

3841
// Config represents runtime configuration parameters
@@ -60,12 +63,13 @@ type Config struct {
6063
// TODO: Add netns field
6164
type JailerConfig struct {
6265
RuncBinaryPath string `json:"runc_binary_path"`
66+
RuncConfigPath string `json:"runc_config_path"`
6367
}
6468

6569
// LoadConfig loads configuration from JSON file at 'path'
6670
func LoadConfig(path string) (*Config, error) {
6771
if path == "" {
68-
path = os.Getenv(configPathEnvName)
72+
path = os.Getenv(ConfigPathEnvName)
6973
}
7074

7175
if path == "" {
@@ -83,6 +87,9 @@ func LoadConfig(path string) (*Config, error) {
8387
RootDrive: defaultRootfsPath,
8488
CPUTemplate: string(defaultCPUTemplate),
8589
ShimBaseDir: defaultShimBaseDir,
90+
JailerConfig: JailerConfig{
91+
RuncConfigPath: runcConfigPath,
92+
},
8693
}
8794

8895
if err := json.Unmarshal(data, cfg); err != nil {

‎runtime/integ_test.go‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ import (
2929
)
3030

3131
const runtimeConfigPath = "/etc/containerd/firecracker-runtime.json"
32-
const shimBaseDir = "/srv/firecracker_containerd_tests"
32+
const shimBaseDirEnvVar = "SHIM_BASE_DIR"
33+
const defaultShimBaseDir = "/srv/firecracker_containerd_tests"
3334

3435
var defaultRuntimeConfig = config.Config{
3536
FirecrackerBinaryPath: "/usr/local/bin/firecracker",
@@ -39,12 +40,21 @@ var defaultRuntimeConfig = config.Config{
3940
CPUTemplate: "T2",
4041
LogLevel: "Debug",
4142
Debug: true,
42-
ShimBaseDir: shimBaseDir,
43+
ShimBaseDir: shimBaseDir(),
4344
JailerConfig: config.JailerConfig{
4445
RuncBinaryPath: "/usr/local/bin/runc",
46+
RuncConfigPath: "/etc/containerd/firecracker-runc-config.json",
4547
},
4648
}
4749

50+
func shimBaseDir() string {
51+
if v := os.Getenv(shimBaseDirEnvVar); v != "" {
52+
return v
53+
}
54+
55+
return defaultShimBaseDir
56+
}
57+
4858
func defaultSnapshotterName() string {
4959
if name := os.Getenv("FICD_SNAPSHOTTER"); name != "" {
5060
return name

‎runtime/jailer.go‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,6 @@ const (
3434
rootfsFolder = "rootfs"
3535
)
3636

37-
var (
38-
runcConfigPath = "/etc/containerd/firecracker-runc-config.json"
39-
)
40-
4137
// jailer will allow modification and provide options to the the Firecracker VM
4238
// to allow for jailing. In addition, this will allow for given files to be exposed
4339
// to the jailed filesystem.
@@ -99,12 +95,13 @@ func newJailer(
9995

10096
l := logger.WithField("jailer", "runc")
10197
config := runcJailerConfig{
102-
OCIBundlePath: ociBundlePath,
103-
RuncBinPath: service.config.JailerConfig.RuncBinaryPath,
104-
UID: request.JailerConfig.UID,
105-
GID: request.JailerConfig.GID,
106-
CPUs: request.JailerConfig.CPUs,
107-
Mems: request.JailerConfig.Mems,
98+
OCIBundlePath: ociBundlePath,
99+
RuncBinPath: service.config.JailerConfig.RuncBinaryPath,
100+
RuncConfigPath: service.config.JailerConfig.RuncConfigPath,
101+
UID: request.JailerConfig.UID,
102+
GID: request.JailerConfig.GID,
103+
CPUs: request.JailerConfig.CPUs,
104+
Mems: request.JailerConfig.Mems,
108105
}
109106
return newRuncJailer(ctx, l, service.vmID, config)
110107
}

0 commit comments

Comments
 (0)