diff --git a/proto/firecracker.pb.go b/proto/firecracker.pb.go index 8c92c26c2..723e77c83 100644 --- a/proto/firecracker.pb.go +++ b/proto/firecracker.pb.go @@ -533,7 +533,13 @@ func (m *GetVMMetadataResponse) GetMetadata() string { } type JailerConfig struct { - NetNS string `protobuf:"bytes,1,opt,name=NetNS,json=netNS,proto3" json:"NetNS,omitempty"` + NetNS string `protobuf:"bytes,1,opt,name=NetNS,json=netNS,proto3" json:"NetNS,omitempty"` + // The list of CPUs to run the jailed application on. This field can only + // be used on numa architectures + CPUs string `protobuf:"bytes,2,opt,name=CPUs,json=cPUs,proto3" json:"CPUs,omitempty"` + // The list of memory nodes to run with the jailed application on. This + // field can only be used on numa architectures + Mems string `protobuf:"bytes,3,opt,name=Mems,json=mems,proto3" json:"Mems,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -570,6 +576,20 @@ func (m *JailerConfig) GetNetNS() string { return "" } +func (m *JailerConfig) GetCPUs() string { + if m != nil { + return m.CPUs + } + return "" +} + +func (m *JailerConfig) GetMems() string { + if m != nil { + return m.Mems + } + return "" +} + func init() { proto.RegisterType((*CreateVMRequest)(nil), "CreateVMRequest") proto.RegisterType((*CreateVMResponse)(nil), "CreateVMResponse") @@ -586,40 +606,41 @@ func init() { func init() { proto.RegisterFile("firecracker.proto", fileDescriptor_a73317e9fb8da571) } var fileDescriptor_a73317e9fb8da571 = []byte{ - // 550 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x54, 0xc1, 0x6e, 0xd3, 0x40, - 0x10, 0x95, 0xdb, 0xb8, 0x4d, 0xc6, 0x6d, 0xd2, 0xac, 0x5a, 0x58, 0x55, 0xa8, 0xb2, 0x2c, 0x54, - 0x59, 0x1c, 0x2a, 0xd1, 0x5e, 0x10, 0x27, 0x4a, 0x42, 0x2b, 0xb7, 0xb8, 0x42, 0x4e, 0xe9, 0x81, - 0xdb, 0xe2, 0x8c, 0x5d, 0x63, 0x67, 0xd7, 0xec, 0x6e, 0x0a, 0x9c, 0xf9, 0x23, 0xf8, 0x41, 0xe4, - 0x4d, 0xea, 0x38, 0x11, 0x54, 0x48, 0x9c, 0x38, 0x65, 0xe7, 0xcd, 0x9b, 0x99, 0xa7, 0x99, 0x17, - 0x43, 0x3f, 0xc9, 0x24, 0xc6, 0x92, 0xc5, 0x39, 0xca, 0xa3, 0x52, 0x0a, 0x2d, 0xf6, 0x1d, 0xfd, - 0xad, 0x44, 0x35, 0x0b, 0xbc, 0xef, 0x2d, 0xe8, 0x0d, 0x24, 0x32, 0x8d, 0x37, 0x61, 0x84, 0x9f, - 0xa7, 0xa8, 0x34, 0x21, 0xd0, 0xba, 0x09, 0x83, 0x21, 0xb5, 0x5c, 0xcb, 0xef, 0x44, 0xad, 0xbb, - 0x30, 0x18, 0x92, 0x57, 0x00, 0x21, 0x8b, 0x6f, 0x33, 0x8e, 0x83, 0x24, 0xa5, 0x6b, 0xae, 0xe5, - 0x3b, 0xc7, 0xee, 0xd1, 0xd9, 0xa2, 0xf9, 0x7d, 0x56, 0xf0, 0x24, 0x4b, 0xa7, 0x92, 0xe9, 0x4c, - 0xf0, 0x08, 0x26, 0x75, 0x0d, 0xf1, 0xa1, 0x77, 0x89, 0x92, 0x63, 0x11, 0x4c, 0x58, 0x8a, 0xef, - 0x98, 0xbe, 0xa5, 0xeb, 0x66, 0x40, 0x2f, 0x5f, 0x86, 0xc9, 0x01, 0xc0, 0x8c, 0x79, 0x2a, 0x53, - 0x45, 0x5b, 0x86, 0x04, 0x79, 0x8d, 0x90, 0x13, 0xe8, 0x44, 0x42, 0xe8, 0xa1, 0xcc, 0xee, 0x90, - 0xda, 0x46, 0xca, 0x5e, 0x53, 0x4a, 0x9d, 0x8c, 0x3a, 0xf2, 0xfe, 0x49, 0x5e, 0x80, 0x63, 0x1e, - 0xa1, 0x98, 0x72, 0xad, 0xe8, 0x86, 0xbb, 0xee, 0x3b, 0xc7, 0x8f, 0x9a, 0x65, 0x8b, 0x74, 0xe4, - 0x8c, 0x17, 0x54, 0x72, 0x01, 0xfd, 0x2b, 0xd4, 0x5f, 0x84, 0xcc, 0x03, 0xae, 0x51, 0x26, 0x2c, - 0x46, 0x45, 0x37, 0x4d, 0xfd, 0x93, 0x66, 0xfd, 0x2a, 0x29, 0xea, 0xf3, 0xd5, 0x32, 0x72, 0x08, - 0xdd, 0x81, 0xe0, 0x9a, 0x65, 0x1c, 0xe5, 0xa0, 0x6a, 0x4f, 0xdb, 0xae, 0xe5, 0xdb, 0x51, 0x37, - 0x5e, 0x42, 0xc9, 0x4b, 0xa0, 0x6f, 0xbe, 0x66, 0xfa, 0x34, 0xd1, 0x28, 0x4f, 0x8b, 0xe2, 0x9a, - 0xa9, 0x5c, 0x0d, 0xb1, 0x40, 0x8d, 0x63, 0xda, 0x71, 0x2d, 0xbf, 0x1d, 0x51, 0xfc, 0x43, 0x9e, - 0x3c, 0x87, 0xad, 0x0b, 0x96, 0x15, 0x55, 0xab, 0xea, 0x16, 0x14, 0xcc, 0x86, 0xb6, 0x8f, 0x9a, - 0x60, 0xb4, 0xf5, 0xa9, 0x11, 0x79, 0x3f, 0x2c, 0xd8, 0x59, 0xb8, 0x40, 0x95, 0x82, 0x2b, 0xfc, - 0xad, 0x0d, 0x0e, 0x00, 0x46, 0x22, 0xce, 0x51, 0x9b, 0xfb, 0xad, 0xcd, 0x4e, 0xa3, 0x6a, 0x84, - 0xb8, 0xe0, 0xbc, 0x15, 0xe9, 0x59, 0x96, 0x88, 0xc6, 0x81, 0x9d, 0x62, 0x01, 0x55, 0x36, 0x08, - 0x51, 0xcb, 0x2c, 0x56, 0x35, 0x6b, 0x76, 0xe1, 0xde, 0x64, 0x19, 0xae, 0x66, 0x0d, 0x52, 0x29, - 0xa6, 0xa5, 0x21, 0xd9, 0xb3, 0x59, 0x71, 0x8d, 0x78, 0x97, 0xb0, 0x3d, 0xd2, 0xa2, 0x7c, 0xd8, - 0xb7, 0x87, 0xd0, 0xbd, 0xce, 0x26, 0x28, 0xa6, 0x7a, 0x84, 0xb1, 0xe0, 0x63, 0x65, 0x44, 0x6f, - 0x47, 0x5d, 0xbd, 0x84, 0x7a, 0x87, 0xb0, 0x73, 0x8e, 0xfa, 0x26, 0x0c, 0x78, 0x22, 0x1e, 0xe8, - 0xe7, 0xfd, 0xb4, 0xa0, 0xdf, 0x20, 0xfe, 0x27, 0xab, 0x3a, 0x83, 0xdd, 0x51, 0x25, 0x3a, 0x44, - 0xcd, 0xc6, 0x4c, 0xb3, 0x87, 0x36, 0xb6, 0x0f, 0xed, 0x7b, 0xda, 0x5c, 0x75, 0x7b, 0x32, 0x8f, - 0xbd, 0x00, 0x1e, 0xbf, 0x2f, 0xc7, 0xc6, 0x26, 0xff, 0xda, 0xea, 0x19, 0xec, 0x9e, 0xff, 0xa5, - 0x24, 0xef, 0x04, 0xf6, 0x56, 0xb8, 0xf3, 0xbd, 0x37, 0x07, 0x58, 0x2b, 0x03, 0x9e, 0x2e, 0xff, - 0x0d, 0xc8, 0x2e, 0xd8, 0x57, 0xa8, 0xaf, 0x46, 0x73, 0xa2, 0xcd, 0xab, 0xe0, 0xf5, 0xe6, 0x07, - 0xdb, 0x7c, 0x08, 0x3f, 0x6e, 0x98, 0x9f, 0x93, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x62, - 0xbf, 0xbf, 0x31, 0x05, 0x00, 0x00, + // 571 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x54, 0xc1, 0x4e, 0xdb, 0x40, + 0x10, 0x95, 0x21, 0x81, 0x64, 0x0c, 0x09, 0x59, 0x41, 0xbb, 0x42, 0x15, 0xb2, 0x7c, 0x40, 0x51, + 0x0f, 0x48, 0x85, 0x4b, 0xd5, 0x53, 0x69, 0x52, 0x50, 0x00, 0x23, 0xe4, 0x00, 0x87, 0xde, 0xb6, + 0xce, 0xd8, 0xb8, 0xb6, 0x77, 0xdd, 0xdd, 0x0d, 0x6d, 0xcf, 0xfd, 0xa3, 0xf6, 0x07, 0x2b, 0x6f, + 0x8c, 0xe3, 0x44, 0x6d, 0x54, 0xa9, 0xa7, 0x9e, 0xb2, 0xf3, 0xe6, 0xcd, 0xcc, 0xd3, 0xbc, 0x89, + 0xa1, 0x17, 0xc6, 0x12, 0x03, 0xc9, 0x82, 0x04, 0xe5, 0x51, 0x2e, 0x85, 0x16, 0xfb, 0xb6, 0xfe, + 0x96, 0xa3, 0x9a, 0x05, 0xee, 0xf7, 0x06, 0x74, 0x07, 0x12, 0x99, 0xc6, 0x7b, 0xcf, 0xc7, 0xcf, + 0x53, 0x54, 0x9a, 0x10, 0x68, 0xdc, 0x7b, 0xa3, 0x21, 0xb5, 0x1c, 0xab, 0xdf, 0xf6, 0x1b, 0x8f, + 0xde, 0x68, 0x48, 0xde, 0x02, 0x78, 0x2c, 0x78, 0x88, 0x39, 0x0e, 0xc2, 0x88, 0xae, 0x39, 0x56, + 0xdf, 0x3e, 0x76, 0x8e, 0xce, 0xe6, 0xcd, 0x9f, 0xb2, 0x82, 0x87, 0x71, 0x34, 0x95, 0x4c, 0xc7, + 0x82, 0xfb, 0x90, 0x55, 0x35, 0xa4, 0x0f, 0xdd, 0x4b, 0x94, 0x1c, 0xd3, 0x51, 0xc6, 0x22, 0xbc, + 0x61, 0xfa, 0x81, 0xae, 0x9b, 0x01, 0xdd, 0x64, 0x11, 0x26, 0x07, 0x00, 0x33, 0xe6, 0xa9, 0x8c, + 0x14, 0x6d, 0x18, 0x12, 0x24, 0x15, 0x42, 0x4e, 0xa0, 0xed, 0x0b, 0xa1, 0x87, 0x32, 0x7e, 0x44, + 0xda, 0x34, 0x52, 0xf6, 0xea, 0x52, 0xaa, 0xa4, 0xdf, 0x96, 0x4f, 0x4f, 0xf2, 0x1a, 0x6c, 0xf3, + 0xf0, 0xc4, 0x94, 0x6b, 0x45, 0x37, 0x9c, 0xf5, 0xbe, 0x7d, 0xfc, 0xac, 0x5e, 0x36, 0x4f, 0xfb, + 0xf6, 0x64, 0x4e, 0x25, 0x17, 0xd0, 0xbb, 0x46, 0xfd, 0x45, 0xc8, 0x64, 0xc4, 0x35, 0xca, 0x90, + 0x05, 0xa8, 0xe8, 0xa6, 0xa9, 0x7f, 0x51, 0xaf, 0x5f, 0x26, 0xf9, 0x3d, 0xbe, 0x5c, 0x46, 0x0e, + 0xa1, 0x33, 0x10, 0x5c, 0xb3, 0x98, 0xa3, 0x1c, 0x14, 0xed, 0x69, 0xcb, 0xb1, 0xfa, 0x4d, 0xbf, + 0x13, 0x2c, 0xa0, 0xe4, 0x0d, 0xd0, 0xf7, 0x5f, 0x63, 0x7d, 0x1a, 0x6a, 0x94, 0xa7, 0x69, 0x7a, + 0xcb, 0x54, 0xa2, 0x86, 0x98, 0xa2, 0xc6, 0x09, 0x6d, 0x3b, 0x56, 0xbf, 0xe5, 0x53, 0xfc, 0x43, + 0x9e, 0xbc, 0x82, 0xad, 0x0b, 0x16, 0xa7, 0x45, 0xab, 0xc2, 0x0b, 0x0a, 0x66, 0x43, 0xdb, 0x47, + 0x75, 0xd0, 0xdf, 0xfa, 0x54, 0x8b, 0xdc, 0x1f, 0x16, 0xec, 0xcc, 0xaf, 0x40, 0xe5, 0x82, 0x2b, + 0xfc, 0xed, 0x19, 0x1c, 0x00, 0x8c, 0x45, 0x90, 0xa0, 0x36, 0xfe, 0xad, 0xcd, 0xac, 0x51, 0x15, + 0x42, 0x1c, 0xb0, 0xaf, 0x44, 0x74, 0x16, 0x87, 0xa2, 0x66, 0xb0, 0x9d, 0xce, 0xa1, 0xe2, 0x0c, + 0x3c, 0xd4, 0x32, 0x0e, 0x54, 0xc5, 0x9a, 0x39, 0xdc, 0xcd, 0x16, 0xe1, 0x62, 0xd6, 0x20, 0x92, + 0x62, 0x9a, 0x1b, 0x52, 0x73, 0x36, 0x2b, 0xa8, 0x10, 0xf7, 0x12, 0xb6, 0xc7, 0x5a, 0xe4, 0xab, + 0xef, 0xf6, 0x10, 0x3a, 0xb7, 0x71, 0x86, 0x62, 0xaa, 0xc7, 0x18, 0x08, 0x3e, 0x51, 0x46, 0xf4, + 0xb6, 0xdf, 0xd1, 0x0b, 0xa8, 0x7b, 0x08, 0x3b, 0xe7, 0xa8, 0xef, 0xbd, 0x11, 0x0f, 0xc5, 0x8a, + 0x7e, 0xee, 0x4f, 0x0b, 0x7a, 0x35, 0xe2, 0x7f, 0xb2, 0xaa, 0x33, 0xd8, 0x1d, 0x17, 0xa2, 0x3d, + 0xd4, 0x6c, 0xc2, 0x34, 0x5b, 0xb5, 0xb1, 0x7d, 0x68, 0x3d, 0xd1, 0x4a, 0xd5, 0xad, 0xac, 0x8c, + 0xdd, 0x11, 0x3c, 0xbf, 0xcb, 0x27, 0xe6, 0x4c, 0xfe, 0xb5, 0xd5, 0x4b, 0xd8, 0x3d, 0xff, 0x4b, + 0x49, 0xee, 0x09, 0xec, 0x2d, 0x71, 0xcb, 0xbd, 0xd7, 0x07, 0x58, 0x4b, 0x03, 0xae, 0x16, 0xff, + 0x06, 0x64, 0x17, 0x9a, 0xd7, 0xa8, 0xaf, 0xc7, 0x25, 0xb1, 0xc9, 0x8b, 0xa0, 0x18, 0x37, 0xb8, + 0xb9, 0x53, 0xa5, 0xbc, 0x46, 0x70, 0x73, 0xa7, 0x0a, 0xcc, 0xc3, 0x4c, 0x95, 0x96, 0x34, 0x32, + 0xcc, 0xd4, 0xbb, 0xcd, 0x0f, 0x4d, 0xf3, 0xc1, 0xfc, 0xb8, 0x61, 0x7e, 0x4e, 0x7e, 0x05, 0x00, + 0x00, 0xff, 0xff, 0xa6, 0xb8, 0xfd, 0xa5, 0x59, 0x05, 0x00, 0x00, } diff --git a/proto/firecracker.proto b/proto/firecracker.proto index 7361ca902..2fc5f9dbb 100644 --- a/proto/firecracker.proto +++ b/proto/firecracker.proto @@ -81,5 +81,25 @@ message GetVMMetadataResponse { message JailerConfig { string NetNS = 1; + + // List of the physical numbers of the CPUs on which processes in that + // cpuset are allowed to execute. See List Format below for a description + // of the format of cpus. + // + // The CPUs allowed to a cpuset may be changed by writing a new list to its + // cpus file. + // Taken from http://man7.org/linux/man-pages/man7/cpuset.7.html + // + // This is formatted as specified in the cpuset man page under "List Format" + // http://man7.org/linux/man-pages/man7/cpuset.7.html + string CPUs = 2; + // List of memory nodes on which processes in this cpuset are allowed to + // allocate memory. See List Format below for a description of the format + // of mems. + // Taken from http://man7.org/linux/man-pages/man7/cpuset.7.html + // + // This is formatted as specified in the cpuset man page under "List Format" + // http://man7.org/linux/man-pages/man7/cpuset.7.html + string Mems = 3; } diff --git a/runtime/cpuset/cpuset_builder.go b/runtime/cpuset/cpuset_builder.go new file mode 100644 index 000000000..3b50f3573 --- /dev/null +++ b/runtime/cpuset/cpuset_builder.go @@ -0,0 +1,143 @@ +// Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cpuset + +import ( + "fmt" + "strconv" + "strings" +) + +// Builder will allow building of cpuset fields such as cpuset.cpus and +// cpuset.mems +// +// Example: +// cset := cpuset.Builder{}. +// AddCPU(0). +// AddMemRange(0). +// Build() +// +// fcClient, err := fcclient.New(containerdTTRPCAddress) +// if err != nil { +// return err +// } +// +// defer fcClient.Close() +// +// vmID := "cpuset-builder-example" +// createVMRequest := &proto.CreateVMRequest{ +// VMID: vmID, +// JailerConfig: proto.JailerConfig{ +// CPUs: cset.CPUs(), +// Mems: cset.Mems(), +// }, +// } +// +// _, err = fcClient.CreateVM(ctx, createVMRequest) +// if err != nil { +// return errors.Wrap(err, "failed to create VM") +// } +type Builder struct { + cpus []int + cpuRanges []_range + + mems []int + memRanges []_range +} + +type _range struct { + min, max int +} + +func (r _range) String() string { + return fmt.Sprintf("%d-%d", r.min, r.max) +} + +// CPUSet represents the linux CPUSet which is a series of configurable values +// that allow processes to run on a specific CPUs and those CPUs are then bound +// to the memory nodes specified. +// +// More information can be found here: http://man7.org/linux/man-pages/man7/cpuset.7.html +type CPUSet struct { + cpus string + mems string +} + +// CPUs returns the cpuset.cpus string +func (c CPUSet) CPUs() string { + return c.cpus +} + +// Mems returns the cpuset.mems string +func (c CPUSet) Mems() string { + return c.mems +} + +// AddCPU will add the physical CPU number that the process is allowed to run +// on. +func (b Builder) AddCPU(cpu int) Builder { + b.cpus = append(b.cpus, cpu) + return b +} + +// AddCPURange adds a range of physical CPU numbers that the process is allowed +// to run on +func (b Builder) AddCPURange(min, max int) Builder { + b.cpuRanges = append(b.cpuRanges, _range{ + min: min, + max: max, + }) + + return b +} + +// AddMem adds a memory node which limits where the cpus can allocate memory +func (b Builder) AddMem(mem int) Builder { + b.mems = append(b.mems, mem) + return b +} + +// AddMemRange adds a range of memory nodes to be used. +func (b Builder) AddMemRange(min, max int) Builder { + b.memRanges = append(b.memRanges, _range{ + min: min, + max: max, + }) + + return b +} + +// Build constructs a new CPUSet +func (b Builder) Build() CPUSet { + cpus := stringify(b.cpus, b.cpuRanges) + mems := stringify(b.mems, b.memRanges) + + return CPUSet{ + cpus: cpus, + mems: mems, + } +} + +func stringify(elems []int, ranges []_range) string { + strs := []string{} + for _, elem := range elems { + strs = append(strs, strconv.Itoa(elem)) + } + + for _, r := range ranges { + strs = append(strs, r.String()) + } + + return strings.Join(strs, ",") +} diff --git a/runtime/cpuset/cpuset_builder_test.go b/runtime/cpuset/cpuset_builder_test.go new file mode 100644 index 000000000..d5bc65d1a --- /dev/null +++ b/runtime/cpuset/cpuset_builder_test.go @@ -0,0 +1,169 @@ +// Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"). You may +// not use this file except in compliance with the License. A copy of the +// License is located at +// +// http://aws.amazon.com/apache2.0/ +// +// or in the "license" file accompanying this file. This file is distributed +// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +// express or implied. See the License for the specific language governing +// permissions and limitations under the License. + +package cpuset + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCPUSet(t *testing.T) { + cases := []struct { + name string + cpus []int + mems []int + cpuRanges []_range + memRanges []_range + expectedCPUSet CPUSet + }{ + { + name: "empty set", + }, + { + name: "single cpu only", + cpus: []int{0}, + expectedCPUSet: CPUSet{ + cpus: "0", + }, + }, + { + name: "cpus only", + cpus: []int{0, 5, 6}, + expectedCPUSet: CPUSet{ + cpus: "0,5,6", + }, + }, + { + name: "single mem only", + mems: []int{2}, + expectedCPUSet: CPUSet{ + mems: "2", + }, + }, + { + name: "mems only", + mems: []int{2, 8, 3}, + expectedCPUSet: CPUSet{ + mems: "2,8,3", + }, + }, + { + name: "cpu single range only", + cpuRanges: []_range{ + { + min: 0, + max: 3, + }, + }, + expectedCPUSet: CPUSet{ + cpus: "0-3", + }, + }, + { + name: "cpu ranges only", + cpuRanges: []_range{ + { + min: 0, + max: 3, + }, + { + min: 5, + max: 10, + }, + }, + expectedCPUSet: CPUSet{ + cpus: "0-3,5-10", + }, + }, + { + name: "mem single range only", + memRanges: []_range{ + { + min: 0, + max: 1, + }, + }, + expectedCPUSet: CPUSet{ + mems: "0-1", + }, + }, + { + name: "mem ranges only", + memRanges: []_range{ + { + min: 0, + max: 1, + }, + { + min: 2, + max: 3, + }, + }, + expectedCPUSet: CPUSet{ + mems: "0-1,2-3", + }, + }, + { + name: "all inclusive", + cpus: []int{15, 17, 31}, + cpuRanges: []_range{ + { + min: 0, + max: 3, + }, + }, + mems: []int{128, 131, 140}, + memRanges: []_range{ + { + min: 0, + max: 1, + }, + { + min: 2, + max: 3, + }, + }, + expectedCPUSet: CPUSet{ + cpus: "15,17,31,0-3", + mems: "128,131,140,0-1,2-3", + }, + }, + } + + for _, _c := range cases { + c := _c + t.Run(c.name, func(t *testing.T) { + b := Builder{} + for _, cpu := range c.cpus { + b = b.AddCPU(cpu) + } + + for _, r := range c.cpuRanges { + b = b.AddCPURange(r.min, r.max) + } + + for _, mem := range c.mems { + b = b.AddMem(mem) + } + + for _, r := range c.memRanges { + b = b.AddMemRange(r.min, r.max) + } + + cpuSet := b.Build() + assert.Equal(t, c.expectedCPUSet, cpuSet) + }) + } +} diff --git a/runtime/jailer.go b/runtime/jailer.go index a37c3997e..ba4c83fc0 100644 --- a/runtime/jailer.go +++ b/runtime/jailer.go @@ -82,13 +82,13 @@ func newJailer( } l := logger.WithField("jailer", "runc") - return newRuncJailer( - ctx, - l, - service.vmID, - ociBundlePath, - service.config.JailerConfig.RuncBinaryPath, - jailerUID, - jailerGID, - ) + config := runcJailerConfig{ + OCIBundlePath: ociBundlePath, + RuncBinPath: service.config.JailerConfig.RuncBinaryPath, + UID: jailerUID, + GID: jailerGID, + CPUs: request.JailerConfig.CPUs, + Mems: request.JailerConfig.Mems, + } + return newRuncJailer(ctx, l, service.vmID, config) } diff --git a/runtime/jailer_integ_test.go b/runtime/jailer_integ_test.go index 3c8021021..4f7d8f33b 100644 --- a/runtime/jailer_integ_test.go +++ b/runtime/jailer_integ_test.go @@ -26,6 +26,7 @@ import ( _ "github.com/firecracker-microvm/firecracker-containerd/firecracker-control" "github.com/firecracker-microvm/firecracker-containerd/proto" fccontrol "github.com/firecracker-microvm/firecracker-containerd/proto/service/fccontrol/ttrpc" + "github.com/firecracker-microvm/firecracker-containerd/runtime/cpuset" "github.com/firecracker-microvm/firecracker-containerd/runtime/firecrackeroci" ) @@ -79,3 +80,17 @@ func testJailer(t *testing.T, jailerConfig *proto.JailerConfig) { require.NoError(err, "failed to delete a container") }() } + +func TestJailerCPUSet_Isolated(t *testing.T) { + prepareIntegTest(t, withJailer()) + + t.Run("TestJailerCPUSet_Isolated", func(t *testing.T) { + b := cpuset.Builder{} + cset := b.AddCPU(0).AddMem(0).Build() + config := &proto.JailerConfig{ + CPUs: cset.CPUs(), + Mems: cset.Mems(), + } + testJailer(t, config) + }) +} diff --git a/runtime/runc_jailer.go b/runtime/runc_jailer.go index 2f0ec31ad..22c092c48 100644 --- a/runtime/runc_jailer.go +++ b/runtime/runc_jailer.go @@ -41,38 +41,33 @@ const ( // runcJailer uses runc to set up a jailed environment for the Firecracker VM. type runcJailer struct { - ctx context.Context - logger *logrus.Entry - // ociBundlePath is the path that will be used to create an OCI bundle, - // https://github.com/opencontainers/runtime-spec/blob/master/bundle.md - ociBundlePath string - // runcBinaryPath is the path used to execute the runc binary from. - runcBinaryPath string - uid uint32 - gid uint32 - vmID string - configSpec specs.Spec + ctx context.Context + logger *logrus.Entry + Config runcJailerConfig + vmID string + configSpec specs.Spec } const firecrackerFileName = "firecracker" -func newRuncJailer( - ctx context.Context, - logger *logrus.Entry, - vmID string, - ociBundlePath string, - runcBinaryPath string, - uid uint32, - gid uint32) (*runcJailer, error) { - l := logger.WithField("ociBundlePath", ociBundlePath).WithField("runcBinaryPath", runcBinaryPath) +type runcJailerConfig struct { + OCIBundlePath string + RuncBinPath string + UID uint32 + GID uint32 + CPUs string + Mems string +} + +func newRuncJailer(ctx context.Context, logger *logrus.Entry, vmID string, cfg runcJailerConfig) (*runcJailer, error) { + l := logger.WithField("ociBundlePath", cfg.OCIBundlePath). + WithField("runcBinaryPath", cfg.RuncBinPath) + j := &runcJailer{ - ctx: ctx, - logger: l, - ociBundlePath: ociBundlePath, - runcBinaryPath: runcBinaryPath, - uid: uid, - gid: gid, - vmID: vmID, + ctx: ctx, + logger: l, + Config: cfg, + vmID: vmID, } spec := specs.Spec{} @@ -93,7 +88,7 @@ func newRuncJailer( const mode = os.FileMode(0700) // Create the proper paths needed for the runc jailer j.logger.WithField("rootPath", rootPath).Debug("Creating root drive path") - if err := mkdirAndChown(rootPath, mode, j.uid, j.gid); err != nil { + if err := mkdirAndChown(rootPath, mode, j.Config.UID, j.Config.GID); err != nil { return nil, errors.Wrapf(err, "%s failed to mkdirAndChown", rootPath) } @@ -103,7 +98,7 @@ func newRuncJailer( // JailPath returns the base directory from where the jail binary will be ran // from func (j *runcJailer) OCIBundlePath() string { - return j.ociBundlePath + return j.Config.OCIBundlePath } // RootPath returns the root fs of the jailed system. @@ -268,7 +263,7 @@ func (j *runcJailer) BuildLinkFifoHandler() firecracker.Handler { func (j runcJailer) StubDrivesOptions() []FileOpt { return []FileOpt{ func(file *os.File) error { - err := unix.Fchown(int(file.Fd()), int(j.uid), int(j.gid)) + err := unix.Fchown(int(file.Fd()), int(j.Config.UID), int(j.Config.GID)) if err != nil { return errors.Wrapf(err, "failed to chown stub file %q", file.Name()) } @@ -283,8 +278,8 @@ func (j runcJailer) StubDrivesOptions() []FileOpt { // set the correct permissions to ensure visibility in the jail. Regular files // will be copied into the jail. func (j *runcJailer) ExposeFileToJail(srcPath string) error { - uid := j.uid - gid := j.gid + uid := j.Config.UID + gid := j.Config.GID stat := syscall.Stat_t{} if err := syscall.Stat(srcPath, &stat); err != nil { @@ -327,7 +322,7 @@ func (j *runcJailer) copyFileToJail(src, dst string, mode os.FileMode) error { if err := copyFile(src, dst, mode); err != nil { return err } - if err := os.Chown(dst, int(j.uid), int(j.gid)); err != nil { + if err := os.Chown(dst, int(j.Config.UID), int(j.Config.GID)); err != nil { return err } return nil @@ -372,7 +367,7 @@ func copyFile(src, dst string, mode os.FileMode) error { } func (j *runcJailer) jailerCommand(containerName string, isDebug bool) *exec.Cmd { - cmd := exec.CommandContext(j.ctx, j.runcBinaryPath, "run", containerName) + cmd := exec.CommandContext(j.ctx, j.Config.RuncBinPath, "run", containerName) cmd.Dir = j.OCIBundlePath() if isDebug { @@ -398,8 +393,8 @@ func (j *runcJailer) overwriteConfig(cfg *Config, machineConfig *firecracker.Con spec = j.setDefaultConfigValues(cfg, socketPath, spec) spec.Root.Path = rootfsFolder spec.Root.Readonly = false - spec.Process.User.UID = j.uid - spec.Process.User.GID = j.gid + spec.Process.User.UID = j.Config.UID + spec.Process.User.GID = j.Config.GID if machineConfig.NetNS != "" { for i, ns := range spec.Linux.Namespaces { @@ -411,6 +406,17 @@ func (j *runcJailer) overwriteConfig(cfg *Config, machineConfig *firecracker.Con } } + if spec.Linux.Resources == nil { + spec.Linux.Resources = &specs.LinuxResources{} + } + + if spec.Linux.Resources.CPU == nil { + spec.Linux.Resources.CPU = &specs.LinuxCPU{} + } + + spec.Linux.Resources.CPU.Cpus = j.Config.CPUs + spec.Linux.Resources.CPU.Mems = j.Config.Mems + configBytes, err := json.Marshal(&spec) if err != nil { return err diff --git a/runtime/runc_jailer_test.go b/runtime/runc_jailer_test.go index 8dcd2762e..a879c68e0 100644 --- a/runtime/runc_jailer_test.go +++ b/runtime/runc_jailer_test.go @@ -53,8 +53,14 @@ func TestBuildJailedRootHandler_Isolated(t *testing.T) { defer firecrackerFd.Close() l := logrus.NewEntry(logrus.New()) + config := runcJailerConfig{ + OCIBundlePath: dir, + RuncBinPath: "bin-path", + UID: 123, + GID: 456, + } vmID := "foo" - jailer, err := newRuncJailer(context.Background(), l, vmID, dir, "path/to/runc", 123, 456) + jailer, err := newRuncJailer(context.Background(), l, vmID, config) require.NoError(t, err, "failed to create runc jailer") cfg := Config{ diff --git a/runtime/service_integ_test.go b/runtime/service_integ_test.go index cd60edbf9..61f14263c 100644 --- a/runtime/service_integ_test.go +++ b/runtime/service_integ_test.go @@ -406,8 +406,10 @@ func testMultipleExecs(ctx context.Context, t *testing.T, vmID int, containerID require.NoError(t, err, "failed to get shim dir") jailer := &runcJailer{ - ociBundlePath: string(shimDir), - vmID: vmIDStr, + Config: runcJailerConfig{ + OCIBundlePath: string(shimDir), + }, + vmID: vmIDStr, } _, err = os.Stat(jailer.RootPath()) require.NoError(t, err, "failed to stat root path of jailer")