From e64043f3adecd1062210550662ff02b73548466d Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Thu, 3 Aug 2023 10:36:32 +0300 Subject: [PATCH 1/3] gpu: refactor nfdhook functionality to plugin NFD v0.14+ doesn't support binary NFD hooks by default, so there is a need to move the label creation away from the GPU nfdhook. Move extended resource label creation to plugin, and drop labels that were already marked deprecated (platform_gen, media_version etc.). Drop init-container from deployment files and operator. It is still possible to use an initcontainer, but the default deployments do not support it. Signed-off-by: Tuomas Katila --- cmd/gpu_nfdhook/README.md | 2 + cmd/gpu_nfdhook/main.go | 19 +- cmd/gpu_plugin/gpu_plugin.go | 35 +- cmd/gpu_plugin/gpu_plugin_test.go | 2 +- cmd/gpu_plugin/render-device.sh | 2 +- .../rm/gpu_plugin_resource_manager.go | 2 +- .../rm/gpu_plugin_resource_manager_test.go | 2 +- .../labeler}/labeler.go | 214 +++++++------ .../labeler}/labeler_test.go | 298 ++++-------------- .../gpu_plugin/base/intel-gpu-plugin.yaml | 20 +- .../add-kubelet-crt-mount.yaml | 17 - .../fractional_resources/add-mounts.yaml | 34 ++ .../add-podresource-mount.yaml | 17 - .../fractional_resources/kustomization.yaml | 5 +- .../deviceplugin_v1_gpudeviceplugin.yaml | 1 - .../v1/gpudeviceplugin_webhook.go | 6 - pkg/controllers/gpu/controller.go | 48 ++- pkg/controllers/gpu/controller_test.go | 129 ++++++-- 18 files changed, 403 insertions(+), 450 deletions(-) rename cmd/{gpu_nfdhook => internal/labeler}/labeler.go (77%) rename cmd/{gpu_nfdhook => internal/labeler}/labeler_test.go (65%) delete mode 100644 deployments/gpu_plugin/overlays/fractional_resources/add-kubelet-crt-mount.yaml create mode 100644 deployments/gpu_plugin/overlays/fractional_resources/add-mounts.yaml delete mode 100644 deployments/gpu_plugin/overlays/fractional_resources/add-podresource-mount.yaml diff --git a/cmd/gpu_nfdhook/README.md b/cmd/gpu_nfdhook/README.md index 2735f8470..3ff387354 100644 --- a/cmd/gpu_nfdhook/README.md +++ b/cmd/gpu_nfdhook/README.md @@ -11,6 +11,8 @@ Table of Contents ## Introduction +***NOTE:*** NFD's binary hook support will be turned off by default in the 0.14 release. The functionality in the GPU NFD hook is moved into a new NFD rule and into GPU plugin, and the capability labels are being removed completely. The GPU plugin deployment doesn't anymore support using init container. This directory will be removed in the future. + This is the [Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery) binary hook implementation for the Intel GPUs. The intel-gpu-initcontainer (which is built with the other images) can be used as part of the gpu-plugin deployment diff --git a/cmd/gpu_nfdhook/main.go b/cmd/gpu_nfdhook/main.go index 4023f9f32..cdd15eb6d 100644 --- a/cmd/gpu_nfdhook/main.go +++ b/cmd/gpu_nfdhook/main.go @@ -15,25 +15,14 @@ package main import ( - "os" - - "k8s.io/klog/v2" + "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler" ) const ( - sysfsDirectory = "/host-sys" - sysfsDRMDirectory = sysfsDirectory + "/class/drm" - debugfsDRIDirectory = sysfsDirectory + "/kernel/debug/dri" + sysfsDirectory = "/host-sys" + sysfsDRMDirectory = sysfsDirectory + "/class/drm" ) func main() { - l := newLabeler(sysfsDRMDirectory, debugfsDRIDirectory) - - err := l.createLabels() - if err != nil { - klog.Errorf("%+v", err) - os.Exit(1) - } - - l.printLabels() + labeler.CreateAndPrintLabels(sysfsDRMDirectory) } diff --git a/cmd/gpu_plugin/gpu_plugin.go b/cmd/gpu_plugin/gpu_plugin.go index 23a497a97..e8687d866 100644 --- a/cmd/gpu_plugin/gpu_plugin.go +++ b/cmd/gpu_plugin/gpu_plugin.go @@ -1,4 +1,4 @@ -// Copyright 2017-2022 Intel Corporation. All Rights Reserved. +// Copyright 2017-2023 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -31,6 +31,7 @@ import ( pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" "github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm" + "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler" "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" ) @@ -38,6 +39,8 @@ import ( const ( sysfsDrmDirectory = "/sys/class/drm" devfsDriDirectory = "/dev/dri" + nfdFeatureDir = "/etc/kubernetes/node-feature-discovery/features.d" + resourceFilename = "intel-gpu-resources.txt" gpuDeviceRE = `^card[0-9]+$` controlDeviceRE = `^controlD[0-9]+$` pciAddressRE = "^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\\.[0-9a-f]{1}$" @@ -53,6 +56,9 @@ const ( // Period of device scans. scanPeriod = 5 * time.Second + + // Labeler's max update interval, 5min. + labelerMaxInterval = 5 * 60 * time.Second ) type cliOptions struct { @@ -242,8 +248,9 @@ type devicePlugin struct { controlDeviceReg *regexp.Regexp pciAddressReg *regexp.Regexp - scanTicker *time.Ticker - scanDone chan bool + scanTicker *time.Ticker + scanDone chan bool + scanResources chan bool resMan rm.ResourceManager @@ -270,6 +277,7 @@ func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugi scanTicker: time.NewTicker(scanPeriod), scanDone: make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it bypathFound: true, + scanResources: make(chan bool, 1), } if options.resourceManagement { @@ -347,17 +355,26 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error { klog.Warning("Failed to scan: ", err) } + countChanged := false + for name, prev := range previousCount { count := devTree.DeviceTypeCount(name) if count != prev { klog.V(1).Infof("GPU scan update: %d->%d '%s' resources found", prev, count, name) previousCount[name] = count + + countChanged = true } } notifier.Notify(devTree) + // Trigger resource scan if it's enabled. + if dp.resMan != nil && countChanged { + dp.scanResources <- true + } + select { case <-dp.scanDone: return nil @@ -515,6 +532,18 @@ func main() { klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy) plugin := newDevicePlugin(prefix+sysfsDrmDirectory, prefix+devfsDriDirectory, opts) + + if plugin.options.resourceManagement { + // Start labeler to export labels file for NFD. + nfdFeatureFile := path.Join(nfdFeatureDir, resourceFilename) + + klog.V(2).Infof("NFD feature file location: %s", nfdFeatureFile) + + // Labeler catches OS signals and calls os.Exit() after receiving any. + go labeler.Run(prefix+sysfsDrmDirectory, nfdFeatureFile, + labelerMaxInterval, plugin.scanResources) + } + manager := dpapi.NewManager(namespace, plugin) manager.Run() } diff --git a/cmd/gpu_plugin/gpu_plugin_test.go b/cmd/gpu_plugin/gpu_plugin_test.go index a5b59519f..707eee438 100644 --- a/cmd/gpu_plugin/gpu_plugin_test.go +++ b/cmd/gpu_plugin/gpu_plugin_test.go @@ -1,4 +1,4 @@ -// Copyright 2017-2021 Intel Corporation. All Rights Reserved. +// Copyright 2017-2023 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/gpu_plugin/render-device.sh b/cmd/gpu_plugin/render-device.sh index 5621c1283..e85d35671 100755 --- a/cmd/gpu_plugin/render-device.sh +++ b/cmd/gpu_plugin/render-device.sh @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright 2021 Intel Corporation. +# Copyright 2021-2023 Intel Corporation. # # SPDX-License-Identifier: Apache-2.0 # diff --git a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go index 98504ba16..60c060406 100644 --- a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go +++ b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go @@ -1,4 +1,4 @@ -// Copyright 2021 Intel Corporation. All Rights Reserved. +// Copyright 2021-2023 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go index 8201c8c57..6f53d4455 100644 --- a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go +++ b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go @@ -1,4 +1,4 @@ -// Copyright 2021 Intel Corporation. All Rights Reserved. +// Copyright 2021-2023 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/gpu_nfdhook/labeler.go b/cmd/internal/labeler/labeler.go similarity index 77% rename from cmd/gpu_nfdhook/labeler.go rename to cmd/internal/labeler/labeler.go index 87496a3cf..08c407cd9 100644 --- a/cmd/gpu_nfdhook/labeler.go +++ b/cmd/internal/labeler/labeler.go @@ -1,4 +1,4 @@ -// Copyright 2020-2021 Intel Corporation. All Rights Reserved. +// Copyright 2020-2023 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,18 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package labeler import ( - "bufio" "fmt" "os" + "os/signal" "path" "path/filepath" + "reflect" "regexp" "sort" "strconv" "strings" + "syscall" + "time" "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils" "github.com/pkg/errors" @@ -57,16 +60,16 @@ type labeler struct { labels labelMap sysfsDRMDir string - debugfsDRIDir string + labelsChanged bool } -func newLabeler(sysfsDRMDir, debugfsDRIDir string) *labeler { +func newLabeler(sysfsDRMDir string) *labeler { return &labeler{ sysfsDRMDir: sysfsDRMDir, - debugfsDRIDir: debugfsDRIDir, gpuDeviceReg: regexp.MustCompile(gpuDeviceRE), controlDeviceReg: regexp.MustCompile(controlDeviceRE), labels: labelMap{}, + labelsChanged: true, } } @@ -222,88 +225,6 @@ func (lm labelMap) addNumericLabel(labelName string, valueToAdd int64) { lm[labelName] = strconv.FormatInt(value, 10) } -// createCapabilityLabels creates labels from the gpu capability file under debugfs. -func (l *labeler) createCapabilityLabels(gpuNum string, numTiles uint64) { - // try to read the capabilities from the i915_capabilities file - file, err := os.Open(filepath.Join(l.debugfsDRIDir, gpuNum, "i915_capabilities")) - if err != nil { - klog.V(3).Infof("Couldn't open file:%s", err.Error()) // debugfs is not stable, there is no need to spam with error level prints - return - } - defer file.Close() - - gen := "" - media := "" - graphics := "" - // define string prefixes to search from the file, and the actions to take in order to create labels from those strings (as funcs) - searchStringActionMap := map[string]func(string){ - "platform:": func(platformName string) { - l.labels.addNumericLabel(labelNamespace+"platform_"+platformName+".count", 1) - l.labels[labelNamespace+"platform_"+platformName+".tiles"] = strconv.FormatInt(int64(numTiles), 10) - l.labels[labelNamespace+"platform_"+platformName+".present"] = "true" - }, - // there's also display block version, but that's not relevant - "media version:": func(version string) { - l.labels[labelNamespace+"media_version"] = version - media = version - }, - "graphics version:": func(version string) { - l.labels[labelNamespace+"graphics_version"] = version - graphics = version - }, - "gen:": func(version string) { - l.labels[labelNamespace+"platform_gen"] = version - gen = version - }, - } - - // Finally, read the file, and try to find the matches. Perform actions and reduce the search map size as we proceed. Return at 0 size. - scanner := bufio.NewScanner(file) -scanning: - for scanner.Scan() { - line := scanner.Text() - for prefix, action := range searchStringActionMap { - if !strings.HasPrefix(line, prefix) { - continue - } - - fields := strings.Split(line, ": ") - if len(fields) == 2 { - action(fields[1]) - } else { - klog.Warningf("invalid '%s' line format: '%s'", file.Name(), line) - } - - delete(searchStringActionMap, prefix) - - if len(searchStringActionMap) == 0 { - break scanning - } - break - } - } - - if gen == "" { - // TODO: drop gen label before engine types - // start to have diverging major gen values - if graphics != "" { - gen = graphics - } else if media != "" { - gen = media - } - - if gen != "" { - // truncate to major value - gen = strings.SplitN(gen, ".", 2)[0] - l.labels[labelNamespace+"platform_gen"] = gen - } - } else if media == "" && graphics == "" { - // 5.14 or older kernels need this - l.labels[labelNamespace+"media_version"] = gen - l.labels[labelNamespace+"graphics_version"] = gen - } -} - // this returns pci groups label value, groups separated by "_", gpus separated by ".". // Example for two groups with 4 gpus: "0.1.2.3_4.5.6.7". func (l *labeler) createPCIGroupLabel(gpuNumList []string) string { @@ -345,6 +266,10 @@ func (l *labeler) createPCIGroupLabel(gpuNumList []string) string { // createLabels is the main function of plugin labeler, it creates label-value pairs for the gpus. func (l *labeler) createLabels() error { + prevLabels := l.labels + + l.labels = labelMap{} + gpuNameList, err := l.scan() if err != nil { return err @@ -380,9 +305,6 @@ func (l *labeler) createLabels() error { numaMapping[numaNode] = numaList } - // try to add capability labels - l.createCapabilityLabels(gpuNum, numTiles) - l.labels.addNumericLabel(labelNamespace+"memory.max", int64(memoryAmount)) } @@ -431,6 +353,8 @@ func (l *labeler) createLabels() error { } } + l.labelsChanged = !reflect.DeepEqual(prevLabels, l.labels) + return nil } @@ -455,8 +379,114 @@ func createNumaNodeMappingLabel(mapping map[int][]string) string { return strings.Join(parts, "_") } -func (l *labeler) printLabels() { +func (l *labeler) atomicPrintLabelsToFile(labelFile string) error { + baseDir := filepath.Dir(labelFile) + + // TODO: Use NFD's "hidden file" feature when it becomes available. + d, err := os.MkdirTemp(baseDir, "labels") + if err != nil { + klog.Warning("could not create temporary directory, writing directly to destination") + + return l.printLabelsToFile(labelFile) + } + + defer os.RemoveAll(d) + + tmpFile := filepath.Join(d, "labels.txt") + + if err := l.printLabelsToFile(tmpFile); err != nil { + return err + } + + return os.Rename(tmpFile, labelFile) +} + +func (l *labeler) printLabelsToFile(labelFile string) error { + f, err := os.OpenFile(labelFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open file (%s): %w", labelFile, err) + } + + defer f.Close() + + for key, val := range l.labels { + if _, err := f.WriteString(key + "=" + val + "\n"); err != nil { + return fmt.Errorf("failed to write label (%s=%s) to file: %w", key, val, err) + } + } + + return nil +} + +func CreateAndPrintLabels(sysfsDRMDir string) { + l := newLabeler(sysfsDRMDir) + + if err := l.createLabels(); err != nil { + klog.Warningf("failed to create labels: %+v", err) + + return + } + for key, val := range l.labels { fmt.Println(key + "=" + val) } } + +// Gathers node's GPU labels on channel trigger or timeout, and write them to a file. +// The created label file is deleted on exit (process dying). +func Run(sysfsDrmDir, nfdFeatureFile string, updateInterval time.Duration, scanResources chan bool) { + l := newLabeler(sysfsDrmDir) + + interruptChan := make(chan os.Signal, 1) + signal.Notify(interruptChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT) + + klog.V(1).Info("Starting GPU labeler") + +Loop: + for { + timeout := time.After(updateInterval) + + select { + case <-timeout: + case <-scanResources: + case interrupt := <-interruptChan: + klog.V(2).Infof("Interrupt %d received", interrupt) + + break Loop + } + + klog.V(1).Info("Ext resources scanning") + + err := l.createLabels() + if err != nil { + klog.Warningf("label creation failed: %+v", err) + + continue + } + + if l.labelsChanged { + klog.V(1).Info("Writing labels") + + if err := l.atomicPrintLabelsToFile(nfdFeatureFile); err != nil { + klog.Warningf("failed to write labels to file: %+v", err) + + // Reset labels so that next time the labeler runs the writing is retried. + l.labels = labelMap{} + } + } + } + + signal.Stop(interruptChan) + + klog.V(2).Info("Removing label file") + + err := os.Remove(nfdFeatureFile) + if err != nil { + klog.Errorf("Failed to cleanup label file: %+v", err) + } + + klog.V(1).Info("Stopping GPU labeler") + + // Close the whole application + os.Exit(0) +} diff --git a/cmd/gpu_nfdhook/labeler_test.go b/cmd/internal/labeler/labeler_test.go similarity index 65% rename from cmd/gpu_nfdhook/labeler_test.go rename to cmd/internal/labeler/labeler_test.go index 188830c46..9833eabe7 100644 --- a/cmd/gpu_nfdhook/labeler_test.go +++ b/cmd/internal/labeler/labeler_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package labeler import ( "os" @@ -24,6 +24,10 @@ import ( "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils" ) +const ( + sysfsDirectory = "/sys/" +) + type testcase struct { capabilityFile map[string][]byte expectedRetval error @@ -49,24 +53,13 @@ func getTestCases() []testcase { }, name: "successful labeling via lmem_total_bytes", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "8086", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "8086", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "1", }, }, { @@ -79,11 +72,6 @@ func getTestCases() []testcase { }, name: "pf with vfs", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ "gpu.intel.com/tiles": "0", @@ -101,24 +89,13 @@ func getTestCases() []testcase { }, name: "successful labeling via card0/lmem_total_bytes and two tiles", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "2", - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "2", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "16000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "2", }, }, { @@ -133,24 +110,13 @@ func getTestCases() []testcase { name: "successful labeling via lmem_total_bytes and reserved memory", memoryOverride: 16000000000, memoryReserved: 86, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "8000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "8000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "1", }, }, { @@ -162,24 +128,13 @@ func getTestCases() []testcase { }, name: "successful labeling via memory override", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "16000000000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "1", }, }, { @@ -191,20 +146,13 @@ func getTestCases() []testcase { }, name: "when gen:capability info is missing", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "16000000000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "1", }, }, { @@ -216,25 +164,13 @@ func getTestCases() []testcase { }, name: "gen version missing, but media & graphics versions present", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "media version: 12.5\n" + - "graphics version: 12.2"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/graphics_version": "12.2", - "gpu.intel.com/media_version": "12.5", - "gpu.intel.com/platform_gen": "12", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "16000000000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "1", }, }, { @@ -246,48 +182,13 @@ func getTestCases() []testcase { }, name: "only media version present", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "media version: 12.5"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "16000000000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/media_version": "12.5", - "gpu.intel.com/platform_gen": "12", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "1", - }, - }, - { - sysfsdirs: []string{ - "card0/device/drm/card0", - "card1/device/drm/card1", - }, - sysfsfiles: map[string][]byte{ - "card0/device/vendor": []byte("0x8086"), - "card1/device/vendor": []byte("0x8086"), - }, - name: "when capability file is missing (foobar), related labels don't appear", - memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "foobar": []byte( - "platform: new\n" + - "gen: 9"), - }, - expectedRetval: nil, - expectedLabels: labelMap{ - "gpu.intel.com/millicores": "2000", - "gpu.intel.com/memory.max": "32000000000", - "gpu.intel.com/cards": "card0.card1", - "gpu.intel.com/gpu-numbers": "0.1", - "gpu.intel.com/tiles": "2", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "16000000000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "1", }, }, { @@ -433,24 +334,13 @@ func getTestCases() []testcase { }, name: "successful labeling via card0/lmem_total_bytes and three tiles", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "24000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "3", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/cards": "card0", - "gpu.intel.com/gpu-numbers": "0", - "gpu.intel.com/tiles": "3", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "24000", + "gpu.intel.com/cards": "card0", + "gpu.intel.com/gpu-numbers": "0", + "gpu.intel.com/tiles": "3", }, }, { @@ -469,30 +359,13 @@ func getTestCases() []testcase { }, name: "successful labeling with two cards and total three tiles", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - "1/i915_capabilities": []byte( - "platform: newnew\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/millicores": "2000", - "gpu.intel.com/memory.max": "24000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "2", - "gpu.intel.com/platform_newnew.count": "1", - "gpu.intel.com/platform_newnew.present": "true", - "gpu.intel.com/platform_newnew.tiles": "1", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/gpu-numbers": "0.1", - "gpu.intel.com/cards": "card0.card1", - "gpu.intel.com/tiles": "3", + "gpu.intel.com/millicores": "2000", + "gpu.intel.com/memory.max": "24000", + "gpu.intel.com/gpu-numbers": "0.1", + "gpu.intel.com/cards": "card0.card1", + "gpu.intel.com/tiles": "3", }, }, { @@ -512,31 +385,14 @@ func getTestCases() []testcase { }, name: "successful labeling with two cards and numa node info", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "0/i915_capabilities": []byte( - "platform: new\n" + - "gen: 9"), - "1/i915_capabilities": []byte( - "platform: newnew\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/millicores": "2000", - "gpu.intel.com/memory.max": "16000", - "gpu.intel.com/platform_new.count": "1", - "gpu.intel.com/platform_new.present": "true", - "gpu.intel.com/platform_new.tiles": "1", - "gpu.intel.com/platform_newnew.count": "1", - "gpu.intel.com/platform_newnew.present": "true", - "gpu.intel.com/platform_newnew.tiles": "1", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/gpu-numbers": "0.1", - "gpu.intel.com/cards": "card0.card1", - "gpu.intel.com/tiles": "2", - "gpu.intel.com/numa-gpu-map": "0-0_1-1", + "gpu.intel.com/millicores": "2000", + "gpu.intel.com/memory.max": "16000", + "gpu.intel.com/gpu-numbers": "0.1", + "gpu.intel.com/cards": "card0.card1", + "gpu.intel.com/tiles": "2", + "gpu.intel.com/numa-gpu-map": "0-0_1-1", }, }, { @@ -554,25 +410,14 @@ func getTestCases() []testcase { }, name: "successful labeling with one 0x8086 card and numa node info", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{ - "1/i915_capabilities": []byte( - "platform: newnew\n" + - "gen: 9"), - }, expectedRetval: nil, expectedLabels: labelMap{ - "gpu.intel.com/graphics_version": "9", - "gpu.intel.com/media_version": "9", - "gpu.intel.com/millicores": "1000", - "gpu.intel.com/memory.max": "8000", - "gpu.intel.com/platform_newnew.count": "1", - "gpu.intel.com/platform_newnew.present": "true", - "gpu.intel.com/platform_newnew.tiles": "1", - "gpu.intel.com/platform_gen": "9", - "gpu.intel.com/gpu-numbers": "1", - "gpu.intel.com/cards": "card1", - "gpu.intel.com/tiles": "1", - "gpu.intel.com/numa-gpu-map": "1-1", + "gpu.intel.com/millicores": "1000", + "gpu.intel.com/memory.max": "8000", + "gpu.intel.com/gpu-numbers": "1", + "gpu.intel.com/cards": "card1", + "gpu.intel.com/tiles": "1", + "gpu.intel.com/numa-gpu-map": "1-1", }, }, { @@ -663,7 +508,6 @@ func getTestCases() []testcase { }, name: "successful labeling with two cards and numa node info", memoryOverride: 16000000000, - capabilityFile: map[string][]byte{}, expectedRetval: nil, expectedLabels: labelMap{ "gpu.intel.com/cards": "card0.card1.card10.card11.card12.card13.card14.card15.card16.ca", @@ -680,22 +524,6 @@ func getTestCases() []testcase { } func (tc *testcase) createFiles(t *testing.T, sysfs, root string) { - var err error - - for filename, body := range tc.capabilityFile { - filePath := path.Join(root, filename) - dirOnly := path.Dir(filePath) - - err = os.MkdirAll(dirOnly, 0750) - if err != nil { - t.Fatalf("Failed to create base directories: %+v", err) - } - - if err = os.WriteFile(filePath, body, 0600); err != nil { - t.Fatalf("Failed to create fake capability file: %+v", err) - } - } - for _, sysfsdir := range tc.sysfsdirs { if err := os.MkdirAll(path.Join(sysfs, sysfsdir), 0750); err != nil { t.Fatalf("Failed to create fake sysfs directory: %+v", err) @@ -768,7 +596,7 @@ func TestLabeling(t *testing.T) { os.Setenv(memoryReservedEnv, strconv.FormatUint(tc.memoryReserved, 10)) os.Setenv(pciGroupingEnv, strconv.FormatUint(tc.pciGroupLevel, 10)) - labeler := newLabeler(sysfs, subroot) + labeler := newLabeler(sysfs) err = labeler.createLabels() if err != nil && tc.expectedRetval == nil || err == nil && tc.expectedRetval != nil { diff --git a/deployments/gpu_plugin/base/intel-gpu-plugin.yaml b/deployments/gpu_plugin/base/intel-gpu-plugin.yaml index 94180d405..30a6158d6 100644 --- a/deployments/gpu_plugin/base/intel-gpu-plugin.yaml +++ b/deployments/gpu_plugin/base/intel-gpu-plugin.yaml @@ -13,18 +13,6 @@ spec: labels: app: intel-gpu-plugin spec: - initContainers: - - name: intel-gpu-initcontainer - image: intel/intel-gpu-initcontainer:devel - imagePullPolicy: IfNotPresent - securityContext: - seLinuxOptions: - type: "container_device_plugin_init_t" - readOnlyRootFilesystem: true - allowPrivilegeEscalation: false - volumeMounts: - - mountPath: /etc/kubernetes/node-feature-discovery/source.d/ - name: nfd-features containers: - name: intel-gpu-plugin env: @@ -47,7 +35,7 @@ spec: - name: devfs mountPath: /dev/dri readOnly: true - - name: sysfs + - name: sysfsdrm mountPath: /sys/class/drm readOnly: true - name: kubeletsockets @@ -56,15 +44,11 @@ spec: - name: devfs hostPath: path: /dev/dri - - name: sysfs + - name: sysfsdrm hostPath: path: /sys/class/drm - name: kubeletsockets hostPath: path: /var/lib/kubelet/device-plugins - - name: nfd-features - hostPath: - path: /etc/kubernetes/node-feature-discovery/source.d/ - type: DirectoryOrCreate nodeSelector: kubernetes.io/arch: amd64 diff --git a/deployments/gpu_plugin/overlays/fractional_resources/add-kubelet-crt-mount.yaml b/deployments/gpu_plugin/overlays/fractional_resources/add-kubelet-crt-mount.yaml deleted file mode 100644 index ff4afafa8..000000000 --- a/deployments/gpu_plugin/overlays/fractional_resources/add-kubelet-crt-mount.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: intel-gpu-plugin -spec: - template: - spec: - containers: - - name: intel-gpu-plugin - volumeMounts: - - name: kubeletcrt - mountPath: /var/lib/kubelet/pki/kubelet.crt - volumes: - - name: kubeletcrt - hostPath: - path: /var/lib/kubelet/pki/kubelet.crt - type: FileOrCreate diff --git a/deployments/gpu_plugin/overlays/fractional_resources/add-mounts.yaml b/deployments/gpu_plugin/overlays/fractional_resources/add-mounts.yaml new file mode 100644 index 000000000..797bdb82f --- /dev/null +++ b/deployments/gpu_plugin/overlays/fractional_resources/add-mounts.yaml @@ -0,0 +1,34 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-gpu-plugin +spec: + template: + spec: + containers: + - name: intel-gpu-plugin + volumeMounts: + - name: kubeletcrt + mountPath: /var/lib/kubelet/pki/kubelet.crt + - mountPath: /etc/kubernetes/node-feature-discovery/features.d/ + name: nfd-features + - mountPath: /sys/devices + name: sysfsdevices + readOnly: true + - name: podresources + mountPath: /var/lib/kubelet/pod-resources + volumes: + - name: kubeletcrt + hostPath: + path: /var/lib/kubelet/pki/kubelet.crt + type: FileOrCreate + - name: sysfsdevices + hostPath: + path: /sys/devices + - name: nfd-features + hostPath: + path: /etc/kubernetes/node-feature-discovery/features.d/ + type: DirectoryOrCreate + - name: podresources + hostPath: + path: /var/lib/kubelet/pod-resources diff --git a/deployments/gpu_plugin/overlays/fractional_resources/add-podresource-mount.yaml b/deployments/gpu_plugin/overlays/fractional_resources/add-podresource-mount.yaml deleted file mode 100644 index d127334f7..000000000 --- a/deployments/gpu_plugin/overlays/fractional_resources/add-podresource-mount.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: intel-gpu-plugin -spec: - template: - spec: - containers: - - name: intel-gpu-plugin - volumeMounts: - - name: podresources - mountPath: /var/lib/kubelet/pod-resources - volumes: - - name: podresources - hostPath: - path: /var/lib/kubelet/pod-resources - \ No newline at end of file diff --git a/deployments/gpu_plugin/overlays/fractional_resources/kustomization.yaml b/deployments/gpu_plugin/overlays/fractional_resources/kustomization.yaml index 6f0cf4107..d5e347974 100644 --- a/deployments/gpu_plugin/overlays/fractional_resources/kustomization.yaml +++ b/deployments/gpu_plugin/overlays/fractional_resources/kustomization.yaml @@ -7,7 +7,7 @@ patches: - path: add-serviceaccount.yaml target: kind: DaemonSet - - path: add-podresource-mount.yaml + - path: add-mounts.yaml target: kind: DaemonSet - path: add-args.yaml @@ -16,6 +16,3 @@ patches: - path: add-nodeselector-intel-gpu.yaml target: kind: DaemonSet - - path: add-kubelet-crt-mount.yaml - target: - kind: DaemonSet diff --git a/deployments/operator/samples/deviceplugin_v1_gpudeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_gpudeviceplugin.yaml index 0c024c2a3..a6ac821ab 100644 --- a/deployments/operator/samples/deviceplugin_v1_gpudeviceplugin.yaml +++ b/deployments/operator/samples/deviceplugin_v1_gpudeviceplugin.yaml @@ -4,7 +4,6 @@ metadata: name: gpudeviceplugin-sample spec: image: intel/intel-gpu-plugin:0.27.1 - initImage: intel/intel-gpu-initcontainer:0.27.1 sharedDevNum: 10 logLevel: 4 nodeSelector: diff --git a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go index 8a7285a6a..6a3f55b4a 100644 --- a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go @@ -86,12 +86,6 @@ func (r *GpuDevicePlugin) ValidateDelete() (admission.Warnings, error) { } func (r *GpuDevicePlugin) validatePlugin() error { - if r.Spec.InitImage != "" { - if err := validatePluginImage(r.Spec.InitImage, "intel-gpu-initcontainer", gpuMinVersion); err != nil { - return err - } - } - if r.Spec.SharedDevNum == 1 && r.Spec.PreferredAllocationPolicy != "none" { return errors.Errorf("PreferredAllocationPolicy is valid only when setting sharedDevNum > 1") } diff --git a/pkg/controllers/gpu/controller.go b/pkg/controllers/gpu/controller.go index fa8ac6178..dca852d35 100644 --- a/pkg/controllers/gpu/controller.go +++ b/pkg/controllers/gpu/controller.go @@ -151,15 +151,19 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { if devicePlugin.Spec.ResourceManager { daemonSet.Spec.Template.Spec.ServiceAccountName = serviceAccountName addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory) - addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources") + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", false) addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", v1.HostPathFileOrCreate) - addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt") + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", true) + addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", v1.HostPathDirectory) + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", false) + addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "sysfsdevices", "/sys/devices", v1.HostPathDirectory) + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "sysfsdevices", "/sys/devices", true) } return daemonSet } -func addVolumeMountIfMissing(spec *v1.PodSpec, name, mountPath string) { +func addVolumeMountIfMissing(spec *v1.PodSpec, name, mountPath string, readOnly bool) { for _, mount := range spec.Containers[0].VolumeMounts { if mount.Name == name { return @@ -169,6 +173,7 @@ func addVolumeMountIfMissing(spec *v1.PodSpec, name, mountPath string) { spec.Containers[0].VolumeMounts = append(spec.Containers[0].VolumeMounts, v1.VolumeMount{ Name: name, MountPath: mountPath, + ReadOnly: readOnly, }) } @@ -206,11 +211,11 @@ func setInitContainer(spec *v1.PodSpec, imageName string) { VolumeMounts: []v1.VolumeMount{ { MountPath: "/etc/kubernetes/node-feature-discovery/source.d/", - Name: "nfd-features", + Name: "nfd-sources", }, }, }} - addVolumeIfMissing(spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/source.d/", v1.HostPathDirectoryOrCreate) + addVolumeIfMissing(spec, "nfd-sources", "/etc/kubernetes/node-feature-discovery/source.d/", v1.HostPathDirectoryOrCreate) } func removeVolume(volumes []v1.Volume, name string) []v1.Volume { @@ -267,11 +272,40 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( } newargs := getPodArgs(dp) - if strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") != strings.Join(newargs, " ") { + oldArgString := strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") + + if oldArgString != strings.Join(newargs, " ") { ds.Spec.Template.Spec.Containers[0].Args = newargs updated = true } + hadRM := strings.Contains(oldArgString, "-resource-manager") + + // Add volumes if they do not exist, or remove them when + // labels are not requested anymore. + if !hadRM && dp.Spec.ResourceManager { + addVolumeIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", false) + addVolumeIfMissing(&ds.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", v1.HostPathFileOrCreate) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", true) + addVolumeIfMissing(&ds.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", v1.HostPathDirectory) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", false) + addVolumeIfMissing(&ds.Spec.Template.Spec, "sysfsdevices", "/sys/devices", v1.HostPathDirectory) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "sysfsdevices", "/sys/devices", true) + } else if hadRM && !dp.Spec.ResourceManager { + volMounts := &ds.Spec.Template.Spec.Containers[0].VolumeMounts + *volMounts = removeVolumeMount(*volMounts, "nfd-features") + *volMounts = removeVolumeMount(*volMounts, "sysfsdevices") + *volMounts = removeVolumeMount(*volMounts, "kubeletcrt") + *volMounts = removeVolumeMount(*volMounts, "podresources") + + volumes := &ds.Spec.Template.Spec.Volumes + *volumes = removeVolume(*volumes, "nfd-features") + *volumes = removeVolume(*volumes, "sysfsdevices") + *volumes = removeVolume(*volumes, "kubeletcrt") + *volumes = removeVolume(*volumes, "podresources") + } + newServiceAccountName := "default" if dp.Spec.ResourceManager { newServiceAccountName = serviceAccountName @@ -281,7 +315,7 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) ( ds.Spec.Template.Spec.ServiceAccountName = newServiceAccountName if dp.Spec.ResourceManager { addVolumeIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory) - addVolumeMountIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources") + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", false) } else { ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "podresources") ds.Spec.Template.Spec.Containers[0].VolumeMounts = removeVolumeMount(ds.Spec.Template.Spec.Containers[0].VolumeMounts, "podresources") diff --git a/pkg/controllers/gpu/controller_test.go b/pkg/controllers/gpu/controller_test.go index 9d4db1f04..94b28b9cc 100644 --- a/pkg/controllers/gpu/controller_test.go +++ b/pkg/controllers/gpu/controller_test.go @@ -17,6 +17,7 @@ package gpu import ( "reflect" + "strings" "testing" apps "k8s.io/api/apps/v1" @@ -100,7 +101,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet ReadOnly: true, }, { - Name: "sysfs", + Name: "sysfsdrm", MountPath: "/sys/class/drm", ReadOnly: true, }, @@ -122,7 +123,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet }, }, { - Name: "sysfs", + Name: "sysfsdrm", VolumeSource: v1.VolumeSource{ HostPath: &v1.HostPathVolumeSource{ Path: "/sys/class/drm", @@ -143,49 +144,106 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet }, } - // add the optional init container - if devicePlugin.Spec.InitImage != "" { - setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage) - } - // add service account if resource manager is enabled if devicePlugin.Spec.ResourceManager { daemonSet.Spec.Template.Spec.ServiceAccountName = serviceAccountName + addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory) - addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources") + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", false) addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", v1.HostPathFileOrCreate) - addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt") + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", true) + addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", v1.HostPathDirectory) + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", false) + addVolumeIfMissing(&daemonSet.Spec.Template.Spec, "sysfsdevices", "/sys/devices", v1.HostPathDirectory) + addVolumeMountIfMissing(&daemonSet.Spec.Template.Spec, "sysfsdevices", "/sys/devices", true) } return &daemonSet } +func (c *controller) updateDaemonSetExpected(rawObj client.Object, ds *apps.DaemonSet) { + dp := rawObj.(*devicepluginv1.GpuDevicePlugin) + + argString := strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") + + hadRM := strings.Contains(argString, "-resource-manager") + + if !hadRM && dp.Spec.ResourceManager { + ds.Spec.Template.Spec.ServiceAccountName = "gpu-manager-sa" + + addVolumeIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", v1.HostPathDirectory) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "podresources", "/var/lib/kubelet/pod-resources", false) + addVolumeIfMissing(&ds.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", v1.HostPathFileOrCreate) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "kubeletcrt", "/var/lib/kubelet/pki/kubelet.crt", true) + addVolumeIfMissing(&ds.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", v1.HostPathDirectory) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/features.d/", false) + addVolumeIfMissing(&ds.Spec.Template.Spec, "sysfsdevices", "/sys/devices", v1.HostPathDirectory) + addVolumeMountIfMissing(&ds.Spec.Template.Spec, "sysfsdevices", "/sys/devices", true) + } else if hadRM && !dp.Spec.ResourceManager { + ds.Spec.Template.Spec.ServiceAccountName = "default" + + volMounts := &ds.Spec.Template.Spec.Containers[0].VolumeMounts + *volMounts = removeVolumeMount(*volMounts, "nfd-features") + *volMounts = removeVolumeMount(*volMounts, "sysfsdevices") + *volMounts = removeVolumeMount(*volMounts, "kubeletcrt") + *volMounts = removeVolumeMount(*volMounts, "podresources") + + volumes := &ds.Spec.Template.Spec.Volumes + *volumes = removeVolume(*volumes, "nfd-features") + *volumes = removeVolume(*volumes, "sysfsdevices") + *volumes = removeVolume(*volumes, "kubeletcrt") + *volumes = removeVolume(*volumes, "podresources") + } + + ds.Spec.Template.Spec.Containers[0].Args = getPodArgs(dp) +} + // Test that GPU daemonsets created by using go:embed // are equal to the expected daemonsets. func TestNewDamonSetGPU(t *testing.T) { tcases := []struct { - name string - resourceManager bool - isInitImage bool + name string + rm bool }{ { - "plugin without resource manager and without initcontainer", - false, - false, + "plugin with resource manager", + true, }, { - "plugin without resource manager and with initcontainer", + "plugin without resource manager", false, - true, }, + } + + c := &controller{} + + for _, tc := range tcases { + plugin := &devicepluginv1.GpuDevicePlugin{} + + plugin.Spec.ResourceManager = tc.rm + + t.Run(tc.name, func(t *testing.T) { + expected := c.newDaemonSetExpected(plugin) + actual := c.NewDaemonSet(plugin) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("expected and actuall daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) + } + }) + } +} + +func TestUpdateDamonSetGPU(t *testing.T) { + tcases := []struct { + name string + rmInitially bool + }{ { - "plugin with resource manager and without initcontainer", - true, + "plugin without rm and then with rm", false, }, { - "plugin with resource manager and with initcontainer", - true, + "plugin with rm and then without rm", true, }, } @@ -193,21 +251,30 @@ func TestNewDamonSetGPU(t *testing.T) { c := &controller{} for _, tc := range tcases { - plugin := &devicepluginv1.GpuDevicePlugin{} - if tc.resourceManager { - plugin.Spec.ResourceManager = true - } + before := &devicepluginv1.GpuDevicePlugin{} + + before.Spec.ResourceManager = tc.rmInitially - if tc.isInitImage { - plugin.Spec.InitImage = "intel/intel-gpu-initcontainer:devel" - } + after := &devicepluginv1.GpuDevicePlugin{} + + after.Spec.ResourceManager = !tc.rmInitially t.Run(tc.name, func(t *testing.T) { - expected := c.newDaemonSetExpected(plugin) - actual := c.NewDaemonSet(plugin) + expected := c.newDaemonSetExpected(before) + actual := c.NewDaemonSet(before) if !reflect.DeepEqual(expected, actual) { - t.Errorf("expected and actuall daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) + t.Errorf("expected and actual daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) + } + + updated := c.UpdateDaemonSet(after, actual) + if updated == false { + t.Error("daemonset didn't update while it should have") + } + c.updateDaemonSetExpected(after, expected) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("updated expected and actual daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) } }) } From 9b3b881cc2946f61235935d4133e1af2be7a0516 Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Tue, 5 Sep 2023 08:47:06 +0300 Subject: [PATCH 2/3] nfd: add rules to label nodes with different GPUs Signed-off-by: Tuomas Katila --- cmd/gpu_nfdhook/README.md | 68 +------ cmd/gpu_plugin/README.md | 4 + cmd/gpu_plugin/labels.md | 87 ++++++++ .../node-feature-rules/kustomization.yaml | 1 + .../platform-labeling-rules.yaml | 189 ++++++++++++++++++ 5 files changed, 282 insertions(+), 67 deletions(-) create mode 100644 cmd/gpu_plugin/labels.md create mode 100644 deployments/nfd/overlays/node-feature-rules/platform-labeling-rules.yaml diff --git a/cmd/gpu_nfdhook/README.md b/cmd/gpu_nfdhook/README.md index 3ff387354..7a4a1796e 100644 --- a/cmd/gpu_nfdhook/README.md +++ b/cmd/gpu_nfdhook/README.md @@ -25,70 +25,4 @@ by the NFD, allowing for finer grained resource management for GPU-using PODs. In the NFD deployment, the hook requires `/host-sys` -folder to have the host `/sys`-folder content mounted. Write access is not necessary. -## GPU memory - -GPU memory amount is read from sysfs `gt/gt*` files and turned into a label. -There are two supported environment variables named `GPU_MEMORY_OVERRIDE` and -`GPU_MEMORY_RESERVED`. Both are supposed to hold numeric byte amounts. For systems with -older kernel drivers or GPUs which do not support reading the GPU memory -amount, the `GPU_MEMORY_OVERRIDE` environment variable value is turned into a GPU -memory amount label instead of a read value. `GPU_MEMORY_RESERVED` value will be -scoped out from the GPU memory amount found from sysfs. - -## Default labels - -Following labels are created by default. You may turn numeric labels into extended resources with NFD. - -name | type | description| ------|------|------| -|`gpu.intel.com/millicores`| number | node GPU count * 1000. Can be used as a finer grained shared execution fraction. -|`gpu.intel.com/memory.max`| number | sum of detected [GPU memory amounts](#gpu-memory) in bytes OR environment variable value * GPU count -|`gpu.intel.com/cards`| string | list of card names separated by '`.`'. The names match host `card*`-folders under `/sys/class/drm/`. Deprecated, use `gpu-numbers`. -|`gpu.intel.com/gpu-numbers`| string | list of numbers separated by '`.`'. The numbers correspond to device file numbers for the primary nodes of given GPUs in kernel DRI subsystem, listed as `/dev/dri/card` in devfs, and `/sys/class/drm/card` in sysfs. -|`gpu.intel.com/tiles`| number | sum of all detected GPU tiles in the system. -|`gpu.intel.com/numa-gpu-map`| string | list of numa node to gpu mappings. - -If the value of the `gpu-numbers` label would not fit into the 63 character length limit, you will also get labels `gpu-numbers2`, -`gpu-numbers3`... until all the gpu numbers have been labeled. - -The tile count `gpu.intel.com/tiles` describes the total amount of tiles on the system. System is expected to be homogeneous, and thus the number of tiles per GPU can be calculated by dividing the tile count with GPU count. - -The `numa-gpu-map` label is a list of numa to gpu mapping items separated by `_`. Each list item has a numa node id combined with a list of gpu indices. e.g. 0-1.2.3 would mean: numa node 0 has gpus 1, 2 and 3. More complex example would be: 0-0.1_1-3.4 where numa node 0 would have gpus 0 and 1, and numa node 1 would have gpus 3 and 4. As with `gpu-numbers`, this label will be extended to multiple labels if the length of the value exceeds the max label length. - -## PCI-groups (optional) - -GPUs which share the same pci paths under `/sys/devices/pci*` can be grouped into a label. GPU nums are separated by '`.`' and -groups are separated by '`_`'. The label is created only if environment variable named `GPU_PCI_GROUPING_LEVEL` has a value greater -than zero. GPUs are considered to belong to the same group, if as many identical folder names are found for the GPUs, as is the value -of the environment variable. Counting starts from the folder name which starts with `pci`. - -For example, the SG1 card has 4 GPUs, which end up sharing pci-folder names under `/sys/devices`. With a `GPU_PCI_GROUPING_LEVEL` -of 3, a node with two such SG1 cards could produce a `pci-groups` label with a value of `0.1.2.3_4.5.6.7`. - -name | type | description| ------|------|------| -|`gpu.intel.com/pci-groups`| string | list of pci-groups separated by '`_`'. GPU numbers in the groups are separated by '`.`'. The numbers correspond to device file numbers for the primary nodes of given GPUs in kernel DRI subsystem, listed as `/dev/dri/card` in devfs, and `/sys/class/drm/card` in sysfs. - -If the value of the `pci-groups` label would not fit into the 63 character length limit, you will also get labels `pci-groups2`, -`pci-groups3`... until all the pci groups have been labeled. - -## Capability labels (optional) - -Capability labels are created from information found inside debugfs, and therefore -unfortunately require running the NFD worker as root. Due to coming from debugfs, -which is not guaranteed to be stable, these are not guaranteed to be stable either. -If you do not need these, simply do not run NFD worker as root, that is also more secure. -Depending on your kernel driver, running the NFD hook as root may introduce following labels: - -name | type | description| ------|------|------| -|`gpu.intel.com/platform_gen`| string | GPU platform generation name, typically an integer. Deprecated. -|`gpu.intel.com/media_version`| string | GPU platform Media pipeline generation name, typically a number. Deprecated. -|`gpu.intel.com/graphics_version`| string | GPU platform graphics/compute pipeline generation name, typically a number. Deprecated. -|`gpu.intel.com/platform_.count`| number | GPU count for the named platform. -|`gpu.intel.com/platform_.tiles`| number | GPU tile count in the GPUs of the named platform. -|`gpu.intel.com/platform_.present`| string | "true" for indicating the presense of the GPU platform. - -## Limitations - -For the above to work as intended, GPUs on the same node must be identical in their capabilities. +For detailed info about the labels created by the NFD hook, see the [labels documentation](../gpu_plugin/labels.md). diff --git a/cmd/gpu_plugin/README.md b/cmd/gpu_plugin/README.md index 484c09e2f..e5e80324b 100644 --- a/cmd/gpu_plugin/README.md +++ b/cmd/gpu_plugin/README.md @@ -23,6 +23,7 @@ Table of Contents * [Fractional resources details](#fractional-resources-details) * [Verify Plugin Registration](#verify-plugin-registration) * [Testing and Demos](#testing-and-demos) +* [Labels created by GPU plugin](#labels-created-by-gpu-plugin) * [Issues with media workloads on multi-GPU setups](#issues-with-media-workloads-on-multi-gpu-setups) * [Workaround for QSV and VA-API](#workaround-for-qsv-and-va-api) @@ -339,6 +340,9 @@ The GPU plugin functionality can be verified by deploying an [OpenCL image](../. Warning FailedScheduling default-scheduler 0/1 nodes are available: 1 Insufficient gpu.intel.com/i915. ``` +## Labels created by GPU plugin + +If installed with NFD and started with resource-management, plugin will export a set of labels for the node. For detailed info, see [labeling documentation](./labels.md). ## Issues with media workloads on multi-GPU setups diff --git a/cmd/gpu_plugin/labels.md b/cmd/gpu_plugin/labels.md new file mode 100644 index 000000000..19a00c66e --- /dev/null +++ b/cmd/gpu_plugin/labels.md @@ -0,0 +1,87 @@ +# Labels + +GPU labels originate from two main sources: NFD rules and GPU plugin (& NFD hook). + +## NFD rules + +NFD rule is a method to instruct NFD to add certain label(s) to node based on the devices detected on it. There is a generic rule to identify all Intel GPUs. It will add labels for each PCI device type. For example, a Tigerlake iGPU (PCI Id 0x9a49) will show up as: + +``` +gpu.intel.com/device-id.0300-9a49.count=1 +gpu.intel.com/device-id.0300-9a49.present=true +``` + +For data center GPUs, there are more specific rules which will create additional labels for GPU family, product and device count. For example, Flex 170: +``` +gpu.intel.com/device.count=1 +gpu.intel.com/family=Flex_Series +gpu.intel.com/product=Flex_170 +``` + +For MAX 1550: +``` +gpu.intel.com/device.count=2 +gpu.intel.com/family=Max_Series +gpu.intel.com/product=Max_1550 +``` + +Current covered platforms/devices are: Flex 140, Flex 170, Max 1100 and Max 1550. + +To identify other GPUs, see the graphics processor table [here](https://dgpu-docs.intel.com/devices/hardware-table.html#graphics-processor-table). + +## GPU Plugin and NFD hook + +In GPU plugin, these labels are only applied when [Resource Management](README.md#fractional-resources-details) is enabled. With the NFD hook, labels are created regardless of how GPU plugin is configured. + +Numeric labels are converted into extended resources for the node (with NFD) and other labels are used directly by [GPU Aware Scheduling (GAS)](https://github.com/intel/platform-aware-scheduling/tree/master/gpu-aware-scheduling). Extended resources should only be used with GAS as Kubernetes scheduler doesn't properly handle resource allocations with multiple GPUs. + +### Default labels + +Following labels are created by default. + +name | type | description| +-----|------|------| +|`gpu.intel.com/millicores`| number | node GPU count * 1000. +|`gpu.intel.com/memory.max`| number | sum of detected [GPU memory amounts](#gpu-memory) in bytes OR environment variable value * GPU count +|`gpu.intel.com/cards`| string | list of card names separated by '`.`'. The names match host `card*`-folders under `/sys/class/drm/`. Deprecated, use `gpu-numbers`. +|`gpu.intel.com/gpu-numbers`| string | list of numbers separated by '`.`'. The numbers correspond to device file numbers for the primary nodes of given GPUs in kernel DRI subsystem, listed as `/dev/dri/card` in devfs, and `/sys/class/drm/card` in sysfs. +|`gpu.intel.com/tiles`| number | sum of all detected GPU tiles in the system. +|`gpu.intel.com/numa-gpu-map`| string | list of numa node to gpu mappings. + +If the value of the `gpu-numbers` label would not fit into the 63 character length limit, you will also get labels `gpu-numbers2`, +`gpu-numbers3`... until all the gpu numbers have been labeled. + +The tile count `gpu.intel.com/tiles` describes the total amount of tiles on the system. System is expected to be homogeneous, and thus the number of tiles per GPU can be calculated by dividing the tile count with GPU count. + +The `numa-gpu-map` label is a list of numa to gpu mapping items separated by `_`. Each list item has a numa node id combined with a list of gpu indices. e.g. 0-1.2.3 would mean: numa node 0 has gpus 1, 2 and 3. More complex example would be: 0-0.1_1-3.4 where numa node 0 would have gpus 0 and 1, and numa node 1 would have gpus 3 and 4. As with `gpu-numbers`, this label will be extended to multiple labels if the length of the value exceeds the max label length. + +### PCI-groups (optional) + +GPUs which share the same pci paths under `/sys/devices/pci*` can be grouped into a label. GPU nums are separated by '`.`' and +groups are separated by '`_`'. The label is created only if environment variable named `GPU_PCI_GROUPING_LEVEL` has a value greater +than zero. GPUs are considered to belong to the same group, if as many identical folder names are found for the GPUs, as is the value +of the environment variable. Counting starts from the folder name which starts with `pci`. + +For example, the SG1 card has 4 GPUs, which end up sharing pci-folder names under `/sys/devices`. With a `GPU_PCI_GROUPING_LEVEL` +of 3, a node with two such SG1 cards could produce a `pci-groups` label with a value of `0.1.2.3_4.5.6.7`. + +name | type | description| +-----|------|------| +|`gpu.intel.com/pci-groups`| string | list of pci-groups separated by '`_`'. GPU numbers in the groups are separated by '`.`'. The numbers correspond to device file numbers for the primary nodes of given GPUs in kernel DRI subsystem, listed as `/dev/dri/card` in devfs, and `/sys/class/drm/card` in sysfs. + +If the value of the `pci-groups` label would not fit into the 63 character length limit, you will also get labels `pci-groups2`, +`pci-groups3`... until all the pci groups have been labeled. + +### Limitations + +For the above to work as intended, GPUs on the same node must be identical in their capabilities. + +### GPU memory + +GPU memory amount is read from sysfs `gt/gt*` files and turned into a label. +There are two supported environment variables named `GPU_MEMORY_OVERRIDE` and +`GPU_MEMORY_RESERVED`. Both are supposed to hold numeric byte amounts. For systems with +older kernel drivers or GPUs which do not support reading the GPU memory +amount, the `GPU_MEMORY_OVERRIDE` environment variable value is turned into a GPU +memory amount label instead of a read value. `GPU_MEMORY_RESERVED` value will be +scoped out from the GPU memory amount found from sysfs. diff --git a/deployments/nfd/overlays/node-feature-rules/kustomization.yaml b/deployments/nfd/overlays/node-feature-rules/kustomization.yaml index 540c771b8..c0bf90957 100644 --- a/deployments/nfd/overlays/node-feature-rules/kustomization.yaml +++ b/deployments/nfd/overlays/node-feature-rules/kustomization.yaml @@ -2,3 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - node-feature-rules.yaml +- platform-labeling-rules.yaml diff --git a/deployments/nfd/overlays/node-feature-rules/platform-labeling-rules.yaml b/deployments/nfd/overlays/node-feature-rules/platform-labeling-rules.yaml new file mode 100644 index 000000000..02ebdbab1 --- /dev/null +++ b/deployments/nfd/overlays/node-feature-rules/platform-labeling-rules.yaml @@ -0,0 +1,189 @@ +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: intel-gpu-platform-labeling +spec: + rules: + # generic rule for older and upcoming devices + - labels: + labelsTemplate: | + {{ range .pci.device }}gpu.intel.com/device-id.{{ .class }}-{{ .device }}.present=true + {{ end }} + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0300" + - "0380" + vendor: + op: In + value: + - "8086" + name: intel.gpu.generic.deviceid + - labels: + labelsTemplate: gpu.intel.com/device-id.0300-{{ (index .pci.device 0).device }}.count={{ len .pci.device }} + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0300" + vendor: + op: In + value: + - "8086" + name: intel.gpu.generic.count.300 + - labels: + labelsTemplate: gpu.intel.com/device-id.0380-{{ (index .pci.device 0).device }}.count={{ len .pci.device }} + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0380" + vendor: + op: In + value: + - "8086" + name: intel.gpu.generic.count.380 + - labels: + gpu.intel.com/product: "Max_1100" + labelsTemplate: "gpu.intel.com/device.count={{ len .pci.device }}" + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0380" + vendor: + op: In + value: + - "8086" + device: + op: In + value: + - "0bda" + name: intel.gpu.max.1100 + - labels: + gpu.intel.com/product: "Max_1550" + labelsTemplate: "gpu.intel.com/device.count={{ len .pci.device }}" + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0380" + vendor: + op: In + value: + - "8086" + device: + op: In + value: + - "0bd5" + name: intel.gpu.max.1550 + - labels: + gpu.intel.com/family: "Max_Series" + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0380" + vendor: + op: In + value: + - "8086" + device: + op: In + value: + - "0bda" + - "0bd5" + - "0bd9" + - "0bdb" + - "0bd7" + - "0bd6" + - "0bd0" + name: intel.gpu.max.series + - labels: + gpu.intel.com/family: "Flex_Series" + gpu.intel.com/product: "Flex_170" + labelsTemplate: "gpu.intel.com/device.count={{ len .pci.device }}" + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0380" + vendor: + op: In + value: + - "8086" + device: + op: In + value: + - "56c0" + name: intel.gpu.flex.170 + - labels: + gpu.intel.com/family: "Flex_Series" + gpu.intel.com/product: "Flex_140" + labelsTemplate: "gpu.intel.com/device.count={{ len .pci.device }}" + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0380" + vendor: + op: In + value: + - "8086" + device: + op: In + value: + - "56c1" + name: intel.gpu.flex.140 + - labels: + gpu.intel.com/family: "A_Series" + matchFeatures: + - feature: pci.device + matchExpressions: + class: + op: In + value: + - "0300" + vendor: + op: In + value: + - "8086" + device: + op: In + value: + - "56a6" + - "56a5" + - "56a1" + - "56a0" + - "5694" + - "5693" + - "5692" + - "5691" + - "5690" + - "56b3" + - "56b2" + - "56a4" + - "56a3" + - "5697" + - "5696" + - "5695" + - "56b1" + - "56b0" + name: intel.gpu.a.series From bd2a35d780278500711c11ee6c711e37e86547de Mon Sep 17 00:00:00 2001 From: Tuomas Katila Date: Wed, 6 Sep 2023 13:15:15 +0300 Subject: [PATCH 3/3] fix crash with rm when kubelet request timeouts Signed-off-by: Tuomas Katila --- cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go index 60c060406..56ed3d3fd 100644 --- a/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go +++ b/cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go @@ -337,7 +337,7 @@ func (rm *resourceManager) listPods() (*v1.PodList, error) { for i := 0; i < kubeletAPIMaxRetries; i++ { if podList, err := rm.listPodsFromKubelet(); err == nil { return podList, nil - } else if errors.As(err, neterr); neterr.Timeout() { + } else if errors.As(err, &neterr) && neterr.Timeout() { continue }