gpu: copy nfdhook functionality to gpu-plugin

In order to support v0.14+ NFD and extended resources,
write node labels into a file and store it for NFD's features.d
directory.

The old functionality still exists in the initcontainer, but if it's
used NFD has to be configured to allow it.

Signed-off-by: Tuomas Katila <[email protected]>

Author: tkatila

cmd/gpu_nfdhook/README.md

@@ -11,6 +11,8 @@ Table of Contents
 
 ## Introduction
 
+***NOTE:*** NFD's binary hook support will be turned off by default in the 0.14 release. If one desires to use it then, NFD's default configuration has to be modified to allow binary hooks. Due to the change, GPU binary hook's functionality has been moved to GPU plugin.
+
 This is the [Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery)
 binary hook implementation for the Intel GPUs. The intel-gpu-initcontainer (which
 is built with the other images) can be used as part of the gpu-plugin deployment

cmd/gpu_nfdhook/main.go

@@ -15,9 +15,7 @@
 package main
 
 import (
-	"os"
-
-	"k8s.io/klog/v2"
+	l "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
 )
 
 const (
@@ -27,13 +25,5 @@ const (
 )
 
 func main() {
-	l := newLabeler(sysfsDRMDirectory, debugfsDRIDirectory)
-
-	err := l.createLabels()
-	if err != nil {
-		klog.Errorf("%+v", err)
-		os.Exit(1)
-	}
-
-	l.printLabels()
+	l.CreateAndPrintLabels(sysfsDRMDirectory, debugfsDRIDirectory)
 }

cmd/gpu_plugin/README.md

@@ -52,6 +52,7 @@ backend libraries can offload compute operations to GPU.
 | -resource-manager | - | disabled | Enable fractional resource management, [see also dependencies](#fractional-resources) |
 | -shared-dev-num | int | 1 | Number of containers that can share the same GPU device |
 | -allocation-policy | string | none | 3 possible values: balanced, packed, none. For shared-dev-num > 1: _balanced_ mode spreads workloads among GPU devices, _packed_ mode fills one GPU fully before moving to next, and _none_ selects first available device from kubelet. Default is _none_. Allocation policy does not have an effect when resource manager is enabled. |
+| -export-resource-lables | bool | true | Export resource labels to NFD via features.d directory. Replaces initcontainer's functionality. |
 
 The plugin also accepts a number of other arguments (common to all plugins) related to logging.
 Please use the -h option to see the complete list of logging related options.

cmd/gpu_plugin/gpu_plugin.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2022 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,6 +31,7 @@ import (
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm"
+	l "github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 )
@@ -53,13 +54,17 @@ const (
 
 	// Period of device scans.
 	scanPeriod = 5 * time.Second
+
+	// Labeler's max update interval, 5min.
+	labelerMaxInterval = 5 * 60 * time.Second
 )
 
 type cliOptions struct {
 	preferredAllocationPolicy string
 	sharedDevNum              int
 	enableMonitoring          bool
 	resourceManagement        bool
+	exportResourceLabels      bool
 }
 
 type preferredAllocationPolicyFunc func(*pluginapi.ContainerPreferredAllocationRequest) []string
@@ -242,8 +247,9 @@ type devicePlugin struct {
 	controlDeviceReg *regexp.Regexp
 	pciAddressReg    *regexp.Regexp
 
-	scanTicker *time.Ticker
-	scanDone   chan bool
+	scanTicker    *time.Ticker
+	scanDone      chan bool
+	scanResources chan bool
 
 	resMan rm.ResourceManager
 
@@ -270,6 +276,7 @@ func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugi
 		scanTicker:       time.NewTicker(scanPeriod),
 		scanDone:         make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it
 		bypathFound:      true,
+		scanResources:    make(chan bool, 1),
 	}
 
 	if options.resourceManagement {
@@ -347,17 +354,26 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 			klog.Warning("Failed to scan: ", err)
 		}
 
+		countChanged := false
+
 		for name, prev := range previousCount {
 			count := devTree.DeviceTypeCount(name)
 			if count != prev {
 				klog.V(1).Infof("GPU scan update: %d->%d '%s' resources found", prev, count, name)
 
 				previousCount[name] = count
+
+				countChanged = true
 			}
 		}
 
 		notifier.Notify(devTree)
 
+		// Trigger resource scan if it's enabled.
+		if dp.options.exportResourceLabels && countChanged {
+			dp.scanResources <- true
+		}
+
 		select {
 		case <-dp.scanDone:
 			return nil
@@ -494,6 +510,7 @@ func main() {
 	flag.BoolVar(&opts.resourceManagement, "resource-manager", false, "fractional GPU resource management")
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
 	flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none")
+	flag.BoolVar(&opts.exportResourceLabels, "export-resource-labels", true, "export labels from node's GPUs and store them for NFD")
 	flag.Parse()
 
 	if opts.sharedDevNum < 1 {
@@ -515,6 +532,12 @@ func main() {
 	klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy)
 
 	plugin := newDevicePlugin(prefix+sysfsDrmDirectory, prefix+devfsDriDirectory, opts)
+
+	if opts.exportResourceLabels {
+		// Start labeler to export labels file for NFD.
+		go l.Run(prefix, labelerMaxInterval, plugin.scanResources)
+	}
+
 	manager := dpapi.NewManager(namespace, plugin)
 	manager.Run()
 }

cmd/gpu_plugin/gpu_plugin_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_plugin/render-device.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2021 Intel Corporation.
+# Copyright 2021-2023 Intel Corporation.
 #
 # SPDX-License-Identifier: Apache-2.0
 #

cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_nfdhook/labeler.go renamed to cmd/internal/labeler/labeler.go

@@ -1,4 +1,4 @@
-// Copyright 2020-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package labeler
 
 import (
 	"bufio"
 	"fmt"
 	"os"
+	"os/signal"
 	"path"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 	"github.com/pkg/errors"
@@ -47,6 +51,10 @@ const (
 	vendorString        = "0x8086"
 	labelMaxLength      = 63
 	labelControlChar    = "Z"
+
+	sysfsDRMDirectory   = "/sys/class/drm"
+	debugfsDRIDirectory = "/sys/kernel/debug/dri"
+	nfdFeatureFile      = "/etc/kubernetes/node-feature-discovery/features.d/intel-i915-resources.txt"
 )
 
 type labelMap map[string]string
@@ -58,6 +66,7 @@ type labeler struct {
 
 	sysfsDRMDir   string
 	debugfsDRIDir string
+	labelsChanged bool
 }
 
 func newLabeler(sysfsDRMDir, debugfsDRIDir string) *labeler {
@@ -67,6 +76,7 @@ func newLabeler(sysfsDRMDir, debugfsDRIDir string) *labeler {
 		gpuDeviceReg:     regexp.MustCompile(gpuDeviceRE),
 		controlDeviceReg: regexp.MustCompile(controlDeviceRE),
 		labels:           labelMap{},
+		labelsChanged:    true,
 	}
 }
 
@@ -345,6 +355,10 @@ func (l *labeler) createPCIGroupLabel(gpuNumList []string) string {
 
 // createLabels is the main function of plugin labeler, it creates label-value pairs for the gpus.
 func (l *labeler) createLabels() error {
+	prevLabels := l.labels
+
+	l.labels = labelMap{}
+
 	gpuNameList, err := l.scan()
 	if err != nil {
 		return err
@@ -431,6 +445,8 @@ func (l *labeler) createLabels() error {
 		}
 	}
 
+	l.labelsChanged = !reflect.DeepEqual(prevLabels, l.labels)
+
 	return nil
 }
 
@@ -455,8 +471,97 @@ func createNumaNodeMappingLabel(mapping map[int][]string) string {
 	return strings.Join(parts, "_")
 }
 
-func (l *labeler) printLabels() {
+func (l *labeler) printLabelsToFile(labelFile string) error {
+	f, err := os.OpenFile(labelFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		klog.Warning("Failed to open file:", err)
+
+		return err
+	}
+
+	for key, val := range l.labels {
+		if _, err := f.WriteString(key + "=" + val + "\n"); err != nil {
+			klog.Warning("Failed to write label to file:", err)
+
+			return err
+		}
+	}
+
+	return nil
+}
+
+func CreateAndPrintLabels(sysfsDRMDir, debugfsDRIDir string) {
+	l := newLabeler(sysfsDRMDir, debugfsDRIDir)
+
+	if err := l.createLabels(); err != nil {
+		klog.Warningf("failed to create labels: %+v", err)
+
+		return
+	}
+
 	for key, val := range l.labels {
 		fmt.Println(key + "=" + val)
 	}
 }
+
+// Gathers node's GPU labels on channel trigger or timeout, and write them to a file.
+// The created label file is deleted on exit (process dying).
+func Run(pathPrefix string, updateInterval time.Duration, scanResources chan bool) {
+	l := newLabeler(pathPrefix+sysfsDRMDirectory, pathPrefix+debugfsDRIDirectory)
+
+	interruptChan := make(chan os.Signal, 1)
+	signal.Notify(interruptChan, os.Interrupt)
+	signal.Notify(interruptChan, syscall.SIGTERM)
+
+	klog.V(1).Info("Starting GPU labeler")
+
+	running := true
+
+	for running {
+		timeout := time.After(updateInterval)
+
+		select {
+		case <-timeout:
+		case <-scanResources:
+		case <-interruptChan:
+			klog.V(2).Info("Interrupt received")
+
+			running = false
+
+			continue
+		}
+
+		klog.V(1).Info("Ext resources scanning")
+
+		err := l.createLabels()
+		if err != nil {
+			klog.Warningf("label creation failed: %+v", err)
+
+			continue
+		}
+
+		if l.labelsChanged {
+			klog.V(1).Info("Writing labels")
+
+			if err := l.printLabelsToFile(nfdFeatureFile); err != nil {
+				klog.Warningf("failed to write labels to file: %+v", err)
+			}
+		}
+	}
+
+	signal.Stop(interruptChan)
+
+	klog.V(2).Info("Removing label file")
+
+	err := os.Remove(nfdFeatureFile)
+	if err != nil {
+		klog.Errorf("Failed to cleanup label file: %+v", err)
+	}
+
+	klog.V(1).Info("Stopping GPU labeler")
+
+	// Trigger sigterm to cause the app to exit without delays
+	if err := syscall.Kill(syscall.Getpid(), syscall.SIGTERM); err != nil {
+		klog.Errorf("Failed to re-send sigterm: %+v", err)
+	}
+}

cmd/gpu_nfdhook/labeler_test.go renamed to cmd/internal/labeler/labeler_test.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package labeler
 
 import (
 	"os"
@@ -24,6 +24,10 @@ import (
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 )
 
+const (
+	sysfsDirectory = "/sys/"
+)
+
 type testcase struct {
 	capabilityFile map[string][]byte
 	expectedRetval error