Add DeviceInfo File management

Save the deviceInfo files on server startup and remove them on server
clean up.

Signed-off-by: Adrian Moreno <[email protected]>

deployments/k8s-v1.16/sriovdp-daemonset.yaml

@@ -51,13 +51,18 @@ spec:
           mountPath: /var/log
         - name: config-volume
           mountPath: /etc/pcidp
+        - name: device-info
+          mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
       volumes:
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/
         - name: log
           hostPath:
             path: /var/log
+        - name: device-info
+          hostPath:
+            path: /var/run/k8s.cni.cncf.io/devinfo/dp
         - name: config-volume
           configMap:
             name: sriovdp-config
@@ -111,13 +116,18 @@ spec:
           mountPath: /var/log
         - name: config-volume
           mountPath: /etc/pcidp
+        - name: device-info
+          mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
       volumes:
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/
         - name: log
           hostPath:
             path: /var/log
+        - name: device-info
+          hostPath:
+            path: /var/run/k8s.cni.cncf.io/devinfo/dp
         - name: config-volume
           configMap:
             name: sriovdp-config
@@ -172,13 +182,18 @@ spec:
           mountPath: /var/log
         - name: config-volume
           mountPath: /etc/pcidp
+        - name: device-info
+          mountPath: /var/run/k8s.cni.cncf.io/devinfo/dp
       volumes:
         - name: devicesock
           hostPath:
             path: /var/lib/kubelet/
         - name: log
           hostPath:
             path: /var/log
+        - name: device-info
+          hostPath:
+            path: /var/run/k8s.cni.cncf.io/devinfo/dp
         - name: config-volume
           configMap:
             name: sriovdp-config

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
 	github.com/jaypipes/ghw v0.6.0
 	github.com/jaypipes/pcidb v0.5.0
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.1-0.20201119153432-9d213757d22d
 	github.com/onsi/ginkgo v1.12.0
 	github.com/onsi/gomega v1.9.0
 	github.com/pkg/errors v0.9.1

pkg/netdevice/netResourcePool.go

@@ -15,8 +15,12 @@
 package netdevice
 
 import (
+	"fmt"
 	"github.com/golang/glog"
+	"strings"
 
+	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/resources"
@@ -80,3 +84,42 @@ func (rp *netResourcePool) GetDeviceSpecs(deviceIDs []string) []*pluginapi.Devic
 	}
 	return devSpecs
 }
+
+// StoreDeviceInfoFile stores the Device Info files according to the
+//  k8snetworkplumbingwg/device-info-spec
+func (rp *netResourcePool) StoreDeviceInfoFile(resourceNamePrefix string) error {
+	for id, dev := range rp.GetDevicePool() {
+		netDev, ok := dev.(types.PciNetDevice)
+		if !ok {
+			return fmt.Errorf("StoreDeviceInfoFile: Only pciNetDevices are supported")
+		}
+		devInfo := nettypes.DeviceInfo{
+			Type:    "pci",
+			Version: nettypes.DeviceInfoVersion,
+			Pci: &nettypes.PciDevice{
+				PciAddress: netDev.GetPciAddr(),
+			},
+		}
+		resource := fmt.Sprintf("%s/%s", resourceNamePrefix, rp.GetConfig().ResourceName)
+		if err := nadutils.SaveDeviceInfoForDP(resource, id, &devInfo); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// CleanDeviceInfoFile cleans the Device Info files
+func (rp *netResourcePool) CleanDeviceInfoFile(resourceNamePrefix string) error {
+	errors := make([]string, 0)
+	for id := range rp.GetDevicePool() {
+		resource := fmt.Sprintf("%s/%s", resourceNamePrefix, rp.GetConfig().ResourceName)
+		if err := nadutils.CleanDeviceInfoForDP(resource, id); err != nil {
+			// Continue trying to clean.
+			errors = append(errors, err.Error())
+		}
+	}
+	if len(errors) > 0 {
+		return fmt.Errorf(strings.Join(errors, ","))
+	}
+	return nil
+}

pkg/resources/pciDevice.go

@@ -17,9 +17,9 @@ package resources
 import (
 	"strconv"
 
+	"github.com/jaypipes/ghw"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/types"
 	"github.com/k8snetworkplumbingwg/sriov-network-device-plugin/pkg/utils"
-	"github.com/jaypipes/ghw"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 

pkg/resources/pool_stub.go

@@ -136,3 +136,15 @@ func (rp *ResourcePoolImpl) DeviceSpecExist(specs []*pluginapi.DeviceSpec, newSp
 func (rp *ResourcePoolImpl) GetDevicePool() map[string]types.PciDevice {
 	return rp.devicePool
 }
+
+// StoreDeviceInfoFile does nothing. DeviceType-specific ResourcePools might
+// store information according to the k8snetworkplumbingwg/device-info-spec
+func (rp *ResourcePoolImpl) StoreDeviceInfoFile(resourceNamePrefix string) error {
+	return nil
+}
+
+// CleanDeviceInfoFile does nothing. DeviceType-specific ResourcePools might
+// clean the Device Info file
+func (rp *ResourcePoolImpl) CleanDeviceInfoFile(resourceNamePrefix string) error {
+	return nil
+}

pkg/resources/server.go

@@ -188,6 +188,11 @@ func (rs *resourceServer) Init() error {
 func (rs *resourceServer) Start() error {
 	resourceName := rs.resourcePool.GetResourceName()
 	_ = rs.cleanUp() // try tp clean up and continue
+
+	if err := rs.resourcePool.StoreDeviceInfoFile(rs.resourceNamePrefix); err != nil {
+		glog.Errorf("%s: error creating DeviceInfo File: %s", rs.resourcePool.GetResourceName(), err.Error())
+	}
+
 	glog.Infof("starting %s device plugin endpoint at: %s\n", resourceName, rs.endPoint)
 	lis, err := net.Listen("unix", rs.sockPath)
 	if err != nil {
@@ -292,8 +297,15 @@ func (rs *resourceServer) Watch() {
 }
 
 func (rs *resourceServer) cleanUp() error {
+	errors := make([]string, 0)
 	if err := os.Remove(rs.sockPath); err != nil && !os.IsNotExist(err) {
-		return err
+		errors = append(errors, err.Error())
+	}
+	if err := rs.resourcePool.CleanDeviceInfoFile(rs.resourceNamePrefix); err != nil {
+		errors = append(errors, err.Error())
+	}
+	if len(errors) > 0 {
+		return fmt.Errorf(strings.Join(errors, ","))
 	}
 	return nil
 }

pkg/types/types.go

@@ -142,6 +142,8 @@ type ResourcePool interface {
 	GetDeviceSpecs(deviceIDs []string) []*pluginapi.DeviceSpec
 	GetEnvs(deviceIDs []string) []string
 	GetMounts(deviceIDs []string) []*pluginapi.Mount
+	StoreDeviceInfoFile(resourceNamePrefix string) error
+	CleanDeviceInfoFile(resourceNamePrefix string) error
 }
 
 // DeviceProvider provides interface for device discovery