Merge pull request #1719 from tkatila/gnrd-qat-support

Add QAT support for 420xx driver and the 4946 device (GNR-D)

Author: hj-johannes-lee

.github/workflows/lib-e2e.yaml

@@ -27,6 +27,13 @@ jobs:
               - intel-idxd-config-initcontainer
               - accel-config-demo
               - intel-deviceplugin-operator
+          - name: e2e-qat-gnrd
+            targetjob: e2e-qat FOCUS="Mode:dpdk.*Resource:(cy|dc)" SKIP=App:crypto-perf
+            runner: simics-gnrd
+            images:
+              - intel-qat-plugin
+              - intel-qat-initcontainer
+              - openssl-qat-engine
           - name: e2e-qat
             targetjob: e2e-qat FOCUS=Resource:generic
             runner: qat

cmd/qat_plugin/README.md

@@ -43,7 +43,7 @@ The QAT plugin can take a number of command line arguments, summarised in the fo
 | Flag | Argument | Meaning |
 |:---- |:-------- |:------- |
 | -dpdk-driver | string | DPDK Device driver for configuring the QAT device (default: `vfio-pci`) |
-| -kernel-vf-drivers | string | Comma separated list of the QuickAssist VFs to search and use in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx/401xx/402xx, C4xxx and D15xx (default: `c6xxvf,4xxxvf`) |
+| -kernel-vf-drivers | string | Comma separated list of the QuickAssist VFs to search and use in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx/401xx/402xx, 420xx, C4xxx and D15xx (default: `c6xxvf,4xxxvf,420xxvf`) |
 | -max-num-devices | int | maximum number of QAT devices to be provided to the QuickAssist device plugin (default: `64`) |
 | -mode | string | Deprecated: plugin mode which can be either `dpdk` or `kernel` (default: `dpdk`).|
 | -allocation-policy | string | 2 possible values: balanced and packed. Balanced mode spreads allocated QAT VF resources balanced among QAT PF devices, and packed mode packs one QAT PF device full of QAT VF resources before allocating resources from the next QAT PF. (There is no default.) |
@@ -132,7 +132,7 @@ In addition to the default configuration, you can add device-specific configurat
 
 | Device | Possible Configuration | How To Customize | Options | Notes |
 |:-------|:-----------------------|:-----------------|:--------|:------|
-| 4xxx, 401xx,402xx | [cfg_services](https://github.com/torvalds/linux/blob/v6.6-rc5/Documentation/ABI/testing/sysfs-driver-qat) reports the configured services (crypto services or compression services) of the QAT device. | `ServicesEnabled=<value>` | compress:`dc`, crypto:`sym;asym`, <br>crypto+compress:`asym;dc`,<br>crypto+compress:`sym;dc` | Linux 6.0+ kernel is required. |
+| 4xxx, 401xx, 402xx, 420xx | [cfg_services](https://github.com/torvalds/linux/blob/v6.6-rc5/Documentation/ABI/testing/sysfs-driver-qat) reports the configured services (crypto services or compression services) of the QAT device. | `ServicesEnabled=<value>` | compress:`dc`, crypto:`sym;asym`, <br>crypto+compress:`asym;dc`,<br>crypto+compress:`sym;dc` | 4xxx/401xx/402xx: Linux 6.0+ kernel. 420xx: Linux 6.8+ kernel. |
 
 To create a provisioning `configMap`, run the following command before deploying initcontainer:
 
@@ -247,7 +247,7 @@ In order to utilise the QAT device plugin, QuickAssist SR-IOV virtual functions
 You can verify this on your nodes by checking for the relevant PCI identifiers:
 
 ```bash
-for i in 0442 0443 18a1 37c9 6f55 19e3 4941 4943; do lspci -d 8086:$i; done
+for i in 0442 0443 18a1 37c9 6f55 19e3 4941 4943 4945 4947; do lspci -d 8086:$i; done
 ```
 
 [1]:https://www-ssl.intel.com/content/www/us/en/design/products-and-solutions/processors-and-chipsets/purley/intel-xeon-scalable-processors.html

cmd/qat_plugin/dpdkdrv/dpdkdrv.go

@@ -66,6 +66,7 @@ var qatDeviceDriver = map[string]string{
 	"4941": "4xxxvf",
 	"4943": "4xxxvf",
 	"4945": "4xxxvf",
+	"4947": "420xxvf",
 	"37c9": "c6xxvf",
 	"6f55": "d15xxvf",
 }
@@ -382,7 +383,7 @@ func readDeviceConfiguration(pfDev string) string {
 	}
 
 	devCfgPath := filepath.Join(filepath.Dir(filepath.Join(pfDev, "../../")), "kernel/debug",
-		fmt.Sprintf("qat_4xxx_%s/dev_cfg", filepath.Base(pfDev)))
+		fmt.Sprintf("qat_%s_%s/dev_cfg", getCurrentDriver(pfDev), filepath.Base(pfDev)))
 
 	devCfg, err := ini.LoadSources(lOpts, devCfgPath)
 	if err != nil {
@@ -434,6 +435,7 @@ func getDeviceCapabilities(device string) (string, error) {
 		"4941": {}, // QAT Gen4 (4xxx) VF PCI ID
 		"4943": {}, // QAT Gen4 (401xx) VF PCI ID
 		"4945": {}, // QAT Gen4 (402xx) VF PCI ID
+		"4947": {}, // QAT Gen4 (420xx) VF PCI ID
 	}
 
 	if _, ok := devicesWithCapabilities[devID]; !ok {

cmd/qat_plugin/qat_plugin.go

@@ -40,7 +40,7 @@ func main() {
 	mode := flag.String("mode", "dpdk", "plugin mode which can be either dpdk (default) or kernel")
 
 	dpdkDriver := flag.String("dpdk-driver", "vfio-pci", "DPDK Device driver for configuring the QAT device")
-	kernelVfDrivers := flag.String("kernel-vf-drivers", "c6xxvf,4xxxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, and D15xx")
+	kernelVfDrivers := flag.String("kernel-vf-drivers", "c6xxvf,4xxxvf,420xxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, 420xxx, and D15xx")
 	preferredAllocationPolicy := flag.String("allocation-policy", "", "Modes of allocating QAT devices: balanced and packed")
 	maxNumDevices := flag.Int("max-num-devices", 64, "maximum number of QAT devices to be provided to the QuickAssist device plugin")
 	flag.Parse()

demo/qat-init.sh

@@ -1,12 +1,13 @@
 #!/usr/bin/env bash
 # This script is based on qatlib's qat_init.sh
 NODE_NAME="${NODE_NAME:-}"
-ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942 4944}
+ENABLED_QAT_PF_PCIIDS=${ENABLED_QAT_PF_PCIIDS:-37c8 4940 4942 4944 4946}
 DEVS=$(for pf in $ENABLED_QAT_PF_PCIIDS; do lspci -n | grep -e "$pf" | grep -o -e "^\\S*"; done)
 SERVICES_LIST="sym asym sym;asym dc sym;dc asym;dc"
 QAT_4XXX_DEVICE_PCI_ID="0x4940"
 QAT_401XX_DEVICE_PCI_ID="0x4942"
 QAT_402XX_DEVICE_PCI_ID="0x4944"
+QAT_420XX_DEVICE_PCI_ID="0x4946"
 SERVICES_ENABLED="NONE"
 SERVICES_ENABLED_FOUND="FALSE"
 
@@ -31,7 +32,7 @@ sysfs_config() {
     for dev in $DEVS; do
       DEVPATH="/sys/bus/pci/devices/0000:$dev"
       PCI_DEV=$(cat "$DEVPATH"/device 2> /dev/null)
-      if [ "$PCI_DEV" != "$QAT_4XXX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_401XX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_402XX_DEVICE_PCI_ID" ]; then
+      if [ "$PCI_DEV" != "$QAT_4XXX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_401XX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_402XX_DEVICE_PCI_ID" ] && [ "$PCI_DEV" != "$QAT_420XX_DEVICE_PCI_ID" ]; then
         continue
       fi
 

deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml

@@ -95,7 +95,7 @@ spec:
         - feature: pci.device
           matchExpressions:
             vendor: {op: In, value: ["8086"]}
-            device: {op: In, value: ["37c8", "4940", "4942", "4944"]}
+            device: {op: In, value: ["37c8", "4940", "4942", "4944", "4946"]}
             class: {op: In, value: ["0b40"]}
         - feature: kernel.loadedmodule
           matchExpressions:

deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml

@@ -81,6 +81,7 @@ spec:
                   - c3xxxvf
                   - d15xxvf
                   - 4xxxvf
+                  - 420xxvf
                   - c4xxxvf
                   type: string
                 type: array

deployments/operator/samples/deviceplugin_v1_qatdeviceplugin.yaml

@@ -15,6 +15,7 @@ spec:
   kernelVfDrivers:
     - c6xxvf
     - 4xxxvf
+    - 420xxvf
   maxNumDevices: 1
   logLevel: 4
   nodeSelector:

pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go

@@ -21,7 +21,7 @@ import (
 
 // NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
 
-// +kubebuilder:validation:Enum={"dh895xccvf","c6xxvf","c3xxxvf","d15xxvf","4xxxvf","c4xxxvf"}
+// +kubebuilder:validation:Enum={"dh895xccvf","c6xxvf","c3xxxvf","d15xxvf","4xxxvf", "420xxvf", "c4xxxvf"}
 
 // KernelVfDriver is a VF device driver for QuickAssist devices.
 type KernelVfDriver string

pkg/apis/deviceplugin/v1/qatdeviceplugin_webhook.go

@@ -92,7 +92,8 @@ func (r *QatDevicePlugin) validatePlugin() error {
 		// check if 4xxxvf is enabled
 		contains := false
 		devicesWithCapabilities := map[KernelVfDriver]struct{}{
-			"4xxxvf": {},
+			"4xxxvf":  {},
+			"420xxvf": {},
 		}
 
 		for _, kernelVfDriver := range r.Spec.KernelVfDrivers {
@@ -103,7 +104,7 @@ func (r *QatDevicePlugin) validatePlugin() error {
 		}
 
 		if !contains {
-			return errors.Errorf("ProvisioningConfig is available only for 4xxx devices")
+			return errors.Errorf("ProvisioningConfig is available only for 4xxx and 420xx devices")
 		}
 	}
 

pkg/controllers/qat/controller.go

@@ -213,6 +213,7 @@ func setInitContainer(dsSpec *v1.PodSpec, dpSpec devicepluginv1.QatDevicePluginS
 		"c6xxvf":     "37c8",
 		"d15xxvf":    "6f54",
 		"4xxxvf":     "4940 4942 4944",
+		"420xxvf":    "4946",
 		"c4xxxvf":    "18a0",
 	}