Skip to content

Commit c645abd

Browse files
tkatilatkatila
committed
cdi: restructure cdi support for more generic use
Move FPGA CDI spec generation from common files to FPGA plugin and allow using CDI device names in device discovery Signed-off-by: Tuomas Katila <[email protected]>
1 parent e01c4e4 commit c645abd

File tree

7 files changed

+116
-78
lines changed

7 files changed

+116
-78
lines changed

‎cmd/fpga_plugin/dfl_test.go‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
2424
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga"
2525
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
26-
cdispec "tags.cncf.io/container-device-interface/specs-go"
2726
)
2827

2928
func TestNewDevicePluginDFL(t *testing.T) {
@@ -188,20 +187,17 @@ func TestGetRegionDevelTreeDFL(t *testing.T) {
188187

189188
func TestGetRegionTreeDFL(t *testing.T) {
190189
expected := dpapi.NewDeviceTree()
191-
hooks := []*cdispec.Hook{
192-
{
193-
HookName: HookName,
194-
Path: HookPath,
195-
},
196-
}
197190
nodes := []pluginapi.DeviceSpec{
198191
{
199192
HostPath: "/dev/dfl-port.0",
200193
ContainerPath: "/dev/dfl-port.0",
201194
Permissions: "rw",
202195
},
203196
}
204-
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, hooks))
197+
cdiDevices := []*pluginapi.CDIDevice{
198+
{Name: CDIKind + "=" + regionMode + "-ce48969398f05f33946d560708be108a"},
199+
}
200+
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, cdiDevices))
205201

206202
nodes = []pluginapi.DeviceSpec{
207203
{
@@ -215,7 +211,7 @@ func TestGetRegionTreeDFL(t *testing.T) {
215211
Permissions: "rw",
216212
},
217213
}
218-
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, hooks))
214+
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "region2", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, cdiDevices))
219215

220216
nodes = []pluginapi.DeviceSpec{
221217
{
@@ -229,7 +225,10 @@ func TestGetRegionTreeDFL(t *testing.T) {
229225
Permissions: "rw",
230226
},
231227
}
232-
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil, nil, hooks))
228+
cdiDevices = []*pluginapi.CDIDevice{
229+
{Name: CDIKind + "=" + regionMode + "-" + unhealthyInterfaceID},
230+
}
231+
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "region3", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil, nil, cdiDevices))
233232

234233
result := getRegionTree(getDevicesDFL())
235234
if !reflect.DeepEqual(result, expected) {

‎cmd/fpga_plugin/fpga_plugin.go‎

Lines changed: 62 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
package main
1616

1717
import (
18+
"encoding/json"
1819
"flag"
1920
"fmt"
2021
"os"
@@ -53,8 +54,11 @@ const (
5354
scanPeriod = 5 * time.Second
5455

5556
// CDI hook attributes.
56-
HookName = "createRuntime"
57-
HookPath = "/opt/intel/fpga-sw/intel-fpga-crihook"
57+
CDIKind = "intel.com/fpga"
58+
CDIVersion = "0.5.0" // Kubernetes 1.27 / CRI-O 1.27 / Containerd 1.7 use this version.
59+
CDIDir = "/var/run/cdi"
60+
HookName = "createRuntime"
61+
HookPath = "/opt/intel/fpga-sw/intel-fpga-crihook"
5862
)
5963

6064
type newPortFunc func(fname string) (fpga.Port, error)
@@ -117,14 +121,11 @@ func getRegionTree(devices []device) dpapi.DeviceTree {
117121
}
118122
}
119123

120-
hooks := []*cdispec.Hook{
121-
{
122-
HookName: HookName,
123-
Path: HookPath,
124-
},
124+
cdiDevices := []*pluginapi.CDIDevice{
125+
{Name: fmt.Sprintf("%s=%s", CDIKind, region.id)},
125126
}
126127

127-
regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil, nil, hooks))
128+
regionTree.AddDevice(devType, region.id, dpapi.NewDeviceInfo(health, devNodes, nil, nil, nil, cdiDevices))
128129
}
129130
}
130131

@@ -187,6 +188,8 @@ type devicePlugin struct {
187188

188189
sysfsDir string
189190
devfsDir string
191+
mode string
192+
cdiPath string
190193

191194
deviceReg *regexp.Regexp
192195
portReg *regexp.Regexp
@@ -225,13 +228,32 @@ func newDevicePlugin(mode string, rootPath string) (*devicePlugin, error) {
225228
return nil, err
226229
}
227230

231+
dp.mode = mode
232+
dp.cdiPath = CDIDir
233+
228234
dp.newPort = fpga.NewPort
229235
dp.scanTicker = time.NewTicker(scanPeriod)
230236
dp.scanDone = make(chan bool, 1) // buffered as we may send to it before Scan starts receiving from it
231237

232238
return dp, nil
233239
}
234240

241+
func (dp *devicePlugin) Allocate(request *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) {
242+
// Write CDI device specs if in region mode.
243+
if dp.mode == regionMode {
244+
for _, cont := range request.ContainerRequests {
245+
for _, id := range cont.DevicesIDs {
246+
if err := generateAndWriteCDISpec(id, dp.cdiPath); err != nil {
247+
return nil, err
248+
}
249+
}
250+
}
251+
}
252+
253+
// And return error to continue with the default Allocate
254+
return nil, &dpapi.UseDefaultMethodError{}
255+
}
256+
235257
func (dp *devicePlugin) PostAllocate(response *pluginapi.AllocateResponse) error {
236258
// Set container annotations when programming is allowed
237259
if len(dp.annotationValue) > 0 {
@@ -358,6 +380,38 @@ func getPluginParams(mode string) (getDevTreeFunc, string, error) {
358380
return getDevTree, annotationValue, nil
359381
}
360382

383+
func generateAndWriteCDISpec(deviceID, cdiPath string) error {
384+
spec := cdispec.Spec{
385+
Version: CDIVersion,
386+
Kind: CDIKind,
387+
Devices: []cdispec.Device{
388+
{
389+
Name: deviceID,
390+
ContainerEdits: cdispec.ContainerEdits{
391+
Hooks: []*cdispec.Hook{
392+
{
393+
HookName: HookName,
394+
Path: HookPath,
395+
},
396+
},
397+
},
398+
},
399+
},
400+
}
401+
402+
jsonSpec, err := json.Marshal(spec)
403+
if err != nil {
404+
return err
405+
}
406+
407+
cdiFileName := path.Join(cdiPath, deviceID) + ".json"
408+
if err = os.WriteFile(cdiFileName, jsonSpec, 0o644); err != nil {
409+
return err
410+
}
411+
412+
return nil
413+
}
414+
361415
func main() {
362416
var (
363417
mode string

‎cmd/fpga_plugin/fpga_plugin_test.go‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,33 @@ func TestPostAllocate(t *testing.T) {
103103
}
104104
}
105105

106+
func TestAllocate(t *testing.T) {
107+
request := new(pluginapi.AllocateRequest)
108+
creq := new(pluginapi.ContainerAllocateRequest)
109+
creq.DevicesIDs = append(creq.DevicesIDs, "fpga-1")
110+
request.ContainerRequests = append(request.ContainerRequests, creq)
111+
112+
tmpDir, err := os.MkdirTemp("/tmp/", "fpga-allocate")
113+
if err != nil {
114+
t.Fatal("failed to create temp directory: ", err)
115+
}
116+
defer os.RemoveAll(tmpDir)
117+
118+
dp := &devicePlugin{
119+
mode: regionMode,
120+
cdiPath: tmpDir,
121+
}
122+
123+
resp, err := dp.Allocate(request)
124+
if resp != nil {
125+
t.Errorf("Unexpected non-nil response %+v", resp)
126+
}
127+
128+
if err == nil {
129+
t.Error("Unexpected non error")
130+
}
131+
}
132+
106133
func TestNewDevicePlugin(t *testing.T) {
107134
root, err := os.MkdirTemp("", "test_new_device_plugin")
108135
if err != nil {

‎cmd/fpga_plugin/opae_test.go‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@ import (
2323
dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
2424
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga"
2525
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
26-
cdispec "tags.cncf.io/container-device-interface/specs-go"
2726
)
2827

2928
func TestNewDevicePluginOPAE(t *testing.T) {
@@ -168,20 +167,17 @@ func TestGetRegionDevelTreeOPAE(t *testing.T) {
168167

169168
func TestGetRegionTreeOPAE(t *testing.T) {
170169
expected := dpapi.NewDeviceTree()
171-
hooks := []*cdispec.Hook{
172-
{
173-
HookName: HookName,
174-
Path: HookPath,
175-
},
176-
}
177170
nodes := []pluginapi.DeviceSpec{
178171
{
179172
HostPath: "/dev/intel-fpga-port.0",
180173
ContainerPath: "/dev/intel-fpga-port.0",
181174
Permissions: "rw",
182175
},
183176
}
184-
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, hooks))
177+
cdiDevices := []*pluginapi.CDIDevice{
178+
{Name: CDIKind + "=" + regionMode + "-ce48969398f05f33946d560708be108a"},
179+
}
180+
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.0", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, cdiDevices))
185181

186182
nodes = []pluginapi.DeviceSpec{
187183
{
@@ -190,7 +186,7 @@ func TestGetRegionTreeOPAE(t *testing.T) {
190186
Permissions: "rw",
191187
},
192188
}
193-
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, hooks))
189+
expected.AddDevice(regionMode+"-ce48969398f05f33946d560708be108a", "intel-fpga-fme.1", dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil, cdiDevices))
194190

195191
nodes = []pluginapi.DeviceSpec{
196192
{
@@ -199,7 +195,10 @@ func TestGetRegionTreeOPAE(t *testing.T) {
199195
Permissions: "rw",
200196
},
201197
}
202-
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil, nil, hooks))
198+
cdiDevices = []*pluginapi.CDIDevice{
199+
{Name: CDIKind + "=" + regionMode + "-" + unhealthyInterfaceID},
200+
}
201+
expected.AddDevice(regionMode+"-"+unhealthyInterfaceID, "intel-fpga-fme.2", dpapi.NewDeviceInfo(pluginapi.Unhealthy, nodes, nil, nil, nil, cdiDevices))
203202

204203
result := getRegionTree(getDevicesOPAE())
205204
if !reflect.DeepEqual(result, expected) {

‎cmd/sgx_plugin/sgx_plugin.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -87,13 +87,13 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
8787
for i := uint(0); i < dp.nEnclave; i++ {
8888
devID := fmt.Sprintf("%s-%d", "sgx-enclave", i)
8989
nodes := []pluginapi.DeviceSpec{{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}}
90-
devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil))
90+
devTree.AddDevice(deviceTypeEnclave, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil, nil))
9191
}
9292

9393
for i := uint(0); i < dp.nProvision; i++ {
9494
devID := fmt.Sprintf("%s-%d", "sgx-provision", i)
9595
nodes := []pluginapi.DeviceSpec{{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}}
96-
devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil))
96+
devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil, nil))
9797
}
9898

9999
return devTree, nil

‎pkg/deviceplugin/api.go‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
"github.com/intel/intel-device-plugins-for-kubernetes/pkg/topology"
2020
"k8s.io/klog/v2"
2121
pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
22-
cdispec "tags.cncf.io/container-device-interface/specs-go"
2322
)
2423

2524
// DeviceInfo contains information about device maintained by Device Plugin.
@@ -30,9 +29,8 @@ type DeviceInfo struct {
3029
topology *pluginapi.TopologyInfo
3130
state string
3231
nodes []pluginapi.DeviceSpec
33-
// Hooks can be passed only through CDI
3432
// https://github.com/kubernetes/enhancements/tree/master/keps/sig-node/4009-add-cdi-devices-to-device-plugin-api
35-
hooks []*cdispec.Hook
33+
cdiDevices []*pluginapi.CDIDevice
3634
}
3735

3836
// UseDefaultMethodError allows the plugin to request running the default
@@ -49,14 +47,14 @@ func init() {
4947
}
5048

5149
// NewDeviceInfo makes DeviceInfo struct and adds topology information to it.
52-
func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs, annotations map[string]string, hooks []*cdispec.Hook) DeviceInfo {
50+
func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs, annotations map[string]string, cdiDevices []*pluginapi.CDIDevice) DeviceInfo {
5351
deviceInfo := DeviceInfo{
5452
state: state,
5553
nodes: nodes,
5654
mounts: mounts,
5755
envs: envs,
5856
annotations: annotations,
59-
hooks: hooks,
57+
cdiDevices: cdiDevices,
6058
}
6159

6260
devPaths := []string{}
@@ -77,14 +75,15 @@ func NewDeviceInfo(state string, nodes []pluginapi.DeviceSpec, mounts []pluginap
7775

7876
// NewDeviceInfoWithTopologyHints makes DeviceInfo struct with topology information provided to it.
7977
func NewDeviceInfoWithTopologyHints(state string, nodes []pluginapi.DeviceSpec, mounts []pluginapi.Mount, envs map[string]string,
80-
annotations map[string]string, topology *pluginapi.TopologyInfo) DeviceInfo {
78+
annotations map[string]string, topology *pluginapi.TopologyInfo, cdiDevices []*pluginapi.CDIDevice) DeviceInfo {
8179
return DeviceInfo{
8280
state: state,
8381
nodes: nodes,
8482
mounts: mounts,
8583
envs: envs,
8684
annotations: annotations,
8785
topology: topology,
86+
cdiDevices: cdiDevices,
8887
}
8988
}
9089

0 commit comments

Comments
 (0)