controller, network status: update pod after interface add/remove

This commit updates the pods network-status annotation whenever an
interface is hot-plugged to the pod (or removed from it ...).

Signed-off-by: Miguel Duarte Barroso <[email protected]>

Author: maiqueb

pkg/annotations/dynamic-network-status.go

@@ -0,0 +1,78 @@
+package annotations
+
+import (
+	"encoding/json"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+
+	nettypes "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	nadutils "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils"
+	multusapi "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/api"
+)
+
+func AddDynamicIfaceToStatus(currentPod *corev1.Pod, networkSelectionElement *nettypes.NetworkSelectionElement, response *multusapi.Response) (string, error) {
+	currentIfaceStatus, err := podDynamicNetworkStatus(currentPod)
+	if err != nil {
+		return "", err
+	}
+
+	if response != nil && response.Result != nil {
+		newIfaceStatus, err := nadutils.CreateNetworkStatus(
+			response.Result,
+			NamespacedName(networkSelectionElement.Namespace, networkSelectionElement.Name),
+			false,
+			nil,
+		)
+		if err != nil {
+			return "", fmt.Errorf("failed to create NetworkStatus from the response: %v", err)
+		}
+
+		newIfaceString, err := json.Marshal(append(currentIfaceStatus, *newIfaceStatus))
+		if err != nil {
+			return "", fmt.Errorf("failed to marshall the dynamic networks status after interface creation")
+		}
+		return string(newIfaceString), nil
+	}
+	return "", fmt.Errorf("got an empty response from multus: %+v", response)
+}
+
+func DeleteDynamicIfaceFromStatus(currentPod *corev1.Pod, netName string, ifaceName string) (string, error) {
+	currentIfaceStatus, err := podDynamicNetworkStatus(currentPod)
+	if err != nil {
+		return "", err
+	}
+
+	var newIfaceStatus []nettypes.NetworkStatus
+	newIfaceStatus = make([]nettypes.NetworkStatus, 0)
+	for i := range currentIfaceStatus {
+		if currentIfaceStatus[i].Name == netName && currentIfaceStatus[i].Interface == ifaceName {
+			continue
+		}
+		newIfaceStatus = append(newIfaceStatus, currentIfaceStatus[i])
+	}
+
+	newIfaceString, err := json.Marshal(newIfaceStatus)
+	if err != nil {
+		return "", fmt.Errorf("failed to marshall the dynamic networks status after deleting interface")
+	}
+	return string(newIfaceString), nil
+}
+
+func podDynamicNetworkStatus(currentPod *corev1.Pod) ([]nettypes.NetworkStatus, error) {
+	var currentIfaceStatus []nettypes.NetworkStatus
+	if currentIfaceStatusString, wasFound := currentPod.Annotations[nettypes.NetworkStatusAnnot]; wasFound {
+		if err := json.Unmarshal([]byte(currentIfaceStatusString), &currentIfaceStatus); err != nil {
+			return nil, fmt.Errorf("could not unmarshall the current dynamic annotations for pod %s: %v", podNameAndNs(currentPod), err)
+		}
+	}
+	return currentIfaceStatus, nil
+}
+
+func podNameAndNs(currentPod *corev1.Pod) string {
+	return fmt.Sprintf("%s/%s", currentPod.GetNamespace(), currentPod.GetName())
+}
+
+func NamespacedName(podNamespace string, podName string) string {
+	return fmt.Sprintf("%s/%s", podNamespace, podName)
+}

pkg/annotations/dynamic-network-status_test.go

@@ -0,0 +1,137 @@
+package annotations
+
+import (
+	"encoding/json"
+	"net"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	cni100 "github.com/containernetworking/cni/pkg/types/100"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	nadv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
+	"gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/server/api"
+)
+
+var _ = Describe("NetworkStatusFromResponse", func() {
+	const (
+		ifaceName   = "ens32"
+		namespace   = "ns1"
+		networkName = "tenantnetwork"
+		podName     = "tpod"
+	)
+
+	DescribeTable("add dynamic interface to network status", func(initialNetStatus []nadv1.NetworkStatus, resultIPs []string, expectedNetworkStatus string) {
+		const (
+			ifaceToAdd = "newiface"
+			macAddr    = "02:03:04:05:06:07"
+		)
+		Expect(
+			AddDynamicIfaceToStatus(
+				newPod(podName, namespace, initialNetStatus...),
+				newNetworkSelectionElementWithIface(networkName, ifaceName, namespace),
+				newResponse(ifaceToAdd, macAddr, resultIPs...),
+			),
+		).To(Equal(expectedNetworkStatus))
+	},
+		Entry("initial empty pod", []nadv1.NetworkStatus{}, nil, `[{"name":"ns1/tenantnetwork","interface":"newiface","mac":"02:03:04:05:06:07","dns":{}}]`),
+		Entry("pod with a network present in the network status", []nadv1.NetworkStatus{
+			{
+				Name:      "net1",
+				Interface: "iface1",
+				Mac:       "00:00:00:20:10:00",
+			}},
+			nil,
+			`[{"name":"net1","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}},{"name":"ns1/tenantnetwork","interface":"newiface","mac":"02:03:04:05:06:07","dns":{}}]`),
+		Entry("result with IPs", []nadv1.NetworkStatus{
+			{
+				Name:      "net1",
+				Interface: "iface1",
+				Mac:       "00:00:00:20:10:00",
+			}},
+			[]string{"10.10.10.10/24"},
+			`[{"name":"net1","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}},{"name":"ns1/tenantnetwork","interface":"newiface","ips":["10.10.10.10"],"mac":"02:03:04:05:06:07","dns":{}}]`))
+
+	DescribeTable("remove an interface to the current network status", func(initialNetStatus []nadv1.NetworkStatus, networkName, ifaceName, expectedNetworkStatus string) {
+		Expect(
+			DeleteDynamicIfaceFromStatus(
+				newPod(podName, namespace, initialNetStatus...),
+				networkName,
+				ifaceName,
+			),
+		).To(Equal(expectedNetworkStatus))
+	},
+		Entry("when there aren't any existing interfaces", nil, "net1", "iface1", "[]"),
+		Entry("when we remove all the currently existing interfaces", []nadv1.NetworkStatus{
+			{
+				Name:      "net1",
+				Interface: "iface1",
+				Mac:       "00:00:00:20:10:00",
+			}}, "net1", "iface1", "[]"),
+		Entry("when there is *not* a matching interface to remove", []nadv1.NetworkStatus{
+			{
+				Name:      "net1",
+				Interface: "iface1",
+				Mac:       "00:00:00:20:10:00",
+			}}, "net2", "iface1", `[{"name":"net1","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}}]`),
+		Entry("when we remove one of the existing interfaces", []nadv1.NetworkStatus{
+			{
+				Name:      "net1",
+				Interface: "iface1",
+				Mac:       "00:00:00:20:10:00",
+			},
+			{
+				Name:      "net2",
+				Interface: "iface2",
+				Mac:       "aa:bb:cc:20:10:00",
+			},
+		}, "net2", "iface2", `[{"name":"net1","interface":"iface1","mac":"00:00:00:20:10:00","dns":{}}]`))
+})
+
+func newPod(podName string, namespace string, netStatus ...nadv1.NetworkStatus) *corev1.Pod {
+	status, err := json.Marshal(netStatus)
+	if err != nil {
+		return nil
+	}
+	return &corev1.Pod{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      podName,
+			Namespace: namespace,
+			Annotations: map[string]string{
+				nadv1.NetworkStatusAnnot: string(status),
+			},
+		},
+	}
+}
+
+func newResponse(ifaceName string, macAddr string, ips ...string) *api.Response {
+	var ipConfs []*cni100.IPConfig
+	for i := range ips {
+		ipConfs = append(ipConfs, &cni100.IPConfig{Address: *ipNet(ips[i])})
+	}
+
+	const sandboxPath = "/over/there"
+	ifaces := []*cni100.Interface{{
+		Name:    ifaceName,
+		Mac:     macAddr,
+		Sandbox: sandboxPath,
+	}}
+	return &api.Response{
+		Result: &cni100.Result{
+			CNIVersion: "1.0.0",
+			Interfaces: ifaces,
+			IPs:        ipConfs,
+		}}
+}
+
+func ipNet(ipString string) *net.IPNet {
+	ip, network, err := net.ParseCIDR(ipString)
+	if err != nil {
+		return nil
+	}
+	network.IP = ip
+	return network
+}

pkg/annotations/network-selection-elements_test.go

@@ -1,11 +1,13 @@
 package annotations
 
 import (
-	v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
 	"strings"
 	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	v1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
 )
 
 func TestController(t *testing.T) {

pkg/controller/pod.go

@@ -1,13 +1,15 @@
 package controller
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"reflect"
 	"strings"
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	v1coreinformerfactory "k8s.io/client-go/informers"
@@ -174,7 +176,7 @@ func (pnc *PodNetworksController) handleResult(err error, dynamicAttachmentReque
 
 	currentRetries := pnc.workqueue.NumRequeues(dynamicAttachmentRequest)
 	if currentRetries <= maxRetries {
-		klog.Errorf("re-queued request for: %v", dynamicAttachmentRequest)
+		klog.Errorf("re-queued request for: %v. Error: %v", dynamicAttachmentRequest, err)
 		pnc.workqueue.AddRateLimited(dynamicAttachmentRequest)
 		return
 	}
@@ -196,7 +198,7 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int
 	}
 	podNamespace := oldPod.GetNamespace()
 	podName := oldPod.GetName()
-	klog.V(logging.Debug).Infof("pod [%s] updated", namespacedName(podNamespace, podName))
+	klog.V(logging.Debug).Infof("pod [%s] updated", annotations.NamespacedName(podNamespace, podName))
 
 	oldNetworkSelectionElements, err := networkSelectionElements(oldPod.Annotations, podNamespace)
 	if err != nil {
@@ -211,7 +213,7 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int
 	}
 
 	toAdd := exclusiveNetworks(newNetworkSelectionElements, oldNetworkSelectionElements)
-	klog.Infof("%d attachments to add to pod %s", len(toAdd), namespacedName(podNamespace, podName))
+	klog.Infof("%d attachments to add to pod %s", len(toAdd), annotations.NamespacedName(podNamespace, podName))
 
 	netnsPath, err := pnc.netnsPath(newPod)
 	if err != nil {
@@ -230,7 +232,7 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int
 	}
 
 	toRemove := exclusiveNetworks(oldNetworkSelectionElements, newNetworkSelectionElements)
-	klog.Infof("%d attachments to remove from pod %s", len(toRemove), namespacedName(podNamespace, podName))
+	klog.Infof("%d attachments to remove from pod %s", len(toRemove), annotations.NamespacedName(podNamespace, podName))
 	if len(toRemove) > 0 {
 		pnc.workqueue.Add(
 			&DynamicAttachmentRequest{
@@ -243,10 +245,6 @@ func (pnc *PodNetworksController) handlePodUpdate(oldObj interface{}, newObj int
 	}
 }
 
-func namespacedName(podNamespace string, podName string) string {
-	return fmt.Sprintf("%s/%s", podNamespace, podName)
-}
-
 func (pnc *PodNetworksController) addNetworks(dynamicAttachmentRequest *DynamicAttachmentRequest, pod *corev1.Pod) error {
 	for i := range dynamicAttachmentRequest.AttachmentNames {
 		netToAdd := dynamicAttachmentRequest.AttachmentNames[i]
@@ -274,6 +272,15 @@ func (pnc *PodNetworksController) addNetworks(dynamicAttachmentRequest *DynamicA
 		}
 		klog.Infof("response: %v", *response.Result)
 
+		newIfaceStatus, err := annotations.AddDynamicIfaceToStatus(pod, netToAdd, response)
+		if err != nil {
+			return fmt.Errorf("failed to compute the updated network status: %v", err)
+		}
+
+		if err := pnc.updatePodNetworkStatus(pod, newIfaceStatus); err != nil {
+			return err
+		}
+
 		pnc.Eventf(pod, corev1.EventTypeNormal, "AddedInterface", addIfaceEventFormat(pod, netToAdd))
 	}
 
@@ -307,12 +314,38 @@ func (pnc *PodNetworksController) removeNetworks(dynamicAttachmentRequest *Dynam
 		}
 		klog.Infof("response: %v", *response)
 
+		newIfaceStatus, err := annotations.DeleteDynamicIfaceFromStatus(
+			pod,
+			annotations.NamespacedName(netToRemove.Namespace, netToRemove.Name),
+			netToRemove.InterfaceRequest,
+		)
+		if err != nil {
+			return fmt.Errorf(
+				"failed to compute the dynamic network attachments after deleting network: %s, iface: %s: %v",
+				netToRemove.Name,
+				netToRemove.InterfaceRequest,
+				err,
+			)
+		}
+		if err := pnc.updatePodNetworkStatus(pod, newIfaceStatus); err != nil {
+			return err
+		}
+
 		pnc.Eventf(pod, corev1.EventTypeNormal, "RemovedInterface", removeIfaceEventFormat(pod, netToRemove))
 	}
 
 	return nil
 }
 
+func (pnc *PodNetworksController) updatePodNetworkStatus(pod *corev1.Pod, newIfaceStatus string) error {
+	pod.Annotations[nadv1.NetworkStatusAnnot] = newIfaceStatus
+
+	if _, err := pnc.k8sClientSet.CoreV1().Pods(pod.GetNamespace()).Update(context.Background(), pod, metav1.UpdateOptions{}); err != nil {
+		return fmt.Errorf("failed to update pod's network-status annotations for %s: %v", pod.GetName(), err)
+	}
+	return nil
+}
+
 func networkSelectionElements(podAnnotations map[string]string, podNamespace string) ([]*nadv1.NetworkSelectionElement, error) {
 	podNetworks, ok := podAnnotations[nadv1.NetworkAttachmentAnnot]
 	if !ok {
@@ -408,7 +441,7 @@ func podContainerID(pod *corev1.Pod) string {
 func addIfaceEventFormat(pod *corev1.Pod, network *nadv1.NetworkSelectionElement) string {
 	return fmt.Sprintf(
 		"pod [%s]: added interface %s to network: %s",
-		namespacedName(pod.GetNamespace(), pod.GetName()),
+		annotations.NamespacedName(pod.GetNamespace(), pod.GetName()),
 		network.InterfaceRequest,
 		network.Name,
 	)
@@ -417,7 +450,7 @@ func addIfaceEventFormat(pod *corev1.Pod, network *nadv1.NetworkSelectionElement
 func removeIfaceEventFormat(pod *corev1.Pod, network *nadv1.NetworkSelectionElement) string {
 	return fmt.Sprintf(
 		"pod [%s]: removed interface %s from network: %s",
-		namespacedName(pod.GetNamespace(), pod.GetName()),
+		annotations.NamespacedName(pod.GetNamespace(), pod.GetName()),
 		network.InterfaceRequest,
 		network.Name,
 	)