diff --git a/cmd/webhook/main.go b/cmd/webhook/main.go index 603a23f0..397c4cba 100644 --- a/cmd/webhook/main.go +++ b/cmd/webhook/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "github.com/harvester/harvester/pkg/indexeres" "os" ctlcni "github.com/harvester/harvester/pkg/generated/controllers/k8s.cni.cncf.io" @@ -11,7 +12,6 @@ import ( ctlcore "github.com/rancher/wrangler/pkg/generated/controllers/core" ctlcorev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/rancher/wrangler/pkg/kubeconfig" - "github.com/rancher/wrangler/pkg/leader" "github.com/rancher/wrangler/pkg/signals" "github.com/rancher/wrangler/pkg/start" "github.com/sirupsen/logrus" @@ -19,7 +19,6 @@ import ( "github.com/yaocw2020/webhook/pkg/config" "github.com/yaocw2020/webhook/pkg/server" "github.com/yaocw2020/webhook/pkg/types" - "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" ctlnetwork "github.com/harvester/harvester-network-controller/pkg/generated/controllers/network.harvesterhci.io" @@ -84,7 +83,6 @@ func main() { logrus.Fatal(err) } - client := kubernetes.NewForConfigOrDie(cfg) ctx := signals.SetupSignalContext() app := cli.NewApp() @@ -96,11 +94,9 @@ func main() { } } - leader.RunOrDie(ctx, options.Namespace, name, client, func(ctx context.Context) { - if err := app.Run(os.Args); err != nil { - logrus.Fatalf("run webhook server failed: %v", err) - } - }) + if err := app.Run(os.Args); err != nil { + logrus.Fatalf("run webhook server failed: %v", err) + } } func run(ctx context.Context, cfg *rest.Config, options *config.Options) error { @@ -144,13 +140,15 @@ func newCaches(ctx context.Context, cfg *rest.Config, threadiness int) (*caches, starters = append(starters, harvesterNetworkFactory) coreFactory := ctlcore.NewFactoryFromConfigOrDie(cfg) starters = append(starters, coreFactory) - // must declare cache before starting the factories + // must declare cache before starting informers c := &caches{ vmCache: kubevirtFactory.Kubevirt().V1().VirtualMachine().Cache(), nadCache: cniFactory.K8s().V1().NetworkAttachmentDefinition().Cache(), vsCache: harvesterNetworkFactory.Network().V1beta1().VlanStatus().Cache(), nodeCache: coreFactory.Core().V1().Node().Cache(), } + // Indexer must be added before starting the informer, otherwise panic `cannot add indexers to running index` happens + c.vmCache.AddIndexer(indexeres.VMByNetworkIndex, indexeres.VMByNetwork) if err := start.All(ctx, threadiness, starters...); err != nil { return nil, err diff --git a/manifests/crds/network.harvesterhci.io_vlanstatuses.yaml b/manifests/crds/network.harvesterhci.io_vlanstatuses.yaml index 7fec8b4e..3b5b1afc 100644 --- a/manifests/crds/network.harvesterhci.io_vlanstatuses.yaml +++ b/manifests/crds/network.harvesterhci.io_vlanstatuses.yaml @@ -91,16 +91,16 @@ spec: type: string masterIndex: type: integer + name: + type: string promiscuous: type: boolean state: type: string - string: - type: string type: type: string required: - - string + - name type: object type: array localAreas: diff --git a/pkg/apis/network.harvesterhci.io/v1beta1/vlanstatus.go b/pkg/apis/network.harvesterhci.io/v1beta1/vlanstatus.go index e114bbb7..664331eb 100644 --- a/pkg/apis/network.harvesterhci.io/v1beta1/vlanstatus.go +++ b/pkg/apis/network.harvesterhci.io/v1beta1/vlanstatus.go @@ -30,14 +30,20 @@ type VlStatus struct { Node string `json:"node"` // +optional - VLANIDs []uint16 `json:"vlanIds,omitempty"` + LocalAreas []LocalArea `json:"localAreas,omitempty"` // +optional LinkStatus []LinkStatus `json:"linkStatus,omitempty"` // +optional Conditions []Condition `json:"conditions,omitempty"` } +type LocalArea struct { + VID uint16 `json:"vlanID"` + CIDR string `json:"cidr,omitempty"` +} + type LinkStatus struct { + Name string `json:"name"` // +optional Index int `json:"index,omitempty"` // +optional diff --git a/pkg/controller/agent/nad/controller.go b/pkg/controller/agent/nad/controller.go index 64665a75..b86e4af6 100644 --- a/pkg/controller/agent/nad/controller.go +++ b/pkg/controller/agent/nad/controller.go @@ -36,7 +36,7 @@ func Register(ctx context.Context, management *config.Management) error { return nil } -func (h Handler) OnChange(_ string, nad *nadv1.NetworkAttachmentDefinition) (*nadv1.NetworkAttachmentDefinition, error) { +func (h Handler) OnChange(key string, nad *nadv1.NetworkAttachmentDefinition) (*nadv1.NetworkAttachmentDefinition, error) { if nad == nil || nad.DeletionTimestamp != nil { return nil, nil } @@ -54,7 +54,7 @@ func (h Handler) OnChange(_ string, nad *nadv1.NetworkAttachmentDefinition) (*na return nad, nil } -func (h Handler) OnRemove(_ string, nad *nadv1.NetworkAttachmentDefinition) (*nadv1.NetworkAttachmentDefinition, error) { +func (h Handler) OnRemove(key string, nad *nadv1.NetworkAttachmentDefinition) (*nadv1.NetworkAttachmentDefinition, error) { if nad == nil { return nil, nil } @@ -78,11 +78,6 @@ func (h Handler) OnRemove(_ string, nad *nadv1.NetworkAttachmentDefinition) (*na } func (h Handler) addLocalArea(nad *nadv1.NetworkAttachmentDefinition) error { - vlanID, err := strconv.Atoi(nad.Labels[utils.KeyVlanLabel]) - if err != nil { - return fmt.Errorf("invalid vlanconfig %s", nad.Labels[utils.KeyVlanLabel]) - } - v, err := vlan.GetVlan(nad.Labels[utils.KeyClusterNetworkLabel]) if err != nil && !errors.As(err, &netlink.LinkNotFoundError{}) { return err @@ -90,14 +85,12 @@ func (h Handler) addLocalArea(nad *nadv1.NetworkAttachmentDefinition) error { return nil } - layer3NetworkConf := &utils.Layer3NetworkConf{} - if nad.Annotations != nil && nad.Annotations[utils.KeyNetworkConf] != "" { - if layer3NetworkConf, err = utils.NewLayer3NetworkConf(nad.Annotations[utils.KeyNetworkConf]); err != nil { - return err - } + localArea, err := GetLocalArea(nad) + if err != nil { + return fmt.Errorf("failed to get local area from nad %s/%s, error: %w", nad.Namespace, nad.Name, err) } - return v.AddLocalArea(uint16(vlanID), layer3NetworkConf.CIDR) + return v.AddLocalArea(localArea) } func (h Handler) existDuplicateNad(vlanIdStr, cn string) (bool, error) { @@ -113,11 +106,6 @@ func (h Handler) existDuplicateNad(vlanIdStr, cn string) (bool, error) { } func (h Handler) removeLocalArea(nad *nadv1.NetworkAttachmentDefinition) error { - vlanID, err := strconv.Atoi(nad.Labels[utils.KeyVlanLabel]) - if err != nil { - return fmt.Errorf("invalid vlanconfig %s", nad.Labels[utils.KeyVlanLabel]) - } - v, err := vlan.GetVlan(nad.Labels[utils.KeyClusterNetworkLabel]) if err != nil && !errors.As(err, &netlink.LinkNotFoundError{}) { return err @@ -125,13 +113,29 @@ func (h Handler) removeLocalArea(nad *nadv1.NetworkAttachmentDefinition) error { return nil } + localArea, err := GetLocalArea(nad) + if err != nil { + return fmt.Errorf("failed to get local area from nad %s/%s, error: %w", nad.Namespace, nad.Name, err) + } + + return v.RemoveLocalArea(localArea) +} + +func GetLocalArea(nad *nadv1.NetworkAttachmentDefinition) (*vlan.LocalArea, error) { + vlanID, err := strconv.Atoi(nad.Labels[utils.KeyVlanLabel]) + if err != nil { + return nil, fmt.Errorf("invalid vlanconfig %s", nad.Labels[utils.KeyVlanLabel]) + } + layer3NetworkConf := &utils.Layer3NetworkConf{} if nad.Annotations != nil && nad.Annotations[utils.KeyNetworkConf] != "" { - layer3NetworkConf, err = utils.NewLayer3NetworkConf(nad.Annotations[utils.KeyNetworkConf]) - if err != nil { - return err + if layer3NetworkConf, err = utils.NewLayer3NetworkConf(nad.Annotations[utils.KeyNetworkConf]); err != nil { + return nil, err } } - return v.RemoveLocalArea(uint16(vlanID), layer3NetworkConf.CIDR) + return &vlan.LocalArea{ + Vid: uint16(vlanID), + Cidr: layer3NetworkConf.CIDR, + }, nil } diff --git a/pkg/controller/agent/vlanconfig/controller.go b/pkg/controller/agent/vlanconfig/controller.go index 9101541a..0585e5a9 100644 --- a/pkg/controller/agent/vlanconfig/controller.go +++ b/pkg/controller/agent/vlanconfig/controller.go @@ -5,8 +5,8 @@ import ( "encoding/json" "errors" "fmt" + "hash/crc32" - ctlcniv1 "github.com/harvester/harvester/pkg/generated/controllers/k8s.cni.cncf.io/v1" ctlcorev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/vishvananda/netlink" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -17,10 +17,12 @@ import ( "github.com/harvester/harvester-network-controller/pkg/apis/network.harvesterhci.io" networkv1 "github.com/harvester/harvester-network-controller/pkg/apis/network.harvesterhci.io/v1beta1" "github.com/harvester/harvester-network-controller/pkg/config" + "github.com/harvester/harvester-network-controller/pkg/controller/agent/nad" ctlnetworkv1 "github.com/harvester/harvester-network-controller/pkg/generated/controllers/network.harvesterhci.io/v1beta1" "github.com/harvester/harvester-network-controller/pkg/network/iface" "github.com/harvester/harvester-network-controller/pkg/network/vlan" "github.com/harvester/harvester-network-controller/pkg/utils" + ctlcniv1 "github.com/harvester/harvester/pkg/generated/controllers/k8s.cni.cncf.io/v1" ) const ( @@ -65,7 +67,7 @@ func Register(ctx context.Context, management *config.Management) error { return nil } -func (h Handler) OnChange(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { +func (h Handler) OnChange(key string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { if vc == nil || vc.DeletionTimestamp != nil { return nil, nil } @@ -95,7 +97,7 @@ func (h Handler) OnChange(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanCo return vc, nil } -func (h Handler) OnRemove(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { +func (h Handler) OnRemove(key string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { klog.Infof("vlan config %s has been removed", vc.Name) if err := h.removeVLAN(vc); err != nil { @@ -105,6 +107,10 @@ func (h Handler) OnRemove(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanCo } func (h Handler) MatchNode(vc *networkv1.VlanConfig) (bool, string, error) { + if vc.Annotations == nil || vc.Annotations[utils.KeyMatchedNodes] == "" { + return false, "", nil + } + var matchedNodes []string if err := json.Unmarshal([]byte(vc.Annotations[utils.KeyMatchedNodes]), &matchedNodes); err != nil { return false, "", nil @@ -116,8 +122,7 @@ func (h Handler) MatchNode(vc *networkv1.VlanConfig) (bool, string, error) { if err != nil { return false, "", err } - key := network.GroupName + "/" + vc.Spec.ClusterNetwork - return true, node.Labels[key], nil + return true, node.Labels[utils.KeyVlanConfigLabel], nil } } @@ -127,10 +132,10 @@ func (h Handler) MatchNode(vc *networkv1.VlanConfig) (bool, string, error) { func (h Handler) setupVLAN(vc *networkv1.VlanConfig) error { var v *vlan.Vlan var setupErr error - var vids []uint16 + var localAreas []*vlan.LocalArea var uplink *iface.Link // get VIDs - vids, setupErr = h.getNadVidList(iface.GenerateName(vc.Spec.ClusterNetwork, iface.BridgeSuffix)) + localAreas, setupErr = h.getLocalAreas(iface.GenerateName(vc.Spec.ClusterNetwork, iface.BridgeSuffix)) if setupErr != nil { goto updateStatus } @@ -140,13 +145,14 @@ func (h Handler) setupVLAN(vc *networkv1.VlanConfig) error { goto updateStatus } // set up VLAN - v = vlan.NewVlan(vc.Spec.ClusterNetwork, vids) + v = vlan.NewVlan(vc.Spec.ClusterNetwork, localAreas) setupErr = v.Setup(uplink) updateStatus: // Update status and still return setup error if not nil if err := h.updateStatus(vc, v, setupErr); err != nil { - return fmt.Errorf("update status into vlanstatus %s", h.statusName(vc.Name)) + return fmt.Errorf("update status into vlanstatus %s failed, error: %w, setup error: %v", + h.statusName(vc.Name), err, setupErr) } if setupErr != nil { return fmt.Errorf("set up VLAN failed, vlanconfig: %s, node: %s, error: %w", vc.Name, h.nodeName, setupErr) @@ -180,7 +186,8 @@ updateStatus: return err } if err := h.deleteStatus(vc, teardownErr); err != nil { - return fmt.Errorf("update status into vlanstatus %s", h.statusName(vc.Name)) + return fmt.Errorf("update status into vlanstatus %s failed, error: %w, teardown error: %v", + h.statusName(vc.Name), err, teardownErr) } if teardownErr != nil { return fmt.Errorf("tear down VLAN failed, vlanconfig: %s, node: %s, error: %w", vc.Name, h.nodeName, teardownErr) @@ -207,7 +214,7 @@ func setUplink(vc *networkv1.VlanConfig) (*iface.Link, error) { // Note: do not use &netlink.Bond{} bond := netlink.NewLinkBond(linkAttrs) // set bonding mode - mode := netlink.BOND_MODE_BALANCE_TLB + mode := netlink.BOND_MODE_ACTIVE_BACKUP if vc.Spec.Uplink.BondOptions != nil && vc.Spec.Uplink.BondOptions.Mode != "" { mode = netlink.StringToBondMode(string(vc.Spec.Uplink.BondOptions.Mode)) } @@ -225,14 +232,13 @@ func setUplink(vc *networkv1.VlanConfig) (*iface.Link, error) { return &iface.Link{Link: b}, nil } -func (h Handler) getNadVidList(bridgeName string) ([]uint16, error) { +func (h Handler) getLocalAreas(bridgeName string) ([]*vlan.LocalArea, error) { nads, err := h.nadCache.List("", labels.Everything()) if err != nil { return nil, fmt.Errorf("list nad failed, error: %v", err) } - vidList := make([]uint16, 0) - + localAreas := make([]*vlan.LocalArea, 0) for _, n := range nads { netconf := &utils.NetConf{} if err := json.Unmarshal([]byte(n.Spec.Config), netconf); err != nil { @@ -241,11 +247,15 @@ func (h Handler) getNadVidList(bridgeName string) ([]uint16, error) { if netconf.Type == bridgeCNIName && netconf.BrName == bridgeName { klog.Infof("add nad:%s with vid:%d to the list", n.Name, netconf.Vlan) - vidList = append(vidList, uint16(netconf.Vlan)) + localArea, err := nad.GetLocalArea(n) + if err != nil { + return nil, fmt.Errorf("failed to get local area from nad %s/%s, error: %w", n.Namespace, n.Name, err) + } + localAreas = append(localAreas, localArea) } } - return vidList, nil + return localAreas, nil } func (h Handler) updateStatus(vc *networkv1.VlanConfig, v *vlan.Vlan, setupErr error) error { @@ -288,9 +298,16 @@ func (h Handler) updateStatus(vc *networkv1.VlanConfig, v *vlan.Vlan, setupErr e if setupErr == nil { networkv1.Ready.SetStatusBool(vStatus, true) networkv1.Ready.Message(vStatus, "") - vStatus.Status.VLANIDs = v.ListLocalArea() + vStatus.Status.LocalAreas = []networkv1.LocalArea{} + for _, la := range v.ListLocalArea() { + vStatus.Status.LocalAreas = append(vStatus.Status.LocalAreas, networkv1.LocalArea{ + VID: la.Vid, + CIDR: la.Cidr, + }) + } vStatus.Status.LinkStatus = []networkv1.LinkStatus{ { + Name: v.Bridge().Name, Index: v.Bridge().Index, Type: v.Bridge().Type(), MAC: v.Bridge().HardwareAddr.String(), @@ -299,6 +316,7 @@ func (h Handler) updateStatus(vc *networkv1.VlanConfig, v *vlan.Vlan, setupErr e MasterIndex: v.Bridge().MasterIndex, }, { + Name: v.Uplink().Attrs().Name, Index: v.Uplink().Attrs().Index, Type: v.Uplink().Type(), MAC: v.Uplink().Attrs().HardwareAddr.String(), @@ -350,13 +368,18 @@ func (h Handler) deleteStatus(vc *networkv1.VlanConfig, teardownErr error) error return nil } +// vlanstatus name: -- func (h Handler) statusName(vcName string) string { name := vcName + "-" + h.nodeName - if len(name) > 63 { - name = name[:63] + digest := crc32.ChecksumIEEE([]byte(name)) + suffix := fmt.Sprintf("%08x", digest) + // The name contains no more than 63 characters + maxLengthOfName := 63 - 1 - len(suffix) + if len(name) > maxLengthOfName { + name = name[:maxLengthOfName] } - return name + return name + "-" + suffix } func (h Handler) addNodeLabel(vc *networkv1.VlanConfig) error { @@ -364,9 +387,10 @@ func (h Handler) addNodeLabel(vc *networkv1.VlanConfig) error { if err != nil { return err } + // Since the length of cluster network isn't bigger than 12, the length of key will less than 63. key := network.GroupName + "/" + vc.Spec.ClusterNetwork - - if node.Labels != nil && node.Labels[key] == vc.Name { + if node.Labels != nil && node.Labels[key] == utils.ValueTrue && + node.Labels[utils.KeyVlanConfigLabel] == vc.Name { return nil } @@ -374,10 +398,11 @@ func (h Handler) addNodeLabel(vc *networkv1.VlanConfig) error { if nodeCopy.Labels == nil { nodeCopy.Labels = make(map[string]string) } - nodeCopy.Labels[key] = vc.Name + nodeCopy.Labels[key] = utils.ValueTrue + nodeCopy.Labels[utils.KeyVlanConfigLabel] = vc.Name if _, err := h.nodeClient.Update(nodeCopy); err != nil { - return fmt.Errorf("add VLAN labels %s=true to node %s failed, error: %w", key, h.nodeName, err) + return fmt.Errorf("add labels for vlanconfig %s to node %s failed, error: %w", vc.Name, h.nodeName, err) } return nil @@ -388,13 +413,15 @@ func (h Handler) removeNodeLabel(vc *networkv1.VlanConfig) error { if err != nil { return err } - key := network.GroupName + vc.Spec.ClusterNetwork - if node.Labels != nil && node.Labels[key] == vc.Name { + key := network.GroupName + "/" + vc.Spec.ClusterNetwork + if node.Labels != nil && (node.Labels[key] == utils.ValueTrue || + node.Labels[utils.KeyVlanConfigLabel] == vc.Name) { nodeCopy := node.DeepCopy() delete(nodeCopy.Labels, key) + delete(nodeCopy.Labels, utils.KeyVlanConfigLabel) if _, err := h.nodeClient.Update(nodeCopy); err != nil { - return fmt.Errorf("remove VLAN labels %s=true from node %s failed, error: %w", key, h.nodeName, err) + return fmt.Errorf("remove labels for vlanconfig %s from node %s failed, error: %w", vc.Name, h.nodeName, err) } } diff --git a/pkg/controller/manager/common/mgmt.go b/pkg/controller/manager/common/mgmt.go new file mode 100644 index 00000000..1e4984c5 --- /dev/null +++ b/pkg/controller/manager/common/mgmt.go @@ -0,0 +1,3 @@ +package common + +const ManagementClusterNetworkName = "mgmt" diff --git a/pkg/controller/manager/nad/controller.go b/pkg/controller/manager/nad/controller.go index eedb281d..0ceddc50 100644 --- a/pkg/controller/manager/nad/controller.go +++ b/pkg/controller/manager/nad/controller.go @@ -3,6 +3,7 @@ package nad import ( "context" "fmt" + "github.com/harvester/harvester-network-controller/pkg/apis/network.harvesterhci.io" "sync" "time" @@ -90,7 +91,7 @@ func Register(ctx context.Context, management *config.Management) error { return nil } -func (h Handler) OnChange(_ string, nad *cniv1.NetworkAttachmentDefinition) (*cniv1.NetworkAttachmentDefinition, error) { +func (h Handler) OnChange(key string, nad *cniv1.NetworkAttachmentDefinition) (*cniv1.NetworkAttachmentDefinition, error) { if nad == nil || nad.DeletionTimestamp != nil { return nil, nil } @@ -104,7 +105,7 @@ func (h Handler) OnChange(_ string, nad *cniv1.NetworkAttachmentDefinition) (*cn return nad, nil } -func (h Handler) OnRemove(_ string, nad *cniv1.NetworkAttachmentDefinition) (*cniv1.NetworkAttachmentDefinition, error) { +func (h Handler) OnRemove(key string, nad *cniv1.NetworkAttachmentDefinition) (*cniv1.NetworkAttachmentDefinition, error) { if nad == nil { return nil, nil } @@ -240,7 +241,26 @@ func constructJob(cur *batchv1.Job, namespace, image, dhcpServerAddr string, nad ImagePullPolicy: corev1.PullIfNotPresent, }, } - + // Add nodeAffinity to prove the job pod is scheduled to the proper node with the specified cluster network + job.Spec.Template.Spec.Affinity = &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: network.GroupName + "/" + nad.Labels[utils.KeyClusterNetworkLabel], + Operator: corev1.NodeSelectorOpIn, + Values: []string{ + utils.ValueTrue, + }, + }, + }, + }, + }, + }, + }, + } job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever job.Spec.Template.Spec.ServiceAccountName = jobServiceAccountName backoffLimit := int32(1) diff --git a/pkg/controller/manager/node/controller.go b/pkg/controller/manager/node/controller.go index d33a9b6e..55c57b83 100644 --- a/pkg/controller/manager/node/controller.go +++ b/pkg/controller/manager/node/controller.go @@ -2,6 +2,10 @@ package node import ( "context" + "fmt" + "github.com/harvester/harvester-network-controller/pkg/apis/network.harvesterhci.io" + "github.com/harvester/harvester-network-controller/pkg/controller/manager/common" + ctlcorev1 "github.com/rancher/wrangler/pkg/generated/controllers/core/v1" "github.com/deckarep/golang-set/v2" corev1 "k8s.io/api/core/v1" @@ -16,8 +20,9 @@ import ( const controllerName = "harvester-network-manager-node-controller" type Handler struct { - vcCache ctlnetworkv1.VlanConfigCache - vcClient ctlnetworkv1.VlanConfigClient + nodeClient ctlcorev1.NodeClient + vcCache ctlnetworkv1.VlanConfigCache + vcClient ctlnetworkv1.VlanConfigClient } func Register(ctx context.Context, management *config.Management) error { @@ -25,8 +30,9 @@ func Register(ctx context.Context, management *config.Management) error { vcs := management.HarvesterNetworkFactory.Network().V1beta1().VlanConfig() h := Handler{ - vcClient: vcs, - vcCache: vcs.Cache(), + nodeClient: nodes, + vcCache: vcs.Cache(), + vcClient: vcs, } nodes.OnChange(ctx, controllerName, h.OnChange) @@ -34,23 +40,27 @@ func Register(ctx context.Context, management *config.Management) error { return nil } -func (h Handler) OnChange(_ string, node *corev1.Node) (*corev1.Node, error) { +func (h Handler) OnChange(key string, node *corev1.Node) (*corev1.Node, error) { if node == nil || node.DeletionTimestamp != nil { return nil, nil } + if err := h.ensureMgmtLabels(node); err != nil { + return nil, err + } + vcs, err := h.vcCache.List(labels.Everything()) if err != nil { return nil, err } - for _, vc := range vcs { if err := h.updateMatchedNodeAnnotation(vc, node); err != nil { - return nil, err + return nil, fmt.Errorf("failed to update matched node annotation, vc: %s, node: %s, error: %w", + vc.Name, node.Name, err) } } - return nil, nil + return node, nil } func (h Handler) updateMatchedNodeAnnotation(vc *networkv1.VlanConfig, node *corev1.Node) error { @@ -90,3 +100,21 @@ func (h Handler) updateMatchedNodeAnnotation(vc *networkv1.VlanConfig, node *cor return nil } + +func (h Handler) ensureMgmtLabels(node *corev1.Node) error { + key := network.GroupName + "/" + common.ManagementClusterNetworkName + if node.Labels != nil && node.Labels[key] == utils.ValueTrue { + return nil + } + + nodeCopy := node.DeepCopy() + if nodeCopy.Labels == nil { + nodeCopy.Labels = make(map[string]string) + } + nodeCopy.Labels[key] = utils.ValueTrue + if _, err := h.nodeClient.Update(nodeCopy); err != nil { + return fmt.Errorf("update node %s failed, error: %w", node.Name, err) + } + + return nil +} diff --git a/pkg/controller/manager/vlanconfig/controller.go b/pkg/controller/manager/vlanconfig/controller.go index 8ef2907b..066ff309 100644 --- a/pkg/controller/manager/vlanconfig/controller.go +++ b/pkg/controller/manager/vlanconfig/controller.go @@ -3,6 +3,8 @@ package vlanconfig import ( "context" "fmt" + "github.com/cenk/backoff" + "github.com/harvester/harvester-network-controller/pkg/controller/manager/common" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -15,8 +17,7 @@ import ( ) const ( - ControllerName = "harvester-network-manager-vlanconfig-controller" - managementClusterNetworkName = "mgmt" + ControllerName = "harvester-network-manager-vlanconfig-controller" ) type Handler struct { @@ -45,7 +46,7 @@ func Register(ctx context.Context, management *config.Management) error { return nil } -func (h Handler) OnChange(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { +func (h Handler) OnChange(key string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { if vc == nil || vc.DeletionTimestamp != nil { return nil, nil } @@ -58,7 +59,7 @@ func (h Handler) OnChange(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanCo return nil, nil } -func (h Handler) OnRemove(_ string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { +func (h Handler) OnRemove(key string, vc *networkv1.VlanConfig) (*networkv1.VlanConfig, error) { klog.Infof("vlan config %s has been removed", vc.Name) // delete clusternetwork if there isn't any other vlanconfigs if err := h.deleteClusterNetwork(vc.Spec.ClusterNetwork); err != nil { @@ -102,13 +103,18 @@ func (h Handler) deleteClusterNetwork(name string) error { } func (h Handler) initialize() error { - // It's not allowed to use the local cache to get the cluster network in the register period - // because the factory hasn't started. We just create the cluster network and ignore the `AlreadyExists` error. - if _, err := h.cnClient.Create(&networkv1.ClusterNetwork{ - ObjectMeta: metav1.ObjectMeta{ - Name: managementClusterNetworkName, - }, - }); err != nil && !apierrors.IsAlreadyExists(err) { + if err := backoff.Retry(func() error { + // It's not allowed to use the local cache to get the cluster network in the register period + // because the factory hasn't started. We just create the cluster network and ignore the `AlreadyExists` error. + if _, err := h.cnClient.Create(&networkv1.ClusterNetwork{ + ObjectMeta: metav1.ObjectMeta{ + Name: common.ManagementClusterNetworkName, + }, + }); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("create %s failed, error: %w", common.ManagementClusterNetworkName, err) + } + return nil + }, backoff.NewExponentialBackOff()); err != nil { return err } diff --git a/pkg/network/iface/bond.go b/pkg/network/iface/bond.go index e1c5ffc3..a4055a46 100644 --- a/pkg/network/iface/bond.go +++ b/pkg/network/iface/bond.go @@ -3,6 +3,7 @@ package iface import ( "errors" "fmt" + "net" "github.com/vishvananda/netlink" ) @@ -81,16 +82,21 @@ func (b *Bond) ensureBondSlaves() error { if l.Attrs().MasterIndex != 0 && l.Attrs().MasterIndex != b.Index { return fmt.Errorf("%s has been enslaved by the link with index %d", l.Attrs().Name, l.Attrs().MasterIndex) } + // The slave link should be down before enslaved, otherwise, there will be error like `operation not permitted`. + if err := netlink.LinkSetDown(l); err != nil { + return fmt.Errorf("set slave %s down failed, error: %w", slave, err) + } if err := netlink.LinkSetBondSlave(l, b.Bond); err != nil { return fmt.Errorf("add slave %s to bond %s failed, error: %w", slave, b.Name, err) } } - if l.Attrs().OperState != netlink.OperUp { + if l.Attrs().Flags&net.FlagUp == 0 { if err := netlink.LinkSetUp(l); err != nil { return err } } + // delete the handled slave delete(slaveMap, slave) } diff --git a/pkg/network/iface/bridge.go b/pkg/network/iface/bridge.go index 9f78036a..bdaac9ba 100644 --- a/pkg/network/iface/bridge.go +++ b/pkg/network/iface/bridge.go @@ -73,7 +73,6 @@ func disableBridgeNF() error { func (br *Bridge) Fetch() error { l, err := netlink.LinkByName(br.Name) - // TODO bridge not found if err != nil { return fmt.Errorf("could not lookup link %s, error: %w", br.Name, err) } diff --git a/pkg/network/iface/link.go b/pkg/network/iface/link.go index 23c388af..c21ea03f 100644 --- a/pkg/network/iface/link.go +++ b/pkg/network/iface/link.go @@ -100,7 +100,7 @@ func (l *Link) clearMacVlan() error { return nil } -func (l *Link) SetMaster(br *Bridge, vids []uint16) error { +func (l *Link) SetMaster(br *Bridge) error { if l.Attrs().MasterIndex == br.Index { return nil } @@ -112,12 +112,6 @@ func (l *Link) SetMaster(br *Bridge, vids []uint16) error { return fmt.Errorf("%s set %s as master failed, error: %w", l.Attrs().Name, br.Name, err) } - for _, vid := range vids { - if err := l.AddBridgeVlan(vid); err != nil { - return err - } - } - return nil } diff --git a/pkg/network/vlan/vlan.go b/pkg/network/vlan/vlan.go index 1ddc0290..5fe2d225 100644 --- a/pkg/network/vlan/vlan.go +++ b/pkg/network/vlan/vlan.go @@ -8,10 +8,15 @@ import ( ) type Vlan struct { - name string - bridge *iface.Bridge - uplink *iface.Link - vids []uint16 + name string + bridge *iface.Bridge + uplink *iface.Link + localAreas []*LocalArea +} + +type LocalArea struct { + Vid uint16 + Cidr string } func (v *Vlan) Type() string { @@ -20,13 +25,13 @@ func (v *Vlan) Type() string { // The bridge of a pure VLAN may have no latest information // The NIC of a pure VLAN can be empty -func NewVlan(name string, vids []uint16) *Vlan { +func NewVlan(name string, localAreas []*LocalArea) *Vlan { br := iface.NewBridge(iface.GenerateName(name, iface.BridgeSuffix)) return &Vlan{ - name: name, - bridge: br, - vids: vids, + name: name, + bridge: br, + localAreas: localAreas, } } @@ -66,11 +71,6 @@ func GetVlan(name string) (*Vlan, error) { } v.uplink = uplink - v.vids, err = v.uplink.ListBridgeVlan() - if err != nil { - return nil, err - } - return v, nil } @@ -81,12 +81,17 @@ func (v *Vlan) Setup(l *iface.Link) error { } // set master - if err := l.SetMaster(v.bridge, v.vids); err != nil { + if err := l.SetMaster(v.bridge); err != nil { return err } - v.uplink = l + for _, la := range v.localAreas { + if err := v.AddLocalArea(la); err != nil { + return fmt.Errorf("add local area %v failed, error: %w", la, err) + } + } + return nil } @@ -114,58 +119,60 @@ func (v *Vlan) Teardown() error { return nil } -func (v *Vlan) AddLocalArea(vid uint16, cidr string) error { +func (v *Vlan) AddLocalArea(la *LocalArea) error { if v.uplink == nil { return fmt.Errorf("bridge %s hasn't attached an uplink", v.bridge.Name) } - if ok, _ := v.findVid(vid); ok { + if ok, _ := v.findVid(la.Vid); ok { return nil } - if err := v.uplink.AddBridgeVlan(vid); err != nil { - return fmt.Errorf("add bridge vlanconfig %d failed, error: %w", vid, err) + if err := v.uplink.AddBridgeVlan(la.Vid); err != nil { + return fmt.Errorf("add bridge vlanconfig %d failed, error: %w", la.Vid, err) } - v.vids = append(v.vids, vid) - if cidr == "" { + if la.Cidr == "" { + v.localAreas = append(v.localAreas, la) return nil } - if err := iface.EnsureRouteViaGateway(cidr); err != nil { - return fmt.Errorf("ensure %s to route via gateway failed, error: %w", cidr, err) + if err := iface.EnsureRouteViaGateway(la.Cidr); err != nil { + return fmt.Errorf("ensure %s to route via gateway failed, error: %w", la.Cidr, err) } + v.localAreas = append(v.localAreas, la) return nil } -func (v *Vlan) RemoveLocalArea(vid uint16, cidr string) error { +func (v *Vlan) RemoveLocalArea(la *LocalArea) error { if v.uplink == nil { return fmt.Errorf("bridge %s hasn't attached an uplink", v.bridge.Name) } - ok, index := v.findVid(vid) + ok, index := v.findVid(la.Vid) if !ok { return nil } - if err := v.uplink.DelBridgeVlan(vid); err != nil { - return fmt.Errorf("remove bridge vlanconfig %d failed, error: %w", vid, err) + if err := v.uplink.DelBridgeVlan(la.Vid); err != nil { + return fmt.Errorf("remove bridge vlanconfig %d failed, error: %w", la.Vid, err) } - v.vids = append(v.vids[:index], v.vids[index+1:]...) - if cidr == "" { + if la.Cidr == "" { + v.localAreas = append(v.localAreas[:index], v.localAreas[index+1:]...) return nil } - if err := iface.DeleteRouteViaGateway(cidr); err != nil { - return fmt.Errorf("delete route with dst %s via gateway failed, error: %w", cidr, err) + if err := iface.DeleteRouteViaGateway(la.Cidr); err != nil { + return fmt.Errorf("delete route with dst %s via gateway failed, error: %w", la.Cidr, err) } + v.localAreas = append(v.localAreas[:index], v.localAreas[index+1:]...) return nil } -func (v *Vlan) ListLocalArea() []uint16 { - return v.vids +func (v *Vlan) ListLocalArea() []*LocalArea { + return v.localAreas } func (v *Vlan) Bridge() *iface.Bridge { @@ -177,8 +184,8 @@ func (v *Vlan) Uplink() *iface.Link { } func (v *Vlan) findVid(vid uint16) (bool, int) { - for i, v := range v.vids { - if v == vid { + for i, la := range v.localAreas { + if la.Vid == vid { return true, i } } diff --git a/pkg/utils/labels.go b/pkg/utils/labels.go index cf54ec11..3d39d7b7 100644 --- a/pkg/utils/labels.go +++ b/pkg/utils/labels.go @@ -10,4 +10,6 @@ const ( KeyNodeLabel = network.GroupName + "/node" KeyMatchedNodes = network.GroupName + "/matched-nodes" + + ValueTrue = "true" ) diff --git a/pkg/webhook/nad/mutator.go b/pkg/webhook/nad/mutator.go index e4c2f3dd..39103fec 100644 --- a/pkg/webhook/nad/mutator.go +++ b/pkg/webhook/nad/mutator.go @@ -24,7 +24,7 @@ func NewNadMutator() *nadMutator { return &nadMutator{} } -func (n *nadMutator) Create(request *types.Request, newObj runtime.Object) (types.Patch, error) { +func (n *nadMutator) Create(_ *types.Request, newObj runtime.Object) (types.Patch, error) { netAttachDef := newObj.(*cniv1.NetworkAttachmentDefinition) labels := netAttachDef.Labels @@ -39,8 +39,9 @@ func (n *nadMutator) Create(request *types.Request, newObj runtime.Object) (type labels[utils.KeyVlanLabel] = strconv.Itoa(netconf.Vlan) lenOfBrName := len(netconf.BrName) - if lenOfBrName < 3 { - return nil, fmt.Errorf(createErr, netAttachDef.Namespace, netAttachDef.Name, fmt.Errorf("the length of brName must be more than 3")) + if lenOfBrName < len(iface.BridgeSuffix) { + return nil, fmt.Errorf(createErr, netAttachDef.Namespace, netAttachDef.Name, + fmt.Errorf("the length of brName must be more than %d", len(iface.BridgeSuffix))) } labels[utils.KeyClusterNetworkLabel] = netconf.BrName[:lenOfBrName-len(iface.BridgeSuffix)] diff --git a/pkg/webhook/nad/validator.go b/pkg/webhook/nad/validator.go index d20288ce..fd38a1dd 100644 --- a/pkg/webhook/nad/validator.go +++ b/pkg/webhook/nad/validator.go @@ -29,7 +29,6 @@ type nadValidator struct { var _ types.Validator = &nadValidator{} func NewNadValidator(vmCache ctlkubevirtv1.VirtualMachineCache) *nadValidator { - vmCache.AddIndexer(indexeres.VMByNetworkIndex, indexeres.VMByNetwork) return &nadValidator{ vmCache: vmCache, } diff --git a/pkg/webhook/vlanconfig/mutator.go b/pkg/webhook/vlanconfig/mutator.go index 70c2a7ce..79683c88 100644 --- a/pkg/webhook/vlanconfig/mutator.go +++ b/pkg/webhook/vlanconfig/mutator.go @@ -59,7 +59,7 @@ func (v *vlanConfigMutator) Update(_ *types.Request, oldObj, newObj runtime.Obje } func getCnLabelPatch(v *networkv1.VlanConfig) types.Patch { - if v.Labels[utils.KeyClusterNetworkLabel] == v.Spec.ClusterNetwork { + if v.Labels != nil && v.Labels[utils.KeyClusterNetworkLabel] == v.Spec.ClusterNetwork { return nil } @@ -87,7 +87,12 @@ func (v *vlanConfigMutator) matchNodes(vc *networkv1.VlanConfig) (types.Patch, e return nil, err } - matchedNodes := make([]string, len(nodes)) + lenOfNodes := len(nodes) + if lenOfNodes == 0 { + return nil, nil + } + + matchedNodes := make([]string, lenOfNodes) for i, node := range nodes { matchedNodes[i] = node.Name } diff --git a/pkg/webhook/vlanconfig/validator.go b/pkg/webhook/vlanconfig/validator.go index 8c6492df..0552de28 100644 --- a/pkg/webhook/vlanconfig/validator.go +++ b/pkg/webhook/vlanconfig/validator.go @@ -19,7 +19,7 @@ import ( const ( createErr = "could not create vlanConfig %s because %w" updateErr = "could not update vlanConfig %s because %w" - deleteErr = "cloud not delete vlanConfig %s because %w" + deleteErr = "could not delete vlanConfig %s because %w" ) type vlanConfigValidator struct { @@ -61,6 +61,7 @@ func (v *vlanConfigValidator) Delete(_ *types.Request, oldObj runtime.Object) er if err != nil { return fmt.Errorf(deleteErr, vc.Name, err) } + nads, err := v.nadCache.List("", labels.Set(map[string]string{ utils.KeyClusterNetworkLabel: vc.Spec.ClusterNetwork, }).AsSelector()) @@ -82,7 +83,7 @@ func (v *vlanConfigValidator) Delete(_ *types.Request, oldObj runtime.Object) er func (v *vlanConfigValidator) Resource() types.Resource { return types.Resource{ - Names: []string{"vlanconfs"}, + Names: []string{"vlanconfigs"}, Scope: admissionregv1.ClusterScope, APIGroup: networkv1.SchemeGroupVersion.Group, APIVersion: networkv1.SchemeGroupVersion.Version,