add vpc support for capl clusters (#159)

* add vpc support for capl clusters

* disable kube-proxy, address review comments

* use updated version of linode-ccm

* address review comments

* don't use /etc/hosts for node-ip

* add additional interface using machinetemplate

* rebase fix and address comments

* reduce cognitive complexity, update tests and use ccm v0.4.3 for updated helm chart

* update linode-ccm to v0.4.4

* address review comments, add vpc note

---------

Co-authored-by: Rahul Sharma <[email protected]>

Author: rahulait

controller/linodemachine_controller.go

@@ -61,6 +61,7 @@ const (
 	ConditionPreflightAdditionalDisksCreated clusterv1.ConditionType = "PreflightAdditionalDisksCreated"
 	ConditionPreflightConfigured             clusterv1.ConditionType = "PreflightConfigured"
 	ConditionPreflightBootTriggered          clusterv1.ConditionType = "PreflightBootTriggered"
+	ConditionPreflightNetworking             clusterv1.ConditionType = "PreflightNetworking"
 	ConditionPreflightReady                  clusterv1.ConditionType = "PreflightReady"
 )
 
@@ -248,7 +249,6 @@ func (r *LinodeMachineReconciler) reconcile(
 	return
 }
 
-//nolint:cyclop // keep top-level preflight condition checks in the same function for readability
 func (r *LinodeMachineReconciler) reconcileCreate(
 	ctx context.Context,
 	logger logr.Logger,
@@ -312,8 +312,17 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 		return ctrl.Result{}, err
 	}
 
+	return r.reconcileInstanceCreate(ctx, logger, machineScope, linodeInstance)
+}
+
+func (r *LinodeMachineReconciler) reconcileInstanceCreate(
+	ctx context.Context,
+	logger logr.Logger,
+	machineScope *scope.MachineScope,
+	linodeInstance *linodego.Instance,
+) (ctrl.Result, error) {
 	if !reconciler.ConditionTrue(machineScope.LinodeMachine, ConditionPreflightConfigured) {
-		if err = r.configureDisks(ctx, logger, machineScope, linodeInstance.ID); err != nil {
+		if err := r.configureDisks(ctx, logger, machineScope, linodeInstance.ID); err != nil {
 			if reconciler.RecordDecayingCondition(machineScope.LinodeMachine,
 				ConditionPreflightConfigured, string(cerrs.CreateMachineError), err.Error(),
 				reconciler.DefaultMachineControllerPreflightTimeout(r.ReconcileTimeout)) {
@@ -327,7 +336,7 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 	}
 
 	if !reconciler.ConditionTrue(machineScope.LinodeMachine, ConditionPreflightBootTriggered) {
-		if err = machineScope.LinodeClient.BootInstance(ctx, linodeInstance.ID, 0); err != nil {
+		if err := machineScope.LinodeClient.BootInstance(ctx, linodeInstance.ID, 0); err != nil {
 			logger.Error(err, "Failed to boot instance")
 
 			if reconciler.RecordDecayingCondition(machineScope.LinodeMachine,
@@ -342,10 +351,27 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 		conditions.MarkTrue(machineScope.LinodeMachine, ConditionPreflightBootTriggered)
 	}
 
-	if !reconciler.ConditionTrue(machineScope.LinodeMachine, ConditionPreflightReady) {
-		if err = services.AddNodeToNB(ctx, logger, machineScope); err != nil {
+	if !reconciler.ConditionTrue(machineScope.LinodeMachine, ConditionPreflightNetworking) {
+		if err := services.AddNodeToNB(ctx, logger, machineScope); err != nil {
 			logger.Error(err, "Failed to add instance to Node Balancer backend")
 
+			if reconciler.RecordDecayingCondition(machineScope.LinodeMachine,
+				ConditionPreflightNetworking, string(cerrs.CreateMachineError), err.Error(),
+				reconciler.DefaultMachineControllerPreflightTimeout(r.ReconcileTimeout)) {
+				return ctrl.Result{}, err
+			}
+
+			return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerWaitForRunningDelay}, nil
+		}
+
+		conditions.MarkTrue(machineScope.LinodeMachine, ConditionPreflightNetworking)
+	}
+
+	if !reconciler.ConditionTrue(machineScope.LinodeMachine, ConditionPreflightReady) {
+		addrs, err := r.buildInstanceAddrs(ctx, machineScope, linodeInstance.ID)
+		if err != nil {
+			logger.Error(err, "Failed to get instance ip addresses")
+
 			if reconciler.RecordDecayingCondition(machineScope.LinodeMachine,
 				ConditionPreflightReady, string(cerrs.CreateMachineError), err.Error(),
 				reconciler.DefaultMachineControllerPreflightTimeout(r.ReconcileTimeout)) {
@@ -354,12 +380,12 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 
 			return ctrl.Result{RequeueAfter: reconciler.DefaultMachineControllerWaitForRunningDelay}, nil
 		}
+		machineScope.LinodeMachine.Status.Addresses = addrs
 
 		conditions.MarkTrue(machineScope.LinodeMachine, ConditionPreflightReady)
 	}
 
 	machineScope.LinodeMachine.Spec.ProviderID = util.Pointer(fmt.Sprintf("linode://%d", linodeInstance.ID))
-	machineScope.LinodeMachine.Status.Addresses = buildInstanceAddrs(linodeInstance)
 
 	// Set the instance state to signal preflight process is done
 	machineScope.LinodeMachine.Status.InstanceState = util.Pointer(linodego.InstanceOffline)

controller/linodemachine_controller_helpers.go

@@ -88,6 +88,7 @@ func (r *LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineSc
 		createConfig.RootPass = uuid.NewString()
 	}
 
+	// if vpc, attach additional interface to linode (eth1)
 	if machineScope.LinodeCluster.Spec.VPCRef != nil {
 		iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger)
 		if err != nil {
@@ -101,20 +102,39 @@ func (r *LinodeMachineReconciler) newCreateConfig(ctx context.Context, machineSc
 	return createConfig, nil
 }
 
-func buildInstanceAddrs(linodeInstance *linodego.Instance) []clusterv1.MachineAddress {
-	addrs := []clusterv1.MachineAddress{}
-	for _, addr := range linodeInstance.IPv4 {
-		addrType := clusterv1.MachineExternalIP
-		if addr.IsPrivate() {
-			addrType = clusterv1.MachineInternalIP
+func (r *LinodeMachineReconciler) buildInstanceAddrs(ctx context.Context, machineScope *scope.MachineScope, instanceID int) ([]clusterv1.MachineAddress, error) {
+	addresses, err := machineScope.LinodeClient.GetInstanceIPAddresses(ctx, instanceID)
+	if err != nil {
+		return nil, fmt.Errorf("get instance ips: %w", err)
+	}
+
+	// get the default instance config
+	configs, err := machineScope.LinodeClient.ListInstanceConfigs(ctx, instanceID, &linodego.ListOptions{})
+	if err != nil || len(configs) == 0 {
+		return nil, fmt.Errorf("list instance configs: %w", err)
+	}
+
+	ips := []clusterv1.MachineAddress{}
+	// check if a node has public ip and store it
+	if len(addresses.IPv4.Public) != 0 {
+		ips = append(ips, clusterv1.MachineAddress{Address: addresses.IPv4.Public[0].Address, Type: clusterv1.MachineExternalIP})
+	}
+
+	// Iterate over interfaces in config and find VPC specific ips
+	for _, iface := range configs[0].Interfaces {
+		if iface.VPCID != nil && iface.IPv4.VPC != "" {
+			ips = append(ips, clusterv1.MachineAddress{Address: iface.IPv4.VPC, Type: clusterv1.MachineInternalIP})
 		}
-		addrs = append(addrs, clusterv1.MachineAddress{
-			Type:    addrType,
-			Address: addr.String(),
-		})
 	}
 
-	return addrs
+	// if a node has private ip, store it as well
+	// NOTE: We specifically store VPC ips first so that they are used first during
+	//       bootstrap when we set `registrationMethod: internal-only-ips`
+	if len(addresses.IPv4.Private) != 0 {
+		ips = append(ips, clusterv1.MachineAddress{Address: addresses.IPv4.Private[0].Address, Type: clusterv1.MachineInternalIP})
+	}
+
+	return ips, nil
 }
 
 func (r *LinodeMachineReconciler) getOwnerMachine(ctx context.Context, linodeMachine infrav1alpha1.LinodeMachine, log logr.Logger) (*clusterv1.Machine, error) {

controller/linodemachine_controller_test.go

@@ -148,10 +148,26 @@ var _ = Describe("create", Label("machine", "create"), func() {
 				IPv4:   []*net.IP{ptr.To(net.IPv4(192, 168, 0, 2))},
 				Status: linodego.InstanceOffline,
 			}, nil)
-		mockLinodeClient.EXPECT().
+		bootInst := mockLinodeClient.EXPECT().
 			BootInstance(ctx, 123, 0).
 			After(createInst).
 			Return(nil)
+		getAddrs := mockLinodeClient.EXPECT().
+			GetInstanceIPAddresses(ctx, 123).
+			After(bootInst).
+			Return(&linodego.InstanceIPAddressResponse{
+				IPv4: &linodego.InstanceIPv4Response{
+					Private: []*linodego.InstanceIP{{Address: "192.168.0.2"}},
+				},
+			}, nil)
+		mockLinodeClient.EXPECT().
+			ListInstanceConfigs(ctx, 123, gomock.Any()).
+			After(getAddrs).
+			Return([]linodego.InstanceConfig{{
+				Devices: &linodego.InstanceConfigDeviceMap{
+					SDA: &linodego.InstanceConfigDevice{DiskID: 100},
+				},
+			}}, nil)
 
 		mScope := scope.MachineScope{
 			Client:        k8sClient,
@@ -308,14 +324,30 @@ var _ = Describe("create", Label("machine", "create"), func() {
 						Private: []*linodego.InstanceIP{{Address: "192.168.0.2"}},
 					},
 				}, nil)
-			mockLinodeClient.EXPECT().
+			createNB := mockLinodeClient.EXPECT().
 				CreateNodeBalancerNode(ctx, 1, 2, linodego.NodeBalancerNodeCreateOptions{
 					Label:   "mock",
 					Address: "192.168.0.2:6443",
 					Mode:    linodego.ModeAccept,
 				}).
 				After(getAddrs).
 				Return(nil, nil)
+			getAddrs = mockLinodeClient.EXPECT().
+				GetInstanceIPAddresses(ctx, 123).
+				After(createNB).
+				Return(&linodego.InstanceIPAddressResponse{
+					IPv4: &linodego.InstanceIPv4Response{
+						Private: []*linodego.InstanceIP{{Address: "192.168.0.2"}},
+					},
+				}, nil)
+			mockLinodeClient.EXPECT().
+				ListInstanceConfigs(ctx, 123, gomock.Any()).
+				After(getAddrs).
+				Return([]linodego.InstanceConfig{{
+					Devices: &linodego.InstanceConfigDeviceMap{
+						SDA: &linodego.InstanceConfigDevice{DiskID: 100},
+					},
+				}}, nil)
 
 			mScope := scope.MachineScope{
 				Client:        k8sClient,
@@ -461,16 +493,38 @@ var _ = Describe("create", Label("machine", "create"), func() {
 				Return(&linodego.InstanceIPAddressResponse{
 					IPv4: &linodego.InstanceIPv4Response{
 						Private: []*linodego.InstanceIP{{Address: "192.168.0.2"}},
+						Public:  []*linodego.InstanceIP{{Address: "172.0.0.2"}},
 					},
 				}, nil)
-			mockLinodeClient.EXPECT().
+			createNB := mockLinodeClient.EXPECT().
 				CreateNodeBalancerNode(ctx, 1, 2, linodego.NodeBalancerNodeCreateOptions{
 					Label:   "mock",
 					Address: "192.168.0.2:6443",
 					Mode:    linodego.ModeAccept,
 				}).
 				After(getAddrs).
 				Return(nil, nil)
+			getAddrs = mockLinodeClient.EXPECT().
+				GetInstanceIPAddresses(ctx, 123).
+				After(createNB).
+				Return(&linodego.InstanceIPAddressResponse{
+					IPv4: &linodego.InstanceIPv4Response{
+						Private: []*linodego.InstanceIP{{Address: "192.168.0.2"}},
+						Public:  []*linodego.InstanceIP{{Address: "172.0.0.2"}},
+					},
+				}, nil)
+			mockLinodeClient.EXPECT().
+				ListInstanceConfigs(ctx, 123, gomock.Any()).
+				After(getAddrs).
+				Return([]linodego.InstanceConfig{{
+					Devices: &linodego.InstanceConfigDeviceMap{
+						SDA: &linodego.InstanceConfigDevice{DiskID: 100},
+					},
+					Interfaces: []linodego.InstanceConfigInterface{{
+						VPCID: ptr.To(1),
+						IPv4:  &linodego.VPCIPv4{VPC: "10.0.0.2"},
+					}},
+				}}, nil)
 
 			_, err = reconciler.reconcileCreate(ctx, logger, &mScope)
 			Expect(err).NotTo(HaveOccurred())
@@ -483,10 +537,20 @@ var _ = Describe("create", Label("machine", "create"), func() {
 			Expect(*linodeMachine.Status.InstanceState).To(Equal(linodego.InstanceOffline))
 			Expect(*linodeMachine.Spec.InstanceID).To(Equal(123))
 			Expect(*linodeMachine.Spec.ProviderID).To(Equal("linode://123"))
-			Expect(linodeMachine.Status.Addresses).To(Equal([]clusterv1.MachineAddress{{
-				Type:    clusterv1.MachineInternalIP,
-				Address: "192.168.0.2",
-			}}))
+			Expect(linodeMachine.Status.Addresses).To(Equal([]clusterv1.MachineAddress{
+				{
+					Type:    clusterv1.MachineExternalIP,
+					Address: "172.0.0.2",
+				},
+				{
+					Type:    clusterv1.MachineInternalIP,
+					Address: "10.0.0.2",
+				},
+				{
+					Type:    clusterv1.MachineInternalIP,
+					Address: "192.168.0.2",
+				},
+			}))
 
 			Expect(testLogs.String()).To(ContainSubstring("creating machine"))
 			Expect(testLogs.String()).To(ContainSubstring("Linode instance already exists"))

docs/src/developers/development.md

@@ -213,7 +213,7 @@ clusterctl generate cluster $CLUSTER_NAME \
   | kubectl apply -f -
 ```
 
-This will provision the cluster with the CNI defaulted to [cilium](../topics/addons.md#cilium)
+This will provision the cluster within VPC with the CNI defaulted to [cilium](../topics/addons.md#cilium)
 and the [linode-ccm](../topics/addons.md#ccm) installed.
 
 ##### Using ClusterClass (alpha)
@@ -245,6 +245,9 @@ To delete the cluster, simply run:
 ```sh
 kubectl delete cluster $CLUSTER_NAME
 ```
+```admonish warning
+VPCs are not deleted when a cluster is deleted using kubectl. One can run `kubectl delete linodevpc <vpcname>` to cleanup VPC once cluster is deleted.
+```
 
 ```admonish question title=""
 For any issues, please refer to the [troubleshooting guide](../topics/troubleshooting.md).

docs/src/topics/getting-started.md

@@ -32,6 +32,9 @@ export LINODE_MACHINE_TYPE=g6-standard-2
 For Regions and Images that do not yet support Akamai's cloud-init datasource CAPL will automatically use a stackscript shim
 to provision the node. If you are using a custom image ensure the [cloud_init](https://www.linode.com/docs/api/images/#image-create) flag is set correctly on it
 ```
+```admonish warning
+By default, clusters are provisioned within VPC. For Regions which do not have [VPC support](https://www.linode.com/docs/products/networking/vpc/#availability) yet, use the VPCLess[TODO] flavor to have clusters provisioned.
+```
 
 ## Register linode as an infrastructure provider
 1. Add `linode` as an infrastructure provider in `~/.cluster-api/clusterctl.yaml`

templates/addons/cilium/cilium.yaml

@@ -18,12 +18,31 @@ spec:
   valuesTemplate: |
     bgpControlPlane:
       enabled: true
+    routingMode: native
+    kubeProxyReplacement: true
+    ipv4NativeRoutingCIDR: 10.0.0.0/8
+    tunnelProtocol: ""
+    enableIPv4Masquerade: true
+    egressMasqueradeInterfaces: eth0
+    k8sServiceHost: {{ .InfraCluster.spec.controlPlaneEndpoint.host }}
+    k8sServicePort: {{ .InfraCluster.spec.controlPlaneEndpoint.port }}
+    extraArgs:
+    - --direct-routing-device=eth1
+    - --nodeport-addresses=0.0.0.0/0
     ipam:
       mode: kubernetes
+    ipv4:
+      enabled: true
+    ipv6:
+      enabled: false
     k8s:
       requireIPv4PodCIDR: true
     hubble:
       relay:
         enabled: true
       ui:
         enabled: true
+#    ipMasqAgent:
+#      enabled: true
+#    bpf:
+#      masquerade: true

templates/addons/provider-linode/linode-ccm.yaml

@@ -9,12 +9,16 @@ spec:
   repoURL: https://linode.github.io/linode-cloud-controller-manager/
   chartName: ccm-linode
   namespace: kube-system
-  version: ${LINODE_CCM_VERSION:=v0.3.24}
+  version: ${LINODE_CCM_VERSION:=v0.4.4}
   options:
     waitForJobs: true
     wait: true
     timeout: 5m
   valuesTemplate: |
+    routeController:
+      vpcName: ${VPC_NAME:=${CLUSTER_NAME}}
+      clusterCIDR: 10.0.0.0/8
+      configureCloudRoutes: true
     secretRef:
       name: "linode-token-region"
     image:

templates/common-init-files/secret.yaml

@@ -38,4 +38,5 @@ stringData:
     modprobe overlay
     modprobe br_netfilter
     sysctl --system
-
+    IPADDR=$(ip a s eth1 |grep 'inet ' |cut -d' ' -f6|cut -d/ -f1)
+    sed -i "s/kubeletExtraArgs:/kubeletExtraArgs:\n    node-ip: $IPADDR/g" /run/kubeadm/kubeadm.yaml

templates/flavors/base/kustomization.yaml

@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - cluster.yaml
+  - linodeVPC.yaml
   - linodeCluster.yaml
   - linodeMachineTemplate.yaml
   - machineDeployment.yaml

templates/flavors/base/linodeCluster.yaml

@@ -7,3 +7,7 @@ spec:
   region: ${LINODE_REGION}
   credentialsRef:
     name: ${CLUSTER_NAME}-credentials
+  vpcRef:
+    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+    kind: LinodeVPC
+    name: ${VPC_NAME:=${CLUSTER_NAME}}

templates/flavors/base/linodeMachineTemplate.yaml

@@ -9,6 +9,9 @@ spec:
       image: ${LINODE_OS:="linode/ubuntu22.04"}
       type: ${LINODE_CONTROL_PLANE_MACHINE_TYPE}
       region: ${LINODE_REGION}
+      interfaces:
+      - purpose: public
+        primary: true
       authorizedKeys:
       # uncomment to include your ssh key in linode provisioning
       # - ${LINODE_SSH_PUBKEY:=""}
@@ -23,6 +26,9 @@ spec:
       image: ${LINODE_OS:="linode/ubuntu22.04"}
       type: ${LINODE_MACHINE_TYPE}
       region: ${LINODE_REGION}
+      interfaces:
+      - purpose: public
+        primary: true
       authorizedKeys:
       # uncomment to include your ssh key in linode provisioning
       # - ${LINODE_SSH_PUBKEY:=""}