Create external managed kubernetes cluster on CloudStack

Author: vishesh92

api/v1beta3/cloudstackcluster_types.go

@@ -43,6 +43,10 @@ type CloudStackClusterStatus struct {
 	// +optional
 	FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"`
 
+	// Id of CAPC managed kubernetes cluster created in CloudStack
+	// +optional
+	CloudStackClusterID string `json:"cloudStackClusterId"`
+
 	// Reflects the readiness of the CS cluster.
 	Ready bool `json:"ready"`
 }

config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml

@@ -422,6 +422,9 @@ spec:
           status:
             description: The actual cluster state reported by CloudStack.
             properties:
+              cloudStackClusterId:
+                description: Id of CAPC managed kubernetes cluster created in CloudStack
+                type: string
               failureDomains:
                 additionalProperties:
                   description: FailureDomainSpec is the Schema for Cluster API failure

controllers/cloudstackcluster_controller.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
@@ -94,9 +95,28 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re
 		r.GetFailureDomains(r.FailureDomains),
 		r.RemoveExtraneousFailureDomains(r.FailureDomains),
 		r.VerifyFailureDomainCRDs,
+		r.GetOrCreateUnmanagedCluster,
 		r.SetReady)
 }
 
+// GetOrCreateUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack else creates one.
+func (r *CloudStackClusterReconciliationRunner) GetOrCreateUnmanagedCluster() (ctrl.Result, error) {
+	res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])()
+	if r.ShouldReturn(res, err) {
+		return res, err
+	}
+	err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec)
+	if err != nil {
+		if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") {
+			r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation")
+			return ctrl.Result{}, nil
+		}
+		// Not requeueing the failure to support CloudStack v4.18 and before
+		r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error()))
+	}
+	return ctrl.Result{}, nil
+}
+
 // SetReady adds a finalizer and sets the cluster status to ready.
 func (r *CloudStackClusterReconciliationRunner) SetReady() (ctrl.Result, error) {
 	controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
@@ -151,10 +171,30 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result,
 		}
 		return r.RequeueWithMessage("Child FailureDomains still present, requeueing.")
 	}
+	if res, err := r.DeleteUnmanagedCluster(); r.ShouldReturn(res, err) {
+		return res, err
+	}
 	controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer)
 	return ctrl.Result{}, nil
 }
 
+// DeleteUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it.
+func (r *CloudStackClusterReconciliationRunner) DeleteUnmanagedCluster() (ctrl.Result, error) {
+	// If field is present and delete fails, then requeue
+	res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])()
+	if r.ShouldReturn(res, err) {
+		return res, err
+	}
+	err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject)
+	if err != nil {
+		if strings.Contains(err.Error(), " not found") {
+			return ctrl.Result{}, nil
+		}
+		return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error()))
+	}
+	return ctrl.Result{}, nil
+}
+
 // Called in main, this registers the cluster reconciler to the CAPI controller manager.
 func (reconciler *CloudStackClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opts controller.Options) error {
 	log := ctrl.LoggerFrom(ctx)

controllers/cloudstackfailuredomain_controller.go

@@ -18,6 +18,8 @@ package controllers
 
 import (
 	"context"
+	"sort"
+
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -26,7 +28,6 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sort"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
 	csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils"

controllers/cloudstackmachine_controller.go

@@ -128,6 +128,7 @@ func (r *CloudStackMachineReconciliationRunner) Reconcile() (retRes ctrl.Result,
 		r.RequeueIfInstanceNotRunning,
 		r.AddToLBIfNeeded,
 		r.GetOrCreateMachineStateChecker,
+		r.AttachVM,
 	)
 }
 
@@ -214,6 +215,20 @@ func (r *CloudStackMachineReconciliationRunner) DeleteMachineIfFailuredomainNotE
 	return ctrl.Result{}, nil
 }
 
+// AttachVM adds the VM to CloudStack Unmanaged kubernetes.
+// No action taken if it fails
+func (r *CloudStackMachineReconciliationRunner) AttachVM() (retRes ctrl.Result, reterr error) {
+	_ = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject)
+	return ctrl.Result{}, nil
+}
+
+// RemoveVM removes the VM from CloudStack Unmanaged kubernetes.
+// No action taken if it fails
+func (r *CloudStackMachineReconciliationRunner) RemoveVM() (retRes ctrl.Result, reterr error) {
+	_ = r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject)
+	return ctrl.Result{}, nil
+}
+
 // GetOrCreateVMInstance gets or creates a VM instance.
 // Implicitly it also fetches its bootstrap secret in order to create said instance.
 func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes ctrl.Result, reterr error) {
@@ -341,6 +356,10 @@ func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.R
 		return ctrl.Result{}, err
 	}
 
+	res, err := r.RemoveVM()
+	if r.ShouldReturn(res, err) {
+		return res, err
+	}
 	controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer)
 	r.Log.Info("VM Deleted", "instanceID", r.ReconciliationSubject.Spec.InstanceID)
 	return ctrl.Result{}, nil

controllers/cloudstackmachine_controller_test.go

@@ -54,6 +54,17 @@ var _ = Describe("CloudStackMachineReconciler", func() {
 			// Setup a failure domain for the machine reconciler to find.
 			Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed())
 			setClusterReady(k8sClient)
+
+			mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(
+				func(arg1, _, _ interface{}) {
+					arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123"
+				}).AnyTimes().Return(nil)
+
+			mockCloudClient.EXPECT().AddVMToUnmanagedCluster(
+				gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
+
+			mockCloudClient.EXPECT().RemoveVMFromUnmanagedCluster(
+				gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 		})
 
 		It("Should call GetOrCreateVMInstance and set Status.Ready to true", func() {
@@ -240,6 +251,8 @@ var _ = Describe("CloudStackMachineReconciler", func() {
 				func(arg1, _, _, _, _, _ interface{}) {
 					arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running"
 				}).AnyTimes()
+			mockCloudClient.EXPECT().AddVMToUnmanagedCluster(
+				gomock.Any(), gomock.Any()).AnyTimes().Return(nil)
 			Ω(fakeCtrlClient.Get(ctx, key, dummies.CSCluster)).Should(Succeed())
 			Ω(fakeCtrlClient.Create(ctx, dummies.CAPIMachine)).Should(Succeed())
 			Ω(fakeCtrlClient.Create(ctx, dummies.CSMachine1)).Should(Succeed())

controllers/controllers_suite_test.go

@@ -21,16 +21,17 @@ import (
 	"flag"
 	"fmt"
 	"go/build"
-	"k8s.io/client-go/tools/record"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
-	"sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes"
 	"strings"
 	"testing"
 	"time"
 
+	"k8s.io/client-go/tools/record"
+	"sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes"
+
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
 	"k8s.io/klog/v2/klogr"

go.mod

@@ -3,7 +3,7 @@ module sigs.k8s.io/cluster-api-provider-cloudstack
 go 1.19
 
 require (
-	github.com/apache/cloudstack-go/v2 v2.15.0
+	github.com/apache/cloudstack-go/v2 v2.16.0-rc.2
 	github.com/go-logr/logr v1.2.4
 	github.com/golang/mock v1.6.0
 	github.com/hashicorp/go-multierror v1.1.1
@@ -91,3 +91,5 @@ require (
 )
 
 replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 // Indirect upgrade to address https://github.com/advisories/GHSA-w73w-5m7g-f7qc
+
+replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510

go.sum

@@ -22,8 +22,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves=
-github.com/apache/cloudstack-go/v2 v2.15.0 h1:oojn1qx0+wBwrFSSmA2rL8XjWd4BXqwYo0RVCrAXoHk=
-github.com/apache/cloudstack-go/v2 v2.15.0/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -304,6 +302,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 h1:FPRBv784robz6sZSqDGfZDZMse31lj96i+enH02Xzds=
+github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

pkg/cloud/client.go

@@ -37,6 +37,7 @@ import (
 //go:generate ../../hack/tools/bin/mockgen -destination=../mocks/mock_client.go -package=mocks sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud Client
 
 type Client interface {
+	ClusterIface
 	VMIface
 	NetworkIface
 	AffinityGroupIface

pkg/cloud/cluster.go

@@ -0,0 +1,139 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cloud
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/apache/cloudstack-go/v2/cloudstack"
+	"github.com/pkg/errors"
+	infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+)
+
+type ClusterIface interface {
+	GetOrCreateUnmanagedCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error
+	DeleteUnmanagedCluster(*infrav1.CloudStackCluster) error
+	AddVMToUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error
+	RemoveVMFromUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error
+}
+
+type ClustertypeSetter interface {
+	SetClustertype(string)
+}
+
+func withExternalManaged() cloudstack.OptionFunc {
+	return func(cs *cloudstack.CloudStackClient, p interface{}) error {
+		ps, ok := p.(ClustertypeSetter)
+		if !ok {
+			return errors.New("invalid params type")
+		}
+		ps.SetClustertype("ExternalManaged")
+		return nil
+	}
+}
+
+func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error {
+	// Get cluster
+	if csCluster.Status.CloudStackClusterID != "" {
+		externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged())
+		if err != nil {
+			return err
+		} else if count > 0 {
+			csCluster.Status.CloudStackClusterID = externalManagedCluster.Id
+			return nil
+		}
+	}
+
+	// Check if a cluster exists with the same name
+	clusterName := fmt.Sprintf("%s - %s - %s", cluster.GetName(), csCluster.GetName(), csCluster.GetUID())
+	externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged())
+	if err != nil && !strings.Contains(err.Error(), "No match found for ") {
+		return err
+	}
+	if count > 0 {
+		csCluster.Status.CloudStackClusterID = externalManagedCluster.Id
+	} else if err == nil || (err != nil && strings.Contains(err.Error(), "No match found for ")) {
+		// Create cluster
+		domain := Domain{Path: rootDomain}
+		if csCluster.Spec.FailureDomains[0].Domain != "" {
+			domain.Path = fd.Domain
+		}
+		_ = c.ResolveDomain(&domain)
+
+		accountName := csCluster.Spec.FailureDomains[0].Account
+		if accountName == "" {
+			userParams := c.cs.User.NewGetUserParams(c.config.APIKey)
+			user, err := c.cs.User.GetUser(userParams)
+			if err != nil {
+				return err
+			}
+			accountName = user.Account
+		}
+		params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), clusterName, fd.Zone.ID)
+
+		setIfNotEmpty(accountName, params.SetAccount)
+		setIfNotEmpty(domain.ID, params.SetDomainid)
+		setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid)
+		setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress)
+		params.SetClustertype("ExternalManaged")
+
+		cloudStackCKSCluster, err := c.cs.Kubernetes.CreateKubernetesCluster(params)
+		if err != nil {
+			return err
+		}
+		csCluster.Status.CloudStackClusterID = cloudStackCKSCluster.Id
+	}
+	return nil
+}
+
+func (c *client) DeleteUnmanagedCluster(csCluster *infrav1.CloudStackCluster) error {
+	if csCluster.Status.CloudStackClusterID != "" {
+		csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged())
+		if err != nil && strings.Contains(err.Error(), " not found") {
+			return nil
+		}
+		if count != 0 {
+			params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csUnmanagedCluster.Id)
+			_, err = c.cs.Kubernetes.DeleteKubernetesCluster(params)
+			if err != nil {
+				return err
+			}
+		}
+		csCluster.Status.CloudStackClusterID = ""
+	}
+	return nil
+}
+
+func (c *client) AddVMToUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error {
+	if csCluster.Status.CloudStackClusterID != "" {
+		params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID})
+		_, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params)
+		return err
+	}
+	return nil
+}
+
+func (c *client) RemoveVMFromUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error {
+	if csCluster.Status.CloudStackClusterID != "" {
+		params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID})
+		_, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params)
+		return err
+	}
+	return nil
+}

test/e2e/common.go

@@ -251,6 +251,17 @@ func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath
 	return result, nil
 }
 
+func GetACSVersion(client *cloudstack.CloudStackClient) (string, error) {
+	msServersResp, err := client.InfrastructureUsage.ListManagementServersMetrics(client.InfrastructureUsage.NewListManagementServersMetricsParams())
+	if err != nil {
+		return "", err
+	}
+	if msServersResp.Count == 0 {
+		return "", errors.New("no management servers found")
+	}
+	return msServersResp.ManagementServersMetrics[0].Version, nil
+}
+
 func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string, machineType string) {
 	matcher := clusterName + "-" + machineType