diff --git a/Makefile b/Makefile index f8bfd389..bdd01637 100644 --- a/Makefile +++ b/Makefile @@ -293,7 +293,7 @@ E2E_TIMEOUT ?= 3h run-e2e: e2e-essentials ## Run e2e testing. JOB is an optional REGEXP to select certainn test cases to run. e.g. JOB=PR-Blocking, JOB=Conformance $(KUBECTL) apply -f cloud-config.yaml && \ cd test/e2e && \ - $(GINKGO) -v --trace --tags=e2e --focus=$(JOB) --timeout=$(E2E_TIMEOUT) --skip=Conformance --skip-package=kubeconfig_helper --nodes=1 --no-color=false ./... -- \ + $(GINKGO) -vv --trace --tags=e2e --focus=$(JOB) --timeout=$(E2E_TIMEOUT) --skip=Conformance --skip-package=kubeconfig_helper --nodes=1 --no-color=false ./... -- \ -e2e.artifacts-folder=${REPO_ROOT}/_artifacts \ -e2e.config=${E2E_CONFIG} \ -e2e.skip-resource-cleanup=false -e2e.use-existing-cluster=true diff --git a/api/v1beta1/conversion.go b/api/v1beta1/conversion.go index 236685f6..3436aa59 100644 --- a/api/v1beta1/conversion.go +++ b/api/v1beta1/conversion.go @@ -51,7 +51,7 @@ func Convert_v1beta1_CloudStackCluster_To_v1beta3_CloudStackCluster(in *CloudSta //nolint:golint,revive,stylecheck func Convert_v1beta3_CloudStackCluster_To_v1beta1_CloudStackCluster(in *v1beta3.CloudStackCluster, out *CloudStackCluster, scope conv.Scope) error { if len(in.Spec.FailureDomains) < 1 { - return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", in.Spec) + return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", fmt.Sprintf("%v", in.Spec)) } out.ObjectMeta = in.ObjectMeta out.Spec = CloudStackClusterSpec{ diff --git a/api/v1beta2/cloudstackcluster_conversion.go b/api/v1beta2/cloudstackcluster_conversion.go index 31fd3c20..dbd8a21a 100644 --- a/api/v1beta2/cloudstackcluster_conversion.go +++ b/api/v1beta2/cloudstackcluster_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + machineryconversion "k8s.io/apimachinery/pkg/conversion" "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" "sigs.k8s.io/controller-runtime/pkg/conversion" ) @@ -30,3 +31,11 @@ func (dst *CloudStackCluster) ConvertFrom(srcRaw conversion.Hub) error { // noli src := srcRaw.(*v1beta3.CloudStackCluster) return Convert_v1beta3_CloudStackCluster_To_v1beta2_CloudStackCluster(src, dst, nil) } + +func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s machineryconversion.Scope) error { // nolint + return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s) +} + +func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s machineryconversion.Scope) error { // nolint + return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s) +} diff --git a/api/v1beta2/zz_generated.conversion.go b/api/v1beta2/zz_generated.conversion.go index 026a8bd5..9573ab25 100644 --- a/api/v1beta2/zz_generated.conversion.go +++ b/api/v1beta2/zz_generated.conversion.go @@ -103,21 +103,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta3.CloudStackClusterSpec)(nil), (*CloudStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(a.(*v1beta3.CloudStackClusterSpec), b.(*CloudStackClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*CloudStackClusterStatus)(nil), (*v1beta3.CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus(a.(*CloudStackClusterStatus), b.(*v1beta3.CloudStackClusterStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta3.CloudStackClusterStatus)(nil), (*CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(a.(*v1beta3.CloudStackClusterStatus), b.(*CloudStackClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*CloudStackFailureDomain)(nil), (*v1beta3.CloudStackFailureDomain)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_CloudStackFailureDomain_To_v1beta3_CloudStackFailureDomain(a.(*CloudStackFailureDomain), b.(*v1beta3.CloudStackFailureDomain), scope) }); err != nil { @@ -358,6 +348,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta3.CloudStackClusterSpec)(nil), (*CloudStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(a.(*v1beta3.CloudStackClusterSpec), b.(*CloudStackClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta3.CloudStackClusterStatus)(nil), (*CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(a.(*v1beta3.CloudStackClusterStatus), b.(*CloudStackClusterStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta3.CloudStackFailureDomainSpec)(nil), (*CloudStackFailureDomainSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_CloudStackFailureDomainSpec_To_v1beta2_CloudStackFailureDomainSpec(a.(*v1beta3.CloudStackFailureDomainSpec), b.(*CloudStackFailureDomainSpec), scope) }); err != nil { @@ -579,14 +579,10 @@ func autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec( out.FailureDomains = nil } out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + // WARNING: in.SyncWithACS requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec is an autogenerated conversion function. -func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s) -} - func autoConvert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus(in *CloudStackClusterStatus, out *v1beta3.CloudStackClusterStatus, s conversion.Scope) error { out.FailureDomains = *(*v1beta1.FailureDomains)(unsafe.Pointer(&in.FailureDomains)) out.Ready = in.Ready @@ -600,15 +596,11 @@ func Convert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus( func autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s conversion.Scope) error { out.FailureDomains = *(*v1beta1.FailureDomains)(unsafe.Pointer(&in.FailureDomains)) + // WARNING: in.CloudStackClusterID requires manual conversion: does not exist in peer-type out.Ready = in.Ready return nil } -// Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus is an autogenerated conversion function. -func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s) -} - func autoConvert_v1beta2_CloudStackFailureDomain_To_v1beta3_CloudStackFailureDomain(in *CloudStackFailureDomain, out *v1beta3.CloudStackFailureDomain, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1beta2_CloudStackFailureDomainSpec_To_v1beta3_CloudStackFailureDomainSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/api/v1beta3/cloudstackcluster_types.go b/api/v1beta3/cloudstackcluster_types.go index 1b47ff89..08b68d58 100644 --- a/api/v1beta3/cloudstackcluster_types.go +++ b/api/v1beta3/cloudstackcluster_types.go @@ -34,6 +34,10 @@ type CloudStackClusterSpec struct { // The kubernetes control plane endpoint. ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` + + // SyncWithACS determines if an externalManaged CKS cluster should be created on ACS. + // +optional + SyncWithACS bool `json:"syncWithACS,omitempty"` } // The status of the CloudStackCluster object. @@ -43,6 +47,10 @@ type CloudStackClusterStatus struct { // +optional FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` + // Id of CAPC managed kubernetes cluster created in CloudStack + // +optional + CloudStackClusterID string `json:"cloudStackClusterId"` + // Reflects the readiness of the CS cluster. Ready bool `json:"ready"` } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml index 882a42eb..81bbca78 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml @@ -415,6 +415,10 @@ spec: - zone type: object type: array + syncWithACS: + description: SyncWithACS determines if an externalManaged CKS cluster + should be created on ACS. + type: boolean required: - controlPlaneEndpoint - failureDomains @@ -422,6 +426,9 @@ spec: status: description: The actual cluster state reported by CloudStack. properties: + cloudStackClusterId: + description: Id of CAPC managed kubernetes cluster created in CloudStack + type: string failureDomains: additionalProperties: description: FailureDomainSpec is the Schema for Cluster API failure diff --git a/controllers/cks_cluster_controller.go b/controllers/cks_cluster_controller.go new file mode 100644 index 00000000..030555b7 --- /dev/null +++ b/controllers/cks_cluster_controller.go @@ -0,0 +1,124 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "strings" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" +) + +const CksClusterFinalizer = "ckscluster.infrastructure.cluster.x-k8s.io" + +// RBAC permissions for CloudStackCluster. +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/status,verbs=create;get;update;patch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/finalizers,verbs=update + +// CksClusterReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters. +// The runner does the actual reconciliation. +type CksClusterReconciliationRunner struct { + *csCtrlrUtils.ReconciliationRunner + FailureDomains *infrav1.CloudStackFailureDomainList + ReconciliationSubject *infrav1.CloudStackCluster +} + +// CksClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster. +// This is primarily to adapt to k8s. +type CksClusterReconciler struct { + csCtrlrUtils.ReconcilerBase +} + +// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields. +func NewCksClusterReconciliationRunner() *CksClusterReconciliationRunner { + // Set concrete type and init pointers. + runner := &CksClusterReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackCluster{}} + runner.FailureDomains = &infrav1.CloudStackFailureDomainList{} + // Setup the base runner. Initializes pointers and links reconciliation methods. + runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSClusterController") + runner.CSCluster = runner.ReconciliationSubject + return runner +} + +// Reconcile is the method k8s will call upon a reconciliation request. +func (reconciler *CksClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { + r := NewCksClusterReconciliationRunner() + r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx) + r.WithAdditionalCommonStages(r.GetFailureDomains(r.FailureDomains)) + return r.RunBaseReconciliationStages() +} + +// Reconcile actually reconciles the CloudStackCluster. +func (r *CksClusterReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) { + if r.CSCluster.Spec.SyncWithACS { + // Prevent premature deletion. + controllerutil.AddFinalizer(r.ReconciliationSubject, CksClusterFinalizer) + + if len(r.FailureDomains.Items) == 0 { + return r.RequeueWithMessage("No failure domains found") + } + + res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + + r.Log.Info("Creating entry with CKS") + err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + if err != nil { + if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { + r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") + return ctrl.Result{}, nil + } + r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) + return r.RequeueWithMessage(fmt.Sprintf("Syncing VMs with CloudStack failed. error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + +// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers. +func (r *CksClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, error) { + if r.ReconciliationSubject.Status.CloudStackClusterID != "" { + if len(r.FailureDomains.Items) == 0 { + return ctrl.Result{}, fmt.Errorf("no failure domains found") + } + res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject) + if err != nil && !strings.Contains(err.Error(), " not found") { + return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error())) + } + } + controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksClusterFinalizer) + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (reconciler *CksClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.CloudStackCluster{}). + Complete(reconciler) +} diff --git a/controllers/cks_cluster_controller_test.go b/controllers/cks_cluster_controller_test.go new file mode 100644 index 00000000..5f3c81da --- /dev/null +++ b/controllers/cks_cluster_controller_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + "sigs.k8s.io/cluster-api-provider-cloudstack/controllers" + "sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud" + dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +var _ = Describe("CksCloudStackClusterReconciler", func() { + Context("With k8s like test environment.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + SetupTestEnvironment() + Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler. + Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler. + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).MinTimes(1).Return(nil) + mockCloudClient.EXPECT().ResolveZone(gomock.Any()).AnyTimes() + mockCloudClient.EXPECT().ResolveNetworkForZone(gomock.Any()).AnyTimes().Do( + func(arg1 interface{}) { + arg1.(*infrav1.CloudStackZoneSpec).Network.ID = "SomeID" + arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared + }).MinTimes(1) + }) + + It("Should create a cluster in CKS.", func() { + Eventually(func() string { + key := client.ObjectKeyFromObject(dummies.CSCluster) + if err := k8sClient.Get(ctx, key, dummies.CSCluster); err != nil { + return "" + } + return dummies.CSCluster.Status.CloudStackClusterID + }, timeout).WithPolling(pollInterval).Should(Equal("cluster-id-123")) + + }) + + }) + + Context("Without a k8s test environment.", func() { + It("Should create a reconciliation runner with a Cloudstack Cluster as the reconciliation subject.", func() { + reconRunner := controllers.NewCksClusterReconciliationRunner() + Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/cks_machine_controller.go b/controllers/cks_machine_controller.go new file mode 100644 index 00000000..80937e82 --- /dev/null +++ b/controllers/cks_machine_controller.go @@ -0,0 +1,108 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + ctrl "sigs.k8s.io/controller-runtime" + + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" +) + +// RBAC permissions for CloudStackCluster. +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/finalizers,verbs=update + +// CksMachineReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters. +// The runner does the actual reconciliation. +type CksMachineReconciliationRunner struct { + *csCtrlrUtils.ReconciliationRunner + FailureDomain *infrav1.CloudStackFailureDomain + ReconciliationSubject *infrav1.CloudStackMachine +} + +// CksMachineReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster. +// This is primarily to adapt to k8s. +type CksMachineReconciler struct { + csCtrlrUtils.ReconcilerBase +} + +// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields. +func NewCksMachineReconciliationRunner() *CksMachineReconciliationRunner { + // Set concrete type and init pointers. + runner := &CksMachineReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackMachine{}} + runner.FailureDomain = &infrav1.CloudStackFailureDomain{} + // Setup the base runner. Initializes pointers and links reconciliation methods. + runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSMachineController") + return runner +} + +// Reconcile is the method k8s will call upon a reconciliation request. +func (reconciler *CksMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { + r := NewCksMachineReconciliationRunner() + r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx) + r.WithAdditionalCommonStages( + r.GetFailureDomainByName(func() string { return r.ReconciliationSubject.Spec.FailureDomainName }, r.FailureDomain), + r.AsFailureDomainUser(&r.FailureDomain.Spec)) + return r.RunBaseReconciliationStages() +} + +// Reconcile actually reconciles the CloudStackCluster. +func (r *CksMachineReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) { + if r.CSCluster.Spec.SyncWithACS { + if r.CSCluster.Status.CloudStackClusterID == "" { + return r.RequeueWithMessage("CloudStackClusterID is not set") + } + + if r.ReconciliationSubject.Spec.InstanceID == nil || *r.ReconciliationSubject.Spec.InstanceID == "" { + return r.RequeueWithMessage("InstanceID is not set") + } + + res, err := r.AsFailureDomainUser(&r.FailureDomain.Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + r.Log.Info("Assigning VM to CKS") + err = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Adding VM to CloudStack Unmanaged kubernetes failed. error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + +// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers. +func (r *CksMachineReconciliationRunner) ReconcileDelete() (ctrl.Result, error) { + if r.ReconciliationSubject.Spec.InstanceID != nil && *r.ReconciliationSubject.Spec.InstanceID != "" { + err := r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Removing VM from CloudStack Unmanaged kubernetes failed. error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (reconciler *CksMachineReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.CloudStackMachine{}). + Complete(reconciler) +} diff --git a/controllers/cks_machine_controller_test.go b/controllers/cks_machine_controller_test.go new file mode 100644 index 00000000..6bb0b63b --- /dev/null +++ b/controllers/cks_machine_controller_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + "sigs.k8s.io/cluster-api-provider-cloudstack/controllers" + dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +var _ = Describe("CloudStackMachineReconciler", func() { + Context("With machine controller running.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + dummies.CSCluster.Spec.SyncWithACS = true + dummies.CSCluster.Spec.FailureDomains = dummies.CSCluster.Spec.FailureDomains[:1] + dummies.CSCluster.Spec.FailureDomains[0].Name = dummies.CSFailureDomain1.Spec.Name + + SetupTestEnvironment() // Must happen before setting up managers/reconcilers. + Ω(MachineReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register the CloudStack MachineReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register the CloudStack MachineReconciler. + Ω(CksMachineReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register the CloudStack MachineReconciler. + + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).MinTimes(1).Return(nil) + // Point CAPI machine Bootstrap secret ref to dummy bootstrap secret. + dummies.CAPIMachine.Spec.Bootstrap.DataSecretName = &dummies.BootstrapSecret.Name + Ω(k8sClient.Create(ctx, dummies.BootstrapSecret)).Should(Succeed()) + + // Setup a failure domain for the machine reconciler to find. + Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) + setClusterReady(k8sClient) + }) + + It("Should call AddVMToUnmanagedCluster", func() { + // Mock a call to GetOrCreateVMInstance and set the machine to running. + mockCloudClient.EXPECT().GetOrCreateVMInstance( + gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _, _, _, _ interface{}) { + arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" + }).AnyTimes() + + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + // Have to do this here or the reconcile call to GetOrCreateVMInstance may happen too early. + setupMachineCRDs() + + // Eventually the machine should set ready to true. + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err == nil { + if tempMachine.Status.Ready == true { + return len(tempMachine.ObjectMeta.Finalizers) > 0 + } + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + }) + + It("Should call RemoveVMFromUnmanagedCluster when CS machine deleted", func() { + // Mock a call to GetOrCreateVMInstance and set the machine to running. + mockCloudClient.EXPECT().GetOrCreateVMInstance( + gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _, _, _, _ interface{}) { + arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" + controllerutil.AddFinalizer(arg1.(*infrav1.CloudStackMachine), infrav1.MachineFinalizer) + }).AnyTimes() + + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + + mockCloudClient.EXPECT().DestroyVMInstance(gomock.Any()).Times(1).Return(nil) + mockCloudClient.EXPECT().RemoveVMFromUnmanagedCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + // Have to do this here or the reconcile call to GetOrCreateVMInstance may happen too early. + setupMachineCRDs() + + // Eventually the machine should set ready to true. + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err == nil { + if tempMachine.Status.Ready == true { + return true + } + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + + Ω(k8sClient.Delete(ctx, dummies.CSMachine1)).Should(Succeed()) + + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err != nil { + return errors.IsNotFound(err) + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + + }) + }) + + Context("Without a k8s test environment.", func() { + It("Should create a reconciliation runner with a Cloudstack Machine as the reconciliation subject.", func() { + reconRunner := controllers.NewCksMachineReconciliationRunner() + Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/cloudstackfailuredomain_controller.go b/controllers/cloudstackfailuredomain_controller.go index 822d4ef7..3084cf6e 100644 --- a/controllers/cloudstackfailuredomain_controller.go +++ b/controllers/cloudstackfailuredomain_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + "sort" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -26,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sort" infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" diff --git a/controllers/cloudstackmachine_controller_test.go b/controllers/cloudstackmachine_controller_test.go index e4195726..dda1acea 100644 --- a/controllers/cloudstackmachine_controller_test.go +++ b/controllers/cloudstackmachine_controller_test.go @@ -54,6 +54,11 @@ var _ = Describe("CloudStackMachineReconciler", func() { // Setup a failure domain for the machine reconciler to find. Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) setClusterReady(k8sClient) + + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).AnyTimes().Return(nil) }) It("Should call GetOrCreateVMInstance and set Status.Ready to true", func() { diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index a9776f64..c1b2d1df 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -21,16 +21,17 @@ import ( "flag" "fmt" "go/build" - "k8s.io/client-go/tools/record" "os" "os/exec" "path/filepath" "regexp" - "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" "strings" "testing" "time" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" @@ -129,6 +130,10 @@ var ( FailureDomainReconciler *csReconcilers.CloudStackFailureDomainReconciler IsoNetReconciler *csReconcilers.CloudStackIsoNetReconciler AffinityGReconciler *csReconcilers.CloudStackAffinityGroupReconciler + + // CKS Reconcilers + CksClusterReconciler *csReconcilers.CksClusterReconciler + CksMachineReconciler *csReconcilers.CksMachineReconciler ) var _ = BeforeSuite(func() { @@ -224,6 +229,9 @@ func SetupTestEnvironment() { IsoNetReconciler = &csReconcilers.CloudStackIsoNetReconciler{ReconcilerBase: base} AffinityGReconciler = &csReconcilers.CloudStackAffinityGroupReconciler{ReconcilerBase: base} + CksClusterReconciler = &csReconcilers.CksClusterReconciler{ReconcilerBase: base} + CksMachineReconciler = &csReconcilers.CksMachineReconciler{ReconcilerBase: base} + ctx, cancel = context.WithCancel(context.TODO()) // Setup mock clients. @@ -237,6 +245,9 @@ func SetupTestEnvironment() { AffinityGReconciler.CSClient = mockCloudClient FailureDomainReconciler.CSClient = mockCloudClient + CksClusterReconciler.CSClient = mockCloudClient + CksMachineReconciler.CSClient = mockCloudClient + setupClusterCRDs() // See reconciliation results. Left commented as it's noisy otherwise. diff --git a/go.mod b/go.mod index e099ebf9..0c7e4099 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module sigs.k8s.io/cluster-api-provider-cloudstack go 1.19 require ( - github.com/apache/cloudstack-go/v2 v2.15.0 + github.com/apache/cloudstack-go/v2 v2.16.1 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -70,11 +70,11 @@ require ( github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/spf13/cobra v1.6.1 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 2dea8353..b04ed6ac 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= -github.com/apache/cloudstack-go/v2 v2.15.0 h1:oojn1qx0+wBwrFSSmA2rL8XjWd4BXqwYo0RVCrAXoHk= -github.com/apache/cloudstack-go/v2 v2.15.0/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.1 h1:2wOE4RKEjWPRZNO7ZNnZYmR3JJ+JJPQwhoc7W1fkiK4= +github.com/apache/cloudstack-go/v2 v2.16.1/go.mod h1:cZsgFe+VmrgLBm7QjeHTJBXYe8E5+yGYkdfwGb+Pu9c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -367,7 +367,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -409,8 +409,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -448,11 +448,11 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/main.go b/main.go index d7046dd2..3e0248d2 100644 --- a/main.go +++ b/main.go @@ -267,4 +267,12 @@ func setupReconcilers(ctx context.Context, base utils.ReconcilerBase, opts manag setupLog.Error(err, "unable to create controller", "controller", "CloudStackFailureDomain") os.Exit(1) } + if err := (&controllers.CksClusterReconciler{ReconcilerBase: base}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CKSClusterController") + os.Exit(1) + } + if err := (&controllers.CksMachineReconciler{ReconcilerBase: base}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CKSMachineController") + os.Exit(1) + } } diff --git a/pkg/cloud/client.go b/pkg/cloud/client.go index 12e61176..76c50256 100644 --- a/pkg/cloud/client.go +++ b/pkg/cloud/client.go @@ -37,6 +37,7 @@ import ( //go:generate ../../hack/tools/bin/mockgen -destination=../mocks/mock_client.go -package=mocks sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud Client type Client interface { + ClusterIface VMIface NetworkIface AffinityGroupIface diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go new file mode 100644 index 00000000..f1831063 --- /dev/null +++ b/pkg/cloud/cluster.go @@ -0,0 +1,137 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "fmt" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/pkg/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +type ClusterIface interface { + GetOrCreateUnmanagedCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error + DeleteUnmanagedCluster(*infrav1.CloudStackCluster) error + AddVMToUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error + RemoveVMFromUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error +} + +type ClustertypeSetter interface { + SetClustertype(string) +} + +func withExternalManaged() cloudstack.OptionFunc { + return func(cs *cloudstack.CloudStackClient, p interface{}) error { + ps, ok := p.(ClustertypeSetter) + if !ok { + return errors.New("invalid params type") + } + ps.SetClustertype("ExternalManaged") + return nil + } +} + +func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { + // Get cluster + if csCluster.Status.CloudStackClusterID != "" { + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) + if err != nil { + return err + } else if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + return nil + } + } + + // Check if a cluster exists with the same name + clusterName := fmt.Sprintf("%s - %s - %s", cluster.GetName(), csCluster.GetName(), csCluster.GetUID()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) + if err != nil && !strings.Contains(err.Error(), "No match found for ") { + return err + } + if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + } else if err == nil || (err != nil && strings.Contains(err.Error(), "No match found for ")) { + // Create cluster + accountName := csCluster.Spec.FailureDomains[0].Account + if accountName == "" { + userParams := c.cs.User.NewGetUserParams(c.config.APIKey) + user, err := c.cs.User.GetUser(userParams) + if err != nil { + return err + } + accountName = user.Account + } + // NewCreateKubernetesClusterParams(description string, kubernetesversionid string, name string, serviceofferingid string, size int64, zoneid string) *CreateKubernetesClusterParams + params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), "", clusterName, "", 0, fd.Zone.ID) + + setIfNotEmpty(c.user.Project.ID, params.SetProjectid) + setIfNotEmpty(accountName, params.SetAccount) + setIfNotEmpty(c.user.Domain.ID, params.SetDomainid) + setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid) + setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress) + params.ResetKubernetesversionid() + params.ResetServiceofferingid() + params.SetClustertype("ExternalManaged") + + cloudStackCKSCluster, err := c.cs.Kubernetes.CreateKubernetesCluster(params) + if err != nil { + return err + } + csCluster.Status.CloudStackClusterID = cloudStackCKSCluster.Id + } + return nil +} + +func (c *client) DeleteUnmanagedCluster(csCluster *infrav1.CloudStackCluster) error { + if csCluster.Status.CloudStackClusterID != "" { + csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + if err != nil && strings.Contains(err.Error(), " not found") { + return nil + } + if count != 0 { + params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csUnmanagedCluster.Id) + _, err = c.cs.Kubernetes.DeleteKubernetesCluster(params) + if err != nil { + return err + } + } + csCluster.Status.CloudStackClusterID = "" + } + return nil +} + +func (c *client) AddVMToUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params) + return err + } + return nil +} + +func (c *client) RemoveVMFromUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params) + return err + } + return nil +} diff --git a/test/dummies/v1beta3/vars.go b/test/dummies/v1beta3/vars.go index e633abf5..86df1f07 100644 --- a/test/dummies/v1beta3/vars.go +++ b/test/dummies/v1beta3/vars.go @@ -80,6 +80,7 @@ var ( // Declare exported dummy vars. LBRuleID string PublicIPID string EndPointHost string + SyncWithACS bool EndPointPort int32 CSConf *simpleyaml.Yaml DiskOffering infrav1.CloudStackResourceDiskOffering @@ -273,6 +274,7 @@ func SetDummyCAPCClusterVars() { CSClusterKind = "CloudStackCluster" ClusterName = "test-cluster" EndPointHost = "EndpointHost" + SyncWithACS = true EndPointPort = int32(5309) PublicIPID = "FakePublicIPID" ClusterNameSpace = "default" @@ -333,6 +335,7 @@ func SetDummyCAPCClusterVars() { Spec: infrav1.CloudStackClusterSpec{ ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: EndPointHost, Port: EndPointPort}, FailureDomains: []infrav1.CloudStackFailureDomainSpec{CSFailureDomain1.Spec, CSFailureDomain2.Spec}, + SyncWithACS: SyncWithACS, }, Status: infrav1.CloudStackClusterStatus{}, } diff --git a/test/e2e/common.go b/test/e2e/common.go index 53518904..ea537200 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -251,6 +251,17 @@ func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath return result, nil } +func GetACSVersion(client *cloudstack.CloudStackClient) (string, error) { + msServersResp, err := client.InfrastructureUsage.ListManagementServersMetrics(client.InfrastructureUsage.NewListManagementServersMetricsParams()) + if err != nil { + return "", err + } + if msServersResp.Count == 0 { + return "", errors.New("no management servers found") + } + return msServersResp.ManagementServersMetrics[0].Version, nil +} + func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string, machineType string) { matcher := clusterName + "-" + machineType diff --git a/test/e2e/config/cloudstack.yaml b/test/e2e/config/cloudstack.yaml index ee06f42c..bd3409db 100644 --- a/test/e2e/config/cloudstack.yaml +++ b/test/e2e/config/cloudstack.yaml @@ -98,6 +98,7 @@ providers: - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-ip.yaml" - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-before.yaml" - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-after.yaml" + - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged.yaml" - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" versions: - name: v1.0.0 diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml index bdd9d70b..2b719aa9 100644 --- a/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml @@ -35,6 +35,7 @@ spec: name : ${CLOUDSTACK_ZONE_NAME} network: name: ${CLOUDSTACK_NETWORK_NAME} + syncWithACS: false --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml new file mode 100644 index 00000000..7bb1a966 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 +kind: CloudStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + controlPlaneEndpoint: + host: "" + port: 6443 + failureDomains: + - name: ${CLOUDSTACK_FD1_NAME} + acsEndpoint: + name: ${CLOUDSTACK_FD1_SECRET_NAME} + namespace: default + zone: + name : ${CLOUDSTACK_ZONE_NAME} + network: + name: ${CLOUDSTACK_NETWORK_NAME} + syncWithACS: true diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml new file mode 100644 index 00000000..7a6d2984 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml @@ -0,0 +1,6 @@ +bases: + - ../bases/cluster-with-kcp.yaml + - ../bases/md.yaml + +patchesStrategicMerge: +- ./cloudstack-cluster.yaml diff --git a/test/e2e/go.mod b/test/e2e/go.mod index a94abae4..4ed90ee5 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( github.com/Shopify/toxiproxy/v2 v2.5.0 - github.com/apache/cloudstack-go/v2 v2.15.0 + github.com/apache/cloudstack-go/v2 v2.16.1 github.com/blang/semver v3.5.1+incompatible github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.8 @@ -97,12 +97,12 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/fastjson v1.6.4 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 1a821219..91201722 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -65,8 +65,8 @@ github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPp github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/cloudstack-go/v2 v2.15.0 h1:oojn1qx0+wBwrFSSmA2rL8XjWd4BXqwYo0RVCrAXoHk= -github.com/apache/cloudstack-go/v2 v2.15.0/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.1 h1:2wOE4RKEjWPRZNO7ZNnZYmR3JJ+JJPQwhoc7W1fkiK4= +github.com/apache/cloudstack-go/v2 v2.16.1/go.mod h1:cZsgFe+VmrgLBm7QjeHTJBXYe8E5+yGYkdfwGb+Pu9c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -528,8 +528,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -609,8 +609,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -693,13 +693,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -710,8 +710,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/test/e2e/unmanaged_k8s.go b/test/e2e/unmanaged_k8s.go new file mode 100644 index 00000000..10987d69 --- /dev/null +++ b/test/e2e/unmanaged_k8s.go @@ -0,0 +1,136 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +// UnmanagedK8SSpec implements a spec that creates a cluster and checks whether an entry is created in ACS. +func UnmanagedK8SSpec(ctx context.Context, inputGetter func() CommonSpecInput) { + var ( + specName = "k8s-unmanaged" + input CommonSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + version, err := GetACSVersion(csClient) + + if err != nil || version == "" { + Skip("Failed to get CloudStack's version") + } + + v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], ".")) + + if err != nil { + Skip("Failed to parse CloudStack version " + version) + } + + expectedRange, _ := semver.ParseRange(">=4.19.0") + + if !expectedRange(v) { + Skip("Cloudstack version " + version + " is less than 4.19.") + } + + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) + + It("Should create a workload cluster", func() { + By("Creating a workload cluster") + + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + CNIManifestPath: input.E2EConfig.GetVariable(CNIPath), + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: specName, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, clusterResources) + + By("checking unmanaged k8s resource is created on ACS") + // Get details from ACS + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + lkcp := csClient.Kubernetes.NewListKubernetesClustersParams() + lkcp.SetListall(true) + + clusters, err := csClient.Kubernetes.ListKubernetesClusters(lkcp) + + if err != nil { + Fail("Failed to get Kubernetes clusters from ACS") + } + + var cluster *cloudstack.KubernetesCluster + + for _, d := range clusters.KubernetesClusters { + if strings.HasPrefix(d.Name, fmt.Sprintf("%s - %s", clusterName, clusterName)) { + cluster = d + } + } + + Expect(cluster).ShouldNot(BeNil(), "Couldn't find the external managed kubernetes in ACS") + Expect(len(cluster.Virtualmachines)).Should(Equal(2), "Expected 2 VMs in the cluster, found %d", len(cluster.Virtualmachines)) + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} diff --git a/test/e2e/unmanaged_k8s_test.go b/test/e2e/unmanaged_k8s_test.go new file mode 100644 index 00000000..5951d163 --- /dev/null +++ b/test/e2e/unmanaged_k8s_test.go @@ -0,0 +1,36 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("When testing creation of unmanaged CKS cluster in ACS", func() { + UnmanagedK8SSpec(ctx, func() CommonSpecInput { + return CommonSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) +})