From b7feb189e0241c1110b14fa5cd7596ca71c88eef Mon Sep 17 00:00:00 2001 From: Vishesh Date: Thu, 27 Jul 2023 17:52:08 +0530 Subject: [PATCH 1/4] Create external managed kubernetes cluster on CloudStack --- api/v1beta3/cloudstackcluster_types.go | 4 + ...e.cluster.x-k8s.io_cloudstackclusters.yaml | 3 + controllers/cloudstackcluster_controller.go | 40 +++++ .../cloudstackfailuredomain_controller.go | 3 +- controllers/cloudstackmachine_controller.go | 19 +++ .../cloudstackmachine_controller_test.go | 13 ++ controllers/controllers_suite_test.go | 5 +- go.mod | 4 +- go.sum | 4 +- pkg/cloud/client.go | 1 + pkg/cloud/cluster.go | 139 ++++++++++++++++++ test/e2e/common.go | 11 ++ test/e2e/go.mod | 6 +- test/e2e/go.sum | 27 +++- test/e2e/unmanaged_k8s.go | 136 +++++++++++++++++ test/e2e/unmanaged_k8s_test.go | 36 +++++ 16 files changed, 442 insertions(+), 9 deletions(-) create mode 100644 pkg/cloud/cluster.go create mode 100644 test/e2e/unmanaged_k8s.go create mode 100644 test/e2e/unmanaged_k8s_test.go diff --git a/api/v1beta3/cloudstackcluster_types.go b/api/v1beta3/cloudstackcluster_types.go index 1b47ff89..9ffafbe0 100644 --- a/api/v1beta3/cloudstackcluster_types.go +++ b/api/v1beta3/cloudstackcluster_types.go @@ -43,6 +43,10 @@ type CloudStackClusterStatus struct { // +optional FailureDomains clusterv1.FailureDomains `json:"failureDomains,omitempty"` + // Id of CAPC managed kubernetes cluster created in CloudStack + // +optional + CloudStackClusterID string `json:"cloudStackClusterId"` + // Reflects the readiness of the CS cluster. Ready bool `json:"ready"` } diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml index 882a42eb..2cead21c 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml @@ -422,6 +422,9 @@ spec: status: description: The actual cluster state reported by CloudStack. properties: + cloudStackClusterId: + description: Id of CAPC managed kubernetes cluster created in CloudStack + type: string failureDomains: additionalProperties: description: FailureDomainSpec is the Schema for Cluster API failure diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index a42acfca..fb896c13 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -20,6 +20,7 @@ import ( "context" "fmt" "reflect" + "strings" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -94,9 +95,28 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re r.GetFailureDomains(r.FailureDomains), r.RemoveExtraneousFailureDomains(r.FailureDomains), r.VerifyFailureDomainCRDs, + r.GetOrCreateUnmanagedCluster, r.SetReady) } +// GetOrCreateUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack else creates one. +func (r *CloudStackClusterReconciliationRunner) GetOrCreateUnmanagedCluster() (ctrl.Result, error) { + res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + if err != nil { + if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { + r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") + return ctrl.Result{}, nil + } + // Not requeueing the failure to support CloudStack v4.18 and before + r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) + } + return ctrl.Result{}, nil +} + // SetReady adds a finalizer and sets the cluster status to ready. func (r *CloudStackClusterReconciliationRunner) SetReady() (ctrl.Result, error) { controllerutil.AddFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer) @@ -151,10 +171,30 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, } return r.RequeueWithMessage("Child FailureDomains still present, requeueing.") } + if res, err := r.DeleteUnmanagedCluster(); r.ShouldReturn(res, err) { + return res, err + } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer) return ctrl.Result{}, nil } +// DeleteUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it. +func (r *CloudStackClusterReconciliationRunner) DeleteUnmanagedCluster() (ctrl.Result, error) { + // If field is present and delete fails, then requeue + res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject) + if err != nil { + if strings.Contains(err.Error(), " not found") { + return ctrl.Result{}, nil + } + return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error())) + } + return ctrl.Result{}, nil +} + // Called in main, this registers the cluster reconciler to the CAPI controller manager. func (reconciler *CloudStackClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opts controller.Options) error { log := ctrl.LoggerFrom(ctx) diff --git a/controllers/cloudstackfailuredomain_controller.go b/controllers/cloudstackfailuredomain_controller.go index 822d4ef7..3084cf6e 100644 --- a/controllers/cloudstackfailuredomain_controller.go +++ b/controllers/cloudstackfailuredomain_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + "sort" + "github.com/pkg/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" @@ -26,7 +28,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" - "sort" infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" diff --git a/controllers/cloudstackmachine_controller.go b/controllers/cloudstackmachine_controller.go index 0a3483af..156c463e 100644 --- a/controllers/cloudstackmachine_controller.go +++ b/controllers/cloudstackmachine_controller.go @@ -128,6 +128,7 @@ func (r *CloudStackMachineReconciliationRunner) Reconcile() (retRes ctrl.Result, r.RequeueIfInstanceNotRunning, r.AddToLBIfNeeded, r.GetOrCreateMachineStateChecker, + r.AttachVM, ) } @@ -214,6 +215,20 @@ func (r *CloudStackMachineReconciliationRunner) DeleteMachineIfFailuredomainNotE return ctrl.Result{}, nil } +// AttachVM adds the VM to CloudStack Unmanaged kubernetes. +// No action taken if it fails +func (r *CloudStackMachineReconciliationRunner) AttachVM() (retRes ctrl.Result, reterr error) { + _ = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) + return ctrl.Result{}, nil +} + +// RemoveVM removes the VM from CloudStack Unmanaged kubernetes. +// No action taken if it fails +func (r *CloudStackMachineReconciliationRunner) RemoveVM() (retRes ctrl.Result, reterr error) { + _ = r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) + return ctrl.Result{}, nil +} + // GetOrCreateVMInstance gets or creates a VM instance. // Implicitly it also fetches its bootstrap secret in order to create said instance. func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes ctrl.Result, reterr error) { @@ -341,6 +356,10 @@ func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.R return ctrl.Result{}, err } + res, err := r.RemoveVM() + if r.ShouldReturn(res, err) { + return res, err + } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer) r.Log.Info("VM Deleted", "instanceID", r.ReconciliationSubject.Spec.InstanceID) return ctrl.Result{}, nil diff --git a/controllers/cloudstackmachine_controller_test.go b/controllers/cloudstackmachine_controller_test.go index e4195726..5e4c223c 100644 --- a/controllers/cloudstackmachine_controller_test.go +++ b/controllers/cloudstackmachine_controller_test.go @@ -54,6 +54,17 @@ var _ = Describe("CloudStackMachineReconciler", func() { // Setup a failure domain for the machine reconciler to find. Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) setClusterReady(k8sClient) + + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).AnyTimes().Return(nil) + + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( + gomock.Any(), gomock.Any()).AnyTimes().Return(nil) + + mockCloudClient.EXPECT().RemoveVMFromUnmanagedCluster( + gomock.Any(), gomock.Any()).AnyTimes().Return(nil) }) It("Should call GetOrCreateVMInstance and set Status.Ready to true", func() { @@ -240,6 +251,8 @@ var _ = Describe("CloudStackMachineReconciler", func() { func(arg1, _, _, _, _, _ interface{}) { arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" }).AnyTimes() + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( + gomock.Any(), gomock.Any()).AnyTimes().Return(nil) Ω(fakeCtrlClient.Get(ctx, key, dummies.CSCluster)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CAPIMachine)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CSMachine1)).Should(Succeed()) diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index a9776f64..05c9a88d 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -21,16 +21,17 @@ import ( "flag" "fmt" "go/build" - "k8s.io/client-go/tools/record" "os" "os/exec" "path/filepath" "regexp" - "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" "strings" "testing" "time" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/cluster-api-provider-cloudstack/test/fakes" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" diff --git a/go.mod b/go.mod index e099ebf9..2675d5b7 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module sigs.k8s.io/cluster-api-provider-cloudstack go 1.19 require ( - github.com/apache/cloudstack-go/v2 v2.15.0 + github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -91,3 +91,5 @@ require ( ) replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 // Indirect upgrade to address https://github.com/advisories/GHSA-w73w-5m7g-f7qc + +replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 diff --git a/go.sum b/go.sum index 2dea8353..5bf484bc 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,6 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= -github.com/apache/cloudstack-go/v2 v2.15.0 h1:oojn1qx0+wBwrFSSmA2rL8XjWd4BXqwYo0RVCrAXoHk= -github.com/apache/cloudstack-go/v2 v2.15.0/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -304,6 +302,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 h1:FPRBv784robz6sZSqDGfZDZMse31lj96i+enH02Xzds= +github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/pkg/cloud/client.go b/pkg/cloud/client.go index 12e61176..76c50256 100644 --- a/pkg/cloud/client.go +++ b/pkg/cloud/client.go @@ -37,6 +37,7 @@ import ( //go:generate ../../hack/tools/bin/mockgen -destination=../mocks/mock_client.go -package=mocks sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud Client type Client interface { + ClusterIface VMIface NetworkIface AffinityGroupIface diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go new file mode 100644 index 00000000..43442a1d --- /dev/null +++ b/pkg/cloud/cluster.go @@ -0,0 +1,139 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package cloud + +import ( + "fmt" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/pkg/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" +) + +type ClusterIface interface { + GetOrCreateUnmanagedCluster(*clusterv1.Cluster, *infrav1.CloudStackCluster, *infrav1.CloudStackFailureDomainSpec) error + DeleteUnmanagedCluster(*infrav1.CloudStackCluster) error + AddVMToUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error + RemoveVMFromUnmanagedCluster(*infrav1.CloudStackCluster, *infrav1.CloudStackMachine) error +} + +type ClustertypeSetter interface { + SetClustertype(string) +} + +func withExternalManaged() cloudstack.OptionFunc { + return func(cs *cloudstack.CloudStackClient, p interface{}) error { + ps, ok := p.(ClustertypeSetter) + if !ok { + return errors.New("invalid params type") + } + ps.SetClustertype("ExternalManaged") + return nil + } +} + +func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { + // Get cluster + if csCluster.Status.CloudStackClusterID != "" { + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + if err != nil { + return err + } else if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + return nil + } + } + + // Check if a cluster exists with the same name + clusterName := fmt.Sprintf("%s - %s - %s", cluster.GetName(), csCluster.GetName(), csCluster.GetUID()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged()) + if err != nil && !strings.Contains(err.Error(), "No match found for ") { + return err + } + if count > 0 { + csCluster.Status.CloudStackClusterID = externalManagedCluster.Id + } else if err == nil || (err != nil && strings.Contains(err.Error(), "No match found for ")) { + // Create cluster + domain := Domain{Path: rootDomain} + if csCluster.Spec.FailureDomains[0].Domain != "" { + domain.Path = fd.Domain + } + _ = c.ResolveDomain(&domain) + + accountName := csCluster.Spec.FailureDomains[0].Account + if accountName == "" { + userParams := c.cs.User.NewGetUserParams(c.config.APIKey) + user, err := c.cs.User.GetUser(userParams) + if err != nil { + return err + } + accountName = user.Account + } + params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), clusterName, fd.Zone.ID) + + setIfNotEmpty(accountName, params.SetAccount) + setIfNotEmpty(domain.ID, params.SetDomainid) + setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid) + setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress) + params.SetClustertype("ExternalManaged") + + cloudStackCKSCluster, err := c.cs.Kubernetes.CreateKubernetesCluster(params) + if err != nil { + return err + } + csCluster.Status.CloudStackClusterID = cloudStackCKSCluster.Id + } + return nil +} + +func (c *client) DeleteUnmanagedCluster(csCluster *infrav1.CloudStackCluster) error { + if csCluster.Status.CloudStackClusterID != "" { + csUnmanagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + if err != nil && strings.Contains(err.Error(), " not found") { + return nil + } + if count != 0 { + params := c.cs.Kubernetes.NewDeleteKubernetesClusterParams(csUnmanagedCluster.Id) + _, err = c.cs.Kubernetes.DeleteKubernetesCluster(params) + if err != nil { + return err + } + } + csCluster.Status.CloudStackClusterID = "" + } + return nil +} + +func (c *client) AddVMToUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewAddVirtualMachinesToKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.AddVirtualMachinesToKubernetesCluster(params) + return err + } + return nil +} + +func (c *client) RemoveVMFromUnmanagedCluster(csCluster *infrav1.CloudStackCluster, csMachine *infrav1.CloudStackMachine) error { + if csCluster.Status.CloudStackClusterID != "" { + params := c.cs.Kubernetes.NewRemoveVirtualMachinesFromKubernetesClusterParams(csCluster.Status.CloudStackClusterID, []string{*csMachine.Spec.InstanceID}) + _, err := c.cs.Kubernetes.RemoveVirtualMachinesFromKubernetesCluster(params) + return err + } + return nil +} diff --git a/test/e2e/common.go b/test/e2e/common.go index 53518904..ea537200 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -251,6 +251,17 @@ func DownloadMetricsFromCAPCManager(ctx context.Context, bootstrapKubeconfigPath return result, nil } +func GetACSVersion(client *cloudstack.CloudStackClient) (string, error) { + msServersResp, err := client.InfrastructureUsage.ListManagementServersMetrics(client.InfrastructureUsage.NewListManagementServersMetricsParams()) + if err != nil { + return "", err + } + if msServersResp.Count == 0 { + return "", errors.New("no management servers found") + } + return msServersResp.ManagementServersMetrics[0].Version, nil +} + func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string, machineType string) { matcher := clusterName + "-" + machineType diff --git a/test/e2e/go.mod b/test/e2e/go.mod index a94abae4..7ab0f640 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -4,8 +4,9 @@ go 1.19 require ( github.com/Shopify/toxiproxy/v2 v2.5.0 - github.com/apache/cloudstack-go/v2 v2.15.0 + github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 github.com/blang/semver v3.5.1+incompatible + github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.8 gopkg.in/yaml.v3 v3.0.1 @@ -18,6 +19,8 @@ require ( sigs.k8s.io/controller-runtime v0.14.5 ) +require gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + require ( github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect @@ -78,6 +81,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 1a821219..7217c3ec 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -65,8 +65,8 @@ github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPp github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/cloudstack-go/v2 v2.15.0 h1:oojn1qx0+wBwrFSSmA2rL8XjWd4BXqwYo0RVCrAXoHk= -github.com/apache/cloudstack-go/v2 v2.15.0/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 h1:Vare0MkgdEbAEY1dZtcz/m6HRdK+NKkuoiF6WwC1t2g= +github.com/apache/cloudstack-go/v2 v2.16.0-rc.2/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -144,6 +144,7 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -171,6 +172,7 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= @@ -290,6 +292,7 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -375,9 +378,18 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -568,6 +580,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -593,6 +606,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -639,6 +653,7 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -651,8 +666,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -676,6 +694,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -765,6 +784,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -887,12 +907,15 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/test/e2e/unmanaged_k8s.go b/test/e2e/unmanaged_k8s.go new file mode 100644 index 00000000..b8298c63 --- /dev/null +++ b/test/e2e/unmanaged_k8s.go @@ -0,0 +1,136 @@ +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/blang/semver" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +// UnmanagedK8SSpec implements a spec that creates a cluster and checks whether an entry is created in ACS. +func UnmanagedK8SSpec(ctx context.Context, inputGetter func() CommonSpecInput) { + var ( + specName = "k8s-unmanaged" + input CommonSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + version, err := GetACSVersion(csClient) + + if err != nil || version == "" { + Skip("Failed to get CloudStack's version") + } + + v, err := semver.ParseTolerant(strings.Join(strings.Split(version, ".")[0:3], ".")) + + if err != nil { + Skip("Failed to parse CloudStack version " + version) + } + + expectedRange, _ := semver.ParseRange(">=4.19.0") + + if !expectedRange(v) { + Skip("Cloudstack version " + version + " is less than 4.19.") + } + + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult) + }) + + It("Should create a workload cluster", func() { + By("Creating a workload cluster") + + clusterName := fmt.Sprintf("%s-%s", specName, util.RandomString(6)) + + clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + CNIManifestPath: input.E2EConfig.GetVariable(CNIPath), + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()), + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, + Flavor: clusterctl.DefaultFlavor, + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: pointer.Int64Ptr(1), + WorkerMachineCount: pointer.Int64Ptr(1), + }, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, clusterResources) + + By("checking unmanaged k8s resource is created on ACS") + // Get details from ACS + csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath()) + lkcp := csClient.Kubernetes.NewListKubernetesClustersParams() + lkcp.SetListall(true) + + clusters, err := csClient.Kubernetes.ListKubernetesClusters(lkcp) + + if err != nil { + Fail("Failed to get Kubernetes clusters from ACS") + } + + var cluster *cloudstack.KubernetesCluster + + for _, d := range clusters.KubernetesClusters { + if strings.HasPrefix(d.Name, fmt.Sprintf("%s - %s", clusterName, clusterName)) { + cluster = d + } + } + + Expect(cluster).ShouldNot(BeNil(), "Couldn't find the external managed kubernetes in ACS") + Expect(len(cluster.Virtualmachines)).Should(Equal(2), "Expected 2 VMs in the cluster, found %d", len(cluster.Virtualmachines)) + By("PASSED!") + }) + + AfterEach(func() { + // Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself. + dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup) + }) +} diff --git a/test/e2e/unmanaged_k8s_test.go b/test/e2e/unmanaged_k8s_test.go new file mode 100644 index 00000000..ee8bd487 --- /dev/null +++ b/test/e2e/unmanaged_k8s_test.go @@ -0,0 +1,36 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2020 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo" +) + +var _ = Describe("When testing creation of unmanaged CKS cluster in ACS", func() { + UnmanagedK8SSpec(ctx, func() CommonSpecInput { + return CommonSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + } + }) +}) From 908364806f75d5df2689e0db05eef94294be7f48 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Wed, 3 Apr 2024 01:09:24 +0530 Subject: [PATCH 2/4] Add syncWithACS to control creation of external managed cluster on ACS --- api/v1beta1/conversion.go | 2 +- api/v1beta3/cloudstackcluster_types.go | 4 +++ ...e.cluster.x-k8s.io_cloudstackclusters.yaml | 4 +++ controllers/cloudstackcluster_controller.go | 25 +++++++++++-------- pkg/cloud/cluster.go | 3 ++- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/api/v1beta1/conversion.go b/api/v1beta1/conversion.go index 236685f6..3436aa59 100644 --- a/api/v1beta1/conversion.go +++ b/api/v1beta1/conversion.go @@ -51,7 +51,7 @@ func Convert_v1beta1_CloudStackCluster_To_v1beta3_CloudStackCluster(in *CloudSta //nolint:golint,revive,stylecheck func Convert_v1beta3_CloudStackCluster_To_v1beta1_CloudStackCluster(in *v1beta3.CloudStackCluster, out *CloudStackCluster, scope conv.Scope) error { if len(in.Spec.FailureDomains) < 1 { - return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", in.Spec) + return fmt.Errorf("v1beta3 to v1beta1 conversion not supported when < 1 failure domain is provided. Input CloudStackCluster spec %s", fmt.Sprintf("%v", in.Spec)) } out.ObjectMeta = in.ObjectMeta out.Spec = CloudStackClusterSpec{ diff --git a/api/v1beta3/cloudstackcluster_types.go b/api/v1beta3/cloudstackcluster_types.go index 9ffafbe0..ee6aadd8 100644 --- a/api/v1beta3/cloudstackcluster_types.go +++ b/api/v1beta3/cloudstackcluster_types.go @@ -34,6 +34,10 @@ type CloudStackClusterSpec struct { // The kubernetes control plane endpoint. ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"` + + // SyncWithACS determines if an externalManaged CKS cluster should be created on ACS. + // +optional + SyncWithACS bool `json:"syncWithACS"` } // The status of the CloudStackCluster object. diff --git a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml index 2cead21c..81bbca78 100644 --- a/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml +++ b/config/crd/bases/infrastructure.cluster.x-k8s.io_cloudstackclusters.yaml @@ -415,6 +415,10 @@ spec: - zone type: object type: array + syncWithACS: + description: SyncWithACS determines if an externalManaged CKS cluster + should be created on ACS. + type: boolean required: - controlPlaneEndpoint - failureDomains diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index fb896c13..e8218e33 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -101,18 +101,21 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re // GetOrCreateUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack else creates one. func (r *CloudStackClusterReconciliationRunner) GetOrCreateUnmanagedCluster() (ctrl.Result, error) { - res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() - if r.ShouldReturn(res, err) { - return res, err - } - err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) - if err != nil { - if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { - r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") - return ctrl.Result{}, nil + if r.CSCluster.Spec.SyncWithACS { + res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() + if r.ShouldReturn(res, err) { + return res, err + } + + err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + if err != nil { + if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { + r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") + return ctrl.Result{}, nil + } + // Not requeueing the failure to support CloudStack v4.18 and before + r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) } - // Not requeueing the failure to support CloudStack v4.18 and before - r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) } return ctrl.Result{}, nil } diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go index 43442a1d..d058f27a 100644 --- a/pkg/cloud/cluster.go +++ b/pkg/cloud/cluster.go @@ -85,7 +85,8 @@ func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csClust } accountName = user.Account } - params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), clusterName, fd.Zone.ID) + // NewCreateKubernetesClusterParams(description string, kubernetesversionid string, name string, serviceofferingid string, size int64, zoneid string) *CreateKubernetesClusterParams + params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), "", clusterName, "", 0, fd.Zone.ID) setIfNotEmpty(accountName, params.SetAccount) setIfNotEmpty(domain.ID, params.SetDomainid) From 3f13fb543c32e92d8806820a1bffbd7a425f0ad7 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Fri, 5 Apr 2024 11:54:20 +0530 Subject: [PATCH 3/4] Add e2e tests --- Makefile | 2 +- api/v1beta2/cloudstackcluster_conversion.go | 9 ++++ api/v1beta2/zz_generated.conversion.go | 32 +++++-------- api/v1beta3/cloudstackcluster_types.go | 2 +- controllers/cloudstackcluster_controller.go | 4 +- go.mod | 10 ++-- go.sum | 18 +++---- pkg/cloud/cluster.go | 15 +++--- test/e2e/config/cloudstack.yaml | 1 + .../v1beta3/bases/cluster-with-kcp.yaml | 1 + .../cloudstack-cluster.yaml | 19 ++++++++ .../kustomization.yaml | 6 +++ test/e2e/go.mod | 16 +++---- test/e2e/go.sum | 47 +++++-------------- test/e2e/unmanaged_k8s.go | 4 +- test/e2e/unmanaged_k8s_test.go | 2 +- 16 files changed, 92 insertions(+), 96 deletions(-) create mode 100644 test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml create mode 100644 test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml diff --git a/Makefile b/Makefile index f8bfd389..bdd01637 100644 --- a/Makefile +++ b/Makefile @@ -293,7 +293,7 @@ E2E_TIMEOUT ?= 3h run-e2e: e2e-essentials ## Run e2e testing. JOB is an optional REGEXP to select certainn test cases to run. e.g. JOB=PR-Blocking, JOB=Conformance $(KUBECTL) apply -f cloud-config.yaml && \ cd test/e2e && \ - $(GINKGO) -v --trace --tags=e2e --focus=$(JOB) --timeout=$(E2E_TIMEOUT) --skip=Conformance --skip-package=kubeconfig_helper --nodes=1 --no-color=false ./... -- \ + $(GINKGO) -vv --trace --tags=e2e --focus=$(JOB) --timeout=$(E2E_TIMEOUT) --skip=Conformance --skip-package=kubeconfig_helper --nodes=1 --no-color=false ./... -- \ -e2e.artifacts-folder=${REPO_ROOT}/_artifacts \ -e2e.config=${E2E_CONFIG} \ -e2e.skip-resource-cleanup=false -e2e.use-existing-cluster=true diff --git a/api/v1beta2/cloudstackcluster_conversion.go b/api/v1beta2/cloudstackcluster_conversion.go index 31fd3c20..dbd8a21a 100644 --- a/api/v1beta2/cloudstackcluster_conversion.go +++ b/api/v1beta2/cloudstackcluster_conversion.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + machineryconversion "k8s.io/apimachinery/pkg/conversion" "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" "sigs.k8s.io/controller-runtime/pkg/conversion" ) @@ -30,3 +31,11 @@ func (dst *CloudStackCluster) ConvertFrom(srcRaw conversion.Hub) error { // noli src := srcRaw.(*v1beta3.CloudStackCluster) return Convert_v1beta3_CloudStackCluster_To_v1beta2_CloudStackCluster(src, dst, nil) } + +func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s machineryconversion.Scope) error { // nolint + return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s) +} + +func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s machineryconversion.Scope) error { // nolint + return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s) +} diff --git a/api/v1beta2/zz_generated.conversion.go b/api/v1beta2/zz_generated.conversion.go index 026a8bd5..9573ab25 100644 --- a/api/v1beta2/zz_generated.conversion.go +++ b/api/v1beta2/zz_generated.conversion.go @@ -103,21 +103,11 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta3.CloudStackClusterSpec)(nil), (*CloudStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(a.(*v1beta3.CloudStackClusterSpec), b.(*CloudStackClusterSpec), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*CloudStackClusterStatus)(nil), (*v1beta3.CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus(a.(*CloudStackClusterStatus), b.(*v1beta3.CloudStackClusterStatus), scope) }); err != nil { return err } - if err := s.AddGeneratedConversionFunc((*v1beta3.CloudStackClusterStatus)(nil), (*CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { - return Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(a.(*v1beta3.CloudStackClusterStatus), b.(*CloudStackClusterStatus), scope) - }); err != nil { - return err - } if err := s.AddGeneratedConversionFunc((*CloudStackFailureDomain)(nil), (*v1beta3.CloudStackFailureDomain)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta2_CloudStackFailureDomain_To_v1beta3_CloudStackFailureDomain(a.(*CloudStackFailureDomain), b.(*v1beta3.CloudStackFailureDomain), scope) }); err != nil { @@ -358,6 +348,16 @@ func RegisterConversions(s *runtime.Scheme) error { }); err != nil { return err } + if err := s.AddConversionFunc((*v1beta3.CloudStackClusterSpec)(nil), (*CloudStackClusterSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(a.(*v1beta3.CloudStackClusterSpec), b.(*CloudStackClusterSpec), scope) + }); err != nil { + return err + } + if err := s.AddConversionFunc((*v1beta3.CloudStackClusterStatus)(nil), (*CloudStackClusterStatus)(nil), func(a, b interface{}, scope conversion.Scope) error { + return Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(a.(*v1beta3.CloudStackClusterStatus), b.(*CloudStackClusterStatus), scope) + }); err != nil { + return err + } if err := s.AddConversionFunc((*v1beta3.CloudStackFailureDomainSpec)(nil), (*CloudStackFailureDomainSpec)(nil), func(a, b interface{}, scope conversion.Scope) error { return Convert_v1beta3_CloudStackFailureDomainSpec_To_v1beta2_CloudStackFailureDomainSpec(a.(*v1beta3.CloudStackFailureDomainSpec), b.(*CloudStackFailureDomainSpec), scope) }); err != nil { @@ -579,14 +579,10 @@ func autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec( out.FailureDomains = nil } out.ControlPlaneEndpoint = in.ControlPlaneEndpoint + // WARNING: in.SyncWithACS requires manual conversion: does not exist in peer-type return nil } -// Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec is an autogenerated conversion function. -func Convert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in *v1beta3.CloudStackClusterSpec, out *CloudStackClusterSpec, s conversion.Scope) error { - return autoConvert_v1beta3_CloudStackClusterSpec_To_v1beta2_CloudStackClusterSpec(in, out, s) -} - func autoConvert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus(in *CloudStackClusterStatus, out *v1beta3.CloudStackClusterStatus, s conversion.Scope) error { out.FailureDomains = *(*v1beta1.FailureDomains)(unsafe.Pointer(&in.FailureDomains)) out.Ready = in.Ready @@ -600,15 +596,11 @@ func Convert_v1beta2_CloudStackClusterStatus_To_v1beta3_CloudStackClusterStatus( func autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s conversion.Scope) error { out.FailureDomains = *(*v1beta1.FailureDomains)(unsafe.Pointer(&in.FailureDomains)) + // WARNING: in.CloudStackClusterID requires manual conversion: does not exist in peer-type out.Ready = in.Ready return nil } -// Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus is an autogenerated conversion function. -func Convert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in *v1beta3.CloudStackClusterStatus, out *CloudStackClusterStatus, s conversion.Scope) error { - return autoConvert_v1beta3_CloudStackClusterStatus_To_v1beta2_CloudStackClusterStatus(in, out, s) -} - func autoConvert_v1beta2_CloudStackFailureDomain_To_v1beta3_CloudStackFailureDomain(in *CloudStackFailureDomain, out *v1beta3.CloudStackFailureDomain, s conversion.Scope) error { out.ObjectMeta = in.ObjectMeta if err := Convert_v1beta2_CloudStackFailureDomainSpec_To_v1beta3_CloudStackFailureDomainSpec(&in.Spec, &out.Spec, s); err != nil { diff --git a/api/v1beta3/cloudstackcluster_types.go b/api/v1beta3/cloudstackcluster_types.go index ee6aadd8..08b68d58 100644 --- a/api/v1beta3/cloudstackcluster_types.go +++ b/api/v1beta3/cloudstackcluster_types.go @@ -37,7 +37,7 @@ type CloudStackClusterSpec struct { // SyncWithACS determines if an externalManaged CKS cluster should be created on ACS. // +optional - SyncWithACS bool `json:"syncWithACS"` + SyncWithACS bool `json:"syncWithACS,omitempty"` } // The status of the CloudStackCluster object. diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index e8218e33..958a57e3 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -95,8 +95,8 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re r.GetFailureDomains(r.FailureDomains), r.RemoveExtraneousFailureDomains(r.FailureDomains), r.VerifyFailureDomainCRDs, - r.GetOrCreateUnmanagedCluster, - r.SetReady) + r.SetReady, + r.GetOrCreateUnmanagedCluster) } // GetOrCreateUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack else creates one. diff --git a/go.mod b/go.mod index 2675d5b7..0c7e4099 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module sigs.k8s.io/cluster-api-provider-cloudstack go 1.19 require ( - github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 + github.com/apache/cloudstack-go/v2 v2.16.1 github.com/go-logr/logr v1.2.4 github.com/golang/mock v1.6.0 github.com/hashicorp/go-multierror v1.1.1 @@ -70,11 +70,11 @@ require ( github.com/rogpeppe/go-internal v1.9.0 // indirect github.com/spf13/cobra v1.6.1 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sync v0.2.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect @@ -91,5 +91,3 @@ require ( ) replace github.com/dgrijalva/jwt-go => github.com/golang-jwt/jwt/v4 v4.0.0 // Indirect upgrade to address https://github.com/advisories/GHSA-w73w-5m7g-f7qc - -replace github.com/apache/cloudstack-go/v2 => github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 diff --git a/go.sum b/go.sum index 5bf484bc..b04ed6ac 100644 --- a/go.sum +++ b/go.sum @@ -22,6 +22,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= +github.com/apache/cloudstack-go/v2 v2.16.1 h1:2wOE4RKEjWPRZNO7ZNnZYmR3JJ+JJPQwhoc7W1fkiK4= +github.com/apache/cloudstack-go/v2 v2.16.1/go.mod h1:cZsgFe+VmrgLBm7QjeHTJBXYe8E5+yGYkdfwGb+Pu9c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -302,8 +304,6 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510 h1:FPRBv784robz6sZSqDGfZDZMse31lj96i+enH02Xzds= -github.com/shapeblue/cloudstack-go/v2 v2.9.1-0.20230717062313-73e4efc8a510/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -367,7 +367,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -409,8 +409,8 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -448,11 +448,11 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= diff --git a/pkg/cloud/cluster.go b/pkg/cloud/cluster.go index d058f27a..f1831063 100644 --- a/pkg/cloud/cluster.go +++ b/pkg/cloud/cluster.go @@ -51,7 +51,7 @@ func withExternalManaged() cloudstack.OptionFunc { func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csCluster *infrav1.CloudStackCluster, fd *infrav1.CloudStackFailureDomainSpec) error { // Get cluster if csCluster.Status.CloudStackClusterID != "" { - externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByID(csCluster.Status.CloudStackClusterID, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) if err != nil { return err } else if count > 0 { @@ -62,7 +62,7 @@ func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csClust // Check if a cluster exists with the same name clusterName := fmt.Sprintf("%s - %s - %s", cluster.GetName(), csCluster.GetName(), csCluster.GetUID()) - externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged()) + externalManagedCluster, count, err := c.cs.Kubernetes.GetKubernetesClusterByName(clusterName, withExternalManaged(), cloudstack.WithProject(c.user.Project.ID)) if err != nil && !strings.Contains(err.Error(), "No match found for ") { return err } @@ -70,12 +70,6 @@ func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csClust csCluster.Status.CloudStackClusterID = externalManagedCluster.Id } else if err == nil || (err != nil && strings.Contains(err.Error(), "No match found for ")) { // Create cluster - domain := Domain{Path: rootDomain} - if csCluster.Spec.FailureDomains[0].Domain != "" { - domain.Path = fd.Domain - } - _ = c.ResolveDomain(&domain) - accountName := csCluster.Spec.FailureDomains[0].Account if accountName == "" { userParams := c.cs.User.NewGetUserParams(c.config.APIKey) @@ -88,10 +82,13 @@ func (c *client) GetOrCreateUnmanagedCluster(cluster *clusterv1.Cluster, csClust // NewCreateKubernetesClusterParams(description string, kubernetesversionid string, name string, serviceofferingid string, size int64, zoneid string) *CreateKubernetesClusterParams params := c.cs.Kubernetes.NewCreateKubernetesClusterParams(fmt.Sprintf("%s managed by CAPC", clusterName), "", clusterName, "", 0, fd.Zone.ID) + setIfNotEmpty(c.user.Project.ID, params.SetProjectid) setIfNotEmpty(accountName, params.SetAccount) - setIfNotEmpty(domain.ID, params.SetDomainid) + setIfNotEmpty(c.user.Domain.ID, params.SetDomainid) setIfNotEmpty(fd.Zone.Network.ID, params.SetNetworkid) setIfNotEmpty(csCluster.Spec.ControlPlaneEndpoint.Host, params.SetExternalloadbalanceripaddress) + params.ResetKubernetesversionid() + params.ResetServiceofferingid() params.SetClustertype("ExternalManaged") cloudStackCKSCluster, err := c.cs.Kubernetes.CreateKubernetesCluster(params) diff --git a/test/e2e/config/cloudstack.yaml b/test/e2e/config/cloudstack.yaml index ee06f42c..bd3409db 100644 --- a/test/e2e/config/cloudstack.yaml +++ b/test/e2e/config/cloudstack.yaml @@ -98,6 +98,7 @@ providers: - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-ip.yaml" - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-before.yaml" - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-after.yaml" + - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged.yaml" - sourcePath: "../data/shared/v1beta1_provider/metadata.yaml" versions: - name: v1.0.0 diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml index bdd9d70b..2b719aa9 100644 --- a/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/bases/cluster-with-kcp.yaml @@ -35,6 +35,7 @@ spec: name : ${CLOUDSTACK_ZONE_NAME} network: name: ${CLOUDSTACK_NETWORK_NAME} + syncWithACS: false --- kind: KubeadmControlPlane apiVersion: controlplane.cluster.x-k8s.io/v1beta1 diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml new file mode 100644 index 00000000..7bb1a966 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/cloudstack-cluster.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1beta3 +kind: CloudStackCluster +metadata: + name: ${CLUSTER_NAME} +spec: + controlPlaneEndpoint: + host: "" + port: 6443 + failureDomains: + - name: ${CLOUDSTACK_FD1_NAME} + acsEndpoint: + name: ${CLOUDSTACK_FD1_SECRET_NAME} + namespace: default + zone: + name : ${CLOUDSTACK_ZONE_NAME} + network: + name: ${CLOUDSTACK_NETWORK_NAME} + syncWithACS: true diff --git a/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml new file mode 100644 index 00000000..7a6d2984 --- /dev/null +++ b/test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-k8s-unmanaged/kustomization.yaml @@ -0,0 +1,6 @@ +bases: + - ../bases/cluster-with-kcp.yaml + - ../bases/md.yaml + +patchesStrategicMerge: +- ./cloudstack-cluster.yaml diff --git a/test/e2e/go.mod b/test/e2e/go.mod index 7ab0f640..4ed90ee5 100644 --- a/test/e2e/go.mod +++ b/test/e2e/go.mod @@ -4,9 +4,8 @@ go 1.19 require ( github.com/Shopify/toxiproxy/v2 v2.5.0 - github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 + github.com/apache/cloudstack-go/v2 v2.16.1 github.com/blang/semver v3.5.1+incompatible - github.com/onsi/ginkgo v1.16.5 github.com/onsi/ginkgo/v2 v2.11.0 github.com/onsi/gomega v1.27.8 gopkg.in/yaml.v3 v3.0.1 @@ -19,8 +18,6 @@ require ( sigs.k8s.io/controller-runtime v0.14.5 ) -require gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - require ( github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect @@ -81,7 +78,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/nxadm/tail v1.4.8 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.2 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -101,12 +97,12 @@ require ( github.com/stoewer/go-strcase v1.2.0 // indirect github.com/subosito/gotenv v1.4.2 // indirect github.com/valyala/fastjson v1.6.4 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.6.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.9.3 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect diff --git a/test/e2e/go.sum b/test/e2e/go.sum index 7217c3ec..91201722 100644 --- a/test/e2e/go.sum +++ b/test/e2e/go.sum @@ -65,8 +65,8 @@ github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPp github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 h1:yL7+Jz0jTC6yykIK/Wh74gnTJnrGr5AyrNMXuA0gves= github.com/antlr/antlr4/runtime/Go/antlr v1.4.10/go.mod h1:F7bn7fEU90QkQ3tnmaTx3LTKLEDqnwWODIYppRQ5hnY= -github.com/apache/cloudstack-go/v2 v2.16.0-rc.2 h1:Vare0MkgdEbAEY1dZtcz/m6HRdK+NKkuoiF6WwC1t2g= -github.com/apache/cloudstack-go/v2 v2.16.0-rc.2/go.mod h1:Mc+tXpujtslBuZFk5atoGT2LanVxOrXS2GGgidAoz1A= +github.com/apache/cloudstack-go/v2 v2.16.1 h1:2wOE4RKEjWPRZNO7ZNnZYmR3JJ+JJPQwhoc7W1fkiK4= +github.com/apache/cloudstack-go/v2 v2.16.1/go.mod h1:cZsgFe+VmrgLBm7QjeHTJBXYe8E5+yGYkdfwGb+Pu9c= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -144,7 +144,6 @@ github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -172,7 +171,6 @@ github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= @@ -292,7 +290,6 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4= github.com/huandu/xstrings v1.3.3/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= @@ -378,18 +375,9 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= @@ -540,8 +528,8 @@ golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -580,7 +568,6 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -606,7 +593,6 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -623,8 +609,8 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -653,7 +639,6 @@ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -666,11 +651,8 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -694,7 +676,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -712,13 +693,13 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -729,8 +710,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -784,7 +765,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -907,15 +887,12 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/test/e2e/unmanaged_k8s.go b/test/e2e/unmanaged_k8s.go index b8298c63..10987d69 100644 --- a/test/e2e/unmanaged_k8s.go +++ b/test/e2e/unmanaged_k8s.go @@ -25,7 +25,7 @@ import ( "github.com/apache/cloudstack-go/v2/cloudstack" "github.com/blang/semver" - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/utils/pointer" @@ -92,7 +92,7 @@ func UnmanagedK8SSpec(ctx context.Context, inputGetter func() CommonSpecInput) { ClusterctlConfigPath: input.ClusterctlConfigPath, KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), InfrastructureProvider: clusterctl.DefaultInfrastructureProvider, - Flavor: clusterctl.DefaultFlavor, + Flavor: specName, Namespace: namespace.Name, ClusterName: clusterName, KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), diff --git a/test/e2e/unmanaged_k8s_test.go b/test/e2e/unmanaged_k8s_test.go index ee8bd487..5951d163 100644 --- a/test/e2e/unmanaged_k8s_test.go +++ b/test/e2e/unmanaged_k8s_test.go @@ -20,7 +20,7 @@ limitations under the License. package e2e import ( - . "github.com/onsi/ginkgo" + . "github.com/onsi/ginkgo/v2" ) var _ = Describe("When testing creation of unmanaged CKS cluster in ACS", func() { From 2290e10d4fa14152864dcd45c15189c279e2ec19 Mon Sep 17 00:00:00 2001 From: Vishesh Date: Sat, 1 Jun 2024 01:18:58 +0530 Subject: [PATCH 4/4] Move CKS integration to a separate controller --- controllers/cks_cluster_controller.go | 124 ++++++++++++++++ controllers/cks_cluster_controller_test.go | 69 +++++++++ controllers/cks_machine_controller.go | 108 ++++++++++++++ controllers/cks_machine_controller_test.go | 135 ++++++++++++++++++ controllers/cloudstackcluster_controller.go | 45 +----- controllers/cloudstackmachine_controller.go | 19 --- .../cloudstackmachine_controller_test.go | 8 -- controllers/controllers_suite_test.go | 10 ++ main.go | 8 ++ test/dummies/v1beta3/vars.go | 3 + 10 files changed, 458 insertions(+), 71 deletions(-) create mode 100644 controllers/cks_cluster_controller.go create mode 100644 controllers/cks_cluster_controller_test.go create mode 100644 controllers/cks_machine_controller.go create mode 100644 controllers/cks_machine_controller_test.go diff --git a/controllers/cks_cluster_controller.go b/controllers/cks_cluster_controller.go new file mode 100644 index 00000000..030555b7 --- /dev/null +++ b/controllers/cks_cluster_controller.go @@ -0,0 +1,124 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "strings" + + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" +) + +const CksClusterFinalizer = "ckscluster.infrastructure.cluster.x-k8s.io" + +// RBAC permissions for CloudStackCluster. +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/status,verbs=create;get;update;patch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackclusters/finalizers,verbs=update + +// CksClusterReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters. +// The runner does the actual reconciliation. +type CksClusterReconciliationRunner struct { + *csCtrlrUtils.ReconciliationRunner + FailureDomains *infrav1.CloudStackFailureDomainList + ReconciliationSubject *infrav1.CloudStackCluster +} + +// CksClusterReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster. +// This is primarily to adapt to k8s. +type CksClusterReconciler struct { + csCtrlrUtils.ReconcilerBase +} + +// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields. +func NewCksClusterReconciliationRunner() *CksClusterReconciliationRunner { + // Set concrete type and init pointers. + runner := &CksClusterReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackCluster{}} + runner.FailureDomains = &infrav1.CloudStackFailureDomainList{} + // Setup the base runner. Initializes pointers and links reconciliation methods. + runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSClusterController") + runner.CSCluster = runner.ReconciliationSubject + return runner +} + +// Reconcile is the method k8s will call upon a reconciliation request. +func (reconciler *CksClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { + r := NewCksClusterReconciliationRunner() + r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx) + r.WithAdditionalCommonStages(r.GetFailureDomains(r.FailureDomains)) + return r.RunBaseReconciliationStages() +} + +// Reconcile actually reconciles the CloudStackCluster. +func (r *CksClusterReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) { + if r.CSCluster.Spec.SyncWithACS { + // Prevent premature deletion. + controllerutil.AddFinalizer(r.ReconciliationSubject, CksClusterFinalizer) + + if len(r.FailureDomains.Items) == 0 { + return r.RequeueWithMessage("No failure domains found") + } + + res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + + r.Log.Info("Creating entry with CKS") + err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) + if err != nil { + if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { + r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") + return ctrl.Result{}, nil + } + r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) + return r.RequeueWithMessage(fmt.Sprintf("Syncing VMs with CloudStack failed. error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + +// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers. +func (r *CksClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, error) { + if r.ReconciliationSubject.Status.CloudStackClusterID != "" { + if len(r.FailureDomains.Items) == 0 { + return ctrl.Result{}, fmt.Errorf("no failure domains found") + } + res, err := r.AsFailureDomainUser(&r.FailureDomains.Items[0].Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject) + if err != nil && !strings.Contains(err.Error(), " not found") { + return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error())) + } + } + controllerutil.RemoveFinalizer(r.ReconciliationSubject, CksClusterFinalizer) + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (reconciler *CksClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.CloudStackCluster{}). + Complete(reconciler) +} diff --git a/controllers/cks_cluster_controller_test.go b/controllers/cks_cluster_controller_test.go new file mode 100644 index 00000000..5f3c81da --- /dev/null +++ b/controllers/cks_cluster_controller_test.go @@ -0,0 +1,69 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + "sigs.k8s.io/cluster-api-provider-cloudstack/controllers" + "sigs.k8s.io/cluster-api-provider-cloudstack/pkg/cloud" + dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" +) + +var _ = Describe("CksCloudStackClusterReconciler", func() { + Context("With k8s like test environment.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + SetupTestEnvironment() + Ω(ClusterReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack ClusterReconciler. + Ω(FailureDomainReconciler.SetupWithManager(k8sManager, controller.Options{})).Should(Succeed()) // Register CloudStack FailureDomainReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register CloudStack Cks ClusterReconciler. + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).MinTimes(1).Return(nil) + mockCloudClient.EXPECT().ResolveZone(gomock.Any()).AnyTimes() + mockCloudClient.EXPECT().ResolveNetworkForZone(gomock.Any()).AnyTimes().Do( + func(arg1 interface{}) { + arg1.(*infrav1.CloudStackZoneSpec).Network.ID = "SomeID" + arg1.(*infrav1.CloudStackZoneSpec).Network.Type = cloud.NetworkTypeShared + }).MinTimes(1) + }) + + It("Should create a cluster in CKS.", func() { + Eventually(func() string { + key := client.ObjectKeyFromObject(dummies.CSCluster) + if err := k8sClient.Get(ctx, key, dummies.CSCluster); err != nil { + return "" + } + return dummies.CSCluster.Status.CloudStackClusterID + }, timeout).WithPolling(pollInterval).Should(Equal("cluster-id-123")) + + }) + + }) + + Context("Without a k8s test environment.", func() { + It("Should create a reconciliation runner with a Cloudstack Cluster as the reconciliation subject.", func() { + reconRunner := controllers.NewCksClusterReconciliationRunner() + Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/cks_machine_controller.go b/controllers/cks_machine_controller.go new file mode 100644 index 00000000..80937e82 --- /dev/null +++ b/controllers/cks_machine_controller.go @@ -0,0 +1,108 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + ctrl "sigs.k8s.io/controller-runtime" + + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + csCtrlrUtils "sigs.k8s.io/cluster-api-provider-cloudstack/controllers/utils" +) + +// RBAC permissions for CloudStackCluster. +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines,verbs=get;list;watch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=cloudstackmachines/finalizers,verbs=update + +// CksMachineReconciliationRunner is a ReconciliationRunner with extensions specific to CloudStackClusters. +// The runner does the actual reconciliation. +type CksMachineReconciliationRunner struct { + *csCtrlrUtils.ReconciliationRunner + FailureDomain *infrav1.CloudStackFailureDomain + ReconciliationSubject *infrav1.CloudStackMachine +} + +// CksMachineReconciler is the k8s controller manager's interface to reconcile a CloudStackCluster. +// This is primarily to adapt to k8s. +type CksMachineReconciler struct { + csCtrlrUtils.ReconcilerBase +} + +// Initialize a new CloudStackCluster reconciliation runner with concrete types and initialized member fields. +func NewCksMachineReconciliationRunner() *CksMachineReconciliationRunner { + // Set concrete type and init pointers. + runner := &CksMachineReconciliationRunner{ReconciliationSubject: &infrav1.CloudStackMachine{}} + runner.FailureDomain = &infrav1.CloudStackFailureDomain{} + // Setup the base runner. Initializes pointers and links reconciliation methods. + runner.ReconciliationRunner = csCtrlrUtils.NewRunner(runner, runner.ReconciliationSubject, "CKSMachineController") + return runner +} + +// Reconcile is the method k8s will call upon a reconciliation request. +func (reconciler *CksMachineReconciler) Reconcile(ctx context.Context, req ctrl.Request) (retRes ctrl.Result, retErr error) { + r := NewCksMachineReconciliationRunner() + r.UsingBaseReconciler(reconciler.ReconcilerBase).ForRequest(req).WithRequestCtx(ctx) + r.WithAdditionalCommonStages( + r.GetFailureDomainByName(func() string { return r.ReconciliationSubject.Spec.FailureDomainName }, r.FailureDomain), + r.AsFailureDomainUser(&r.FailureDomain.Spec)) + return r.RunBaseReconciliationStages() +} + +// Reconcile actually reconciles the CloudStackCluster. +func (r *CksMachineReconciliationRunner) Reconcile() (res ctrl.Result, reterr error) { + if r.CSCluster.Spec.SyncWithACS { + if r.CSCluster.Status.CloudStackClusterID == "" { + return r.RequeueWithMessage("CloudStackClusterID is not set") + } + + if r.ReconciliationSubject.Spec.InstanceID == nil || *r.ReconciliationSubject.Spec.InstanceID == "" { + return r.RequeueWithMessage("InstanceID is not set") + } + + res, err := r.AsFailureDomainUser(&r.FailureDomain.Spec)() + if r.ShouldReturn(res, err) { + return res, err + } + r.Log.Info("Assigning VM to CKS") + err = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Adding VM to CloudStack Unmanaged kubernetes failed. error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + +// ReconcileDelete cleans up resources used by the cluster and finally removes the CloudStackCluster's finalizers. +func (r *CksMachineReconciliationRunner) ReconcileDelete() (ctrl.Result, error) { + if r.ReconciliationSubject.Spec.InstanceID != nil && *r.ReconciliationSubject.Spec.InstanceID != "" { + err := r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) + if err != nil { + return r.RequeueWithMessage(fmt.Sprintf("Removing VM from CloudStack Unmanaged kubernetes failed. error: %s", err.Error())) + } + } + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (reconciler *CksMachineReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&infrav1.CloudStackMachine{}). + Complete(reconciler) +} diff --git a/controllers/cks_machine_controller_test.go b/controllers/cks_machine_controller_test.go new file mode 100644 index 00000000..6bb0b63b --- /dev/null +++ b/controllers/cks_machine_controller_test.go @@ -0,0 +1,135 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers_test + +import ( + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" + infrav1 "sigs.k8s.io/cluster-api-provider-cloudstack/api/v1beta3" + "sigs.k8s.io/cluster-api-provider-cloudstack/controllers" + dummies "sigs.k8s.io/cluster-api-provider-cloudstack/test/dummies/v1beta3" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +var _ = Describe("CloudStackMachineReconciler", func() { + Context("With machine controller running.", func() { + BeforeEach(func() { + dummies.SetDummyVars() + dummies.CSCluster.Spec.SyncWithACS = true + dummies.CSCluster.Spec.FailureDomains = dummies.CSCluster.Spec.FailureDomains[:1] + dummies.CSCluster.Spec.FailureDomains[0].Name = dummies.CSFailureDomain1.Spec.Name + + SetupTestEnvironment() // Must happen before setting up managers/reconcilers. + Ω(MachineReconciler.SetupWithManager(ctx, k8sManager, controller.Options{})).Should(Succeed()) // Register the CloudStack MachineReconciler. + Ω(CksClusterReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register the CloudStack MachineReconciler. + Ω(CksMachineReconciler.SetupWithManager(k8sManager)).Should(Succeed()) // Register the CloudStack MachineReconciler. + + mockCloudClient.EXPECT().GetOrCreateUnmanagedCluster(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_, arg1, _ interface{}) { + arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" + }).MinTimes(1).Return(nil) + // Point CAPI machine Bootstrap secret ref to dummy bootstrap secret. + dummies.CAPIMachine.Spec.Bootstrap.DataSecretName = &dummies.BootstrapSecret.Name + Ω(k8sClient.Create(ctx, dummies.BootstrapSecret)).Should(Succeed()) + + // Setup a failure domain for the machine reconciler to find. + Ω(k8sClient.Create(ctx, dummies.CSFailureDomain1)).Should(Succeed()) + setClusterReady(k8sClient) + }) + + It("Should call AddVMToUnmanagedCluster", func() { + // Mock a call to GetOrCreateVMInstance and set the machine to running. + mockCloudClient.EXPECT().GetOrCreateVMInstance( + gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _, _, _, _ interface{}) { + arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" + }).AnyTimes() + + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + // Have to do this here or the reconcile call to GetOrCreateVMInstance may happen too early. + setupMachineCRDs() + + // Eventually the machine should set ready to true. + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err == nil { + if tempMachine.Status.Ready == true { + return len(tempMachine.ObjectMeta.Finalizers) > 0 + } + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + }) + + It("Should call RemoveVMFromUnmanagedCluster when CS machine deleted", func() { + // Mock a call to GetOrCreateVMInstance and set the machine to running. + mockCloudClient.EXPECT().GetOrCreateVMInstance( + gomock.Any(), gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Do( + func(arg1, _, _, _, _, _ interface{}) { + arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" + controllerutil.AddFinalizer(arg1.(*infrav1.CloudStackMachine), infrav1.MachineFinalizer) + }).AnyTimes() + + mockCloudClient.EXPECT().AddVMToUnmanagedCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + + mockCloudClient.EXPECT().DestroyVMInstance(gomock.Any()).Times(1).Return(nil) + mockCloudClient.EXPECT().RemoveVMFromUnmanagedCluster( + gomock.Any(), gomock.Any()).MinTimes(1).Return(nil) + // Have to do this here or the reconcile call to GetOrCreateVMInstance may happen too early. + setupMachineCRDs() + + // Eventually the machine should set ready to true. + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err == nil { + if tempMachine.Status.Ready == true { + return true + } + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + + Ω(k8sClient.Delete(ctx, dummies.CSMachine1)).Should(Succeed()) + + Eventually(func() bool { + tempMachine := &infrav1.CloudStackMachine{} + key := client.ObjectKey{Namespace: dummies.ClusterNameSpace, Name: dummies.CSMachine1.Name} + if err := k8sClient.Get(ctx, key, tempMachine); err != nil { + return errors.IsNotFound(err) + } + return false + }, timeout).WithPolling(pollInterval).Should(BeTrue()) + + }) + }) + + Context("Without a k8s test environment.", func() { + It("Should create a reconciliation runner with a Cloudstack Machine as the reconciliation subject.", func() { + reconRunner := controllers.NewCksMachineReconciliationRunner() + Ω(reconRunner.ReconciliationSubject).ShouldNot(BeNil()) + }) + }) +}) diff --git a/controllers/cloudstackcluster_controller.go b/controllers/cloudstackcluster_controller.go index 958a57e3..a42acfca 100644 --- a/controllers/cloudstackcluster_controller.go +++ b/controllers/cloudstackcluster_controller.go @@ -20,7 +20,6 @@ import ( "context" "fmt" "reflect" - "strings" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -95,29 +94,7 @@ func (r *CloudStackClusterReconciliationRunner) Reconcile() (res ctrl.Result, re r.GetFailureDomains(r.FailureDomains), r.RemoveExtraneousFailureDomains(r.FailureDomains), r.VerifyFailureDomainCRDs, - r.SetReady, - r.GetOrCreateUnmanagedCluster) -} - -// GetOrCreateUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack else creates one. -func (r *CloudStackClusterReconciliationRunner) GetOrCreateUnmanagedCluster() (ctrl.Result, error) { - if r.CSCluster.Spec.SyncWithACS { - res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() - if r.ShouldReturn(res, err) { - return res, err - } - - err = r.CSUser.GetOrCreateUnmanagedCluster(r.CAPICluster, r.ReconciliationSubject, &r.FailureDomains.Items[0].Spec) - if err != nil { - if strings.Contains(err.Error(), "Kubernetes Service plugin is disabled") { - r.Log.Info("Kubernetes Service plugin is disabled on CloudStack. Skipping ExternalManaged kubernetes cluster creation") - return ctrl.Result{}, nil - } - // Not requeueing the failure to support CloudStack v4.18 and before - r.Log.Info(fmt.Sprintf("Failed creating ExternalManaged kubernetes cluster on CloudStack. Error: %s", err.Error())) - } - } - return ctrl.Result{}, nil + r.SetReady) } // SetReady adds a finalizer and sets the cluster status to ready. @@ -174,30 +151,10 @@ func (r *CloudStackClusterReconciliationRunner) ReconcileDelete() (ctrl.Result, } return r.RequeueWithMessage("Child FailureDomains still present, requeueing.") } - if res, err := r.DeleteUnmanagedCluster(); r.ShouldReturn(res, err) { - return res, err - } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.ClusterFinalizer) return ctrl.Result{}, nil } -// DeleteUnmanagedCluster checks if an unmanaged cluster is present in Cloudstack and then deletes it. -func (r *CloudStackClusterReconciliationRunner) DeleteUnmanagedCluster() (ctrl.Result, error) { - // If field is present and delete fails, then requeue - res, err := r.AsFailureDomainUser(&r.CSCluster.Spec.FailureDomains[0])() - if r.ShouldReturn(res, err) { - return res, err - } - err = r.CSUser.DeleteUnmanagedCluster(r.ReconciliationSubject) - if err != nil { - if strings.Contains(err.Error(), " not found") { - return ctrl.Result{}, nil - } - return r.RequeueWithMessage(fmt.Sprintf("Deleting unmanaged kubernetes cluster on CloudStack failed. error: %s", err.Error())) - } - return ctrl.Result{}, nil -} - // Called in main, this registers the cluster reconciler to the CAPI controller manager. func (reconciler *CloudStackClusterReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager, opts controller.Options) error { log := ctrl.LoggerFrom(ctx) diff --git a/controllers/cloudstackmachine_controller.go b/controllers/cloudstackmachine_controller.go index 156c463e..0a3483af 100644 --- a/controllers/cloudstackmachine_controller.go +++ b/controllers/cloudstackmachine_controller.go @@ -128,7 +128,6 @@ func (r *CloudStackMachineReconciliationRunner) Reconcile() (retRes ctrl.Result, r.RequeueIfInstanceNotRunning, r.AddToLBIfNeeded, r.GetOrCreateMachineStateChecker, - r.AttachVM, ) } @@ -215,20 +214,6 @@ func (r *CloudStackMachineReconciliationRunner) DeleteMachineIfFailuredomainNotE return ctrl.Result{}, nil } -// AttachVM adds the VM to CloudStack Unmanaged kubernetes. -// No action taken if it fails -func (r *CloudStackMachineReconciliationRunner) AttachVM() (retRes ctrl.Result, reterr error) { - _ = r.CSUser.AddVMToUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) - return ctrl.Result{}, nil -} - -// RemoveVM removes the VM from CloudStack Unmanaged kubernetes. -// No action taken if it fails -func (r *CloudStackMachineReconciliationRunner) RemoveVM() (retRes ctrl.Result, reterr error) { - _ = r.CSUser.RemoveVMFromUnmanagedCluster(r.CSCluster, r.ReconciliationSubject) - return ctrl.Result{}, nil -} - // GetOrCreateVMInstance gets or creates a VM instance. // Implicitly it also fetches its bootstrap secret in order to create said instance. func (r *CloudStackMachineReconciliationRunner) GetOrCreateVMInstance() (retRes ctrl.Result, reterr error) { @@ -356,10 +341,6 @@ func (r *CloudStackMachineReconciliationRunner) ReconcileDelete() (retRes ctrl.R return ctrl.Result{}, err } - res, err := r.RemoveVM() - if r.ShouldReturn(res, err) { - return res, err - } controllerutil.RemoveFinalizer(r.ReconciliationSubject, infrav1.MachineFinalizer) r.Log.Info("VM Deleted", "instanceID", r.ReconciliationSubject.Spec.InstanceID) return ctrl.Result{}, nil diff --git a/controllers/cloudstackmachine_controller_test.go b/controllers/cloudstackmachine_controller_test.go index 5e4c223c..dda1acea 100644 --- a/controllers/cloudstackmachine_controller_test.go +++ b/controllers/cloudstackmachine_controller_test.go @@ -59,12 +59,6 @@ var _ = Describe("CloudStackMachineReconciler", func() { func(arg1, _, _ interface{}) { arg1.(*infrav1.CloudStackCluster).Status.CloudStackClusterID = "cluster-id-123" }).AnyTimes().Return(nil) - - mockCloudClient.EXPECT().AddVMToUnmanagedCluster( - gomock.Any(), gomock.Any()).AnyTimes().Return(nil) - - mockCloudClient.EXPECT().RemoveVMFromUnmanagedCluster( - gomock.Any(), gomock.Any()).AnyTimes().Return(nil) }) It("Should call GetOrCreateVMInstance and set Status.Ready to true", func() { @@ -251,8 +245,6 @@ var _ = Describe("CloudStackMachineReconciler", func() { func(arg1, _, _, _, _, _ interface{}) { arg1.(*infrav1.CloudStackMachine).Status.InstanceState = "Running" }).AnyTimes() - mockCloudClient.EXPECT().AddVMToUnmanagedCluster( - gomock.Any(), gomock.Any()).AnyTimes().Return(nil) Ω(fakeCtrlClient.Get(ctx, key, dummies.CSCluster)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CAPIMachine)).Should(Succeed()) Ω(fakeCtrlClient.Create(ctx, dummies.CSMachine1)).Should(Succeed()) diff --git a/controllers/controllers_suite_test.go b/controllers/controllers_suite_test.go index 05c9a88d..c1b2d1df 100644 --- a/controllers/controllers_suite_test.go +++ b/controllers/controllers_suite_test.go @@ -130,6 +130,10 @@ var ( FailureDomainReconciler *csReconcilers.CloudStackFailureDomainReconciler IsoNetReconciler *csReconcilers.CloudStackIsoNetReconciler AffinityGReconciler *csReconcilers.CloudStackAffinityGroupReconciler + + // CKS Reconcilers + CksClusterReconciler *csReconcilers.CksClusterReconciler + CksMachineReconciler *csReconcilers.CksMachineReconciler ) var _ = BeforeSuite(func() { @@ -225,6 +229,9 @@ func SetupTestEnvironment() { IsoNetReconciler = &csReconcilers.CloudStackIsoNetReconciler{ReconcilerBase: base} AffinityGReconciler = &csReconcilers.CloudStackAffinityGroupReconciler{ReconcilerBase: base} + CksClusterReconciler = &csReconcilers.CksClusterReconciler{ReconcilerBase: base} + CksMachineReconciler = &csReconcilers.CksMachineReconciler{ReconcilerBase: base} + ctx, cancel = context.WithCancel(context.TODO()) // Setup mock clients. @@ -238,6 +245,9 @@ func SetupTestEnvironment() { AffinityGReconciler.CSClient = mockCloudClient FailureDomainReconciler.CSClient = mockCloudClient + CksClusterReconciler.CSClient = mockCloudClient + CksMachineReconciler.CSClient = mockCloudClient + setupClusterCRDs() // See reconciliation results. Left commented as it's noisy otherwise. diff --git a/main.go b/main.go index d7046dd2..3e0248d2 100644 --- a/main.go +++ b/main.go @@ -267,4 +267,12 @@ func setupReconcilers(ctx context.Context, base utils.ReconcilerBase, opts manag setupLog.Error(err, "unable to create controller", "controller", "CloudStackFailureDomain") os.Exit(1) } + if err := (&controllers.CksClusterReconciler{ReconcilerBase: base}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CKSClusterController") + os.Exit(1) + } + if err := (&controllers.CksMachineReconciler{ReconcilerBase: base}).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CKSMachineController") + os.Exit(1) + } } diff --git a/test/dummies/v1beta3/vars.go b/test/dummies/v1beta3/vars.go index e633abf5..86df1f07 100644 --- a/test/dummies/v1beta3/vars.go +++ b/test/dummies/v1beta3/vars.go @@ -80,6 +80,7 @@ var ( // Declare exported dummy vars. LBRuleID string PublicIPID string EndPointHost string + SyncWithACS bool EndPointPort int32 CSConf *simpleyaml.Yaml DiskOffering infrav1.CloudStackResourceDiskOffering @@ -273,6 +274,7 @@ func SetDummyCAPCClusterVars() { CSClusterKind = "CloudStackCluster" ClusterName = "test-cluster" EndPointHost = "EndpointHost" + SyncWithACS = true EndPointPort = int32(5309) PublicIPID = "FakePublicIPID" ClusterNameSpace = "default" @@ -333,6 +335,7 @@ func SetDummyCAPCClusterVars() { Spec: infrav1.CloudStackClusterSpec{ ControlPlaneEndpoint: clusterv1.APIEndpoint{Host: EndPointHost, Port: EndPointPort}, FailureDomains: []infrav1.CloudStackFailureDomainSpec{CSFailureDomain1.Spec, CSFailureDomain2.Spec}, + SyncWithACS: SyncWithACS, }, Status: infrav1.CloudStackClusterStatus{}, }