Add e2e test for project

Author: vishesh92

docs/book/src/getting-started.md

@@ -35,7 +35,6 @@
          secret-key: <cloudstackSecretKey>
          api-url: <cloudstackApiUrl>
          verify-ssl: "false"
-         project-id: <project id>  # Optional. If provided, the provider will use this project id for all operations
 
        ```
    - Apply this secret to the management cluster:

test/e2e/common.go

@@ -280,12 +280,16 @@ func DestroyOneMachine(client *cloudstack.CloudStackClient, clusterName string,
 }
 
 func CheckAffinityGroupsDeleted(client *cloudstack.CloudStackClient, affinityIds []string) error {
+	return CheckAffinityGroupsDeletedInProject(client, affinityIds, "")
+}
+
+func CheckAffinityGroupsDeletedInProject(client *cloudstack.CloudStackClient, affinityIds []string, project string) error {
 	if len(affinityIds) == 0 {
 		return errors.New("affinityIds are empty")
 	}
 
 	for _, affinityId := range affinityIds {
-		affinity, count, _ := client.AffinityGroup.GetAffinityGroupByID(affinityId)
+		affinity, count, _ := client.AffinityGroup.GetAffinityGroupByID(affinityId, cloudstack.WithProject(project))
 		if count > 0 {
 			return errors.New("Affinity group " + affinity.Name + " still exists")
 		}
@@ -312,9 +316,21 @@ func GetHostCount(client *cloudstack.CloudStackClient, zoneName string) int {
 }
 
 func CheckAffinityGroup(client *cloudstack.CloudStackClient, clusterName string, affinityType string) []string {
+	return CheckAffinityGroupInProject(client, clusterName, affinityType, "")
+}
+
+func CheckAffinityGroupInProject(client *cloudstack.CloudStackClient, clusterName string, affinityType string, project string) []string {
 	By("Listing all machines")
 	p := client.VirtualMachine.NewListVirtualMachinesParams()
 	p.SetListall(true)
+
+	if project != "" {
+		projectID, _, err := client.Project.GetProjectID(project)
+		if err != nil {
+			Fail("Failed to get project: " + err.Error())
+		}
+		p.SetProjectid(projectID)
+	}
 	listResp, err := client.VirtualMachine.ListVirtualMachines(p)
 	if err != nil {
 		Fail("Failed to list machines: " + err.Error())
@@ -334,7 +350,7 @@ func CheckAffinityGroup(client *cloudstack.CloudStackClient, clusterName string,
 
 			for _, affinity := range vm.Affinitygroup {
 				affinityIds = append(affinityIds, affinity.Id)
-				affinity, _, _ := client.AffinityGroup.GetAffinityGroupByID(affinity.Id)
+				affinity, _, _ := client.AffinityGroup.GetAffinityGroupByID(affinity.Id, cloudstack.WithProject(project))
 				if err != nil {
 					Fail("Failed to get affinity group for " + affinity.Id + " : " + err.Error())
 				}

test/e2e/config/cloudstack.yaml

@@ -94,6 +94,7 @@ providers:
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-disk-offering.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-custom-disk-offering.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-subdomain.yaml"
+      - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-project.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-invalid-ip.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-before.yaml"
       - sourcePath: "../data/infrastructure-cloudstack/v1beta3/cluster-template-kubernetes-version-upgrade-after.yaml"
@@ -124,6 +125,7 @@ variables:
   CLOUDSTACK_INVALID_ZONE_NAME: zoneXXXX
   CLOUDSTACK_INVALID_NETWORK_NAME: networkXXXX
   CLOUDSTACK_ACCOUNT_NAME: admin
+  CLOUDSTACK_PROJECT_NAME: capc-e2e-test
   CLOUDSTACK_INVALID_ACCOUNT_NAME: accountXXXX
   CLOUDSTACK_DOMAIN_NAME: ROOT
   CLOUDSTACK_INVALID_DOMAIN_NAME: domainXXXX

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-project/cloudstack-cluster.yaml

@@ -0,0 +1,18 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackCluster
+metadata:
+  name: ${CLUSTER_NAME}
+spec:
+  failureDomains:
+  - name: ${CLOUDSTACK_FD1_NAME}
+    acsEndpoint:
+      name: ${CLOUDSTACK_FD1_SECRET_NAME}
+      namespace: default
+    zone:
+      name :  ${CLOUDSTACK_ZONE_NAME}
+      network: 
+        name: ${CLOUDSTACK_NETWORK_NAME}
+    project: ${CLOUDSTACK_PROJECT_NAME}
+  controlPlaneEndpoint:
+    host: ""
+    port: 6443

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-project/kustomization.yaml

@@ -0,0 +1,7 @@
+bases:
+  - ../bases/cluster-with-kcp.yaml
+  - ../bases/md.yaml
+
+patchesStrategicMerge:
+- ./cloudstack-cluster.yaml
+- ./md.yaml

test/e2e/data/infrastructure-cloudstack/v1beta3/cluster-template-project/md.yaml

@@ -0,0 +1,28 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-control-plane
+spec:
+  template:
+    spec:
+      offering:
+        name: ${CLOUDSTACK_CONTROL_PLANE_MACHINE_OFFERING}
+      template:
+        name: ${CLOUDSTACK_TEMPLATE_NAME}
+      sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
+      affinity: pro
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta3
+kind: CloudStackMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-md-0
+spec:
+  template:
+    spec:
+      offering:
+        name: ${CLOUDSTACK_WORKER_MACHINE_OFFERING}
+      template:
+        name: ${CLOUDSTACK_TEMPLATE_NAME}
+      sshKey: ${CLOUDSTACK_SSH_KEY_NAME}
+      affinity: pro

test/e2e/project.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/pointer"
+
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// ProjectSpec implements a test that verifies that an app deployed to the workload cluster works.
+func ProjectSpec(ctx context.Context, inputGetter func() CommonSpecInput) {
+	var (
+		specName         = "project"
+		projectName      string
+		input            CommonSpecInput
+		namespace        *corev1.Namespace
+		cancelWatches    context.CancelFunc
+		clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
+		affinityIds      []string
+	)
+
+	BeforeEach(func() {
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
+		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
+		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
+		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
+
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
+		clusterResources = new(clusterctl.ApplyClusterTemplateAndWaitResult)
+
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+
+		projectName = os.Getenv("CLOUDSTACK_PROJECT_NAME")
+		csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
+		project, _, err := csClient.Project.GetProjectByName(projectName)
+		if (err != nil) || (project == nil) {
+			Skip("Failed to fetch project")
+		}
+	})
+
+	It("Should create a cluster in a project", func() {
+		clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
+			ClusterProxy:    input.BootstrapClusterProxy,
+			CNIManifestPath: input.E2EConfig.GetVariable(CNIPath),
+			ConfigCluster: clusterctl.ConfigClusterInput{
+				LogFolder:                filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()),
+				ClusterctlConfigPath:     input.ClusterctlConfigPath,
+				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+				InfrastructureProvider:   clusterctl.DefaultInfrastructureProvider,
+				Flavor:                   specName,
+				Namespace:                namespace.Name,
+				ClusterName:              fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
+				ControlPlaneMachineCount: pointer.Int64Ptr(1),
+				WorkerMachineCount:       pointer.Int64Ptr(1),
+			},
+			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+		}, clusterResources)
+
+		csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
+		affinityIds = CheckAffinityGroupInProject(csClient, clusterResources.Cluster.Name, "pro", projectName)
+	})
+
+	AfterEach(func() {
+		// Dumps all the resources in the spec namespace, then cleanups the cluster object and the spec namespace itself.
+		dumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, clusterResources.Cluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
+
+		csClient := CreateCloudStackClient(ctx, input.BootstrapClusterProxy.GetKubeconfigPath())
+		err := CheckAffinityGroupsDeletedInProject(csClient, affinityIds, projectName)
+		if err != nil {
+			Fail(err.Error())
+		}
+		By("PASSED!")
+	})
+}

test/e2e/project_test.go

@@ -0,0 +1,39 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("When testing project", func() {
+	ProjectSpec(context.TODO(), func() CommonSpecInput {
+		return CommonSpecInput{
+			E2EConfig:             e2eConfig,
+			ClusterctlConfigPath:  clusterctlConfigPath,
+			BootstrapClusterProxy: bootstrapClusterProxy,
+			ArtifactFolder:        artifactFolder,
+			SkipCleanup:           skipCleanup,
+		}
+	})
+
+})