Create test suite for EKS addons and components.

operator/pkg/awsprovider/iam/reconciler.go

@@ -46,6 +46,7 @@ var (
 		"arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
 		"arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
 		"arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess",
+		"arn:aws:iam::aws:policy/AmazonEC2FullAccess",
 	}
 )
 

substrate/pkg/controller/substrate/cluster/instanceprofile.go

@@ -325,6 +325,7 @@ func desiredRolesFor(substrate *v1alpha1.Substrate) []role {
 			"arn:aws:iam::aws:policy/AmazonEKSClusterPolicy",
 			"arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
 			"arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess",
+			"arn:aws:iam::aws:policy/AmazonS3FullAccess",
 		},
 	}}
 }

tests/addon-tests/pipelineruns/create/create-test-cluster-run.yaml

@@ -0,0 +1,34 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: create-test-cluster-run
+  namespace: tekton-pipelines
+spec:
+  workspaces:
+    - name: config
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+          - ReadWriteOnce
+          storageClassName: kit-gp2
+          resources:
+            requests:
+              storage: 1Gi
+  params:
+  - name: cluster-name
+    value: addon-test-cluster-3
+  - name: endpoint
+    value: ""
+  - name: desired-nodes
+    value: "50"
+  - name: vpc-cfn-url
+    value: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json"
+  - name: kubernetes-version
+    value: "1.25"
+  podTemplate:
+    nodeSelector:
+      kubernetes.io/arch: amd64
+  serviceAccountName: tekton-pipelines-executor
+  pipelineRef:
+    name: create-test-cluster
+  timeout: "0"

tests/addon-tests/pipelineruns/load/load-test-cluster-run.yaml

@@ -0,0 +1,48 @@
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: load-test-cluster-run
+  namespace: tekton-pipelines
+spec:
+  workspaces:
+    - name: source
+      emptyDir: {}
+    - name: results
+      emptyDir: {}
+    - name: config
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+          - ReadWriteOnce
+          storageClassName: kit-gp2
+          resources:
+            requests:
+              storage: 1Gi
+  params:
+  - name: cluster-name
+    value: addon-test-cluster-3
+  - name: endpoint
+    value: ""
+  - name: desired-nodes
+    value: "50"
+  - name: vpc-cfn-url
+    value: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json"
+  - name: pods-per-node
+    value: "10"
+  - name: nodes-per-namespace
+    value: "10"
+  - name: cl2-load-test-throughput
+    value: "20"
+  - name: results-bucket
+    value: ""
+  - name: amp-workspace-id
+    value: ""
+  - name: kubernetes-version
+    value: "1.25"
+  podTemplate:
+    nodeSelector:
+      kubernetes.io/arch: amd64
+  serviceAccountName: tekton-pipelines-executor
+  pipelineRef:
+    name: load-test-cluster
+  timeout: "0"

tests/addon-tests/pipelines/create/create-test-cluster.yaml

@@ -0,0 +1,119 @@
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: create-test-cluster
+  namespace: tekton-pipelines
+spec:
+  description: |
+    This pipeline creates a cluster for testing new addons, webhooks, or controllers.
+  params:
+  - name: cluster-name
+  - name: endpoint
+  - name: desired-nodes
+  - name: vpc-cfn-url
+  - name: kubernetes-version
+    default: "1.25"
+  - name: service-role-cfn-url
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_service_role.json"
+  - name: node-role-cfn-url
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_role.json"
+  tasks:
+  - name: awscli-vpc-create
+    params:
+    - name: stack-name
+      value: $(params.cluster-name)
+    - name: vpc-cfn-url
+      value: "$(params.vpc-cfn-url)"
+    taskRef:
+      kind: Task
+      name:  awscli-vpc-create
+  - name: create-cluster-service-role
+    params:
+    - name: stack-name
+      value: $(params.cluster-name)-service-role
+    - name: role-cfn-url
+      value: $(params.service-role-cfn-url)
+    - name: role-name
+      value: "$(params.cluster-name)-service-role"
+    taskRef:
+      kind: Task
+      name: awscli-role-create
+  - name: create-cluster-node-role
+    params:
+    - name: stack-name
+      value: $(params.cluster-name)-node-role
+    - name: role-cfn-url
+      value: $(params.node-role-cfn-url)
+    - name: role-name
+      value: "$(params.cluster-name)-node-role"
+    taskRef:
+      kind: Task
+      name: awscli-role-create
+  - name: create-eks-cluster
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: service-role-name
+      value: "$(params.cluster-name)-service-role"
+    - name: endpoint
+      value: $(params.endpoint)
+    - name: vpc-stack-name
+      value: $(params.cluster-name)
+    - name: kubernetes-version
+      value: "$(params.kubernetes-version)"
+    runAfter:
+    - create-cluster-node-role
+    - create-cluster-service-role
+    - awscli-vpc-create
+    taskRef:
+      kind: Task
+      name:  awscli-eks-cluster-create-with-vpc-stack
+    workspaces:
+    - name: config    
+      workspace: config
+  - name: create-mng-monitoring-nodes
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: host-cluster-node-role-name
+      value: "$(params.cluster-name)-node-role"
+    - name: endpoint
+      value: $(params.endpoint)
+    - name: desired-nodes
+      value: "1"
+    - name: max-nodes
+      value: "1"
+    - name: host-instance-types
+      value: "m5.4xlarge"
+    - name: host-taints
+      value: "key=monitoring,value=true,effect=NO_SCHEDULE"
+    - name: nodegroup-prefix
+      value: "monitoring-"
+    runAfter:
+    - create-eks-cluster
+    taskRef:
+      kind: Task
+      name:  awscli-eks-nodegroup-create
+    workspaces:
+    - name: config    
+      workspace: config
+  - name: create-mng-nodes
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: desired-nodes
+      value: $(params.desired-nodes)
+    - name: host-cluster-node-role-name
+      value: "$(params.cluster-name)-node-role"
+    - name: endpoint
+      value: $(params.endpoint)
+    runAfter:
+    - create-mng-monitoring-nodes
+    taskRef:
+      kind: Task
+      name:  awscli-eks-nodegroup-create
+    workspaces:
+    - name: config    
+      workspace: config
+  workspaces:
+  - name: config

tests/addon-tests/pipelines/load/load-test-cluster.yaml

@@ -0,0 +1,86 @@
+apiVersion: tekton.dev/v1beta1
+kind: Pipeline
+metadata:
+  name: load-test-cluster
+  namespace: tekton-pipelines
+spec:
+  description: |
+    This pipeline sends slack notifcation before it spins up an EKS cluster with in it's own VPC 
+    and runs cl2 loadtest and upload results to s3 and tearsdown the cluster and sends slack notification.
+  params:
+  - name: cluster-name
+  - name: endpoint
+  - name: desired-nodes
+  - name: pods-per-node
+  - name: nodes-per-namespace
+  - name: cl2-load-test-throughput
+  - name: results-bucket
+  - name: vpc-cfn-url
+  - name: kubernetes-version
+    default: "1.25"
+  - name: amp-workspace-id
+  tasks:
+  - name: create-eks-cluster
+    params:
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: service-role-name
+      value: "$(params.cluster-name)-service-role"
+    - name: endpoint
+      value: $(params.endpoint)
+    - name: vpc-stack-name
+      value: $(params.cluster-name)
+    - name: kubernetes-version
+      value: "$(params.kubernetes-version)"
+    taskRef:
+      kind: Task
+      name:  awscli-eks-cluster-create-with-vpc-stack
+    workspaces:
+    - name: config    
+      workspace: config
+  - name: generate
+    params:
+    - name: pods-per-node
+      value: $(params.pods-per-node)
+    - name: nodes-per-namespace
+      value: $(params.nodes-per-namespace)
+    - name: cl2-load-test-throughput
+      value: $(params.cl2-load-test-throughput)
+    - name: results-bucket
+      value: $(params.results-bucket)
+    - name: nodes
+      value: $(params.desired-nodes)
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: amp-workspace-id
+      value: '$(params.amp-workspace-id)'
+    runAfter:
+    - create-eks-cluster
+    taskRef:
+      kind: Task
+      name: load
+    workspaces:
+    - name: source
+      workspace: source
+    - name: results
+      workspace: results
+    - name: config    
+      workspace: config
+  finally:    
+  - name: teardown
+    params:   
+    - name: cluster-name
+      value: $(params.cluster-name)
+    - name: endpoint
+      value: $(params.endpoint)
+    - name: service-role-stack-name
+      value: $(params.cluster-name)-service-role
+    - name: node-role-stack-name
+      value: $(params.cluster-name)-node-role
+    taskRef:
+      kind: Task
+      name:  addon-test-cluster-teardown
+  workspaces:
+  - name: source
+  - name: results
+  - name: config

tests/pipelineruns/eks/run.yaml

@@ -0,0 +1,45 @@
+---
+apiVersion: tekton.dev/v1beta1
+kind: PipelineRun
+metadata:
+  name: awscli-eks-load-15
+  namespace: tekton-pipelines
+spec:
+  pipelineRef:
+    name: awscli-eks-cl2loadtest-with-addons
+  timeout: 9h0m0s
+  workspaces:
+    - name: source
+      emptyDir: {}
+    - name: config
+      volumeClaimTemplate:
+        spec:
+          accessModes:
+            - ReadWriteOnce
+          storageClassName: gp2
+          resources:
+            requests:
+              storage: 1Gi
+    - name: results
+      emptyDir: {}
+  params:
+    - name: cluster-name
+      value: "awscli-eks-load-15"
+    - name: desired-nodes
+      value: "15"
+    - name: pods-per-node
+      value: "10"
+    - name: nodes-per-namespace
+      value: "15"
+    - name: cl2-load-test-throughput
+      value: "20"
+    - name: results-bucket
+      value: ""
+    - name: vpc-cfn-url
+      value: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json"
+    - name: endpoint
+      value: ""
+  podTemplate:
+    nodeSelector:
+      kubernetes.io/arch: amd64
+  serviceAccountName: tekton-pipelines-executor

tests/pipelines/kitctl/run.yaml

@@ -12,4 +12,4 @@ spec:
       kubernetes.io/arch: amd64
   serviceAccountName: tekton-pipelines-executor
   pipelineRef:
-    name: pipeline-template
+    name: pipeline-template

tests/pipelines/kitctl/template.yaml

@@ -23,7 +23,7 @@ spec:
     params:
       - name: name
         value: '$(params.name)'
-  finally:      
+  finally:
   - name: teardown
     taskRef:
       name: teardown

tests/tasks/generators/clusterloader/load.yaml

@@ -128,7 +128,7 @@ spec:
       fi
       # Building clusterloader2 binary 
       cd $(workspaces.source.path)/perf-tests/clusterloader2/
-      GOPROXY=direct GOOS=linux CGO_ENABLED=0  go build -v -o ./clusterloader ./cmd
+      GOOS=linux CGO_ENABLED=0  go build -v -o ./clusterloader ./cmd
   - name: run-loadtest
     image: alpine/k8s:1.23.7
     onError: continue

tests/tasks/setup/kitctl/controlplane.yaml

@@ -129,5 +129,7 @@ spec:
       done
       echo "Installing CNI"
       kubectl --kubeconfig=/tmp/kubeconfig apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.10/config/master/aws-k8s-cni.yaml
+      echo "Installing CSI"
+      kubectl --kubeconfig=/tmp/kubeconfig apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-1.13"
       echo "Approving KCM requests"
       kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}')  2>/dev/null  || true

tests/tasks/setup/kitctl/dataplane.yaml

@@ -47,5 +47,5 @@ spec:
           ready_node=$(kubectl --kubeconfig=/tmp/kubeconfig get nodes 2>/dev/null | grep -w Ready | wc -l)
           if [[ "$ready_node" -eq $(params.node-count) ]]; then break; fi
           sleep 5
-      done 
+      done
       kubectl --kubeconfig=/tmp/kubeconfig get nodes