From 7c28a303c1e3d13a3c875ece5496d941f254b59a Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Fri, 31 Mar 2023 17:47:52 -0700 Subject: [PATCH] Load and run eks (#2) * Kitctl-Test.yml Github workflow that uses kitctl to bootstrap a test environment and deletes it . * Rename main.yml to Test-kitctl.yml * Update Test-kitctl.yml * Update Test-kitctl.yml * Update Test-kitctl.yml * Update Test-kitctl.yml * Update Test-kitctl.yml * Simple workflow files to dummy test * Update Kitctl.yml * Update Kitctl.yml * Update Kitctl.yml * Update Kitctl.yml * AB tests finished, no load yet * add changes * Delete action.yml * Delete First-workflow.yml * Delete Kitctl.yml * Delete depend.yml * Delete second-workflow.yml * Delete awscli-vpc-delete.yaml * Serial AB cluster setup with one load test." * parallel testing AB clusters * create addon tests * change default name * fix merge conflicts * fix spacing * remove idea * formatting * fix merges * reduce diff --------- Co-authored-by: Ganesh Putta Co-authored-by: Ganesh Putta --- operator/pkg/awsprovider/iam/reconciler.go | 1 + .../substrate/cluster/instanceprofile.go | 1 + .../create/create-test-cluster-run.yaml | 34 +++++ .../load/load-test-cluster-run.yaml | 48 +++++++ .../pipelines/create/create-test-cluster.yaml | 119 ++++++++++++++++++ .../pipelines/load/load-test-cluster.yaml | 86 +++++++++++++ tests/pipelineruns/eks/run.yaml | 45 +++++++ tests/pipelines/kitctl/run.yaml | 2 +- tests/pipelines/kitctl/template.yaml | 2 +- .../tasks/generators/clusterloader/load.yaml | 2 +- tests/tasks/setup/kitctl/controlplane.yaml | 2 + tests/tasks/setup/kitctl/dataplane.yaml | 2 +- tests/tasks/teardown/addon-test-cluster.yaml | 42 +++++++ tests/tasks/teardown/awscli-vpc-delete.yaml | 2 +- 14 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 tests/addon-tests/pipelineruns/create/create-test-cluster-run.yaml create mode 100644 tests/addon-tests/pipelineruns/load/load-test-cluster-run.yaml create mode 100644 tests/addon-tests/pipelines/create/create-test-cluster.yaml create mode 100644 tests/addon-tests/pipelines/load/load-test-cluster.yaml create mode 100644 tests/pipelineruns/eks/run.yaml create mode 100644 tests/tasks/teardown/addon-test-cluster.yaml diff --git a/operator/pkg/awsprovider/iam/reconciler.go b/operator/pkg/awsprovider/iam/reconciler.go index 8cdd72a2..71e8ef3a 100644 --- a/operator/pkg/awsprovider/iam/reconciler.go +++ b/operator/pkg/awsprovider/iam/reconciler.go @@ -46,6 +46,7 @@ var ( "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy", "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly", "arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess", + "arn:aws:iam::aws:policy/AmazonEC2FullAccess", } ) diff --git a/substrate/pkg/controller/substrate/cluster/instanceprofile.go b/substrate/pkg/controller/substrate/cluster/instanceprofile.go index 5a79e1e3..2faef10a 100644 --- a/substrate/pkg/controller/substrate/cluster/instanceprofile.go +++ b/substrate/pkg/controller/substrate/cluster/instanceprofile.go @@ -325,6 +325,7 @@ func desiredRolesFor(substrate *v1alpha1.Substrate) []role { "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy", "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy", "arn:aws:iam::aws:policy/AmazonPrometheusRemoteWriteAccess", + "arn:aws:iam::aws:policy/AmazonS3FullAccess", }, }} } diff --git a/tests/addon-tests/pipelineruns/create/create-test-cluster-run.yaml b/tests/addon-tests/pipelineruns/create/create-test-cluster-run.yaml new file mode 100644 index 00000000..1944b08e --- /dev/null +++ b/tests/addon-tests/pipelineruns/create/create-test-cluster-run.yaml @@ -0,0 +1,34 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: create-test-cluster-run + namespace: tekton-pipelines +spec: + workspaces: + - name: config + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + storageClassName: kit-gp2 + resources: + requests: + storage: 1Gi + params: + - name: cluster-name + value: addon-test-cluster-3 + - name: endpoint + value: "" + - name: desired-nodes + value: "50" + - name: vpc-cfn-url + value: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json" + - name: kubernetes-version + value: "1.25" + podTemplate: + nodeSelector: + kubernetes.io/arch: amd64 + serviceAccountName: tekton-pipelines-executor + pipelineRef: + name: create-test-cluster + timeout: "0" diff --git a/tests/addon-tests/pipelineruns/load/load-test-cluster-run.yaml b/tests/addon-tests/pipelineruns/load/load-test-cluster-run.yaml new file mode 100644 index 00000000..709ff6ea --- /dev/null +++ b/tests/addon-tests/pipelineruns/load/load-test-cluster-run.yaml @@ -0,0 +1,48 @@ +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: load-test-cluster-run + namespace: tekton-pipelines +spec: + workspaces: + - name: source + emptyDir: {} + - name: results + emptyDir: {} + - name: config + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + storageClassName: kit-gp2 + resources: + requests: + storage: 1Gi + params: + - name: cluster-name + value: addon-test-cluster-3 + - name: endpoint + value: "" + - name: desired-nodes + value: "50" + - name: vpc-cfn-url + value: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json" + - name: pods-per-node + value: "10" + - name: nodes-per-namespace + value: "10" + - name: cl2-load-test-throughput + value: "20" + - name: results-bucket + value: "" + - name: amp-workspace-id + value: "" + - name: kubernetes-version + value: "1.25" + podTemplate: + nodeSelector: + kubernetes.io/arch: amd64 + serviceAccountName: tekton-pipelines-executor + pipelineRef: + name: load-test-cluster + timeout: "0" diff --git a/tests/addon-tests/pipelines/create/create-test-cluster.yaml b/tests/addon-tests/pipelines/create/create-test-cluster.yaml new file mode 100644 index 00000000..1cf1d938 --- /dev/null +++ b/tests/addon-tests/pipelines/create/create-test-cluster.yaml @@ -0,0 +1,119 @@ +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: create-test-cluster + namespace: tekton-pipelines +spec: + description: | + This pipeline creates a cluster for testing new addons, webhooks, or controllers. + params: + - name: cluster-name + - name: endpoint + - name: desired-nodes + - name: vpc-cfn-url + - name: kubernetes-version + default: "1.25" + - name: service-role-cfn-url + default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_service_role.json" + - name: node-role-cfn-url + default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_role.json" + tasks: + - name: awscli-vpc-create + params: + - name: stack-name + value: $(params.cluster-name) + - name: vpc-cfn-url + value: "$(params.vpc-cfn-url)" + taskRef: + kind: Task + name: awscli-vpc-create + - name: create-cluster-service-role + params: + - name: stack-name + value: $(params.cluster-name)-service-role + - name: role-cfn-url + value: $(params.service-role-cfn-url) + - name: role-name + value: "$(params.cluster-name)-service-role" + taskRef: + kind: Task + name: awscli-role-create + - name: create-cluster-node-role + params: + - name: stack-name + value: $(params.cluster-name)-node-role + - name: role-cfn-url + value: $(params.node-role-cfn-url) + - name: role-name + value: "$(params.cluster-name)-node-role" + taskRef: + kind: Task + name: awscli-role-create + - name: create-eks-cluster + params: + - name: cluster-name + value: $(params.cluster-name) + - name: service-role-name + value: "$(params.cluster-name)-service-role" + - name: endpoint + value: $(params.endpoint) + - name: vpc-stack-name + value: $(params.cluster-name) + - name: kubernetes-version + value: "$(params.kubernetes-version)" + runAfter: + - create-cluster-node-role + - create-cluster-service-role + - awscli-vpc-create + taskRef: + kind: Task + name: awscli-eks-cluster-create-with-vpc-stack + workspaces: + - name: config + workspace: config + - name: create-mng-monitoring-nodes + params: + - name: cluster-name + value: $(params.cluster-name) + - name: host-cluster-node-role-name + value: "$(params.cluster-name)-node-role" + - name: endpoint + value: $(params.endpoint) + - name: desired-nodes + value: "1" + - name: max-nodes + value: "1" + - name: host-instance-types + value: "m5.4xlarge" + - name: host-taints + value: "key=monitoring,value=true,effect=NO_SCHEDULE" + - name: nodegroup-prefix + value: "monitoring-" + runAfter: + - create-eks-cluster + taskRef: + kind: Task + name: awscli-eks-nodegroup-create + workspaces: + - name: config + workspace: config + - name: create-mng-nodes + params: + - name: cluster-name + value: $(params.cluster-name) + - name: desired-nodes + value: $(params.desired-nodes) + - name: host-cluster-node-role-name + value: "$(params.cluster-name)-node-role" + - name: endpoint + value: $(params.endpoint) + runAfter: + - create-mng-monitoring-nodes + taskRef: + kind: Task + name: awscli-eks-nodegroup-create + workspaces: + - name: config + workspace: config + workspaces: + - name: config \ No newline at end of file diff --git a/tests/addon-tests/pipelines/load/load-test-cluster.yaml b/tests/addon-tests/pipelines/load/load-test-cluster.yaml new file mode 100644 index 00000000..c554abb5 --- /dev/null +++ b/tests/addon-tests/pipelines/load/load-test-cluster.yaml @@ -0,0 +1,86 @@ +apiVersion: tekton.dev/v1beta1 +kind: Pipeline +metadata: + name: load-test-cluster + namespace: tekton-pipelines +spec: + description: | + This pipeline sends slack notifcation before it spins up an EKS cluster with in it's own VPC + and runs cl2 loadtest and upload results to s3 and tearsdown the cluster and sends slack notification. + params: + - name: cluster-name + - name: endpoint + - name: desired-nodes + - name: pods-per-node + - name: nodes-per-namespace + - name: cl2-load-test-throughput + - name: results-bucket + - name: vpc-cfn-url + - name: kubernetes-version + default: "1.25" + - name: amp-workspace-id + tasks: + - name: create-eks-cluster + params: + - name: cluster-name + value: $(params.cluster-name) + - name: service-role-name + value: "$(params.cluster-name)-service-role" + - name: endpoint + value: $(params.endpoint) + - name: vpc-stack-name + value: $(params.cluster-name) + - name: kubernetes-version + value: "$(params.kubernetes-version)" + taskRef: + kind: Task + name: awscli-eks-cluster-create-with-vpc-stack + workspaces: + - name: config + workspace: config + - name: generate + params: + - name: pods-per-node + value: $(params.pods-per-node) + - name: nodes-per-namespace + value: $(params.nodes-per-namespace) + - name: cl2-load-test-throughput + value: $(params.cl2-load-test-throughput) + - name: results-bucket + value: $(params.results-bucket) + - name: nodes + value: $(params.desired-nodes) + - name: cluster-name + value: $(params.cluster-name) + - name: amp-workspace-id + value: '$(params.amp-workspace-id)' + runAfter: + - create-eks-cluster + taskRef: + kind: Task + name: load + workspaces: + - name: source + workspace: source + - name: results + workspace: results + - name: config + workspace: config + finally: + - name: teardown + params: + - name: cluster-name + value: $(params.cluster-name) + - name: endpoint + value: $(params.endpoint) + - name: service-role-stack-name + value: $(params.cluster-name)-service-role + - name: node-role-stack-name + value: $(params.cluster-name)-node-role + taskRef: + kind: Task + name: addon-test-cluster-teardown + workspaces: + - name: source + - name: results + - name: config \ No newline at end of file diff --git a/tests/pipelineruns/eks/run.yaml b/tests/pipelineruns/eks/run.yaml new file mode 100644 index 00000000..cfb1bd4d --- /dev/null +++ b/tests/pipelineruns/eks/run.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: PipelineRun +metadata: + name: awscli-eks-load-15 + namespace: tekton-pipelines +spec: + pipelineRef: + name: awscli-eks-cl2loadtest-with-addons + timeout: 9h0m0s + workspaces: + - name: source + emptyDir: {} + - name: config + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + storageClassName: gp2 + resources: + requests: + storage: 1Gi + - name: results + emptyDir: {} + params: + - name: cluster-name + value: "awscli-eks-load-15" + - name: desired-nodes + value: "15" + - name: pods-per-node + value: "10" + - name: nodes-per-namespace + value: "15" + - name: cl2-load-test-throughput + value: "20" + - name: results-bucket + value: "" + - name: vpc-cfn-url + value: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json" + - name: endpoint + value: "" + podTemplate: + nodeSelector: + kubernetes.io/arch: amd64 + serviceAccountName: tekton-pipelines-executor \ No newline at end of file diff --git a/tests/pipelines/kitctl/run.yaml b/tests/pipelines/kitctl/run.yaml index 89513605..0fb47a93 100644 --- a/tests/pipelines/kitctl/run.yaml +++ b/tests/pipelines/kitctl/run.yaml @@ -12,4 +12,4 @@ spec: kubernetes.io/arch: amd64 serviceAccountName: tekton-pipelines-executor pipelineRef: - name: pipeline-template + name: pipeline-template \ No newline at end of file diff --git a/tests/pipelines/kitctl/template.yaml b/tests/pipelines/kitctl/template.yaml index 6f8e8017..dfc9b92e 100644 --- a/tests/pipelines/kitctl/template.yaml +++ b/tests/pipelines/kitctl/template.yaml @@ -23,7 +23,7 @@ spec: params: - name: name value: '$(params.name)' - finally: + finally: - name: teardown taskRef: name: teardown diff --git a/tests/tasks/generators/clusterloader/load.yaml b/tests/tasks/generators/clusterloader/load.yaml index 755bb897..687cafa2 100644 --- a/tests/tasks/generators/clusterloader/load.yaml +++ b/tests/tasks/generators/clusterloader/load.yaml @@ -128,7 +128,7 @@ spec: fi # Building clusterloader2 binary cd $(workspaces.source.path)/perf-tests/clusterloader2/ - GOPROXY=direct GOOS=linux CGO_ENABLED=0 go build -v -o ./clusterloader ./cmd + GOOS=linux CGO_ENABLED=0 go build -v -o ./clusterloader ./cmd - name: run-loadtest image: alpine/k8s:1.23.7 onError: continue diff --git a/tests/tasks/setup/kitctl/controlplane.yaml b/tests/tasks/setup/kitctl/controlplane.yaml index 98d7a124..ef6d4d0c 100644 --- a/tests/tasks/setup/kitctl/controlplane.yaml +++ b/tests/tasks/setup/kitctl/controlplane.yaml @@ -129,5 +129,7 @@ spec: done echo "Installing CNI" kubectl --kubeconfig=/tmp/kubeconfig apply -f https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/release-1.10/config/master/aws-k8s-cni.yaml + echo "Installing CSI" + kubectl --kubeconfig=/tmp/kubeconfig apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-1.13" echo "Approving KCM requests" kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}') 2>/dev/null || true diff --git a/tests/tasks/setup/kitctl/dataplane.yaml b/tests/tasks/setup/kitctl/dataplane.yaml index cda0c129..95d0798c 100644 --- a/tests/tasks/setup/kitctl/dataplane.yaml +++ b/tests/tasks/setup/kitctl/dataplane.yaml @@ -47,5 +47,5 @@ spec: ready_node=$(kubectl --kubeconfig=/tmp/kubeconfig get nodes 2>/dev/null | grep -w Ready | wc -l) if [[ "$ready_node" -eq $(params.node-count) ]]; then break; fi sleep 5 - done + done kubectl --kubeconfig=/tmp/kubeconfig get nodes diff --git a/tests/tasks/teardown/addon-test-cluster.yaml b/tests/tasks/teardown/addon-test-cluster.yaml new file mode 100644 index 00000000..38551310 --- /dev/null +++ b/tests/tasks/teardown/addon-test-cluster.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: addon-test-cluster-teardown + namespace: tekton-pipelines +spec: + description: | + Teardown an EKS cluster. + This Task can be used to teardown an EKS cluster with mng in an AWS account. + params: + - name: cluster-name + description: The name of the EKS cluster which will be teared down. + - name: region + default: us-west-2 + description: The region where the cluster is in. + - name: endpoint + default: "" + - name: service-role-stack-name + - name: node-role-stack-name + steps: + - name: delete-cluster + image: alpine/k8s:1.22.6 + script: | + echo "Approving KCM requests" + kubectl certificate approve $(kubectl get csr | grep "Pending" | awk '{print $1}') 2>/dev/null || true + ENDPOINT_FLAG="" + if [ -n "$(params.endpoint)" ]; then + ENDPOINT_FLAG="--endpoint $(params.endpoint)" + fi + + for i in `aws eks list-nodegroups --cluster-name $(params.cluster-name) $ENDPOINT_FLAG --region $(params.region) | jq -r '.nodegroups[]'`; + do + aws eks delete-nodegroup --nodegroup-name $i --cluster-name $(params.cluster-name) $ENDPOINT_FLAG --region $(params.region); + aws eks wait nodegroup-deleted --nodegroup-name $i --cluster-name $(params.cluster-name) $ENDPOINT_FLAG --region $(params.region); + done; + aws eks delete-cluster --name $(params.cluster-name) --region $(params.region) $ENDPOINT_FLAG + - name: teardown-eks-role-stack + image: alpine/k8s:1.23.13 + script: | + aws cloudformation delete-stack --stack-name $(params.service-role-stack-name) --region $(params.region) + aws cloudformation delete-stack --stack-name $(params.node-role-stack-name) --region $(params.region) diff --git a/tests/tasks/teardown/awscli-vpc-delete.yaml b/tests/tasks/teardown/awscli-vpc-delete.yaml index ae2e9c7f..e3e167d8 100644 --- a/tests/tasks/teardown/awscli-vpc-delete.yaml +++ b/tests/tasks/teardown/awscli-vpc-delete.yaml @@ -29,4 +29,4 @@ spec: aws cloudformation delete-stack --region $(params.region) --stack-name $(params.stack-name) # Wait for the stack to be deleted aws cloudformation wait stack-delete-complete --region $(params.region) --stack-name $(params.stack-name) - echo "Stack deleted successfully!" + echo "Stack deleted successfully!" \ No newline at end of file