Bug fix ; Update heml repo;CP spec and DP spec; (#51)

* bug fix; add kit dp spec and cp spec for small cluster type.

Co-authored-by: Harish Kuna <[email protected]>

Author: hakuna-matatah

testbed/addons/kit/construct.ts

@@ -77,7 +77,7 @@ export class Kit extends cdk.Construct {
         const chart = props.cluster.addHelmChart('kit', {
             chart: 'kit-operator',
             release: 'kit-operator',
-            repository: 'https://github.com/awslabs/kubernetes-iteration-toolkit/tree/main/operator/charts/kit-operator',
+            repository: 'https://awslabs.github.io/kubernetes-iteration-toolkit/',
             namespace: namespace,
             createNamespace: false,
             values: {

tests/pipelines/kit/upstream-load.yaml

@@ -17,10 +17,27 @@ spec:
   - name: guest-cluster-region
     default: "us-west-2"
     description: The region where the kit cluster is created.
-  - name: control-plane-spec
-    description:  url to the controlplane spec.
-  - name: dataplane-spec
-    description: url to the dataplane spec.
+  - name: version
+    default: "1.21"
+    description:  kubernetes version.
+  - name: cp-instance-type
+    default: "m5.8xlarge"
+    description: control plane instance type.
+  - name: max-requests-inflight
+    default: "400"
+    description: maximum number of inflight read request that apiserver could allow
+  - name: max-mutating-requests
+    default: "200"
+    description: maximum number of mutating requests in flight that apiserver could allow
+  - name: kcm-qps
+    default: "20"
+    description: Kubernetes-Controller-Manager QPS setting
+  - name: scheduler-qps
+    default: "20"
+    description: Kubernetes-Scheduler QPS setting
+  - name: node-count
+    default: "1000"
+    description: desired node count for Dataplane.
   - name: giturl
     description: "git url to clone the package"
     default: https://github.com/kubernetes/perf-tests.git
@@ -35,8 +52,6 @@ spec:
     default: "15"
   - name: results-bucket
     description: "Results bucket with path of s3 to upload results"
-  - name: desired-nodes
-    description: The desired number of nodes in the cluster which we define in DataPlane spec.
   tasks:
   - name: create-kit-cluster
     taskRef:
@@ -48,10 +63,20 @@ spec:
         value: '$(params.host-cluster-region)'
       - name: guest-cluster-region
         value: '$(params.guest-cluster-region)'
-      - name: control-plane-spec
-        value: '$(params.control-plane-spec)'
-      - name: dataplane-spec
-        value: '$(params.dataplane-spec)'
+      - name: version
+        value: '$(params.version)'    
+      - name: cp-instance-type
+        value: '$(params.cp-instance-type)' 
+      - name: max-requests-inflight
+        value: '$(params.max-requests-inflight)' 
+      - name: max-mutating-requests
+        value: '$(params.max-mutating-requests)' 
+      - name: kcm-qps
+        value: '$(params.kcm-qps)' 
+      - name: scheduler-qps
+        value: '$(params.scheduler-qps)'
+      - name: node-count
+        value: '$(params.node-count)'      
     workspaces:
       - name: config    
         workspace: config
@@ -71,7 +96,7 @@ spec:
       - name: results-bucket
         value: '$(params.results-bucket)'
       - name: nodes
-        value: '$(params.desired-nodes)'
+        value: '$(params.node-count)'
     workspaces:
       - name: source
         workspace: source

tests/tasks/setup/kit-cluster/default.yaml

@@ -17,10 +17,27 @@ spec:
   - name: guest-cluster-region
     default: "us-west-2"
     description: The region where the kit cluster is created.
-  - name: control-plane-spec
-    description:  url to the controlplane spec.
-  - name: dataplane-spec
-    description: url to the dataplane spec.
+  - name: version
+    default: "1.21"
+    description:  kubernetes version.
+  - name: cp-instance-type
+    default: "m5.8xlarge"
+    description: control plane instance type.
+  - name: max-requests-inflight
+    default: "400"
+    description: maximum number of inflight read request that apiserver could allow
+  - name: max-mutating-requests
+    default: "200"
+    description: maximum number of mutating requests in flight that apiserver could allow
+  - name: kcm-qps
+    default: "20"
+    description: Kubernetes-Controller-Manager QPS setting
+  - name: scheduler-qps
+    default: "20"
+    description: Kubernetes-Scheduler QPS setting
+  - name: node-count
+    default: "1000"
+    description: desired node count for Dataplane.
   workspaces:
   - name: config
     description: |
@@ -35,9 +52,51 @@ spec:
       aws eks update-kubeconfig --name testbed --region $(params.host-cluster-region)
       kubectl config current-context
       #Provision a control plane for the guest cluster
-      kubectl create -f $(params.control-plane-spec)
+      cat > "$(workspaces.config.path)/cp.yaml" <<EOL
+      apiVersion: kit.k8s.sh/v1alpha1
+      kind: ControlPlane
+      metadata:
+        name: $(params.cluster-name) # Desired Cluster Name
+      spec:
+        kubernetesVersion: $(params.version)
+        master:
+          apiServer:
+            spec:
+              nodeSelector: 
+                node.kubernetes.io/instance-type: $(params.cp-instance-type)
+              containers: 
+              - name: apiserver
+                args: 
+                  - --max-requests-inflight=$(params.max-requests-inflight)
+                  - --max-mutating-requests-inflight=$(params.max-mutating-requests)
+          controllerManager:
+            spec:
+              containers:
+              - name: controller-manager
+                args:
+                  - --kube-api-qps=$(params.kcm-qps)
+          scheduler:
+            spec:
+              containers:
+              - name: scheduler
+                args:
+                  - --kube-api-qps=$(params.scheduler-qps)
+      EOL
+      kubectl create -f $(workspaces.config.path)/cp.yaml
+
+      cat > "$(workspaces.config.path)/dp.yaml" <<EOL
+      apiVersion: kit.k8s.sh/v1alpha1
+      kind: DataPlane
+      metadata:
+        name: $(params.cluster-name)-nodes 
+      spec: 
+        clusterName: $(params.cluster-name) # Desired Cluster Name
+        nodeCount: $(params.node-count)
+      EOL
+
       #Provision a dataplane plane for the guest cluster
-      kubectl create -f $(params.dataplane-spec)
+      kubectl create -f $(workspaces.config.path)/dp.yaml
+
       #Get the admin KUBECONFIG for the guest cluster from the substrate cluster
       kubectl get secret $(params.cluster-name)-kube-admin-config -ojsonpath='{.data.config}' | base64 -d > $(workspaces.config.path)/kubeconfig
       #Deploy CNI plugin to the guest cluster for the nodes to be ready.

tests/tasks/teardown/kit.yaml

@@ -9,13 +9,16 @@ spec:
     This Task can be used to teardown an KIT cluster in an AWS account.
   params:
   - name: cluster-name
-    description: The name of the EKS cluster which will be teared down.
+    description: The name of the kit cluster which will be teared down.
+  - name: region
+    default: us-west-2
+    description: The region where the EKS/host cluster is in.
   steps:
   - name: delete-cluster
     image: amazon/aws-cli
     script: |
       kubectl version
-      aws eks update-kubeconfig --name testbed --region $(params.host-cluster-region)
+      aws eks update-kubeconfig --name testbed --region $(params.region)
       kubectl config current-context 
       #delete kit controlplane spec and dataplane spec crds
       kubectl delete ControlPlane --name $(params.cluster-name)

tests/triggers/kit/small/upstream-load.yaml

@@ -56,6 +56,6 @@ spec:
         emptyDir: {}
       params:
       - name: cluster-name
-        value: "kit-small"
+        value: "kit-small-upstream-load-$(context.pipelineRun.uid)"
       - name: results-bucket
         value: "kit-scalability-dev/$(date +'%Y/%m/%d/%HH/%mm)/$(context.pipelineRun.uid)"

tests/triggers/kustomization.yaml

@@ -1,6 +1,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-- triggers/eks/small/upstream-load.yaml
-- triggers/rbac.yaml
-- triggers/kit/small/upstream-load.yaml
+- eks/small/upstream-load.yaml
+- rbac.yaml
+- kit/small/upstream-load.yaml