Add neuron-device-plugin load test to the test bed

tests/assets/neuron/config.yaml

@@ -0,0 +1,65 @@
+{{$uniformQps := DefaultParam .CL2_UNIFORM_QPS 500}}
+{{$neuronResourcesPerPod := DefaultParam .CL2_NEURON_RESOURCES_PER_POD 64}}
+{{$neuronPods := DefaultParam .CL2_NEURON_PODS .Nodes}}
+
+name: neuron-workers
+namespace:
+  number: 1
+tuningSets:
+- name: UniformQPS
+  qpsLoad:
+    qps: {{$uniformQps}}
+
+steps:
+- name: Start measurements
+  measurements:
+  - Identifier: PodStartupLatency
+    Method: PodStartupLatency
+    Params:
+      action: start
+      labelSelector: group = neuron-worker
+      threshold: 25s
+- name: Create pods
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 1
+    replicasPerNamespace: {{$neuronPods}}
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: neuron-worker
+      objectTemplatePath: pod.yaml
+      templateFillMap:
+        Group: neuron-worker
+        NeuronResources: {{$neuronResourcesPerPod}}
+
+- name: Wait for pods to be running
+  measurements:
+  - Identifier: WaitForRunningPods
+    Method: WaitForRunningPods
+    Params:
+      action: gather
+      desiredPodCount: {{$neuronPods}}
+      labelSelector: group = neuron-worker
+      timeout: 5m
+
+- name: Measure pod startup latency
+  measurements:
+  - Identifier: PodStartupLatency
+    Method: PodStartupLatency
+    Params:
+      action: gather
+
+- name: Delete pods
+  phases:
+  - namespaceRange:
+      min: 1
+      max: 1
+    replicasPerNamespace: 0
+    tuningSet: UniformQPS
+    objectBundle:
+    - basename: neuron-worker
+      objectTemplatePath: pod.yaml
+      templateFillMap:
+        Group: neuron-worker
+        NeuronResources: {{$neuronResourcesPerPod}}

tests/assets/neuron/pod.yaml

@@ -0,0 +1,19 @@
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: neuron-worker-
+  labels:
+    group: {{.Group}}
+spec:
+  containers:
+  - name: main
+    image: public.ecr.aws/amazonlinux/amazonlinux:2023
+    command:
+    - "sleep"
+    - "infinity"
+    resources:
+      requests:
+        aws.amazon.com/neuron: "{{.NeuronResources}}"
+      limits:
+        aws.amazon.com/neuron: "{{.NeuronResources}}"

tests/tekton-resources/pipelines/eks/awscli-cl2-load-with-addons-slos.yaml

@@ -88,6 +88,16 @@ spec:
     default: "20m"
   - name: timeout-pia-pod-startup
     default: "5m"
+  - name: neuron-test-config-url
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/neuron/config.yaml"
+  - name: neuron-test-pod-spec-url
+    default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/neuron/pod.yaml"
+  - name: cl2-neuron-uniform-qps
+    description: "Rate at which pods are created/deleted. Defaults to 500 QPS."
+    default: "500"
+  - name: cl2-neuron-resources-per-pod
+    description: "Neuron device units requested per pod. Defaults to 64 units."
+    default: "64"
   - name: unmanaged-nodegroup-cfn-url
     default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/asg_node_group.yaml"
   - name: launch-template-ami
@@ -261,6 +271,50 @@ spec:
     workspaces:
     - name: config
       workspace: config
+  - name: install-neuron-device-plugin
+    params:
+      - name: cluster-name
+        value: $(params.cluster-name)
+      - name: endpoint
+        value: $(params.endpoint)
+    runAfter: 
+    - create-mng-nodes
+    taskRef:
+      kind: Task
+      name: install-neuron-device-plugin
+    workspaces:
+      - name: config
+        workspace: config
+  - name: generate-neuron-load
+    params:
+      - name: cluster-name
+        value: $(params.cluster-name)
+      - name: results-bucket
+        value: $(params.results-bucket)
+      - name: nodes
+        value: $(params.desired-nodes)
+      - name: cl2-neuron-pods
+        value: $(params.desired-nodes)
+      - name: cl2-uniform-qps
+        value: $(params.cl2-neuron-uniform-qps)
+      - name: cl2-neuron-resources-per-pod
+        value: $(params.cl2-neuron-resources-per-pod)
+      - name: neuron-config-url
+        value: $(params.neuron-test-config-url) 
+      - name: neuron-pod-url
+        value: $(params.neuron-test-pod-spec-url)
+    runAfter:
+    - install-neuron-device-plugin
+    taskRef:
+      kind: Task
+      name: load-neuron-device-plugin
+    workspaces:
+      - name: source
+        workspace: source
+      - name: results
+        workspace: results
+      - name: config
+        workspace: config
   - name: create-pod-identity-association
     params:
     - name: cluster-name
@@ -274,7 +328,7 @@ spec:
     - name: pia-trust-policy-url
       value: $(params.pia-trust-policy-url)
     runAfter:
-    - create-mng-nodes
+    - generate-neuron-load
     taskRef:
       kind: Task
       name:  awscli-eks-pia-create
@@ -377,6 +431,72 @@ spec:
     taskRef:
       kind: Task
       name: cloudwatch
+  - name: cw-metrics-neuron-device-plugin-latency-p50
+    params:
+    - name: dimensions
+      value: $(params.desired-nodes)
+    - name: value
+      value: $(tasks.generate-neuron-load.results.pod_startup_p50)
+    - name: namespace
+      value: neuron-device-plugin-$(params.kubernetes-version)
+    - name: metric-name
+      value: pod_startup_latency_p50 
+    - name: unit
+      value: Milliseconds
+    runAfter:
+    - generate-neuron-load
+    taskRef:
+      kind: Task
+      name: cloudwatch
+  - name: cw-metrics-neuron-device-plugin-latency-p90
+    params:
+    - name: dimensions
+      value: $(params.desired-nodes)
+    - name: value
+      value: $(tasks.generate-neuron-load.results.pod_startup_p90)
+    - name: namespace
+      value: neuron-device-plugin-$(params.kubernetes-version)
+    - name: metric-name
+      value: pod_startup_latency_p90
+    - name: unit
+      value: Milliseconds
+    runAfter:
+    - generate-neuron-load
+    taskRef:
+      kind: Task
+      name: cloudwatch
+  - name: cw-metrics-neuron-device-plugin-latency-p99
+    params:
+    - name: dimensions
+      value: $(params.desired-nodes)
+    - name: value
+      value: $(tasks.generate-neuron-load.results.pod_startup_p99)
+    - name: namespace
+      value: neuron-device-plugin-$(params.kubernetes-version)
+    - name: metric-name
+      value: pod_startup_latency_p99
+    - name: unit
+      value: Milliseconds
+    runAfter:
+    - generate-neuron-load
+    taskRef:
+      kind: Task
+      name: cloudwatch
+  - name: cw-metrics-neuron-load-test-outcome
+    params:
+    - name: dimensions
+      value: $(params.desired-nodes)
+    - name: value
+      value: $(tasks.generate-neuron-load.results.datapoint)
+    - name: metric-name
+      value: outcome
+    - name: namespace
+      value: neuron-device-plugin-$(params.kubernetes-version)
+    runAfter:
+    - generate-neuron-load
+    taskRef:
+      kind: Task
+      name: cloudwatch
   workspaces:
   - name: source
   - name: results