Skip to content

Commit 597cd30

Browse files
ashishranjan738hakuna-matatah
ashishranjan738
authored and
hakuna-matatah
committed
Adds changes for using custom launch template
This commit adds changes for using custom launch template for worker nodes to PVRE and IMDSV2 issues. Signed-off-by: Ashish Ranjan <[email protected]>
1 parent 04e3a4e commit 597cd30

8 files changed

+214
-5
lines changed

‎tests/assets/eks_node_group_launch_template.json

+44
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"AWSTemplateFormatVersion": "2010-09-09",
3+
"Description": "Create an EKS Node Group Launch Template",
4+
"Parameters": {
5+
"LaunchTemplateName": {
6+
"Type": "String",
7+
"Description": "Name of the Launch Template"
8+
},
9+
"ClusterName": {
10+
"Type": "String",
11+
"Description": "Name of the Cluster"
12+
}
13+
},
14+
"Resources": {
15+
"NodeGroupLaunchTemplate": {
16+
"Type": "AWS::EC2::LaunchTemplate",
17+
"Properties": {
18+
"LaunchTemplateName": { "Ref": "LaunchTemplateName" },
19+
"LaunchTemplateData": {
20+
"BlockDeviceMappings": [
21+
{
22+
"DeviceName": "/dev/xvda",
23+
"Ebs": {
24+
"VolumeSize": 20,
25+
"VolumeType": "gp2"
26+
}
27+
}
28+
],
29+
"MetadataOptions": {
30+
"HttpPutResponseHopLimit": 2,
31+
"HttpEndpoint": "enabled",
32+
"HttpTokens": "required"
33+
}
34+
}
35+
}
36+
}
37+
},
38+
"Outputs": {
39+
"NodeGroupLaunchTemplateName": {
40+
"Description": "Name of the Node Group Launch Template",
41+
"Value": { "Ref": "NodeGroupLaunchTemplate" }
42+
}
43+
}
44+
}

‎tests/pipelines/eks/awscli-cl2-load-with-addons-slos.yaml

+24-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ spec:
1919
value: $(params.cluster-name)-service-role
2020
- name: node-role-stack-name
2121
value: $(params.cluster-name)-node-role
22+
- name: launch-template-stack-name
23+
value: $(params.cluster-name)-launch-template
2224
retries: 10
2325
taskRef:
2426
kind: Task
@@ -49,7 +51,7 @@ spec:
4951
default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/amazon-eks-vpc.json"
5052
type: string
5153
- name: ng-cfn-url
52-
default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_nodeGroup_launch_template.json"
54+
default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_group_launch_template.json"
5355
type: string
5456
- name: kubernetes-version
5557
type: string
@@ -127,6 +129,26 @@ spec:
127129
workspaces:
128130
- name: config
129131
workspace: config
132+
- name: create-launch-template
133+
params:
134+
- name: cluster-name
135+
value: $(params.cluster-name)
136+
- name: stack-name
137+
value: $(params.cluster-name)-launch-template
138+
- name: kubernetes-version
139+
value: "$(params.kubernetes-version)"
140+
- name: ng-cfn-url
141+
value: "$(params.ng-cfn-url)"
142+
- name: endpoint
143+
value: $(params.endpoint)
144+
runAfter:
145+
- create-eks-cluster
146+
taskRef:
147+
kind: Task
148+
name: awscli-eks-cfn-launch-template
149+
workspaces:
150+
- name: config
151+
workspace: config
130152
- name: create-mng-monitoring-nodes
131153
params:
132154
- name: cluster-name
@@ -146,7 +168,7 @@ spec:
146168
- name: nodegroup-prefix
147169
value: monitoring-
148170
runAfter:
149-
- create-eks-cluster
171+
- create-launch-template
150172
taskRef:
151173
kind: Task
152174
name: awscli-eks-nodegroup-create

‎tests/pipelines/eks/awscli-cl2-load-with-addons.yaml

+26-1
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@ spec:
1616
- name: slack-message
1717
- name: amp-workspace-id
1818
- name: vpc-cfn-url
19+
- name: ng-cfn-url
20+
default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_group_launch_template.json"
21+
type: string
1922
- name: kubernetes-version
2023
- name: service-role-cfn-url
2124
default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_service_role.json"
@@ -89,6 +92,26 @@ spec:
8992
workspaces:
9093
- name: config
9194
workspace: config
95+
- name: create-launch-template
96+
params:
97+
- name: cluster-name
98+
value: $(params.cluster-name)
99+
- name: stack-name
100+
value: $(params.cluster-name)-launch-template
101+
- name: kubernetes-version
102+
value: "$(params.kubernetes-version)"
103+
- name: ng-cfn-url
104+
value: "$(params.ng-cfn-url)"
105+
- name: endpoint
106+
value: $(params.endpoint)
107+
runAfter:
108+
- create-eks-cluster
109+
taskRef:
110+
kind: Task
111+
name: awscli-eks-cfn-launch-template
112+
workspaces:
113+
- name: config
114+
workspace: config
92115
- name: create-mng-monitoring-nodes
93116
params:
94117
- name: cluster-name
@@ -108,7 +131,7 @@ spec:
108131
- name: nodegroup-prefix
109132
value: "monitoring-"
110133
runAfter:
111-
- create-eks-cluster
134+
- create-launch-template
112135
taskRef:
113136
kind: Task
114137
name: awscli-eks-nodegroup-create
@@ -190,6 +213,8 @@ spec:
190213
value: $(params.cluster-name)-service-role
191214
- name: node-role-stack-name
192215
value: $(params.cluster-name)-node-role
216+
- name: launch-template-stack-name
217+
value: $(params.cluster-name)-launch-template
193218
taskRef:
194219
kind: Task
195220
name: awscli-eks-cluster-teardown

‎tests/pipelines/eks/awscli-eks-cl2-load.yaml

+26-1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ spec:
1818
- name: slack-hook
1919
- name: slack-message
2020
- name: vpc-cfn-url
21+
- name: ng-cfn-url
22+
default: "https://raw.githubusercontent.com/awslabs/kubernetes-iteration-toolkit/main/tests/assets/eks_node_group_launch_template.json"
23+
type: string
2124
- name: kubernetes-version
2225
default: "1.23"
2326
- name: amp-workspace-id
@@ -88,6 +91,26 @@ spec:
8891
workspaces:
8992
- name: config
9093
workspace: config
94+
- name: create-cfn-launch-template
95+
params:
96+
- name: cluster-name
97+
value: $(params.cluster-name)
98+
- name: stack-name
99+
value: $(params.cluster-name)-launch-template
100+
- name: kubernetes-version
101+
value: "$(params.kubernetes-version)"
102+
- name: ng-cfn-url
103+
value: "$(params.ng-cfn-url)"
104+
- name: endpoint
105+
value: $(params.endpoint)
106+
runAfter:
107+
- create-eks-cluster
108+
taskRef:
109+
kind: Task
110+
name: awscli-eks-cfn-launch-template
111+
workspaces:
112+
- name: config
113+
workspace: config
91114
- name: create-mng-monitoring-nodes
92115
params:
93116
- name: cluster-name
@@ -107,7 +130,7 @@ spec:
107130
- name: nodegroup-prefix
108131
value: "monitoring-"
109132
runAfter:
110-
- create-eks-cluster
133+
- create-cfn-launch-template
111134
taskRef:
112135
kind: Task
113136
name: awscli-eks-nodegroup-create
@@ -201,6 +224,8 @@ spec:
201224
value: $(params.cluster-name)-service-role
202225
- name: node-role-stack-name
203226
value: $(params.cluster-name)-node-role
227+
- name: launch-template-stack-name
228+
value: $(params.cluster-name)-launch-template
204229
taskRef:
205230
kind: Task
206231
name: awscli-eks-cluster-teardown

‎tests/tasks/setup/eks/awscli-cfn-lt.yaml

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
---
2+
apiVersion: tekton.dev/v1beta1
3+
kind: Task
4+
metadata:
5+
name: awscli-eks-cfn-launch-template
6+
namespace: scalability
7+
spec:
8+
description: |
9+
Create an EKS CFN stack to output a launch template.
10+
This Task can be used to create an EKS CFN stack that outputs a launch template.
11+
params:
12+
- name: cluster-name
13+
description: EKS cluster you want to create CFN stack for.
14+
- name: stack-name
15+
description: Stack name you want to spin.
16+
- name: region
17+
default: "us-west-2"
18+
description: The region where the cluster is in.
19+
- name: kubernetes-version
20+
default: "1.28"
21+
description: The EKS version to install.
22+
- name: ng-cfn-url
23+
description: The url of the CFN YAML/JSON to create CFN stack for NG launch template
24+
- name: endpoint
25+
default: ""
26+
workspaces:
27+
- name: config
28+
mountPath: /config/
29+
stepTemplate:
30+
env:
31+
- name: KUBECONFIG
32+
value: /config/kubeconfig
33+
steps:
34+
- name: create-launch-template
35+
image: alpine/k8s:1.23.7
36+
script: |
37+
set -x
38+
ENDPOINT_FLAG=""
39+
if [ -n "$(params.endpoint)" ]; then
40+
ENDPOINT_FLAG="--endpoint $(params.endpoint)"
41+
fi
42+
43+
curl -s $(params.ng-cfn-url) -o ./amazon-ng-cfn
44+
45+
launch_template_name=$(params.cluster-name)-launchTemplate
46+
STACK_NAME=$(params.stack-name)
47+
STACK_STATUS=$(aws cloudformation describe-stacks --query 'Stacks[?StackName==`'${STACK_NAME}'`].StackStatus' --output text --region $(params.region))
48+
49+
if [[ "$STACK_STATUS" == "" ]]; then
50+
aws cloudformation create-stack \
51+
--stack-name $STACK_NAME \
52+
--template-body file://$(pwd)/amazon-ng-cfn \
53+
--parameters ParameterKey=LaunchTemplateName,ParameterValue=$launch_template_name\
54+
ParameterKey=ClusterName,ParameterValue=$(params.cluster-name)\
55+
--region $(params.region)
56+
57+
aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region)
58+
echo "CREATED_CFN_STACK=$STACK_NAME"
59+
else
60+
echo "$STACK_NAME Already exists"
61+
fi

‎tests/tasks/setup/eks/awscli-cp-with-vpc.yaml

+27
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,33 @@ spec:
7878
# install csi drivers.
7979
kubectl apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=$(params.aws-ebs-csi-driver-version)"
8080
# TODO: Calculate replicas based on the cluster size going forward.
81+
# Patching the coredns not to get scheduled on the monitoring node.
82+
kubectl patch deployment coredns --patch '{
83+
"spec": {
84+
"template": {
85+
"spec": {
86+
"affinity": {
87+
"podAntiAffinity": {
88+
"requiredDuringSchedulingIgnoredDuringExecution": [
89+
{
90+
"labelSelector": {
91+
"matchExpressions": [
92+
{
93+
"key": "eks.amazonaws.com/nodegroup",
94+
"operator": "In",
95+
"values": ["monitoring-$(params.cluster-name)-nodes-1"]
96+
}
97+
]
98+
},
99+
"topologyKey": "kubernetes.io/hostname"
100+
}
101+
]
102+
}
103+
}
104+
}
105+
}
106+
}
107+
}' -n kube-system
81108
kubectl scale --replicas 1000 deploy coredns -n kube-system
82109
#ToDo - remove these comments after experimentation
83110
# Install EKS Pod Identity Agent

‎tests/tasks/setup/eks/awscli-mng.yaml

+4-1
Original file line numberDiff line numberDiff line change
@@ -69,15 +69,17 @@ spec:
6969
NG_SUBNETS=$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-cluster --name $(params.cluster-name) \
7070
--query cluster.resourcesVpcConfig.subnetIds --output text \
7171
)
72-
72+
7373
max_nodes=$(params.max-nodes)
7474
nodes=$(params.desired-nodes)
7575
asgs=$((nodes/max_nodes))
7676
echo "asgs: $asgs"
7777
node_group=$(params.nodegroup-prefix)$(params.cluster-name)-nodes
78+
7879
create_and_validate_dp_nodes()
7980
{
8081
node_group_name=$node_group-$1
82+
launch_template_name=$(params.cluster-name)-launchTemplate
8183
CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name) --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
8284
EC2_INSTANCES=$3
8385
if [ "$CREATED_NODEGROUP" == "" ]; then
@@ -86,6 +88,7 @@ spec:
8688
--cluster-name $(params.cluster-name) \
8789
--nodegroup-name $node_group_name \
8890
--node-role $NODE_ROLE_ARN \
91+
--launch-template name=$launch_template_name\
8992
--region $(params.region) \
9093
--instance-types $EC2_INSTANCES \
9194
--scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \

‎tests/tasks/teardown/awscli-eks.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ spec:
2222
default: "Job is completed"
2323
- name: service-role-stack-name
2424
- name: node-role-stack-name
25+
- name: launch-template-stack-name
2526
steps:
2627
- name: delete-cluster
2728
image: alpine/k8s:1.23.7
@@ -42,6 +43,7 @@ spec:
4243
script: |
4344
aws cloudformation delete-stack --stack-name $(params.service-role-stack-name)
4445
aws cloudformation delete-stack --stack-name $(params.node-role-stack-name)
46+
aws cloudformation delete-stack --stack-name $(params.launch-template-stack-name)
4547
- name: send-slack-notification
4648
image: alpine/k8s:1.23.7
4749
script: |

0 commit comments

Comments
 (0)