Skip to content

Commit f9be768

Browse files
Add unmanaged nodegroup support
1 parent cb3ee5c commit f9be768

File tree

4 files changed

+150
-31
lines changed

4 files changed

+150
-31
lines changed

tests/assets/asg_node_group.yaml

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
---
2+
AWSTemplateFormatVersion: '2010-09-09'
3+
Description: 'Unmanaged EKS nodegroup using EC2 AutoScaling'
4+
Parameters:
5+
ClusterName:
6+
Type: String
7+
Description: Name of EKS cluster.
8+
AutoScalingGroupName:
9+
Description: Name of ASG.
10+
Type: String
11+
VpcId:
12+
Type: AWS::EC2::VPC::Id
13+
SubnetIds:
14+
Type: List<AWS::EC2::Subnet::Id>
15+
SecurityGroup:
16+
Type: AWS::EC2::SecurityGroup::Id
17+
LaunchTemplateName:
18+
Type: String
19+
Description: Launch template name.
20+
LaunchTemplateVersion:
21+
Type: String
22+
Description: Launch template version.
23+
Default: "$Default"
24+
NodeCount:
25+
Type: Number
26+
Resources:
27+
AutoScalingGroup:
28+
Type: AWS::AutoScaling::AutoScalingGroup
29+
UpdatePolicy:
30+
AutoScalingRollingUpdate:
31+
WaitOnResourceSignals: true
32+
PauseTime: PT15M
33+
Properties:
34+
AutoScalingGroupName: !Ref AutoScalingGroupName
35+
DesiredCapacity: !Ref NodeCount
36+
MinSize: !Ref NodeCount
37+
MaxSize: !Ref NodeCount
38+
MixedInstancesPolicy:
39+
LaunchTemplate:
40+
LaunchTemplateSpecification:
41+
LaunchTemplateName: !Ref LaunchTemplateName
42+
Version: "1"
43+
Overrides: PLACEHOLDER_LAUNCH_TEMPLATE_OVERRIDES
44+
VPCZoneIdentifier:
45+
!Ref SubnetIds
46+
Tags:
47+
# necessary for kubelet's legacy, in-tree cloud provider
48+
- Key: !Sub kubernetes.io/cluster/${ClusterName}
49+
Value: owned
50+
PropagateAtLaunch: true

tests/assets/eks_node_group_launch_template_al2023.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,22 @@ Parameters:
2929
Type: String
3030
Description: Launch template ImageId value, which may be an AMI ID or resolve:ssm reference.
3131
Default: ''
32+
NodeRoleName:
33+
Type: String
34+
Description: Name of the IAM Role for the node instances.
35+
SecurityGroup:
36+
Type: AWS::EC2::SecurityGroup::Id
37+
Description: EKS-created cluster security group that allows node communication with the control plane.
3238
Conditions:
3339
AMIProvided:
3440
!Not [!Equals [!Ref AMI, '']]
3541
Resources:
42+
NodeInstanceProfile:
43+
Type: AWS::IAM::InstanceProfile
44+
Properties:
45+
Path: "/"
46+
Roles:
47+
- !Ref NodeRoleName
3648
LaunchTemplate:
3749
Type: AWS::EC2::LaunchTemplate
3850
Properties:
@@ -50,6 +62,10 @@ Resources:
5062
HttpPutResponseHopLimit: 2
5163
HttpEndpoint: enabled
5264
HttpTokens: required
65+
IamInstanceProfile:
66+
Arn: !GetAtt NodeInstanceProfile.Arn
67+
SecurityGroupIds:
68+
- !Ref SecurityGroup
5369
ImageId:
5470
!If
5571
- AMIProvided

tests/tasks/setup/eks/awscli-cfn-lt-al2023.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ spec:
3030
- name: ami
3131
default: ""
3232
description: The AMI ID (or SSM parameter) to use for the launch template. If not provided, the launch template will not specify an AMI.
33+
- name: node-role-name
34+
description: The name of the IAM role to use for the node's instance profile, specified in the launch template.
3335
workspaces:
3436
- name: config
3537
mountPath: /config/
@@ -80,6 +82,8 @@ spec:
8082
--arg CertificateAuthority "$(jq -r .cluster.certificateAuthority.data cluster.json)" \
8183
--arg KubeletConfig '$(params.kubelet-config)' \
8284
--arg AMI "$(params.ami)" \
85+
--arg SecurityGroup "$(jq -r .cluster.resourcesVpcConfig.clusterSecurityGroupId cluster.json)" \
86+
--arg NodeRoleName '$(params.node-role-name)' \
8387
'$ARGS.named | to_entries | map({"ParameterKey": .key, "ParameterValue": .value})' \
8488
> parameters.json
8589
@@ -88,6 +92,7 @@ spec:
8892
--stack-name $STACK_NAME \
8993
--template-body file://$(pwd)/amazon-ng-cfn \
9094
--parameters file://$(pwd)/parameters.json \
95+
--capabilities CAPABILITY_IAM \
9196
--region $(params.region)
9297
9398
aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region)

tests/tasks/setup/eks/awscli-mng.yaml

Lines changed: 79 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,8 @@ metadata:
66
namespace: scalability
77
spec:
88
description: |
9-
Create an EKS managed nodegroup for a given cluster.
10-
This Task can be used to create an EKS managed nodegroup for a given VPC Subnets, security groups and service role in an AWS account.
9+
Create an EKS nodegroup, managed or unamaged, for a given cluster.
10+
This Task can be used to create an EKS managed or unmanaged nodegroup for a given VPC Subnets, security groups and service role in an AWS account.
1111
params:
1212
- name: cluster-name
1313
description: The name of the EKS cluster you want to spin managed nodegroups for.
@@ -36,6 +36,12 @@ spec:
3636
- name: nodegroup-prefix
3737
description: Prefix that needs to be appended to asg names.
3838
default: ""
39+
- name: unmanaged-nodegroup-cfn-url
40+
default: ""
41+
description: URL for "unmanaged nodegroup" (AutoScaling group) CloudFormation template. If not specified, a managed nodegroup will be created.
42+
- name: launch-template-name
43+
default: "$(params.cluster-name)-launchTemplate"
44+
description: Name of the launch template to be used for the nodegroup.
3945
workspaces:
4046
- name: config
4147
mountPath: /config/
@@ -47,6 +53,11 @@ spec:
4753
- name: create-nodegroup
4854
image: alpine/k8s:1.23.7
4955
script: |
56+
set -o xtrace
57+
set -o errexit
58+
set -o pipefail
59+
set -o nounset
60+
5061
ENDPOINT_FLAG=""
5162
5263
NODE_ROLE_NAME=$(params.host-cluster-node-role-name)
@@ -60,9 +71,8 @@ spec:
6071
TAINTS_FLAG="--taints $(params.host-taints)"
6172
fi
6273
63-
NG_SUBNETS=$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-cluster --name $(params.cluster-name) \
64-
--query cluster.resourcesVpcConfig.subnetIds --output text \
65-
)
74+
aws eks $ENDPOINT_FLAG --region $(params.region) describe-cluster --name $(params.cluster-name) --output json > cluster.json
75+
NG_SUBNETS=$(jq -r '.cluster.resourcesVpcConfig.subnetIds | join(" ")' cluster.json)
6676
6777
max_nodes=$(params.max-nodes)
6878
nodes=$(params.desired-nodes)
@@ -72,33 +82,71 @@ spec:
7282
create_and_validate_dp_nodes()
7383
{
7484
node_group_name=$node_group-$1
75-
launch_template_name=$(params.cluster-name)-launchTemplate
76-
CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name) --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
7785
EC2_INSTANCES=$3
78-
if [ "$CREATED_NODEGROUP" == "" ]; then
79-
#create node group
80-
aws eks $ENDPOINT_FLAG create-nodegroup \
81-
--cluster-name $(params.cluster-name) \
82-
--nodegroup-name $node_group_name \
83-
--node-role $NODE_ROLE_ARN \
84-
--launch-template name=$launch_template_name\
85-
--region $(params.region) \
86-
--instance-types $EC2_INSTANCES \
87-
--scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
88-
--subnets $NG_SUBNETS $TAINTS_FLAG
86+
# if no unmanaged nodegroup cfn template is provided, assume we want managed nodegroups
87+
if [ "$(params.unmanaged-nodegroup-cfn-url)" = "" ]; then
88+
CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name) --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
89+
if [ "$CREATED_NODEGROUP" == "" ]; then
90+
aws eks $ENDPOINT_FLAG create-nodegroup \
91+
--cluster-name $(params.cluster-name) \
92+
--nodegroup-name $node_group_name \
93+
--node-role $NODE_ROLE_ARN \
94+
--launch-template name=$(params.launch-template-name) \
95+
--region $(params.region) \
96+
--instance-types $EC2_INSTANCES \
97+
--scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
98+
--subnets $NG_SUBNETS $TAINTS_FLAG
99+
fi
100+
echo "CREATED_NODEGROUP=$node_group_name"
101+
while [[ "$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-nodegroup --cluster-name $(params.cluster-name) --nodegroup-name $node_group_name --query nodegroup.status --output text)" == "CREATING" ]]
102+
do
103+
echo "$node_group_name is "CREATING" at $(date)"
104+
sleep 2
105+
done
106+
# TODO: do this for unmanaged nodes as well
107+
# right now we don't have an appropriate label to filter on for unmanaged nodes
108+
while true; do
109+
ready_node=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$node_group_name --no-headers 2>/dev/null | grep -w Ready | wc -l)
110+
echo "ready-nodes=$ready_node out of $2, for nodegroup: $node_group_name"
111+
if [[ "$ready_node" -eq $2 ]]; then break; fi
112+
sleep 5
113+
done
114+
else
115+
STACK_NAME=$node_group_name
116+
STACK_STATUS=$(aws cloudformation describe-stacks --query 'Stacks[?StackName==`'${STACK_NAME}'`].StackStatus' --output text --region $(params.region))
117+
if [[ "$STACK_STATUS" == "" ]]; then
118+
curl -s $(params.unmanaged-nodegroup-cfn-url) -o ./cfn-template
119+
120+
# assemble the stack parameters as a JSON file
121+
# the AWS CLI can't handle a JSON string as a ParameterValue in the flag representation
122+
# and we need that for kubelet-config
123+
jq --null-input \
124+
--arg LaunchTemplateName "$(params.launch-template-name)" \
125+
--arg ClusterName "$(params.cluster-name)" \
126+
--arg AutoScalingGroupName "${node_group_name}" \
127+
--arg NodeCount "$2" \
128+
--arg SubnetIds $(jq -r '.cluster.resourcesVpcConfig.subnetIds | join(",")' cluster.json) \
129+
--arg SecurityGroup "$(jq -r '.cluster.resourcesVpcConfig.clusterSecurityGroupId' cluster.json)" \
130+
--arg VpcId $(jq -r '.cluster.resourcesVpcConfig.vpcId' cluster.json) \
131+
'$ARGS.named | to_entries | map({"ParameterKey": .key, "ParameterValue": .value})' \
132+
> parameters.json
133+
134+
# cloudformation really fights you every step of the way to pass JSON in, so let's just hack it
135+
LAUNCH_TEMPLATE_OVERRIDES=$(echo "$EC2_INSTANCES" | jq -R -c 'split(" ") | map({"InstanceType": .})')
136+
sed -i "s/PLACEHOLDER_LAUNCH_TEMPLATE_OVERRIDES/$LAUNCH_TEMPLATE_OVERRIDES/g" cfn-template
137+
138+
aws cloudformation create-stack \
139+
--region $(params.region) \
140+
--stack-name $STACK_NAME \
141+
--template-body file://$(pwd)/cfn-template \
142+
--parameters file://$(pwd)/parameters.json
143+
144+
aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region)
145+
echo "CREATED_CFN_STACK=$STACK_NAME"
146+
else
147+
echo "$STACK_NAME Already exists"
148+
fi
89149
fi
90-
echo "CREATED_NODEGROUP=$node_group_name"
91-
while [[ "$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-nodegroup --cluster-name $(params.cluster-name) --nodegroup-name $node_group_name --query nodegroup.status --output text)" == "CREATING" ]]
92-
do
93-
echo "$node_group_name is "CREATING" at $(date)"
94-
sleep 2
95-
done
96-
while true; do
97-
ready_node=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$node_group_name --no-headers 2>/dev/null | grep -w Ready | wc -l)
98-
echo "ready-nodes=$ready_node out of $2, for nodegroup: $node_group_name"
99-
if [[ "$ready_node" -eq $2 ]]; then break; fi
100-
sleep 5
101-
done
102150
}
103151
for i in $(seq 1 $asgs)
104152
do
@@ -119,4 +167,4 @@ spec:
119167
kubectl describe clusterrole eks:node-manager
120168
kubectl get nodes -o wide
121169
kubectl get ns
122-
kubectl get cs
170+
kubectl get cs

0 commit comments

Comments
 (0)