Add unmanaged nodegroup support

cartermckinnon · cartermckinnon · commit 777d772c379a · 2025-02-28T21:03:56.000Z
diff --git a/tests/assets/asg_node_group.yaml b/tests/assets/asg_node_group.yaml
@@ -0,0 +1,52 @@
+---
+AWSTemplateFormatVersion: '2010-09-09'
+Description: 'Unmanaged EKS nodegroup using EC2 AutoScaling'
+Parameters:
+  AutoScalingGroupName:
+    Description: Name of ASG.
+    Type: String
+  VpcId:
+    Type: AWS::EC2::VPC::Id
+  SubnetIds:
+    Type: List<AWS::EC2::Subnet::Id>
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup::Id
+  LaunchTemplateName:
+    Type: String
+    Description: Launch template name.
+  LaunchTemplateVersion:
+    Type: String
+    Description: Launch template version. Default is 0 as the launch template being used is generally ephemeral/single-use.
+    Default: "0"
+  NodeCount:
+    Type: Number
+  LaunchTemplateOverrides:
+    Type: List<AWS::AutoScaling::AutoScalingGroup::LaunchTemplateOverrides>
+  InstanceType:
+    Description: EC2 instance type.
+    Type: String
+Resources:
+  AutoScalingGroup:
+    Type: AWS::AutoScaling::AutoScalingGroup
+    UpdatePolicy:
+      AutoScalingRollingUpdate:
+        WaitOnResourceSignals: true
+        PauseTime: PT15M
+    Properties:
+      AutoScalingGroupName: !Ref AutoScalingGroupName
+      DesiredCapacity: !Ref NodeCount
+      MinSize: !Ref NodeCount
+      MaxSize: !Ref NodeCount
+      MixedInstancesPolicy:
+        LaunchTemplate:
+          LaunchTemplateSpecification:
+            LaunchTemplateName: !Ref LaunchTemplateName
+            Version: !Ref LaunchTemplateVersion
+          Overrides: !Ref LaunchTemplateOverrides
+      VPCZoneIdentifier:
+        !Ref SubnetIds
+      Tags:
+        # necessary for kubelet's legacy, in-tree cloud provider
+        - Key: !Sub kubernetes.io/cluster/${ClusterName}
+          Value: owned
+          PropagateAtLaunch: true
diff --git a/tests/assets/eks_node_group_launch_template_al2023.yaml b/tests/assets/eks_node_group_launch_template_al2023.yaml
@@ -29,10 +29,22 @@ Parameters:
     Type: String
     Description: Launch template ImageId value, which may be an AMI ID or resolve:ssm reference.
     Default: ''
+  NodeRoleName:
+    Type: String
+    Description: Name of the IAM Role for the node instances.
+  SecurityGroup:
+    Type: AWS::EC2::SecurityGroup::Id
+    Description: EKS-created cluster security group that allows node communication with the control plane.
 Conditions:
   AMIProvided:
     !Not [!Equals [!Ref AMI, '']]
 Resources:
+  NodeInstanceProfile:
+    Type: AWS::IAM::InstanceProfile
+    Properties:
+      Path: "/"
+      Roles:
+      - !Ref NodeRoleName
   LaunchTemplate:
     Type: AWS::EC2::LaunchTemplate
     Properties:
@@ -50,6 +62,10 @@ Resources:
           HttpPutResponseHopLimit: 2
           HttpEndpoint: enabled
           HttpTokens: required
+        IamInstanceProfile:
+          Arn: !GetAtt NodeInstanceProfile.Arn
+        SecurityGroupIds:
+          - !Ref SecurityGroup
         ImageId:
           !If
           - AMIProvided
diff --git a/tests/tasks/setup/eks/awscli-cfn-lt-al2023.yaml b/tests/tasks/setup/eks/awscli-cfn-lt-al2023.yaml
@@ -80,6 +80,7 @@ spec:
             --arg CertificateAuthority "$(jq -r .cluster.certificateAuthority.data cluster.json)" \
             --arg KubeletConfig '$(params.kubelet-config)' \
             --arg AMI "$(params.ami)" \
+            --arg SecurityGroup "$(jq -r .cluster.resourcesVpcConfig.clusterSecurityGroupId)" \
             '$ARGS.named | to_entries | map({"ParameterKey": .key, "ParameterValue": .value})' \
           > parameters.json
 
diff --git a/tests/tasks/setup/eks/awscli-mng.yaml b/tests/tasks/setup/eks/awscli-mng.yaml
@@ -6,8 +6,8 @@ metadata:
   namespace: scalability
 spec:
   description: |
-    Create an EKS managed nodegroup for a given cluster.
-    This Task can be used to create an EKS managed nodegroup for a given VPC Subnets, security groups and service role in an AWS account.
+    Create an EKS nodegroup, managed or unamaged, for a given cluster.
+    This Task can be used to create an EKS managed or unmanaged nodegroup for a given VPC Subnets, security groups and service role in an AWS account.
   params:
   - name: cluster-name
     description: The name of the EKS cluster you want to spin managed nodegroups for.
@@ -36,6 +36,9 @@ spec:
   - name: nodegroup-prefix
     description: Prefix that needs to be appended to asg names.
     default: ""
+  - name: unmanaged-nodegroup-cfn-url
+    default: ""
+    description: URL for "unmanaged nodegroup" (AutoScaling group) CloudFormation template. If not specified, a managed nodegroup will be created.
   workspaces:
   - name: config
     mountPath: /config/
@@ -60,9 +63,12 @@ spec:
         TAINTS_FLAG="--taints $(params.host-taints)"
       fi
 
-      NG_SUBNETS=$(aws eks $ENDPOINT_FLAG --region $(params.region)  describe-cluster --name $(params.cluster-name) \
+      NG_SUBNETS=$( \
       --query cluster.resourcesVpcConfig.subnetIds --output text \
       )
+
+      aws eks $ENDPOINT_FLAG --region $(params.region)  describe-cluster --name $(params.cluster-name) --output json > cluster.json
+      NG_SUBNETS=$(jq -r '.cluster.resourcesVpcConfig.subnetIds | join(" ")' cluster.json)
       
       max_nodes=$(params.max-nodes)
       nodes=$(params.desired-nodes)
@@ -73,32 +79,60 @@ spec:
       {
         node_group_name=$node_group-$1
         launch_template_name=$(params.cluster-name)-launchTemplate
-        CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name)  --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
-        EC2_INSTANCES=$3
-        if [ "$CREATED_NODEGROUP" == "" ]; then
-          #create node group
-          aws eks $ENDPOINT_FLAG create-nodegroup \
-          --cluster-name $(params.cluster-name) \
-          --nodegroup-name $node_group_name \
-          --node-role $NODE_ROLE_ARN \
-          --launch-template name=$launch_template_name\
-          --region $(params.region) \
-          --instance-types $EC2_INSTANCES \
-          --scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
-          --subnets $NG_SUBNETS $TAINTS_FLAG
+        # if no unmanaged nodegroup cfn template is provided, assume we want managed nodegroups
+        if [ "$(params.unmanaged-nodegroup-cfn-url)" = "" ]; then
+          CREATED_NODEGROUP=$(aws eks $ENDPOINT_FLAG --region $(params.region) list-nodegroups --cluster-name $(params.cluster-name)  --query 'nodegroups[?@==`'$node_group_name'`]' --output text)
+          EC2_INSTANCES=$3
+          if [ "$CREATED_NODEGROUP" == "" ]; then
+            aws eks $ENDPOINT_FLAG create-nodegroup \
+            --cluster-name $(params.cluster-name) \
+            --nodegroup-name $node_group_name \
+            --node-role $NODE_ROLE_ARN \
+            --launch-template name=$launch_template_name\
+            --region $(params.region) \
+            --instance-types $EC2_INSTANCES \
+            --scaling-config minSize=$(params.min-nodes),maxSize=$2,desiredSize=$2 \
+            --subnets $NG_SUBNETS $TAINTS_FLAG
+          fi
+          echo "CREATED_NODEGROUP=$node_group_name"
+          while [[ "$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-nodegroup --cluster-name $(params.cluster-name) --nodegroup-name $node_group_name --query nodegroup.status --output text)" == "CREATING" ]]
+          do
+            echo "$node_group_name is "CREATING" at $(date)"
+            sleep 2
+          done
+          # TODO: do this for unmanaged nodes as well
+          # right now we don't have an appropriate label to filter on for unmanaged nodes
+          while true; do
+            ready_node=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$node_group_name --no-headers 2>/dev/null | grep -w Ready | wc -l)
+            echo "ready-nodes=$ready_node out of $2, for nodegroup: $node_group_name"
+            if [[ "$ready_node" -eq $2 ]]; then break; fi
+            sleep 5
+          done
+        else
+          STACK_NAME=$node_group_name
+          STACK_STATUS=$(aws cloudformation describe-stacks --query 'Stacks[?StackName==`'${STACK_NAME}'`].StackStatus' --output text  --region $(params.region))
+          if [[ "$STACK_STATUS" == "" ]]; then
+            curl -s $(params.unmanaged-nodegroup-cfn-url) -o ./cfn-template
+            aws cloudformation create-stack \
+              --region $(params.region)
+              --stack-name $STACK_NAME \
+              --template-body file://$(pwd)/cfn-template \
+              --parameters ParameterKey=LaunchTemplateName,ParameterValue=$launch_template_name \
+                ParameterKey=ClusterName,ParameterValue=$(params.cluster-name) \
+                ParameterKey=SSHKeyName,ParameterValue=$SSH_KEY_NAME \
+                ParameterKey=AutoScalingGroupName,ParameterValue=$node_group_name \
+                ParameterKey=NodeCount,ParameterValue=$2 \
+                ParameterKey=LaunchTemplateOverrides,ParameterValue=$(echo $EC2_INSTANCES | jq -R 'split(" ") | .[] | {"InstanceType":.}' | jq -s '.') \
+                ParameterKey=SubnetIds,ParameterValue=$(jq -r '.cluster.resourcesVpcConfig.subnetIds | join(",")' cluster.json) \
+                ParameterKey=SecurityGroup,ParameterValue=$(jq -r '.cluster.resourcesVpcConfig.clusterSecurityGroupId' cluster.json) \
+                ParameterKey=VpcId,ParameterValue=$(jq -r '.cluster.resourcesVpcConfig.vpcId' cluster.json)
+
+              aws cloudformation wait stack-create-complete --stack-name $STACK_NAME --region $(params.region)
+              echo "CREATED_CFN_STACK=$STACK_NAME"
+          else
+              echo "$STACK_NAME Already exists"
+          fi
         fi
-        echo "CREATED_NODEGROUP=$node_group_name"
-        while [[ "$(aws eks $ENDPOINT_FLAG --region $(params.region) describe-nodegroup --cluster-name $(params.cluster-name) --nodegroup-name $node_group_name --query nodegroup.status --output text)" == "CREATING" ]]
-        do
-          echo "$node_group_name is "CREATING" at $(date)"
-          sleep 2
-        done
-        while true; do
-          ready_node=$(kubectl get nodes -l eks.amazonaws.com/nodegroup=$node_group_name --no-headers 2>/dev/null | grep -w Ready | wc -l)
-          echo "ready-nodes=$ready_node out of $2, for nodegroup: $node_group_name"
-          if [[ "$ready_node" -eq $2 ]]; then break; fi
-          sleep 5
-        done
       }
       for i in $(seq 1 $asgs)
       do
@@ -119,4 +153,4 @@ spec:
       kubectl describe clusterrole eks:node-manager
       kubectl get nodes -o wide
       kubectl get ns
-      kubectl get cs
+      kubectl get cs
