IGW routes for KIT CP LB; Custom tags for KIT CP and DP;update karp (#62)

Co-authored-by: Harish Kuna <[email protected]>

Author: hakuna-matatah

testbed/addons/karpenter/construct.ts

@@ -62,7 +62,7 @@ export class Karpenter extends cdk.Construct {
         const chart = props.cluster.addHelmChart('karpenter', {
             chart: 'karpenter',
             release: 'karpenter',
-            version: 'v0.3.1',
+            version: 'v0.4.1',
             repository: 'https://awslabs.github.io/karpenter/charts',
             namespace: namespace,
             createNamespace: false,
@@ -77,21 +77,5 @@ export class Karpenter extends cdk.Construct {
             }
         })
         chart.node.addDependency(ns)
-
-        // Default Provisioner
-        props.cluster.addManifest("default-provisioner", {
-            apiVersion: 'karpenter.sh/v1alpha3',
-            kind: 'Provisioner',
-            metadata: {
-                name: 'default',
-            },
-            spec: {
-                cluster: {
-                    name: props.cluster.clusterName,
-                    endpoint: props.cluster.clusterEndpoint,
-                },
-                ttlSecondsAfterEmpty: 30,
-            }
-        }).node.addDependency(chart)
     }
 }

testbed/addons/kit/construct.ts

@@ -52,27 +52,6 @@ export class Kit extends cdk.Construct {
             ],
         }))
 
-        // Node Role
-        const nodeRole = new iam.Role(this, 'kit-node-role', {
-            assumedBy: new iam.ServicePrincipal('ec2.amazonaws.com'),
-            managedPolicies: [
-                iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEKSWorkerNodePolicy'),
-                iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEC2ContainerRegistryReadOnly'),
-                iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonEKS_CNI_Policy'),
-                iam.ManagedPolicy.fromAwsManagedPolicyName('AmazonSSMManagedInstanceCore')
-            ]
-        })
-
-        props.cluster.awsAuth.addRoleMapping(nodeRole, {
-            username: 'system:node:{{EC2PrivateDNSName}}',
-            groups: ['system:bootstrappers', 'system:nodes']
-        })
-
-        new iam.CfnInstanceProfile(this, 'kit-instance-profile', {
-            roles: [nodeRole.roleName],
-            instanceProfileName: 'KitNodeInstanceProfile'
-        })
-
         // Install kit
         const chart = props.cluster.addHelmChart('kit', {
             chart: 'kit-operator',
@@ -92,5 +71,27 @@ export class Kit extends cdk.Construct {
             }
         })
         chart.node.addDependency(ns)
+
+        //Karp Provisioner for kit
+        props.cluster.addManifest("default-provisioner", {
+            apiVersion: 'karpenter.sh/v1alpha5',
+            kind: 'Provisioner',
+            metadata: {
+                name: 'default',
+            },
+            spec: {
+                provider: {
+                    cluster: {
+                        name: props.cluster.clusterName,
+                        endpoint: props.cluster.clusterEndpoint,
+                    },
+                    subnetSelector: {
+                        "kit/hostcluster": `${props.cluster.clusterName}-controlplane`
+                    }
+                },
+                ttlSecondsAfterEmpty: 30,
+            }
+        })
+
     }
 }

testbed/stack.ts

@@ -11,7 +11,7 @@ export interface TestbedProps extends cdk.StackProps {
 }
 
 export class Testbed extends cdk.Stack {
-    constructor(scope: cdk.Construct, id: string="testbed", props: TestbedProps) {
+    constructor(scope: cdk.Construct, id: string = "testbed", props: TestbedProps) {
         super(scope, id, props)
 
         const vpc = new ec2.Vpc(this, id, {
@@ -30,6 +30,15 @@ export class Testbed extends cdk.Stack {
                 },
             ],
         });
+
+        //Tag pub subnets for KIT CP
+        const selection = vpc.selectSubnets({
+            subnetType: ec2.SubnetType.PUBLIC
+        });
+        selection.subnets.forEach(subnet => {
+            Tags.of(subnet).add('kit/hostcluster', `${id}-controlplane`)
+        })
+
         //ToDo: revisit once this is resolved - https://github.com/aws/aws-cdk/issues/5927
         // index<=8 will give us 9  /16 cidrs additionally to make a mega VPC.
         for (let index = 0; index <= 8; index++) {
@@ -38,26 +47,37 @@ export class Testbed extends cdk.Stack {
                 cidrBlock: `10.${index + 1}.0.0/16`
             });
             let privateSubnet = new ec2.PrivateSubnet(this, `${id}-private-subnet-${index}`, {
-                availabilityZone: cdk.Stack.of(this).availabilityZones[index%cdk.Stack.of(this).availabilityZones.length],
+                availabilityZone: cdk.Stack.of(this).availabilityZones[index % cdk.Stack.of(this).availabilityZones.length],
                 vpcId: vpc.vpcId,
                 cidrBlock: `10.${index + 1}.0.0/16`
             })
             privateSubnet.node.addDependency(additionalCidr);
+            //Tag pub subnets for KIT DP
+            Tags.of(privateSubnet).add('kit/hostcluster', `${id}-dataplane`)
+            let natSubnet = new ec2.PublicSubnet(this, `${id}-nat-subnet-${index}`, {
+                availabilityZone: cdk.Stack.of(this).availabilityZones[index % cdk.Stack.of(this).availabilityZones.length],
+                vpcId: vpc.vpcId,
+                cidrBlock: `10.0.64.${index * 16}/28`
+            })
+            //add igw route for nat subnets
+            let routeTableId = natSubnet.routeTable.routeTableId
+            new ec2.CfnRoute(this, 'publicIGWRoute' + index, {
+                routeTableId,
+                gatewayId: vpc.internetGatewayId,
+                destinationCidrBlock: "0.0.0.0/0"
+            })
 
             ec2.NatProvider.gateway().configureNat({
                 natSubnets: [
-                    new ec2.PublicSubnet(this, `${id}-nat-subnet-${index}`, {
-                        availabilityZone: cdk.Stack.of(this).availabilityZones[index%cdk.Stack.of(this).availabilityZones.length],
-                        vpcId: vpc.vpcId,
-                        cidrBlock: `10.0.64.${index*16}/28`
-                    })
+                    natSubnet
                 ],
                 privateSubnets: [
                     privateSubnet
                 ],
                 vpc: vpc
             })
         }
+
         const cluster = new eks.Cluster(this, 'cluster', {
             clusterName: id,
             vpc: vpc,
@@ -87,7 +107,6 @@ export class Testbed extends cdk.Stack {
                 ]
             }),
         })
-
         // service account used by tekton workflows.
         cluster.addServiceAccount('test-executor', { name: 'test-executor' })
             .role.addManagedPolicy({ managedPolicyArn: 'arn:aws:iam::aws:policy/AdministratorAccess' })