Verify IAC deployment using eks (#255)

* Moving to a compatible version * adding vpc cni specific version * Upgrading to new version * addnig vpc cni service account role * private zone change * ns record * Changes for public_int_domain * fixing zone * fixing zone * temprly setting the flag to true * removing ns record * try using defaults from self managed * rm configmap * cleanup and add ns record * fix typo on ns * fix output for eks module for int domain * add zone for int to post config * missed local var * add prefix delegation and sgs * just use primary * adding try for taints and labels * adding try for node pool ref * Fixing null nodepool * correcting the condition * use latest cni * revert * go back to latest cni addon --------- Co-authored-by: David Fry <[email protected]>
mojaloop · May 2, 2024 · 9432e9a · 9432e9a
1 parent a44db1c
commit 9432e9a
Show file tree

Hide file tree

Showing 10 changed files with 75 additions and 24 deletions.
diff --git a/terraform/aws/base-infra/data.tf b/terraform/aws/base-infra/data.tf
@@ -1,23 +1,28 @@
 data "aws_route53_zone" "public" {
   count = (var.create_public_zone || !var.configure_route_53) ? 0 : 1
-  name = "${local.cluster_domain}."
+  name  = "${local.cluster_domain}."
+}
+
+data "aws_route53_zone" "public_int" {
+  count = (var.create_public_zone || !var.configure_route_53) ? 0 : 1
+  name  = "${var.private_subdomain_string}.${local.cluster_domain}."
 }
 
 data "aws_route53_zone" "private" {
   count = (var.create_private_zone || !var.configure_route_53) ? 0 : 1
-  name = "${local.cluster_domain}.internal."
+  name  = "${local.cluster_domain}.internal."
 }
 
 data "aws_route53_zone" "cluster_parent" {
   count = (var.manage_parent_domain || !var.configure_route_53) ? 0 : 1
-  name = "${local.cluster_parent_domain}."
+  name  = "${local.cluster_parent_domain}."
 }
 
 data "aws_route53_zone" "cluster_parent_parent" {
   count = (var.manage_parent_domain && var.manage_parent_domain_ns && var.configure_route_53) ? 1 : 0
-  name = "${local.cluster_parent_parent_domain}."
+  name  = "${local.cluster_parent_parent_domain}."
 }
 
 data "aws_availability_zones" "available" {
   state = "available"
-}
+}
diff --git a/terraform/aws/base-infra/outputs.tf b/terraform/aws/base-infra/outputs.tf
@@ -22,6 +22,10 @@ output "private_zone" {
   value = local.private_zone
 }
 
+output "public_int_zone" {
+  value = local.public_int_zone
+}
+
 output "public_zone" {
   value = local.public_zone
 }
@@ -62,4 +66,4 @@ output "key_pair_name" {
 output "haproxy_server_fqdn" {
   description = "haproxy server Hostname"
   value       = var.create_haproxy_dns_record ? aws_route53_record.haproxy_server_private[0].fqdn : ""
-}
+}
diff --git a/terraform/aws/base-infra/route53.tf b/terraform/aws/base-infra/route53.tf
@@ -1,7 +1,8 @@
 resource "aws_route53_zone" "private" {
   force_destroy = var.route53_zone_force_destroy
-  count = (var.configure_route_53 && var.create_private_zone) ? 1 : 0
-  name  = "${var.private_subdomain_string}.${local.cluster_domain}."
+  count         = (var.configure_route_53 && var.create_private_zone) ? 1 : 0
+  name          = "${local.cluster_domain}.internal."
+
   vpc {
     vpc_id = module.vpc.vpc_id
   }
@@ -10,29 +11,45 @@ resource "aws_route53_zone" "private" {
 
 resource "aws_route53_zone" "public" {
   force_destroy = var.route53_zone_force_destroy
-  count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
-  name  = "${local.cluster_domain}."
-  tags = merge({ Name = "${local.cluster_domain}-public" }, local.common_tags)
+  count         = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  name          = "${local.cluster_domain}."
+  tags          = merge({ Name = "${local.cluster_domain}-public" }, local.common_tags)
+}
+
+resource "aws_route53_zone" "public_int" {
+  force_destroy = var.route53_zone_force_destroy
+  count         = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  name          = "${var.private_subdomain_string}.${local.cluster_domain}."
+  tags          = merge({ Name = "${local.cluster_domain}-public-int" }, local.common_tags)
 }
 
 resource "aws_route53_record" "public_ns" {
-  count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  count   = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
   zone_id = local.cluster_parent_zone_id
   name    = local.cluster_domain
   type    = "NS"
   ttl     = "30"
   records = aws_route53_zone.public[0].name_servers
 }
 
+resource "aws_route53_record" "public_int_ns" {
+  count   = (var.configure_route_53 && var.create_public_zone) ? 1 : 0
+  zone_id = aws_route53_zone.public[0].zone_id
+  name    = "${var.private_subdomain_string}.${local.cluster_domain}"
+  type    = "NS"
+  ttl     = "30"
+  records = aws_route53_zone.public_int[0].name_servers
+}
+
 resource "aws_route53_zone" "cluster_parent" {
   force_destroy = var.route53_zone_force_destroy
-  count = (var.configure_route_53 && var.manage_parent_domain) ? 1 : 0
-  name  = "${local.cluster_parent_domain}."
-  tags = merge({ Name = "${local.cluster_domain}-cluster-parent" }, local.common_tags)
+  count         = (var.configure_route_53 && var.manage_parent_domain) ? 1 : 0
+  name          = "${local.cluster_parent_domain}."
+  tags          = merge({ Name = "${local.cluster_domain}-cluster-parent" }, local.common_tags)
 }
 
 resource "aws_route53_record" "cluster_ns" {
-  count = (var.configure_route_53 && var.manage_parent_domain && var.manage_parent_domain_ns) ? 1 : 0
+  count   = (var.configure_route_53 && var.manage_parent_domain && var.manage_parent_domain_ns) ? 1 : 0
   zone_id = data.aws_route53_zone.cluster_parent_parent[0].zone_id
   name    = local.cluster_parent_domain
   type    = "NS"
@@ -41,10 +58,10 @@ resource "aws_route53_record" "cluster_ns" {
 }
 
 resource "aws_route53_record" "haproxy_server_private" {
-  count = (var.configure_route_53 && var.create_haproxy_dns_record) ? 1 : 0
+  count   = (var.configure_route_53 && var.create_haproxy_dns_record) ? 1 : 0
   zone_id = local.public_zone.id
   name    = "haproxy"
   type    = "A"
   ttl     = "300"
   records = [aws_instance.bastion.private_ip]
-}
+}
diff --git a/terraform/aws/base-infra/variables.tf b/terraform/aws/base-infra/variables.tf
@@ -113,6 +113,7 @@ locals {
   azs                           = slice(data.aws_availability_zones.available.names, 0, var.az_count)
   public_zone                   = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public[0] : data.aws_route53_zone.public[0]) : null
   private_zone                  = var.configure_route_53 ? (var.create_private_zone ? aws_route53_zone.private[0] : data.aws_route53_zone.private[0]) : null
+  public_int_zone               = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public_int[0] : data.aws_route53_zone.public_int[0]) : null
   cluster_parent_zone_id        = var.configure_route_53 ? (var.manage_parent_domain ? aws_route53_zone.cluster_parent[0].zone_id : data.aws_route53_zone.cluster_parent[0].zone_id) : null
   cluster_parent_parent_zone_id = var.configure_route_53 ? ((var.manage_parent_domain && var.manage_parent_domain_ns) ? data.aws_route53_zone.cluster_parent_parent[0].zone_id : null) : null
   ssh_keys                      = []

diff --git a/terraform/aws/base-k8s/infra.tf b/terraform/aws/base-k8s/infra.tf
@@ -25,7 +25,7 @@ module "post_config" {
   name            = var.cluster_name
   domain          = var.domain
   tags            = var.tags
-  private_zone_id = module.base_infra.private_zone.id
+  private_zone_id = module.base_infra.public_int_zone.id
   public_zone_id  = module.base_infra.public_zone.id
 }
 

diff --git a/terraform/aws/base-k8s/outputs.tf b/terraform/aws/base-k8s/outputs.tf
@@ -12,7 +12,7 @@ output "external_load_balancer_dns" {
 }
 
 output "private_subdomain" {
-  value = module.base_infra.private_zone.name
+  value = module.base_infra.public_int_zone.name
 }
 
 output "public_subdomain" {

diff --git a/terraform/aws/eks/infra.tf b/terraform/aws/eks/infra.tf
@@ -46,7 +46,9 @@ module "k6s_test_harness" {
 }
 
 module "eks" {
-  source = "terraform-aws-modules/eks/aws"
+  source      = "terraform-aws-modules/eks/aws"
+  version     = "~> 19.21"
+  enable_irsa = true
 
   cluster_name                    = local.eks_name
   cluster_version                 = var.kubernetes_version
@@ -62,6 +64,9 @@ module "eks" {
       # See README for further details
       before_compute = true
       most_recent    = true # To ensure access to the latest settings provided
+      #addon_version            = "v1.18.0-eksbuild.1" #https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html#vpc-add-on-self-managed-update
+      resolve_conflicts        = "OVERWRITE"
+      service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
       configuration_values = jsonencode({
         env = {
           # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html
@@ -135,6 +140,7 @@ locals {
       launch_template_use_name_prefix = false
       iam_role_name                   = "${local.eks_name}-${node_pool_key}"
       iam_role_use_name_prefix        = false
+      iam_role_attach_cni_policy      = true
       bootstrap_extra_args            = "--use-max-pods false --kubelet-extra-args '--max-pods=110 --node-labels=${join(",", local.node_labels[node_pool_key].extra_args)} --register-with-taints=${join(",", local.node_taints[node_pool_key].extra_args)}'"
       post_bootstrap_user_data        = <<-EOT
         yum install iscsi-initiator-utils -y && sudo systemctl enable iscsid && sudo systemctl start iscsid
@@ -172,6 +178,23 @@ locals {
   }
 }
 
+module "vpc_cni_irsa" {
+  source                = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version               = "~> 5.39"
+  role_name             = "AmazonEKSVPCCNIRole"
+  attach_vpc_cni_policy = true
+  vpc_cni_enable_ipv4   = true
+
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:aws-node"]
+    }
+  }
+
+  tags = var.tags
+}
+
 data "aws_ami" "eks_default" {
   most_recent = true
   owners      = ["amazon"]

diff --git a/terraform/aws/eks/outputs.tf b/terraform/aws/eks/outputs.tf
@@ -12,7 +12,7 @@ output "external_load_balancer_dns" {
 }
 
 output "private_subdomain" {
-  value = module.base_infra.private_zone.name
+  value = module.base_infra.public_int_zone.name
 }
 
 output "public_subdomain" {

diff --git a/terraform/aws/eks/variables.tf b/terraform/aws/eks/variables.tf
@@ -15,7 +15,7 @@ variable "domain" {
 variable "kubernetes_version" {
   description = "version of k8s"
   type        = string
-  default     = "1.27"
+  default     = "1.29"
 }
 variable "ext_interop_switch_subdomain" {
   description = "subdomain for interop ext"

diff --git a/terraform/k8s/k8s-deploy/terragrunt.hcl b/terraform/k8s/k8s-deploy/terragrunt.hcl
@@ -14,7 +14,7 @@ inputs = {
   domain                               = local.CLUSTER_DOMAIN
   dns_zone_force_destroy               = local.env_vars.dns_zone_force_destroy
   longhorn_backup_object_store_destroy = local.env_vars.longhorn_backup_object_store_destroy
-  node_pools                           = local.env_vars.nodes
+  node_pools                           = local.enabled_node_pools
   enable_k6s_test_harness              = local.env_vars.enable_k6s_test_harness
   k6s_docker_server_instance_type      = local.env_vars.k6s_docker_server_instance_type
   vpc_cidr                             = local.env_vars.vpc_cidr
@@ -32,6 +32,7 @@ locals {
   cloud_platform_vars = yamldecode(
     file("${find_in_parent_folders("${get_env("CONFIG_PATH")}/${get_env("cloud_platform")}-vars.yaml")}")
   )
+  enabled_node_pools = [for node in local.env_vars.nodes : node if node != null]
   total_agent_count = try(sum([for node in local.env_vars.nodes : node.node_count if !node.master]), 0)
   total_master_count = try(sum([for node in local.env_vars.nodes : node.node_count if node.master]), 0)
   tags                      = local.env_vars.tags