From 9432e9a8c57a23acfef87182450b7fe72887949a Mon Sep 17 00:00:00 2001 From: Sijo George <87609749+sijo5722-2021@users.noreply.github.com> Date: Thu, 2 May 2024 20:32:04 +0530 Subject: [PATCH] Verify IAC deployment using eks (#255) * Moving to a compatible version * adding vpc cni specific version * Upgrading to new version * addnig vpc cni service account role * private zone change * ns record * Changes for public_int_domain * fixing zone * fixing zone * temprly setting the flag to true * removing ns record * try using defaults from self managed * rm configmap * cleanup and add ns record * fix typo on ns * fix output for eks module for int domain * add zone for int to post config * missed local var * add prefix delegation and sgs * just use primary * adding try for taints and labels * adding try for node pool ref * Fixing null nodepool * correcting the condition * use latest cni * revert * go back to latest cni addon --------- Co-authored-by: David Fry --- terraform/aws/base-infra/data.tf | 15 ++++++--- terraform/aws/base-infra/outputs.tf | 6 +++- terraform/aws/base-infra/route53.tf | 41 +++++++++++++++++-------- terraform/aws/base-infra/variables.tf | 1 + terraform/aws/base-k8s/infra.tf | 2 +- terraform/aws/base-k8s/outputs.tf | 2 +- terraform/aws/eks/infra.tf | 25 ++++++++++++++- terraform/aws/eks/outputs.tf | 2 +- terraform/aws/eks/variables.tf | 2 +- terraform/k8s/k8s-deploy/terragrunt.hcl | 3 +- 10 files changed, 75 insertions(+), 24 deletions(-) diff --git a/terraform/aws/base-infra/data.tf b/terraform/aws/base-infra/data.tf index cda6ec52d..1c1c6cd49 100644 --- a/terraform/aws/base-infra/data.tf +++ b/terraform/aws/base-infra/data.tf @@ -1,23 +1,28 @@ data "aws_route53_zone" "public" { count = (var.create_public_zone || !var.configure_route_53) ? 0 : 1 - name = "${local.cluster_domain}." + name = "${local.cluster_domain}." +} + +data "aws_route53_zone" "public_int" { + count = (var.create_public_zone || !var.configure_route_53) ? 0 : 1 + name = "${var.private_subdomain_string}.${local.cluster_domain}." } data "aws_route53_zone" "private" { count = (var.create_private_zone || !var.configure_route_53) ? 0 : 1 - name = "${local.cluster_domain}.internal." + name = "${local.cluster_domain}.internal." } data "aws_route53_zone" "cluster_parent" { count = (var.manage_parent_domain || !var.configure_route_53) ? 0 : 1 - name = "${local.cluster_parent_domain}." + name = "${local.cluster_parent_domain}." } data "aws_route53_zone" "cluster_parent_parent" { count = (var.manage_parent_domain && var.manage_parent_domain_ns && var.configure_route_53) ? 1 : 0 - name = "${local.cluster_parent_parent_domain}." + name = "${local.cluster_parent_parent_domain}." } data "aws_availability_zones" "available" { state = "available" -} \ No newline at end of file +} diff --git a/terraform/aws/base-infra/outputs.tf b/terraform/aws/base-infra/outputs.tf index 94cc97702..9ff8d4134 100644 --- a/terraform/aws/base-infra/outputs.tf +++ b/terraform/aws/base-infra/outputs.tf @@ -22,6 +22,10 @@ output "private_zone" { value = local.private_zone } +output "public_int_zone" { + value = local.public_int_zone +} + output "public_zone" { value = local.public_zone } @@ -62,4 +66,4 @@ output "key_pair_name" { output "haproxy_server_fqdn" { description = "haproxy server Hostname" value = var.create_haproxy_dns_record ? aws_route53_record.haproxy_server_private[0].fqdn : "" -} \ No newline at end of file +} diff --git a/terraform/aws/base-infra/route53.tf b/terraform/aws/base-infra/route53.tf index f7ad4abd5..efd469fe2 100644 --- a/terraform/aws/base-infra/route53.tf +++ b/terraform/aws/base-infra/route53.tf @@ -1,7 +1,8 @@ resource "aws_route53_zone" "private" { force_destroy = var.route53_zone_force_destroy - count = (var.configure_route_53 && var.create_private_zone) ? 1 : 0 - name = "${var.private_subdomain_string}.${local.cluster_domain}." + count = (var.configure_route_53 && var.create_private_zone) ? 1 : 0 + name = "${local.cluster_domain}.internal." + vpc { vpc_id = module.vpc.vpc_id } @@ -10,13 +11,20 @@ resource "aws_route53_zone" "private" { resource "aws_route53_zone" "public" { force_destroy = var.route53_zone_force_destroy - count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0 - name = "${local.cluster_domain}." - tags = merge({ Name = "${local.cluster_domain}-public" }, local.common_tags) + count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0 + name = "${local.cluster_domain}." + tags = merge({ Name = "${local.cluster_domain}-public" }, local.common_tags) +} + +resource "aws_route53_zone" "public_int" { + force_destroy = var.route53_zone_force_destroy + count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0 + name = "${var.private_subdomain_string}.${local.cluster_domain}." + tags = merge({ Name = "${local.cluster_domain}-public-int" }, local.common_tags) } resource "aws_route53_record" "public_ns" { - count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0 + count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0 zone_id = local.cluster_parent_zone_id name = local.cluster_domain type = "NS" @@ -24,15 +32,24 @@ resource "aws_route53_record" "public_ns" { records = aws_route53_zone.public[0].name_servers } +resource "aws_route53_record" "public_int_ns" { + count = (var.configure_route_53 && var.create_public_zone) ? 1 : 0 + zone_id = aws_route53_zone.public[0].zone_id + name = "${var.private_subdomain_string}.${local.cluster_domain}" + type = "NS" + ttl = "30" + records = aws_route53_zone.public_int[0].name_servers +} + resource "aws_route53_zone" "cluster_parent" { force_destroy = var.route53_zone_force_destroy - count = (var.configure_route_53 && var.manage_parent_domain) ? 1 : 0 - name = "${local.cluster_parent_domain}." - tags = merge({ Name = "${local.cluster_domain}-cluster-parent" }, local.common_tags) + count = (var.configure_route_53 && var.manage_parent_domain) ? 1 : 0 + name = "${local.cluster_parent_domain}." + tags = merge({ Name = "${local.cluster_domain}-cluster-parent" }, local.common_tags) } resource "aws_route53_record" "cluster_ns" { - count = (var.configure_route_53 && var.manage_parent_domain && var.manage_parent_domain_ns) ? 1 : 0 + count = (var.configure_route_53 && var.manage_parent_domain && var.manage_parent_domain_ns) ? 1 : 0 zone_id = data.aws_route53_zone.cluster_parent_parent[0].zone_id name = local.cluster_parent_domain type = "NS" @@ -41,10 +58,10 @@ resource "aws_route53_record" "cluster_ns" { } resource "aws_route53_record" "haproxy_server_private" { - count = (var.configure_route_53 && var.create_haproxy_dns_record) ? 1 : 0 + count = (var.configure_route_53 && var.create_haproxy_dns_record) ? 1 : 0 zone_id = local.public_zone.id name = "haproxy" type = "A" ttl = "300" records = [aws_instance.bastion.private_ip] -} \ No newline at end of file +} diff --git a/terraform/aws/base-infra/variables.tf b/terraform/aws/base-infra/variables.tf index 0bee6ada0..5a6532911 100644 --- a/terraform/aws/base-infra/variables.tf +++ b/terraform/aws/base-infra/variables.tf @@ -113,6 +113,7 @@ locals { azs = slice(data.aws_availability_zones.available.names, 0, var.az_count) public_zone = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public[0] : data.aws_route53_zone.public[0]) : null private_zone = var.configure_route_53 ? (var.create_private_zone ? aws_route53_zone.private[0] : data.aws_route53_zone.private[0]) : null + public_int_zone = var.configure_route_53 ? (var.create_public_zone ? aws_route53_zone.public_int[0] : data.aws_route53_zone.public_int[0]) : null cluster_parent_zone_id = var.configure_route_53 ? (var.manage_parent_domain ? aws_route53_zone.cluster_parent[0].zone_id : data.aws_route53_zone.cluster_parent[0].zone_id) : null cluster_parent_parent_zone_id = var.configure_route_53 ? ((var.manage_parent_domain && var.manage_parent_domain_ns) ? data.aws_route53_zone.cluster_parent_parent[0].zone_id : null) : null ssh_keys = [] diff --git a/terraform/aws/base-k8s/infra.tf b/terraform/aws/base-k8s/infra.tf index ff375eb95..99112894a 100644 --- a/terraform/aws/base-k8s/infra.tf +++ b/terraform/aws/base-k8s/infra.tf @@ -25,7 +25,7 @@ module "post_config" { name = var.cluster_name domain = var.domain tags = var.tags - private_zone_id = module.base_infra.private_zone.id + private_zone_id = module.base_infra.public_int_zone.id public_zone_id = module.base_infra.public_zone.id } diff --git a/terraform/aws/base-k8s/outputs.tf b/terraform/aws/base-k8s/outputs.tf index 8989f1559..7c2fd3b18 100644 --- a/terraform/aws/base-k8s/outputs.tf +++ b/terraform/aws/base-k8s/outputs.tf @@ -12,7 +12,7 @@ output "external_load_balancer_dns" { } output "private_subdomain" { - value = module.base_infra.private_zone.name + value = module.base_infra.public_int_zone.name } output "public_subdomain" { diff --git a/terraform/aws/eks/infra.tf b/terraform/aws/eks/infra.tf index d5a98371d..356c74130 100644 --- a/terraform/aws/eks/infra.tf +++ b/terraform/aws/eks/infra.tf @@ -46,7 +46,9 @@ module "k6s_test_harness" { } module "eks" { - source = "terraform-aws-modules/eks/aws" + source = "terraform-aws-modules/eks/aws" + version = "~> 19.21" + enable_irsa = true cluster_name = local.eks_name cluster_version = var.kubernetes_version @@ -62,6 +64,9 @@ module "eks" { # See README for further details before_compute = true most_recent = true # To ensure access to the latest settings provided + #addon_version = "v1.18.0-eksbuild.1" #https://docs.aws.amazon.com/eks/latest/userguide/managing-vpc-cni.html#vpc-add-on-self-managed-update + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn configuration_values = jsonencode({ env = { # Reference docs https://docs.aws.amazon.com/eks/latest/userguide/cni-increase-ip-addresses.html @@ -135,6 +140,7 @@ locals { launch_template_use_name_prefix = false iam_role_name = "${local.eks_name}-${node_pool_key}" iam_role_use_name_prefix = false + iam_role_attach_cni_policy = true bootstrap_extra_args = "--use-max-pods false --kubelet-extra-args '--max-pods=110 --node-labels=${join(",", local.node_labels[node_pool_key].extra_args)} --register-with-taints=${join(",", local.node_taints[node_pool_key].extra_args)}'" post_bootstrap_user_data = <<-EOT yum install iscsi-initiator-utils -y && sudo systemctl enable iscsid && sudo systemctl start iscsid @@ -172,6 +178,23 @@ locals { } } +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 5.39" + role_name = "AmazonEKSVPCCNIRole" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } + } + + tags = var.tags +} + data "aws_ami" "eks_default" { most_recent = true owners = ["amazon"] diff --git a/terraform/aws/eks/outputs.tf b/terraform/aws/eks/outputs.tf index 5506ee66e..e56c94d5e 100644 --- a/terraform/aws/eks/outputs.tf +++ b/terraform/aws/eks/outputs.tf @@ -12,7 +12,7 @@ output "external_load_balancer_dns" { } output "private_subdomain" { - value = module.base_infra.private_zone.name + value = module.base_infra.public_int_zone.name } output "public_subdomain" { diff --git a/terraform/aws/eks/variables.tf b/terraform/aws/eks/variables.tf index 83dd8cd8a..6572ad9f7 100644 --- a/terraform/aws/eks/variables.tf +++ b/terraform/aws/eks/variables.tf @@ -15,7 +15,7 @@ variable "domain" { variable "kubernetes_version" { description = "version of k8s" type = string - default = "1.27" + default = "1.29" } variable "ext_interop_switch_subdomain" { description = "subdomain for interop ext" diff --git a/terraform/k8s/k8s-deploy/terragrunt.hcl b/terraform/k8s/k8s-deploy/terragrunt.hcl index 33dcd9a2a..c69395f26 100644 --- a/terraform/k8s/k8s-deploy/terragrunt.hcl +++ b/terraform/k8s/k8s-deploy/terragrunt.hcl @@ -14,7 +14,7 @@ inputs = { domain = local.CLUSTER_DOMAIN dns_zone_force_destroy = local.env_vars.dns_zone_force_destroy longhorn_backup_object_store_destroy = local.env_vars.longhorn_backup_object_store_destroy - node_pools = local.env_vars.nodes + node_pools = local.enabled_node_pools enable_k6s_test_harness = local.env_vars.enable_k6s_test_harness k6s_docker_server_instance_type = local.env_vars.k6s_docker_server_instance_type vpc_cidr = local.env_vars.vpc_cidr @@ -32,6 +32,7 @@ locals { cloud_platform_vars = yamldecode( file("${find_in_parent_folders("${get_env("CONFIG_PATH")}/${get_env("cloud_platform")}-vars.yaml")}") ) + enabled_node_pools = [for node in local.env_vars.nodes : node if node != null] total_agent_count = try(sum([for node in local.env_vars.nodes : node.node_count if !node.master]), 0) total_master_count = try(sum([for node in local.env_vars.nodes : node.node_count if node.master]), 0) tags = local.env_vars.tags