Merge pull request #267 from mojaloop/feature/upgrade-netmaker-v24

Netmaker upgrade and managed services connections

terraform/ansible/k8s-deploy/ansible.tf

@@ -12,7 +12,7 @@ resource "local_sensitive_file" "ansible_inventory" {
       all_hosts_var_maps          = merge(var.all_hosts_var_maps, local.ssh_private_key_file_map, local.all_hosts_var_maps),
       agent_hosts_yaml_maps       = var.agent_hosts_yaml_maps,
       master_hosts_yaml_maps      = var.master_hosts_yaml_maps,
-      bastion_hosts_yaml_maps     = var.bastion_hosts_yaml_maps,
+      bastion_hosts_yaml_maps     = merge(var.bastion_hosts_yaml_maps, local.bastion_hosts_yaml_maps)
       test_harness_hosts          = var.test_harness_hosts,
       test_harness_hosts_var_maps = merge(var.test_harness_hosts_var_maps, local.jumphostmap)
     }
@@ -36,7 +36,7 @@ resource "null_resource" "run_ansible" {
     EOT
     working_dir = path.module
   }
- 
+
   depends_on = [
     local_sensitive_file.ansible_inventory,
     local_sensitive_file.ec2_ssh_key
@@ -53,12 +53,12 @@ resource "null_resource" "destroy_ansible_actions" {
           ansible-playbook "$destroy_ansible_playbook" -i "$destroy_ansible_inventory"
     EOT
     working_dir = path.module
-  } 
+  }
 
- depends_on = [
+  depends_on = [
     local_sensitive_file.ansible_inventory,
     local_sensitive_file.ec2_ssh_key,
- ]
+  ]
 
 }
 
@@ -68,6 +68,19 @@ resource "local_sensitive_file" "ec2_ssh_key" {
   file_permission = "0600"
 }
 
+data "gitlab_project_variable" "external_rds_stateful_resource_instance_address" {
+  for_each = local.managed_rds_stateful_resources
+  project  = var.current_gitlab_project_id
+  key      = each.value.external_resource_config.instance_address_key_name
+}
+
+data "gitlab_project_variable" "external_kafka_stateful_resource_instance_address" {
+  for_each = local.managed_kafka_stateful_resources
+  project  = var.current_gitlab_project_id
+  key      = each.value.external_resource_config.instance_address_key_name
+}
+
+
 locals {
   jumphostmap = {
     ansible_ssh_common_args = "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ProxyCommand=\"ssh -W %h:%p -i ${local_sensitive_file.ec2_ssh_key.filename} -o StrictHostKeyChecking=no -q ${var.ansible_bastion_os_username}@${var.ansible_bastion_public_ip}\""
@@ -80,4 +93,41 @@ locals {
     kubeconfig_local_location = local.ansible_output_dir
   }
 
+  stateful_resources               = jsondecode(file(var.stateful_resources_config_file))
+  enabled_stateful_resources       = { for stateful_resource in local.stateful_resources : stateful_resource.resource_name => stateful_resource if stateful_resource.enabled }
+  managed_rds_stateful_resources   = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service && managed_resource.resource_type == "mysql" }
+  managed_kafka_stateful_resources = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service && managed_resource.resource_type == "kafka" }
+
+
+  external_rds_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_rds_stateful_resource_instance_address : address.key => address.value }
+  external_kafka_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_kafka_stateful_resource_instance_address : address.key => address.value }
+
+
+  managed_kafka_brokers_list                    = { for service in local.managed_kafka_stateful_resources : service.resource_name => split(",", local.external_kafka_stateful_resource_instance_addresses[service.external_resource_config.instance_address_key_name]) }
+
+
+  managed_rds_svc_port_maps = [for service in local.managed_rds_stateful_resources :
+    {
+      "local_listening_port" = service.logical_service_config.logical_service_port
+      "mode"                 = service.communication_mode
+      "name"                 = service.resource_name
+      "dest_fqdn"            = local.external_rds_stateful_resource_instance_addresses[service.external_resource_config.instance_address_key_name]
+      "dest_port"            = service.external_resource_config.port
+    }
+  ]
+
+  managed_kafka_svc_maps = [for service in local.managed_kafka_stateful_resources :
+    {
+      "local_listening_port"       = service.logical_service_config.logical_service_port
+      "managed_kafka_brokers_list" = local.managed_kafka_brokers_list[service.resource_name]
+      "mode"                       = service.communication_mode
+      "name"                       = service.resource_name
+      "dest_port"                  = service.external_resource_config.port
+    }
+
+  ]
+  bastion_hosts_yaml_maps = {
+    managed_rds_svc   = yamlencode(local.managed_rds_svc_port_maps)
+    managed_kafka_svc = yamlencode(local.managed_kafka_svc_maps)
+  }
 }

terraform/ansible/k8s-deploy/variables.tf

@@ -105,3 +105,12 @@ variable "ansible_debug" {
   type    = string
   default = ""
 }
+
+variable "stateful_resources_config_file" {
+  type = string
+}
+
+variable "current_gitlab_project_id" {
+  type        = string
+  description = "current_gitlab_project_id"
+}

terraform/aws/support-svcs/deploy-managed-svcs/deploy.tf

@@ -14,7 +14,7 @@ module "deploy_msk" {
   deployment_name = var.deployment_name
   tags = var.tags
   msk_services = local.msk_services
-  security_group_id = aws_security_group.managed_svcs[0].id
+  security_group_id = aws_security_group.managed_svcs.*.id
   private_subnets = module.base_infra[0].private_subnets
 }
 

terraform/aws/support-svcs/deploy-managed-svcs/outputs.tf

@@ -4,7 +4,7 @@ output "secrets_var_map" {
 }
 
 output "properties_var_map" {
-  value = length(local.rds_services) > 0 ? module.deploy_rds[0].properties_var_map : {}
+  value = ( length(local.rds_services) > 0 || length(local.msk_services) > 0 ) ? merge(try(module.deploy_rds[0].properties_var_map, {}), try(module.deploy_msk[0].properties_var_map , {})) : {}
 }
 
 output "secrets_key_map" {

terraform/aws/support-svcs/deploy-managed-svcs/security-groups.tf

@@ -20,6 +20,17 @@ resource "aws_security_group_rule" "mysql" {
   description       = "mysql client access"
 }
 
+resource "aws_security_group_rule" "kafka" {
+  count             = length(local.msk_services) > 0 ? 1 : 0
+  type              = "ingress"
+  from_port         = 9094
+  to_port           = 9094
+  protocol          = "tcp"
+  cidr_blocks       = ["0.0.0.0/0"]
+  security_group_id = aws_security_group.managed_svcs[0].id
+  description       = "kafka broker access"
+}
+
 resource "aws_security_group_rule" "egress_all" {
   count             = length(local.external_services) > 0 ? 1 : 0
   type              = "egress"

terraform/aws/support-svcs/deploy-msk/outputs.tf

@@ -5,12 +5,13 @@ output "secrets_var_map" {
     for index, rds_module in module.rds : 
       var.rds_services[index].external_resource_config.password_key_name => jsondecode(data.aws_secretsmanager_secret_version.rds_passwords[index].secret_string)["password"]
   }
-}
+}*/
+
+#var.msk_services[index].external_resource_config.instance_address_key_name => ( var.msk_services[index].external_resource_config.encryption_in_transit_client_broker == "PLAINTEXT" )  ? msk_module.bootstrap_brokers : msk_module.bootstrap_brokers_tls  
 
 output "properties_var_map" {
   value = {
-    for index, rds_module in module.rds : 
-      var.rds_services[index].external_resource_config.instance_address_key_name => rds_module.db_instance_address
+    for index, msk_module in module.msk : 
+      var.msk_services[index].external_resource_config.instance_address_key_name => join(",", msk_module.bootstrap_brokers)
   }
 }
-*/

terraform/control-center/init/environment.yaml

@@ -10,7 +10,7 @@ smtp_server_enable: false
 gitlab_version: 16.0.5
 gitlab_runner_version: 16.0.2
 iac_group_name: iac_admin
-netmaker_version: 0.18.7
+netmaker_version: 0.24.0
 letsencrypt_email: [email protected]
 delete_storage_on_term: true
 docker_server_extra_vol_size: 100

terraform/gitops/k8s-cluster-config/app-deploy.tf

@@ -55,6 +55,7 @@ module "mojaloop" {
   mojaloop_values_override_file        = var.mojaloop_values_override_file
   finance_portal_values_override_file  = var.finance_portal_values_override_file
   fspiop_use_ory_for_auth              = var.app_var_map.fspiop_use_ory_for_auth
+  managed_db_host                      = var.managed_db_host
 }
 
 module "pm4ml" {
@@ -155,6 +156,7 @@ module "vnext" {
   keycloak_hubop_realm_name            = var.keycloak_hubop_realm_name
   rbac_api_resources_file              = var.rbac_api_resources_file
   fspiop_use_ory_for_auth              = var.app_var_map.fspiop_use_ory_for_auth
+  managed_db_host                      = var.managed_db_host
 }
 
 variable "app_var_map" {

terraform/gitops/k8s-cluster-config/common-stateful-resources-config.tf

@@ -10,7 +10,8 @@ module "common_stateful_resources" {
   stateful_resources_namespace                  = var.stateful_resources_namespace
   create_stateful_resources_ns                  = true
   kv_path                                       = var.kv_path
-  external_stateful_resource_instance_addresses = length(data.gitlab_project_variable.external_stateful_resource_instance_address) > 0 ? data.gitlab_project_variable.external_stateful_resource_instance_address.*.value : []
+  external_stateful_resource_instance_addresses = local.external_stateful_resource_instance_addresses
+  managed_db_host                               = var.managed_db_host
 }
 
 variable "stateful_resources_config_file" {
@@ -32,4 +33,5 @@ locals {
   stateful_resources         = jsondecode(file(var.stateful_resources_config_file))
   enabled_stateful_resources = { for stateful_resource in local.stateful_resources : stateful_resource.resource_name => stateful_resource if stateful_resource.enabled }
   managed_stateful_resources = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service }
+  external_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_stateful_resource_instance_address : address.key => address.value }
 }

terraform/gitops/k8s-cluster-config/variables.tf

@@ -114,6 +114,11 @@ variable "default_internal_ssl_certificate" {
   default     = "lets-enc-internal-tls"
 }
 
+variable "managed_db_host" {
+  type        = string
+  description = "url to managed db based on haproxy"
+}
+
 locals {
   cloud_region                                     = data.gitlab_project_variable.cloud_region.value
   k8s_cluster_type                                 = data.gitlab_project_variable.k8s_cluster_type.value

terraform/gitops/mojaloop/stateful-resources-config.tf

@@ -10,7 +10,8 @@ module "mojaloop_stateful_resources" {
   stateful_resources_namespace                  = var.stateful_resources_namespace
   create_stateful_resources_ns                  = false
   kv_path                                       = var.kv_path
-  external_stateful_resource_instance_addresses = length(data.gitlab_project_variable.external_stateful_resource_instance_address) > 0 ? data.gitlab_project_variable.external_stateful_resource_instance_address.*.value : []
+  external_stateful_resource_instance_addresses = local.external_stateful_resource_instance_addresses
+  managed_db_host                               = var.managed_db_host
 }
 
 variable "stateful_resources_config_file" {
@@ -32,4 +33,5 @@ locals {
   stateful_resources         = jsondecode(file(var.stateful_resources_config_file))
   enabled_stateful_resources = { for stateful_resource in local.stateful_resources : stateful_resource.resource_name => stateful_resource if stateful_resource.enabled }
   managed_stateful_resources = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service }
+  external_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_stateful_resource_instance_address : address.key => address.value }
 }

terraform/gitops/mojaloop/variables.tf

@@ -161,3 +161,8 @@ variable "switch_dfspid" {
   type    = string
   default = "switch"
 }
+
+variable "managed_db_host" {
+  type        = string
+  description = "url to managed db based on haproxy"
+}

terraform/gitops/stateful-resources/stateful-resources-config.tf

@@ -65,7 +65,7 @@ locals {
   managed_stateful_resources         = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service }
   local_stateful_resources           = { for local_stateful_resource in local.enabled_stateful_resources : local_stateful_resource.resource_name => local_stateful_resource if !local_stateful_resource.external_service }
   local_external_name_map            = { for stateful_resource in local.local_stateful_resources : stateful_resource.logical_service_config.logical_service_name => stateful_resource.local_resource_config.override_service_name != null ? "${stateful_resource.local_resource_config.override_service_name}.${stateful_resource.local_resource_config.resource_namespace}.svc.cluster.local" : "${stateful_resource.resource_name}.${stateful_resource.local_resource_config.resource_namespace}.svc.cluster.local" }
-  managed_external_name_map          = { for index, stateful_resource in local.managed_stateful_resources : stateful_resource.logical_service_config.logical_service_name => var.external_stateful_resource_instance_addresses[index] }
+  managed_external_name_map          = { for index, stateful_resource in local.managed_stateful_resources : stateful_resource.logical_service_config.logical_service_name => var.managed_db_host }  
   external_name_map                  = merge(local.local_external_name_map, local.managed_external_name_map)
   managed_resource_password_map = { for index, stateful_resource in local.managed_stateful_resources : stateful_resource.resource_name => {
     vault_path  = "${var.kv_path}/${var.cluster_name}/${stateful_resource.external_resource_config.password_key_name}"
@@ -87,8 +87,6 @@ locals {
 }
 
 variable "external_stateful_resource_instance_addresses" {
-  type    = list(string)
-  default = []
 }
 
 variable "create_stateful_resources_ns" {
@@ -149,3 +147,8 @@ variable "stateful_resources_sync_wave" {
   description = "stateful_resources_sync_wave, wait for vault config operator"
   default     = "-5"
 }
+
+variable "managed_db_host" {
+  type        = string
+  description = "url to managed db based on haproxy"
+}

terraform/gitops/vnext/stateful-resources-config.tf

@@ -10,7 +10,9 @@ module "vnext_stateful_resources" {
   stateful_resources_namespace                  = var.stateful_resources_namespace
   create_stateful_resources_ns                  = false
   kv_path                                       = var.kv_path
-  external_stateful_resource_instance_addresses = length(data.gitlab_project_variable.external_stateful_resource_instance_address) > 0 ? data.gitlab_project_variable.external_stateful_resource_instance_address.*.value : []
+  external_stateful_resource_instance_addresses = local.external_stateful_resource_instance_addresses
+  managed_db_host                               = var.managed_db_host
+
 }
 
 variable "stateful_resources_config_file" {
@@ -32,4 +34,6 @@ locals {
   stateful_resources         = jsondecode(file(var.stateful_resources_config_file))
   enabled_stateful_resources = { for stateful_resource in local.stateful_resources : stateful_resource.resource_name => stateful_resource if stateful_resource.enabled }
   managed_stateful_resources = { for managed_resource in local.enabled_stateful_resources : managed_resource.resource_name => managed_resource if managed_resource.external_service }
+  external_stateful_resource_instance_addresses = { for address in data.gitlab_project_variable.external_stateful_resource_instance_address : address.key => address.value }
+
 }

terraform/gitops/vnext/variables.tf

@@ -162,3 +162,8 @@ variable "switch_dfspid" {
   type    = string
   default = "switch"
 }
+
+variable "managed_db_host" {
+  type        = string
+  description = "url to managed db based on haproxy"
+}

terraform/k8s/ansible-k8s-deploy/terragrunt.hcl

@@ -61,6 +61,9 @@ inputs = {
   ansible_playbook_name         = "argo${local.K8S_CLUSTER_TYPE}_cluster_deploy"
   ansible_destroy_playbook_name = "argo${local.K8S_CLUSTER_TYPE}_cluster_destroy"
   master_node_supports_traffic = (local.total_agent_count == 0) ? true : false
+  stateful_resources_config_file  = find_in_parent_folders("${get_env("CONFIG_PATH")}/mojaloop-stateful-resources.json")
+  current_gitlab_project_id       = local.GITLAB_CURRENT_PROJECT_ID
+
 }
 
 locals {
@@ -74,19 +77,20 @@ locals {
   CLUSTER_NAME                     = get_env("cluster_name")
   NEXUS_DOCKER_REPO_LISTENING_PORT = get_env("NEXUS_DOCKER_REPO_LISTENING_PORT")
   NEXUS_FQDN                       = get_env("NEXUS_FQDN")
+  GITLAB_CURRENT_PROJECT_ID        = get_env("GITLAB_CURRENT_PROJECT_ID")
 
   total_agent_count  = try(sum([for node in local.env_vars.nodes : node.node_count if !node.master]), 0)
   total_master_count = try(sum([for node in local.env_vars.nodes : node.node_count if node.master]), 0)
 
   bastion_hosts_yaml_maps = {
-    netmaker_join_tokens = yamlencode([get_env("NETMAKER_OPS_TOKEN")])
+    netmaker_join_tokens = yamlencode(concat([get_env("NETMAKER_OPS_TOKEN")], [get_env("NETMAKER_ENV_TOKEN")]))
   }
   bastion_hosts_var_maps = {
     netmaker_image_version       = get_env("NETMAKER_VERSION")
     nexus_fqdn                   = get_env("NEXUS_FQDN")
     minio_fqdn                   = get_env("MINIO_FQDN")
     vault_fqdn                   = get_env("VAULT_FQDN")
-    netmaker_master_key          = get_env("METMAKER_MASTER_KEY")
+    netmaker_master_key          = get_env("METMAKER_MASTER_KEY") 
     netmaker_api_host            = get_env("NETMAKER_HOST_NAME")
     root_app_path                = "${local.ARGO_CD_ROOT_APP_PATH}/app-yamls"
     external_secrets_version     = local.common_vars.external_secrets_version

terraform/k8s/default-config/mojaloop-stateful-resources.json

@@ -247,6 +247,7 @@
         "enabled": true,
         "resource_name": "account-lookup-db",
         "resource_type": "mysql",
+        "communication_mode": "tcp", 
         "external_service": false,
         "logical_service_config": {
             "logical_service_port": 3306,
@@ -338,7 +339,7 @@
             "engine": "mysql",
             "engine_version": "5.7",
             "family": "mysql5.7",
-            "instance_class": "db.t2.micro",
+            "instance_class": "db.m5.large",
             "maintenance_window": "Sun:04:00-Sun:06:00",
             "major_engine_version": "5.7",
             "monitoring_interval": "30",
@@ -384,6 +385,7 @@
         "enabled": true,
         "resource_name": "central-ledger-db",
         "resource_type": "mysql",
+        "communication_mode": "tcp", 
         "external_service": false,
         "logical_service_config": {
             "logical_service_port": 3306,
@@ -475,7 +477,7 @@
             "engine": "mysql",
             "engine_version": "5.7",
             "family": "mysql5.7",
-            "instance_class": "db.t2.micro",
+            "instance_class": "db.m5.large",
             "maintenance_window": "Sun:04:00-Sun:06:00",
             "major_engine_version": "5.7",
             "monitoring_interval": "30",
@@ -653,6 +655,8 @@
         "enabled": true,
         "resource_name": "mojaloop-kafka",
         "resource_type": "kafka",
+        "instance_address_key_name": "mojaloop-kafka-brokers",
+        "communication_mode": "tcp",
         "external_service": false,
         "logical_service_config": {
             "logical_service_port": 9092,
@@ -907,6 +911,7 @@
         },
         "external_resource_config": {
             "name": "mks",
+            "port": 9092,
             "kafka_version": "3.4.0",
             "number_of_broker_nodes": 3,
             "enhanced_monitoring": "PER_TOPIC_PER_PARTITION",
@@ -916,7 +921,7 @@
                 }
             },
             "broker_node_instance_type": "kafka.t3.small",
-            "encryption_in_transit_client_broker": "TLS",
+            "encryption_in_transit_client_broker": "PLAINTEXT",
             "encryption_in_transit_in_cluster": true,
             "configuration_name": "example-configuration",
             "configuration_description": "Example configuration",