From 4d8d141afd35bba18cb9855ae382a0f6891dbb17 Mon Sep 17 00:00:00 2001 From: Siddharth-Ponugoti <85951337+Siddharth-Ponugoti@users.noreply.github.com> Date: Mon, 7 Apr 2025 10:17:15 +0100 Subject: [PATCH] Update- I have updated the AWS modular private link repo to allow the creation of multiple PL and Non-PL enabled workspaces --- .../README.md | 4 +++- .../main.tf | 7 +++++-- .../modules/mws_workspace/main.tf | 5 ++--- .../mws_workspace/modules/mws_network/main.tf | 11 ++++++---- .../modules/mws_network/variables.tf | 4 ++++ .../modules/mws_workspace/variables.tf | 4 ++++ .../variables.tf | 20 ++++++++++++++++++- 7 files changed, 44 insertions(+), 11 deletions(-) diff --git a/examples/aws-databricks-modular-privatelink/README.md b/examples/aws-databricks-modular-privatelink/README.md index a681f331..5359de32 100644 --- a/examples/aws-databricks-modular-privatelink/README.md +++ b/examples/aws-databricks-modular-privatelink/README.md @@ -1,7 +1,7 @@ Deploy Multiple AWS Databricks Workspace with CMK, Customer-managed VPC, Private Links, IP Access Lists ========================= -In this example, we created modules and root level template to deploy multiple (e.g. 10+) E2 Databricks workspaces at scale easily. Users of this template minimally should do these: +In this example, we created modules and root level template to deploy multiple (e.g. 10+) E2 Privatelink and Non-privatelink Databricks workspaces at scale easily. Users of this template minimally should do these: 1. Supply credentials (aws+databricks) and configuration variables for each workspaces 2. Edit the locals block in `main.tf` to decide what & how many workspaces to deploy 3. Run `terraform init` and `terraform apply` to deploy 1 or more workspaces into your VPC. @@ -72,6 +72,7 @@ variable "workspace_1_config" { prefix = "ws1" // prefix decides subnets name region = "ap-southeast-1" root_bucket_name = "test-workspace-1-rootbucket" + enable_privatelink = true // Switch to false if you don't want your workspace to use Privatelink. Please note once PL is enabled you can't disable it block_list = ["58.133.93.159"] allow_list = [] // if allow_list empty, all public IP not blocked by block_list are allowed tags = { @@ -92,6 +93,7 @@ workspace_confs = { workspace_1 = var.workspace_1_config workspace_2 = var.workspace_2_config workspace_3 = var.workspace_3_config + workspace_4 = var.workspace_4_config } ``` diff --git a/examples/aws-databricks-modular-privatelink/main.tf b/examples/aws-databricks-modular-privatelink/main.tf index 0c2e41d3..047fe485 100644 --- a/examples/aws-databricks-modular-privatelink/main.tf +++ b/examples/aws-databricks-modular-privatelink/main.tf @@ -13,6 +13,7 @@ locals { workspace_confs = { //add more workspaces here, remove from here to delete specific workspace workspace_1 = var.workspace_1_config workspace_2 = var.workspace_2_config + workspace_3 = var.workspace_3_config } } @@ -43,13 +44,14 @@ module "workspace_collection" { tags = each.value.tags existing_vpc_id = aws_vpc.mainvpc.id nat_gateways_id = aws_nat_gateway.nat_gateways[0].id + enable_privatelink = each.value.enable_privatelink security_group_ids = [aws_security_group.sg.id] private_subnet_pair = [each.value.private_subnet_pair.subnet1_cidr, each.value.private_subnet_pair.subnet2_cidr] workspace_storage_cmk = module.databricks_cmk.workspace_storage_cmk managed_services_cmk = module.databricks_cmk.managed_services_cmk root_bucket_name = each.value.root_bucket_name - relay_vpce_id = [databricks_mws_vpc_endpoint.relay.vpc_endpoint_id] - rest_vpce_id = [databricks_mws_vpc_endpoint.backend_rest_vpce.vpc_endpoint_id] + relay_vpce_id = each.value.enable_privatelink ? [databricks_mws_vpc_endpoint.relay.vpc_endpoint_id] : null + rest_vpce_id = each.value.enable_privatelink ? [databricks_mws_vpc_endpoint.backend_rest_vpce.vpc_endpoint_id] : null depends_on = [ databricks_mws_vpc_endpoint.relay, databricks_mws_vpc_endpoint.backend_rest_vpce @@ -72,3 +74,4 @@ resource "local_file" "deployment_information" { }) filename = "./artifacts/${each.key}.json" } + diff --git a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/main.tf b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/main.tf index 0b782ef8..8282c18a 100644 --- a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/main.tf +++ b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/main.tf @@ -9,6 +9,7 @@ module "my_mws_network" { prefix = "${var.prefix}-network" relay_vpce_id = var.relay_vpce_id rest_vpce_id = var.rest_vpce_id + enable_privatelink = var.enable_privatelink tags = var.tags } @@ -38,7 +39,6 @@ resource "databricks_mws_customer_managed_keys" "managed_services" { use_cases = ["MANAGED_SERVICES"] } - resource "databricks_mws_private_access_settings" "pas" { account_id = var.databricks_account_id private_access_settings_name = "Private Access Settings for ${var.prefix}" @@ -47,12 +47,11 @@ resource "databricks_mws_private_access_settings" "pas" { private_access_level = "ACCOUNT" // a fix for recent changes - 202209 } - resource "databricks_mws_workspaces" "this" { account_id = var.databricks_account_id aws_region = var.region workspace_name = var.workspace_name - private_access_settings_id = databricks_mws_private_access_settings.pas.private_access_settings_id + private_access_settings_id = var.enable_privatelink ? databricks_mws_private_access_settings.pas.private_access_settings_id : null pricing_tier = "ENTERPRISE" # deployment_name = local.prefix diff --git a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/main.tf b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/main.tf index fc534cd2..8b8f851f 100644 --- a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/main.tf +++ b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/main.tf @@ -34,11 +34,14 @@ resource "databricks_mws_networks" "mwsnetwork" { account_id = var.databricks_account_id network_name = "${var.prefix}-network" vpc_id = var.existing_vpc_id - subnet_ids = [aws_subnet.private_subnets.0.id, aws_subnet.private_subnets.1.id] + subnet_ids = [aws_subnet.private_subnets[0].id, aws_subnet.private_subnets[1].id] security_group_ids = var.security_group_ids - vpc_endpoints { - dataplane_relay = var.relay_vpce_id - rest_api = var.rest_vpce_id + dynamic "vpc_endpoints" { + for_each = var.enable_privatelink ? [1] : [] + content { + dataplane_relay = var.relay_vpce_id + rest_api = var.rest_vpce_id + } } } diff --git a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/variables.tf b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/variables.tf index 899ec83e..25f3c0d3 100644 --- a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/variables.tf +++ b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/modules/mws_network/variables.tf @@ -39,3 +39,7 @@ variable "rest_vpce_id" { variable "tags" { type = map(string) } + +variable "enable_privatelink" { + type = bool +} \ No newline at end of file diff --git a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/variables.tf b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/variables.tf index 775b905b..4c22b01e 100644 --- a/examples/aws-databricks-modular-privatelink/modules/mws_workspace/variables.tf +++ b/examples/aws-databricks-modular-privatelink/modules/mws_workspace/variables.tf @@ -56,3 +56,7 @@ variable "rest_vpce_id" { variable "tags" { type = map(string) } + +variable "enable_privatelink" { + type = bool +} \ No newline at end of file diff --git a/examples/aws-databricks-modular-privatelink/variables.tf b/examples/aws-databricks-modular-privatelink/variables.tf index 10cf2d13..0f009710 100644 --- a/examples/aws-databricks-modular-privatelink/variables.tf +++ b/examples/aws-databricks-modular-privatelink/variables.tf @@ -61,8 +61,9 @@ variable "workspace_1_config" { workspace_name = "test-workspace-1" prefix = "ws1" // prefix decides subnets name region = "ap-southeast-1" + enable_privatelink = true root_bucket_name = "test-workspace-1-rootbucket" - block_list = ["58.133.93.159"] + block_list = ["54.112.179.135", "195.78.164.130"] allow_list = ["65.184.145.97"] // if allow_list empty, all public IP not blocked by block_list are allowed tags = { "Name" = "test-workspace-1-tags", @@ -78,6 +79,7 @@ variable "workspace_2_config" { prefix = "ws2" // prefix decides subnets name region = "ap-southeast-1" root_bucket_name = "test-workspace-2-rootbucket" + enable_privatelink = true block_list = ["54.112.179.135", "195.78.164.130"] allow_list = ["65.184.145.97"] // if allow_list empty, all public IP not blocked by block_list are allowed tags = { @@ -85,3 +87,19 @@ variable "workspace_2_config" { } } } + +variable "workspace_3_config" { + default = { + private_subnet_pair = { subnet1_cidr = "10.109.18.0/23", subnet2_cidr = "10.109.20.0/23" } + workspace_name = "test-workspace-3" + prefix = "ws3" // prefix decides subnets name + region = "ap-southeast-1" + root_bucket_name = "test-workspace-3-rootbucket" + enable_privatelink = false + block_list = ["54.112.179.135", "195.78.164.130"] + allow_list = ["65.184.145.97"] // if allow_list empty, all public IP not blocked by block_list are allowed + tags = { + "Name" = "test-workspace-3-tags" + } + } +}