Update adb-external-hive-metastore to use azurerm v4 (#151)

Signed-off-by: Niko <[email protected]>

Author: r3stl355

examples/adb-external-hive-metastore/README.md

@@ -11,7 +11,7 @@ This template will complete 99% process for external hive metastore deployment w
 On your local machine:
 
 1. Clone this repository to local.
-2. Provide values to variables, some variabes will have default values defined. See inputs section below on optional/required variables.
+2. Update `terraform.tfvars` file and provide values to each defined variable. Some variabes may have default values defined in `variables.tf` file. 
 3. For step 2, variables for db_username and db_password, you can also use your environment variables: terraform will automatically look for environment variables with name format TF_VAR_xxxxx.
 
     `export TF_VAR_db_username=yoursqlserveradminuser`
@@ -56,4 +56,4 @@ for db in dbs:
         f.write(DDL.first()[0])
         f.write("\n")
     f.close()
-```
+```

examples/adb-external-hive-metastore/akv.tf

@@ -7,12 +7,12 @@ resource "azurerm_key_vault" "akv1" {
   soft_delete_retention_days  = 7
   purge_protection_enabled    = false
   enabled_for_disk_encryption = true
+}
 
-  access_policy {
-    tenant_id = data.azurerm_client_config.current.tenant_id
-    object_id = data.azurerm_client_config.current.object_id
-
-    key_permissions    = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore"]
-    secret_permissions = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set"]
-  }
+resource "azurerm_key_vault_access_policy" "this" {
+  key_vault_id       = azurerm_key_vault.akv1.id
+  tenant_id          = data.azurerm_client_config.current.tenant_id
+  object_id          = data.azurerm_client_config.current.object_id
+  key_permissions    = ["Delete", "Get", "List", "Purge", "Recover", "Restore"]
+  secret_permissions = ["Delete", "Get", "List", "Purge", "Recover", "Restore", "Set"]
 }

examples/adb-external-hive-metastore/cold_start_metastore.tf

@@ -6,8 +6,11 @@ resource "databricks_notebook" "ddl" {
 
 resource "databricks_job" "metastoresetup" {
   name                = "Initialize external hive metastore"
-  existing_cluster_id = databricks_cluster.coldstart.id
-  notebook_task {
-    notebook_path = databricks_notebook.ddl.path
+  task {
+    task_key = "task-1"
+    existing_cluster_id = databricks_cluster.coldstart.id
+    notebook_task {
+      notebook_path = databricks_notebook.ddl.path
+    }
   }
 }

examples/adb-external-hive-metastore/main.tf

@@ -1,12 +1,3 @@
-/**
- * this example creates:
- * * Resource group with random prefix
- * * Tags, including `Owner`, which is taken from `az account show --query user`
- * * VNet with public and private subnet
- * * Databricks workspace
- * * External Hive Metastore for ADB workspace
- */
-
 resource "random_string" "naming" {
   special = false
   upper   = false
@@ -33,17 +24,16 @@ data "databricks_spark_version" "latest_lts" {
 
 
 locals {
-  // dltp - databricks labs terraform provider
-  prefix   = join("-", [var.workspace_prefix, "${random_string.naming.result}"])
-  location = var.rglocation
-  cidr     = var.spokecidr
-  sqlcidr  = var.sqlvnetcidr
-  dbfsname = join("", [var.dbfs_prefix, "${random_string.naming.result}"]) // dbfs name must not have special chars
-  db_url   = "jdbc:sqlserver://${azurerm_mssql_server.metastoreserver.name}.database.windows.net:1433;database=${azurerm_mssql_database.sqlmetastore.name};user=${var.db_username}@${azurerm_mssql_server.metastoreserver.name};password={${var.db_password}};encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;"
-
+  prefix      = join("-", [var.workspace_prefix, "${random_string.naming.result}"])
+  location    = var.rglocation
+  cidr        = var.spokecidr
+  sqlcidr     = var.sqlvnetcidr
+  dbfsname    = join("", [var.dbfs_prefix, "${random_string.naming.result}"]) // dbfs name must not have special chars
+  db_url      = "jdbc:sqlserver://${azurerm_mssql_server.metastoreserver.name}.database.windows.net:1433;database=${azurerm_mssql_database.sqlmetastore.name};user=${var.db_username}@${azurerm_mssql_server.metastoreserver.name};password={${var.db_password}};encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;"
+  my_username = lookup(data.external.me.result, "name")
   tags = {
     Environment = "Testing"
-    Owner       = lookup(data.external.me.result, "name")
+    Owner       = local.my_username
     Epoch       = random_string.naming.result
   }
 }

examples/adb-external-hive-metastore/outputs.tf

@@ -1,13 +1,29 @@
 output "databricks_azure_workspace_resource_id" {
-  value = azurerm_databricks_workspace.this.id
+  description = "**Depricated**"
+  value       = azurerm_databricks_workspace.this.id
 }
 
-output "workspace_url" {
-  // The workspace URL which is of the format 'adb-{workspaceId}.{random}.azuredatabricks.net'
-  // this is not named as DATABRICKS_HOST, because it affect authentication
-  value = "https://${azurerm_databricks_workspace.this.workspace_url}/"
+output "resource_group" {
+  description = "**Depricated**"
+  value       = azurerm_resource_group.this.name
 }
 
-output "resource_group" {
-  value = azurerm_resource_group.this.name
+output "azure_resource_group_id" {
+  description = "The Azure resource group ID"
+  value       = azurerm_resource_group.this.id
+}
+
+output "workspace_id" {
+  description = "The Databricks workspace ID"
+  value       = azurerm_databricks_workspace.this.workspace_id
 }
+
+output "workspace_url" {
+  description = "The Databricks workspace URL"
+  value       = "https://${azurerm_databricks_workspace.this.workspace_url}/"
+}
+
+output "keyvault_id" {
+  description = "The Azure KeyVault ID"
+  value       = azurerm_key_vault.akv1.id
+}

examples/adb-external-hive-metastore/providers.tf

@@ -2,28 +2,25 @@ terraform {
   required_providers {
     databricks = {
       source  = "databricks/databricks"
-      version = ">=1.27.0"
+      version = ">=1.52.0"
     }
-
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = ">=3.76.0"
+      version = ">=4.0.0"
     }
   }
 }
 
-provider "random" {
-}
-
 provider "azurerm" {
+  subscription_id = var.subscription_id
   features {
     key_vault {
       purge_soft_delete_on_destroy = true
     }
   }
 }
 
-# Use Azure CLI to authenticate at Azure Databricks account level, and the Azure Databricks workspace level
+# This will be used to manage Azure Databricks workspace resources (Azure Databricks workspace itself is managed by `azurerm` provider)
 provider "databricks" {
   host = azurerm_databricks_workspace.this.workspace_url
 }

examples/adb-external-hive-metastore/secrets.tf

@@ -1,6 +1,7 @@
 resource "databricks_secret_scope" "kv" {
   # akv backed secret scope
-  name = "hive"
+  name                      = "hive"
+  initial_manage_principal  = "users"
   keyvault_metadata {
     resource_id = azurerm_key_vault.akv1.id
     dns_name    = azurerm_key_vault.akv1.vault_uri
@@ -11,19 +12,19 @@ resource "azurerm_key_vault_secret" "hiveurl" {
   name         = "HIVE-URL"
   value        = local.db_url
   key_vault_id = azurerm_key_vault.akv1.id
-  depends_on   = [azurerm_key_vault.akv1]
+  depends_on   = [azurerm_key_vault_access_policy.this]
 }
 
 resource "azurerm_key_vault_secret" "hiveuser" {
   name         = "HIVE-USER"
   value        = var.db_username
   key_vault_id = azurerm_key_vault.akv1.id
-  depends_on   = [azurerm_key_vault.akv1]
+  depends_on   = [azurerm_key_vault_access_policy.this]
 }
 
 resource "azurerm_key_vault_secret" "hivepwd" {
   name         = "HIVE-PASSWORD"
   value        = var.db_password
   key_vault_id = azurerm_key_vault.akv1.id
-  depends_on   = [azurerm_key_vault.akv1]
+  depends_on   = [azurerm_key_vault_access_policy.this]
 }

examples/adb-external-hive-metastore/terraform.tfvars

@@ -0,0 +1,3 @@
+subscription_id = "<your Azure Subscription ID here>"
+db_username     = "<yoursqlserveradminuser>"
+db_password     = "<yoursqlserveradminpassword>"

examples/adb-external-hive-metastore/variables.tf

@@ -1,3 +1,8 @@
+variable "subscription_id" {
+  type        = string
+  description = "Azure Subscription ID to deploy the workspace into"
+}
+
 variable "spokecidr" {
   type    = string
   default = "10.179.0.0/20"
@@ -8,11 +13,6 @@ variable "sqlvnetcidr" {
   default = "10.178.0.0/20"
 }
 
-variable "no_public_ip" {
-  type    = bool
-  default = true
-}
-
 variable "rglocation" {
   type    = string
   default = "southeastasia"

examples/adb-external-hive-metastore/vnet.tf

@@ -42,7 +42,7 @@ resource "azurerm_subnet" "private" {
   virtual_network_name = azurerm_virtual_network.this.name
   address_prefixes     = [cidrsubnet(local.cidr, 3, 1)]
 
-  private_endpoint_network_policies_enabled     = true
+  private_endpoint_network_policies             = "Enabled"
   private_link_service_network_policies_enabled = true
 
   delegation {
@@ -70,7 +70,7 @@ resource "azurerm_subnet" "plsubnet" {
   resource_group_name                       = azurerm_resource_group.this.name
   virtual_network_name                      = azurerm_virtual_network.this.name
   address_prefixes                          = [cidrsubnet(local.cidr, 3, 2)]
-  private_endpoint_network_policies_enabled = true
+  private_endpoint_network_policies         = "Enabled"
 }
 
 

examples/adb-external-hive-metastore/workspace.tf

@@ -6,7 +6,6 @@ resource "azurerm_databricks_workspace" "this" {
   tags                         = local.tags
   customer_managed_key_enabled = true
   custom_parameters {
-    no_public_ip                                         = var.no_public_ip
     virtual_network_id                                   = azurerm_virtual_network.this.id
     private_subnet_name                                  = azurerm_subnet.private.name
     public_subnet_name                                   = azurerm_subnet.public.name
@@ -26,6 +25,8 @@ resource "databricks_cluster" "coldstart" {
   cluster_name            = "cluster - external metastore"
   spark_version           = data.databricks_spark_version.latest_lts.id
   node_type_id            = var.node_type
+  data_security_mode      = "SINGLE_USER"
+  single_user_name        = local.my_username
   autotermination_minutes = 30
   autoscale {
     min_workers = 1