reorg external hive metastore example and simplify steps (#96)

* reorg external hive metastore example and simplify steps

* removing file

Author: hwang-db

examples/adb-external-hive-metastore/.terraform.lock.hcl

@@ -6,9 +6,8 @@ This architecture will be deployed:
 ![alt text](https://raw.githubusercontent.com/databricks/terraform-databricks-examples/main/examples/adb-external-hive-metastore/images/adb-external-hive-metastore.png?raw=true)
 
 # Get Started:
-There are 2 stages of deployment: stage 1 will deploy all the major infra components including the Databricks workspace and the sql server & database that serves as your external hive metastore. After stage 1 is complete, you need to log into your workspace (this will turn you into the first workspace admin), then you need to navigate into `stage-2-workspace-objects` to deploy remaining components like secret scope, cluster, job, notebook, etc. These are the workspace objects that since we are using `az cli` auth type with Databricks provider at workspace level, we rely on having the caller identity being inside the workspace before stage 2. 
+This template will complete 99% process for external hive metastore deployment with Azure Databricks, using hive version 3.1.0. The last 1% step is just to `run only once` a pre-deployed Databricks job to initialize the external hive metastore. After successful deployment, your cluster can connect to external hive metastore (using azure sql database). 
 
-Stage 1:
 On your local machine:
 
 1. Clone this repository to local.
@@ -24,20 +23,9 @@ On your local machine:
 
     `terraform apply`
 
-After the deployment of stage 1 completes, you should have a Databricks workspace running in your own VNet, a sql server and azure sql database in another VNet, and private link connection from your Databricks VNet to your sql server.
+Now we log into the Databricks workspace, such that you are added into the workspace (since the user identity deployed workspace and have at least Contributor role on the workspace, upon lauching workspace, user identity will be added as workspace admin).
 
-Now we need to manually log into the Databricks workspace, such that you are added into the workspace (since you have Azure contributor role on the workspace resource, at lauch workspace time, you will be added as workspace admin). After first login, you can now proceed to stage 2.
-
-Stage 2:
-1. Navigate into `stage-2-workspace-objects` folder.
-2. Configure input variables, see samples inside provided `terraform.tfvars`. You can get the values from stage 1 outputs.
-3. Init terraform and apply to deploy resources:
-    
-    `terraform init`
-    
-    `terraform apply`
-
-At this step, we've completes most of the work. The final step is to manually trigger the deployed job to run it only once.
+Once logged into workspace, the final step is to manually trigger the pre-deployed job. 
 
 Go to databricks workspace - Job - run the auto-deployed job only once; this is to initialize the database with metastore schema.
 

examples/adb-external-hive-metastore/README.md

@@ -2,17 +2,17 @@ resource "azurerm_key_vault" "akv1" {
   name                        = "${local.prefix}-akv"
   location                    = azurerm_resource_group.this.location
   resource_group_name         = azurerm_resource_group.this.name
-  enabled_for_disk_encryption = true
   tenant_id                   = data.azurerm_client_config.current.tenant_id
+  sku_name                    = "premium"
   soft_delete_retention_days  = 7
   purge_protection_enabled    = false
-  sku_name                    = "standard"
-}
+  enabled_for_disk_encryption = true
+
+  access_policy {
+    tenant_id = data.azurerm_client_config.current.tenant_id
+    object_id = data.azurerm_client_config.current.object_id
 
-resource "azurerm_key_vault_access_policy" "example" {
-  key_vault_id       = azurerm_key_vault.akv1.id
-  tenant_id          = data.azurerm_client_config.current.tenant_id
-  object_id          = data.azurerm_client_config.current.object_id
-  key_permissions    = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore"]
-  secret_permissions = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set"]
+    key_permissions    = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore"]
+    secret_permissions = ["Backup", "Delete", "Get", "List", "Purge", "Recover", "Restore", "Set"]
+  }
 }

examples/adb-external-hive-metastore/akv.tf

@@ -1,6 +1,6 @@
 # automate job to init schema of the database, prep to be hive metastore
 resource "databricks_notebook" "ddl" {
-  source = "../coldstart/metastore_coldstart.py"               #local notebook
+  source = "./coldstart/metastore_coldstart.py"                #local notebook
   path   = "${data.databricks_current_user.me.home}/coldstart" #remote notebook
 }
 

examples/adb-external-hive-metastore/stage-2-workspace-objects/cold_start_metastore.tf renamed to examples/adb-external-hive-metastore/cold_start_metastore.tf

@@ -7,22 +7,6 @@
  * * External Hive Metastore for ADB workspace
  */
 
-provider "azurerm" {
-  features {
-    key_vault {
-      purge_soft_delete_on_destroy = true
-    }
-  }
-}
-
-provider "random" {
-}
-
-# Use Azure CLI to authenticate at Azure Databricks account level, and the Azure Databricks workspace level
-provider "databricks" {
-  host = azurerm_databricks_workspace.this.workspace_url
-}
-
 resource "random_string" "naming" {
   special = false
   upper   = false
@@ -36,17 +20,26 @@ data "external" "me" {
   program = ["az", "account", "show", "--query", "user"]
 }
 
+# Retrieve information about the current user (the caller of tf apply)
+data "databricks_current_user" "me" {
+  depends_on    = [azurerm_databricks_workspace.this]
+}
+
+data "databricks_spark_version" "latest_lts" {
+  long_term_support = true
+  latest            = true
+  depends_on        = [azurerm_databricks_workspace.this]
+}
+
+
 locals {
   // dltp - databricks labs terraform provider
   prefix   = join("-", [var.workspace_prefix, "${random_string.naming.result}"])
   location = var.rglocation
   cidr     = var.spokecidr
   sqlcidr  = var.sqlvnetcidr
   dbfsname = join("", [var.dbfs_prefix, "${random_string.naming.result}"]) // dbfs name must not have special chars
-
-  db_url            = "jdbc:sqlserver://${azurerm_mssql_server.metastoreserver.name}.database.windows.net:1433;database=${azurerm_mssql_database.sqlmetastore.name};user=${var.db_username}@${azurerm_mssql_server.metastoreserver.name};password={${var.db_password}};encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;"
-  db_username_local = var.db_username
-  db_password_local = var.db_password
+  db_url   = "jdbc:sqlserver://${azurerm_mssql_server.metastoreserver.name}.database.windows.net:1433;database=${azurerm_mssql_database.sqlmetastore.name};user=${var.db_username}@${azurerm_mssql_server.metastoreserver.name};password={${var.db_password}};encrypt=true;trustServerCertificate=false;hostNameInCertificate=*.database.windows.net;loginTimeout=30;"
 
   tags = {
     Environment = "Testing"
@@ -56,7 +49,7 @@ locals {
 }
 
 resource "azurerm_resource_group" "this" {
-  name     = "adb-dev-${local.prefix}-rg"
+  name     = "adb-test-${local.prefix}-rg"
   location = local.location
   tags     = local.tags
 }

examples/adb-external-hive-metastore/images/test-metastore.png

@@ -1,5 +1,4 @@
 output "databricks_azure_workspace_resource_id" {
-  // The ID of the Databricks Workspace in the Azure management plane.
   value = azurerm_databricks_workspace.this.id
 }
 
@@ -12,19 +11,3 @@ output "workspace_url" {
 output "resource_group" {
   value = azurerm_resource_group.this.name
 }
-
-output "vault_uri" {
-  value = azurerm_key_vault.akv1.vault_uri
-}
-
-output "key_vault_id" {
-  value = azurerm_key_vault.akv1.id
-}
-
-output "metastoreserver" {
-  value = azurerm_mssql_server.metastoreserver.name
-}
-
-output "metastoredbname" {
-  value = azurerm_mssql_database.sqlmetastore.name
-}

examples/adb-external-hive-metastore/main.tf

@@ -0,0 +1,29 @@
+terraform {
+  required_providers {
+    databricks = {
+      source  = "databricks/databricks"
+      version = ">=1.27.0"
+    }
+
+    azurerm = {
+      source  = "hashicorp/azurerm"
+      version = ">=3.76.0"
+    }
+  }
+}
+
+provider "random" {
+}
+
+provider "azurerm" {
+  features {
+    key_vault {
+      purge_soft_delete_on_destroy = true
+    }
+  }
+}
+
+# Use Azure CLI to authenticate at Azure Databricks account level, and the Azure Databricks workspace level
+provider "databricks" {
+  host = azurerm_databricks_workspace.this.workspace_url
+}

examples/adb-external-hive-metastore/outputs.tf

@@ -2,25 +2,28 @@ resource "databricks_secret_scope" "kv" {
   # akv backed secret scope
   name = "hive"
   keyvault_metadata {
-    resource_id = var.key_vault_id
-    dns_name    = var.vault_uri
+    resource_id = azurerm_key_vault.akv1.id
+    dns_name    = azurerm_key_vault.akv1.vault_uri
   }
 }
 
 resource "azurerm_key_vault_secret" "hiveurl" {
   name         = "HIVE-URL"
   value        = local.db_url
-  key_vault_id = var.key_vault_id
+  key_vault_id = azurerm_key_vault.akv1.id
+  depends_on   = [azurerm_key_vault.akv1]
 }
 
 resource "azurerm_key_vault_secret" "hiveuser" {
   name         = "HIVE-USER"
-  value        = var.db_username # use local group instead of var
-  key_vault_id = var.key_vault_id
+  value        = var.db_username
+  key_vault_id = azurerm_key_vault.akv1.id
+  depends_on   = [azurerm_key_vault.akv1]
 }
 
 resource "azurerm_key_vault_secret" "hivepwd" {
   name         = "HIVE-PASSWORD"
   value        = var.db_password
-  key_vault_id = var.key_vault_id
+  key_vault_id = azurerm_key_vault.akv1.id
+  depends_on   = [azurerm_key_vault.akv1]
 }

examples/adb-external-hive-metastore/providers.tf

@@ -13,7 +13,7 @@ resource "azurerm_mssql_server" "metastoreserver" {
   version                       = "12.0"
   administrator_login           = var.db_username // sensitive data stored as env variables locally
   administrator_login_password  = var.db_password
-  public_network_access_enabled = true // consider to disable public access to the server, to set as false
+  public_network_access_enabled = true // set to false to remove public access
 }
 
 resource "azurerm_mssql_database" "sqlmetastore" {