Fix: Quick fix to solve some issues with adb-lakehouse example (#59)

yessawab · web-flow · commit 501a4ce2c8f4 · 2023-04-24T18:30:47.000+02:00
* Fix: Quick fix to solve some issues with adb-lakehouse example

* Fix: Quick fix to solve some issues with adb-lakehouse example
diff --git a/examples/adb-lakehouse/README.md b/examples/adb-lakehouse/README.md
@@ -1,7 +1,10 @@
 # Lakehouse terraform blueprints
 
 This example contains Terraform code used to provision a Lakehouse platform using the [adb-lakehouse module](../../modules/adb-lakehouse).
-It also contains Terraform code to create Unity Catalog metastore and multiple UC resources, it also creates principals in Databricks account and assign them to Databricks workspace.
+It also contains Terraform code to create the following: 
+* Unity Catalog metastore 
+* Unity Catalog resources: Catalog, Schema, table, storage credential and external location
+* New principals in the Databricks account and assign them to the Databricks workspace.
 
 ## Deployed resources
 
diff --git a/examples/adb-lakehouse/main.tf b/examples/adb-lakehouse/main.tf
@@ -26,6 +26,7 @@ module "adb-lakehouse-uc-metastore" {
   access_connector_name  = var.access_connector_name
   metastore_id           = module.adb-lakehouse-uc-metastore.metastore_id
   workspace_id           = module.adb-lakehouse.workspace_id
+  metastore_admins       = var.metastore_admins
   providers = {
     databricks = databricks.workspace
   }
@@ -48,8 +49,10 @@ module "adb-lakehouse-data-assets" {
   environment_name               = var.environment_name
   storage_credential_name        = var.access_connector_name
   metastore_id                   = module.adb-lakehouse-uc-metastore.metastore_id
+  access_connector_id            = module.adb-lakehouse.access_connector_principal_id
   landing_external_location_name = var.landing_external_location_name
   landing_adls_path              = var.landing_adls_path
+  landing_adls_rg                = var.landing_adls_rg
   metastore_admins               = var.metastore_admins
   providers = {
     databricks = databricks.workspace
diff --git a/examples/adb-lakehouse/providers.tf b/examples/adb-lakehouse/providers.tf
@@ -11,5 +11,5 @@ provider "databricks" {
 
 provider "databricks" {
   alias = "workspace"
-  host  = "<workspace_url>"
+  host  = module.adb-lakehouse.workspace_url
 }
diff --git a/examples/adb-lakehouse/variables.tf b/examples/adb-lakehouse/variables.tf
@@ -109,6 +109,11 @@ variable "landing_adls_path" {
   description = "The ADLS path of the landing zone"
 }
 
+variable "landing_adls_rg" {
+  type        = string
+  description = "The resource group name of the landing zone"
+}
+
 variable "metastore_admins" {
   type        = list(string)
   description = "list of principals: service principals or groups that have metastore admin privileges"
diff --git a/examples/adb-lakehouse/versions.tf b/examples/adb-lakehouse/versions.tf
@@ -1,10 +1,10 @@
 terraform {
   required_providers {
     azurerm = {
-      source  = "hashicorp/azurerm"
+      source = "hashicorp/azurerm"
     }
     databricks = {
-      source  = "databricks/databricks"
+      source = "databricks/databricks"
     }
   }
 }
diff --git a/modules/adb-lakehouse-uc/account-principals/providers.tf b/modules/adb-lakehouse-uc/account-principals/providers.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     azurerm = {
-      source  = "hashicorp/azurerm"
+      source = "hashicorp/azurerm"
     }
     databricks = {
       source = "databricks/databricks"
diff --git a/modules/adb-lakehouse-uc/uc-data-assets/providers.tf b/modules/adb-lakehouse-uc/uc-data-assets/providers.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     azurerm = {
-      source  = "hashicorp/azurerm"
+      source = "hashicorp/azurerm"
     }
     databricks = {
       source = "databricks/databricks"
diff --git a/modules/adb-lakehouse-uc/uc-data-assets/uc-data-assets.tf b/modules/adb-lakehouse-uc/uc-data-assets/uc-data-assets.tf
@@ -12,8 +12,20 @@ resource "databricks_schema" "bronze_source1-schema" {
   force_destroy = true
 }
 
+data "azurerm_storage_account" "ext_storage" {
+  name                = var.landing_external_location_name
+  resource_group_name = var.landing_adls_rg
+}
+
+resource "azurerm_role_assignment" "ext_storage" {
+  scope                = data.azurerm_storage_account.ext_storage.id
+  role_definition_name = "Storage Blob Data Contributor"
+  principal_id         = var.access_connector_id
+}
+
 resource "databricks_external_location" "landing-external-location" {
   name            = var.landing_external_location_name
   url             = var.landing_adls_path
   credential_name = var.storage_credential_name
+  comment         = "Created by TF"
 }
diff --git a/modules/adb-lakehouse-uc/uc-data-assets/variables.tf b/modules/adb-lakehouse-uc/uc-data-assets/variables.tf
@@ -13,6 +13,11 @@ variable "landing_adls_path" {
   description = "The ADLS path of the landing zone"
 }
 
+variable "landing_adls_rg" {
+  type        = string
+  description = "The resource group name of the landing zone"
+}
+
 variable "storage_credential_name" {
   type        = string
   description = "the name of the storage credential"
@@ -27,3 +32,8 @@ variable "metastore_admins" {
   type        = list(string)
   description = "list of principals: service principals or groups that have metastore admin privileges"
 }
+
+variable "access_connector_id" {
+  type        = string
+  description = "the id of the access connector"
+}
diff --git a/modules/adb-lakehouse-uc/uc-metastore/providers.tf b/modules/adb-lakehouse-uc/uc-metastore/providers.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     azurerm = {
-      source  = "hashicorp/azurerm"
+      source = "hashicorp/azurerm"
     }
     databricks = {
       source = "databricks/databricks"
diff --git a/modules/adb-lakehouse-uc/uc-metastore/variables.tf b/modules/adb-lakehouse-uc/uc-metastore/variables.tf
@@ -29,3 +29,7 @@ variable "workspace_id" {
   description = "the id of the workspace"
 }
 
+variable "metastore_admins" {
+  type        = list(string)
+  description = "list of principals: service principals or groups that have metastore admin privileges"
+}
diff --git a/modules/adb-lakehouse/azure_data_factory.tf b/modules/adb-lakehouse/azure_data_factory.tf
@@ -1,6 +1,6 @@
 resource "azurerm_data_factory" "adf" {
   name                = var.data_factory_name
   location            = var.location
-  resource_group_name = var.spoke_resource_group_name
+  resource_group_name = azurerm_resource_group.this.name
   tags                = var.tags
 }
diff --git a/modules/adb-lakehouse/key_vault.tf b/modules/adb-lakehouse/key_vault.tf
@@ -1,7 +1,7 @@
 resource "azurerm_key_vault" "example" {
   name                        = var.key_vault_name
   location                    = var.location
-  resource_group_name         = var.spoke_resource_group_name
+  resource_group_name         = azurerm_resource_group.this.name
   enabled_for_disk_encryption = true
   tenant_id                   = data.azurerm_client_config.current.tenant_id
   soft_delete_retention_days  = 7
diff --git a/modules/adb-lakehouse/outputs.tf b/modules/adb-lakehouse/outputs.tf
@@ -41,4 +41,9 @@ output "workspace_url" {
 output "access_connector_id" {
   value       = azurerm_databricks_access_connector.access_connector.id
   description = "the id of the access connector"
+}
+
+output "access_connector_principal_id" {
+  value       = azurerm_databricks_access_connector.access_connector.identity[0].principal_id
+  description = "The Principal ID of the System Assigned Managed Service Identity that is configured on this Access Connector"
 }
diff --git a/modules/adb-lakehouse/versions.tf b/modules/adb-lakehouse/versions.tf
@@ -1,7 +1,7 @@
 terraform {
   required_providers {
     azurerm = {
-      source  = "hashicorp/azurerm"
+      source = "hashicorp/azurerm"
     }
   }
 }

Original file line number	Diff line number	Diff line change
`@@ -11,5 +11,5 @@ provider "databricks" {`
`11`	`11`
`12`	`12`	`provider "databricks" {`
`13`	`13`	`alias = "workspace"`
`14`		`- host = "<workspace_url>"`
	`14`	`+ host = module.adb-lakehouse.workspace_url`
`15`	`15`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,10 +1,10 @@`
`1`	`1`	`terraform {`
`2`	`2`	`required_providers {`
`3`	`3`	`azurerm = {`
`4`		`- source = "hashicorp/azurerm"`
	`4`	`+ source = "hashicorp/azurerm"`
`5`	`5`	`}`
`6`	`6`	`databricks = {`
`7`		`- source = "databricks/databricks"`
	`7`	`+ source = "databricks/databricks"`
`8`	`8`	`}`
`9`	`9`	`}`
`10`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,7 @@`
`1`	`1`	`terraform {`
`2`	`2`	`required_providers {`
`3`	`3`	`azurerm = {`
`4`		`- source = "hashicorp/azurerm"`
	`4`	`+ source = "hashicorp/azurerm"`
`5`	`5`	`}`
`6`	`6`	`databricks = {`
`7`	`7`	`source = "databricks/databricks"`
Original file line number	Diff line number	Diff line change
`@@ -29,3 +29,7 @@ variable "workspace_id" {`
`29`	`29`	`description = "the id of the workspace"`
`30`	`30`	`}`
`31`	`31`
	`32`	`+variable "metastore_admins" {`
	`33`	`+ type = list(string)`
	`34`	`+ description = "list of principals: service principals or groups that have metastore admin privileges"`
	`35`	`+}`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`resource "azurerm_data_factory" "adf" {`
`2`	`2`	`name = var.data_factory_name`
`3`	`3`	`location = var.location`
`4`		`- resource_group_name = var.spoke_resource_group_name`
	`4`	`+ resource_group_name = azurerm_resource_group.this.name`
`5`	`5`	`tags = var.tags`
`6`	`6`	`}`