Add multiple AWS and Azure Databricks examples (#17)

* add more aws examples and teradata demo example

* change VM type to support nested virtualization

* update VM size type to work with gen2 image

* add workspace and vnet configs

* add config script for td vm

* update module and readme for Teradata

* add readme content

* update readme

* readme update

* update vm nsg for TD

* clean up repo

* readme

* readme

* replace underscore with dash for foldernames

* add adb-kafka example

* remove simple stuff

Author: hwang-db

examples/adb-basic-demo/outputs.tf

@@ -1,15 +1,11 @@
 output "databricks_azure_workspace_resource_id" {
-  // The ID of the Databricks Workspace in the Azure management plane.
   value = azurerm_databricks_workspace.example.id
 }
 
 output "workspace_url" {
-  // The workspace URL which is of the format 'adb-{workspaceId}.{random}.azuredatabricks.net'
-  // this is not named as DATABRICKS_HOST, because it affect authentication
   value = "https://${azurerm_databricks_workspace.example.workspace_url}/"
 }
 
 output "module_cluster_id" {
-  // reference to module's outputs: value = module.module_name.output_attr_name
   value = module.auto_scaling_cluster_example.cluster_id
 }

examples/adb-kafka/.terraform.lock.hcl

@@ -0,0 +1,92 @@
+## ADB - Kafka Single VM Demo environment
+
+This template provisions a single VM and Azure Databricks workspace, installation of Kafka service is a manual step. Major components to deploy include:
+- 1 Vnet with 3 subnets (2 for Databricks, 1 for Kafka VM)
+- 1 Azure VM (to host Kafka and Zookeeper services), with port 9092 exposed to other devices in same VNet (allowed by default NSG rules).
+- 1 VNet injected Azure Databricks Workspace
+- NSGs for Databricks and Kafka subnets
+
+## Folder Structure
+    .
+    ├── main.tf
+    ├── outputs.tf
+    ├── data.tf
+    ├── providers.tf
+    ├── variables.tf
+    ├── vnet.tf
+    ├── workspace.tf
+    ├── terraform.tfvars
+    ├── charts
+    ├── modules
+        ├── general_vm
+            ├── main.tf
+            ├── outputs.tf      
+            ├── providers.tf
+            ├── variables.tf
+
+`terraform.tfvars` is provided as reference variable values, you should change it based on your need.
+
+## Getting Started
+
+> Step 1: Preparation
+
+Clone this repo to your local, and run `az login` to interactively login thus get authenticated with `azurerm` provider.
+
+> Step 2: Deploy resources
+
+Change the `terraform.tfvars` to your need (you can also leave as default values as a random string will be generated in prefix), then run:
+```bash
+terraform init
+terraform apply
+```
+This will deploy all resources wrapped in a new resource group to your the default subscription of your `az login` profile; you will see the public ip address of the VM after the deployment is done. After deployment, you will get below resources:
+
+![alt text](./charts/resources.png?raw=true)
+
+> Step 3: Configure your VM to run Kafka and Zookeeper services
+
+At this moment, you have a vanilla VM without any bootstraping performed. We are to manually log into the VM and install Kafka and Zookeeper services.
+
+The VM's private key has been generated for you in local folder; replace the public ip accordingly. SSH into VM by (azureuser is the hardcoded username for VMs in this template):
+
+```bash
+ssh -i <private_key_local_path> azureuser@<public_ip>
+```
+
+Now you should follow this [guide from DigitalOcean](https://www.digitalocean.com/community/tutorials/how-to-install-apache-kafka-on-ubuntu-20-04) to install Kafka on the VM. Note that a few commands need to be updated:
+1. When downloading the kafka binary, go to https://kafka.apache.org/downloads.html and copy the latest binary link and replace it here:
+```bash
+curl "https://downloads.apache.org/kafka/3.3.2/kafka_2.12-3.3.2.tgz" -o ~/Downloads/kafka.tgz
+```
+
+![alt text](./charts/kafka-download.png?raw=true)
+
+2. When testing your Kafka installation, --zookeeper is deprecated, use --bootstrap-server instead:
+   
+```bash
+~/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic TutorialTopic
+```
+
+At the end of the guide, you should have a running Kafka service on your VM. You can test it by running the following command:
+```bash
+sudo systemctl status kafka
+```
+
+![alt text](./charts/test-kafka.png?raw=true)
+
+> Step 4: Integration with Azure Databricks
+
+Now your Kafka Broker is running; let's connect to it in Databricks. 
+We first create a topic `TutorialTopic2` in Kafka via your VM's Command Line:
+
+```bash
+~/kafka/bin/kafka-topics.sh --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic TutorialTopic2
+```
+
+Then we can write from Spark DataFrame to this topic; you can also test the connection by `telnet vm-private-ip 9092` first.
+
+![alt text](./charts/write-to-kafka.png?raw=true)
+
+Read from this topic in another stream job:
+
+![alt text](./charts/read-kafka.png?raw=true)

examples/adb-kafka/README.md

@@ -0,0 +1,6 @@
+data "azurerm_client_config" "current" {
+}
+
+data "external" "me" {
+  program = ["az", "account", "show", "--query", "user"]
+}

examples/adb-kafka/charts/kafka-download.png

@@ -0,0 +1,31 @@
+resource "random_string" "naming" {
+  special = false
+  upper   = false
+  length  = 6
+}
+
+locals {
+  prefix   = join("-", [var.workspace_prefix, "${random_string.naming.result}"])
+  location = var.rglocation
+  dbfsname = join("", [var.dbfs_prefix, "${random_string.naming.result}"]) // dbfs name must not have special chars
+
+  tags = {
+    Environment = "Testing"
+    Owner       = lookup(data.external.me.result, "name")
+    Epoch       = random_string.naming.result
+  }
+}
+
+resource "azurerm_resource_group" "this" {
+  name     = "adb-${local.prefix}-rg"
+  location = local.location
+  tags     = local.tags
+}
+
+module "kafka_broker" {
+  source              = "./modules/general_vm"
+  resource_group_name = azurerm_resource_group.this.name
+  vm_name             = "broker01"
+  region              = local.location
+  subnet_id           = azurerm_subnet.vm_subnet.id
+}

examples/adb-kafka/charts/read-kafka.png

@@ -0,0 +1,79 @@
+resource "azurerm_network_interface" "general-nic" {
+  name                = "${var.vm_name}-nic"
+  location            = var.region
+  resource_group_name = var.resource_group_name
+
+  ip_configuration {
+    name                          = "internal"
+    subnet_id                     = var.subnet_id
+    private_ip_address_allocation = "Dynamic"
+    public_ip_address_id          = azurerm_public_ip.general-nic-pubip.id
+  }
+}
+
+resource "tls_private_key" "general_ssh" {
+  algorithm = "RSA"
+  rsa_bits  = 4096
+}
+
+resource "local_file" "private_key" {
+  content         = tls_private_key.general_ssh.private_key_pem
+  filename        = "${var.vm_name}_ssh_private.pem"
+  file_permission = "0600"
+}
+
+resource "azurerm_public_ip" "general-nic-pubip" {
+  name                = "${var.vm_name}-nic-pubip"
+  resource_group_name = var.resource_group_name
+  location            = var.region
+  allocation_method   = "Static"
+}
+
+resource "azurerm_linux_virtual_machine" "general_vm" {
+  name                = "${var.vm_name}-vm"
+  resource_group_name = var.resource_group_name
+  location            = var.region
+  size                = "Standard_D16s_v3"
+  admin_username      = "azureuser"
+
+  network_interface_ids = [
+    azurerm_network_interface.general-nic.id,
+  ]
+
+  admin_ssh_key {
+    username   = "azureuser"
+    public_key = tls_private_key.general_ssh.public_key_openssh // using generated ssh key
+  }
+
+  os_disk {
+    caching              = "ReadWrite"
+    storage_account_type = "Standard_LRS"
+  }
+
+  source_image_reference {
+    publisher = "Canonical"
+    offer     = "0001-com-ubuntu-server-focal"
+    sku       = "20_04-lts-gen2"
+    version   = "latest"
+  }
+
+  depends_on = [
+    local_file.private_key,
+  ]
+}
+
+resource "azurerm_managed_disk" "general_disk" {
+  name                 = "${var.vm_name}-disk"
+  location             = var.region
+  resource_group_name  = var.resource_group_name
+  storage_account_type = "Standard_LRS"
+  create_option        = "Empty"
+  disk_size_gb         = 60
+}
+
+resource "azurerm_virtual_machine_data_disk_attachment" "diskattachment" {
+  managed_disk_id    = azurerm_managed_disk.general_disk.id
+  virtual_machine_id = azurerm_linux_virtual_machine.general_vm.id
+  lun                = "10"
+  caching            = "ReadWrite"
+}

examples/adb-kafka/charts/resources.png

@@ -0,0 +1,3 @@
+output "vm_public_ip" {
+  value = azurerm_public_ip.general-nic-pubip.ip_address
+}

examples/adb-kafka/charts/test-kafka.png

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source = "hashicorp/azurerm"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}

examples/adb-kafka/charts/write-to-kafka.png

@@ -0,0 +1,15 @@
+variable "resource_group_name" {
+  type = string
+}
+
+variable "region" {
+  type = string
+}
+
+variable "subnet_id" {
+  type = string
+}
+
+variable "vm_name" {
+  type = string
+}

examples/adb-kafka/data.tf

@@ -0,0 +1,3 @@
+output "pip" {
+  value = module.kafka_broker.vm_public_ip
+}

examples/adb-kafka/main.tf

@@ -0,0 +1,14 @@
+terraform {
+  required_providers {
+    azurerm = {
+      source = "hashicorp/azurerm"
+    }
+  }
+}
+
+provider "azurerm" {
+  features {}
+}
+
+provider "random" {
+}

examples/adb-kafka/modules/general_vm/main.tf

@@ -0,0 +1,5 @@
+spokecidr="10.179.0.0/20"
+no_public_ip=true
+rglocation="southeastasia"
+dbfs_prefix="dbfs"
+workspace_prefix="adb-kafka"