Add ScyllaDB Enterprise demo

README.md

@@ -20,71 +20,9 @@ Currently supported DEMOs:
     ```
     git clone https://github.com/scylladb/1m-ops-demo.git
     ```
-1. Open the DEMO folder that you want to run (e.g. for ScyllaDB Cloud 1M ops/sec demo):
-    ```
-    cd scylladb-cloud
-    ```
-1. Edit the `variables.tf` file, for example:
-    ```terraform
-    # ScyllaDB Cloud API token
-    variable "scylla_cloud_token" {
-    description = "ScyllaDB Cloud API token"
-    type        = string
-    default     = "YOUR-API-TOKEN"
-    }
-
-    # ScyllaDB Cloud region
-    variable "scylla_cloud_region" {
-    description = "ScyllaDB Cloud region of the cluster"
-    type        = string
-    default     = "us-east-1"
-    }
-
-    # SSH private key for EC2 instance access
-    variable "ssh_private_key" {
-    description = "SSH private key location for EC2 instance access"
-    type        = string
-    default     = "key.pem"
-    }
-
-    variable "aws_key_pair" {
-    description = "Key pair name in AWS"
-    type        = string
-    default     = "my-key-pair"
-    }
-
-    # AWS credentials file
-    variable "aws_creds" {
-    description = "AWS credentials location"
-    type        = string
-    default     = "/home/user/.aws/credentials"
-    }
-
-    # AWS Profile to Use
-    variable "aws_profile" {
-    description = "AWS Profile to Use"
-    type        = string
-    default     = "DeveloperAccessRole"
-    }
-    ```
-1. Run Terraform
-
-    Initialize Terraform project:
-    ```
-    terraform init
-    ```
-    Review infrastrcuture that Terraform is planning to build out:
-    ```
-    terraform plan
-    ```
-    Finally, start Terraform:
-    ```
-    terraform apply
-    ```
-    Wait for Terraform to finish setting up
 1. Install DEMO UI application dependencies
 
-    Make sure your are in the root folder.
+    Make sure your are in the root folder of the project.
     
     Install backend dependencies (virtual environment is recommended):
     ```bash
@@ -111,9 +49,16 @@ Currently supported DEMOs:
 1. Open DEMO UI application
     
     Go to http://localhost:5173
-    ![demo ui](/docs/source/_static/img/demo_ui.png)
-1. Click through the scenarios one by one.
+1. Select a demo you want to try (this will run `terraform init` under the hood)
+    ![demo ui](/docs/source/_static/img/demo_ui.jpg)
+1. (Optional) Configure cluster size and workload settings on the left side
+    ![demo ui workload](/docs/source/_static/img/demo_ui2.jpg)
+1. Hit `APPLY` (this runs `terraform apply`)
+1. Keep an eye on the console output to see when Terraform finishes
+1. Click on the different dashboard tabs to monitor the cluster.
+1. If you are done, don't forget to run `DESTROY` to remove infrastructure elements and avoid unnecessary costs.
 
 
 ## Relevant links
-* [ScyllaDB docs](https://docs.scylladb.com/stable/)
+* [ScyllaDB docs](https://docs.scylladb.com/stable/)
+* [ScyllaDB Cloud](https://cloud.scylladb.com)

scylladb-enterprise/monitoring.tf

@@ -1,7 +1,7 @@
 resource "aws_instance" "scylladb-monitoring" {
   ami           = var.monitoring_ami_id
   instance_type = var.monitoring_instance_type
-  key_name      = var.aws_key_pair_name
+  key_name      = var.aws_key_pair
 
   security_groups = [aws_security_group.sg.id]
   subnet_id       = element(aws_subnet.public_subnet.*.id, 0)
@@ -29,7 +29,7 @@ resource "aws_instance" "scylladb-monitoring" {
     content = templatefile("monitoring/configure_monitoring.sh.tpl", {
       CLUSTER_NAME   = var.custom_name,
       scylla_servers = join("\n", formatlist("  - %s:9180", concat(aws_instance.scylladb_seed.*.private_ip, aws_instance.scylladb_nonseeds.*.private_ip))),
-      DC_NAME        = var.aws_region
+      DC_NAME        = var.region
     })
     destination = "/tmp/configure_monitoring.sh"
 
@@ -61,7 +61,7 @@ resource "aws_instance" "scylladb-monitoring" {
       "export docker=/usr/bin/docker",
       "chmod +x ./start-all.sh",
       "chmod +x ./versions.sh",
-      "bash ./start-all.sh -d prometheus_data",
+      "bash ./start-all.sh -d prometheus_data --no-loki --no-renderer --no-alertmanager --no-cas --no-cdc -c GF_SECURITY_ALLOW_EMBEDDING=true",
       "echo 'done'",
     ]
 
@@ -81,6 +81,23 @@ resource "aws_instance" "scylladb-monitoring" {
   }
 }
 
+locals {
+  scylla_grafana_url = format("http://%s:3000", aws_instance.scylladb-monitoring.*.public_ip[0])
+
+  template_file_init = templatefile("${path.module}/../terraform-data.tftpl", {
+    monitoring_url = local.scylla_grafana_url
+    scylla_version = "6-2"
+    is_scylla_cloud = "false"
+    scenario_steps = "\"original_cluster\", \"sample_data\", \"start_stress\", \"scale_out\", \"stop_stress\", \"scale_in\""
+    demo = "scylladb-enterprise"
+  })
+}
+
+resource "local_file" "grafana_urls" {
+  content  = local.template_file_init
+  filename = "${path.module}/../frontend/public/data/terraform-data.json"
+}
+
 output "monitoring_url" {
-  value = format("http://%s:3000", aws_instance.scylladb-monitoring.*.public_ip[0])
-}
+  value = local.scylla_grafana_url
+}

scylladb-enterprise/provider.tf

@@ -1,5 +1,5 @@
 provider "aws" {
-  region                   = var.aws_region
-  shared_credentials_files = ["${var.path_to_aws_cred_file}"]
+  region                   = var.region
+  shared_credentials_files = ["${var.aws_creds_file}"]
   profile                  = var.aws_creds_profile
 }

scylladb-enterprise/scylladb-instances.tf

@@ -2,8 +2,8 @@
 resource "aws_instance" "scylladb_seed" {
   count         = 1
   ami           = var.scylla_ami_id
-  instance_type = var.scylla_instance_type
-  key_name      = var.aws_key_pair_name
+  instance_type = var.scylla_node_type
+  key_name      = var.aws_key_pair
 
   subnet_id       = element(aws_subnet.public_subnet.*.id, count.index)
   security_groups = [aws_security_group.sg.id]
@@ -19,7 +19,6 @@ resource "aws_instance" "scylladb_seed" {
 {
     "scylla_yaml": {
         "cluster_name": "${var.custom_name}",
-        "internode_compression": "all",
         },
     "start_scylla_on_first_boot": true
 }
@@ -29,8 +28,8 @@ EOF
 resource "aws_instance" "scylladb_nonseeds" {
   count         = var.scylla_node_count - 1
   ami           = var.scylla_ami_id
-  instance_type = var.scylla_instance_type
-  key_name      = var.aws_key_pair_name
+  instance_type = var.scylla_node_type
+  key_name      = var.aws_key_pair
 
   subnet_id       = element(aws_subnet.public_subnet.*.id, count.index)
   security_groups = [aws_security_group.sg.id]
@@ -45,10 +44,10 @@ resource "aws_instance" "scylladb_nonseeds" {
 {
     "scylla_yaml": {
         "cluster_name": "${var.custom_name}",
+        "start_scylla_on_first_boot": true
          "seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider",
                             "parameters": [{"seeds": "${aws_instance.scylladb_seed[0].private_ip}"}]}]
         },
-        "start_scylla_on_first_boot": true
 }
 EOF
 

scylladb-enterprise/scylladb-loaders.tf

@@ -7,7 +7,7 @@ resource "aws_instance" "loader_instance" {
   instance_type   = var.loader_instance_type
   subnet_id       = element(aws_subnet.public_subnet.*.id, count.index)
   security_groups = [aws_security_group.sg.id, ]
-  key_name        = var.aws_key_pair_name
+  key_name        = var.aws_key_pair
   tags = {
     "Name"      = "${var.custom_name}-Loader-${count.index}"
     "CreatedBy" = "scylladb-demo"
@@ -17,45 +17,48 @@ resource "aws_instance" "loader_instance" {
     create = "10m"
   }
 
-  # Provision files to each instance. Copy three files from the current directory 
-  # to the remote instance: stress.yml, cassandra-stress.service, and cassandra-stress-benchmark.service.
+  # Provision files to each instance. Copy loader files.
 
   provisioner "file" {
-    source      = "./stress/stress.yml"
-    destination = "/home/scyllaadm/stress.yml"
+    source      = "../loader/Dockerfile"
+    destination = "/home/scyllaadm/Dockerfile"
   }
   provisioner "file" {
-    source      = "./stress/cassandra-stress.service"
-    destination = "/home/scyllaadm/cassandra-stress.service"
+    source      = "../loader/loader.sh"
+    destination = "/home/scyllaadm/loader.sh"
   }
   provisioner "file" {
-    source      = "./stress/cassandra-stress-benchmark.service"
-    destination = "/home/scyllaadm/cassandra-stress-benchmark.service"
+    source      = "../loader/docker.sh"
+    destination = "/home/scyllaadm/docker.sh"
   }
 
-  # Run remote-exec commands on each instance. It stops the scylla-server, creates a start.sh script, 
-  # creates a benchmark.sh script, sets permissions on the scripts, moves two files to /etc/systemd/system/, 
-  # runs daemon-reload, and starts the cassandra-stress service.
+  provisioner "file" {
+    source      = "../loader/cql-stress.service"
+    destination = "/home/scyllaadm/cql-stress.service"
+  }
+
+  # Build loader app 
 
   provisioner "remote-exec" {
     inline = [
       "sudo systemctl stop scylla-server |tee scylla.log",
-      "echo '/usr/bin/cassandra-stress user profile=./stress.yml n=${var.num_of_ops} cl=local_quorum no-warmup \"ops(insert=1)\" -rate threads=${var.num_threads} fixed=120000/s -mode native cql3 user=scylla password=pass -log file=populating.log -pop seq=1..100M -node ${local.scylla_ips}' > start.sh",
-      "echo '/usr/bin/cassandra-stress user profile=./stress.yml duration=24h no-warmup cl=local_quorum \"ops(add_sensor_data=1,get_sensor_data=3)\" -rate threads=${var.num_threads} fixed=${var.throttle} -mode native cql3 user=scylla password=pass -log file=benchmarking.log -node ${local.scylla_ips}' > benchmark.sh",
-      "sudo chmod +x start.sh benchmark.sh",
-      "echo '/home/scyllaadm/benchmark.sh' >> /home/scyllaadm/start.sh",
-      "sudo mv /home/scyllaadm/cassandra-stress.service /etc/systemd/system/cassandra-stress.service ",
-      "sudo mv /home/scyllaadm/cassandra-stress-benchmark.service /etc/systemd/system/cassandra-stress-benchmark.service ",
+      "sudo chmod +x /home/scyllaadm/docker.sh",
+      "sudo /home/scyllaadm/docker.sh",
+      "sudo systemctl restart docker",
+      "cd /home/scyllaadm/ && sudo docker build -t loader .",
+      "echo 'sudo docker run loader ${var.loader_ops_per_sec} ${var.loader_read_ratio} ${var.loader_write_ratio} \"${local.scylla_ips}\" \"scylla\" \"pass\"' > start.sh",
+      "sudo chmod +x start.sh",
+      "sudo mv /home/scyllaadm/cql-stress.service /etc/systemd/system/cql-stress.service ",
       "sudo systemctl daemon-reload ",
-      "sudo systemctl start cassandra-stress.service",
+      "sudo systemctl start cql-stress.service",
     ]
   }
 
   # Set up an SSH connection to each EC2 instance using the scyllaadm user and the private key. 
   # The coalesce function is used to select the public IP address of ScyllaDB Nodes.
   connection {
     type        = "ssh"
-    user        = var.instance_username
+    user        = var.scylla_user
     private_key = file(var.ssh_private_key)
     host        = coalesce(self.public_ip, self.private_ip)
     agent       = true

scylladb-enterprise/variables.tf

@@ -3,36 +3,36 @@
 #
 
 # AWS credentials file
-variable "path_to_aws_cred_file" {
+variable "aws_creds_file" {
   description = "AWS credentials location"
   type        = string
-  default     = "/home/user/.aws/credentials"
+  default     = ""
 }
 
 # AWS credentials file
 variable "aws_creds_profile" {
   description = "AWS credentials profile"
   type        = string
-  default     = "DeveloperAccessRole"
+  default     = ""
 }
 
 # SSH private key for EC2 instance access
 variable "ssh_private_key" {
   description = "SSH private key location for EC2 instance access"
   type        = string
-  default     = "/home/user/key.pem"
+  default     = ""
 }
 
-variable "aws_key_pair_name" {
+variable "aws_key_pair" {
   description = "Key pair name in AWS"
   type        = string
-  default     = "key-pair"
+  default     = ""
 }
 
-variable "aws_region" {
-  description = "AWS region"
+variable "region" {
+  description = "Region of the cluster"
   type        = string
-  default     = "us-east-2"
+  default     = ""
 }
 
 # Amazon Machine Image (AMI) ID
@@ -64,17 +64,17 @@ variable "loader_ami_id" {
 #
 
 # Scylla instance type
-variable "scylla_instance_type" {
+variable "scylla_node_type" {
   description = "Type of the EC2 instance"
   type        = string
-  default     = "i4i.2xlarge"
+  default     = ""
 }
 
 # Loader instance type
 variable "loader_instance_type" {
   description = "Type of the EC2 instance"
   type        = string
-  default     = "i4i.8xlarge"
+  default     = "i4i.12xlarge"
 }
 
 # Virtual Private Cloud (VPC) IP range
@@ -92,7 +92,7 @@ variable "subnet_count" {
 }
 
 # Amazon Machine Image (AMI) Username
-variable "instance_username" {
+variable "scylla_user" {
   description = "username for the ScyllaDB AMI"
   type        = string
   default     = "scyllaadm"
@@ -127,10 +127,22 @@ variable "num_of_ops" {
 }
 
 # Throttling for the Cassandra stress tool
-variable "throttle" {
+variable "loader_ops_per_sec" {
   description = "Throttling for the Cassandra stress tool (in ops/sec)"
   type        = string
-  default     = "1200000/s"
+  default     = ""
+}
+
+variable "loader_read_ratio" {
+  description = "Read ratio"
+  type        = string
+  default     = "7"
+}
+
+variable "loader_write_ratio" {
+  description = "Write ratio"
+  type        = string
+  default     = "3"
 }
 
 # Environment name
@@ -157,4 +169,12 @@ variable "loader_node_count" {
 
 locals {
   scylla_ips = join(",", concat(aws_instance.scylladb_seed.*.private_ip, aws_instance.scylladb_nonseeds.*.private_ip))
+}
+
+# ScyllaDB Cloud API token
+# Not needed for this demo, but needed by DEMO UI
+variable "scylla_cloud_token" {
+  description = "ScyllaDB Cloud API token"
+  type        = string
+  default     = ""
 }