New Rust-based loader and minor fixes

.github/workflows/docs-pr.yaml

@@ -22,7 +22,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: '3.10'
 
       - name: Set up env
         run: make -C docs setupenv

README.md

@@ -1,19 +1,119 @@
 # ScyllaDB 1 million operations/second DEMOs
 
-Set up 1 million operations/second DEMO with different flavours of ScyllaDB:
-* [1 million operations/second with ScyllaDB Cloud](/scylladb-cloud)
-* [1 million operations/second with ScyllaDB Enterprise](/scylladb-enterprise)
+This repository contains Terraform and Ansible based projects to help you
+set up DEMOs and POCs with ScyllaDB in a cloud environment.
 
-Scale a cluster from 3 to 6 nodes with Tablets enabled:
-* [ScyllaDB Tablets DEMO on AWS](/tablets-scaling)
+Currently supported DEMOs:
+* [ScyllaDB Cloud 1 million operations/second (AWS and ScyllaDB Cloud account needed)](/scylladb-cloud)
+* [ScyllaDB Enterprise 1 million operations/second (AWS account needed)](/scylladb-enterprise)
+* [Scaling from 3 to 6 nodes (AWS account needed))](/tablets-scaling)
+
+
+## Prerequisites
+* [Terraform](https://developer.hashicorp.com/terraform/install)
+* [Python 3](https://www.python.org/downloads/)
+* [NodeJS](https://nodejs.org/en/download)
+* [AWS CLI](https://aws.amazon.com/cli/) and [ScyllaDB Cloud API key](https://cloud.scylladb.com/)
 
 ## Usage
 1. Clone the repository
     ```
     git clone https://github.com/scylladb/1m-ops-demo.git
     ```
-1. Open a folder and follow the instructions in the corresponding readme.
+1. Open the DEMO folder that you want to run (e.g. for ScyllaDB Cloud 1M ops/sec demo):
+    ```
+    cd scylladb-cloud
+    ```
+1. Edit the `variables.tf` file, for example:
+    ```terraform
+    # ScyllaDB Cloud API token
+    variable "scylla_cloud_token" {
+    description = "ScyllaDB Cloud API token"
+    type        = string
+    default     = "YOUR-API-TOKEN"
+    }
+
+    # ScyllaDB Cloud region
+    variable "scylla_cloud_region" {
+    description = "ScyllaDB Cloud region of the cluster"
+    type        = string
+    default     = "us-east-1"
+    }
+
+    # SSH private key for EC2 instance access
+    variable "ssh_private_key" {
+    description = "SSH private key location for EC2 instance access"
+    type        = string
+    default     = "key.pem"
+    }
+
+    variable "aws_key_pair" {
+    description = "Key pair name in AWS"
+    type        = string
+    default     = "my-key-pair"
+    }
+
+    # AWS credentials file
+    variable "aws_creds" {
+    description = "AWS credentials location"
+    type        = string
+    default     = "/home/user/.aws/credentials"
+    }
+
+    # AWS Profile to Use
+    variable "aws_profile" {
+    description = "AWS Profile to Use"
+    type        = string
+    default     = "DeveloperAccessRole"
+    }
+    ```
+1. Run Terraform
+
+    Initialize Terraform project:
+    ```
+    terraform init
+    ```
+    Review infrastrcuture that Terraform is planning to build out:
+    ```
+    terraform plan
+    ```
+    Finally, start Terraform:
+    ```
+    terraform apply
+    ```
+    Wait for Terraform to finish setting up
+1. Install DEMO UI application dependencies
+
+    Make sure your are in the root folder.
+    
+    Install backend dependencies (virtual environment is recommended):
+    ```bash
+    virtualenv env && source env/bin/activate && pip install -r requirements.txt
+    ```
+
+    Install frontend dependencies (use `npm` or `yarn`):
+    ```bash
+    cd frontend
+    npm install
+    ```
+1. Run DEMO UI application
+    
+    Start backend
+    ```bash
+    python app.py
+    ```
+
+    Start frontend
+    ```bash
+    cd frontend
+    npm run dev 
+    ```
+1. Open DEMO UI application
+    
+    Go to http://localhost:5173
+    ![demo ui](/docs/source/_static/img/demo_ui.png)
+1. Click through the scenarios one by one.
+
 
 ## Relevant links
-* [ScyllaDB Open Source docs](https://opensource.docs.scylladb.com/stable/)
-* [ScyllaDB Enterprise docs](https://enterprise.docs.scylladb.com)
+* [ScyllaDB docs](https://docs.scylladb.com/stable/)

frontend/src/components/About/index.tsx

@@ -22,13 +22,13 @@ const Banner = (): ReactElement => (
       />
 
       <h2>
-        1 million ops/sec <br />
-        ScyllaDB demos with Terraform
+        ScyllaDB DEMOs with Terraform
       </h2>
 
       <p className="lead">
-        Test and benchmark ScyllaDB under a 1 million operations per second
-        workload.
+        Test and benchmark ScyllaDB under different scenarios and
+        workloads.
+        Repository: <a target="_blank" href="https://github.com/scylladb/1m-ops-demo">https://github.com/scylladb/1m-ops-demo</a>
       </p>
     </Card.Body>
   </Card>

frontend/src/components/HeaderNav.tsx

@@ -8,6 +8,6 @@ export const HeaderNav = (): ReactElement => (
       alt="ScyllaDB"
     />
 
-    <h3>Demo</h3>
+    <h3>DEMO UI</h3>
   </div>
 );

frontend/src/components/TabsLayout.tsx

@@ -16,12 +16,12 @@ interface SectionTab {
 }
 
 const tabs: readonly SectionTab[] = [
-  {
-    key: 'dashboard',
-    title: 'Dashboard',
-    Icon: FaGaugeSimple,
-    Component: Dashboard,
-  },
+ // {
+   // key: 'dashboard',
+   // title: 'Dashboard',
+   // Icon: FaGaugeSimple,
+   // Component: Dashboard,
+  //},
   {
     key: 'scenarios',
     title: 'Scenarios',
@@ -38,7 +38,7 @@ const tabs: readonly SectionTab[] = [
 
 export const TabsLayout = (): ReactElement => (
   <Tabs
-    defaultActiveKey="dashboard"
+    defaultActiveKey="scenarios"
     id="controlTabs"
     className="nav-tabs nav-fill"
   >

loader/Dockerfile

@@ -0,0 +1,20 @@
+FROM ubuntu:20.04
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+
+RUN apt-get update -y && \
+    apt-get install -y build-essential libssl-dev git-all pkg-config curl && \
+    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
+    git clone https://github.com/scylladb/cql-stress.git /home/ubuntu/cql-stress && \
+    cd /home/ubuntu/cql-stress && \
+    cargo build --release
+
+WORKDIR /home/ubuntu/cql-stress
+
+COPY loader.sh /loader.sh
+RUN chmod +x /loader.sh
+
+ENTRYPOINT ["/loader.sh"]

loader/cql-stress.service

@@ -0,0 +1,10 @@
+[Unit]
+Description="Start cql-stress to populate ScyllaDB"
+
+[Service]
+User=scyllaadm
+WorkingDirectory=/home/scyllaadm/
+ExecStart=/usr/bin/bash /home/scyllaadm/start.sh
+Type=simple
+[Install]
+WantedBy=multi-user.target

loader/docker.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+echo "\$nrconf{restart} = 'a';" | sudo tee /etc/needrestart/needrestart.conf
+
+sudo apt-get update
+sudo apt-get install -y ca-certificates curl gnupg
+
+sudo install -m 0755 -d /etc/apt/keyrings
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
+sudo chmod a+r /etc/apt/keyrings/docker.gpg
+
+
+echo \
+    "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \
+    $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
+    sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+sudo apt-get update
+sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+
+sudo usermod -aG docker scyllaadm
+sudo systemctl enable docker
+sudo systemctl start docker

loader/loader.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+threads=${1:-100}   
+read_ratio=${2:-7}   
+write_ratio=${3:-3}  
+node=${4:-"node"}
+user=${5:-"scylla"}
+pass=${6:-"pass"}
+
+# Create schema and load initial dataset
+cargo run --release --bin cql-stress-cassandra-stress -- write n=10000 cl=local_quorum keysize=100 -rate throttle=10000/s threads="$threads" -pop seq=1..10000 -mode user="$user" password="$pass" -node "$node"
+
+# Start stress
+cargo run --release --bin cql-stress-cassandra-stress -- mixed duration=6h cl=local_quorum keysize=100 'ratio(read='$read_ratio',write='$write_ratio')' -rate threads="$threads" -pop seq=1..10000 -mode user="$user" password="$pass" -node "$node"

scylladb-cloud/ansible/3_stress.yml

@@ -6,7 +6,7 @@
   tasks: 
     - name: Start CQL Stress
       ansible.builtin.systemd_service:
-        name: cassandra-stress.service
+        name: cql-stress.service
         state: started
         daemon_reload: True
 

scylladb-cloud/ansible/5_stop_stress.yml

@@ -6,5 +6,5 @@
   tasks: 
     - name: Stop CQL Stress
       ansible.builtin.systemd_service:
-        name: cassandra-stress.service
+        name: cql-stress.service
         state: stopped

scylladb-enterprise/ansible/3_stress.yml

@@ -0,0 +1,12 @@
+---
+- name: Start loader
+  hosts: stress
+  become: True
+
+  tasks: 
+    - name: Start CQL Stress
+      ansible.builtin.systemd_service:
+        name: cql-stress.service
+        state: started
+        daemon_reload: True
+

scylladb-enterprise/ansible/5_stop_stress.yml

@@ -0,0 +1,10 @@
+---
+- name: Stop loader
+  hosts: stress
+  become: True
+
+  tasks: 
+    - name: Stop CQL Stress
+      ansible.builtin.systemd_service:
+        name: cql-stress.service
+        state: stopped