feat: integration test for cardinality performance testing (#113)

**Summary**: An integration test for cardinality performance testing.

**Demo**:


https://github.com/cmu-db/optd/assets/20631215/8eda2a04-0c23-4646-932d-8c0e22b249e4

**Details**:
* The integration test requires Postgres to be running.
* The integration test runs in its own workspace that's fully deleted at
the start of each test.
* The test runs the cardtest twice to test that it works both with and
without cached files in the workspace.
* Changed parts of CLI ("," delimiter for `--query-ids` instead of " ",
added defaults for `--query-ids`, etc.).
* Refactored modules into `lib.rs` so that `tests/` can access all
modules.
* Added `pguser` and `pgpassword` args to CLI.
* Changed COPY IN to be from STDIN instead of from a file so that we can
load even to Postgres processes in a container or on another machine.

Author: wangpatrick57

.github/workflows/CI.yaml

@@ -14,6 +14,22 @@ env:
 jobs:
   check:
     runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_USER: test_user
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
     steps:
       - uses: actions/checkout@v2
       - uses: actions-rs/toolchain@v1

.gitignore

@@ -2,4 +2,5 @@
 /.vscode
 /.DS_Store
 /.idea
-.history
+.history
+**/*_workspace/**

Cargo.lock

@@ -0,0 +1,22 @@
+#!/bin/bash
+if [[ "$(whoami)" == "patrickwang" ]]; then
+    # a sketchy way to kill "orphaned" postgres processes (i.e. processes whose pgdata dir has already been deleted)
+    # use this script to avoid having to manually do "ps aux" and "kill [pid]" every time cardtest_integration (or something similar) fails
+    # the "main" process has bin/postgres in it
+    # we kill it in a loop because sometimes it's required for some reason I don't understand
+    while pid=$(ps aux | grep bin/postgres | grep -v grep | head -n1 | awk '{print $2}'); do
+    if [ -z "$pid" ]; then
+        break
+    else
+        # we'll sometimes kill pids that don't exist. don't show the error in these cases
+        kill $pid &>/dev/null
+        sleep 1 # sleep so we don't loop too fast
+    fi
+    done
+
+    # even after it's gone, wait for it to completely shut down
+    sleep 1
+else
+    echo "unimplemented" >&2
+    exit 1
+fi

dev_scripts/kill_postgres.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+if [[ "$(whoami)" == "patrickwang" ]]; then
+    rm -rf ~/pgdata
+    cd ~/pgdata
+    initdb
+    cd -
+    pg_ctl start
+    # default_user is used for cargo run --bin optd-perftest
+    psql -d postgres -c "CREATE USER default_user WITH SUPERUSER PASSWORD 'password';"
+    # test_user is used for cargo test --package optd-perftest
+    psql -d postgres -c "CREATE USER test_user WITH SUPERUSER PASSWORD 'password';"
+else
+    echo "unimplemented" >&2
+    exit 1
+fi

dev_scripts/start_postgres.sh

@@ -36,3 +36,8 @@ clap = { version = "4.5", features = [
 log = "0.4"
 env_logger = "0.11"
 lazy_static = "1.4.0"
+tokio-util = "0.7"
+futures-util = "0.3"
+
+[dev_dependencies]
+assert_cmd = "2.0"

mac_kill_postgres.sh

@@ -1,10 +1,12 @@
 use std::collections::HashMap;
+use std::path::Path;
+
+use crate::postgres_db::PostgresDb;
+use crate::{benchmark::Benchmark, datafusion_db_cardtest::DatafusionDb, tpch::TpchConfig};
 
 use anyhow::{self};
 use async_trait::async_trait;
 
-use crate::benchmark::Benchmark;
-
 /// This struct performs cardinality testing across one or more databases.
 /// Another design would be for the CardtestRunnerDBHelper trait to expose a function
 ///   to evaluate the Q-error. However, I chose not to do this design for reasons
@@ -86,3 +88,21 @@ pub trait CardtestRunnerDBHelper {
         benchmark: &Benchmark,
     ) -> anyhow::Result<Vec<usize>>;
 }
+
+pub async fn cardtest<P: AsRef<Path> + Clone>(
+    workspace_dpath: P,
+    pguser: &str,
+    pgpassword: &str,
+    tpch_config: TpchConfig,
+) -> anyhow::Result<HashMap<String, Vec<f64>>> {
+    let pg_db = PostgresDb::new(workspace_dpath.clone(), pguser, pgpassword);
+    let df_db = DatafusionDb::new(workspace_dpath).await?;
+    let databases: Vec<Box<dyn CardtestRunnerDBHelper>> = vec![Box::new(pg_db), Box::new(df_db)];
+
+    let tpch_benchmark = Benchmark::Tpch(tpch_config.clone());
+    let mut cardtest_runner = CardtestRunner::new(databases).await?;
+    let qerrors = cardtest_runner
+        .eval_benchmark_qerrors_alldbs(&tpch_benchmark)
+        .await?;
+    Ok(qerrors)
+}

optd-perftest/Cargo.toml

@@ -0,0 +1,6 @@
+mod benchmark;
+pub mod cardtest;
+mod datafusion_db_cardtest;
+mod postgres_db;
+pub mod shell;
+pub mod tpch;

optd-perftest/src/cardtest.rs

@@ -1,30 +1,19 @@
-use std::{fs, path::Path};
+use optd_perftest::cardtest;
+use optd_perftest::shell;
+use optd_perftest::tpch::{TpchConfig, TPCH_KIT_POSTGRES};
+use std::fs;
 
-use cardtest::{CardtestRunner, CardtestRunnerDBHelper};
 use clap::{Parser, Subcommand};
-use postgres_db::PostgresDb;
-
-use crate::{
-    benchmark::Benchmark,
-    datafusion_db_cardtest::DatafusionDb,
-    tpch::{TpchConfig, TPCH_KIT_POSTGRES},
-};
-
-mod benchmark;
-mod cardtest;
-mod datafusion_db_cardtest;
-mod postgres_db;
-mod shell;
-mod tpch;
 
 #[derive(Parser)]
 struct Cli {
     #[arg(long)]
-    #[clap(default_value = "../optd_perftest_workspace")]
+    #[clap(default_value = "optd_perftest_workspace")]
     #[clap(
         help = "The directory where artifacts required for performance testing (such as pgdata or TPC-H queries) are generated. See comment of parse_pathstr() to see what paths are allowed (TLDR: absolute and relative both ok)."
     )]
     workspace: String,
+
     #[command(subcommand)]
     command: Commands,
 }
@@ -35,12 +24,26 @@ enum Commands {
         #[arg(long)]
         #[clap(default_value = "0.01")]
         scale_factor: f64,
+
         #[arg(long)]
         #[clap(default_value = "15721")]
         seed: i32,
+
         #[arg(long)]
-        #[clap(value_delimiter = ' ', num_args = 1..)]
+        #[clap(value_delimiter = ',', num_args = 1..)]
+        // this is the current list of all queries that work in perftest
+        #[clap(default_value = "2,3,5,7,8,9,10,12,14,17")]
         query_ids: Vec<u32>,
+
+        #[arg(long)]
+        #[clap(default_value = "default_user")]
+        #[clap(help = "The name of a user with superuser privileges")]
+        pguser: String,
+
+        #[arg(long)]
+        #[clap(default_value = "password")]
+        #[clap(help = "The name of a user with superuser privileges")]
+        pgpassword: String,
     },
 }
 
@@ -59,31 +62,20 @@ async fn main() -> anyhow::Result<()> {
             scale_factor,
             seed,
             query_ids,
+            pguser,
+            pgpassword,
         } => {
             let tpch_config = TpchConfig {
                 database: String::from(TPCH_KIT_POSTGRES),
                 scale_factor,
                 seed,
                 query_ids,
             };
-            cardtest(&workspace_dpath, tpch_config).await
+            let qerrors =
+                cardtest::cardtest(&workspace_dpath, &pguser, &pgpassword, tpch_config).await?;
+            println!("qerrors={:?}", qerrors);
         }
     }
-}
-
-async fn cardtest<P: AsRef<Path> + Clone>(
-    workspace_dpath: P,
-    tpch_config: TpchConfig,
-) -> anyhow::Result<()> {
-    let pg_db = PostgresDb::new(workspace_dpath.clone());
-    let df_db = DatafusionDb::new(workspace_dpath).await?;
-    let databases: Vec<Box<dyn CardtestRunnerDBHelper>> = vec![Box::new(pg_db), Box::new(df_db)];
 
-    let tpch_benchmark = Benchmark::Tpch(tpch_config.clone());
-    let mut cardtest_runner = CardtestRunner::new(databases).await?;
-    let qerrors = cardtest_runner
-        .eval_benchmark_qerrors_alldbs(&tpch_benchmark)
-        .await?;
-    println!("qerrors: {:?}", qerrors);
     Ok(())
 }