improve individual test matching and output (#9)

* improve individual test matching and output

Signed-off-by: Yuchen Liang <[email protected]>

* newline at end of test files

Signed-off-by: Yuchen Liang <[email protected]>

* apply clippy fixes

Signed-off-by: Yuchen Liang <[email protected]>

* add tasks

Signed-off-by: Alex Chi <[email protected]>

* piggy-back version bump to 0.4

Signed-off-by: Alex Chi <[email protected]>

---------

Signed-off-by: Yuchen Liang <[email protected]>
Signed-off-by: Alex Chi <[email protected]>
Co-authored-by: Alex Chi <[email protected]>

Author: yliang412

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "sqlplannertest"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 description = "A yaml-based SQL planner test framework."
 license = "MIT OR Apache-2.0"
@@ -16,6 +16,7 @@ async-trait = "0.1"
 console = "0.15"
 futures-util = { version = "0.3", default-features = false, features = ["alloc"] }
 glob = "0.3"
+itertools = "0.13"
 libtest-mimic = "0.7"
 serde = { version = "1.0", features = ["derive"] }
 serde_yaml = "0.9"

naivedb/Cargo.toml

@@ -7,6 +7,7 @@ edition = "2021"
 [dependencies]
 anyhow = { version = "1", features = ["backtrace"] }
 async-trait = "0.1"
+clap = { version = "4.5", features = ["derive"] }
 sqlplannertest = { path = ".." }
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "fs"] }
 

naivedb/src/bin/apply.rs

@@ -1,12 +1,31 @@
 use std::path::Path;
 
 use anyhow::Result;
+use clap::Parser;
+use sqlplannertest::PlannerTestApplyOptions;
+
+#[derive(Parser)]
+#[command(version, about, long_about = None)]
+struct Cli {
+    /// Optional list of selections to apply the test; if empty, apply all tests
+    selections: Vec<String>,
+    /// Execute tests in serial
+    #[clap(long)]
+    serial: bool,
+}
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    sqlplannertest::planner_test_apply(
+    let cli = Cli::parse();
+
+    let options = PlannerTestApplyOptions {
+        serial: cli.serial,
+        selections: cli.selections,
+    };
+    sqlplannertest::planner_test_apply_with_options(
         Path::new(env!("CARGO_MANIFEST_DIR")).join("tests"),
         || async { Ok(naivedb::NaiveDb::new()) },
+        options,
     )
     .await?;
     Ok(())

naivedb/tests/basics/create.planner.sql

@@ -0,0 +1,14 @@
+-- Test whether select stmt works correctly.
+CREATE TABLE t1(v1 int);
+
+/*
+=== logical
+I'm a naive db, so I don't now how to process
+CREATE TABLE t1(v1 int);
+
+
+=== physical
+I'm a naive db, so I don't now how to process
+CREATE TABLE t1(v1 int);
+*/
+

naivedb/tests/basics/create.yml

@@ -0,0 +1,6 @@
+- sql: |
+    CREATE TABLE t1(v1 int);
+  desc: Test whether select stmt works correctly.
+  tasks:
+  - logical
+  - physical

naivedb/tests/basics/select.planner.sql

@@ -0,0 +1,16 @@
+-- Test whether select stmt works correctly.
+SELECT * FROM t1;
+
+/*
+=== logical
+I'm a naive db, so I don't now how to process
+CREATE TABLE t1(v1 int);
+SELECT * FROM t1;
+
+
+=== physical
+I'm a naive db, so I don't now how to process
+CREATE TABLE t1(v1 int);
+SELECT * FROM t1;
+*/
+

naivedb/tests/basics/select.yml

@@ -0,0 +1,7 @@
+- sql: |
+    SELECT * FROM t1;
+  desc: Test whether select stmt works correctly.
+  before: ["CREATE TABLE t1(v1 int);"]
+  tasks:
+    - logical
+    - physical

src/apply.rs

@@ -7,12 +7,15 @@ use console::style;
 use futures_util::{stream, StreamExt, TryFutureExt};
 
 use crate::{
-    discover_tests, parse_test_cases, ParsedTestCase, PlannerTestRunner, TestCase, RESULT_SUFFIX,
+    discover_tests_with_selections, parse_test_cases, ParsedTestCase, PlannerTestRunner, TestCase,
+    RESULT_SUFFIX,
 };
 
 #[derive(Default, Clone)]
 pub struct PlannerTestApplyOptions {
     pub serial: bool,
+    /// A selection of test modules or files.
+    pub selections: Vec<String>,
 }
 
 pub async fn planner_test_apply<F, Ft, R>(path: impl AsRef<Path>, runner_fn: F) -> Result<()>
@@ -25,7 +28,7 @@ where
 }
 
 pub async fn planner_test_apply_with_options<F, Ft, R>(
-    path: impl AsRef<Path>,
+    tests_dir: impl AsRef<Path>,
     runner_fn: F,
     options: PlannerTestApplyOptions,
 ) -> Result<()>
@@ -34,14 +37,14 @@ where
     Ft: Future<Output = Result<R>> + Send,
     R: PlannerTestRunner + 'static,
 {
-    let tests = discover_tests(path)?
+    let tests = discover_tests_with_selections(&tests_dir, &options.selections)?
         .map(|path| {
             let path = path?;
-            let filename = path
-                .file_name()
-                .context("unable to extract filename")?
-                .to_os_string();
-            let testname = filename
+            let relative_path = path
+                .strip_prefix(&tests_dir)
+                .context("unable to relative path")?
+                .as_os_str();
+            let testname = relative_path
                 .to_str()
                 .context("unable to convert to string")?
                 .to_string();

src/lib.rs

@@ -8,6 +8,7 @@ use anyhow::{Context, Result};
 pub use apply::{planner_test_apply, planner_test_apply_with_options, PlannerTestApplyOptions};
 use async_trait::async_trait;
 use glob::Paths;
+use itertools::Itertools;
 use resolve_id::resolve_testcase_id;
 use serde::{Deserialize, Serialize};
 pub use test_runner::planner_test_runner;
@@ -51,12 +52,51 @@ pub fn parse_test_cases(
 const TEST_SUFFIX: &str = ".yml";
 const RESULT_SUFFIX: &str = "planner.sql";
 
-pub fn discover_tests(path: impl AsRef<Path>) -> Result<Paths> {
-    let pattern = format!("**/[!_]*{}", TEST_SUFFIX);
-    let path = path.as_ref().join(pattern);
-    let path = path.to_str().context("non utf-8 path")?;
-    let paths = glob::glob(path).context("failed to discover test")?;
-    Ok(paths)
+pub fn discover_tests(path: impl AsRef<Path>) -> Result<impl Iterator<Item = glob::GlobResult>> {
+    discover_tests_with_selections(path, &[])
+}
+
+pub fn discover_tests_with_selections(
+    path: impl AsRef<Path>,
+    selections: &[String],
+) -> Result<impl Iterator<Item = glob::GlobResult>> {
+    let patterns = mk_patterns(&path, selections);
+    let paths: Vec<Paths> = patterns
+        .into_iter()
+        .map(|pattern| glob::glob(&pattern).context("input pattern is invalid"))
+        .try_collect()?;
+
+    Ok(paths.into_iter().flatten())
+}
+
+/// Make glob patterns based on `selections`.
+///
+/// If `selections` is empty, returns the glob pattern that select all tests within `path`.
+/// Otherwise returns the glob pattterns that matches the selected test modules or files.
+fn mk_patterns(path: impl AsRef<Path>, selections: &[String]) -> Vec<String> {
+    let mk_pattern = |glob: String| {
+        let path = path.as_ref().join(glob);
+        path.to_str().expect("non utf-8 path").to_string()
+    };
+
+    if selections.is_empty() {
+        // Select all tests
+        return vec![mk_pattern(format!("**/[!_]*{}", TEST_SUFFIX))];
+    }
+
+    // Select matching tests.
+    selections
+        .iter()
+        .flat_map(|s| {
+            let path_segment = s.replace("::", "/");
+            // e.g. tests/<..>/path_segment.yml
+            let file_match = mk_pattern(format!("**/{path_segment}{}", TEST_SUFFIX));
+            // Module match, needs to start at the top level.
+            // e.g. tests/path_segment/<..>/<some>.yml
+            let module_match = mk_pattern(format!("{path_segment}/**/[!_]*{}", TEST_SUFFIX));
+            std::iter::once(file_match).chain(std::iter::once(module_match))
+        })
+        .collect()
 }
 
 #[cfg(test)]

src/test_runner.rs

@@ -27,21 +27,26 @@ impl fmt::Display for Line {
 // End Copyright 2022 Armin Ronacher
 
 /// Test runner based on libtest-mimic.
-pub fn planner_test_runner<F, Ft, R>(path: impl AsRef<Path>, runner_fn: F) -> Result<()>
+pub fn planner_test_runner<F, Ft, R>(tests_dir: impl AsRef<Path>, runner_fn: F) -> Result<()>
 where
     F: Fn() -> Ft + Send + Sync + 'static + Clone,
     Ft: Future<Output = Result<R>> + Send,
     R: PlannerTestRunner,
 {
-    let paths = discover_tests(path)?;
+    let paths = discover_tests(&tests_dir)?;
 
     let args = Arguments::from_args();
 
     let mut tests = vec![];
     for entry in paths {
         let path = entry.context("failed to read glob entry")?;
-        let filename = path.file_name().context("unable to extract filename")?;
-        let testname = filename.to_str().context("unable to convert to string")?;
+        let relative_path = path
+            .strip_prefix(&tests_dir)
+            .context("failed to extract relative path")?
+            .as_os_str();
+        let testname = relative_path
+            .to_str()
+            .context("unable to convert to string")?;
 
         let nocapture = args.nocapture;
         let runner_fn = runner_fn.clone();
@@ -50,7 +55,7 @@ where
             testname
                 .strip_suffix(TEST_SUFFIX)
                 .unwrap()
-                .replace('/', "_"),
+                .replace('/', "::"),
             move || run(path, nocapture, runner_fn),
         ));
     }