From 3c4ee7202ef47d0d4b4dcf15eacf18ee9c7f276c Mon Sep 17 00:00:00 2001 From: xxchan Date: Fri, 10 Jan 2025 14:56:02 +0800 Subject: [PATCH] feat(bin): support --fail-fast (#246) * feat(bin): support --fail-fast close #110 Signed-off-by: xxchan * add env var Signed-off-by: xxchan --------- Signed-off-by: xxchan --- CHANGELOG.md | 4 +++ Cargo.lock | 6 ++-- Cargo.toml | 2 +- sqllogictest-bin/src/main.rs | 53 ++++++++++++++++++++++++++++++++++-- 4 files changed, 58 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 356db82..78b14ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +## [0.26.2] - 2025-01-08 + +* bin: support `--fail-fast`, and add env vars `SLT_FAIL_FAST` and `SLT_KEEP_DB_ON_FAILURE` + ## [0.26.1] - 2025-01-08 * parser/runner: support `system ok retry` diff --git a/Cargo.lock b/Cargo.lock index ec50a54..61011e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "sqllogictest" -version = "0.26.1" +version = "0.26.2" dependencies = [ "async-trait", "educe", @@ -1907,7 +1907,7 @@ dependencies = [ [[package]] name = "sqllogictest-bin" -version = "0.26.1" +version = "0.26.2" dependencies = [ "anyhow", "async-trait", @@ -1929,7 +1929,7 @@ dependencies = [ [[package]] name = "sqllogictest-engines" -version = "0.26.1" +version = "0.26.2" dependencies = [ "async-trait", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 3a81a2b..b8c8356 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = ["sqllogictest", "sqllogictest-bin", "sqllogictest-engines", "tests"] [workspace.package] -version = "0.26.1" +version = "0.26.2" edition = "2021" homepage = "https://github.com/risinglightdb/sqllogictest-rs" keywords = ["sql", "database", "parser", "cli"] diff --git a/sqllogictest-bin/src/main.rs b/sqllogictest-bin/src/main.rs index 9376648..f6e48e2 100644 --- a/sqllogictest-bin/src/main.rs +++ b/sqllogictest-bin/src/main.rs @@ -63,9 +63,13 @@ struct Opt { #[clap(long, short)] jobs: Option, /// When using `-j`, whether to keep the temporary database when a test case fails. - #[clap(long, default_value = "false")] + #[clap(long, default_value = "false", env = "SLT_KEEP_DB_ON_FAILURE")] keep_db_on_failure: bool, + /// Whether to exit immediately when a test case fails. + #[clap(long, default_value = "false", env = "SLT_FAIL_FAST")] + fail_fast: bool, + /// Report to junit XML. #[clap(long)] junit: Option, @@ -150,6 +154,7 @@ pub async fn main() -> Result<()> { color, jobs, keep_db_on_failure, + fail_fast, junit, host, port, @@ -239,6 +244,7 @@ pub async fn main() -> Result<()> { config, &labels, junit.clone(), + fail_fast, ) .await } else { @@ -249,6 +255,7 @@ pub async fn main() -> Result<()> { config, &labels, junit.clone(), + fail_fast, ) .await }; @@ -272,6 +279,7 @@ async fn run_parallel( config: DBConfig, labels: &[String], junit: Option, + fail_fast: bool, ) -> Result<()> { let mut create_databases = BTreeMap::new(); let mut filenames = BTreeSet::new(); @@ -332,11 +340,14 @@ async fn run_parallel( let mut failed_case = vec![]; let mut failed_db: HashSet = HashSet::new(); + let mut remaining_files: HashSet = HashSet::from_iter(filenames.clone()); let start = Instant::now(); while let Some((db_name, file, res, mut buf)) = stream.next().await { + remaining_files.remove(&file); let test_case_name = file.replace(['/', ' ', '.', '-'], "_"); + let mut failed = false; let case = match res { Ok(duration) => { let mut case = TestCase::new(test_case_name, TestCaseStatus::success()); @@ -346,6 +357,7 @@ async fn run_parallel( case } Err(e) => { + failed = true; writeln!(buf, "{}\n\n{:?}", style("[FAILED]").red().bold(), e)?; writeln!(buf)?; failed_case.push(file.clone()); @@ -363,6 +375,20 @@ async fn run_parallel( }; test_suite.add_test_case(case); tokio::task::block_in_place(|| stdout().write_all(&buf))?; + if fail_fast && failed { + println!("early exit after failure..."); + break; + } + } + + for file in remaining_files { + println!("{file} is not finished, skipping"); + let test_case_name = file.replace(['/', ' ', '.', '-'], "_"); + let mut case = TestCase::new(test_case_name, TestCaseStatus::skipped()); + case.set_time(Duration::from_millis(0)); + case.set_timestamp(Local::now()); + case.set_classname(junit.as_deref().unwrap_or_default()); + test_suite.add_test_case(case); } eprintln!( @@ -404,10 +430,12 @@ async fn run_serial( config: DBConfig, labels: &[String], junit: Option, + fail_fast: bool, ) -> Result<()> { let mut failed_case = vec![]; - - for file in files { + let mut skipped_case = vec![]; + let mut files = files.into_iter(); + for file in &mut files { let mut runner = Runner::new(|| engines::connect(engine, &config)); for label in labels { runner.add_label(label); @@ -415,6 +443,7 @@ async fn run_serial( let filename = file.to_string_lossy().to_string(); let test_case_name = filename.replace(['/', ' ', '.', '-'], "_"); + let mut failed = false; let case = match run_test_file(&mut std::io::stdout(), runner, &file).await { Ok(duration) => { let mut case = TestCase::new(test_case_name, TestCaseStatus::success()); @@ -424,6 +453,7 @@ async fn run_serial( case } Err(e) => { + failed = true; println!("{}\n\n{:?}", style("[FAILED]").red().bold(), e); println!(); failed_case.push(filename.clone()); @@ -439,6 +469,23 @@ async fn run_serial( } }; test_suite.add_test_case(case); + if fail_fast && failed { + println!("early exit after failure..."); + break; + } + } + for file in files { + let filename = file.to_string_lossy().to_string(); + let test_case_name = filename.replace(['/', ' ', '.', '-'], "_"); + let mut case = TestCase::new(test_case_name, TestCaseStatus::skipped()); + case.set_time(Duration::from_millis(0)); + case.set_timestamp(Local::now()); + case.set_classname(junit.as_deref().unwrap_or_default()); + test_suite.add_test_case(case); + skipped_case.push(filename.clone()); + } + if !skipped_case.is_empty() { + println!("some test case skipped:\n{:#?}", skipped_case); } if !failed_case.is_empty() {