diff --git a/Cargo.lock b/Cargo.lock index e644eed..c3f3c63 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", + "toml 0.8.14", "tracing", "tracing-subscriber", ] @@ -169,7 +170,7 @@ checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996" dependencies = [ "semver", "serde", - "toml", + "toml 0.5.11", "url", ] @@ -841,6 +842,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_spanned" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0" +dependencies = [ + "serde", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -1048,6 +1058,40 @@ dependencies = [ "serde", ] +[[package]] +name = "toml" +version = "0.8.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f49eb2ab21d2f26bd6db7bf383edc527a7ebaee412d17af4d40fdccd442f335" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59a3a72298453f564e2b111fa896f8d07fabb36f51f06d7e875fc5e0b5a3ef1" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "tracing" version = "0.1.40" @@ -1331,3 +1375,12 @@ name = "windows_x86_64_msvc" version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "winnow" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59b5e5f6c299a3c7890b876a2a587f3115162487e704907d9b6cd29473052ba1" +dependencies = [ + "memchr", +] diff --git a/Cargo.toml b/Cargo.toml index ad8d6c9..8d34963 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ cc.workspace = true clap.workspace = true console.workspace = true kdl.workspace = true +include_dir.workspace = true libloading.workspace = true linked-hash-map.workspace = true miette.workspace = true @@ -32,7 +33,8 @@ thiserror.workspace = true tokio.workspace = true tracing.workspace = true tracing-subscriber.workspace = true -include_dir = "0.7.4" +toml.workspace = true + [build-dependencies] built.workspace = true @@ -60,6 +62,7 @@ camino = { version = "1.1.7", features = ["serde1"] } cc = { version = "1.1.0" } clap = { version = "4.5.4", features = ["cargo", "wrap_help", "derive"] } console = "0.15.8" +include_dir = "0.7.4" kdl = "4.6.0" libloading = "0.7.3" linked-hash-map = { version = "0.5.6", features = ["serde", "serde_impl"] } @@ -73,6 +76,7 @@ serde = { version = "1.0.136", features = ["derive"] } serde_json = "1.0.83" thiserror = "1.0.30" tokio = { version = "1.37.0", features = ["full", "tracing"] } +toml = "0.8.14" tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } # build diff --git a/abi-cafe-rules-example.toml b/abi-cafe-rules-example.toml new file mode 100644 index 0000000..6d35176 --- /dev/null +++ b/abi-cafe-rules-example.toml @@ -0,0 +1,28 @@ +# Here are some example annotations for test expecations + +# this test fails on this platform, with this toolchain pairing +[targets.x86_64-pc-windows-msvc."simple::cc_calls_rustc"] +fail = "check" + +# this test has random results on this platform, whenever rustc is the caller (callee also supported) +[targets.x86_64-pc-windows-msvc."simple::rustc_caller"] +random = true + +# whenever this test involves cc, only link it, and expect linking to fail +[targets.x86_64-pc-windows-msvc."EmptyStruct::cc_toolchain"] +run = "link" +fail = "link" + +# any repr(c) version of this test fails to run +[targets.x86_64-unknown-linux-gnu."simple::repr_c"] +busted = "run" + +# for this pairing, with the rust calling convention, only generate the test, and expect it to work +[targets.x86_64-unknown-linux-gnu."simple::rustc_calls_rustc::conv_rust"] +run = "generate" +pass = "generate" + +# can match all tests with leading :: +[targets.x86_64-unknown-linux-gnu."::rustc_calls_rustc"] +run = "generate" +pass = "generate" diff --git a/src/cli.rs b/src/cli.rs index 0e08daf..6c658c0 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -130,6 +130,12 @@ struct Cli { #[clap(long)] add_tests: Option, + /// Add the test expectations at the given path + /// + /// (If not specified we'll look for a file called abi-cafe-rules.toml in the working dir) + #[clap(long)] + rules: Option, + /// disable the builtin tests /// /// See also `--add-tests` @@ -154,6 +160,7 @@ pub fn make_app() -> Config { output_format, add_rustc_codegen_backend, add_tests, + rules, disable_builtin_tests, // unimplemented select_vals: _, @@ -228,12 +235,14 @@ Hint: Try using `--pairs {name}_calls_rustc` or `--pairs rustc_calls_{name}`. let out_dir = target_dir.join("temp"); let generated_src_dir = target_dir.join("generated_impls"); let runtime_test_input_dir = add_tests; + let runtime_rules_file = rules.unwrap_or_else(|| "abi-cafe-rules.toml".into()); let paths = Paths { target_dir, out_dir, generated_src_dir, runtime_test_input_dir, + runtime_rules_file, }; Config { output_format, diff --git a/src/error.rs b/src/error.rs index 723da86..f007ec5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -26,6 +26,8 @@ pub enum GenerateError { #[error(transparent)] #[diagnostic(transparent)] KdlScriptError(#[from] kdl_script::KdlScriptError), + #[error(transparent)] + TomlError(#[from] toml::de::Error), /// Used to signal we just skipped it #[error("")] Skipped, diff --git a/src/files.rs b/src/files.rs index f5108b1..669b6db 100644 --- a/src/files.rs +++ b/src/files.rs @@ -13,6 +13,7 @@ pub struct Paths { pub out_dir: Utf8PathBuf, pub generated_src_dir: Utf8PathBuf, pub runtime_test_input_dir: Option, + pub runtime_rules_file: Utf8PathBuf, } impl Paths { pub fn harness_dylib_main_file(&self) -> Utf8PathBuf { diff --git a/src/harness/build.rs b/src/harness/build.rs index ed70068..4d80b13 100644 --- a/src/harness/build.rs +++ b/src/harness/build.rs @@ -5,8 +5,8 @@ use camino::Utf8Path; use tracing::info; use crate::error::*; +use crate::harness::report::*; use crate::harness::test::*; -use crate::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/check.rs b/src/harness/check.rs index 38cf7f5..8bc6cbf 100644 --- a/src/harness/check.rs +++ b/src/harness/check.rs @@ -5,7 +5,6 @@ use kdl_script::types::Ty; use tracing::{error, info}; use crate::error::*; -use crate::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/mod.rs b/src/harness/mod.rs index 01f53da..49bab66 100644 --- a/src/harness/mod.rs +++ b/src/harness/mod.rs @@ -17,11 +17,12 @@ mod build; mod check; mod generate; mod read; +pub mod report; mod run; pub mod test; pub mod vals; -pub use read::{find_tests, spawn_read_test}; +pub use read::{find_test_rules, find_tests, spawn_read_test}; pub use run::TestBuffer; pub type Memoized = Mutex>>>; @@ -30,6 +31,7 @@ pub struct TestHarness { paths: Paths, toolchains: Toolchains, tests: SortedMap>, + test_rules: ExpectFile, tests_with_vals: Memoized<(TestId, ValueGeneratorKind), Arc>, tests_with_toolchain: Memoized<(TestId, ValueGeneratorKind, ToolchainId), Arc>, @@ -39,11 +41,12 @@ pub struct TestHarness { } impl TestHarness { - pub fn new(tests: SortedMap>, cfg: &Config) -> Self { + pub fn new(test_rules: ExpectFile, tests: SortedMap>, cfg: &Config) -> Self { let toolchains = toolchains::create_toolchains(cfg); Self { paths: cfg.paths.clone(), tests, + test_rules, toolchains, tests_with_vals: Default::default(), tests_with_toolchain: Default::default(), @@ -109,13 +112,6 @@ impl TestHarness { .clone(); Ok(output) } - pub fn get_test_rules(&self, test_key: &TestKey) -> TestRules { - let caller = self.toolchains[&test_key.caller].clone(); - let callee = self.toolchains[&test_key.callee].clone(); - - get_test_rules(test_key, &*caller, &*callee) - } - pub fn spawn_test( self: Arc, rt: &tokio::runtime::Runtime, @@ -279,64 +275,69 @@ impl TestHarness { output } - pub fn parse_test_key(&self, input: &str) -> Result { + pub fn parse_test_pattern(&self, input: &str) -> Result { let separator = "::"; let parts = input.split(separator).collect::>(); let [test, rest @ ..] = &parts[..] else { todo!(); }; - let mut key = TestKey { - test: test.to_string(), - caller: String::new(), - callee: String::new(), - options: TestOptions { - convention: CallingConvention::C, - repr: LangRepr::C, - functions: FunctionSelector::All, - val_writer: WriteImpl::HarnessCallback, - val_generator: ValueGeneratorKind::Graffiti, + let mut key = TestKeyPattern { + test: (!test.is_empty()).then(|| test.to_string()), + caller: None, + callee: None, + toolchain: None, + options: TestOptionsPattern { + convention: None, + repr: None, + functions: None, + val_writer: None, + val_generator: None, }, }; for part in rest { // pairs if let Some((caller, callee)) = part.split_once("_calls_") { - key.caller = caller.to_owned(); - key.callee = callee.to_owned(); + key.caller = Some(caller.to_owned()); + key.callee = Some(callee.to_owned()); continue; } if let Some(caller) = part.strip_suffix("_caller") { - key.caller = caller.to_owned(); + key.caller = Some(caller.to_owned()); continue; } if let Some(callee) = part.strip_suffix("_callee") { - key.callee = callee.to_owned(); + key.callee = Some(callee.to_owned()); + continue; + } + if let Some(toolchain) = part.strip_suffix("_toolchain") { + key.toolchain = Some(toolchain.to_owned()); continue; } // repr if let Some(repr) = part.strip_prefix("repr_") { - key.options.repr = repr.parse()?; + key.options.repr = Some(repr.parse()?); continue; } // conv if let Some(conv) = part.strip_prefix("conv_") { - key.options.convention = conv.parse()?; + key.options.convention = Some(conv.parse()?); continue; } // generator if let Ok(val_generator) = part.parse() { - key.options.val_generator = val_generator; + key.options.val_generator = Some(val_generator); continue; } // writer if let Ok(val_writer) = part.parse() { - key.options.val_writer = val_writer; + key.options.val_writer = Some(val_writer); continue; } - return Err(format!("unknown testkey part: {part}")) + return Err(format!("unknown testkey part: {part}")); } Ok(key) } @@ -351,4 +352,4 @@ impl TestHarness { let base = self.full_test_name(key); format!("{base}::{func_name}") } -} \ No newline at end of file +} diff --git a/src/harness/read.rs b/src/harness/read.rs index a05fc95..cfd56f5 100644 --- a/src/harness/read.rs +++ b/src/harness/read.rs @@ -32,6 +32,21 @@ impl Pathish { } } +pub fn find_test_rules(cfg: &Config) -> Result { + let rules = find_test_rules_runtime(&cfg.paths.runtime_rules_file)?; + Ok(rules) +} + +pub fn find_test_rules_runtime(rule_file: &Utf8Path) -> Result { + if rule_file.exists() { + let data = read_runtime_file_to_string(rule_file)?; + let rules = toml::from_str(&data)?; + Ok(rules) + } else { + Ok(ExpectFile::default()) + } +} + pub fn find_tests(cfg: &Config) -> Result, GenerateError> { let mut tests = find_tests_runtime(cfg.paths.runtime_test_input_dir.as_deref())?; let mut more_tests = find_tests_static(cfg.disable_builtin_tests)?; diff --git a/src/report.rs b/src/harness/report.rs similarity index 74% rename from src/report.rs rename to src/harness/report.rs index 1865e80..461c29f 100644 --- a/src/report.rs +++ b/src/harness/report.rs @@ -1,68 +1,83 @@ use camino::Utf8PathBuf; use console::Style; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::json; use crate::error::*; use crate::harness::test::*; -use crate::toolchains::*; use crate::*; /// These are the builtin test-expectations, edit these if there are new rules! -#[allow(unused_variables)] -pub fn get_test_rules(test: &TestKey, caller: &dyn Toolchain, callee: &dyn Toolchain) -> TestRules { - use TestCheckMode::*; - use TestRunMode::*; - - // By default, require tests to run completely and pass - let mut result = TestRules { - run: Check, - check: Pass(Check), - }; +impl TestHarness { + #[allow(unused_variables)] + pub fn get_test_rules(&self, key: &TestKey) -> TestRules { + let caller = self.toolchain_by_test_key(key, CallSide::Caller); + let callee = self.toolchain_by_test_key(key, CallSide::Caller); - // Now apply specific custom expectations for platforms/suites - let is_c = caller.lang() == "c" || callee.lang() == "c"; - let is_rust = caller.lang() == "rust" || callee.lang() == "rust"; - let is_rust_and_c = is_c && is_rust; - - // i128 types are fake on windows so this is all random garbage that might - // not even compile, but that datapoint is a little interesting/useful - // so let's keep running them and just ignore the result for now. - // - // Anyone who cares about this situation more can make the expectations more precise. - if cfg!(windows) && (test.test == "i128" || test.test == "u128") { - result.check = Random; - } + use TestCheckMode::*; + use TestRunMode::*; - // CI GCC is too old to support `_Float16`. - if cfg!(all(target_arch = "x86_64", target_os = "linux")) && is_c && test.test == "f16" { - result.check = Random; - } + // By default, require tests to run completely and pass + let mut result = TestRules { + run: Check, + check: Pass(Check), + }; - // FIXME: investigate why this is failing to build - if cfg!(windows) && is_c && (test.test == "EmptyStruct" || test.test == "EmptyStructInside") { - result.check = Busted(Build); - } + // Now apply specific custom expectations for platforms/suites + let is_c = caller.lang() == "c" || callee.lang() == "c"; + let is_rust = caller.lang() == "rust" || callee.lang() == "rust"; + let is_rust_and_c = is_c && is_rust; + + // i128 types are fake on windows so this is all random garbage that might + // not even compile, but that datapoint is a little interesting/useful + // so let's keep running them and just ignore the result for now. + // + // Anyone who cares about this situation more can make the expectations more precise. + if cfg!(windows) && (key.test == "i128" || key.test == "u128") { + result.check = Random(true); + } - // - // - // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + // CI GCC is too old to support `_Float16`. + if cfg!(all(target_arch = "x86_64", target_os = "linux")) && is_c && key.test == "f16" { + result.check = Random(true); + } - // END OF VENDOR RESERVED AREA - // - // + // FIXME: investigate why this is failing to build + if cfg!(windows) && is_c && (key.test == "EmptyStruct" || key.test == "EmptyStructInside") { + result.check = Busted(Build); + } - result + // + // + // THIS AREA RESERVED FOR VENDORS TO APPLY PATCHES + + // END OF VENDOR RESERVED AREA + // + // + + if let Some(rules) = self.test_rules.targets.get(built_info::TARGET) { + for (pattern, rules) in rules { + let pat = self + .parse_test_pattern(pattern) + .expect("failed to parse test pattern"); + if pat.matches(key) { + if let Some(run) = rules.run.clone() { + result.run = run; + } + if let Some(check) = rules.check.clone() { + result.check = check; + } + } + } + } + result + } } -struct ExpectFile { - targets: IndexMap, +#[derive(Debug, Default, Clone, Serialize, Deserialize)] +pub struct ExpectFile { + pub targets: SortedMap>, } -struct ExpectEntry { - tests: IndexMap -} - -R impl Serialize for BuildError { fn serialize(&self, serializer: S) -> Result @@ -156,7 +171,7 @@ pub fn report_test(results: TestRunResults) -> TestReport { .map(|r| !r.all_passed) .unwrap_or(false), }, - TestCheckMode::Random => true, + TestCheckMode::Random(_) => true, }; if passed { if matches!(results.rules.check, TestCheckMode::Busted(_)) { @@ -209,6 +224,23 @@ pub struct TestKey { pub callee: ToolchainId, pub options: TestOptions, } + +#[derive(Debug, Clone)] +pub struct TestKeyPattern { + pub test: Option, + pub caller: Option, + pub callee: Option, + pub toolchain: Option, + pub options: TestOptionsPattern, +} +#[derive(Debug, Clone)] +pub struct TestOptionsPattern { + pub convention: Option, + pub functions: Option, + pub val_writer: Option, + pub val_generator: Option, + pub repr: Option, +} impl TestKey { pub(crate) fn toolchain_id(&self, call_side: CallSide) -> &str { match call_side { @@ -218,12 +250,88 @@ impl TestKey { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +impl TestKeyPattern { + fn matches(&self, key: &TestKey) -> bool { + let TestKeyPattern { + test, + caller, + callee, + toolchain, + options: + TestOptionsPattern { + convention, + functions, + val_writer, + val_generator, + repr, + }, + } = self; + + if let Some(test) = test { + if test != &key.test { + return false; + } + } + + if let Some(caller) = caller { + if caller != &key.caller { + return false; + } + } + if let Some(callee) = callee { + if callee != &key.callee { + return false; + } + } + if let Some(toolchain) = toolchain { + if toolchain != &key.caller && toolchain != &key.callee { + return false; + } + } + + if let Some(convention) = convention { + if convention != &key.options.convention { + return false; + } + } + if let Some(functions) = functions { + if functions != &key.options.functions { + return false; + } + } + if let Some(val_writer) = val_writer { + if val_writer != &key.options.val_writer { + return false; + } + } + if let Some(val_generator) = val_generator { + if val_generator != &key.options.val_generator { + return false; + } + } + if let Some(repr) = repr { + if repr != &key.options.repr { + return false; + } + } + + true + } +} + +#[derive(Debug, Clone, Serialize)] pub struct TestRules { pub run: TestRunMode, + #[serde(flatten)] pub check: TestCheckMode, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TestRulesPattern { + pub run: Option, + #[serde(flatten)] + pub check: Option, +} /// How far the test should be executed /// /// Each case implies all the previous cases. @@ -247,7 +355,7 @@ pub enum TestRunMode { /// To what level of correctness should the test be graded? /// /// Tests that are Skipped ignore this. -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize)] +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] pub enum TestCheckMode { /// The test must successfully complete this phase, @@ -315,6 +423,7 @@ pub struct CheckOutput { } #[derive(Debug, Copy, Clone, Serialize, PartialEq, Eq, PartialOrd, Ord)] +#[serde(rename_all = "kebab-case")] pub enum TestConclusion { Skipped, Passed, @@ -350,7 +459,7 @@ impl FullReport { } (Passed, Pass(_)) => write!(f, "passed")?, - (Passed, Random) => write!(f, "passed (random, result ignored)")?, + (Passed, Random(_)) => write!(f, "passed (random, result ignored)")?, (Passed, Fail(_)) => write!(f, "passed (failed as expected)")?, (Failed, Pass(_)) => { @@ -374,7 +483,7 @@ impl FullReport { writeln!(f, " {}", red.apply_to(err))?; } } - (Failed, Random) => { + (Failed, Random(_)) => { write!(f, "{}", red.apply_to("failed!? (failed but random!?)"))? } (Failed, Fail(_)) => { diff --git a/src/harness/run.rs b/src/harness/run.rs index cf7df25..e2e31e2 100644 --- a/src/harness/run.rs +++ b/src/harness/run.rs @@ -6,7 +6,7 @@ use serde::Serialize; use tracing::info; use crate::error::*; -use crate::report::*; +use crate::harness::report::*; use crate::*; impl TestHarness { diff --git a/src/harness/test.rs b/src/harness/test.rs index f13642a..cce0d39 100644 --- a/src/harness/test.rs +++ b/src/harness/test.rs @@ -114,19 +114,19 @@ impl FunctionSelector { } } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] pub enum FunctionSelector { All, One { idx: FuncIdx, args: ArgSelector }, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] pub enum ArgSelector { All, One { idx: usize, vals: ValSelector }, } -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, PartialEq, Eq)] pub enum ValSelector { All, One { idx: usize }, diff --git a/src/main.rs b/src/main.rs index e0e7849..f85d9ef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,17 +6,15 @@ mod harness; mod log; mod toolchains; -mod report; - use error::*; use files::Paths; +use harness::report::*; use harness::test::*; use harness::vals::*; use harness::*; use toolchains::*; use kdl_script::parse::LangRepr; -use report::*; use std::error::Error; use std::process::Command; use std::sync::Arc; @@ -92,6 +90,7 @@ fn main() -> Result<(), Box> { let _handle = rt.enter(); // Grab all the tests + let test_rules = harness::find_test_rules(&cfg)?; let test_sources = harness::find_tests(&cfg)?; let read_tasks = test_sources .into_iter() @@ -119,7 +118,7 @@ fn main() -> Result<(), Box> { } debug!("loaded tests!"); - let harness = Arc::new(TestHarness::new(tests, &cfg)); + let harness = Arc::new(TestHarness::new(test_rules, tests, &cfg)); debug!("initialized test harness!"); // Run the tests