Add cli test framework

Cargo.toml

@@ -35,3 +35,5 @@ colored="1.9"
 
 [dev-dependencies]
 quickcheck = "0.9.2"
+tempfile = "3.1.0"
+test_bin=  "0.3.0"

src/main.rs

@@ -240,33 +240,39 @@ impl Config {
             status, stdout_utf8, stderr_utf8
         );
 
-        // for commit message:
-        // no text but 101 https://github.com/rust-lang/rust/issues/21599
-        // no text, signal https://github.com/rust-lang/rust/issues/13368
+        let saw_ice = || -> bool { stderr_utf8.contains("error: internal compiler error") };
 
-        const SUCCESS: Option<i32> = Some(0);
-        const ICE: Option<i32> = Some(101);
-
-        let input = (self.output_processing_mode(), status.code());
+        let input = (self.output_processing_mode(), status.success());
         let result = match input {
-            (OutputProcessingMode::RegressOnErrorStatus, SUCCESS) => TestOutcome::Baseline,
-            (OutputProcessingMode::RegressOnErrorStatus, _) => TestOutcome::Regressed,
-
-            (OutputProcessingMode::RegressOnSuccessStatus, SUCCESS) => TestOutcome::Regressed,
-            (OutputProcessingMode::RegressOnSuccessStatus, _) => TestOutcome::Baseline,
+            (OutputProcessingMode::RegressOnErrorStatus, true) => TestOutcome::Baseline,
+            (OutputProcessingMode::RegressOnErrorStatus, false) => TestOutcome::Regressed,
 
-            (OutputProcessingMode::RegressOnIceAlone, ICE) => TestOutcome::Regressed,
-            (OutputProcessingMode::RegressOnIceAlone, None) => TestOutcome::Regressed,
-            (OutputProcessingMode::RegressOnIceAlone, _) => TestOutcome::Baseline,
+            (OutputProcessingMode::RegressOnSuccessStatus, true) => TestOutcome::Regressed,
+            (OutputProcessingMode::RegressOnSuccessStatus, false) => TestOutcome::Baseline,
 
-            (OutputProcessingMode::RegressOnNotIce, ICE) => TestOutcome::Baseline,
-            (OutputProcessingMode::RegressOnNotIce, None) => TestOutcome::Baseline,
-            (OutputProcessingMode::RegressOnNotIce, _) => TestOutcome::Regressed,
+            (OutputProcessingMode::RegressOnIceAlone, _) => {
+                if saw_ice() {
+                    TestOutcome::Regressed
+                } else {
+                    TestOutcome::Baseline
+                }
+            }
+            (OutputProcessingMode::RegressOnNotIce, _) => {
+                if saw_ice() {
+                    TestOutcome::Baseline
+                } else {
+                    TestOutcome::Regressed
+                }
+            }
 
-            (OutputProcessingMode::RegressOnNonCleanError, SUCCESS) => TestOutcome::Regressed,
-            (OutputProcessingMode::RegressOnNonCleanError, ICE) => TestOutcome::Regressed,
-            (OutputProcessingMode::RegressOnNonCleanError, None) => TestOutcome::Regressed,
-            (OutputProcessingMode::RegressOnNonCleanError, _) => TestOutcome::Baseline,
+            (OutputProcessingMode::RegressOnNonCleanError, true) => TestOutcome::Regressed,
+            (OutputProcessingMode::RegressOnNonCleanError, false) => {
+                if saw_ice() {
+                    TestOutcome::Regressed
+                } else {
+                    TestOutcome::Baseline
+                }
+            }
         };
         debug!(
             "default_outcome_of_output: input: {:?} result: {:?}",
@@ -311,27 +317,27 @@ enum OutputProcessingMode {
     RegressOnSuccessStatus,
 
     /// `RegressOnIceAlone`: Marks test outcome as `Regressed` if and only if
-    /// the `rustc` process crashes or reports an interal compiler error (ICE)
-    /// has occurred. This covers the use case for when you want to bisect to
-    /// see when an ICE was introduced pon a codebase that is meant to produce a
-    /// clean error.
+    /// the `rustc` process issues a diagnostic indicating that an internal
+    /// compiler error (ICE) occurred. This covers the use case for when you
+    /// want to bisect to see when an ICE was introduced pon a codebase that is
+    /// meant to produce a clean error.
     ///
     /// You explicitly opt into this seting via `--regress=ice`.
     RegressOnIceAlone,
 
     /// `RegressOnNotIce`: Marks test outcome as `Regressed` if and only if
-    /// the `rustc` process does not crash or report that an internal compiler
-    /// error (ICE) has occurred. This covers the use case for when you want to
-    /// bisect to see when an ICE was fixed.
+    /// the `rustc` process does not issue a diagnostic indicating that an
+    /// internal compiler error (ICE) occurred. This covers the use case for
+    /// when you want to bisect to see when an ICE was fixed.
     ///
     /// You explicitly opt into this setting via `--regress=non-ice`
     RegressOnNotIce,
 
     /// `RegressOnNonCleanError`: Marks test outcome as `Baseline` if and only
-    /// if the `rustc` process reports error status that is not an internal
-    /// compiler error (ICE). This is the use case if the regression is a case
-    /// where an ill-formed program has stopped being properly rejected by the
-    /// compiler.
+    /// if the `rustc` process reports error status and does not issue any
+    /// diagnostic indicating that an internal compiler error (ICE) occurred.
+    /// This is the use case if the regression is a case where an ill-formed
+    /// program has stopped being properly rejected by the compiler.
     ///
     /// (The main difference between this case and `RegressOnSuccessStatus` is
     /// the handling of ICE: `RegressOnSuccessStatus` assumes that ICE should be
@@ -346,10 +352,11 @@ impl OutputProcessingMode {
     fn must_process_stderr(&self) -> bool {
         match self {
             OutputProcessingMode::RegressOnErrorStatus
-            | OutputProcessingMode::RegressOnSuccessStatus
-            | OutputProcessingMode::RegressOnNonCleanError
+            | OutputProcessingMode::RegressOnSuccessStatus => false,
+
+            OutputProcessingMode::RegressOnNonCleanError
             | OutputProcessingMode::RegressOnIceAlone
-            | OutputProcessingMode::RegressOnNotIce => false,
+            | OutputProcessingMode::RegressOnNotIce => true,
         }
     }
 }

tests/cli/meta_build.rs

@@ -0,0 +1,84 @@
+use std::fs::{DirBuilder};
+use std::path::{Path};
+
+pub struct InjectionPoint {
+    pub date: YearMonthDay,
+    pub associated_sha: &'static str,
+}
+
+pub struct Test<'a> {
+    pub crate_name: &'a str,
+    pub cli_params: &'a [&'a str],
+    pub delta_date: InjectionPoint,
+    pub delta_kind: DeltaKind,
+}
+
+impl<'a> Test<'a> {
+    pub fn expected_sha(&self) -> &str {
+        self.delta_date.associated_sha
+    }
+}
+
+pub fn make_crate_files(
+    dir_builder: &DirBuilder,
+    dir: &Path,
+    test: &Test)
+    -> Result<(), failure::Error>
+{
+    (crate::make_a_crate::Crate {
+        dir,
+        name: test.crate_name,
+        build_rs: Some(meta_build(test).into()),
+        cargo_toml: format!(r##"
+[package]
+name = "{NAME}"
+version = "0.1.0"
+authors = ["Felix S. Klock II <[email protected]>"]
+"##, NAME=test.crate_name).into(),
+        main_rs: MAIN_RS.into(),
+    }).make_files(dir_builder)?;
+
+    Ok(())
+}
+
+// A test crate to exercise `cargo-bisect-rustc` has three basic components: a
+// Cargo.toml file, a build.rs script that inspects the current version of Rust
+// and injects an error for the appropriate versions into a build-time generated
+// version.rs file, and a main.rs file that include!'s the version.rs file
+//
+// We only inject errors based on YYYY-MM-DD date comparison (<, <=, >=, >), and
+// having that conditonally add a `#[rustc_error]` to the (injected) `fn main()`
+// function.
+
+const MAIN_RS: &'static str = std::include_str!("meta_build/included_main.rs");
+
+#[derive(Copy, Clone)]
+pub struct YearMonthDay(pub u32, pub u32, pub u32);
+
+#[derive(Copy, Clone)]
+pub enum DeltaKind { Fix, Err }
+
+fn meta_build(test: &Test) -> String {
+    let YearMonthDay(year, month, day) = test.delta_date.date;
+    let delta_kind = test.delta_kind;
+    let date_item = format!(r##"
+/// `DELTA_DATE` identfies nightly where simulated change was injected.
+const DELTA_DATE: YearMonthDay = YearMonthDay({YEAR}, {MONTH}, {DAY});
+"##,
+                            YEAR=year, MONTH=month, DAY=day);
+
+    let kind_variant = match delta_kind {
+        DeltaKind::Fix => "Fix",
+        DeltaKind::Err => "Err",
+    };
+    let kind_item = format!(r##"
+/// `DELTA_KIND` identfies whether simulated change is new error, or a fix to ancient error.
+const DELTA_KIND: DeltaKind = DeltaKind::{VARIANT};
+"##,
+                            VARIANT=kind_variant);
+
+    format!("{DATE_ITEM}{KIND_ITEM}{SUFFIX}",
+            DATE_ITEM=date_item, KIND_ITEM=kind_item, SUFFIX=BUILD_SUFFIX)
+}
+
+const BUILD_SUFFIX: &'static str = std::include_str!("meta_build/included_build_suffix.rs");

tests/cli/meta_build/included_build_suffix.rs

@@ -0,0 +1,80 @@
+// Strategy inspired by dtolnay/rustversion: run `rustc --version` at build time
+// to observe version info.
+//
+// (The dtolnay/rustversion is dual-licensed under APACHE/MIT as of January 2020.)
+
+use std::env;
+use std::ffi::OsString;
+use std::fs;
+use std::path::Path;
+use std::process::{self, Command};
+
+#[derive(PartialOrd, Ord, PartialEq, Eq, Debug)]
+struct YearMonthDay(u32, u32, u32);
+
+enum DeltaKind { Fix, Err }
+
+fn main() {
+    let mut context = Context::introspect();
+    context.generate();
+}
+
+struct Context {
+    commit: Commit,
+    rustc_date: YearMonthDay,
+}
+
+#[derive(PartialOrd, Ord, PartialEq, Eq, Debug)]
+struct Commit(String);
+
+impl Context {
+    fn introspect() -> Context {
+        let rustc = env::var_os("RUSTC").unwrap_or_else(|| OsString::from("rustc"));
+        let output = Command::new(&rustc).arg("--version").output().unwrap_or_else(|e| {
+            let rustc = rustc.to_string_lossy();
+            eprintln!("Error: failed to run `{} --version`: {}", rustc, e);
+            process::exit(1);
+        });
+        let output = String::from_utf8(output.stdout).unwrap();
+        let mut tokens = output.split(' ');
+
+        let _rustc = tokens.next().unwrap();
+        let _version = tokens.next().unwrap();
+        let open_paren_commit = tokens.next().unwrap();
+        let date_close_paren = tokens.next().unwrap();
+
+        let commit = Commit(open_paren_commit[1..].to_string());
+
+        let date_str: String =
+            date_close_paren.matches(|c: char| c.is_numeric() || c == '-').collect();
+        let mut date_parts = date_str.split('-');
+        let year: u32 = date_parts.next().unwrap().parse().unwrap();
+        let month: u32 = date_parts.next().unwrap().parse().unwrap();
+        let day: u32 = date_parts.next().unwrap().parse().unwrap();
+
+        Context { commit, rustc_date: YearMonthDay(year, month, day) }
+    }
+
+    fn generate(&mut self) {
+        let inject_with_error = match DELTA_KIND {
+            DeltaKind::Err => self.rustc_date >= DELTA_DATE,
+            DeltaKind::Fix => self.rustc_date < DELTA_DATE,
+        };
+        let prefix = if inject_with_error { "#[rustc_error] " } else { "" };
+        let maybe_static_error = format!("{PREFIX}{ITEM}", PREFIX=prefix, ITEM="fn main() { }");
+
+        let content = format!(r#"{MAIN}
+pub const COMMIT: &'static str = "{COMMIT}";
+pub const DATE: &'static str = "{Y:04}-{M:02}-{D:02}";
+"#,
+                              MAIN=maybe_static_error,
+                              COMMIT=self.commit.0,
+                              Y=self.rustc_date.0,
+                              M=self.rustc_date.1,
+                              D=self.rustc_date.2);
+
+        let out_dir = env::var_os("OUT_DIR").expect("OUT_DIR not set");
+        let out_file = Path::new(&out_dir).join("version.rs");
+        fs::write(out_file, content).expect("failed to write version.rs");
+    }
+}

tests/cli/meta_build/included_main.rs

@@ -0,0 +1,2 @@
+#![feature(rustc_attrs)]
+include!(concat!(env!("OUT_DIR"), "/version.rs"));