Rustfix edition testing setup

Also refactors the test setup to finally run as many tests as possible
and give useful errors.

Author: killercup

Cargo.toml

@@ -26,9 +26,9 @@ failure = "0.1.1"
 duct = "0.8.2"
 env_logger = "0.5.0-rc.1"
 log = "0.4.1"
-pretty_assertions = "0.4.1"
 tempdir = "0.3.5"
 proptest = "0.7.0"
+difference = "2.0.0"
 
 [workspace]
 members = [

tests/parse_and_replace.rs

@@ -1,39 +1,61 @@
 #![cfg(not(windows))] // TODO: should fix these tests on Windows
 
-#[macro_use]
 extern crate duct;
 extern crate env_logger;
 #[macro_use]
 extern crate log;
-#[macro_use]
-extern crate pretty_assertions;
 extern crate rustfix;
 extern crate serde_json;
 extern crate tempdir;
+#[macro_use]
+extern crate failure;
+extern crate difference;
 
+use std::ffi::OsString;
 use std::{env, fs};
-use std::error::Error;
 use std::path::{Path, PathBuf};
 use std::collections::HashSet;
 use std::process::Output;
+
+use failure::{Error, ResultExt};
 use tempdir::TempDir;
 
 use rustfix::apply_suggestions;
 
-fn compile(file: &Path) -> Result<Output, Box<Error>> {
+mod fixmode {
+    pub const EVERYTHING: &str = "yolo";
+    pub const EDITION: &str = "edition";
+}
+
+mod settings {
+    // can be set as env var to debug
+    pub const CHECK_JSON: &str = "RUSTFIX_TEST_CHECK_JSON";
+    pub const RECORD_JSON: &str = "RUSTFIX_TEST_RECORD_JSON";
+    pub const RECORD_FIXED_RUST: &str = "RUSTFIX_TEST_RECORD_FIXED_RUST";
+
+    // set automatically
+    pub const MODE: &str = "RUSTFIX_MODE";
+}
+
+fn compile(file: &Path, mode: &str) -> Result<Output, Error> {
     let tmp = TempDir::new("rustfix-tests")?;
-    let better_call_clippy = cmd!(
-        "rustc",
-        file,
-        "--error-format=pretty-json",
-        "-Zunstable-options",
-        "--emit=metadata",
-        "--crate-name=rustfix_test",
-        "-Zsuggestion-applicability",
-        "--out-dir",
-        tmp.path()
-    );
-    let res = better_call_clippy
+
+    let mut args: Vec<OsString> = vec![
+        file.into(),
+        "--error-format=pretty-json".into(),
+        "-Zunstable-options".into(),
+        "--emit=metadata".into(),
+        "--crate-name=rustfix_test".into(),
+        "-Zsuggestion-applicability".into(),
+        "--out-dir".into(),
+        tmp.path().into(),
+    ];
+
+    if mode == fixmode::EDITION {
+        args.push("--edition=2018".into());
+    }
+
+    let res = duct::cmd("rustc", &args)
         .env("CLIPPY_DISABLE_DOCS_LINKS", "true")
         .stdout_capture()
         .stderr_capture()
@@ -43,24 +65,19 @@ fn compile(file: &Path) -> Result<Output, Box<Error>> {
     Ok(res)
 }
 
-fn compile_and_get_json_errors(file: &Path) -> Result<String, Box<Error>> {
-    let res = compile(file)?;
+fn compile_and_get_json_errors(file: &Path, mode: &str) -> Result<String, Error> {
+    let res = compile(file, mode)?;
     let stderr = String::from_utf8(res.stderr)?;
 
-    use std::io::{Error, ErrorKind};
     match res.status.code() {
         Some(0) | Some(1) | Some(101) => Ok(stderr),
-        _ => Err(Box::new(Error::new(
-            ErrorKind::Other,
-            format!("failed with status {:?}: {}", res.status.code(), stderr),
-        ))),
+        _ => Err(format_err!("failed with status {:?}: {}", res.status.code(), stderr)),
     }
 }
 
-fn compiles_without_errors(file: &Path) -> Result<(), Box<Error>> {
-    let res = compile(file)?;
+fn compiles_without_errors(file: &Path, mode: &str) -> Result<(), Error> {
+    let res = compile(file, mode)?;
 
-    use std::io::{Error, ErrorKind};
     match res.status.code() {
         Some(0) => Ok(()),
         _ => {
@@ -69,18 +86,15 @@ fn compiles_without_errors(file: &Path) -> Result<(), Box<Error>> {
                 file,
                 String::from_utf8(res.stderr)?
             );
-            Err(Box::new(Error::new(
-                ErrorKind::Other,
-                format!(
-                    "failed with status {:?} (`env RUST_LOG=everything=info` for more info)",
-                    res.status.code(),
-                ),
-            )))
+            Err(format_err!(
+                "failed with status {:?} (`env RUST_LOG=parse_and_replace=info` for more info)",
+                res.status.code(),
+            ))
         }
     }
 }
 
-fn read_file(path: &Path) -> Result<String, Box<Error>> {
+fn read_file(path: &Path) -> Result<String, Error> {
     use std::io::Read;
 
     let mut buffer = String::new();
@@ -89,53 +103,90 @@ fn read_file(path: &Path) -> Result<String, Box<Error>> {
     Ok(buffer)
 }
 
-fn test_rustfix_with_file<P: AsRef<Path>>(file: P) -> Result<(), Box<Error>> {
+fn diff(expected: &str, actual: &str) -> String {
+    use std::fmt::Write;
+    use difference::{Changeset, Difference};
+
+    let mut res = String::new();
+    let changeset = Changeset::new(expected.trim(), actual.trim(), "\n");
+
+    let mut different = false;
+    for diff in changeset.diffs {
+        let (prefix, diff) = match diff {
+            Difference::Same(_) => continue,
+            Difference::Add(add) => ("+", add),
+            Difference::Rem(rem) => ("-", rem),
+        };
+        if !different {
+            write!(&mut res, "differences found (+ == actual, - == expected):\n");
+            different = true;
+        }
+        for diff in diff.lines() {
+            writeln!(&mut res, "{} {}", prefix, diff);
+        }
+    }
+    if different {
+        write!(&mut res, "");
+    }
+
+    res
+}
+
+fn test_rustfix_with_file<P: AsRef<Path>>(file: P, mode: &str) -> Result<(), Error> {
     let file: &Path = file.as_ref();
     let json_file = file.with_extension("json");
     let fixed_file = file.with_extension("fixed.rs");
 
     debug!("next up: {:?}", file);
-    let code = read_file(file)?;
-    let errors = compile_and_get_json_errors(file)?;
+    let code = read_file(file)
+        .context(format!("could not read {}", file.display()))?;
+    let errors = compile_and_get_json_errors(file, mode)
+        .context(format!("could compile {}", file.display()))?;
     let suggestions = rustfix::get_suggestions_from_json(&errors, &HashSet::new())
-        .expect("could not load suggestions");
+        .context("could not load suggestions")?;
 
-    if std::env::var("RUSTFIX_TEST_RECORD_JSON").is_ok() {
+    if std::env::var(settings::RECORD_JSON).is_ok() {
         use std::io::Write;
-        let mut recorded_json = fs::File::create(&file.with_extension("recorded.json"))?;
+        let mut recorded_json = fs::File::create(&file.with_extension("recorded.json"))
+            .context(format!("could not create recorded.json for {}", file.display()))?;
         recorded_json.write_all(errors.as_bytes())?;
     }
 
-    let expected_json = read_file(&json_file)?;
-    let expected_suggestions = rustfix::get_suggestions_from_json(&expected_json, &HashSet::new())
-        .expect("could not load expected suggesitons");
-    assert_eq!(
-        expected_suggestions, suggestions,
-        "got unexpected suggestions from clippy",
-    );
+    if std::env::var(settings::CHECK_JSON).is_ok() {
+        let expected_json = read_file(&json_file)
+            .context(format!("could not load json fixtures for {}", file.display()))?;;
+        let expected_suggestions = rustfix::get_suggestions_from_json(&expected_json, &HashSet::new())
+            .context("could not load expected suggesitons")?;
+
+        ensure!(
+            expected_suggestions == suggestions,
+            "got unexpected suggestions from clippy:\n{}",
+            diff(&format!("{:?}", expected_suggestions), &format!("{:?}", suggestions))
+        );
+    }
 
-    let fixed = apply_suggestions(&code, &suggestions)?;
+    let fixed = apply_suggestions(&code, &suggestions)
+        .context(format!("could not apply suggestions to {}", file.display()))?;
 
-    if std::env::var("RUSTFIX_TEST_RECORD_FIXED_RUST").is_ok() {
+    if std::env::var(settings::RECORD_FIXED_RUST).is_ok() {
         use std::io::Write;
         let mut recorded_rust = fs::File::create(&file.with_extension("recorded.rs"))?;
         recorded_rust.write_all(fixed.as_bytes())?;
     }
 
-    let expected_fixed = read_file(&fixed_file)?;
-    assert_eq!(
-        fixed.trim(),
-        expected_fixed.trim(),
-        "file {} doesn't look fixed",
-        file.display()
+    let expected_fixed = read_file(&fixed_file)
+        .context(format!("could read fixed file for {}", file.display()))?;
+    ensure!(
+        fixed.trim() == expected_fixed.trim(),
+        "file {} doesn't look fixed:\n{}", file.display(), diff(fixed.trim(), expected_fixed.trim())
     );
 
-    compiles_without_errors(&fixed_file)?;
+    compiles_without_errors(&fixed_file, mode)?;
 
     Ok(())
 }
 
-fn get_fixture_files(p: &str) -> Result<Vec<PathBuf>, Box<Error>> {
+fn get_fixture_files(p: &str) -> Result<Vec<PathBuf>, Error> {
     Ok(fs::read_dir(&p)?
         .into_iter()
         .map(|e| e.unwrap().path())
@@ -148,16 +199,40 @@ fn get_fixture_files(p: &str) -> Result<Vec<PathBuf>, Box<Error>> {
 }
 
 fn assert_fixtures(dir: &str, mode: &str) {
-    for file in &get_fixture_files(&dir).unwrap() {
-        env::set_var("RUSTFIX_MODE", mode);
-        test_rustfix_with_file(file).unwrap();
-        env::remove_var("RUSTFIX_MODE")
+    let files = get_fixture_files(&dir)
+        .context(format!("couldn't load dir `{}`", dir))
+        .unwrap();
+    let mut failures = 0;
+
+    for file in &files {
+        if let Err(err) = test_rustfix_with_file(file, mode) {
+            println!("failed: {}", file.display());
+            warn!("{}", err);
+            for cause in err.causes().skip(1) {
+                info!("\tcaused by: {}", cause);
+            }
+            failures += 1;
+        }
         info!("passed: {:?}", file);
     }
+
+    if failures > 0 {
+        panic!(
+            "{} out of {} fixture asserts failed\n\
+            (run with `env RUST_LOG=parse_and_replace=info` to get more details)",
+            failures, files.len(),
+        );
+    }
 }
 
 #[test]
 fn everything() {
     let _ = env_logger::try_init();
-    assert_fixtures("./tests/everything", "yolo");
+    assert_fixtures("./tests/everything", fixmode::EVERYTHING);
+}
+
+#[test]
+fn edition() {
+    let _ = env_logger::try_init();
+    assert_fixtures("./tests/edition", fixmode::EDITION);
 }