Auto merge of #4837 - flip1995:integration, r=<try>

[WIP] RIIR: Integration tests

In #4825 the `rust-lang/chalk` test failed because the output was too large. I didn't want to completely disabling the output, since showing the backtrace of an ICE directly in travis is pretty useful. Since finding strings in command outputs is easier in Rust, than in bash, I just RIIRed it.

This and also rewriting our tests in Rust may help with trying out new CI platforms (cc #4577)

changelog: none

Author: bors

.travis.yml

@@ -61,14 +61,12 @@ matrix:
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=rust-lang/cargo
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
-    # FIXME: Output too large
-    # - env: INTEGRATION=rust-lang-nursery/chalk
-    #   if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
+    - env: INTEGRATION=rust-lang/chalk
+      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=Geal/nom
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
-    # FIXME blocked on https://github.com/rust-lang/rust-clippy/issues/4727
-    #- env: INTEGRATION=rust-lang/rustfmt
-    #  if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
+    - env: INTEGRATION=rust-lang/rustfmt
+      if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=hyperium/hyper
       if: repo =~ /^rust-lang\/rust-clippy$/ AND branch IN (auto, try)
     - env: INTEGRATION=bluss/rust-itertools
@@ -127,7 +125,7 @@ before_script:
 script:
   - |
     if [[ -n ${INTEGRATION} ]]; then
-      ./ci/integration-tests.sh && sleep 5
+      cargo test --test integration --features integration && sleep 5
     else
       ./ci/base-tests.sh && sleep 5
     fi

Cargo.toml

@@ -38,6 +38,8 @@ clippy_lints = { version = "0.0.212", path = "clippy_lints" }
 regex = "1"
 semver = "0.9"
 rustc_tools_util = { version = "0.2.0", path = "rustc_tools_util"}
+git2 = { version = "0.10", optional = true }
+tempfile = { version = "3.1.0", optional = true }
 
 [dev-dependencies]
 cargo_metadata = "0.9.0"
@@ -59,3 +61,4 @@ rustc_tools_util = { version = "0.2.0", path = "rustc_tools_util"}
 [features]
 deny-warnings = []
 debugging = []
+integration = ["git2", "tempfile"]

ci/integration-tests.sh

@@ -0,0 +1,75 @@
+#![cfg(feature = "integration")]
+
+use git2::Repository;
+use tempfile;
+
+use std::env;
+use std::process::Command;
+
+#[cfg_attr(feature = "integration", test)]
+fn integration_test() {
+    let repo_name = env::var("INTEGRATION").expect("`INTEGRATION` var not set");
+    let repo_url = format!("https://github.com/{}", repo_name);
+    let crate_name = repo_name
+        .split('/')
+        .nth(1)
+        .expect("repo name should have format `<org>/<name>`");
+
+    let repo_dir = tempfile::tempdir()
+        .expect("couldn't create temp dir")
+        .path()
+        .join(crate_name);
+
+    Repository::clone(&repo_url, &repo_dir).expect("clone of repo failed");
+
+    let root_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+    let target_dir = std::path::Path::new(&root_dir).join("target");
+    let clippy_binary = target_dir.join(env!("PROFILE")).join("cargo-clippy");
+
+    let output = Command::new(clippy_binary)
+        .current_dir(repo_dir)
+        .env("RUST_BACKTRACE", "full")
+        .env("CARGO_TARGET_DIR", target_dir)
+        .args(&[
+            "clippy",
+            "--all-targets",
+            "--all-features",
+            "--",
+            "--cap-lints",
+            "warn",
+            "-Wclippy::pedantic",
+            "-Wclippy::nursery",
+        ])
+        .output()
+        .expect("unable to run clippy");
+
+    let stderr = String::from_utf8_lossy(&output.stderr);
+    if stderr.contains("internal compiler error") {
+        let backtrace_start = stderr
+            .find("thread 'rustc' panicked at")
+            .expect("start of backtrace not found");
+        let backtrace_end = stderr
+            .rfind("error: internal compiler error")
+            .expect("end of backtrace not found");
+
+        panic!(
+            "internal compiler error\nBacktrace:\n\n{}",
+            &stderr[backtrace_start..backtrace_end]
+        );
+    } else if stderr.contains("query stack during panic") {
+        panic!("query stack during panic in the output");
+    } else if stderr.contains("E0463") {
+        panic!("error: E0463");
+    }
+
+    match output.status.code() {
+        Some(code) => {
+            if code != 0 {
+                eprintln!("Compilation failed. Exit code: {}", code);
+            } else {
+                eprintln!("Compilation successful");
+            }
+        },
+        None => panic!("Process terminated by signal"),
+    }
+}