Add a way to track Rustfix UI test coverage

This came out of the first Rustfix WG meeting.

One of the goals is to enable Rustfix tests for all UI tests that
trigger lints with `MachineApplicable` suggestions. In order to do that
we first want to create a tracking issue that lists all files with
missing `// run-rustfix` headers.

This PR adds a `--rustfix-coverage` flag to `./x.py` and compiletest to
list the files with the missing headers in `/tmp/rustfix_missing_coverage.txt`.

From that file we can create the tracking issue and at some point also
enforce the `// run-rustfix` flag on UI tests with `MachineApplicable`
lints.

Author: phansch

src/bootstrap/flags.rs

@@ -56,6 +56,7 @@ pub enum Subcommand {
         rustc_args: Vec<String>,
         fail_fast: bool,
         doc_tests: DocTests,
+        rustfix_coverage: bool,
     },
     Bench {
         paths: Vec<PathBuf>,
@@ -188,6 +189,12 @@ To learn more about a subcommand, run `./x.py <subcommand> -h`"
                     "mode describing what file the actual ui output will be compared to",
                     "COMPARE MODE",
                 );
+                opts.optflag(
+                    "",
+                    "rustfix-coverage",
+                    "enable this to generate a Rustfix coverage file, which is saved in \
+                        `/tmp/rustfix_missing_coverage.txt`",
+                );
             }
             "bench" => {
                 opts.optmulti("", "test-args", "extra arguments", "ARGS");
@@ -363,6 +370,7 @@ Arguments:
                 test_args: matches.opt_strs("test-args"),
                 rustc_args: matches.opt_strs("rustc-args"),
                 fail_fast: !matches.opt_present("no-fail-fast"),
+                rustfix_coverage: matches.opt_present("rustfix-coverage"),
                 doc_tests: if matches.opt_present("doc") {
                     DocTests::Only
                 } else if matches.opt_present("no-doc") {
@@ -467,6 +475,13 @@ impl Subcommand {
         }
     }
 
+    pub fn rustfix_coverage(&self) -> bool {
+        match *self {
+            Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage,
+            _ => false,
+        }
+    }
+
     pub fn compare_mode(&self) -> Option<&str> {
         match *self {
             Subcommand::Test {

src/bootstrap/test.rs

@@ -1284,6 +1284,10 @@ impl Step for Compiletest {
             cmd.arg("--android-cross-path").arg("");
         }
 
+        if builder.config.cmd.rustfix_coverage() {
+            cmd.arg("--rustfix-coverage");
+        }
+
         builder.ci_env.force_coloring_in_ci(&mut cmd);
 
         let _folder = builder.fold_output(|| format!("test_{}", suite));

src/tools/compiletest/src/common.rs

@@ -245,6 +245,11 @@ pub struct Config {
     /// mode describing what file the actual ui output will be compared to
     pub compare_mode: Option<CompareMode>,
 
+    /// If true, this will generate a coverage file with UI test files that run `MachineApplicable`
+    /// lints but are missing `run-rustfix` annotations. The generated coverage file is created in
+    /// `/tmp/rustfix_missing_coverage.txt`
+    pub rustfix_coverage: bool,
+
     // Configuration for various run-make tests frobbing things like C compilers
     // or querying about various LLVM component information.
     pub cc: String,

src/tools/compiletest/src/main.rs

@@ -233,6 +233,12 @@ pub fn parse_config(args: Vec<String>) -> Config {
             "mode describing what file the actual ui output will be compared to",
             "COMPARE MODE",
         )
+        .optflag(
+            "",
+            "rustfix-coverage",
+            "enable this to generate a Rustfix coverage file, which is saved in \
+                `/tmp/rustfix_missing_coverage.txt`",
+        )
         .optflag("h", "help", "show this message");
 
     let (argv0, args_) = args.split_first().unwrap();
@@ -336,6 +342,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         color,
         remote_test_client: matches.opt_str("remote-test-client").map(PathBuf::from),
         compare_mode: matches.opt_str("compare-mode").map(CompareMode::parse),
+        rustfix_coverage: matches.opt_present("rustfix-coverage"),
 
         cc: matches.opt_str("cc").unwrap(),
         cxx: matches.opt_str("cxx").unwrap(),
@@ -475,6 +482,18 @@ pub fn run_tests(config: &Config) {
         let _ = fs::remove_dir_all("tmp/partitioning-tests");
     }
 
+    // If we want to collect rustfix coverage information,
+    // we first make sure that the coverage file does not exist.
+    // It will be created later on.
+    if config.rustfix_coverage {
+        let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt");
+        if coverage_file_path.exists() {
+            if let Err(e) = fs::remove_file(coverage_file_path) {
+                panic!("Could not delete {} due to {}", coverage_file_path.display(), e)
+            }
+        }
+    }
+
     let opts = test_opts(config);
     let tests = make_tests(config);
     // sadly osx needs some file descriptor limits raised for running tests in

src/tools/compiletest/src/runtest.rs

@@ -19,7 +19,7 @@ use std::collections::{HashMap, HashSet, VecDeque};
 use std::env;
 use std::ffi::{OsStr, OsString};
 use std::fmt;
-use std::fs::{self, create_dir_all, File};
+use std::fs::{self, create_dir_all, File, OpenOptions};
 use std::hash::{Hash, Hasher};
 use std::io::prelude::*;
 use std::io::{self, BufReader};
@@ -2818,6 +2818,28 @@ impl<'test> TestCx<'test> {
 
         if self.config.compare_mode.is_some() {
             // don't test rustfix with nll right now
+        } else if self.config.rustfix_coverage {
+            // Find out which tests have `MachineApplicable` suggestions but are missing
+            // `run-rustfix` or `run-rustfix-only-machine-applicable` headers
+            let suggestions = get_suggestions_from_json(
+                &proc_res.stderr,
+                &HashSet::new(),
+                Filter::MachineApplicableOnly
+            ).unwrap();
+            if suggestions.len() > 0
+                && !self.props.run_rustfix
+                && !self.props.rustfix_only_machine_applicable {
+                    let coverage_file_path = Path::new("/tmp/rustfix_missing_coverage.txt");
+                    let mut file = OpenOptions::new()
+                        .create(true)
+                        .append(true)
+                        .open(coverage_file_path)
+                        .expect("could not create or open file");
+
+                    if let Err(_) = writeln!(file, "{}", self.testpaths.file.display()) {
+                        panic!("couldn't write to {}", coverage_file_path.display());
+                    }
+            }
         } else if self.props.run_rustfix {
             // Apply suggestions from rustc to the code itself
             let unfixed_code = self