Rollup merge of rust-lang#37016 - alexcrichton:workspaces, r=japaric

Leverage Cargo workspaces in rustbuild

This is a continuation of rust-lang#36032 which implements the change to use `cargo metadata` to learn about the crate graph.

Author: Manishearth

src/Cargo.lock

@@ -0,0 +1,13 @@
+[workspace]
+members = [
+  "bootstrap",
+  "rustc",
+  "rustc/std_shim",
+  "rustc/test_shim",
+  "tools/cargotest",
+  "tools/compiletest",
+  "tools/error_index_generator",
+  "tools/linkchecker",
+  "tools/rustbook",
+  "tools/tidy",
+]

src/Cargo.toml

@@ -13,19 +13,44 @@
 //! This file implements the various regression test suites that we execute on
 //! our CI.
 
+use std::collections::{HashMap, HashSet};
 use std::env;
-use std::fs::{self, File};
-use std::io::prelude::*;
+use std::fs;
 use std::path::{PathBuf, Path};
 use std::process::Command;
 
 use build_helper::output;
+use rustc_serialize::json;
 
 use {Build, Compiler, Mode};
 use util::{self, dylib_path, dylib_path_var};
 
 const ADB_TEST_DIR: &'static str = "/data/tmp";
 
+#[derive(RustcDecodable)]
+struct Output {
+    packages: Vec<Package>,
+    resolve: Resolve,
+}
+
+#[derive(RustcDecodable)]
+struct Package {
+    id: String,
+    name: String,
+    source: Option<String>,
+}
+
+#[derive(RustcDecodable)]
+struct Resolve {
+    nodes: Vec<ResolveNode>,
+}
+
+#[derive(RustcDecodable)]
+struct ResolveNode {
+    id: String,
+    dependencies: Vec<String>,
+}
+
 /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
 ///
 /// This tool in `src/tools` will verify the validity of all our links in the
@@ -263,56 +288,74 @@ fn markdown_test(build: &Build, compiler: &Compiler, markdown: &Path) {
 /// It essentially is the driver for running `cargo test`.
 ///
 /// Currently this runs all tests for a DAG by passing a bunch of `-p foo`
-/// arguments, and those arguments are discovered from `Cargo.lock`.
+/// arguments, and those arguments are discovered from `cargo metadata`.
 pub fn krate(build: &Build,
              compiler: &Compiler,
              target: &str,
              mode: Mode) {
-    let (name, path, features) = match mode {
-        Mode::Libstd => ("libstd", "src/rustc/std_shim", build.std_features()),
-        Mode::Libtest => ("libtest", "src/rustc/test_shim", String::new()),
-        Mode::Librustc => ("librustc", "src/rustc", build.rustc_features()),
+    let (name, path, features, root) = match mode {
+        Mode::Libstd => {
+            ("libstd", "src/rustc/std_shim", build.std_features(), "std_shim")
+        }
+        Mode::Libtest => {
+            ("libtest", "src/rustc/test_shim", String::new(), "test_shim")
+        }
+        Mode::Librustc => {
+            ("librustc", "src/rustc", build.rustc_features(), "rustc-main")
+        }
         _ => panic!("can only test libraries"),
     };
     println!("Testing {} stage{} ({} -> {})", name, compiler.stage,
              compiler.host, target);
 
+    // Run `cargo metadata` to figure out what crates we're testing.
+    //
+    // Down below we're going to call `cargo test`, but to test the right set
+    // of packages we're going to have to know what `-p` arguments to pass it
+    // to know what crates to test. Here we run `cargo metadata` to learn about
+    // the dependency graph and what `-p` arguments there are.
+    let mut cargo = Command::new(&build.cargo);
+    cargo.arg("metadata")
+         .arg("--manifest-path").arg(build.src.join(path).join("Cargo.toml"));
+    let output = output(&mut cargo);
+    let output: Output = json::decode(&output).unwrap();
+    let id2pkg = output.packages.iter()
+                        .map(|pkg| (&pkg.id, pkg))
+                        .collect::<HashMap<_, _>>();
+    let id2deps = output.resolve.nodes.iter()
+                        .map(|node| (&node.id, &node.dependencies))
+                        .collect::<HashMap<_, _>>();
+
     // Build up the base `cargo test` command.
+    //
+    // Pass in some standard flags then iterate over the graph we've discovered
+    // in `cargo metadata` with the maps above and figure out what `-p`
+    // arguments need to get passed.
     let mut cargo = build.cargo(compiler, mode, target, "test");
     cargo.arg("--manifest-path")
          .arg(build.src.join(path).join("Cargo.toml"))
          .arg("--features").arg(features);
 
-    // Generate a list of `-p` arguments to pass to the `cargo test` invocation
-    // by crawling the corresponding Cargo.lock file.
-    let lockfile = build.src.join(path).join("Cargo.lock");
-    let mut contents = String::new();
-    t!(t!(File::open(&lockfile)).read_to_string(&mut contents));
-    let mut lines = contents.lines();
-    while let Some(line) = lines.next() {
-        let prefix = "name = \"";
-        if !line.starts_with(prefix) {
+    let mut visited = HashSet::new();
+    let root_pkg = output.packages.iter().find(|p| p.name == root).unwrap();
+    let mut next = vec![&root_pkg.id];
+    while let Some(id) = next.pop() {
+        // Skip any packages with sources listed, as these come from crates.io
+        // and we shouldn't be testing them.
+        if id2pkg[id].source.is_some() {
             continue
         }
-        lines.next(); // skip `version = ...`
-
-        // skip crates.io or otherwise non-path crates
-        if let Some(line) = lines.next() {
-            if line.starts_with("source") {
-                continue
-            }
-        }
-
-        let crate_name = &line[prefix.len()..line.len() - 1];
-
         // Right now jemalloc is our only target-specific crate in the sense
         // that it's not present on all platforms. Custom skip it here for now,
         // but if we add more this probably wants to get more generalized.
-        if crate_name.contains("jemalloc") {
-            continue
+        if !id.contains("jemalloc") {
+            cargo.arg("-p").arg(&id2pkg[id].name);
+        }
+        for dep in id2deps[id] {
+            if visited.insert(dep) {
+                next.push(dep);
+            }
         }
-
-        cargo.arg("-p").arg(crate_name);
     }
 
     // The tests are going to run with the *target* libraries, so we need to