Rollup merge of rust-lang#139978 - Kobzol:ci-test-summary, r=jieyouxu

Add citool command for generating a test dashboard

This PR implements an initial version of a test suite dashboard, which shows which tests were executed on CI and which tests were ignored. This was discussed [here](https://rust-lang.zulipchat.com/#narrow/channel/238009-t-compiler.2Fmeetings/topic/.5Bsteering.5D.202025-04-11.20Dealing.20with.20disabled.20tests/with/512799036). The dashboard is still quite bare-bones, but I think that it could already be useful.

The next step is to create a job index, similarly to the post-merge report, and link from the individual tests to the job that executed them.

You can try it locally like this:
```bash
$ cargo run --manifest-path src/ci/citool/Cargo.toml --release \
    -- test-dashboard 38c560a --output-dir dashboard
```
and then open `dashboard/index.html` in a web browser.

CC ````@wesleywiser````

r? ````@jieyouxu````

Author: ChrisDenton

src/ci/citool/Cargo.lock

@@ -64,12 +64,63 @@ version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
 
+[[package]]
+name = "askama"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d4744ed2eef2645831b441d8f5459689ade2ab27c854488fbab1fbe94fce1a7"
+dependencies = [
+ "askama_derive",
+ "itoa",
+ "percent-encoding",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "askama_derive"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d661e0f57be36a5c14c48f78d09011e67e0cb618f269cca9f2fd8d15b68c46ac"
+dependencies = [
+ "askama_parser",
+ "basic-toml",
+ "memchr",
+ "proc-macro2",
+ "quote",
+ "rustc-hash",
+ "serde",
+ "serde_derive",
+ "syn",
+]
+
+[[package]]
+name = "askama_parser"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf315ce6524c857bb129ff794935cf6d42c82a6cff60526fe2a63593de4d0d4f"
+dependencies = [
+ "memchr",
+ "serde",
+ "serde_derive",
+ "winnow",
+]
+
 [[package]]
 name = "base64"
 version = "0.22.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
 
+[[package]]
+name = "basic-toml"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba62675e8242a4c4e806d12f11d136e626e6c8361d6b829310732241652a178a"
+dependencies = [
+ "serde",
+]
+
 [[package]]
 name = "build_helper"
 version = "0.1.0"
@@ -104,6 +155,7 @@ name = "citool"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "askama",
  "build_helper",
  "clap",
  "csv",
@@ -646,6 +698,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+
 [[package]]
 name = "rustls"
 version = "0.23.23"
@@ -1026,6 +1084,15 @@ version = "0.52.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
 
+[[package]]
+name = "winnow"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "63d3fcd9bba44b03821e7d699eeee959f3126dcc4aa8e4ae18ec617c2a5cea10"
+dependencies = [
+ "memchr",
+]
+
 [[package]]
 name = "write16"
 version = "1.0.0"

src/ci/citool/Cargo.toml

@@ -5,6 +5,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "1"
+askama = "0.13"
 clap = { version = "4.5", features = ["derive"] }
 csv = "1"
 diff = "0.1"

src/ci/citool/src/analysis.rs

@@ -8,9 +8,9 @@ use build_helper::metrics::{
 };
 
 use crate::github::JobInfoResolver;
-use crate::metrics;
 use crate::metrics::{JobMetrics, JobName, get_test_suites};
 use crate::utils::{output_details, pluralize};
+use crate::{metrics, utils};
 
 /// Outputs durations of individual bootstrap steps from the gathered bootstrap invocations,
 /// and also a table with summarized information about executed tests.
@@ -394,18 +394,17 @@ fn aggregate_tests(metrics: &JsonRoot) -> TestSuiteData {
             // Poor man's detection of doctests based on the "(line XYZ)" suffix
             let is_doctest = matches!(suite.metadata, TestSuiteMetadata::CargoPackage { .. })
                 && test.name.contains("(line");
-            let test_entry = Test { name: generate_test_name(&test.name), stage, is_doctest };
+            let test_entry = Test {
+                name: utils::normalize_path_delimiters(&test.name).to_string(),
+                stage,
+                is_doctest,
+            };
             tests.insert(test_entry, test.outcome.clone());
         }
     }
     TestSuiteData { tests }
 }
 
-/// Normalizes Windows-style path delimiters to Unix-style paths.
-fn generate_test_name(name: &str) -> String {
-    name.replace('\\', "/")
-}
-
 /// Prints test changes in Markdown format to stdout.
 fn report_test_diffs(
     diff: AggregatedTestDiffs,

src/ci/citool/src/main.rs

@@ -4,6 +4,7 @@ mod datadog;
 mod github;
 mod jobs;
 mod metrics;
+mod test_dashboard;
 mod utils;
 
 use std::collections::{BTreeMap, HashMap};
@@ -22,7 +23,8 @@ use crate::datadog::upload_datadog_metric;
 use crate::github::JobInfoResolver;
 use crate::jobs::RunType;
 use crate::metrics::{JobMetrics, download_auto_job_metrics, download_job_metrics, load_metrics};
-use crate::utils::load_env_var;
+use crate::test_dashboard::generate_test_dashboard;
+use crate::utils::{load_env_var, output_details};
 
 const CI_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/..");
 const DOCKER_DIRECTORY: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../docker");
@@ -180,12 +182,26 @@ fn postprocess_metrics(
 }
 
 fn post_merge_report(db: JobDatabase, current: String, parent: String) -> anyhow::Result<()> {
-    let metrics = download_auto_job_metrics(&db, &parent, &current)?;
+    let metrics = download_auto_job_metrics(&db, Some(&parent), &current)?;
 
     println!("\nComparing {parent} (parent) -> {current} (this PR)\n");
 
     let mut job_info_resolver = JobInfoResolver::new();
     output_test_diffs(&metrics, &mut job_info_resolver);
+
+    output_details("Test dashboard", || {
+        println!(
+            r#"\nRun
+
+```bash
+cargo run --manifest-path src/ci/citool/Cargo.toml -- \
+    test-dashboard {current} --output-dir test-dashboard
+```
+And then open `test-dashboard/index.html` in your browser to see an overview of all executed tests.
+"#
+        );
+    });
+
     output_largest_duration_changes(&metrics, &mut job_info_resolver);
 
     Ok(())
@@ -234,6 +250,14 @@ enum Args {
         /// Current commit that will be compared to `parent`.
         current: String,
     },
+    /// Generate a directory containing a HTML dashboard of test results from a CI run.
+    TestDashboard {
+        /// Commit SHA that was tested on CI to analyze.
+        current: String,
+        /// Output path for the HTML directory.
+        #[clap(long)]
+        output_dir: PathBuf,
+    },
 }
 
 #[derive(clap::ValueEnum, Clone)]
@@ -275,7 +299,11 @@ fn main() -> anyhow::Result<()> {
             postprocess_metrics(metrics_path, parent, job_name)?;
         }
         Args::PostMergeReport { current, parent } => {
-            post_merge_report(load_db(default_jobs_file)?, current, parent)?;
+            post_merge_report(load_db(&default_jobs_file)?, current, parent)?;
+        }
+        Args::TestDashboard { current, output_dir } => {
+            let db = load_db(&default_jobs_file)?;
+            generate_test_dashboard(db, &current, &output_dir)?;
         }
     }
 

src/ci/citool/src/metrics.rs

@@ -46,24 +46,25 @@ pub struct JobMetrics {
 /// `parent` and `current` should be commit SHAs.
 pub fn download_auto_job_metrics(
     job_db: &JobDatabase,
-    parent: &str,
+    parent: Option<&str>,
     current: &str,
 ) -> anyhow::Result<HashMap<JobName, JobMetrics>> {
     let mut jobs = HashMap::default();
 
     for job in &job_db.auto_jobs {
         eprintln!("Downloading metrics of job {}", job.name);
-        let metrics_parent = match download_job_metrics(&job.name, parent) {
-            Ok(metrics) => Some(metrics),
-            Err(error) => {
-                eprintln!(
-                    r#"Did not find metrics for job `{}` at `{parent}`: {error:?}.
+        let metrics_parent =
+            parent.and_then(|parent| match download_job_metrics(&job.name, parent) {
+                Ok(metrics) => Some(metrics),
+                Err(error) => {
+                    eprintln!(
+                        r#"Did not find metrics for job `{}` at `{parent}`: {error:?}.
 Maybe it was newly added?"#,
-                    job.name
-                );
-                None
-            }
-        };
+                        job.name
+                    );
+                    None
+                }
+            });
         let metrics_current = download_job_metrics(&job.name, current)?;
         jobs.insert(
             job.name.clone(),