DO NOT MERGE: what I'm currently driving

Author: ErichDonGubler

moz-webgpu-cts/src/main.rs

@@ -123,6 +123,10 @@ enum Subcommand {
         /// `implementation-status`es that changes should be applied to.
         #[clap(value_enum, long, default_value = "backlog")]
         implementation_status: Vec<ImplementationStatus>,
+        /// Assumptions about report coverage that affect cases where tests are present in
+        /// metadata, but not in reports.
+        #[clap(flatten)]
+        on_missing: Option<OnMissing>,
         /// What do when only `SKIP` outcomes are found for tests and subtests.
         #[clap(value_enum, long, default_value_t = OnSkipOnly::Reconcile)]
         on_skip_only: OnSkipOnly,
@@ -205,6 +209,32 @@ impl From<UpdateExpectedPreset> for process_reports::ReportProcessingPreset {
     }
 }
 
+/// See [`Subcommand::UpdateExpected::on_missing`].
+#[derive(Clone, Debug, Parser)]
+pub(crate) struct OnMissing {
+    #[clap(long = "on-missing-log-level")]
+    log_level: Option<log::Level>,
+    #[clap(long = "on-missing-delete-because")]
+    delete_because: Option<String>,
+}
+
+impl From<OnMissing> for process_reports::OnMissing {
+    fn from(value: OnMissing) -> Self {
+        let OnMissing {
+            log_level,
+            delete_because,
+        } = value;
+
+        let delete_because = delete_because.map(|s| &*s.leak());
+
+        // OPT: big hack, oh goodness
+        Self {
+            log_level,
+            delete_because,
+        }
+    }
+}
+
 /// See [`Subcommand::UpdateExpected::on_skip_only`].
 #[derive(Clone, Copy, Debug, ValueEnum)]
 pub(crate) enum OnSkipOnly {
@@ -290,6 +320,10 @@ fn run(cli: Cli) -> ExitCode {
             exec_report_spec,
             process_reports::ReportProcessingPreset::MigrateTestStructure,
             &mut should_update_expected::NeverUpdateExpected,
+            process_reports::OnMissing {
+                log_level: Some(log::Level::Info),
+                delete_because: None,
+            },
             OnSkipOnly::Ignore.into(),
         ) {
             Ok(()) => ExitCode::SUCCESS,
@@ -299,6 +333,7 @@ fn run(cli: Cli) -> ExitCode {
             exec_report_spec,
             preset,
             implementation_status,
+            on_missing,
             on_skip_only,
         } => {
             assert!(
@@ -309,6 +344,20 @@ fn run(cli: Cli) -> ExitCode {
                 )
             );
             let allowed_implementation_statuses = EnumSet::from_iter(implementation_status);
+
+            let on_missing = on_missing.map(Into::into).unwrap_or_else(|| match preset {
+                UpdateExpectedPreset::Merge => process_reports::OnMissing {
+                    log_level: Some(log::Level::Warn),
+                    delete_because: None,
+                },
+                UpdateExpectedPreset::ResetAll | UpdateExpectedPreset::ResetContradictory => {
+                    process_reports::OnMissing {
+                        log_level: Some(log::Level::Warn),
+                        delete_because: Some("expecting full test coverage"),
+                    }
+                }
+            });
+
             match process_reports(
                 browser,
                 &checkout,
@@ -317,6 +366,7 @@ fn run(cli: Cli) -> ExitCode {
                 &mut should_update_expected::ImplementationStatusFilter {
                     allowed: allowed_implementation_statuses,
                 },
+                on_missing,
                 on_skip_only.into(),
             ) {
                 Ok(()) => ExitCode::SUCCESS,
@@ -1372,6 +1422,7 @@ fn process_reports(
     exec_report_spec: ExecReportSpec,
     preset: process_reports::ReportProcessingPreset,
     should_update_expected: &mut dyn ShouldUpdateExpected,
+    on_missing: process_reports::OnMissing,
     on_skip_only: process_reports::OnSkipOnly,
 ) -> Result<(), AlreadyReportedToCommandline> {
     let exec_report_paths = exec_report_spec.paths()?;
@@ -1386,6 +1437,7 @@ fn process_reports(
         preset,
         should_update_expected,
         meta_files_by_path,
+        on_missing,
         on_skip_only,
     })?;
 

moz-webgpu-cts/src/process_reports.rs

@@ -11,7 +11,7 @@ use std::{
 use camino::Utf8PathBuf;
 use enumset::EnumSetType;
 use indexmap::IndexMap;
-use lazy_format::lazy_format;
+use lazy_format::make_lazy_format;
 use miette::{IntoDiagnostic, Report, WrapErr};
 use rayon::prelude::{IntoParallelIterator, ParallelIterator};
 use whippit::metadata::SectionHeader;
@@ -57,6 +57,7 @@ pub(crate) struct ProcessReportsArgs<'a> {
     pub preset: ReportProcessingPreset,
     pub should_update_expected: &'a mut dyn should_update_expected::ShouldUpdateExpected,
     pub meta_files_by_path: IndexMap<Arc<PathBuf>, File>,
+    pub on_missing: OnMissing,
     pub on_skip_only: OnSkipOnly,
 }
 
@@ -68,6 +69,40 @@ pub(crate) enum ReportProcessingPreset {
     MigrateTestStructure,
 }
 
+#[derive(Clone, Copy, Debug)]
+pub(crate) struct OnMissing {
+    pub log_level: Option<log::Level>,
+    pub delete_because: Option<&'static str>,
+}
+
+impl OnMissing {
+    fn should_delete<F>(&self, msg: F) -> bool
+    where
+        F: Fn(&mut std::fmt::Formatter<'_>) -> std::fmt::Result,
+    {
+        let &Self {
+            log_level,
+            delete_because,
+        } = self;
+
+        if let Some(level) = log_level {
+            log::log!(
+                level,
+                "{}",
+                make_lazy_format!(|f| {
+                    if let Some(reason) = delete_because {
+                        write!(f, "removing metadata after {reason}, and ")?;
+                    }
+
+                    msg(f)
+                })
+            );
+        }
+
+        delete_because.is_some()
+    }
+}
+
 #[derive(Clone, Copy, Debug)]
 pub(crate) enum OnSkipOnly {
     /// Do not change metadata.
@@ -170,6 +205,7 @@ pub(crate) fn process_reports(
         preset,
         should_update_expected,
         meta_files_by_path,
+        on_missing,
         on_skip_only,
     } = args;
 
@@ -462,18 +498,14 @@ pub(crate) fn process_reports(
 
             if test_reported.is_empty() {
                 let test_entry_path = &test_entry_path;
-                let msg = lazy_format!("no entries found in reports for {:?}", test_entry_path);
-                match preset {
-                    ReportProcessingPreset::MergeOutcomes => log::warn!("{msg}"),
-                    ReportProcessingPreset::ResetAllOutcomes
-                    | ReportProcessingPreset::ResetContradictoryOutcomes => {
-                        log::warn!("removing metadata after {msg}");
-                        return None;
-                    }
-                    ReportProcessingPreset::MigrateTestStructure => {
-                        log::info!("removing metadata after {msg}");
-                        return None;
-                    }
+                if on_missing.should_delete(|f| {
+                    write!(
+                        f,
+                        "no entries were found in reports for {:?}",
+                        test_entry_path
+                    )
+                }) {
+                    return None;
                 }
             }
 
@@ -543,22 +575,17 @@ pub(crate) fn process_reports(
                     if subtest_reported.is_empty() {
                         let test_entry_path = &test_entry_path;
                         let subtest_name = &subtest_name;
-                        let msg = lazy_format!(
-                            "no subtest entries found in reports for {:?}, subtest {:?}",
-                            test_entry_path,
-                            subtest_name,
-                        );
-                        match preset {
-                            ReportProcessingPreset::MergeOutcomes => log::warn!("{msg}"),
-                            ReportProcessingPreset::ResetAllOutcomes
-                            | ReportProcessingPreset::ResetContradictoryOutcomes => {
-                                log::warn!("removing metadata after {msg}");
-                                return None;
-                            }
-                            ReportProcessingPreset::MigrateTestStructure => {
-                                log::info!("removing metadata after {msg}");
-                                return None;
-                            }
+                        if on_missing.should_delete(|f| {
+                            write!(
+                                f,
+                                concat!(
+                                    "no subtest entries found in reports ",
+                                    "for {:?}, subtest {:?}"
+                                ),
+                                test_entry_path, subtest_name,
+                            )
+                        }) {
+                            return None;
                         }
                     }