From aeed1f590934564c0c9d1c599d5c6ec9df378d9b Mon Sep 17 00:00:00 2001
From: Adam Gastineau <adam@agg.im>
Date: Mon, 10 Feb 2025 12:52:53 -0800
Subject: [PATCH] fix(runner): display no projects error across all `test`
 modes (#34676)

---
 packages/playwright/src/program.ts            | 32 +++++++++++--------
 tests/playwright-test/config.spec.ts          | 19 +++++++++++
 .../ui-mode-test-setup.spec.ts                | 11 -------
 3 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index a759e6f366a5b..7aef21faee8e7 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -29,6 +29,7 @@ export { program } from 'playwright-core/lib/cli/program';
 import { prepareErrorStack } from './reporters/base';
 import { showHTMLReport } from './reporters/html';
 import { createMergedReport } from './reporters/merge';
+import { filterProjects } from './runner/projectUtils';
 import { Runner } from './runner/runner';
 import * as testServer from './runner/testServer';
 import { runWatchModeLoop } from './runner/watchMode';
@@ -161,6 +162,23 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   await startProfiling();
   const cliOverrides = overridesFromOptions(opts);
 
+  const config = await loadConfigFromFileRestartIfNeeded(opts.config, cliOverrides, opts.deps === false);
+  if (!config)
+    return;
+
+  config.cliArgs = args;
+  config.cliGrep = opts.grep as string | undefined;
+  config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
+  config.cliGrepInvert = opts.grepInvert as string | undefined;
+  config.cliListOnly = !!opts.list;
+  config.cliProjectFilter = opts.project || undefined;
+  config.cliPassWithNoTests = !!opts.passWithNoTests;
+  config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
+  config.cliLastFailed = !!opts.lastFailed;
+
+  // Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
+  filterProjects(config.projects, config.cliProjectFilter);
+
   if (opts.ui || opts.uiHost || opts.uiPort) {
     if (opts.onlyChanged)
       throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`);
@@ -202,20 +220,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
     return;
   }
 
-  const config = await loadConfigFromFileRestartIfNeeded(opts.config, cliOverrides, opts.deps === false);
-  if (!config)
-    return;
-
-  config.cliArgs = args;
-  config.cliGrep = opts.grep as string | undefined;
-  config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
-  config.cliGrepInvert = opts.grepInvert as string | undefined;
-  config.cliListOnly = !!opts.list;
-  config.cliProjectFilter = opts.project || undefined;
-  config.cliPassWithNoTests = !!opts.passWithNoTests;
-  config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
-  config.cliLastFailed = !!opts.lastFailed;
-
   const runner = new Runner(config);
   const status = await runner.runAllTests();
   await stopProfiling('runner');
diff --git a/tests/playwright-test/config.spec.ts b/tests/playwright-test/config.spec.ts
index 7870d7470677e..518bf33839449 100644
--- a/tests/playwright-test/config.spec.ts
+++ b/tests/playwright-test/config.spec.ts
@@ -327,6 +327,25 @@ test('should print nice error when project is unknown', async ({ runInlineTest }
   expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
 });
 
+test('should print nice error when project is unknown and launching UI mode', async ({ runInlineTest }) => {
+  // Prevent UI mode from opening and the test never finishing
+  test.setTimeout(5000);
+  const { output, exitCode } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('pass', async ({}, testInfo) => {});
+    `
+  }, { project: 'suite3', ui: true });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
+});
+
 test('should filter by project list, case-insensitive', async ({ runInlineTest }) => {
   const { passed, failed, outputLines, skipped } = await runInlineTest({
     'playwright.config.ts': `
diff --git a/tests/playwright-test/ui-mode-test-setup.spec.ts b/tests/playwright-test/ui-mode-test-setup.spec.ts
index d3fefb27fcd36..c912038571bdd 100644
--- a/tests/playwright-test/ui-mode-test-setup.spec.ts
+++ b/tests/playwright-test/ui-mode-test-setup.spec.ts
@@ -95,17 +95,6 @@ test('should teardown on sigint', async ({ runUITest, nodeVersion }) => {
   ]);
 });
 
-test('should show errors in config', async ({ runUITest }) => {
-  const { page } = await runUITest({
-    'playwright.config.ts': `
-      import { defineConfig, devices } from '@playwright/test';
-      throw new Error("URL is empty")
-    `,
-  });
-  await page.getByText('playwright.config.ts').click();
-  await expect(page.getByText('Error: URL is empty')).toBeInViewport();
-});
-
 const testsWithSetup = {
   'playwright.config.ts': `
     import { defineConfig } from '@playwright/test';