fix(runner): display no projects error across all test modes (#34676)

Author: agg23

packages/playwright/src/program.ts

@@ -29,6 +29,7 @@ export { program } from 'playwright-core/lib/cli/program';
 import { prepareErrorStack } from './reporters/base';
 import { showHTMLReport } from './reporters/html';
 import { createMergedReport } from './reporters/merge';
+import { filterProjects } from './runner/projectUtils';
 import { Runner } from './runner/runner';
 import * as testServer from './runner/testServer';
 import { runWatchModeLoop } from './runner/watchMode';
@@ -161,6 +162,23 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
   await startProfiling();
   const cliOverrides = overridesFromOptions(opts);
 
+  const config = await loadConfigFromFileRestartIfNeeded(opts.config, cliOverrides, opts.deps === false);
+  if (!config)
+    return;
+
+  config.cliArgs = args;
+  config.cliGrep = opts.grep as string | undefined;
+  config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
+  config.cliGrepInvert = opts.grepInvert as string | undefined;
+  config.cliListOnly = !!opts.list;
+  config.cliProjectFilter = opts.project || undefined;
+  config.cliPassWithNoTests = !!opts.passWithNoTests;
+  config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
+  config.cliLastFailed = !!opts.lastFailed;
+
+  // Evaluate project filters against config before starting execution. This enables a consistent error message across run modes
+  filterProjects(config.projects, config.cliProjectFilter);
+
   if (opts.ui || opts.uiHost || opts.uiPort) {
     if (opts.onlyChanged)
       throw new Error(`--only-changed is not supported in UI mode. If you'd like that to change, see https://github.com/microsoft/playwright/issues/15075 for more details.`);
@@ -202,20 +220,6 @@ async function runTests(args: string[], opts: { [key: string]: any }) {
     return;
   }
 
-  const config = await loadConfigFromFileRestartIfNeeded(opts.config, cliOverrides, opts.deps === false);
-  if (!config)
-    return;
-
-  config.cliArgs = args;
-  config.cliGrep = opts.grep as string | undefined;
-  config.cliOnlyChanged = opts.onlyChanged === true ? 'HEAD' : opts.onlyChanged;
-  config.cliGrepInvert = opts.grepInvert as string | undefined;
-  config.cliListOnly = !!opts.list;
-  config.cliProjectFilter = opts.project || undefined;
-  config.cliPassWithNoTests = !!opts.passWithNoTests;
-  config.cliFailOnFlakyTests = !!opts.failOnFlakyTests;
-  config.cliLastFailed = !!opts.lastFailed;
-
   const runner = new Runner(config);
   const status = await runner.runAllTests();
   await stopProfiling('runner');

tests/playwright-test/config.spec.ts

@@ -327,6 +327,25 @@ test('should print nice error when project is unknown', async ({ runInlineTest }
   expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
 });
 
+test('should print nice error when project is unknown and launching UI mode', async ({ runInlineTest }) => {
+  // Prevent UI mode from opening and the test never finishing
+  test.setTimeout(5000);
+  const { output, exitCode } = await runInlineTest({
+    'playwright.config.ts': `
+      module.exports = { projects: [
+        { name: 'suite1' },
+        { name: 'suite2' },
+      ] };
+    `,
+    'a.test.ts': `
+      import { test, expect } from '@playwright/test';
+      test('pass', async ({}, testInfo) => {});
+    `
+  }, { project: 'suite3', ui: true });
+  expect(exitCode).toBe(1);
+  expect(output).toContain('Project(s) "suite3" not found. Available projects: "suite1", "suite2"');
+});
+
 test('should filter by project list, case-insensitive', async ({ runInlineTest }) => {
   const { passed, failed, outputLines, skipped } = await runInlineTest({
     'playwright.config.ts': `

tests/playwright-test/ui-mode-test-setup.spec.ts

@@ -95,17 +95,6 @@ test('should teardown on sigint', async ({ runUITest, nodeVersion }) => {
   ]);
 });
 
-test('should show errors in config', async ({ runUITest }) => {
-  const { page } = await runUITest({
-    'playwright.config.ts': `
-      import { defineConfig, devices } from '@playwright/test';
-      throw new Error("URL is empty")
-    `,
-  });
-  await page.getByText('playwright.config.ts').click();
-  await expect(page.getByText('Error: URL is empty')).toBeInViewport();
-});
-
 const testsWithSetup = {
   'playwright.config.ts': `
     import { defineConfig } from '@playwright/test';