Run Karma Chrome tests headlessly (#6997)

Avoid opening a bunch of browser pages when running karma chrome tests where possible. This excludes WebGPU on Linux because headless webgpu is not supported on linux yet.

Add the flag `--//:headless=true|false` (default true) to control this and set it to false on Windows.

Author: mattsoulanille

.bazelrc

@@ -68,11 +68,12 @@ build --action_env=BROWSERSTACK_USERNAME --action_env=BROWSERSTACK_KEY
 run --action_env=BROWSERSTACK_USERNAME --action_env=BROWSERSTACK_KEY
 test --action_env=BROWSERSTACK_USERNAME --action_env=BROWSERSTACK_KEY
 
-# Platform specific DISPLAY environment variable for webgl
+# Platform specific DISPLAY environment variable for webgl and headless setting
+# for browser tests.
 test --enable_platform_specific_config
 test:linux --test_env=DISPLAY
 test:macos --define DISPLAY=true
-test:windows --define DISPLAY=true
+test:windows --define DISPLAY=true --//:headless=false
 
 # Enable debugging tests with --config=debug
 run:debug --test_arg=--node_options=--inspect-brk --test_output=streamed --test_strategy=exclusive --test_timeout=9999 --nocache_test_results

BUILD.bazel

@@ -14,7 +14,7 @@
 # =============================================================================
 
 load("@npm//@bazel/concatjs:index.bzl", "ts_config")
-load("//tools:tfjs_web_test.bzl", "grep_flag")
+load("//tools:tfjs_web_test.bzl", "grep_flag", "headless_flag")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -42,6 +42,12 @@ grep_flag(
     visibility = ["//visibility:public"],
 )
 
+headless_flag(
+    name = "headless",
+    build_setting_default = True,
+    visibility = ["//visibility:public"],
+)
+
 test_suite(
     name = "tests",
     tests = [

DEVELOPMENT.md

@@ -67,7 +67,7 @@ $ yarn test --//:grep=multinomial
 > Chrome 62.0.3202 (Mac OS X 10.12.6): Executed 28 of 1891 (skipped 1863) SUCCESS (6.914 secs / 0.634 secs)
 ```
 
-By default, the tests run once and exit. To keep the browser window open, run the following:
+By default, the tests run once and exit. To keep the karma server active, run the following:
 
 ```bash
 # For packages with only browser tests
@@ -76,6 +76,17 @@ $ yarn test-debug
 $ yarn test-browser-debug
 ```
 
+You can now connect to the server with your own browser.
+
+On most platforms, Karma tests will run with a headless Chrome browser. This can be overridden by setting the
+`--//:headless` flag.
+
+```bash
+# Run tests with a visible browser window. Note this does not work well with
+# 'watch' modes such as 'test-dev' or 'ibazel'.
+$ yarn test --//:headless=false
+```
+
 #### Packaging (browser and npm)
 
 In any of the directories the following commands build the NPM tarball:

tools/karma_template.conf.js

@@ -20,6 +20,9 @@ const browserstackConfig = {
   port: 9876,
 };
 
+// Select Chrome or ChromeHeadless based on the value of the --//:headless flag.
+const CHROME = TEMPLATE_headless ? 'ChromeHeadless' : 'Chrome';
+
 const CUSTOM_LAUNCHERS = {
   // For browserstack configs see:
   // https://www.browserstack.com/automate/node
@@ -66,21 +69,28 @@ const CUSTOM_LAUNCHERS = {
     os_version: '10'
   },
   chrome_with_swift_shader: {
-    base: 'Chrome',
+    base: CHROME,
     flags: ['--blacklist-accelerated-compositing', '--blacklist-webgl']
   },
   chrome_autoplay: {
-    base: 'Chrome',
-    flags: ['--autoplay-policy=no-user-gesture-required'],
+    base: CHROME,
+    flags: [
+      '--autoplay-policy=no-user-gesture-required',
+      '--no-sandbox',
+    ],
   },
   chrome_webgpu_linux: {
     base: 'ChromeCanary',
     flags: [
-      '--disable-dawn-features=disallow_unsafe_apis',
-      '--flag-switches-begin',
-      '--enable-unsafe-webgpu',
+      // See https://bugs.chromium.org/p/chromium/issues/detail?id=765284
       '--enable-features=Vulkan,UseSkiaRenderer',
-      '--flag-switches-end',
+      '--use-vulkan=native',
+      '--enable-unsafe-webgpu',
+      '--disable-vulkan-fallback-to-gl-for-testing',
+      '--disable-vulkan-surface',
+      '--disable-features=VaapiVideoDecoder',
+      '--ignore-gpu-blocklist',
+      '--use-angle=vulkan',
     ]
   },
   chrome_webgpu: {
@@ -90,10 +100,17 @@ const CUSTOM_LAUNCHERS = {
       '--flag-switches-begin',
       '--enable-unsafe-webgpu',
       '--flag-switches-end',
+      '--no-sandbox',
     ]
   },
-  chrome_debugging:
-      {base: 'Chrome', flags: ['--remote-debugging-port=9333']}
+  chrome_debugging: {
+    base: 'Chrome',
+    flags: ['--remote-debugging-port=9333'],
+  },
+  chrome_no_sandbox: {
+    base: CHROME,
+    flags: ['--no-sandbox'],
+  }
 };
 
 module.exports = function(config) {
@@ -106,6 +123,9 @@ module.exports = function(config) {
       throw new Error(`Missing launcher for ${browser}`);
     }
     extraConfig.browsers = [browser];
+  } else {
+    // Use no sandbox by default. This has better support on MacOS.
+    extraConfig.browsers = ['chrome_no_sandbox'];
   }
   if (browserLauncher?.base === 'BrowserStack') {
     const username = process.env.BROWSERSTACK_USERNAME;

tools/tfjs_web_test.bzl

@@ -29,6 +29,7 @@ PEER_DEPS = [
     "@npm//karma-jasmine-order-reporter",
 ]
 
+# @//:grep flag greps for a set of tests to run
 GrepProvider = provider(fields = ["grep"])
 
 def _grep_flag_impl(ctx):
@@ -39,8 +40,20 @@ grep_flag = rule(
     build_setting = config.string(flag = True),
 )
 
+# @//:headless flag runs tests headlessly. Defaults to true.
+HeadlessProvider = provider(fields = ["headless"])
+
+def _headless_flag_impl(ctx):
+    return HeadlessProvider(headless = ctx.build_setting_value)
+
+headless_flag = rule(
+    implementation = _headless_flag_impl,
+    build_setting = config.bool(flag = True),
+)
+
 def _make_karma_config_impl(ctx):
     grep = ctx.attr._grep[GrepProvider].grep
+    headless = ctx.attr.headless[HeadlessProvider].headless
     output_file_path = ctx.label.name + ".js"
     output_file = ctx.actions.declare_file(output_file_path)
     args = ctx.attr.args
@@ -54,6 +67,7 @@ def _make_karma_config_impl(ctx):
         substitutions = {
             "TEMPLATE_args": str(args),
             "TEMPLATE_browser": ctx.attr.browser,
+            "TEMPLATE_headless": "true" if headless else "false",
             "TEMPLATE_jasmine_random": "false" if seed else "true",
             "TEMPLATE_jasmine_seed": seed if seed else "undefined",
         },
@@ -74,6 +88,15 @@ _make_karma_config = rule(
             default = "",
             doc = "The browser to run",
         ),
+        "headless": attr.label(
+            default = "@//:headless",
+            doc = """Whether to run chrome tests headlessly.
+
+            Defaults to true on most platforms. Note that not all browsers
+            support headless mode. Check //tools/karma_template.conf.js for
+            more details.
+            """,
+        ),
         "seed": attr.string(
             default = "",
             doc = """Use this seed for test order.