DevTools: skip specific tests for backwards compatibility testing

Created a new TestExpectations file for the backwards compatibility tests
with specific reasons why certain types of tests are skipped.

BUG=682078

Review-Url: https://codereview.chromium.org/2661573002
Cr-Commit-Position: refs/heads/master@{#446826}

Author: wwwillchen

package.json

@@ -7,6 +7,7 @@
     "server": "node scripts/hosted_mode/server.js",
     "test": "node scripts/npm_test.js",
     "debug-test": "node scripts/npm_test.js --debug-devtools",
+    "compat-test": "node scripts/npm_test.js --compat-protocol=1.2",
     "lint": "eslint front_end",
     "format": "node scripts/format.js",
     "closure": "python scripts/compile_frontend.py",

scripts/npm_test.js

@@ -13,7 +13,8 @@ var Flags = {
   DEBUG_DEVTOOLS: '--debug-devtools',
   DEBUG_DEVTOOLS_SHORTHAND: '-d',
   FETCH_CONTENT_SHELL: '--fetch-content-shell',
-  COMPAT_PROTOCOL: '--compat-protocol',
+  COMPAT_PROTOCOL: '--compat-protocol',  // backwards compatibility testing
+  CHROMIUM_PATH: '--chromium-path'       // useful for bisecting
 };
 
 var COMPAT_URL_MAPPING = {
@@ -23,17 +24,18 @@ var COMPAT_URL_MAPPING = {
 var IS_DEBUG_ENABLED =
     utils.includes(process.argv, Flags.DEBUG_DEVTOOLS) || utils.includes(process.argv, Flags.DEBUG_DEVTOOLS_SHORTHAND);
 var COMPAT_PROTOCOL = utils.parseArgs(process.argv)[Flags.COMPAT_PROTOCOL];
+var CUSTOM_CHROMIUM_PATH = utils.parseArgs(process.argv)[Flags.CHROMIUM_PATH];
 var IS_FETCH_CONTENT_SHELL = utils.includes(process.argv, Flags.FETCH_CONTENT_SHELL);
 
 var CONTENT_SHELL_ZIP = 'content-shell.zip';
 var MAX_CONTENT_SHELLS = 10;
 var PLATFORM = getPlatform();
 var PYTHON = process.platform === 'win32' ? 'python.bat' : 'python';
 
-var CHROMIUM_SRC_PATH = path.resolve(__dirname, '..', '..', '..', '..', '..');
+var CHROMIUM_SRC_PATH = CUSTOM_CHROMIUM_PATH || path.resolve(__dirname, '..', '..', '..', '..', '..');
 var RELEASE_PATH = path.resolve(CHROMIUM_SRC_PATH, 'out', 'Release');
 var BLINK_TEST_PATH = path.resolve(CHROMIUM_SRC_PATH, 'blink', 'tools', 'run_layout_tests.py');
-var DEVTOOLS_PATH = path.resolve(__dirname, '..');
+var DEVTOOLS_PATH = path.resolve(CHROMIUM_SRC_PATH, 'third_party', 'WebKit', 'Source', 'devtools');
 var CACHE_PATH = path.resolve(DEVTOOLS_PATH, '.test_cache');
 var SOURCE_PATH = path.resolve(DEVTOOLS_PATH, 'front_end');
 
@@ -67,6 +69,7 @@ main();
 
 function runCompatibilityTests() {
   const folder = `compat-protocol-${COMPAT_PROTOCOL}`;
+  utils.removeRecursive(path.resolve(RELEASE_PATH, 'resources', 'inspector'));
   compileFrontend();
   var outPath = path.resolve(CACHE_PATH, folder, 'out');
   var contentShellDirPath = path.resolve(outPath, 'Release');
@@ -87,7 +90,7 @@ function runCompatibilityTests() {
 function compileFrontend() {
   console.log('Compiling devtools frontend');
   try {
-    shell(`ninja -C ${RELEASE_PATH} devtools_frontend_resources`);
+    shell(`ninja -C ${RELEASE_PATH} devtools_frontend_resources`, {cwd: CHROMIUM_SRC_PATH});
   } catch (err) {
     console.log(err.stdout.toString());
     console.log('ERROR: Cannot compile frontend\n' + err);
@@ -271,9 +274,12 @@ function runTests(buildDirectoryPath, useDebugDevtools) {
     console.log('TIP: You can debug a test using: npm run debug-test inspector/test-name.html');
 
   if (COMPAT_PROTOCOL) {
-    let platform = `protocol-${COMPAT_PROTOCOL}`;
-    let compatBaselinePath = path.resolve(DEVTOOLS_PATH, 'tests', 'baseline', platform);
+    const platform = `protocol-${COMPAT_PROTOCOL}`;
+    const testsPath = path.resolve(DEVTOOLS_PATH, 'tests');
+    const compatBaselinePath = path.resolve(testsPath, 'baseline', platform);
     testArgs.push(`--additional-platform-directory=${compatBaselinePath}`);
+    const expectationsPath = path.resolve(testsPath, 'TestExpectations');
+    testArgs.push(`--additional-expectations=${expectationsPath}`);
   }
   if (IS_DEBUG_ENABLED) {
     testArgs.push('--additional-driver-flag=--remote-debugging-port=9222');

tests/TestExpectations

@@ -0,0 +1,55 @@
+# Skipping service worker tests b/c migration from ServiceWorker domain to Target domain
+# https://crrev.com/c2cea7d490e0b9ec93b72b0e00502bb9fa79f10a
+http/tests/inspector/elements/event-listeners-framework-with-service-worker.html [ Skip ]
+http/tests/inspector/service-workers/lazy-addeventlisteners.html [ Skip ]
+http/tests/inspector/service-workers/service-worker-agents.html [ Skip ]
+http/tests/inspector/service-workers/service-worker-manager.html [ Skip ]
+http/tests/inspector/service-workers/service-worker-pause.html [ Skip ]
+http/tests/inspector/service-workers/service-workers-force-update-on-page-load.html [ Skip ]
+http/tests/inspector/service-workers/service-workers-navigation-preload.html [ Skip ]
+http/tests/inspector/service-workers/service-workers-redundant.html [ Skip ]
+http/tests/inspector/service-workers/service-workers-view.html [ Skip ]
+http/tests/inspector/service-workers/service-workers-bypass-for-network-redirect.html [ Skip ]
+
+# Skipping worker tests b/c migration from Worker domain to Target domain
+# https://crrev.com/14d5ae1493499885e5f9820490e77aef43d0d083
+http/tests/inspector-enabled/dedicated-workers-list.html [ Skip ]
+http/tests/inspector/sources/debugger/worker-debugging-script-mapping.html [ Skip ]
+http/tests/inspector/sources/debugger/worker-debugging.html [ Skip ]
+inspector/console/console-revoke-error-in-worker.html [ Skip ]
+inspector/console/worker-eval-contains-stack.html [ Skip ]
+inspector/sources/debugger-frameworks/frameworks-with-worker.html [ Skip ]
+inspector/sources/debugger-ui/script-snippet-model.html [ Skip ] # TODO(chenwilliam): extract the worker test
+
+# Skipping accessibility tests b/c of protocol change
+# https://crrev.com/3a46cc72d58945804dd4919a679a8ce26690bfc0
+inspector-protocol/accessibility/accessibility-getNodeWithNoAXNode.html [ Skip ]
+inspector-protocol/accessibility/accessibility-getRelationships.html [ Skip ]
+inspector-protocol/accessibility/accessibility-ignoredNodes.html [ Skip ]
+inspector-protocol/accessibility/accessibility-ignoredNodesModal.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-buttons.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-img-figure.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-input-buttons.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-input.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-labelledby.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-summary.html [ Skip ]
+inspector-protocol/accessibility/accessibility-nameSources-visiblity.html [ Skip ]
+
+# Skipping async await tests b/c 1.2 content shell (DevTools runtime) does not
+# support async await
+inspector/sources/debugger-async/async-await/async-callstack-async-await1.html [ Skip ]
+inspector/sources/debugger-async/async-await/async-callstack-async-await2.html [ Skip ]
+inspector/sources/debugger-async/async-await/async-callstack-async-await3.html [ Skip ]
+inspector/sources/debugger-async/async-await/async-pause-on-exception.html [ Skip ]
+inspector/sources/debugger-ui/async-call-stack-async-function.html [ Skip ]
+
+# 1.2 content shell does not support inspector-unit
+http/tests/inspector-unit/filtered-item-selection-dialog-filtering.js [ Skip ]
+http/tests/inspector-unit/list-control-equal-height.js [ Skip ]
+http/tests/inspector-unit/list-control-non-viewport.js [ Skip ]
+http/tests/inspector-unit/list-control-various-height.js [ Skip ]
+http/tests/inspector-unit/suggest-box.js [ Skip ]
+http/tests/inspector-unit/test-failure.js [ Skip ]
+http/tests/inspector-unit/text-prompt-hint.js [ Skip ]
+http/tests/inspector-unit/text-prompt.js [ Skip ]
+http/tests/inspector-unit/trie.js [ Skip ]