[test] Adds 'hello world' screenshot test

This CL is the initial commit for a screenshot test. A further CL will
provide tooling for updating the goldens.

Bug: 1051458
Change-Id: Id61dd0a804ad33566451e0f56f6981e3e7bd9067
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/2051984
Commit-Queue: Paul Lewis <[email protected]>
Reviewed-by: Tim van der Lippe <[email protected]>

Author: paullewis

.gitignore

@@ -35,6 +35,12 @@ npm-debug.log
 !/third_party/pyjson5
 !/third_party/typescript
 
-test/e2e/**/*[.js, .tsbuildinfo]
+test/e2e/**/*.js
 !test/e2e/resources/**/*.js
-test/shared/**/*[.js, .tsbuildinfo]
+test/e2e/**/*.tsbuildinfo
+test/shared/**/*.js
+test/shared/**/helper.d.ts
+test/shared/**/*.tsbuildinfo
+test/screenshots/**/*.js
+test/screenshots/**/*.tsbuildinfo
+test/screenshots/.generated

scripts/test/run_screenshot_tests.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+#
+# Copyright 2020 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+"""
+Run screenshot tests on a pre-built chrome or one specified via --chrome-binary.
+"""
+
+import argparse
+import os
+import platform
+import re
+import subprocess
+import sys
+import signal
+
+scripts_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+sys.path.append(scripts_path)
+
+import devtools_paths
+
+is_cygwin = sys.platform == 'cygwin'
+
+
+def parse_options(cli_args):
+    parser = argparse.ArgumentParser(description='Run boot perf test')
+    parser.add_argument('--runs', help='Number of runs', type=int)
+    parser.add_argument('--chrome-binary', dest='chrome_binary', help='path to Chromium binary')
+    return parser.parse_args(cli_args)
+
+
+def check_chrome_binary(chrome_binary):
+    return os.path.exists(chrome_binary) and os.path.isfile(chrome_binary) and os.access(chrome_binary, os.X_OK)
+
+
+def popen(arguments, cwd=None, env=None, capture=False):
+    process = subprocess.Popen(arguments, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env)
+    if not capture:
+        return process
+
+    def handle_signal(signum, frame):
+        print 'Sending signal (%i) to process' % signum
+        process.send_signal(signum)
+        process.terminate()
+
+    # Propagate sigterm / int to the child process.
+    original_sigint = signal.getsignal(signal.SIGINT)
+    original_sigterm = signal.getsignal(signal.SIGTERM)
+    signal.signal(signal.SIGINT, handle_signal)
+    signal.signal(signal.SIGTERM, handle_signal)
+
+    for line in iter(process.stdout.readline, ''):
+        sys.stdout.write(line)
+        if process.returncode == 0:
+            sys.stdout.write('done')
+
+    # Restore the original sigterm / int handlers.
+    signal.signal(signal.SIGINT, original_sigint)
+    signal.signal(signal.SIGTERM, original_sigterm)
+
+    return process
+
+
+def to_platform_path_exact(filepath):
+    if not is_cygwin:
+        return filepath
+    output, _ = popen(['cygpath', '-w', filepath]).communicate()
+    # pylint: disable=E1103
+    return output.strip().replace('\\', '\\\\')
+
+
+def compile_typescript_test_files():
+    tsc_compile_errors_found = False
+    cwd = devtools_paths.devtools_root_path()
+    env = os.environ.copy()
+    shared_path = os.path.join(cwd, 'test', 'shared')
+    screenshots_test_path = os.path.join(cwd, 'test', 'screenshots')
+
+    # Compile shared code, e.g. helper and runner.
+    print("Compiling shared TypeScript")
+    exec_command = [devtools_paths.node_path(), devtools_paths.typescript_compiler_path(), '-p', shared_path]
+    tsc_compile_proc = popen(exec_command, cwd=cwd, env=env, capture=True)
+    tsc_compile_proc.communicate()
+    if tsc_compile_proc.returncode != 0:
+        tsc_compile_errors_found = True
+
+    # Compile screenshots tests, e.g. helper and runner.
+    print("Compiling screenshots TypeScript")
+    exec_command = [devtools_paths.node_path(), devtools_paths.typescript_compiler_path(), '-p', screenshots_test_path]
+    tsc_compile_proc = popen(exec_command, cwd=cwd, env=env, capture=True)
+    tsc_compile_proc.communicate()
+    if tsc_compile_proc.returncode != 0:
+        tsc_compile_errors_found = True
+
+    return tsc_compile_errors_found
+
+
+def run_screenshots_test(chrome_binary):
+    screenshots_errors_found = False
+    cwd = devtools_paths.devtools_root_path()
+    screenshots_test_path = os.path.join(cwd, 'test', 'shared', 'runner.js')
+    screenshots_test_list = os.path.join(cwd, 'test', 'screenshots', 'test-list.js')
+    exec_command = [devtools_paths.node_path(), screenshots_test_path]
+
+    env = os.environ.copy()
+    env['CHROME_BIN'] = chrome_binary
+    env['TEST_LIST'] = screenshots_test_list
+
+    screenshots_proc = popen(exec_command, cwd=cwd, env=env, capture=True)
+    screenshots_proc.communicate()
+    if screenshots_proc.returncode != 0:
+        screenshots_errors_found = True
+
+    return screenshots_errors_found
+
+
+def main():
+    OPTIONS = parse_options(sys.argv[1:])
+    is_cygwin = sys.platform == 'cygwin'
+    chrome_binary = None
+
+    # Default to the downloaded / pinned Chromium binary
+    downloaded_chrome_binary = devtools_paths.downloaded_chrome_binary_path()
+    if check_chrome_binary(downloaded_chrome_binary):
+        chrome_binary = downloaded_chrome_binary
+
+    # Override with the arg value if provided.
+    if OPTIONS.chrome_binary:
+        chrome_binary = OPTIONS.chrome_binary
+        if not check_chrome_binary(chrome_binary):
+            print('Unable to find a Chrome binary at \'%s\'' % chrome_binary)
+            sys.exit(1)
+
+    if (chrome_binary is None):
+        print('Unable to run, no Chrome binary provided')
+        sys.exit(1)
+
+    print('Using Chromium binary (%s)\n' % chrome_binary)
+
+    errors_found = False
+    try:
+        errors_found = compile_typescript_test_files()
+        if (errors_found):
+            raise Exception('Typescript failed to compile')
+        errors_found = run_screenshots_test(chrome_binary)
+    except Exception as err:
+        print(err)
+
+    if errors_found:
+        print('ERRORS DETECTED')
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()

test/screenshots/goldens/hello-world.png

@@ -0,0 +1,21 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {describe, it} from 'mocha';
+import {click, getBrowserAndPages, resetPages, resourcesPath, assertScreenshotUnchanged} from '../../shared/helper.js';
+
+describe('hello world', () => {
+  beforeEach(async() => {
+    await resetPages();
+  });
+
+  it('takes a screenshot', async () => {
+    const {target, frontend} = getBrowserAndPages();
+    await target.goto(`${resourcesPath}/console/big-int.html`);
+    await click('#tab-console');
+    await frontend.waitForSelector('.console-group-messages');
+
+    await assertScreenshotUnchanged(frontend, 'hello-world.png');
+  });
+});

test/screenshots/hello-world/hello-world.ts

@@ -0,0 +1,9 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import {join} from 'path';
+
+export const testList = [
+  join(__dirname, '.', 'hello-world', 'hello-world.js'),
+];

test/screenshots/test-list.ts

@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "baseUrl": ".",
+
+    "checkJs": false,
+    "allowJs": false
+  },
+  "references": [
+    {
+      "path": "../shared/tsconfig.json"
+    }
+  ],
+  "extends": "../../tsconfig.json"
+}

test/screenshots/tsconfig.json

@@ -4,6 +4,11 @@
 
 import * as puppeteer from 'puppeteer';
 import {performance} from 'perf_hooks';
+import {join} from 'path';
+import * as fs from 'fs';
+import * as rimraf from 'rimraf';
+import * as resemblejs from 'resemblejs';
+import {assert} from 'chai';
 
 interface BrowserAndPages {
   browser: puppeteer.Browser;
@@ -176,4 +181,44 @@ export const getBrowserAndPages = (): BrowserAndPages => {
   };
 };
 
+const goldensScreenshotFolder = join(__dirname, '..', 'screenshots', 'goldens');
+const generatedScreenshotFolder = join(__dirname, '..', 'screenshots', '.generated');
+
+// Delete and create the generated images.
+if (fs.existsSync(generatedScreenshotFolder)) {
+  rimraf.sync(generatedScreenshotFolder);
+}
+fs.mkdirSync(generatedScreenshotFolder);
+
+const defaultScreenshotOpts: puppeteer.ScreenshotOptions = {
+  type: 'png',
+  fullPage: true,
+  encoding: 'binary'
+};
+export const assertScreenshotUnchanged = async (page: puppeteer.Page, fileName: string, options: Partial<puppeteer.ScreenshotOptions> = {}) => {
+  const goldensScreenshotPath = join(goldensScreenshotFolder, fileName);
+  const generatedScreenshotPath = join(generatedScreenshotFolder, fileName);
+
+  if (fs.existsSync(generatedScreenshotPath)) {
+    throw new Error(`${generatedScreenshotPath} already exists.`);
+  }
+
+  const opts = {...defaultScreenshotOpts, ...options, path: generatedScreenshotPath};
+  await page.screenshot(opts);
+
+  return new Promise((resolve, reject) => {
+    resemblejs.compare(generatedScreenshotPath, goldensScreenshotPath, {}, (err, data) => {
+      if (err) {
+        reject(err);
+        return;
+      }
+
+      const {dimensionDifference, rawMisMatchPercentage} = data;
+      assert.deepEqual(dimensionDifference, { width: 0, height: 0});
+      assert.isBelow(rawMisMatchPercentage, 1);
+      resolve();
+    });
+  })
+};
+
 export const resourcesPath = 'http://localhost:8090/test/e2e/resources';

test/shared/helper.ts

@@ -0,0 +1,13 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+declare module 'resemblejs' {
+  interface ResembleComparisonResult {
+    dimensionDifference: { width: number, height: number };
+    rawMisMatchPercentage: number;
+  }
+
+  export function compare(file1: string, file2: string, options: {},
+      cb:(err: Error, data: ResembleComparisonResult) => void): void;
+}

test/shared/resemblejs.d.ts

@@ -0,0 +1,7 @@
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+declare module 'rimraf' {
+  export function sync(path: string): void;
+}

test/shared/rimraf.d.ts

@@ -0,0 +1 @@
+export {};