feat: Browser SDK Integration Tests (#3989)

Initial structure of new integration tests for Sentry's Browser SDK.

New integration tests internally use Playwright and run on recent
versions of Chromium, Webkit and Firefox linked with the Playwright
release.

This new test structure aims to create a modern and intuitive
environment to test @sentry/browser and potentially other browser-side
SDKs like @sentry/react and @sentry/vue.

Co-authored-by: Rodolfo Carvalho <[email protected]>
Co-authored-by: Abhijeet Prasad <[email protected]>

Author: 3 people

.github/workflows/build.yml

@@ -271,6 +271,33 @@ jobs:
             ${{ github.workspace }}/packages/**/*.tgz
             ${{ github.workspace }}/packages/serverless/dist-awslambda-layer/*.zip
 
+  job_browser_playwright_tests:
+    name: Browser Playwright Tests
+    needs: job_build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out current commit (${{ github.sha }})
+        uses: actions/checkout@v2
+      - name: Set up Node
+        uses: actions/setup-node@v1
+        with:
+          node-version: '16'
+      - name: Check dependency cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CACHED_DEPENDENCY_PATHS }}
+          key: ${{ needs.job_build.outputs.dependency_cache_key }}
+      - name: Check build cache
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.CACHED_BUILD_PATHS }}
+          key: ${{ env.BUILD_CACHE_KEY }}
+      - name: Run Playwright tests
+        run: |
+          cd packages/integration-tests
+          yarn run playwright install-deps webkit
+          yarn test:ci
+
   job_browser_integration_tests:
     name: Browser Integration Tests (${{ matrix.browser }})
     needs: job_build

package.json

@@ -19,7 +19,7 @@
     "pack:changed": "lerna run pack --since",
     "prepublishOnly": "lerna run --stream --concurrency 1 prepublishOnly",
     "postpublish": "make publish-docs && lerna run --stream --concurrency 1 postpublish",
-    "test": "lerna run --stream --concurrency 1 --sort test"
+    "test": "lerna run --ignore @sentry-internal/browser-integration-tests --stream --concurrency 1 --sort test"
   },
   "volta": {
     "node": "14.17.0",
@@ -34,6 +34,7 @@
     "packages/eslint-plugin-sdk",
     "packages/gatsby",
     "packages/hub",
+    "packages/integration-tests",
     "packages/integrations",
     "packages/minimal",
     "packages/nextjs",

packages/eslint-config-sdk/src/index.js

@@ -164,7 +164,7 @@ module.exports = {
       env: {
         jest: true,
       },
-      files: ['*.test.ts', '*.test.tsx', '*.test.js', '*.test.jsx', 'test/**/*.ts', 'test/**/*.js'],
+      files: ['test.ts', '*.test.ts', '*.test.tsx', '*.test.js', '*.test.jsx', 'test/**/*.ts', 'test/**/*.js'],
       rules: {
         'max-lines': 'off',
         '@typescript-eslint/explicit-function-return-type': 'off',

packages/integration-tests/.eslintrc.js

@@ -0,0 +1,11 @@
+module.exports = {
+  env: {
+    browser: true,
+    node: true,
+  },
+  extends: ['../../.eslintrc.js'],
+  ignorePatterns: ['suites/**/subject.js', 'suites/**/dist/*'],
+  parserOptions: {
+    sourceType: 'module',
+  },
+};

packages/integration-tests/.gitignore

@@ -0,0 +1 @@
+dist

packages/integration-tests/README.md

@@ -0,0 +1,72 @@
+# Integration Tests for Sentry Browser SDK
+
+Integration tests for Sentry's Browser SDK use [Playwright](https://playwright.dev/) internally. These tests are run on latest stable versions of Chromium, Firefox and Webkit.
+
+## Structure
+
+The tests are grouped by their scope such as `breadcrumbs` or `onunhandledrejection`. In every group of tests, there are multiple folders containing test cases with their optional supporting assets.
+
+Each case group has a default HTML skeleton named `template.hbs`, and also a default initialization script named `init.js `, which contains the `Sentry.init()` call. These defaults are used as fallbacks when a specific `template.hbs` or `init.js` is not defined in a case folder.
+
+`subject.js` contains the logic that sets up the environment to be tested. It also can be defined locally and as a group fallback. Unlike `template.hbs` and `init.js`, it's not required to be defined for a group, as there may be cases that does not require a subject, instead the logic is injected using `injectScriptAndGetEvents` from `utils/helpers.ts`.
+
+`test.ts` is required for each test case, which contains the assertions (and if required the script injection logic). For every case, any set of `init.js`, `template.hbs` and `subject.js` can be defined locally, and each one of them  will have precedence over the default definitions of the test group.
+
+```
+suites/
+|---- breadcrumbs/
+      |---- template.hbs [fallback template for breadcrumb tests]
+      |---- init.js [fallback init for breadcrumb tests]
+      |---- subject.js [optional fallback subject for breadcrumb tests]
+      |---- click_event_tree/
+            |---- template.hbs [optional case specific template]
+            |---- init.js [optional case specific init]
+            |---- subject.js [optional case specific subject]
+            |---- test.ts [assertions]
+```
+
+## Writing Tests
+
+### Helpers
+
+`utils/helpers.ts` contains helpers that could be used in assertions (`test.ts`). These helpers define a convenient and reliable API to interact with Playwright's native API. It's highly recommended to define all common patterns of Playwright usage in helpers.
+
+### Fixtures
+
+[Fixtures](https://playwright.dev/docs/api/class-fixtures) allows us to define the globals and test-specific information in assertion groups (`test.ts` files). In it's current state, `fixtures.ts` contains an extension over the pure version of `test()` function of Playwright. All the tests should import `sentryTest` function from `utils/fixtures.ts` instead of `@playwright/test` to be able to access the extra fixtures.
+
+## Running Tests Locally
+
+Tests can be run locally using the latest version of Chromium with:
+
+`yarn test`
+
+To run tests with a different browser such as `firefox` or `webkit`:
+
+`yarn test --browser='firefox'`
+`yarn test --browser='webkit'`
+
+Or to run on all three browsers:
+
+`yarn test --browser='all'`
+
+To filter tests by their title:
+
+`yarn test -g "XMLHttpRequest without any handlers set"`
+
+You can refer to [Playwright documentation](https://playwright.dev/docs/test-cli) for other CLI options.
+
+### Troubleshooting
+
+Apart from [Playwright-specific issues](https://playwright.dev/docs/troubleshooting), below are common issues that might occur while writing tests for Sentry Browser SDK.
+
+- #### Flaky Tests
+  If a test fails randomly, giving a `Page Closed`, `Target Closed` or a similar error, most of the times, the reason is a race condition between the page action defined in the `subject` and the listeners of the Sentry event / request. It's recommended to firstly check `utils/helpers.ts` whether if that async logic can be replaced by one of the helpers. If not, whether the awaited (or non-awaited on purpose in some cases) Playwright methods can be orchestrated by [`Promise.all`](http://mdn.io/promise.all). Manually-defined waiting logic such as timeouts are not recommended, and should not be required in most of the cases.
+
+- #### Build Errors
+  Before running, a page for each test case is built under the case folder inside `dist`. If a page build is failed, it's recommended to check:
+
+  - If both default `template.hbs` and `init.js` are defined for the test group.
+  - If a `subject.js` is defined for the test case.
+  - If either of `init.js` or `subject.js` contain non-browser code.
+  - If the webpack configuration is valid.

packages/integration-tests/package.json

@@ -0,0 +1,30 @@
+{
+  "name": "@sentry-internal/browser-integration-tests",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "engines": {
+    "node": ">=10"
+  },
+  "private": true,
+  "scripts": {
+    "clean": "rimraf -g suites/**/dist",
+    "install-browsers": "playwright install --with-deps",
+    "lint": "run-s lint:prettier lint:eslint",
+    "lint:eslint": "eslint . --cache --cache-location '../../eslintcache/' --format stylish",
+    "lint:prettier": "prettier --check \"{suites,utils}/**/*.ts\"",
+    "test:ci": "playwright test ./suites --browser='all' --reporter='line'",
+    "type-check": "tsc",
+    "pretest": "yarn clean && yarn type-check",
+    "test": "playwright test ./suites"
+  },
+  "dependencies": {
+    "@playwright/test": "^1.17.0",
+    "babel-loader": "^8.2.2",
+    "handlebars-loader": "^1.7.1",
+    "html-webpack-plugin": "^5.5.0",
+    "playwright": "^1.17.1",
+    "typescript": "^4.5.2",
+    "webpack": "^5.52.0"
+  }
+}

packages/integration-tests/playwright.config.ts

@@ -0,0 +1,8 @@
+import { PlaywrightTestConfig } from '@playwright/test';
+
+const config: PlaywrightTestConfig = {
+  retries: 2,
+  timeout: 12000,
+  workers: 3,
+};
+export default config;

packages/integration-tests/suites/demo/init.js

@@ -0,0 +1,7 @@
+import * as Sentry from '@sentry/browser';
+
+window.Sentry = Sentry;
+
+Sentry.init({
+  dsn: 'https://[email protected]/1337',
+});

packages/integration-tests/suites/demo/template.hbs

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title></title>
+  <script src="{{htmlWebpackPlugin.options.initialization}}"></script>
+  </head>
+  <body>
+    <script src="{{htmlWebpackPlugin.options.subject}}"></script>
+  </body>
+</html>

packages/integration-tests/suites/demo/tmp/subject.js

@@ -0,0 +1 @@
+Sentry.captureMessage(1);

packages/integration-tests/suites/demo/tmp/test.ts

@@ -0,0 +1,12 @@
+import { expect } from '@playwright/test';
+
+import { sentryTest } from '../../../utils/fixtures';
+import { getSentryRequest } from '../../../utils/helpers';
+
+sentryTest('should fail', async ({ getLocalTestPath, page }) => {
+  const url = await getLocalTestPath({ testDir: __dirname });
+
+  const eventData = await getSentryRequest(page, url);
+
+  expect(eventData.message).toBe('1');
+});

packages/integration-tests/tsconfig.json

@@ -0,0 +1,12 @@
+{
+  "extends": "../../tsconfig.json",
+
+  "compilerOptions": {
+    "lib": ["dom", "es2019"],
+    "moduleResolution": "node",
+    "noEmit": true,
+    "strict": true
+  },
+  "include": ["**/*.ts"],
+  "exclude": ["node_modules"]
+}

packages/integration-tests/utils/fixtures.ts

@@ -0,0 +1,46 @@
+import { test as base } from '@playwright/test';
+import fs from 'fs';
+import path from 'path';
+
+import { generatePage } from './generatePage';
+
+const getAsset = (assetDir: string, asset: string): string => {
+  const assetPath = `${assetDir}/${asset}`;
+
+  if (fs.existsSync(assetPath)) {
+    return assetPath;
+  }
+
+  return `${path.dirname(assetDir)}/${asset}`;
+};
+
+export type TestOptions = {
+  testDir: string;
+};
+
+export type TestFixtures = {
+  testDir: string;
+  getLocalTestPath: (options: TestOptions) => Promise<string>;
+};
+
+const sentryTest = base.extend<TestFixtures>({
+  // eslint-disable-next-line no-empty-pattern
+  getLocalTestPath: ({}, use, testInfo) => {
+    return use(async ({ testDir }) => {
+      const pagePath = `file:///${path.resolve(testDir, './dist/index.html')}`;
+
+      // Build test page if it doesn't exist
+      if (!fs.existsSync(pagePath)) {
+        const testDir = path.dirname(testInfo.file);
+        const subject = getAsset(testDir, 'subject.js');
+        const template = getAsset(testDir, 'template.hbs');
+        const init = getAsset(testDir, 'init.js');
+
+        await generatePage(init, subject, template, testDir);
+      }
+      return pagePath;
+    });
+  },
+});
+
+export { sentryTest };

packages/integration-tests/utils/generatePage.ts

@@ -0,0 +1,89 @@
+import { Package } from '@sentry/types';
+import { existsSync, mkdirSync, promises } from 'fs';
+import HtmlWebpackPlugin from 'html-webpack-plugin';
+import path from 'path';
+import webpack from 'webpack';
+
+import webpackConfig from '../webpack.config';
+
+const PACKAGE_PATH = '../../packages';
+
+/**
+ * Generate webpack aliases based on packages in monorepo
+ * Example of an alias: '@sentry/serverless': 'path/to/sentry-javascript/packages/serverless',
+ */
+async function generateSentryAlias(): Promise<Record<string, string>> {
+  const dirents = (await promises.readdir(PACKAGE_PATH, { withFileTypes: true }))
+    .filter(dirent => dirent.isDirectory())
+    .map(dir => dir.name);
+
+  return Object.fromEntries(
+    await Promise.all(
+      dirents.map(async d => {
+        const packageJSON: Package = JSON.parse(
+          (await promises.readFile(path.resolve(PACKAGE_PATH, d, 'package.json'), { encoding: 'utf-8' })).toString(),
+        );
+        return [packageJSON['name'], path.resolve(PACKAGE_PATH, d)];
+      }),
+    ),
+  );
+}
+
+export async function generatePage(
+  initializationPath: string,
+  subjectPath: string,
+  templatePath: string,
+  outPath: string,
+): Promise<void> {
+  const localPath = `${outPath}/dist`;
+  const bundlePath = `${localPath}/index.html`;
+
+  const alias = await generateSentryAlias();
+
+  if (!existsSync(localPath)) {
+    mkdirSync(localPath, { recursive: true });
+  }
+
+  if (!existsSync(bundlePath)) {
+    await new Promise<void>((resolve, reject) => {
+      const compiler = webpack(
+        webpackConfig({
+          resolve: {
+            alias,
+          },
+          entry: {
+            initialization: initializationPath,
+            subject: subjectPath,
+          },
+          output: {
+            path: localPath,
+            filename: '[name].bundle.js',
+          },
+          plugins: [
+            new HtmlWebpackPlugin({
+              filename: 'index.html',
+              template: templatePath,
+              initialization: 'initialization.bundle.js',
+              subject: `subject.bundle.js`,
+              inject: false,
+            }),
+          ],
+        }),
+      );
+
+      compiler.run(err => {
+        if (err) {
+          reject(err);
+        }
+
+        compiler.close(err => {
+          if (err) {
+            reject(err);
+          }
+
+          resolve();
+        });
+      });
+    });
+  }
+}