graphics e2e: run sequentially and recreate browser on fails

scripts/run-graphics-tests.sh

@@ -30,6 +30,8 @@ fi
 
 npm install
 npm run $BUILD_SCRIPT
+# Remove existing merge-base-dist if it exists
+rm -rf ./merge-base-dist
 mv ./dist ./merge-base-dist
 
 echo "Checkout to HEAD back and build..."

tests/e2e/graphics/graphics-test-cases.ts

@@ -144,7 +144,6 @@ function registerTestCases(testCases: TestCase[], screenshoter: Screenshoter, ou
 	for (const testCase of testCases) {
 		void it(testCase.name, { timeout: TEST_CASE_TIMEOUT * NUMBER_RETRIES + 1000 }, async () => {
 			await retryTest(NUMBER_RETRIES, async () => {
-				const previousAttempts = attempts[testCase.name];
 				attempts[testCase.name] += 1;
 
 				const testCaseOutDir = path.join(outDir, testCase.name);
@@ -164,36 +163,24 @@ function registerTestCases(testCases: TestCase[], screenshoter: Screenshoter, ou
 				const errors: string[] = [];
 				const failedPages: string[] = [];
 
-				// run in parallel to increase speed
-				const goldenScreenshotPromise = withTimeout(screenshoter.generateScreenshot(goldenPageContent), TEST_CASE_TIMEOUT);
-
-				if (previousAttempts) {
-					try {
-						// If a test has previously failed then attempt to run the tests in series (one at a time).
-						await goldenScreenshotPromise;
-					} catch {
-						// error will be caught again below and handled correctly there.
-					}
-				}
-
-				const testScreenshotPromise = withTimeout(screenshoter.generateScreenshot(testPageContent), TEST_CASE_TIMEOUT);
-
 				let goldenScreenshot: PNG | null = null;
 				try {
-					goldenScreenshot = await goldenScreenshotPromise;
+					goldenScreenshot = await withTimeout(screenshoter.generateScreenshot(goldenPageContent), TEST_CASE_TIMEOUT);
 					writeTestDataItem('1.golden.png', PNG.sync.write(goldenScreenshot));
 				} catch (e: unknown) {
 					errors.push(`=== Golden page ===\n${(e as Error).message}`);
 					failedPages.push('golden');
+					await screenshoter.close();
 				}
 
 				let testScreenshot: PNG | null = null;
 				try {
-					testScreenshot = await testScreenshotPromise;
+					testScreenshot = await withTimeout(screenshoter.generateScreenshot(testPageContent), TEST_CASE_TIMEOUT);
 					writeTestDataItem('2.test.png', PNG.sync.write(testScreenshot));
 				} catch (e: unknown) {
 					errors.push(`=== Test page ===\n${(e as Error).message}`);
 					failedPages.push('test');
+					await screenshoter.close();
 				}
 
 				if (goldenScreenshot !== null && testScreenshot !== null) {

tests/e2e/graphics/helpers/screenshoter.ts

@@ -1,11 +1,10 @@
 /// <reference types="node" />
 
 import { PNG } from 'pngjs';
-import {
+import puppeteer, {
 	Browser,
 	ConsoleMessage,
 	HTTPResponse,
-	launch as launchPuppeteer,
 	Page,
 	PuppeteerLaunchOptions,
 } from 'puppeteer';
@@ -18,33 +17,29 @@ const viewportHeight = 600;
 
 export class Screenshoter {
 	private _browserPromise: Promise<Browser>;
+	private _noSandbox: boolean;
+	private _devicePixelRatio: number;
+	private _created: boolean = false;
 
 	public constructor(noSandbox: boolean, devicePixelRatio: number = 1) {
-		const puppeteerOptions: PuppeteerLaunchOptions = {
-			defaultViewport: {
-				deviceScaleFactor: devicePixelRatio,
-				width: viewportWidth,
-				height: viewportHeight,
-			},
-			headless: true,
-		};
-
-		if (noSandbox) {
-			puppeteerOptions.args = ['--no-sandbox', '--disable-setuid-sandbox'];
-		}
-
-		this._browserPromise = launchPuppeteer(puppeteerOptions);
+		this._noSandbox = noSandbox;
+		this._devicePixelRatio = devicePixelRatio;
+		this._browserPromise = this._createBrowser();
 	}
 
 	public async close(): Promise<void> {
 		const browser = await this._browserPromise;
 		await browser.close();
+		this._created = false;
 	}
 
 	public async generateScreenshot(pageContent: string): Promise<PNG> {
 		let page: Page | undefined;
 
 		try {
+			if (!this._created) {
+				this._browserPromise = this._createBrowser();
+			}
 			const browser = await this._browserPromise;
 			page = await browser.newPage();
 
@@ -77,29 +72,37 @@ export class Screenshoter {
 				return Boolean((window as unknown as TestCaseWindow).ignoreMouseMove);
 			});
 
+			const hasChartGlobal = await page.evaluate(() => {
+				return Boolean((window as unknown as TestCaseWindow).chart);
+			});
+
+			if (!shouldIgnoreMouseMove && !hasChartGlobal) {
+				throw new Error('Either an error occurred during the chart creation, or the window variable `chart` was required because `ignoreMouseMove` was not set to true');
+			}
+
 			if (!shouldIgnoreMouseMove) {
 				// move mouse to top-left corner
 				await page.mouse.move(0, 0);
 			}
 
-			const waitForMouseMove = page.evaluate(() => {
-				if ((window as unknown as TestCaseWindow).ignoreMouseMove) { return Promise.resolve(); }
-				return new Promise<number[]>((resolve: (value: number[]) => void) => {
-					const chart = (window as unknown as TestCaseWindow).chart;
-					if (!chart) {
-						throw new Error('window variable `chart` is required unless `ignoreMouseMove` is set to true');
-					}
-					chart.subscribeCrosshairMove((param: MouseEventParams) => {
-						const point = param.point;
-						if (!point) { return; }
-						if (point.x > 0 && point.y > 0) {
-							requestAnimationFrame(() => resolve([point.x, point.y] as number[]));
+			if (!shouldIgnoreMouseMove) {
+				const waitForMouseMove = page.evaluate(() => {
+					if ((window as unknown as TestCaseWindow).ignoreMouseMove) { return Promise.resolve(); }
+					return new Promise<number[]>((resolve: (value: number[]) => void) => {
+						const chart = (window as unknown as TestCaseWindow).chart;
+						if (!chart) {
+							// An error occurred during the chart creation
+							return;
 						}
+						chart.subscribeCrosshairMove((param: MouseEventParams) => {
+							const point = param.point;
+							if (!point) { return; }
+							if (point.x > 0 && point.y > 0) {
+								requestAnimationFrame(() => resolve([point.x, point.y] as number[]));
+							}
+						});
 					});
 				});
-			});
-
-			if (!shouldIgnoreMouseMove) {
 				// to avoid random cursor position
 				await page.mouse.move(viewportWidth / 2, viewportHeight / 2);
 				await waitForMouseMove;
@@ -169,4 +172,21 @@ export class Screenshoter {
 			}
 		}
 	}
+
+	private _createBrowser(): Promise<Browser> {
+		const puppeteerOptions: PuppeteerLaunchOptions = {
+			defaultViewport: {
+				deviceScaleFactor: this._devicePixelRatio,
+				width: viewportWidth,
+				height: viewportHeight,
+			},
+			headless: true,
+		};
+
+		if (this._noSandbox) {
+			puppeteerOptions.args = ['--no-sandbox', '--disable-setuid-sandbox'];
+		}
+		this._created = true;
+		return puppeteer.launch(puppeteerOptions);
+	}
 }

tests/e2e/runner.ts

@@ -41,6 +41,8 @@ export async function runTests(
 		}
 	}
 
+	await new Promise<void>((resolver: () => void) => setTimeout(resolver, 1000));
+	console.log('filesToServeLocally', Object.fromEntries(filesToServeLocally));
 	const closeServer = await serveLocalFiles(filesToServeLocally, port, hostname);
 
 	const spec = new Spec();