refactor and add cmdline args for benchmark driver

Author: krausest

webdriver-ts/src/benchmarkConfiguration.ts

@@ -1,41 +1,36 @@
 import { fork } from "child_process";
 import * as fs from "fs";
 import * as yargs from "yargs";
-import { BenchmarkInfo, BenchmarkType, TBenchmark } from "./benchmarksCommon";
-import { CPUBenchmarkPuppeteer, MemBenchmarkPuppeteer, TBenchmarkPuppeteer } from "./benchmarksPuppeteer";
-import { CPUBenchmarkPlaywright, MemBenchmarkPlaywright, TBenchmarkPlaywright } from "./benchmarksPlaywright";
-import { CPUBenchmarkWebdriver } from "./benchmarksWebdriver";
-import { BenchmarkDriverOptions, BenchmarkOptions, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common";
+import { BenchmarkInfo, benchmarkInfos, BenchmarkType, CPUBenchmarkInfo, MemBenchmarkInfo, StartupBenchmarkInfo } from "./benchmarksCommon";
+import { StartupBenchmarkResult } from "./benchmarksLighthouse";
+import { BenchmarkOptions, BENCHMARK_RUNNER, config, ErrorAndWarning, FrameworkData, initializeFrameworks } from "./common";
 import { writeResults } from "./writeResults";
-import {benchmarks, TBenchmarkImplementation} from "./benchmarkConfiguration";
-import { BenchmarkLighthouse, StartupBenchmarkResult } from "./benchmarksLighthouse";
-import { CPUBenchmarkWebdriverCDP } from "./benchmarksWebdriverCDP";
 
 function forkAndCallBenchmark(
   framework: FrameworkData,
-  benchmark: CPUBenchmarkWebdriver|TBenchmarkPuppeteer|BenchmarkLighthouse,
+  benchmarkInfo: BenchmarkInfo,
   benchmarkOptions: BenchmarkOptions
 ): Promise<ErrorAndWarning> {
   return new Promise((resolve, reject) => {
     let forkedRunner = null;
-    if (benchmark instanceof BenchmarkLighthouse) {
+    if (benchmarkInfo.type === BenchmarkType.STARTUP_MAIN) {
       forkedRunner = "dist/forkedBenchmarkRunnerLighthouse.js";
-    } else if (benchmark instanceof CPUBenchmarkWebdriverCDP) {
+    } else if (config.BENCHMARK_RUNNER == BENCHMARK_RUNNER.WEBDRIVER_CDP) {
       forkedRunner = "dist/forkedBenchmarkRunnerWebdriverCDP.js";
-    } else if (benchmark instanceof CPUBenchmarkPlaywright || benchmark instanceof MemBenchmarkPlaywright) {
+    } else if (config.BENCHMARK_RUNNER == BENCHMARK_RUNNER.PLAYWRIGHT) {
       forkedRunner = "dist/forkedBenchmarkRunnerPlaywright.js";
-    } else if (benchmark instanceof CPUBenchmarkPuppeteer || benchmark instanceof MemBenchmarkPuppeteer) {
-      forkedRunner = "dist/forkedBenchmarkRunnerPuppeteer.js";
-    } else {
+    } else if (config.BENCHMARK_RUNNER == BENCHMARK_RUNNER.WEBDRIVER) {
       forkedRunner = "dist/forkedBenchmarkRunnerWebdriver.js";
+    } else {
+      forkedRunner = "dist/forkedBenchmarkRunnerPuppeteer.js";
     }
     console.log("forking ",forkedRunner);
     const forked = fork(forkedRunner);
     if (config.LOG_DETAILS) console.log("FORKING:  forked child process");
     forked.send({
       config,
       framework,
-      benchmarkId: benchmark.benchmarkInfo.id,
+      benchmarkId: benchmarkInfo.id,
       benchmarkOptions,
     });
     forked.on("message", async (msg: ErrorAndWarning) => {
@@ -57,7 +52,7 @@ function forkAndCallBenchmark(
 
 async function runBenchmakLoopStartup(
   framework: FrameworkData,
-  benchmark: BenchmarkLighthouse,
+  benchmarkInfo: StartupBenchmarkInfo,
   benchmarkOptions: BenchmarkOptions
 ): Promise<{ errors: String[]; warnings: String[] }> {
   let warnings: String[] = [];
@@ -70,12 +65,12 @@ async function runBenchmakLoopStartup(
   let retries = 0;
   let done = 0;
 
-  console.log("runBenchmakLoopStartup", framework, benchmark);
+  console.log("runBenchmakLoopStartup", framework, benchmarkInfo);
 
 
   while (done < count) {
-    console.log("FORKING: ", benchmark.benchmarkInfo.id, " BatchSize ", benchmarkOptions.batchSize);
-    let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
+    console.log("FORKING: ", benchmarkInfo.id, " BatchSize ", benchmarkOptions.batchSize);
+    let res = await forkAndCallBenchmark(framework, benchmarkInfo, benchmarkOptions);
     if (Array.isArray(res.result)) {
       results = results.concat(res.result as StartupBenchmarkResult[]);
     } else results.push(res.result);
@@ -86,7 +81,7 @@ async function runBenchmakLoopStartup(
         retries++;
         if (retries == 3) break;
       } else {
-        errors.push(`Executing ${framework.uri} and benchmark ${benchmark.benchmarkInfo.id} failed: ` + res.error);
+        errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
         break;
       }
     }
@@ -95,7 +90,7 @@ async function runBenchmakLoopStartup(
   console.log("******* result ", results);
   await writeResults(config, {
     framework: framework,
-    benchmark: benchmark,
+    benchmark: benchmarkInfo,
     results: results,
     type: BenchmarkType.STARTUP
   });
@@ -106,7 +101,7 @@ async function runBenchmakLoopStartup(
 
 async function runBenchmakLoop(
   framework: FrameworkData,
-  benchmark: CPUBenchmarkWebdriver | TBenchmarkPuppeteer,
+  benchmarkInfo: CPUBenchmarkInfo|MemBenchmarkInfo,
   benchmarkOptions: BenchmarkOptions
 ): Promise<{ errors: String[]; warnings: String[] }> {
   let warnings: String[] = [];
@@ -115,23 +110,23 @@ async function runBenchmakLoop(
   let results: Array<number> = [];
   let count = 0;
 
-  if (benchmark.type == BenchmarkType.CPU) {
+  if (benchmarkInfo.type == BenchmarkType.CPU) {
     count = benchmarkOptions.numIterationsForCPUBenchmarks;
     // FIXME
-    benchmarkOptions.batchSize = config.ALLOW_BATCHING && (benchmark.benchmarkInfo as any).allowBatching ? count : 1;
-  } else if (benchmark.type == BenchmarkType.MEM) {
+    benchmarkOptions.batchSize = config.ALLOW_BATCHING && benchmarkInfo.allowBatching ? count : 1;
+  } else if (benchmarkInfo.type == BenchmarkType.MEM) {
     count = benchmarkOptions.numIterationsForMemBenchmarks;
     benchmarkOptions.batchSize = 1;
   }
 
   let retries = 0;
 
-  console.log("runBenchmakLoop", framework, benchmark);
+  console.log("runBenchmakLoop", framework, benchmarkInfo);
 
   while (results.length < count) {
     benchmarkOptions.batchSize = Math.min(benchmarkOptions.batchSize, count - results.length);
-    console.log("FORKING: ", benchmark.benchmarkInfo.id, " BatchSize ", benchmarkOptions.batchSize);
-    let res = await forkAndCallBenchmark(framework, benchmark, benchmarkOptions);
+    console.log("FORKING: ", benchmarkInfo.id, " BatchSize ", benchmarkOptions.batchSize);
+    let res = await forkAndCallBenchmark(framework, benchmarkInfo, benchmarkOptions);
     if (Array.isArray(res.result)) {
       results = results.concat(res.result as number[]);
     } else results.push(res.result);
@@ -142,12 +137,12 @@ async function runBenchmakLoop(
         retries++;
         if (retries == 3) break;
       } else {
-        errors.push(`Executing ${framework.uri} and benchmark ${benchmark.benchmarkInfo.id} failed: ` + res.error);
+        errors.push(`Executing ${framework.uri} and benchmark ${benchmarkInfo.id} failed: ` + res.error);
         break;
       }
     }
   }
-  if (benchmark.type == BenchmarkType.CPU) {
+  if (benchmarkInfo.type == BenchmarkType.CPU) {
     console.log("CPU results before: ", results);
     (results as number[]).sort((a: number, b: number) => a - b);
     results = results.slice(0, config.NUM_ITERATIONS_FOR_BENCHMARK_CPU);
@@ -157,23 +152,19 @@ async function runBenchmakLoop(
   console.log("******* result ", results);
   await writeResults(config, {
     framework: framework,
-    benchmark: benchmark.benchmarkInfo,
+    benchmark: benchmarkInfo,
     results: results,
-    type: benchmark.type as typeof BenchmarkType.CPU|BenchmarkType.MEM
+    type: benchmarkInfo.type as typeof BenchmarkType.CPU|BenchmarkType.MEM
   });
   return { errors, warnings };
   // } else {
   //     return executeBenchmark(frameworks, keyed, frameworkName, benchmarkName, benchmarkOptions);
 }
 
-async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]) {
+async function runBench(runFrameworks: FrameworkData[], benchmarkInfos: BenchmarkInfo[]) {
   let errors: String[] = [];
   let warnings: String[] = [];
 
-  let runBenchmarks: Array<TBenchmarkImplementation> = benchmarks.filter((b) =>
-    benchmarkNames.some((name) => b.benchmarkInfo.id.toLowerCase().indexOf(name) > -1)
-  );
-
   let restart: string = undefined;
   let index = runFrameworks.findIndex((f) => f.fullNameWithKeyedAndVersion === restart);
   if (index > -1) {
@@ -186,13 +177,14 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
   );
   console.log(
     "Benchmarks that will be run",
-    runBenchmarks.map((b) => b.benchmarkInfo.id)
+    benchmarkInfos.map((b) => b.id)
   );
 
     console.log("HEADLESS*** ", args.headless);
 
   let benchmarkOptions: BenchmarkOptions = {
     port: config.PORT.toFixed(),
+    HOST: config.HOST,
     remoteDebuggingPort: config.REMOTE_DEBUGGING_PORT,
     chromePort: config.CHROME_PORT,
     headless: args.headless,
@@ -203,13 +195,20 @@ async function runBench(runFrameworks: FrameworkData[], benchmarkNames: string[]
     batchSize: 1,
   };
 
+  console.log("benchmarkOptions", benchmarkOptions);
+
   for (let i = 0; i < runFrameworks.length; i++) {
-    for (let j = 0; j < runBenchmarks.length; j++) {
+    for (let j = 0; j < benchmarkInfos.length; j++) {
       try {
-        console.log("****** runBenchmarks[j].type", runBenchmarks[j].type, runBenchmarks[j].type == BenchmarkType.STARTUP_MAIN)
-        let result = (runBenchmarks[j].type == BenchmarkType.STARTUP_MAIN) ?
-            await runBenchmakLoopStartup(runFrameworks[i], runBenchmarks[j] as BenchmarkLighthouse, benchmarkOptions)
-          : await runBenchmakLoop(runFrameworks[i], runBenchmarks[j] as TBenchmarkPuppeteer|CPUBenchmarkWebdriver, benchmarkOptions);
+        let result;
+
+        if (benchmarkInfos[j].type == BenchmarkType.STARTUP_MAIN) {
+          result = await runBenchmakLoopStartup(runFrameworks[i], benchmarkInfos[j] as StartupBenchmarkInfo, benchmarkOptions)
+        } else if (benchmarkInfos[j].type == BenchmarkType.CPU) {
+          result = await runBenchmakLoop(runFrameworks[i], benchmarkInfos[j] as CPUBenchmarkInfo, benchmarkOptions);
+        } else {
+          result = await runBenchmakLoop(runFrameworks[i], benchmarkInfos[j] as MemBenchmarkInfo, benchmarkOptions);
+        }
         errors = errors.concat(result.errors);
         warnings = warnings.concat(result.warnings);
       } catch (e) {
@@ -252,18 +251,37 @@ let args: any = yargs(process.argv)
   .help("help")
   .boolean("headless").default("headless", false)
   .boolean("smoketest")
+  .string("runner").default("runner","playwright")
   .array("framework")
   .array("benchmark")
   .string("chromeBinary").argv;
 
+let runner = args.runner;
+if ([BENCHMARK_RUNNER.WEBDRIVER_CDP,BENCHMARK_RUNNER.WEBDRIVER,BENCHMARK_RUNNER.PLAYWRIGHT,BENCHMARK_RUNNER.PUPPETEER].includes(runner)) {
+  console.log(`INFO: Using ${runner} benchmark runner`)
+  config.BENCHMARK_RUNNER = runner;
+} else {
+  console.log("ERROR: argument driver has illegal value "+runner, [BENCHMARK_RUNNER.WEBDRIVER_CDP,BENCHMARK_RUNNER.WEBDRIVER,BENCHMARK_RUNNER.PLAYWRIGHT,BENCHMARK_RUNNER.PUPPETEER]);
+  process.exit(1);
+}
+
 let allArgs = args._.length <= 2 ? [] : args._.slice(2, args._.length);
 let frameworkArgument = !args.framework ? allArgs : args.framework;
 console.log("args", args, "allArgs", allArgs);
 
+if (process.env.HOST) {
+  config.HOST = process.env.HOST;
+  console.log(`INFO: Using host ${config.HOST} instead of localhost`);
+}
+
 async function main() {
-  let runBenchmarks = (args.benchmark && args.benchmark.length > 0 ? args.benchmark : [""]).map((v: string) => v.toString());
+  let runBenchmarksArgs: string[] =  (args.benchmark && args.benchmark.length > 0) ? args.benchmark : [""];
+  let runBenchmarks: Array<BenchmarkInfo> = Object.values(benchmarkInfos).filter((b) =>
+    runBenchmarksArgs.some((name) => b.id.toLowerCase().indexOf(name) > -1)
+  );
+  
+  
   let runFrameworks: FrameworkData[];
-
   let matchesDirectoryArg = (directoryName: string) =>
     frameworkArgument.length == 0 || frameworkArgument.some((arg: string) => arg == directoryName);
   runFrameworks = await initializeFrameworks(matchesDirectoryArg);
@@ -297,4 +315,4 @@ main()
   .catch((error) => {
     console.log("run was not completely sucessful", error);
     process.exit(1);
-  });
+  });