From 8fc2f05e770ec38c3c1f8d8357dde193714830ca Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 13 Mar 2024 11:23:05 +0100 Subject: [PATCH 1/9] Remove unused methods from BenchmarksRunner --- .../org/enso/interpreter/bench/BenchmarksRunner.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java index b7a694240188..8d098aebdb9f 100644 --- a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -21,15 +21,6 @@ public class BenchmarksRunner { public static final File REPORT_FILE = new File("./bench-report.xml"); - /** - * @return A list of qualified names of all benchmarks visible to JMH. - */ - public List getAvailable() { - return BenchmarkList.defaultList().getAll(null, new ArrayList<>()).stream() - .map(BenchmarkListEntry::getUsername) - .collect(Collectors.toList()); - } - public static void run(String[] args) throws RunnerException { CommandLineOptions cmdOpts = null; try { From cdd51f0361661587de909bb0117811ac3e530125 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 13 Mar 2024 11:36:33 +0100 Subject: [PATCH 2/9] Fix typo in sbt command for runtime benches --- build/build/src/engine/context.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs index dfde5101339d..4e000836f942 100644 --- a/build/build/src/engine/context.rs +++ b/build/build/src/engine/context.rs @@ -453,7 +453,7 @@ impl RunContext { // TODO [mwu] It should be possible to run them through context config option. if self.config.build_benchmarks { tasks.extend([ - "runtime/Benchmark/compile", + "runtime-benchmarks/compile", "language-server/Benchmark/compile", "searcher/Benchmark/compile", "std-benchmarks/Benchmark/compile", From b5d4a7225e27f355f680f56d9c38150da9e50e48 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 5 Mar 2024 17:41:46 +0100 Subject: [PATCH 3/9] runtime-benchmarks/benchOnly depends on runtime-fat-jar/assembly --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index cd67f6c7318d..064bbd4ff938 100644 --- a/build.sbt +++ b/build.sbt @@ -1857,6 +1857,7 @@ lazy val `runtime-benchmarks` = "-Xlint:unchecked" ), Compile / compile := (Compile / compile) + .dependsOn(`runtime-fat-jar` / assembly) .dependsOn(Def.task { (Compile / sourceManaged).value.mkdirs }) .value, parallelExecution := false, From d470fa1094dfa8cba240d91e5064ece85a3ff4dd Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 5 Mar 2024 17:48:06 +0100 Subject: [PATCH 4/9] Increase warmup for IfVsCaseBenchmarks --- .../bench/benchmarks/semantic/IfVsCaseBenchmarks.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java index 3a838ae688f6..097786e2d038 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java @@ -25,8 +25,8 @@ @BenchmarkMode(Mode.AverageTime) @Fork(1) -@Warmup(iterations = 5, time = 1) -@Measurement(iterations = 3, time = 3) +@Warmup(iterations = 5) +@Measurement(iterations = 3) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class IfVsCaseBenchmarks { From 4e4ff62cf852e7bea1cda121f3b2b212d2873418 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 13 Mar 2024 12:07:18 +0100 Subject: [PATCH 5/9] Improve BenchmarksRunner runCompileOnly message --- .../java/org/enso/interpreter/bench/BenchmarksRunner.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java index 8d098aebdb9f..0b6c55919a9a 100644 --- a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -68,7 +68,11 @@ public static void run(String[] args) throws RunnerException { private static Collection runCompileOnly(List includes) throws RunnerException { - System.out.println("Running benchmarks " + includes + " in compileOnly mode"); + if (includes.isEmpty()) { + System.out.println("Running all benchmarks in compileOnly mode"); + } else { + System.out.println("Running benchmarks " + includes + " in compileOnly mode"); + } var optsBuilder = new OptionsBuilder() .measurementTime(TimeValue.seconds(1)) From f3d2eed7bafd7a33c0eb1803580a98a1f486522b Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 13 Mar 2024 12:36:37 +0100 Subject: [PATCH 6/9] sbt graalVMVersionCheck does not fail during reload --- build.sbt | 11 +++-------- project/GraalVM.scala | 15 +++++++++------ 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/build.sbt b/build.sbt index 064bbd4ff938..d8811dcfcb18 100644 --- a/build.sbt +++ b/build.sbt @@ -47,20 +47,15 @@ val currentEdition = sys.env.getOrElse( val stdLibVersion = defaultDevEnsoVersion val targetStdlibVersion = ensoVersion -lazy val graalVMVersionCheck = taskKey[Unit]("Check GraalVM and Java versions") -graalVMVersionCheck := { +// Inspired by https://www.scala-sbt.org/1.x/docs/Howto-Startup.html#How+to+take+an+action+on+startup +lazy val startupStateTransition: State => State = { s: State => GraalVM.versionCheck( graalVersion, graalMavenPackagesVersion, javaVersion, - state.value.log + s ) } - -// Inspired by https://www.scala-sbt.org/1.x/docs/Howto-Startup.html#How+to+take+an+action+on+startup -lazy val startupStateTransition: State => State = { s: State => - "graalVMVersionCheck" :: s -} Global / onLoad := { val old = (Global / onLoad).value startupStateTransition compose old diff --git a/project/GraalVM.scala b/project/GraalVM.scala index 5c9ff49d1621..e271d7822d5e 100644 --- a/project/GraalVM.scala +++ b/project/GraalVM.scala @@ -120,14 +120,15 @@ object GraalVM { graalVersion: String, graalPackagesVersion: String, javaVersion: String, - log: ManagedLogger - ): Unit = { + oldState: State + ): State = { + val log = oldState.log if (graalPackagesVersion != version) { log.error( s"Expected GraalVM packages version $version, but got $graalPackagesVersion. " + s"Version specified in build.sbt and GraalVM.scala must be in sync" ) - throw new IllegalStateException("GraalVM version check failed") + return oldState.fail } val javaVendor = System.getProperty("java.vendor") if (!allowedJavaVendors.contains(javaVendor)) { @@ -143,7 +144,7 @@ object GraalVM { s"Running on Java version $javaSpecVersion. " + s"Expected Java version $javaVersion." ) - throw new IllegalStateException("java.specification.vendor check failed") + return oldState.fail } val vmVersion = System.getProperty("java.vm.version") @@ -154,13 +155,15 @@ object GraalVM { s"Running on GraalVM version $version. " + s"Expected GraalVM version $graalVersion." ) - throw new IllegalStateException("java.vm.version check failed") + oldState.fail + } else { + oldState } case None => log.error( s"Could not parse GraalVM version from java.vm.version: $vmVersion." ) - throw new IllegalStateException("java.vm.version check failed") + oldState.fail } } From a8f475ba341df89fff9e251d328eeaeea29139e7 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 13 Mar 2024 12:52:08 +0100 Subject: [PATCH 7/9] BenchmarksRunner.runCompileOnly exits if some benchmark failed --- .../interpreter/bench/BenchmarksRunner.java | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java index 0b6c55919a9a..8c154bc6120a 100644 --- a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -66,8 +66,12 @@ public static void run(String[] args) throws RunnerException { } } - private static Collection runCompileOnly(List includes) - throws RunnerException { + /** + * Results from compileOnly mode are not reported. Moreover, if some of the benchmarks in this + * mode fails, the whole process immediately fails. This behavior is different to *normal* + * benchmarks, where a single failure does not stop the whole process. + */ + private static void runCompileOnly(List includes) { if (includes.isEmpty()) { System.out.println("Running all benchmarks in compileOnly mode"); } else { @@ -78,27 +82,19 @@ private static Collection runCompileOnly(List includes) .measurementTime(TimeValue.seconds(1)) .measurementIterations(1) .warmupIterations(0) + .shouldFailOnError(true) .forks(0); includes.forEach(optsBuilder::include); var opts = optsBuilder.build(); var runner = new Runner(opts); - return runner.run(); - } - - public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { - String includeRegex = "^" + label + "$"; - if (Boolean.getBoolean("bench.compileOnly")) { - var results = runCompileOnly(List.of(includeRegex)); - var firstResult = results.iterator().next(); - return reportResult(label, firstResult); - } else { - var opts = - new OptionsBuilder() - .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") - .include(includeRegex) - .build(); - RunResult benchmarksResult = new Runner(opts).runSingle(); - return reportResult(label, benchmarksResult); + try { + runner.run(); + System.out.println( + "benchmarks run successfully in compileOnly mode. Results are not reported."); + } catch (RunnerException e) { + System.err.println("Benchmark run failed: " + e.getMessage()); + e.printStackTrace(System.err); + System.exit(1); } } From ba79096249ad08887d46570ead715c160af79004 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 14 Mar 2024 14:26:03 +0100 Subject: [PATCH 8/9] Fix LANGUAGE_HOME_OVERRIDE option in compiler benchmarks --- .../src/main/java/org/enso/compiler/benchmarks/Utils.java | 4 ++++ .../benchmarks/module/ImportStandardLibrariesBenchmark.java | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java index ba110b3c7e91..4b5f31946002 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.logging.Level; import org.enso.interpreter.runtime.EnsoContext; import org.enso.polyglot.LanguageInfo; @@ -21,6 +22,9 @@ public class Utils { public static Context.Builder createDefaultContextBuilder() { return Context.newBuilder() .allowExperimentalOptions(true) + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath()) .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) .option(RuntimeOptions.DISABLE_IR_CACHES, "true") .option(RuntimeOptions.STRICT_ERRORS, "true") diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java index 01698de4da21..8cb86738fe84 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java @@ -3,7 +3,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.file.Paths; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -93,9 +92,6 @@ public void setup(BenchmarkParams params) throws IOException { this.out = new ByteArrayOutputStream(); this.context = Utils.createDefaultContextBuilder() - .option( - RuntimeOptions.LANGUAGE_HOME_OVERRIDE, - Paths.get("../../distribution/component").toFile().getAbsolutePath()) // Enable IR caches - we don't want to compile the imported modules from the standard // libraries .option(RuntimeOptions.DISABLE_IR_CACHES, "false") From 951064437ef465cd48aa9dfb388d8b3ca9198aee Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 14 Mar 2024 14:29:49 +0100 Subject: [PATCH 9/9] fmt --- .../java/org/enso/interpreter/bench/BenchmarksRunner.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java index 8c154bc6120a..680957228e8c 100644 --- a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -3,13 +3,9 @@ import jakarta.xml.bind.JAXBException; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import org.openjdk.jmh.results.RunResult; -import org.openjdk.jmh.runner.BenchmarkList; -import org.openjdk.jmh.runner.BenchmarkListEntry; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.CommandLineOptionException;