diff --git a/build.sbt b/build.sbt index 434afce22a2d..f4c949ec4b87 100644 --- a/build.sbt +++ b/build.sbt @@ -1850,6 +1850,7 @@ lazy val `runtime-benchmarks` = "-Xlint:unchecked" ), Compile / compile := (Compile / compile) + .dependsOn(`runtime-fat-jar` / assembly) .dependsOn(Def.task { (Compile / sourceManaged).value.mkdirs }) .value, parallelExecution := false, diff --git a/build/build/src/engine/context.rs b/build/build/src/engine/context.rs index dfde5101339d..4e000836f942 100644 --- a/build/build/src/engine/context.rs +++ b/build/build/src/engine/context.rs @@ -453,7 +453,7 @@ impl RunContext { // TODO [mwu] It should be possible to run them through context config option. if self.config.build_benchmarks { tasks.extend([ - "runtime/Benchmark/compile", + "runtime-benchmarks/compile", "language-server/Benchmark/compile", "searcher/Benchmark/compile", "std-benchmarks/Benchmark/compile", diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java index ba110b3c7e91..4b5f31946002 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -8,6 +8,7 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.logging.Level; import org.enso.interpreter.runtime.EnsoContext; import org.enso.polyglot.LanguageInfo; @@ -21,6 +22,9 @@ public class Utils { public static Context.Builder createDefaultContextBuilder() { return Context.newBuilder() .allowExperimentalOptions(true) + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath()) .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) .option(RuntimeOptions.DISABLE_IR_CACHES, "true") .option(RuntimeOptions.STRICT_ERRORS, "true") diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java index 01698de4da21..8cb86738fe84 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java @@ -3,7 +3,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.nio.file.Paths; import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -93,9 +92,6 @@ public void setup(BenchmarkParams params) throws IOException { this.out = new ByteArrayOutputStream(); this.context = Utils.createDefaultContextBuilder() - .option( - RuntimeOptions.LANGUAGE_HOME_OVERRIDE, - Paths.get("../../distribution/component").toFile().getAbsolutePath()) // Enable IR caches - we don't want to compile the imported modules from the standard // libraries .option(RuntimeOptions.DISABLE_IR_CACHES, "false") diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java index 3a838ae688f6..097786e2d038 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/IfVsCaseBenchmarks.java @@ -25,8 +25,8 @@ @BenchmarkMode(Mode.AverageTime) @Fork(1) -@Warmup(iterations = 5, time = 1) -@Measurement(iterations = 3, time = 3) +@Warmup(iterations = 5) +@Measurement(iterations = 3) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class IfVsCaseBenchmarks { diff --git a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java index b7a694240188..680957228e8c 100644 --- a/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/lib/java/benchmarks-common/src/main/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -3,13 +3,9 @@ import jakarta.xml.bind.JAXBException; import java.io.File; import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.stream.Collectors; import org.openjdk.jmh.results.RunResult; -import org.openjdk.jmh.runner.BenchmarkList; -import org.openjdk.jmh.runner.BenchmarkListEntry; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.CommandLineOptionException; @@ -21,15 +17,6 @@ public class BenchmarksRunner { public static final File REPORT_FILE = new File("./bench-report.xml"); - /** - * @return A list of qualified names of all benchmarks visible to JMH. - */ - public List getAvailable() { - return BenchmarkList.defaultList().getAll(null, new ArrayList<>()).stream() - .map(BenchmarkListEntry::getUsername) - .collect(Collectors.toList()); - } - public static void run(String[] args) throws RunnerException { CommandLineOptions cmdOpts = null; try { @@ -75,35 +62,35 @@ public static void run(String[] args) throws RunnerException { } } - private static Collection runCompileOnly(List includes) - throws RunnerException { - System.out.println("Running benchmarks " + includes + " in compileOnly mode"); + /** + * Results from compileOnly mode are not reported. Moreover, if some of the benchmarks in this + * mode fails, the whole process immediately fails. This behavior is different to *normal* + * benchmarks, where a single failure does not stop the whole process. + */ + private static void runCompileOnly(List includes) { + if (includes.isEmpty()) { + System.out.println("Running all benchmarks in compileOnly mode"); + } else { + System.out.println("Running benchmarks " + includes + " in compileOnly mode"); + } var optsBuilder = new OptionsBuilder() .measurementTime(TimeValue.seconds(1)) .measurementIterations(1) .warmupIterations(0) + .shouldFailOnError(true) .forks(0); includes.forEach(optsBuilder::include); var opts = optsBuilder.build(); var runner = new Runner(opts); - return runner.run(); - } - - public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { - String includeRegex = "^" + label + "$"; - if (Boolean.getBoolean("bench.compileOnly")) { - var results = runCompileOnly(List.of(includeRegex)); - var firstResult = results.iterator().next(); - return reportResult(label, firstResult); - } else { - var opts = - new OptionsBuilder() - .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") - .include(includeRegex) - .build(); - RunResult benchmarksResult = new Runner(opts).runSingle(); - return reportResult(label, benchmarksResult); + try { + runner.run(); + System.out.println( + "benchmarks run successfully in compileOnly mode. Results are not reported."); + } catch (RunnerException e) { + System.err.println("Benchmark run failed: " + e.getMessage()); + e.printStackTrace(System.err); + System.exit(1); } }