From 61a79669437dedc429f80240671541bde8f65c84 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 23 Feb 2024 13:31:36 +0100 Subject: [PATCH 01/21] runtime-benchmarks run with Truffle DSL processor --- build.sbt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build.sbt b/build.sbt index 92bd4f1ed074..e1b147d59b6f 100644 --- a/build.sbt +++ b/build.sbt @@ -1833,6 +1833,7 @@ lazy val `runtime-benchmarks` = "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion, + "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided", "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "slf4j-nop" % slf4jVersion ), @@ -1848,6 +1849,14 @@ lazy val `runtime-benchmarks` = frgaalSourceLevel, "--enable-preview" ), + javacOptions ++= Seq( + "-s", + (Compile / sourceManaged).value.getAbsolutePath, + "-Xlint:unchecked" + ), + Compile / compile := (Compile / compile) + .dependsOn(Def.task { (Compile / sourceManaged).value.mkdirs }) + .value, parallelExecution := false, modulePath := { val requiredModIds = GraalVM.modules ++ GraalVM.langsPkgs ++ Seq( From 95962c32eb36577e7b83ff18a0f5848071480949 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 23 Feb 2024 20:25:36 +0100 Subject: [PATCH 02/21] Add skeleton of some compiler benchmarks --- .../compiler/benchmarks/CodeGenerator.java | 166 ++++++++++++++++++ .../benchmarks/InlineCompilerBenchmarks.java | 103 +++++++++++ .../benchmarks/ModuleCompilerBenchmarks.java | 145 +++++++++++++++ .../org/enso/compiler/benchmarks/Utils.java | 79 +++++++++ 4 files changed, 493 insertions(+) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java new file mode 100644 index 000000000000..197b6e18ccd1 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -0,0 +1,166 @@ +package org.enso.compiler.benchmarks; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +// TODO: Make sure all identifiers are used, so that there is no unused variable warning +public class CodeGenerator { + private static final int MAX_IDENT_SIZE = 10; + private static final int MAX_TEXT_LITERAL_SIZE = 20; + private static final int SMALL_LETTERS_CNT = 26; + private static final int SEED = 42; + + private static final List OPERATORS = List.of( + "+", "-", "*", "/", "&&", "||", "!", "==", "!=", "<", ">", "<=", ">=" + ); + + private static final List METHODS = List.of( + "map", "filter", "foldl", "foldr", "head", "tail", "init", "last", "length", "reverse" + ); + + private final Random random = new Random(SEED); + private int identifierCnt = 0; + + + List createIdentifiers(int count) { + List idents = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + idents.add(nextIdentifier()); + } + return idents; + } + + private String nextIdentifier() { + identifierCnt++; + return "operator" + identifierCnt; + } + + private String nextMethod() { + var idx = random.nextInt(METHODS.size()); + return METHODS.get(idx); + } + + private String nextTextLiteral() { + var size = random.nextInt(MAX_TEXT_LITERAL_SIZE); + var sb = new StringBuilder(); + for (int i = 0; i < size; i++) { + var rndChar = (char) ('a' + random.nextInt(SMALL_LETTERS_CNT)); + sb.append(rndChar); + } + return sb.toString(); + } + + private String nextIntLiteral() { + return Integer.toString(random.nextInt()); + } + + private String nextDecimalLiteral() { + var part1 = random.nextInt(); + var decimalPart = Math.abs(random.nextInt()); + return part1 + "." + decimalPart; + } + + private String nextLiteral() { + var rndInt = random.nextInt(3); + return switch (rndInt) { + case 0 -> nextTextLiteral(); + case 1 -> nextIntLiteral(); + case 2 -> nextDecimalLiteral(); + default -> throw new UnsupportedOperationException("unimplemented"); + }; + } + + String createExpression(List identifiers, int size) { + switch (size) { + // Literal + case 0 -> { + return nextLiteral(); + } + // Either a single identifier or a method call on the identifier + case 1 -> { + var sb = new StringBuilder(); + var ident = chooseIdentifier(identifiers); + sb.append(ident); + var shouldCallMethod = random.nextBoolean(); + if (shouldCallMethod) { + sb.append(".") + .append(nextMethod()); + } + return sb.toString(); + } + // Method call or binary operator + case 2 -> { + var sb = new StringBuilder(); + var shouldCallMethod = random.nextBoolean(); + var ident1 = chooseIdentifier(identifiers); + if (shouldCallMethod) { + var methodArg = createExpression(identifiers, 1); + sb.append(ident1) + .append(".") + .append(nextMethod()) + .append(" (") + .append(methodArg) + .append(")"); + } else { + var ident2 = chooseIdentifier(identifiers); + // Binary operator + sb.append(ident1) + .append(nextOperator()) + .append(ident2); + } + return sb.toString(); + } + // Split into two expressions with random size + default -> { + var sb = new StringBuilder(); + var shouldCallMethod = random.nextBoolean(); + if (shouldCallMethod) { + var ident = chooseIdentifier(identifiers); + var methodArity = size - 1; + List methodArgs = new ArrayList<>(); + for (int i = 0; i < methodArity; i++) { + methodArgs.add( + createExpression(identifiers, size - 1) + ); + } + sb.append(ident) + .append(".") + .append(nextMethod()) + .append(" "); + for (var methodArg : methodArgs) { + sb.append(methodArg) + .append(" "); + } + } else { + var rndIdx = Math.max(2, random.nextInt(size)); + var size1 = rndIdx; + var size2 = size - rndIdx; + var expr1 = createExpression(identifiers, size1); + var expr2 = createExpression(identifiers, size2); + var op = nextOperator(); + sb.append("(") + .append(expr1) + .append(")") + .append(op) + .append("(") + .append(expr2) + .append(")"); + } + return sb.toString(); + } + } + } + + private String nextOperator() { + var idx = random.nextInt(OPERATORS.size()); + return OPERATORS.get(idx); + } + + private String chooseIdentifier(List identifiers) { + assert !identifiers.isEmpty(); + var randomIdx = random.nextInt(identifiers.size()); + return identifiers.get(randomIdx); + } + +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java new file mode 100644 index 000000000000..4a52896b687b --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java @@ -0,0 +1,103 @@ +package org.enso.compiler.benchmarks; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import org.enso.compiler.context.InlineContext; +import org.enso.interpreter.node.MethodRootNode; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames.Module; +import org.enso.polyglot.MethodNames.TopScope; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.io.IOAccess; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Measures the inline compilation, that is the compilation that is requested inside a method. + * Simulates a scenario where there is an existing method and we are trying to insert a new + * expression into it. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(0) +@Warmup(iterations = 1) +@Measurement(iterations = 1) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class InlineCompilerBenchmarks { + @Setup + public void setup() throws IOException { + var ctx = + Context.newBuilder() + .allowExperimentalOptions(true) + .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .option(RuntimeOptions.STRICT_ERRORS, "true") + .logHandler(System.err) + .allowIO(IOAccess.ALL) + .allowAllAccess(true) + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath()) + .build(); + var ensoCtx = ctx + .getBindings(LanguageInfo.ID) + .invokeMember(TopScope.LEAK_CONTEXT) + .as(EnsoContext.class); + var code = """ + main = 1 + 1 + """; + var srcFile = createSrcFile(code, "bench-1.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = ctx.eval(src); + var assocType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); + var assocTypeReceiver = (Type) Utils.unwrapReceiver(ctx, assocType); + var moduleScope = assocTypeReceiver.getDefinitionScope(); + var mainFunc = moduleScope.getMethodForType(assocTypeReceiver, "main"); + var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); + var localScope = mainFuncRootNode.getLocalScope(); + var compiler = ensoCtx.getCompiler(); + InlineContext inlineContext = + InlineContext.fromJava( + localScope, + moduleScope.getModule().asCompilerModule(), + scala.Option.apply(false), + ensoCtx.getCompilerConfig(), + scala.Option.apply(compiler.packageRepository())); + var inlineExpr = "42 * 2"; + var tuppleOpt = compiler.runInline(inlineExpr, inlineContext); + assert tuppleOpt.isDefined(); + var newInlineContext = tuppleOpt.get()._1(); + var ir = tuppleOpt.get()._2(); + var newSrc = tuppleOpt.get()._3(); + + // + System.out.println("Done bench init"); + } + + private static File createSrcFile(String code, String name) { + var benchDataDir = Path.of(".", "target", "bench-data"); + var srcFile = benchDataDir.resolve(name).toFile(); + try { + Files.writeString(srcFile.toPath(), code); + } catch (IOException e) { + throw new AssertionError(e); + } + return srcFile; + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java new file mode 100644 index 000000000000..8c42316ef555 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java @@ -0,0 +1,145 @@ +package org.enso.compiler.benchmarks; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.interpreter.runtime.Module; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.interpreter.runtime.data.Type; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures how long does it take for the compiler passes to run on a various modules. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(0) +@Warmup(iterations = 1) +@Measurement(iterations = 1) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ModuleCompilerBenchmarks { + + private static final int IDENTIFIERS_CNT = 10; + private Compiler compiler; + private Module module; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + var ctx = Utils.createDefaultContext(); + var ensoCtx = Utils.leakEnsoContext(ctx); + switch (params.getBenchmark()) { + case "org.enso.compiler.benchmarks.ModuleCompilerBenchmarks.longMethodWithLotOfLocalVars" -> { + var sb = new StringBuilder(); + var codeGen = new CodeGenerator(); + var allIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); + var firstIdent = allIdentifiers.get(0); + List initializedIdentifiers = new ArrayList<>(); + initializedIdentifiers.add(firstIdent); + sb.append("main = ").append(System.lineSeparator()); + sb.append(" ") + .append(firstIdent) + .append(" = ") + .append("42") + .append(System.lineSeparator()); + + allIdentifiers + .stream() + .skip(1) + .forEach( + identifier -> { + var maxExprSize = Math.min(5, initializedIdentifiers.size() - 1); + sb.append(" ") + .append(identifier) + .append(" = ") + .append(codeGen.createExpression(initializedIdentifiers, maxExprSize)) + .append(System.lineSeparator()); + initializedIdentifiers.add(identifier); + }); + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = ctx.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(ctx, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + default -> { + throw new UnsupportedOperationException("unimplemented: Benchmark " + params.getBenchmark()); + } + } + } + + /** + * Measure compilation of a module with a single long method with a format like: + * 
+   * main =
+   *    obj1 = ...
+   *    obj2 = ...
+   *    obj3 = ...
+   *
+ * This is the format that is used by the IDE. + * This should measure mostly the performance of the dataflow analysis pass. + */ + @Benchmark + public void longMethodWithLotOfLocalVars(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + if (compilerResult.compiledModules().size() != 1) { + throw new AssertionError("Module compilation failed"); + } + } + + + /** + * Measure compilation of a module with a lot of small methods with + * variable number of arguments. + * @param blackhole + */ + @Benchmark + public void manySmallMethods(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + if (compilerResult.compiledModules().size() != 1) { + throw new AssertionError("Module compilation failed"); + } + } + + /** + * Measure compilation of a module with one method that contains a lot of + * nested blocks. + * @param blackhole + */ + @Benchmark + public void manyNestedBlocks(Blackhole blackhole) { + throw new UnsupportedOperationException("unimplemented"); + } + + /** + * Measure compilation of a module with one method that contains a lot of errors - + * syntactical errors and unknown identifiers. The compiler should be able to recover from errors + * and so it should compile the whole module and not stop after the first error. + * @param blackhole + */ + @Benchmark + public void manyErrors(Blackhole blackhole) { + throw new UnsupportedOperationException("unimplemented"); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java new file mode 100644 index 000000000000..48e5dacb8df9 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -0,0 +1,79 @@ +package org.enso.compiler.benchmarks; + +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.TruffleObject; +import com.oracle.truffle.api.library.ExportLibrary; +import com.oracle.truffle.api.library.ExportMessage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.logging.Level; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; + +public class Utils { + public static Context createDefaultContext() { + var ctx = + Context.newBuilder() + .allowExperimentalOptions(true) + .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .option(RuntimeOptions.STRICT_ERRORS, "true") + .logHandler(System.err) + .allowIO(IOAccess.ALL) + .allowAllAccess(true) + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath()) + .build(); + return ctx; + } + + public static EnsoContext leakEnsoContext(Context ctx) { + return ctx + .getBindings(LanguageInfo.ID) + .invokeMember(MethodNames.TopScope.LEAK_CONTEXT) + .as(EnsoContext.class); + } + + static Object unwrapReceiver(Context ctx, Value value) { + var unwrapper = new Unwrapper(); + var unwrapperValue = ctx.asValue(unwrapper); + unwrapperValue.execute(value); + return unwrapper.args[0]; + } + + static File createSrcFile(String code, String name) { + var benchDataDir = Path.of(".", "target", "bench-data"); + var srcFile = benchDataDir.resolve(name).toFile(); + try { + Files.writeString(srcFile.toPath(), code); + } catch (IOException e) { + throw new AssertionError(e); + } + return srcFile; + } + + @ExportLibrary(InteropLibrary.class) + static final class Unwrapper implements TruffleObject { + Object[] args; + + @ExportMessage + Object execute(Object[] args) { + this.args = args; + return this; + } + + @ExportMessage + boolean isExecutable() { + return true; + } + } +} From 7cc217c242d1cc02bc8a67c28b0fd36716101fa4 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 26 Feb 2024 12:25:39 +0100 Subject: [PATCH 03/21] ManyLocalVars is a separate benchmark --- .../compiler/benchmarks/CodeGenerator.java | 6 +- .../benchmarks/ModuleCompilerBenchmarks.java | 58 ---------- .../org/enso/compiler/benchmarks/Utils.java | 4 +- .../module/ManyLocalVarsBenchmark.java | 104 ++++++++++++++++++ 4 files changed, 109 insertions(+), 63 deletions(-) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java index 197b6e18ccd1..fd4a98df85dd 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -12,7 +12,7 @@ public class CodeGenerator { private static final int SEED = 42; private static final List OPERATORS = List.of( - "+", "-", "*", "/", "&&", "||", "!", "==", "!=", "<", ">", "<=", ">=" + "+", "-", "*", "/", "&&", "||", "==", "!=", "<", ">", "<=", ">=" ); private static final List METHODS = List.of( @@ -23,7 +23,7 @@ public class CodeGenerator { private int identifierCnt = 0; - List createIdentifiers(int count) { + public List createIdentifiers(int count) { List idents = new ArrayList<>(count); for (int i = 0; i < count; i++) { idents.add(nextIdentifier()); @@ -71,7 +71,7 @@ private String nextLiteral() { }; } - String createExpression(List identifiers, int size) { + public String createExpression(List identifiers, int size) { switch (size) { // Literal case 0 -> { diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java index 8c42316ef555..1798821c0df5 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java @@ -35,8 +35,6 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class ModuleCompilerBenchmarks { - - private static final int IDENTIFIERS_CNT = 10; private Compiler compiler; private Module module; @@ -45,69 +43,13 @@ public void setup(BenchmarkParams params) throws IOException { var ctx = Utils.createDefaultContext(); var ensoCtx = Utils.leakEnsoContext(ctx); switch (params.getBenchmark()) { - case "org.enso.compiler.benchmarks.ModuleCompilerBenchmarks.longMethodWithLotOfLocalVars" -> { - var sb = new StringBuilder(); - var codeGen = new CodeGenerator(); - var allIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); - var firstIdent = allIdentifiers.get(0); - List initializedIdentifiers = new ArrayList<>(); - initializedIdentifiers.add(firstIdent); - sb.append("main = ").append(System.lineSeparator()); - sb.append(" ") - .append(firstIdent) - .append(" = ") - .append("42") - .append(System.lineSeparator()); - - allIdentifiers - .stream() - .skip(1) - .forEach( - identifier -> { - var maxExprSize = Math.min(5, initializedIdentifiers.size() - 1); - sb.append(" ") - .append(identifier) - .append(" = ") - .append(codeGen.createExpression(initializedIdentifiers, maxExprSize)) - .append(System.lineSeparator()); - initializedIdentifiers.add(identifier); - }); - var code = sb.toString(); - var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); - var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); - var module = ctx.eval(src); - var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); - var assocType = (Type) Utils.unwrapReceiver(ctx, assocTypeValue); - var moduleScope = assocType.getDefinitionScope(); - this.module = moduleScope.getModule(); - this.compiler = ensoCtx.getCompiler(); - } default -> { throw new UnsupportedOperationException("unimplemented: Benchmark " + params.getBenchmark()); } } } - /** - * Measure compilation of a module with a single long method with a format like: - * 
-   * main =
-   *    obj1 = ...
-   *    obj2 = ...
-   *    obj3 = ...
-   *
- * This is the format that is used by the IDE. - * This should measure mostly the performance of the dataflow analysis pass. - */ - @Benchmark - public void longMethodWithLotOfLocalVars(Blackhole blackhole) { - var compilerResult = compiler.run(module.asCompilerModule()); - if (compilerResult.compiledModules().size() != 1) { - throw new AssertionError("Module compilation failed"); - } - } - /** * Measure compilation of a module with a lot of small methods with diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java index 48e5dacb8df9..5b23b394ff33 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -43,14 +43,14 @@ public static EnsoContext leakEnsoContext(Context ctx) { .as(EnsoContext.class); } - static Object unwrapReceiver(Context ctx, Value value) { + public static Object unwrapReceiver(Context ctx, Value value) { var unwrapper = new Unwrapper(); var unwrapperValue = ctx.asValue(unwrapper); unwrapperValue.execute(value); return unwrapper.args[0]; } - static File createSrcFile(String code, String name) { + public static File createSrcFile(String code, String name) { var benchDataDir = Path.of(".", "target", "bench-data"); var srcFile = benchDataDir.resolve(name).toFile(); try { diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java new file mode 100644 index 000000000000..fc034dad9320 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -0,0 +1,104 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measure compilation of a module with a single long method with a format like: + * 
+ * main =
+ *    obj1 = ...
+ *    obj2 = ...
+ *    obj3 = ...
+ *
+ * This is the format that is used by the IDE. + * This should measure mostly the performance of the dataflow analysis pass. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(0) +@Warmup(iterations = 1) +@Measurement(iterations = 1) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManyLocalVarsBenchmark { + + /** + * Total count of local variables in the `main` method. Every variable is defined on + * a new line. + */ + private static final int IDENTIFIERS_CNT = 10; + private Compiler compiler; + private Module module; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + var ctx = Utils.createDefaultContext(); + var ensoCtx = Utils.leakEnsoContext(ctx); + var sb = new StringBuilder(); + var codeGen = new CodeGenerator(); + var allIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); + var firstIdent = allIdentifiers.get(0); + List initializedIdentifiers = new ArrayList<>(); + initializedIdentifiers.add(firstIdent); + sb.append("main = ").append(System.lineSeparator()); + sb.append(" ") + .append(firstIdent) + .append(" = ") + .append("42") + .append(System.lineSeparator()); + + allIdentifiers + .stream() + .skip(1) + .forEach( + identifier -> { + var maxExprSize = Math.min(5, initializedIdentifiers.size() - 1); + sb.append(" ") + .append(identifier) + .append(" = ") + .append(codeGen.createExpression(initializedIdentifiers, maxExprSize)) + .append(System.lineSeparator()); + initializedIdentifiers.add(identifier); + }); + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = ctx.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(ctx, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + @Benchmark + public void longMethodWithLotOfLocalVars(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + if (compilerResult.compiledModules().size() != 1) { + throw new AssertionError("Module compilation failed"); + } + } +} From 38170a1754752cf6f23fce3b184107bacdf4c9eb Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 26 Feb 2024 12:38:14 +0100 Subject: [PATCH 04/21] Make sure there are no unused variables in ManyLocalVarsBenchmark --- .../compiler/benchmarks/CodeGenerator.java | 22 ++++++++++++---- .../module/ManyLocalVarsBenchmark.java | 26 ++++++++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java index fd4a98df85dd..93a660a5b0fc 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -3,6 +3,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Random; +import java.util.Set; // TODO: Make sure all identifiers are used, so that there is no unused variable warning public class CodeGenerator { @@ -71,7 +72,14 @@ private String nextLiteral() { }; } - public String createExpression(List identifiers, int size) { + /** + * Creates an expression with the given size. + * @param identifiers A collection of all the identifiers that can be used in the expression. + * @param usedIdentifiers Will be filled with the identifiers used in the expression. + * @param size Arity of the expression. + * @return A string representing the expression. + */ + public String createExpression(List identifiers, Set usedIdentifiers, int size) { switch (size) { // Literal case 0 -> { @@ -81,6 +89,7 @@ public String createExpression(List identifiers, int size) { case 1 -> { var sb = new StringBuilder(); var ident = chooseIdentifier(identifiers); + usedIdentifiers.add(ident); sb.append(ident); var shouldCallMethod = random.nextBoolean(); if (shouldCallMethod) { @@ -94,8 +103,9 @@ public String createExpression(List identifiers, int size) { var sb = new StringBuilder(); var shouldCallMethod = random.nextBoolean(); var ident1 = chooseIdentifier(identifiers); + usedIdentifiers.add(ident1); if (shouldCallMethod) { - var methodArg = createExpression(identifiers, 1); + var methodArg = createExpression(identifiers, usedIdentifiers, 1); sb.append(ident1) .append(".") .append(nextMethod()) @@ -104,6 +114,7 @@ public String createExpression(List identifiers, int size) { .append(")"); } else { var ident2 = chooseIdentifier(identifiers); + usedIdentifiers.add(ident2); // Binary operator sb.append(ident1) .append(nextOperator()) @@ -117,11 +128,12 @@ public String createExpression(List identifiers, int size) { var shouldCallMethod = random.nextBoolean(); if (shouldCallMethod) { var ident = chooseIdentifier(identifiers); + usedIdentifiers.add(ident); var methodArity = size - 1; List methodArgs = new ArrayList<>(); for (int i = 0; i < methodArity; i++) { methodArgs.add( - createExpression(identifiers, size - 1) + createExpression(identifiers, usedIdentifiers, size - 1) ); } sb.append(ident) @@ -136,8 +148,8 @@ public String createExpression(List identifiers, int size) { var rndIdx = Math.max(2, random.nextInt(size)); var size1 = rndIdx; var size2 = size - rndIdx; - var expr1 = createExpression(identifiers, size1); - var expr2 = createExpression(identifiers, size2); + var expr1 = createExpression(identifiers, usedIdentifiers, size1); + var expr2 = createExpression(identifiers, usedIdentifiers, size2); var op = nextOperator(); sb.append("(") .append(expr1) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index fc034dad9320..d301d666d24b 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -2,7 +2,9 @@ import java.io.IOException; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.enso.compiler.Compiler; import org.enso.compiler.benchmarks.CodeGenerator; @@ -69,6 +71,7 @@ public void setup(BenchmarkParams params) throws IOException { .append("42") .append(System.lineSeparator()); + Set usedIdentifiers = new HashSet<>(); allIdentifiers .stream() .skip(1) @@ -78,11 +81,32 @@ public void setup(BenchmarkParams params) throws IOException { sb.append(" ") .append(identifier) .append(" = ") - .append(codeGen.createExpression(initializedIdentifiers, maxExprSize)) + .append(codeGen.createExpression(initializedIdentifiers, usedIdentifiers, maxExprSize)) .append(System.lineSeparator()); initializedIdentifiers.add(identifier); }); + assert initializedIdentifiers.size() == IDENTIFIERS_CNT; + assert usedIdentifiers.size() <= IDENTIFIERS_CNT; + if (usedIdentifiers.size() < IDENTIFIERS_CNT) { + // Add a final line that uses the rest of the identifiers, so that there is no "Unused binding" + // warning. + List unusedIdentifiers = new ArrayList<>(allIdentifiers); + unusedIdentifiers.removeAll(usedIdentifiers); + sb.append(" ") + .append("result = "); + sb.append( + unusedIdentifiers + .stream() + .reduce((a, b) -> a + " + " + b) + .orElseThrow() + ); + sb.append(System.lineSeparator()); + // result is the return value from the main method + sb.append(" ") + .append("result") + .append(System.lineSeparator()); + } var code = sb.toString(); var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); From 5f79c1aaed123a7c8904ce07f777629b7036af94 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 26 Feb 2024 13:34:16 +0100 Subject: [PATCH 05/21] Compiler stdout and stderr is checked in teardown --- .../org/enso/compiler/benchmarks/Utils.java | 24 +++++-------- .../module/ManyLocalVarsBenchmark.java | 34 +++++++++++++++---- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java index 5b23b394ff33..2560edbe94c6 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -19,21 +19,15 @@ import org.graalvm.polyglot.io.IOAccess; public class Utils { - public static Context createDefaultContext() { - var ctx = - Context.newBuilder() - .allowExperimentalOptions(true) - .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) - .option(RuntimeOptions.DISABLE_IR_CACHES, "true") - .option(RuntimeOptions.STRICT_ERRORS, "true") - .logHandler(System.err) - .allowIO(IOAccess.ALL) - .allowAllAccess(true) - .option( - RuntimeOptions.LANGUAGE_HOME_OVERRIDE, - Paths.get("../../distribution/component").toFile().getAbsolutePath()) - .build(); - return ctx; + public static Context.Builder createDefaultContextBuilder() { + return Context.newBuilder() + .allowExperimentalOptions(true) + .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .option(RuntimeOptions.STRICT_ERRORS, "true") + .logHandler(System.err) + .allowIO(IOAccess.ALL) + .allowAllAccess(true); } public static EnsoContext leakEnsoContext(Context ctx) { diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index d301d666d24b..414ef47605e1 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -1,6 +1,8 @@ package org.enso.compiler.benchmarks.module; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.io.OutputStream; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -13,6 +15,7 @@ import org.enso.interpreter.runtime.data.Type; import org.enso.polyglot.LanguageInfo; import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Source; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -23,6 +26,7 @@ import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; import org.openjdk.jmh.annotations.Warmup; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.infra.Blackhole; @@ -39,9 +43,8 @@ * This should measure mostly the performance of the dataflow analysis pass. */ @BenchmarkMode(Mode.AverageTime) -@Fork(0) -@Warmup(iterations = 1) -@Measurement(iterations = 1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class ManyLocalVarsBenchmark { @@ -51,13 +54,21 @@ public class ManyLocalVarsBenchmark { * a new line. */ private static final int IDENTIFIERS_CNT = 10; + private Context context; private Compiler compiler; private Module module; + private OutputStream out; @Setup public void setup(BenchmarkParams params) throws IOException { - var ctx = Utils.createDefaultContext(); - var ensoCtx = Utils.leakEnsoContext(ctx); + this.out = new ByteArrayOutputStream(); + this.context = Utils + .createDefaultContextBuilder() + .logHandler(out) + .out(out) + .err(out) + .build(); + var ensoCtx = Utils.leakEnsoContext(context); var sb = new StringBuilder(); var codeGen = new CodeGenerator(); var allIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); @@ -110,14 +121,23 @@ public void setup(BenchmarkParams params) throws IOException { var code = sb.toString(); var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); - var module = ctx.eval(src); + var module = context.eval(src); var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); - var assocType = (Type) Utils.unwrapReceiver(ctx, assocTypeValue); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); var moduleScope = assocType.getDefinitionScope(); this.module = moduleScope.getModule(); this.compiler = ensoCtx.getCompiler(); } + @TearDown + public void tearDown() throws IOException { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out.toString()); + } + out.close(); + context.close(); + } + @Benchmark public void longMethodWithLotOfLocalVars(Blackhole blackhole) { var compilerResult = compiler.run(module.asCompilerModule()); From 5c8da151d570bbca7dfd5ff4d6dffa1d264ded8e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 26 Feb 2024 13:34:40 +0100 Subject: [PATCH 06/21] Set sensible measure and warmup params for ManyLocalVarsBenchmark --- .../compiler/benchmarks/module/ManyLocalVarsBenchmark.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index 414ef47605e1..dc70a3f0046f 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -19,7 +19,6 @@ import org.graalvm.polyglot.Source; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; @@ -53,7 +52,7 @@ public class ManyLocalVarsBenchmark { * Total count of local variables in the `main` method. Every variable is defined on * a new line. */ - private static final int IDENTIFIERS_CNT = 10; + private static final int IDENTIFIERS_CNT = 40; private Context context; private Compiler compiler; private Module module; @@ -144,5 +143,6 @@ public void longMethodWithLotOfLocalVars(Blackhole blackhole) { if (compilerResult.compiledModules().size() != 1) { throw new AssertionError("Module compilation failed"); } + blackhole.consume(compilerResult); } } From e4396a157727b4147feafa576213e525b3c35ec2 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 26 Feb 2024 16:54:58 +0100 Subject: [PATCH 07/21] Add ManyErrorsBenchmark --- .../compiler/benchmarks/CodeGenerator.java | 8 +- .../module/ManyErrorsBenchmark.java | 153 ++++++++++++++++++ .../module/ManyLocalVarsBenchmark.java | 3 +- 3 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java index 93a660a5b0fc..2d251b872704 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -1,6 +1,7 @@ package org.enso.compiler.benchmarks; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Random; import java.util.Set; @@ -49,7 +50,7 @@ private String nextTextLiteral() { var rndChar = (char) ('a' + random.nextInt(SMALL_LETTERS_CNT)); sb.append(rndChar); } - return sb.toString(); + return "\"" + sb + "\""; } private String nextIntLiteral() { @@ -62,7 +63,7 @@ private String nextDecimalLiteral() { return part1 + "." + decimalPart; } - private String nextLiteral() { + public String nextLiteral() { var rndInt = random.nextInt(3); return switch (rndInt) { case 0 -> nextTextLiteral(); @@ -71,6 +72,9 @@ private String nextLiteral() { default -> throw new UnsupportedOperationException("unimplemented"); }; } + public String createExpression(List identifiers, int size) { + return createExpression(identifiers, new HashSet<>(), size); + } /** * Creates an expression with the given size. diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java new file mode 100644 index 000000000000..544a88334b5e --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java @@ -0,0 +1,153 @@ +package org.enso.compiler.benchmarks.module; + + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measure compilation of a module with one method that contains a lot of errors - + * syntactical errors and unknown identifiers. The compiler should be able to recover from errors + * and so it should compile the whole module and not stop after the first error. + */ +@BenchmarkMode(Mode.AverageTime) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManyErrorsBenchmark { + private static final int ERRORS_CNT = 30; + private static final int IDENTIFIERS_CNT = 40; + private static final int EXPR_SIZE = 5; + private static final List UNDEFINED_IDENTIFIERS = List.of( + "FOO_BAR", + "Bazzzzz", + "Type.Constructor.Foo.Bar.Baz" + ); + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + private final Random random = new Random(42); + + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + // TODO: Compiler with non-strict errors? + this.context = Utils + .createDefaultContextBuilder() + .option(RuntimeOptions.STRICT_ERRORS, "false") + .logHandler(out) + .out(out) + .err(out) + .build(); + var ensoCtx = Utils.leakEnsoContext(context); + var sb = new StringBuilder(); + var codeGen = new CodeGenerator(); + var definedIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); + sb.append("main = ").append(System.lineSeparator()); + for (String ident : definedIdentifiers) { + sb.append(" ") + .append(ident) + .append(" = ") + .append(codeGen.nextLiteral()) + .append(System.lineSeparator()); + } + for (int i = 0; i < ERRORS_CNT; i++) { + var rndInt = random.nextInt(0, 3); + switch (rndInt) { + // Expression with unknown identifiers + case 0 -> { + var expr = codeGen.createExpression(UNDEFINED_IDENTIFIERS, EXPR_SIZE); + sb.append(" ") + .append(expr) + .append(System.lineSeparator()); + } + // Inline type ascription with unknown identifier + case 1 -> { + var expr = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); + sb.append(" ") + .append(" var : (Type.Constructor.Foo Bar.Baz A B) = ") + .append(expr) + .append(System.lineSeparator()); + } + // Put arrows before, after, and between expressions + case 2 -> { + var expr1 = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); + var expr2 = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); + sb.append(" ") + .append(" -> ") + .append(expr1) + .append(" -> ") + .append(expr2) + .append(" -> ") + .append(System.lineSeparator()); + } + default -> throw new AssertionError("Unexpected random integer: " + rndInt); + } + } + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manyErrorsBenchmark.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + @TearDown + public void tearDown() throws IOException { + // Teardown is called at the end of all the iterations. This means that the number of errors + // reported into `out` should be much bigger than `ERRORS_CNT`. Let's iterate over some lines + // and see if this is true. + var errorsFound = 0; + for (String line : out.toString().split(System.lineSeparator())) { + if (errorsFound == ERRORS_CNT) { + break; + } + if (line.matches(".*: error: .*")) { + errorsFound++; + } + } + if (errorsFound != ERRORS_CNT) { + throw new AssertionError("Expected " + ERRORS_CNT + " errors, but found " + errorsFound); + } + out.close(); + context.close(); + } + + @Benchmark + public void longMethodWithLotOfLocalVars(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index dc70a3f0046f..9bf0f5416da8 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -53,6 +53,7 @@ public class ManyLocalVarsBenchmark { * a new line. */ private static final int IDENTIFIERS_CNT = 40; + private static final int MAX_EXPR_SIZE = 5; private Context context; private Compiler compiler; private Module module; @@ -87,7 +88,7 @@ public void setup(BenchmarkParams params) throws IOException { .skip(1) .forEach( identifier -> { - var maxExprSize = Math.min(5, initializedIdentifiers.size() - 1); + var maxExprSize = Math.min(MAX_EXPR_SIZE, initializedIdentifiers.size() - 1); sb.append(" ") .append(identifier) .append(" = ") From a188e1955cb22f6eae29ddb51d126f8699f90ba6 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 10:58:07 +0100 Subject: [PATCH 08/21] Explicitly specify Fork to ManyErrorsBenchmark. Otherwise, the JMH framework may choose to fork it more than 4 times, which is unnecessary. --- .../enso/compiler/benchmarks/module/ManyErrorsBenchmark.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java index 544a88334b5e..fefd9ed15396 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java @@ -19,6 +19,7 @@ import org.graalvm.polyglot.Source; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; @@ -36,6 +37,7 @@ * and so it should compile the whole module and not stop after the first error. */ @BenchmarkMode(Mode.AverageTime) +@Fork(1) @Warmup(iterations = 6) @Measurement(iterations = 4) @OutputTimeUnit(TimeUnit.MILLISECONDS) @@ -60,7 +62,6 @@ public class ManyErrorsBenchmark { @Setup public void setup(BenchmarkParams params) throws IOException { this.out = new ByteArrayOutputStream(); - // TODO: Compiler with non-strict errors? this.context = Utils .createDefaultContextBuilder() .option(RuntimeOptions.STRICT_ERRORS, "false") From a737cb78d48dd3815b7d1339a3847e59be224571 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 12:49:45 +0100 Subject: [PATCH 09/21] Ensure bench-data dir is created --- .../org/enso/compiler/benchmarks/Utils.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java index 2560edbe94c6..ba110b3c7e91 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -8,7 +8,6 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.logging.Level; import org.enso.interpreter.runtime.EnsoContext; import org.enso.polyglot.LanguageInfo; @@ -20,19 +19,18 @@ public class Utils { public static Context.Builder createDefaultContextBuilder() { - return Context.newBuilder() - .allowExperimentalOptions(true) - .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) - .option(RuntimeOptions.DISABLE_IR_CACHES, "true") - .option(RuntimeOptions.STRICT_ERRORS, "true") - .logHandler(System.err) - .allowIO(IOAccess.ALL) - .allowAllAccess(true); + return Context.newBuilder() + .allowExperimentalOptions(true) + .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .option(RuntimeOptions.STRICT_ERRORS, "true") + .logHandler(System.err) + .allowIO(IOAccess.ALL) + .allowAllAccess(true); } public static EnsoContext leakEnsoContext(Context ctx) { - return ctx - .getBindings(LanguageInfo.ID) + return ctx.getBindings(LanguageInfo.ID) .invokeMember(MethodNames.TopScope.LEAK_CONTEXT) .as(EnsoContext.class); } @@ -46,6 +44,7 @@ public static Object unwrapReceiver(Context ctx, Value value) { public static File createSrcFile(String code, String name) { var benchDataDir = Path.of(".", "target", "bench-data"); + benchDataDir.toFile().mkdirs(); var srcFile = benchDataDir.resolve(name).toFile(); try { Files.writeString(srcFile.toPath(), code); From 33e8edcd6d7698e1cef7f2c7d361000b93d3e1b3 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 12:51:15 +0100 Subject: [PATCH 10/21] CodeGenerator keeps track of defined and used variables --- .../compiler/benchmarks/CodeGenerator.java | 127 ++++++++++++------ .../module/ManyErrorsBenchmark.java | 41 +++--- .../module/ManyLocalVarsBenchmark.java | 81 ++++------- 3 files changed, 126 insertions(+), 123 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java index 2d251b872704..ae243c9e6f6b 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -6,24 +6,78 @@ import java.util.Random; import java.util.Set; -// TODO: Make sure all identifiers are used, so that there is no unused variable warning public class CodeGenerator { - private static final int MAX_IDENT_SIZE = 10; private static final int MAX_TEXT_LITERAL_SIZE = 20; private static final int SMALL_LETTERS_CNT = 26; private static final int SEED = 42; - private static final List OPERATORS = List.of( - "+", "-", "*", "/", "&&", "||", "==", "!=", "<", ">", "<=", ">=" - ); + private static final List OPERATORS = + List.of("+", "-", "*", "/", "&&", "||", "==", "!=", "<", ">", "<=", ">="); - private static final List METHODS = List.of( - "map", "filter", "foldl", "foldr", "head", "tail", "init", "last", "length", "reverse" - ); + private static final List METHODS = + List.of( + "map", "filter", "foldl", "foldr", "head", "tail", "init", "last", "length", "reverse"); + + /** + * A collection of identifiers that were defined. I.e., for these identifiers, there exist + * assignments in the generated code. + */ + private final List definedIdentifiers = new ArrayList<>(); + + /** A collection of identifiers that were used in some expression. */ + private final Set usedIdentifiers = new HashSet<>(); private final Random random = new Random(SEED); private int identifierCnt = 0; + /** + * Creates an expression that initializes a new variable. + * + * @param expressionSize Size of the expression that will be used to initialize the variable. + * @return The assignment expression that defines the new variable + */ + public String defineNewVariable(int expressionSize) { + assert expressionSize >= 0; + var ident = nextIdentifier(); + definedIdentifiers.add(ident); + var initExpr = createExpressionFromDefinedIdentifiers(expressionSize); + return ident + " = " + initExpr; + } + + /** Returns a new identifiers and records it as a defined variable. */ + public String defineNewVariable() { + var ident = nextIdentifier(); + definedIdentifiers.add(ident); + return ident; + } + + /** + * Randomly chooses one of the defined identifiers. + * + * @return The chosen identifier. + */ + public String chooseDefinedIdentifier() { + assert !definedIdentifiers.isEmpty(); + var randomIdx = random.nextInt(definedIdentifiers.size()); + return definedIdentifiers.get(randomIdx); + } + + /** + * Returns a collection of identifiers that are not used in any expression. These identifiers were + * defined, but not used, so the compiler should generate a warning about them. + * + * @return A collection of unused identifiers. + */ + public Set getUnusedIdentifiers() { + var allDefined = new HashSet<>(definedIdentifiers); + allDefined.removeAll(usedIdentifiers); + return allDefined; + } + + public void markVariableAsUsed(String variable) { + assert definedIdentifiers.contains(variable); + usedIdentifiers.add(variable); + } public List createIdentifiers(int count) { List idents = new ArrayList<>(count); @@ -72,24 +126,31 @@ public String nextLiteral() { default -> throw new UnsupportedOperationException("unimplemented"); }; } - public String createExpression(List identifiers, int size) { - return createExpression(identifiers, new HashSet<>(), size); + + /** + * Creates an expression using only the defined identifiers. + * + * @param size Arity of the expression. + */ + public String createExpressionFromDefinedIdentifiers(int size) { + return createExpression(definedIdentifiers, size); } /** * Creates an expression with the given size. + * * @param identifiers A collection of all the identifiers that can be used in the expression. - * @param usedIdentifiers Will be filled with the identifiers used in the expression. + * These may be undefined identifiers. In that case, the compiler should throw an error. * @param size Arity of the expression. * @return A string representing the expression. */ - public String createExpression(List identifiers, Set usedIdentifiers, int size) { + public String createExpression(List identifiers, int size) { switch (size) { - // Literal + // Literal case 0 -> { return nextLiteral(); } - // Either a single identifier or a method call on the identifier + // Either a single identifier or a method call on the identifier case 1 -> { var sb = new StringBuilder(); var ident = chooseIdentifier(identifiers); @@ -97,19 +158,18 @@ public String createExpression(List identifiers, Set usedIdentif sb.append(ident); var shouldCallMethod = random.nextBoolean(); if (shouldCallMethod) { - sb.append(".") - .append(nextMethod()); + sb.append(".").append(nextMethod()); } return sb.toString(); } - // Method call or binary operator + // Method call or binary operator case 2 -> { var sb = new StringBuilder(); var shouldCallMethod = random.nextBoolean(); var ident1 = chooseIdentifier(identifiers); usedIdentifiers.add(ident1); if (shouldCallMethod) { - var methodArg = createExpression(identifiers, usedIdentifiers, 1); + var methodArg = createExpression(identifiers, 1); sb.append(ident1) .append(".") .append(nextMethod()) @@ -120,13 +180,11 @@ public String createExpression(List identifiers, Set usedIdentif var ident2 = chooseIdentifier(identifiers); usedIdentifiers.add(ident2); // Binary operator - sb.append(ident1) - .append(nextOperator()) - .append(ident2); + sb.append(ident1).append(nextOperator()).append(ident2); } return sb.toString(); } - // Split into two expressions with random size + // Split into two expressions with random size default -> { var sb = new StringBuilder(); var shouldCallMethod = random.nextBoolean(); @@ -136,32 +194,20 @@ public String createExpression(List identifiers, Set usedIdentif var methodArity = size - 1; List methodArgs = new ArrayList<>(); for (int i = 0; i < methodArity; i++) { - methodArgs.add( - createExpression(identifiers, usedIdentifiers, size - 1) - ); + methodArgs.add(createExpression(identifiers, size - 1)); } - sb.append(ident) - .append(".") - .append(nextMethod()) - .append(" "); + sb.append(ident).append(".").append(nextMethod()).append(" "); for (var methodArg : methodArgs) { - sb.append(methodArg) - .append(" "); + sb.append(methodArg).append(" "); } } else { var rndIdx = Math.max(2, random.nextInt(size)); var size1 = rndIdx; var size2 = size - rndIdx; - var expr1 = createExpression(identifiers, usedIdentifiers, size1); - var expr2 = createExpression(identifiers, usedIdentifiers, size2); + var expr1 = createExpression(identifiers, size1); + var expr2 = createExpression(identifiers, size2); var op = nextOperator(); - sb.append("(") - .append(expr1) - .append(")") - .append(op) - .append("(") - .append(expr2) - .append(")"); + sb.append("(").append(expr1).append(")").append(op).append("(").append(expr2).append(")"); } return sb.toString(); } @@ -178,5 +224,4 @@ private String chooseIdentifier(List identifiers) { var randomIdx = random.nextInt(identifiers.size()); return identifiers.get(randomIdx); } - } diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java index fefd9ed15396..d8a236d1253b 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java @@ -1,6 +1,5 @@ package org.enso.compiler.benchmarks.module; - import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; @@ -32,9 +31,9 @@ import org.openjdk.jmh.infra.Blackhole; /** - * Measure compilation of a module with one method that contains a lot of errors - - * syntactical errors and unknown identifiers. The compiler should be able to recover from errors - * and so it should compile the whole module and not stop after the first error. + * Measure compilation of a module with one method that contains a lot of errors - syntactical + * errors and unknown identifiers. The compiler should be able to recover from errors and so it + * should compile the whole module and not stop after the first error. */ @BenchmarkMode(Mode.AverageTime) @Fork(1) @@ -46,11 +45,8 @@ public class ManyErrorsBenchmark { private static final int ERRORS_CNT = 30; private static final int IDENTIFIERS_CNT = 40; private static final int EXPR_SIZE = 5; - private static final List UNDEFINED_IDENTIFIERS = List.of( - "FOO_BAR", - "Bazzzzz", - "Type.Constructor.Foo.Bar.Baz" - ); + private static final List UNDEFINED_IDENTIFIERS = + List.of("FOO_BAR", "Bazzzzz", "Type.Constructor.Foo.Bar.Baz"); private Context context; private Compiler compiler; private Module module; @@ -58,17 +54,16 @@ public class ManyErrorsBenchmark { private final Random random = new Random(42); - @Setup public void setup(BenchmarkParams params) throws IOException { this.out = new ByteArrayOutputStream(); - this.context = Utils - .createDefaultContextBuilder() - .option(RuntimeOptions.STRICT_ERRORS, "false") - .logHandler(out) - .out(out) - .err(out) - .build(); + this.context = + Utils.createDefaultContextBuilder() + .option(RuntimeOptions.STRICT_ERRORS, "false") + .logHandler(out) + .out(out) + .err(out) + .build(); var ensoCtx = Utils.leakEnsoContext(context); var sb = new StringBuilder(); var codeGen = new CodeGenerator(); @@ -84,14 +79,12 @@ public void setup(BenchmarkParams params) throws IOException { for (int i = 0; i < ERRORS_CNT; i++) { var rndInt = random.nextInt(0, 3); switch (rndInt) { - // Expression with unknown identifiers + // Expression with unknown identifiers case 0 -> { var expr = codeGen.createExpression(UNDEFINED_IDENTIFIERS, EXPR_SIZE); - sb.append(" ") - .append(expr) - .append(System.lineSeparator()); + sb.append(" ").append(expr).append(System.lineSeparator()); } - // Inline type ascription with unknown identifier + // Inline type ascription with unknown identifier case 1 -> { var expr = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); sb.append(" ") @@ -99,7 +92,7 @@ public void setup(BenchmarkParams params) throws IOException { .append(expr) .append(System.lineSeparator()); } - // Put arrows before, after, and between expressions + // Put arrows before, after, and between expressions case 2 -> { var expr1 = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); var expr2 = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); @@ -147,7 +140,7 @@ public void tearDown() throws IOException { } @Benchmark - public void longMethodWithLotOfLocalVars(Blackhole blackhole) { + public void manyErrors(Blackhole blackhole) { var compilerResult = compiler.run(module.asCompilerModule()); blackhole.consume(compilerResult); } diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index 9bf0f5416da8..b1e1e4110c93 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -3,10 +3,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; +import java.util.Random; import java.util.concurrent.TimeUnit; import org.enso.compiler.Compiler; import org.enso.compiler.benchmarks.CodeGenerator; @@ -32,14 +29,16 @@ /** * Measure compilation of a module with a single long method with a format like: + * * 
  * main =
  *    obj1 = ...
  *    obj2 = ...
  *    obj3 = ...
  *
- * This is the format that is used by the IDE. - * This should measure mostly the performance of the dataflow analysis pass. + * + * This is the format that is used by the IDE. This should measure mostly the performance of the + * dataflow analysis pass. */ @BenchmarkMode(Mode.AverageTime) @Warmup(iterations = 6) @@ -49,11 +48,13 @@ public class ManyLocalVarsBenchmark { /** - * Total count of local variables in the `main` method. Every variable is defined on - * a new line. + * Total count of local variables in the `main` method. Every variable is defined on a new line. */ private static final int IDENTIFIERS_CNT = 40; + private static final int MAX_EXPR_SIZE = 5; + private final CodeGenerator codeGen = new CodeGenerator(); + private final Random random = new Random(42); private Context context; private Compiler compiler; private Module module; @@ -62,62 +63,26 @@ public class ManyLocalVarsBenchmark { @Setup public void setup(BenchmarkParams params) throws IOException { this.out = new ByteArrayOutputStream(); - this.context = Utils - .createDefaultContextBuilder() - .logHandler(out) - .out(out) - .err(out) - .build(); + this.context = Utils.createDefaultContextBuilder().logHandler(out).out(out).err(out).build(); var ensoCtx = Utils.leakEnsoContext(context); var sb = new StringBuilder(); var codeGen = new CodeGenerator(); var allIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); - var firstIdent = allIdentifiers.get(0); - List initializedIdentifiers = new ArrayList<>(); - initializedIdentifiers.add(firstIdent); - sb.append("main = ").append(System.lineSeparator()); - sb.append(" ") - .append(firstIdent) - .append(" = ") - .append("42") - .append(System.lineSeparator()); - - Set usedIdentifiers = new HashSet<>(); - allIdentifiers - .stream() - .skip(1) - .forEach( - identifier -> { - var maxExprSize = Math.min(MAX_EXPR_SIZE, initializedIdentifiers.size() - 1); - sb.append(" ") - .append(identifier) - .append(" = ") - .append(codeGen.createExpression(initializedIdentifiers, usedIdentifiers, maxExprSize)) - .append(System.lineSeparator()); - initializedIdentifiers.add(identifier); - }); - assert initializedIdentifiers.size() == IDENTIFIERS_CNT; - assert usedIdentifiers.size() <= IDENTIFIERS_CNT; - if (usedIdentifiers.size() < IDENTIFIERS_CNT) { - // Add a final line that uses the rest of the identifiers, so that there is no "Unused binding" - // warning. - List unusedIdentifiers = new ArrayList<>(allIdentifiers); - unusedIdentifiers.removeAll(usedIdentifiers); - sb.append(" ") - .append("result = "); - sb.append( - unusedIdentifiers - .stream() - .reduce((a, b) -> a + " + " + b) - .orElseThrow() - ); - sb.append(System.lineSeparator()); - // result is the return value from the main method - sb.append(" ") - .append("result") - .append(System.lineSeparator()); + for (int i = 0; i < IDENTIFIERS_CNT; i++) { + int exprSize = random.nextInt(0, MAX_EXPR_SIZE); + var assignmentExpr = codeGen.defineNewVariable(exprSize); + sb.append(" ").append(assignmentExpr).append(System.lineSeparator()); } + + // Add a final line that uses the rest of the identifiers, so that there is no "Unused binding" + // warning. + sb.append(" ").append("result = "); + sb.append( + codeGen.getUnusedIdentifiers().stream().reduce((a, b) -> a + " + " + b).orElseThrow()); + sb.append(System.lineSeparator()); + // result is the return value from the main method + sb.append(" ").append("result").append(System.lineSeparator()); var code = sb.toString(); var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); From 1ab1fc88432f5cf89dd0f7576160723f27ba679a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 12:53:57 +0100 Subject: [PATCH 11/21] Created ManyNestedBlocksBenchmark --- .../module/ManyNestedBlocksBenchmark.java | 202 ++++++++++++++++++ 1 file changed, 202 insertions(+) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java new file mode 100644 index 000000000000..e6a28f480493 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java @@ -0,0 +1,202 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** Measure compilation of a module with one method that contains a lot of nested blocks. */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManyNestedBlocksBenchmark { + + /** Maximum nesting. */ + private static final int NESTED_BLOCKS_CNT = 40; + + /** + * Number of cases in a case expression. Only the last case contains nested blocks, the rest are + * simple literals. + */ + private static final int CASES_CNT = 5; + + /** Number of lines per block. */ + private static final int BLOCK_SIZE = 5; + + /** How many new identifiers will be defined in each block. */ + private static final int IDENTIFIERS_CNT = 2; + + /** Maximum size of an expression on a single line. */ + private static final int MAX_EXPR_SIZE = 5; + + /** Maximum number of arguments of an anonymous function */ + private final int ANONYMOUS_FUNC_MAX_ARGS = 5; + + private final Random random = new Random(42); + private final CodeGenerator codeGen = new CodeGenerator(); + private final StringBuilder sb = new StringBuilder(); + + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = Utils.createDefaultContextBuilder().logHandler(out).out(out).err(out).build(); + var ensoCtx = Utils.leakEnsoContext(context); + sb.append("main = ").append(System.lineSeparator()); + createNestedBlocks(1); + + assert codeGen.getUnusedIdentifiers().isEmpty() + : "All variables should be used in simple blocks"; + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manyNestedBlocks.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + /** + * All the unused variables are used in the last expression in the block + * + * @param nestedBlockIdx Nesting index. + */ + private void createSimpleBlock(int nestedBlockIdx) { + var spaces = " ".repeat(nestedBlockIdx); + var exprsCnt = random.nextInt(1, BLOCK_SIZE); + var exprSize = random.nextInt(0, MAX_EXPR_SIZE); + for (int i = 0; i < IDENTIFIERS_CNT; i++) { + var newVar = codeGen.defineNewVariable(); + sb.append(spaces) + .append(newVar) + .append(" = ") + .append(codeGen.nextLiteral()) + .append(System.lineSeparator()); + } + for (int i = 0; i < exprsCnt; i++) { + var expr = codeGen.createExpressionFromDefinedIdentifiers(exprSize); + sb.append(spaces).append(expr).append(System.lineSeparator()); + } + Set unusedIdentifiers = codeGen.getUnusedIdentifiers(); + var lastExpr = + unusedIdentifiers.stream().reduce((acc, ident) -> acc + " + " + ident).orElse("42"); + sb.append(spaces).append(lastExpr).append(System.lineSeparator()); + unusedIdentifiers.forEach(codeGen::markVariableAsUsed); + } + + private void createNestedBlocks(int nestedBlockIdx) { + if (nestedBlockIdx >= NESTED_BLOCKS_CNT) { + return; + } + createSimpleBlock(nestedBlockIdx); + // Introduce a new nested block at the end of the current block + if (nestedBlockIdx < NESTED_BLOCKS_CNT - 1) { + var rndInt = random.nextInt(0, 4); + var spaces = " ".repeat(nestedBlockIdx); + switch (rndInt) { + case 0 -> { + var ident = codeGen.chooseDefinedIdentifier(); + sb.append(spaces) + .append("case ") + .append(ident) + .append(" of") + .append(System.lineSeparator()); + // Only the last case expression contains nested blocks + for (int i = 0; i < CASES_CNT - 1; i++) { + sb.append(spaces) + .append(" ") + .append(codeGen.nextLiteral()) + .append(" -> ") + .append(codeGen.nextLiteral()) + .append(System.lineSeparator()); + } + sb.append(spaces) + .append(" ") + .append(codeGen.nextLiteral()) + .append(" -> ") + .append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 2); + } + case 1 -> { + var ident = codeGen.chooseDefinedIdentifier(); + sb.append(spaces) + .append("if ") + .append(ident) + .append(" == 42 then 42 else") + .append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 1); + } + case 2 -> { + var ident = codeGen.chooseDefinedIdentifier(); + sb.append(spaces) + .append("if ") + .append(ident) + .append(" == 42 then") + .append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 1); + } + // Create a nested anonymous function + case 3 -> { + var argsCnt = random.nextInt(1, ANONYMOUS_FUNC_MAX_ARGS); + sb.append(spaces); + for (int i = 0; i < argsCnt; i++) { + var newIdent = codeGen.defineNewVariable(); + sb.append(newIdent).append("-> "); + } + sb.append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 1); + } + default -> throw new IllegalStateException("Unexpected random int: " + rndInt); + } + } + } + + @TearDown + public void tearDown() throws IOException { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out.toString()); + } + out.close(); + context.close(); + } + + @Benchmark + public void manyNestedBlocks(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} From a62396c87c1c8b1cca48be8d380b9b0d0c2a678d Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 17:03:33 +0100 Subject: [PATCH 12/21] Add ManySmallMethodsBenchmark --- .../compiler/benchmarks/CodeGenerator.java | 37 +++-- .../module/ManyLocalVarsBenchmark.java | 1 - .../module/ManySmallMethodsBenchmark.java | 150 ++++++++++++++++++ 3 files changed, 177 insertions(+), 11 deletions(-) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java index ae243c9e6f6b..37736f9c16ec 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -1,6 +1,7 @@ package org.enso.compiler.benchmarks; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Random; @@ -30,6 +31,21 @@ public class CodeGenerator { private final Random random = new Random(SEED); private int identifierCnt = 0; + /** + * Creates a new empty code generator. + */ + public CodeGenerator() { + + } + + /** + * + * @param identifiers Collection of already defined identifiers, like arguments to a function. + */ + public CodeGenerator(Collection identifiers) { + definedIdentifiers.addAll(identifiers); + } + /** * Creates an expression that initializes a new variable. * @@ -145,11 +161,12 @@ public String createExpressionFromDefinedIdentifiers(int size) { * @return A string representing the expression. */ public String createExpression(List identifiers, int size) { - switch (size) { - // Literal - case 0 -> { - return nextLiteral(); - } + if (identifiers.isEmpty()) { + return nextLiteral(); + } + + return switch (size) { + case 0 -> nextLiteral(); // Either a single identifier or a method call on the identifier case 1 -> { var sb = new StringBuilder(); @@ -160,9 +177,9 @@ public String createExpression(List identifiers, int size) { if (shouldCallMethod) { sb.append(".").append(nextMethod()); } - return sb.toString(); + yield sb.toString(); } - // Method call or binary operator + // Method call or binary operator case 2 -> { var sb = new StringBuilder(); var shouldCallMethod = random.nextBoolean(); @@ -182,7 +199,7 @@ public String createExpression(List identifiers, int size) { // Binary operator sb.append(ident1).append(nextOperator()).append(ident2); } - return sb.toString(); + yield sb.toString(); } // Split into two expressions with random size default -> { @@ -209,9 +226,9 @@ public String createExpression(List identifiers, int size) { var op = nextOperator(); sb.append("(").append(expr1).append(")").append(op).append("(").append(expr2).append(")"); } - return sb.toString(); + yield sb.toString(); } - } + }; } private String nextOperator() { diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index b1e1e4110c93..46b558d2127c 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -53,7 +53,6 @@ public class ManyLocalVarsBenchmark { private static final int IDENTIFIERS_CNT = 40; private static final int MAX_EXPR_SIZE = 5; - private final CodeGenerator codeGen = new CodeGenerator(); private final Random random = new Random(42); private Context context; private Compiler compiler; diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java new file mode 100644 index 000000000000..62b0087332e1 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java @@ -0,0 +1,150 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measure compilation of a module with a lot of small methods with variable number of arguments. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManySmallMethodsBenchmark { + + private static final int METHODS_CNT = 30; + + /** Maximum number of arguments of a single method. */ + private static final int MAX_ARGS = 10; + + /** Maximum number of local variables per method */ + private static final int MAX_LOCAL_VARS = 5; + + /** Maximum number of expressions per method, excluding new variable definitions. */ + private static final int MAX_METHOD_SIZE = 7; + + /** Maximum arity of an expression in method's body. */ + private static final int MAX_EXPR_SIZE = 5; + + private final Random random = new Random(42); + private final StringBuilder sb = new StringBuilder(); + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = Utils.createDefaultContextBuilder().logHandler(out).out(out).err(out).build(); + var ensoCtx = Utils.leakEnsoContext(context); + List methods = new ArrayList<>(); + + for (int methodIdx = 0; methodIdx < METHODS_CNT; methodIdx++) { + methods.add(createMethod(methodIdx)); + } + + sb.append(System.lineSeparator()); + sb.append("main = ").append(System.lineSeparator()); + // Make sure that every method is called, with correct number of arguments + var codeGen = new CodeGenerator(); + for (var method : methods) { + sb.append(" ").append(method.name); + for (int i = 0; i < method.argCount; i++) { + sb.append(" ").append(codeGen.nextLiteral()); + } + sb.append(System.lineSeparator()); + } + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manySmallMethods.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + private Method createMethod(int methodIdx) { + var methodName = "method_" + methodIdx; + sb.append(methodName).append(" "); + var argCount = random.nextInt(0, MAX_ARGS + 1); + Set args = new HashSet<>(); + for (int argIdx = 0; argIdx < argCount; argIdx++) { + var argName = "arg_" + argIdx; + sb.append(argName).append(" "); + args.add(argName); + } + sb.append(" = ").append(System.lineSeparator()); + var codeGen = new CodeGenerator(args); + var localVarsCnt = random.nextInt(0, MAX_LOCAL_VARS); + for (int i = 0; i < localVarsCnt; i++) { + sb.append(" ") + .append(codeGen.defineNewVariable(MAX_EXPR_SIZE)) + .append(System.lineSeparator()); + } + var methodSize = random.nextInt(0, MAX_METHOD_SIZE); + for (int i = 0; i < methodSize; i++) { + var exprSize = random.nextInt(0, MAX_EXPR_SIZE); + var expr = codeGen.createExpressionFromDefinedIdentifiers(exprSize); + sb.append(" ").append(expr).append(System.lineSeparator()); + } + var lastExpr = + codeGen.getUnusedIdentifiers().stream() + .reduce((acc, ident) -> acc + " + " + ident) + .orElse("42"); + sb.append(" ").append(lastExpr).append(System.lineSeparator()); + sb.append(System.lineSeparator()); + return new Method(methodName, argCount); + } + + @TearDown + public void tearDown() throws IOException { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out.toString()); + } + out.close(); + context.close(); + } + + @Benchmark + public void manySmallMethods(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } + + private record Method(String name, int argCount) {} +} From 73777025e959cd924c2ba3f0f8f4b211842f476b Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 17:10:46 +0100 Subject: [PATCH 13/21] Remove unused ModuleCompilerBenchmarks. Every benchmark is now in a separate class --- .../benchmarks/ModuleCompilerBenchmarks.java | 87 ------------------- 1 file changed, 87 deletions(-) delete mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java deleted file mode 100644 index 1798821c0df5..000000000000 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/ModuleCompilerBenchmarks.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.enso.compiler.benchmarks; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.enso.compiler.Compiler; -import org.enso.interpreter.runtime.Module; -import org.enso.polyglot.LanguageInfo; -import org.enso.polyglot.MethodNames; -import org.enso.interpreter.runtime.data.Type; -import org.graalvm.polyglot.Source; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.infra.Blackhole; - -/** - * Measures how long does it take for the compiler passes to run on a various modules. - */ -@BenchmarkMode(Mode.AverageTime) -@Fork(0) -@Warmup(iterations = 1) -@Measurement(iterations = 1) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Benchmark) -public class ModuleCompilerBenchmarks { - private Compiler compiler; - private Module module; - - @Setup - public void setup(BenchmarkParams params) throws IOException { - var ctx = Utils.createDefaultContext(); - var ensoCtx = Utils.leakEnsoContext(ctx); - switch (params.getBenchmark()) { - - default -> { - throw new UnsupportedOperationException("unimplemented: Benchmark " + params.getBenchmark()); - } - } - } - - - /** - * Measure compilation of a module with a lot of small methods with - * variable number of arguments. - * @param blackhole - */ - @Benchmark - public void manySmallMethods(Blackhole blackhole) { - var compilerResult = compiler.run(module.asCompilerModule()); - if (compilerResult.compiledModules().size() != 1) { - throw new AssertionError("Module compilation failed"); - } - } - - /** - * Measure compilation of a module with one method that contains a lot of - * nested blocks. - * @param blackhole - */ - @Benchmark - public void manyNestedBlocks(Blackhole blackhole) { - throw new UnsupportedOperationException("unimplemented"); - } - - /** - * Measure compilation of a module with one method that contains a lot of errors - - * syntactical errors and unknown identifiers. The compiler should be able to recover from errors - * and so it should compile the whole module and not stop after the first error. - * @param blackhole - */ - @Benchmark - public void manyErrors(Blackhole blackhole) { - throw new UnsupportedOperationException("unimplemented"); - } -} From c4edb0bf6ee787f3153fb00df2f3edcdfb27692b Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 17:10:57 +0100 Subject: [PATCH 14/21] fmt --- .../org/enso/compiler/benchmarks/CodeGenerator.java | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java index 37736f9c16ec..5e6b86b1742f 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -31,15 +31,10 @@ public class CodeGenerator { private final Random random = new Random(SEED); private int identifierCnt = 0; - /** - * Creates a new empty code generator. - */ - public CodeGenerator() { - - } + /** Creates a new empty code generator. */ + public CodeGenerator() {} /** - * * @param identifiers Collection of already defined identifiers, like arguments to a function. */ public CodeGenerator(Collection identifiers) { @@ -179,7 +174,7 @@ public String createExpression(List identifiers, int size) { } yield sb.toString(); } - // Method call or binary operator + // Method call or binary operator case 2 -> { var sb = new StringBuilder(); var shouldCallMethod = random.nextBoolean(); From d9c3fb606c0eb9703111db7f582836f43be9d674 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 17:20:02 +0100 Subject: [PATCH 15/21] Finetune parameters of ManyLocalVarsBenchmark --- .../benchmarks/module/ManyLocalVarsBenchmark.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index 46b558d2127c..56dd06875623 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -16,6 +16,7 @@ import org.graalvm.polyglot.Source; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.annotations.OutputTimeUnit; @@ -41,6 +42,7 @@ * dataflow analysis pass. */ @BenchmarkMode(Mode.AverageTime) +@Fork(1) @Warmup(iterations = 6) @Measurement(iterations = 4) @OutputTimeUnit(TimeUnit.MILLISECONDS) @@ -50,7 +52,7 @@ public class ManyLocalVarsBenchmark { /** * Total count of local variables in the `main` method. Every variable is defined on a new line. */ - private static final int IDENTIFIERS_CNT = 40; + private static final int IDENTIFIERS_CNT = 100; private static final int MAX_EXPR_SIZE = 5; private final Random random = new Random(42); @@ -66,8 +68,9 @@ public void setup(BenchmarkParams params) throws IOException { var ensoCtx = Utils.leakEnsoContext(context); var sb = new StringBuilder(); var codeGen = new CodeGenerator(); - var allIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); + sb.append("main =") + .append(System.lineSeparator()); for (int i = 0; i < IDENTIFIERS_CNT; i++) { int exprSize = random.nextInt(0, MAX_EXPR_SIZE); var assignmentExpr = codeGen.defineNewVariable(exprSize); @@ -83,7 +86,7 @@ public void setup(BenchmarkParams params) throws IOException { // result is the return value from the main method sb.append(" ").append("result").append(System.lineSeparator()); var code = sb.toString(); - var srcFile = Utils.createSrcFile(code, "longMethodWithLotOfLocalVars.enso"); + var srcFile = Utils.createSrcFile(code, "manyLocalVars.enso"); var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); var module = context.eval(src); var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); @@ -105,9 +108,6 @@ public void tearDown() throws IOException { @Benchmark public void longMethodWithLotOfLocalVars(Blackhole blackhole) { var compilerResult = compiler.run(module.asCompilerModule()); - if (compilerResult.compiledModules().size() != 1) { - throw new AssertionError("Module compilation failed"); - } blackhole.consume(compilerResult); } } From 23434b816b57865ab32690c99c6eb7371ed87c0a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 27 Feb 2024 18:15:34 +0100 Subject: [PATCH 16/21] Fix InlineCompilerBenchmarks --- .../benchmarks/InlineCompilerBenchmarks.java | 103 ----------- .../inline/InlineCompilerBenchmarks.java | 160 ++++++++++++++++++ 2 files changed, 160 insertions(+), 103 deletions(-) delete mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java deleted file mode 100644 index 4a52896b687b..000000000000 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/InlineCompilerBenchmarks.java +++ /dev/null @@ -1,103 +0,0 @@ -package org.enso.compiler.benchmarks; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import org.enso.compiler.context.InlineContext; -import org.enso.interpreter.node.MethodRootNode; -import org.enso.interpreter.runtime.EnsoContext; -import org.enso.interpreter.runtime.data.Type; -import org.enso.polyglot.LanguageInfo; -import org.enso.polyglot.MethodNames.Module; -import org.enso.polyglot.MethodNames.TopScope; -import org.enso.polyglot.RuntimeOptions; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.Source; -import org.graalvm.polyglot.io.IOAccess; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; - -/** - * Measures the inline compilation, that is the compilation that is requested inside a method. - * Simulates a scenario where there is an existing method and we are trying to insert a new - * expression into it. - */ -@BenchmarkMode(Mode.AverageTime) -@Fork(0) -@Warmup(iterations = 1) -@Measurement(iterations = 1) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Benchmark) -public class InlineCompilerBenchmarks { - @Setup - public void setup() throws IOException { - var ctx = - Context.newBuilder() - .allowExperimentalOptions(true) - .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) - .option(RuntimeOptions.DISABLE_IR_CACHES, "true") - .option(RuntimeOptions.STRICT_ERRORS, "true") - .logHandler(System.err) - .allowIO(IOAccess.ALL) - .allowAllAccess(true) - .option( - RuntimeOptions.LANGUAGE_HOME_OVERRIDE, - Paths.get("../../distribution/component").toFile().getAbsolutePath()) - .build(); - var ensoCtx = ctx - .getBindings(LanguageInfo.ID) - .invokeMember(TopScope.LEAK_CONTEXT) - .as(EnsoContext.class); - var code = """ - main = 1 + 1 - """; - var srcFile = createSrcFile(code, "bench-1.enso"); - var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); - var module = ctx.eval(src); - var assocType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); - var assocTypeReceiver = (Type) Utils.unwrapReceiver(ctx, assocType); - var moduleScope = assocTypeReceiver.getDefinitionScope(); - var mainFunc = moduleScope.getMethodForType(assocTypeReceiver, "main"); - var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); - var localScope = mainFuncRootNode.getLocalScope(); - var compiler = ensoCtx.getCompiler(); - InlineContext inlineContext = - InlineContext.fromJava( - localScope, - moduleScope.getModule().asCompilerModule(), - scala.Option.apply(false), - ensoCtx.getCompilerConfig(), - scala.Option.apply(compiler.packageRepository())); - var inlineExpr = "42 * 2"; - var tuppleOpt = compiler.runInline(inlineExpr, inlineContext); - assert tuppleOpt.isDefined(); - var newInlineContext = tuppleOpt.get()._1(); - var ir = tuppleOpt.get()._2(); - var newSrc = tuppleOpt.get()._3(); - - // - System.out.println("Done bench init"); - } - - private static File createSrcFile(String code, String name) { - var benchDataDir = Path.of(".", "target", "bench-data"); - var srcFile = benchDataDir.resolve(name).toFile(); - try { - Files.writeString(srcFile.toPath(), code); - } catch (IOException e) { - throw new AssertionError(e); - } - return srcFile; - } -} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java new file mode 100644 index 000000000000..b631a85dcba8 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java @@ -0,0 +1,160 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.enso.interpreter.node.MethodRootNode; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames.Module; +import org.enso.polyglot.MethodNames.TopScope; +import org.enso.polyglot.RuntimeOptions; +import org.enso.polyglot.runtime.Runtime; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.io.IOAccess; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the inline compilation, that is the compilation that is requested inside a method. + * Simulates a scenario where there is an existing method and we are trying to insert a new + * expression into it. + *

+ * There is just a single `main` method that contains {@link #LOCAL_VARS_CNT} local variables. + * One benchmark measures the time it takes to inline compile a long expression that uses all the + * local variables. The other benchmark measures the time it takes to inline compile an expression + * that contains some undefined identifiers and thus, should fail to compile. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(0) +@Warmup(iterations = 1) +@Measurement(iterations = 1) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class InlineCompilerBenchmarks { + private static final int LONG_EXPR_SIZE = 5; + + private final OutputStream out = new ByteArrayOutputStream(); + /** + * How many variables should be declared in the main method. + */ + private static final int LOCAL_VARS_CNT = 40; + private Compiler compiler; + private InlineContext mainInlineContext; + private String longExpression; + private String expressionWithErrors; + + @Setup + public void setup() throws IOException { + var ctx = Utils.createDefaultContextBuilder() + .out(out) + .err(out) + .logHandler(out) + .option(RuntimeOptions.STRICT_ERRORS, "false") + .build(); + + var sb = new StringBuilder(); + sb.append("main = ") + .append(System.lineSeparator()); + var codeGen = new CodeGenerator(); + Set localVarNames = new HashSet<>(); + for (int i = 0; i < LOCAL_VARS_CNT; i++) { + var varName = "loc_var_" + i; + localVarNames.add(varName); + var literal = codeGen.nextLiteral(); + sb.append(" ") + .append(varName) + .append(" = ") + .append(literal) + .append(System.lineSeparator()); + } + // In the last expression of main method, use all the variables, so that there is no unused + // variable warning + var lastExpr = localVarNames.stream() + .reduce((acc, varName) -> acc + " + " + varName) + .orElseThrow(); + sb.append(" ") + .append(lastExpr) + .append(System.lineSeparator()); + + + var ensoCtx = Utils.leakEnsoContext(ctx); + var srcFile = Utils.createSrcFile(sb.toString(), "inlineBenchmark.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = ctx.eval(src); + var moduleAssocType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); + var assocTypeReceiver = (Type) Utils.unwrapReceiver(ctx, moduleAssocType); + var moduleScope = assocTypeReceiver.getDefinitionScope(); + var mainFunc = moduleScope.getMethodForType(assocTypeReceiver, "main"); + var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); + var mainLocalScope = mainFuncRootNode.getLocalScope(); + compiler = ensoCtx.getCompiler(); + mainInlineContext = InlineContext.fromJava( + mainLocalScope, + moduleScope.getModule().asCompilerModule(), + scala.Option.apply(false), + ensoCtx.getCompilerConfig(), + scala.Option.apply(compiler.packageRepository())); + longExpression = createLongExpression(localVarNames); + expressionWithErrors = "UNDEFINED * " + longExpression + " * UNDEFINED"; + } + + private String createLongExpression(Set localVars) { + var codeGen = new CodeGenerator(localVars); + return codeGen.createExpressionFromDefinedIdentifiers(LONG_EXPR_SIZE); + } + + @TearDown + public void tearDown() { + try { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out); + } + out.close(); + } catch (IOException e) { + throw new AssertionError("Failed to close the output stream", e); + } + } + + @Benchmark + public void longExpression(Blackhole blackhole) { + var tuppleOpt = compiler.runInline(longExpression, mainInlineContext); + if (tuppleOpt.isEmpty()) { + throw new AssertionError("Unexpected: inline compilation should succeed"); + } + blackhole.consume(tuppleOpt); + } + + @Benchmark + public void expressionWithErrors() { + var tuppleOpt = compiler.runInline(expressionWithErrors, mainInlineContext); + if (tuppleOpt.isDefined()) { + throw new AssertionError("Unexpected: inline compilation should fail"); + } + } + +} From 52a35efe95546b80c951148a48ab8193e0fe0d55 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 29 Feb 2024 18:23:31 +0100 Subject: [PATCH 17/21] Add inline compiler benchmark for errors --- .../inline/InlineCompilerBenchmark.java | 92 ++++++++++ .../inline/InlineCompilerBenchmarks.java | 160 ------------------ .../inline/InlineCompilerErrorBenchmark.java | 86 ++++++++++ .../benchmarks/inline/InlineContextUtils.java | 74 ++++++++ .../benchmarks/inline/InlineSource.java | 12 ++ 5 files changed, 264 insertions(+), 160 deletions(-) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java delete mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java new file mode 100644 index 000000000000..81db6851b043 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java @@ -0,0 +1,92 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the inline compilation, that is the compilation that is requested inside a method. + * Simulates a scenario where there is an existing method and we are trying to insert a new + * expression into it. + *

+ * There is just a single `main` method that contains {@link #LOCAL_VARS_CNT} local variables. + * One benchmark measures the time it takes to inline compile a long expression that uses all the + * local variables. The other benchmark measures the time it takes to inline compile an expression + * that contains some undefined identifiers and thus, should fail to compile. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class InlineCompilerBenchmark { + /** + * How many variables should be declared in the main method. + */ + private static final int LOCAL_VARS_CNT = 40; + private static final int LONG_EXPR_SIZE = 5; + + private final OutputStream out = new ByteArrayOutputStream(); + private Compiler compiler; + private Context ctx; + private InlineContext mainInlineContext; + private String longExpression; + + @Setup + public void setup() throws IOException { + ctx = Utils.createDefaultContextBuilder() + .out(out) + .err(out) + .logHandler(out) + .build(); + var ensoCtx = Utils.leakEnsoContext(ctx); + compiler = ensoCtx.getCompiler(); + + var inlineSource = InlineContextUtils.createMainMethodWithLocalVars(ctx, LOCAL_VARS_CNT); + mainInlineContext = inlineSource.mainInlineContext(); + longExpression = InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); + } + + @TearDown + public void tearDown() { + ctx.close(); + try { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out); + } + out.close(); + } catch (IOException e) { + throw new AssertionError("Failed to close the output stream", e); + } + } + + @Benchmark + public void longExpression(Blackhole blackhole) { + var tuppleOpt = compiler.runInline(longExpression, mainInlineContext); + if (tuppleOpt.isEmpty()) { + throw new AssertionError("Unexpected: inline compilation should succeed"); + } + blackhole.consume(tuppleOpt); + } + + +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java deleted file mode 100644 index b631a85dcba8..000000000000 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmarks.java +++ /dev/null @@ -1,160 +0,0 @@ -package org.enso.compiler.benchmarks.inline; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import org.enso.compiler.Compiler; -import org.enso.compiler.benchmarks.CodeGenerator; -import org.enso.compiler.benchmarks.Utils; -import org.enso.compiler.context.InlineContext; -import org.enso.interpreter.node.MethodRootNode; -import org.enso.interpreter.runtime.EnsoContext; -import org.enso.interpreter.runtime.data.Type; -import org.enso.polyglot.LanguageInfo; -import org.enso.polyglot.MethodNames.Module; -import org.enso.polyglot.MethodNames.TopScope; -import org.enso.polyglot.RuntimeOptions; -import org.enso.polyglot.runtime.Runtime; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.Source; -import org.graalvm.polyglot.io.IOAccess; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -/** - * Measures the inline compilation, that is the compilation that is requested inside a method. - * Simulates a scenario where there is an existing method and we are trying to insert a new - * expression into it. - *

- * There is just a single `main` method that contains {@link #LOCAL_VARS_CNT} local variables. - * One benchmark measures the time it takes to inline compile a long expression that uses all the - * local variables. The other benchmark measures the time it takes to inline compile an expression - * that contains some undefined identifiers and thus, should fail to compile. - */ -@BenchmarkMode(Mode.AverageTime) -@Fork(0) -@Warmup(iterations = 1) -@Measurement(iterations = 1) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Benchmark) -public class InlineCompilerBenchmarks { - private static final int LONG_EXPR_SIZE = 5; - - private final OutputStream out = new ByteArrayOutputStream(); - /** - * How many variables should be declared in the main method. - */ - private static final int LOCAL_VARS_CNT = 40; - private Compiler compiler; - private InlineContext mainInlineContext; - private String longExpression; - private String expressionWithErrors; - - @Setup - public void setup() throws IOException { - var ctx = Utils.createDefaultContextBuilder() - .out(out) - .err(out) - .logHandler(out) - .option(RuntimeOptions.STRICT_ERRORS, "false") - .build(); - - var sb = new StringBuilder(); - sb.append("main = ") - .append(System.lineSeparator()); - var codeGen = new CodeGenerator(); - Set localVarNames = new HashSet<>(); - for (int i = 0; i < LOCAL_VARS_CNT; i++) { - var varName = "loc_var_" + i; - localVarNames.add(varName); - var literal = codeGen.nextLiteral(); - sb.append(" ") - .append(varName) - .append(" = ") - .append(literal) - .append(System.lineSeparator()); - } - // In the last expression of main method, use all the variables, so that there is no unused - // variable warning - var lastExpr = localVarNames.stream() - .reduce((acc, varName) -> acc + " + " + varName) - .orElseThrow(); - sb.append(" ") - .append(lastExpr) - .append(System.lineSeparator()); - - - var ensoCtx = Utils.leakEnsoContext(ctx); - var srcFile = Utils.createSrcFile(sb.toString(), "inlineBenchmark.enso"); - var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); - var module = ctx.eval(src); - var moduleAssocType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); - var assocTypeReceiver = (Type) Utils.unwrapReceiver(ctx, moduleAssocType); - var moduleScope = assocTypeReceiver.getDefinitionScope(); - var mainFunc = moduleScope.getMethodForType(assocTypeReceiver, "main"); - var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); - var mainLocalScope = mainFuncRootNode.getLocalScope(); - compiler = ensoCtx.getCompiler(); - mainInlineContext = InlineContext.fromJava( - mainLocalScope, - moduleScope.getModule().asCompilerModule(), - scala.Option.apply(false), - ensoCtx.getCompilerConfig(), - scala.Option.apply(compiler.packageRepository())); - longExpression = createLongExpression(localVarNames); - expressionWithErrors = "UNDEFINED * " + longExpression + " * UNDEFINED"; - } - - private String createLongExpression(Set localVars) { - var codeGen = new CodeGenerator(localVars); - return codeGen.createExpressionFromDefinedIdentifiers(LONG_EXPR_SIZE); - } - - @TearDown - public void tearDown() { - try { - if (!out.toString().isEmpty()) { - throw new AssertionError("Unexpected output from the compiler: " + out); - } - out.close(); - } catch (IOException e) { - throw new AssertionError("Failed to close the output stream", e); - } - } - - @Benchmark - public void longExpression(Blackhole blackhole) { - var tuppleOpt = compiler.runInline(longExpression, mainInlineContext); - if (tuppleOpt.isEmpty()) { - throw new AssertionError("Unexpected: inline compilation should succeed"); - } - blackhole.consume(tuppleOpt); - } - - @Benchmark - public void expressionWithErrors() { - var tuppleOpt = compiler.runInline(expressionWithErrors, mainInlineContext); - if (tuppleOpt.isDefined()) { - throw new AssertionError("Unexpected: inline compilation should fail"); - } - } - -} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java new file mode 100644 index 000000000000..5fc56c272c60 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java @@ -0,0 +1,86 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * + * Measures the inline compilation, that is the compilation that is requested inside a method. + * Simulates a scenario where there is an existing method and we are trying to insert a new + * expression into it. + *

+ * The expression inserted into the method contains some undefined identifiers and thus, should fail + * to compile. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class InlineCompilerErrorBenchmark { + /** + * How many variables should be declared in the main method. + */ + private static final int LOCAL_VARS_CNT = 40; + private static final int LONG_EXPR_SIZE = 5; + + private final OutputStream out = new ByteArrayOutputStream(); + /** + * How many variables should be declared in the main method. + */ + private Compiler compiler; + private Context ctx; + private InlineContext mainInlineContext; + private String expressionWithErrors; + + @Setup + public void setup() throws IOException { + ctx = Utils.createDefaultContextBuilder() + .out(out) + .err(out) + .logHandler(out) + .option(RuntimeOptions.STRICT_ERRORS, "false") + .build(); + var ensoCtx = Utils.leakEnsoContext(ctx); + compiler = ensoCtx.getCompiler(); + + var inlineSource = InlineContextUtils.createMainMethodWithLocalVars(ctx, LOCAL_VARS_CNT); + var longExpression = InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); + expressionWithErrors = "UNDEFINED * " + longExpression + " * UNDEFINED"; + mainInlineContext = inlineSource.mainInlineContext(); + } + + @TearDown + public void teardown() { + ctx.close(); + if (out.toString().isEmpty()) { + throw new AssertionError("Expected some output (some errors) from the compiler"); + } + } + + @Benchmark + public void expressionWithErrors(Blackhole blackhole) { + var tuppleOpt = compiler.runInline(expressionWithErrors, mainInlineContext); + blackhole.consume(tuppleOpt); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java new file mode 100644 index 000000000000..d84b08526613 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java @@ -0,0 +1,74 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.enso.interpreter.node.MethodRootNode; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; + +class InlineContextUtils { + private InlineContextUtils() {} + + /** + * Creates a main method, generates some local variables, and fills their identifiers in the given + * set. + * + * @param localVarsCnt How many local variables should be initialized in the main method + * @return Body of the main method + */ + static InlineSource createMainMethodWithLocalVars(Context ctx, int localVarsCnt) throws IOException { + var sb = new StringBuilder(); + sb.append("main = ").append(System.lineSeparator()); + var codeGen = new CodeGenerator(); + Set localVarNames = new HashSet<>(); + for (int i = 0; i < localVarsCnt; i++) { + var varName = "loc_var_" + i; + localVarNames.add(varName); + var literal = codeGen.nextLiteral(); + sb.append(" ") + .append(varName) + .append(" = ") + .append(literal) + .append(System.lineSeparator()); + } + // In the last expression of main method, use all the variables, so that there is no unused + // variable warning + var lastExpr = + localVarNames.stream().reduce((acc, varName) -> acc + " + " + varName).orElseThrow(); + sb.append(" ").append(lastExpr).append(System.lineSeparator()); + var ensoCtx = Utils.leakEnsoContext(ctx); + var srcFile = Utils.createSrcFile(sb.toString(), "inlineBenchmark.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = ctx.eval(src); + var moduleAssocType = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocTypeReceiver = (Type) Utils.unwrapReceiver(ctx, moduleAssocType); + var moduleScope = assocTypeReceiver.getDefinitionScope(); + var mainFunc = moduleScope.getMethodForType(assocTypeReceiver, "main"); + var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); + var mainLocalScope = mainFuncRootNode.getLocalScope(); + var compiler = ensoCtx.getCompiler(); + var mainInlineContext = InlineContext.fromJava( + mainLocalScope, + moduleScope.getModule().asCompilerModule(), + scala.Option.apply(false), + ensoCtx.getCompilerConfig(), + scala.Option.apply(compiler.packageRepository())); + return new InlineSource( + sb.toString(), + mainInlineContext, + localVarNames + ); + } + + static String createLongExpression(Set localVars, int exprSize) { + var codeGen = new CodeGenerator(localVars); + return codeGen.createExpressionFromDefinedIdentifiers(exprSize); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java new file mode 100644 index 000000000000..a81ef6c98fa2 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java @@ -0,0 +1,12 @@ +package org.enso.compiler.benchmarks.inline; + +import java.util.Set; +import org.enso.compiler.context.InlineContext; + +record InlineSource( + String source, + // InlineContext for the main method + InlineContext mainInlineContext, + // Local variables in main method + Set localVarNames +) {} From 4bda14a71bc32b1a8ff1efec10e179e4734cfc87 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 29 Feb 2024 18:53:45 +0100 Subject: [PATCH 18/21] Add ImportStandardLibrariesBenchmark --- .../inline/InlineCompilerBenchmark.java | 21 +-- .../inline/InlineCompilerErrorBenchmark.java | 34 ++--- .../benchmarks/inline/InlineContextUtils.java | 22 ++- .../benchmarks/inline/InlineSource.java | 3 +- .../ImportStandardLibrariesBenchmark.java | 142 ++++++++++++++++++ .../module/ManyLocalVarsBenchmark.java | 3 +- 6 files changed, 178 insertions(+), 47 deletions(-) create mode 100644 engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java index 81db6851b043..25c813447f53 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java @@ -7,7 +7,6 @@ import org.enso.compiler.Compiler; import org.enso.compiler.benchmarks.Utils; import org.enso.compiler.context.InlineContext; -import org.enso.polyglot.RuntimeOptions; import org.graalvm.polyglot.Context; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -26,8 +25,8 @@ * Measures the inline compilation, that is the compilation that is requested inside a method. * Simulates a scenario where there is an existing method and we are trying to insert a new * expression into it. - *

- * There is just a single `main` method that contains {@link #LOCAL_VARS_CNT} local variables. + * + *

There is just a single `main` method that contains {@link #LOCAL_VARS_CNT} local variables. * One benchmark measures the time it takes to inline compile a long expression that uses all the * local variables. The other benchmark measures the time it takes to inline compile an expression * that contains some undefined identifiers and thus, should fail to compile. @@ -39,10 +38,9 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class InlineCompilerBenchmark { - /** - * How many variables should be declared in the main method. - */ + /** How many variables should be declared in the main method. */ private static final int LOCAL_VARS_CNT = 40; + private static final int LONG_EXPR_SIZE = 5; private final OutputStream out = new ByteArrayOutputStream(); @@ -53,17 +51,14 @@ public class InlineCompilerBenchmark { @Setup public void setup() throws IOException { - ctx = Utils.createDefaultContextBuilder() - .out(out) - .err(out) - .logHandler(out) - .build(); + ctx = Utils.createDefaultContextBuilder().out(out).err(out).logHandler(out).build(); var ensoCtx = Utils.leakEnsoContext(ctx); compiler = ensoCtx.getCompiler(); var inlineSource = InlineContextUtils.createMainMethodWithLocalVars(ctx, LOCAL_VARS_CNT); mainInlineContext = inlineSource.mainInlineContext(); - longExpression = InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); + longExpression = + InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); } @TearDown @@ -87,6 +82,4 @@ public void longExpression(Blackhole blackhole) { } blackhole.consume(tuppleOpt); } - - } diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java index 5fc56c272c60..6c55131ecf6e 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java @@ -23,13 +23,12 @@ import org.openjdk.jmh.infra.Blackhole; /** - * * Measures the inline compilation, that is the compilation that is requested inside a method. * Simulates a scenario where there is an existing method and we are trying to insert a new * expression into it. - *

- * The expression inserted into the method contains some undefined identifiers and thus, should fail - * to compile. + * + *

The expression inserted into the method contains some undefined identifiers and thus, should + * fail to compile. */ @BenchmarkMode(Mode.AverageTime) @Fork(1) @@ -38,34 +37,35 @@ @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class InlineCompilerErrorBenchmark { - /** - * How many variables should be declared in the main method. - */ + /** How many variables should be declared in the main method. */ private static final int LOCAL_VARS_CNT = 40; + private static final int LONG_EXPR_SIZE = 5; private final OutputStream out = new ByteArrayOutputStream(); - /** - * How many variables should be declared in the main method. - */ + + /** How many variables should be declared in the main method. */ private Compiler compiler; + private Context ctx; private InlineContext mainInlineContext; private String expressionWithErrors; @Setup public void setup() throws IOException { - ctx = Utils.createDefaultContextBuilder() - .out(out) - .err(out) - .logHandler(out) - .option(RuntimeOptions.STRICT_ERRORS, "false") - .build(); + ctx = + Utils.createDefaultContextBuilder() + .out(out) + .err(out) + .logHandler(out) + .option(RuntimeOptions.STRICT_ERRORS, "false") + .build(); var ensoCtx = Utils.leakEnsoContext(ctx); compiler = ensoCtx.getCompiler(); var inlineSource = InlineContextUtils.createMainMethodWithLocalVars(ctx, LOCAL_VARS_CNT); - var longExpression = InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); + var longExpression = + InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); expressionWithErrors = "UNDEFINED * " + longExpression + " * UNDEFINED"; mainInlineContext = inlineSource.mainInlineContext(); } diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java index d84b08526613..b15e98a92263 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java @@ -23,7 +23,8 @@ private InlineContextUtils() {} * @param localVarsCnt How many local variables should be initialized in the main method * @return Body of the main method */ - static InlineSource createMainMethodWithLocalVars(Context ctx, int localVarsCnt) throws IOException { + static InlineSource createMainMethodWithLocalVars(Context ctx, int localVarsCnt) + throws IOException { var sb = new StringBuilder(); sb.append("main = ").append(System.lineSeparator()); var codeGen = new CodeGenerator(); @@ -54,17 +55,14 @@ static InlineSource createMainMethodWithLocalVars(Context ctx, int localVarsCnt) var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); var mainLocalScope = mainFuncRootNode.getLocalScope(); var compiler = ensoCtx.getCompiler(); - var mainInlineContext = InlineContext.fromJava( - mainLocalScope, - moduleScope.getModule().asCompilerModule(), - scala.Option.apply(false), - ensoCtx.getCompilerConfig(), - scala.Option.apply(compiler.packageRepository())); - return new InlineSource( - sb.toString(), - mainInlineContext, - localVarNames - ); + var mainInlineContext = + InlineContext.fromJava( + mainLocalScope, + moduleScope.getModule().asCompilerModule(), + scala.Option.apply(false), + ensoCtx.getCompilerConfig(), + scala.Option.apply(compiler.packageRepository())); + return new InlineSource(sb.toString(), mainInlineContext, localVarNames); } static String createLongExpression(Set localVars, int exprSize) { diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java index a81ef6c98fa2..50ae5a572beb 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java @@ -8,5 +8,4 @@ record InlineSource( // InlineContext for the main method InlineContext mainInlineContext, // Local variables in main method - Set localVarNames -) {} + Set localVarNames) {} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java new file mode 100644 index 000000000000..921c144bf43e --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java @@ -0,0 +1,142 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Paths; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * This benchmark imports all the symbols from libraries, that are normally imported in the IDE + * template. The + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ImportStandardLibrariesBenchmark { + + /** Manually maintained list of symbols that are known to be imported. */ + private static final List KNOWN_IMPORTED_SYMBOLS = + List.of( + // from Base + "True", + "False", + "Vector", + "Vector.new_builder", + // from Table + "Table", + "Column.from_vector", + "Value_Type.Boolean", + "Aggregate_Column.Count", + "Excel_Format.read", + "Previous_Value", + "Join_Kind.Inner", + // From Database + "In_Memory", + "SQL_Query", + "Postgres", + "SQLite_Details.connect", + "Credentials.Username_And_Password", + "Connection_Options.Connection_Options"); + + private static final String IMPORTS = + """ +from Standard.Base import all +from Standard.Table import all +from Standard.Database import all +from Standard.Visualization import all +"""; + + /** + * Number of symbols that will be resolved. I.e., an expression will be created for them, so that + * we are sure that they are imported and resolved. + */ + private static final int RESOLVED_SYMBOLS_CNT = 20; + + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = + Utils.createDefaultContextBuilder() + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath()) + // Enable IR caches - we don't want to compile the imported modules from the standard + // libraries + .option(RuntimeOptions.DISABLE_IR_CACHES, "false") + .logHandler(out) + .out(out) + .err(out) + .build(); + var ensoCtx = Utils.leakEnsoContext(context); + + Set symbolsToUse = + KNOWN_IMPORTED_SYMBOLS.stream().limit(RESOLVED_SYMBOLS_CNT).collect(Collectors.toSet()); + + var sb = new StringBuilder(); + sb.append(IMPORTS); + sb.append(System.lineSeparator()); + sb.append("main =").append(System.lineSeparator()); + + for (var symbolToUse : symbolsToUse) { + sb.append(" ").append(symbolToUse).append(System.lineSeparator()); + } + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "importStandardLibraries.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + @TearDown + public void teardown() { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output (errors?) from the compiler: " + out.toString()); + } + context.close(); + } + + @Benchmark + public void importStandardLibraries(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index 56dd06875623..4f5744e43e97 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -69,8 +69,7 @@ public void setup(BenchmarkParams params) throws IOException { var sb = new StringBuilder(); var codeGen = new CodeGenerator(); - sb.append("main =") - .append(System.lineSeparator()); + sb.append("main =").append(System.lineSeparator()); for (int i = 0; i < IDENTIFIERS_CNT; i++) { int exprSize = random.nextInt(0, MAX_EXPR_SIZE); var assignmentExpr = codeGen.defineNewVariable(exprSize); From 43d0fc228fb5b6f31e142154064a4f58e548625b Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 1 Mar 2024 13:22:30 +0100 Subject: [PATCH 19/21] Add docs --- .../benchmarks/module/ImportStandardLibrariesBenchmark.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java index 921c144bf43e..01698de4da21 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java @@ -33,7 +33,9 @@ /** * This benchmark imports all the symbols from libraries, that are normally imported in the IDE - * template. The + * template. This benchmark focuses on performance of import/export resolution compiler phase. The + * IR cache is enabled, so that modules that are imported from standard libraries are not + * re-compiled. */ @BenchmarkMode(Mode.AverageTime) @Fork(1) From e20fd3ce556c18a2b9ce6050e377b1322aeac3fd Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 1 Mar 2024 13:31:57 +0100 Subject: [PATCH 20/21] fmt --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index e1b147d59b6f..4b1e763e6c18 100644 --- a/build.sbt +++ b/build.sbt @@ -1833,7 +1833,7 @@ lazy val `runtime-benchmarks` = "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion, - "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided", "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "slf4j-nop" % slf4jVersion ), From b2ddf953f20bf7750c99f6b4864888f9e92e18cb Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 1 Mar 2024 13:33:19 +0100 Subject: [PATCH 21/21] Add docs --- .../enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java index 4f5744e43e97..c30ca4b79c98 100644 --- a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -39,7 +39,7 @@ * * * This is the format that is used by the IDE. This should measure mostly the performance of the - * dataflow analysis pass. + * alias analysis pass. */ @BenchmarkMode(Mode.AverageTime) @Fork(1)