diff --git a/build.sbt b/build.sbt index 92bd4f1ed074..4b1e763e6c18 100644 --- a/build.sbt +++ b/build.sbt @@ -1833,6 +1833,7 @@ lazy val `runtime-benchmarks` = "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion, + "org.graalvm.truffle" % "truffle-dsl-processor" % graalMavenPackagesVersion % "provided", "org.slf4j" % "slf4j-api" % slf4jVersion, "org.slf4j" % "slf4j-nop" % slf4jVersion ), @@ -1848,6 +1849,14 @@ lazy val `runtime-benchmarks` = frgaalSourceLevel, "--enable-preview" ), + javacOptions ++= Seq( + "-s", + (Compile / sourceManaged).value.getAbsolutePath, + "-Xlint:unchecked" + ), + Compile / compile := (Compile / compile) + .dependsOn(Def.task { (Compile / sourceManaged).value.mkdirs }) + .value, parallelExecution := false, modulePath := { val requiredModIds = GraalVM.modules ++ GraalVM.langsPkgs ++ Seq( diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java new file mode 100644 index 000000000000..5e6b86b1742f --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/CodeGenerator.java @@ -0,0 +1,239 @@ +package org.enso.compiler.benchmarks; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; + +public class CodeGenerator { + private static final int MAX_TEXT_LITERAL_SIZE = 20; + private static final int SMALL_LETTERS_CNT = 26; + private static final int SEED = 42; + + private static final List OPERATORS = + List.of("+", "-", "*", "/", "&&", "||", "==", "!=", "<", ">", "<=", ">="); + + private static final List METHODS = + List.of( + "map", "filter", "foldl", "foldr", "head", "tail", "init", "last", "length", "reverse"); + + /** + * A collection of identifiers that were defined. I.e., for these identifiers, there exist + * assignments in the generated code. + */ + private final List definedIdentifiers = new ArrayList<>(); + + /** A collection of identifiers that were used in some expression. */ + private final Set usedIdentifiers = new HashSet<>(); + + private final Random random = new Random(SEED); + private int identifierCnt = 0; + + /** Creates a new empty code generator. */ + public CodeGenerator() {} + + /** + * @param identifiers Collection of already defined identifiers, like arguments to a function. + */ + public CodeGenerator(Collection identifiers) { + definedIdentifiers.addAll(identifiers); + } + + /** + * Creates an expression that initializes a new variable. + * + * @param expressionSize Size of the expression that will be used to initialize the variable. + * @return The assignment expression that defines the new variable + */ + public String defineNewVariable(int expressionSize) { + assert expressionSize >= 0; + var ident = nextIdentifier(); + definedIdentifiers.add(ident); + var initExpr = createExpressionFromDefinedIdentifiers(expressionSize); + return ident + " = " + initExpr; + } + + /** Returns a new identifiers and records it as a defined variable. */ + public String defineNewVariable() { + var ident = nextIdentifier(); + definedIdentifiers.add(ident); + return ident; + } + + /** + * Randomly chooses one of the defined identifiers. + * + * @return The chosen identifier. + */ + public String chooseDefinedIdentifier() { + assert !definedIdentifiers.isEmpty(); + var randomIdx = random.nextInt(definedIdentifiers.size()); + return definedIdentifiers.get(randomIdx); + } + + /** + * Returns a collection of identifiers that are not used in any expression. These identifiers were + * defined, but not used, so the compiler should generate a warning about them. + * + * @return A collection of unused identifiers. + */ + public Set getUnusedIdentifiers() { + var allDefined = new HashSet<>(definedIdentifiers); + allDefined.removeAll(usedIdentifiers); + return allDefined; + } + + public void markVariableAsUsed(String variable) { + assert definedIdentifiers.contains(variable); + usedIdentifiers.add(variable); + } + + public List createIdentifiers(int count) { + List idents = new ArrayList<>(count); + for (int i = 0; i < count; i++) { + idents.add(nextIdentifier()); + } + return idents; + } + + private String nextIdentifier() { + identifierCnt++; + return "operator" + identifierCnt; + } + + private String nextMethod() { + var idx = random.nextInt(METHODS.size()); + return METHODS.get(idx); + } + + private String nextTextLiteral() { + var size = random.nextInt(MAX_TEXT_LITERAL_SIZE); + var sb = new StringBuilder(); + for (int i = 0; i < size; i++) { + var rndChar = (char) ('a' + random.nextInt(SMALL_LETTERS_CNT)); + sb.append(rndChar); + } + return "\"" + sb + "\""; + } + + private String nextIntLiteral() { + return Integer.toString(random.nextInt()); + } + + private String nextDecimalLiteral() { + var part1 = random.nextInt(); + var decimalPart = Math.abs(random.nextInt()); + return part1 + "." + decimalPart; + } + + public String nextLiteral() { + var rndInt = random.nextInt(3); + return switch (rndInt) { + case 0 -> nextTextLiteral(); + case 1 -> nextIntLiteral(); + case 2 -> nextDecimalLiteral(); + default -> throw new UnsupportedOperationException("unimplemented"); + }; + } + + /** + * Creates an expression using only the defined identifiers. + * + * @param size Arity of the expression. + */ + public String createExpressionFromDefinedIdentifiers(int size) { + return createExpression(definedIdentifiers, size); + } + + /** + * Creates an expression with the given size. + * + * @param identifiers A collection of all the identifiers that can be used in the expression. + * These may be undefined identifiers. In that case, the compiler should throw an error. + * @param size Arity of the expression. + * @return A string representing the expression. + */ + public String createExpression(List identifiers, int size) { + if (identifiers.isEmpty()) { + return nextLiteral(); + } + + return switch (size) { + case 0 -> nextLiteral(); + // Either a single identifier or a method call on the identifier + case 1 -> { + var sb = new StringBuilder(); + var ident = chooseIdentifier(identifiers); + usedIdentifiers.add(ident); + sb.append(ident); + var shouldCallMethod = random.nextBoolean(); + if (shouldCallMethod) { + sb.append(".").append(nextMethod()); + } + yield sb.toString(); + } + // Method call or binary operator + case 2 -> { + var sb = new StringBuilder(); + var shouldCallMethod = random.nextBoolean(); + var ident1 = chooseIdentifier(identifiers); + usedIdentifiers.add(ident1); + if (shouldCallMethod) { + var methodArg = createExpression(identifiers, 1); + sb.append(ident1) + .append(".") + .append(nextMethod()) + .append(" (") + .append(methodArg) + .append(")"); + } else { + var ident2 = chooseIdentifier(identifiers); + usedIdentifiers.add(ident2); + // Binary operator + sb.append(ident1).append(nextOperator()).append(ident2); + } + yield sb.toString(); + } + // Split into two expressions with random size + default -> { + var sb = new StringBuilder(); + var shouldCallMethod = random.nextBoolean(); + if (shouldCallMethod) { + var ident = chooseIdentifier(identifiers); + usedIdentifiers.add(ident); + var methodArity = size - 1; + List methodArgs = new ArrayList<>(); + for (int i = 0; i < methodArity; i++) { + methodArgs.add(createExpression(identifiers, size - 1)); + } + sb.append(ident).append(".").append(nextMethod()).append(" "); + for (var methodArg : methodArgs) { + sb.append(methodArg).append(" "); + } + } else { + var rndIdx = Math.max(2, random.nextInt(size)); + var size1 = rndIdx; + var size2 = size - rndIdx; + var expr1 = createExpression(identifiers, size1); + var expr2 = createExpression(identifiers, size2); + var op = nextOperator(); + sb.append("(").append(expr1).append(")").append(op).append("(").append(expr2).append(")"); + } + yield sb.toString(); + } + }; + } + + private String nextOperator() { + var idx = random.nextInt(OPERATORS.size()); + return OPERATORS.get(idx); + } + + private String chooseIdentifier(List identifiers) { + assert !identifiers.isEmpty(); + var randomIdx = random.nextInt(identifiers.size()); + return identifiers.get(randomIdx); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java new file mode 100644 index 000000000000..ba110b3c7e91 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/Utils.java @@ -0,0 +1,72 @@ +package org.enso.compiler.benchmarks; + +import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.TruffleObject; +import com.oracle.truffle.api.library.ExportLibrary; +import com.oracle.truffle.api.library.ExportMessage; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.logging.Level; +import org.enso.interpreter.runtime.EnsoContext; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; + +public class Utils { + public static Context.Builder createDefaultContextBuilder() { + return Context.newBuilder() + .allowExperimentalOptions(true) + .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.getName()) + .option(RuntimeOptions.DISABLE_IR_CACHES, "true") + .option(RuntimeOptions.STRICT_ERRORS, "true") + .logHandler(System.err) + .allowIO(IOAccess.ALL) + .allowAllAccess(true); + } + + public static EnsoContext leakEnsoContext(Context ctx) { + return ctx.getBindings(LanguageInfo.ID) + .invokeMember(MethodNames.TopScope.LEAK_CONTEXT) + .as(EnsoContext.class); + } + + public static Object unwrapReceiver(Context ctx, Value value) { + var unwrapper = new Unwrapper(); + var unwrapperValue = ctx.asValue(unwrapper); + unwrapperValue.execute(value); + return unwrapper.args[0]; + } + + public static File createSrcFile(String code, String name) { + var benchDataDir = Path.of(".", "target", "bench-data"); + benchDataDir.toFile().mkdirs(); + var srcFile = benchDataDir.resolve(name).toFile(); + try { + Files.writeString(srcFile.toPath(), code); + } catch (IOException e) { + throw new AssertionError(e); + } + return srcFile; + } + + @ExportLibrary(InteropLibrary.class) + static final class Unwrapper implements TruffleObject { + Object[] args; + + @ExportMessage + Object execute(Object[] args) { + this.args = args; + return this; + } + + @ExportMessage + boolean isExecutable() { + return true; + } + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java new file mode 100644 index 000000000000..25c813447f53 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerBenchmark.java @@ -0,0 +1,85 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.graalvm.polyglot.Context; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the inline compilation, that is the compilation that is requested inside a method. + * Simulates a scenario where there is an existing method and we are trying to insert a new + * expression into it. + * + *

There is just a single `main` method that contains {@link #LOCAL_VARS_CNT} local variables. + * One benchmark measures the time it takes to inline compile a long expression that uses all the + * local variables. The other benchmark measures the time it takes to inline compile an expression + * that contains some undefined identifiers and thus, should fail to compile. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class InlineCompilerBenchmark { + /** How many variables should be declared in the main method. */ + private static final int LOCAL_VARS_CNT = 40; + + private static final int LONG_EXPR_SIZE = 5; + + private final OutputStream out = new ByteArrayOutputStream(); + private Compiler compiler; + private Context ctx; + private InlineContext mainInlineContext; + private String longExpression; + + @Setup + public void setup() throws IOException { + ctx = Utils.createDefaultContextBuilder().out(out).err(out).logHandler(out).build(); + var ensoCtx = Utils.leakEnsoContext(ctx); + compiler = ensoCtx.getCompiler(); + + var inlineSource = InlineContextUtils.createMainMethodWithLocalVars(ctx, LOCAL_VARS_CNT); + mainInlineContext = inlineSource.mainInlineContext(); + longExpression = + InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); + } + + @TearDown + public void tearDown() { + ctx.close(); + try { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out); + } + out.close(); + } catch (IOException e) { + throw new AssertionError("Failed to close the output stream", e); + } + } + + @Benchmark + public void longExpression(Blackhole blackhole) { + var tuppleOpt = compiler.runInline(longExpression, mainInlineContext); + if (tuppleOpt.isEmpty()) { + throw new AssertionError("Unexpected: inline compilation should succeed"); + } + blackhole.consume(tuppleOpt); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java new file mode 100644 index 000000000000..6c55131ecf6e --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineCompilerErrorBenchmark.java @@ -0,0 +1,86 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measures the inline compilation, that is the compilation that is requested inside a method. + * Simulates a scenario where there is an existing method and we are trying to insert a new + * expression into it. + * + *

The expression inserted into the method contains some undefined identifiers and thus, should + * fail to compile. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class InlineCompilerErrorBenchmark { + /** How many variables should be declared in the main method. */ + private static final int LOCAL_VARS_CNT = 40; + + private static final int LONG_EXPR_SIZE = 5; + + private final OutputStream out = new ByteArrayOutputStream(); + + /** How many variables should be declared in the main method. */ + private Compiler compiler; + + private Context ctx; + private InlineContext mainInlineContext; + private String expressionWithErrors; + + @Setup + public void setup() throws IOException { + ctx = + Utils.createDefaultContextBuilder() + .out(out) + .err(out) + .logHandler(out) + .option(RuntimeOptions.STRICT_ERRORS, "false") + .build(); + var ensoCtx = Utils.leakEnsoContext(ctx); + compiler = ensoCtx.getCompiler(); + + var inlineSource = InlineContextUtils.createMainMethodWithLocalVars(ctx, LOCAL_VARS_CNT); + var longExpression = + InlineContextUtils.createLongExpression(inlineSource.localVarNames(), LONG_EXPR_SIZE); + expressionWithErrors = "UNDEFINED * " + longExpression + " * UNDEFINED"; + mainInlineContext = inlineSource.mainInlineContext(); + } + + @TearDown + public void teardown() { + ctx.close(); + if (out.toString().isEmpty()) { + throw new AssertionError("Expected some output (some errors) from the compiler"); + } + } + + @Benchmark + public void expressionWithErrors(Blackhole blackhole) { + var tuppleOpt = compiler.runInline(expressionWithErrors, mainInlineContext); + blackhole.consume(tuppleOpt); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java new file mode 100644 index 000000000000..b15e98a92263 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineContextUtils.java @@ -0,0 +1,72 @@ +package org.enso.compiler.benchmarks.inline; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.compiler.context.InlineContext; +import org.enso.interpreter.node.MethodRootNode; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; + +class InlineContextUtils { + private InlineContextUtils() {} + + /** + * Creates a main method, generates some local variables, and fills their identifiers in the given + * set. + * + * @param localVarsCnt How many local variables should be initialized in the main method + * @return Body of the main method + */ + static InlineSource createMainMethodWithLocalVars(Context ctx, int localVarsCnt) + throws IOException { + var sb = new StringBuilder(); + sb.append("main = ").append(System.lineSeparator()); + var codeGen = new CodeGenerator(); + Set localVarNames = new HashSet<>(); + for (int i = 0; i < localVarsCnt; i++) { + var varName = "loc_var_" + i; + localVarNames.add(varName); + var literal = codeGen.nextLiteral(); + sb.append(" ") + .append(varName) + .append(" = ") + .append(literal) + .append(System.lineSeparator()); + } + // In the last expression of main method, use all the variables, so that there is no unused + // variable warning + var lastExpr = + localVarNames.stream().reduce((acc, varName) -> acc + " + " + varName).orElseThrow(); + sb.append(" ").append(lastExpr).append(System.lineSeparator()); + var ensoCtx = Utils.leakEnsoContext(ctx); + var srcFile = Utils.createSrcFile(sb.toString(), "inlineBenchmark.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = ctx.eval(src); + var moduleAssocType = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocTypeReceiver = (Type) Utils.unwrapReceiver(ctx, moduleAssocType); + var moduleScope = assocTypeReceiver.getDefinitionScope(); + var mainFunc = moduleScope.getMethodForType(assocTypeReceiver, "main"); + var mainFuncRootNode = (MethodRootNode) mainFunc.getCallTarget().getRootNode(); + var mainLocalScope = mainFuncRootNode.getLocalScope(); + var compiler = ensoCtx.getCompiler(); + var mainInlineContext = + InlineContext.fromJava( + mainLocalScope, + moduleScope.getModule().asCompilerModule(), + scala.Option.apply(false), + ensoCtx.getCompilerConfig(), + scala.Option.apply(compiler.packageRepository())); + return new InlineSource(sb.toString(), mainInlineContext, localVarNames); + } + + static String createLongExpression(Set localVars, int exprSize) { + var codeGen = new CodeGenerator(localVars); + return codeGen.createExpressionFromDefinedIdentifiers(exprSize); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java new file mode 100644 index 000000000000..50ae5a572beb --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/inline/InlineSource.java @@ -0,0 +1,11 @@ +package org.enso.compiler.benchmarks.inline; + +import java.util.Set; +import org.enso.compiler.context.InlineContext; + +record InlineSource( + String source, + // InlineContext for the main method + InlineContext mainInlineContext, + // Local variables in main method + Set localVarNames) {} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java new file mode 100644 index 000000000000..01698de4da21 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ImportStandardLibrariesBenchmark.java @@ -0,0 +1,144 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Paths; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * This benchmark imports all the symbols from libraries, that are normally imported in the IDE + * template. This benchmark focuses on performance of import/export resolution compiler phase. The + * IR cache is enabled, so that modules that are imported from standard libraries are not + * re-compiled. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ImportStandardLibrariesBenchmark { + + /** Manually maintained list of symbols that are known to be imported. */ + private static final List KNOWN_IMPORTED_SYMBOLS = + List.of( + // from Base + "True", + "False", + "Vector", + "Vector.new_builder", + // from Table + "Table", + "Column.from_vector", + "Value_Type.Boolean", + "Aggregate_Column.Count", + "Excel_Format.read", + "Previous_Value", + "Join_Kind.Inner", + // From Database + "In_Memory", + "SQL_Query", + "Postgres", + "SQLite_Details.connect", + "Credentials.Username_And_Password", + "Connection_Options.Connection_Options"); + + private static final String IMPORTS = + """ +from Standard.Base import all +from Standard.Table import all +from Standard.Database import all +from Standard.Visualization import all +"""; + + /** + * Number of symbols that will be resolved. I.e., an expression will be created for them, so that + * we are sure that they are imported and resolved. + */ + private static final int RESOLVED_SYMBOLS_CNT = 20; + + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = + Utils.createDefaultContextBuilder() + .option( + RuntimeOptions.LANGUAGE_HOME_OVERRIDE, + Paths.get("../../distribution/component").toFile().getAbsolutePath()) + // Enable IR caches - we don't want to compile the imported modules from the standard + // libraries + .option(RuntimeOptions.DISABLE_IR_CACHES, "false") + .logHandler(out) + .out(out) + .err(out) + .build(); + var ensoCtx = Utils.leakEnsoContext(context); + + Set symbolsToUse = + KNOWN_IMPORTED_SYMBOLS.stream().limit(RESOLVED_SYMBOLS_CNT).collect(Collectors.toSet()); + + var sb = new StringBuilder(); + sb.append(IMPORTS); + sb.append(System.lineSeparator()); + sb.append("main =").append(System.lineSeparator()); + + for (var symbolToUse : symbolsToUse) { + sb.append(" ").append(symbolToUse).append(System.lineSeparator()); + } + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "importStandardLibraries.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + @TearDown + public void teardown() { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output (errors?) from the compiler: " + out.toString()); + } + context.close(); + } + + @Benchmark + public void importStandardLibraries(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java new file mode 100644 index 000000000000..d8a236d1253b --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyErrorsBenchmark.java @@ -0,0 +1,147 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measure compilation of a module with one method that contains a lot of errors - syntactical + * errors and unknown identifiers. The compiler should be able to recover from errors and so it + * should compile the whole module and not stop after the first error. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManyErrorsBenchmark { + private static final int ERRORS_CNT = 30; + private static final int IDENTIFIERS_CNT = 40; + private static final int EXPR_SIZE = 5; + private static final List UNDEFINED_IDENTIFIERS = + List.of("FOO_BAR", "Bazzzzz", "Type.Constructor.Foo.Bar.Baz"); + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + private final Random random = new Random(42); + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = + Utils.createDefaultContextBuilder() + .option(RuntimeOptions.STRICT_ERRORS, "false") + .logHandler(out) + .out(out) + .err(out) + .build(); + var ensoCtx = Utils.leakEnsoContext(context); + var sb = new StringBuilder(); + var codeGen = new CodeGenerator(); + var definedIdentifiers = codeGen.createIdentifiers(IDENTIFIERS_CNT); + sb.append("main = ").append(System.lineSeparator()); + for (String ident : definedIdentifiers) { + sb.append(" ") + .append(ident) + .append(" = ") + .append(codeGen.nextLiteral()) + .append(System.lineSeparator()); + } + for (int i = 0; i < ERRORS_CNT; i++) { + var rndInt = random.nextInt(0, 3); + switch (rndInt) { + // Expression with unknown identifiers + case 0 -> { + var expr = codeGen.createExpression(UNDEFINED_IDENTIFIERS, EXPR_SIZE); + sb.append(" ").append(expr).append(System.lineSeparator()); + } + // Inline type ascription with unknown identifier + case 1 -> { + var expr = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); + sb.append(" ") + .append(" var : (Type.Constructor.Foo Bar.Baz A B) = ") + .append(expr) + .append(System.lineSeparator()); + } + // Put arrows before, after, and between expressions + case 2 -> { + var expr1 = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); + var expr2 = codeGen.createExpression(definedIdentifiers, EXPR_SIZE); + sb.append(" ") + .append(" -> ") + .append(expr1) + .append(" -> ") + .append(expr2) + .append(" -> ") + .append(System.lineSeparator()); + } + default -> throw new AssertionError("Unexpected random integer: " + rndInt); + } + } + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manyErrorsBenchmark.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + @TearDown + public void tearDown() throws IOException { + // Teardown is called at the end of all the iterations. This means that the number of errors + // reported into `out` should be much bigger than `ERRORS_CNT`. Let's iterate over some lines + // and see if this is true. + var errorsFound = 0; + for (String line : out.toString().split(System.lineSeparator())) { + if (errorsFound == ERRORS_CNT) { + break; + } + if (line.matches(".*: error: .*")) { + errorsFound++; + } + } + if (errorsFound != ERRORS_CNT) { + throw new AssertionError("Expected " + ERRORS_CNT + " errors, but found " + errorsFound); + } + out.close(); + context.close(); + } + + @Benchmark + public void manyErrors(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java new file mode 100644 index 000000000000..c30ca4b79c98 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyLocalVarsBenchmark.java @@ -0,0 +1,112 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Random; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measure compilation of a module with a single long method with a format like: + * + *

+ * main =
+ *    obj1 = ...
+ *    obj2 = ...
+ *    obj3 = ...
+ * 
+ * + * This is the format that is used by the IDE. This should measure mostly the performance of the + * alias analysis pass. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManyLocalVarsBenchmark { + + /** + * Total count of local variables in the `main` method. Every variable is defined on a new line. + */ + private static final int IDENTIFIERS_CNT = 100; + + private static final int MAX_EXPR_SIZE = 5; + private final Random random = new Random(42); + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = Utils.createDefaultContextBuilder().logHandler(out).out(out).err(out).build(); + var ensoCtx = Utils.leakEnsoContext(context); + var sb = new StringBuilder(); + var codeGen = new CodeGenerator(); + + sb.append("main =").append(System.lineSeparator()); + for (int i = 0; i < IDENTIFIERS_CNT; i++) { + int exprSize = random.nextInt(0, MAX_EXPR_SIZE); + var assignmentExpr = codeGen.defineNewVariable(exprSize); + sb.append(" ").append(assignmentExpr).append(System.lineSeparator()); + } + + // Add a final line that uses the rest of the identifiers, so that there is no "Unused binding" + // warning. + sb.append(" ").append("result = "); + sb.append( + codeGen.getUnusedIdentifiers().stream().reduce((a, b) -> a + " + " + b).orElseThrow()); + sb.append(System.lineSeparator()); + // result is the return value from the main method + sb.append(" ").append("result").append(System.lineSeparator()); + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manyLocalVars.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + @TearDown + public void tearDown() throws IOException { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out.toString()); + } + out.close(); + context.close(); + } + + @Benchmark + public void longMethodWithLotOfLocalVars(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java new file mode 100644 index 000000000000..e6a28f480493 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManyNestedBlocksBenchmark.java @@ -0,0 +1,202 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** Measure compilation of a module with one method that contains a lot of nested blocks. */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManyNestedBlocksBenchmark { + + /** Maximum nesting. */ + private static final int NESTED_BLOCKS_CNT = 40; + + /** + * Number of cases in a case expression. Only the last case contains nested blocks, the rest are + * simple literals. + */ + private static final int CASES_CNT = 5; + + /** Number of lines per block. */ + private static final int BLOCK_SIZE = 5; + + /** How many new identifiers will be defined in each block. */ + private static final int IDENTIFIERS_CNT = 2; + + /** Maximum size of an expression on a single line. */ + private static final int MAX_EXPR_SIZE = 5; + + /** Maximum number of arguments of an anonymous function */ + private final int ANONYMOUS_FUNC_MAX_ARGS = 5; + + private final Random random = new Random(42); + private final CodeGenerator codeGen = new CodeGenerator(); + private final StringBuilder sb = new StringBuilder(); + + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = Utils.createDefaultContextBuilder().logHandler(out).out(out).err(out).build(); + var ensoCtx = Utils.leakEnsoContext(context); + sb.append("main = ").append(System.lineSeparator()); + createNestedBlocks(1); + + assert codeGen.getUnusedIdentifiers().isEmpty() + : "All variables should be used in simple blocks"; + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manyNestedBlocks.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + /** + * All the unused variables are used in the last expression in the block + * + * @param nestedBlockIdx Nesting index. + */ + private void createSimpleBlock(int nestedBlockIdx) { + var spaces = " ".repeat(nestedBlockIdx); + var exprsCnt = random.nextInt(1, BLOCK_SIZE); + var exprSize = random.nextInt(0, MAX_EXPR_SIZE); + for (int i = 0; i < IDENTIFIERS_CNT; i++) { + var newVar = codeGen.defineNewVariable(); + sb.append(spaces) + .append(newVar) + .append(" = ") + .append(codeGen.nextLiteral()) + .append(System.lineSeparator()); + } + for (int i = 0; i < exprsCnt; i++) { + var expr = codeGen.createExpressionFromDefinedIdentifiers(exprSize); + sb.append(spaces).append(expr).append(System.lineSeparator()); + } + Set unusedIdentifiers = codeGen.getUnusedIdentifiers(); + var lastExpr = + unusedIdentifiers.stream().reduce((acc, ident) -> acc + " + " + ident).orElse("42"); + sb.append(spaces).append(lastExpr).append(System.lineSeparator()); + unusedIdentifiers.forEach(codeGen::markVariableAsUsed); + } + + private void createNestedBlocks(int nestedBlockIdx) { + if (nestedBlockIdx >= NESTED_BLOCKS_CNT) { + return; + } + createSimpleBlock(nestedBlockIdx); + // Introduce a new nested block at the end of the current block + if (nestedBlockIdx < NESTED_BLOCKS_CNT - 1) { + var rndInt = random.nextInt(0, 4); + var spaces = " ".repeat(nestedBlockIdx); + switch (rndInt) { + case 0 -> { + var ident = codeGen.chooseDefinedIdentifier(); + sb.append(spaces) + .append("case ") + .append(ident) + .append(" of") + .append(System.lineSeparator()); + // Only the last case expression contains nested blocks + for (int i = 0; i < CASES_CNT - 1; i++) { + sb.append(spaces) + .append(" ") + .append(codeGen.nextLiteral()) + .append(" -> ") + .append(codeGen.nextLiteral()) + .append(System.lineSeparator()); + } + sb.append(spaces) + .append(" ") + .append(codeGen.nextLiteral()) + .append(" -> ") + .append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 2); + } + case 1 -> { + var ident = codeGen.chooseDefinedIdentifier(); + sb.append(spaces) + .append("if ") + .append(ident) + .append(" == 42 then 42 else") + .append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 1); + } + case 2 -> { + var ident = codeGen.chooseDefinedIdentifier(); + sb.append(spaces) + .append("if ") + .append(ident) + .append(" == 42 then") + .append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 1); + } + // Create a nested anonymous function + case 3 -> { + var argsCnt = random.nextInt(1, ANONYMOUS_FUNC_MAX_ARGS); + sb.append(spaces); + for (int i = 0; i < argsCnt; i++) { + var newIdent = codeGen.defineNewVariable(); + sb.append(newIdent).append("-> "); + } + sb.append(System.lineSeparator()); + createNestedBlocks(nestedBlockIdx + 1); + } + default -> throw new IllegalStateException("Unexpected random int: " + rndInt); + } + } + } + + @TearDown + public void tearDown() throws IOException { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out.toString()); + } + out.close(); + context.close(); + } + + @Benchmark + public void manyNestedBlocks(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } +} diff --git a/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java new file mode 100644 index 000000000000..62b0087332e1 --- /dev/null +++ b/engine/runtime-benchmarks/src/main/java/org/enso/compiler/benchmarks/module/ManySmallMethodsBenchmark.java @@ -0,0 +1,150 @@ +package org.enso.compiler.benchmarks.module; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.Compiler; +import org.enso.compiler.benchmarks.CodeGenerator; +import org.enso.compiler.benchmarks.Utils; +import org.enso.interpreter.runtime.Module; +import org.enso.interpreter.runtime.data.Type; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Measure compilation of a module with a lot of small methods with variable number of arguments. + */ +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 6) +@Measurement(iterations = 4) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class ManySmallMethodsBenchmark { + + private static final int METHODS_CNT = 30; + + /** Maximum number of arguments of a single method. */ + private static final int MAX_ARGS = 10; + + /** Maximum number of local variables per method */ + private static final int MAX_LOCAL_VARS = 5; + + /** Maximum number of expressions per method, excluding new variable definitions. */ + private static final int MAX_METHOD_SIZE = 7; + + /** Maximum arity of an expression in method's body. */ + private static final int MAX_EXPR_SIZE = 5; + + private final Random random = new Random(42); + private final StringBuilder sb = new StringBuilder(); + private Context context; + private Compiler compiler; + private Module module; + private OutputStream out; + + @Setup + public void setup(BenchmarkParams params) throws IOException { + this.out = new ByteArrayOutputStream(); + this.context = Utils.createDefaultContextBuilder().logHandler(out).out(out).err(out).build(); + var ensoCtx = Utils.leakEnsoContext(context); + List methods = new ArrayList<>(); + + for (int methodIdx = 0; methodIdx < METHODS_CNT; methodIdx++) { + methods.add(createMethod(methodIdx)); + } + + sb.append(System.lineSeparator()); + sb.append("main = ").append(System.lineSeparator()); + // Make sure that every method is called, with correct number of arguments + var codeGen = new CodeGenerator(); + for (var method : methods) { + sb.append(" ").append(method.name); + for (int i = 0; i < method.argCount; i++) { + sb.append(" ").append(codeGen.nextLiteral()); + } + sb.append(System.lineSeparator()); + } + + var code = sb.toString(); + var srcFile = Utils.createSrcFile(code, "manySmallMethods.enso"); + var src = Source.newBuilder(LanguageInfo.ID, srcFile).build(); + var module = context.eval(src); + var assocTypeValue = module.invokeMember(MethodNames.Module.GET_ASSOCIATED_TYPE); + var assocType = (Type) Utils.unwrapReceiver(context, assocTypeValue); + var moduleScope = assocType.getDefinitionScope(); + this.module = moduleScope.getModule(); + this.compiler = ensoCtx.getCompiler(); + } + + private Method createMethod(int methodIdx) { + var methodName = "method_" + methodIdx; + sb.append(methodName).append(" "); + var argCount = random.nextInt(0, MAX_ARGS + 1); + Set args = new HashSet<>(); + for (int argIdx = 0; argIdx < argCount; argIdx++) { + var argName = "arg_" + argIdx; + sb.append(argName).append(" "); + args.add(argName); + } + sb.append(" = ").append(System.lineSeparator()); + var codeGen = new CodeGenerator(args); + var localVarsCnt = random.nextInt(0, MAX_LOCAL_VARS); + for (int i = 0; i < localVarsCnt; i++) { + sb.append(" ") + .append(codeGen.defineNewVariable(MAX_EXPR_SIZE)) + .append(System.lineSeparator()); + } + var methodSize = random.nextInt(0, MAX_METHOD_SIZE); + for (int i = 0; i < methodSize; i++) { + var exprSize = random.nextInt(0, MAX_EXPR_SIZE); + var expr = codeGen.createExpressionFromDefinedIdentifiers(exprSize); + sb.append(" ").append(expr).append(System.lineSeparator()); + } + var lastExpr = + codeGen.getUnusedIdentifiers().stream() + .reduce((acc, ident) -> acc + " + " + ident) + .orElse("42"); + sb.append(" ").append(lastExpr).append(System.lineSeparator()); + sb.append(System.lineSeparator()); + return new Method(methodName, argCount); + } + + @TearDown + public void tearDown() throws IOException { + if (!out.toString().isEmpty()) { + throw new AssertionError("Unexpected output from the compiler: " + out.toString()); + } + out.close(); + context.close(); + } + + @Benchmark + public void manySmallMethods(Blackhole blackhole) { + var compilerResult = compiler.run(module.asCompilerModule()); + blackhole.consume(compilerResult); + } + + private record Method(String name, int argCount) {} +}