From df5fdfa3cee51755e66133772a45bb51049b7d09 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 28 Jan 2025 18:56:43 +0100 Subject: [PATCH 1/4] Dump compiler IR to IGV (#12061) Compiler dumps IR passes into IGV. # Important Notes New documentation with screenshots in https://github.com/enso-org/enso/blob/2e714a70ddf12456e9f3fa9e132fd2ac43aa3b77/docs/runtime/compiler-ir.md IR graphs can be dumped to IGV in tests as well. See https://github.com/enso-org/enso/pull/12061#issuecomment-2610596092 --- .jvmopts | 1 + build.sbt | 70 +- docs/runtime/compiler-ir.md | 84 +- .../java/org/enso/common/RuntimeOptions.java | 2 + .../src/main/java/module-info.java | 10 + .../org/enso/compiler/dump/igv/ASTBlock.java | 56 ++ .../compiler/dump/igv/ASTDumpStructure.java | 278 +++++++ .../org/enso/compiler/dump/igv/ASTEdge.java | 13 + .../enso/compiler/dump/igv/ASTLocation.java | 51 ++ .../org/enso/compiler/dump/igv/ASTMethod.java | 15 + .../org/enso/compiler/dump/igv/ASTNode.java | 135 +++ .../enso/compiler/dump/igv/ASTNodeClass.java | 3 + .../enso/compiler/dump/igv/EnsoModuleAST.java | 511 ++++++++++++ .../org/enso/compiler/dump/igv/IGVDumper.java | 177 ++++ .../compiler/dump/igv/IGVDumperFactory.java | 42 + .../org/enso/compiler/dump/igv/Utils.java | 49 ++ .../enso/compiler/dump/igv/package-info.java | 6 + .../src/main/java/module-info.java | 7 + .../dump/service/IRDumpFactoryService.java | 13 + .../dump/service/IRDumpSingleton.java | 37 + .../enso/compiler/dump/service/IRDumper.java | 36 + .../src/main/java/module-info.java | 5 +- .../compiler/{dump => docs}/DocsDispatch.java | 2 +- .../{dump => docs}/DocsEmitMarkdown.java | 2 +- .../{dump => docs}/DocsEmitSignatures.java | 2 +- .../compiler/{dump => docs}/DocsGenerate.java | 2 +- .../compiler/{dump => docs}/DocsUtils.java | 2 +- .../compiler/{dump => docs}/DocsVisit.java | 2 +- .../org/enso/compiler/dump/GraphVizEdge.java | 63 -- .../org/enso/compiler/dump/GraphVizNode.java | 188 ----- .../java/org/enso/compiler/dump/IRDumper.java | 772 ------------------ .../org/enso/compiler/dump/IRDumperPass.java | 64 -- .../java/org/enso/compiler/dump/Utils.java | 19 - .../scala/org/enso/compiler/Compiler.scala | 103 ++- .../main/scala/org/enso/compiler/Passes.scala | 5 +- .../enso/compiler/data/CompilerConfig.scala | 14 +- .../org/enso/compiler/pass/PassManager.scala | 54 +- .../compiler/phase/BuiltinsIrBuilder.scala | 9 +- .../compiler/test/CompilerTestSetup.scala | 3 +- .../compiler/dump/test/DocsGenerateTest.java | 4 +- .../enso/compiler/dump/test/IRDumpTest.java | 123 ++- .../org/enso/compiler/test/WithIRDumper.java | 27 + .../test/mini/passes/MiniPassTester.java | 3 +- .../org/enso/compiler/test/CompilerTest.scala | 6 +- .../org/enso/interpreter/EnsoLanguage.java | 2 +- .../enso/interpreter/runtime/EnsoContext.java | 5 +- .../enso/test/utils/IRDumperTestWrapper.java | 40 + 47 files changed, 1921 insertions(+), 1196 deletions(-) create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/module-info.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTBlock.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTDumpStructure.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTEdge.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTLocation.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTMethod.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNode.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNodeClass.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/EnsoModuleAST.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumper.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumperFactory.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/Utils.java create mode 100644 engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/package-info.java create mode 100644 engine/runtime-compiler-dump/src/main/java/module-info.java create mode 100644 engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpFactoryService.java create mode 100644 engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpSingleton.java create mode 100644 engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumper.java rename engine/runtime-compiler/src/main/java/org/enso/compiler/{dump => docs}/DocsDispatch.java (98%) rename engine/runtime-compiler/src/main/java/org/enso/compiler/{dump => docs}/DocsEmitMarkdown.java (98%) rename engine/runtime-compiler/src/main/java/org/enso/compiler/{dump => docs}/DocsEmitSignatures.java (98%) rename engine/runtime-compiler/src/main/java/org/enso/compiler/{dump => docs}/DocsGenerate.java (99%) rename engine/runtime-compiler/src/main/java/org/enso/compiler/{dump => docs}/DocsUtils.java (99%) rename engine/runtime-compiler/src/main/java/org/enso/compiler/{dump => docs}/DocsVisit.java (98%) delete mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizEdge.java delete mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizNode.java delete mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumper.java delete mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumperPass.java delete mode 100644 engine/runtime-compiler/src/main/java/org/enso/compiler/dump/Utils.java create mode 100644 engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/WithIRDumper.java create mode 100644 lib/java/test-utils/src/main/java/org/enso/test/utils/IRDumperTestWrapper.java diff --git a/.jvmopts b/.jvmopts index 2e0a02c98ea4..4245db4e437c 100644 --- a/.jvmopts +++ b/.jvmopts @@ -7,3 +7,4 @@ --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED +--add-exports=jdk.internal.vm.compiler/org.graalvm.graphio=ALL-UNNAMED diff --git a/build.sbt b/build.sbt index 170ec271e8fb..61548978c7d5 100644 --- a/build.sbt +++ b/build.sbt @@ -351,6 +351,8 @@ lazy val enso = (project in file(".")) `runtime-and-langs`, `runtime-benchmarks`, `runtime-compiler`, + `runtime-compiler-dump`, + `runtime-compiler-dump-igv`, `runtime-parser`, `runtime-parser-dsl`, `runtime-parser-processor`, @@ -722,6 +724,8 @@ lazy val componentModulesPaths = (`runtime` / Compile / exportedModuleBin).value, (`syntax-rust-definition` / Compile / exportedModuleBin).value, (`runtime-compiler` / Compile / exportedModuleBin).value, + (`runtime-compiler-dump` / Compile / exportedModuleBin).value, + (`runtime-compiler-dump-igv` / Compile / exportedModuleBin).value, (`runtime-parser` / Compile / exportedModuleBin).value, (`runtime-suggestions` / Compile / exportedModuleBin).value, (`runtime-instrument-common` / Compile / exportedModuleBin).value, @@ -2398,6 +2402,7 @@ lazy val `language-server` = (project in file("engine/language-server")) (`runtime-suggestions` / Compile / exportedModule).value, (`runtime-parser` / Compile / exportedModule).value, (`runtime-compiler` / Compile / exportedModule).value, + (`runtime-compiler-dump` / Compile / exportedModule).value, (`polyglot-api` / Compile / exportedModule).value, (`polyglot-api-macros` / Compile / exportedModule).value, (`pkg` / Compile / exportedModule).value, @@ -2940,6 +2945,8 @@ lazy val `runtime-integration-tests` = (`runtime-suggestions` / Compile / exportedModule).value, (`runtime-parser` / Compile / exportedModule).value, (`runtime-compiler` / Compile / exportedModule).value, + (`runtime-compiler-dump` / Compile / exportedModule).value, + (`runtime-compiler-dump-igv` / Compile / exportedModule).value, (`polyglot-api` / Compile / exportedModule).value, (`polyglot-api-macros` / Compile / exportedModule).value, (`pkg` / Compile / exportedModule).value, @@ -3018,6 +3025,13 @@ lazy val `runtime-integration-tests` = ) }, Test / addExports := { + // Add necessary exports for IR module dumping to IGV + // Which is used in the test utils + val irDumperExports = Map( + "jdk.internal.vm.compiler/org.graalvm.graphio" -> Seq( + (`runtime-compiler-dump-igv` / javaModuleName).value + ) + ) val runtimeModName = (`runtime` / javaModuleName).value val exports = Map( (`runtime-instrument-common` / javaModuleName).value + "/org.enso.interpreter.instrument.job" -> Seq( @@ -3033,7 +3047,7 @@ lazy val `runtime-integration-tests` = val testPkgsExports = testPkgs.map { pkg => runtimeModName + "/" + pkg -> Seq("ALL-UNNAMED") }.toMap - exports ++ testPkgsExports + exports ++ testPkgsExports ++ irDumperExports } ) .dependsOn(`runtime`) @@ -3115,6 +3129,7 @@ lazy val `runtime-benchmarks` = (`runtime-suggestions` / Compile / exportedModule).value, (`runtime-parser` / Compile / exportedModule).value, (`runtime-compiler` / Compile / exportedModule).value, + (`runtime-compiler-dump` / Compile / exportedModule).value, (`polyglot-api` / Compile / exportedModule).value, (`polyglot-api-macros` / Compile / exportedModule).value, (`pkg` / Compile / exportedModule).value, @@ -3338,6 +3353,7 @@ lazy val `runtime-compiler` = (`engine-common` / Compile / exportedModule).value, (`pkg` / Compile / exportedModule).value, (`runtime-parser` / Compile / exportedModule).value, + (`runtime-compiler-dump` / Compile / exportedModule).value, (`syntax-rust-definition` / Compile / exportedModule).value, (`scala-libs-wrapper` / Compile / exportedModule).value, (`persistance` / Compile / exportedModule).value, @@ -3393,11 +3409,61 @@ lazy val `runtime-compiler` = } ) .dependsOn(`runtime-parser`) + .dependsOn(`runtime-compiler-dump`) .dependsOn(pkg) .dependsOn(`engine-common`) .dependsOn(editions) .dependsOn(`persistance-dsl` % "provided") +/** This project contains only a single service (interface) definition. + */ +lazy val `runtime-compiler-dump` = + (project in file("engine/runtime-compiler-dump")) + .enablePlugins(JPMSPlugin) + .settings( + frgaalJavaCompilerSetting, + javaModuleName := "org.enso.runtime.compiler.dump", + Compile / internalModuleDependencies := { + val transitiveDeps = + (`runtime-parser` / Compile / internalModuleDependencies).value + Seq( + (`runtime-parser` / Compile / exportedModule).value + ) ++ transitiveDeps + } + ) + .dependsOn(`runtime-parser`) + +/** This is a standalone project that is not compiled with Frgaal on purpose. + * It depends on jdk.internal.vm.compiler module, which cannot be included in Frgaal. + * It includes a service provider for service definition in `runtime-compiler-dump`. + */ +lazy val `runtime-compiler-dump-igv` = + (project in file("engine/runtime-compiler-dump-igv")) + .enablePlugins(JPMSPlugin) + .settings( + scalaModuleDependencySetting, + javaModuleName := "org.enso.runtime.compiler.dump.igv", + Compile / internalModuleDependencies := { + val transitiveDeps = + (`runtime-compiler` / Compile / internalModuleDependencies).value + Seq( + (`runtime-compiler` / Compile / exportedModule).value + ) ++ transitiveDeps + }, + Compile / moduleDependencies ++= Seq( + "org.slf4j" % "slf4j-api" % slf4jVersion + ), + Compile / addExports ++= { + Map( + "jdk.internal.vm.compiler/org.graalvm.graphio" -> Seq( + javaModuleName.value + ) + ) + } + ) + .dependsOn(`runtime-compiler-dump`) + .dependsOn(`runtime-compiler`) + lazy val `runtime-suggestions` = (project in file("engine/runtime-suggestions")) .enablePlugins(JPMSPlugin) @@ -3463,6 +3529,7 @@ lazy val `runtime-instrument-common` = (`refactoring-utils` / Compile / exportedModule).value, (`runtime` / Compile / exportedModule).value, (`runtime-compiler` / Compile / exportedModule).value, + (`runtime-compiler-dump` / Compile / exportedModule).value, (`runtime-parser` / Compile / exportedModule).value, (`runtime-suggestions` / Compile / exportedModule).value, (`text-buffer` / Compile / exportedModule).value, @@ -3492,6 +3559,7 @@ lazy val `runtime-instrument-id-execution` = Compile / internalModuleDependencies := Seq( (`runtime` / Compile / exportedModule).value, (`runtime-compiler` / Compile / exportedModule).value, + (`runtime-compiler-dump` / Compile / exportedModule).value, (`polyglot-api` / Compile / exportedModule).value ) ) diff --git a/docs/runtime/compiler-ir.md b/docs/runtime/compiler-ir.md index 3fd435ee30f1..f841a7caeedf 100644 --- a/docs/runtime/compiler-ir.md +++ b/docs/runtime/compiler-ir.md @@ -9,15 +9,75 @@ multiple passes. Every pass is a class implementing the See [Runtime roadmap - static analysis](../runtime-roadmap.md#static-analysis) for future goals. -## Visualization - -The IR can be visualized using `--vm.D=enso.compiler.dumpIr` system property. -This will output a `.dot` file in [GraphViz](www.graphviz.org) format in the -`ir-dumps` directory for each IR in the program. The _dot_ file format is a -minimal textual format, that can be converted to a graphical representation -using the `dot` command from the GraphViz package. For example, on Ubuntu, -install `dot` with `sudo apt install graphviz`. Then, convert the `.dot` file to -a `.svg` image with `dot -Tsvg -o .svg .dot`. An example is: -![image.svg](https://github.com/user-attachments/assets/26ab8415-72cf-46da-bc63-f475e9fa628e) - -See `org.enso.compiler.dump.IRDumper`. +## Dumping IR + +The IR can be visualized using the `enso.compiler.dumpIr` system property. The +value of the property is a substring of a module name to dump. IRs are dumped +into the [IGV tool](https://www.graalvm.org/latest/tools/igv/) in a similar way +to how GraalVM graphs are dumped, which is documented in +[enso4igv](https://github.com/enso-org/enso/blob/2e714a70ddf12456e9f3fa9e132fd2ac43aa3b77/tools/enso4igv/IGV.md#using-the-igv). + +When using the `enso.compiler.dumpIr` property, one has to add +`--add-exports jdk.internal.vm.compiler/org.graalvm.graphio=org.enso.runtime.compiler.dump.igv` +to the `JAVA_OPTS` env var, because the IGV dumper uses an internal package of +GraalVM JDK's module which is not exported by default. + +Usage example: + +``` +$ env JAVA_OPTS='--add-exports jdk.internal.vm.compiler/org.graalvm.graphio=org.enso.runtime.compiler.dump.igv' ./built-distribution/*/bin/enso --vm.D enso.compiler.dumpIr=Vector --no-ir-caches --run tmp.enso +``` + +The IR graphs are dumped directly to IGV, if it is running, or to the `ir-dumps` +directory in the +[BGV](https://www.graalvm.org/graphio/javadoc/jdk/graal/compiler/graphio/package-summary.html) +format. + +### Description of the graphs + +For a module, multiple graphs are dumped. Names of the graphs correspond to the +names of the Compiler passes, as can be seen on the screenshot: +![1](https://github.com/user-attachments/assets/eb9bc883-c482-4461-8e38-0b615bec2e83) + +Opening the first graph for the `Vector` module is overwhelming, since it has +more than 3000 nodes: +![2](https://github.com/user-attachments/assets/296c0b63-a28c-4b9b-bed7-22c52f9f1cd3) +However, nodes are structured in _blocks_. Blocks can be seen on the right side +in the `Control Flow` tool window. + +Zoom into a particular block: + +- Double-click on the block with id 1 in the `Control Flow` tool window. +- Click on the `Zoom to Selection` button in the toolbar. + ![3](https://github.com/user-attachments/assets/dd8a4c7f-b8e4-4356-b1fd-64fa7824134c) + +Below the tab, there are all the passes displayed as points (_Phase toolbar_). +Hovering over a point shows the pass name: +![4](https://github.com/user-attachments/assets/b5449f4c-de35-46d4-8fee-e8963cf3d4f1) + +Click-and-drag the mouse to select a region of passes. This will display the +difference in the graph between those passes. In the following screenshot, we +can see the difference between the very first pass, and `MethodCalls` pass: +![5](https://github.com/user-attachments/assets/b320d706-e074-4509-a836-5e88e5305a55) +Orange nodes represent those with changed properties. + +Clicking on a single changed node, we can see that there is `NEW_passData` +property: +![6](https://github.com/user-attachments/assets/e3ce8f3c-a227-4626-96f8-910dc4d4c31a) +In this case, we can see that `AliasMetadata.ChildScope` passData was added to +that node between the selected passes. + +In the following screenshot, we can see that the `8 Function$Lambda` node was +removed (displayed in red) between passes `15: AliasAnalysis` and +`18: SuspendedArguments`: +![7](https://github.com/user-attachments/assets/21164dec-5a2b-449a-a335-9f0f4b60e4d0) +Nodes that were added are displayed in green. + +Clicking on a single node, the points representing passes in the _Phase toolbar_ +change colors: + +- White: No change +- Orange: Property on the node changed. +- Black: Node was removed +- Green: Node was added + ![8](https://github.com/user-attachments/assets/befc2083-e262-4933-90bc-f1729387d1ee) diff --git a/engine/common/src/main/java/org/enso/common/RuntimeOptions.java b/engine/common/src/main/java/org/enso/common/RuntimeOptions.java index 65bec92ef9bb..ffe3fbff1705 100644 --- a/engine/common/src/main/java/org/enso/common/RuntimeOptions.java +++ b/engine/common/src/main/java/org/enso/common/RuntimeOptions.java @@ -142,6 +142,8 @@ private RuntimeOptions() {} private static final OptionDescriptor WARNINGS_LIMIT_DESCRIPTOR = OptionDescriptor.newBuilder(WARNINGS_LIMIT_KEY, WARNINGS_LIMIT).build(); + public static final String IR_DUMPER_SYSTEM_PROP = "enso.compiler.dumpIr"; + public static final OptionDescriptors OPTION_DESCRIPTORS = OptionDescriptors.create( Arrays.asList( diff --git a/engine/runtime-compiler-dump-igv/src/main/java/module-info.java b/engine/runtime-compiler-dump-igv/src/main/java/module-info.java new file mode 100644 index 000000000000..0418652630cb --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/module-info.java @@ -0,0 +1,10 @@ +module org.enso.runtime.compiler.dump.igv { + requires jdk.internal.vm.compiler; + requires org.enso.runtime.parser; + requires org.enso.runtime.compiler.dump; + requires org.slf4j; + requires scala.library; + + provides org.enso.compiler.dump.service.IRDumpFactoryService with + org.enso.compiler.dump.igv.IGVDumperFactory; +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTBlock.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTBlock.java new file mode 100644 index 000000000000..00c06043b79a --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTBlock.java @@ -0,0 +1,56 @@ +package org.enso.compiler.dump.igv; + +import java.util.ArrayList; +import java.util.List; + +final class ASTBlock { + private final int id; + private final List nodes; + private final List successors; + + ASTBlock(int id, List nodes, List successors) { + this.id = id; + this.nodes = nodes; + this.successors = successors; + } + + public int getId() { + return id; + } + + public List getNodes() { + return nodes; + } + + public List getSuccessors() { + return successors; + } + + static final class Builder { + private int id; + private final List nodes = new ArrayList<>(); + private final List successors = new ArrayList<>(); + + static Builder fromId(int id) { + var bldr = new Builder(); + bldr.id = id; + return bldr; + } + + Builder addNode(ASTNode node) { + nodes.add(node); + return this; + } + + void addSuccessor(ASTBlock successor) { + successors.add(successor); + } + + ASTBlock build() { + if (nodes.isEmpty()) { + throw new IllegalArgumentException("Cannot build an empty block"); + } + return new ASTBlock(id, nodes, successors); + } + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTDumpStructure.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTDumpStructure.java new file mode 100644 index 000000000000..b179c2b999f8 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTDumpStructure.java @@ -0,0 +1,278 @@ +package org.enso.compiler.dump.igv; + +import java.net.URI; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import org.enso.compiler.dump.igv.ASTEdge.EdgeType; +import org.graalvm.graphio.GraphBlocks; +import org.graalvm.graphio.GraphElements; +import org.graalvm.graphio.GraphLocations; +import org.graalvm.graphio.GraphStructure; + +final class ASTDumpStructure + implements GraphStructure>, + GraphBlocks, + GraphElements, + GraphLocations { + + @Override + public EnsoModuleAST graph(EnsoModuleAST currentGraph, Object obj) { + if (obj instanceof EnsoModuleAST ensoAST) { + return ensoAST; + } + return null; + } + + @Override + public Iterable nodes(EnsoModuleAST graph) { + return graph.getNodes(); + } + + @Override + public int nodesCount(EnsoModuleAST graph) { + return graph.getNodes().size(); + } + + @Override + public int nodeId(ASTNode node) { + return node.getId(); + } + + @Override + public boolean nodeHasPredecessor(ASTNode node) { + return false; + } + + @Override + public void nodeProperties( + EnsoModuleAST graph, ASTNode node, Map properties) { + properties.putAll(node.getProperties()); + } + + @Override + public ASTNode node(Object obj) { + if (obj instanceof ASTNode astNode) { + return astNode; + } + return null; + } + + @Override + public ASTNodeClass nodeClass(Object obj) { + if (obj instanceof ASTNodeClass astNodeClass) { + return astNodeClass; + } + return null; + } + + @Override + public ASTNodeClass classForNode(ASTNode node) { + return node.getNodeClass(); + } + + @Override + public String nameTemplate(ASTNodeClass nodeClass) { + return "{p#label}"; + } + + @Override + public Object nodeClassType(ASTNodeClass nodeClass) { + return nodeClass.getClass(); + } + + @Override + public List portInputs(ASTNodeClass nodeClass) { + return List.of(); + } + + @Override + public List portOutputs(ASTNodeClass nodeClass) { + return nodeClass.node().getEdges(); + } + + @Override + public int portSize(List port) { + return port.size(); + } + + @Override + public boolean edgeDirect(List port, int index) { + return true; + } + + @Override + public String edgeName(List port, int index) { + return port.get(index).label(); + } + + @Override + public Object edgeType(List port, int index) { + return EdgeType.EDGE_TYPE; + } + + @Override + public Collection edgeNodes( + EnsoModuleAST graph, ASTNode node, List port, int index) { + return List.of(port.get(index).node()); + } + + @Override + public Collection blocks(EnsoModuleAST graph) { + return graph.getBlocks(); + } + + @Override + public int blockId(ASTBlock block) { + return block.getId(); + } + + @Override + public Collection blockNodes(EnsoModuleAST info, ASTBlock block) { + return block.getNodes(); + } + + @Override + public Collection blockSuccessors(ASTBlock block) { + return block.getSuccessors(); + } + + @Override + public ASTMethod method(Object obj) { + if (obj instanceof ASTMethod m) { + return m; + } + return null; + } + + @Override + public byte[] methodCode(ASTMethod method) { + return new byte[0]; + } + + @Override + public int methodModifiers(ASTMethod method) { + return 0; + } + + @Override + public ASTMethod.Signature methodSignature(ASTMethod method) { + return ASTMethod.Signature.NONE; + } + + @Override + public String methodName(ASTMethod method) { + return method.getName(); + } + + @Override + public Object methodDeclaringClass(ASTMethod method) { + return ASTMethod.class; + } + + @Override + public Object field(Object object) { + return null; + } + + @Override + public int fieldModifiers(Object field) { + return 0; + } + + @Override + public String fieldTypeName(Object field) { + return null; + } + + @Override + public String fieldName(Object field) { + return null; + } + + @Override + public Object fieldDeclaringClass(Object field) { + return null; + } + + @Override + public ASTMethod.Signature signature(Object object) { + return object instanceof ASTMethod.Signature s ? s : null; + } + + @Override + public int signatureParameterCount(ASTMethod.Signature signature) { + return 0; + } + + @Override + public String signatureParameterTypeName(ASTMethod.Signature signature, int index) { + return null; + } + + @Override + public String signatureReturnTypeName(ASTMethod.Signature signature) { + return null; + } + + @Override + public ASTLocation nodeSourcePosition(Object object) { + if (object instanceof ASTLocation location) { + return location; + } + if (object instanceof ASTNode node) { + return node.getLocation(); + } + return null; + } + + @Override + public ASTMethod nodeSourcePositionMethod(ASTLocation pos) { + return ASTMethod.UNKNOWN; + } + + @Override + public ASTLocation nodeSourcePositionCaller(ASTLocation pos) { + return null; + } + + @Override + public int nodeSourcePositionBCI(ASTLocation pos) { + return 0; + } + + @Override + public StackTraceElement methodStackTraceElement(ASTMethod method, int bci, ASTLocation pos) { + return null; + } + + @Override + public Iterable methodLocation(ASTMethod method, int bci, ASTLocation pos) { + return List.of(pos); + } + + @Override + public String locationLanguage(ASTLocation location) { + return "enso"; + } + + @Override + public URI locationURI(ASTLocation location) { + return location.getLocationUri(); + } + + @Override + public int locationLineNumber(ASTLocation location) { + return location.getLineNum(); + } + + @Override + public int locationOffsetStart(ASTLocation location) { + return location.getOffsetStart(); + } + + @Override + public int locationOffsetEnd(ASTLocation location) { + return location.getOffsetEnd(); + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTEdge.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTEdge.java new file mode 100644 index 000000000000..75be82bc5f54 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTEdge.java @@ -0,0 +1,13 @@ +package org.enso.compiler.dump.igv; + +/** + * Edge is unidirectional - from paretn to child + * + * @param node Pointer to child + * @param label + */ +record ASTEdge(ASTNode node, String label) { + enum EdgeType { + EDGE_TYPE; + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTLocation.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTLocation.java new file mode 100644 index 000000000000..7b5493945eda --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTLocation.java @@ -0,0 +1,51 @@ +package org.enso.compiler.dump.igv; + +import java.io.File; +import java.net.URI; +import org.enso.compiler.core.ir.IdentifiedLocation; + +final class ASTLocation { + private final int lineNum; + // May be null + private final URI locationUri; + private final int offsetStart; + private final int offsetEnd; + + private ASTLocation(int lineNum, URI locationUri, int offsetStart, int offsetEnd) { + this.lineNum = lineNum; + this.locationUri = locationUri; + this.offsetStart = offsetStart; + this.offsetEnd = offsetEnd; + } + + public static ASTLocation fromIdentifiedLocation(IdentifiedLocation loc, File srcFile) { + int offStart = -1; + int offEnd = -1; + int lineNum = 1; + URI uri = null; + if (loc != null) { + offStart = loc.start(); + offEnd = loc.end(); + } + if (srcFile != null) { + uri = srcFile.toURI(); + } + return new ASTLocation(lineNum, uri, offStart, offEnd); + } + + public int getLineNum() { + return lineNum; + } + + public int getOffsetStart() { + return offsetStart; + } + + public int getOffsetEnd() { + return offsetEnd; + } + + public URI getLocationUri() { + return locationUri; + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTMethod.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTMethod.java new file mode 100644 index 000000000000..e4af51c00bd9 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTMethod.java @@ -0,0 +1,15 @@ +package org.enso.compiler.dump.igv; + +final class ASTMethod { + static final ASTMethod UNKNOWN = new ASTMethod(); + + public String getName() { + return ""; + } + + static final class Signature { + static final Signature NONE = new Signature(); + + private Signature() {} + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNode.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNode.java new file mode 100644 index 000000000000..c1d9f34a6618 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNode.java @@ -0,0 +1,135 @@ +package org.enso.compiler.dump.igv; + +import java.io.File; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.MetadataStorage; + +final class ASTNode { + + private final List edges = new ArrayList<>(); + private final int id; + private final Object object; + // Frozen + private final Map properties; + private final ASTNodeClass nodeClass; + private final List children = new ArrayList<>(); + // May be null; + private final ASTLocation location; + + ASTNode(int id, Object object, Map props, ASTLocation location) { + this.id = id; + this.object = object; + props.put("nodeSourcePosition", location); + this.properties = props; + this.nodeClass = new ASTNodeClass(this); + this.location = location; + if (object instanceof IR && location == null) { + throw new IllegalArgumentException("IR object must have location"); + } + } + + public Object getObject() { + return object; + } + + public int getId() { + return id; + } + + public List getEdges() { + return edges; + } + + public ASTNodeClass getNodeClass() { + return nodeClass; + } + + public Map getProperties() { + return properties; + } + + public ASTLocation getLocation() { + return location; + } + + public void addChild(ASTNode child) { + children.add(child); + } + + public void addEdge(ASTEdge edge) { + edges.add(edge); + } + + static final class Builder { + private int id = -1; + private Object object; + private final Map properties = new LinkedHashMap<>(); + private ASTLocation location; + + public static Builder fromIr(IR ir, File srcFile) { + var bldr = new Builder(); + var label = Utils.label(ir); + var location = ASTLocation.fromIdentifiedLocation(ir.identifiedLocation(), srcFile); + bldr.object = ir; + bldr.location = location; + bldr.property("label", label); + bldr.property("IRClassName", ir.getClass().getName()); + bldr.property("location", ir.identifiedLocation()); + bldr.property("passData", ir.passData()); + bldr.property("uuid", ir.getId()); + return bldr; + } + + public Builder id(int id) { + this.id = id; + return this; + } + + public Builder property(String key, Object value) { + properties.put(key, value); + return this; + } + + public Builder location(ASTLocation location) { + this.location = location; + return this; + } + + public ASTNode build() { + if (id == -1) { + throw new IllegalArgumentException("ID must be set"); + } + if (object == null) { + throw new IllegalArgumentException("IR must be set"); + } + return new ASTNode(id, object, properties, location); + } + + private static String simpleClassName(Object obj) { + return Arrays.stream(obj.getClass().getName().split("\\.")) + .dropWhile( + item -> + item.equals("org") + || item.equals("enso") + || item.equals("compiler") + || item.equals("core")) + .collect(Collectors.joining(".")); + } + + private static boolean isPassDataEmpty(MetadataStorage passData) { + int[] counter = new int[] {0}; + passData.map( + (pass, data) -> { + counter[0]++; + return null; + }); + return counter[0] == 0; + } + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNodeClass.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNodeClass.java new file mode 100644 index 000000000000..9dc47d116bef --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/ASTNodeClass.java @@ -0,0 +1,3 @@ +package org.enso.compiler.dump.igv; + +record ASTNodeClass(ASTNode node) {} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/EnsoModuleAST.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/EnsoModuleAST.java new file mode 100644 index 000000000000..96bd5ee65422 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/EnsoModuleAST.java @@ -0,0 +1,511 @@ +package org.enso.compiler.dump.igv; + +import java.io.File; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.UUID; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.CallArgument; +import org.enso.compiler.core.ir.DefinitionArgument; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Function; +import org.enso.compiler.core.ir.Literal.Number; +import org.enso.compiler.core.ir.Literal.Text; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.core.ir.Name; +import org.enso.compiler.core.ir.Pattern; +import org.enso.compiler.core.ir.Type; +import org.enso.compiler.core.ir.expression.Application; +import org.enso.compiler.core.ir.expression.Case; +import org.enso.compiler.core.ir.expression.Comment; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.Definition.Data; +import org.enso.compiler.core.ir.module.scope.Export; +import org.enso.compiler.core.ir.module.scope.Import; +import org.enso.compiler.core.ir.module.scope.definition.Method; +import org.enso.compiler.core.ir.module.scope.imports.Polyglot; + +/** + * Implemented only for dumping the AST to IGV. Heavily inspired by the internal {@code + * org.graalvm.compiler.truffle.compiler.TruffleAST}. + */ +final class EnsoModuleAST { + static final ASTDumpStructure AST_DUMP_STRUCTURE = new ASTDumpStructure(); + + private final ASTNode root; + + /** Underlying source file. May be null. */ + private final File srcFile; + + private final String moduleName; + + private final Map nodes = new HashMap<>(); + + /** List of blocks that are already built. */ + private final List blocks = new ArrayList<>(); + + /** Stack of blocks that are being built. */ + private final Queue blockStack = new ArrayDeque<>(); + + private final Map nodeIds; + + private EnsoModuleAST( + Module moduleIr, File srcFile, String moduleName, Map nodeIds) { + this.nodeIds = nodeIds; + this.srcFile = srcFile; + this.moduleName = moduleName; + this.root = buildTree(moduleIr); + } + + private EnsoModuleAST(Expression expr, String moduleName, Map nodeIds) { + this.nodeIds = nodeIds; + this.srcFile = null; + this.moduleName = moduleName; + this.root = buildTree(expr); + } + + /** + * @param srcFile Source file for the module. May be null. + * @param moduleName FQN of the module. + * @param nodeIds Mapping of IR node UUIDs to sequential IDs expected by the IGV. + */ + static EnsoModuleAST fromModuleIR( + Module module, File srcFile, String moduleName, Map nodeIds) { + return new EnsoModuleAST(module, srcFile, moduleName, nodeIds); + } + + static EnsoModuleAST fromExpressionIR( + Expression expr, String moduleName, Map nodeIds) { + return new EnsoModuleAST(expr, moduleName, nodeIds); + } + + public File getSrcFile() { + return srcFile; + } + + public String getModuleName() { + return moduleName; + } + + public List getNodes() { + return nodes.values().stream().toList(); + } + + public List getBlocks() { + return blocks; + } + + private void createEdge(ASTNode from, ASTNode to, String label) { + if (!nodes.containsKey(from.getId())) { + throw new IllegalArgumentException("Node " + from.getId() + " is not defined."); + } + if (!nodes.containsKey(to.getId())) { + throw new IllegalArgumentException("Node " + to.getId() + " is not defined."); + } + from.addChild(to); + from.addEdge(new ASTEdge(to, label)); + } + + private ASTNode buildTree(Module module) { + var root = newNode(module); + for (var i = 0; i < module.bindings().size(); i++) { + var bindingIr = module.bindings().apply(i); + var bindingNode = buildTree(bindingIr); + var edgeDescr = "binding[" + i + "]"; + createEdge(root, bindingNode, edgeDescr); + } + for (var i = 0; i < module.imports().size(); i++) { + var imp = module.imports().apply(i); + var impNode = buildTree(imp); + var edgeDescr = "import[" + i + "]"; + createEdge(root, impNode, edgeDescr); + } + for (var i = 0; i < module.exports().size(); i++) { + var export = module.exports().apply(i); + var exportNode = buildTree(export); + var edgeDescr = "export[" + i + "]"; + createEdge(root, exportNode, edgeDescr); + } + return root; + } + + private ASTNode buildTree(Import importIr) { + return switch (importIr) { + case Import.Module importModIr -> { + Map props = + Map.of( + "isSynthetic", importModIr.isSynthetic(), + "importName", importModIr.name().name(), + "isAll", importModIr.isAll(), + "hiddenName", importModIr.hiddenNames(), + "rename", importModIr.rename()); + yield newNode(importModIr, props); + } + case Polyglot polyImport -> { + Map props = + Map.of( + "entity", polyImport.entity(), + "visibleName", polyImport.getVisibleName(), + "rename", polyImport.rename()); + yield newNode(polyImport, props); + } + default -> throw unimpl(importIr); + }; + } + + private ASTNode buildTree(Export exportIr) { + return switch (exportIr) { + case Export.Module exportModIr -> { + Map props = + Map.of( + "isSynthetic", exportModIr.isSynthetic(), + "exportName", exportModIr.name().name()); + yield newNode(exportModIr, props); + } + default -> throw unimpl(exportIr); + }; + } + + private ASTNode buildTree(Definition definitionIr) { + return switch (definitionIr) { + case Method.Explicit explicitMethodIr -> { + startBlock(); + Map props = + Map.of( + "methodName", explicitMethodIr.methodName().name(), + "isStatic", explicitMethodIr.isStatic(), + "isPrivate", explicitMethodIr.isPrivate(), + "typeName", explicitMethodIr.typeName()); + var methodNode = newNode(explicitMethodIr, props); + var methodRefNode = buildTree(explicitMethodIr.methodReference()); + var bodyNode = buildTree(explicitMethodIr.body()); + createEdge(methodNode, bodyNode, "body"); + createEdge(methodNode, methodRefNode, "methodReference"); + endBlock(); + yield methodNode; + } + case Method.Conversion conversionMethod -> { + startBlock(); + Map props = + Map.of( + "methodName", conversionMethod.methodName().name(), + "isPrivate", conversionMethod.isPrivate()); + var methodNode = newNode(conversionMethod, props); + var methodRefNode = buildTree(conversionMethod.methodReference()); + var body = conversionMethod.body(); + var bodyNode = buildTree(body); + createEdge(methodNode, bodyNode, "body"); + createEdge(methodNode, methodRefNode, "methodReference"); + endBlock(); + yield methodNode; + } + case Method.Binding binding -> { + startBlock(); + Map props = + Map.of( + "methodName", binding.methodName().name(), + "isPrivate", binding.isPrivate()); + var methodNode = newNode(binding, props); + var methodRefNode = buildTree(binding.methodReference()); + createEdge(methodNode, methodRefNode, "methodReference"); + for (var i = 0; i < binding.arguments().size(); i++) { + var arg = binding.arguments().apply(i); + var argNode = buildTree(arg); + createEdge(methodNode, argNode, "arg[" + i + "]"); + } + var body = binding.body(); + var bodyNode = buildTree(body); + createEdge(methodNode, bodyNode, "body"); + endBlock(); + yield methodNode; + } + case Definition.Type type -> { + startBlock(); + Map props = Map.of("typeName", type.name().name()); + var typeNode = newNode(type, props); + for (var i = 0; i < type.members().size(); i++) { + var member = type.members().apply(i); + var memberNode = buildTree(member); + createEdge(typeNode, memberNode, "member[" + i + "]"); + } + endBlock(); + yield typeNode; + } + case Definition.SugaredType type -> { + Map props = Map.of("typeName", type.name().name()); + var node = newNode(type, props); + for (var i = 0; i < type.arguments().size(); i++) { + var arg = type.arguments().apply(i); + var argNode = buildTree(arg); + createEdge(node, argNode, "arg[" + i + "]"); + } + yield node; + } + case Name.GenericAnnotation genericAnnotation -> { + Map props = + Map.of( + "annotationName", genericAnnotation.name(), + "isMethod", genericAnnotation.isMethod()); + var anotNode = newNode(genericAnnotation, props); + var expr = genericAnnotation.expression(); + var exprNode = buildTree(expr); + createEdge(anotNode, exprNode, "expression"); + yield anotNode; + } + case Name.BuiltinAnnotation builtinAnnotation -> { + Map props = Map.of("annotationName", builtinAnnotation.name()); + yield newNode(builtinAnnotation, props); + } + case Type.Ascription ascription -> { + var ascrNode = newNode(ascription); + var typed = ascription.typed(); + var typedNode = buildTree(typed); + createEdge(ascrNode, typedNode, "typed"); + var signature = ascription.signature(); + var signatureNode = buildTree(signature); + createEdge(ascrNode, signatureNode, "signature"); + yield ascrNode; + } + case Comment.Documentation doc -> { + Map props = Map.of("doc", doc.doc()); + yield newNode(doc, props); + } + default -> throw unimpl(definitionIr); + }; + } + + private ASTNode buildTree(Data atomCons) { + Map props = Map.of("consName", atomCons.name().name()); + var consNode = newNode(atomCons, props); + for (var i = 0; i < atomCons.arguments().size(); i++) { + var arg = atomCons.arguments().apply(i); + var argNode = buildTree(arg); + createEdge(consNode, argNode, "arg[" + i + "]"); + } + return consNode; + } + + private ASTNode buildTree(Expression expression) { + return switch (expression) { + case Expression.Block block -> { + var blockNode = newNode(block); + for (var i = 0; i < block.expressions().size(); i++) { + var expr = block.expressions().apply(i); + var exprNode = buildTree(expr); + createEdge(blockNode, exprNode, "expression[" + i + "]"); + } + var retValNode = buildTree(block.returnValue()); + createEdge(blockNode, retValNode, "returnValue"); + yield blockNode; + } + case Case.Expr caseExpr -> { + Map props = Map.of("isNested", caseExpr.isNested()); + var caseNode = newNode(caseExpr, props); + var scrutineeNode = buildTree(caseExpr.scrutinee()); + createEdge(caseNode, scrutineeNode, "scrutinee"); + for (var i = 0; i < caseExpr.branches().size(); i++) { + var branch = caseExpr.branches().apply(i); + var branchNode = buildTree(branch); + createEdge(caseNode, branchNode, "branch[" + i + "]"); + } + yield caseNode; + } + case Case.Branch caseBranch -> { + Map props = Map.of("isTerminal", caseBranch.terminalBranch()); + var branchNode = newNode(caseBranch, props); + var patternNode = buildTree(caseBranch.pattern()); + createEdge(branchNode, patternNode, "pattern"); + var exprNode = buildTree(caseBranch.expression()); + createEdge(branchNode, exprNode, "expression"); + yield branchNode; + } + case Application.Prefix prefixApp -> { + Map props = + Map.of("hasDefaultsSuspended", prefixApp.hasDefaultsSuspended()); + var prefixAppNode = newNode(prefixApp, props); + var funcNode = buildTree(prefixApp.function()); + createEdge(prefixAppNode, funcNode, "function"); + for (var i = 0; i < prefixApp.arguments().size(); i++) { + var arg = prefixApp.arguments().apply(i); + var argNode = buildTree(arg); + createEdge(prefixAppNode, argNode, "arg[" + i + "]"); + } + yield prefixAppNode; + } + case Function.Lambda lambda -> { + var lambdaNode = newNode(lambda); + var bodyNode = buildTree(lambda.body()); + createEdge(lambdaNode, bodyNode, "body"); + for (var i = 0; i < lambda.arguments().size(); i++) { + var arg = lambda.arguments().apply(i); + var argNode = buildTree(arg); + createEdge(lambdaNode, argNode, "arg[" + i + "]"); + } + yield lambdaNode; + } + case Expression.Binding exprBinding -> { + Map props = Map.of("bindingName", exprBinding.name().name()); + var node = newNode(exprBinding, props); + var exprNode = buildTree(exprBinding.expression()); + createEdge(node, exprNode, "expression"); + yield node; + } + case Number number -> { + Map props = Map.of("value", number.value()); + yield newNode(number, props); + } + case Text text -> { + Map props = Map.of("text", text.text()); + yield newNode(text, props); + } + case Name.Literal literal -> { + Map props = + Map.of( + "literalName", literal.name(), + "isMethod", literal.isMethod(), + "originalName", literal.originalName()); + yield newNode(literal, props); + } + case Name.Qualified qualName -> { + Map props = + Map.of( + "qualName", qualName.name(), + "isMethod", qualName.isMethod(), + "parts", qualName.parts()); + yield newNode(qualName, props); + } + case Name.MethodReference methodRef -> { + Map props = + Map.of( + "methodName", methodRef.methodName().name(), + "typePointer", methodRef.typePointer()); + yield newNode(methodRef, props); + } + default -> newNode(expression); + }; + } + + private ASTNode buildTree(Pattern pattern) { + return switch (pattern) { + case Pattern.Constructor constrPat -> { + Map props = Map.of("constructor", constrPat.constructor().name()); + var node = newNode(constrPat, props); + for (var i = 0; i < constrPat.fields().size(); i++) { + var field = constrPat.fields().apply(i); + var fieldNode = buildTree(field); + createEdge(node, fieldNode, "field[" + i + "]"); + } + yield node; + } + case Pattern.Type tp -> { + var node = newNode(tp); + var nameNode = buildTree(tp.name()); + var tpeNode = buildTree(tp.tpe()); + createEdge(node, nameNode, "name"); + createEdge(node, tpeNode, "tpe"); + yield node; + } + case Pattern.Literal litPat -> { + var node = newNode(litPat); + var litNode = buildTree(litPat.literal()); + createEdge(node, litNode, "literal"); + yield node; + } + case Pattern.Name name -> { + Map props = Map.of("patternName", name.name().name()); + yield newNode(name, props); + } + case Pattern.Documentation doc -> { + Map props = Map.of("doc", doc.doc()); + yield newNode(doc, props); + } + default -> throw unimpl(pattern); + }; + } + + private ASTNode buildTree(CallArgument argument) { + return switch (argument) { + case CallArgument.Specified specifiedArg -> { + Map props = Map.of("argName", specifiedArg.name()); + var node = newNode(specifiedArg, props); + var valueNode = buildTree(specifiedArg.value()); + createEdge(node, valueNode, "value"); + yield node; + } + default -> throw unimpl(argument); + }; + } + + private ASTNode buildTree(DefinitionArgument argument) { + return switch (argument) { + case DefinitionArgument.Specified specifiedArg -> { + Map props = + Map.of( + "argName", specifiedArg.name().name(), + "suspended", specifiedArg.suspended()); + var node = newNode(specifiedArg, props); + if (specifiedArg.ascribedType().isDefined()) { + var ascribedTypeNode = buildTree(specifiedArg.ascribedType().get()); + createEdge(node, ascribedTypeNode, "ascribedType"); + } + if (specifiedArg.defaultValue().isDefined()) { + var defaultValueNode = buildTree(specifiedArg.defaultValue().get()); + createEdge(node, defaultValueNode, "defaultValue"); + } + yield node; + } + default -> throw unimpl(argument); + }; + } + + private ASTNode newNode(IR ir, Map props) { + ASTNode.Builder bldr = ASTNode.Builder.fromIr(ir, srcFile); + var nodeId = nodeIds.get(ir.getId()); + if (nodeId == null) { + var lastSeqId = nodeIds.size(); + nodeIds.put(ir.getId(), lastSeqId); + nodeId = lastSeqId; + } + bldr.id(nodeId); + props.forEach(bldr::property); + var node = bldr.build(); + assert !nodes.containsKey(node.getId()); + nodes.put(node.getId(), node); + if (currentBlockBldr() != null) { + currentBlockBldr().addNode(node); + } + return node; + } + + private ASTNode newNode(IR ir) { + return newNode(ir, Map.of()); + } + + private void startBlock() { + var blockBldr = ASTBlock.Builder.fromId(blocks.size()); + blockStack.add(blockBldr); + } + + private ASTBlock.Builder currentBlockBldr() { + return blockStack.peek(); + } + + private void endBlock() { + assert !blockStack.isEmpty(); + var block = blockStack.remove().build(); + var parentBlock = blockStack.peek(); + if (parentBlock != null) { + parentBlock.addSuccessor(block); + } + blocks.add(block); + } + + private static RuntimeException unimpl(Object obj) { + throw new UnsupportedOperationException( + "Not implemented IR processing for class: " + obj.getClass().getName()); + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumper.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumper.java new file mode 100644 index 000000000000..25c3fcdf9c20 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumper.java @@ -0,0 +1,177 @@ +package org.enso.compiler.dump.igv; + +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.channels.SocketChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.dump.service.IRDumper; +import org.graalvm.graphio.GraphOutput; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class IGVDumper implements IRDumper { + + private static final String DEFAULT_DUMP_DIR = "ir-dumps"; + private static final Logger LOGGER = LoggerFactory.getLogger(IGVDumper.class); + private static final int DEFAULT_IGV_PORT = 4445; + private final String moduleName; + // Created lazily + private GraphOutput graphOutput; + private int currGraphId; + private boolean groupCreated; + + private final Map nodeIds = new HashMap<>(); + + private IGVDumper(String moduleName) { + this.moduleName = moduleName; + } + + static IGVDumper createForModule(String moduleName) { + return new IGVDumper(moduleName); + } + + private GraphOutput graphOutput() { + if (graphOutput == null) { + var channel = createChannel(moduleName); + try { + graphOutput = + GraphOutput.newBuilder(EnsoModuleAST.AST_DUMP_STRUCTURE) + .blocks(EnsoModuleAST.AST_DUMP_STRUCTURE) + .elementsAndLocations( + EnsoModuleAST.AST_DUMP_STRUCTURE, EnsoModuleAST.AST_DUMP_STRUCTURE) + .attr("type", "Enso IR") + .build(channel); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to create graph output for module " + moduleName, e); + } + } + return graphOutput; + } + + private static WritableByteChannel createChannel(String moduleName) { + WritableByteChannel channel; + try { + channel = SocketChannel.open(new InetSocketAddress(DEFAULT_IGV_PORT)); + LOGGER.info("Connected to IGV"); + } catch (IOException e) { + var outPath = outputForModule(moduleName); + LOGGER.info("Failed to connect to IGV. Graph will be dumped in {}", outPath); + channel = createFileChannel(outPath); + } + return channel; + } + + @Override + public void dumpModule(Module ir, String graphName, File srcFile, String afterPass) { + assert graphName.equals(this.moduleName); + dumpTask(ir, graphName, srcFile, afterPass); + } + + @Override + public void dumpExpression(Expression expr, String graphName, String afterPass) { + dumpTask(expr, graphName, null, afterPass); + } + + private void dumpTask(IR ir, String moduleName, File srcFile, String afterPass) { + LOGGER.trace("[{}] Creating EnsoModuleAST after pass {}", moduleName, afterPass); + EnsoModuleAST moduleAst; + if (ir instanceof Module moduleIr) { + moduleAst = EnsoModuleAST.fromModuleIR(moduleIr, srcFile, moduleName, nodeIds); + } else if (ir instanceof Expression expr) { + moduleAst = EnsoModuleAST.fromExpressionIR(expr, moduleName, nodeIds); + } else { + throw new IllegalArgumentException("Unsupported IR type: " + ir.getClass()); + } + try { + if (!groupCreated) { + var groupProps = groupProps(moduleName, moduleAst); + graphOutput().beginGroup(moduleAst, moduleName, moduleName, null, 0, groupProps); + groupCreated = true; + } + LOGGER.trace("[{}] Printing module AST with ID {}", moduleName, currGraphId); + var graphProps = graphProps(afterPass); + graphOutput().print(moduleAst, graphProps, currGraphId, "%s", afterPass); + } catch (IOException e) { + LOGGER.error("[{}] Failed to dump the graph for pass {}", moduleName, afterPass); + throw new RuntimeException(e); + } + currGraphId++; + LOGGER.trace("[{}] Dumped after pass {}", moduleName, afterPass); + } + + private static Map groupProps(String moduleName, EnsoModuleAST graph) { + var props = new HashMap(); + props.put("moduleName", moduleName); + props.put("date", LocalDateTime.now()); + props.put("srcFile", graph.getSrcFile() == null ? null : graph.getSrcFile().getAbsolutePath()); + return props; + } + + private static Map graphProps(String afterPass) { + var props = new HashMap(); + props.put("passName", afterPass); + return props; + } + + @Override + public void close() { + if (groupCreated) { + assert graphOutput != null; + try { + graphOutput.endGroup(); + } catch (IOException e) { + LOGGER.error("[%s] Failed to end the group".formatted(moduleName), e); + return; + } + graphOutput.close(); + LOGGER.trace("[{}] Graph dumped", moduleName); + } else { + LOGGER.trace("[{}] No graphs to dump", moduleName); + } + } + + private static Path outputForModule(String moduleName) { + var irDumpsDir = Path.of(DEFAULT_DUMP_DIR); + if (!irDumpsDir.toFile().exists()) { + try { + Files.createDirectory(irDumpsDir); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + var outPath = irDumpsDir.resolve(moduleName + ".bgv"); + if (!outPath.toFile().exists()) { + try { + Files.createFile(outPath); + } catch (IOException e) { + LOGGER.error("Failed to create output: {}", outPath, e); + } + } + return outPath; + } + + private static WritableByteChannel createFileChannel(Path path) { + try { + return Files.newByteChannel( + path, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to create byte channel to file " + path.toAbsolutePath(), e); + } + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumperFactory.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumperFactory.java new file mode 100644 index 000000000000..0a9c13b92a68 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/IGVDumperFactory.java @@ -0,0 +1,42 @@ +package org.enso.compiler.dump.igv; + +import org.enso.compiler.dump.service.IRDumpFactoryService; +import org.enso.compiler.dump.service.IRDumper; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class IGVDumperFactory implements IRDumpFactoryService { + private static final Logger LOGGER = LoggerFactory.getLogger(IGVDumperFactory.class); + private static final String GRAPHIO_PKG = "org.graalvm.graphio"; + private static final String COMPILER_MOD = "jdk.internal.vm.compiler"; + + public IGVDumperFactory() { + ensureInternalModuleIsExported(); + } + + private static void ensureInternalModuleIsExported() { + var internalCompilerMod = ModuleLayer.boot().findModule(COMPILER_MOD); + if (internalCompilerMod.isEmpty()) { + throw new IllegalStateException( + "Module " + COMPILER_MOD + " is not present. Is this a GraalVM JDK?"); + } + var thisModule = IGVDumperFactory.class.getModule(); + if (!internalCompilerMod.get().isExported(GRAPHIO_PKG, thisModule)) { + throw new IllegalStateException( + "Package " + + GRAPHIO_PKG + + " is not exported to this module. You must explicitly provide " + + "--add-exports %s/%s=%s" + .formatted(COMPILER_MOD, GRAPHIO_PKG, thisModule.getName())); + } + } + + @Override + public IRDumper create(String moduleName) { + LOGGER.trace("Creating IGV dumper for module {}", moduleName); + return IGVDumper.createForModule(moduleName); + } + + @Override + public void shutdown() {} +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/Utils.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/Utils.java new file mode 100644 index 000000000000..383ddea8dac3 --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/Utils.java @@ -0,0 +1,49 @@ +package org.enso.compiler.dump.igv; + +import org.enso.compiler.core.ir.DefinitionArgument; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Literal; +import org.enso.compiler.core.ir.Name; +import org.enso.compiler.core.ir.Pattern; +import org.enso.compiler.core.ir.module.scope.Definition; +import org.enso.compiler.core.ir.module.scope.Export; +import org.enso.compiler.core.ir.module.scope.Import; +import org.enso.compiler.core.ir.module.scope.definition.Method; + +final class Utils { + private Utils() {} + + static String label(Object obj) { + return switch (obj) { + case Literal.Text txt -> "Literal.Text ('" + txt.text() + "')"; + case Literal.Number num -> "Literal.Number (" + num.value() + ")"; + case Name.Literal lit -> "Name.Literal ('" + lit.name() + "')"; + case Definition.Type tp -> "Definition.Type ('" + tp.name().name() + "')"; + case Definition.SugaredType tp -> "Definition.SugaredType ('" + tp.name().name() + "')"; + case Import.Module imp -> "Import.Module ('" + imp.name().name() + "')"; + case Export.Module exp -> "Export.Module ('" + exp.name().name() + "')"; + case Method.Explicit m -> "Method.Explicit ('" + m.methodName().name() + "')"; + case Method.Conversion c -> "Method.Conversion ('" + c.methodName().name() + "')"; + case Method.Binding b -> "Method.Binding ('" + b.methodName().name() + "')"; + case DefinitionArgument.Specified arg -> "DefinitionArgument.Specified ('" + + arg.name().name() + + "')"; + case Expression.Binding b -> "Expression.Binding ('" + b.name().name() + "')"; + case Pattern.Name n -> "Pattern.Name ('" + n.name().name() + "')"; + case Pattern.Constructor c -> "Pattern.Constructor ('" + c.constructor().name() + "')"; + default -> defaultLabel(obj); + }; + } + + static String hash(Object obj) { + return Integer.toHexString(System.identityHashCode(obj)); + } + + private static String strippedClassName(String fqn) { + return fqn.replace("org.enso.compiler.core.ir.", ""); + } + + private static String defaultLabel(Object obj) { + return strippedClassName(obj.getClass().getName()); + } +} diff --git a/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/package-info.java b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/package-info.java new file mode 100644 index 000000000000..79a82d2450df --- /dev/null +++ b/engine/runtime-compiler-dump-igv/src/main/java/org/enso/compiler/dump/igv/package-info.java @@ -0,0 +1,6 @@ +/** + * Dumps the {@link org.enso.compiler.core.IR IR tree} into the IGV + * format. + */ +package org.enso.compiler.dump.igv; diff --git a/engine/runtime-compiler-dump/src/main/java/module-info.java b/engine/runtime-compiler-dump/src/main/java/module-info.java new file mode 100644 index 000000000000..c1d964abe20b --- /dev/null +++ b/engine/runtime-compiler-dump/src/main/java/module-info.java @@ -0,0 +1,7 @@ +module org.enso.runtime.compiler.dump { + requires org.enso.runtime.parser; + + exports org.enso.compiler.dump.service; + + uses org.enso.compiler.dump.service.IRDumpFactoryService; +} diff --git a/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpFactoryService.java b/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpFactoryService.java new file mode 100644 index 000000000000..5551399c96ba --- /dev/null +++ b/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpFactoryService.java @@ -0,0 +1,13 @@ +package org.enso.compiler.dump.service; + +/** A service that creates {@link org.enso.compiler.core.IR} dumpers for modules. */ +public interface IRDumpFactoryService { + static final IRDumpFactoryService DEFAULT = IRDumpSingleton.DEFAULT; + + String SYSTEM_PROP = "enso.compiler.dumpIr"; + String DEFAULT_DUMP_DIR = "ir-dumps"; + + IRDumper create(String moduleName); + + void shutdown(); +} diff --git a/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpSingleton.java b/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpSingleton.java new file mode 100644 index 000000000000..62e0b842621f --- /dev/null +++ b/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumpSingleton.java @@ -0,0 +1,37 @@ +package org.enso.compiler.dump.service; + +import java.io.File; +import java.util.ServiceLoader; +import org.enso.compiler.core.ir.Module; + +final class IRDumpSingleton { + static final IRDumpFactoryService DEFAULT = find(); + + private static IRDumpFactoryService find() { + IRDumpFactoryService service = new NoDumping(); + var loader = ServiceLoader.load(IRDumpFactoryService.class); + var it = loader.iterator(); + while (it.hasNext()) { + service = it.next(); + assert service != null; + break; + } + return service; + } + + private static final class NoDumping implements IRDumpFactoryService, IRDumper { + @Override + public IRDumper create(String moduleName) { + return this; + } + + @Override + public void shutdown() {} + + @Override + public void dumpModule(Module ir, String graphName, File srcFile, String afterPass) {} + + @Override + public void close() {} + } +} diff --git a/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumper.java b/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumper.java new file mode 100644 index 000000000000..567803f79b7c --- /dev/null +++ b/engine/runtime-compiler-dump/src/main/java/org/enso/compiler/dump/service/IRDumper.java @@ -0,0 +1,36 @@ +package org.enso.compiler.dump.service; + +import java.io.File; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Module; + +public interface IRDumper { + + /** + * Dumps module IR. + * + * @param ir IR of the module to dump + * @param graphName Name of the graph to be dumped. Usually a fully-qualified module name. + * @param srcFile Source file of the module. May be null. + * @param afterPass Name of the pass that this dumper runs after. Corresponds to name of the + * subgraph. + */ + void dumpModule(Module ir, String graphName, File srcFile, String afterPass); + + /** + * Dumps a single expression IR. + * + * @param expr the IR. + * @param graphName Name of the graph to be dumped. May contain spaces. + * @param afterPass Name of the subgraph. + */ + default void dumpExpression(Expression expr, String graphName, String afterPass) { + // nop + } + + /** + * Close and flush all the underlying resources. There will be no more dumps for the module after + * this call. + */ + void close(); +} diff --git a/engine/runtime-compiler/src/main/java/module-info.java b/engine/runtime-compiler/src/main/java/module-info.java index e834881f7825..d88961c29a23 100644 --- a/engine/runtime-compiler/src/main/java/module-info.java +++ b/engine/runtime-compiler/src/main/java/module-info.java @@ -5,6 +5,7 @@ requires org.enso.engine.common; requires org.enso.editions; requires org.enso.pkg; + requires org.enso.runtime.compiler.dump; requires org.enso.runtime.parser; requires static org.enso.persistance; requires org.enso.syntax; @@ -16,7 +17,7 @@ exports org.enso.compiler; exports org.enso.compiler.context; exports org.enso.compiler.data; - exports org.enso.compiler.dump; + exports org.enso.compiler.docs; exports org.enso.compiler.exception; exports org.enso.compiler.pass; exports org.enso.compiler.pass.analyse; @@ -31,4 +32,6 @@ exports org.enso.compiler.phase.exports; exports org.enso.compiler.refactoring; exports org.enso.compiler.common; + + uses org.enso.compiler.dump.service.IRDumpFactoryService; } diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsDispatch.java similarity index 98% rename from engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java rename to engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsDispatch.java index c253917837d0..ae9b14c40583 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsDispatch.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsDispatch.java @@ -1,4 +1,4 @@ -package org.enso.compiler.dump; +package org.enso.compiler.docs; import java.io.IOException; import org.enso.compiler.core.ir.Module; diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsEmitMarkdown.java similarity index 98% rename from engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java rename to engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsEmitMarkdown.java index fa16256b150c..d24ea52d2b26 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitMarkdown.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsEmitMarkdown.java @@ -1,4 +1,4 @@ -package org.enso.compiler.dump; +package org.enso.compiler.docs; import java.io.IOException; import java.io.PrintWriter; diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsEmitSignatures.java similarity index 98% rename from engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java rename to engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsEmitSignatures.java index cf8397e8ca85..2d8184b52787 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsEmitSignatures.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsEmitSignatures.java @@ -1,4 +1,4 @@ -package org.enso.compiler.dump; +package org.enso.compiler.docs; import static org.enso.scala.wrapper.ScalaConversions.asJava; diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsGenerate.java similarity index 99% rename from engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java rename to engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsGenerate.java index bf27672d0a28..b1348e8e08c1 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsGenerate.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsGenerate.java @@ -1,4 +1,4 @@ -package org.enso.compiler.dump; +package org.enso.compiler.docs; import java.io.IOException; import java.io.PrintWriter; diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsUtils.java similarity index 99% rename from engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java rename to engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsUtils.java index 5659c243e006..895f46f8a567 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsUtils.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsUtils.java @@ -1,4 +1,4 @@ -package org.enso.compiler.dump; +package org.enso.compiler.docs; import static org.enso.scala.wrapper.ScalaConversions.asJava; diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsVisit.java similarity index 98% rename from engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java rename to engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsVisit.java index 78b95fec8d52..b48a2f6ac11f 100644 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/DocsVisit.java +++ b/engine/runtime-compiler/src/main/java/org/enso/compiler/docs/DocsVisit.java @@ -1,4 +1,4 @@ -package org.enso.compiler.dump; +package org.enso.compiler.docs; import java.io.IOException; import java.io.PrintWriter; diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizEdge.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizEdge.java deleted file mode 100644 index 2007d140505d..000000000000 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizEdge.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.enso.compiler.dump; - -import java.util.Map; -import java.util.Objects; - -/** - * Represents an edge in the GraphViz graph. - * - * @param from Identifier of the source node. - * @param to Identifier of the target node. - * @param label The label to display on the edge. - * @param additionalAttrs Additional attributes to specify for the edge, except for label which is - * handled by the {@code label} field. - */ -record GraphVizEdge(String from, String to, String label, Map additionalAttrs) { - static GraphVizEdge newEdge(String from, String to) { - Objects.requireNonNull(from); - Objects.requireNonNull(to); - return new GraphVizEdge(from, to, "", Map.of()); - } - - static GraphVizEdge newEdgeWithLabel(String from, String to, String label) { - Objects.requireNonNull(from); - Objects.requireNonNull(to); - Objects.requireNonNull(label); - return new GraphVizEdge(from, to, label, Map.of()); - } - - static GraphVizEdge newEdgeWithAttributes( - String from, String to, String label, Map attrs) { - Objects.requireNonNull(from); - Objects.requireNonNull(to); - Objects.requireNonNull(label); - Objects.requireNonNull(attrs); - return new GraphVizEdge(from, to, label, attrs); - } - - String toGraphViz() { - var sb = new StringBuilder(); - sb.append(from).append(" -> ").append(to); - if (!additionalAttrs.isEmpty()) { - sb.append(" ["); - additionalAttrs.forEach( - (k, v) -> { - assert Utils.hasOneLine(k) : k; - assert Utils.hasOneLine(v) : v; - sb.append(k); - sb.append("="); - if (!Utils.isSurroundedByQuotes(v)) { - sb.append("\"").append(v).append("\""); - } else { - sb.append(v); - } - sb.append(", "); - }); - sb.append("label=\""); - } else { - sb.append(" [label=\""); - } - sb.append(label).append("\"];"); - return sb.toString(); - } -} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizNode.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizNode.java deleted file mode 100644 index c62eafb70af4..000000000000 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/GraphVizNode.java +++ /dev/null @@ -1,188 +0,0 @@ -package org.enso.compiler.dump; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import org.enso.compiler.core.IR; -import org.enso.compiler.core.ir.MetadataStorage; -import org.enso.compiler.pass.IRPass.IRMetadata; -import org.enso.compiler.pass.resolve.DocumentationComments; - -/** - * Represents a node in the GraphViz graph. - * - * @param id Identifier of the node. Used to refer to the node in edges. Must be unique. - * @param header The first line of the label. It is not justified to the left. Can be null. - * @param multiLineLabel A label in GraphViz is a simple textual attribute. To make it multi-line, - * we need to escape newlines with "\\n". - * @param additionalAttrs Additional attributes to specify for the node, apart from `label`. - * @param object The underlying object from which the node was created. - */ -record GraphVizNode( - String id, - String header, - List multiLineLabel, - Map additionalAttrs, - Object object) { - public String toGraphViz() { - var sb = new StringBuilder(); - sb.append(id); - if (!additionalAttrs.isEmpty()) { - sb.append(" ["); - additionalAttrs.forEach( - (k, v) -> { - assert Utils.hasOneLine(k) : k; - assert Utils.hasOneLine(v) : v; - sb.append(k); - sb.append("="); - if (!Utils.isSurroundedByQuotes(v)) { - sb.append("\"").append(v).append("\""); - } else { - sb.append(v); - } - sb.append(", "); - }); - sb.append("label=\""); - } else { - sb.append(" [label=\""); - } - // Id is the "header" of the node - the first line of the label. - // It is not justified to the left. - if (header != null) { - sb.append(header).append("\\n"); - } - for (var line : multiLineLabel) { - var formattedLine = line.replace("\"", "\\\""); - assert Utils.hasOneLine(formattedLine); - sb.append(formattedLine); - // Justify every line to the left - it looks better in the resulting graph. - sb.append("\\l"); - } - sb.append("\"];"); - return sb.toString(); - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - @Override - public boolean equals(Object otherObj) { - if (otherObj instanceof GraphVizNode otherNode) { - return id.equals(otherNode.id); - } - return false; - } - - static class Builder { - private String id; - private String header; - private List labelLines = new ArrayList<>(); - private Map additionalAttrs = new HashMap<>(); - private Object object; - - private static final List> metadataToSkip = - List.of(DocumentationComments.Doc.class); - - static Builder fromObject(Object obj) { - var className = className(obj); - var id = Utils.id(obj); - var bldr = new Builder(); - bldr.object = obj; - bldr.id = id; - bldr.header = id; - bldr.addLabelLine("className: " + className); - return bldr; - } - - /** - * Does not include some common info in the labels like class name, only create an empty - * builder. - */ - static Builder fromObjectPlain(Object obj) { - var id = Utils.id(obj); - var bldr = new Builder(); - bldr.object = obj; - bldr.id = id; - bldr.header = null; - return bldr; - } - - static Builder fromIr(IR ir) { - var className = className(ir); - var bldr = new Builder(); - var id = Utils.id(ir); - bldr.object = ir; - bldr.id = id; - bldr.header = id; - bldr.addLabelLine("className: " + className); - if (ir.location().isDefined()) { - var loc = ir.location().get(); - bldr.addLabelLine("location_start: " + loc.start()); - bldr.addLabelLine("location_end: " + loc.end()); - } else { - bldr.addLabelLine("location: null"); - } - bldr.addLabelLine("id: " + ir.getId()); - if (!isPassDataEmpty(ir.passData())) { - bldr.addLabelLine("pass_data: "); - ir.passData() - .map( - (pass, metadata) -> { - if (!metadataToSkip.contains(metadata.getClass())) { - var metaName = metadata.metadataName(); - bldr.addLabelLine(" - " + metaName); - } - return null; - }); - } else { - bldr.addLabelLine("pass_data: []"); - } - return bldr; - } - - private static boolean isPassDataEmpty(MetadataStorage passData) { - int[] counter = new int[] {0}; - passData.map( - (pass, data) -> { - counter[0]++; - return null; - }); - return counter[0] == 0; - } - - Builder addLabelLine(String line) { - labelLines.add(line); - return this; - } - - Builder addAttribute(String key, String value) { - additionalAttrs.put(key, value); - return this; - } - - GraphVizNode build() { - Objects.requireNonNull(id); - Objects.requireNonNull(labelLines); - Objects.requireNonNull(additionalAttrs); - Objects.requireNonNull(object); - return new GraphVizNode(id, header, labelLines, additionalAttrs, object); - } - - private static String className(Object obj) { - return Arrays.stream(obj.getClass().getName().split("\\.")) - .dropWhile( - item -> - item.equals("org") - || item.equals("enso") - || item.equals("compiler") - || item.equals("core")) - .collect(Collectors.joining(".")); - } - } -} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumper.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumper.java deleted file mode 100644 index 93d3077ddb2e..000000000000 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumper.java +++ /dev/null @@ -1,772 +0,0 @@ -package org.enso.compiler.dump; - -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.HashSet; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import org.enso.compiler.core.IR; -import org.enso.compiler.core.ir.CallArgument; -import org.enso.compiler.core.ir.DefinitionArgument; -import org.enso.compiler.core.ir.Expression; -import org.enso.compiler.core.ir.Function; -import org.enso.compiler.core.ir.Literal.Number; -import org.enso.compiler.core.ir.Literal.Text; -import org.enso.compiler.core.ir.Module; -import org.enso.compiler.core.ir.Name; -import org.enso.compiler.core.ir.Pattern; -import org.enso.compiler.core.ir.expression.Application; -import org.enso.compiler.core.ir.expression.Case; -import org.enso.compiler.core.ir.module.scope.Definition; -import org.enso.compiler.core.ir.module.scope.Definition.Data; -import org.enso.compiler.core.ir.module.scope.Export; -import org.enso.compiler.core.ir.module.scope.Import; -import org.enso.compiler.core.ir.module.scope.definition.Method; -import org.enso.compiler.core.ir.module.scope.imports.Polyglot; -import org.enso.compiler.data.BindingsMap; -import org.enso.compiler.data.BindingsMap.ResolvedConstructor; -import org.enso.compiler.data.BindingsMap.ResolvedModuleMethod; -import org.enso.compiler.data.BindingsMap.ResolvedPolyglotField; -import org.enso.compiler.data.BindingsMap.ResolvedPolyglotSymbol; -import org.enso.compiler.data.BindingsMap.ResolvedType; -import org.enso.compiler.pass.analyse.alias.AliasMetadata; -import org.enso.compiler.pass.analyse.alias.graph.Graph; -import org.enso.compiler.pass.resolve.FullyQualifiedNames.FQNResolution; -import org.enso.compiler.pass.resolve.FullyQualifiedNames.ResolvedLibrary; -import org.enso.compiler.pass.resolve.FullyQualifiedNames.ResolvedModule; - -/** - * Utility class that dumps {@link IR} to a GraphViz file. This - * file can be processed by a {@code dot} command to generate a visual representation of the IR. The - * GraphViz command line utilities are easy to install. See the download page. Alternatively, the resulting file can be - * interactivelly visualized in VSCode with the GraphViz - * Interactive Preview extension. - */ -public class IRDumper { - /** - * Whether to include the code of the IR nodes in the Graphviz file. This can make the file very - * large. - */ - private static final boolean INCLUDE_CODE = true; - - /** Whether to include some pass data in the GraphViz file. */ - private static final boolean INCLUDE_PASS_DATA = true; - - public static final String DEFAULT_DUMP_DIR = "ir-dumps"; - public static final String SYSTEM_PROP = "enso.compiler.dumpIr"; - - private final OutputStream out; - private final Set nodes = new HashSet<>(); - private final Set edges = new HashSet<>(); - - /** - * @param out the output stream to write the Graphviz file to. - */ - private IRDumper(OutputStream out) { - Objects.requireNonNull(out); - this.out = out; - } - - /** - * Creates a new {@link IRDumper} that dumps the graph into the given {@code path}. - * - * @param path the path to write the Graphviz file to. - */ - public static IRDumper fromPath(Path path) { - OutputStream out = null; - try { - out = - Files.newOutputStream( - path, - StandardOpenOption.CREATE, - StandardOpenOption.WRITE, - StandardOpenOption.TRUNCATE_EXISTING); - } catch (IOException e) { - throw new IllegalStateException(e); - } - return new IRDumper(out); - } - - /** - * Creates a new {@link IRDumper} that dumps the graph into the given {@code out}. - * - * @param out the output stream to write the Graphviz file to. - */ - public static IRDumper fromOut(OutputStream out) { - return new IRDumper(out); - } - - /** - * Dumps the given IR into the Graphviz file. Any {@link IOException} is translated to a {@link - * IllegalStateException} within this class. - * - * @param ir the IR to dump. - */ - public void dump(IR ir) { - createIRGraph(ir); - dumpGraph(); - try { - out.flush(); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - private void createIRGraph(IR ir) { - switch (ir) { - case Module moduleIr -> createIRGraph(moduleIr); - default -> throw unimpl(ir); - } - } - - private void createIRGraph(Module moduleIr) { - var moduleNode = GraphVizNode.Builder.fromIr(moduleIr).build(); - addNode(moduleNode); - - for (int i = 0; i < moduleIr.bindings().size(); i++) { - var bindingIr = moduleIr.bindings().apply(i); - createIRGraph(bindingIr); - var edgeDescr = "binding[" + i + "]"; - createEdge(moduleIr, bindingIr, edgeDescr); - } - - for (int i = 0; i < moduleIr.imports().size(); i++) { - var importIr = moduleIr.imports().apply(i); - createIRGraph(importIr); - var edgeDescr = "import[" + i + "]"; - createEdge(moduleIr, importIr, edgeDescr); - } - - for (int i = 0; i < moduleIr.exports().size(); i++) { - var exportIr = moduleIr.exports().apply(i); - createIRGraph(exportIr); - var edgeDescr = "export[" + i + "]"; - createEdge(moduleIr, exportIr, edgeDescr); - } - } - - private void createIRGraph(Definition definitionIr) { - switch (definitionIr) { - case Method.Explicit explicitMethodIr -> { - var bldr = - GraphVizNode.Builder.fromIr(explicitMethodIr) - .addLabelLine("methodName: " + explicitMethodIr.methodName().name()) - .addLabelLine("isStatic: " + explicitMethodIr.isStatic()); - if (explicitMethodIr.typeName().isDefined()) { - bldr.addLabelLine("typeName: " + explicitMethodIr.typeName().get().name()); - } else { - bldr.addLabelLine("typeName: null"); - } - addNode(bldr.build()); - var body = explicitMethodIr.body(); - createIRGraph(body); - createEdge(explicitMethodIr, body, "body"); - var methodRef = explicitMethodIr.methodReference(); - createIRGraph(methodRef); - createEdge(explicitMethodIr, methodRef, "methodReference"); - } - case Method.Conversion conversionMethod -> { - var bldr = - GraphVizNode.Builder.fromIr(conversionMethod) - .addLabelLine("methodName: " + conversionMethod.methodName().name()); - addNode(bldr.build()); - var body = conversionMethod.body(); - createIRGraph(body); - createEdge(conversionMethod, body, "body"); - var methodRef = conversionMethod.methodReference(); - createIRGraph(methodRef); - createEdge(conversionMethod, methodRef, "methodReference"); - } - case Method.Binding binding -> { - var bldr = GraphVizNode.Builder.fromIr(binding); - addNode(bldr.build()); - for (int i = 0; i < binding.arguments().size(); i++) { - var arg = binding.arguments().apply(i); - createIRGraph(arg); - createEdge(binding, arg, "arg[" + i + "]"); - } - var body = binding.body(); - createIRGraph(body); - createEdge(binding, body, "body"); - var methodRef = binding.methodReference(); - createIRGraph(methodRef); - createEdge(binding, methodRef, "methodReference"); - } - case Definition.Type type -> { - var typeNode = - GraphVizNode.Builder.fromIr(type).addLabelLine("name: " + type.name().name()).build(); - addNode(typeNode); - for (int i = 0; i < type.members().size(); i++) { - var member = type.members().apply(i); - createIRGraph(member); - createEdge(type, member, "member[" + i + "]"); - } - } - case Name.GenericAnnotation genericAnnotation -> { - var bldr = - GraphVizNode.Builder.fromIr(genericAnnotation) - .addLabelLine("name: " + genericAnnotation.name()) - .addLabelLine("isMethod: " + genericAnnotation.isMethod()); - addNode(bldr.build()); - var expr = genericAnnotation.expression(); - createIRGraph(expr); - createEdge(genericAnnotation, expr, "expression"); - } - case Name.BuiltinAnnotation builtinAnnotation -> { - var bldr = - GraphVizNode.Builder.fromIr(builtinAnnotation) - .addLabelLine("name: " + builtinAnnotation.name()); - addNode(bldr.build()); - } - case org.enso.compiler.core.ir.Type.Ascription ascription -> { - var ascriptionNode = GraphVizNode.Builder.fromIr(ascription).build(); - addNode(ascriptionNode); - var typed = ascription.typed(); - createIRGraph(typed); - createEdge(ascription, typed, "typed"); - var signature = ascription.signature(); - createIRGraph(signature); - createEdge(ascription, signature, "signature"); - } - default -> throw unimpl(definitionIr); - } - } - - private void createIRGraph(Data atomCons) { - var consNode = - GraphVizNode.Builder.fromIr(atomCons) - .addLabelLine("name: " + atomCons.name().name()) - .build(); - addNode(consNode); - for (int i = 0; i < atomCons.arguments().size(); i++) { - var arg = atomCons.arguments().apply(i); - createIRGraph(arg); - createEdge(atomCons, arg, "arg[" + i + "]"); - } - } - - private void createIRGraph(Expression expression) { - switch (expression) { - case Expression.Block block -> { - var blockNode = GraphVizNode.Builder.fromIr(block).build(); - addNode(blockNode); - for (int i = 0; i < block.expressions().size(); i++) { - var expr = block.expressions().apply(i); - createIRGraph(expr); - createEdge(block, expr, "expression[" + i + "]"); - } - var retVal = block.returnValue(); - createIRGraph(retVal); - createEdge(block, retVal, "returnValue"); - } - case Case.Expr caseExpr -> { - var isNested = caseExpr.isNested(); - var caseNode = - GraphVizNode.Builder.fromIr(caseExpr).addLabelLine("isNested: " + isNested).build(); - addNode(caseNode); - var scrutineeExpr = caseExpr.scrutinee(); - createIRGraph(scrutineeExpr); - createEdge(caseExpr, scrutineeExpr, "scrutinee"); - var branches = caseExpr.branches(); - for (int i = 0; i < branches.size(); i++) { - var branch = branches.apply(i); - createIRGraph(branch); - createEdge(caseExpr, branch, "branch[" + i + "]"); - } - } - case Case.Branch caseBranch -> { - var isTerminalBranch = caseBranch.terminalBranch(); - var caseBranchNode = - GraphVizNode.Builder.fromIr(caseBranch) - .addLabelLine("terminalBranch: " + isTerminalBranch) - .build(); - addNode(caseBranchNode); - var pattern = caseBranch.pattern(); - createIRGraph(pattern); - createEdge(caseBranch, pattern, "pattern"); - var expr = caseBranch.expression(); - createIRGraph(expr); - createEdge(caseBranch, expr, "expression"); - } - case Application.Prefix prefixApp -> { - var prefixAppNode = - GraphVizNode.Builder.fromIr(prefixApp) - .addLabelLine("hasDefaultsSuspended: " + prefixApp.hasDefaultsSuspended()) - .build(); - addNode(prefixAppNode); - - var func = prefixApp.function(); - createIRGraph(func); - createEdge(prefixApp, func, "function"); - - for (int i = 0; i < prefixApp.arguments().size(); i++) { - var arg = prefixApp.arguments().apply(i); - createIRGraph(arg); - createEdge(prefixApp, arg, "arg[" + i + "]"); - } - } - case Function.Lambda lambda -> { - var lambdaNode = GraphVizNode.Builder.fromIr(lambda).build(); - addNode(lambdaNode); - var body = lambda.body(); - createIRGraph(body); - createEdge(lambda, body, "body"); - for (int i = 0; i < lambda.arguments().size(); i++) { - var arg = lambda.arguments().apply(i); - createIRGraph(arg); - createEdge(lambda, arg, "arg[" + i + "]"); - } - } - case Expression.Binding exprBinding -> { - var exprBindNode = - GraphVizNode.Builder.fromIr(exprBinding) - .addLabelLine("name: " + exprBinding.name().name()) - .build(); - addNode(exprBindNode); - createIRGraph(exprBinding.expression()); - createEdge(exprBinding, exprBinding.expression(), "expression"); - } - case Number number -> { - var numNode = - GraphVizNode.Builder.fromIr(number).addLabelLine("value: " + number.value()).build(); - addNode(numNode); - } - case Text text -> { - var textNode = - GraphVizNode.Builder.fromIr(text).addLabelLine("text: " + text.text()).build(); - addNode(textNode); - } - case Name.Literal literal -> { - var bldr = GraphVizNode.Builder.fromIr(literal); - bldr.addLabelLine("name: " + literal.name()); - bldr.addLabelLine("isMethod: " + literal.isMethod()); - if (literal.originalName().isDefined()) { - var origName = literal.originalName().get(); - bldr.addLabelLine("originalName: " + origName.name()); - } else { - bldr.addLabelLine("originalName: null"); - } - var literalNode = bldr.build(); - addNode(literalNode); - } - case Name.MethodReference methodRef -> { - var bldr = GraphVizNode.Builder.fromIr(methodRef); - bldr.addLabelLine("methodName: " + methodRef.methodName().name()); - if (methodRef.typePointer().isDefined()) { - bldr.addLabelLine("typePointer: " + methodRef.typePointer().get().name()); - } else { - bldr.addLabelLine("typePointer: null"); - } - var methodRefNode = bldr.build(); - addNode(methodRefNode); - } - default -> { - var node = GraphVizNode.Builder.fromIr(expression).build(); - addNode(node); - } - } - } - - private void createIRGraph(Pattern pattern) { - var bldr = GraphVizNode.Builder.fromIr(pattern); - switch (pattern) { - case Pattern.Constructor constrPat -> { - var constr = constrPat.constructor(); - bldr.addLabelLine("constructor: " + constr.name()); - addNode(bldr.build()); - var fields = constrPat.fields(); - for (int i = 0; i < fields.size(); i++) { - var field = fields.apply(i); - createIRGraph(field); - createEdge(constrPat, field, "field[" + i + "]"); - } - } - case Pattern.Type tp -> { - addNode(bldr.build()); - var name = tp.name(); - var tpe = tp.tpe(); - createIRGraph(name); - createIRGraph(tpe); - createEdge(tp, name, "name"); - createEdge(tp, tpe, "tpe"); - } - case Pattern.Literal litPat -> { - addNode(bldr.build()); - var lit = litPat.literal(); - createIRGraph(lit); - createEdge(litPat, lit, "literal"); - } - case Pattern.Name name -> { - bldr.addLabelLine("name: " + name.name().name()); - addNode(bldr.build()); - } - case Pattern.Documentation doc -> { - bldr.addLabelLine("doc: " + doc.doc()); - addNode(bldr.build()); - } - default -> throw unimpl(pattern); - } - } - - private void createIRGraph(CallArgument argument) { - switch (argument) { - case CallArgument.Specified specifiedArg -> { - var bldr = GraphVizNode.Builder.fromIr(specifiedArg); - if (specifiedArg.name().isDefined()) { - bldr.addLabelLine("name: " + specifiedArg.name().get().name()); - } else { - bldr.addLabelLine("name: null"); - } - addNode(bldr.build()); - - var value = specifiedArg.value(); - createIRGraph(value); - createEdge(specifiedArg, value, "value"); - } - default -> throw unimpl(argument); - } - } - - private void createIRGraph(DefinitionArgument argument) { - switch (argument) { - case DefinitionArgument.Specified specifiedArg -> { - var bldr = - GraphVizNode.Builder.fromIr(specifiedArg) - .addLabelLine("name: " + specifiedArg.name().name()) - .addLabelLine("suspended: " + specifiedArg.suspended()); - var node = bldr.build(); - addNode(node); - - if (specifiedArg.ascribedType().isDefined()) { - var ascribedType = specifiedArg.ascribedType().get(); - createIRGraph(ascribedType); - createEdge(specifiedArg, ascribedType, "ascribedType"); - } - if (specifiedArg.defaultValue().isDefined()) { - var defaultValue = specifiedArg.defaultValue().get(); - createIRGraph(defaultValue); - createEdge(specifiedArg, defaultValue, "defaultValue"); - } - } - default -> throw unimpl(argument); - } - } - - private void createIRGraph(Import importIr) { - switch (importIr) { - case Import.Module importModIr -> { - var bldr = - GraphVizNode.Builder.fromIr(importModIr) - .addLabelLine("isSynthetic: " + importModIr.isSynthetic()) - .addLabelLine("name: " + importModIr.name().name()) - .addLabelLine("isAll: " + importModIr.isAll()); - if (importModIr.rename().isDefined()) { - var rename = importModIr.rename().get(); - bldr.addLabelLine("rename: " + rename.name()); - } else { - bldr.addLabelLine("rename: null"); - } - addNode(bldr.build()); - } - case Polyglot polyImport -> { - var bldr = GraphVizNode.Builder.fromIr(polyImport); - bldr.addLabelLine( - "entity: Entity(langName=" - + polyImport.entity().langName() - + ", visibleName=" - + polyImport.entity().getVisibleName() - + ")"); - if (polyImport.rename().isDefined()) { - var rename = polyImport.rename().get(); - bldr.addLabelLine("rename: " + rename); - } else { - bldr.addLabelLine("rename: null"); - } - addNode(bldr.build()); - } - default -> throw unimpl(importIr); - } - } - - private void createIRGraph(Export exportIr) { - switch (exportIr) { - case Export.Module exportModIr -> { - var node = - GraphVizNode.Builder.fromIr(exportIr) - .addLabelLine("isSynthetic: " + exportModIr.isSynthetic()) - .addLabelLine("name: " + exportModIr.name().name()) - .build(); - addNode(node); - } - default -> throw unimpl(exportIr); - } - } - - private void createPassDataGraph(IR ir) { - var passData = ir.passData(); - passData.map( - (pass, data) -> { - var bldr = GraphVizNode.Builder.fromObject(data); - bldr.addAttribute("shape", "box"); - bldr.addAttribute("color", "blue"); - bldr.addLabelLine("metadataName: " + data.metadataName()); - switch (data) { - case BindingsMap.Resolution resolution -> { - switch (resolution.target()) { - case BindingsMap.ResolvedModule resolvedModule -> { - bldr.addLabelLine( - "target: ResolvedModule(" - + resolvedModule.module().getName().toString() - + ")"); - } - case ResolvedConstructor resolvedConstructor -> { - bldr.addLabelLine( - "target: ResolvedConstructor(" + resolvedConstructor.cons().name() + ")"); - } - case ResolvedModuleMethod resolvedModuleMethod -> { - bldr.addLabelLine( - "target: ResolvedMethod(" + resolvedModuleMethod.method().name() + ")"); - } - case ResolvedPolyglotField resolvedPolyglotField -> { - bldr.addLabelLine( - "target: ResolvedPolyglotField(" + resolvedPolyglotField.name() + ")"); - } - case ResolvedPolyglotSymbol resolvedPolyglotSymbol -> { - bldr.addLabelLine( - "target: ResolvedPolyglotSymbol(" - + resolvedPolyglotSymbol.symbol().name() - + ")"); - } - case ResolvedType resolvedType -> { - bldr.addLabelLine("target: ResolvedType(" + resolvedType.tp().name() + ")"); - } - default -> throw unimpl(resolution.target()); - } - var metaNode = bldr.build(); - addNode(metaNode); - createEdge(ir, resolution, "BindingsMap.Resolution"); - } - case FQNResolution fqnResolution -> { - switch (fqnResolution.target()) { - case ResolvedLibrary resolvedLibrary -> { - bldr.addLabelLine("target: ResolvedLibrary(" + resolvedLibrary.namespace() + ")"); - } - case ResolvedModule resolvedModule -> { - bldr.addLabelLine( - "target: ResolvedModule(" - + resolvedModule.moduleRef().getName().toString() - + ")"); - } - default -> throw unimpl(fqnResolution.target()); - } - var fqnMetaNode = bldr.build(); - addNode(fqnMetaNode); - createEdge(ir, fqnResolution, "FullyQualifiedNames.FQNResolution"); - } - case BindingsMap bindingsMap -> { - if (bindingsMap.definedEntities().isEmpty()) { - bldr.addLabelLine("definedEntities: []"); - } else { - bldr.addLabelLine("definedEntities: "); - for (int i = 0; i < bindingsMap.definedEntities().size(); i++) { - var entity = bindingsMap.definedEntities().apply(i); - switch (entity) { - case BindingsMap.Type tp -> bldr.addLabelLine(" - Type(" + tp.name() + ")"); - case BindingsMap.ModuleMethod method -> bldr.addLabelLine( - " - ModuleMethod(" + method.name() + ")"); - case BindingsMap.PolyglotSymbol polySym -> bldr.addLabelLine( - " - PolyglotSymbol(" + polySym.name() + ")"); - case BindingsMap.ExtensionMethod extensionMethod -> bldr.addLabelLine( - " - ExtensionMethod(" + extensionMethod.name() + ")"); - case BindingsMap.ConversionMethod conversionMethod -> bldr.addLabelLine( - " - ConversionMethod(" + conversionMethod.name() + ")"); - default -> throw unimpl(entity); - } - } - } - - if (bindingsMap.resolvedImports().isEmpty()) { - bldr.addLabelLine("resolvedImports: []"); - } else { - bldr.addLabelLine("resolvedImports: "); - for (int i = 0; i < bindingsMap.resolvedImports().size(); i++) { - var resolvedImport = bindingsMap.resolvedImports().apply(i); - var firstImpTarget = resolvedImport.targets().head(); - switch (firstImpTarget) { - case ResolvedType resolvedType -> bldr.addLabelLine( - " - ResolvedType(" + resolvedType.tp().name() + ")"); - case BindingsMap.ResolvedModule resolvedModule -> bldr.addLabelLine( - " - ResolvedModule(" + resolvedModule.qualifiedName() + ")"); - default -> throw unimpl(firstImpTarget); - } - } - } - var bmNode = bldr.build(); - addNode(bmNode); - createEdge(ir, bindingsMap, "BindingsMap"); - } - case AliasMetadata.Occurrence occurence -> { - bldr.addLabelLine("occurenceId: " + occurence.id()); - addNode(bldr.build()); - createEdge(ir, occurence, "Alias.Info.Occurence"); - } - case AliasMetadata.RootScope rootScope -> { - addAliasGraphScopeLabels(bldr, rootScope.graph().rootScope()); - var aliasNode = bldr.build(); - addNode(aliasNode); - createEdge(ir, rootScope, "Alias.Info.Scope.Root"); - } - case AliasMetadata.ChildScope childScope -> { - addAliasGraphScopeLabels(bldr, childScope.scope()); - var aliasNode = bldr.build(); - addNode(aliasNode); - createEdge(ir, childScope, "Alias.Info.Scope.Child"); - } - // The rest is ignored - default -> {} - } - return null; - }); - } - - private void addAliasGraphScopeLabels(GraphVizNode.Builder bldr, Graph.Scope scope) { - var parent = scope.parent(); - if (parent.isDefined()) { - var parentId = Utils.id(parent.get()); - bldr.addLabelLine("parent: " + parentId); - } else { - bldr.addLabelLine("parent: null"); - } - var occurences = scope.occurrences(); - if (occurences.isEmpty()) { - bldr.addLabelLine("occurrences: []"); - } else { - bldr.addLabelLine("occurrences: "); - occurences - .values() - .foreach( - occ -> { - bldr.addLabelLine(" - " + occ); - return null; - }); - } - var childScopes = scope.childScopes(); - if (childScopes.isEmpty()) { - bldr.addLabelLine("childScopes: []"); - } else { - bldr.addLabelLine("childScopes: "); - childScopes.foreach( - childScope -> { - var id = Utils.id(childScope); - bldr.addLabelLine(" - " + id); - return null; - }); - } - } - - private void addNode(GraphVizNode node) { - var isNodeAlreadyDefined = nodes.stream().anyMatch(n -> n.equals(node)); - if (isNodeAlreadyDefined) { - // Skip duplicate nodes. - return; - } - nodes.add(node); - if (INCLUDE_CODE) { - if (node.object() instanceof IR ir) { - var code = new Code(ir.showCode()); - var codeNode = - GraphVizNode.Builder.fromObjectPlain(code) - .addAttribute("shape", "box") - .addAttribute("color", "grey") - .addLabelLine(code.code) - .build(); - nodes.add(codeNode); - var edgeAttrs = Map.of("color", "grey", "style", "dotted"); - createEdge(ir, code, "code", edgeAttrs); - } - } - if (INCLUDE_PASS_DATA) { - if (node.object() instanceof IR ir) { - createPassDataGraph(ir); - } - } - } - - private void createEdge(Object from, Object to, String label, Map attrs) { - assert !(from instanceof String); - assert !(to instanceof String); - assert !(from instanceof GraphVizNode); - assert !(to instanceof GraphVizNode); - var fromId = Utils.id(from); - var toId = Utils.id(to); - var edge = GraphVizEdge.newEdgeWithAttributes(fromId, toId, label, attrs); - var nodesContainsFrom = nodes.stream().anyMatch(node -> node.id().equals(fromId)); - var nodesContainsTo = nodes.stream().anyMatch(node -> node.id().equals(toId)); - assert nodesContainsFrom - : "Node " + fromId + " not found. You must first create it before creating an edge from it"; - assert nodesContainsTo - : "Node " + toId + " not found. You must first create it before creating an edge to it"; - var edgeAlreadyExists = edges.stream().anyMatch(e -> e.equals(edge)); - if (!edgeAlreadyExists) { - edges.add(edge); - } - } - - private void createEdge(Object from, Object to, String label) { - createEdge(from, to, label, Map.of()); - } - - /** Dump all the nodes and edges definitions into the GraphViz format. */ - private void dumpGraph() { - write("digraph {"); - write(System.lineSeparator()); - for (GraphVizNode node : nodes) { - var nodeRepr = node.toGraphViz(); - write(nodeRepr); - write(System.lineSeparator()); - } - for (GraphVizEdge edge : edges) { - var containsFromNode = nodes.stream().anyMatch(node -> node.id().equals(edge.from())); - var containsToNode = nodes.stream().anyMatch(node -> node.id().equals(edge.to())); - assert containsFromNode; - assert containsToNode; - var edgeRepr = edge.toGraphViz(); - write(edgeRepr); - write(System.lineSeparator()); - } - write("}"); - } - - private static RuntimeException unimpl(Object obj) { - throw new UnsupportedOperationException(obj.getClass().getName()); - } - - private void write(String str) { - try { - out.write(str.getBytes()); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - /** Just a wrapper for code, we need this to be able to add the code to the graph. */ - private record Code(String code) { - - private Code(String code) { - // Replace new lines with left-justify literals, so that all the lines - // in the code are justified to the left side of the box. - String formattedCode = code.replace("\r", "\\l").replace("\n", "\\l"); - if (code.contains("\"")) { - formattedCode = formattedCode.replace("\"", "\\\""); - } - assert Utils.hasOneLine(formattedCode); - this.code = formattedCode; - } - } -} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumperPass.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumperPass.java deleted file mode 100644 index acd86fccbe80..000000000000 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/IRDumperPass.java +++ /dev/null @@ -1,64 +0,0 @@ -package org.enso.compiler.dump; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import org.enso.compiler.context.InlineContext; -import org.enso.compiler.context.ModuleContext; -import org.enso.compiler.core.IR; -import org.enso.compiler.core.ir.Expression; -import org.enso.compiler.core.ir.Module; -import org.enso.compiler.pass.IRPass; -import org.enso.compiler.pass.IRProcessingPass; -import scala.collection.immutable.Seq; - -/** A pass that just dumps IR to the local {@code ir-dumps} directory. See {@link IRDumper}. */ -public class IRDumperPass implements IRPass { - public static final IRDumperPass INSTANCE = new IRDumperPass(); - - private IRDumperPass() {} - - @Override - public Seq precursorPasses() { - return nil(); - } - - @Override - public Seq invalidatedPasses() { - return nil(); - } - - @Override - public Module runModule(Module ir, ModuleContext moduleContext) { - var irDumpsDir = Path.of(IRDumper.DEFAULT_DUMP_DIR); - if (!irDumpsDir.toFile().exists()) { - try { - Files.createDirectory(irDumpsDir); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - var modName = moduleContext.getName().toString(); - var irPath = irDumpsDir.resolve(modName + ".dot"); - var irDumper = IRDumper.fromPath(irPath); - irDumper.dump(ir); - System.out.println("IR dumped to " + irPath); - return ir; - } - - @Override - public Expression runExpression(Expression ir, InlineContext inlineContext) { - return ir; - } - - @Override - public T updateMetadataInDuplicate(T sourceIr, T copyOfIr) { - return IRPass.super.updateMetadataInDuplicate(sourceIr, copyOfIr); - } - - @SuppressWarnings("unchecked") - private static scala.collection.immutable.List nil() { - Object obj = scala.collection.immutable.Nil$.MODULE$; - return (scala.collection.immutable.List) obj; - } -} diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/Utils.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/Utils.java deleted file mode 100644 index 9eaf5463e1a4..000000000000 --- a/engine/runtime-compiler/src/main/java/org/enso/compiler/dump/Utils.java +++ /dev/null @@ -1,19 +0,0 @@ -package org.enso.compiler.dump; - -final class Utils { - private Utils() {} - - static String id(Object obj) { - var className = obj.getClass().getSimpleName(); - var hash = Integer.toHexString(System.identityHashCode(obj)); - return className + "_" + hash; - } - - static boolean hasOneLine(String label) { - return label.lines().count() == 1; - } - - static boolean isSurroundedByQuotes(String str) { - return str.startsWith("\"") && str.endsWith("\""); - } -} diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala index a00d0cce24b6..46750f80866d 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala @@ -7,8 +7,6 @@ import org.enso.compiler.context.{ InlineContext, ModuleContext } -import org.enso.compiler.dump.DocsGenerate -import org.enso.compiler.dump.DocsVisit import org.enso.compiler.context.CompilerContext.Module import org.enso.compiler.core.CompilerError import org.enso.compiler.core.Implicits.AsMetadata @@ -32,6 +30,8 @@ import org.enso.compiler.phase.{ImportResolver, ImportResolverAlgorithm} import org.enso.editions.LibraryName import org.enso.pkg.QualifiedName import org.enso.common.CompilationStage +import org.enso.compiler.docs.{DocsGenerate, DocsVisit} +import org.enso.compiler.dump.service.{IRDumpFactoryService, IRDumper} import org.enso.compiler.phase.exports.{ ExportCycleException, ExportSymbolAnalysis, @@ -50,6 +50,7 @@ import java.util.concurrent.{ TimeUnit } import java.util.logging.Level +import scala.collection.immutable.HashMap /** This class encapsulates the static transformation processes that take place * on source code, including parsing, desugaring, type-checking, static @@ -74,9 +75,6 @@ class Compiler( new PrintStream(config.outputRedirect.get) else context.getOut - /** Java accessor */ - def getConfig(): CompilerConfig = config - /** The thread pool that handles parsing of modules. */ private val pool: ExecutorService = if (config.parallelParsing) { new ThreadPoolExecutor( @@ -91,6 +89,9 @@ class Compiler( ) } else null + /** Java accessor */ + def getConfig(): CompilerConfig = config + /** Duplicates this compiler with a different config. * @param newConfig Configuration to be used in the duplicated Compiler. */ @@ -292,6 +293,28 @@ class Compiler( } ) + var moduleIrDumpers: HashMap[Module, IRDumper] = new HashMap() + def getOrCreateDumper(module: Module): Option[IRDumper] = { + config.dumpModuleIR.flatMap(pattern => { + if (module.getName().toString.contains(pattern)) { + moduleIrDumpers.get(module) match { + case Some(existing) => Some(existing) + case None => + val dumper = + IRDumpFactoryService.DEFAULT.create(module.getName.toString) + moduleIrDumpers = moduleIrDumpers.updated(module, dumper) + Some(dumper) + } + } else { + None + } + }) + } + + def closeAllDumpers(): Unit = { + moduleIrDumpers.foreach { case (_, dumper) => dumper.close() } + } + val requiredModules = modules.flatMap { module => val isLoadedFromSource = (m: Module) => !context.wasLoadedFromCache(m) && !context.isSynthetic(m) @@ -320,7 +343,8 @@ class Compiler( parseModule( module, irCachingEnabled && !context.isInteractive(module), - generateDocs + generateDocs, + irDumper = getOrCreateDumper(module) ) importedModules .filter(isLoadedFromSource) @@ -329,7 +353,8 @@ class Compiler( parseModule( m, irCachingEnabled && !context.isInteractive(module), - generateDocs + generateDocs, + irDumper = getOrCreateDumper(module) ) } }) @@ -360,7 +385,11 @@ class Compiler( isGeneratingDocs = generateDocs ) val compilerOutput = - runGlobalTypingPasses(context.getIr(module), moduleContext) + runGlobalTypingPasses( + context.getIr(module), + moduleContext, + irDumper = getOrCreateDumper(module) + ) context.updateModule( module, @@ -390,7 +419,11 @@ class Compiler( isGeneratingDocs = generateDocs ) val compilerOutput = - runMethodBodyPasses(context.getIr(module), moduleContext) + runMethodBodyPasses( + context.getIr(module), + moduleContext, + irDumper = getOrCreateDumper(module) + ) context.updateModule( module, { u => @@ -418,7 +451,11 @@ class Compiler( isGeneratingDocs = generateDocs ) val compilerOutput = - runFinalTypeInferencePasses(context.getIr(module), moduleContext) + runFinalTypeInferencePasses( + context.getIr(module), + moduleContext, + irDumper = getOrCreateDumper(module) + ) context.updateModule( module, { u => @@ -510,8 +547,10 @@ class Compiler( } } } - } + + closeAllDumpers() + requiredModules } @@ -629,7 +668,8 @@ class Compiler( private def parseModule( module: Module, useCaches: Boolean, - generateDocs: Boolean + generateDocs: Boolean, + irDumper: Option[IRDumper] = None ): Unit = { context.log( Compiler.defaultLogLevel, @@ -645,7 +685,7 @@ class Compiler( return } - uncachedParseModule(module, generateDocs) + uncachedParseModule(module, generateDocs, irDumper) } /** Retrieve module bindings from cache, if available. @@ -664,7 +704,8 @@ class Compiler( private def uncachedParseModule( module: Module, - generateDocs: Boolean + generateDocs: Boolean, + irDumper: Option[IRDumper] ): Unit = { context.log( Compiler.defaultLogLevel, @@ -691,7 +732,7 @@ class Compiler( injectSyntheticModuleExports(expr, module.getDirectModulesRefs) context.updateModule(module, _.ir(exprWithModuleExports)) val discoveredModule = - recognizeBindings(exprWithModuleExports, moduleContext) + recognizeBindings(exprWithModuleExports, moduleContext, irDumper) if (context.wasLoadedFromCache(module)) { if (module.getBindingsMap() != null) { discoveredModule.passData.update( @@ -857,12 +898,14 @@ class Compiler( private def recognizeBindings( module: IRModule, - moduleContext: ModuleContext + moduleContext: ModuleContext, + irDumper: Option[IRDumper] ): IRModule = { passManager.runPassesOnModule( module, moduleContext, - passes.moduleDiscoveryPasses + passes.moduleDiscoveryPasses, + irDumper ) } @@ -873,26 +916,38 @@ class Compiler( */ private def runMethodBodyPasses( ir: IRModule, - moduleContext: ModuleContext + moduleContext: ModuleContext, + irDumper: Option[IRDumper] ): IRModule = { context.log( Level.FINEST, "Passing module {0} with method body passes", moduleContext.module.getName ) - passManager.runPassesOnModule(ir, moduleContext, passes.functionBodyPasses) + passManager.runPassesOnModule( + ir, + moduleContext, + passes.functionBodyPasses, + irDumper + ) } private def runGlobalTypingPasses( ir: IRModule, - moduleContext: ModuleContext + moduleContext: ModuleContext, + irDumper: Option[IRDumper] ): IRModule = { context.log( Level.FINEST, "Passing module {0} with global typing passes", moduleContext.module.getName ) - passManager.runPassesOnModule(ir, moduleContext, passes.globalTypingPasses) + passManager.runPassesOnModule( + ir, + moduleContext, + passes.globalTypingPasses, + irDumper + ) } /** Runs the final type inference passes, if they are enabled. @@ -901,12 +956,14 @@ class Compiler( */ private def runFinalTypeInferencePasses( ir: IRModule, - moduleContext: ModuleContext + moduleContext: ModuleContext, + irDumper: Option[IRDumper] ): IRModule = { passManager.runPassesOnModule( ir, moduleContext, - passes.typeInferenceFinalPasses + passes.typeInferenceFinalPasses, + irDumper ) } diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Passes.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Passes.scala index 9d0f42809031..0e81614e5252 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Passes.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Passes.scala @@ -1,7 +1,6 @@ package org.enso.compiler import org.enso.compiler.data.CompilerConfig -import org.enso.compiler.dump.IRDumperPass import org.enso.compiler.pass.PassConfiguration._ import org.enso.compiler.pass.analyse._ import org.enso.compiler.pass.analyse.types.scope.StaticModuleScopeAnalysis @@ -109,9 +108,7 @@ class Passes(config: CompilerConfig) { TypeInferenceSignatures.INSTANCE, StaticModuleScopeAnalysis.INSTANCE ) - } else Nil) ++ (if (config.dumpIrs) { - List(IRDumperPass.INSTANCE) - } else Nil) + } else Nil) ) val typeInferenceFinalPasses = new PassGroup( diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/data/CompilerConfig.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/data/CompilerConfig.scala index 2739953d1f0d..ba327f830b45 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/data/CompilerConfig.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/data/CompilerConfig.scala @@ -4,13 +4,13 @@ import java.io.PrintStream /** Configuration for the compiler. * - * @param autoParallelismEnabled whether or not automatic parallelism detection - * is enabled. - * @param warningsEnabled whether or not warnings are enabled - * @param privateCheckEnabled whether or not private keyword is enabled + * @param autoParallelismEnabled whether or not automatic parallelism detection + * is enabled. + * @param warningsEnabled whether or not warnings are enabled + * @param privateCheckEnabled whether or not private keyword is enabled * @param staticTypeInferenceEnabled whether or not type inference is enabled - * @param dumpIrs whether or not to dump IRs. See [[org.enso.compiler.dump.IRDumper]]. - * @param isStrictErrors if true, presence of any Error in IR will result in an exception + * @param dumpModuleIR identification (name) of a module to dump + * @param isStrictErrors if true, presence of any Error in IR will result in an exception * @oaram isLintingDisabled if true, compilation should not run any linting passes * @param outputRedirect redirection of the output of warnings and errors of compiler */ @@ -19,7 +19,7 @@ case class CompilerConfig( warningsEnabled: Boolean = true, privateCheckEnabled: Boolean = true, staticTypeInferenceEnabled: Boolean = false, - dumpIrs: Boolean = false, + dumpModuleIR: Option[String] = None, isStrictErrors: Boolean = false, isLintingDisabled: Boolean = false, outputRedirect: Option[PrintStream] = None diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/PassManager.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/PassManager.scala index 8f0bbeaa99ea..bf0f57221b03 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/PassManager.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/PassManager.scala @@ -4,7 +4,9 @@ import org.slf4j.LoggerFactory import org.enso.compiler.context.{InlineContext, ModuleContext} import org.enso.compiler.core.ir.{Expression, Module} import org.enso.compiler.core.{CompilerError, IR} +import org.enso.compiler.dump.service.IRDumper +import java.io.File import scala.collection.mutable.ListBuffer // TODO [AA] In the future, the pass ordering should be _computed_ from the list @@ -69,7 +71,8 @@ class PassManager( def runPassesOnModule( ir: Module, moduleContext: ModuleContext, - passGroup: PassGroup + passGroup: PassGroup, + irDumper: Option[IRDumper] ): Module = { if (!passes.contains(passGroup)) { throw new CompilerError("Cannot run an unvalidated pass group.") @@ -84,10 +87,22 @@ class PassManager( val newContext = moduleContext.copy(passConfiguration = Some(passConfiguration)) + def getSrcFile(): File = { + val modPath = newContext.module.getPath + if (modPath == null) { + null + } else { + new File(modPath) + } + } + runPasses[Module, ModuleContext]( ir, newContext, passGroup, + moduleName = Some(moduleContext.getName().toString), + irDumper = irDumper, + getSrcFile = getSrcFile, createMiniPass = (factory, ctx) => factory.createForModuleCompilation(ctx), miniPassCompile = (miniPass, ir) => @@ -130,10 +145,22 @@ class PassManager( val newContext = inlineContext.copy(passConfiguration = Some(passConfiguration)) + def getSrcFile(): File = { + val modPath = inlineContext.getModule().getPath + if (modPath == null) { + null + } else { + new File(modPath) + } + } + runPasses[Expression, InlineContext]( ir, newContext, passGroup, + moduleName = null, + getSrcFile = getSrcFile, + irDumper = None, createMiniPass = (factory, ctx) => factory.createForInlineCompilation(ctx), miniPassCompile = (miniPass, ir) => @@ -142,6 +169,20 @@ class PassManager( ) } + private def dump( + ir: IR, + moduleName: Option[String], + irDumper: Option[IRDumper], + passName: String, + getSrcFile: () => File + ): Unit = { + (ir, moduleName, irDumper) match { + case (moduleIr: Module, Some(modName), Some(dumper)) => + dumper.dumpModule(moduleIr, modName, getSrcFile(), passName) + case _ => () + } + } + /** Runs all the passes in the given `passGroup` on `ir` with `context`. * @param createMiniPass Function that creates a minipass. * @param miniPassCompile Function that compiles IR with mini pass. @@ -155,6 +196,9 @@ class PassManager( ir: IRType, context: ContextType, passGroup: PassGroup, + moduleName: Option[String], + irDumper: Option[IRDumper], + getSrcFile: () => File, createMiniPass: (MiniPassFactory, ContextType) => MiniIRPass, miniPassCompile: (MiniIRPass, IRType) => IRType, megaPassCompile: (IRPass, IRType, ContextType) => IRType @@ -169,7 +213,9 @@ class PassManager( pendingMiniPasses.clear() if (combinedPass != null) { logger.trace(" flushing pending mini pass: {}", combinedPass) - miniPassCompile(combinedPass, in) + val ret = miniPassCompile(combinedPass, in) + dump(ret, moduleName, irDumper, combinedPass.toString, getSrcFile) + ret } else { in } @@ -216,7 +262,9 @@ class PassManager( " mega running: {}", megaPass ) - megaPassCompile(megaPass, flushedIR, context) + val ret = megaPassCompile(megaPass, flushedIR, context) + dump(ret, moduleName, irDumper, megaPass.toString, getSrcFile) + ret } } diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala index 2f1785d2f898..bac54b99d885 100644 --- a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala +++ b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/BuiltinsIrBuilder.scala @@ -46,7 +46,8 @@ object BuiltinsIrBuilder { val irAfterModDiscovery = passManager.runPassesOnModule( initialIr, moduleContext, - passes.moduleDiscoveryPasses + passes.moduleDiscoveryPasses, + None ) context.updateModule( module, @@ -60,12 +61,14 @@ object BuiltinsIrBuilder { val irAfterTypes = passManager.runPassesOnModule( irAfterModDiscovery, moduleContext, - passes.globalTypingPasses + passes.globalTypingPasses, + None ) val irAfterCompilation = passManager.runPassesOnModule( irAfterTypes, moduleContext, - passes.functionBodyPasses + passes.functionBodyPasses, + None ) context.updateModule( module, diff --git a/engine/runtime-instrument-common/src/test/scala/org/enso/compiler/test/CompilerTestSetup.scala b/engine/runtime-instrument-common/src/test/scala/org/enso/compiler/test/CompilerTestSetup.scala index 62edc72e4fcf..039bc45a4377 100644 --- a/engine/runtime-instrument-common/src/test/scala/org/enso/compiler/test/CompilerTestSetup.scala +++ b/engine/runtime-instrument-common/src/test/scala/org/enso/compiler/test/CompilerTestSetup.scala @@ -49,7 +49,8 @@ trait CompilerTestSetup { // IR on the runtime module, as the pass manager will not do this for us. // This is to ensure consistency between the curIr and IR stored in moduleContext ModuleTestUtils.unsafeSetIr(runtimeMod, curIr) - val newIr = passManager.runPassesOnModule(curIr, moduleContext, group) + val newIr = + passManager.runPassesOnModule(curIr, moduleContext, group, None) newIr }) } diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java index 80a18fa1a512..ac20a576897c 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/DocsGenerateTest.java @@ -14,8 +14,8 @@ import org.enso.compiler.core.ir.Module; import org.enso.compiler.core.ir.module.scope.Definition; import org.enso.compiler.core.ir.module.scope.definition.Method; -import org.enso.compiler.dump.DocsGenerate; -import org.enso.compiler.dump.DocsVisit; +import org.enso.compiler.docs.DocsGenerate; +import org.enso.compiler.docs.DocsVisit; import org.enso.interpreter.runtime.EnsoContext; import org.enso.pkg.QualifiedName; import org.enso.test.utils.ContextUtils; diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/IRDumpTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/IRDumpTest.java index ffcef2cfc8f8..0951336bd3f9 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/IRDumpTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/dump/test/IRDumpTest.java @@ -1,44 +1,127 @@ package org.enso.compiler.dump.test; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; -import org.enso.compiler.dump.IRDumper; +import java.util.logging.Level; +import org.enso.common.RuntimeOptions; +import org.enso.compiler.core.ir.MetadataStorage; +import org.enso.compiler.core.ir.Name; +import org.enso.compiler.dump.service.IRDumpFactoryService; import org.enso.test.utils.ContextUtils; +import org.enso.test.utils.IRDumperTestWrapper; import org.enso.test.utils.ProjectUtils; +import org.junit.After; +import org.junit.Assume; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; +import scala.Option; +/** + * If run locally, make sure that no IGV instance is running. Otherwise, the IRDumper will try to + * connect to that instance and the tests will fail. + */ public class IRDumpTest { + private static final Path irDumpsDir = Path.of(IRDumpFactoryService.DEFAULT_DUMP_DIR); + private ByteArrayOutputStream out; + + @Rule public PrintOutRule printOutRule = new PrintOutRule(); + + @BeforeClass + public static void skipOnWindows() { + Assume.assumeFalse( + "This test suite should be skipped on Windows", + System.getProperty("os.name").toLowerCase().contains("win")); + } + + @Before + public void before() { + this.out = new ByteArrayOutputStream(); + cleanIrDumpsDir(); + } + + @After + public void after() { + cleanIrDumpsDir(); + } + + private void cleanIrDumpsDir() { + try { + ProjectUtils.deleteRecursively(irDumpsDir); + } catch (IOException e) { + // Ignore. The ir-dumps directory should be deleted eventually. + } + } + @Test - public void testIrDump() { - var irDumpsDir = Path.of(IRDumper.DEFAULT_DUMP_DIR); - var out = new ByteArrayOutputStream(); - System.setProperty(IRDumper.SYSTEM_PROP, "true"); - try (var ctx = ContextUtils.defaultContextBuilder().out(out).build()) { + public void dumpVectorModule() throws IOException { + System.setProperty(IRDumpFactoryService.SYSTEM_PROP, "Vector"); + try (var ctx = + ContextUtils.defaultContextBuilder() + .option(RuntimeOptions.LOG_LEVEL, Level.WARNING.toString()) + .out(out) + .build()) { // Dumping is done in the compiler, so it is enough just to compile the module - ContextUtils.compileModule(ctx, """ + ContextUtils.compileModule( + ctx, + """ + import Standard.Base.Data.Vector.Vector main = 42 - """, "MyMainModule"); + """, + "MyMainModule"); assertThat( "ir-dumps directory was generated in current working directory", irDumpsDir.toFile().exists(), is(true)); - var mainModDump = irDumpsDir.resolve("MyMainModule.dot"); - assertThat( - "MyMainModule.dot file was generated in ir-dumps directory", - mainModDump.toFile().exists(), - is(true)); + var vectorDump = + Files.list(irDumpsDir) + .filter(file -> file.getFileName().toString().contains("Vector")) + .findFirst(); + assertTrue("Vector dump file was generated in ir-dumps directory", vectorDump.isPresent()); } finally { - System.setProperty(IRDumper.SYSTEM_PROP, "false"); - try { - ProjectUtils.deleteRecursively(irDumpsDir); - } catch (IOException e) { - // Ignore. The ir-dumps directory should be deleted eventually. - } - out.reset(); + System.setProperty(IRDumpFactoryService.SYSTEM_PROP, "false"); + } + } + + @Test + public void dumpExpression() throws Exception { + var irDumpsDir = Path.of(IRDumpFactoryService.DEFAULT_DUMP_DIR); + try (var irDumper = new IRDumperTestWrapper()) { + var lit = new Name.Literal("method", true, null, Option.empty(), new MetadataStorage()); + irDumper.dump(lit, "MyModule", "AfterPass"); + } + assertTrue("ir-dumps directory was created", irDumpsDir.toFile().exists()); + var dumpsCnt = Files.list(irDumpsDir).count(); + assertThat("ir-dumps directory is not empty", dumpsCnt, is(greaterThan(0L))); + } + + public final class PrintOutRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() { + try { + base.evaluate(); + } catch (Throwable e) { + var errMsg = new StringBuilder(); + errMsg.append("Test failed with output:").append(System.lineSeparator()); + errMsg.append(out.toString()); + throw new AssertionError(errMsg.toString(), e); + } + } + }; } } } diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/WithIRDumper.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/WithIRDumper.java new file mode 100644 index 000000000000..5d18a612633a --- /dev/null +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/WithIRDumper.java @@ -0,0 +1,27 @@ +package org.enso.compiler.test; + +import java.util.function.Function; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Module; +import org.enso.test.utils.IRDumperTestWrapper; + +/** A mixin interface to be used in the tests to dump IR graphs in IGV. */ +public interface WithIRDumper { + IRDumperTestWrapper dumper = new IRDumperTestWrapper(); + + default Expression processExprWithDump( + Expression expr, String graphName, Function transition) { + dumper.dump(expr, graphName, "before"); + var newIr = transition.apply(expr); + dumper.dump(newIr, graphName, "after"); + return newIr; + } + + default Module processModuleWithDump( + Module expr, String graphName, Function transition) { + dumper.dump(expr, graphName, "before"); + var newIr = transition.apply(expr); + dumper.dump(newIr, graphName, "after"); + return newIr; + } +} diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/mini/passes/MiniPassTester.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/mini/passes/MiniPassTester.java index 85e774b634d6..e067f684ba4b 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/mini/passes/MiniPassTester.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/mini/passes/MiniPassTester.java @@ -58,6 +58,7 @@ protected ModuleContext buildModuleContext(QualifiedName moduleName) { } private static CompilerConfig defaultCompilerConfig() { - return CompilerConfig.apply(false, true, true, false, false, false, false, Option.empty()); + return CompilerConfig.apply( + false, true, true, false, Option.empty(), false, false, Option.empty()); } } diff --git a/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/CompilerTest.scala b/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/CompilerTest.scala index f82944c7f4af..cb8d21dbddea 100644 --- a/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/CompilerTest.scala +++ b/engine/runtime-integration-tests/src/test/scala/org/enso/compiler/test/CompilerTest.scala @@ -69,7 +69,8 @@ trait CompilerRunner { // IR on the runtime module, as the pass manager will not do this for us. // This is to ensure consistency between the curIr and IR stored in moduleContext ModuleTestUtils.unsafeSetIr(runtimeMod, curIr) - val newIr = passManager.runPassesOnModule(curIr, moduleContext, group) + val newIr = + passManager.runPassesOnModule(curIr, moduleContext, group, None) newIr }) } @@ -81,7 +82,8 @@ trait CompilerRunner { ): Module = { val runtimeMod = runtime.Module.fromCompilerModule(moduleContext.module) ModuleTestUtils.unsafeSetIr(runtimeMod, ir) - val newIr = passManager.runPassesOnModule(ir, moduleContext, passGroup) + val newIr = + passManager.runPassesOnModule(ir, moduleContext, passGroup, None) newIr } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java b/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java index 9832d7fcdae6..7b1bb6032bf1 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/EnsoLanguage.java @@ -263,7 +263,7 @@ protected ExecutableNode parse(InlineParsingRequest request) throws InlineParsin false, true, false, - false, + scala.Option.empty(), true, false, scala.Option.apply(new PrintStream(outputRedirect))); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index a70c94046676..2413601ce338 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -44,7 +44,6 @@ import org.enso.compiler.Compiler; import org.enso.compiler.core.EnsoParser; import org.enso.compiler.data.CompilerConfig; -import org.enso.compiler.dump.IRDumper; import org.enso.distribution.DistributionManager; import org.enso.distribution.locking.LockManager; import org.enso.editions.LibraryName; @@ -149,14 +148,14 @@ public EnsoContext( this.assertionsEnabled = shouldAssertionsBeEnabled(); this.shouldWaitForPendingSerializationJobs = getOption(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS_KEY); - var dumpIrs = Boolean.parseBoolean(System.getProperty(IRDumper.SYSTEM_PROP)); + var dumpModuleIR = System.getProperty(RuntimeOptions.IR_DUMPER_SYSTEM_PROP); this.compilerConfig = new CompilerConfig( isParallelismEnabled, true, !isPrivateCheckDisabled, isStaticTypeAnalysisEnabled, - dumpIrs, + scala.Option.apply(dumpModuleIR), getOption(RuntimeOptions.STRICT_ERRORS_KEY), getOption(RuntimeOptions.DISABLE_LINTING_KEY), scala.Option.empty()); diff --git a/lib/java/test-utils/src/main/java/org/enso/test/utils/IRDumperTestWrapper.java b/lib/java/test-utils/src/main/java/org/enso/test/utils/IRDumperTestWrapper.java new file mode 100644 index 000000000000..3cc74d67b550 --- /dev/null +++ b/lib/java/test-utils/src/main/java/org/enso/test/utils/IRDumperTestWrapper.java @@ -0,0 +1,40 @@ +package org.enso.test.utils; + +import java.util.HashMap; +import java.util.Map; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.ir.Expression; +import org.enso.compiler.core.ir.Module; +import org.enso.compiler.dump.service.IRDumpFactoryService; + +/** Utility class for {@link org.enso.compiler.dump.service.IRDumper}. */ +public final class IRDumperTestWrapper implements AutoCloseable { + private final Map dumpers = new HashMap<>(); + + /** + * @param ir Either {@link Module} or {@link Expression}. If it is Expression, a synthetic Module + * IR is created and dumped. + * @param moduleName + * @param passName + */ + public void dump(IR ir, String moduleName, String passName) { + var dumper = dumpers.get(moduleName); + if (dumper == null) { + dumper = IRDumpFactoryService.DEFAULT.create(moduleName); + dumpers.put(moduleName, dumper); + } + if (ir instanceof Module modIr) { + dumper.dumpModule(modIr, moduleName, null, passName); + } else if (ir instanceof Expression expr) { + dumper.dumpExpression(expr, moduleName, passName); + } else { + throw new IllegalArgumentException("Unsupported IR type: " + ir.getClass()); + } + } + + @Override + public void close() throws Exception { + dumpers.values().forEach(org.enso.compiler.dump.service.IRDumper::close); + dumpers.clear(); + } +} From 6f3f187778854be6703bbc03c59337cb310b2cd3 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Tue, 28 Jan 2025 19:59:04 +0100 Subject: [PATCH 2/4] Always attempt to resolve to a default edition (#12157) * Always attempt to resolve to a default edition It appears that we still fail when some old edition is requested and ignore the default editions. This changes adds (a last one?) fallback to the default edition. * PR feedback --- .../runtime/DefaultPackageRepository.scala | 27 ++++++++- .../enso/editions/EditionResolverSpec.scala | 59 +++++++++++++++++-- 2 files changed, 78 insertions(+), 8 deletions(-) diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index 41cd630104fb..f2f984aa38d9 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -29,7 +29,7 @@ import org.enso.common.CompilationStage import java.nio.file.Path import scala.collection.immutable.ListSet import scala.jdk.CollectionConverters.{IterableHasAsJava, SeqHasAsJava} -import scala.util.{Failure, Try, Using} +import scala.util.{Failure, Success, Try, Using} import org.enso.distribution.locking.ResourceManager import org.enso.distribution.{DistributionManager, LanguageHome} import org.enso.editions.updater.EditionManager @@ -38,6 +38,7 @@ import org.enso.interpreter.runtime.builtin.Builtins import org.enso.interpreter.runtime.instrument.NotificationHandler import org.enso.librarymanager.DefaultLibraryProvider import org.enso.pkg.{ComponentGroups, Package} +import org.slf4j.LoggerFactory /** The default [[PackageRepository]] implementation. * @@ -661,7 +662,27 @@ private object DefaultPackageRepository { val homeManager = languageHome.map { home => LanguageHome(Path.of(home)) } val editionManager = EditionManager(distributionManager, homeManager) - val edition = editionManager.resolveEdition(rawEdition).get + val logger = LoggerFactory.getLogger(classOf[DefaultPackageRepository]) + val edition = editionManager + .resolveEdition(rawEdition) + .transform( + e => Success(e), + { err => + logger + .warn( + "Failed to resolve original edition. Trying fallback to the default one", + err + ) + editionManager.resolveEdition(DefaultEdition.getDefaultEdition) + } + ) + + edition.failed.foreach { err => + logger.error( + "Failed to resolve original edition. Fallback failed. Aborting", + err + ) + } val projectRoot = projectPackage.map { pkg => val root = pkg.root @@ -675,7 +696,7 @@ private object DefaultPackageRepository { lockUserInterface = notificationHandler, progressReporter = notificationHandler, languageHome = homeManager, - edition = edition, + edition = edition.get, preferLocalLibraries = projectPackage.exists(_.getConfig().preferLocalLibraries), projectRoot = projectRoot diff --git a/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala b/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala index b0187a3e2416..be012beaf315 100644 --- a/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala +++ b/lib/scala/editions/src/test/scala/org/enso/editions/EditionResolverSpec.scala @@ -1,8 +1,11 @@ package org.enso.editions import org.enso.semver.SemVer -import org.enso.editions.EditionResolutionError.EditionResolutionCycle -import org.enso.editions.Editions.Repository +import org.enso.editions.EditionResolutionError.{ + CannotLoadEdition, + EditionResolutionCycle +} +import org.enso.editions.Editions.{RawEdition, Repository} import org.enso.editions.provider.{ EditionLoadingError, EditionNotFound, @@ -17,9 +20,9 @@ class EditionResolverSpec with Matchers with Inside with OptionValues { - object FakeEditionProvider extends EditionProvider { + class FakeEditionProvider extends EditionProvider { val mainRepo = Repository("main", "https://example.com/main") - val editions: Map[String, Editions.RawEdition] = Map( + val _editions: Map[String, Editions.RawEdition] = Map( "2021.0" -> Editions.Raw.Edition( parent = None, engineVersion = Some(SemVer.of(1, 2, 3)), @@ -48,6 +51,7 @@ class EditionResolverSpec libraries = Map() ) ) + def editions = _editions override def findEditionForName( name: String @@ -58,7 +62,25 @@ class EditionResolverSpec editions.keys.toSeq } - val resolver = EditionResolver(FakeEditionProvider) + object FakeEditionProvider { + val mainRepo = Repository("main", "https://example.com/main") + } + + class FakeEditionProviderWithDefault extends FakeEditionProvider { + override lazy val editions: Map[String, RawEdition] = + super.editions + ("0.0.0-dev" -> Editions.Raw.Edition( + parent = None, + engineVersion = Some(SemVer.of(0, 0, 0, "dev")), + repositories = Map(), + libraries = Map() + )) + } + + class WithDefaultContext { + val resolver = EditionResolver(new FakeEditionProviderWithDefault) + } + + val resolver = EditionResolver(new FakeEditionProvider) "EditionResolver" should { "resolve a simple edition" in { @@ -189,5 +211,32 @@ class EditionResolverSpec editions shouldEqual Seq("cycleA", "cycleB", "cycleA") } } + + "not defer to default version when parent is missing" in new WithDefaultContext { + ctx => + val localRepo = Repository("main", "http://example.com/local") + + localRepo should not equal FakeEditionProvider.mainRepo + + val edition = Editions.Raw.Edition( + parent = Some("2020.2"), + engineVersion = None, + repositories = Map("main" -> localRepo), + libraries = Map( + LibraryName("foo", "bar") -> Editions.Raw + .PublishedLibrary( + LibraryName("foo", "bar"), + SemVer.of(1, 2, 3), + "main" + ) + ) + ) + + inside(ctx.resolver.resolve(edition)) { + case Left(CannotLoadEdition(name, _)) => + name shouldEqual "2020.2" + } + } + } } From 166ad909247fcc9754a5253342bd062aeb85538e Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 28 Jan 2025 22:49:16 +0100 Subject: [PATCH 3/4] Avoiding duplicated code inside of the with index and `-NoIndex` tasks (#12169) I am seeing [CI failures like this one](https://github.com/enso-org/enso/actions/runs/13010150839/job/36285908565?pr=12163#step:8:1085): ``` java.io.FileNotFoundException: built-distribution/enso-engine-2025.1.1-dev-linux-amd64/enso-2025.1.1-dev/lib/Standard/Base/2025.1.1-dev/package.yaml (No such file or directory) at java.base/java.io.FileInputStream.open(FileInputStream.java:213) at java.base/java.io.FileInputStream.(FileInputStream.java:152) at sbt.io.IO$.read(IO.scala:973) at DistributionPackage$.fixLibraryManifest(DistributionPackage.scala:492) at DistributionPackage$.$anonfun$copyLibraryCacheIncremental$2(DistributionPackage.scala:535) at DistributionPackage$.$anonfun$copyLibraryCacheIncremental$2$adapted(DistributionPackage.scala:527) at DistributionPackage$.$anonfun$copyLibraryCacheIncremental$1(DistributionPackage.scala:527) at DistributionPackage$.$anonfun$copyLibraryCacheIncremental$1$adapted(DistributionPackage.scala:526) at DistributionPackage$.copyLibraryCacheIncremental(DistributionPackage.scala:526) at DistributionPackage$.createEnginePackage(DistributionPackage.scala:177) ``` I believe it was introduced recently when [bunch of tasks got duplicated](https://github.com/enso-org/enso/pull/12117/files#diff-5634c415cd8c8504fdb973a3ed092300b43c4b8fc1e184f7249eb29a55511f91R5145) in `build.sbt`. This change removes the duplication: - `DistributionPackage.createEnginePackage` no longer build the indexes - there is separete `DistributionPackage.indexStdLibs` function for that wrapped in `createStdLibsIndexes` task - `buildEngineDistribution` just depends on all three sub tasks: - `buildEngineDistributionNoIndex` - `createEnginePackageNoIndex` - `createStdLibsIndexes` This shall avoid a situation when two different tasks are mangling with the same package/libraries structure. --- build.sbt | 69 +++++++++++-------------------- project/DistributionPackage.scala | 14 +------ 2 files changed, 24 insertions(+), 59 deletions(-) diff --git a/build.sbt b/build.sbt index 61548978c7d5..8878032c76ce 100644 --- a/build.sbt +++ b/build.sbt @@ -5193,34 +5193,29 @@ launcherDistributionRoot := packageBuilder.localArtifact("launcher") / "enso" projectManagerDistributionRoot := packageBuilder.localArtifact("project-manager") / "enso" -lazy val createEnginePackage = - taskKey[Unit]("Creates the engine distribution package") -createEnginePackage := { +lazy val createStdLibsIndexes = + taskKey[Unit]("Creates index files for standard libraries") +createStdLibsIndexes := { updateLibraryManifests.value buildEngineDistributionNoIndex.value - val modulesToCopy = componentModulesPaths.value - val root = engineDistributionRoot.value - val log = streams.value.log - val cacheFactory = streams.value.cacheStoreFactory - DistributionPackage.createEnginePackage( - distributionRoot = root, - cacheFactory = cacheFactory, - log = log, - jarModulesToCopy = modulesToCopy, - graalVersion = graalMavenPackagesVersion, - javaVersion = graalVersion, - ensoVersion = ensoVersion, - editionName = currentEdition, - sourceStdlibVersion = stdLibVersion, - targetStdlibVersion = targetStdlibVersion, - targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value, - generateIndex = true - ) - log.info(s"Engine package created at $root") + val modulesToCopy = componentModulesPaths.value + val distributionRoot = engineDistributionRoot.value + val log = streams.value.log + val cacheFactory = streams.value.cacheStoreFactory + + DistributionPackage.indexStdLibs( + stdLibVersion = targetStdlibVersion, + ensoVersion = ensoVersion, + stdLibRoot = distributionRoot / "lib", + ensoExecutable = distributionRoot / "bin" / "enso", + cacheFactory = cacheFactory.sub("stdlib"), + log = log + ) + log.info(s"Standard library indexes create for $distributionRoot") } -ThisBuild / createEnginePackage := { - createEnginePackage.result.value +ThisBuild / createStdLibsIndexes := { + createStdLibsIndexes.result.value } lazy val createEnginePackageNoIndex = @@ -5242,8 +5237,7 @@ createEnginePackageNoIndex := { editionName = currentEdition, sourceStdlibVersion = stdLibVersion, targetStdlibVersion = targetStdlibVersion, - targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value, - generateIndex = false + targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value ) log.info(s"Engine package created at $root") } @@ -5267,25 +5261,7 @@ buildEngineDistributionNoIndex := Def.taskIf { // of other tasks. ThisBuild / buildEngineDistributionNoIndex := { updateLibraryManifests.value - val modulesToCopy = componentModulesPaths.value - val root = engineDistributionRoot.value - val log = streams.value.log - val cacheFactory = streams.value.cacheStoreFactory - DistributionPackage.createEnginePackage( - distributionRoot = root, - cacheFactory = cacheFactory, - log = log, - jarModulesToCopy = modulesToCopy, - graalVersion = graalMavenPackagesVersion, - javaVersion = graalVersion, - ensoVersion = ensoVersion, - editionName = currentEdition, - sourceStdlibVersion = stdLibVersion, - targetStdlibVersion = targetStdlibVersion, - targetDir = (`syntax-rust-definition` / rustParserTargetDirectory).value, - generateIndex = false - ) - log.info(s"Engine package created at $root") + createEnginePackageNoIndex.value } lazy val shouldBuildNativeImage = taskKey[Boolean]( @@ -5324,7 +5300,8 @@ lazy val buildEngineDistribution = taskKey[Unit]("Builds the engine distribution") buildEngineDistribution := { buildEngineDistributionNoIndex.value - createEnginePackage.value + createEnginePackageNoIndex.value + createStdLibsIndexes.value } // This makes the buildEngineDistributionNoIndex task usable as a dependency diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index 5af65ec471e3..510d489facc8 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -137,8 +137,7 @@ object DistributionPackage { editionName: String, sourceStdlibVersion: String, targetStdlibVersion: String, - targetDir: File, - generateIndex: Boolean + targetDir: File ): Unit = { copyDirectoryIncremental( file("distribution/engine/THIRD-PARTY"), @@ -195,17 +194,6 @@ object DistributionPackage { graalVersion = graalVersion, javaVersion = javaVersion ) - - if (generateIndex) { - indexStdLibs( - stdLibVersion = targetStdlibVersion, - ensoVersion = ensoVersion, - stdLibRoot = distributionRoot / "lib", - ensoExecutable = distributionRoot / "bin" / "enso", - cacheFactory = cacheFactory.sub("stdlib"), - log = log - ) - } } def indexStdLibs( From 54ccd1a6ef92b35a5460c65c51a2a93550554e17 Mon Sep 17 00:00:00 2001 From: Hubert Plociniczak Date: Wed, 29 Jan 2025 10:03:17 +0100 Subject: [PATCH 4/4] Enable Standard.AWS and Standard.Microsoft in native image (#12156) * Enable Standard.AWS in native image `com.amazonaws` was required to be initialized at runtime due to logging being already present on that list. Closes #12149. * Add Standard.Microsoft * update CI * revert accidental change * Update resources for Microsoft * fmt --- build.sbt | 10 ++++++++-- build_tools/build/src/engine/context.rs | 14 +++++++++++++- .../org/enso/microsoft/resource-config.json | 9 +++++++++ test/Microsoft_Tests/README.md | 2 +- 4 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 std-bits/microsoft/src/main/resources/META-INF/native-image/org/enso/microsoft/resource-config.json diff --git a/build.sbt b/build.sbt index 8878032c76ce..5c0028a5b9dc 100644 --- a/build.sbt +++ b/build.sbt @@ -3787,7 +3787,12 @@ lazy val `engine-runner` = project `base-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++ `image-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++ `table-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++ - `database-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) + `database-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++ + `std-aws-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++ + `std-microsoft-polyglot-root` + .listFiles("*.jar") + .map(_.getAbsolutePath()) + core ++ stdLibsJars }, buildSmallJdk := { @@ -3905,7 +3910,8 @@ lazy val `engine-runner` = project "org.enso.image", "org.enso.table", "org.enso.database", - "org.eclipse.jgit" + "org.eclipse.jgit", + "com.amazonaws" ) ) } diff --git a/build_tools/build/src/engine/context.rs b/build_tools/build/src/engine/context.rs index 1d9bb9557a24..bbaabfbc2659 100644 --- a/build_tools/build/src/engine/context.rs +++ b/build_tools/build/src/engine/context.rs @@ -702,6 +702,18 @@ pub async fn runner_sanity_test( .run_ok() .await; + let test_aws = Command::new(&enso) + .args(["--run", repo_root.test.join("AWS_Tests").as_str()]) + .set_env(ENSO_DATA_DIRECTORY, engine_package)? + .run_ok() + .await; + + let test_microsoft = Command::new(&enso) + .args(["--run", repo_root.test.join("Microsoft_Tests").as_str()]) + .set_env(ENSO_DATA_DIRECTORY, engine_package)? + .run_ok() + .await; + let test_geo = Command::new(&enso) .args(["--run", repo_root.test.join("Geo_Tests").as_str()]) .set_env(ENSO_DATA_DIRECTORY, engine_package)? @@ -714,7 +726,7 @@ pub async fn runner_sanity_test( .run_ok() .await; - let all_cmds = test_base.and(test_internal_base).and(test_table).and(test_geo).and(test_image); + let all_cmds = test_base.and(test_internal_base).and(test_table).and(test_aws).and(test_microsoft).and(test_geo).and(test_image); // The following test does not actually run anything, it just checks if the engine // can accept `--jvm` argument and evaluates something. diff --git a/std-bits/microsoft/src/main/resources/META-INF/native-image/org/enso/microsoft/resource-config.json b/std-bits/microsoft/src/main/resources/META-INF/native-image/org/enso/microsoft/resource-config.json new file mode 100644 index 000000000000..ef3099149de5 --- /dev/null +++ b/std-bits/microsoft/src/main/resources/META-INF/native-image/org/enso/microsoft/resource-config.json @@ -0,0 +1,9 @@ +{ + "resources":{ + "includes":[{ + "pattern":"\\QMETA-INF/java.sql.Driver\\E" + }]}, + "bundles":[{ + "name": "com.microsoft.sqlserver.jdbc.SQLServerResource" + }] +} diff --git a/test/Microsoft_Tests/README.md b/test/Microsoft_Tests/README.md index 6fd823f01fd9..cead4ad918e5 100644 --- a/test/Microsoft_Tests/README.md +++ b/test/Microsoft_Tests/README.md @@ -10,7 +10,7 @@ Please set the following environment variables: - 'ENSO_SQLSERVER_HOST' - the name of the server hosting SQLServer, - 'ENSO_SQLSERVER_PORT' - the port SQLServer is on, - 'ENSO_SQLSERVER_USER' - the user name to use to connect, -- 'ENSO_SQLSERVER_PASSWORD' - the pasword for that user, +- 'ENSO_SQLSERVER_PASSWORD' - the password for that user, - 'ENSO_SQLSERVER_DATABASE' - the database on the SQLServer to use. ## Docker