Merge branch 'develop' into wip/mk/dq-metric

.jvmopts

@@ -7,3 +7,4 @@
 --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED
 --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED
 --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+--add-exports=jdk.internal.vm.compiler/org.graalvm.graphio=ALL-UNNAMED

build.sbt

@@ -351,6 +351,8 @@ lazy val enso = (project in file("."))
     `runtime-and-langs`,
     `runtime-benchmarks`,
     `runtime-compiler`,
+    `runtime-compiler-dump`,
+    `runtime-compiler-dump-igv`,
     `runtime-parser`,
     `runtime-parser-dsl`,
     `runtime-parser-processor`,
@@ -722,6 +724,8 @@ lazy val componentModulesPaths =
     (`runtime` / Compile / exportedModuleBin).value,
     (`syntax-rust-definition` / Compile / exportedModuleBin).value,
     (`runtime-compiler` / Compile / exportedModuleBin).value,
+    (`runtime-compiler-dump` / Compile / exportedModuleBin).value,
+    (`runtime-compiler-dump-igv` / Compile / exportedModuleBin).value,
     (`runtime-parser` / Compile / exportedModuleBin).value,
     (`runtime-suggestions` / Compile / exportedModuleBin).value,
     (`runtime-instrument-common` / Compile / exportedModuleBin).value,
@@ -2398,6 +2402,7 @@ lazy val `language-server` = (project in file("engine/language-server"))
       (`runtime-suggestions` / Compile / exportedModule).value,
       (`runtime-parser` / Compile / exportedModule).value,
       (`runtime-compiler` / Compile / exportedModule).value,
+      (`runtime-compiler-dump` / Compile / exportedModule).value,
       (`polyglot-api` / Compile / exportedModule).value,
       (`polyglot-api-macros` / Compile / exportedModule).value,
       (`pkg` / Compile / exportedModule).value,
@@ -2940,6 +2945,8 @@ lazy val `runtime-integration-tests` =
         (`runtime-suggestions` / Compile / exportedModule).value,
         (`runtime-parser` / Compile / exportedModule).value,
         (`runtime-compiler` / Compile / exportedModule).value,
+        (`runtime-compiler-dump` / Compile / exportedModule).value,
+        (`runtime-compiler-dump-igv` / Compile / exportedModule).value,
         (`polyglot-api` / Compile / exportedModule).value,
         (`polyglot-api-macros` / Compile / exportedModule).value,
         (`pkg` / Compile / exportedModule).value,
@@ -3018,6 +3025,13 @@ lazy val `runtime-integration-tests` =
         )
       },
       Test / addExports := {
+        // Add necessary exports for IR module dumping to IGV
+        // Which is used in the test utils
+        val irDumperExports = Map(
+          "jdk.internal.vm.compiler/org.graalvm.graphio" -> Seq(
+            (`runtime-compiler-dump-igv` / javaModuleName).value
+          )
+        )
         val runtimeModName = (`runtime` / javaModuleName).value
         val exports = Map(
           (`runtime-instrument-common` / javaModuleName).value + "/org.enso.interpreter.instrument.job" -> Seq(
@@ -3033,7 +3047,7 @@ lazy val `runtime-integration-tests` =
         val testPkgsExports = testPkgs.map { pkg =>
           runtimeModName + "/" + pkg -> Seq("ALL-UNNAMED")
         }.toMap
-        exports ++ testPkgsExports
+        exports ++ testPkgsExports ++ irDumperExports
       }
     )
     .dependsOn(`runtime`)
@@ -3115,6 +3129,7 @@ lazy val `runtime-benchmarks` =
         (`runtime-suggestions` / Compile / exportedModule).value,
         (`runtime-parser` / Compile / exportedModule).value,
         (`runtime-compiler` / Compile / exportedModule).value,
+        (`runtime-compiler-dump` / Compile / exportedModule).value,
         (`polyglot-api` / Compile / exportedModule).value,
         (`polyglot-api-macros` / Compile / exportedModule).value,
         (`pkg` / Compile / exportedModule).value,
@@ -3338,6 +3353,7 @@ lazy val `runtime-compiler` =
         (`engine-common` / Compile / exportedModule).value,
         (`pkg` / Compile / exportedModule).value,
         (`runtime-parser` / Compile / exportedModule).value,
+        (`runtime-compiler-dump` / Compile / exportedModule).value,
         (`syntax-rust-definition` / Compile / exportedModule).value,
         (`scala-libs-wrapper` / Compile / exportedModule).value,
         (`persistance` / Compile / exportedModule).value,
@@ -3393,11 +3409,61 @@ lazy val `runtime-compiler` =
       }
     )
     .dependsOn(`runtime-parser`)
+    .dependsOn(`runtime-compiler-dump`)
     .dependsOn(pkg)
     .dependsOn(`engine-common`)
     .dependsOn(editions)
     .dependsOn(`persistance-dsl` % "provided")
 
+/** This project contains only a single service (interface) definition.
+  */
+lazy val `runtime-compiler-dump` =
+  (project in file("engine/runtime-compiler-dump"))
+    .enablePlugins(JPMSPlugin)
+    .settings(
+      frgaalJavaCompilerSetting,
+      javaModuleName := "org.enso.runtime.compiler.dump",
+      Compile / internalModuleDependencies := {
+        val transitiveDeps =
+          (`runtime-parser` / Compile / internalModuleDependencies).value
+        Seq(
+          (`runtime-parser` / Compile / exportedModule).value
+        ) ++ transitiveDeps
+      }
+    )
+    .dependsOn(`runtime-parser`)
+
+/** This is a standalone project that is not compiled with Frgaal on purpose.
+  * It depends on jdk.internal.vm.compiler module, which cannot be included in Frgaal.
+  * It includes a service provider for service definition in `runtime-compiler-dump`.
+  */
+lazy val `runtime-compiler-dump-igv` =
+  (project in file("engine/runtime-compiler-dump-igv"))
+    .enablePlugins(JPMSPlugin)
+    .settings(
+      scalaModuleDependencySetting,
+      javaModuleName := "org.enso.runtime.compiler.dump.igv",
+      Compile / internalModuleDependencies := {
+        val transitiveDeps =
+          (`runtime-compiler` / Compile / internalModuleDependencies).value
+        Seq(
+          (`runtime-compiler` / Compile / exportedModule).value
+        ) ++ transitiveDeps
+      },
+      Compile / moduleDependencies ++= Seq(
+        "org.slf4j" % "slf4j-api" % slf4jVersion
+      ),
+      Compile / addExports ++= {
+        Map(
+          "jdk.internal.vm.compiler/org.graalvm.graphio" -> Seq(
+            javaModuleName.value
+          )
+        )
+      }
+    )
+    .dependsOn(`runtime-compiler-dump`)
+    .dependsOn(`runtime-compiler`)
+
 lazy val `runtime-suggestions` =
   (project in file("engine/runtime-suggestions"))
     .enablePlugins(JPMSPlugin)
@@ -3463,6 +3529,7 @@ lazy val `runtime-instrument-common` =
         (`refactoring-utils` / Compile / exportedModule).value,
         (`runtime` / Compile / exportedModule).value,
         (`runtime-compiler` / Compile / exportedModule).value,
+        (`runtime-compiler-dump` / Compile / exportedModule).value,
         (`runtime-parser` / Compile / exportedModule).value,
         (`runtime-suggestions` / Compile / exportedModule).value,
         (`text-buffer` / Compile / exportedModule).value,
@@ -3492,6 +3559,7 @@ lazy val `runtime-instrument-id-execution` =
       Compile / internalModuleDependencies := Seq(
         (`runtime` / Compile / exportedModule).value,
         (`runtime-compiler` / Compile / exportedModule).value,
+        (`runtime-compiler-dump` / Compile / exportedModule).value,
         (`polyglot-api` / Compile / exportedModule).value
       )
     )
@@ -3719,7 +3787,12 @@ lazy val `engine-runner` = project
         `base-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++
         `image-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++
         `table-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++
-        `database-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath())
+        `database-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++
+        `std-aws-polyglot-root`.listFiles("*.jar").map(_.getAbsolutePath()) ++
+        `std-microsoft-polyglot-root`
+          .listFiles("*.jar")
+          .map(_.getAbsolutePath())
+
       core ++ stdLibsJars
     },
     buildSmallJdk := {
@@ -3837,7 +3910,8 @@ lazy val `engine-runner` = project
               "org.enso.image",
               "org.enso.table",
               "org.enso.database",
-              "org.eclipse.jgit"
+              "org.eclipse.jgit",
+              "com.amazonaws"
             )
           )
       }
@@ -5125,34 +5199,29 @@ launcherDistributionRoot := packageBuilder.localArtifact("launcher") / "enso"
 projectManagerDistributionRoot :=
   packageBuilder.localArtifact("project-manager") / "enso"
 
-lazy val createEnginePackage =
-  taskKey[Unit]("Creates the engine distribution package")
-createEnginePackage := {
+lazy val createStdLibsIndexes =
+  taskKey[Unit]("Creates index files for standard libraries")
+createStdLibsIndexes := {
   updateLibraryManifests.value
   buildEngineDistributionNoIndex.value
-  val modulesToCopy = componentModulesPaths.value
-  val root          = engineDistributionRoot.value
-  val log           = streams.value.log
-  val cacheFactory  = streams.value.cacheStoreFactory
-  DistributionPackage.createEnginePackage(
-    distributionRoot    = root,
-    cacheFactory        = cacheFactory,
-    log                 = log,
-    jarModulesToCopy    = modulesToCopy,
-    graalVersion        = graalMavenPackagesVersion,
-    javaVersion         = graalVersion,
-    ensoVersion         = ensoVersion,
-    editionName         = currentEdition,
-    sourceStdlibVersion = stdLibVersion,
-    targetStdlibVersion = targetStdlibVersion,
-    targetDir           = (`syntax-rust-definition` / rustParserTargetDirectory).value,
-    generateIndex       = true
-  )
-  log.info(s"Engine package created at $root")
+  val modulesToCopy    = componentModulesPaths.value
+  val distributionRoot = engineDistributionRoot.value
+  val log              = streams.value.log
+  val cacheFactory     = streams.value.cacheStoreFactory
+
+  DistributionPackage.indexStdLibs(
+    stdLibVersion  = targetStdlibVersion,
+    ensoVersion    = ensoVersion,
+    stdLibRoot     = distributionRoot / "lib",
+    ensoExecutable = distributionRoot / "bin" / "enso",
+    cacheFactory   = cacheFactory.sub("stdlib"),
+    log            = log
+  )
+  log.info(s"Standard library indexes create for $distributionRoot")
 }
 
-ThisBuild / createEnginePackage := {
-  createEnginePackage.result.value
+ThisBuild / createStdLibsIndexes := {
+  createStdLibsIndexes.result.value
 }
 
 lazy val createEnginePackageNoIndex =
@@ -5174,8 +5243,7 @@ createEnginePackageNoIndex := {
     editionName         = currentEdition,
     sourceStdlibVersion = stdLibVersion,
     targetStdlibVersion = targetStdlibVersion,
-    targetDir           = (`syntax-rust-definition` / rustParserTargetDirectory).value,
-    generateIndex       = false
+    targetDir           = (`syntax-rust-definition` / rustParserTargetDirectory).value
   )
   log.info(s"Engine package created at $root")
 }
@@ -5199,25 +5267,7 @@ buildEngineDistributionNoIndex := Def.taskIf {
 // of other tasks.
 ThisBuild / buildEngineDistributionNoIndex := {
   updateLibraryManifests.value
-  val modulesToCopy = componentModulesPaths.value
-  val root          = engineDistributionRoot.value
-  val log           = streams.value.log
-  val cacheFactory  = streams.value.cacheStoreFactory
-  DistributionPackage.createEnginePackage(
-    distributionRoot    = root,
-    cacheFactory        = cacheFactory,
-    log                 = log,
-    jarModulesToCopy    = modulesToCopy,
-    graalVersion        = graalMavenPackagesVersion,
-    javaVersion         = graalVersion,
-    ensoVersion         = ensoVersion,
-    editionName         = currentEdition,
-    sourceStdlibVersion = stdLibVersion,
-    targetStdlibVersion = targetStdlibVersion,
-    targetDir           = (`syntax-rust-definition` / rustParserTargetDirectory).value,
-    generateIndex       = false
-  )
-  log.info(s"Engine package created at $root")
+  createEnginePackageNoIndex.value
 }
 
 lazy val shouldBuildNativeImage = taskKey[Boolean](
@@ -5256,7 +5306,8 @@ lazy val buildEngineDistribution =
   taskKey[Unit]("Builds the engine distribution")
 buildEngineDistribution := {
   buildEngineDistributionNoIndex.value
-  createEnginePackage.value
+  createEnginePackageNoIndex.value
+  createStdLibsIndexes.value
 }
 
 // This makes the buildEngineDistributionNoIndex task usable as a dependency

build_tools/build/src/engine/context.rs

@@ -702,6 +702,18 @@ pub async fn runner_sanity_test(
             .run_ok()
             .await;
 
+        let test_aws = Command::new(&enso)
+            .args(["--run", repo_root.test.join("AWS_Tests").as_str()])
+            .set_env(ENSO_DATA_DIRECTORY, engine_package)?
+            .run_ok()
+            .await;
+
+        let test_microsoft = Command::new(&enso)
+            .args(["--run", repo_root.test.join("Microsoft_Tests").as_str()])
+            .set_env(ENSO_DATA_DIRECTORY, engine_package)?
+            .run_ok()
+            .await;
+
         let test_geo = Command::new(&enso)
             .args(["--run", repo_root.test.join("Geo_Tests").as_str()])
             .set_env(ENSO_DATA_DIRECTORY, engine_package)?
@@ -714,7 +726,7 @@ pub async fn runner_sanity_test(
             .run_ok()
             .await;
 
-        let all_cmds = test_base.and(test_internal_base).and(test_table).and(test_geo).and(test_image);
+        let all_cmds = test_base.and(test_internal_base).and(test_table).and(test_aws).and(test_microsoft).and(test_geo).and(test_image);
 
         // The following test does not actually run anything, it just checks if the engine
         // can accept `--jvm` argument and evaluates something.

docs/runtime/compiler-ir.md

@@ -9,15 +9,75 @@ multiple passes. Every pass is a class implementing the
 See [Runtime roadmap - static analysis](../runtime-roadmap.md#static-analysis)
 for future goals.
 
-## Visualization
-
-The IR can be visualized using `--vm.D=enso.compiler.dumpIr` system property.
-This will output a `.dot` file in [GraphViz](www.graphviz.org) format in the
-`ir-dumps` directory for each IR in the program. The _dot_ file format is a
-minimal textual format, that can be converted to a graphical representation
-using the `dot` command from the GraphViz package. For example, on Ubuntu,
-install `dot` with `sudo apt install graphviz`. Then, convert the `.dot` file to
-a `.svg` image with `dot -Tsvg -o <output>.svg <input>.dot`. An example is:
-![image.svg](https://github.com/user-attachments/assets/26ab8415-72cf-46da-bc63-f475e9fa628e)
-
-See `org.enso.compiler.dump.IRDumper`.
+## Dumping IR
+
+The IR can be visualized using the `enso.compiler.dumpIr` system property. The
+value of the property is a substring of a module name to dump. IRs are dumped
+into the [IGV tool](https://www.graalvm.org/latest/tools/igv/) in a similar way
+to how GraalVM graphs are dumped, which is documented in
+[enso4igv](https://github.com/enso-org/enso/blob/2e714a70ddf12456e9f3fa9e132fd2ac43aa3b77/tools/enso4igv/IGV.md#using-the-igv).
+
+When using the `enso.compiler.dumpIr` property, one has to add
+`--add-exports jdk.internal.vm.compiler/org.graalvm.graphio=org.enso.runtime.compiler.dump.igv`
+to the `JAVA_OPTS` env var, because the IGV dumper uses an internal package of
+GraalVM JDK's module which is not exported by default.
+
+Usage example:
+
+```
+$ env JAVA_OPTS='--add-exports jdk.internal.vm.compiler/org.graalvm.graphio=org.enso.runtime.compiler.dump.igv' ./built-distribution/*/bin/enso --vm.D enso.compiler.dumpIr=Vector --no-ir-caches --run tmp.enso
+```
+
+The IR graphs are dumped directly to IGV, if it is running, or to the `ir-dumps`
+directory in the
+[BGV](https://www.graalvm.org/graphio/javadoc/jdk/graal/compiler/graphio/package-summary.html)
+format.
+
+### Description of the graphs
+
+For a module, multiple graphs are dumped. Names of the graphs correspond to the
+names of the Compiler passes, as can be seen on the screenshot:
+![1](https://github.com/user-attachments/assets/eb9bc883-c482-4461-8e38-0b615bec2e83)
+
+Opening the first graph for the `Vector` module is overwhelming, since it has
+more than 3000 nodes:
+![2](https://github.com/user-attachments/assets/296c0b63-a28c-4b9b-bed7-22c52f9f1cd3)
+However, nodes are structured in _blocks_. Blocks can be seen on the right side
+in the `Control Flow` tool window.
+
+Zoom into a particular block:
+
+- Double-click on the block with id 1 in the `Control Flow` tool window.
+- Click on the `Zoom to Selection` button in the toolbar.
+  ![3](https://github.com/user-attachments/assets/dd8a4c7f-b8e4-4356-b1fd-64fa7824134c)
+
+Below the tab, there are all the passes displayed as points (_Phase toolbar_).
+Hovering over a point shows the pass name:
+![4](https://github.com/user-attachments/assets/b5449f4c-de35-46d4-8fee-e8963cf3d4f1)
+
+Click-and-drag the mouse to select a region of passes. This will display the
+difference in the graph between those passes. In the following screenshot, we
+can see the difference between the very first pass, and `MethodCalls` pass:
+![5](https://github.com/user-attachments/assets/b320d706-e074-4509-a836-5e88e5305a55)
+Orange nodes represent those with changed properties.
+
+Clicking on a single changed node, we can see that there is `NEW_passData`
+property:
+![6](https://github.com/user-attachments/assets/e3ce8f3c-a227-4626-96f8-910dc4d4c31a)
+In this case, we can see that `AliasMetadata.ChildScope` passData was added to
+that node between the selected passes.
+
+In the following screenshot, we can see that the `8 Function$Lambda` node was
+removed (displayed in red) between passes `15: AliasAnalysis` and
+`18: SuspendedArguments`:
+![7](https://github.com/user-attachments/assets/21164dec-5a2b-449a-a335-9f0f4b60e4d0)
+Nodes that were added are displayed in green.
+
+Clicking on a single node, the points representing passes in the _Phase toolbar_
+change colors:
+
+- White: No change
+- Orange: Property on the node changed.
+- Black: Node was removed
+- Green: Node was added
+  ![8](https://github.com/user-attachments/assets/befc2083-e262-4933-90bc-f1729387d1ee)