Merge remote-tracking branch 'origin/wip/jtulach/MultiType11846' into…

… wip/jtulach/FirstDispatch11846

.gitignore

@@ -110,6 +110,7 @@ bench-report*.xml
 /enso.lib
 
 *.dll
+*.dylib
 *.exe
 *.pdb
 *.so

CHANGELOG.md

@@ -21,11 +21,15 @@
 - A constructor or type definition with a single inline argument definition was
   previously allowed to use spaces in the argument definition without
   parentheses. [This is now a syntax error.][11856]
+- [Native libraries of projects can be added to `polyglot/lib` directory][11874]
+- [Redo stack is no longer lost when interacting with text literals][11908].
 - Symetric, transitive and reflexive [equality for intersection types][11897]
 
 [11777]: https://github.com/enso-org/enso/pull/11777
 [11600]: https://github.com/enso-org/enso/pull/11600
 [11856]: https://github.com/enso-org/enso/pull/11856
+[11874]: https://github.com/enso-org/enso/pull/11874
+[11908]: https://github.com/enso-org/enso/pull/11908
 [11897]: https://github.com/enso-org/enso/pull/11897
 
 # Enso 2024.5

build.sbt

@@ -3712,7 +3712,8 @@ lazy val `engine-runner` = project
               // "-H:-DeleteLocalSymbols",
               // you may need to set smallJdk := None to use following flags:
               // "--trace-class-initialization=org.enso.syntax2.Parser",
-              "-Dnic=nic"
+              "-Dnic=nic",
+              "-Dorg.enso.feature.native.lib.output=" + (engineDistributionRoot.value / "bin")
             ),
             mainClass = Some("org.enso.runner.Main"),
             initializeAtRuntime = Seq(
@@ -4487,6 +4488,7 @@ def stdLibComponentRoot(name: String): File =
 val `base-polyglot-root`  = stdLibComponentRoot("Base") / "polyglot" / "java"
 val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java"
 val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java"
+val `image-native-libs`   = stdLibComponentRoot("Image") / "polyglot" / "lib"
 val `google-api-polyglot-root` =
   stdLibComponentRoot("Google_Api") / "polyglot" / "java"
 val `database-polyglot-root` =
@@ -4654,6 +4656,10 @@ lazy val `std-table` = project
   )
   .dependsOn(`std-base` % "provided")
 
+lazy val extractNativeLibs = taskKey[Unit](
+  "Helper task to extract native libraries from OpenCV JAR"
+)
+
 lazy val `std-image` = project
   .in(file("std-bits") / "image")
   .settings(
@@ -4669,15 +4675,21 @@ lazy val `std-image` = project
       "org.netbeans.api"     % "org-openide-util-lookup" % netbeansApiVersion        % "provided",
       "org.openpnp"          % "opencv"                  % opencvVersion
     ),
-    Compile / packageBin := Def.task {
-      val result = (Compile / packageBin).value
-      val _ = StdBits
-        .copyDependencies(
+    // Extract native libraries from opencv.jar, and put them under
+    // Standard/Image/polyglot/lib directory. The minimized opencv.jar will
+    // be put under Standard/Image/polyglot/java directory.
+    extractNativeLibs := {
+      StdBits
+        .extractNativeLibsFromOpenCV(
           `image-polyglot-root`,
-          Seq("std-image.jar"),
-          ignoreScalaLibrary = true
+          `image-native-libs`,
+          opencvVersion
         )
         .value
+    },
+    Compile / packageBin := Def.task {
+      val result = (Compile / packageBin).value
+      val _      = extractNativeLibs.value
       result
     }.value
   )

docs/polyglot/java.md

@@ -44,27 +44,60 @@ The dynamic polyglot system is a dynamic runtime lookup for Java objects,
 allowing Enso code to work with them through a runtime reflection-style
 mechanism. It is comprised of the following components:
 
-- `Java.lookup_class : Class.Path -> Maybe Class`: A function that lets users
-  look up a class by a given name on the runtime classpath.
-- `Polyglot.instantiate : Class -> Object`: A function that lets users
-  instantiate a class into an object.
+- `Java.lookup_class : Text -> Any`: A function that lets users look up a class
+  by a given name on the runtime classpath.
 - A whole host of functions on the polyglot type that let you dynamically work
   with object bindings.
 
 An example can be found below:
 
 ```ruby
+from Standard.Base.Polyglot import Java, polyglot
+
 main =
     class = Java.lookup_class "org.enso.example.TestClass"
-    instance = Polyglot.instantiate1 class (x -> x * 2)
+    instance = class.new (x -> x * 2)
     method = Polyglot.get_member instance "callFunctionAndIncrement"
-    Polyglot.execute1 method 10
+    Polyglot.execute method 10
 ```
 
 > The actionables for this section are:
 >
 > - Expand on the detail when there is time.
 
+## Native libraries
+
+Java can load native libraries using, e.g., the
+[System.loadLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/System.html#loadLibrary(java.lang.String)>)
+or
+[ClassLoader.findLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)>)
+methods. If a Java method loaded from the `polyglot/java` directory in project
+`Proj` tries to load a native library via one of the aforementioned mechanisms,
+the runtime system will look for the native library in the `polyglot/lib`
+directory within the project `Proj`. The runtime system implements this by
+overriding the
+[ClassLoader.findLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)>)
+method on the `ClassLoader` used to load the Java class.
+
+The algorithm used to search for the native libraries within the `polyglot/lib`
+directory hierarchy conforms to the
+[NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni):
+Lookup of library with name `native` works roughly in these steps:
+
+- Add platform-specific prefix and/or suffix to the library name, e.g.,
+  `libnative.so` on Linux.
+- Search for the library in the `polyglot/lib` directory.
+- Search for the library in the `polyglot/lib/<arch>` directory, where `<arch>`
+  is the name of the architecture.
+- Search for the library in the `polyglot/lib/<arch>/<os>` directory, where
+  `<os>` is the name of the operating system.
+
+Supported names:
+
+- Names for `<os>` are `linux`, `macos`, `windows`.
+  - Note that for simplicity we omit the versions of the operating systems.
+- Names for architectures `<arch>` are `amd64`, `x86_64`, `x86_32`, `aarch64`.
+
 ## Download a Java Library from Maven Central
 
 A typical use-case when bringing in some popular Java library into Enso

engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java

@@ -2,27 +2,41 @@
 
 import static scala.jdk.javaapi.CollectionConverters.asJava;
 
+import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.LinkedHashSet;
 import java.util.TreeSet;
 import org.enso.compiler.core.EnsoParser;
 import org.enso.compiler.core.ir.module.scope.imports.Polyglot;
+import org.enso.filesystem.FileSystem$;
+import org.enso.pkg.NativeLibraryFinder;
 import org.enso.pkg.PackageManager$;
 import org.graalvm.nativeimage.hosted.Feature;
 import org.graalvm.nativeimage.hosted.RuntimeProxyCreation;
 import org.graalvm.nativeimage.hosted.RuntimeReflection;
-import org.graalvm.nativeimage.hosted.RuntimeResourceAccess;
 
 public final class EnsoLibraryFeature implements Feature {
+  private static final String LIB_OUTPUT = "org.enso.feature.native.lib.output";
+  private final File nativeLibDir;
+
+  public EnsoLibraryFeature() {
+    var nativeLibOut = System.getProperty(LIB_OUTPUT);
+    if (nativeLibOut == null) {
+      throw new IllegalStateException("Missing system property: " + LIB_OUTPUT);
+    }
+    nativeLibDir = new File(nativeLibOut);
+    if (!nativeLibDir.exists() || !nativeLibDir.isDirectory()) {
+      var created = nativeLibDir.mkdirs();
+      if (!created) {
+        throw new IllegalStateException("Cannot create directory: " + nativeLibDir);
+      }
+    }
+  }
+
   @Override
   public void beforeAnalysis(BeforeAnalysisAccess access) {
-    try {
-      registerOpenCV(access.getApplicationClassLoader());
-    } catch (ReflectiveOperationException ex) {
-      ex.printStackTrace();
-      throw new IllegalStateException(ex);
-    }
+
     var libs = new LinkedHashSet<Path>();
     for (var p : access.getApplicationClassPath()) {
       var p1 = p.getParent();
@@ -53,6 +67,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
     */
 
     var classes = new TreeSet<String>();
+    var nativeLibPaths = new TreeSet<String>();
     try {
       for (var p : libs) {
         var result = PackageManager$.MODULE$.Default().loadPackage(p.toFile());
@@ -90,46 +105,26 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
               }
             }
           }
+          if (pkg.nativeLibraryDir().exists()) {
+            var nativeLibs =
+                NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs());
+            for (var nativeLib : nativeLibs) {
+              var out = new File(nativeLibDir, nativeLib.getName());
+              Files.copy(nativeLib.toPath(), out.toPath());
+              nativeLibPaths.add(out.getAbsolutePath());
+            }
+          }
         }
       }
     } catch (Exception ex) {
-      ex.printStackTrace();
+      ex.printStackTrace(System.err);
       throw new IllegalStateException(ex);
     }
     System.err.println("Summary for polyglot import java:");
     for (var className : classes) {
       System.err.println("  " + className);
     }
     System.err.println("Registered " + classes.size() + " classes for reflection");
-  }
-
-  private static void registerOpenCV(ClassLoader cl) throws ReflectiveOperationException {
-    var moduleOpenCV = cl.getUnnamedModule();
-    var currentOS = System.getProperty("os.name").toUpperCase().replaceAll(" .*$", "");
-
-    var libOpenCV =
-        switch (currentOS) {
-          case "LINUX" -> "nu/pattern/opencv/linux/x86_64/libopencv_java470.so";
-          case "WINDOWS" -> "nu/pattern/opencv/windows/x86_64/opencv_java470.dll";
-          case "MAC" -> {
-            var arch = System.getProperty("os.arch").toUpperCase();
-            yield switch (arch) {
-              case "X86_64" -> "nu/pattern/opencv/osx/x86_64/libopencv_java470.dylib";
-              case "AARCH64" -> "nu/pattern/opencv/osx/ARMv8/libopencv_java470.dylib";
-              default -> null;
-            };
-          }
-          default -> null;
-        };
-
-    if (libOpenCV != null) {
-      var verify = cl.getResource(libOpenCV);
-      if (verify == null) {
-        throw new IllegalStateException("Cannot find " + libOpenCV + " resource in " + cl);
-      }
-      RuntimeResourceAccess.addResource(moduleOpenCV, libOpenCV);
-    } else {
-      throw new IllegalStateException("No resource suggested for " + currentOS);
-    }
+    System.err.println("Copied native libraries: " + nativeLibPaths + " into " + nativeLibDir);
   }
 }

engine/runtime-benchmarks/src/main/java/org/enso/interpreter/bench/benchmarks/semantic/VectorBenchmarks.java

@@ -20,8 +20,8 @@
 
 @BenchmarkMode(Mode.AverageTime)
 @Fork(1)
-@Warmup(iterations = 3)
-@Measurement(iterations = 5)
+@Warmup(iterations = 3, time = 3, timeUnit = TimeUnit.SECONDS)
+@Measurement(iterations = 5, time = 2, timeUnit = TimeUnit.SECONDS)
 @OutputTimeUnit(TimeUnit.MILLISECONDS)
 @State(Scope.Benchmark)
 public class VectorBenchmarks {
@@ -35,10 +35,8 @@ public void initializeBenchmark(BenchmarkParams params) throws Exception {
     var benchmarkName = SrcUtil.findName(params);
     var code =
         """
-        import Standard.Base.Data.Vector.Builder
-        import Standard.Base.Data.Vector.Vector
+        from Standard.Base import all
         import Standard.Base.Data.Array_Proxy.Array_Proxy
-        from Standard.Base.Data.Boolean import False
 
         avg arr =
             sum acc i =
@@ -62,7 +60,16 @@ public void initializeBenchmark(BenchmarkParams params) throws Exception {
 
         to_vector arr = Vector.from_polyglot_array arr
         to_array vec = vec.to_array
-        slice vec = vec.slice
+        slice n vec start end =
+            initial = vec.slice start end
+
+            complicate round v = if round != 0 then complicate round-1 (v.slice 1 v.length) else
+                Runtime.assert v==initial
+                v
+
+            if n==0 then initial else
+                complicate n (0.up_to n).to_vector+initial
+
         fill_proxy proxy vec =
           size v =
             _ = v
@@ -91,7 +98,17 @@ public void initializeBenchmark(BenchmarkParams params) throws Exception {
         }
       case "averageOverSlice":
         {
-          this.arrayOfFibNumbers = getMethod.apply("slice").execute(self, vec, 1, length);
+          this.arrayOfFibNumbers = getMethod.apply("slice").execute(self, 0, vec, 1, length);
+          break;
+        }
+      case "averageOverSliceWrapped10":
+        {
+          this.arrayOfFibNumbers = getMethod.apply("slice").execute(self, 10, vec, 1, length);
+          break;
+        }
+      case "averageOverSliceWrapped100":
+        {
+          this.arrayOfFibNumbers = getMethod.apply("slice").execute(self, 100, vec, 1, length);
           break;
         }
       case "averageOverArray":
@@ -149,11 +166,24 @@ public void averageOverVector(Blackhole matter) {
     performBenchmark(matter);
   }
 
+  /** Measures performance of a single {@code Vector.slice}. */
   @Benchmark
   public void averageOverSlice(Blackhole matter) {
     performBenchmark(matter);
   }
 
+  /** Measures performance of {@code Vector.slice} applied ten times on the same array. */
+  @Benchmark
+  public void averageOverSliceWrapped10(Blackhole matter) {
+    performBenchmark(matter);
+  }
+
+  /** Measures performance of {@code Vector.slice} applied hundred times on the same array. */
+  @Benchmark
+  public void averageOverSliceWrapped100(Blackhole matter) {
+    performBenchmark(matter);
+  }
+
   @Benchmark
   public void averageOverPolyglotVector(Blackhole matter) {
     performBenchmark(matter);

engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java

@@ -34,7 +34,7 @@ public class SerializationManagerTest {
 
   @Before
   public void setup() {
-    packageManager = new PackageManager<>(new TruffleFileSystem());
+    packageManager = new PackageManager<>(TruffleFileSystem.INSTANCE);
     interpreterContext = new InterpreterContext(x -> x);
     ensoContext =
         interpreterContext