From 305db07ac6cf3f06a4e02d3bb5d16967e21c78b9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 16 Dec 2024 12:21:52 +0100 Subject: [PATCH 01/39] Update java.md docs --- docs/polyglot/java.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index 9efb70568ddc..547a562a9b73 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -44,21 +44,21 @@ The dynamic polyglot system is a dynamic runtime lookup for Java objects, allowing Enso code to work with them through a runtime reflection-style mechanism. It is comprised of the following components: -- `Java.lookup_class : Class.Path -> Maybe Class`: A function that lets users +- `Java.lookup_class : Text -> Any`: A function that lets users look up a class by a given name on the runtime classpath. -- `Polyglot.instantiate : Class -> Object`: A function that lets users - instantiate a class into an object. - A whole host of functions on the polyglot type that let you dynamically work with object bindings. An example can be found below: ```ruby +from Standard.Base.Polyglot import Java, polyglot + main = class = Java.lookup_class "org.enso.example.TestClass" - instance = Polyglot.instantiate1 class (x -> x * 2) + instance = class.new (x -> x * 2) method = Polyglot.get_member instance "callFunctionAndIncrement" - Polyglot.execute1 method 10 + Polyglot.execute method 10 ``` > The actionables for this section are: @@ -73,7 +73,7 @@ ecosystem is to download it (including is **transitive dependencies**) from libraries. Let's **start from scratch** by creating an _empty Enso project_: ```bash -$ bin/enso --new polydemo +$ bin/ensoup --new polydemo $ cd polydemo polydemo$ find . . From a67afdab3319d21ba1eaca1332cfcd118f48763c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 16 Dec 2024 12:44:59 +0100 Subject: [PATCH 02/39] Add docs --- docs/polyglot/java.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index 547a562a9b73..d901ee73b978 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -65,6 +65,21 @@ main = > > - Expand on the detail when there is time. +## Native libraries +Java can load native libraries using, e.g., the [System.loadLibrary](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/System.html#loadLibrary(java.lang.String)) +or [ClassLoader.findLibrary](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)) +methods. +If a Java method loaded from the `polyglot/java` directory in project `Proj` tries to load a +native library via one of the aforementioned mechanisms, the runtime system will look for the +native library in the `polyglot/lib` directory within the project `Proj`. +The algorithm used to search for the native libraries within the `polyglot/lib` directory hierarchy conforms to the [NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni). +The runtime system implements this by overriding the [ClassLoader.findLibrary](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)) +method on the `ClassLoader` used to load the Java class. + +Only the `polyglot/lib` directory within the project is searched for the native libraries. +This means that the native library search path is effectively _isolated_ for every project. + + ## Download a Java Library from Maven Central A typical use-case when bringing in some popular Java library into Enso From c98bdaf4e2b4fd0d79b735874e2430ec23526645 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 16 Dec 2024 17:09:19 +0100 Subject: [PATCH 03/39] Separate instance of HostClassLoader per project --- .../interop/java/AddToClassPathNode.java | 9 +++- .../enso/interpreter/runtime/EnsoContext.java | 44 ++++++++++++++----- .../interpreter/runtime/HostClassLoader.java | 14 +++++- .../runtime/DefaultPackageRepository.scala | 2 +- 4 files changed, 55 insertions(+), 14 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java index 4d757a703502..b19dc338e7b8 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java @@ -27,7 +27,12 @@ static AddToClassPathNode build() { Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) { var ctx = EnsoContext.get(this); var file = ctx.getTruffleFile(new File(expectStringNode.execute(path))); - ctx.addToClassPath(file); - return ctx.getBuiltins().nothing(); + var pkg = ctx.getPackageOf(file); + if (pkg.isEmpty()) { + throw ctx.raiseAssertionPanic(this, "File " + file + "should be in a package", null); + } else { + ctx.addToClassPath(file, pkg.get()); + return ctx.getBuiltins().nothing(); + } } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index c3f3bebd0f44..5abee4234697 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -30,7 +30,9 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -82,7 +84,7 @@ public final class EnsoContext { private final EnsoLanguage language; private final Env environment; - private final HostClassLoader hostClassLoader = new HostClassLoader(); + private final Map hostClassLoaders = new HashMap<>(); private final boolean assertionsEnabled; private final boolean isPrivateCheckDisabled; private final boolean isStaticTypeAnalysisEnabled; @@ -297,7 +299,7 @@ public void shutdown() { packageRepository.shutdown(); guestJava = null; topScope = null; - hostClassLoader.close(); + hostClassLoaders.values().forEach(HostClassLoader::close); EnsoParser.freeAll(); } @@ -466,10 +468,16 @@ public Optional findModuleByExpressionId(UUID expressionId) { * Modifies the classpath to use to lookup {@code polyglot java} imports. * * @param file the file to register + * @param pkg Package the file belongs to */ @TruffleBoundary - public void addToClassPath(TruffleFile file) { + public void addToClassPath(TruffleFile file, Package pkg) { if (findGuestJava() == null) { + var hostClassLoader = hostClassLoaders.get(pkg.libraryName()); + if (hostClassLoader == null) { + hostClassLoader = new HostClassLoader(pkg); + hostClassLoaders.put(pkg.libraryName(), hostClassLoader); + } try { var url = file.toUri().toURL(); hostClassLoader.add(url); @@ -589,16 +597,32 @@ public TruffleObject lookupJavaClass(String className) { return getBuiltins().error().makeMissingPolyglotImportError(className); } + /** + * Tries to find a Java class by its fully qualified name in all the {@link HostClassLoader host + * class loaders}. + */ private Object lookupHostSymbol(String fqn) throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException { - try { - if (findGuestJava() == null) { - return environment.asHostSymbol(hostClassLoader.loadClass(fqn)); - } else { - return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); + if (findGuestJava() == null) { + Class clazz = null; + ClassNotFoundException lastEx = null; + for (var hostClassLoader : hostClassLoaders.values()) { + try { + clazz = hostClassLoader.loadClass(fqn); + if (clazz != null) { + break; + } + } catch (ClassNotFoundException e) { + lastEx = e; + continue; + } + } + if (clazz == null) { + throw new ClassNotFoundException("Error loading host symbol " + fqn, lastEx); } - } catch (Error e) { - throw new ClassNotFoundException("Error loading " + fqn, e); + return environment.asHostSymbol(clazz); + } else { + return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index 4c2ffb2a2b6e..69c61274454d 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -1,9 +1,12 @@ package org.enso.interpreter.runtime; +import com.oracle.truffle.api.TruffleFile; import java.net.URL; import java.net.URLClassLoader; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.enso.pkg.Package; import org.graalvm.polyglot.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,12 +28,16 @@ final class HostClassLoader extends URLClassLoader implements AutoCloseable { // module layer's class loader. private static final ClassLoader polyglotClassLoader = Context.class.getClassLoader(); + private final Package pkg; + // polyglotClassLoader will be used only iff `org.enso.runtime` module is not in the // boot module layer. private static final boolean isRuntimeModInBootLayer; - public HostClassLoader() { + public HostClassLoader(Package pkg) { super(new URL[0]); + this.pkg = pkg; + assert pkg.polyglotDir().exists(); } static { @@ -85,4 +92,9 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE public void close() { loadedClasses.clear(); } + + @Override + public String toString() { + return "HostClassLoader(" + pkg.libraryName() + ")"; + } } diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index 3723cab9b1ae..ca6b5b8a3c1e 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -197,7 +197,7 @@ private class DefaultPackageRepository( isLibrary: Boolean ): Unit = { val extensions = pkg.listPolyglotExtensions("java") - extensions.foreach(context.addToClassPath) + extensions.foreach(ext => context.addToClassPath(ext, pkg)) val (regularModules, syntheticModulesMetadata) = pkg .listSources() From 84be17527b08eafac2f1b47d9d906fe59ad26fda Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 17 Dec 2024 13:14:51 +0100 Subject: [PATCH 04/39] Insert default HostClassLoader for builtin lib --- .../java/org/enso/interpreter/runtime/EnsoContext.java | 3 +++ .../org/enso/interpreter/runtime/HostClassLoader.java | 9 ++++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 5abee4234697..c72db6dcb655 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -164,6 +164,9 @@ public EnsoContext( scala.Option.empty()); this.home = home; this.builtins = new Builtins(this); + // Insert default host class loader. + var builtinsLibName = new LibraryName(Builtins.NAMESPACE, Builtins.PACKAGE_NAME); + this.hostClassLoaders.put(builtinsLibName, new HostClassLoader(null)); this.notificationHandler = notificationHandler; this.lockManager = lockManager; this.distributionManager = distributionManager; diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index 69c61274454d..0ac2923b8e1e 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -3,7 +3,6 @@ import com.oracle.truffle.api.TruffleFile; import java.net.URL; import java.net.URLClassLoader; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.enso.pkg.Package; @@ -37,7 +36,7 @@ final class HostClassLoader extends URLClassLoader implements AutoCloseable { public HostClassLoader(Package pkg) { super(new URL[0]); this.pkg = pkg; - assert pkg.polyglotDir().exists(); + assert pkg == null || pkg.polyglotDir().exists(); } static { @@ -95,6 +94,10 @@ public void close() { @Override public String toString() { - return "HostClassLoader(" + pkg.libraryName() + ")"; + if (pkg != null) { + return "HostClassLoader(" + pkg.libraryName() + ")"; + } else { + return "HostClassLoader"; + } } } From 1649ddfae87f3c12744e6a45dc2425b48afe685e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 17 Dec 2024 18:03:08 +0100 Subject: [PATCH 05/39] Revert "Insert default HostClassLoader for builtin lib" This reverts commit 84be17527b08eafac2f1b47d9d906fe59ad26fda. --- .../java/org/enso/interpreter/runtime/EnsoContext.java | 3 --- .../org/enso/interpreter/runtime/HostClassLoader.java | 9 +++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index c72db6dcb655..5abee4234697 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -164,9 +164,6 @@ public EnsoContext( scala.Option.empty()); this.home = home; this.builtins = new Builtins(this); - // Insert default host class loader. - var builtinsLibName = new LibraryName(Builtins.NAMESPACE, Builtins.PACKAGE_NAME); - this.hostClassLoaders.put(builtinsLibName, new HostClassLoader(null)); this.notificationHandler = notificationHandler; this.lockManager = lockManager; this.distributionManager = distributionManager; diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index 0ac2923b8e1e..69c61274454d 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -3,6 +3,7 @@ import com.oracle.truffle.api.TruffleFile; import java.net.URL; import java.net.URLClassLoader; +import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.enso.pkg.Package; @@ -36,7 +37,7 @@ final class HostClassLoader extends URLClassLoader implements AutoCloseable { public HostClassLoader(Package pkg) { super(new URL[0]); this.pkg = pkg; - assert pkg == null || pkg.polyglotDir().exists(); + assert pkg.polyglotDir().exists(); } static { @@ -94,10 +95,6 @@ public void close() { @Override public String toString() { - if (pkg != null) { - return "HostClassLoader(" + pkg.libraryName() + ")"; - } else { - return "HostClassLoader"; - } + return "HostClassLoader(" + pkg.libraryName() + ")"; } } From 05851f56a2fec49485c2da7ef1e6b95f91d1687f Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 17 Dec 2024 18:03:11 +0100 Subject: [PATCH 06/39] Revert "Separate instance of HostClassLoader per project" This reverts commit c98bdaf4e2b4fd0d79b735874e2430ec23526645. --- .../interop/java/AddToClassPathNode.java | 9 +--- .../enso/interpreter/runtime/EnsoContext.java | 44 +++++-------------- .../interpreter/runtime/HostClassLoader.java | 14 +----- .../runtime/DefaultPackageRepository.scala | 2 +- 4 files changed, 14 insertions(+), 55 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java index b19dc338e7b8..4d757a703502 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java @@ -27,12 +27,7 @@ static AddToClassPathNode build() { Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) { var ctx = EnsoContext.get(this); var file = ctx.getTruffleFile(new File(expectStringNode.execute(path))); - var pkg = ctx.getPackageOf(file); - if (pkg.isEmpty()) { - throw ctx.raiseAssertionPanic(this, "File " + file + "should be in a package", null); - } else { - ctx.addToClassPath(file, pkg.get()); - return ctx.getBuiltins().nothing(); - } + ctx.addToClassPath(file); + return ctx.getBuiltins().nothing(); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 5abee4234697..c3f3bebd0f44 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -30,9 +30,7 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -84,7 +82,7 @@ public final class EnsoContext { private final EnsoLanguage language; private final Env environment; - private final Map hostClassLoaders = new HashMap<>(); + private final HostClassLoader hostClassLoader = new HostClassLoader(); private final boolean assertionsEnabled; private final boolean isPrivateCheckDisabled; private final boolean isStaticTypeAnalysisEnabled; @@ -299,7 +297,7 @@ public void shutdown() { packageRepository.shutdown(); guestJava = null; topScope = null; - hostClassLoaders.values().forEach(HostClassLoader::close); + hostClassLoader.close(); EnsoParser.freeAll(); } @@ -468,16 +466,10 @@ public Optional findModuleByExpressionId(UUID expressionId) { * Modifies the classpath to use to lookup {@code polyglot java} imports. * * @param file the file to register - * @param pkg Package the file belongs to */ @TruffleBoundary - public void addToClassPath(TruffleFile file, Package pkg) { + public void addToClassPath(TruffleFile file) { if (findGuestJava() == null) { - var hostClassLoader = hostClassLoaders.get(pkg.libraryName()); - if (hostClassLoader == null) { - hostClassLoader = new HostClassLoader(pkg); - hostClassLoaders.put(pkg.libraryName(), hostClassLoader); - } try { var url = file.toUri().toURL(); hostClassLoader.add(url); @@ -597,32 +589,16 @@ public TruffleObject lookupJavaClass(String className) { return getBuiltins().error().makeMissingPolyglotImportError(className); } - /** - * Tries to find a Java class by its fully qualified name in all the {@link HostClassLoader host - * class loaders}. - */ private Object lookupHostSymbol(String fqn) throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException { - if (findGuestJava() == null) { - Class clazz = null; - ClassNotFoundException lastEx = null; - for (var hostClassLoader : hostClassLoaders.values()) { - try { - clazz = hostClassLoader.loadClass(fqn); - if (clazz != null) { - break; - } - } catch (ClassNotFoundException e) { - lastEx = e; - continue; - } - } - if (clazz == null) { - throw new ClassNotFoundException("Error loading host symbol " + fqn, lastEx); + try { + if (findGuestJava() == null) { + return environment.asHostSymbol(hostClassLoader.loadClass(fqn)); + } else { + return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); } - return environment.asHostSymbol(clazz); - } else { - return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); + } catch (Error e) { + throw new ClassNotFoundException("Error loading " + fqn, e); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index 69c61274454d..4c2ffb2a2b6e 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -1,12 +1,9 @@ package org.enso.interpreter.runtime; -import com.oracle.truffle.api.TruffleFile; import java.net.URL; import java.net.URLClassLoader; -import java.util.Locale; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import org.enso.pkg.Package; import org.graalvm.polyglot.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,16 +25,12 @@ final class HostClassLoader extends URLClassLoader implements AutoCloseable { // module layer's class loader. private static final ClassLoader polyglotClassLoader = Context.class.getClassLoader(); - private final Package pkg; - // polyglotClassLoader will be used only iff `org.enso.runtime` module is not in the // boot module layer. private static final boolean isRuntimeModInBootLayer; - public HostClassLoader(Package pkg) { + public HostClassLoader() { super(new URL[0]); - this.pkg = pkg; - assert pkg.polyglotDir().exists(); } static { @@ -92,9 +85,4 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE public void close() { loadedClasses.clear(); } - - @Override - public String toString() { - return "HostClassLoader(" + pkg.libraryName() + ")"; - } } diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index ca6b5b8a3c1e..3723cab9b1ae 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -197,7 +197,7 @@ private class DefaultPackageRepository( isLibrary: Boolean ): Unit = { val extensions = pkg.listPolyglotExtensions("java") - extensions.foreach(ext => context.addToClassPath(ext, pkg)) + extensions.foreach(context.addToClassPath) val (regularModules, syntheticModulesMetadata) = pkg .listSources() From c17b62d93d580b1ebe9d1c24377cfe88d3feb854 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 17 Dec 2024 18:23:55 +0100 Subject: [PATCH 07/39] Add NativeLibraryFinder --- .../interpreter/runtime/HostClassLoader.java | 25 +++++++++ .../runtime/NativeLibraryFinder.java | 53 +++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index 4c2ffb2a2b6e..bfb1d7b87017 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -81,6 +81,31 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE } } + /** + * Find the library with the specified name inside the {@code polyglot/lib} directory of caller's + * project. The search inside the {@code polyglot/lib} directory hierarchy is specified by NetBeans + * JNI specification. + * + *

Note: The current implementation iterates all the {@code polyglot/lib} directories of all + * the packages. + * + * @param libname The library name. Without platform-specific suffix or prefix. + * @return Absolute path to the library if found, or null. + */ + @Override + protected String findLibrary(String libname) { + var pkgRepo = EnsoContext.get(null).getPackageRepository(); + for (var pkg : pkgRepo.getLoadedPackagesJava()) { + var libPath = NativeLibraryFinder.findNativeLibrary(libname, pkg); + if (libPath != null) { + return libPath; + } + } + logger.trace("Native library {} not found in any package", libname); + return null; + } + @Override public void close() { loadedClasses.clear(); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java new file mode 100644 index 000000000000..3cf20d86e9b5 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java @@ -0,0 +1,53 @@ +package org.enso.interpreter.runtime; + +import com.oracle.truffle.api.TruffleFile; +import java.util.Locale; +import org.enso.pkg.Package; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Helper class to find native libraries in packages. The search algorithm complies to the NetBeans + * JNI specification. + */ +final class NativeLibraryFinder { + + private static final Logger logger = LoggerFactory.getLogger(NativeLibraryFinder.class); + + private NativeLibraryFinder() {} + + /** + * Tries to find native library in the given package. + * + * @param libName the name of the library to find, without platform specific prefix or suffix. + * @param pkg the package to search in. + * @return null if not found, absolute path otherwise. + */ + static String findNativeLibrary(String libName, Package pkg) { + var arch = System.getProperty("os.arch"); + var osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + var libNameWithSuffix = System.mapLibraryName(libName); + var libDir = pkg.polyglotDir().resolve("lib"); + if (!libDir.exists()) { + logger.trace("Native library directory {} does not exist", libDir); + return null; + } + var nativeLib = libDir.resolve(libNameWithSuffix); + if (nativeLib.exists()) { + logger.trace("Found native library {}", nativeLib); + return nativeLib.getAbsoluteFile().getPath(); + } + nativeLib = libDir.resolve(arch).resolve(libNameWithSuffix); + if (nativeLib.exists()) { + logger.trace("Found native library {}", nativeLib); + return nativeLib.getAbsoluteFile().getPath(); + } + nativeLib = libDir.resolve(arch).resolve(osName).resolve(libNameWithSuffix); + if (nativeLib.exists()) { + logger.trace("Found native library {}", nativeLib); + return nativeLib.getAbsoluteFile().getPath(); + } + return null; + } +} From 83c43e73f8094747ad50c5856d1b14105882bdc0 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 18 Dec 2024 10:56:58 +0100 Subject: [PATCH 08/39] OpenCV native lib is not copied to a temporary directory, but loaded from a dll --- std-bits/image/src/main/java/org/enso/image/Codecs.java | 2 +- std-bits/image/src/main/java/org/enso/image/data/Image.java | 2 +- std-bits/image/src/main/java/org/enso/image/data/Matrix.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/std-bits/image/src/main/java/org/enso/image/Codecs.java b/std-bits/image/src/main/java/org/enso/image/Codecs.java index 9706d6b38878..83f74eb7d1af 100644 --- a/std-bits/image/src/main/java/org/enso/image/Codecs.java +++ b/std-bits/image/src/main/java/org/enso/image/Codecs.java @@ -11,7 +11,7 @@ public class Codecs { public static final int READ_FLAG_EMPTY = -127; static { - OpenCV.loadLocally(); + OpenCV.loadShared(); } /** An error occurred when reading a file. */ diff --git a/std-bits/image/src/main/java/org/enso/image/data/Image.java b/std-bits/image/src/main/java/org/enso/image/data/Image.java index 74d0f6d25072..2fd690060001 100644 --- a/std-bits/image/src/main/java/org/enso/image/data/Image.java +++ b/std-bits/image/src/main/java/org/enso/image/data/Image.java @@ -11,7 +11,7 @@ public class Image { static { - OpenCV.loadLocally(); + OpenCV.loadShared(); } private static final byte MAX_SIGNED_BYTE = -1; diff --git a/std-bits/image/src/main/java/org/enso/image/data/Matrix.java b/std-bits/image/src/main/java/org/enso/image/data/Matrix.java index 65d46569515d..c87c6d830193 100644 --- a/std-bits/image/src/main/java/org/enso/image/data/Matrix.java +++ b/std-bits/image/src/main/java/org/enso/image/data/Matrix.java @@ -7,7 +7,7 @@ public class Matrix { static { - OpenCV.loadLocally(); + OpenCV.loadShared(); } /** From c51a2cca362a276a21b0fc50eb5fe2e4413fb0b3 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 18 Dec 2024 18:57:52 +0100 Subject: [PATCH 09/39] std-image/compile does not copy opencv.jar --- build.sbt | 3 ++- project/StdBits.scala | 26 +++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 758ae5cd9415..499f9bbe4950 100644 --- a/build.sbt +++ b/build.sbt @@ -4671,7 +4671,8 @@ lazy val `std-image` = project .copyDependencies( `image-polyglot-root`, Seq("std-image.jar"), - ignoreScalaLibrary = true + ignoreScalaLibrary = true, + ignoreDependency = Some("org.openpnp" % "opencv" % opencvVersion) ) .value result diff --git a/project/StdBits.scala b/project/StdBits.scala index 2e34470e3b84..f5500b2f589d 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -2,7 +2,11 @@ import sbt.Keys._ import sbt._ import sbt.internal.util.ManagedLogger import sbt.io.IO -import sbt.librarymanagement.{ConfigurationFilter, DependencyFilter} +import sbt.librarymanagement.{ + ConfigurationFilter, + DependencyFilter, + ModuleFilter +} import java.io.File @@ -19,11 +23,13 @@ object StdBits { * @param ignoreScalaLibrary whether to ignore Scala dependencies that are * added by default be SBT and are not relevant in * pure-Java projects + * @param ignoreDependency A dependency that should be ignored - not copied to the destination */ def copyDependencies( destination: File, providedJarNames: Seq[String], - ignoreScalaLibrary: Boolean + ignoreScalaLibrary: Boolean, + ignoreDependency: Option[ModuleID] = None ): Def.Initialize[Task[Unit]] = Def.task { val libraryUpdates = (Compile / update).value @@ -48,12 +54,26 @@ object StdBits { !graalVmOrgs.contains(orgName) }) ) + val moduleFilter = ignoreDependency match { + case None => graalModuleFilter + case Some(ignoreDepID) => + DependencyFilter.moduleFilter( + organization = new SimpleFilter(orgName => { + !graalVmOrgs.contains( + orgName + ) && orgName != ignoreDepID.organization + }), + name = new SimpleFilter(nm => { + nm != ignoreDepID.name + }) + ) + } val unmanagedFiles = (Compile / unmanagedJars).value.map(_.data) val relevantFiles = libraryUpdates .select( configuration = configFilter, - module = graalModuleFilter, + module = moduleFilter, artifact = DependencyFilter.artifactFilter() ) ++ unmanagedFiles val dependencyStore = From 1d7c0bb47aef99c41607483738b0b6e77e4cf8dc Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 18 Dec 2024 19:10:38 +0100 Subject: [PATCH 10/39] Add JARUtils to sbt project --- project/JARUtils.scala | 55 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 project/JARUtils.scala diff --git a/project/JARUtils.scala b/project/JARUtils.scala new file mode 100644 index 000000000000..b014c353098c --- /dev/null +++ b/project/JARUtils.scala @@ -0,0 +1,55 @@ +import java.io.IOException +import java.nio.file.{Files, Path} +import java.util.jar.{JarEntry, JarFile, JarOutputStream} +import scala.util.{Try, Using} + +object JARUtils { + + /** Extracts all file entries starting with `extractPrefix` from `inputJarPath` to `extractedFilesDir`, + * optionally renaming them with `renameFunc`. + * The rest is copied into `outputJarPath`. + * + * @param inputJarPath Path to the JAR archive. Will not be modified. + * @param extractPrefix Prefix of the files to extract. + * @param outputJarPath Path to the output JAR. Input JAR will be copied here without the files + * starting with `extractPrefix`. + * @param extractedFilesDir Destination directory for the extracted files. The prefix from the + * extracted files is tripped. + * @param renameFunc Function that renames the extracted files. The extracted file name is taken + * from the jar entry, and thus may contain slashes. If None is returned, the + * file is ignored and not extracted. + */ + def extractFilesFromJar( + inputJarPath: Path, + extractPrefix: String, + outputJarPath: Path, + extractedFilesDir: Path, + renameFunc: String => Option[String] + ): Unit = { + Using(new JarFile(inputJarPath.toFile)) { inputJar => + Using(new JarOutputStream(Files.newOutputStream(outputJarPath))) { + outputJar => + inputJar.stream().forEach { entry => + if (entry.getName.startsWith(extractPrefix) && !entry.isDirectory) { + renameFunc(entry.getName) match { + case Some(strippedEntryName) => + assert(!strippedEntryName.startsWith("/")) + val destFile = extractedFilesDir.resolve(strippedEntryName) + Files.createDirectories(destFile.getParent) + Using(inputJar.getInputStream(entry)) { is => + Files.copy(is, destFile) + } + case None => () + } + } else { + outputJar.putNextEntry(new JarEntry(entry.getName)) + Using(inputJar.getInputStream(entry)) { is => + is.transferTo(outputJar) + } + outputJar.closeEntry() + } + } + } + } + } +} From 98c8a61e5d5b6080ad65d2fed3c2904557c0d640 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 18 Dec 2024 19:30:21 +0100 Subject: [PATCH 11/39] Add *.dylib to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 9d63591daa09..444645571d9b 100644 --- a/.gitignore +++ b/.gitignore @@ -110,6 +110,7 @@ bench-report*.xml /enso.lib *.dll +*.dylib *.exe *.pdb *.so From 5f0ce0a3d4cb8687a4bb77d8c5aae8a0c9dfc1ba Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 19 Dec 2024 12:21:13 +0100 Subject: [PATCH 12/39] Add error recovery to JARUtils --- project/JARUtils.scala | 51 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/project/JARUtils.scala b/project/JARUtils.scala index b014c353098c..f8d14e9d652d 100644 --- a/project/JARUtils.scala +++ b/project/JARUtils.scala @@ -24,8 +24,11 @@ object JARUtils { extractPrefix: String, outputJarPath: Path, extractedFilesDir: Path, - renameFunc: String => Option[String] + renameFunc: String => Option[String], + logger: sbt.util.Logger ): Unit = { + ensureDirExists(outputJarPath.getParent, logger) + ensureDirExists(extractedFilesDir, logger) Using(new JarFile(inputJarPath.toFile)) { inputJar => Using(new JarOutputStream(Files.newOutputStream(outputJarPath))) { outputJar => @@ -34,21 +37,61 @@ object JARUtils { renameFunc(entry.getName) match { case Some(strippedEntryName) => assert(!strippedEntryName.startsWith("/")) + assert(extractedFilesDir.toFile.exists) val destFile = extractedFilesDir.resolve(strippedEntryName) - Files.createDirectories(destFile.getParent) + if (!destFile.getParent.toFile.exists) { + Files.createDirectories(destFile.getParent) + } Using(inputJar.getInputStream(entry)) { is => Files.copy(is, destFile) - } + }.recover({ case e: IOException => + logger.err( + s"Failed to extract $entry to $destFile: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) case None => () } } else { outputJar.putNextEntry(new JarEntry(entry.getName)) Using(inputJar.getInputStream(entry)) { is => is.transferTo(outputJar) - } + }.recover({ case e: IOException => + logger.err( + s"Failed to copy $entry to output JAR: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) outputJar.closeEntry() } } + }.recover({ case e: IOException => + logger.err( + s"Failed to create output JAR at $outputJarPath: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) + }.recover({ case e: IOException => + logger.err( + s"Failed to extract files from $inputJarPath to $extractedFilesDir: ${e.getMessage}" + ) + e.printStackTrace(System.err) + }) + } + + private def ensureDirExists( + path: Path, + logger: sbt.util.Logger + ): Unit = { + if (path != null) { + try { + Files.createDirectories(path) + } catch { + case e: IOException => + logger.err( + s"Failed to create parent directories for $path: ${e.getMessage}" + ) + e.printStackTrace(System.err) } } } From e2d7099486f852bcb17522466314d19249246ca9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 19 Dec 2024 12:22:11 +0100 Subject: [PATCH 13/39] std-image/compile extract native libraries from OpenCV.jar --- build.sbt | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 499f9bbe4950..5044a93664aa 100644 --- a/build.sbt +++ b/build.sbt @@ -4483,6 +4483,7 @@ def stdLibComponentRoot(name: String): File = val `base-polyglot-root` = stdLibComponentRoot("Base") / "polyglot" / "java" val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java" val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java" +val `image-native-libs` = stdLibComponentRoot("Image") / "polyglot" / "lib" val `google-api-polyglot-root` = stdLibComponentRoot("Google_Api") / "polyglot" / "java" val `database-polyglot-root` = @@ -4650,6 +4651,10 @@ lazy val `std-table` = project ) .dependsOn(`std-base` % "provided") +lazy val extractNativeLibs = taskKey[Unit]( + "Helper task to extract native libraries from OpenCV JAR" +) + lazy val `std-image` = project .in(file("std-bits") / "image") .settings( @@ -4665,8 +4670,11 @@ lazy val `std-image` = project "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", "org.openpnp" % "opencv" % opencvVersion ), - Compile / packageBin := Def.task { - val result = (Compile / packageBin).value + // Extract native libraries from opencv.jar, and put them under + // Standard/Image/polyglot/lib directory. The minimized opencv.jar will + // be put under Standard/Image/polyglot/java directory. + extractNativeLibs := { + // Ensure dependencies are first copied. val _ = StdBits .copyDependencies( `image-polyglot-root`, @@ -4675,6 +4683,44 @@ lazy val `std-image` = project ignoreDependency = Some("org.openpnp" % "opencv" % opencvVersion) ) .value + val extractPrefix = "nu/pattern/opencv" + def renameFunc(entryName: String): Option[String] = { + val strippedEntryName = entryName.substring(extractPrefix.length + 1) + if ( + strippedEntryName.contains("linux/ARM") || + strippedEntryName.contains("linux/x86_32") || + strippedEntryName.contains("README.md") + ) { + None + } else { + Some(strippedEntryName.replace("osx", "macos")) + } + } + val logger = streams.value.log + val openCvJar = JPMSUtils + .filterModulesFromUpdate( + update.value, + Seq("org.openpnp" % "opencv" % opencvVersion), + logger, + moduleName.value, + scalaBinaryVersion.value, + shouldContainAll = true + ) + .head + val outputJarPath = (`image-polyglot-root` / "opencv.jar").toPath + val extractedFilesDir = `image-native-libs`.toPath + JARUtils.extractFilesFromJar( + openCvJar.toPath, + extractPrefix, + outputJarPath, + extractedFilesDir, + renameFunc, + logger + ) + }, + Compile / packageBin := Def.task { + val result = (Compile / packageBin).value + val _ = extractNativeLibs.value result }.value ) From 561bcdcb31780c26d0cf7d56155a61e3250d4c58 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 19 Dec 2024 12:23:04 +0100 Subject: [PATCH 14/39] fmt --- project/StdBits.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/project/StdBits.scala b/project/StdBits.scala index f5500b2f589d..2d41dcdced10 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -5,7 +5,6 @@ import sbt.io.IO import sbt.librarymanagement.{ ConfigurationFilter, DependencyFilter, - ModuleFilter } import java.io.File From f0c930e93b6ef879bc386aeac38c3fb9843798ff Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 13:43:43 +0100 Subject: [PATCH 15/39] JAR extraction uses sbt cache. So that no extraction is done unnecessarily. --- build.sbt | 5 +++-- project/JARUtils.scala | 42 +++++++++++++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/build.sbt b/build.sbt index 5044a93664aa..5417a4bbc396 100644 --- a/build.sbt +++ b/build.sbt @@ -4678,7 +4678,7 @@ lazy val `std-image` = project val _ = StdBits .copyDependencies( `image-polyglot-root`, - Seq("std-image.jar"), + Seq("std-image.jar", "opencv.jar"), ignoreScalaLibrary = true, ignoreDependency = Some("org.openpnp" % "opencv" % opencvVersion) ) @@ -4715,7 +4715,8 @@ lazy val `std-image` = project outputJarPath, extractedFilesDir, renameFunc, - logger + logger, + streams.value.cacheStoreFactory ) }, Compile / packageBin := Def.task { diff --git a/project/JARUtils.scala b/project/JARUtils.scala index f8d14e9d652d..5b977066d2dd 100644 --- a/project/JARUtils.scala +++ b/project/JARUtils.scala @@ -1,3 +1,7 @@ +import sbt.{IO, Tracked} +import sbt.std.Streams +import sbt.util.{CacheStoreFactory, FileInfo} + import java.io.IOException import java.nio.file.{Files, Path} import java.util.jar.{JarEntry, JarFile, JarOutputStream} @@ -25,10 +29,29 @@ object JARUtils { outputJarPath: Path, extractedFilesDir: Path, renameFunc: String => Option[String], - logger: sbt.util.Logger + logger: sbt.util.Logger, + cacheStoreFactory: CacheStoreFactory ): Unit = { - ensureDirExists(outputJarPath.getParent, logger) - ensureDirExists(extractedFilesDir, logger) + val dependencyStore = cacheStoreFactory.make("extract-jar-files") + // Make sure that the actual file extraction is done only iff some of the cached files change. + val cachedFiles = Set( + inputJarPath.toFile, + ) + var shouldExtract = false + Tracked.diffInputs(dependencyStore, FileInfo.hash)(cachedFiles) { + report => + shouldExtract = report.modified.nonEmpty || report.removed.nonEmpty || report.added.nonEmpty + } + + if (!shouldExtract) { + logger.debug("No changes in the input JAR, skipping extraction.") + return + } else { + logger.info(s"Extracting files with prefix '${extractPrefix}' from $inputJarPath to $extractedFilesDir.") + } + + ensureDirExistsAndIsClean(outputJarPath.getParent, logger) + ensureDirExistsAndIsClean(extractedFilesDir, logger) Using(new JarFile(inputJarPath.toFile)) { inputJar => Using(new JarOutputStream(Files.newOutputStream(outputJarPath))) { outputJar => @@ -79,17 +102,22 @@ object JARUtils { }) } - private def ensureDirExists( + private def ensureDirExistsAndIsClean( path: Path, logger: sbt.util.Logger ): Unit = { - if (path != null) { + require(path != null) + val dir = path.toFile + if (dir.exists && dir.isDirectory) { + // Clean previous contents + IO.delete(IO.listFiles(dir)) + } else { try { - Files.createDirectories(path) + IO.createDirectory(dir) } catch { case e: IOException => logger.err( - s"Failed to create parent directories for $path: ${e.getMessage}" + s"Failed to create directory $path: ${e.getMessage}" ) e.printStackTrace(System.err) } From 5f158b8a48a6329ad002e1295b74a7814e17dfad Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 14:07:34 +0100 Subject: [PATCH 16/39] Move extractNativeLibs task to StdBits --- build.sbt | 45 +++-------------------------- project/StdBits.scala | 67 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 67 insertions(+), 45 deletions(-) diff --git a/build.sbt b/build.sbt index 5417a4bbc396..d8e2847307c0 100644 --- a/build.sbt +++ b/build.sbt @@ -4674,50 +4674,13 @@ lazy val `std-image` = project // Standard/Image/polyglot/lib directory. The minimized opencv.jar will // be put under Standard/Image/polyglot/java directory. extractNativeLibs := { - // Ensure dependencies are first copied. - val _ = StdBits - .copyDependencies( + StdBits + .extractNativeLibsFromOpenCV( `image-polyglot-root`, - Seq("std-image.jar", "opencv.jar"), - ignoreScalaLibrary = true, - ignoreDependency = Some("org.openpnp" % "opencv" % opencvVersion) + `image-native-libs`, + opencvVersion ) .value - val extractPrefix = "nu/pattern/opencv" - def renameFunc(entryName: String): Option[String] = { - val strippedEntryName = entryName.substring(extractPrefix.length + 1) - if ( - strippedEntryName.contains("linux/ARM") || - strippedEntryName.contains("linux/x86_32") || - strippedEntryName.contains("README.md") - ) { - None - } else { - Some(strippedEntryName.replace("osx", "macos")) - } - } - val logger = streams.value.log - val openCvJar = JPMSUtils - .filterModulesFromUpdate( - update.value, - Seq("org.openpnp" % "opencv" % opencvVersion), - logger, - moduleName.value, - scalaBinaryVersion.value, - shouldContainAll = true - ) - .head - val outputJarPath = (`image-polyglot-root` / "opencv.jar").toPath - val extractedFilesDir = `image-native-libs`.toPath - JARUtils.extractFilesFromJar( - openCvJar.toPath, - extractPrefix, - outputJarPath, - extractedFilesDir, - renameFunc, - logger, - streams.value.cacheStoreFactory - ) }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/project/StdBits.scala b/project/StdBits.scala index 2d41dcdced10..3b1152720254 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -2,10 +2,7 @@ import sbt.Keys._ import sbt._ import sbt.internal.util.ManagedLogger import sbt.io.IO -import sbt.librarymanagement.{ - ConfigurationFilter, - DependencyFilter, -} +import sbt.librarymanagement.{ConfigurationFilter, DependencyFilter} import java.io.File @@ -105,6 +102,68 @@ object StdBits { } } + /** Extract native libraries from `opencv.jar` and put them under + * `Standard/Image/polyglot/lib` directory. The minimized `opencv.jar` will + * be put under `Standard/Image/polyglot/java` directory. + * @param imagePolyglotRoot root dir of Std image polyglot dir + * @param imageNativeLibs root dir of Std image lib dir + * @return + */ + def extractNativeLibsFromOpenCV( + imagePolyglotRoot: File, + imageNativeLibs: File, + opencvVersion: String + ): Def.Initialize[Task[Unit]] = Def.task { + // Ensure dependencies are first copied. + val _ = StdBits + .copyDependencies( + imagePolyglotRoot, + Seq("std-image.jar", "opencv.jar"), + ignoreScalaLibrary = true, + ignoreDependency = Some("org.openpnp" % "opencv" % opencvVersion) + ) + .value + val extractPrefix = "nu/pattern/opencv" + def renameFunc(entryName: String): Option[String] = { + val strippedEntryName = entryName.substring(extractPrefix.length + 1) + if ( + strippedEntryName.contains("linux/ARM") || + strippedEntryName.contains("linux/x86_32") || + strippedEntryName.contains("README.md") + ) { + None + } else { + Some( + strippedEntryName + .replace("osx", "macos") + .replace("x86_64", "amd64") + ) + } + } + val logger = streams.value.log + val openCvJar = JPMSUtils + .filterModulesFromUpdate( + update.value, + Seq("org.openpnp" % "opencv" % opencvVersion), + logger, + moduleName.value, + scalaBinaryVersion.value, + shouldContainAll = true + ) + .head + val outputJarPath = (imagePolyglotRoot / "opencv.jar").toPath + val extractedFilesDir = imageNativeLibs.toPath + JARUtils.extractFilesFromJar( + openCvJar.toPath, + extractPrefix, + outputJarPath, + extractedFilesDir, + renameFunc, + logger, + streams.value.cacheStoreFactory + ) + } + private def updateDependency( jar: File, destinationDir: File, From c6d596341f2e8ece1dd64106db7d132c05054b5f Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 14:07:51 +0100 Subject: [PATCH 17/39] NativeLibraryFinder strips version from os.arch --- .../org/enso/interpreter/runtime/NativeLibraryFinder.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java index 3cf20d86e9b5..45d3aa4d4198 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java @@ -25,7 +25,11 @@ private NativeLibraryFinder() {} * @return null if not found, absolute path otherwise. */ static String findNativeLibrary(String libName, Package pkg) { - var arch = System.getProperty("os.arch"); + var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); + if (arch.contains(" ")) { + // Strip version information from the architecture string. + arch = arch.substring(0, arch.indexOf(' ')); + } var osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); var libNameWithSuffix = System.mapLibraryName(libName); var libDir = pkg.polyglotDir().resolve("lib"); From 7edc3be8688eda364d645bc35093c4fae1196d77 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 14:07:59 +0100 Subject: [PATCH 18/39] fmt --- project/JARUtils.scala | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/project/JARUtils.scala b/project/JARUtils.scala index 5b977066d2dd..8d7a122bf2d7 100644 --- a/project/JARUtils.scala +++ b/project/JARUtils.scala @@ -35,19 +35,21 @@ object JARUtils { val dependencyStore = cacheStoreFactory.make("extract-jar-files") // Make sure that the actual file extraction is done only iff some of the cached files change. val cachedFiles = Set( - inputJarPath.toFile, + inputJarPath.toFile ) var shouldExtract = false - Tracked.diffInputs(dependencyStore, FileInfo.hash)(cachedFiles) { - report => - shouldExtract = report.modified.nonEmpty || report.removed.nonEmpty || report.added.nonEmpty + Tracked.diffInputs(dependencyStore, FileInfo.hash)(cachedFiles) { report => + shouldExtract = + report.modified.nonEmpty || report.removed.nonEmpty || report.added.nonEmpty } if (!shouldExtract) { logger.debug("No changes in the input JAR, skipping extraction.") return } else { - logger.info(s"Extracting files with prefix '${extractPrefix}' from $inputJarPath to $extractedFilesDir.") + logger.info( + s"Extracting files with prefix '${extractPrefix}' from $inputJarPath to $extractedFilesDir." + ) } ensureDirExistsAndIsClean(outputJarPath.getParent, logger) From fc325f295e84ba2f132cefed02bb2160e95721d7 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 14:26:39 +0100 Subject: [PATCH 19/39] NativeLibFinder simplifies os name --- .../runtime/NativeLibraryFinder.java | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java index 45d3aa4d4198..3b7f8b261515 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java @@ -26,11 +26,7 @@ private NativeLibraryFinder() {} */ static String findNativeLibrary(String libName, Package pkg) { var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); - if (arch.contains(" ")) { - // Strip version information from the architecture string. - arch = arch.substring(0, arch.indexOf(' ')); - } - var osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + var osName = simpleOsName(); var libNameWithSuffix = System.mapLibraryName(libName); var libDir = pkg.polyglotDir().resolve("lib"); if (!libDir.exists()) { @@ -54,4 +50,21 @@ static String findNativeLibrary(String libName, Package pkg) { } return null; } + + private static String simpleOsName() { + var osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); + if (osName.contains(" ")) { + // Strip version + osName = osName.substring(0, osName.indexOf(' ')); + } + if (osName.contains("linux")) { + return "linux"; + } else if (osName.contains("mac")) { + return "macos"; + } else if (osName.contains("windows")) { + return "windows"; + } else { + throw new IllegalStateException("Unsupported OS: " + osName); + } + } } From 3cdad6ea5a33546513c05dd3a0cd5ef5b219e85e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 14:27:06 +0100 Subject: [PATCH 20/39] Extracted native lib dir conforms to the hierarchy naming convention --- project/StdBits.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/project/StdBits.scala b/project/StdBits.scala index 3b1152720254..b110c5f8ab95 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -124,6 +124,9 @@ object StdBits { ) .value val extractPrefix = "nu/pattern/opencv" + + // Make sure that the native libs in the `lib` directory complies with + // `org.enso.interpreter.runtime.NativeLibraryFinder` def renameFunc(entryName: String): Option[String] = { val strippedEntryName = entryName.substring(extractPrefix.length + 1) if ( @@ -135,11 +138,15 @@ object StdBits { } else { Some( strippedEntryName - .replace("osx", "macos") - .replace("x86_64", "amd64") + .replace("linux/x86_64", "amd64/linux") + .replace("windows/x86_64", "amd64/windows") + .replace("windows/x86_32", "x86_32/windows") + .replace("osx/ARMv8", "ARMv8/macos") + .replace("osx/x86_64", "amd64/macos") ) } } + val logger = streams.value.log val openCvJar = JPMSUtils .filterModulesFromUpdate( From 51cc81104f755b75cffabab0ba58ed46f155e567 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 17:08:15 +0100 Subject: [PATCH 21/39] Move NativeLibraryFinder to pkg and make it generic. --- .../interpreter/runtime/HostClassLoader.java | 4 +- .../runtime/util/TruffleFileSystem.java | 11 +++++- .../runtime/DefaultPackageRepository.scala | 2 +- .../org/enso/pkg}/NativeLibraryFinder.java | 39 +++++++------------ .../org/enso/filesystem/FileSystem.scala | 10 +++++ 5 files changed, 39 insertions(+), 27 deletions(-) rename {engine/runtime/src/main/java/org/enso/interpreter/runtime => lib/scala/pkg/src/main/java/org/enso/pkg}/NativeLibraryFinder.java (53%) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java index bfb1d7b87017..bac13e762752 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java @@ -4,6 +4,8 @@ import java.net.URLClassLoader; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import org.enso.interpreter.runtime.util.TruffleFileSystem; +import org.enso.pkg.NativeLibraryFinder; import org.graalvm.polyglot.Context; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,7 +99,7 @@ protected Class loadClass(String name, boolean resolve) throws ClassNotFoundE protected String findLibrary(String libname) { var pkgRepo = EnsoContext.get(null).getPackageRepository(); for (var pkg : pkgRepo.getLoadedPackagesJava()) { - var libPath = NativeLibraryFinder.findNativeLibrary(libname, pkg); + var libPath = NativeLibraryFinder.findNativeLibrary(libname, pkg, TruffleFileSystem.INSTANCE); if (libPath != null) { return libPath; } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java index 56cfb1e4550c..157af944132b 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/util/TruffleFileSystem.java @@ -14,7 +14,11 @@ import org.enso.filesystem.FileSystem; /** A {@link TruffleFile}-based implementation of {@link FileSystem}. */ -public class TruffleFileSystem implements FileSystem { +public final class TruffleFileSystem implements FileSystem { + private TruffleFileSystem() {} + + public static final TruffleFileSystem INSTANCE = new TruffleFileSystem(); + @Override public TruffleFile getChild(TruffleFile parent, String childName) { return parent.resolve(childName); @@ -45,6 +49,11 @@ public List getSegments(TruffleFile file) { return Arrays.asList(file.toRelativeUri().getPath().split("/")); } + @Override + public String getAbsolutePath(TruffleFile file) { + return file.getAbsoluteFile().getPath(); + } + @Override public String getName(TruffleFile file) { return file.getName(); diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index 3723cab9b1ae..41cd630104fb 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -58,7 +58,7 @@ private class DefaultPackageRepository( private val logger = Logger[DefaultPackageRepository] - implicit private val fs: TruffleFileSystem = new TruffleFileSystem + implicit private val fs: TruffleFileSystem = TruffleFileSystem.INSTANCE private val packageManager = new PackageManager[TruffleFile] private var projectPackage: Option[Package[TruffleFile]] = None diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java similarity index 53% rename from engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java rename to lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java index 3b7f8b261515..87a52c476b0f 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/NativeLibraryFinder.java +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -1,19 +1,14 @@ -package org.enso.interpreter.runtime; +package org.enso.pkg; -import com.oracle.truffle.api.TruffleFile; import java.util.Locale; -import org.enso.pkg.Package; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +import org.enso.filesystem.FileSystem; /** * Helper class to find native libraries in packages. The search algorithm complies to the NetBeans * JNI specification. */ -final class NativeLibraryFinder { - - private static final Logger logger = LoggerFactory.getLogger(NativeLibraryFinder.class); +public final class NativeLibraryFinder { private NativeLibraryFinder() {} @@ -24,29 +19,25 @@ private NativeLibraryFinder() {} * @param pkg the package to search in. * @return null if not found, absolute path otherwise. */ - static String findNativeLibrary(String libName, Package pkg) { + public static String findNativeLibrary(String libName, Package pkg, FileSystem fs) { var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); var osName = simpleOsName(); var libNameWithSuffix = System.mapLibraryName(libName); - var libDir = pkg.polyglotDir().resolve("lib"); - if (!libDir.exists()) { - logger.trace("Native library directory {} does not exist", libDir); + var libDir = fs.getChild(pkg.polyglotDir(), "lib"); + if (!fs.exists(libDir)) { return null; } - var nativeLib = libDir.resolve(libNameWithSuffix); - if (nativeLib.exists()) { - logger.trace("Found native library {}", nativeLib); - return nativeLib.getAbsoluteFile().getPath(); + var nativeLib = fs.getChild(libDir, libNameWithSuffix); + if (fs.exists(nativeLib)) { + return fs.getAbsolutePath(nativeLib); } - nativeLib = libDir.resolve(arch).resolve(libNameWithSuffix); - if (nativeLib.exists()) { - logger.trace("Found native library {}", nativeLib); - return nativeLib.getAbsoluteFile().getPath(); + nativeLib = fs.getChild(fs.getChild(libDir, arch), libNameWithSuffix); + if (fs.exists(nativeLib)) { + return fs.getAbsolutePath(nativeLib); } - nativeLib = libDir.resolve(arch).resolve(osName).resolve(libNameWithSuffix); - if (nativeLib.exists()) { - logger.trace("Found native library {}", nativeLib); - return nativeLib.getAbsoluteFile().getPath(); + nativeLib = fs.getChild(fs.getChild(fs.getChild(libDir, arch), osName), libName); + if (fs.exists(nativeLib)) { + return fs.getAbsolutePath(nativeLib); } return null; } diff --git a/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala b/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala index e6f99509253e..c751f48b20e7 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala @@ -64,6 +64,12 @@ trait FileSystem[F] { */ def getSegments(file: F): java.lang.Iterable[String] + /** Returns absolute path of the given file. + * @param file + * @return + */ + def getAbsolutePath(file: F): String + /** Gets the name of the given file. * * @param file @@ -231,5 +237,9 @@ object FileSystem { Files .readAttributes(file.toPath, classOf[BasicFileAttributes]) .creationTime() + + override def getAbsolutePath(file: File): String = { + file.getAbsolutePath + } } } From 361ab14e3b99d1f4103dbbc870896a792a4f6667 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 17:11:36 +0100 Subject: [PATCH 22/39] Package has nativeLibraryDir method --- .../pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java | 2 +- lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java index 87a52c476b0f..34d384f85fd3 100644 --- a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -23,7 +23,7 @@ public static String findNativeLibrary(String libName, Package pkg, FileS var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); var osName = simpleOsName(); var libNameWithSuffix = System.mapLibraryName(libName); - var libDir = fs.getChild(pkg.polyglotDir(), "lib"); + var libDir = pkg.nativeLibraryDir(); if (!fs.exists(libDir)) { return null; } diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala index 32279c2764b0..645c89c47277 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala @@ -40,6 +40,7 @@ class Package[F]( val configFile: F = root.getChild(Package.configFileName) val thumbFile: F = root.getChild(Package.thumbFileName) val polyglotDir: F = root.getChild(Package.polyglotExtensionsDirName) + val nativeLibraryDir: F = root.getChild(Package.nativeLibraryDirName) val internalDirectory: F = root.getChild(Package.internalDirName) val irCacheDirectory: F = internalDirectory .getChild(Package.cacheDirName) @@ -599,6 +600,7 @@ object Package { val configFileName = "package.yaml" val sourceDirName = "src" val polyglotExtensionsDirName = "polyglot" + val nativeLibraryDirName = "lib" val internalDirName = ".enso" val mainFileName = "Main.enso" val thumbFileName = "thumb.png" From 218c19fbe0d897c1c32f05ac3016e88d84d6b90c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 17:16:36 +0100 Subject: [PATCH 23/39] Fix typo in NativeLibraryFinder --- .../pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java index 34d384f85fd3..7a40996fd149 100644 --- a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -35,7 +35,7 @@ public static String findNativeLibrary(String libName, Package pkg, FileS if (fs.exists(nativeLib)) { return fs.getAbsolutePath(nativeLib); } - nativeLib = fs.getChild(fs.getChild(fs.getChild(libDir, arch), osName), libName); + nativeLib = fs.getChild(fs.getChild(fs.getChild(libDir, arch), osName), libNameWithSuffix); if (fs.exists(nativeLib)) { return fs.getAbsolutePath(nativeLib); } From f9b71bcc63d14ac03c97cf9d91a01a12534b837a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 17:16:49 +0100 Subject: [PATCH 24/39] Fix path to nativeLibDir in pkg --- lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala index 645c89c47277..a13c44bd81ba 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala @@ -40,7 +40,7 @@ class Package[F]( val configFile: F = root.getChild(Package.configFileName) val thumbFile: F = root.getChild(Package.thumbFileName) val polyglotDir: F = root.getChild(Package.polyglotExtensionsDirName) - val nativeLibraryDir: F = root.getChild(Package.nativeLibraryDirName) + val nativeLibraryDir: F = polyglotDir.getChild(Package.nativeLibraryDirName) val internalDirectory: F = root.getChild(Package.internalDirName) val irCacheDirectory: F = internalDirectory .getChild(Package.cacheDirName) From 9ed04b1d97721405373f249ce3789e7f40f9db75 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 17:16:58 +0100 Subject: [PATCH 25/39] Fix build --- .../java/org/enso/compiler/test/SerializationManagerTest.java | 2 +- .../src/main/java/org/enso/interpreter/runtime/EnsoContext.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java index bb41f3d6a129..24e0595f3653 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java @@ -34,7 +34,7 @@ public class SerializationManagerTest { @Before public void setup() { - packageManager = new PackageManager<>(new TruffleFileSystem()); + packageManager = new PackageManager<>(TruffleFileSystem.INSTANCE); interpreterContext = new InterpreterContext(x -> x); ensoContext = interpreterContext diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index c3f3bebd0f44..a70c94046676 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -170,7 +170,7 @@ public EnsoContext( /** Perform expensive initialization logic for the context. */ public void initialize() { - TruffleFileSystem fs = new TruffleFileSystem(); + TruffleFileSystem fs = TruffleFileSystem.INSTANCE; PackageManager packageManager = new PackageManager<>(fs); Optional projectRoot = OptionsHelper.getProjectRoot(environment); From 3d9a3771544275f01acc363a1c0dd86d6e2ef3fc Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 20 Dec 2024 18:25:05 +0100 Subject: [PATCH 26/39] Add NativeLibraryFinder.listAllNativeLibraries --- .../org/enso/pkg/NativeLibraryFinder.java | 63 ++++++++++++++----- 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java index 7a40996fd149..1aed64fc34bf 100644 --- a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -1,6 +1,11 @@ package org.enso.pkg; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Locale; +import java.util.Set; import org.enso.filesystem.FileSystem; /** @@ -20,28 +25,52 @@ private NativeLibraryFinder() {} * @return null if not found, absolute path otherwise. */ public static String findNativeLibrary(String libName, Package pkg, FileSystem fs) { - var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); - var osName = simpleOsName(); var libNameWithSuffix = System.mapLibraryName(libName); - var libDir = pkg.nativeLibraryDir(); - if (!fs.exists(libDir)) { - return null; - } - var nativeLib = fs.getChild(libDir, libNameWithSuffix); - if (fs.exists(nativeLib)) { - return fs.getAbsolutePath(nativeLib); - } - nativeLib = fs.getChild(fs.getChild(libDir, arch), libNameWithSuffix); - if (fs.exists(nativeLib)) { - return fs.getAbsolutePath(nativeLib); - } - nativeLib = fs.getChild(fs.getChild(fs.getChild(libDir, arch), osName), libNameWithSuffix); - if (fs.exists(nativeLib)) { - return fs.getAbsolutePath(nativeLib); + for (var dir : searchPath(pkg, fs)) { + if (!fs.exists(dir)) { + return null; + } + var nativeLib = fs.getChild(dir, libNameWithSuffix); + if (fs.exists(nativeLib)) { + return fs.getAbsolutePath(nativeLib); + } } return null; } + /** Returns set of native libraries for the given package for the current OS and architecture. */ + public static Set listAllNativeLibraries(Package pkg, FileSystem fs) { + var nativeLibs = new HashSet(); + for (var dir : searchPath(pkg, fs)) { + if (!fs.exists(dir)) { + return nativeLibs; + } + try { + fs.list(dir) + .forEach( + file -> { + if (fs.isRegularFile(file)) { + nativeLibs.add(file); + } + }); + } catch (IOException e) { + throw new IllegalStateException(e); + } + } + return nativeLibs; + } + + private static List searchPath(Package pkg, FileSystem fs) { + var searchPath = new ArrayList(); + var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); + var osName = simpleOsName(); + var libDir = pkg.nativeLibraryDir(); + searchPath.add(libDir); + searchPath.add(fs.getChild(libDir, arch)); + searchPath.add(fs.getChild(fs.getChild(libDir, arch), osName)); + return searchPath; + } + private static String simpleOsName() { var osName = System.getProperty("os.name").toLowerCase(Locale.ENGLISH); if (osName.contains(" ")) { From 7bc8768b98262594e41fa6b50bf51e249441c610 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 16:46:40 +0100 Subject: [PATCH 27/39] Gather native libraries in EnsoLibraryFeature --- .../org/enso/runner/EnsoLibraryFeature.java | 58 +++++++------------ .../org/enso/filesystem/FileSystem.scala | 2 + 2 files changed, 23 insertions(+), 37 deletions(-) diff --git a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java index 7b25fd8a11d5..79b43d25e16f 100644 --- a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java +++ b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java @@ -2,27 +2,25 @@ import static scala.jdk.javaapi.CollectionConverters.asJava; +import java.io.File; import java.nio.file.Files; import java.nio.file.Path; import java.util.LinkedHashSet; import java.util.TreeSet; import org.enso.compiler.core.EnsoParser; import org.enso.compiler.core.ir.module.scope.imports.Polyglot; +import org.enso.filesystem.FileSystem$; +import org.enso.pkg.NativeLibraryFinder; import org.enso.pkg.PackageManager$; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.hosted.RuntimeProxyCreation; import org.graalvm.nativeimage.hosted.RuntimeReflection; -import org.graalvm.nativeimage.hosted.RuntimeResourceAccess; +import org.graalvm.nativeimage.hosted.RuntimeSystemProperties; public final class EnsoLibraryFeature implements Feature { @Override public void beforeAnalysis(BeforeAnalysisAccess access) { - try { - registerOpenCV(access.getApplicationClassLoader()); - } catch (ReflectiveOperationException ex) { - ex.printStackTrace(); - throw new IllegalStateException(ex); - } + var libs = new LinkedHashSet(); for (var p : access.getApplicationClassPath()) { var p1 = p.getParent(); @@ -53,6 +51,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { */ var classes = new TreeSet(); + var nativeLibPaths = new TreeSet(); try { for (var p : libs) { var result = PackageManager$.MODULE$.Default().loadPackage(p.toFile()); @@ -90,6 +89,19 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { } } } + if (pkg.nativeLibraryDir().exists()) { + var nativeLibs = + NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs()); + for (var nativeLib : nativeLibs) { + assert nativeLib.exists() && nativeLib.isFile(); + var dir = nativeLib.toPath().getParent().toFile(); + assert dir.exists() && dir.isDirectory(); + nativeLibPaths.add(dir.getAbsolutePath()); + var current = System.getProperty("java.library.path"); + RuntimeSystemProperties.register( + "java.library.path", current + File.pathSeparator + dir.getAbsolutePath()); + } + } } } } catch (Exception ex) { @@ -101,35 +113,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { System.err.println(" " + className); } System.err.println("Registered " + classes.size() + " classes for reflection"); - } - - private static void registerOpenCV(ClassLoader cl) throws ReflectiveOperationException { - var moduleOpenCV = cl.getUnnamedModule(); - var currentOS = System.getProperty("os.name").toUpperCase().replaceAll(" .*$", ""); - - var libOpenCV = - switch (currentOS) { - case "LINUX" -> "nu/pattern/opencv/linux/x86_64/libopencv_java470.so"; - case "WINDOWS" -> "nu/pattern/opencv/windows/x86_64/opencv_java470.dll"; - case "MAC" -> { - var arch = System.getProperty("os.arch").toUpperCase(); - yield switch (arch) { - case "X86_64" -> "nu/pattern/opencv/osx/x86_64/libopencv_java470.dylib"; - case "AARCH64" -> "nu/pattern/opencv/osx/ARMv8/libopencv_java470.dylib"; - default -> null; - }; - } - default -> null; - }; - - if (libOpenCV != null) { - var verify = cl.getResource(libOpenCV); - if (verify == null) { - throw new IllegalStateException("Cannot find " + libOpenCV + " resource in " + cl); - } - RuntimeResourceAccess.addResource(moduleOpenCV, libOpenCV); - } else { - throw new IllegalStateException("No resource suggested for " + currentOS); - } + System.err.println( + "Registered native libraries into runtime's java.library.path: " + nativeLibPaths); } } diff --git a/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala b/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala index c751f48b20e7..b593f7977b25 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/filesystem/FileSystem.scala @@ -150,6 +150,8 @@ trait FileSystem[F] { object FileSystem { + val defaultFs = Default + /** Exposes [[FileSystem]] operations through method call syntax. * All methods have the same semantics as the corresponding [[FileSystem]] * methods. From 5a4432850c1aadb7e61991abe8b9a93a1b8bbb3b Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 17:36:17 +0100 Subject: [PATCH 28/39] Stack of exception in EnsoLibraryFeature is printed to System.err. The current printing is not visible on CI --- .../src/main/java/org/enso/runner/EnsoLibraryFeature.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java index 79b43d25e16f..66f1d960d073 100644 --- a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java +++ b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java @@ -105,7 +105,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { } } } catch (Exception ex) { - ex.printStackTrace(); + ex.printStackTrace(System.err); throw new IllegalStateException(ex); } System.err.println("Summary for polyglot import java:"); From 158110cc4ec3253f4fd84c3718d14818daad3e98 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 17:41:00 +0100 Subject: [PATCH 29/39] Update docs --- docs/polyglot/java.md | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index d901ee73b978..a37c099f7419 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -44,8 +44,8 @@ The dynamic polyglot system is a dynamic runtime lookup for Java objects, allowing Enso code to work with them through a runtime reflection-style mechanism. It is comprised of the following components: -- `Java.lookup_class : Text -> Any`: A function that lets users - look up a class by a given name on the runtime classpath. +- `Java.lookup_class : Text -> Any`: A function that lets users look up a class + by a given name on the runtime classpath. - A whole host of functions on the polyglot type that let you dynamically work with object bindings. @@ -66,19 +66,31 @@ main = > - Expand on the detail when there is time. ## Native libraries -Java can load native libraries using, e.g., the [System.loadLibrary](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/System.html#loadLibrary(java.lang.String)) -or [ClassLoader.findLibrary](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)) -methods. -If a Java method loaded from the `polyglot/java` directory in project `Proj` tries to load a -native library via one of the aforementioned mechanisms, the runtime system will look for the -native library in the `polyglot/lib` directory within the project `Proj`. -The algorithm used to search for the native libraries within the `polyglot/lib` directory hierarchy conforms to the [NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni). -The runtime system implements this by overriding the [ClassLoader.findLibrary](https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)) -method on the `ClassLoader` used to load the Java class. -Only the `polyglot/lib` directory within the project is searched for the native libraries. -This means that the native library search path is effectively _isolated_ for every project. +Java can load native libraries using, e.g., the +[System.loadLibrary]() +or +[ClassLoader.findLibrary]() +methods. If a Java method loaded from the `polyglot/java` directory in project +`Proj` tries to load a native library via one of the aforementioned mechanisms, +the runtime system will look for the native library in the `polyglot/lib` +directory within the project `Proj`. The runtime system implements this by +overriding the +[ClassLoader.findLibrary]() +method on the `ClassLoader` used to load the Java class. +The algorithm used to search for the native libraries within the `polyglot/lib` +directory hierarchy conforms to the +[NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni): +Lookup of library with name `native` works roughly in these steps: + +- Add platform-specific prefix and/or suffix to the library name, e.g., + `libnative.so` on Linux. +- Search for the library in the `polyglot/lib` directory. +- Search for the library in the `polyglot/lib/` directory, where `` + is the name of the architecture. +- Search for the library in the `polyglot/lib//` directory, where + `` is the name of the operating system. ## Download a Java Library from Maven Central From a9c60a5f8ef734e4f4ccfdf63cb2a1d989c5e14a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 17:44:30 +0100 Subject: [PATCH 30/39] Update docs --- docs/polyglot/java.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index a37c099f7419..74bdd5fdb757 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -100,7 +100,7 @@ ecosystem is to download it (including is **transitive dependencies**) from libraries. Let's **start from scratch** by creating an _empty Enso project_: ```bash -$ bin/ensoup --new polydemo +$ bin/enso --new polydemo $ cd polydemo polydemo$ find . . From 829c0a9128129ee7e881726300bae49aabe25bce Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 18:54:44 +0100 Subject: [PATCH 31/39] Add unit test for searching native lib --- .../test/NativeLibraryFinderTest.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java new file mode 100644 index 000000000000..0d3ab26ddb5f --- /dev/null +++ b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java @@ -0,0 +1,34 @@ +package org.enso.interpreter.test; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; + +import org.enso.editions.LibraryName; +import org.enso.interpreter.runtime.util.TruffleFileSystem; +import org.enso.pkg.NativeLibraryFinder; +import org.enso.test.utils.ContextUtils; +import org.junit.Test; + +public class NativeLibraryFinderTest { + @Test + public void standardImageShouldHaveNativeLib() { + try (var ctx = ContextUtils.createDefaultContext()) { + // Evaluate dummy sources to force loading Standard.Image + ContextUtils.evalModule( + ctx, """ + from Standard.Image import all + main = 42 + """); + var ensoCtx = ContextUtils.leakContext(ctx); + var stdImg = + ensoCtx + .getPackageRepository() + .getPackageForLibraryJava(LibraryName.apply("Standard", "Image")); + assertThat(stdImg.isPresent(), is(true)); + var nativeLibs = + NativeLibraryFinder.listAllNativeLibraries(stdImg.get(), TruffleFileSystem.INSTANCE); + assertThat( + "There should be just single native lib in Standard.Image", nativeLibs.size(), is(1)); + } + } +} From 5d7d74827117f8f5b133602dcb9cbae8d3dcbce7 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 19:13:20 +0100 Subject: [PATCH 32/39] Add printing system info to NativeLibraryFinderTest --- .../test/NativeLibraryFinderTest.java | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java index 0d3ab26ddb5f..aeee72203ad9 100644 --- a/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java +++ b/engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java @@ -3,13 +3,31 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.is; +import com.oracle.truffle.api.TruffleFile; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; import org.enso.editions.LibraryName; import org.enso.interpreter.runtime.util.TruffleFileSystem; import org.enso.pkg.NativeLibraryFinder; +import org.enso.pkg.Package; import org.enso.test.utils.ContextUtils; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; public class NativeLibraryFinderTest { + + @Rule public final TestRule printContextRule = new PrintSystemInfoRule(); + private Package stdImgPkg; + @Test public void standardImageShouldHaveNativeLib() { try (var ctx = ContextUtils.createDefaultContext()) { @@ -25,10 +43,66 @@ public void standardImageShouldHaveNativeLib() { .getPackageRepository() .getPackageForLibraryJava(LibraryName.apply("Standard", "Image")); assertThat(stdImg.isPresent(), is(true)); + this.stdImgPkg = stdImg.get(); var nativeLibs = NativeLibraryFinder.listAllNativeLibraries(stdImg.get(), TruffleFileSystem.INSTANCE); assertThat( "There should be just single native lib in Standard.Image", nativeLibs.size(), is(1)); } } + + public final class PrintSystemInfoRule implements TestRule { + @Override + public Statement apply(Statement base, Description description) { + return new Statement() { + @Override + public void evaluate() { + try { + base.evaluate(); + } catch (Throwable e) { + var sb = new StringBuilder(); + sb.append(System.lineSeparator()); + sb.append(" os.name: ") + .append(System.getProperty("os.name")) + .append(System.lineSeparator()); + sb.append(" os.arch: ") + .append(System.getProperty("os.arch")) + .append(System.lineSeparator()); + var mappedLibName = System.mapLibraryName("opencv_java470"); + sb.append(" Mapped library name: ") + .append(mappedLibName) + .append(System.lineSeparator()); + if (stdImgPkg != null) { + sb.append(" Contents of Standard.Image native library dir:") + .append(System.lineSeparator()); + var nativeLibDir = stdImgPkg.nativeLibraryDir(); + var nativeLibPath = Path.of(nativeLibDir.getAbsoluteFile().getPath()); + var contents = contentsOfDir(nativeLibPath); + contents.forEach( + path -> sb.append(" ").append(path).append(System.lineSeparator())); + } + throw new AssertionError(sb.toString(), e); + } + } + }; + } + } + + private static List contentsOfDir(Path dir) { + var contents = new ArrayList(); + var fileVisitor = + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + contents.add(file.toAbsolutePath().toString()); + return FileVisitResult.CONTINUE; + } + }; + try { + Files.walkFileTree(dir, fileVisitor); + } catch (IOException e) { + throw new AssertionError(e); + } + return contents; + } } From a1521cd23edbb62fecb332d53931ddc93c72d169 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 23 Dec 2024 20:33:21 +0100 Subject: [PATCH 33/39] Always replace x86_64 by amd64 --- .../src/main/java/org/enso/pkg/NativeLibraryFinder.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java index 1aed64fc34bf..0e0772061dbe 100644 --- a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -62,7 +62,7 @@ public static Set listAllNativeLibraries(Package pkg, FileSystem fs private static List searchPath(Package pkg, FileSystem fs) { var searchPath = new ArrayList(); - var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); + var arch = arch(); var osName = simpleOsName(); var libDir = pkg.nativeLibraryDir(); searchPath.add(libDir); @@ -87,4 +87,9 @@ private static String simpleOsName() { throw new IllegalStateException("Unsupported OS: " + osName); } } + + private static String arch() { + var arch = System.getProperty("os.arch").toLowerCase(Locale.ENGLISH); + return arch.replace("x86_64", "amd64"); + } } From f4bec69ff2694e26936bb0b3a58659c036a72be9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 24 Dec 2024 13:55:13 +0100 Subject: [PATCH 34/39] Add list of supported names in docs --- docs/polyglot/java.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index 74bdd5fdb757..869d271d74d9 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -92,6 +92,11 @@ Lookup of library with name `native` works roughly in these steps: - Search for the library in the `polyglot/lib//` directory, where `` is the name of the operating system. +Supported names: +- Names for `` are `linux`, `macos`, `windows`. + - Note that for simplicity we omit the versions of the operating systems. +- Names for architectures `` are `amd64`, `x86_64`, `x86_32`. + ## Download a Java Library from Maven Central A typical use-case when bringing in some popular Java library into Enso From ceb3326d197db8b80a48d45373425217b691b7b9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 24 Dec 2024 13:59:01 +0100 Subject: [PATCH 35/39] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d4db090dd76..d555ba0177fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,10 +17,12 @@ - A constructor or type definition with a single inline argument definition was previously allowed to use spaces in the argument definition without parentheses. [This is now a syntax error.][11856] +- [Native libraries of projects can be added to `polyglot/lib` directory][11874] [11777]: https://github.com/enso-org/enso/pull/11777 [11600]: https://github.com/enso-org/enso/pull/11600 [11856]: https://github.com/enso-org/enso/pull/11856 +[11874]: https://github.com/enso-org/enso/pull/11874 # Next Release From b4d3f98d8f6e0f190ec1d73861b4699ccf4c21bc Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 24 Dec 2024 13:59:07 +0100 Subject: [PATCH 36/39] fmt docs --- docs/polyglot/java.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index 869d271d74d9..50a1c12b1463 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -93,6 +93,7 @@ Lookup of library with name `native` works roughly in these steps: `` is the name of the operating system. Supported names: + - Names for `` are `linux`, `macos`, `windows`. - Note that for simplicity we omit the versions of the operating systems. - Names for architectures `` are `amd64`, `x86_64`, `x86_32`. From e6956db258c63572c989c803ece33254aed7166a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 31 Dec 2024 18:30:23 +0100 Subject: [PATCH 37/39] EnsoLibraryFeature copies native libs next to the generated binary --- build.sbt | 3 +- .../org/enso/runner/EnsoLibraryFeature.java | 31 +++++++++++++------ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/build.sbt b/build.sbt index 43d8fedb9804..e13b2b32d9f0 100644 --- a/build.sbt +++ b/build.sbt @@ -3712,7 +3712,8 @@ lazy val `engine-runner` = project // "-H:-DeleteLocalSymbols", // you may need to set smallJdk := None to use following flags: // "--trace-class-initialization=org.enso.syntax2.Parser", - "-Dnic=nic" + "-Dnic=nic", + "-Dorg.enso.feature.native.lib.output=" + (engineDistributionRoot.value / "bin") ), mainClass = Some("org.enso.runner.Main"), initializeAtRuntime = Seq( diff --git a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java index 66f1d960d073..f9fa7eb67875 100644 --- a/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java +++ b/engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java @@ -15,9 +15,25 @@ import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.nativeimage.hosted.RuntimeProxyCreation; import org.graalvm.nativeimage.hosted.RuntimeReflection; -import org.graalvm.nativeimage.hosted.RuntimeSystemProperties; public final class EnsoLibraryFeature implements Feature { + private static final String LIB_OUTPUT = "org.enso.feature.native.lib.output"; + private final File nativeLibDir; + + public EnsoLibraryFeature() { + var nativeLibOut = System.getProperty(LIB_OUTPUT); + if (nativeLibOut == null) { + throw new IllegalStateException("Missing system property: " + LIB_OUTPUT); + } + nativeLibDir = new File(nativeLibOut); + if (!nativeLibDir.exists() || !nativeLibDir.isDirectory()) { + var created = nativeLibDir.mkdirs(); + if (!created) { + throw new IllegalStateException("Cannot create directory: " + nativeLibDir); + } + } + } + @Override public void beforeAnalysis(BeforeAnalysisAccess access) { @@ -93,13 +109,9 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { var nativeLibs = NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs()); for (var nativeLib : nativeLibs) { - assert nativeLib.exists() && nativeLib.isFile(); - var dir = nativeLib.toPath().getParent().toFile(); - assert dir.exists() && dir.isDirectory(); - nativeLibPaths.add(dir.getAbsolutePath()); - var current = System.getProperty("java.library.path"); - RuntimeSystemProperties.register( - "java.library.path", current + File.pathSeparator + dir.getAbsolutePath()); + var out = new File(nativeLibDir, nativeLib.getName()); + Files.copy(nativeLib.toPath(), out.toPath()); + nativeLibPaths.add(out.getAbsolutePath()); } } } @@ -113,7 +125,6 @@ public void beforeAnalysis(BeforeAnalysisAccess access) { System.err.println(" " + className); } System.err.println("Registered " + classes.size() + " classes for reflection"); - System.err.println( - "Registered native libraries into runtime's java.library.path: " + nativeLibPaths); + System.err.println("Copied native libraries: " + nativeLibPaths + " into " + nativeLibDir); } } From 7d89629ec91a1da265a79a6cc394498cc1fe1159 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 1 Jan 2025 12:56:49 +0100 Subject: [PATCH 38/39] Flatten hierarchy of native libs in opencv --- docs/polyglot/java.md | 2 +- project/StdBits.scala | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/polyglot/java.md b/docs/polyglot/java.md index 50a1c12b1463..b698aff493cd 100644 --- a/docs/polyglot/java.md +++ b/docs/polyglot/java.md @@ -96,7 +96,7 @@ Supported names: - Names for `` are `linux`, `macos`, `windows`. - Note that for simplicity we omit the versions of the operating systems. -- Names for architectures `` are `amd64`, `x86_64`, `x86_32`. +- Names for architectures `` are `amd64`, `x86_64`, `x86_32`, `aarch64`. ## Download a Java Library from Maven Central diff --git a/project/StdBits.scala b/project/StdBits.scala index b110c5f8ab95..e52de6c98e88 100644 --- a/project/StdBits.scala +++ b/project/StdBits.scala @@ -138,11 +138,11 @@ object StdBits { } else { Some( strippedEntryName - .replace("linux/x86_64", "amd64/linux") - .replace("windows/x86_64", "amd64/windows") - .replace("windows/x86_32", "x86_32/windows") - .replace("osx/ARMv8", "ARMv8/macos") - .replace("osx/x86_64", "amd64/macos") + .replace("linux/x86_64", "amd64") + .replace("windows/x86_64", "amd64") + .replace("windows/x86_32", "x86_32") + .replace("osx/ARMv8", "aarch64") + .replace("osx/x86_64", "amd64") ) } } From 3ca596c6af48bed107871f8521235da3c9472000 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 3 Jan 2025 18:50:25 +0100 Subject: [PATCH 39/39] native libs must have the correct platform-specific suffix --- .../src/main/java/org/enso/pkg/NativeLibraryFinder.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java index 0e0772061dbe..851dcd158b81 100644 --- a/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java +++ b/lib/scala/pkg/src/main/java/org/enso/pkg/NativeLibraryFinder.java @@ -49,7 +49,8 @@ public static Set listAllNativeLibraries(Package pkg, FileSystem fs fs.list(dir) .forEach( file -> { - if (fs.isRegularFile(file)) { + var fname = fs.getName(file); + if (fs.isRegularFile(file) && fname.endsWith(nativeLibSuffix())) { nativeLibs.add(file); } }); @@ -60,6 +61,11 @@ public static Set listAllNativeLibraries(Package pkg, FileSystem fs return nativeLibs; } + private static String nativeLibSuffix() { + var libName = System.mapLibraryName(""); + return libName.substring(libName.lastIndexOf('.')); + } + private static List searchPath(Package pkg, FileSystem fs) { var searchPath = new ArrayList(); var arch = arch();