From 9c149b9b93948d3fc75284b34c9dc6cb6d5271a6 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 29 Jul 2024 15:57:29 +0200 Subject: [PATCH 01/89] Must specify --repl to enable debug server --- .../src/main/java/org/enso/runner/Main.java | 74 ++++++++++--------- .../java/org/enso/runner/EngineMainTest.java | 46 ++++++++++++ .../Base_Tests/src/Semantic/Runtime_Spec.enso | 4 + 3 files changed, 88 insertions(+), 36 deletions(-) create mode 100644 engine/runner/src/test/java/org/enso/runner/EngineMainTest.java diff --git a/engine/runner/src/main/java/org/enso/runner/Main.java b/engine/runner/src/main/java/org/enso/runner/Main.java index 98c882bfd8d1..7e9bc456c55f 100644 --- a/engine/runner/src/main/java/org/enso/runner/Main.java +++ b/engine/runner/src/main/java/org/enso/runner/Main.java @@ -56,7 +56,7 @@ import scala.runtime.BoxedUnit; /** The main CLI entry point class. */ -public final class Main { +public class Main { private static final String JVM_OPTION = "jvm"; private static final String RUN_OPTION = "run"; private static final String INSPECT_OPTION = "inspect"; @@ -98,6 +98,8 @@ public final class Main { private static final org.slf4j.Logger logger = LoggerFactory.getLogger(Main.class); + Main() {} + private static boolean isDevBuild() { return Info.ensoVersion().matches(".+-SNAPSHOT$"); } @@ -106,11 +108,8 @@ private static Option.Builder cliOptionBuilder() { return Option.builder(); } - /** - * Builds the [[Options]] object representing the CLI syntax. - * - * @return an [[Options]] object representing the CLI syntax - */ + private static final Options CLI_OPTIONS = buildOptions(); + private static Options buildOptions() { var help = cliOptionBuilder().option("h").longOpt(HELP_OPTION).desc("Displays this message.").build(); @@ -512,22 +511,22 @@ private static Options buildOptions() { * * @param options object representing the CLI syntax */ - private static void printHelp(Options options) { - new HelpFormatter().printHelp(LanguageInfo.ID, options); + private static void printHelp() { + new HelpFormatter().printHelp(LanguageInfo.ID, CLI_OPTIONS); } /** Terminates the process with a failure exit code. */ - private static RuntimeException exitFail() { + private RuntimeException exitFail() { return doExit(1); } /** Terminates the process with a success exit code. */ - private static RuntimeException exitSuccess() { + private final RuntimeException exitSuccess() { return doExit(0); } /** Shuts down the logging service and terminates the process. */ - private static RuntimeException doExit(int exitCode) { + RuntimeException doExit(int exitCode) { RunnerLogging.tearDown(); System.exit(exitCode); return null; @@ -665,7 +664,7 @@ private void compile( * @param executionEnvironment name of the execution environment to use during execution or {@code * null} */ - private void run( + private void handleRun( String path, java.util.List additionalArgs, String projectPath, @@ -675,6 +674,7 @@ private void run( boolean disablePrivateCheck, boolean enableAutoParallelism, boolean enableStaticAnalysis, + boolean enableDebugServer, boolean inspect, boolean dump, String executionEnvironment, @@ -708,9 +708,13 @@ private void run( .options(options); if (inspect) { + if (enableDebugServer) { + println("Cannot use --inspect and --repl and --run at once"); + throw exitFail(); + } options.put("inspect", ""); - } else { - // by default running with debug server enabled + } + if (enableDebugServer) { factory.messageTransport(replTransport()); options.put(DebugServerInfo.ENABLE_OPTION, "true"); } @@ -960,7 +964,7 @@ private void displayVersion(boolean useJson) { } /** Parses the log level option. */ - private static Level parseLogLevel(String levelOption) { + private Level parseLogLevel(String levelOption) { var name = levelOption.toLowerCase(); var found = Stream.of(Level.values()).filter(x -> name.equals(x.name().toLowerCase())).findFirst(); @@ -977,7 +981,7 @@ private static Level parseLogLevel(String levelOption) { } /** Parses an URI that specifies the logging service connection. */ - private static URI parseUri(String string) { + private URI parseUri(String string) { try { return new URI(string); } catch (URISyntaxException ex) { @@ -1001,15 +1005,13 @@ public static void main(String[] args) throws Exception { /** * Main entry point for the CLI program. * - * @param options the command line options * @param line the provided command line arguments * @param logLevel the provided log level * @param logMasking the flag indicating if the log masking is enabled */ - private void runMain(Options options, CommandLine line, Level logLevel, boolean logMasking) - throws IOException { + final void mainEntry(CommandLine line, Level logLevel, boolean logMasking) throws IOException { if (line.hasOption(HELP_OPTION)) { - printHelp(options); + printHelp(); throw exitSuccess(); } if (line.hasOption(VERSION_OPTION)) { @@ -1089,7 +1091,7 @@ private void runMain(Options options, CommandLine line, Level logLevel, boolean } if (line.hasOption(RUN_OPTION)) { - run( + handleRun( line.getOptionValue(RUN_OPTION), Arrays.asList(line.getArgs()), line.getOptionValue(IN_PROJECT_OPTION), @@ -1099,6 +1101,7 @@ private void runMain(Options options, CommandLine line, Level logLevel, boolean line.hasOption(DISABLE_PRIVATE_CHECK_OPTION), line.hasOption(AUTO_PARALLELISM_OPTION), line.hasOption(ENABLE_STATIC_ANALYSIS_OPTION), + line.hasOption(REPL_OPTION), line.hasOption(INSPECT_OPTION), line.hasOption(DUMP_GRAPHS_OPTION), line.getOptionValue(EXECUTION_ENVIRONMENT_OPTION), @@ -1106,7 +1109,7 @@ private void runMain(Options options, CommandLine line, Level logLevel, boolean .map(Integer::parseInt) .getOrElse(() -> 100)); } - if (line.hasOption(REPL_OPTION)) { + if (line.hasOption(REPL_OPTION) && !line.hasOption(RUN_OPTION)) { runRepl( line.getOptionValue(IN_PROJECT_OPTION), logLevel, @@ -1122,7 +1125,7 @@ private void runMain(Options options, CommandLine line, Level logLevel, boolean preinstallDependencies(line.getOptionValue(IN_PROJECT_OPTION), logLevel); } if (line.getOptions().length == 0) { - printHelp(options); + printHelp(); throw exitFail(); } } @@ -1240,16 +1243,15 @@ private static final scala.collection.immutable.List join( return scala.collection.immutable.$colon$colon$.MODULE$.apply(head, tail); } - private void println(String msg) { + void println(String msg) { System.out.println(msg); } private void launch(String[] args) throws IOException, InterruptedException, URISyntaxException { - var options = buildOptions(); - var line = preprocessArguments(options, args); + var line = preprocessArguments(args); var logMasking = new boolean[1]; - var logLevel = setupLogging(options, line, logMasking); + var logLevel = setupLogging(line, logMasking); if (line.hasOption(JVM_OPTION)) { var jvm = line.getOptionValue(JVM_OPTION); @@ -1332,36 +1334,36 @@ private void launch(String[] args) throws IOException, InterruptedException, URI } } - launch(options, line, logLevel, logMasking[0]); + launch(line, logLevel, logMasking[0]); } - protected CommandLine preprocessArguments(Options options, String[] args) { + final CommandLine preprocessArguments(String... args) { var parser = new DefaultParser(); try { var startParsing = System.currentTimeMillis(); - var line = parser.parse(options, args); + var line = parser.parse(CLI_OPTIONS, args); logger.trace( "Parsing Language Server arguments took {0}ms", System.currentTimeMillis() - startParsing); return line; } catch (Exception e) { - printHelp(options); + printHelp(); throw exitFail(); } } - private static Level setupLogging(Options options, CommandLine line, boolean[] logMasking) { + private Level setupLogging(CommandLine line, boolean[] logMasking) { var logLevel = scala.Option.apply(line.getOptionValue(LOG_LEVEL)) - .map(Main::parseLogLevel) + .map(this::parseLogLevel) .getOrElse(() -> defaultLogLevel); - var connectionUri = scala.Option.apply(line.getOptionValue(LOGGER_CONNECT)).map(Main::parseUri); + var connectionUri = scala.Option.apply(line.getOptionValue(LOGGER_CONNECT)).map(this::parseUri); logMasking[0] = !line.hasOption(NO_LOG_MASKING); RunnerLogging.setup(connectionUri, logLevel, logMasking[0]); return logLevel; } - private void launch(Options options, CommandLine line, Level logLevel, boolean logMasking) { + private void launch(CommandLine line, Level logLevel, boolean logMasking) { if (line.hasOption(LANGUAGE_SERVER_OPTION)) { try { var conf = parseProfilingConfig(line); @@ -1379,7 +1381,7 @@ private void launch(Options options, CommandLine line, Level logLevel, boolean l conf, ExecutionContext.global(), () -> { - runMain(options, line, logLevel, logMasking); + mainEntry(line, logLevel, logMasking); return BoxedUnit.UNIT; }); } catch (IOException ex) { diff --git a/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java b/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java new file mode 100644 index 000000000000..5b496f38ef32 --- /dev/null +++ b/engine/runner/src/test/java/org/enso/runner/EngineMainTest.java @@ -0,0 +1,46 @@ +package org.enso.runner; + +import static org.junit.Assert.assertEquals; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import org.junit.Test; +import org.slf4j.event.Level; + +public class EngineMainTest { + private final List linesOut = new ArrayList<>(); + + @Test + public void cannotUseReplAndInspectAtOnce() throws Exception { + try { + var m = + new Main() { + @Override + RuntimeException doExit(int code) { + throw new ExitCode(code); + } + + void println(String line) { + linesOut.add(line); + } + }; + var file = File.createTempFile("some", ".enso"); + file.deleteOnExit(); + var line = m.preprocessArguments("--repl", "--inspect", "--run", file.getAbsolutePath()); + m.mainEntry(line, Level.INFO, false); + } catch (ExitCode ex) { + assertEquals("Execution fails", 1, ex.exitCode); + assertEquals("One line printed", 1, linesOut.size()); + assertEquals("Cannot use --inspect and --repl and --run at once", linesOut.get(0)); + } + } + + private static final class ExitCode extends RuntimeException { + final int exitCode; + + ExitCode(int exitCode) { + this.exitCode = exitCode; + } + } +} diff --git a/test/Base_Tests/src/Semantic/Runtime_Spec.enso b/test/Base_Tests/src/Semantic/Runtime_Spec.enso index 5c4ec3955915..3b0c4977e118 100644 --- a/test/Base_Tests/src/Semantic/Runtime_Spec.enso +++ b/test/Base_Tests/src/Semantic/Runtime_Spec.enso @@ -1,4 +1,5 @@ import Standard.Base.Runtime +import Standard.Base.Runtime.Debug import Standard.Base.Data.Numbers.Integer import Standard.Base.Any.Any import Standard.Base.Nothing.Nothing @@ -38,6 +39,9 @@ add_specs suite_builder = r2 = Panic.catch Any (Runtime.with_disabled_context Input environment=Runtime.current_execution_environment <| in_fn (out_fn 10)) p-> p.payload.to_text r2 . should_equal "(Forbidden_Operation.Error 'The Input context is disabled.')" + suite_builder.group "Debug" group_builder-> + group_builder.specify "Debug.breakpoint doesn't stop indefinitly" <| + Debug.breakpoint main filter=Nothing = suite = Test.build suite_builder-> From 76070ba54bfc57ce84a60b6215c1138df41fa8b9 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 29 Jul 2024 16:41:06 +0200 Subject: [PATCH 02/89] Use --repl to enable REPL changelog note --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a0ce5c320d0..af58be175a5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,12 @@ - [Enforce conversion method return type][10468] - [Renaming launcher executable to ensoup][10535] - [Space-precedence does not apply to value-level operators][10597] +- [Must specify `--repl` to enable debug server][10709] [10468]: https://github.com/enso-org/enso/pull/10468 [10535]: https://github.com/enso-org/enso/pull/10535 [10597]: https://github.com/enso-org/enso/pull/10597 +[10709]: https://github.com/enso-org/enso/pull/10709 #### Enso IDE From 873c41834f1c9c975afaae0b4ca66bf3bc45a1c9 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 30 Jul 2024 08:06:57 +0200 Subject: [PATCH 03/89] Towards a ClassPath per Package --- .../interop/java/AddToClassPathNode.java | 3 +- .../builtin/interop/java/LookupClassNode.java | 2 +- .../enso/interpreter/runtime/EnsoContext.java | 52 ++++++++++++------- .../runtime/TruffleCompilerContext.java | 2 +- .../runtime/DefaultPackageRepository.scala | 2 +- .../interpreter/runtime/IrToTruffle.scala | 16 ++++-- 6 files changed, 50 insertions(+), 27 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java index 4d757a703502..3307a0ef6335 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java @@ -5,6 +5,7 @@ import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.Node; import java.io.File; +import java.util.Collections; import org.enso.interpreter.dsl.BuiltinMethod; import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; import org.enso.interpreter.runtime.EnsoContext; @@ -27,7 +28,7 @@ static AddToClassPathNode build() { Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) { var ctx = EnsoContext.get(this); var file = ctx.getTruffleFile(new File(expectStringNode.execute(path))); - ctx.addToClassPath(file); + ctx.createClassPath(Collections.singletonList(file)); return ctx.getBuiltins().nothing(); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java index 60e84837fbb4..9012a7c95645 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java @@ -21,7 +21,7 @@ static LookupClassNode build() { @Specialization @CompilerDirectives.TruffleBoundary Object doExecute(Object name, @Cached("build()") ExpectStringNode expectStringNode) { - return EnsoContext.get(this).lookupJavaClass(expectStringNode.execute(name)); + return EnsoContext.get(this).lookupJavaClass(null, expectStringNode.execute(name)); } abstract Object execute(Object name); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 5c3b13b15423..162d14389aed 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -424,28 +425,42 @@ public Optional findModuleByExpressionId(UUID expressionId) { } /** - * Modifies the classpath to use to lookup {@code polyglot java} imports. + * Creates a class path. A class path is capable to load Java classes in an isolated manner. * - * @param file the file to register + * @param files files to register */ @TruffleBoundary - public void addToClassPath(TruffleFile file) { - if (findGuestJava() == null) { - try { - var url = file.toUri().toURL(); - hostClassLoader.add(url); - } catch (MalformedURLException ex) { - throw new IllegalStateException(ex); + public EnsoClassPath createClassPath(List files) { + return new EnsoClassPath(files); + } + + public final class EnsoClassPath { + // private final TruffleFile[] files; + + private EnsoClassPath(List files) { + for (var f : files) { + add(f); } - } else { - try { - var path = new File(file.toUri()).getAbsoluteFile(); - if (!path.exists()) { - throw new IllegalStateException("File not found " + path); + } + + private void add(TruffleFile file) { + if (findGuestJava() == null) { + try { + var url = file.toUri().toURL(); + hostClassLoader.add(url); + } catch (MalformedURLException ex) { + throw new IllegalStateException(ex); + } + } else { + try { + var path = new File(file.toUri()).getAbsoluteFile(); + if (!path.exists()) { + throw new IllegalStateException("File not found " + path); + } + InteropLibrary.getUncached().invokeMember(findGuestJava(), "addPath", path.getPath()); + } catch (InteropException ex) { + throw new IllegalStateException(ex); } - InteropLibrary.getUncached().invokeMember(findGuestJava(), "addPath", path.getPath()); - } catch (InteropException ex) { - throw new IllegalStateException(ex); } } } @@ -518,11 +533,12 @@ public boolean isColorTerminalOutput() { * resolves to an inner class, then the import of the outer class is resolved, and the inner class * is looked up by iterating the members of the outer class via Truffle's interop protocol. * + * @param pkg Enso package to load the class for * @param className Fully qualified class name, can also be nested static inner class. * @return If the java class is found, return it, otherwise return null. */ @TruffleBoundary - public Object lookupJavaClass(String className) { + public Object lookupJavaClass(Package pkg, String className) { var binaryName = new StringBuilder(className); var collectedExceptions = new ArrayList(); for (; ; ) { diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java index cfed0237247f..8428cd847059 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java @@ -147,7 +147,7 @@ public void truffleRunCodegen( var s = org.enso.interpreter.runtime.scope.ModuleScope.Builder.fromCompilerModuleScopeBuilder( scopeBuilder); - new IrToTruffle(context, m.getSource(), s, config).run(module.getIr()); + new IrToTruffle(context, m.getSource(), s, config).run(module.getPackage(), module.getIr()); } // module related diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index 5c7773944f54..e4816434ed87 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -197,7 +197,7 @@ private class DefaultPackageRepository( isLibrary: Boolean ): Unit = { val extensions = pkg.listPolyglotExtensions("java") - extensions.foreach(context.addToClassPath) + context.createClassPath(extensions.asJava) val (regularModules, syntheticModulesMetadata) = pkg .listSources() diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala index 5ce3330212ea..bbfed346c7f2 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala @@ -153,7 +153,7 @@ class IrToTruffle( * * @param ir the IR to generate code for */ - def run(ir: Module): Unit = processModule(ir) + def run(p: org.enso.pkg.Package[_], ir: Module): Unit = processModule(p, ir) /** Executes the codegen pass on an inline input. * @@ -186,7 +186,10 @@ class IrToTruffle( * * @param module the module for which code should be generated */ - private def processModule(module: Module): Unit = { + private def processModule( + p: org.enso.pkg.Package[_], + module: Module + ): Unit = { generateReExportBindings(module) val bindingsMap = module @@ -197,7 +200,7 @@ class IrToTruffle( registerModuleExports(bindingsMap) registerModuleImports(bindingsMap) - registerPolyglotImports(module) + registerPolyglotImports(p, module) registerTypeDefinitions(module) registerMethodDefinitions(module) @@ -232,10 +235,13 @@ class IrToTruffle( } } - private def registerPolyglotImports(module: Module): Unit = + private def registerPolyglotImports( + p: org.enso.pkg.Package[_], + module: Module + ): Unit = module.imports.foreach { case poly @ imports.Polyglot(i: imports.Polyglot.Java, _, _, _, _) => - var hostSymbol = context.lookupJavaClass(i.getJavaName) + var hostSymbol = context.lookupJavaClass(p, i.getJavaName) if (hostSymbol == null) { val err = Text.create( s"Incorrect polyglot java import: ${i.getJavaName}" From 3b1bc98e37e0a81a9bbe1b976e6eead3b24b5bc9 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 30 Jul 2024 09:58:23 +0200 Subject: [PATCH 04/89] One ClassLoader per each Package --- .../interop/java/AddToClassPathNode.java | 4 +-- .../enso/interpreter/runtime/EnsoContext.java | 29 ++++++++++--------- .../runtime/DefaultPackageRepository.scala | 3 +- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java index 3307a0ef6335..e301da72ea7d 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java @@ -5,7 +5,6 @@ import com.oracle.truffle.api.dsl.Specialization; import com.oracle.truffle.api.nodes.Node; import java.io.File; -import java.util.Collections; import org.enso.interpreter.dsl.BuiltinMethod; import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; import org.enso.interpreter.runtime.EnsoContext; @@ -28,7 +27,8 @@ static AddToClassPathNode build() { Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) { var ctx = EnsoContext.get(this); var file = ctx.getTruffleFile(new File(expectStringNode.execute(path))); - ctx.createClassPath(Collections.singletonList(file)); + var cp = ctx.findClassPath(null); + cp.addToClassPath(file); return ctx.getBuiltins().nothing(); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 162d14389aed..2b15205cc211 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -27,8 +27,9 @@ import java.net.MalformedURLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; -import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.UUID; @@ -75,7 +76,7 @@ public final class EnsoContext { private final EnsoLanguage language; private final Env environment; - private final HostClassLoader hostClassLoader = new HostClassLoader(); + private final Map, EnsoClassPath> classPaths = new HashMap<>(); private final boolean assertionsEnabled; private final boolean isPrivateCheckDisabled; private final boolean isStaticTypeAnalysisEnabled; @@ -430,20 +431,21 @@ public Optional findModuleByExpressionId(UUID expressionId) { * @param files files to register */ @TruffleBoundary - public EnsoClassPath createClassPath(List files) { - return new EnsoClassPath(files); + public synchronized EnsoClassPath findClassPath(Package pkg) { + var l = classPaths.get(pkg); + if (l == null) { + l = new EnsoClassPath(); + classPaths.put(pkg, l); + } + return l; } public final class EnsoClassPath { - // private final TruffleFile[] files; + private final HostClassLoader hostClassLoader = new HostClassLoader(); - private EnsoClassPath(List files) { - for (var f : files) { - add(f); - } - } + private EnsoClassPath() {} - private void add(TruffleFile file) { + public void addToClassPath(TruffleFile file) { if (findGuestJava() == null) { try { var url = file.toUri().toURL(); @@ -541,10 +543,11 @@ public boolean isColorTerminalOutput() { public Object lookupJavaClass(Package pkg, String className) { var binaryName = new StringBuilder(className); var collectedExceptions = new ArrayList(); + var cp = findClassPath(pkg); for (; ; ) { var fqn = binaryName.toString(); try { - var hostSymbol = lookupHostSymbol(fqn); + var hostSymbol = lookupHostSymbol(cp.hostClassLoader, fqn); if (hostSymbol != null) { return hostSymbol; } @@ -566,7 +569,7 @@ public Object lookupJavaClass(Package pkg, String className) { return null; } - private Object lookupHostSymbol(String fqn) + private Object lookupHostSymbol(ClassLoader hostClassLoader, String fqn) throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException { try { if (findGuestJava() == null) { diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index e4816434ed87..1402690d47a2 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -197,7 +197,8 @@ private class DefaultPackageRepository( isLibrary: Boolean ): Unit = { val extensions = pkg.listPolyglotExtensions("java") - context.createClassPath(extensions.asJava) + val cp = context.findClassPath(pkg) + extensions.map(cp.addToClassPath) val (regularModules, syntheticModulesMetadata) = pkg .listSources() From 45f62cf98f7867be30a5c2e1dc745851c28f9561 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 30 Jul 2024 11:32:01 +0200 Subject: [PATCH 05/89] Define ModuleLayer for each Package --- .../enso/interpreter/runtime/EnsoContext.java | 34 +++++++++++++++---- .../runtime/TruffleCompilerContext.java | 3 +- .../interpreter/runtime/IrToTruffle.scala | 8 +++-- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 2b15205cc211..dd06f2a4b7f4 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -24,9 +24,13 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; +import java.lang.module.Configuration; +import java.lang.module.ModuleFinder; import java.net.MalformedURLException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -428,22 +432,38 @@ public Optional findModuleByExpressionId(UUID expressionId) { /** * Creates a class path. A class path is capable to load Java classes in an isolated manner. * - * @param files files to register + * @param pkg package to find class path for */ @TruffleBoundary - public synchronized EnsoClassPath findClassPath(Package pkg) { + public synchronized EnsoClassPath findClassPath(Package pkg) { var l = classPaths.get(pkg); if (l == null) { - l = new EnsoClassPath(); + var ch = pkg.polyglotDir().resolve("java"); + l = new EnsoClassPath(ch); classPaths.put(pkg, l); } return l; } public final class EnsoClassPath { - private final HostClassLoader hostClassLoader = new HostClassLoader(); - - private EnsoClassPath() {} + private final ModuleLayer.Controller cntrl; + private final ModuleLayer layer; + private final HostClassLoader hostClassLoader; + + private EnsoClassPath(TruffleFile file) { + var parent = ModuleLayer.boot(); + var parentCfgs = Collections.singletonList(parent.configuration()); + var parentModules = Collections.singletonList(parent); + var parentLoader = parent.findLoader("java.base"); + var finder = ModuleFinder.of(Paths.get(file.toUri())); + var cfg = Configuration.resolve(finder, parentCfgs, finder, Collections.emptyList()); + + this.hostClassLoader = new HostClassLoader(); + + this.cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); + this.layer = cntrl.layer(); + System.out.println("modules: " + layer.modules()); + } public void addToClassPath(TruffleFile file) { if (findGuestJava() == null) { @@ -540,7 +560,7 @@ public boolean isColorTerminalOutput() { * @return If the java class is found, return it, otherwise return null. */ @TruffleBoundary - public Object lookupJavaClass(Package pkg, String className) { + public Object lookupJavaClass(Package pkg, String className) { var binaryName = new StringBuilder(className); var collectedExceptions = new ArrayList(); var cp = findClassPath(pkg); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java index 8428cd847059..160cc7231bc4 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/TruffleCompilerContext.java @@ -147,7 +147,8 @@ public void truffleRunCodegen( var s = org.enso.interpreter.runtime.scope.ModuleScope.Builder.fromCompilerModuleScopeBuilder( scopeBuilder); - new IrToTruffle(context, m.getSource(), s, config).run(module.getPackage(), module.getIr()); + var pkg = ((Module) module).getPackage(); + new IrToTruffle(context, m.getSource(), s, config).run(pkg, module.getIr()); } // module related diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala index bbfed346c7f2..8222f2b38ccc 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala @@ -1,5 +1,6 @@ package org.enso.interpreter.runtime +import com.oracle.truffle.api.TruffleFile import com.oracle.truffle.api.source.{Source, SourceSection} import com.oracle.truffle.api.interop.InteropLibrary import org.enso.compiler.context.{ @@ -153,7 +154,8 @@ class IrToTruffle( * * @param ir the IR to generate code for */ - def run(p: org.enso.pkg.Package[_], ir: Module): Unit = processModule(p, ir) + def run(p: org.enso.pkg.Package[TruffleFile], ir: Module): Unit = + processModule(p, ir) /** Executes the codegen pass on an inline input. * @@ -187,7 +189,7 @@ class IrToTruffle( * @param module the module for which code should be generated */ private def processModule( - p: org.enso.pkg.Package[_], + p: org.enso.pkg.Package[TruffleFile], module: Module ): Unit = { generateReExportBindings(module) @@ -236,7 +238,7 @@ class IrToTruffle( } private def registerPolyglotImports( - p: org.enso.pkg.Package[_], + p: org.enso.pkg.Package[TruffleFile], module: Module ): Unit = module.imports.foreach { From 8700cc81415a8f917bd9dc93f6eb1dc20f449666 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 30 Jul 2024 14:05:18 +0200 Subject: [PATCH 06/89] Load classes via ModuleLayer --- .../interop/java/AddToClassPathNode.java | 4 +- .../interpreter/runtime/EnsoClassPath.java | 65 +++++++++++++++++++ .../enso/interpreter/runtime/EnsoContext.java | 58 +++-------------- .../runtime/DefaultPackageRepository.scala | 4 -- 4 files changed, 74 insertions(+), 57 deletions(-) create mode 100644 engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java index e301da72ea7d..3b7f7341c58c 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java @@ -27,8 +27,6 @@ static AddToClassPathNode build() { Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) { var ctx = EnsoContext.get(this); var file = ctx.getTruffleFile(new File(expectStringNode.execute(path))); - var cp = ctx.findClassPath(null); - cp.addToClassPath(file); - return ctx.getBuiltins().nothing(); + throw ctx.raiseAssertionPanic(this, "Cannot add_to_class_path anymore " + file, null); } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java new file mode 100644 index 000000000000..6296b0ddc424 --- /dev/null +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -0,0 +1,65 @@ +package org.enso.interpreter.runtime; + +import java.lang.module.Configuration; +import java.lang.module.ModuleFinder; +import java.nio.file.Path; +import java.util.Collections; + +/** Representation of an Enso library class path. */ +public final class EnsoClassPath { + private static final EnsoClassPath EMPTY = new EnsoClassPath(null, null, null); + private final ModuleLayer.Controller cntrl; + private final ModuleLayer layer; + final ClassLoader loader; + + private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoader loader) { + this.cntrl = cntrl; + this.layer = layer; + this.loader = loader; + } + + static EnsoClassPath create(Path file) { + java.lang.module.ModuleFinder finder = ModuleFinder.of(file); + java.util.List moduleNames = + finder.findAll().stream().map(mod -> mod.descriptor().name()).toList(); + if (moduleNames.isEmpty()) { + return EMPTY; + } else { + java.lang.ModuleLayer parent = ModuleLayer.boot(); + java.util.List parentCfgs = + Collections.singletonList(parent.configuration()); + java.util.List parentModules = Collections.singletonList(parent); + java.lang.ClassLoader parentLoader = parent.findLoader("java.base"); + java.lang.module.Configuration cfg = + Configuration.resolve(finder, parentCfgs, finder, moduleNames); + java.lang.ModuleLayer.Controller cntrl = + ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); + java.lang.ModuleLayer layer = cntrl.layer(); + java.lang.ClassLoader loader = layer.findLoader(moduleNames.get(0)); + return new EnsoClassPath(cntrl, layer, loader); + } + } + /* + public void addToClassPath(TruffleFile file) { + if (findGuestJava() == null) { + try { + var url = file.toUri().toURL(); + loader.add(url); + } catch (MalformedURLException ex) { + throw new IllegalStateException(ex); + } + } else { + try { + var path = new File(file.toUri()).getAbsoluteFile(); + if (!path.exists()) { + throw new IllegalStateException("File not found " + path); + } + InteropLibrary.getUncached().invokeMember(findGuestJava(), "addPath", path.getPath()); + } catch (InteropException ex) { + throw new IllegalStateException(ex); + } + } + } + */ + +} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index dd06f2a4b7f4..ab1a83476b8d 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -24,13 +24,9 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.PrintStream; -import java.lang.module.Configuration; -import java.lang.module.ModuleFinder; -import java.net.MalformedURLException; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -433,60 +429,19 @@ public Optional findModuleByExpressionId(UUID expressionId) { * Creates a class path. A class path is capable to load Java classes in an isolated manner. * * @param pkg package to find class path for + * @return representation of an Enso library Java class path for given package */ @TruffleBoundary public synchronized EnsoClassPath findClassPath(Package pkg) { - var l = classPaths.get(pkg); + org.enso.interpreter.runtime.EnsoClassPath l = classPaths.get(pkg); if (l == null) { var ch = pkg.polyglotDir().resolve("java"); - l = new EnsoClassPath(ch); + l = EnsoClassPath.create(Paths.get(ch.toUri())); classPaths.put(pkg, l); } return l; } - public final class EnsoClassPath { - private final ModuleLayer.Controller cntrl; - private final ModuleLayer layer; - private final HostClassLoader hostClassLoader; - - private EnsoClassPath(TruffleFile file) { - var parent = ModuleLayer.boot(); - var parentCfgs = Collections.singletonList(parent.configuration()); - var parentModules = Collections.singletonList(parent); - var parentLoader = parent.findLoader("java.base"); - var finder = ModuleFinder.of(Paths.get(file.toUri())); - var cfg = Configuration.resolve(finder, parentCfgs, finder, Collections.emptyList()); - - this.hostClassLoader = new HostClassLoader(); - - this.cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); - this.layer = cntrl.layer(); - System.out.println("modules: " + layer.modules()); - } - - public void addToClassPath(TruffleFile file) { - if (findGuestJava() == null) { - try { - var url = file.toUri().toURL(); - hostClassLoader.add(url); - } catch (MalformedURLException ex) { - throw new IllegalStateException(ex); - } - } else { - try { - var path = new File(file.toUri()).getAbsoluteFile(); - if (!path.exists()) { - throw new IllegalStateException("File not found " + path); - } - InteropLibrary.getUncached().invokeMember(findGuestJava(), "addPath", path.getPath()); - } catch (InteropException ex) { - throw new IllegalStateException(ex); - } - } - } - } - /** * Checks whether provided object comes from Java. Either Java system libraries or libraries added * by {@link #addToClassPath(TruffleFile)}. @@ -563,11 +518,11 @@ public boolean isColorTerminalOutput() { public Object lookupJavaClass(Package pkg, String className) { var binaryName = new StringBuilder(className); var collectedExceptions = new ArrayList(); - var cp = findClassPath(pkg); + org.enso.interpreter.runtime.EnsoClassPath cp = findClassPath(pkg); for (; ; ) { var fqn = binaryName.toString(); try { - var hostSymbol = lookupHostSymbol(cp.hostClassLoader, fqn); + var hostSymbol = lookupHostSymbol(cp.loader, fqn); if (hostSymbol != null) { return hostSymbol; } @@ -593,6 +548,9 @@ private Object lookupHostSymbol(ClassLoader hostClassLoader, String fqn) throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException { try { if (findGuestJava() == null) { + if (hostClassLoader == null) { + throw new ClassNotFoundException(fqn); + } return environment.asHostSymbol(hostClassLoader.loadClass(fqn)); } else { return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala index 1402690d47a2..312a6696e3c4 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/DefaultPackageRepository.scala @@ -196,10 +196,6 @@ private class DefaultPackageRepository( pkg: Package[TruffleFile], isLibrary: Boolean ): Unit = { - val extensions = pkg.listPolyglotExtensions("java") - val cp = context.findClassPath(pkg) - extensions.map(cp.addToClassPath) - val (regularModules, syntheticModulesMetadata) = pkg .listSources() .map(srcFile => From 74f81117db0f6ec13f7b11e4493657983330045f Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 30 Jul 2024 16:01:00 +0200 Subject: [PATCH 07/89] Turning std-base and std-aws into Java modules --- std-bits/aws/src/main/java/module-info.java | 7 +++++++ .../org/enso/aws/file_system/S3FileSystemSPI.java | 1 - std-bits/base/src/main/java/module-info.java | 14 ++++++++++++++ .../base/enso_cloud/EnsoPathFileSystemSPI.java | 1 - 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 std-bits/aws/src/main/java/module-info.java create mode 100644 std-bits/base/src/main/java/module-info.java diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java new file mode 100644 index 000000000000..e7c11323bdb5 --- /dev/null +++ b/std-bits/aws/src/main/java/module-info.java @@ -0,0 +1,7 @@ +module org.enso.aws { + requires java.base; + requires org.enso.base; + + provides FileSystemSPI with + org.enso.aws.file_system.S3FileSystemSPI; +} diff --git a/std-bits/aws/src/main/java/org/enso/aws/file_system/S3FileSystemSPI.java b/std-bits/aws/src/main/java/org/enso/aws/file_system/S3FileSystemSPI.java index 360c4cfb2f70..c448ab2f158b 100644 --- a/std-bits/aws/src/main/java/org/enso/aws/file_system/S3FileSystemSPI.java +++ b/std-bits/aws/src/main/java/org/enso/aws/file_system/S3FileSystemSPI.java @@ -2,7 +2,6 @@ import org.enso.base.file_system.FileSystemSPI; -@org.openide.util.lookup.ServiceProvider(service = FileSystemSPI.class) public class S3FileSystemSPI extends FileSystemSPI { @Override protected String getModuleName() { diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java new file mode 100644 index 000000000000..0d32e5bf0d4c --- /dev/null +++ b/std-bits/base/src/main/java/module-info.java @@ -0,0 +1,14 @@ +import org.enso.base.enso_cloud.EnsoPathFileSystemSPI; +import org.enso.base.file_system.FileSystemSPI; + +module org.enso.base { + requires java.logging; + requires org.graalvm.collections; + + uses FileSystemSPI; + + exports org.enso.base.file_system; + + provides FileSystemSPI with + EnsoPathFileSystemSPI; +} diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoPathFileSystemSPI.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoPathFileSystemSPI.java index 2008cb61a9d5..aa325acfb39f 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoPathFileSystemSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoPathFileSystemSPI.java @@ -7,7 +7,6 @@ * *

See `Enso_File.new` for more information on path resolution. */ -@org.openide.util.lookup.ServiceProvider(service = FileSystemSPI.class) public class EnsoPathFileSystemSPI extends FileSystemSPI { @Override protected String getModuleName() { From 7e865ce07265ecff8bc4dbb9c8f300535b312136 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 30 Jul 2024 19:14:05 +0200 Subject: [PATCH 08/89] Turn common-polyglot-core-utils into JPMS module --- build.sbt | 13 ++++++++++++- .../src/main/java/module-info.java | 5 +++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 lib/scala/common-polyglot-core-utils/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index cde4f6c20126..10a2c3aec3bb 100644 --- a/build.sbt +++ b/build.sbt @@ -3294,6 +3294,7 @@ lazy val `std-base` = project lazy val `common-polyglot-core-utils` = project .in(file("lib/scala/common-polyglot-core-utils")) + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -3302,7 +3303,17 @@ lazy val `common-polyglot-core-utils` = project libraryDependencies ++= Seq( "com.ibm.icu" % "icu4j" % icuVersion, "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided" - ) + ), + modulePath := { + JPMSUtils.filterModulesFromUpdate( + (Compile / update).value, + Seq( + "com.ibm.icu" % "icu4j" % icuVersion + ), + streams.value.log, + shouldContainAll = true + ) + } ) lazy val `enso-test-java-helpers` = project diff --git a/lib/scala/common-polyglot-core-utils/src/main/java/module-info.java b/lib/scala/common-polyglot-core-utils/src/main/java/module-info.java new file mode 100644 index 000000000000..fcbb6dd02023 --- /dev/null +++ b/lib/scala/common-polyglot-core-utils/src/main/java/module-info.java @@ -0,0 +1,5 @@ +module org.enso.polyglot.common_utils { + exports org.enso.polyglot.common_utils; + + requires com.ibm.icu; +} From 3a6c33558a6dbdca4262eef7af9eb82707a97008 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 30 Jul 2024 19:14:39 +0200 Subject: [PATCH 09/89] Add depedencies to org.enso.std.base JPMS module --- build.sbt | 23 ++++++++++++++++++++ std-bits/base/src/main/java/module-info.java | 13 ++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 10a2c3aec3bb..1b7cc51a5652 100644 --- a/build.sbt +++ b/build.sbt @@ -3263,6 +3263,7 @@ val `std-snowflake-polyglot-root` = lazy val `std-base` = project .in(file("std-bits") / "base") + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -3276,6 +3277,28 @@ lazy val `std-base` = project "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion ), + modulePath := { + val updateReport = (Compile / update).value + val requiredModIds = Seq( + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, + "com.ibm.icu" % "icu4j" % icuVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion + ) + val externalRequiredMods = JPMSUtils.filterModulesFromUpdate( + updateReport, + requiredModIds, + streams.value.log, + shouldContainAll = true + ) + val ourRequiredMods = + (`common-polyglot-core-utils` / Compile / exportedProducts).value + .map(_.data) + externalRequiredMods ++ ourRequiredMods + }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ensureCoreIsCompiled = diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index 0d32e5bf0d4c..b1f29ced30a5 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -1,13 +1,24 @@ import org.enso.base.enso_cloud.EnsoPathFileSystemSPI; import org.enso.base.file_system.FileSystemSPI; -module org.enso.base { +module org.enso.std.base { requires java.logging; + requires java.xml; + requires java.net.http; + requires java.sql; requires org.graalvm.collections; + requires com.ibm.icu; + requires org.graalvm.polyglot; + requires com.fasterxml.jackson.databind; + requires org.openide.util.lookup.RELEASE180; + requires org.enso.polyglot.common_utils; uses FileSystemSPI; exports org.enso.base.file_system; + exports org.enso.base.file_format; + exports org.enso.base.enso_cloud; + exports org.enso.base.enso_cloud.audit; provides FileSystemSPI with EnsoPathFileSystemSPI; From b23be3de330483b011f1e1a719af74b7a824e965 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 30 Jul 2024 19:15:21 +0200 Subject: [PATCH 10/89] Turn std-database into JPMS module --- build.sbt | 20 +++++++++++++++++++ .../database/src/main/java/module-info.java | 12 +++++++++++ 2 files changed, 32 insertions(+) create mode 100644 std-bits/database/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index 1b7cc51a5652..19ea499dca6c 100644 --- a/build.sbt +++ b/build.sbt @@ -3504,6 +3504,7 @@ lazy val `std-google-api` = project lazy val `std-database` = project .in(file("std-bits") / "database") + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -3518,6 +3519,25 @@ lazy val `std-database` = project "org.xerial" % "sqlite-jdbc" % sqliteVersion, "org.postgresql" % "postgresql" % "42.4.0" ), + modulePath := { + val externalModIds = libraryDependencies.value ++ Seq( + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion + ) + val externalMods = JPMSUtils.filterModulesFromUpdate( + (Compile / update).value, + externalModIds, + streams.value.log, + shouldContainAll = true + ) + val ourMods = + (`std-base` / Compile / exportedProducts).value.map(_.data) ++ + (`common-polyglot-core-utils` / Compile / exportedProducts).value + .map(_.data) + externalMods ++ ourMods + }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/std-bits/database/src/main/java/module-info.java b/std-bits/database/src/main/java/module-info.java new file mode 100644 index 000000000000..8bba39864e3e --- /dev/null +++ b/std-bits/database/src/main/java/module-info.java @@ -0,0 +1,12 @@ +module org.enso.std.database { + requires java.logging; + requires java.sql; + requires org.enso.std.base; + requires org.enso.polyglot.common_utils; + requires org.graalvm.collections; + requires org.graalvm.polyglot; + requires com.fasterxml.jackson.databind; + requires org.openide.util.lookup.RELEASE180; + + exports org.enso.database; +} From f0dd3f7e466096672aa9e65ddec38b2b31060c77 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 30 Jul 2024 19:15:49 +0200 Subject: [PATCH 11/89] Add depedencies to org.enso.std.aws JPMS module --- build.sbt | 26 +++++++++++++++++++++ std-bits/aws/src/main/java/module-info.java | 16 +++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 19ea499dca6c..739b5bb179ac 100644 --- a/build.sbt +++ b/build.sbt @@ -3555,6 +3555,7 @@ lazy val `std-database` = project lazy val `std-aws` = project .in(file("std-bits") / "aws") + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -3575,6 +3576,31 @@ lazy val `std-aws` = project "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version ), + modulePath := { + val externalModIds = Seq( + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, + "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version + ) + val externalMods = JPMSUtils.filterModulesFromUpdate( + (Compile / update).value, + externalModIds, + streams.value.log, + shouldContainAll = true + ) + val ourMods = + (`std-base` / Compile / exportedProducts).value.map(_.data) ++ + (`std-database` / Compile / exportedProducts).value.map(_.data) + externalMods ++ ourMods + }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index e7c11323bdb5..86952e7d733d 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -1,6 +1,18 @@ -module org.enso.aws { +import org.enso.base.file_system.FileSystemSPI; + +module org.enso.std.aws { requires java.base; - requires org.enso.base; + requires org.enso.std.base; + requires org.enso.std.database; + requires java.logging; + requires software.amazon.awssdk.http; + requires software.amazon.awssdk.regions; + requires software.amazon.awssdk.services.s3; + requires software.amazon.awssdk.auth; + requires software.amazon.awssdk.core; + requires software.amazon.awssdk.profiles; + requires software.amazon.awssdk.awscore; + requires org.openide.util.lookup.RELEASE180; provides FileSystemSPI with org.enso.aws.file_system.S3FileSystemSPI; From c4c1842cc756b3c127cf92e0f0504358687bcb76 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 31 Jul 2024 07:33:40 +0200 Subject: [PATCH 12/89] Using module-info to register SPI services --- build.sbt | 60 +++++++++---------- std-bits/aws/src/main/java/module-info.java | 8 ++- .../RedshiftConnectionDetailsSPI.java | 3 +- .../enso/aws/file_system/S3DataLinkSPI.java | 3 +- std-bits/base/src/main/java/module-info.java | 32 +++++++++- .../base/enso_cloud/EnsoFileDataLinkSPI.java | 3 +- .../enso/base/file_format/ByteFormatSPI.java | 3 +- .../enso/base/file_format/JSONFormatSPI.java | 3 +- .../enso/base/file_format/TextFormatSPI.java | 3 +- .../enso/base/file_format/XMLFormatSPI.java | 3 +- .../base/net/http/HTTPFetchDataLinkSPI.java | 3 +- .../database/src/main/java/module-info.java | 21 ++++++- .../PostgresConnectionDetailsSPI.java | 3 +- .../postgres/PostgresDataLinkSPI.java | 3 +- .../sqlite/SQLiteConnectionDetailsSPI.java | 3 +- .../enso/database/sqlite/SQLiteFormatSPI.java | 3 +- .../sqlite/SQLiteInMemoryDetailsSPI.java | 3 +- 17 files changed, 98 insertions(+), 62 deletions(-) diff --git a/build.sbt b/build.sbt index 739b5bb179ac..215c63afbea1 100644 --- a/build.sbt +++ b/build.sbt @@ -3280,13 +3280,12 @@ lazy val `std-base` = project modulePath := { val updateReport = (Compile / update).value val requiredModIds = Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, - "com.ibm.icu" % "icu4j" % icuVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "com.ibm.icu" % "icu4j" % icuVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion ) val externalRequiredMods = JPMSUtils.filterModulesFromUpdate( updateReport, @@ -3514,10 +3513,9 @@ lazy val `std-database` = project Compile / packageBin / artifactPath := `database-polyglot-root` / "std-database.jar", libraryDependencies ++= Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "org.xerial" % "sqlite-jdbc" % sqliteVersion, - "org.postgresql" % "postgresql" % "42.4.0" + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", + "org.xerial" % "sqlite-jdbc" % sqliteVersion, + "org.postgresql" % "postgresql" % "42.4.0" ), modulePath := { val externalModIds = libraryDependencies.value ++ Seq( @@ -3565,30 +3563,28 @@ lazy val `std-aws` = project Compile / packageBin / artifactPath := `std-aws-polyglot-root` / "std-aws.jar", libraryDependencies ++= Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, - "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version, - "com.amazonaws" % "aws-java-sdk-redshift" % awsJavaSdkV1Version, - "com.amazonaws" % "aws-java-sdk-sts" % awsJavaSdkV1Version, - "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "bom" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version + "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, + "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version, + "com.amazonaws" % "aws-java-sdk-redshift" % awsJavaSdkV1Version, + "com.amazonaws" % "aws-java-sdk-sts" % awsJavaSdkV1Version, + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "bom" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version ), modulePath := { val externalModIds = Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, - "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, - "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version + "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version ) val externalMods = JPMSUtils.filterModulesFromUpdate( (Compile / update).value, diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index 86952e7d733d..15d6c2398c9d 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -1,4 +1,7 @@ +import org.enso.aws.database.RedshiftConnectionDetailsSPI; +import org.enso.aws.file_system.S3FileSystemSPI; import org.enso.base.file_system.FileSystemSPI; +import org.enso.database.DatabaseConnectionDetailsSPI; module org.enso.std.aws { requires java.base; @@ -12,8 +15,9 @@ requires software.amazon.awssdk.core; requires software.amazon.awssdk.profiles; requires software.amazon.awssdk.awscore; - requires org.openide.util.lookup.RELEASE180; provides FileSystemSPI with - org.enso.aws.file_system.S3FileSystemSPI; + S3FileSystemSPI; + provides DatabaseConnectionDetailsSPI with + RedshiftConnectionDetailsSPI; } diff --git a/std-bits/aws/src/main/java/org/enso/aws/database/RedshiftConnectionDetailsSPI.java b/std-bits/aws/src/main/java/org/enso/aws/database/RedshiftConnectionDetailsSPI.java index ed0623931443..10c7e3912d7f 100644 --- a/std-bits/aws/src/main/java/org/enso/aws/database/RedshiftConnectionDetailsSPI.java +++ b/std-bits/aws/src/main/java/org/enso/aws/database/RedshiftConnectionDetailsSPI.java @@ -2,8 +2,7 @@ import org.enso.database.DatabaseConnectionDetailsSPI; -@org.openide.util.lookup.ServiceProvider(service = DatabaseConnectionDetailsSPI.class) -public class RedshiftConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { +public final class RedshiftConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { @Override protected String getModuleName() { return "Standard.AWS.Database.Redshift.Redshift_Details"; diff --git a/std-bits/aws/src/main/java/org/enso/aws/file_system/S3DataLinkSPI.java b/std-bits/aws/src/main/java/org/enso/aws/file_system/S3DataLinkSPI.java index 0f6692e93421..de9af591d790 100644 --- a/std-bits/aws/src/main/java/org/enso/aws/file_system/S3DataLinkSPI.java +++ b/std-bits/aws/src/main/java/org/enso/aws/file_system/S3DataLinkSPI.java @@ -2,8 +2,7 @@ import org.enso.base.enso_cloud.DataLinkSPI; -@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class) -public class S3DataLinkSPI extends DataLinkSPI { +public final class S3DataLinkSPI extends DataLinkSPI { @Override protected String getModuleName() { return "Standard.AWS.S3.S3_Data_Link"; diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index b1f29ced30a5..24fc2f66a2ed 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -1,5 +1,13 @@ +import org.enso.base.enso_cloud.DataLinkSPI; +import org.enso.base.enso_cloud.EnsoFileDataLinkSPI; import org.enso.base.enso_cloud.EnsoPathFileSystemSPI; +import org.enso.base.file_format.ByteFormatSPI; +import org.enso.base.file_format.FileFormatSPI; +import org.enso.base.file_format.JSONFormatSPI; +import org.enso.base.file_format.TextFormatSPI; +import org.enso.base.file_format.XMLFormatSPI; import org.enso.base.file_system.FileSystemSPI; +import org.enso.base.net.http.HTTPFetchDataLinkSPI; module org.enso.std.base { requires java.logging; @@ -10,16 +18,38 @@ requires com.ibm.icu; requires org.graalvm.polyglot; requires com.fasterxml.jackson.databind; - requires org.openide.util.lookup.RELEASE180; requires org.enso.polyglot.common_utils; uses FileSystemSPI; + uses FileFormatSPI; + uses DataLinkSPI; + // following packages are accessed by Java code in other Enso modules exports org.enso.base.file_system; exports org.enso.base.file_format; exports org.enso.base.enso_cloud; exports org.enso.base.enso_cloud.audit; + // following packages are accessed by Enso via polyglot java import + opens org.enso.base; + opens org.enso.base.arrays; + opens org.enso.base.encoding; + opens org.enso.base.numeric; + opens org.enso.base.net; + opens org.enso.base.polyglot; + opens org.enso.base.random; + opens org.enso.base.statistics; + opens org.enso.base.text; + opens org.enso.base.time; + provides FileSystemSPI with EnsoPathFileSystemSPI; + provides FileFormatSPI with + ByteFormatSPI, + JSONFormatSPI, + TextFormatSPI, + XMLFormatSPI; + provides DataLinkSPI with + EnsoFileDataLinkSPI, + HTTPFetchDataLinkSPI; } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java index 6e496d4c61fb..4841d839ec21 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoFileDataLinkSPI.java @@ -1,7 +1,6 @@ package org.enso.base.enso_cloud; -@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class) -public class EnsoFileDataLinkSPI extends DataLinkSPI { +public final class EnsoFileDataLinkSPI extends DataLinkSPI { @Override protected String getModuleName() { return "Standard.Base.Enso_Cloud.Internal.Enso_File_Data_Link"; diff --git a/std-bits/base/src/main/java/org/enso/base/file_format/ByteFormatSPI.java b/std-bits/base/src/main/java/org/enso/base/file_format/ByteFormatSPI.java index 35df9626cab6..fccd140ff7e9 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_format/ByteFormatSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_format/ByteFormatSPI.java @@ -1,7 +1,6 @@ package org.enso.base.file_format; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class ByteFormatSPI extends FileFormatSPI { +public final class ByteFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Base.System.File_Format"; diff --git a/std-bits/base/src/main/java/org/enso/base/file_format/JSONFormatSPI.java b/std-bits/base/src/main/java/org/enso/base/file_format/JSONFormatSPI.java index 575e6d4bc23f..d1a88c0b852d 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_format/JSONFormatSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_format/JSONFormatSPI.java @@ -1,7 +1,6 @@ package org.enso.base.file_format; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class JSONFormatSPI extends FileFormatSPI { +public final class JSONFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Base.System.File_Format"; diff --git a/std-bits/base/src/main/java/org/enso/base/file_format/TextFormatSPI.java b/std-bits/base/src/main/java/org/enso/base/file_format/TextFormatSPI.java index 2ee2ce353b9a..6e92b4ac8653 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_format/TextFormatSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_format/TextFormatSPI.java @@ -1,7 +1,6 @@ package org.enso.base.file_format; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class TextFormatSPI extends FileFormatSPI { +public final class TextFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Base.System.File_Format"; diff --git a/std-bits/base/src/main/java/org/enso/base/file_format/XMLFormatSPI.java b/std-bits/base/src/main/java/org/enso/base/file_format/XMLFormatSPI.java index a1ff39077660..8a2abaa95ef9 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_format/XMLFormatSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_format/XMLFormatSPI.java @@ -1,7 +1,6 @@ package org.enso.base.file_format; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class XMLFormatSPI extends FileFormatSPI { +public final class XMLFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Base.Data.XML.XML_Format"; diff --git a/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java b/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java index 027b157de63c..175f5b50e316 100644 --- a/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/net/http/HTTPFetchDataLinkSPI.java @@ -2,8 +2,7 @@ import org.enso.base.enso_cloud.DataLinkSPI; -@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class) -public class HTTPFetchDataLinkSPI extends DataLinkSPI { +public final class HTTPFetchDataLinkSPI extends DataLinkSPI { @Override protected String getModuleName() { return "Standard.Base.Network.HTTP.Internal.HTTP_Fetch_Data_Link"; diff --git a/std-bits/database/src/main/java/module-info.java b/std-bits/database/src/main/java/module-info.java index 8bba39864e3e..701f17a52760 100644 --- a/std-bits/database/src/main/java/module-info.java +++ b/std-bits/database/src/main/java/module-info.java @@ -1,3 +1,12 @@ +import org.enso.base.enso_cloud.DataLinkSPI; +import org.enso.base.file_format.FileFormatSPI; +import org.enso.database.DatabaseConnectionDetailsSPI; +import org.enso.database.postgres.PostgresConnectionDetailsSPI; +import org.enso.database.postgres.PostgresDataLinkSPI; +import org.enso.database.sqlite.SQLiteConnectionDetailsSPI; +import org.enso.database.sqlite.SQLiteFormatSPI; +import org.enso.database.sqlite.SQLiteInMemoryDetailsSPI; + module org.enso.std.database { requires java.logging; requires java.sql; @@ -6,7 +15,17 @@ requires org.graalvm.collections; requires org.graalvm.polyglot; requires com.fasterxml.jackson.databind; - requires org.openide.util.lookup.RELEASE180; + + uses DatabaseConnectionDetailsSPI; exports org.enso.database; + + provides DatabaseConnectionDetailsSPI with + PostgresConnectionDetailsSPI, + SQLiteConnectionDetailsSPI, + SQLiteInMemoryDetailsSPI; + provides DataLinkSPI with + PostgresDataLinkSPI; + provides FileFormatSPI with + SQLiteFormatSPI; } diff --git a/std-bits/database/src/main/java/org/enso/database/postgres/PostgresConnectionDetailsSPI.java b/std-bits/database/src/main/java/org/enso/database/postgres/PostgresConnectionDetailsSPI.java index ee357856e205..a0f0c98a67a0 100644 --- a/std-bits/database/src/main/java/org/enso/database/postgres/PostgresConnectionDetailsSPI.java +++ b/std-bits/database/src/main/java/org/enso/database/postgres/PostgresConnectionDetailsSPI.java @@ -2,8 +2,7 @@ import org.enso.database.DatabaseConnectionDetailsSPI; -@org.openide.util.lookup.ServiceProvider(service = DatabaseConnectionDetailsSPI.class) -public class PostgresConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { +public final class PostgresConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { @Override protected String getModuleName() { return "Standard.Database.Connection.Postgres"; diff --git a/std-bits/database/src/main/java/org/enso/database/postgres/PostgresDataLinkSPI.java b/std-bits/database/src/main/java/org/enso/database/postgres/PostgresDataLinkSPI.java index 589f6202b74a..b953898b1372 100644 --- a/std-bits/database/src/main/java/org/enso/database/postgres/PostgresDataLinkSPI.java +++ b/std-bits/database/src/main/java/org/enso/database/postgres/PostgresDataLinkSPI.java @@ -2,8 +2,7 @@ import org.enso.base.enso_cloud.DataLinkSPI; -@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class) -public class PostgresDataLinkSPI extends DataLinkSPI { +public final class PostgresDataLinkSPI extends DataLinkSPI { @Override protected String getModuleName() { return "Standard.Database.Connection.Data_Link.Postgres_Data_Link"; diff --git a/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteConnectionDetailsSPI.java b/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteConnectionDetailsSPI.java index a758ffd5980f..f81805a2f27e 100644 --- a/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteConnectionDetailsSPI.java +++ b/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteConnectionDetailsSPI.java @@ -2,8 +2,7 @@ import org.enso.database.DatabaseConnectionDetailsSPI; -@org.openide.util.lookup.ServiceProvider(service = DatabaseConnectionDetailsSPI.class) -public class SQLiteConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { +public final class SQLiteConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { @Override protected String getModuleName() { return "Standard.Database.Connection.SQLite"; diff --git a/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteFormatSPI.java b/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteFormatSPI.java index f8d3a9b9b463..e6149777791a 100644 --- a/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteFormatSPI.java +++ b/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteFormatSPI.java @@ -2,8 +2,7 @@ import org.enso.base.file_format.FileFormatSPI; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class SQLiteFormatSPI extends FileFormatSPI { +public final class SQLiteFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Database.Connection.SQLite_Format"; diff --git a/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteInMemoryDetailsSPI.java b/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteInMemoryDetailsSPI.java index ea91624afead..490ba2094322 100644 --- a/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteInMemoryDetailsSPI.java +++ b/std-bits/database/src/main/java/org/enso/database/sqlite/SQLiteInMemoryDetailsSPI.java @@ -2,8 +2,7 @@ import org.enso.database.DatabaseConnectionDetailsSPI; -@org.openide.util.lookup.ServiceProvider(service = DatabaseConnectionDetailsSPI.class) -public class SQLiteInMemoryDetailsSPI extends DatabaseConnectionDetailsSPI { +public final class SQLiteInMemoryDetailsSPI extends DatabaseConnectionDetailsSPI { @Override protected String getModuleName() { return "Standard.Database.Connection.SQLite"; From 149dc7e0286e01e604f8e59132af95ae57cb4dc3 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 31 Jul 2024 09:07:06 +0200 Subject: [PATCH 13/89] Removing another no longer needed dependency on NetBeans Lookup API --- build.sbt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 215c63afbea1..4d8be1dc213c 100644 --- a/build.sbt +++ b/build.sbt @@ -3273,9 +3273,8 @@ lazy val `std-base` = project Compile / packageBin / artifactPath := `base-polyglot-root` / "std-base.jar", libraryDependencies ++= Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion ), modulePath := { val updateReport = (Compile / update).value From 48b29c71aaa86358a97dfe1cd744889020773239 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 31 Jul 2024 10:01:38 +0200 Subject: [PATCH 14/89] Removing add to class path concept --- .../lib/Standard/Base/0.0.0-dev/src/Meta.enso | 3 +- .../Standard/Base/0.0.0-dev/src/Polyglot.enso | 47 ---------- .../interop/java/AddToClassPathNode.java | 32 ------- .../builtin/interop/java/LookupClassNode.java | 28 ------ .../interpreter/runtime/EnsoClassPath.java | 23 ----- .../enso/interpreter/runtime/EnsoContext.java | 6 +- .../interpreter/runtime/HostClassLoader.java | 87 ------------------- 7 files changed, 4 insertions(+), 222 deletions(-) delete mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java delete mode 100644 engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java delete mode 100644 engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso index 49d4095ef40c..e2b84eb05976 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Meta.enso @@ -13,7 +13,6 @@ import project.Error.Error as Base_Error import project.Errors.Common.Not_Found import project.Function.Function import project.Nothing.Nothing -import project.Polyglot.Java from project.Data.Boolean import Boolean, False, True from project.Runtime.Managed_Resource import Managed_Resource @@ -445,7 +444,7 @@ is_a value typ = @Builtin_Method "Meta.is_a" java_instance_check value typ = val_java = get_polyglot_language value == "java" typ_java = get_polyglot_language typ == "java" - val_java && typ_java && Java.is_instance value typ + val_java && typ_java && typ.class.isInstance value ## PRIVATE ADVANCED diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso index 55f4684c7bd8..0fb2d57240e0 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Polyglot.enso @@ -132,50 +132,3 @@ type Polyglot Returns the executable name of a polyglot object. get_executable_name : Any -> Text get_executable_name value = @Builtin_Method "Polyglot.get_executable_name" - -## Utilities for working with Java polyglot objects. -type Java - ## PRIVATE - ADVANCED - Adds the provided entry to the host class path. - - Arguments: - - path: The java classpath entry to add. - - Use of the actual polyglot imports system should be preferred to use of - this method. - - > Example - Adding Random to the classpath. - - Java.add_to_class_path "java.util.Random" - add_to_class_path : Text -> Nothing - add_to_class_path path = @Builtin_Method "Java.add_to_class_path" - - ## PRIVATE - ADVANCED - Looks up a java symbol on the classpath by name. - - Arguments: - - name: The name of the java symbol to look up. - - Use of the actual polyglot imports system should be preferred to use of - this method. - - > Example - Look up java's Random class. - - Java.lookup_class "java.util.Random" - lookup_class : Text -> Any - lookup_class name = @Builtin_Method "Java.lookup_class" - - ## PRIVATE - Checks whether an object is an instance of a given class. - - Arguments: - - object: The object to check for class membership. - - class: The java class to check for membership in. - is_instance : Any -> Any -> Boolean - is_instance object class = - class_object = class.class - class_object.isInstance object diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java deleted file mode 100644 index 3b7f7341c58c..000000000000 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/AddToClassPathNode.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.enso.interpreter.node.expression.builtin.interop.java; - -import com.oracle.truffle.api.CompilerDirectives; -import com.oracle.truffle.api.dsl.Cached; -import com.oracle.truffle.api.dsl.Specialization; -import com.oracle.truffle.api.nodes.Node; -import java.io.File; -import org.enso.interpreter.dsl.BuiltinMethod; -import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; -import org.enso.interpreter.runtime.EnsoContext; - -@BuiltinMethod( - type = "Java", - name = "add_to_class_path", - description = "Adds a path to the host class path.", - autoRegister = false) -public abstract class AddToClassPathNode extends Node { - - static AddToClassPathNode build() { - return AddToClassPathNodeGen.create(); - } - - abstract Object execute(Object path); - - @CompilerDirectives.TruffleBoundary - @Specialization - Object doExecute(Object path, @Cached ExpectStringNode expectStringNode) { - var ctx = EnsoContext.get(this); - var file = ctx.getTruffleFile(new File(expectStringNode.execute(path))); - throw ctx.raiseAssertionPanic(this, "Cannot add_to_class_path anymore " + file, null); - } -} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java b/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java deleted file mode 100644 index 9012a7c95645..000000000000 --- a/engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.enso.interpreter.node.expression.builtin.interop.java; - -import com.oracle.truffle.api.CompilerDirectives; -import com.oracle.truffle.api.dsl.Cached; -import com.oracle.truffle.api.dsl.Specialization; -import com.oracle.truffle.api.nodes.Node; -import org.enso.interpreter.dsl.BuiltinMethod; -import org.enso.interpreter.node.expression.builtin.text.util.ExpectStringNode; -import org.enso.interpreter.runtime.EnsoContext; - -@BuiltinMethod( - type = "Java", - name = "lookup_class", - description = "Looks up a Java symbol.", - autoRegister = false) -public abstract class LookupClassNode extends Node { - static LookupClassNode build() { - return LookupClassNodeGen.create(); - } - - @Specialization - @CompilerDirectives.TruffleBoundary - Object doExecute(Object name, @Cached("build()") ExpectStringNode expectStringNode) { - return EnsoContext.get(this).lookupJavaClass(null, expectStringNode.execute(name)); - } - - abstract Object execute(Object name); -} diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 6296b0ddc424..c491082355d8 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -39,27 +39,4 @@ static EnsoClassPath create(Path file) { return new EnsoClassPath(cntrl, layer, loader); } } - /* - public void addToClassPath(TruffleFile file) { - if (findGuestJava() == null) { - try { - var url = file.toUri().toURL(); - loader.add(url); - } catch (MalformedURLException ex) { - throw new IllegalStateException(ex); - } - } else { - try { - var path = new File(file.toUri()).getAbsoluteFile(); - if (!path.exists()) { - throw new IllegalStateException("File not found " + path); - } - InteropLibrary.getUncached().invokeMember(findGuestJava(), "addPath", path.getPath()); - } catch (InteropException ex) { - throw new IllegalStateException(ex); - } - } - } - */ - } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index ab1a83476b8d..9f3ccbd33d21 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -544,14 +544,14 @@ public Object lookupJavaClass(Package pkg, String className) { return null; } - private Object lookupHostSymbol(ClassLoader hostClassLoader, String fqn) + private Object lookupHostSymbol(ClassLoader loader, String fqn) throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException { try { if (findGuestJava() == null) { - if (hostClassLoader == null) { + if (loader == null) { throw new ClassNotFoundException(fqn); } - return environment.asHostSymbol(hostClassLoader.loadClass(fqn)); + return environment.asHostSymbol(loader.loadClass(fqn)); } else { return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java deleted file mode 100644 index 8a8886809865..000000000000 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/HostClassLoader.java +++ /dev/null @@ -1,87 +0,0 @@ -package org.enso.interpreter.runtime; - -import java.io.IOException; -import java.net.URL; -import java.net.URLClassLoader; -import java.util.Enumeration; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import org.enso.ClassLoaderConstants; -import org.graalvm.polyglot.Context; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Host class loader that serves as a replacement for {@link - * com.oracle.truffle.host.HostClassLoader}. Add URLs to Jar archives with {@link #add(URL)}. All - * the classes that are loaded via this class loader are first searched inside those archives. If - * not found, delegates to parent class loaders. - */ -final class HostClassLoader extends URLClassLoader { - - private final Map> loadedClasses = new ConcurrentHashMap<>(); - private static final Logger logger = LoggerFactory.getLogger(HostClassLoader.class); - // Classes from "org.graalvm" packages are loaded either by a class loader for the boot - // module layer, or by a specific class loader, depending on how enso is run. For example, - // if enso is run via `org.graalvm.polyglot.Context.eval` from `javac`, then the graalvm - // classes are loaded via a class loader somehow created by `javac` and not by the boot - // module layer's class loader. - private static final ClassLoader polyglotClassLoader = Context.class.getClassLoader(); - - public HostClassLoader() { - super(new URL[0], polyglotClassLoader); - } - - void add(URL u) { - logger.debug("Adding URL '{}' to class path", u); - addURL(u); - } - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - return loadClass(name, false); - } - - @Override - protected Class loadClass(String name, boolean resolve) throws ClassNotFoundException { - logger.trace("Loading class {}", name); - var l = loadedClasses.get(name); - if (l != null) { - logger.trace("Class {} found in cache", name); - return l; - } - if (ClassLoaderConstants.CLASS_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) { - return polyglotClassLoader.loadClass(name); - } - try { - l = findClass(name); - if (resolve) { - l.getMethods(); - } - logger.trace("Class {} found, putting in cache", name); - loadedClasses.put(name, l); - return l; - } catch (ClassNotFoundException ex) { - logger.trace("Class {} not found, delegating to super", name); - return super.loadClass(name, resolve); - } - } - - @Override - public URL findResource(String name) { - if (ClassLoaderConstants.CLASS_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) { - return polyglotClassLoader.getResource(name); - } else { - return super.findResource(name); - } - } - - @Override - public Enumeration findResources(String name) throws IOException { - if (ClassLoaderConstants.CLASS_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) { - return polyglotClassLoader.getResources(name); - } else { - return super.findResources(name); - } - } -} From dd5723e2245144397fecd1e198fe1b4996a7c5cc Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 31 Jul 2024 16:54:27 +0200 Subject: [PATCH 15/89] Turn std-table into JPMS module --- build.sbt | 28 +++++++++++++++++++ std-bits/base/src/main/java/module-info.java | 7 ++++- std-bits/table/src/main/java/module-info.java | 15 ++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 std-bits/table/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index 4d8be1dc213c..63f485704fa2 100644 --- a/build.sbt +++ b/build.sbt @@ -3405,6 +3405,7 @@ lazy val `benchmark-java-helpers` = project lazy val `std-table` = project .in(file("std-bits") / "table") .enablePlugins(Antlr4Plugin) + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -3429,6 +3430,33 @@ lazy val `std-table` = project "org.antlr" % "antlr4-runtime" % antlrVersion, "org.apache.logging.log4j" % "log4j-to-slf4j" % "2.18.0" // org.apache.poi uses log4j ), + modulePath := { + val updateReport = (Compile / update).value + val requiredModIds = Seq( + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.antlr" % "antlr4-runtime" % antlrVersion, + "com.univocity" % "univocity-parsers" % univocityParsersVersion, + "org.apache.poi" % "poi" % poiOoxmlVersion, + "org.apache.poi" % "poi-ooxml" % poiOoxmlVersion, + "org.apache.poi" % "poi-ooxml-lite" % poiOoxmlVersion, + "org.apache.xmlbeans" % "xmlbeans" % xmlbeansVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "com.ibm.icu" % "icu4j" % icuVersion, + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion + ) + val externalRequiredMods = JPMSUtils.filterModulesFromUpdate( + updateReport, + requiredModIds, + streams.value.log, + shouldContainAll = true + ) + val ourRequiredMods = + (`common-polyglot-core-utils` / Compile / exportedProducts).value + .map(_.data) ++ + (`std-base` / Compile / exportedProducts).value + .map(_.data) + externalRequiredMods ++ ourRequiredMods + }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index 24fc2f66a2ed..e82a8a168aeb 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -25,10 +25,16 @@ uses DataLinkSPI; // following packages are accessed by Java code in other Enso modules + exports org.enso.base; exports org.enso.base.file_system; exports org.enso.base.file_format; exports org.enso.base.enso_cloud; exports org.enso.base.enso_cloud.audit; + exports org.enso.base.polyglot; + exports org.enso.base.time; + exports org.enso.base.text; + exports org.enso.base.arrays; + exports org.enso.base.statistics; // following packages are accessed by Enso via polyglot java import opens org.enso.base; @@ -40,7 +46,6 @@ opens org.enso.base.random; opens org.enso.base.statistics; opens org.enso.base.text; - opens org.enso.base.time; provides FileSystemSPI with EnsoPathFileSystemSPI; diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java new file mode 100644 index 000000000000..abb14a0b6cec --- /dev/null +++ b/std-bits/table/src/main/java/module-info.java @@ -0,0 +1,15 @@ +module enso.std.table { + requires java.xml; + requires com.ibm.icu; + requires org.antlr.antlr4.runtime; + requires org.apache.poi.ooxml; + requires org.apache.poi.ooxml.schemas; + requires org.apache.poi.poi; + requires org.apache.xmlbeans; + requires org.graalvm.collections; + requires org.graalvm.polyglot; + requires org.openide.util.lookup.RELEASE180; + requires org.enso.polyglot.common_utils; + requires org.enso.std.base; + requires univocity.parsers; +} From 74fa9e14d573ffa414f174f2428f4efa5586d9f8 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 31 Jul 2024 17:30:53 +0200 Subject: [PATCH 16/89] Move tests that need internal access to Standard.Base into test/Base_Internal_Tests --- build/build/src/enso.rs | 3 + .../0.0.0-dev/src/Internal/Extra_Imports.enso | 22 ++++- project/DistributionPackage.scala | 16 ++++ test/Base_Internal_Tests/README.md | 2 + test/Base_Internal_Tests/package.yaml | 9 ++ .../src}/Comparator_Spec.enso | 6 +- .../src/Grapheme_Spec.enso} | 10 +-- .../src/Input_Output_Spec.enso | 86 +++++++++++++++++++ test/Base_Internal_Tests/src/Main.enso | 15 ++++ test/Base_Tests/src/Data/Dictionary_Spec.enso | 8 -- test/Base_Tests/src/Main.enso | 4 - test/Base_Tests/src/Semantic/Equals_Spec.enso | 7 -- .../src/System/Input_Stream_Spec.enso | 47 ---------- 13 files changed, 155 insertions(+), 80 deletions(-) create mode 100644 test/Base_Internal_Tests/README.md create mode 100644 test/Base_Internal_Tests/package.yaml rename test/{Base_Tests/src/Data/Ordering => Base_Internal_Tests/src}/Comparator_Spec.enso (96%) rename test/{Base_Tests/src/Data/Text/Utils_Spec.enso => Base_Internal_Tests/src/Grapheme_Spec.enso} (96%) create mode 100644 test/Base_Internal_Tests/src/Input_Output_Spec.enso create mode 100644 test/Base_Internal_Tests/src/Main.enso diff --git a/build/build/src/enso.rs b/build/build/src/enso.rs index 51fb4115ab00..e2b84f74dbbf 100644 --- a/build/build/src/enso.rs +++ b/build/build/src/enso.rs @@ -100,6 +100,9 @@ impl BuiltEnso { // This flag enables assertions in the JVM. Some of our stdlib tests had in the past // failed on Graal/Truffle assertions, so we want to have them triggered. .set_env(JAVA_OPTS, &ide_ci::programs::java::Option::EnableAssertions.as_ref())?; + if test_path.as_str().contains("_Internal_") { + command.arg("--disable-private-check"); + } Ok(command) } diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Extra_Imports.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Extra_Imports.enso index 7771401cf43d..ae8b02fda49f 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Extra_Imports.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Internal/Extra_Imports.enso @@ -9,11 +9,10 @@ private polyglot java import java.io.PrintStream # needed by Util_Spec -polyglot java import org.enso.base.text.CaseFoldedString +polyglot java import org.enso.base.text.CaseFoldedString as JCaseFoldedString polyglot java import org.enso.base.text.CaseFoldedString.Grapheme - -# needed by Comparator_Spec: -polyglot java import org.enso.base.ObjectComparator +polyglot java import org.enso.base.Text_Utils as JText_Utils +polyglot java import com.ibm.icu.text.BreakIterator as JBreakIterator # often used in tests polyglot java import java.util.ArrayList @@ -26,6 +25,21 @@ polyglot java import java.math.BigInteger polyglot java import java.time.LocalDate polyglot java import java.time.LocalDateTime polyglot java import java.util.function.Function +polyglot java import java.lang.Exception as JException polyglot java import java.lang.Thread polyglot java import java.lang.Thread.State polyglot java import java.lang.Float + +polyglot java import org.enso.base.Stream_Utils as JStream_Utils +polyglot java import org.enso.base.file_system.File_Utils as JFile_Utils +polyglot java import org.enso.base.CompareException as JCompareException +polyglot java import org.enso.base.ObjectComparator as JObjectComparator + +Stream_Utils=JStream_Utils +File_Utils=JFile_Utils +CompareException=JCompareException +ObjectComparator=JObjectComparator +CaseFoldedString=JCaseFoldedString +Text_Utils=JText_Utils +BreakIterator=JBreakIterator +Exception=JException diff --git a/project/DistributionPackage.scala b/project/DistributionPackage.scala index 66f5bbcbbc65..26f5d487d73c 100644 --- a/project/DistributionPackage.scala +++ b/project/DistributionPackage.scala @@ -273,6 +273,19 @@ object DistributionPackage { log.info(s"Executing $enso ${args.mkString(" ")}") val pb = new java.lang.ProcessBuilder() val all = new java.util.ArrayList[String]() + val disablePrivateCheck = { + val findRun = args.indexOf("--run") + if (findRun >= 0 && findRun + 1 < args.size) { + val whatToRun = args(findRun + 1) + if (whatToRun.startsWith("test/") && whatToRun.endsWith("_Tests")) { + whatToRun.contains("_Internal_") + } else { + false + } + } else { + false + } + } all.add(enso.getAbsolutePath()) all.addAll(args.asJava) pb.command(all) @@ -282,6 +295,9 @@ object DistributionPackage { } else { pb.environment().put("JAVA_OPTS", "-ea") } + if (disablePrivateCheck) { + all.add("--disable-private-check") + } pb.inheritIO() val p = pb.start() val exitCode = p.waitFor() diff --git a/test/Base_Internal_Tests/README.md b/test/Base_Internal_Tests/README.md new file mode 100644 index 000000000000..69382c8a07b6 --- /dev/null +++ b/test/Base_Internal_Tests/README.md @@ -0,0 +1,2 @@ +This is a set of tests for the `Base` library for Enso that need access +to library internals. Run with `--disable-private-checks`. diff --git a/test/Base_Internal_Tests/package.yaml b/test/Base_Internal_Tests/package.yaml new file mode 100644 index 000000000000..f0d6318995ce --- /dev/null +++ b/test/Base_Internal_Tests/package.yaml @@ -0,0 +1,9 @@ +name: Base_Internal_Tests +namespace: enso_dev +enso-version: default +version: 0.0.1 +license: MIT +author: enso-dev@enso.org +maintainer: enso-dev@enso.org +# Base_Internal_Tests may import stuff from Helpers sibling project. So we need this property here. +prefer-local-libraries: true diff --git a/test/Base_Tests/src/Data/Ordering/Comparator_Spec.enso b/test/Base_Internal_Tests/src/Comparator_Spec.enso similarity index 96% rename from test/Base_Tests/src/Data/Ordering/Comparator_Spec.enso rename to test/Base_Internal_Tests/src/Comparator_Spec.enso index ab0010b22a99..3ca9a40f24af 100644 --- a/test/Base_Tests/src/Data/Ordering/Comparator_Spec.enso +++ b/test/Base_Internal_Tests/src/Comparator_Spec.enso @@ -1,12 +1,10 @@ from Standard.Base import all import Standard.Base.Errors.Common.Incomparable_Values -polyglot java import org.enso.base.ObjectComparator - -polyglot java import org.enso.base.CompareException - from Standard.Test import all +import Standard.Base.Internal.Extra_Imports.ObjectComparator +import Standard.Base.Internal.Extra_Imports.CompareException # === Test Resources === diff --git a/test/Base_Tests/src/Data/Text/Utils_Spec.enso b/test/Base_Internal_Tests/src/Grapheme_Spec.enso similarity index 96% rename from test/Base_Tests/src/Data/Text/Utils_Spec.enso rename to test/Base_Internal_Tests/src/Grapheme_Spec.enso index d0b987bea246..c1a24dbc507b 100644 --- a/test/Base_Tests/src/Data/Text/Utils_Spec.enso +++ b/test/Base_Internal_Tests/src/Grapheme_Spec.enso @@ -1,14 +1,12 @@ from Standard.Base import all -polyglot java import org.enso.base.Text_Utils -polyglot java import org.enso.base.text.CaseFoldedString -polyglot java import com.ibm.icu.text.BreakIterator +import Standard.Base.Internal.Extra_Imports.Text_Utils +import Standard.Base.Internal.Extra_Imports.CaseFoldedString +import Standard.Base.Internal.Extra_Imports.BreakIterator +import Standard.Base.Internal.Extra_Imports.Exception as JException from Standard.Test import all - -polyglot java import java.lang.Exception as JException - add_specs suite_builder = suite_builder.group "Text_Utils" group_builder-> kshi = '\u0915\u094D\u0937\u093F' diff --git a/test/Base_Internal_Tests/src/Input_Output_Spec.enso b/test/Base_Internal_Tests/src/Input_Output_Spec.enso new file mode 100644 index 000000000000..b2a8f12ae2bd --- /dev/null +++ b/test/Base_Internal_Tests/src/Input_Output_Spec.enso @@ -0,0 +1,86 @@ +from Standard.Base import all + +from Standard.Test import all +import Standard.Test.Test_Environment + +import Standard.Base.Errors.Illegal_State.Illegal_State +import Standard.Base.Runtime.Managed_Resource.Managed_Resource +import Standard.Base.System.Input_Stream.Input_Stream + +polyglot java import org.enso.base_test_helpers.RangeStream +import Standard.Base.Internal.Extra_Imports.Stream_Utils +import Standard.Base.Internal.Extra_Imports.File_Utils + + +add_specs suite_builder = + suite_builder.group "I/O Checks" group_builder-> + group_builder.specify "read without available" <| + stream_like = is_like [20, 5, 1, 10] False + is = Stream_Utils.asInputStream stream_like + is.available . should_equal 0 + + is.read . should_equal 20 + is.read . should_equal 5 + + is.available . should_equal 0 + + is.read . should_equal 1 + is.read . should_equal 10 + + is.available . should_equal 0 + + group_builder.specify "read with available" <| + stream_like = is_like [20, 6, 8, 23] True + is = Stream_Utils.asInputStream stream_like + is.available . should_equal 4 + + is.read . should_equal 20 + is.read . should_equal 6 + + is.available . should_equal 2 + + is.read . should_equal 8 + is.read . should_equal 23 + + is.available . should_equal 0 + + group_builder.specify "File in Dictionary supports host objects as keys" <| + # java.nio.path.Path has proper implementation of hashCode + dict = Dictionary.singleton (File_Utils.toPath "/home/user/file.txt") 42 + dict.get "X" . should_equal Nothing + dict.get "A" . should_equal Nothing + dict.get (File_Utils.toPath "/home/user/file.txt") . should_equal 42 + + group_builder.specify "File equals on host values" <| + path1 = File_Utils.toPath "home" . resolve "user" . resolve "file.txt" + path2 = File_Utils.toPath "home" . resolve "user" . resolve "file.txt" + (path1 == path2).should_be_true + path3 = path1.resolve "subfile.txt" + (path3 == path2).should_be_false + + + suite_builder.group "Internal comparators" group_builder-> + +foreign js is_like data available = """ + let at = 0 + let is = { + read : function(arr, off, len) { + let cnt = 0; + while (len-- > 0) { + arr[off++] = data[at++]; + cnt++; + } + return cnt; + } + } + if (available) { + is.available = function() { + return data.length - at; + }; + } + return is; + +main filter=Nothing = + suite = Test.build suite_builder-> + add_specs suite_builder + suite.run_with_filter filter diff --git a/test/Base_Internal_Tests/src/Main.enso b/test/Base_Internal_Tests/src/Main.enso new file mode 100644 index 000000000000..47d57de1c006 --- /dev/null +++ b/test/Base_Internal_Tests/src/Main.enso @@ -0,0 +1,15 @@ +from Standard.Base import all + +from Standard.Test import all + +import project.Input_Output_Spec +import project.Comparator_Spec +import project.Grapheme_Spec + +main filter=Nothing = + suite = Test.build suite_builder-> + Input_Output_Spec.add_specs suite_builder + Comparator_Spec.add_specs suite_builder + Grapheme_Spec.add_specs suite_builder + + suite.run_with_filter filter diff --git a/test/Base_Tests/src/Data/Dictionary_Spec.enso b/test/Base_Tests/src/Data/Dictionary_Spec.enso index a8b28e654435..ef612ddcb67f 100644 --- a/test/Base_Tests/src/Data/Dictionary_Spec.enso +++ b/test/Base_Tests/src/Data/Dictionary_Spec.enso @@ -5,7 +5,6 @@ import Standard.Base.Errors.No_Such_Key.No_Such_Key from Standard.Test import all polyglot java import java.util.Map as JavaMap -polyglot java import org.enso.base.file_system.File_Utils ## Type that violates reflexivity type My_Nan @@ -276,13 +275,6 @@ add_specs suite_builder = dict.get "A" . should_equal 42 dict.get (js_str "A") . should_equal 42 - group_builder.specify "should support host objects as keys" <| - # java.nio.path.Path has proper implementation of hashCode - dict = Dictionary.singleton (File_Utils.toPath "/home/user/file.txt") 42 - dict.get "X" . should_equal Nothing - dict.get "A" . should_equal Nothing - dict.get (File_Utils.toPath "/home/user/file.txt") . should_equal 42 - group_builder.specify "should support Python objects as keys" pending=pending_python_missing <| py_obj = py_wrapper 42 dict = Dictionary.singleton py_obj "Value" diff --git a/test/Base_Tests/src/Main.enso b/test/Base_Tests/src/Main.enso index 30ef067b105b..a12834f9fdb6 100644 --- a/test/Base_Tests/src/Main.enso +++ b/test/Base_Tests/src/Main.enso @@ -39,7 +39,6 @@ import project.Data.Locale_Spec import project.Data.Maybe_Spec import project.Data.Numbers_Spec import project.Data.Ordering_Spec -import project.Data.Ordering.Comparator_Spec import project.Data.Ordering.Natural_Order_Spec import project.Data.Ordering.Vector_Lexicographic_Order_Spec import project.Data.Pair_Spec @@ -57,7 +56,6 @@ import project.Data.Text.Encoding_Spec import project.Data.Text.Parse_Spec import project.Data.Text.Regex_Spec import project.Data.Text.Span_Spec -import project.Data.Text.Utils_Spec import project.Data.XML.XML_Spec import project.Data.Vector.Slicing_Helpers_Spec @@ -137,7 +135,6 @@ main filter=Nothing = Numbers_Spec.add_specs suite_builder Equals_Spec.add_specs suite_builder Ordering_Spec.add_specs suite_builder - Comparator_Spec.add_specs suite_builder Natural_Order_Spec.add_specs suite_builder Vector_Lexicographic_Order_Spec.add_specs suite_builder Process_Spec.add_specs suite_builder @@ -163,7 +160,6 @@ main filter=Nothing = Missing_Required_Arguments_Spec.add_specs suite_builder Lazy_Generator_Spec.add_specs suite_builder Stack_Traces_Spec.add_specs suite_builder - Utils_Spec.add_specs suite_builder Text_Spec.add_specs suite_builder Time_Spec.add_specs suite_builder URI_Spec.add_specs suite_builder diff --git a/test/Base_Tests/src/Semantic/Equals_Spec.enso b/test/Base_Tests/src/Semantic/Equals_Spec.enso index 9210fbd178e1..e9385cfdc1c7 100644 --- a/test/Base_Tests/src/Semantic/Equals_Spec.enso +++ b/test/Base_Tests/src/Semantic/Equals_Spec.enso @@ -179,13 +179,6 @@ add_specs suite_builder = ((CustomEqType.C1 10) == (CustomEqType.C2 7 3)).should_be_true ((CustomEqType.C1 0) == (CustomEqType.C2 7 3)).should_be_false - group_builder.specify "should dispatch to equals on host values" <| - path1 = File_Utils.toPath "home" . resolve "user" . resolve "file.txt" - path2 = File_Utils.toPath "home" . resolve "user" . resolve "file.txt" - (path1 == path2).should_be_true - path3 = path1.resolve "subfile.txt" - (path3 == path2).should_be_false - group_builder.specify "should return False for different Atoms with same fields" <| rect = Rect.Value (Point.Value 1 2) (Point.Value 3 4) four_field = FourFieldType.Value 1 2 3 4 diff --git a/test/Base_Tests/src/System/Input_Stream_Spec.enso b/test/Base_Tests/src/System/Input_Stream_Spec.enso index 8121dbf587e6..e32edc8e8c2b 100644 --- a/test/Base_Tests/src/System/Input_Stream_Spec.enso +++ b/test/Base_Tests/src/System/Input_Stream_Spec.enso @@ -14,24 +14,6 @@ main filter=Nothing = suite.run_with_filter filter -foreign js is_like data available = """ - let at = 0 - let is = { - read : function(arr, off, len) { - let cnt = 0; - while (len-- > 0) { - arr[off++] = data[at++]; - cnt++; - } - return cnt; - } - } - if (available) { - is.available = function() { - return data.length - at; - }; - } - return is; add_specs suite_builder = suite_builder.group "Input Stream" group_builder-> group_builder.specify "should be peekable if backed by memory" <| @@ -83,32 +65,3 @@ add_specs suite_builder = suite_builder.group "Input Stream" group_builder-> # The read still succeeds - ensuring there isn't some early EOF promoted_stream.read_n_bytes 10 . should_equal [100, 101, 102, 103, 104] - group_builder.specify "read without available" <| - stream_like = is_like [20, 5, 1, 10] False - is = Stream_Utils.asInputStream stream_like - is.available . should_equal 0 - - is.read . should_equal 20 - is.read . should_equal 5 - - is.available . should_equal 0 - - is.read . should_equal 1 - is.read . should_equal 10 - - is.available . should_equal 0 - - group_builder.specify "read with available" <| - stream_like = is_like [20, 6, 8, 23] True - is = Stream_Utils.asInputStream stream_like - is.available . should_equal 4 - - is.read . should_equal 20 - is.read . should_equal 6 - - is.available . should_equal 2 - - is.read . should_equal 8 - is.read . should_equal 23 - - is.available . should_equal 0 From bbe754c690a2c870be7cfa9df45c592108d23ce8 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 1 Aug 2024 05:56:08 +0200 Subject: [PATCH 17/89] No more of Polyglot.Java --- distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso | 1 - .../micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso | 1 - 2 files changed, 2 deletions(-) diff --git a/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso b/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso index c825197549b8..563b217f9076 100644 --- a/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso +++ b/distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso @@ -169,7 +169,6 @@ export project.Nothing.Nothing export project.Panic.Panic -export project.Polyglot.Java export project.Polyglot.Polyglot export project.Random.Random diff --git a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso index 247d1defdcae..f7e519c17090 100644 --- a/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso +++ b/test/micro-distribution/lib/Standard/Base/0.0.0-dev/src/Main.enso @@ -13,7 +13,6 @@ export project.Data.Numbers.Number export project.Data.Numbers.Integer export project.Data.Vector.Vector export project.Function.Function -export project.Polyglot.Java export project.Polyglot.Polyglot export project.Runtime export project.Warning From 1514ad247f4a1d69526fb3f5aafe34adfaac4ae2 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 1 Aug 2024 08:12:05 +0200 Subject: [PATCH 18/89] Nicer formatting with var --- .../interpreter/runtime/EnsoClassPath.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index c491082355d8..f64842de1076 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -25,17 +25,14 @@ static EnsoClassPath create(Path file) { if (moduleNames.isEmpty()) { return EMPTY; } else { - java.lang.ModuleLayer parent = ModuleLayer.boot(); - java.util.List parentCfgs = - Collections.singletonList(parent.configuration()); - java.util.List parentModules = Collections.singletonList(parent); - java.lang.ClassLoader parentLoader = parent.findLoader("java.base"); - java.lang.module.Configuration cfg = - Configuration.resolve(finder, parentCfgs, finder, moduleNames); - java.lang.ModuleLayer.Controller cntrl = - ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); - java.lang.ModuleLayer layer = cntrl.layer(); - java.lang.ClassLoader loader = layer.findLoader(moduleNames.get(0)); + var parent = ModuleLayer.boot(); + var parentCfgs = Collections.singletonList(parent.configuration()); + var parentModules = Collections.singletonList(parent); + var parentLoader = parent.findLoader("java.base"); + var cfg = Configuration.resolve(finder, parentCfgs, finder, moduleNames); + var cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); + var layer = cntrl.layer(); + var loader = layer.findLoader(moduleNames.get(0)); return new EnsoClassPath(cntrl, layer, loader); } } From 09c69464fe109933d0a2417b2f387fbd8851ffdb Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 1 Aug 2024 10:45:08 +0200 Subject: [PATCH 19/89] Allow Standard.Table to depend on modules from Standard.Base --- build.sbt | 19 +++++++------- .../Standard/Database/0.0.0-dev/package.yaml | 3 +++ .../lib/Standard/Table/0.0.0-dev/package.yaml | 2 ++ .../src/main/java/module-info.java | 2 ++ .../interpreter/runtime/EnsoClassPath.java | 26 ++++++++++++++----- .../enso/interpreter/runtime/EnsoContext.java | 18 ++++++++++++- .../src/main/scala/org/enso/pkg/Config.scala | 18 +++++++++++-- .../src/main/scala/org/enso/pkg/Package.scala | 3 ++- std-bits/table/src/main/java/module-info.java | 10 ++++++- .../enso/table/read/DelimitedFormatSPI.java | 3 +-- .../org/enso/table/read/ExcelFormatSPI.java | 3 +-- 11 files changed, 81 insertions(+), 26 deletions(-) diff --git a/build.sbt b/build.sbt index a0f3e7fab158..4adf73afe097 100644 --- a/build.sbt +++ b/build.sbt @@ -3442,16 +3442,15 @@ lazy val `std-table` = project modulePath := { val updateReport = (Compile / update).value val requiredModIds = Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.antlr" % "antlr4-runtime" % antlrVersion, - "com.univocity" % "univocity-parsers" % univocityParsersVersion, - "org.apache.poi" % "poi" % poiOoxmlVersion, - "org.apache.poi" % "poi-ooxml" % poiOoxmlVersion, - "org.apache.poi" % "poi-ooxml-lite" % poiOoxmlVersion, - "org.apache.xmlbeans" % "xmlbeans" % xmlbeansVersion, - "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, - "com.ibm.icu" % "icu4j" % icuVersion, - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.antlr" % "antlr4-runtime" % antlrVersion, + "com.univocity" % "univocity-parsers" % univocityParsersVersion, + "org.apache.poi" % "poi" % poiOoxmlVersion, + "org.apache.poi" % "poi-ooxml" % poiOoxmlVersion, + "org.apache.poi" % "poi-ooxml-lite" % poiOoxmlVersion, + "org.apache.xmlbeans" % "xmlbeans" % xmlbeansVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "com.ibm.icu" % "icu4j" % icuVersion ) val externalRequiredMods = JPMSUtils.filterModulesFromUpdate( updateReport, diff --git a/distribution/lib/Standard/Database/0.0.0-dev/package.yaml b/distribution/lib/Standard/Database/0.0.0-dev/package.yaml index d6a92a2b8338..c959d0381154 100644 --- a/distribution/lib/Standard/Database/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Database/0.0.0-dev/package.yaml @@ -8,3 +8,6 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Base + - Standard.Table diff --git a/distribution/lib/Standard/Table/0.0.0-dev/package.yaml b/distribution/lib/Standard/Table/0.0.0-dev/package.yaml index 9fda257cdb0d..88af7cb11961 100644 --- a/distribution/lib/Standard/Table/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Table/0.0.0-dev/package.yaml @@ -8,3 +8,5 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Base diff --git a/engine/runtime-fat-jar/src/main/java/module-info.java b/engine/runtime-fat-jar/src/main/java/module-info.java index 1f5b2eee274d..1cdfee23bc47 100644 --- a/engine/runtime-fat-jar/src/main/java/module-info.java +++ b/engine/runtime-fat-jar/src/main/java/module-info.java @@ -38,4 +38,6 @@ // Could not resolve type id 'org.enso.polyglot.data.Tree$Node' as a subtype of // `org.enso.polyglot.data.Tree$Node`: Not a subtype requires java.se; + // needed by org.apache.xmlbeans + requires jdk.xml.dom; } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index f64842de1076..e506c5bd2689 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -4,6 +4,7 @@ import java.lang.module.ModuleFinder; import java.nio.file.Path; import java.util.Collections; +import java.util.List; /** Representation of an Enso library class path. */ public final class EnsoClassPath { @@ -18,19 +19,30 @@ private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoad this.loader = loader; } - static EnsoClassPath create(Path file) { + static EnsoClassPath create(Path file, List parents) { java.lang.module.ModuleFinder finder = ModuleFinder.of(file); java.util.List moduleNames = finder.findAll().stream().map(mod -> mod.descriptor().name()).toList(); if (moduleNames.isEmpty()) { return EMPTY; } else { - var parent = ModuleLayer.boot(); - var parentCfgs = Collections.singletonList(parent.configuration()); - var parentModules = Collections.singletonList(parent); - var parentLoader = parent.findLoader("java.base"); - var cfg = Configuration.resolve(finder, parentCfgs, finder, moduleNames); - var cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); + ModuleLayer.Controller cntrl; + if (parents.isEmpty()) { + var parent = ModuleLayer.boot(); + var parentLoader = parent.findLoader("java.base"); + var parentCfgs = Collections.singletonList(parent.configuration()); + var parentModules = Collections.singletonList(parent); + var cfg = + Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); + cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); + } else { + var parentCfgs = parents.stream().map(cp -> cp.layer.configuration()).toList(); + var parentLayers = parents.stream().map(cp -> cp.layer).toList(); + var parentLoader = ModuleLayer.boot().findLoader("java.base"); + var cfg = + Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); + cntrl = ModuleLayer.defineModulesWithManyLoaders(cfg, parentLayers, parentLoader); + } var layer = cntrl.layer(); var loader = layer.findLoader(moduleNames.get(0)); return new EnsoClassPath(cntrl, layer, loader); diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 7f08301b4037..8dafe37f9158 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -463,7 +463,23 @@ public synchronized EnsoClassPath findClassPath(Package pkg) { org.enso.interpreter.runtime.EnsoClassPath l = classPaths.get(pkg); if (l == null) { var ch = pkg.polyglotDir().resolve("java"); - l = EnsoClassPath.create(Paths.get(ch.toUri())); + var requires = Arrays.asList(pkg.getConfig().requires()); + var parents = + requires.stream() + .map( + n -> { + var name = LibraryName.fromModuleName(n); + if (name.isDefined()) { + var ln = name.get(); + var lib = this.packageRepository.getPackageForLibrary(ln); + if (lib.isDefined()) { + return findClassPath(lib.get()); + } + } + return findClassPath(null); + }) + .toList(); + l = EnsoClassPath.create(Paths.get(ch.toUri()), parents); classPaths.put(pkg, l); } return l; diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala index 827bdf35b953..4ef4daa9b117 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala @@ -109,7 +109,8 @@ case class Config( maintainers: List[Contact], edition: Option[Editions.RawEdition], preferLocalLibraries: Boolean, - componentGroups: Option[ComponentGroups] + componentGroups: Option[ComponentGroups], + requires: Array[String] ) { /** Converts the configuration into a YAML representation. */ @@ -144,6 +145,7 @@ object Config { val Author: String = "authors" val Namespace: String = "namespace" val Maintainer: String = "maintainers" + val Requires: String = "requires" val Edition: String = "edition" val PreferLocalLibraries = "prefer-local-libraries" val ComponentGroups = "component-groups" @@ -158,6 +160,7 @@ object Config { val normalizedNameDecoder = implicitly[YamlDecoder[Option[String]]] val contactDecoder = implicitly[YamlDecoder[List[Contact]]] + val requiresDecoder = implicitly[YamlDecoder[List[String]]] val editionNameDecoder = implicitly[YamlDecoder[EditionName]] val editionDecoder = implicitly[YamlDecoder[Option[Editions.RawEdition]]] @@ -195,6 +198,10 @@ object Config { .get(JsonFields.Maintainer) .map(contactDecoder.decode) .getOrElse(Right(Nil)) + requires <- clazzMap + .get(JsonFields.Requires) + .map(requiresDecoder.decode) + .getOrElse(Right(Nil)) rawEdition = clazzMap .get(JsonFields.Edition) .flatMap(x => editionNameDecoder.decode(x).toOption.map(Left(_))) @@ -230,7 +237,8 @@ object Config { maintainers, edition, preferLocalLibraries, - componentGroups + componentGroups, + requires.toArray ) } } @@ -239,6 +247,7 @@ object Config { new YamlEncoder[Config] { override def encode(value: Config) = { val contactsEncoder = implicitly[YamlEncoder[List[Contact]]] + val requiresEncoder = implicitly[YamlEncoder[List[String]]] val editionEncoder = implicitly[YamlEncoder[Editions.RawEdition]] val booleanEncoder = implicitly[YamlEncoder[Boolean]] val componentGroupsEncoder = @@ -268,6 +277,11 @@ object Config { (JsonFields.Maintainer, contactsEncoder.encode(value.maintainers)) ) } + if (value.requires.nonEmpty) { + elements.add( + (JsonFields.Requires, requiresEncoder.encode(value.requires.toList)) + ) + } value.edition.foreach { edition => if (edition.isDerivingWithoutOverrides) diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala index a4ff24d07c81..d368341d145b 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala @@ -306,7 +306,8 @@ class PackageManager[F](implicit val fileSystem: FileSystem[F]) { edition = edition, preferLocalLibraries = true, maintainers = maintainers, - componentGroups = componentGroups + componentGroups = componentGroups, + requires = Array() ) create(root, config, template) } diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index abb14a0b6cec..e188becb5f09 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -1,5 +1,10 @@ +import org.enso.base.file_format.FileFormatSPI; +import org.enso.table.read.DelimitedFormatSPI; +import org.enso.table.read.ExcelFormatSPI; + module enso.std.table { requires java.xml; + requires jdk.xml.dom; requires com.ibm.icu; requires org.antlr.antlr4.runtime; requires org.apache.poi.ooxml; @@ -8,8 +13,11 @@ requires org.apache.xmlbeans; requires org.graalvm.collections; requires org.graalvm.polyglot; - requires org.openide.util.lookup.RELEASE180; requires org.enso.polyglot.common_utils; requires org.enso.std.base; requires univocity.parsers; + + provides FileFormatSPI with + ExcelFormatSPI, + DelimitedFormatSPI; } diff --git a/std-bits/table/src/main/java/org/enso/table/read/DelimitedFormatSPI.java b/std-bits/table/src/main/java/org/enso/table/read/DelimitedFormatSPI.java index c4ddbbda9a42..220624e14649 100644 --- a/std-bits/table/src/main/java/org/enso/table/read/DelimitedFormatSPI.java +++ b/std-bits/table/src/main/java/org/enso/table/read/DelimitedFormatSPI.java @@ -2,8 +2,7 @@ import org.enso.base.file_format.FileFormatSPI; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class DelimitedFormatSPI extends FileFormatSPI { +public final class DelimitedFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Table.Delimited.Delimited_Format"; diff --git a/std-bits/table/src/main/java/org/enso/table/read/ExcelFormatSPI.java b/std-bits/table/src/main/java/org/enso/table/read/ExcelFormatSPI.java index 3f554a2780a0..4256a92aefbb 100644 --- a/std-bits/table/src/main/java/org/enso/table/read/ExcelFormatSPI.java +++ b/std-bits/table/src/main/java/org/enso/table/read/ExcelFormatSPI.java @@ -2,8 +2,7 @@ import org.enso.base.file_format.FileFormatSPI; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) -public class ExcelFormatSPI extends FileFormatSPI { +public final class ExcelFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { return "Standard.Table.Excel.Excel_Format"; From 3148976a7af4fbd9b15f8ff2f8e45a6cfba272e8 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 1 Aug 2024 14:55:29 +0200 Subject: [PATCH 20/89] language-server/test compiles again --- .../languageserver/libraries/ComponentGroupsResolverSpec.scala | 2 ++ .../languageserver/libraries/ComponentGroupsValidatorSpec.scala | 1 + 2 files changed, 3 insertions(+) diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala index f5de8b56a05f..e7f6d31a2375 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala @@ -278,6 +278,7 @@ object ComponentGroupsResolverSpec { license = "", authors = Nil, maintainers = Nil, + requires = Array(), edition = None, preferLocalLibraries = true, componentGroups = Some(componentGroups) @@ -296,6 +297,7 @@ object ComponentGroupsResolverSpec { license = "", authors = Nil, maintainers = Nil, + requires = Array(), edition = None, preferLocalLibraries = true, componentGroups = None diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala index f8b7aa7b580c..ff72f86f9822 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala @@ -188,6 +188,7 @@ object ComponentGroupsValidatorSpec { authors = Nil, maintainers = Nil, edition = None, + requires = Array(), preferLocalLibraries = true, componentGroups = Some( ComponentGroups( From ae64b8acd4780a181dd28162ec9a2abecb767616 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 5 Aug 2024 08:21:34 +0200 Subject: [PATCH 21/89] Host Files values group_builder --- test/Base_Internal_Tests/src/Input_Output_Spec.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Base_Internal_Tests/src/Input_Output_Spec.enso b/test/Base_Internal_Tests/src/Input_Output_Spec.enso index b2a8f12ae2bd..045094fbf9a3 100644 --- a/test/Base_Internal_Tests/src/Input_Output_Spec.enso +++ b/test/Base_Internal_Tests/src/Input_Output_Spec.enso @@ -44,6 +44,7 @@ add_specs suite_builder = is.available . should_equal 0 + suite_builder.group "Host File values behaviour in Enso" group_builder-> group_builder.specify "File in Dictionary supports host objects as keys" <| # java.nio.path.Path has proper implementation of hashCode dict = Dictionary.singleton (File_Utils.toPath "/home/user/file.txt") 42 @@ -59,7 +60,6 @@ add_specs suite_builder = (path3 == path2).should_be_false - suite_builder.group "Internal comparators" group_builder-> foreign js is_like data available = """ let at = 0 From cfdda7b9d5719ec51c4b9f1e3d0009d033515c42 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 26 Sep 2024 14:27:04 +0200 Subject: [PATCH 22/89] Using moduleDependencies and internalModuleDependencies for standard libraries --- build.sbt | 158 +++++++----------- std-bits/base/src/main/java/module-info.java | 3 +- .../database/src/main/java/module-info.java | 2 +- std-bits/table/src/main/java/module-info.java | 4 +- 4 files changed, 61 insertions(+), 106 deletions(-) diff --git a/build.sbt b/build.sbt index 3a80ca75a609..1e4325d79919 100644 --- a/build.sbt +++ b/build.sbt @@ -4353,29 +4353,17 @@ lazy val `std-base` = project "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion ), - modulePath := { - val updateReport = (Compile / update).value - val requiredModIds = Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, - "com.ibm.icu" % "icu4j" % icuVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion - ) - val externalRequiredMods = JPMSUtils.filterModulesFromUpdate( - updateReport, - requiredModIds, - streams.value.log, - moduleName.value, - scalaBinaryVersion.value, - shouldContainAll = true - ) - val ourRequiredMods = - (`common-polyglot-core-utils` / Compile / exportedProducts).value - .map(_.data) - externalRequiredMods ++ ourRequiredMods - }, + Compile / moduleDependencies ++= Seq( + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "com.ibm.icu" % "icu4j" % icuVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion + ), + Compile / internalModuleDependencies := Seq( + (`common-polyglot-core-utils` / Compile / exportedModule).value + ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ensureCoreIsCompiled = @@ -4502,34 +4490,21 @@ lazy val `std-table` = project "org.antlr" % "antlr4-runtime" % antlrVersion, "org.apache.logging.log4j" % "log4j-to-slf4j" % "2.18.0" // org.apache.poi uses log4j ), - modulePath := { - val updateReport = (Compile / update).value - val requiredModIds = Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, - "org.antlr" % "antlr4-runtime" % antlrVersion, - "com.univocity" % "univocity-parsers" % univocityParsersVersion, - "org.apache.poi" % "poi" % poiOoxmlVersion, - "org.apache.poi" % "poi-ooxml" % poiOoxmlVersion, - "org.apache.poi" % "poi-ooxml-lite" % poiOoxmlVersion, - "org.apache.xmlbeans" % "xmlbeans" % xmlbeansVersion, - "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, - "com.ibm.icu" % "icu4j" % icuVersion - ) - val externalRequiredMods = JPMSUtils.filterModulesFromUpdate( - updateReport, - requiredModIds, - streams.value.log, - moduleName.value, - scalaBinaryVersion.value, - shouldContainAll = true - ) - val ourRequiredMods = - (`common-polyglot-core-utils` / Compile / exportedProducts).value - .map(_.data) ++ - (`std-base` / Compile / exportedProducts).value - .map(_.data) - externalRequiredMods ++ ourRequiredMods - }, + Compile / moduleDependencies ++= Seq( + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.antlr" % "antlr4-runtime" % antlrVersion, + "com.univocity" % "univocity-parsers" % univocityParsersVersion, + "org.apache.poi" % "poi" % poiOoxmlVersion, + "org.apache.poi" % "poi-ooxml" % poiOoxmlVersion, + "org.apache.poi" % "poi-ooxml-lite" % poiOoxmlVersion, + "org.apache.xmlbeans" % "xmlbeans" % xmlbeansVersion, + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "com.ibm.icu" % "icu4j" % icuVersion + ), + Compile / internalModuleDependencies := Seq( + (`common-polyglot-core-utils` / Compile / exportedModule).value, + (`std-base` / Compile / exportedModule).value + ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits @@ -4613,32 +4588,21 @@ lazy val `std-database` = project Compile / packageBin / artifactPath := `database-polyglot-root` / "std-database.jar", libraryDependencies ++= Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "org.xerial" % "sqlite-jdbc" % sqliteVersion, - "org.postgresql" % "postgresql" % postgresVersion + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "org.xerial" % "sqlite-jdbc" % sqliteVersion, + "org.postgresql" % "postgresql" % postgresVersion + ), + Compile / moduleDependencies ++= Seq( + "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, + "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion + ), + Compile / internalModuleDependencies := Seq( + (`common-polyglot-core-utils` / Compile / exportedModule).value, + (`std-base` / Compile / exportedModule).value ), - modulePath := { - val externalModIds = libraryDependencies.value ++ Seq( - "org.graalvm.sdk" % "collections" % graalMavenPackagesVersion, - "com.fasterxml.jackson.core" % "jackson-databind" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-annotations" % jacksonVersion, - "com.fasterxml.jackson.core" % "jackson-core" % jacksonVersion - ) - val externalMods = JPMSUtils.filterModulesFromUpdate( - (Compile / update).value, - externalModIds, - streams.value.log, - moduleName.value, - scalaBinaryVersion.value, - shouldContainAll = true - ) - val ourMods = - (`std-base` / Compile / exportedProducts).value.map(_.data) ++ - (`common-polyglot-core-utils` / Compile / exportedProducts).value - .map(_.data) - externalMods ++ ourMods - }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits @@ -4676,32 +4640,22 @@ lazy val `std-aws` = project "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version ), - modulePath := { - val externalModIds = Seq( - "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, - "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version - ) - val externalMods = JPMSUtils.filterModulesFromUpdate( - (Compile / update).value, - externalModIds, - streams.value.log, - moduleName.value, - scalaBinaryVersion.value, - shouldContainAll = true - ) - val ourMods = - (`std-base` / Compile / exportedProducts).value.map(_.data) ++ - (`std-database` / Compile / exportedProducts).value.map(_.data) - externalMods ++ ourMods - }, + Compile / moduleDependencies ++= Seq( + "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version + ), + Compile / internalModuleDependencies := Seq( + (`std-base` / Compile / exportedModule).value, + (`std-database` / Compile / exportedModule).value + ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index e82a8a168aeb..5eb9f1dcfdfd 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -18,7 +18,7 @@ requires com.ibm.icu; requires org.graalvm.polyglot; requires com.fasterxml.jackson.databind; - requires org.enso.polyglot.common_utils; + requires org.enso.common.polyglot.core.utils; uses FileSystemSPI; uses FileFormatSPI; @@ -30,6 +30,7 @@ exports org.enso.base.file_format; exports org.enso.base.enso_cloud; exports org.enso.base.enso_cloud.audit; + exports org.enso.base.numeric; exports org.enso.base.polyglot; exports org.enso.base.time; exports org.enso.base.text; diff --git a/std-bits/database/src/main/java/module-info.java b/std-bits/database/src/main/java/module-info.java index 701f17a52760..5f5164e0f881 100644 --- a/std-bits/database/src/main/java/module-info.java +++ b/std-bits/database/src/main/java/module-info.java @@ -11,7 +11,7 @@ requires java.logging; requires java.sql; requires org.enso.std.base; - requires org.enso.polyglot.common_utils; + requires org.enso.common.polyglot.core.utils; requires org.graalvm.collections; requires org.graalvm.polyglot; requires com.fasterxml.jackson.databind; diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index e188becb5f09..ab6790691c0a 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -2,7 +2,7 @@ import org.enso.table.read.DelimitedFormatSPI; import org.enso.table.read.ExcelFormatSPI; -module enso.std.table { +module org.enso.std.table { requires java.xml; requires jdk.xml.dom; requires com.ibm.icu; @@ -13,7 +13,7 @@ requires org.apache.xmlbeans; requires org.graalvm.collections; requires org.graalvm.polyglot; - requires org.enso.polyglot.common_utils; + requires org.enso.common.polyglot.core.utils; requires org.enso.std.base; requires univocity.parsers; From d50f0069ed2ad7c4d8364a4acf350926e99378c0 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 26 Sep 2024 14:57:22 +0200 Subject: [PATCH 23/89] There can be Enso sources without associated package --- .../interpreter/runtime/EnsoClassPath.java | 2 +- .../enso/interpreter/runtime/EnsoContext.java | 54 +++++++++++-------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index e506c5bd2689..7bad184e98f2 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -8,7 +8,7 @@ /** Representation of an Enso library class path. */ public final class EnsoClassPath { - private static final EnsoClassPath EMPTY = new EnsoClassPath(null, null, null); + static final EnsoClassPath EMPTY = new EnsoClassPath(null, null, null); private final ModuleLayer.Controller cntrl; private final ModuleLayer layer; final ClassLoader loader; diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index af88d8448852..56b88300b859 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -470,24 +470,32 @@ public Optional findModuleByExpressionId(UUID expressionId) { public synchronized EnsoClassPath findClassPath(Package pkg) { org.enso.interpreter.runtime.EnsoClassPath l = classPaths.get(pkg); if (l == null) { - var ch = pkg.polyglotDir().resolve("java"); - var requires = Arrays.asList(pkg.getConfig().requires()); - var parents = - requires.stream() - .map( - n -> { - var name = LibraryName.fromModuleName(n); - if (name.isDefined()) { - var ln = name.get(); - var lib = this.packageRepository.getPackageForLibrary(ln); - if (lib.isDefined()) { - return findClassPath(lib.get()); + if (pkg != null) { + var polyDir = pkg.polyglotDir(); + if (polyDir == null) { + throw new NullPointerException("No polyglot dir for " + pkg); + } + var ch = polyDir.resolve("java"); + var requires = Arrays.asList(pkg.getConfig().requires()); + var parents = + requires.stream() + .map( + n -> { + var name = LibraryName.fromModuleName(n); + if (name.isDefined()) { + var ln = name.get(); + var lib = this.packageRepository.getPackageForLibrary(ln); + if (lib.isDefined()) { + return findClassPath(lib.get()); + } } - } - return findClassPath(null); - }) - .toList(); - l = EnsoClassPath.create(Paths.get(ch.toUri()), parents); + return findClassPath(null); + }) + .toList(); + l = EnsoClassPath.create(Paths.get(ch.toUri()), parents); + } else { + l = EnsoClassPath.EMPTY; + } classPaths.put(pkg, l); } return l; @@ -574,8 +582,8 @@ public TruffleObject lookupJavaClass(Package pkg, String className) var fqn = binaryName.toString(); try { var hostSymbol = lookupHostSymbol(cp.loader, fqn); - if (hostSymbol != null) { - return (TruffleObject) hostSymbol; + if (hostSymbol instanceof TruffleObject truffleSymbol) { + return truffleSymbol; } } catch (ClassNotFoundException | RuntimeException | InteropException ex) { collectedExceptions.add(ex); @@ -599,10 +607,14 @@ private Object lookupHostSymbol(ClassLoader loader, String fqn) throws ClassNotFoundException, UnknownIdentifierException, UnsupportedMessageException { try { if (findGuestJava() == null) { + Class clazz; if (loader == null) { - throw new ClassNotFoundException(fqn); + var baseModule = Object.class.getModule(); + clazz = Class.forName(baseModule, fqn); + } else { + clazz = loader.loadClass(fqn); } - return environment.asHostSymbol(loader.loadClass(fqn)); + return environment.asHostSymbol(clazz); } else { return InteropLibrary.getUncached().readMember(findGuestJava(), fqn); } From b04ce5c6b158fd0cf612758428f66b6253c639bb Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 06:59:51 +0200 Subject: [PATCH 24/89] Avoid duplicated modules. Use the one from boot layer. --- .../interpreter/runtime/EnsoClassPath.java | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 7bad184e98f2..14e1d37d914f 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -3,6 +3,7 @@ import java.lang.module.Configuration; import java.lang.module.ModuleFinder; import java.nio.file.Path; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -20,12 +21,26 @@ private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoad } static EnsoClassPath create(Path file, List parents) { - java.lang.module.ModuleFinder finder = ModuleFinder.of(file); + java.util.List locations = new ArrayList<>(); + java.util.List moduleNames = - finder.findAll().stream().map(mod -> mod.descriptor().name()).toList(); + ModuleFinder.of(file).findAll().stream() + .filter( + (mod) -> { + if (ModuleLayer.boot().findModule(mod.descriptor().name()).isPresent()) { + return false; + } else { + var uri = mod.location().get(); + locations.add(Path.of(uri)); + return true; + } + }) + .map(mod -> mod.descriptor().name()) + .toList(); if (moduleNames.isEmpty()) { return EMPTY; } else { + var finder = ModuleFinder.of(locations.toArray(new Path[0])); ModuleLayer.Controller cntrl; if (parents.isEmpty()) { var parent = ModuleLayer.boot(); From d94a44db07eb662c7ddb4d18b147b3cffb22b3e2 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 07:47:21 +0200 Subject: [PATCH 25/89] Load all library Java modules with a single class loader --- .../main/java/org/enso/interpreter/runtime/EnsoClassPath.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 14e1d37d914f..26d63c5fdae3 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -56,7 +56,7 @@ static EnsoClassPath create(Path file, List parents) { var parentLoader = ModuleLayer.boot().findLoader("java.base"); var cfg = Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); - cntrl = ModuleLayer.defineModulesWithManyLoaders(cfg, parentLayers, parentLoader); + cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentLayers, parentLoader); } var layer = cntrl.layer(); var loader = layer.findLoader(moduleNames.get(0)); From c71183c4f73ab073aab60be70d215d28b8d0be95 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 07:51:38 +0200 Subject: [PATCH 26/89] Opening std-table.jar for reflection from Enso code --- std-bits/table/src/main/java/module-info.java | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index ab6790691c0a..3cfbef27282d 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -20,4 +20,15 @@ provides FileFormatSPI with ExcelFormatSPI, DelimitedFormatSPI; + + // + // opening up for reflection from Enso code + // + opens org.enso.table.problems; + opens org.enso.table.data.column.storage; + opens org.enso.table.data.column.storage.numeric; + opens org.enso.table.data.column.builder; + opens org.enso.table.data.table; + opens org.enso.table.data.table.problems; + opens org.enso.table.error; } From 9bc7a3ac3e9c9880c929cec274171e090acfcef7 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 08:13:55 +0200 Subject: [PATCH 27/89] Visualization_Tests run and yield just nine errors --- std-bits/database/src/main/java/module-info.java | 3 +++ std-bits/table/src/main/java/module-info.java | 1 + 2 files changed, 4 insertions(+) diff --git a/std-bits/database/src/main/java/module-info.java b/std-bits/database/src/main/java/module-info.java index 5f5164e0f881..d389a3c12fff 100644 --- a/std-bits/database/src/main/java/module-info.java +++ b/std-bits/database/src/main/java/module-info.java @@ -17,6 +17,7 @@ requires com.fasterxml.jackson.databind; uses DatabaseConnectionDetailsSPI; + uses java.sql.Driver; exports org.enso.database; @@ -28,4 +29,6 @@ PostgresDataLinkSPI; provides FileFormatSPI with SQLiteFormatSPI; + + opens org.enso.database.dryrun; } diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index 3cfbef27282d..0e73ddfa2a8d 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -31,4 +31,5 @@ opens org.enso.table.data.table; opens org.enso.table.data.table.problems; opens org.enso.table.error; + opens org.enso.table.data.column.storage.type; } From 16b82c3513046a8042aa947909af5e89d8353092 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 09:15:58 +0200 Subject: [PATCH 28/89] Standard.Test loads classes from Standard.Base --- distribution/lib/Standard/Test/0.0.0-dev/package.yaml | 2 ++ .../java/org/enso/interpreter/runtime/EnsoClassPath.java | 5 +++-- .../base/src/main/java/org/enso/base/Environment_Utils.java | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Test/0.0.0-dev/package.yaml b/distribution/lib/Standard/Test/0.0.0-dev/package.yaml index 6aa554fb2bea..0f3233e77da7 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Test/0.0.0-dev/package.yaml @@ -8,3 +8,5 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Base diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 26d63c5fdae3..2196832c4567 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -37,7 +37,7 @@ static EnsoClassPath create(Path file, List parents) { }) .map(mod -> mod.descriptor().name()) .toList(); - if (moduleNames.isEmpty()) { + if (moduleNames.isEmpty() && parents.isEmpty()) { return EMPTY; } else { var finder = ModuleFinder.of(locations.toArray(new Path[0])); @@ -59,7 +59,8 @@ static EnsoClassPath create(Path file, List parents) { cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentLayers, parentLoader); } var layer = cntrl.layer(); - var loader = layer.findLoader(moduleNames.get(0)); + var loader = + !moduleNames.isEmpty() ? layer.findLoader(moduleNames.get(0)) : parents.get(0).loader; return new EnsoClassPath(cntrl, layer, loader); } } diff --git a/std-bits/base/src/main/java/org/enso/base/Environment_Utils.java b/std-bits/base/src/main/java/org/enso/base/Environment_Utils.java index 8e90645c0e9f..80bce96f90a1 100644 --- a/std-bits/base/src/main/java/org/enso/base/Environment_Utils.java +++ b/std-bits/base/src/main/java/org/enso/base/Environment_Utils.java @@ -2,7 +2,9 @@ import java.util.HashMap; -public class Environment_Utils { +public final class Environment_Utils { + private Environment_Utils() {} + /** Gets the environment variable, including any overrides. */ public static String get_environment_variable(String name) { String override = overrides.get(name); From 405e88de540cfa6ee9c7254724f853825e96bf50 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 09:55:40 +0200 Subject: [PATCH 29/89] Always prefer modules from parent layers --- .../org/enso/interpreter/runtime/EnsoClassPath.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 2196832c4567..60f41a48543d 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -29,11 +29,15 @@ static EnsoClassPath create(Path file, List parents) { (mod) -> { if (ModuleLayer.boot().findModule(mod.descriptor().name()).isPresent()) { return false; - } else { - var uri = mod.location().get(); - locations.add(Path.of(uri)); - return true; } + for (var p : parents) { + if (p.layer.findModule(mod.descriptor().name()).isPresent()) { + return false; + } + } + var uri = mod.location().get(); + locations.add(Path.of(uri)); + return true; }) .map(mod -> mod.descriptor().name()) .toList(); From 57e64e0d69fc332a93664593d765c35ba9ba10b5 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 09:56:05 +0200 Subject: [PATCH 30/89] AWS requires access to Standard.Base and Standard.Database JARs --- distribution/lib/Standard/AWS/0.0.0-dev/package.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/distribution/lib/Standard/AWS/0.0.0-dev/package.yaml b/distribution/lib/Standard/AWS/0.0.0-dev/package.yaml index d9ac9519a586..6fd17c0347a3 100644 --- a/distribution/lib/Standard/AWS/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/AWS/0.0.0-dev/package.yaml @@ -8,3 +8,6 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Base + - Standard.Database From efda30fc6115e1e3a2024090da57d4fd34a171a0 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 1 Oct 2024 12:24:57 +0200 Subject: [PATCH 31/89] Opening up std-table JAR, so Enso can access all Java classes --- std-bits/table/src/main/java/module-info.java | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index 0e73ddfa2a8d..6d7222b42d58 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -24,12 +24,32 @@ // // opening up for reflection from Enso code // + opens org.enso.table.aggregations; + opens org.enso.table.expressions; + opens org.enso.table.formatting; + opens org.enso.table.excel; opens org.enso.table.problems; + opens org.enso.table.parsing; + opens org.enso.table.parsing.problems; opens org.enso.table.data.column.storage; opens org.enso.table.data.column.storage.numeric; opens org.enso.table.data.column.builder; + opens org.enso.table.data.column.operation; + opens org.enso.table.data.column.operation.cast; + opens org.enso.table.data.column.operation.map; + opens org.enso.table.data.column.operation.unary; + opens org.enso.table.data.column.storage.type; + opens org.enso.table.data.mask; + opens org.enso.table.data.index; opens org.enso.table.data.table; opens org.enso.table.data.table.problems; + opens org.enso.table.data.table.join; + opens org.enso.table.data.table.join.conditions; + opens org.enso.table.data.table.join.lookup; opens org.enso.table.error; - opens org.enso.table.data.column.storage.type; + opens org.enso.table.read; + opens org.enso.table.operations; + opens org.enso.table.util; + opens org.enso.table.util.problems; + opens org.enso.table.write; } From e9a045666b975d97a17ae48f607d16d06d81f0d0 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 2 Oct 2024 06:27:44 +0200 Subject: [PATCH 32/89] Less functional boilerplate --- .../interpreter/runtime/EnsoClassPath.java | 36 +++++++++---------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 60f41a48543d..c1e69c886926 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -21,26 +21,22 @@ private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoad } static EnsoClassPath create(Path file, List parents) { - java.util.List locations = new ArrayList<>(); - - java.util.List moduleNames = - ModuleFinder.of(file).findAll().stream() - .filter( - (mod) -> { - if (ModuleLayer.boot().findModule(mod.descriptor().name()).isPresent()) { - return false; - } - for (var p : parents) { - if (p.layer.findModule(mod.descriptor().name()).isPresent()) { - return false; - } - } - var uri = mod.location().get(); - locations.add(Path.of(uri)); - return true; - }) - .map(mod -> mod.descriptor().name()) - .toList(); + var locations = new ArrayList(); + var moduleNames = new ArrayList(); + MODULE_LOOP: + for (var mod : ModuleFinder.of(file).findAll()) { + if (ModuleLayer.boot().findModule(mod.descriptor().name()).isPresent()) { + continue; + } + for (var p : parents) { + if (p.layer.findModule(mod.descriptor().name()).isPresent()) { + continue MODULE_LOOP; + } + } + moduleNames.add(mod.descriptor().name()); + var uri = mod.location().get(); + locations.add(Path.of(uri)); + } if (moduleNames.isEmpty() && parents.isEmpty()) { return EMPTY; } else { From 5b14f89bdcfa1701dc702bbfefa7233efbfc5914 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 2 Oct 2024 14:35:49 +0200 Subject: [PATCH 33/89] Special Lookup class and registration via a property to allow searching all the available ModuleLayers --- .../interpreter/runtime/EnsoClassPath.java | 17 ++++++ .../enso/base/file_format/FileFormatSPI.java | 22 ++++---- .../java/org/enso/base/lookup/Lookup.java | 54 +++++++++++++++++++ 3 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index c1e69c886926..71d8edc818d3 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -4,7 +4,9 @@ import java.lang.module.ModuleFinder; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; +import java.util.LinkedHashSet; import java.util.List; /** Representation of an Enso library class path. */ @@ -61,7 +63,22 @@ static EnsoClassPath create(Path file, List parents) { var layer = cntrl.layer(); var loader = !moduleNames.isEmpty() ? layer.findLoader(moduleNames.get(0)) : parents.get(0).loader; + + registerLayer(layer); return new EnsoClassPath(cntrl, layer, loader); } } + + @SuppressWarnings("unchecked") + private static void registerLayer(ModuleLayer moduleLayer) { + var props = System.getProperties(); + Collection layers; + if (props.get("enso.class.path") instanceof Collection registeredLayers) { + layers = registeredLayers; + } else { + layers = new LinkedHashSet<>(); + props.put("enso.class.path", layers); + } + layers.add(moduleLayer); + } } diff --git a/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java b/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java index e3a3fb867775..8318990a2122 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java @@ -1,14 +1,14 @@ package org.enso.base.file_format; +import java.util.ArrayList; import java.util.Objects; -import java.util.ServiceLoader; import java.util.stream.Collectors; +import org.enso.base.lookup.Lookup; import org.enso.base.polyglot.EnsoMeta; import org.graalvm.polyglot.Value; public abstract class FileFormatSPI { - private static final ServiceLoader loader = - ServiceLoader.load(FileFormatSPI.class, FileFormatSPI.class.getClassLoader()); + private static final Lookup loader = Lookup.lookup(FileFormatSPI.class); public static Value[] get_types(boolean refresh) { if (refresh) { @@ -20,19 +20,19 @@ public static Value[] get_types(boolean refresh) { public static Value findFormatForDataLinkSubType(String subType) { Objects.requireNonNull(subType, "subType must not be null/Nothing."); - var providers = - loader.stream() - .filter(provider -> subType.equalsIgnoreCase(provider.get().getDataLinkFormatName())) - .toList(); + var providers = new ArrayList(); + for (var s : loader) { + if (subType.equalsIgnoreCase(s.getDataLinkFormatName())) { + providers.add(s); + } + } if (providers.isEmpty()) { return null; } if (providers.size() > 1) { var modules = - providers.stream() - .map(provider -> provider.get().getModuleName()) - .collect(Collectors.joining(", ")); + providers.stream().map(s -> s.getModuleName()).collect(Collectors.joining(", ")); throw new IllegalStateException( "Error: Multiple Format providers found for format: " + subType @@ -41,7 +41,7 @@ public static Value findFormatForDataLinkSubType(String subType) { + "."); } - return providers.get(0).get().getTypeObject(); + return providers.get(0).getTypeObject(); } public Value getTypeObject() { diff --git a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java new file mode 100644 index 000000000000..0f315498bdcc --- /dev/null +++ b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java @@ -0,0 +1,54 @@ +package org.enso.base.lookup; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.ServiceLoader; +import java.util.stream.Stream; + +public final class Lookup implements Iterable { + private final Class service; + private List> found; + + private Lookup(Class service) { + this.service = service; + } + + @Override + public Iterator iterator() { + return stream().map(p -> p.get()).toList().iterator(); + } + + public Stream> stream() { + var tmp = found; + if (tmp == null) { + found = tmp = findAll(); + } + return tmp.stream(); + } + + public void reload() { + found = null; + } + + private List> findAll() { + var arr = new ArrayList>(); + if (System.getProperties().get("enso.class.path") instanceof Collection layers) { + for (var obj : layers) { + if (obj instanceof ModuleLayer layer) { + ServiceLoader.load(layer, service).stream() + .forEach( + (p) -> { + arr.add(p); + }); + } + } + } + return arr; + } + + public static Lookup lookup(Class service) { + return new Lookup<>(service); + } +} From 841de5ff5b1223e3a14124e6263171cb2d01e5e0 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 2 Oct 2024 14:36:10 +0200 Subject: [PATCH 34/89] Turning Standard.Image into Java module --- build.sbt | 13 ++++++++++--- .../lib/Standard/Image/0.0.0-dev/package.yaml | 2 ++ .../main/java/org/enso/image/ImageFormatSPI.java | 1 - 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 900f49bcd61c..60561b77d013 100644 --- a/build.sbt +++ b/build.sbt @@ -4540,6 +4540,7 @@ lazy val `std-table` = project lazy val `std-image` = project .in(file("std-bits") / "image") + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -4549,9 +4550,15 @@ lazy val `std-image` = project Compile / packageBin / artifactPath := `image-polyglot-root` / "std-image.jar", libraryDependencies ++= Seq( - "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "org.openpnp" % "opencv" % opencvVersion + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided", + "org.openpnp" % "opencv" % opencvVersion + ), + Compile / moduleDependencies ++= Seq( + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, + "org.openpnp" % "opencv" % opencvVersion + ), + Compile / internalModuleDependencies := Seq( + (`std-base` / Compile / exportedModule).value ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/distribution/lib/Standard/Image/0.0.0-dev/package.yaml b/distribution/lib/Standard/Image/0.0.0-dev/package.yaml index ad9365fc61e5..f21f0a66e737 100644 --- a/distribution/lib/Standard/Image/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Image/0.0.0-dev/package.yaml @@ -8,3 +8,5 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Base diff --git a/std-bits/image/src/main/java/org/enso/image/ImageFormatSPI.java b/std-bits/image/src/main/java/org/enso/image/ImageFormatSPI.java index 3893bcdc4e49..b57024996843 100644 --- a/std-bits/image/src/main/java/org/enso/image/ImageFormatSPI.java +++ b/std-bits/image/src/main/java/org/enso/image/ImageFormatSPI.java @@ -2,7 +2,6 @@ import org.enso.base.file_format.FileFormatSPI; -@org.openide.util.lookup.ServiceProvider(service = FileFormatSPI.class) public class ImageFormatSPI extends FileFormatSPI { @Override protected String getModuleName() { From e6e5c098c87809aa6b9c774d4c5d3fc22efc7fa7 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 2 Oct 2024 16:40:53 +0200 Subject: [PATCH 35/89] Making test/Image_Tests pass OK --- build.sbt | 5 +++-- std-bits/base/src/main/java/module-info.java | 1 + .../org/enso/base/file_format/FileFormatSPI.java | 4 +++- .../src/main/java/org/enso/base/lookup/Lookup.java | 13 +++++++------ std-bits/image/src/main/java/module-info.java | 14 ++++++++++++++ 5 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 std-bits/image/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index 60561b77d013..a2ce7ad26bae 100644 --- a/build.sbt +++ b/build.sbt @@ -4700,6 +4700,7 @@ lazy val `std-aws` = project lazy val `std-snowflake` = project .in(file("std-bits") / "snowflake") + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -4730,6 +4731,7 @@ lazy val `std-snowflake` = project lazy val `std-microsoft` = project .in(file("std-bits") / "microsoft") + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -4739,8 +4741,7 @@ lazy val `std-microsoft` = project Compile / packageBin / artifactPath := `std-microsoft-polyglot-root` / "std-microsoft.jar", libraryDependencies ++= Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "com.microsoft.sqlserver" % "mssql-jdbc" % mssqlserverJDBCVersion + "com.microsoft.sqlserver" % "mssql-jdbc" % mssqlserverJDBCVersion ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index 5eb9f1dcfdfd..73555508dcf8 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -30,6 +30,7 @@ exports org.enso.base.file_format; exports org.enso.base.enso_cloud; exports org.enso.base.enso_cloud.audit; + exports org.enso.base.lookup; exports org.enso.base.numeric; exports org.enso.base.polyglot; exports org.enso.base.time; diff --git a/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java b/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java index 8318990a2122..a1b19bfc0013 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_format/FileFormatSPI.java @@ -2,13 +2,15 @@ import java.util.ArrayList; import java.util.Objects; +import java.util.ServiceLoader; import java.util.stream.Collectors; import org.enso.base.lookup.Lookup; import org.enso.base.polyglot.EnsoMeta; import org.graalvm.polyglot.Value; public abstract class FileFormatSPI { - private static final Lookup loader = Lookup.lookup(FileFormatSPI.class); + private static final Lookup loader = + Lookup.lookup((l) -> ServiceLoader.load(l, FileFormatSPI.class)); public static Value[] get_types(boolean refresh) { if (refresh) { diff --git a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java index 0f315498bdcc..d74f58407de9 100644 --- a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java +++ b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java @@ -5,14 +5,15 @@ import java.util.Iterator; import java.util.List; import java.util.ServiceLoader; +import java.util.function.Function; import java.util.stream.Stream; public final class Lookup implements Iterable { - private final Class service; + private final Function> factory; private List> found; - private Lookup(Class service) { - this.service = service; + private Lookup(Function> factory) { + this.factory = factory; } @Override @@ -37,7 +38,7 @@ private List> findAll() { if (System.getProperties().get("enso.class.path") instanceof Collection layers) { for (var obj : layers) { if (obj instanceof ModuleLayer layer) { - ServiceLoader.load(layer, service).stream() + factory.apply(layer).stream() .forEach( (p) -> { arr.add(p); @@ -48,7 +49,7 @@ private List> findAll() { return arr; } - public static Lookup lookup(Class service) { - return new Lookup<>(service); + public static Lookup lookup(Function> factory) { + return new Lookup<>(factory); } } diff --git a/std-bits/image/src/main/java/module-info.java b/std-bits/image/src/main/java/module-info.java new file mode 100644 index 000000000000..6132173e4e3e --- /dev/null +++ b/std-bits/image/src/main/java/module-info.java @@ -0,0 +1,14 @@ +import org.enso.base.file_format.FileFormatSPI; +import org.enso.image.ImageFormatSPI; + +module org.enso.std.image { + requires org.graalvm.polyglot; + requires org.enso.std.base; + requires opencv; + + provides FileFormatSPI with + ImageFormatSPI; + + opens org.enso.image; + opens org.enso.image.data; +} From 3194f6d0bb6e69e7d530dc7f0baa826974c0a904 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 2 Oct 2024 17:35:33 +0200 Subject: [PATCH 36/89] Avoid DriverManager. Directly talk to Driver instances. --- .../enso/base/enso_cloud/EnsoSecretHelper.java | 16 +++++++++++++--- .../main/java/org/enso/database/JDBCProxy.java | 16 ++++++---------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java index b4134ed1b9bb..d88d4a0039da 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/EnsoSecretHelper.java @@ -7,7 +7,7 @@ import java.net.http.HttpRequest.Builder; import java.net.http.HttpResponse; import java.sql.Connection; -import java.sql.DriverManager; +import java.sql.Driver; import java.sql.SQLException; import java.util.List; import java.util.Properties; @@ -20,13 +20,23 @@ public final class EnsoSecretHelper extends SecretValueResolver { /** Gets a JDBC connection resolving EnsoKeyValuePair into the properties. */ public static Connection getJDBCConnection( - String url, List> properties) throws SQLException { + Iterable drivers, String url, List> properties) + throws SQLException { var javaProperties = new Properties(); for (var pair : properties) { javaProperties.setProperty(pair.getLeft(), resolveValue(pair.getRight())); } - return DriverManager.getConnection(url, javaProperties); + var err = new StringBuilder(); + for (var driver : drivers) { + err.append("\nTrying ").append(driver.getClass().getName()); + var c = driver.connect(url, javaProperties); + if (c != null) { + return c; + } + } + err.insert(0, "No handler to connect to " + url); + throw new SQLException(err.toString()); } /** diff --git a/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java b/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java index f76b0689ef44..26251736eca7 100644 --- a/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java +++ b/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java @@ -1,16 +1,17 @@ package org.enso.database; import java.sql.Connection; -import java.sql.DriverManager; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.ServiceLoader; + import org.enso.base.enso_cloud.EnsoSecretAccessDenied; import org.enso.base.enso_cloud.EnsoSecretHelper; import org.enso.base.enso_cloud.HideableValue; +import org.enso.base.lookup.Lookup; import org.enso.database.audit.CloudAuditedConnection; import org.enso.database.audit.LocalAuditedConnection; import org.graalvm.collections.Pair; @@ -31,7 +32,8 @@ public final class JDBCProxy { * @return an array of JDBC drivers that are currently registered */ public static Object[] getDrivers() { - return DriverManager.drivers().toArray(); + var drivers = Lookup.lookup((l) -> ServiceLoader.load(l, java.sql.Driver.class)); + return drivers.stream().toArray(); } /** @@ -46,17 +48,11 @@ public static Object[] getDrivers() { */ public static Connection getConnection(String url, List> properties) throws SQLException { - // We need to manually register all the drivers because the DriverManager is not able - // to correctly use our class loader, it only delegates to the platform class loader when - // loading the java.sql.Driver service. - var sl = ServiceLoader.load(java.sql.Driver.class, JDBCProxy.class.getClassLoader()); - for (var driver : sl) { - DriverManager.registerDriver(driver); - } + var drivers = Lookup.lookup((l) -> ServiceLoader.load(l, java.sql.Driver.class)); PartitionedProperties partitionedProperties = PartitionedProperties.parse(properties); var rawConnection = - EnsoSecretHelper.getJDBCConnection(url, partitionedProperties.jdbcProperties); + EnsoSecretHelper.getJDBCConnection(drivers, url, partitionedProperties.jdbcProperties); return switch (partitionedProperties.audited()) { case "local" -> new LocalAuditedConnection(rawConnection); case "cloud" -> new CloudAuditedConnection( From cbe625f6effa1014fe607d445f6abb9e526ea332 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 10:39:07 +0200 Subject: [PATCH 37/89] Register SQLServerConnectionDetailsSPI in std-bits/microsoft/*/module-info.java --- build.sbt | 6 +++++- .../src/main/java/org/enso/database/JDBCProxy.java | 1 - std-bits/microsoft/src/main/java/module-info.java | 10 ++++++++++ .../enso/microsoft/SQLServerConnectionDetailsSPI.java | 3 +-- 4 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 std-bits/microsoft/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index a2ce7ad26bae..a1b7bb02ff99 100644 --- a/build.sbt +++ b/build.sbt @@ -4753,7 +4753,11 @@ lazy val `std-microsoft` = project ) .value result - }.value + }.value, + Compile / internalModuleDependencies := Seq( + (`std-base` / Compile / exportedModule).value, + (`std-database` / Compile / exportedModule).value + ) ) .dependsOn(`std-base` % "provided") .dependsOn(`std-table` % "provided") diff --git a/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java b/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java index 26251736eca7..62cf5618faa2 100644 --- a/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java +++ b/std-bits/database/src/main/java/org/enso/database/JDBCProxy.java @@ -7,7 +7,6 @@ import java.util.List; import java.util.Map; import java.util.ServiceLoader; - import org.enso.base.enso_cloud.EnsoSecretAccessDenied; import org.enso.base.enso_cloud.EnsoSecretHelper; import org.enso.base.enso_cloud.HideableValue; diff --git a/std-bits/microsoft/src/main/java/module-info.java b/std-bits/microsoft/src/main/java/module-info.java new file mode 100644 index 000000000000..487571496634 --- /dev/null +++ b/std-bits/microsoft/src/main/java/module-info.java @@ -0,0 +1,10 @@ +import org.enso.database.DatabaseConnectionDetailsSPI; +import org.enso.microsoft.SQLServerConnectionDetailsSPI; + +module org.enso.std.microsoft { + requires org.enso.std.base; + requires org.enso.std.database; + + provides DatabaseConnectionDetailsSPI with + SQLServerConnectionDetailsSPI; +} diff --git a/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerConnectionDetailsSPI.java b/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerConnectionDetailsSPI.java index ad426745eb60..e123cb47e00d 100644 --- a/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerConnectionDetailsSPI.java +++ b/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerConnectionDetailsSPI.java @@ -2,8 +2,7 @@ import org.enso.database.DatabaseConnectionDetailsSPI; -@org.openide.util.lookup.ServiceProvider(service = DatabaseConnectionDetailsSPI.class) -public class SQLServerConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { +public final class SQLServerConnectionDetailsSPI extends DatabaseConnectionDetailsSPI { @Override protected String getModuleName() { From c72ab8e49a3c61a099ec52b078618515fc2d3279 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 10:55:22 +0200 Subject: [PATCH 38/89] No polyglot symbol for org.enso.base.encoding.NewlineDetector in Table_Tests fix --- std-bits/base/src/main/java/module-info.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index 73555508dcf8..87a99d26d9aa 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -41,7 +41,9 @@ // following packages are accessed by Enso via polyglot java import opens org.enso.base; opens org.enso.base.arrays; - opens org.enso.base.encoding; + + exports org.enso.base.encoding; + opens org.enso.base.numeric; opens org.enso.base.net; opens org.enso.base.polyglot; From 062b26e9a206742df3de0d8be88bdd9f6346712f Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 10:56:02 +0200 Subject: [PATCH 39/89] No polyglot symbol for org.enso.table.data.table.Table in Table_Tests --- std-bits/table/src/main/java/module-info.java | 4 +++- test/Table_Tests/package.yaml | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index 6d7222b42d58..134ea20eed6e 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -41,7 +41,9 @@ opens org.enso.table.data.column.storage.type; opens org.enso.table.data.mask; opens org.enso.table.data.index; - opens org.enso.table.data.table; + + exports org.enso.table.data.table; + opens org.enso.table.data.table.problems; opens org.enso.table.data.table.join; opens org.enso.table.data.table.join.conditions; diff --git a/test/Table_Tests/package.yaml b/test/Table_Tests/package.yaml index d1174d2d9404..5c31ea8db335 100644 --- a/test/Table_Tests/package.yaml +++ b/test/Table_Tests/package.yaml @@ -5,3 +5,5 @@ license: MIT author: enso-dev@enso.org maintainer: enso-dev@enso.org prefer-local-libraries: true +requires: + - Standard.Table From e4cc4b43b86cf70a81b9cb40a7e5b713ee17719e Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 11:11:23 +0200 Subject: [PATCH 40/89] No polyglot symbol for org.enso.table.data.table.join.between.SortedListIndex reported in Table_Tests --- std-bits/table/src/main/java/module-info.java | 1 + 1 file changed, 1 insertion(+) diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index 134ea20eed6e..7d2a1ed1d576 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -46,6 +46,7 @@ opens org.enso.table.data.table.problems; opens org.enso.table.data.table.join; + exports org.enso.table.data.table.join.between; opens org.enso.table.data.table.join.conditions; opens org.enso.table.data.table.join.lookup; opens org.enso.table.error; From 9c86f7a382752f26569b43323144d0b680cf7f29 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 11:47:04 +0200 Subject: [PATCH 41/89] Using for loop --- .../interpreter/runtime/EnsoClassPath.java | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 71d8edc818d3..b93ecff4174d 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -17,6 +17,11 @@ public final class EnsoClassPath { final ClassLoader loader; private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoader loader) { + if (cntrl != null) { + // cannot be null + layer.getClass(); + loader.getClass(); + } this.cntrl = cntrl; this.layer = layer; this.loader = loader; @@ -31,7 +36,7 @@ static EnsoClassPath create(Path file, List parents) { continue; } for (var p : parents) { - if (p.layer.findModule(mod.descriptor().name()).isPresent()) { + if (p.layer != null && p.layer.findModule(mod.descriptor().name()).isPresent()) { continue MODULE_LOOP; } } @@ -53,8 +58,15 @@ static EnsoClassPath create(Path file, List parents) { Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentModules, parentLoader); } else { - var parentCfgs = parents.stream().map(cp -> cp.layer.configuration()).toList(); - var parentLayers = parents.stream().map(cp -> cp.layer).toList(); + var parentCfgs = new ArrayList(); + var parentLayers = new ArrayList(); + for (var cp : parents) { + if (cp.layer == null) { + continue; + } + parentLayers.add(cp.layer); + parentCfgs.add(cp.layer.configuration()); + } var parentLoader = ModuleLayer.boot().findLoader("java.base"); var cfg = Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); From 6992103b2081621e703824d07da4a9284660e2e3 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 11:48:10 +0200 Subject: [PATCH 42/89] Make enso-test-java-helpers available to Table_Tests --- build.sbt | 8 ++++++++ std-bits/table/src/main/java/module-info.java | 16 ++++++++++++---- test/Base_Tests/package.yaml | 3 +++ .../src/main/java/module-info.java | 7 +++++++ test/Table_Tests/package.yaml | 1 + 5 files changed, 31 insertions(+), 4 deletions(-) create mode 100644 test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index a1b7bb02ff99..a3700f03cc21 100644 --- a/build.sbt +++ b/build.sbt @@ -4418,6 +4418,7 @@ lazy val `common-polyglot-core-utils` = project lazy val `enso-test-java-helpers` = project .in(file("test/Base_Tests/polyglot-sources/enso-test-java-helpers")) + .enablePlugins(JPMSPlugin) .settings( frgaalJavaCompilerSetting, autoScalaLibrary := false, @@ -4426,6 +4427,13 @@ lazy val `enso-test-java-helpers` = project libraryDependencies ++= Seq( "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion % "provided" ), + Compile / moduleDependencies ++= Seq( + "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion + ), + Compile / internalModuleDependencies := Seq( + (`std-base` / Compile / exportedModule).value, + (`std-table` / Compile / exportedModule).value + ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val primaryLocation = (Compile / packageBin / artifactPath).value diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index 7d2a1ed1d576..d1a93cb03cd1 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -31,22 +31,30 @@ opens org.enso.table.problems; opens org.enso.table.parsing; opens org.enso.table.parsing.problems; - opens org.enso.table.data.column.storage; + + exports org.enso.table.data.column.storage; + opens org.enso.table.data.column.storage.numeric; opens org.enso.table.data.column.builder; opens org.enso.table.data.column.operation; opens org.enso.table.data.column.operation.cast; - opens org.enso.table.data.column.operation.map; + + exports org.enso.table.data.column.operation.map; + opens org.enso.table.data.column.operation.unary; - opens org.enso.table.data.column.storage.type; - opens org.enso.table.data.mask; + + exports org.enso.table.data.column.storage.type; + exports org.enso.table.data.mask; + opens org.enso.table.data.index; exports org.enso.table.data.table; opens org.enso.table.data.table.problems; opens org.enso.table.data.table.join; + exports org.enso.table.data.table.join.between; + opens org.enso.table.data.table.join.conditions; opens org.enso.table.data.table.join.lookup; opens org.enso.table.error; diff --git a/test/Base_Tests/package.yaml b/test/Base_Tests/package.yaml index 71c3b3be8894..032a106d7330 100644 --- a/test/Base_Tests/package.yaml +++ b/test/Base_Tests/package.yaml @@ -7,3 +7,6 @@ author: enso-dev@enso.org maintainer: enso-dev@enso.org # Base_Tests import stuff from Helpers sibling project. So we need this property here. prefer-local-libraries: true +requires: + - Standard.Base + - Standard.Table diff --git a/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java b/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java new file mode 100644 index 000000000000..309554a01397 --- /dev/null +++ b/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java @@ -0,0 +1,7 @@ +module org.enso.test.java_helpers { + requires static org.graalvm.polyglot; + requires org.enso.std.table; + + exports org.enso.base_test_helpers; + exports org.enso.table_test_helpers; +} diff --git a/test/Table_Tests/package.yaml b/test/Table_Tests/package.yaml index 5c31ea8db335..51d8f8fa7c17 100644 --- a/test/Table_Tests/package.yaml +++ b/test/Table_Tests/package.yaml @@ -7,3 +7,4 @@ maintainer: enso-dev@enso.org prefer-local-libraries: true requires: - Standard.Table + - Standard.Base_Tests From 98eeb64e3249bf56b9d84e67e4ca4ec7a0f3b112 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 15:59:03 +0200 Subject: [PATCH 43/89] No need to generate .bindings index to run benchmarks --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a3700f03cc21..c40a1d9cfe8e 100644 --- a/build.sbt +++ b/build.sbt @@ -3976,7 +3976,7 @@ lazy val `std-benchmarks` = (project in file("std-bits/benchmarks")) (Compile / run).toTask("").tag(Exclusive).value } .dependsOn( - buildEngineDistribution + buildEngineDistributionNoIndex ) .value, benchOnly := Def.inputTaskDyn { From 60b4277654e01ad08022849140ac22748b552bd3 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Thu, 3 Oct 2024 16:00:57 +0200 Subject: [PATCH 44/89] BenchProcessor puts runtime.jar into own layer. Use that one as bootLayer() when specified. --- .../interpreter/runtime/EnsoClassPath.java | 52 ++++++++++++++++--- .../enso/interpreter/runtime/EnsoContext.java | 2 +- 2 files changed, 46 insertions(+), 8 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index b93ecff4174d..5f2d85e29ad9 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -1,22 +1,29 @@ package org.enso.interpreter.runtime; +import com.oracle.truffle.api.TruffleLogger; import java.lang.module.Configuration; +import java.lang.module.FindException; import java.lang.module.ModuleFinder; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashSet; import java.util.List; +import java.util.logging.Level; /** Representation of an Enso library class path. */ public final class EnsoClassPath { - static final EnsoClassPath EMPTY = new EnsoClassPath(null, null, null); + static final EnsoClassPath EMPTY = new EnsoClassPath("empty", null, null, null); + private final Object id; private final ModuleLayer.Controller cntrl; private final ModuleLayer layer; final ClassLoader loader; - private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoader loader) { + private EnsoClassPath( + Object id, ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoader loader) { + this.id = id; if (cntrl != null) { // cannot be null layer.getClass(); @@ -27,12 +34,38 @@ private EnsoClassPath(ModuleLayer.Controller cntrl, ModuleLayer layer, ClassLoad this.loader = loader; } - static EnsoClassPath create(Path file, List parents) { + private static ModuleLayer bootLayer() { + var myLayer = EnsoClassPath.class.getModule().getLayer(); + if (myLayer != null) { + return myLayer; + } else { + return ModuleLayer.boot(); + } + } + + static EnsoClassPath create(Path file, List parents, TruffleLogger log) { + try { + return createImpl(file, parents); + } catch (FindException e) { + var sb = new StringBuilder(); + sb.append("Cannot instantiate modules at ").append(file); + log.log(Level.FINE, sb.toString(), e); + for (var cp : parents) { + sb.append("\n with parent ").append(cp); + } + e.setStackTrace( + Arrays.asList(e.getStackTrace()).stream().limit(10).toArray(StackTraceElement[]::new)); + log.log(Level.SEVERE, sb.toString(), e); + return EnsoClassPath.EMPTY; + } + } + + private static EnsoClassPath createImpl(Path file, List parents) { var locations = new ArrayList(); var moduleNames = new ArrayList(); MODULE_LOOP: for (var mod : ModuleFinder.of(file).findAll()) { - if (ModuleLayer.boot().findModule(mod.descriptor().name()).isPresent()) { + if (bootLayer().findModule(mod.descriptor().name()).isPresent()) { continue; } for (var p : parents) { @@ -50,7 +83,7 @@ static EnsoClassPath create(Path file, List parents) { var finder = ModuleFinder.of(locations.toArray(new Path[0])); ModuleLayer.Controller cntrl; if (parents.isEmpty()) { - var parent = ModuleLayer.boot(); + var parent = bootLayer(); var parentLoader = parent.findLoader("java.base"); var parentCfgs = Collections.singletonList(parent.configuration()); var parentModules = Collections.singletonList(parent); @@ -67,7 +100,7 @@ static EnsoClassPath create(Path file, List parents) { parentLayers.add(cp.layer); parentCfgs.add(cp.layer.configuration()); } - var parentLoader = ModuleLayer.boot().findLoader("java.base"); + var parentLoader = bootLayer().findLoader("java.base"); var cfg = Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); cntrl = ModuleLayer.defineModulesWithOneLoader(cfg, parentLayers, parentLoader); @@ -77,7 +110,7 @@ static EnsoClassPath create(Path file, List parents) { !moduleNames.isEmpty() ? layer.findLoader(moduleNames.get(0)) : parents.get(0).loader; registerLayer(layer); - return new EnsoClassPath(cntrl, layer, loader); + return new EnsoClassPath(file, cntrl, layer, loader); } } @@ -93,4 +126,9 @@ private static void registerLayer(ModuleLayer moduleLayer) { } layers.add(moduleLayer); } + + @Override + public String toString() { + return "EnsoClassPath[id=" + id + "]"; + } } diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 51ef9ee11d47..3b5a7621da92 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -494,7 +494,7 @@ public synchronized EnsoClassPath findClassPath(Package pkg) { return findClassPath(null); }) .toList(); - l = EnsoClassPath.create(Paths.get(ch.toUri()), parents); + l = EnsoClassPath.create(Paths.get(ch.toUri()), parents, logger); } else { l = EnsoClassPath.EMPTY; } From f482b72b63f64600910003cc61f4a4a24c52486c Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 4 Oct 2024 18:14:40 +0200 Subject: [PATCH 45/89] test/Benchmarks need access to org.enso.table.data.index.OrderedMultiValueKey --- std-bits/table/src/main/java/module-info.java | 4 +--- test/Benchmarks/package.yaml | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index d1a93cb03cd1..f1986f44de85 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -45,9 +45,7 @@ exports org.enso.table.data.column.storage.type; exports org.enso.table.data.mask; - - opens org.enso.table.data.index; - + exports org.enso.table.data.index; exports org.enso.table.data.table; opens org.enso.table.data.table.problems; diff --git a/test/Benchmarks/package.yaml b/test/Benchmarks/package.yaml index cc894674bf77..a2c0639e5f2a 100644 --- a/test/Benchmarks/package.yaml +++ b/test/Benchmarks/package.yaml @@ -4,3 +4,5 @@ version: 0.0.1 license: MIT author: enso-dev@enso.org maintainer: enso-dev@enso.org +requires: + - Standard.Table From 31121581fc891127ada36445ac61db16fad479cf Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 7 Oct 2024 08:09:09 +0200 Subject: [PATCH 46/89] Introducing AWS wrapper module --- build.sbt | 99 +++++++++++++++---- .../src/main/java/module-info.java | 7 ++ std-bits/aws/src/main/java/module-info.java | 8 +- 3 files changed, 88 insertions(+), 26 deletions(-) create mode 100644 lib/java/aws-wrapper/src/main/java/module-info.java diff --git a/build.sbt b/build.sbt index c40a1d9cfe8e..54761439e5df 100644 --- a/build.sbt +++ b/build.sbt @@ -1531,6 +1531,84 @@ lazy val `zio-wrapper` = project } ) +/** JPMS module wrapper for AWS SDK. + */ +lazy val `aws-wrapper` = project + .in(file("lib/java/aws-wrapper")) + .enablePlugins(JPMSPlugin) + .settings( + modularFatJarWrapperSettings, + libraryDependencies ++= Seq( + "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, + "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version, + "com.amazonaws" % "aws-java-sdk-redshift" % awsJavaSdkV1Version, + "com.amazonaws" % "aws-java-sdk-sts" % awsJavaSdkV1Version, + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "bom" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, + "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", + "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" + ), + javaModuleName := "org.enso.aws.wrapper", + Compile / moduleDependencies ++= Seq( + "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", + "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" + ), + assembly / assemblyExcludedJars := { + val excludedJars = JPMSUtils.filterModulesFromUpdate( + update.value, + Seq( + "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", + "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", + "com.fasterxml.jackson.dataformat" % "jackson-cbor" % "2.12.6" + ), + streams.value.log, + moduleName.value, + scalaBinaryVersion.value, + shouldContainAll = true + ) + excludedJars + .map(Attributed.blank) + }, + assembly / assemblyMergeStrategy := { + case PathList("META-INF", "io.netty.versions.properties") => + MergeStrategy.first + case PathList("META-INF", "MANIFEST.MF") => MergeStrategy.discard + case PathList("META-INF", "LICENSE") => MergeStrategy.concat + case PathList("META-INF", "LICENSE.txt") => MergeStrategy.concat + case PathList("META-INF", "NOTICE") => MergeStrategy.concat + case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat + case PathList("META-INF", "DEPENDENCIES") => MergeStrategy.discard + case PathList("META-INF", "INDEX.LIST") => MergeStrategy.discard + case PathList("module-info.class") => MergeStrategy.preferProject + case _ => MergeStrategy.deduplicate + }, + autoScalaLibrary := false, + Compile / patchModules := { + val aws = JPMSUtils.filterModulesFromUpdate( + update.value, + scalaLibrary ++ + Seq( + "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version + ), + streams.value.log, + moduleName.value, + scalaBinaryVersion.value, + shouldContainAll = true + ) + Map( + javaModuleName.value -> aws + ) + } + ) + lazy val cli = project .in(file("lib/scala/cli")) .enablePlugins(JPMSPlugin) @@ -4664,29 +4742,11 @@ lazy val `std-aws` = project Compile / packageBin / artifactPath := `std-aws-polyglot-root` / "std-aws.jar", libraryDependencies ++= Seq( - "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, - "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version, - "com.amazonaws" % "aws-java-sdk-redshift" % awsJavaSdkV1Version, - "com.amazonaws" % "aws-java-sdk-sts" % awsJavaSdkV1Version, - "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "bom" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version ), Compile / moduleDependencies ++= Seq( - "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, - "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sso" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "ssooidc" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, - "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version ), Compile / internalModuleDependencies := Seq( + (`aws-wrapper` / Compile / exportedModule).value, (`std-base` / Compile / exportedModule).value, (`std-database` / Compile / exportedModule).value ), @@ -4702,6 +4762,7 @@ lazy val `std-aws` = project result }.value ) + .dependsOn(`aws-wrapper`) .dependsOn(`std-base` % "provided") .dependsOn(`std-table` % "provided") .dependsOn(`std-database` % "provided") diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java new file mode 100644 index 000000000000..6f1a9b16b4c0 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -0,0 +1,7 @@ +module org.enso.wrapper.aws { + exports software.amazon.awssdk.http; + exports software.amazon.awssdk.services.s3.model; + exports software.amazon.awssdk.auth.credentials; + exports software.amazon.awssdk.regions; + exports software.amazon.awssdk.regions.providers; +} diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index 15d6c2398c9d..b2bff6a72cad 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -8,13 +8,7 @@ requires org.enso.std.base; requires org.enso.std.database; requires java.logging; - requires software.amazon.awssdk.http; - requires software.amazon.awssdk.regions; - requires software.amazon.awssdk.services.s3; - requires software.amazon.awssdk.auth; - requires software.amazon.awssdk.core; - requires software.amazon.awssdk.profiles; - requires software.amazon.awssdk.awscore; + requires org.enso.wrapper.aws; provides FileSystemSPI with S3FileSystemSPI; From e51eb3e981d9a31faa728f45e4e1d34377f6ff80 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 7 Oct 2024 11:44:22 +0200 Subject: [PATCH 47/89] Placeholders for exported packages --- build.sbt | 2 +- lib/java/aws-wrapper/src/main/java/module-info.java | 6 +++++- .../java/software/amazon/awssdk/auth/credentials/Place.java | 3 +++ .../software/amazon/awssdk/awscore/exception/Place.java | 3 +++ .../java/software/amazon/awssdk/core/exception/Place.java | 3 +++ .../src/main/java/software/amazon/awssdk/http/Place.java | 3 +++ .../main/java/software/amazon/awssdk/profiles/Place.java | 3 +++ .../src/main/java/software/amazon/awssdk/regions/Place.java | 3 +++ .../software/amazon/awssdk/regions/providers/Place.java | 3 +++ .../main/java/software/amazon/awssdk/services/s3/Place.java | 3 +++ .../software/amazon/awssdk/services/s3/model/Place.java | 3 +++ 11 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java create mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java diff --git a/build.sbt b/build.sbt index 54761439e5df..7ada9fab3931 100644 --- a/build.sbt +++ b/build.sbt @@ -1596,7 +1596,7 @@ lazy val `aws-wrapper` = project update.value, scalaLibrary ++ Seq( - "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version ), streams.value.log, moduleName.value, diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index 6f1a9b16b4c0..9977cacfdac9 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -1,7 +1,11 @@ module org.enso.wrapper.aws { + exports software.amazon.awssdk.auth.credentials; + exports software.amazon.awssdk.core.exception; + exports software.amazon.awssdk.awscore.exception; exports software.amazon.awssdk.http; + exports software.amazon.awssdk.services.s3; exports software.amazon.awssdk.services.s3.model; - exports software.amazon.awssdk.auth.credentials; + exports software.amazon.awssdk.profiles; exports software.amazon.awssdk.regions; exports software.amazon.awssdk.regions.providers; } diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java new file mode 100644 index 000000000000..e6b35961d0cd --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.auth.credentials; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java new file mode 100644 index 000000000000..ad9da449d0c6 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.awscore.exception; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java new file mode 100644 index 000000000000..541a298948c4 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.core.exception; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java new file mode 100644 index 000000000000..40305211d570 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.http; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java new file mode 100644 index 000000000000..89531189b513 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.profiles; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java new file mode 100644 index 000000000000..927bbd49a070 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.regions; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java new file mode 100644 index 000000000000..c4b07b18d2f2 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.regions.providers; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java new file mode 100644 index 000000000000..b4187342eebb --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.services.s3.model; + +class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java new file mode 100644 index 000000000000..f9d2a41892f7 --- /dev/null +++ b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java @@ -0,0 +1,3 @@ +package software.amazon.awssdk.services.s3; + +class Place {} From a2634498a7ba3b3c33f14dc94a651bcd0162bfc3 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 14:16:08 +0200 Subject: [PATCH 48/89] Remove placeholders --- .../java/software/amazon/awssdk/auth/credentials/Place.java | 3 --- .../java/software/amazon/awssdk/awscore/exception/Place.java | 3 --- .../main/java/software/amazon/awssdk/core/exception/Place.java | 3 --- .../src/main/java/software/amazon/awssdk/http/Place.java | 3 --- .../src/main/java/software/amazon/awssdk/profiles/Place.java | 3 --- .../src/main/java/software/amazon/awssdk/regions/Place.java | 3 --- .../java/software/amazon/awssdk/regions/providers/Place.java | 3 --- .../main/java/software/amazon/awssdk/services/s3/Place.java | 3 --- .../java/software/amazon/awssdk/services/s3/model/Place.java | 3 --- 9 files changed, 27 deletions(-) delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java delete mode 100644 lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java deleted file mode 100644 index e6b35961d0cd..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/auth/credentials/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.auth.credentials; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java deleted file mode 100644 index ad9da449d0c6..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/awscore/exception/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.awscore.exception; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java deleted file mode 100644 index 541a298948c4..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/core/exception/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.core.exception; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java deleted file mode 100644 index 40305211d570..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/http/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.http; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java deleted file mode 100644 index 89531189b513..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/profiles/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.profiles; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java deleted file mode 100644 index 927bbd49a070..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.regions; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java deleted file mode 100644 index c4b07b18d2f2..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/regions/providers/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.regions.providers; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java deleted file mode 100644 index b4187342eebb..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.services.s3.model; - -class Place {} diff --git a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java b/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java deleted file mode 100644 index f9d2a41892f7..000000000000 --- a/lib/java/aws-wrapper/src/main/java/software/amazon/awssdk/services/s3/model/Place.java +++ /dev/null @@ -1,3 +0,0 @@ -package software.amazon.awssdk.services.s3; - -class Place {} From d161f9e8489d088368281e6e23e3b6caa807ae12 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 14:18:38 +0200 Subject: [PATCH 49/89] Remove aws-wrapper/Compile/moduleDependencies. They are not required - we don't want anything on module-path --- build.sbt | 6 ------ 1 file changed, 6 deletions(-) diff --git a/build.sbt b/build.sbt index 7ada9fab3931..8115965b1b3d 100644 --- a/build.sbt +++ b/build.sbt @@ -1554,12 +1554,6 @@ lazy val `aws-wrapper` = project "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" ), javaModuleName := "org.enso.aws.wrapper", - Compile / moduleDependencies ++= Seq( - "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", - "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", - "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" - ), assembly / assemblyExcludedJars := { val excludedJars = JPMSUtils.filterModulesFromUpdate( update.value, From c03825c959e22f75395ae70be9c313f6e16d1cce Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 14:21:20 +0200 Subject: [PATCH 50/89] javaModuleName is the first setting --- build.sbt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 8115965b1b3d..503b54c20a96 100644 --- a/build.sbt +++ b/build.sbt @@ -1538,6 +1538,7 @@ lazy val `aws-wrapper` = project .enablePlugins(JPMSPlugin) .settings( modularFatJarWrapperSettings, + javaModuleName := "org.enso.aws.wrapper", libraryDependencies ++= Seq( "com.amazon.redshift" % "redshift-jdbc42" % redshiftVersion, "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version, @@ -1553,7 +1554,6 @@ lazy val `aws-wrapper` = project "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" ), - javaModuleName := "org.enso.aws.wrapper", assembly / assemblyExcludedJars := { val excludedJars = JPMSUtils.filterModulesFromUpdate( update.value, @@ -1588,7 +1588,6 @@ lazy val `aws-wrapper` = project Compile / patchModules := { val aws = JPMSUtils.filterModulesFromUpdate( update.value, - scalaLibrary ++ Seq( "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version ), From 3ec8ed01d374acd977ce9b37ce10f074801e1842 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 14:24:03 +0200 Subject: [PATCH 51/89] Fix typo in cbor library dependency --- build.sbt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/build.sbt b/build.sbt index 503b54c20a96..c8a37b606f7b 100644 --- a/build.sbt +++ b/build.sbt @@ -1558,10 +1558,10 @@ lazy val `aws-wrapper` = project val excludedJars = JPMSUtils.filterModulesFromUpdate( update.value, Seq( - "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", - "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", - "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", - "com.fasterxml.jackson.dataformat" % "jackson-cbor" % "2.12.6" + "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", + "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" ), streams.value.log, moduleName.value, From 41fd4c1d2f6742e59e9c30290b2369d997caa3d5 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 14:43:20 +0200 Subject: [PATCH 52/89] Typo in module name. The module name in module-info.java must be the same as `javaModuleName` sbt setting. --- lib/java/aws-wrapper/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index 9977cacfdac9..7815cdee2684 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -1,4 +1,4 @@ -module org.enso.wrapper.aws { +module org.enso.aws.wrapper { exports software.amazon.awssdk.auth.credentials; exports software.amazon.awssdk.core.exception; exports software.amazon.awssdk.awscore.exception; From 4bec7d99a4d803797ab63eafb6ade5cb0a5878f2 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 15:22:39 +0200 Subject: [PATCH 53/89] Fix compilation of aws-wrapper - Patch all AWS modules - Fix assemblyExcludedJars to remove all transitive dependencies --- build.sbt | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/build.sbt b/build.sbt index c8a37b606f7b..03b1f98b04e8 100644 --- a/build.sbt +++ b/build.sbt @@ -1544,6 +1544,7 @@ lazy val `aws-wrapper` = project "com.amazonaws" % "aws-java-sdk-core" % awsJavaSdkV1Version, "com.amazonaws" % "aws-java-sdk-redshift" % awsJavaSdkV1Version, "com.amazonaws" % "aws-java-sdk-sts" % awsJavaSdkV1Version, + "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version, "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, "software.amazon.awssdk" % "bom" % awsJavaSdkV2Version, "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version, @@ -1552,24 +1553,45 @@ lazy val `aws-wrapper` = project "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "commons-logging" % "commons-logging" % "1.2", + "commons-codec" % "commons-codec" % "1.15" ), + // Remove all the transitive dependencies of AWS SDK and leave only those that + // include `software.amazon.awssdk.*` classes. assembly / assemblyExcludedJars := { - val excludedJars = JPMSUtils.filterModulesFromUpdate( - update.value, + val excludedJars = JPMSUtils.filterModulesFromClasspath( + (Compile / dependencyClasspath).value, Seq( - "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", - "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", - "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", - "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6" + "com.fasterxml.jackson.core" % "jackson-databind" % "2.12.7.1", + "com.fasterxml.jackson.core" % "jackson-annotations" % "2.12.7", + "com.fasterxml.jackson.core" % "jackson-core" % "2.12.7", + "com.fasterxml.jackson.dataformat" % "jackson-dataformat-cbor" % "2.12.6", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.apache.httpcomponents" % "httpclient" % "4.5.13", + "org.apache.httpcomponents" % "httpcore" % "4.4.13", + "commons-logging" % "commons-logging" % "1.2", + "commons-codec" % "commons-codec" % "1.15", + "org.reactivestreams" % "reactive-streams" % "1.0.4", + "joda-time" % "joda-time" % "2.8.1", + "io.netty" % "netty-common" % "4.1.108.Final", + "io.netty" % "netty-codec" % "4.1.108.Final", + "io.netty" % "netty-codec-http" % "4.1.108.Final", + "io.netty" % "netty-codec-http2" % "4.1.108.Final", + "io.netty" % "netty-buffer" % "4.1.108.Final", + "io.netty" % "netty-handler" % "4.1.108.Final", + "io.netty" % "netty-resolver" % "4.1.108.Final", + "io.netty" % "netty-transport" % "4.1.108.Final", + "io.netty" % "netty-transport-classes-epoll" % "4.1.108.Final", + "io.netty" % "netty-transport-native-unix-common" % "4.1.108.Final" ), streams.value.log, - moduleName.value, scalaBinaryVersion.value, + moduleName.value, shouldContainAll = true ) excludedJars - .map(Attributed.blank) }, assembly / assemblyMergeStrategy := { case PathList("META-INF", "io.netty.versions.properties") => @@ -1589,7 +1611,13 @@ lazy val `aws-wrapper` = project val aws = JPMSUtils.filterModulesFromUpdate( update.value, Seq( - "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version + "software.amazon.awssdk" % "auth" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "aws-core" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "sdk-core" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version ), streams.value.log, moduleName.value, From 88aaa76d61249f8102b4821467f852e384865d69 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 15:25:29 +0200 Subject: [PATCH 54/89] Remove empty settings from std-aws --- build.sbt | 4 ---- 1 file changed, 4 deletions(-) diff --git a/build.sbt b/build.sbt index 03b1f98b04e8..96cc08a7c6ee 100644 --- a/build.sbt +++ b/build.sbt @@ -4762,10 +4762,6 @@ lazy val `std-aws` = project .value, Compile / packageBin / artifactPath := `std-aws-polyglot-root` / "std-aws.jar", - libraryDependencies ++= Seq( - ), - Compile / moduleDependencies ++= Seq( - ), Compile / internalModuleDependencies := Seq( (`aws-wrapper` / Compile / exportedModule).value, (`std-base` / Compile / exportedModule).value, From be66f1b5e93f5b831ae9dc412e9831bc8e83da02 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 15:26:00 +0200 Subject: [PATCH 55/89] Fix typo in module name --- std-bits/aws/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index b2bff6a72cad..fabda613072a 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -8,7 +8,7 @@ requires org.enso.std.base; requires org.enso.std.database; requires java.logging; - requires org.enso.wrapper.aws; + requires org.enso.aws.wrapper; provides FileSystemSPI with S3FileSystemSPI; From c97ec776ffd6f43f01f0b7dd35f14db91d12a166 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 15:31:22 +0200 Subject: [PATCH 56/89] aws-wrapper-assembly.jar gets into component directory. --- build.sbt | 1 + 1 file changed, 1 insertion(+) diff --git a/build.sbt b/build.sbt index 96cc08a7c6ee..8974d070bb43 100644 --- a/build.sbt +++ b/build.sbt @@ -686,6 +686,7 @@ lazy val componentModulesPaths = ) val thirdPartyModFiles = thirdPartyMods.map(_.data) val ourMods = Seq( + (`aws-wrapper` / Compile / exportedModuleBin).value, (`common-polyglot-core-utils` / Compile / exportedModuleBin).value, (`engine-common` / Compile / exportedModuleBin).value, (`engine-runner` / Compile / exportedModuleBin).value, From 4a6e55c50f9490fb48162cb68e91f26638f4e11e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 15:37:33 +0200 Subject: [PATCH 57/89] Remove signature from aws-wrapper-assembly jar --- build.sbt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sbt b/build.sbt index 8974d070bb43..adb3ef2bb316 100644 --- a/build.sbt +++ b/build.sbt @@ -1604,6 +1604,8 @@ lazy val `aws-wrapper` = project case PathList("META-INF", "NOTICE.txt") => MergeStrategy.concat case PathList("META-INF", "DEPENDENCIES") => MergeStrategy.discard case PathList("META-INF", "INDEX.LIST") => MergeStrategy.discard + case PathList("META-INF", "SIGNER.RSA") => MergeStrategy.discard + case PathList("META-INF", "SIGNER.SF") => MergeStrategy.discard case PathList("module-info.class") => MergeStrategy.preferProject case _ => MergeStrategy.deduplicate }, From 011494e4c4ced36d8b836f96f0a618a0ad055f5c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 15:54:54 +0200 Subject: [PATCH 58/89] aws-wrapper dependency of std-aws is "provided". So there is just a single jar in AWS/polyglot/java directory --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index adb3ef2bb316..07f6e3311588 100644 --- a/build.sbt +++ b/build.sbt @@ -4782,7 +4782,7 @@ lazy val `std-aws` = project result }.value ) - .dependsOn(`aws-wrapper`) + .dependsOn(`aws-wrapper` % "provided") .dependsOn(`std-base` % "provided") .dependsOn(`std-table` % "provided") .dependsOn(`std-database` % "provided") From 92bc7b4040c1089195c0741271d4d4821a480a2d Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 16:04:57 +0200 Subject: [PATCH 59/89] Ensure aws-wrapper-assembly.jar gets into AWS/polyglot/java directory --- build.sbt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.sbt b/build.sbt index 07f6e3311588..68a7d2f7ca35 100644 --- a/build.sbt +++ b/build.sbt @@ -4770,6 +4770,12 @@ lazy val `std-aws` = project (`std-base` / Compile / exportedModule).value, (`std-database` / Compile / exportedModule).value ), + // This will cause `aws-wrapper-assembly.jar` to be copied into the + // `AWS/polyglot/java` directory. + Compile / unmanagedJars := { + val wrapperJar = (`aws-wrapper` / Compile / exportedModuleBin).value + Seq(Attributed.blank(wrapperJar)) + }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits From 06ca5cef3ca96eda7f366e27efa0ef378fdad152 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 16:05:22 +0200 Subject: [PATCH 60/89] Fix NoSuchElementException in JPMSUtils --- project/JPMSUtils.scala | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/project/JPMSUtils.scala b/project/JPMSUtils.scala index 14f2ea1fe76a..64d5d19e1543 100644 --- a/project/JPMSUtils.scala +++ b/project/JPMSUtils.scala @@ -53,8 +53,11 @@ object JPMSUtils { val distinctModules = modules.distinct val ret = cp.filter(dep => { - val moduleID = dep.metadata.get(AttributeKey[ModuleID]("moduleID")).get - shouldFilterModule(distinctModules, scalaBinaryVersion)(moduleID) + dep.metadata.get(AttributeKey[ModuleID]("moduleID")) match { + case Some(moduleID) => + shouldFilterModule(distinctModules, scalaBinaryVersion)(moduleID) + case None => false + } }) if (shouldContainAll) { From 34dda82d0f18b2515faef0f80053fd0215d22ffd Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 16:07:19 +0200 Subject: [PATCH 61/89] Revert "aws-wrapper-assembly.jar gets into component directory." This reverts commit c97ec776ffd6f43f01f0b7dd35f14db91d12a166. --- build.sbt | 1 - 1 file changed, 1 deletion(-) diff --git a/build.sbt b/build.sbt index 68a7d2f7ca35..804032bf590f 100644 --- a/build.sbt +++ b/build.sbt @@ -686,7 +686,6 @@ lazy val componentModulesPaths = ) val thirdPartyModFiles = thirdPartyMods.map(_.data) val ourMods = Seq( - (`aws-wrapper` / Compile / exportedModuleBin).value, (`common-polyglot-core-utils` / Compile / exportedModuleBin).value, (`engine-common` / Compile / exportedModuleBin).value, (`engine-runner` / Compile / exportedModuleBin).value, From e788644b8ceaf1c307e45bb2fb5b8fa4417a87ea Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 16:10:53 +0200 Subject: [PATCH 62/89] org.enso.std.aws exports org.enso.aws package --- std-bits/aws/src/main/java/module-info.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index fabda613072a..f286abb012e4 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -10,6 +10,8 @@ requires java.logging; requires org.enso.aws.wrapper; + exports org.enso.aws; + provides FileSystemSPI with S3FileSystemSPI; provides DatabaseConnectionDetailsSPI with From 644357dd58d57a73b0dea8ccda7b2e68ef2f11b4 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 16:21:21 +0200 Subject: [PATCH 63/89] slf4j is module dependency of aws-wrapper --- build.sbt | 15 ++++++++++++++- .../aws-wrapper/src/main/java/module-info.java | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 804032bf590f..e1b1c591ddf3 100644 --- a/build.sbt +++ b/build.sbt @@ -1558,6 +1558,9 @@ lazy val `aws-wrapper` = project "commons-logging" % "commons-logging" % "1.2", "commons-codec" % "commons-codec" % "1.15" ), + Compile / moduleDependencies := Seq( + "org.slf4j" % "slf4j-api" % slf4jVersion + ), // Remove all the transitive dependencies of AWS SDK and leave only those that // include `software.amazon.awssdk.*` classes. assembly / assemblyExcludedJars := { @@ -4773,7 +4776,17 @@ lazy val `std-aws` = project // `AWS/polyglot/java` directory. Compile / unmanagedJars := { val wrapperJar = (`aws-wrapper` / Compile / exportedModuleBin).value - Seq(Attributed.blank(wrapperJar)) + val slf4jJar = JPMSUtils.filterModulesFromUpdate( + update.value, + Seq( + "org.slf4j" % "slf4j-api" % slf4jVersion + ), + streams.value.log, + scalaBinaryVersion.value, + moduleName.value, + shouldContainAll = true + ) + Seq(Attributed.blank(wrapperJar)) ++ slf4jJar.map(Attributed.blank) }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index 7815cdee2684..f0757ea65cf6 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -1,4 +1,6 @@ module org.enso.aws.wrapper { + requires org.slf4j; + exports software.amazon.awssdk.auth.credentials; exports software.amazon.awssdk.core.exception; exports software.amazon.awssdk.awscore.exception; From 6c33fefd7005b0c149c4c4e7b3d2f5366302d4a1 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 18:15:44 +0200 Subject: [PATCH 64/89] aws-wrapper depends on org.reactivestreams --- build.sbt | 18 +++++------------- .../aws-wrapper/src/main/java/module-info.java | 1 + 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/build.sbt b/build.sbt index e1b1c591ddf3..3372b03b50bd 100644 --- a/build.sbt +++ b/build.sbt @@ -1559,7 +1559,8 @@ lazy val `aws-wrapper` = project "commons-codec" % "commons-codec" % "1.15" ), Compile / moduleDependencies := Seq( - "org.slf4j" % "slf4j-api" % slf4jVersion + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.reactivestreams" % "reactive-streams" % "1.0.4", ), // Remove all the transitive dependencies of AWS SDK and leave only those that // include `software.amazon.awssdk.*` classes. @@ -4772,21 +4773,12 @@ lazy val `std-aws` = project (`std-base` / Compile / exportedModule).value, (`std-database` / Compile / exportedModule).value ), - // This will cause `aws-wrapper-assembly.jar` to be copied into the + // This will cause `aws-wrapper-assembly.jar` and its dependencies to be copied into the // `AWS/polyglot/java` directory. Compile / unmanagedJars := { val wrapperJar = (`aws-wrapper` / Compile / exportedModuleBin).value - val slf4jJar = JPMSUtils.filterModulesFromUpdate( - update.value, - Seq( - "org.slf4j" % "slf4j-api" % slf4jVersion - ), - streams.value.log, - scalaBinaryVersion.value, - moduleName.value, - shouldContainAll = true - ) - Seq(Attributed.blank(wrapperJar)) ++ slf4jJar.map(Attributed.blank) + val wrapperDeps = (`aws-wrapper` / Compile / modulePath).value + Seq(Attributed.blank(wrapperJar)) ++ wrapperDeps.map(Attributed.blank) }, Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index f0757ea65cf6..97466e3080f7 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -1,5 +1,6 @@ module org.enso.aws.wrapper { requires org.slf4j; + requires org.reactivestreams; // Automatic module exports software.amazon.awssdk.auth.credentials; exports software.amazon.awssdk.core.exception; From 4c98a58c1b062592ca7362304d81ce2f5e42f186 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 18:16:07 +0200 Subject: [PATCH 65/89] aws-wrapper uses its own service. Fixes ServiceConfigurationError. --- build.sbt | 1 + lib/java/aws-wrapper/src/main/java/module-info.java | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/build.sbt b/build.sbt index 3372b03b50bd..0dd5f3660dfb 100644 --- a/build.sbt +++ b/build.sbt @@ -1623,6 +1623,7 @@ lazy val `aws-wrapper` = project "software.amazon.awssdk" % "profiles" % awsJavaSdkV2Version, "software.amazon.awssdk" % "regions" % awsJavaSdkV2Version, "software.amazon.awssdk" % "http-client-spi" % awsJavaSdkV2Version, + "software.amazon.awssdk" % "apache-client" % awsJavaSdkV2Version, "software.amazon.awssdk" % "s3" % awsJavaSdkV2Version ), streams.value.log, diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index 97466e3080f7..4af5c884b77c 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -11,4 +11,8 @@ exports software.amazon.awssdk.profiles; exports software.amazon.awssdk.regions; exports software.amazon.awssdk.regions.providers; + + uses software.amazon.awssdk.http.SdkHttpService; + provides software.amazon.awssdk.http.SdkHttpService with + software.amazon.awssdk.http.apache.ApacheSdkHttpService; } From e81e27c48857a32de61a9c92bbdbb3f0bd9f4e51 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 18:35:52 +0200 Subject: [PATCH 66/89] Add some dependencies to aws-wrapper --- build.sbt | 3 +++ lib/java/aws-wrapper/src/main/java/module-info.java | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/build.sbt b/build.sbt index 0dd5f3660dfb..551a1bf1bedc 100644 --- a/build.sbt +++ b/build.sbt @@ -1561,6 +1561,9 @@ lazy val `aws-wrapper` = project Compile / moduleDependencies := Seq( "org.slf4j" % "slf4j-api" % slf4jVersion, "org.reactivestreams" % "reactive-streams" % "1.0.4", + "org.apache.httpcomponents" % "httpcore" % "4.4.13", + "org.apache.httpcomponents" % "httpclient" % "4.5.13", + "commons-logging" % "commons-logging" % "1.2", ), // Remove all the transitive dependencies of AWS SDK and leave only those that // include `software.amazon.awssdk.*` classes. diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index 4af5c884b77c..5f0159989b43 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -1,6 +1,11 @@ module org.enso.aws.wrapper { + // Class software.amazon.awssdk.protocols.query.unmarshall.XmlDomParser acesses javax.xml.stream.XMLInputFactory + requires java.xml; requires org.slf4j; requires org.reactivestreams; // Automatic module + requires org.apache.httpcomponents.httpcore; // Automatic module + requires org.apache.httpcomponents.httpclient; // Automatic module + requires commons.logging; // Automatic module (with derived name) exports software.amazon.awssdk.auth.credentials; exports software.amazon.awssdk.core.exception; From 686f24d2e37ddcf8fab569bd3ff3c8fadd3ad252 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 18:36:43 +0200 Subject: [PATCH 67/89] Add necessary exports --- lib/java/aws-wrapper/src/main/java/module-info.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/java/aws-wrapper/src/main/java/module-info.java b/lib/java/aws-wrapper/src/main/java/module-info.java index 5f0159989b43..5b3382a370ec 100644 --- a/lib/java/aws-wrapper/src/main/java/module-info.java +++ b/lib/java/aws-wrapper/src/main/java/module-info.java @@ -8,7 +8,9 @@ requires commons.logging; // Automatic module (with derived name) exports software.amazon.awssdk.auth.credentials; + exports software.amazon.awssdk.core; exports software.amazon.awssdk.core.exception; + exports software.amazon.awssdk.core.sync; exports software.amazon.awssdk.awscore.exception; exports software.amazon.awssdk.http; exports software.amazon.awssdk.services.s3; From c5b1959fe51186cfdd19bf8f7b22aa76df72d154 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 18:42:08 +0200 Subject: [PATCH 68/89] fmt --- build.sbt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/build.sbt b/build.sbt index 551a1bf1bedc..61feb95bd6eb 100644 --- a/build.sbt +++ b/build.sbt @@ -1559,11 +1559,11 @@ lazy val `aws-wrapper` = project "commons-codec" % "commons-codec" % "1.15" ), Compile / moduleDependencies := Seq( - "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.reactivestreams" % "reactive-streams" % "1.0.4", - "org.apache.httpcomponents" % "httpcore" % "4.4.13", - "org.apache.httpcomponents" % "httpclient" % "4.5.13", - "commons-logging" % "commons-logging" % "1.2", + "org.slf4j" % "slf4j-api" % slf4jVersion, + "org.reactivestreams" % "reactive-streams" % "1.0.4", + "org.apache.httpcomponents" % "httpcore" % "4.4.13", + "org.apache.httpcomponents" % "httpclient" % "4.5.13", + "commons-logging" % "commons-logging" % "1.2" ), // Remove all the transitive dependencies of AWS SDK and leave only those that // include `software.amazon.awssdk.*` classes. @@ -4780,7 +4780,7 @@ lazy val `std-aws` = project // This will cause `aws-wrapper-assembly.jar` and its dependencies to be copied into the // `AWS/polyglot/java` directory. Compile / unmanagedJars := { - val wrapperJar = (`aws-wrapper` / Compile / exportedModuleBin).value + val wrapperJar = (`aws-wrapper` / Compile / exportedModuleBin).value val wrapperDeps = (`aws-wrapper` / Compile / modulePath).value Seq(Attributed.blank(wrapperJar)) ++ wrapperDeps.map(Attributed.blank) }, From 2bd96d4e46aade0ca44bc08d6dab0a918502dc75 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 19:04:32 +0200 Subject: [PATCH 69/89] FileSystemSPI uses Lookup instead of ServiceLoader --- .../main/java/org/enso/base/file_system/FileSystemSPI.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/file_system/FileSystemSPI.java b/std-bits/base/src/main/java/org/enso/base/file_system/FileSystemSPI.java index 78ba411c31d9..0017d1d42b40 100644 --- a/std-bits/base/src/main/java/org/enso/base/file_system/FileSystemSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/file_system/FileSystemSPI.java @@ -1,14 +1,13 @@ package org.enso.base.file_system; import java.util.ServiceLoader; +import org.enso.base.lookup.Lookup; import org.enso.base.polyglot.EnsoMeta; import org.graalvm.polyglot.Value; public abstract class FileSystemSPI { - private static final ServiceLoader loader = - ServiceLoader.load( - org.enso.base.file_system.FileSystemSPI.class, - org.enso.base.file_format.FileFormatSPI.class.getClassLoader()); + private static final Lookup loader = + Lookup.lookup((l) -> ServiceLoader.load(l, org.enso.base.file_system.FileSystemSPI.class)); public static Value get_type(String protocol, boolean refresh) { if (refresh) { From cf605cd05fb88c4d1eefe59897f6576b3f69859f Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 19:26:03 +0200 Subject: [PATCH 70/89] org.enso.std.aws provides DataLinkSPI. And DataLinkSPI is implemented with Lookup instead of ServiceLoader. --- std-bits/aws/src/main/java/module-info.java | 3 +++ .../src/main/java/org/enso/base/enso_cloud/DataLinkSPI.java | 5 +++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index f286abb012e4..1654568e813b 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -1,5 +1,6 @@ import org.enso.aws.database.RedshiftConnectionDetailsSPI; import org.enso.aws.file_system.S3FileSystemSPI; +import org.enso.base.enso_cloud.DataLinkSPI; import org.enso.base.file_system.FileSystemSPI; import org.enso.database.DatabaseConnectionDetailsSPI; @@ -14,6 +15,8 @@ provides FileSystemSPI with S3FileSystemSPI; + provides DataLinkSPI with + org.enso.aws.file_system.S3DataLinkSPI; provides DatabaseConnectionDetailsSPI with RedshiftConnectionDetailsSPI; } diff --git a/std-bits/base/src/main/java/org/enso/base/enso_cloud/DataLinkSPI.java b/std-bits/base/src/main/java/org/enso/base/enso_cloud/DataLinkSPI.java index 84cbe3cf341b..4ec005335e92 100644 --- a/std-bits/base/src/main/java/org/enso/base/enso_cloud/DataLinkSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/enso_cloud/DataLinkSPI.java @@ -2,6 +2,7 @@ import java.util.ServiceLoader; import java.util.stream.Collectors; +import org.enso.base.lookup.Lookup; import org.enso.base.polyglot.EnsoMeta; import org.graalvm.polyglot.Value; @@ -11,8 +12,8 @@ * type should return a configured datalink instance that can later be `read`. */ public abstract class DataLinkSPI { - private static final ServiceLoader loader = - ServiceLoader.load(DataLinkSPI.class, DataLinkSPI.class.getClassLoader()); + private static final Lookup loader = + Lookup.lookup((layer) -> ServiceLoader.load(layer, DataLinkSPI.class)); public void reload() { loader.reload(); From 12ebe804437d4c98faacc4b196ee08f2311b3957 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 20:26:02 +0200 Subject: [PATCH 71/89] Better error message on duplicated service providers --- .../java/org/enso/base/lookup/Lookup.java | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java index d74f58407de9..f3d7c5f80d3b 100644 --- a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java +++ b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java @@ -1,8 +1,8 @@ package org.enso.base.lookup; -import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.ServiceLoader; import java.util.function.Function; @@ -34,22 +34,48 @@ public void reload() { } private List> findAll() { - var arr = new ArrayList>(); + var serviceProviders = new LinkedHashMap, ModuleLayer>(); if (System.getProperties().get("enso.class.path") instanceof Collection layers) { for (var obj : layers) { if (obj instanceof ModuleLayer layer) { factory.apply(layer).stream() .forEach( (p) -> { - arr.add(p); + if (serviceProviders.containsKey(p)) { + var prevLayer = serviceProviders.get(p); + throw new IllegalStateException( + "Error: Duplicate provider found: " + + providerToString(p) + + ". " + + "Previous provider in layer '" + + prevLayer + + "'. " + + "Current provider in layer '" + + layer + + "'."); + } + serviceProviders.put(p, layer); }); } } } - return arr; + return serviceProviders.keySet().stream().toList(); } public static Lookup lookup(Function> factory) { return new Lookup<>(factory); } + + private static String providerToString(ServiceLoader.Provider provider) { + var mod = provider.type().getModule(); + var modLayer = mod.getLayer(); + var tp = provider.type(); + return "Provider[type='" + + tp + + "', module='" + + mod.getName() + + "', moduleLayer={" + + modLayer + + "}]"; + } } From a31fb9c676efe316655af21493ad04a0fb5401ed Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Oct 2024 20:49:07 +0200 Subject: [PATCH 72/89] Duplicate provider is just a warning --- .../java/org/enso/base/lookup/Lookup.java | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java index f3d7c5f80d3b..439bf03f22bb 100644 --- a/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java +++ b/std-bits/base/src/main/java/org/enso/base/lookup/Lookup.java @@ -6,11 +6,14 @@ import java.util.List; import java.util.ServiceLoader; import java.util.function.Function; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Stream; public final class Lookup implements Iterable { private final Function> factory; private List> found; + private static final Logger logger = Logger.getLogger(Lookup.class.getName()); private Lookup(Function> factory) { this.factory = factory; @@ -43,16 +46,12 @@ private List> findAll() { (p) -> { if (serviceProviders.containsKey(p)) { var prevLayer = serviceProviders.get(p); - throw new IllegalStateException( - "Error: Duplicate provider found: " - + providerToString(p) - + ". " - + "Previous provider in layer '" - + prevLayer - + "'. " - + "Current provider in layer '" - + layer - + "'."); + logger.log( + Level.WARNING, + String.format( + "Duplicate provider found: %s. Previous provider in layer '%s'." + + " Current provider in layer '%s'.", + providerToString(p), prevLayer, layer)); } serviceProviders.put(p, layer); }); From 77dc8de70f891be66d203e7fde23e2d19010f134 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 04:56:02 +0200 Subject: [PATCH 73/89] List is easier to == than Array --- .../main/java/org/enso/interpreter/runtime/EnsoContext.java | 3 ++- lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala | 4 ++-- lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala | 2 +- lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala | 3 ++- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index 3b5a7621da92..dfc4f6cff22a 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -70,6 +70,7 @@ import org.enso.pkg.QualifiedName; import org.enso.polyglot.debugger.IdExecutionService; import org.graalvm.options.OptionKey; +import scala.jdk.javaapi.CollectionConverters; import scala.jdk.javaapi.OptionConverters; /** @@ -478,7 +479,7 @@ public synchronized EnsoClassPath findClassPath(Package pkg) { throw new NullPointerException("No polyglot dir for " + pkg); } var ch = polyDir.resolve("java"); - var requires = Arrays.asList(pkg.getConfig().requires()); + var requires = CollectionConverters.asJava(pkg.getConfig().requires()); var parents = requires.stream() .map( diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala index 7f078df32705..ab3f4e5c920d 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Config.scala @@ -110,7 +110,7 @@ case class Config( edition: Option[Editions.RawEdition], preferLocalLibraries: Boolean, componentGroups: Option[ComponentGroups], - requires: Array[String] + requires: List[String] ) { /** Converts the configuration into a YAML representation. */ @@ -238,7 +238,7 @@ object Config { edition, preferLocalLibraries, componentGroups, - requires.toArray + requires ) } } diff --git a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala index ca60dcb7dadb..7a0050bd2036 100644 --- a/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala +++ b/lib/scala/pkg/src/main/scala/org/enso/pkg/Package.scala @@ -301,7 +301,7 @@ class PackageManager[F](implicit val fileSystem: FileSystem[F]) { preferLocalLibraries = true, maintainers = maintainers, componentGroups = componentGroups, - requires = Array() + requires = List() ) create(root, config, template) } diff --git a/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala b/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala index d813d7fc0912..8722d3842060 100644 --- a/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala +++ b/lib/scala/pkg/src/test/scala/org/enso/pkg/ConfigSpec.scala @@ -32,7 +32,8 @@ class ConfigSpec Contact(None, Some("c@example.com")) ), preferLocalLibraries = true, - componentGroups = None + componentGroups = None, + requires = List() ) val deserialized = Config.fromYaml(config.toYaml).get deserialized shouldEqual config From 6388881410f9044f36c1677ea42245454afebc4b Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 05:18:02 +0200 Subject: [PATCH 74/89] runtime-integration-tests are loading classes with empty EnsoClassPath --- .../main/java/org/enso/interpreter/runtime/EnsoContext.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index dfc4f6cff22a..cc4a02cc8aeb 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -612,8 +612,7 @@ private Object lookupHostSymbol(ClassLoader loader, String fqn) if (findGuestJava() == null) { Class clazz; if (loader == null) { - var baseModule = Object.class.getModule(); - clazz = Class.forName(baseModule, fqn); + clazz = Class.forName(fqn); } else { clazz = loader.loadClass(fqn); } From 00b47b4b1ad74654744344b84e28748965a3b646 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 05:22:01 +0200 Subject: [PATCH 75/89] Java.lookup_class has been removed --- .../enso/interpreter/test/semantic/PolyglotTest.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/semantic/PolyglotTest.scala b/engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/semantic/PolyglotTest.scala index c0f8463a0b21..21a5114435c0 100644 --- a/engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/semantic/PolyglotTest.scala +++ b/engine/runtime-integration-tests/src/test/scala/org/enso/interpreter/test/semantic/PolyglotTest.scala @@ -17,9 +17,10 @@ class PolyglotTest extends InterpreterTest { val code = """from Standard.Base import all |import Standard.Base.Data.Array.Array + |polyglot java import org.enso.example.TestClass | |main = - | class = Java.lookup_class "org.enso.example.TestClass" + | class = TestClass | method = Polyglot.get_member class "add" | Polyglot.execute method ([1, 2].to_array) |""".stripMargin @@ -60,9 +61,10 @@ class PolyglotTest extends InterpreterTest { "allow instantiating objects and calling methods on them" in { val code = """from Standard.Base import all + |polyglot java import org.enso.example.TestClass | |main = - | class = Java.lookup_class "org.enso.example.TestClass" + | class = TestClass | instance = Polyglot.new class [x -> x * 2] | Polyglot.invoke instance "callFunctionAndIncrement" [10] |""".stripMargin @@ -72,9 +74,10 @@ class PolyglotTest extends InterpreterTest { "allow listing available members of an object" in { val code = """from Standard.Base import all + |polyglot java import org.enso.example.TestClass | |main = - | class = Java.lookup_class "org.enso.example.TestClass" + | class = TestClass | instance = Polyglot.new class [] | members = Polyglot.get_members instance | IO.println members.length From 4173822983087d0ee59adf15a5c863a8e7e585b9 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 06:05:38 +0200 Subject: [PATCH 76/89] Using List() at the right places --- .../libraries/ComponentGroupsResolverSpec.scala | 4 ++-- .../libraries/ComponentGroupsValidatorSpec.scala | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala index 929a39a51cca..e0a1be8fdcb1 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsResolverSpec.scala @@ -278,7 +278,7 @@ object ComponentGroupsResolverSpec { license = "", authors = Nil, maintainers = Nil, - requires = Array(), + requires = List(), edition = None, preferLocalLibraries = true, componentGroups = Some(componentGroups) @@ -297,7 +297,7 @@ object ComponentGroupsResolverSpec { license = "", authors = Nil, maintainers = Nil, - requires = Array(), + requires = List(), edition = None, preferLocalLibraries = true, componentGroups = None diff --git a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala index 84e9253e3039..69ee367981fb 100644 --- a/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala +++ b/engine/language-server/src/test/scala/org/enso/languageserver/libraries/ComponentGroupsValidatorSpec.scala @@ -188,7 +188,7 @@ object ComponentGroupsValidatorSpec { authors = Nil, maintainers = Nil, edition = None, - requires = Array(), + requires = List(), preferLocalLibraries = true, componentGroups = Some( ComponentGroups( From f0c2ae450d1b27aa4779f14fa34f57553cbcc992 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 08:16:48 +0200 Subject: [PATCH 77/89] std.table module isn't present in NI enso build --- .../enso-test-java-helpers/src/main/java/module-info.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java b/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java index 309554a01397..6cabb380f852 100644 --- a/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java +++ b/test/Base_Tests/polyglot-sources/enso-test-java-helpers/src/main/java/module-info.java @@ -1,6 +1,6 @@ module org.enso.test.java_helpers { requires static org.graalvm.polyglot; - requires org.enso.std.table; + requires static org.enso.std.table; exports org.enso.base_test_helpers; exports org.enso.table_test_helpers; From f11a26aed231df8a66a930f889b735b3dec8ea70 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 08:17:08 +0200 Subject: [PATCH 78/89] Use Lookup to lookup DatabaseConnectionDetailsSPI --- .../org/enso/database/DatabaseConnectionDetailsSPI.java | 6 +++--- test/AWS_Tests/package.yaml | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/std-bits/database/src/main/java/org/enso/database/DatabaseConnectionDetailsSPI.java b/std-bits/database/src/main/java/org/enso/database/DatabaseConnectionDetailsSPI.java index ac075118f3b3..c9de669d1c7a 100644 --- a/std-bits/database/src/main/java/org/enso/database/DatabaseConnectionDetailsSPI.java +++ b/std-bits/database/src/main/java/org/enso/database/DatabaseConnectionDetailsSPI.java @@ -1,13 +1,13 @@ package org.enso.database; import java.util.ServiceLoader; +import org.enso.base.lookup.Lookup; import org.enso.base.polyglot.EnsoMeta; import org.graalvm.polyglot.Value; public abstract class DatabaseConnectionDetailsSPI { - private static final ServiceLoader loader = - ServiceLoader.load( - DatabaseConnectionDetailsSPI.class, DatabaseConnectionDetailsSPI.class.getClassLoader()); + private static final Lookup loader = + Lookup.lookup((l) -> ServiceLoader.load(l, DatabaseConnectionDetailsSPI.class)); /** * Returns an array of pairs, where the first element is the user facing connection name and the diff --git a/test/AWS_Tests/package.yaml b/test/AWS_Tests/package.yaml index 3e7516ddccaa..729ec7948b63 100644 --- a/test/AWS_Tests/package.yaml +++ b/test/AWS_Tests/package.yaml @@ -5,3 +5,5 @@ license: MIT author: enso-dev@enso.org maintainer: enso-dev@enso.org prefer-local-libraries: true +requires: + - Standard.Database From b7f654270b209ab2dc99c4895080fb41dfa3562e Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 14:09:55 +0200 Subject: [PATCH 79/89] An unexpected panic was thrown: (No_Such_Method.Error org.enso.base.net.http.UrlencodedBodyBuilder UnresolvedSymbol) --- std-bits/base/src/main/java/module-info.java | 1 + 1 file changed, 1 insertion(+) diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index 87a99d26d9aa..256b240f79e7 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -46,6 +46,7 @@ opens org.enso.base.numeric; opens org.enso.base.net; + opens org.enso.base.net.http; opens org.enso.base.polyglot; opens org.enso.base.random; opens org.enso.base.statistics; From baeff36a708de1017157c81d190a2b54840b4d78 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 14:47:44 +0200 Subject: [PATCH 80/89] Initialize ModuleLayer for a library sooner than a class is loaded from it --- build.sbt | 3 +++ distribution/lib/Standard/Microsoft/0.0.0-dev/package.yaml | 3 +++ .../java/org/enso/interpreter/runtime/EnsoContext.java | 7 +++++-- .../scala/org/enso/interpreter/runtime/IrToTruffle.scala | 4 +++- std-bits/microsoft/src/main/java/module-info.java | 1 + test/Microsoft_Tests/package.yaml | 2 ++ 6 files changed, 17 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 61feb95bd6eb..08698a8a4428 100644 --- a/build.sbt +++ b/build.sbt @@ -4846,6 +4846,9 @@ lazy val `std-microsoft` = project libraryDependencies ++= Seq( "com.microsoft.sqlserver" % "mssql-jdbc" % mssqlserverJDBCVersion ), + Compile / moduleDependencies ++= Seq( + "com.microsoft.sqlserver" % "mssql-jdbc" % mssqlserverJDBCVersion + ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value val _ = StdBits diff --git a/distribution/lib/Standard/Microsoft/0.0.0-dev/package.yaml b/distribution/lib/Standard/Microsoft/0.0.0-dev/package.yaml index b5b48b74c9c7..89ba5ede1e86 100644 --- a/distribution/lib/Standard/Microsoft/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Microsoft/0.0.0-dev/package.yaml @@ -8,3 +8,6 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Base + - Standard.Database diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java index cc4a02cc8aeb..42a22342c9ed 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java @@ -578,9 +578,12 @@ public boolean isColorTerminalOutput() { */ @TruffleBoundary public TruffleObject lookupJavaClass(Package pkg, String className) { - var binaryName = new StringBuilder(className); - var collectedExceptions = new ArrayList(); org.enso.interpreter.runtime.EnsoClassPath cp = findClassPath(pkg); + if (className == null) { + return null; + } + var collectedExceptions = new ArrayList(); + var binaryName = new StringBuilder(className); for (; ; ) { var fqn = binaryName.toString(); try { diff --git a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala index 667f4e3af81e..8cde4ceb38a3 100644 --- a/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala +++ b/engine/runtime/src/main/scala/org/enso/interpreter/runtime/IrToTruffle.scala @@ -240,7 +240,8 @@ class IrToTruffle( module: Module ): Unit = module.imports.foreach { - case poly @ imports.Polyglot(i: imports.Polyglot.Java, _, _, _) => + case poly @ imports.Polyglot(i: imports.Polyglot.Java, _, _, _) => { + context.lookupJavaClass(p, null) this.scopeBuilder.registerPolyglotSymbol( poly.getVisibleName, () => { @@ -248,6 +249,7 @@ class IrToTruffle( hostSymbol } ) + } case _: Import.Module => case _: Error => } diff --git a/std-bits/microsoft/src/main/java/module-info.java b/std-bits/microsoft/src/main/java/module-info.java index 487571496634..f6abccc42e77 100644 --- a/std-bits/microsoft/src/main/java/module-info.java +++ b/std-bits/microsoft/src/main/java/module-info.java @@ -4,6 +4,7 @@ module org.enso.std.microsoft { requires org.enso.std.base; requires org.enso.std.database; + requires com.microsoft.sqlserver.jdbc; provides DatabaseConnectionDetailsSPI with SQLServerConnectionDetailsSPI; diff --git a/test/Microsoft_Tests/package.yaml b/test/Microsoft_Tests/package.yaml index d6db8970ee87..a9c6b2cf9fbd 100644 --- a/test/Microsoft_Tests/package.yaml +++ b/test/Microsoft_Tests/package.yaml @@ -5,3 +5,5 @@ license: MIT author: enso-dev@enso.org maintainer: enso-dev@enso.org prefer-local-libraries: true +requires: + - Standard.Microsoft From a9bc32f04452587719571dd04640c6fa2c5682d7 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 14:52:04 +0200 Subject: [PATCH 81/89] There is no runtime-fat-jar anymore --- .../src/main/java/module-info.java | 43 ------------------- 1 file changed, 43 deletions(-) delete mode 100644 engine/runtime-fat-jar/src/main/java/module-info.java diff --git a/engine/runtime-fat-jar/src/main/java/module-info.java b/engine/runtime-fat-jar/src/main/java/module-info.java deleted file mode 100644 index 1cdfee23bc47..000000000000 --- a/engine/runtime-fat-jar/src/main/java/module-info.java +++ /dev/null @@ -1,43 +0,0 @@ -open module org.enso.runtime { - requires java.base; - requires java.net.http; - // Because of akka.util.Unsafe - requires jdk.unsupported; - requires org.enso.syntax; - // org.enso.profiling and org.enso.doc are not needed for runtime. It is needed for runner.jar, - // but we have to include it here so that the forwarding in IsolatedClassLoader - // works. - requires org.enso.profiling; - requires org.enso.ydoc; - requires org.graalvm.polyglot; - requires org.graalvm.truffle; - requires static org.slf4j; - // ydoc-server - requires io.helidon.webclient; - requires io.helidon.webclient.websocket; - requires io.helidon.webserver; - requires io.helidon.webserver.websocket; - - uses org.slf4j.spi.SLF4JServiceProvider; - uses org.enso.interpreter.instrument.HandlerFactory; - - provides com.oracle.truffle.api.provider.TruffleLanguageProvider with - org.enso.interpreter.EnsoLanguageProvider, - org.enso.interpreter.epb.EpbLanguageProvider; - - provides com.oracle.truffle.api.instrumentation.provider.TruffleInstrumentProvider with - org.enso.interpreter.instrument.ReplDebuggerInstrumentProvider, - org.enso.interpreter.instrument.RuntimeServerInstrumentProvider, - org.enso.interpreter.instrument.IdExecutionInstrumentProvider; - - - // java.beans.Transient needed by Jackson jackson.databind.ext.Java7SupportImpl - requires java.desktop; - // also needed by Jackson to avoid - // com.fasterxml.jackson.databind.exc.InvalidTypeIdException: - // Could not resolve type id 'org.enso.polyglot.data.Tree$Node' as a subtype of - // `org.enso.polyglot.data.Tree$Node`: Not a subtype - requires java.se; - // needed by org.apache.xmlbeans - requires jdk.xml.dom; -} From ec03e1080fbe19f7b7363206e876e72298bbc0f2 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 8 Oct 2024 16:33:12 +0200 Subject: [PATCH 82/89] Add JDK_MODULES to small JDK of engine-runner native image --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 08698a8a4428..801effd9c6a0 100644 --- a/build.sbt +++ b/build.sbt @@ -3633,7 +3633,7 @@ lazy val `engine-runner` = project val NI_MODULES = "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise" val JDK_MODULES = - "jdk.localedata,jdk.httpserver,java.naming,java.net.http" + "jdk.localedata,jdk.httpserver,java.naming,java.net.http,java.desktop,java.xml.crypto,jdk.xml.dom,java.sql.rowset" val DEBUG_MODULES = "jdk.jdwp.agent" val PYTHON_MODULES = "jdk.security.auth,java.naming" From 1d4b172bebc3e8761b80aef8d6b2a3f2653b6587 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Tue, 8 Oct 2024 17:24:47 +0200 Subject: [PATCH 83/89] Catch also ResolutionException --- .../main/java/org/enso/interpreter/runtime/EnsoClassPath.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 5f2d85e29ad9..0fc2169c73b0 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -4,6 +4,7 @@ import java.lang.module.Configuration; import java.lang.module.FindException; import java.lang.module.ModuleFinder; +import java.lang.module.ResolutionException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; @@ -46,7 +47,7 @@ private static ModuleLayer bootLayer() { static EnsoClassPath create(Path file, List parents, TruffleLogger log) { try { return createImpl(file, parents); - } catch (FindException e) { + } catch (FindException | ResolutionException e) { var sb = new StringBuilder(); sb.append("Cannot instantiate modules at ").append(file); log.log(Level.FINE, sb.toString(), e); From 2bf4df1a3d88d507373cd801e427dad4267bf110 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 9 Oct 2024 06:34:29 +0200 Subject: [PATCH 84/89] Always add at least boot layer as parent --- .../java/org/enso/interpreter/runtime/EnsoClassPath.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java index 0fc2169c73b0..2d249e3401e8 100644 --- a/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java +++ b/engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoClassPath.java @@ -28,7 +28,6 @@ private EnsoClassPath( if (cntrl != null) { // cannot be null layer.getClass(); - loader.getClass(); } this.cntrl = cntrl; this.layer = layer; @@ -101,6 +100,10 @@ private static EnsoClassPath createImpl(Path file, List parents) parentLayers.add(cp.layer); parentCfgs.add(cp.layer.configuration()); } + if (parentLayers.isEmpty()) { + parentLayers.add(ModuleLayer.boot()); + parentCfgs.add(ModuleLayer.boot().configuration()); + } var parentLoader = bootLayer().findLoader("java.base"); var cfg = Configuration.resolveAndBind(finder, parentCfgs, ModuleFinder.ofSystem(), moduleNames); From e8c220b8218148d864388639175179f32b3c430a Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Wed, 9 Oct 2024 07:10:37 +0200 Subject: [PATCH 85/89] Make Tableau_Tests pass --- build.sbt | 2 +- distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml | 2 ++ std-bits/table/src/main/java/module-info.java | 8 ++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/build.sbt b/build.sbt index 801effd9c6a0..863994d7c8c3 100644 --- a/build.sbt +++ b/build.sbt @@ -4967,7 +4967,7 @@ lazy val `std-tableau` = project `std-tableau-polyglot-root` / "std-tableau.jar", libraryDependencies ++= Seq( "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "net.java.dev.jna" % "jna-platform" % jnaVersion + "net.java.dev.jna" % "jna-platform" % jnaVersion % "provided" ), Compile / packageBin := Def.task { val result = (Compile / packageBin).value diff --git a/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml b/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml index d757c865dabe..74483152f96e 100644 --- a/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml +++ b/distribution/lib/Standard/Tableau/0.0.0-dev/package.yaml @@ -8,3 +8,5 @@ authors: maintainers: - name: Enso Team email: contact@enso.org +requires: + - Standard.Table diff --git a/std-bits/table/src/main/java/module-info.java b/std-bits/table/src/main/java/module-info.java index f1986f44de85..c717596c58e8 100644 --- a/std-bits/table/src/main/java/module-info.java +++ b/std-bits/table/src/main/java/module-info.java @@ -28,14 +28,18 @@ opens org.enso.table.expressions; opens org.enso.table.formatting; opens org.enso.table.excel; - opens org.enso.table.problems; + + exports org.enso.table.problems; + opens org.enso.table.parsing; opens org.enso.table.parsing.problems; exports org.enso.table.data.column.storage; opens org.enso.table.data.column.storage.numeric; - opens org.enso.table.data.column.builder; + + exports org.enso.table.data.column.builder; + opens org.enso.table.data.column.operation; opens org.enso.table.data.column.operation.cast; From ee79b07ee6a8bdaa669e1966c37944ed65618daf Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 18 Oct 2024 18:18:22 +0200 Subject: [PATCH 86/89] SQLServerDataLinkSPI is registered in module-info --- .../src/main/java/org/enso/microsoft/SQLServerDataLinkSPI.java | 1 - 1 file changed, 1 deletion(-) diff --git a/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerDataLinkSPI.java b/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerDataLinkSPI.java index c8acfd42cef7..b9cd5693a0f2 100644 --- a/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerDataLinkSPI.java +++ b/std-bits/microsoft/src/main/java/org/enso/microsoft/SQLServerDataLinkSPI.java @@ -2,7 +2,6 @@ import org.enso.base.enso_cloud.DataLinkSPI; -@org.openide.util.lookup.ServiceProvider(service = DataLinkSPI.class) public class SQLServerDataLinkSPI extends DataLinkSPI { @Override protected String getModuleName() { From 4c466573dd4592b4193e45b43c1f4d358ad86f85 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 21 Oct 2024 14:11:44 +0200 Subject: [PATCH 87/89] Backing out ec03e1080fbe19f7b7363206e876e72298bbc0f2 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index c4db92a21d41..e9c9ad7ce3f6 100644 --- a/build.sbt +++ b/build.sbt @@ -3655,7 +3655,7 @@ lazy val `engine-runner` = project val NI_MODULES = "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise" val JDK_MODULES = - "jdk.localedata,jdk.httpserver,java.naming,java.net.http,java.desktop,java.xml.crypto,jdk.xml.dom,java.sql.rowset" + "jdk.localedata,jdk.httpserver,java.naming,java.net.http" val DEBUG_MODULES = "jdk.jdwp.agent" val PYTHON_MODULES = "jdk.security.auth,java.naming" From 208d05d9b48ac24bb88b88191ab8d262c2125339 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 21 Oct 2024 16:41:34 +0200 Subject: [PATCH 88/89] Include chrome and DAP debugging tools in the enso executable --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index e9c9ad7ce3f6..bc79ae2dfeb7 100644 --- a/build.sbt +++ b/build.sbt @@ -3572,7 +3572,7 @@ lazy val `engine-runner` = project Compile / run / mainClass := Some("org.enso.runner.Main"), commands += WithDebugCommand.withDebug, inConfig(Compile)(truffleRunOptionsSettings), - libraryDependencies ++= GraalVM.modules ++ Seq( + libraryDependencies ++= GraalVM.modules ++ GraalVM.toolsPkgs ++ Seq( "org.graalvm.polyglot" % "polyglot" % graalMavenPackagesVersion, "org.graalvm.sdk" % "polyglot-tck" % graalMavenPackagesVersion % Provided, "commons-cli" % "commons-cli" % commonsCliVersion, @@ -3655,7 +3655,7 @@ lazy val `engine-runner` = project val NI_MODULES = "org.graalvm.nativeimage,org.graalvm.nativeimage.builder,org.graalvm.nativeimage.base,org.graalvm.nativeimage.driver,org.graalvm.nativeimage.librarysupport,org.graalvm.nativeimage.objectfile,org.graalvm.nativeimage.pointsto,com.oracle.graal.graal_enterprise,com.oracle.svm.svm_enterprise" val JDK_MODULES = - "jdk.localedata,jdk.httpserver,java.naming,java.net.http" + "jdk.localedata,jdk.httpserver,java.naming,java.net.http,java.xml" val DEBUG_MODULES = "jdk.jdwp.agent" val PYTHON_MODULES = "jdk.security.auth,java.naming" From 986fb62193171ace020a1f8f3adce8139a999b16 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 22 Nov 2024 16:22:18 +0100 Subject: [PATCH 89/89] uses/provides instead of @ServiceProvider --- std-bits/aws/src/main/java/module-info.java | 1 + std-bits/base/src/main/java/module-info.java | 6 ++++++ .../main/java/org/enso/base/read/BaseReadManyReturnSPI.java | 3 +-- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/std-bits/aws/src/main/java/module-info.java b/std-bits/aws/src/main/java/module-info.java index 1654568e813b..0881f23a454a 100644 --- a/std-bits/aws/src/main/java/module-info.java +++ b/std-bits/aws/src/main/java/module-info.java @@ -10,6 +10,7 @@ requires org.enso.std.database; requires java.logging; requires org.enso.aws.wrapper; + requires java.net.http; exports org.enso.aws; diff --git a/std-bits/base/src/main/java/module-info.java b/std-bits/base/src/main/java/module-info.java index 256b240f79e7..b9283a26e08e 100644 --- a/std-bits/base/src/main/java/module-info.java +++ b/std-bits/base/src/main/java/module-info.java @@ -8,6 +8,8 @@ import org.enso.base.file_format.XMLFormatSPI; import org.enso.base.file_system.FileSystemSPI; import org.enso.base.net.http.HTTPFetchDataLinkSPI; +import org.enso.base.read.BaseReadManyReturnSPI; +import org.enso.base.read.ReadManyReturnSPI; module org.enso.std.base { requires java.logging; @@ -23,6 +25,7 @@ uses FileSystemSPI; uses FileFormatSPI; uses DataLinkSPI; + uses ReadManyReturnSPI; // following packages are accessed by Java code in other Enso modules exports org.enso.base; @@ -32,6 +35,7 @@ exports org.enso.base.enso_cloud.audit; exports org.enso.base.lookup; exports org.enso.base.numeric; + exports org.enso.base.parser; exports org.enso.base.polyglot; exports org.enso.base.time; exports org.enso.base.text; @@ -62,4 +66,6 @@ provides DataLinkSPI with EnsoFileDataLinkSPI, HTTPFetchDataLinkSPI; + provides ReadManyReturnSPI with + BaseReadManyReturnSPI; } diff --git a/std-bits/base/src/main/java/org/enso/base/read/BaseReadManyReturnSPI.java b/std-bits/base/src/main/java/org/enso/base/read/BaseReadManyReturnSPI.java index 55308ce81f3c..9f79817d9a18 100644 --- a/std-bits/base/src/main/java/org/enso/base/read/BaseReadManyReturnSPI.java +++ b/std-bits/base/src/main/java/org/enso/base/read/BaseReadManyReturnSPI.java @@ -1,7 +1,6 @@ package org.enso.base.read; -@org.openide.util.lookup.ServiceProvider(service = ReadManyReturnSPI.class) -public class BaseReadManyReturnSPI extends ReadManyReturnSPI { +public final class BaseReadManyReturnSPI extends ReadManyReturnSPI { @Override protected String getModuleName() { return "Standard.Base.Data.Read.Return_As";