Merge pull request #335 from marklogic/release/1.2.1

Release 1.2.1

Author: rjrudin

NOTICE.txt

@@ -1,6 +1,6 @@
 MarkLogic® Flux™
 
-Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
+Copyright © 2025 MarkLogic Corporation. All Rights Reserved.
 
 To the extent required by the applicable open-source license, a complete machine-readable copy of the source code corresponding to such code is available upon request. This offer is valid to anyone in receipt of this information and shall expire three years following the date of the final distribution of this product version by Progress Software Corporation. To obtain such source code, send an email to [email protected]. Please specify the product and version for which you are requesting source code.
 
@@ -9,7 +9,7 @@ Third Party Notices
 aws-java-sdk-s3 1.12.262 (Apache-2.0)
 hadoop-aws 3.3.4 (Apache-2.0)
 hadoop-client 3.3.4 (Apache-2.0)
-marklogic-spark-connector 2.5.0 (Apache-2.0)
+marklogic-spark-connector 2.5.1 (Apache-2.0)
 picocli 4.7.6 (Apache-2.0)
 spark-avro_2.12 3.5.3 (Apache-2.0)
 spark-sql_2.12 3.5.3 (Apache-2.0)
@@ -20,7 +20,7 @@ Apache License 2.0 (Apache-2.0)
 
 Third-Party Components
 
-The following is a list of the third-party components used by MarkLogic® Flux™ 1.2.0 (last updated December 17, 2024):
+The following is a list of the third-party components used by MarkLogic® Flux™ 1.2.1 (last updated January 7, 2025):
 
 aws-java-sdk-s3 1.12.262 (Apache-2.0)
 https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3
@@ -34,7 +34,7 @@ hadoop-client 3.3.4 (Apache-2.0)
 https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
-marklogic-spark-connector 2.5.0 (Apache-2.0)
+marklogic-spark-connector 2.5.1 (Apache-2.0)
 https://repo1.maven.org/maven2/com/marklogic/marklogic-spark-connector
 For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
@@ -53,7 +53,7 @@ For the full text of the Apache-2.0 license, see Apache License 2.0 (Apache-2.0)
 
 Common Licenses
 
-This section shows the text of common third-party licenses used by MarkLogic® Flux™ v1 (last updated July 2, 2024):
+This section shows the text of common third-party licenses used by MarkLogic® Flux™ 1.2.1 (last updated January 7, 2025):
 
 Apache License 2.0 (Apache-2.0)
 https://spdx.org/licenses/Apache-2.0.html

docs/api.md

@@ -22,15 +22,15 @@ To add Flux as a dependency to your application, add the following to your Maven
 <dependency>
   <groupId>com.marklogic</groupId>
   <artifactId>flux-api</artifactId>
-  <version>1.2.0</version>
+  <version>1.2.1</version>
 </dependency>
 ```
 
 Or if you are using Gradle, add the following to your `build.gradle` file:
 
 ```
 dependencies {
-  implementation "com.marklogic:flux-api:1.2.0"
+  implementation "com.marklogic:flux-api:1.2.1"
 }
 ```
 
@@ -97,7 +97,7 @@ buildscript {
     mavenCentral()
   }
   dependencies {
-    classpath "com.marklogic:flux-api:1.2.0"
+    classpath "com.marklogic:flux-api:1.2.1"
   }
 }
 ```

docs/getting-started.md

@@ -15,18 +15,18 @@ This guide describes how to get started with Flux with some examples demonstrati
 ## Setup
 
 You can download the latest release of the Flux application zip from [the latest Flux release page](https://github.com/marklogic/flux/releases). 
-The Flux application zip is titled `marklogic-flux-1.2.0.zip`. You can extract this zip to any location on your 
+The Flux application zip is titled `marklogic-flux-1.2.1.zip`. You can extract this zip to any location on your 
 filesystem that you prefer.
 
 ### Deploying the example application
 
 The examples in this guide, along with examples found throughout this documentation, depend on a small MarkLogic 
 application that can be deployed to your own instance of MarkLogic server. The application can be downloaded from 
 [the latest Flux release page](https://github.com/marklogic/flux/releases) in a zip titled 
-`marklogic-flux-getting-started-1.2.0.zip`. To use Flux with this example application, perform the following steps:
+`marklogic-flux-getting-started-1.2.1.zip`. To use Flux with this example application, perform the following steps:
 
-1. Extract the `marklogic-flux-getting-started-1.2.0.zip` file to any location on your local filesystem.
-2. Run `cd marklogic-flux-getting-started-1.2.0` to change to the directory created by extracting the ZIP file.
+1. Extract the `marklogic-flux-getting-started-1.2.1.zip` file to any location on your local filesystem.
+2. Run `cd marklogic-flux-getting-started-1.2.1` to change to the directory created by extracting the ZIP file.
 3. Create a file named `gradle-local.properties` and add `mlPassword=your MarkLogic admin user password` to it.
 4. Examine the contents of the `gradle.properties` file to ensure that the value of `mlHost` points to your MarkLogic 
 server and that the value of `mlRestPort` is a port available for a new MarkLogic app server to use.
@@ -38,15 +38,15 @@ privileges for running the examples in this guide. Finally, the application incl
 [MarkLogic TDE template](https://docs.marklogic.com/guide/app-dev/TDE) that creates a view in MarkLogic for the purpose
 of demonstrating commands that utilize a [MarkLogic Optic query](https://docs.marklogic.com/guide/app-dev/OpticAPI).
 
-It is recommended to extract the Flux application zip into the `marklogic-flux-getting-started-1.2.0` directory so that 
+It is recommended to extract the Flux application zip into the `marklogic-flux-getting-started-1.2.1` directory so that 
 you can easily execute the examples in this guide. After extracting the application zip, the directory should have a 
 structure similar to this (not all files may be shown):
 
 ```
-./marklogic-flux-getting-started-1.2.0
+./marklogic-flux-getting-started-1.2.1
     build.gradle
     ./data
-    ./marklogic-flux-1.2.0
+    ./marklogic-flux-1.2.1
     ./gradle
     gradle.properties
     gradlew
@@ -59,7 +59,7 @@ structure similar to this (not all files may be shown):
 You can run Flux without any options to see the list of available commands. If you are using Flux to run these examples, 
 first change your current directory to where you extract Flux:
 
-    cd marklogic-flux-1.2.0
+    cd marklogic-flux-1.2.1
 
 And then run the Flux executable without any options:
 

docs/import/embedder/embedder.md

@@ -46,9 +46,9 @@ refers to the Flux support for generating and adding embeddings. The value of th
 
 As of the 1.2.0 release, Flux recognizes the following abbreviations as a matter of convenience to avoid typing the full class name:
 
-- `--embedder azure` refers to the embedding model provided by the `flux-embedding-model-azure-open-ai-1.2.0.jar` file.
-- `--embedder minilm` refers to the embedding model provided by the `flux-embedding-model-minilm-1.2.0.jar` file.
-- `--embedder ollama` refers to the embedding model provided by the `flux-embedding-model-ollama-1.2.0.jar` file. 
+- `--embedder azure` refers to the embedding model provided by the `flux-embedding-model-azure-open-ai-1.2.1.jar` file.
+- `--embedder minilm` refers to the embedding model provided by the `flux-embedding-model-minilm-1.2.1.jar` file.
+- `--embedder ollama` refers to the embedding model provided by the `flux-embedding-model-ollama-1.2.1.jar` file. 
 
 The JAR files associated with each of the above abbreviations can be downloaded from the 
 [Flux releases site](https://github.com/marklogic/flux/releases) and added to the `./ext` directory in a Flux 
@@ -73,7 +73,7 @@ file in your Flux installation and change the `logger.langchain4j.level` logger
 
 ### Azure OpenAI options
 
-The `flux-embedding-model-azure-open-ai-1.2.0.jar` file uses 
+The `flux-embedding-model-azure-open-ai-1.2.1.jar` file uses 
 [LangChain4's support](https://docs.langchain4j.dev/integrations/embedding-models/azure-open-ai) for 
 [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service) and
 supports the following options:
@@ -119,7 +119,7 @@ require any setup or configuration.
 
 ### Ollama options
 
-The `flux-embedding-model-ollama-1.2.0.jar` file uses 
+The `flux-embedding-model-ollama-1.2.1.jar` file uses 
 [LangChain4j's support](https://docs.langchain4j.dev/integrations/embedding-models/ollama) for 
 [Ollama](https://ollama.com/) and supports the following options:
 

docs/import/embedder/embedding-query.md

@@ -31,7 +31,7 @@ instructions in the [Getting Started guide](../../getting-started.md).
 
 This example uses the ["minilm" embedding model](https://docs.langchain4j.dev/integrations/embedding-models/in-process). 
 This embedding model is useful for testing vector queries with MarkLogic as it has no external dependencies and requires
-no setup or configuration. To use this embedding model, you must include the `flux-embedding-model-minilm-1.2.0.jar`
+no setup or configuration. To use this embedding model, you must include the `flux-embedding-model-minilm-1.2.1.jar`
 in the `./ext` directory of your Flux installation. Please see the [embedder guide](embedder.md) for more information 
 on embedding models. 
 

docs/reprocess.md

@@ -234,6 +234,9 @@ setting either option will adjust the default value of the `--spark-master-url`
 the number of Spark worker threads. This ensures that each partition processing items will have a Spark worker thread
 available to it. 
 
+When using an option such as `--limit`, where you do not want to wait for all the data to be read first, 
+you may want to include `--thread-count 0` to prevent the data from being repartitioned.
+
 You may also be able to improve reader performance by partitioning your query by database forests. The section on 
 creating partitions above uses the following, which is a simple way to both create and use forest-based partitions:
 

docs/spark-integration.md

@@ -35,8 +35,8 @@ Flux integrates with [spark-submit](https://spark.apache.org/docs/latest/submitt
 submit a Flux command invocation to a remote Spark cluster. Every Flux command is a Spark application, and thus every
 Flux command, along with all of its option, can be invoked via `spark-submit`. 
 
-To use Flux with `spark-submit`, first download the `marklogic-flux-1.2.0-all.jar` file from the 
-[GitHub release page](https://github.com/marklogic/flux/releases/tag/1.2.0). This jar file includes Flux and all of 
+To use Flux with `spark-submit`, first download the `marklogic-flux-1.2.1-all.jar` file from the 
+[GitHub release page](https://github.com/marklogic/flux/releases/tag/1.2.1). This jar file includes Flux and all of 
 its dependencies, excluding those of Spark itself, which will be provided via the Spark cluster that you connect to 
 via `spark-submit`. 
 
@@ -48,7 +48,7 @@ The following shows a notional example of running the Flux `import-files` comman
 ```
 $SPARK_HOME/bin/spark-submit --class com.marklogic.flux.spark.Submit \
     --master spark://changeme:7077 \
-    marklogic-flux-1.2.0-all.jar \
+    marklogic-flux-1.2.1-all.jar \
     import-files \
     --path path/to/data \
     --connection-string user:password@host:8000 \
@@ -59,7 +59,7 @@ $SPARK_HOME/bin/spark-submit --class com.marklogic.flux.spark.Submit \
 ```
 $SPARK_HOME\bin\spark-submit --class com.marklogic.flux.spark.Submit ^
     --master spark://changeme:7077 ^
-    marklogic-flux-1.2.0-all.jar ^
+    marklogic-flux-1.2.1-all.jar ^
     import-files ^
     --path path/to/data ^
     --connection-string user:password@host:8000 ^

examples/client-project/build.gradle

@@ -6,7 +6,7 @@ buildscript {
     mavenLocal()
   }
   dependencies {
-    classpath "com.marklogic:flux-api:1.2.0"
+    classpath "com.marklogic:flux-api:1.2.1"
   }
 }
 
@@ -20,8 +20,8 @@ repositories {
 }
 
 dependencies {
-  implementation "com.marklogic:flux-api:1.2.0"
-  implementation "com.marklogic:flux-embedding-model-minilm:1.2.0"
+  implementation "com.marklogic:flux-api:1.2.1"
+  implementation "com.marklogic:flux-embedding-model-minilm:1.2.1"
 }
 
 tasks.register("runApp", JavaExec) {

flux-cli/build.gradle

@@ -29,7 +29,7 @@ dependencies {
     exclude module: "rocksdbjni"
   }
 
-  implementation "com.marklogic:marklogic-spark-connector:2.5.0"
+  implementation "com.marklogic:marklogic-spark-connector:2.5.1"
   implementation "info.picocli:picocli:4.7.6"
 
   // Spark 3.4.3 depends on Hadoop 3.3.4, which depends on AWS SDK 1.12.262. As of August 2024, all public releases of
@@ -88,7 +88,7 @@ dependencies {
   testImplementation project(path: ":flux-embedding-model-azure-open-ai", configuration: "shadow")
   testImplementation project(path: ":flux-embedding-model-ollama", configuration: "shadow")
 
-  shadowDependencies "com.marklogic:marklogic-spark-connector:2.5-SNAPSHOT"
+  shadowDependencies "com.marklogic:marklogic-spark-connector:2.5.1"
   shadowDependencies "info.picocli:picocli:4.7.6"
 }
 

flux-cli/src/main/java/com/marklogic/flux/api/EmbedderOptions.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
+ * Copyright © 2025 MarkLogic Corporation. All Rights Reserved.
  */
 package com.marklogic.flux.api;
 
@@ -20,9 +20,7 @@ public interface EmbedderOptions {
      *
      * @param name either one of the abbreviations listed above, or the fully-qualified class name of an implementation of
      *             {@code java.util.Function<Map<String, String>, dev.langchain4j.model.embedding.EmbeddingModel>} or
-     *             an abbreviation associated with the class name of an implementation. Due to a bug in Flux 1.2.0,
-     *             the class name must not contain any uppercase characters. This will be fixed in the next Flux
-     *             release.
+     *             an abbreviation associated with the class name of an implementation.
      * @return
      */
     EmbedderOptions embedder(String name);

flux-cli/src/main/java/com/marklogic/flux/impl/ConnectionInputs.java

@@ -4,11 +4,10 @@
 package com.marklogic.flux.impl;
 
 import com.marklogic.client.DatabaseClient;
+import com.marklogic.client.impl.ConnectionString;
 import com.marklogic.flux.api.AuthenticationType;
 import com.marklogic.flux.api.FluxException;
 import com.marklogic.flux.api.SslHostnameVerifier;
-import com.marklogic.spark.ConnectionString;
-import com.marklogic.spark.ConnectorException;
 import com.marklogic.spark.Options;
 import picocli.CommandLine;
 
@@ -26,7 +25,7 @@ public static class ConnectionStringValidator implements CommandLine.ITypeConver
         public String convert(String value) {
             try {
                 new ConnectionString(value, "connection string");
-            } catch (ConnectorException e) {
+            } catch (IllegalArgumentException e) {
                 // See https://picocli.info/#_handling_invalid_input .
                 throw new CommandLine.TypeConversionException(e.getMessage());
             }
@@ -75,7 +74,7 @@ public void validateConnectionString(String inputNameForErrorMessage) {
         if (connectionString != null && connectionString.trim().length() > 0) {
             try {
                 new ConnectionString(connectionString, inputNameForErrorMessage);
-            } catch (ConnectorException e) {
+            } catch (IllegalArgumentException e) {
                 throw new FluxException(e.getMessage());
             }
         }

flux-cli/src/main/java/com/marklogic/flux/impl/importdata/EmbedderParams.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
+ * Copyright © 2025 MarkLogic Corporation. All Rights Reserved.
  */
 package com.marklogic.flux.impl.importdata;
 
@@ -154,9 +154,9 @@ public EmbedderOptions embedderOptions(Map<String, String> options) {
         return this;
     }
 
-    private String determineClassName(String embedderValue) {
-        embedderValue = embedderValue.toLowerCase();
-        switch (embedderValue) {
+    private String determineClassName(final String embedderValue) {
+        String abbreviation = embedderValue.toLowerCase();
+        switch (abbreviation) {
             case "minilm":
                 return "com.marklogic.flux.langchain4j.embedding.MinilmEmbeddingModelFunction";
             case "azure":

flux-cli/src/test/java/com/marklogic/flux/CopyrightTest.java

@@ -41,7 +41,8 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IO
                         String message = String.format("Does not start with copyright comment: %s", file.toFile().getAbsolutePath());
                         assertTrue(content.startsWith("/*"), message);
                         assertTrue(
-                            content.contains("Copyright © 2024 MarkLogic Corporation. All Rights Reserved."),
+                            content.contains("Copyright © 2024 MarkLogic Corporation. All Rights Reserved.") ||
+                                content.contains("Copyright © 2025 MarkLogic Corporation. All Rights Reserved."),
                             message
                         );
                     }

flux-cli/src/test/java/com/marklogic/flux/api/GenericFilesImporterTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright © 2024 MarkLogic Corporation. All Rights Reserved.
+ * Copyright © 2025 MarkLogic Corporation. All Rights Reserved.
  */
 package com.marklogic.flux.api;
 
@@ -198,4 +198,20 @@ void splitAndEmbed() {
             assertEquals(JsonNodeType.ARRAY, chunk.get("embedding").getNodeType());
         });
     }
+
+    @Test
+    void splitAndEmbedWithFullClassName() {
+        Flux.importGenericFiles()
+            .connectionString(makeConnectionString())
+            .from("src/test/resources/json-files/java-client-intro.json")
+            .to(writeOptions -> writeOptions
+                .permissionsString(DEFAULT_PERMISSIONS)
+                .uriTemplate("/split-test.json")
+                .splitter(splitterOptions -> splitterOptions.jsonPointers("/text"))
+                .embedder(embedderOptions -> embedderOptions.embedder("com.marklogic.flux.langchain4j.embedding.MinilmEmbeddingModelFunction"))
+            ).execute();
+
+        JsonNode doc = readJsonDocument("/split-test.json");
+        assertEquals(2, doc.get("chunks").size(), "Should get 2 chunks with the default max chunk size of 1000.");
+    }
 }

flux-embedding-model-azure-open-ai/build.gradle

@@ -32,8 +32,10 @@ publishing {
     mainJava(MavenPublication) {
       groupId = group
       version = version
+      artifactId = project.name
       from components.java
       pom {
+        name = "${group}:${project.name}"
         description = "Integration for Azure OpenAI with the MarkLogic Flux API"
         packaging = "jar"
         url = "https://github.com/marklogic/flux"

flux-embedding-model-minilm/build.gradle

@@ -17,8 +17,10 @@ publishing {
     mainJava(MavenPublication) {
       groupId = group
       version = version
+      artifactId = project.name
       from components.java
       pom {
+        name = "${group}:${project.name}"
         description = "Integration for minilm with the MarkLogic Flux API"
         packaging = "jar"
         url = "https://github.com/marklogic/flux"

flux-embedding-model-ollama/build.gradle

@@ -33,8 +33,10 @@ publishing {
     mainJava(MavenPublication) {
       groupId = group
       version = version
+      artifactId = project.name
       from components.java
       pom {
+        name = "${group}:${project.name}"
         description = "Integration for Ollama with the MarkLogic Flux API"
         packaging = "jar"
         url = "https://github.com/marklogic/flux"

gradle.properties

@@ -1,4 +1,4 @@
-version=1.2.0
+version=1.2.1
 
 # Define these on the command line to publish to OSSRH
 # See https://central.sonatype.org/publish/publish-gradle/#credentials for more information