[AUTO] Incremented version to 2.14.1.

.github/workflows/ci.yml

@@ -9,27 +9,9 @@ on:
 
 jobs:
   Get-CI-Image-Tag:
-    runs-on: ubuntu-latest
-    outputs:
-      ci-image-version-linux: ${{ steps.step-ci-image-version-linux.outputs.ci-image-version-linux }}
-    steps:
-      - name: Install crane
-        uses: iarekylew00t/crane-installer@v1
-        with:
-          crane-release: v0.15.2
-      - name: Checkout opensearch-build repository
-        uses: actions/checkout@v2
-        with:
-          repository: 'opensearch-project/opensearch-build'
-          ref: 'main'
-          path: 'opensearch-build'
-      - name: Get ci image version from opensearch-build repository scripts
-        id: step-ci-image-version-linux
-        run: |
-          crane version
-          CI_IMAGE_VERSION=`opensearch-build/docker/ci/get-ci-images.sh -p centos7 -u opensearch -t build | head -1`
-          echo $CI_IMAGE_VERSION
-          echo "ci-image-version-linux=$CI_IMAGE_VERSION" >> $GITHUB_OUTPUT 
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+        product: opensearch
 
   build-linux:
     needs: Get-CI-Image-Tag
@@ -42,17 +24,20 @@ jobs:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
+
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Java ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
+          distribution: temurin
 
       - name: Build and Test
         run: |
@@ -61,7 +46,7 @@ jobs:
                                ./gradlew publishToMavenLocal"
 
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
@@ -76,18 +61,19 @@ jobs:
 
     steps:
       - name: Checkout skills
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
       - name: Setup Java ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
+          distribution: temurin
       - name: Install dependencies on macos
         run: |
           brew reinstall gcc
           export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran
       - name: Run build
         run: |
-          ./gradlew build
+          ./gradlew build -Dos.arch=x86_64
 
   build-windows:
     strategy:
@@ -99,12 +85,13 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Setup Java ${{ matrix.java }}
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: ${{ matrix.java }}
+          distribution: temurin
 
       - name: Build and Test
         run: |
@@ -115,7 +102,7 @@ jobs:
           ./gradlew publishToMavenLocal
 
       - name: Upload Coverage Report
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/test_security.yml

@@ -17,22 +17,24 @@ jobs:
     strategy:
       matrix:
         java: [11, 17, 21]
-
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     name: Run Security Integration Tests on Linux
     runs-on: ubuntu-latest
     needs: Get-CI-Image-Tag
     container:
       # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
       # this image tag is subject to change as more dependencies and updates will arrive over time
       image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
-      # need to switch to root so that github actions can install runner binary on container without permission issues.
-      options: --user root
+      options: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-options }}
 
     steps:
+      - name: Run start commands
+        run: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-start-command }}
       - name: Checkout Skills
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Setup Java ${{ matrix.java }}
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: ${{ matrix.java }}

MAINTAINERS.md

@@ -22,5 +22,8 @@
 | Surya Sashank Nistala | [eirsep](https://github.com/eirsep)                 | Amazon      |
 | Subhobrata DEY        | [sbcd90](https://github.com/sbcd90)                 | Amazon      |
 | Zhichao Geng          | [zhichao-aws](https://github.com/zhichao-aws)       | Amazon      |
+| Joshua Li             | [joshuali925](https://github.com/joshuali925)       | Amazon      |
+| Binlong Gao           | [gaobinlong](https://github.com/gaobinlong)         | Amazon      |
+| Hailong Cui             | [hailong-am](https://github.com/hailong-am)       | Amazon      |
 
-[This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).
+[This document](https://github.com/opensearch-project/.github/blob/main/MAINTAINERS.md) explains what maintainers do in this repo, and how they should be doing it. If you're interested in contributing, see [CONTRIBUTING](CONTRIBUTING.md).

build.gradle

@@ -11,7 +11,7 @@ import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 buildscript {
     ext {
         opensearch_group = "org.opensearch"
-        opensearch_version = System.getProperty("opensearch.version", "2.13.0-SNAPSHOT")
+        opensearch_version = System.getProperty("opensearch.version", "2.19.2-SNAPSHOT")
         buildVersionQualifier = System.getProperty("build.version_qualifier", "")
         isSnapshot = "true" == System.getProperty("build.snapshot", "true")
         version_tokens = opensearch_version.tokenize('-')
@@ -43,12 +43,12 @@ buildscript {
 plugins {
     id 'java-library'
     id 'com.diffplug.spotless' version '6.25.0'
-    id "io.freefair.lombok" version "8.6"
+    id "io.freefair.lombok" version "8.10.2"
     id "de.undercouch.download" version "5.6.0"
 }
 
 lombok {
-    version = "1.18.30"
+    version = "1.18.34"
 }
 
 repositories {
@@ -80,14 +80,18 @@ apply plugin: 'opensearch.pluginzip'
 def sqlJarDirectory = "$buildDir/dependencies/opensearch-sql-plugin"
 def jsJarDirectory = "$buildDir/dependencies/opensearch-job-scheduler"
 def adJarDirectory = "$buildDir/dependencies/opensearch-anomaly-detection"
+def sparkDir = "$buildDir/dependencies/spark"
 
 configurations {
     zipArchive
+    spark {
+        transitive = false
+    }
     secureIntegTestPluginArchive
     all {
         resolutionStrategy {
             force "org.mockito:mockito-core:${versions.mockito}"
-            force "com.google.guava:guava:33.0.0-jre" // CVE for 31.1
+            force "com.google.guava:guava:33.2.1-jre" // CVE for 31.1
             force("org.eclipse.platform:org.eclipse.core.runtime:3.30.0") // CVE for < 3.29.0, forces JDK17 for spotless
         }
     }
@@ -107,33 +111,60 @@ task addJarsToClasspath(type: Copy) {
     into("$buildDir/classes")
 
     from(fileTree(dir: adJarDirectory)) {
-        include "opensearch-time-series-analytics-${opensearch_build}.jar"
+        include "opensearch-anomaly-detection-${opensearch_build}.jar"
+    }
+    into("$buildDir/classes")
+
+    // spark jar
+    from(fileTree(dir: sparkDir)) {
+        include("*.jar")
     }
     into("$buildDir/classes")
 }
 
+
+
 dependencies {
     // 3P dependencies
     compileOnly group: 'com.google.code.gson', name: 'gson', version: '2.10.1'
-    compileOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.22.0"
+
+    compileOnly "org.apache.logging.log4j:log4j-slf4j-impl:2.23.1"
+    compileOnly group: 'org.slf4j', name: 'slf4j-api', version: '1.7.36'
     compileOnly group: 'org.json', name: 'json', version: '20240205'
-    compileOnly("com.google.guava:guava:33.0.0-jre")
-    compileOnly group: 'org.apache.commons', name: 'commons-lang3', version: '3.14.0'
+    compileOnly("com.google.guava:guava:33.2.1-jre")
+    compileOnly group: 'org.apache.commons', name: 'commons-lang3', version: '3.16.0'
     compileOnly group: 'org.apache.commons', name: 'commons-text', version: '1.11.0'
+    compileOnly("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
+    compileOnly("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
+
+    spark 'org.apache.spark:spark-sql-api_2.13:3.5.4'
+    spark 'org.apache.spark:spark-core_2.13:3.5.4'
+    spark group: 'org.apache.spark', name: 'spark-common-utils_2.13', version: '3.5.4'
+
+    implementation 'org.scala-lang:scala-library:2.13.9'
+    implementation group: 'org.antlr', name: 'antlr4-runtime', version: '4.9.3'
+    implementation("org.json4s:json4s-ast_2.13:3.7.0-M11")
+    implementation("org.json4s:json4s-core_2.13:3.7.0-M11")
+    implementation("org.json4s:json4s-jackson_2.13:3.7.0-M11")
+    implementation 'com.fasterxml.jackson.module:jackson-module-scala_3:2.18.2'
+    implementation group: 'org.scala-lang', name: 'scala3-library_3', version: '3.7.0-RC1-bin-20250119-bd699fc-NIGHTLY'
+    implementation("com.thoughtworks.paranamer:paranamer:2.8")
 
     // Plugin dependencies
     compileOnly group: 'org.opensearch', name:'opensearch-ml-client', version: "${opensearch_build}"
-    implementation fileTree(dir: jsJarDirectory, include: ["opensearch-job-scheduler-${opensearch_build}.jar"])
-    implementation fileTree(dir: adJarDirectory, include: ["opensearch-time-series-analytics-${opensearch_build}.jar"])
+    compileOnly fileTree(dir: jsJarDirectory, include: ["opensearch-job-scheduler-${opensearch_build}.jar"])
+    implementation fileTree(dir: adJarDirectory, include: ["opensearch-anomaly-detection-${opensearch_build}.jar"])
     implementation fileTree(dir: sqlJarDirectory, include: ["opensearch-sql-${opensearch_build}.jar", "ppl-${opensearch_build}.jar", "protocol-${opensearch_build}.jar"])
+    implementation fileTree(dir: sparkDir, include: ["spark*.jar"])
     compileOnly "org.opensearch:common-utils:${opensearch_build}"
     compileOnly "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compileOnly "org.opensearch:opensearch-job-scheduler-spi:${opensearch_build}"
 
 
     // ZipArchive dependencies used for integration tests
-    zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
+    // The order is important here, we need to make sure opensearch-job-scheduler is before ml-plugin and AD
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-job-scheduler', version: "${opensearch_build}"
+    zipArchive group: 'org.opensearch.plugin', name:'opensearch-ml-plugin', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-anomaly-detection', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${opensearch_build}"
     zipArchive group: 'org.opensearch.plugin', name:'opensearch-knn', version: "${opensearch_build}"
@@ -145,18 +176,78 @@ dependencies {
     testImplementation "org.opensearch.test:framework:${opensearch_version}"
     testImplementation group: 'junit', name: 'junit', version: '4.13.2'
     testImplementation group: 'org.json', name: 'json', version: '20240205'
-    testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.10.0'
+    testImplementation group: 'org.mockito', name: 'mockito-core', version: '5.14.2'
     testImplementation group: 'org.mockito', name: 'mockito-inline', version: '5.2.0'
-    testImplementation("net.bytebuddy:byte-buddy:1.14.9")
-    testImplementation("net.bytebuddy:byte-buddy-agent:1.14.7")
-    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.10.1'
-    testImplementation 'org.mockito:mockito-junit-jupiter:5.10.0'
+    testImplementation("net.bytebuddy:byte-buddy:1.15.10")
+    testImplementation("net.bytebuddy:byte-buddy-agent:1.15.10")
+    testImplementation 'org.junit.jupiter:junit-jupiter-api:5.11.2'
+    testImplementation 'org.mockito:mockito-junit-jupiter:5.14.2'
     testImplementation "com.nhaarman.mockitokotlin2:mockito-kotlin:2.2.0"
     testImplementation "com.cronutils:cron-utils:9.2.1"
+    testImplementation 'com.jayway.jsonpath:json-path:2.9.0'
     testImplementation "commons-validator:commons-validator:1.8.0"
-    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.10.1'
+    testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.11.2'
+}
+task addSparkJar(type: Copy) {
+    mustRunAfter()
+    from(configurations.spark)
+    into sparkDir
+
+    doLast {
+        def jarA = file("$sparkDir/spark-sql-api_2.13-3.5.4.jar")
+        def jarB = file("$sparkDir/spark-core_2.13-3.5.4.jar")
+        def jarC = file("$sparkDir/spark-common-utils_2.13-3.5.4.jar")
+
+        // 3a. Extract jar A to manipulate it
+        def jarAContents = file("$buildDir/tmp/JarAContents")
+        delete(jarAContents)
+        jarAContents.mkdirs()
+        copy {
+            from zipTree(jarA)
+            into jarAContents
+        }
+        // Remove the unwanted directory from jar A
+        delete file("${jarAContents}/org/apache/spark/unused")
+
+        // Re-compress jar A
+        ant.zip(destfile: jarA, baseDir: jarAContents)
+
+        // 3b. Repeat for jar B
+        def jarBContents = file("$buildDir/tmp/JarBContents")
+        delete(jarBContents)
+        jarBContents.mkdirs()
+        copy {
+            from zipTree(jarB)
+            into jarBContents
+        }
+        // Remove the unwanted directory from jar B
+        delete file("${jarBContents}/org/apache/spark/unused")
+        delete file("${jarBContents}/org/sparkproject/jetty/http")
+        delete file("${jarBContents}/META-INF/maven/org.eclipse.jetty/jetty-http")
+        // Re-compress jar B
+        ant.zip(destfile: jarB, baseDir: jarBContents)
+
+        def jarCContents = file("$buildDir/tmp/JarBContents")
+        delete(jarCContents)
+        jarCContents.mkdirs()
+        copy {
+            from zipTree(jarC)
+            into jarCContents
+        }
+        // Remove the unwanted directory from jar C
+        delete file("${jarCContents}/org/apache/spark/unused")
+        delete file("${jarCContents}/org/apache/spark/SparkDriverExecutionException.class")
+        delete file("${jarCContents}/org/apache/spark/SparkUserAppException.class")
+        delete file("${jarCContents}/org/apache/spark/SparkUserAppException\$.class")
+        delete file("${jarCContents}/org/apache/spark/SparkUserAppException*class")
+
+
+        // Re-compress jar C
+        ant.zip(destfile: jarC, baseDir: jarCContents)
+    }
 }
 
+
 task extractSqlJar(type: Copy) {
     mustRunAfter()
     from(zipTree(configurations.zipArchive.find { it.name.startsWith("opensearch-sql-plugin")}))
@@ -178,6 +269,7 @@ task extractAdJar(type: Copy) {
 tasks.addJarsToClasspath.dependsOn(extractSqlJar)
 tasks.addJarsToClasspath.dependsOn(extractJsJar)
 tasks.addJarsToClasspath.dependsOn(extractAdJar)
+tasks.addJarsToClasspath.dependsOn(addSparkJar)
 project.tasks.delombok.dependsOn(addJarsToClasspath)
 tasks.publishNebulaPublicationToMavenLocal.dependsOn ':generatePomFileForPluginZipPublication'
 tasks.validateNebulaPom.dependsOn ':generatePomFileForPluginZipPublication'
@@ -337,7 +429,6 @@ task integTest(type: RestIntegTestTask) {
 tasks.named("check").configure { dependsOn(integTest) }
 
 integTest {
-
     dependsOn "bundlePlugin"
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'java.io.tmpdir', opensearch_tmp_dir.absolutePath
@@ -396,7 +487,6 @@ integTest {
         jvmArgs '-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005'
     }
 }
-
 // Set up integration test clusters, installs all zipArchive dependencies and this plugin
 testClusters.integTest {
     testDistribution = "ARCHIVE"

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,8 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.4-bin.zip
+distributionSha256Sum=31c55713e40233a8303827ceb42ca48a47267a0ad4bab9177123121e71524c26
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
 networkTimeout=10000
+validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=3e1af3ae886920c3ac87f7a91f816c0c7c436f276a6eefdb3da152100fef72ae