[Backport 2.x] Add expand_nested_docs Parameter support to NMSLIB engine

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @opensearch-project/k-nn
+*   @heemin32 @navneet1v @VijayanB @vamshin @jmazanec15 @naveentatikonda @junqiu-lei @martin-gaievski @ryanbogan @luyuncheng

.github/workflows/CI.yml

@@ -6,79 +6,190 @@ on:
     branches:
       - "*"
       - "feature/**"
+    paths:
+      - 'build.gradle'
+      - 'settings.gradle'
+      - 'src/**'
+      - 'build-tools/**'
+      - 'buildSrc/**'
+      - 'gradle/**'
+      - 'jni/**'
+      - 'micro-benchmarks/**'
+      - '.github/workflows/CI.yml'
   pull_request:
     branches:
       - "*"
       - "feature/**"
+    paths:
+      - 'build.gradle'
+      - 'settings.gradle'
+      - 'src/**'
+      - 'build-tools/**'
+      - 'buildSrc/**'
+      - 'gradle/**'
+      - 'jni/**'
+      - 'micro-benchmarks/**'
+      - '.github/workflows/CI.yml'
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 
 jobs:
-  Build-k-NN:
+  Get-CI-Image-Tag:
+    uses: opensearch-project/opensearch-build/.github/workflows/get-ci-image-tag.yml@main
+    with:
+      product: opensearch
+
+  Build-k-NN-Linux:
     strategy:
       matrix:
-        java: [11, 17]
+        java: [11, 17, 21]
 
-    name: Build and Test k-NN Plugin
+    name: Build and Test k-NN Plugin on Linux
     runs-on: ubuntu-latest
+    needs: Get-CI-Image-Tag
+    container:
+      # using the same image which is used by opensearch-build team to build the OpenSearch Distribution
+      # this image tag is subject to change as more dependencies and updates will arrive over time
+      image: ${{ needs.Get-CI-Image-Tag.outputs.ci-image-version-linux }}
+      # need to switch to root so that github actions can install runner binary on container without permission issues.
+      options: --user root
 
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v1
 
+      # Setup git user so that patches for native libraries can be applied and committed
+      - name: Setup git user
+        run: |
+          su `id -un 1000` -c 'git config --global user.name "github-actions[bot]"'
+          su `id -un 1000` -c 'git config --global user.email "github-actions[bot]@users.noreply.github.com"'
+
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
           java-version: ${{ matrix.java }}
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get install libopenblas-dev gfortran -y
-
       - name: Run build
+        # switching the user, as OpenSearch cluster can only be started as root/Administrator on linux-deb/linux-rpm/windows-zip.
         run: |
-          ./gradlew build
+          chown -R 1000:1000 `pwd`
+          if lscpu  | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw          
+          then
+            echo "avx512 available on system"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc`"
+          elif lscpu  | grep -i avx2
+          then
+            echo "avx2 available on system"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Dnproc.count=`nproc` -Davx512.enabled=false"
+          else
+            echo "avx512 and avx2 not available on system"
+            su `id -un 1000` -c "whoami && java -version && ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=`nproc`"
+          fi  
+
 
       - name: Upload Coverage Report
         uses: codecov/codecov-action@v1
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
 
-#      - name: Pull and Run Docker for security tests
-#        run: |
-#          plugin=`ls build/distributions/*.zip`
-#          version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-3`
-#          plugin_version=`echo $plugin|awk -F- '{print $4}'| cut -d. -f 1-4`
-#          echo $version
-#          cd ..
-#          if docker pull opendistroforelasticsearch/opendistroforelasticsearch:$version
-#          then
-#            echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$version" >> Dockerfile
-#            echo "RUN if [ -d /usr/share/elasticsearch/plugins/opendistro-knn ]; then /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro-knn; fi" >> Dockerfile
-#            echo "RUN yum -y update \ && yum -y groupinstall "Development Tools" \ && yum install -y unzip glibc.x86_64 cmake \ && yum clean all" >> Dockerfile
-#            echo "RUN git clone --recursive --branch ${GITHUB_REF##*/} https://github.com/opendistro-for-elasticsearch/k-NN.git /usr/share/elasticsearch/k-NN \ " >> Dockerfile
-#            echo "&& cd /usr/share/elasticsearch/k-NN/jni \ && sed -i 's/-march=native/-march=x86-64/g' external/nmslib/similarity_search/CMakeLists.txt \ && cmake . \ && make \ " >> Dockerfile
-#            echo "&& mkdir /tmp/jni/ && cp release/*.so /tmp/jni/ && ls -ltr /tmp/jni/ \ && cp /tmp/jni/libKNNIndex*.so /usr/lib \ && rm -rf /usr/share/elasticsearch/k-NN" >> Dockerfile
-#            echo "RUN cd /usr/share/elasticsearch/" >> Dockerfile
-#            echo "ADD k-NN/build/distributions/opendistro-knn-$plugin_version.zip /tmp/" >> Dockerfile
-#            echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin install --batch file:/tmp/opendistro-knn-$plugin_version.zip" >> Dockerfile
-#            docker build -t odfe-knn:test .
-#            echo "imagePresent=true" >> $GITHUB_ENV
-#          else
-#            echo "imagePresent=false" >> $GITHUB_ENV
-#          fi
-#      - name: Run Docker Image
-#        if: env.imagePresent == 'true'
-#        run: |
-#          cd ..
-#          docker run -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" odfe-knn:test
-#          sleep 90
-#      - name: Run k-NN Test
-#        if: env.imagePresent == 'true'
-#        run: |
-#          security=`curl -XGET https://localhost:9200/_cat/plugins?v -u admin:admin --insecure |grep opendistro_security|wc -l`
-#          if [ $security -gt 0 ]
-#          then
-#            echo "Security plugin is available. Running tests in security mode"
-#            ./gradlew :integTest -Dtests.rest.cluster=localhost:9200 -Dtests.cluster=localhost:9200 -Dtests.clustername="docker-cluster" -Dhttps=true -Duser=admin -Dpassword=admin
-#          else
-#            echo "Security plugin is NOT available. Skipping tests as they are already ran part of ./gradlew build"
-#          fi
+  Build-k-NN-MacOS:
+    strategy:
+      matrix:
+        java: [ 11, 17, 21 ]
+
+    name: Build and Test k-NN Plugin on MacOS
+    needs: Get-CI-Image-Tag
+    runs-on: macos-13
+
+    steps:
+      - name: Checkout k-NN
+        uses: actions/checkout@v1
+
+      # Setup git user so that patches for native libraries can be applied and committed
+      - name: Setup git user
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Install dependencies on macos
+        run: |
+          brew install libomp
+          brew reinstall gcc
+          export FC=/usr/local/Cellar/gcc/12.2.0/bin/gfortran
+
+      # TODO: Detect processor count and set the value of nproc.count
+      - name: Run build
+        run: |
+          if sysctl -n machdep.cpu.features machdep.cpu.leaf7_features | grep -i AVX2
+          then
+              echo "avx2 available on system"
+              ./gradlew build -Dnproc.count=3
+          else
+              echo "avx2 not available on system"
+              ./gradlew build -Davx2.enabled=false -Davx512.enabled=false -Dnproc.count=3
+          fi
+
+  Build-k-NN-Windows:
+    strategy:
+      matrix:
+        java: [ 11, 17, 21 ]
+
+    name: Build and Test k-NN Plugin on Windows
+    needs: Get-CI-Image-Tag
+    runs-on: windows-latest
+
+    steps:
+      - name: Checkout k-NN
+        uses: actions/checkout@v1
+
+      # Setup git user so that patches for native libraries can be applied and committed
+      - name: Setup git user
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Setup Java ${{ matrix.java }}
+        uses: actions/setup-java@v1
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Install MinGW Using Scoop
+        run: |
+          iex "& {$(irm get.scoop.sh)} -RunAsAdmin"
+          scoop bucket add main
+          scoop install mingw
+
+      - name: Add MinGW to PATH
+        run: |
+          echo "C:/Users/runneradmin/scoop/apps/mingw/current/bin" >> $env:GITHUB_PATH
+          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+          refreshenv
+
+      - name: Install Zlib Using Scoop
+        run: |
+          echo "C:/Users/runneradmin/scoop/shims" >> $env:GITHUB_PATH
+          Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
+          refreshenv
+          scoop bucket add extras
+          scoop install zlib
+          regedit /s "C:\\Users\\runneradmin\\scoop\\apps\\zlib\\current\\register.reg"
+
+      - name: Download OpenBLAS
+        run: |
+          curl -L -O https://github.com/xianyi/OpenBLAS/releases/download/v0.3.21/OpenBLAS-0.3.21-x64.zip
+          mkdir OpenBLAS
+          Expand-Archive -Path .\OpenBLAS-0.3.21-x64.zip -DestinationPath .\OpenBLAS\
+          mkdir ./src/main/resources/windowsDependencies
+          cp ./OpenBLAS/bin/libopenblas.dll ./src/main/resources/windowsDependencies/
+          rm .\OpenBLAS-0.3.21-x64.zip
+          rm -r .\OpenBLAS\
+
+      # TODO: Detect processor count and set the value of nproc.count
+      - name: Run build
+        run: |
+          ./gradlew.bat build -D'avx2.enabled=false' -D'avx512.enabled=false' -D'nproc.count=4'

.github/workflows/auto-release.yml

@@ -0,0 +1,28 @@
+name: Releases
+
+on:
+  push:
+    tags:
+      - '*'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
+      - name: Get tag
+        id: tag
+        uses: dawidd6/action-get-tag@v1
+      - uses: actions/checkout@v2
+      - uses: ncipollo/release-action@v1
+        with:
+          github_token: ${{ steps.github_app_token.outputs.token }}
+          bodyFile: release-notes/opensearch-knn.release-notes-${{steps.tag.outputs.tag}}.md

.github/workflows/backwards_compatibility_tests_workflow.yml

@@ -4,18 +4,38 @@ on:
     branches:
       - "*"
       - "feature/**"
+    paths:
+      - 'build.gradle'
+      - 'settings.gradle'
+      - 'src/**'
+      - 'build-tools/**'
+      - 'buildSrc/**'
+      - 'gradle/**'
+      - 'jni/**'
+      - 'qa/**'
+      - '.github/workflows/backwards_compatibility_tests_workflow.yml'
   pull_request:
     branches:
       - "*"
       - "feature/**"
+    paths:
+      - 'build.gradle'
+      - 'settings.gradle'
+      - 'src/**'
+      - 'build-tools/**'
+      - 'buildSrc/**'
+      - 'gradle/**'
+      - 'jni/**'
+      - 'qa/**'
+      - '.github/workflows/backwards_compatibility_tests_workflow.yml'
 
 jobs:
   Restart-Upgrade-BWCTests-k-NN:
     strategy:
       matrix:
         java: [ 11, 17 ]
-        bwc_version : [ "1.0.0", "1.1.0", "1.2.4", "1.3.2", "2.0.0" ]
-        opensearch_version : [ "2.1.0-SNAPSHOT" ]
+        bwc_version : [ "1.1.0", "1.2.4", "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0" ]
+        opensearch_version : [ "2.19.0-SNAPSHOT" ]
 
     name: k-NN Restart-Upgrade BWC Tests
     runs-on: ubuntu-latest
@@ -26,6 +46,12 @@ jobs:
       - name: Checkout k-NN
         uses: actions/checkout@v1
 
+      # Setup git user so that patches for native libraries can be applied and committed
+      - name: Setup git user
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
@@ -38,26 +64,42 @@ jobs:
 
       - name: Run k-NN Restart-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }}
         run: |
-          echo "Running restart-upgrade backwards compatibility tests ..."
-          ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE
-
+          echo "Running restart-upgrade backwards compatibility tests ..."          
+          if lscpu  | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw          
+          then
+            echo "avx512 available on system"
+            ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Dnproc.count=`nproc`
+          elif lscpu  | grep -i avx2
+          then
+            echo "avx2 available on system"
+            ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Dnproc.count=`nproc` -Davx512.enabled=false
+          else
+            echo "avx512 and avx2 not available on system"
+            ./gradlew :qa:restart-upgrade:testRestartUpgrade -Dtests.bwc.version=$BWC_VERSION_RESTART_UPGRADE -Davx2.enabled=false -Davx512.enabled=false -Dsimd.enabled=false -Dnproc.count=`nproc`
+          fi 
 
   Rolling-Upgrade-BWCTests-k-NN:
     strategy:
       matrix:
         java: [ 11, 17 ]
-        bwc_version: [ "1.3.2", "2.0.0" ]
-        opensearch_version: [ "2.1.0-SNAPSHOT" ]
+        bwc_version: [ "1.3.8", "2.0.1", "2.1.0", "2.2.1", "2.3.0", "2.4.1", "2.5.0", "2.6.0", "2.7.0", "2.8.0", "2.9.0", "2.10.0", "2.11.0", "2.12.0", "2.13.0", "2.14.0", "2.15.0", "2.16.0", "2.17.0", "2.18.0"]
+        opensearch_version: [ "2.19.0-SNAPSHOT" ]
 
     name: k-NN Rolling-Upgrade BWC Tests
     runs-on: ubuntu-latest
     env:
-      OPENSEARCH_VERSION_ROLLING_UPGRADE: ${{ matrix.opensearch_version }}
+      BWC_VERSION_ROLLING_UPGRADE: ${{ matrix.bwc_version }}
 
     steps:
       - name: Checkout k-NN
         uses: actions/checkout@v1
 
+      # Setup git user so that patches for native libraries can be applied and committed
+      - name: Setup git user
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+
       - name: Setup Java ${{ matrix.java }}
         uses: actions/setup-java@v1
         with:
@@ -70,4 +112,15 @@ jobs:
       - name: Run k-NN Rolling-Upgrade BWC Tests from BWCVersion-${{ matrix.bwc_version }} to OpenSearch Version-${{ matrix.opensearch_version }}
         run: |
           echo "Running rolling-upgrade backwards compatibility tests ..."
-          ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Drolling.bwctests.opensearch.version=$OPENSEARCH_VERSION_ROLLING_UPGRADE
+          if lscpu  | grep -i avx512f | grep -i avx512cd | grep -i avx512vl | grep -i avx512dq | grep -i avx512bw          
+          then
+            echo "avx512 available on system"
+            ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Dnproc.count=`nproc`
+          elif lscpu  | grep -i avx2
+          then
+            echo "avx2 available on system"
+            ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Dnproc.count=`nproc` -Davx512.enabled=false
+          else
+            echo "avx512 and avx2 not available on system"
+            ./gradlew :qa:rolling-upgrade:testRollingUpgrade -Dtests.bwc.version=$BWC_VERSION_ROLLING_UPGRADE -Davx2.enabled=false -Davx512.enabled=false -Dsimd.enabled=false -Dnproc.count=`nproc`
+          fi