fix broken build flag, move build to one directory (opensearch-projec…

…t#2442) * move build to one directory, fix broken flag Signed-off-by: Samuel Herman <[email protected]> * fix make path Signed-off-by: Samuel Herman <[email protected]> * changelog update Signed-off-by: Samuel Herman <[email protected]> * add fix for classpath change and for cmake discovery on macos Signed-off-by: Samuel Herman <[email protected]> * fix make discovery for gradle Signed-off-by: Samuel Herman <[email protected]> * fix cmake path for macOS Signed-off-by: Samuel Herman <[email protected]> --------- Signed-off-by: Samuel Herman <[email protected]>
rchitale7 · Feb 10, 2025 · d83057d · d83057d
1 parent 55652ca
commit d83057d
Show file tree

Hide file tree

Showing 18 changed files with 83 additions and 27 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -45,7 +45,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Fixing it to retrieve space_type from index setting when both method and top level don't have the value. [#2374](https://github.com/opensearch-project/k-NN/pull/2374)
 * Fixing the bug where setting rescore as false for on_disk knn_vector query is a no-op (#2399)[https://github.com/opensearch-project/k-NN/pull/2399]
 * Fixing bug where mapping accepts both dimension and model-id (#2410)[https://github.com/opensearch-project/k-NN/pull/2410]
-* Add version check for full field name validation (#2477)[https://github.com/opensearch-project/k-NN/pull/2477]
 ### Infrastructure
 * Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259)
 * Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279)
@@ -58,5 +57,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 * Upgrade jsonpath from 2.8.0 to 2.9.0[2325](https://github.com/opensearch-project/k-NN/pull/2325)
 * Bump Faiss commit from 1f42e81 to 0cbc2a8 to accelerate hamming distance calculation using _mm512_popcnt_epi64  intrinsic and also add avx512-fp16 instructions to boost performance [#2381](https://github.com/opensearch-project/k-NN/pull/2381)
 * Enabled indices.breaker.total.use_real_memory setting via build.gradle for integTest Cluster to catch heap CB in local ITs and github CI actions [#2395](https://github.com/opensearch-project/k-NN/pull/2395/) 
-* Fixing Lucene912Codec Issue with BWC for Lucene 10.0.1 upgrade[#2429](https://github.com/opensearch-project/k-NN/pull/2429)
 ### Refactoring
diff --git a/build.gradle b/build.gradle
@@ -329,11 +329,52 @@ task windowsPatches(type:Exec) {
     commandLine 'cmd', '/c', "Powershell -File $rootDir\\scripts\\windowsScript.ps1"
 }
 
-task cmakeJniLib(type:Exec) {
-    workingDir 'jni'
+def findExecutable(String executableName, List<String> additionalPaths = []) {
+    // Print the task's environment before setting it
+    // Print PATH specifically
+    logger.lifecycle("\nSystem PATH:")
+    logger.lifecycle(System.getenv("PATH"))
+
+    def commonBasePaths = [
+            "/opt/homebrew/bin",
+            "/usr/local/bin",
+            "/usr/bin"
+    ]
+
+    // Start with just the executable name (will use system PATH)
+    def execPath = executableName
+
+    if (Os.isFamily(Os.FAMILY_MAC)) {
+        def searchPaths = []
+        // Add common paths
+        commonBasePaths.each { basePath ->
+            searchPaths.add("${basePath}/${executableName}")
+        }
+        // Add any additional specific paths
+        searchPaths.addAll(additionalPaths)
+
+        for (path in searchPaths) {
+            if (new File(path).exists()) {
+                logger.lifecycle("Found ${executableName} at: ${path}")
+                execPath = path
+                break
+            }
+        }
+    }
+
+    return execPath
+}
+
+tasks.register('cmakeJniLib', Exec) {
+    def cmakePath = findExecutable("cmake")
+    logger.lifecycle("Using cmake at: ${cmakePath}")
+    // Inherit the current environment
+    environment System.getenv()
+
     def args = []
-    args.add("cmake")
-    args.add(".")
+    args.add(cmakePath)
+    args.add("-S jni")
+    args.add("-B jni/build")
     args.add("-DKNN_PLUGIN_VERSION=${opensearch_version}")
     args.add("-DAVX2_ENABLED=${avx2_enabled}")
     args.add("-DAVX512_ENABLED=${avx512_enabled}")
@@ -343,7 +384,7 @@ task cmakeJniLib(type:Exec) {
     def javaHome = Jvm.current().getJavaHome()
     logger.lifecycle("Java home directory used by gradle: $javaHome")
     if (Os.isFamily(Os.FAMILY_MAC)) {
-        environment('JAVA_HOME',javaHome)
+        environment('JAVA_HOME', javaHome)
     }
     if (Os.isFamily(Os.FAMILY_WINDOWS)) {
         dependsOn windowsPatches
@@ -353,13 +394,30 @@ task cmakeJniLib(type:Exec) {
         args.add("-DLAPACK_LIBRARIES=$rootDir\\src\\main\\resources\\windowsDependencies\\libopenblas.dll")
     }
 
+    // Print the task's environment after setting it
+    logger.lifecycle("\nTask Environment PATH:")
+    logger.lifecycle(environment.get('PATH'))
+
+    // Print the command that will be executed
+    logger.lifecycle("CMake command: ${args.join(' ')}")
+    def outputStream = new ByteArrayOutputStream()
     commandLine args
+    standardOutput = outputStream
 }
 
 task buildJniLib(type:Exec) {
     dependsOn cmakeJniLib
-    workingDir 'jni'
-    commandLine 'make', 'opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
+    def makePath = findExecutable("make")
+    logger.lifecycle("Using make at: ${makePath}")
+    println "Make path: ${makePath}"
+    // Ensure makePath is treated as a String. If parsing to an int is required,
+    // handle it explicitly, though a path typically should not be parsed as an int.
+    if (makePath.isEmpty()) {
+        throw new GradleException("Make not found in PATH. Please install Make.")
+    }
+    def outputStream = new ByteArrayOutputStream()
+    commandLine makePath, '-Cjni/build','opensearchknn_nmslib', 'opensearchknn_faiss', 'opensearchknn_common', '-j', "${nproc_count}"
+    standardOutput = outputStream
 }
 
 test {

diff --git a/jni/cmake/init-faiss.cmake b/jni/cmake/init-faiss.cmake
@@ -111,10 +111,10 @@ endif()
 if(NOT DEFINED AVX512_SPR_ENABLED)
     # Check if the system is Intel(R) Sapphire Rapids or a newer-generation processor
     execute_process(COMMAND bash -c "lscpu | grep -q 'GenuineIntel' && lscpu | grep -i 'avx512_fp16' | grep -i 'avx512_bf16' | grep -i 'avx512_vpopcntdq'" OUTPUT_VARIABLE SPR_FLAGS OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if (AND NOT "${SPR_FLAGS}" STREQUAL "")
-	set(AVX512_SPR_ENABLED true)
+    if (NOT "${SPR_FLAGS}" STREQUAL "")
+	      set(AVX512_SPR_ENABLED true)
     else()
-	set(AVX512_SPR_ENABLED false)
+	      set(AVX512_SPR_ENABLED false)
     endif()
 endif()
 

diff --git a/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java b/src/main/java/org/opensearch/knn/index/KNNCircuitBreaker.java
@@ -13,7 +13,7 @@
 import org.opensearch.knn.plugin.transport.KNNStatsResponse;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.threadpool.ThreadPool;

diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java
@@ -11,7 +11,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Booleans;

diff --git a/src/main/java/org/opensearch/knn/indices/ModelDao.java b/src/main/java/org/opensearch/knn/indices/ModelDao.java
@@ -36,7 +36,7 @@
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.WriteRequest;
 import org.opensearch.action.support.clustermanager.AcknowledgedResponse;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.health.ClusterHealthStatus;
 import org.opensearch.cluster.health.ClusterIndexHealth;
 import org.opensearch.cluster.metadata.IndexMetadata;

diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java
@@ -49,7 +49,7 @@
 import com.google.common.collect.ImmutableList;
 
 import org.opensearch.action.ActionRequest;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.service.ClusterService;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestClearCacheHandler.java
@@ -8,7 +8,7 @@
 import com.google.common.collect.ImmutableList;
 import lombok.AllArgsConstructor;
 import lombok.extern.log4j.Log4j2;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.core.common.Strings;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestDeleteModelHandler.java
@@ -12,7 +12,7 @@
 package org.opensearch.knn.plugin.rest;
 
 import com.google.common.collect.ImmutableList;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.core.common.Strings;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.transport.DeleteModelAction;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestGetModelHandler.java
@@ -13,7 +13,7 @@
 
 import com.google.common.collect.ImmutableList;
 import org.apache.commons.lang.StringUtils;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.transport.GetModelAction;
 import org.opensearch.knn.plugin.transport.GetModelRequest;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNStatsHandler.java
@@ -11,7 +11,7 @@
 import org.opensearch.knn.plugin.transport.KNNStatsAction;
 import org.opensearch.knn.plugin.transport.KNNStatsRequest;
 import com.google.common.collect.ImmutableList;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.RestActions;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestKNNWarmupHandler.java
@@ -13,7 +13,7 @@
 import com.google.common.collect.ImmutableList;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestSearchModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestSearchModelHandler.java
@@ -13,7 +13,7 @@
 
 import com.google.common.collect.ImmutableList;
 import org.opensearch.action.search.SearchRequest;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.knn.plugin.KNNPlugin;
 import org.opensearch.knn.plugin.transport.SearchModelAction;
 import org.opensearch.rest.BaseRestHandler;

diff --git a/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java b/src/main/java/org/opensearch/knn/plugin/rest/RestTrainModelHandler.java
@@ -12,7 +12,7 @@
 package org.opensearch.knn.plugin.rest;
 
 import com.google.common.collect.ImmutableList;
-import org.opensearch.client.node.NodeClient;
+import org.opensearch.transport.client.node.NodeClient;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.mapper.NumberFieldMapper;
 import org.opensearch.knn.common.KNNConstants;

diff --git a/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java b/src/main/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportAction.java
@@ -17,7 +17,7 @@
 import org.opensearch.action.search.SearchRequest;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.HandledTransportAction;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.ValidationException;

diff --git a/src/main/java/org/opensearch/knn/training/VectorReader.java b/src/main/java/org/opensearch/knn/training/VectorReader.java
@@ -17,7 +17,7 @@
 import org.opensearch.action.search.SearchRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchScrollRequestBuilder;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.ValidationException;

diff --git a/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java b/src/test/java/org/opensearch/knn/plugin/transport/TrainingJobRouterTransportActionTests.java
@@ -16,7 +16,7 @@
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.support.ActionFilters;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.node.DiscoveryNode;

diff --git a/...org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java b/...org/opensearch/knn/quantization/models/quantizationState/QuantizationStateCacheTests.java
@@ -9,7 +9,7 @@
 import lombok.SneakyThrows;
 import org.junit.After;
 import org.junit.Before;
-import org.opensearch.client.Client;
+import org.opensearch.transport.client.Client;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;