diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a5fda2a21..a19b53fd84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Bug Fixes * Fixing the bug when a segment has no vector field present for disk based vector search (#2282)[https://github.com/opensearch-project/k-NN/pull/2282] * Allow validation for non knn index only after 2.17.0 (#2315)[https://github.com/opensearch-project/k-NN/pull/2315] +* Release query vector memory after execution (#2346)[https://github.com/opensearch-project/k-NN/pull/2346] +* Fix shard level rescoring disabled setting flag (#2352)[https://github.com/opensearch-project/k-NN/pull/2352] ### Infrastructure * Updated C++ version in JNI from c++11 to c++17 [#2259](https://github.com/opensearch-project/k-NN/pull/2259) * Upgrade bytebuddy and objenesis version to match OpenSearch core and, update github ci runner for macos [#2279](https://github.com/opensearch-project/k-NN/pull/2279) diff --git a/jni/src/faiss_wrapper.cpp b/jni/src/faiss_wrapper.cpp index 98c40cf6b3..c02c410c1f 100644 --- a/jni/src/faiss_wrapper.cpp +++ b/jni/src/faiss_wrapper.cpp @@ -1180,6 +1180,7 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); throw; } + jniUtil->ReleaseLongArrayElements(env, filterIdsJ, filteredIdsArray, JNI_ABORT); } else { faiss::SearchParameters *searchParameters = nullptr; faiss::SearchParametersHNSW hnswParams; @@ -1202,6 +1203,7 @@ jobjectArray knn_jni::faiss_wrapper::RangeSearchWithFilter(knn_jni::JNIUtilInter throw; } } + jniUtil->ReleaseFloatArrayElements(env, queryVectorJ, rawQueryVector, JNI_ABORT); // lims is structured to support batched queries, it has a length of nq + 1 (where nq is the number of queries), // lims[i] - lims[i-1] gives the number of results for the i-th query. With a single query we used in k-NN, diff --git a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java index e79e2abefd..4ba1fe03aa 100644 --- a/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java +++ b/qa/restart-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRestartUpgradeTestCase.java @@ -61,4 +61,13 @@ protected static final boolean isRunningAgainstOldCluster() { protected final Optional getBWCVersion() { return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); } + + @Override + protected Settings getKNNDefaultIndexSettings() { + if (isApproximateThresholdSupported(getBWCVersion())) { + return super.getKNNDefaultIndexSettings(); + } + // for bwc will return old default setting without approximate value threshold setting + return getDefaultIndexSettings(); + } } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java index 235e3e4df3..16e4a9c3d2 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/AbstractRollingUpgradeTestCase.java @@ -85,4 +85,13 @@ protected final Optional getBWCVersion() { return Optional.ofNullable(System.getProperty(BWC_VERSION, null)); } + @Override + protected Settings getKNNDefaultIndexSettings() { + if (isApproximateThresholdSupported(getBWCVersion())) { + return super.getKNNDefaultIndexSettings(); + } + // for bwc will return old default setting without approximate value threshold setting + return getDefaultIndexSettings(); + } + } diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java index d0ee16c99f..c4bf6082cf 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/knn/bwc/IndexingIT.java @@ -22,7 +22,7 @@ public void testKNNDefaultIndexSettings() throws Exception { waitForClusterHealthGreen(NODES_BWC_CLUSTER); switch (getClusterType()) { case OLD: - createKnnIndex(testIndex, getKNNDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); + createKnnIndex(testIndex, getDefaultIndexSettings(), createKnnIndexMapping(TEST_FIELD, DIMENSIONS)); int docIdOld = 0; addKNNDocs(testIndex, TEST_FIELD, DIMENSIONS, docIdOld, NUM_DOCS); break; diff --git a/src/main/java/org/opensearch/knn/index/KNNSettings.java b/src/main/java/org/opensearch/knn/index/KNNSettings.java index b81a541248..6dc72a22bd 100644 --- a/src/main/java/org/opensearch/knn/index/KNNSettings.java +++ b/src/main/java/org/opensearch/knn/index/KNNSettings.java @@ -577,7 +577,7 @@ public static Integer getFilteredExactSearchThreshold(final String indexName) { .getAsInt(ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD, ADVANCED_FILTERED_EXACT_SEARCH_THRESHOLD_DEFAULT_VALUE); } - public static boolean isShardLevelRescoringEnabledForDiskBasedVector(String indexName) { + public static boolean isShardLevelRescoringDisabledForDiskBasedVector(String indexName) { return KNNSettings.state().clusterService.state() .getMetadata() .index(indexName) diff --git a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java index 47ea215f35..5b4d6e7a1d 100644 --- a/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java +++ b/src/main/java/org/opensearch/knn/index/query/nativelib/NativeEngineKnnVectorQuery.java @@ -63,11 +63,11 @@ public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, flo if (rescoreContext == null) { perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, finalK); } else { - boolean isShardLevelRescoringEnabled = KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(knnQuery.getIndexName()); + boolean isShardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(knnQuery.getIndexName()); int dimension = knnQuery.getQueryVector().length; - int firstPassK = rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension); + int firstPassK = rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension); perLeafResults = doSearch(indexSearcher, leafReaderContexts, knnWeight, firstPassK); - if (isShardLevelRescoringEnabled == true) { + if (isShardLevelRescoringDisabled == false) { ResultUtil.reduceToTopK(perLeafResults, firstPassK); } diff --git a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java index 09aeb75911..03c03a87fb 100644 --- a/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java +++ b/src/main/java/org/opensearch/knn/index/query/rescore/RescoreContext.java @@ -61,17 +61,17 @@ public static RescoreContext getDefault() { * based on the vector dimension if shard-level rescoring is disabled. * * @param finalK The final number of results to return for the entire shard. - * @param isShardLevelRescoringEnabled A boolean flag indicating whether shard-level rescoring is enabled. - * If true, the dimension-based oversampling logic is bypassed. + * @param isShardLevelRescoringDisabled A boolean flag indicating whether shard-level rescoring is disabled. + * If false, the dimension-based oversampling logic is bypassed. * @param dimension The dimension of the vector. This is used to determine the oversampling factor when * shard-level rescoring is disabled. * @return The number of results to return for the first pass of rescoring, adjusted by the oversample factor. */ - public int getFirstPassK(int finalK, boolean isShardLevelRescoringEnabled, int dimension) { + public int getFirstPassK(int finalK, boolean isShardLevelRescoringDisabled, int dimension) { // Only apply default dimension-based oversampling logic when: // 1. Shard-level rescoring is disabled // 2. The oversample factor was not provided by the user - if (!isShardLevelRescoringEnabled && !userProvided) { + if (isShardLevelRescoringDisabled && !userProvided) { // Apply new dimension-based oversampling logic when shard-level rescoring is disabled if (dimension >= DIMENSION_THRESHOLD_1000) { oversampleFactor = OVERSAMPLE_FACTOR_1000; // No oversampling for dimensions >= 1000 diff --git a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java index 7bfce5b94a..41cd4e8a55 100644 --- a/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java +++ b/src/test/java/org/opensearch/knn/KNNSingleNodeTestCase.java @@ -96,7 +96,7 @@ public void tearDown() throws Exception { * Create a k-NN index with default settings */ protected IndexService createKNNIndex(String indexName) { - return createIndex(indexName, getKNNDefaultIndexSettings()); + return createIndex(indexName, getKNNDefaultIndexSettingsBuildsGraphAlways()); } /** @@ -161,13 +161,6 @@ protected void createKnnNestedIndexMapping(String indexName, String fieldPath, I OpenSearchAssertions.assertAcked(client().admin().indices().putMapping(request).actionGet()); } - /** - * Get default k-NN settings for test cases - */ - protected Settings getKNNDefaultIndexSettings() { - return Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", true).build(); - } - /** * Get default k-NN settings for test cases with build graph always */ diff --git a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java index 20940f151d..b6bbb748a4 100644 --- a/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java +++ b/src/test/java/org/opensearch/knn/index/KNNESSettingsTestIT.java @@ -106,7 +106,11 @@ public void testCreateIndexWithInvalidSpaceType() throws IOException { } public void testUpdateIndexSetting() throws IOException { - Settings settings = Settings.builder().put("index.knn", true).put(KNNSettings.KNN_ALGO_PARAM_EF_SEARCH, 512).build(); + Settings settings = Settings.builder() + .put("index.knn", true) + .put(KNNSettings.KNN_ALGO_PARAM_EF_SEARCH, 512) + .put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) + .build(); createKnnIndex(INDEX_NAME, settings, createKnnIndexMapping(FIELD_NAME, 2)); assertEquals("512", getIndexSettingByName(INDEX_NAME, KNNSettings.KNN_ALGO_PARAM_EF_SEARCH)); @@ -122,7 +126,7 @@ public void testUpdateIndexSetting() throws IOException { @SuppressWarnings("unchecked") public void testCacheRebuiltAfterUpdateIndexSettings() throws Exception { - createKnnIndex(INDEX_NAME, buildKNNIndexSettings(0), createKnnIndexMapping(FIELD_NAME, 2)); + createKnnIndex(INDEX_NAME, getKNNDefaultIndexSettings(), createKnnIndexMapping(FIELD_NAME, 2)); Float[] vector = { 6.0f, 6.0f }; addKnnDoc(INDEX_NAME, "1", FIELD_NAME, vector); diff --git a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java index c7a8e7ed83..24990dd364 100644 --- a/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNSettingsTests.java @@ -159,7 +159,7 @@ public void testGetEfSearch_whenEFSearchValueSetByUser_thenReturnValue() { } @SneakyThrows - public void testShardLevelRescoringEnabled_whenNoValuesProvidedByUser_thenDefaultSettingsUsed() { + public void testShardLevelRescoringDisabled_whenNoValuesProvidedByUser_thenDefaultSettingsUsed() { Node mockNode = createMockNode(Collections.emptyMap()); mockNode.start(); ClusterService clusterService = mockNode.injector().getInstance(ClusterService.class); @@ -167,7 +167,7 @@ public void testShardLevelRescoringEnabled_whenNoValuesProvidedByUser_thenDefaul mockNode.client().admin().indices().create(new CreateIndexRequest(INDEX_NAME)).actionGet(); KNNSettings.state().setClusterService(clusterService); - boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(INDEX_NAME); + boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(INDEX_NAME); mockNode.close(); assertFalse(shardLevelRescoringDisabled); } @@ -188,7 +188,7 @@ public void testShardLevelRescoringDisabled_whenValueProvidedByUser_thenSettingA mockNode.client().admin().indices().updateSettings(new UpdateSettingsRequest(rescoringDisabledSetting, INDEX_NAME)).actionGet(); - boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(INDEX_NAME); + boolean shardLevelRescoringDisabled = KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(INDEX_NAME); mockNode.close(); assertEquals(userDefinedRescoringDisabled, shardLevelRescoringDisabled); } diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 9ca0415802..88dd908d3d 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -640,7 +640,7 @@ public void testKNNIndex_whenBuildGraphThresholdIsPresent_thenGetThresholdValue( public void testKNNIndex_whenBuildThresholdIsNotProvided_thenShouldNotReturnSetting() throws Exception { final String knnIndexMapping = createKnnIndexMapping(FIELD_NAME, KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT)); final String indexName = "test-index-with-build-graph-settings"; - createKnnIndex(indexName, knnIndexMapping); + createKnnIndex(indexName, getDefaultIndexSettings(), knnIndexMapping); final String buildVectorDataStructureThresholdSetting = getIndexSettingByName( indexName, KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD @@ -655,7 +655,7 @@ public void testKNNIndex_whenBuildThresholdIsNotProvided_thenShouldNotReturnSett public void testKNNIndex_whenGetIndexSettingWithDefaultIsCalled_thenReturnDefaultBuildGraphThresholdValue() throws Exception { final String knnIndexMapping = createKnnIndexMapping(FIELD_NAME, KNNEngine.getMaxDimensionByEngine(KNNEngine.DEFAULT)); final String indexName = "test-index-with-build-vector-graph-settings"; - createKnnIndex(indexName, knnIndexMapping); + createKnnIndex(indexName, getDefaultIndexSettings(), knnIndexMapping); final String buildVectorDataStructureThresholdSetting = getIndexSettingByName( indexName, KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, diff --git a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java index 789bd10542..87c4a50148 100644 --- a/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java +++ b/src/test/java/org/opensearch/knn/index/query/nativelib/NativeEngineKNNVectorQueryTests.java @@ -183,7 +183,7 @@ public void testRescoreWhenShardLevelRescoringEnabled() { ) { // When shard-level re-scoring is enabled - mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(any())).thenReturn(true); + mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(any())).thenReturn(false); // Mock ResultUtil to return valid TopDocs mockedResultUtil.when(() -> ResultUtil.resultMapToTopDocs(any(), anyInt())) @@ -265,7 +265,7 @@ public void testRescore() { ) { // When shard-level re-scoring is enabled - mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringEnabledForDiskBasedVector(any())).thenReturn(true); + mockedKnnSettings.when(() -> KNNSettings.isShardLevelRescoringDisabledForDiskBasedVector(any())).thenReturn(false); mockedResultUtil.when(() -> ResultUtil.reduceToTopK(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); mockedResultUtil.when(() -> ResultUtil.resultMapToDocIds(any(), anyInt())).thenAnswer(InvocationOnMock::callRealMethod); diff --git a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java index 2b309e4abd..a0a5cc5466 100644 --- a/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java +++ b/src/test/java/org/opensearch/knn/index/query/rescore/RescoreContextTests.java @@ -16,23 +16,23 @@ public void testGetFirstPassK() { float oversample = 2.6f; RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).userProvided(true).build(); int finalK = 100; - boolean isShardLevelRescoringEnabled = true; + boolean isShardLevelRescoringDisabled = false; int dimension = 500; // Case 1: Test with standard oversample factor when shard-level rescoring is enabled - assertEquals(260, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + assertEquals(260, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension)); // Case 2: Test with a very small finalK that should result in a value less than MIN_FIRST_PASS_RESULTS finalK = 1; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension)); // Case 3: Test with finalK = 0, should return MIN_FIRST_PASS_RESULTS finalK = 0; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension)); // Case 4: Test with finalK = MAX_FIRST_PASS_RESULTS, should cap at MAX_FIRST_PASS_RESULTS finalK = MAX_FIRST_PASS_RESULTS; - assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + assertEquals(MAX_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension)); } public void testGetFirstPassKWithDimensionBasedOversampling() { @@ -42,44 +42,44 @@ public void testGetFirstPassKWithDimensionBasedOversampling() { // Case 1: Test no oversampling for dimensions >= 1000 when shard-level rescoring is disabled dimension = 1000; RescoreContext rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensuring dimension-based logic applies - assertEquals(100, rescoreContext.getFirstPassK(finalK, false, dimension)); // No oversampling + assertEquals(100, rescoreContext.getFirstPassK(finalK, true, dimension)); // No oversampling // Case 2: Test 2x oversampling for dimensions >= 768 but < 1000 when shard-level rescoring is disabled dimension = 800; rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensure previous values don't carry over - assertEquals(200, rescoreContext.getFirstPassK(finalK, false, dimension)); // 2x oversampling + assertEquals(200, rescoreContext.getFirstPassK(finalK, true, dimension)); // 2x oversampling // Case 3: Test 3x oversampling for dimensions < 768 when shard-level rescoring is disabled dimension = 700; rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensure previous values don't carry over - assertEquals(300, rescoreContext.getFirstPassK(finalK, false, dimension)); // 3x oversampling + assertEquals(300, rescoreContext.getFirstPassK(finalK, true, dimension)); // 3x oversampling // Case 4: Shard-level rescoring enabled, oversample factor should be used as provided by the user (ignore dimension) rescoreContext = RescoreContext.builder().oversampleFactor(5.0f).userProvided(true).build(); // Provided by user dimension = 500; - assertEquals(500, rescoreContext.getFirstPassK(finalK, true, dimension)); // User-defined oversample factor should be used + assertEquals(500, rescoreContext.getFirstPassK(finalK, false, dimension)); // User-defined oversample factor should be used // Case 5: Test finalK where oversampling factor results in a value less than MIN_FIRST_PASS_RESULTS finalK = 10; dimension = 700; rescoreContext = RescoreContext.builder().userProvided(false).build(); // Ensure dimension-based logic applies - assertEquals(100, rescoreContext.getFirstPassK(finalK, false, dimension)); // 3x oversampling results in 30 + assertEquals(100, rescoreContext.getFirstPassK(finalK, true, dimension)); // 3x oversampling results in 30 } public void testGetFirstPassKWithMinPassK() { float oversample = 0.5f; RescoreContext rescoreContext = RescoreContext.builder().oversampleFactor(oversample).userProvided(true).build(); // User provided - boolean isShardLevelRescoringEnabled = false; + boolean isShardLevelRescoringDisabled = true; // Case 1: Test where finalK * oversample is smaller than MIN_FIRST_PASS_RESULTS int finalK = 10; int dimension = 700; - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension)); // Case 2: Test where finalK * oversample results in exactly MIN_FIRST_PASS_RESULTS finalK = 100; oversample = 1.0f; // This will result in exactly 100 (MIN_FIRST_PASS_RESULTS) rescoreContext = RescoreContext.builder().oversampleFactor(oversample).userProvided(true).build(); // User provided - assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringEnabled, dimension)); + assertEquals(MIN_FIRST_PASS_RESULTS, rescoreContext.getFirstPassK(finalK, isShardLevelRescoringDisabled, dimension)); } } diff --git a/src/test/java/org/opensearch/knn/integ/ExpandNestedDocsIT.java b/src/test/java/org/opensearch/knn/integ/ExpandNestedDocsIT.java index 38e528e955..decd74ab95 100644 --- a/src/test/java/org/opensearch/knn/integ/ExpandNestedDocsIT.java +++ b/src/test/java/org/opensearch/knn/integ/ExpandNestedDocsIT.java @@ -19,6 +19,7 @@ import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.NestedKnnDocBuilder; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.mapper.Mode; @@ -329,6 +330,7 @@ private void createKnnIndex( .put("number_of_shards", numOfShards) .put("number_of_replicas", 0) .put("index.knn", true) + .put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) .build(); createKnnIndex(INDEX_NAME, settings, mapping); } diff --git a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java index d1288c5f34..42a4f2b956 100644 --- a/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java +++ b/src/test/java/org/opensearch/knn/integ/KNNScriptScoringIT.java @@ -14,6 +14,7 @@ import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.KNNConstants; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.opensearch.client.Request; @@ -840,7 +841,11 @@ private void createIndexAndAssertScriptScore( /* * Create knn index and populate data */ - Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", enableKnn).build(); + Settings.Builder builder = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", enableKnn); + if (enableKnn) { + builder.put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0); + } + Settings settings = builder.build(); createKnnIndex(INDEX_NAME, settings, mapper); try { final int numDocsWithField = randomIntBetween(4, 10); diff --git a/src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java b/src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java index 9daef76e95..4d5248a452 100644 --- a/src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java +++ b/src/test/java/org/opensearch/knn/integ/PainlessScriptScoreIT.java @@ -9,6 +9,7 @@ import org.opensearch.common.settings.Settings; import org.opensearch.knn.KNNRestTestCase; import org.opensearch.knn.KNNResult; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; import org.opensearch.knn.index.SpaceType; @@ -622,7 +623,11 @@ private Response buildIndexAndRunPainlessScript( /* * Create knn index and populate data */ - Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", enableKnn).build(); + Settings.Builder builder = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", enableKnn); + if (enableKnn) { + builder.put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0); + } + Settings settings = builder.build(); createKnnIndex(INDEX_NAME, settings, mapper); try { for (Map.Entry data : documents.entrySet()) { diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java index 671185abfe..e8da155892 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestKNNStatsHandlerIT.java @@ -24,6 +24,7 @@ import org.opensearch.index.query.QueryBuilder; import org.opensearch.index.query.QueryBuilders; import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.plugin.stats.KNNStats; @@ -292,7 +293,12 @@ public void testScriptStats_multipleShards() throws Exception { // Create an index with a single vector createKnnIndex( INDEX_NAME, - Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 0).put("index.knn", true).build(), + Settings.builder() + .put("number_of_shards", 2) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) + .build(), createKnnIndexMapping(FIELD_NAME, 2) ); diff --git a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java index bea93e13c7..e00d90056d 100644 --- a/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java +++ b/src/test/java/org/opensearch/knn/plugin/action/RestLegacyKNNStatsHandlerIT.java @@ -12,6 +12,7 @@ package org.opensearch.knn.plugin.action; import org.opensearch.knn.KNNRestTestCase; +import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.SpaceType; import org.opensearch.knn.index.query.KNNQueryBuilder; import org.opensearch.knn.plugin.KNNPlugin; @@ -266,7 +267,12 @@ public void testScriptStats_multipleShards() throws Exception { // Create an index with a single vector createKnnIndex( INDEX_NAME, - Settings.builder().put("number_of_shards", 2).put("number_of_replicas", 0).put("index.knn", true).build(), + Settings.builder() + .put("number_of_shards", 2) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put(KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) + .build(), createKnnIndexMapping(FIELD_NAME, 2) ); diff --git a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java index 3dfb674153..be2dd7b82b 100644 --- a/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java +++ b/src/test/java/org/opensearch/knn/recall/RecallTestsIT.java @@ -46,6 +46,7 @@ import static org.opensearch.knn.common.KNNConstants.PARAMETERS; import static org.opensearch.knn.common.KNNConstants.TYPE; import static org.opensearch.knn.common.KNNConstants.TYPE_KNN_VECTOR; +import static org.opensearch.knn.index.KNNSettings.INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD; import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_EF_SEARCH; import static org.opensearch.knn.index.KNNSettings.KNN_ALGO_PARAM_INDEX_THREAD_QTY; import static org.opensearch.knn.index.KNNSettings.KNN_MEMORY_CIRCUIT_BREAKER_ENABLED; @@ -152,6 +153,7 @@ public void testRecall_whenNmslibHnswFP32_thenRecallAbove75percent() { .put("number_of_shards", SHARD_COUNT) .put("number_of_replicas", REPLICA_COUNT) .put("index.knn", true) + .put(INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) .put(KNN_ALGO_PARAM_EF_SEARCH, HNSW_EF_SEARCH) .build(), builder.toString() @@ -531,6 +533,7 @@ private Settings getSettings() { .put("number_of_shards", SHARD_COUNT) .put("number_of_replicas", REPLICA_COUNT) .put("index.knn", true) + .put(INDEX_KNN_ADVANCED_APPROXIMATE_THRESHOLD, 0) .build(); } } diff --git a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java index a5b4b06923..896674a18c 100644 --- a/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java +++ b/src/testFixtures/java/org/opensearch/knn/KNNRestTestCase.java @@ -797,7 +797,14 @@ protected void updateClusterSettings(String settingKey, Object value) throws Exc * Return default index settings for index creation */ protected Settings getKNNDefaultIndexSettings() { - return Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", true).build(); + return buildKNNIndexSettings(0); + } + + /** + * Return default index settings for index creation + */ + protected Settings getDefaultIndexSettings() { + return Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put(KNN_INDEX, true).build(); } protected Settings getKNNSegmentReplicatedIndexSettings() {