From ea34e81e7c74a8c386dfdc494c1c5132d966af07 Mon Sep 17 00:00:00 2001 From: Jan Schatteman Date: Wed, 9 Apr 2025 22:49:46 +0200 Subject: [PATCH] HSEARCH-5305 - adjust vector search max dimensions for Lucene backend Signed-off-by: Jan Schatteman --- .../codec/impl/HibernateSearchKnnVectorsFormat.java | 4 +--- documentation/src/main/asciidoc/migration/index.adoc | 4 ++++ .../public/reference/_mapping-directfieldmapping.adoc | 2 +- .../backend/lucene/mapping/LuceneVectorFieldIT.java | 4 ++-- .../standalone/realbackend/mapping/VectorFieldIT.java | 8 ++++---- .../codec/impl/HibernateSearchKnnVectorsFormat.java | 4 +--- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java index aa67fc8e851..27e64edfe85 100644 --- a/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java +++ b/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java @@ -19,9 +19,7 @@ public class HibernateSearchKnnVectorsFormat extends KnnVectorsFormat { // OpenSearch has a limit of 16000 - // Elasticsearch has a limit of 4096 - // We'll keep it at 4096 for now as well: - public static final int DEFAULT_MAX_DIMENSIONS = 4096; + public static final int DEFAULT_MAX_DIMENSIONS = 16000; private static final KnnVectorsFormat DEFAULT_KNN_VECTORS_FORMAT = new HibernateSearchKnnVectorsFormat(); public static KnnVectorsFormat defaultFormat() { diff --git a/documentation/src/main/asciidoc/migration/index.adoc b/documentation/src/main/asciidoc/migration/index.adoc index d49be52deb4..e6d5210bacf 100644 --- a/documentation/src/main/asciidoc/migration/index.adoc +++ b/documentation/src/main/asciidoc/migration/index.adoc @@ -77,6 +77,10 @@ in Hibernate Search {hibernateSearchVersion} are backward-compatible with Hibernate Search {hibernateSearchPreviousStableVersionShort}: no database schema update is necessary for these tables. +[[vectorsize]] +=== Vector search max dimension +For the Lucene backend, the vector search maximum dimension has been increased from 4096 to 16000. Increasing the vector size, to leverage a different, higher dimension model, implies a larger memory requirement on behalf of the JVM. + [[configuration]] == Configuration diff --git a/documentation/src/main/asciidoc/public/reference/_mapping-directfieldmapping.adoc b/documentation/src/main/asciidoc/public/reference/_mapping-directfieldmapping.adoc index 2f0120e1e0b..546c67fa310 100644 --- a/documentation/src/main/asciidoc/public/reference/_mapping-directfieldmapping.adoc +++ b/documentation/src/main/asciidoc/public/reference/_mapping-directfieldmapping.adoc @@ -372,7 +372,7 @@ include::../components/_incubating-warning.adoc[] The size of the stored vectors. This is a required field. This size should match the vector size of the vectors produced by the model used to convert the data into vector representation. It is expected to be a positive integer value. Maximum accepted value is backend-specific. -For the <> the dimension must be in `[1, 4096]` range. +For the <> the dimension must be in `[1, 16000]` range. As for the <> the range depends on the distribution. See the link:{elasticsearchDocUrl}/dense-vector.html#dense-vector-params[Elasticsearch]/link:{openSearchDocUrl}/search-plugins/knn/approximate-knn/#get-started-with-approximate-k-nn[OpenSearch] specific documentation to learn about the vector types of these distributions. diff --git a/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/mapping/LuceneVectorFieldIT.java b/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/mapping/LuceneVectorFieldIT.java index ef7c4218742..485b83a920f 100644 --- a/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/mapping/LuceneVectorFieldIT.java +++ b/integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/mapping/LuceneVectorFieldIT.java @@ -20,9 +20,9 @@ class LuceneVectorFieldIT { public final SearchSetupHelper setupHelper = SearchSetupHelper.create(); @ParameterizedTest - @ValueSource(ints = { -1, -1000, 4097, 10000, Integer.MAX_VALUE, Integer.MIN_VALUE }) + @ValueSource(ints = { -1, -1000, 16001, Integer.MAX_VALUE, Integer.MIN_VALUE }) void assertDimension(int dimension) { - test( dimension, 5, 10, "dimension", dimension, 4096 ); + test( dimension, 5, 10, "dimension", dimension, 16000 ); } @ParameterizedTest diff --git a/integrationtest/mapper/pojo-standalone-realbackend/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/standalone/realbackend/mapping/VectorFieldIT.java b/integrationtest/mapper/pojo-standalone-realbackend/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/standalone/realbackend/mapping/VectorFieldIT.java index d7e3bf4066c..d794c9157a8 100644 --- a/integrationtest/mapper/pojo-standalone-realbackend/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/standalone/realbackend/mapping/VectorFieldIT.java +++ b/integrationtest/mapper/pojo-standalone-realbackend/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/standalone/realbackend/mapping/VectorFieldIT.java @@ -52,8 +52,8 @@ class VectorFieldIT { private static final String INDEX_NAME = "IndexName"; - private static final int BATCHES = 20; - private static final int BATCH_SIZE = 1_000; + private static final int BATCHES = 50; + private static final int BATCH_SIZE = 200; @RegisterExtension public StandalonePojoMappingSetupHelper setupHelper = StandalonePojoMappingSetupHelper.withSingleBackend( @@ -78,7 +78,7 @@ static void beforeAll() { @RetryExtension.TestWithRetry void vectorSizeLimits_max_allowed_dimension_with_lots_of_documents() { // with OpenSearch 2.12 it allows up to 16000 which will lead to an OOM in this particular test: - int maxDimension = Math.min( 4096, maxDimension() ); + int maxDimension = Math.min( 16000, maxDimension() ); @Indexed(index = INDEX_NAME) class IndexedEntity { @DocumentId @@ -212,7 +212,7 @@ class IndexedEntity { private static int maxDimension() { if ( BackendConfiguration.isLucene() ) { - return 4096; + return 16000; } else { ElasticsearchVersion actualVersion = ElasticsearchTestDialect.getActualVersion(); diff --git a/lucene-next/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java b/lucene-next/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java index aa67fc8e851..27e64edfe85 100644 --- a/lucene-next/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java +++ b/lucene-next/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java @@ -19,9 +19,7 @@ public class HibernateSearchKnnVectorsFormat extends KnnVectorsFormat { // OpenSearch has a limit of 16000 - // Elasticsearch has a limit of 4096 - // We'll keep it at 4096 for now as well: - public static final int DEFAULT_MAX_DIMENSIONS = 4096; + public static final int DEFAULT_MAX_DIMENSIONS = 16000; private static final KnnVectorsFormat DEFAULT_KNN_VECTORS_FORMAT = new HibernateSearchKnnVectorsFormat(); public static KnnVectorsFormat defaultFormat() {