HSEARCH-5305 - adjust vector search max dimensions for Lucene backend

Signed-off-by: Jan Schatteman <[email protected]>

Author: jrenaat

backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java

@@ -19,9 +19,7 @@
 
 public class HibernateSearchKnnVectorsFormat extends KnnVectorsFormat {
 	// OpenSearch has a limit of 16000
-	// Elasticsearch has a limit of 4096
-	// We'll keep it at 4096 for now as well:
-	public static final int DEFAULT_MAX_DIMENSIONS = 4096;
+	public static final int DEFAULT_MAX_DIMENSIONS = 16000;
 	private static final KnnVectorsFormat DEFAULT_KNN_VECTORS_FORMAT = new HibernateSearchKnnVectorsFormat();
 
 	public static KnnVectorsFormat defaultFormat() {

documentation/src/main/asciidoc/migration/index.adoc

@@ -77,6 +77,10 @@ in Hibernate Search {hibernateSearchVersion}
 are backward-compatible with Hibernate Search {hibernateSearchPreviousStableVersionShort}:
 no database schema update is necessary for these tables.
 
+[[vectorsize]]
+=== Vector search max dimension
+The vector search maximum dimension has been increased from 4096 to 16000. The increased vector size implies a larger memory requirement on behalf of the JVM.
+
 [[configuration]]
 == Configuration
 

documentation/src/main/asciidoc/public/reference/_mapping-directfieldmapping.adoc

@@ -372,7 +372,7 @@ include::../components/_incubating-warning.adoc[]
 The size of the stored vectors. This is a required field. This size should match the vector size of the vectors produced by
 the model used to convert the data into vector representation.
 It is expected to be a positive integer value. Maximum accepted value is backend-specific.
-For the <<backend-lucene, Lucene backend>> the dimension must be in `[1, 4096]` range.
+For the <<backend-lucene, Lucene backend>> the dimension must be in `[1, 16000]` range.
 As for the <<backend-elasticsearch, Elasticsearch backend>> the range depends on the distribution.
 See the link:{elasticsearchDocUrl}/dense-vector.html#dense-vector-params[Elasticsearch]/link:{openSearchDocUrl}/search-plugins/knn/approximate-knn/#get-started-with-approximate-k-nn[OpenSearch]
 specific documentation to learn about the vector types of these distributions.

integrationtest/backend/lucene/src/test/java/org/hibernate/search/integrationtest/backend/lucene/mapping/LuceneVectorFieldIT.java

@@ -20,9 +20,9 @@ class LuceneVectorFieldIT {
 	public final SearchSetupHelper setupHelper = SearchSetupHelper.create();
 
 	@ParameterizedTest
-	@ValueSource(ints = { -1, -1000, 4097, 10000, Integer.MAX_VALUE, Integer.MIN_VALUE })
+	@ValueSource(ints = { -1, -1000, 16001, Integer.MAX_VALUE, Integer.MIN_VALUE })
 	void assertDimension(int dimension) {
-		test( dimension, 5, 10, "dimension", dimension, 4096 );
+		test( dimension, 5, 10, "dimension", dimension, 16000 );
 	}
 
 	@ParameterizedTest

integrationtest/mapper/pojo-standalone-realbackend/src/test/java/org/hibernate/search/integrationtest/mapper/pojo/standalone/realbackend/mapping/VectorFieldIT.java

@@ -52,8 +52,8 @@
 class VectorFieldIT {
 
 	private static final String INDEX_NAME = "IndexName";
-	private static final int BATCHES = 20;
-	private static final int BATCH_SIZE = 1_000;
+	private static final int BATCHES = 100;
+	private static final int BATCH_SIZE = 200;
 
 	@RegisterExtension
 	public StandalonePojoMappingSetupHelper setupHelper = StandalonePojoMappingSetupHelper.withSingleBackend(
@@ -78,7 +78,7 @@ static void beforeAll() {
 	@RetryExtension.TestWithRetry
 	void vectorSizeLimits_max_allowed_dimension_with_lots_of_documents() {
 		// with OpenSearch 2.12 it allows up to 16000 which will lead to an OOM in this particular test:
-		int maxDimension = Math.min( 4096, maxDimension() );
+		int maxDimension = Math.min( 16000, maxDimension() );
 		@Indexed(index = INDEX_NAME)
 		class IndexedEntity {
 			@DocumentId
@@ -212,7 +212,7 @@ class IndexedEntity {
 
 	private static int maxDimension() {
 		if ( BackendConfiguration.isLucene() ) {
-			return 4096;
+			return 16000;
 		}
 		else {
 			ElasticsearchVersion actualVersion = ElasticsearchTestDialect.getActualVersion();

lucene-next/backend/lucene/src/main/java/org/hibernate/search/backend/lucene/lowlevel/codec/impl/HibernateSearchKnnVectorsFormat.java

@@ -19,9 +19,7 @@
 
 public class HibernateSearchKnnVectorsFormat extends KnnVectorsFormat {
 	// OpenSearch has a limit of 16000
-	// Elasticsearch has a limit of 4096
-	// We'll keep it at 4096 for now as well:
-	public static final int DEFAULT_MAX_DIMENSIONS = 4096;
+	public static final int DEFAULT_MAX_DIMENSIONS = 16000;
 	private static final KnnVectorsFormat DEFAULT_KNN_VECTORS_FORMAT = new HibernateSearchKnnVectorsFormat();
 
 	public static KnnVectorsFormat defaultFormat() {