Skip to content

Commit

Permalink
HSEARCH-4950 Fail faster for older dialects not supporting vectors
Browse files Browse the repository at this point in the history
  • Loading branch information
marko-bekhta committed Dec 20, 2023
1 parent f22f4a8 commit a4d1449
Show file tree
Hide file tree
Showing 20 changed files with 222 additions and 28 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@
*/
package org.hibernate.search.backend.elasticsearch.dialect.model.impl;

import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.Elasticsearch7IndexFieldTypeFactoryProvider;
import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.Elasticsearch8IndexFieldTypeFactoryProvider;
import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.ElasticsearchIndexFieldTypeFactoryProvider;
import org.hibernate.search.backend.elasticsearch.validation.impl.Elasticsearch7PropertyMappingValidatorProvider;
import org.hibernate.search.backend.elasticsearch.validation.impl.Elasticsearch8PropertyMappingValidatorProvider;
import org.hibernate.search.backend.elasticsearch.validation.impl.ElasticsearchPropertyMappingValidatorProvider;

import com.google.gson.Gson;
Expand All @@ -20,11 +20,11 @@ public class Elasticsearch8ModelDialect implements ElasticsearchModelDialect {

@Override
public ElasticsearchIndexFieldTypeFactoryProvider createIndexTypeFieldFactoryProvider(Gson userFacingGson) {
return new Elasticsearch7IndexFieldTypeFactoryProvider( userFacingGson );
return new Elasticsearch8IndexFieldTypeFactoryProvider( userFacingGson );
}

@Override
public ElasticsearchPropertyMappingValidatorProvider createElasticsearchPropertyMappingValidatorProvider() {
return new Elasticsearch7PropertyMappingValidatorProvider();
return new Elasticsearch8PropertyMappingValidatorProvider();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@
package org.hibernate.search.backend.elasticsearch.dialect.model.impl;

import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.ElasticsearchIndexFieldTypeFactoryProvider;
import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.OpenSearch1IndexFieldTypeFactoryProvider;
import org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl.OpenSearch2IndexFieldTypeFactoryProvider;
import org.hibernate.search.backend.elasticsearch.validation.impl.ElasticsearchPropertyMappingValidatorProvider;
import org.hibernate.search.backend.elasticsearch.validation.impl.OpenSearch1PropertyMappingValidatorProvider;
import org.hibernate.search.backend.elasticsearch.validation.impl.OpenSearch2PropertyMappingValidatorProvider;

import com.google.gson.Gson;

Expand All @@ -20,11 +20,11 @@ public class OpenSearch2ModelDialect implements ElasticsearchModelDialect {

@Override
public ElasticsearchIndexFieldTypeFactoryProvider createIndexTypeFieldFactoryProvider(Gson userFacingGson) {
return new OpenSearch1IndexFieldTypeFactoryProvider( userFacingGson );
return new OpenSearch2IndexFieldTypeFactoryProvider( userFacingGson );
}

@Override
public ElasticsearchPropertyMappingValidatorProvider createElasticsearchPropertyMappingValidatorProvider() {
return new OpenSearch1PropertyMappingValidatorProvider();
return new OpenSearch2PropertyMappingValidatorProvider();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -874,4 +874,10 @@ SearchException vectorKnnMatchVectorTypeDiffersFromField(String absoluteFieldPat
value = "An OpenSearch distribution does not allow specifying the `number of candidates` option. "
+ "This option is only applicable to an Elasticsearch distribution of an Elasticsearch backend.")
SearchException knnNumberOfCandidatesUnsupportedOption();

@Message(id = ID_OFFSET + 186,
value = "An %1$s distribution version in use is not compatible with the Hibernate Search integration of vector search. "
+ "Update your %1$s cluster to a %2$s series to get vector search integration enabled.")
SearchException searchBackendVersionIncompatibleWithVectorIntegration(String distribution, String version);

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,13 @@
*/
package org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl;

import org.hibernate.search.backend.elasticsearch.types.mapping.impl.Elasticsearch7VectorFieldTypeMappingContributor;
import java.lang.invoke.MethodHandles;

import org.hibernate.search.backend.elasticsearch.logging.impl.Log;
import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping;
import org.hibernate.search.backend.elasticsearch.types.impl.ElasticsearchIndexValueFieldType;
import org.hibernate.search.backend.elasticsearch.types.mapping.impl.ElasticsearchVectorFieldTypeMappingContributor;
import org.hibernate.search.util.common.logging.impl.LoggerFactory;

import com.google.gson.Gson;

Expand All @@ -16,8 +21,21 @@
*/
public class Elasticsearch7IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {

private final Elasticsearch7VectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
new Elasticsearch7VectorFieldTypeMappingContributor();
private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() );

private final ElasticsearchVectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
new ElasticsearchVectorFieldTypeMappingContributor() {

@Override
public void contribute(PropertyMapping mapping, Context context) {
throw log.searchBackendVersionIncompatibleWithVectorIntegration( "Elasticsearch", "8.x" );
}

@Override
public <F> void contribute(ElasticsearchIndexValueFieldType.Builder<F> builder, Context context) {
throw log.searchBackendVersionIncompatibleWithVectorIntegration( "Elasticsearch", "8.x" );
}
};

public Elasticsearch7IndexFieldTypeFactoryProvider(Gson userFacingGson) {
super( userFacingGson );
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl;

import org.hibernate.search.backend.elasticsearch.types.mapping.impl.Elasticsearch8VectorFieldTypeMappingContributor;
import org.hibernate.search.backend.elasticsearch.types.mapping.impl.ElasticsearchVectorFieldTypeMappingContributor;

import com.google.gson.Gson;

/**
* The index field type factory provider for ES8.x.
*/
public class Elasticsearch8IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {

private final Elasticsearch8VectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
new Elasticsearch8VectorFieldTypeMappingContributor();

public Elasticsearch8IndexFieldTypeFactoryProvider(Gson userFacingGson) {
super( userFacingGson );
}

@Override
protected ElasticsearchVectorFieldTypeMappingContributor vectorFieldTypeMappingContributor() {
return vectorFieldTypeMappingContributor;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,36 @@
*/
package org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl;

import java.lang.invoke.MethodHandles;

import org.hibernate.search.backend.elasticsearch.logging.impl.Log;
import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping;
import org.hibernate.search.backend.elasticsearch.types.impl.ElasticsearchIndexValueFieldType;
import org.hibernate.search.backend.elasticsearch.types.mapping.impl.ElasticsearchVectorFieldTypeMappingContributor;
import org.hibernate.search.backend.elasticsearch.types.mapping.impl.OpenSearch1VectorFieldTypeMappingContributor;
import org.hibernate.search.util.common.logging.impl.LoggerFactory;

import com.google.gson.Gson;

/**
* The index field type factory provider for OpenSearch 1.x/2.x.
* The index field type factory provider for OpenSearch 1.x.
*/
public class OpenSearch1IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {

private final OpenSearch1VectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
new OpenSearch1VectorFieldTypeMappingContributor();
private static final Log log = LoggerFactory.make( Log.class, MethodHandles.lookup() );

private final ElasticsearchVectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
new ElasticsearchVectorFieldTypeMappingContributor() {

@Override
public void contribute(PropertyMapping mapping, Context context) {
throw log.searchBackendVersionIncompatibleWithVectorIntegration( "OpenSearch", "2.x" );
}

@Override
public <F> void contribute(ElasticsearchIndexValueFieldType.Builder<F> builder, Context context) {
throw log.searchBackendVersionIncompatibleWithVectorIntegration( "OpenSearch", "2.x" );
}
};

public OpenSearch1IndexFieldTypeFactoryProvider(Gson userFacingGson) {
super( userFacingGson );
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.backend.elasticsearch.types.dsl.provider.impl;

import org.hibernate.search.backend.elasticsearch.types.mapping.impl.ElasticsearchVectorFieldTypeMappingContributor;
import org.hibernate.search.backend.elasticsearch.types.mapping.impl.OpenSearch1VectorFieldTypeMappingContributor;

import com.google.gson.Gson;

/**
* The index field type factory provider for OpenSearch 2.x.
*/
public class OpenSearch2IndexFieldTypeFactoryProvider extends AbstractIndexFieldTypeFactoryProvider {

private final OpenSearch1VectorFieldTypeMappingContributor vectorFieldTypeMappingContributor =
new OpenSearch1VectorFieldTypeMappingContributor();

public OpenSearch2IndexFieldTypeFactoryProvider(Gson userFacingGson) {
super( userFacingGson );
}

@Override
protected ElasticsearchVectorFieldTypeMappingContributor vectorFieldTypeMappingContributor() {
return vectorFieldTypeMappingContributor;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

import com.google.gson.JsonObject;

public class Elasticsearch7VectorFieldTypeMappingContributor implements ElasticsearchVectorFieldTypeMappingContributor {
public class Elasticsearch8VectorFieldTypeMappingContributor implements ElasticsearchVectorFieldTypeMappingContributor {
@Override
public void contribute(PropertyMapping mapping, Context context) {
mapping.setType( DataTypes.DENSE_VECTOR );
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@
public class Elasticsearch7PropertyMappingValidatorProvider implements ElasticsearchPropertyMappingValidatorProvider {
@Override
public Validator<PropertyMapping> create() {
return new PropertyMappingValidator.ElasticsearchPropertyMappingValidator();
return new PropertyMappingValidator.Elasticsearch7PropertyMappingValidator();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.backend.elasticsearch.validation.impl;

import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping;

public class Elasticsearch8PropertyMappingValidatorProvider implements ElasticsearchPropertyMappingValidatorProvider {
@Override
public Validator<PropertyMapping> create() {
return new PropertyMappingValidator.Elasticsearch8PropertyMappingValidator();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,6 @@
public class OpenSearch1PropertyMappingValidatorProvider implements ElasticsearchPropertyMappingValidatorProvider {
@Override
public Validator<PropertyMapping> create() {
return new PropertyMappingValidator.ElasticsearchPropertyMappingValidator();
return new PropertyMappingValidator.OpenSearch1PropertyMappingValidator();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
/*
* Hibernate Search, full-text search for your domain model
*
* License: GNU Lesser General Public License (LGPL), version 2.1 or later
* See the lgpl.txt file in the root directory or <http://www.gnu.org/licenses/lgpl-2.1.html>.
*/
package org.hibernate.search.backend.elasticsearch.validation.impl;

import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping;

public class OpenSearch2PropertyMappingValidatorProvider implements ElasticsearchPropertyMappingValidatorProvider {
@Override
public Validator<PropertyMapping> create() {
return new PropertyMappingValidator.OpenSearch2PropertyMappingValidator();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,16 @@ private void validateIndexOptions(ValidationErrorCollector errorCollector, Prope
protected abstract void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping,
PropertyMapping actualMapping);

static class ElasticsearchPropertyMappingValidator extends PropertyMappingValidator {
static class Elasticsearch7PropertyMappingValidator extends PropertyMappingValidator {

@Override
protected void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping,
PropertyMapping actualMapping) {

}
}

static class Elasticsearch8PropertyMappingValidator extends PropertyMappingValidator {

@Override
protected void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping,
Expand Down Expand Up @@ -164,7 +173,16 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr
}
}

static class OpenSearchPropertyMappingValidator extends PropertyMappingValidator {
static class OpenSearch1PropertyMappingValidator extends PropertyMappingValidator {

@Override
protected void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping,
PropertyMapping actualMapping) {

}
}

static class OpenSearch2PropertyMappingValidator extends PropertyMappingValidator {

@Override
protected void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@
import org.hibernate.search.engine.search.predicate.dsl.TermsPredicateFieldStep;
import org.hibernate.search.engine.search.predicate.dsl.WildcardPredicateFieldStep;
import org.hibernate.search.engine.search.predicate.dsl.impl.BooleanPredicateClausesStepImpl;
import org.hibernate.search.engine.search.predicate.dsl.impl.ExistsPredicateFieldStepImpl;
import org.hibernate.search.engine.search.predicate.dsl.impl.DefaultKnnPredicateFieldStep;
import org.hibernate.search.engine.search.predicate.dsl.impl.ExistsPredicateFieldStepImpl;
import org.hibernate.search.engine.search.predicate.dsl.impl.MatchAllPredicateOptionsStepImpl;
import org.hibernate.search.engine.search.predicate.dsl.impl.MatchIdPredicateMatchingStepImpl;
import org.hibernate.search.engine.search.predicate.dsl.impl.MatchNonePredicateFinalStepImpl;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import static org.hibernate.search.integrationtest.backend.elasticsearch.schema.management.ElasticsearchIndexSchemaManagerTestUtils.defaultMetadataMappingAndCommaForInitialization;
import static org.hibernate.search.integrationtest.backend.elasticsearch.schema.management.ElasticsearchIndexSchemaManagerTestUtils.hasValidationFailureReport;
import static org.hibernate.search.integrationtest.backend.elasticsearch.schema.management.ElasticsearchIndexSchemaManagerTestUtils.simpleMappingForInitialization;
import static org.junit.jupiter.api.Assumptions.assumeTrue;

import java.util.List;
import java.util.Optional;
Expand All @@ -22,6 +23,7 @@
import org.hibernate.search.backend.elasticsearch.cfg.ElasticsearchIndexSettings;
import org.hibernate.search.engine.backend.document.model.dsl.IndexSchemaObjectField;
import org.hibernate.search.engine.backend.types.VectorSimilarity;
import org.hibernate.search.integrationtest.backend.tck.testsupport.util.TckConfiguration;
import org.hibernate.search.integrationtest.backend.tck.testsupport.util.extension.SearchSetupHelper;
import org.hibernate.search.util.common.SearchException;
import org.hibernate.search.util.common.impl.Futures;
Expand Down Expand Up @@ -380,6 +382,10 @@ void multipleErrors(ElasticsearchIndexSchemaManagerValidationOperation operation
@ParameterizedTest(name = "With operation {0}")
@MethodSource("params")
void vector_success_1(ElasticsearchIndexSchemaManagerValidationOperation operation) {
assumeTrue(
TckConfiguration.get().getBackendFeatures().supportsVectorSearch(),
"These tests only make sense for a backend where Vector Search is supported and implemented."
);
StubMappedIndex index = StubMappedIndex.ofNonRetrievable( root -> {
root.field( "vector", f -> f.asByteVector( 100 ) )
.toReference();
Expand All @@ -403,6 +409,10 @@ void vector_success_1(ElasticsearchIndexSchemaManagerValidationOperation operati
@ParameterizedTest(name = "With operation {0}")
@MethodSource("params")
void vector_success_2(ElasticsearchIndexSchemaManagerValidationOperation operation) {
assumeTrue(
TckConfiguration.get().getBackendFeatures().supportsVectorSearch(),
"These tests only make sense for a backend where Vector Search is supported and implemented."
);
StubMappedIndex index = StubMappedIndex.ofNonRetrievable( root -> {
root.field( "vector",
f -> f.asFloatVector( 50 ).vectorSimilarity( VectorSimilarity.L2 ).maxConnections( 2 ).beamWidth( 10 ) )
Expand All @@ -428,6 +438,10 @@ void vector_success_2(ElasticsearchIndexSchemaManagerValidationOperation operati
@ParameterizedTest(name = "With operation {0}")
@MethodSource("params")
void vector_invalid_dimension(ElasticsearchIndexSchemaManagerValidationOperation operation) {
assumeTrue(
TckConfiguration.get().getBackendFeatures().supportsVectorSearch(),
"These tests only make sense for a backend where Vector Search is supported and implemented."
);
StubMappedIndex index = StubMappedIndex.ofNonRetrievable( root -> {
root.field( "vectorB", f -> f.asByteVector( 2 ) ).toReference();
root.field( "vectorF", f -> f.asFloatVector( 2 ) ).toReference();
Expand Down Expand Up @@ -462,6 +476,10 @@ void vector_invalid_dimension(ElasticsearchIndexSchemaManagerValidationOperation
@ParameterizedTest(name = "With operation {0}")
@MethodSource("params")
void vector_invalid_elementType(ElasticsearchIndexSchemaManagerValidationOperation operation) {
assumeTrue(
TckConfiguration.get().getBackendFeatures().supportsVectorSearch(),
"These tests only make sense for a backend where Vector Search is supported and implemented."
);
StubMappedIndex index = StubMappedIndex.ofNonRetrievable( root -> {
root.field( "vectorB", f -> f.asByteVector( 2 ) ).toReference();
root.field( "vectorF", f -> f.asFloatVector( 2 ) ).toReference();
Expand Down Expand Up @@ -496,6 +514,10 @@ void vector_invalid_elementType(ElasticsearchIndexSchemaManagerValidationOperati
@ParameterizedTest(name = "With operation {0}")
@MethodSource("params")
void vector_invalid_similarity(ElasticsearchIndexSchemaManagerValidationOperation operation) {
assumeTrue(
TckConfiguration.get().getBackendFeatures().supportsVectorSearch(),
"These tests only make sense for a backend where Vector Search is supported and implemented."
);
StubMappedIndex index = StubMappedIndex.ofNonRetrievable( root -> {
root.field( "vectorB", f -> f.asByteVector( 2 ).vectorSimilarity( VectorSimilarity.COSINE ) ).toReference();
root.field( "vectorF", f -> f.asFloatVector( 2 ).vectorSimilarity( VectorSimilarity.COSINE ) ).toReference();
Expand Down Expand Up @@ -532,6 +554,10 @@ void vector_invalid_similarity(ElasticsearchIndexSchemaManagerValidationOperatio
@ParameterizedTest(name = "With operation {0}")
@MethodSource("params")
void vector_invalid_m_ef(ElasticsearchIndexSchemaManagerValidationOperation operation) {
assumeTrue(
TckConfiguration.get().getBackendFeatures().supportsVectorSearch(),
"These tests only make sense for a backend where Vector Search is supported and implemented."
);
StubMappedIndex index = StubMappedIndex.ofNonRetrievable( root -> {
root.field( "vectorB", f -> f.asByteVector( 2 ).beamWidth( 2 ).maxConnections( 20 ) ).toReference();
root.field( "vectorF", f -> f.asFloatVector( 2 ).beamWidth( 5 ).maxConnections( 50 ) ).toReference();
Expand Down
Loading

0 comments on commit a4d1449

Please sign in to comment.