diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClasses.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClasses.java index 723ae9bd572..23a38540211 100644 --- a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClasses.java +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClasses.java @@ -54,7 +54,13 @@ public static Set typesRequiringReflection() { "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMappingJsonAdapterFactory", "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.RootTypeMappingJsonAdapterFactory", "org.hibernate.search.backend.elasticsearch.lowlevel.index.settings.impl.AnalysisJsonAdapterFactory", - "org.hibernate.search.backend.elasticsearch.lowlevel.index.settings.impl.IndexSettingsJsonAdapterFactory" + "org.hibernate.search.backend.elasticsearch.lowlevel.index.settings.impl.IndexSettingsJsonAdapterFactory", + "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.ElasticsearchDenseVectorIndexOptions", + "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.OpenSearchVectorTypeMethod", + "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.OpenSearchVectorTypeMethod$Parameters", + "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.ElasticsearchDenseVectorIndexOptionsJsonAdapterFactory", + "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.OpenSearchVectorTypeMethodJsonAdapterFactory", + "org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.OpenSearchVectorTypeMethodJsonAdapterFactory$ParametersJsonAdapterFactory" ) ); } diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/ElasticsearchDenseVectorIndexOptions.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/ElasticsearchDenseVectorIndexOptions.java new file mode 100644 index 00000000000..d748983183e --- /dev/null +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/ElasticsearchDenseVectorIndexOptions.java @@ -0,0 +1,73 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl; + +import java.util.Map; + +import org.hibernate.search.backend.elasticsearch.gson.impl.SerializeExtraProperties; + +import com.google.gson.JsonElement; +import com.google.gson.annotations.JsonAdapter; +import com.google.gson.annotations.SerializedName; + +/** + * An object representing Elasticsearch dense vector-specific index options attributes. + * + * See https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html + */ +/* + * CAUTION: + * 1. JSON serialization is controlled by a specific adapter, which must be + * updated whenever fields of this class are added, renamed or removed. + * + * 2. Whenever adding more properties consider adding property validation to PropertyMappingValidator#ElasticsearchDenseVectorIndexOptionsValidator. + */ +@JsonAdapter(ElasticsearchDenseVectorIndexOptionsJsonAdapterFactory.class) +public class ElasticsearchDenseVectorIndexOptions { + + private String type; + + private Integer m; + + @SerializedName("ef_construction") + private Integer efConstruction; + + @SerializeExtraProperties + private Map extraAttributes; + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public Integer getM() { + return m; + } + + public void setM(Integer m) { + this.m = m; + } + + public Integer getEfConstruction() { + return efConstruction; + } + + public void setEfConstruction(Integer efConstruction) { + this.efConstruction = efConstruction; + } + + public Map getExtraAttributes() { + return extraAttributes; + } + + public void setExtraAttributes(Map extraAttributes) { + this.extraAttributes = extraAttributes; + } +} diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/ElasticsearchDenseVectorIndexOptionsJsonAdapterFactory.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/ElasticsearchDenseVectorIndexOptionsJsonAdapterFactory.java new file mode 100644 index 00000000000..dcfa9827f31 --- /dev/null +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/ElasticsearchDenseVectorIndexOptionsJsonAdapterFactory.java @@ -0,0 +1,21 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl; + +import org.hibernate.search.backend.elasticsearch.gson.impl.AbstractConfiguredExtraPropertiesJsonAdapterFactory; + +public class ElasticsearchDenseVectorIndexOptionsJsonAdapterFactory + extends + AbstractConfiguredExtraPropertiesJsonAdapterFactory { + + @Override + protected void addFields(Builder builder) { + builder.add( "type", String.class ); + builder.add( "m", Integer.class ); + builder.add( "efConstruction", Integer.class ); + } +} diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/OpenSearchVectorTypeMethod.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/OpenSearchVectorTypeMethod.java new file mode 100644 index 00000000000..86773233b7f --- /dev/null +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/OpenSearchVectorTypeMethod.java @@ -0,0 +1,118 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl; + +import java.util.Map; + +import org.hibernate.search.backend.elasticsearch.gson.impl.SerializeExtraProperties; + +import com.google.gson.JsonElement; +import com.google.gson.annotations.JsonAdapter; +import com.google.gson.annotations.SerializedName; + +/** + * An object representing OpenSearch K-NN vector Method attributes. + * + * See https://opensearch.org/docs/latest/field-types/supported-field-types/knn-vector/ + */ +/* + * CAUTION: + * 1. JSON serialization is controlled by a specific adapter, which must be + * updated whenever fields of this class are added, renamed or removed. + * + * 2. Whenever adding more properties consider adding property validation to PropertyMappingValidator. + */ +@JsonAdapter(OpenSearchVectorTypeMethodJsonAdapterFactory.class) +public class OpenSearchVectorTypeMethod { + + private String name; + + @SerializedName("space_type") + private String spaceType; + + private String engine; + + private Parameters parameters; + + @SerializeExtraProperties + private Map extraAttributes; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getSpaceType() { + return spaceType; + } + + public void setSpaceType(String spaceType) { + this.spaceType = spaceType; + } + + public String getEngine() { + return engine; + } + + public void setEngine(String engine) { + this.engine = engine; + } + + public Parameters getParameters() { + return parameters; + } + + public void setParameters(Parameters parameters) { + this.parameters = parameters; + } + + public Map getExtraAttributes() { + return extraAttributes; + } + + public void setExtraAttributes(Map extraAttributes) { + this.extraAttributes = extraAttributes; + } + + @JsonAdapter(OpenSearchVectorTypeMethodJsonAdapterFactory.ParametersJsonAdapterFactory.class) + public static class Parameters { + + @SerializedName("ef_construction") + private Integer efConstruction; + private Integer m; + + @SerializeExtraProperties + private Map extraAttributes; + + public Integer getEfConstruction() { + return efConstruction; + } + + public void setEfConstruction(Integer efConstruction) { + this.efConstruction = efConstruction; + } + + public Integer getM() { + return m; + } + + public void setM(Integer m) { + this.m = m; + } + + public Map getExtraAttributes() { + return extraAttributes; + } + + public void setExtraAttributes(Map extraAttributes) { + this.extraAttributes = extraAttributes; + } + } +} diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/OpenSearchVectorTypeMethodJsonAdapterFactory.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/OpenSearchVectorTypeMethodJsonAdapterFactory.java new file mode 100644 index 00000000000..e7133480a14 --- /dev/null +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/OpenSearchVectorTypeMethodJsonAdapterFactory.java @@ -0,0 +1,32 @@ +/* + * Hibernate Search, full-text search for your domain model + * + * License: GNU Lesser General Public License (LGPL), version 2.1 or later + * See the lgpl.txt file in the root directory or . + */ +package org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl; + +import org.hibernate.search.backend.elasticsearch.gson.impl.AbstractConfiguredExtraPropertiesJsonAdapterFactory; + +public class OpenSearchVectorTypeMethodJsonAdapterFactory + extends + AbstractConfiguredExtraPropertiesJsonAdapterFactory { + + @Override + protected void addFields(Builder builder) { + builder.add( "name", String.class ); + builder.add( "spaceType", String.class ); + builder.add( "engine", String.class ); + builder.add( "parameters", OpenSearchVectorTypeMethod.Parameters.class ); + } + + public static class ParametersJsonAdapterFactory + extends + AbstractConfiguredExtraPropertiesJsonAdapterFactory { + @Override + protected void addFields(Builder builder) { + builder.add( "m", Integer.class ); + builder.add( "efConstruction", Integer.class ); + } + } +} diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMapping.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMapping.java index 3552c97ab49..eee51476e64 100644 --- a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMapping.java +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMapping.java @@ -113,7 +113,7 @@ public class PropertyMapping extends AbstractTypeMapping { * https://www.elastic.co/guide/en/elasticsearch/reference/current/dense-vector.html */ @SerializedName("index_options") - private JsonElement indexOptions; + private ElasticsearchDenseVectorIndexOptions indexOptions; /* * k-NN vector datatype @@ -125,7 +125,7 @@ public class PropertyMapping extends AbstractTypeMapping { * k-NN vector datatype * https://opensearch.org/docs/latest/field-types/supported-field-types/knn-vector/ */ - private JsonElement method; + private OpenSearchVectorTypeMethod method; /* * k-NN vector datatype @@ -247,11 +247,11 @@ public void setSimilarity(String similarity) { this.similarity = similarity; } - public JsonElement getIndexOptions() { + public ElasticsearchDenseVectorIndexOptions getIndexOptions() { return indexOptions; } - public void setIndexOptions(JsonElement indexOptions) { + public void setIndexOptions(ElasticsearchDenseVectorIndexOptions indexOptions) { this.indexOptions = indexOptions; } @@ -263,11 +263,11 @@ public void setDimension(Integer dimension) { this.dimension = dimension; } - public JsonElement getMethod() { + public OpenSearchVectorTypeMethod getMethod() { return method; } - public void setMethod(JsonElement method) { + public void setMethod(OpenSearchVectorTypeMethod method) { this.method = method; } diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMappingJsonAdapterFactory.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMappingJsonAdapterFactory.java index 9868a80fa40..9c3db3bfee9 100644 --- a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMappingJsonAdapterFactory.java +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/lowlevel/index/mapping/impl/PropertyMappingJsonAdapterFactory.java @@ -27,9 +27,9 @@ protected void addFields(Builder builder) { builder.add( "elementType", String.class ); builder.add( "dims", Integer.class ); builder.add( "similarity", String.class ); - builder.add( "indexOptions", JsonElement.class ); + builder.add( "indexOptions", ElasticsearchDenseVectorIndexOptions.class ); builder.add( "dimension", Integer.class ); - builder.add( "method", JsonElement.class ); + builder.add( "method", OpenSearchVectorTypeMethod.class ); builder.add( "dataType", String.class ); } } diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch8VectorFieldTypeMappingContributor.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch8VectorFieldTypeMappingContributor.java index e6bdd96f654..55e406bb522 100644 --- a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch8VectorFieldTypeMappingContributor.java +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/Elasticsearch8VectorFieldTypeMappingContributor.java @@ -7,6 +7,7 @@ package org.hibernate.search.backend.elasticsearch.types.mapping.impl; import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.DataTypes; +import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.ElasticsearchDenseVectorIndexOptions; import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping; import org.hibernate.search.backend.elasticsearch.search.predicate.impl.ElasticsearchKnnPredicate; import org.hibernate.search.backend.elasticsearch.types.impl.ElasticsearchIndexValueFieldType; @@ -14,8 +15,6 @@ import org.hibernate.search.engine.search.predicate.spi.PredicateTypeKeys; import org.hibernate.search.util.common.AssertionFailure; -import com.google.gson.JsonObject; - public class Elasticsearch8VectorFieldTypeMappingContributor implements ElasticsearchVectorFieldTypeMappingContributor { @Override public void contribute(PropertyMapping mapping, Context context) { @@ -27,13 +26,13 @@ public void contribute(PropertyMapping mapping, Context context) { mapping.setSimilarity( resolvedVectorSimilarity ); } if ( context.maxConnections() != null || context.beamWidth() != null ) { - JsonObject indexOptions = new JsonObject(); - indexOptions.addProperty( "type", "hnsw" ); + ElasticsearchDenseVectorIndexOptions indexOptions = new ElasticsearchDenseVectorIndexOptions(); + indexOptions.setType( "hnsw" ); if ( context.maxConnections() != null ) { - indexOptions.addProperty( "m", context.maxConnections() ); + indexOptions.setM( context.maxConnections() ); } if ( context.beamWidth() != null ) { - indexOptions.addProperty( "ef_construction", context.beamWidth() ); + indexOptions.setEfConstruction( context.beamWidth() ); } mapping.setIndexOptions( indexOptions ); } diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/OpenSearch2VectorFieldTypeMappingContributor.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/OpenSearch2VectorFieldTypeMappingContributor.java index 75ec2d3327f..417b549078d 100644 --- a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/OpenSearch2VectorFieldTypeMappingContributor.java +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/types/mapping/impl/OpenSearch2VectorFieldTypeMappingContributor.java @@ -7,6 +7,7 @@ package org.hibernate.search.backend.elasticsearch.types.mapping.impl; import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.DataTypes; +import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.OpenSearchVectorTypeMethod; import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping; import org.hibernate.search.backend.elasticsearch.search.predicate.impl.ElasticsearchKnnPredicate; import org.hibernate.search.backend.elasticsearch.types.impl.ElasticsearchIndexValueFieldType; @@ -14,8 +15,6 @@ import org.hibernate.search.engine.search.predicate.spi.PredicateTypeKeys; import org.hibernate.search.util.common.AssertionFailure; -import com.google.gson.JsonObject; - public class OpenSearch2VectorFieldTypeMappingContributor implements ElasticsearchVectorFieldTypeMappingContributor { private static final String BYTE_TYPE = "BYTE"; @@ -33,21 +32,21 @@ public void contribute(PropertyMapping mapping, Context context) { String resolvedVectorSimilarity = resolveDefault( context.vectorSimilarity() ); // we want to always set Lucene as an engine: - JsonObject method = new JsonObject(); - method.addProperty( "name", "hnsw" ); - method.addProperty( "engine", "lucene" ); + OpenSearchVectorTypeMethod method = new OpenSearchVectorTypeMethod(); + method.setName( "hnsw" ); + method.setEngine( "lucene" ); if ( resolvedVectorSimilarity != null ) { - method.addProperty( "space_type", resolvedVectorSimilarity ); + method.setSpaceType( resolvedVectorSimilarity ); } if ( context.maxConnections() != null || context.beamWidth() != null ) { - JsonObject parameters = new JsonObject(); + OpenSearchVectorTypeMethod.Parameters parameters = new OpenSearchVectorTypeMethod.Parameters(); if ( context.maxConnections() != null ) { - parameters.addProperty( "m", context.maxConnections() ); + parameters.setM( context.maxConnections() ); } if ( context.beamWidth() != null ) { - parameters.addProperty( "ef_construction", context.beamWidth() ); + parameters.setEfConstruction( context.beamWidth() ); } - method.add( "parameters", parameters ); + method.setParameters( parameters ); } mapping.setMethod( method ); diff --git a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/validation/impl/PropertyMappingValidator.java b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/validation/impl/PropertyMappingValidator.java index 4e4fe9bd47a..036231e37b7 100644 --- a/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/validation/impl/PropertyMappingValidator.java +++ b/backend/elasticsearch/src/main/java/org/hibernate/search/backend/elasticsearch/validation/impl/PropertyMappingValidator.java @@ -9,14 +9,17 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.DataTypes; +import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.ElasticsearchDenseVectorIndexOptions; +import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.OpenSearchVectorTypeMethod; import org.hibernate.search.backend.elasticsearch.lowlevel.index.mapping.impl.PropertyMapping; +import org.hibernate.search.backend.elasticsearch.reporting.impl.ElasticsearchValidationMessages; import org.hibernate.search.engine.backend.analysis.AnalyzerNames; import org.hibernate.search.util.common.impl.CollectionHelper; import com.google.gson.JsonElement; -import com.google.gson.JsonObject; abstract class PropertyMappingValidator extends AbstractTypeMappingValidator { @@ -135,6 +138,9 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr static class Elasticsearch8PropertyMappingValidator extends PropertyMappingValidator { + private final ElasticsearchDenseVectorIndexOptionsValidator indexOptionsValidator = + new ElasticsearchDenseVectorIndexOptionsValidator(); + @Override protected void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping, PropertyMapping actualMapping) { @@ -153,22 +159,9 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr expectedMapping.getSimilarity(), actualMapping.getSimilarity(), "cosine" ); - JsonElement expectedIndexOptionsElement = expectedMapping.getIndexOptions(); - if ( expectedIndexOptionsElement != null && expectedIndexOptionsElement.isJsonObject() ) { - JsonObject expectedIndexOptions = expectedIndexOptionsElement.getAsJsonObject(); - JsonObject actualIndexOptions = actualMapping.getIndexOptions().getAsJsonObject(); - LeafValidators.EQUAL.validate( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "index_options.type", - expectedIndexOptions.get( "type" ), actualIndexOptions.get( "type" ) - ); - LeafValidators.EQUAL.validateWithDefault( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "index_options.m", - expectedIndexOptions.get( "m" ), actualIndexOptions.get( "m" ), 16 - ); - LeafValidators.EQUAL.validateWithDefault( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "index_options.ef_construction", - expectedIndexOptions.get( "ef_construction" ), actualIndexOptions.get( "ef_construction" ), 100 - ); + ElasticsearchDenseVectorIndexOptions indexOptions = expectedMapping.getIndexOptions(); + if ( indexOptions != null ) { + indexOptionsValidator.validate( errorCollector, indexOptions, actualMapping.getIndexOptions() ); } } } @@ -184,6 +177,8 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr static class OpenSearch2PropertyMappingValidator extends PropertyMappingValidator { + private final OpenSearchVectorTypeMethodValidator methodValidator = new OpenSearchVectorTypeMethodValidator(); + @Override protected void validateVectorMapping(ValidationErrorCollector errorCollector, PropertyMapping expectedMapping, PropertyMapping actualMapping) { @@ -197,54 +192,155 @@ protected void validateVectorMapping(ValidationErrorCollector errorCollector, Pr expectedMapping.getDataType(), actualMapping.getDataType() ); - JsonElement methodElement = expectedMapping.getMethod(); - if ( methodElement != null && methodElement.isJsonObject() ) { - JsonObject expectedMethod = methodElement.getAsJsonObject(); - JsonObject actualMethod = actualMapping.getMethod().getAsJsonObject(); + OpenSearchVectorTypeMethod expectedMethod = expectedMapping.getMethod(); + if ( expectedMethod != null ) { + methodValidator.validate( errorCollector, expectedMethod, actualMapping.getMethod() ); + } + } + } - LeafValidators.EQUAL.validate( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "method.name", - getAsString( expectedMethod, "name" ), getAsString( actualMethod, "name" ) - ); + private static class ElasticsearchDenseVectorIndexOptionsValidator + extends AbstractVectorAttributesValidator { - LeafValidators.EQUAL.validateWithDefault( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "method.space_type", - getAsString( expectedMethod, "space_type" ), getAsString( actualMethod, "space_type" ), - "l2" - ); + @Override + protected String propertyName() { + return "index_options"; + } - LeafValidators.EQUAL.validate( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "method.engine", - getAsString( expectedMethod, "engine" ), getAsString( actualMethod, "engine" ) - ); + @Override + public void doValidate(ValidationErrorCollector errorCollector, ElasticsearchDenseVectorIndexOptions expected, + ElasticsearchDenseVectorIndexOptions actual) { + LeafValidators.EQUAL.validate( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "type", + expected.getType(), actual.getType() + ); + LeafValidators.EQUAL.validateWithDefault( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "m", + expected.getM(), actual.getM(), 16 + ); + LeafValidators.EQUAL.validateWithDefault( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "ef_construction", + expected.getEfConstruction(), actual.getEfConstruction(), 100 + ); + } + + @Override + protected Map expectedMappingExtraAttributes(ElasticsearchDenseVectorIndexOptions expected) { + return expected.getExtraAttributes(); + } + + @Override + protected Map actualMappingExtraAttributes(ElasticsearchDenseVectorIndexOptions actual) { + return actual.getExtraAttributes(); + } + } + + private static class OpenSearchVectorTypeMethodValidator + extends AbstractVectorAttributesValidator { - JsonElement parametersElement = expectedMethod.get( "parameters" ); - if ( parametersElement != null && parametersElement.isJsonObject() ) { - JsonObject expectedParameters = parametersElement.getAsJsonObject(); - JsonObject actualParameters = actualMethod.get( "parameters" ).getAsJsonObject(); - - LeafValidators.EQUAL.validate( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "method.parameters.m", - getAsInteger( expectedParameters, "m" ), getAsInteger( actualParameters, "m" ) - ); - - LeafValidators.EQUAL.validate( - errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "method.parameters.ef_construction", - getAsInteger( expectedParameters, "ef_construction" ), - getAsInteger( actualParameters, "ef_construction" ) - ); - } + private final OpenSearchVectorTypeMethodParametersValidator parametersValidator = + new OpenSearchVectorTypeMethodParametersValidator(); + + @Override + protected String propertyName() { + return "method"; + } + + @Override + public void doValidate(ValidationErrorCollector errorCollector, OpenSearchVectorTypeMethod expected, + OpenSearchVectorTypeMethod actual) { + LeafValidators.EQUAL.validate( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "name", + expected.getName(), actual.getName() + ); + + LeafValidators.EQUAL.validateWithDefault( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "space_type", + expected.getSpaceType(), actual.getSpaceType(), + "l2" + ); + + LeafValidators.EQUAL.validate( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "engine", + expected.getEngine(), actual.getEngine() + ); + + OpenSearchVectorTypeMethod.Parameters expectedParameters = expected.getParameters(); + if ( expectedParameters != null ) { + parametersValidator.validate( errorCollector, expectedParameters, actual.getParameters() ); } } - private static String getAsString(JsonObject object, String property) { - JsonElement element = object.get( property ); - return element == null || element.isJsonNull() ? null : element.getAsString(); + @Override + protected Map expectedMappingExtraAttributes(OpenSearchVectorTypeMethod expected) { + return expected.getExtraAttributes(); } - private static Integer getAsInteger(JsonObject object, String property) { - JsonElement element = object.get( property ); - return element == null || element.isJsonNull() ? null : element.getAsInt(); + @Override + protected Map actualMappingExtraAttributes(OpenSearchVectorTypeMethod actual) { + return actual.getExtraAttributes(); + } + } + + private static class OpenSearchVectorTypeMethodParametersValidator + extends AbstractVectorAttributesValidator { + @Override + protected String propertyName() { + return "parameters"; + } + + @Override + public void doValidate(ValidationErrorCollector errorCollector, OpenSearchVectorTypeMethod.Parameters expected, + OpenSearchVectorTypeMethod.Parameters actual) { + LeafValidators.EQUAL.validate( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "m", + expected.getM(), actual.getM() + ); + + LeafValidators.EQUAL.validate( + errorCollector, ValidationContextType.MAPPING_ATTRIBUTE, "ef_construction", + expected.getEfConstruction(), + actual.getEfConstruction() + ); } + + @Override + protected Map expectedMappingExtraAttributes(OpenSearchVectorTypeMethod.Parameters expected) { + return expected.getExtraAttributes(); + } + + @Override + protected Map actualMappingExtraAttributes(OpenSearchVectorTypeMethod.Parameters actual) { + return actual.getExtraAttributes(); + } + } + + abstract static class AbstractVectorAttributesValidator implements Validator { + private final Validator extraAttributeValidator = new JsonElementValidator( new JsonElementEquivalence() ); + + @Override + public final void validate(ValidationErrorCollector errorCollector, T expected, T actual) { + errorCollector.push( ValidationContextType.MAPPING_ATTRIBUTE, propertyName() ); + try { + doValidate( errorCollector, expected, actual ); + + extraAttributeValidator.validateAllIgnoreUnexpected( + errorCollector, ValidationContextType.CUSTOM_INDEX_MAPPING_ATTRIBUTE, + ElasticsearchValidationMessages.INSTANCE.customIndexMappingAttributeMissing(), + expectedMappingExtraAttributes( expected ), actualMappingExtraAttributes( actual ) + ); + } + finally { + errorCollector.pop(); + } + } + + protected abstract String propertyName(); + + protected abstract void doValidate(ValidationErrorCollector errorCollector, T expected, T actual); + + protected abstract Map expectedMappingExtraAttributes(T expected); + + protected abstract Map actualMappingExtraAttributes(T actual); } } diff --git a/backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClassesTest.java b/backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClassesTest.java index 99e26b811cd..6fd93382a13 100644 --- a/backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClassesTest.java +++ b/backend/elasticsearch/src/test/java/org/hibernate/search/backend/elasticsearch/gson/spi/GsonClassesTest.java @@ -80,6 +80,9 @@ void testNoMissingGsonContractImplementations() { for ( ClassInfo implementor : backendElasticsearchIndex.getAllKnownSubclasses( gsonContract ) ) { classes.add( implementor.name() ); } + for ( ClassInfo implementor : backendElasticsearchIndex.getAllKnownImplementors( gsonContract ) ) { + classes.add( implementor.name() ); + } } Set classesAndSubclasses = JandexTestUtils.toStrings( diff --git a/integrationtest/backend/elasticsearch/src/test/java/org/hibernate/search/integrationtest/backend/elasticsearch/schema/management/ElasticsearchIndexSchemaManagerValidationMappingBaseIT.java b/integrationtest/backend/elasticsearch/src/test/java/org/hibernate/search/integrationtest/backend/elasticsearch/schema/management/ElasticsearchIndexSchemaManagerValidationMappingBaseIT.java index a1386e5e34d..b6774ff712a 100644 --- a/integrationtest/backend/elasticsearch/src/test/java/org/hibernate/search/integrationtest/backend/elasticsearch/schema/management/ElasticsearchIndexSchemaManagerValidationMappingBaseIT.java +++ b/integrationtest/backend/elasticsearch/src/test/java/org/hibernate/search/integrationtest/backend/elasticsearch/schema/management/ElasticsearchIndexSchemaManagerValidationMappingBaseIT.java @@ -17,6 +17,7 @@ import java.util.OptionalInt; import java.util.stream.Collectors; +import org.hibernate.search.backend.elasticsearch.ElasticsearchDistributionName; import org.hibernate.search.backend.elasticsearch.ElasticsearchExtension; import org.hibernate.search.backend.elasticsearch.analysis.ElasticsearchAnalysisConfigurationContext; import org.hibernate.search.backend.elasticsearch.analysis.ElasticsearchAnalysisConfigurer; @@ -27,6 +28,7 @@ import org.hibernate.search.integrationtest.backend.tck.testsupport.util.extension.SearchSetupHelper; import org.hibernate.search.util.common.SearchException; import org.hibernate.search.util.common.impl.Futures; +import org.hibernate.search.util.impl.integrationtest.backend.elasticsearch.dialect.ElasticsearchTestDialect; import org.hibernate.search.util.impl.integrationtest.backend.elasticsearch.extension.TestElasticsearchClient; import org.hibernate.search.util.impl.integrationtest.common.reporting.FailureReportChecker; import org.hibernate.search.util.impl.integrationtest.mapper.stub.StubMappedIndex; @@ -545,16 +547,30 @@ void vector_invalid_similarity(ElasticsearchIndexSchemaManagerValidationOperatio ) ); - setupAndValidateExpectingFailure( index, - hasValidationFailureReport() - .indexFieldContext( "vectorB" ) - .mappingAttributeContext( elasticSearchClient.getDialect().vectorFieldNames().get( "similarity" ) ) - .failure( "Invalid value. Expected '" + cosine + "', actual is '" + l2 + "'" ) - .indexFieldContext( "vectorF" ) - .mappingAttributeContext( elasticSearchClient.getDialect().vectorFieldNames().get( "similarity" ) ) - .failure( "Invalid value. Expected '" + cosine + "', actual is '" + l2 + "'" ), - operation - ); + FailureReportChecker reportChecker; + + if ( ElasticsearchDistributionName.ELASTIC.equals( ElasticsearchTestDialect.getActualVersion().distribution() ) ) { + reportChecker = hasValidationFailureReport() + .indexFieldContext( "vectorB" ) + .mappingAttributeContext( "similarity" ) + .failure( "Invalid value. Expected '" + cosine + "', actual is '" + l2 + "'" ) + .indexFieldContext( "vectorF" ) + .mappingAttributeContext( "similarity" ) + .failure( "Invalid value. Expected '" + cosine + "', actual is '" + l2 + "'" ); + } + else { + reportChecker = hasValidationFailureReport() + .indexFieldContext( "vectorB" ) + .mappingAttributeContext( "method" ) + .mappingAttributeContext( "space_type" ) + .failure( "Invalid value. Expected '" + cosine + "', actual is '" + l2 + "'" ) + .indexFieldContext( "vectorF" ) + .mappingAttributeContext( "method" ) + .mappingAttributeContext( "space_type" ) + .failure( "Invalid value. Expected '" + cosine + "', actual is '" + l2 + "'" ); + } + + setupAndValidateExpectingFailure( index, reportChecker, operation ); } @ParameterizedTest(name = "With operation {0}") @@ -582,21 +598,48 @@ void vector_invalid_m_ef(ElasticsearchIndexSchemaManagerValidationOperation oper Optional.empty() ) ) ); - - setupAndValidateExpectingFailure( index, - hasValidationFailureReport() - .indexFieldContext( "vectorB" ) - .mappingAttributeContext( elasticSearchClient.getDialect().vectorFieldNames().get( "m" ) ) - .failure( "Invalid value. Expected '20', actual is '30'" ) - .mappingAttributeContext( elasticSearchClient.getDialect().vectorFieldNames().get( "ef_construction" ) ) - .failure( "Invalid value. Expected '2', actual is '3" ) - .indexFieldContext( "vectorF" ) - .mappingAttributeContext( elasticSearchClient.getDialect().vectorFieldNames().get( "m" ) ) - .failure( "Invalid value. Expected '50', actual is '60'" ) - .mappingAttributeContext( elasticSearchClient.getDialect().vectorFieldNames().get( "ef_construction" ) ) - .failure( "Invalid value. Expected '5', actual is '6'" ), - operation - ); + FailureReportChecker reportChecker; + + if ( ElasticsearchDistributionName.ELASTIC.equals( ElasticsearchTestDialect.getActualVersion().distribution() ) ) { + FailureReportChecker indexOptionsB = hasValidationFailureReport() + .indexFieldContext( "vectorB" ) + .mappingAttributeContext( "index_options" ); + indexOptionsB.mappingAttributeContext( "m" ) + .failure( "Invalid value. Expected '20', actual is '30'" ); + indexOptionsB.mappingAttributeContext( "ef_construction" ) + .failure( "Invalid value. Expected '2', actual is '3" ); + + FailureReportChecker indexOptionsF = indexOptionsB.indexFieldContext( "vectorF" ) + .mappingAttributeContext( "index_options" ); + indexOptionsF.mappingAttributeContext( "m" ) + .failure( "Invalid value. Expected '50', actual is '60'" ); + indexOptionsF.mappingAttributeContext( "ef_construction" ) + .failure( "Invalid value. Expected '5', actual is '6'" ); + + reportChecker = indexOptionsF; + } + else { + FailureReportChecker indexOptionsB = hasValidationFailureReport() + .indexFieldContext( "vectorB" ) + .mappingAttributeContext( "method" ) + .mappingAttributeContext( "parameters" ); + indexOptionsB.mappingAttributeContext( "m" ) + .failure( "Invalid value. Expected '20', actual is '30'" ); + indexOptionsB.mappingAttributeContext( "ef_construction" ) + .failure( "Invalid value. Expected '2', actual is '3" ); + + FailureReportChecker indexOptionsF = indexOptionsB.indexFieldContext( "vectorF" ) + .mappingAttributeContext( "method" ) + .mappingAttributeContext( "parameters" ); + indexOptionsF.mappingAttributeContext( "m" ) + .failure( "Invalid value. Expected '50', actual is '60'" ); + indexOptionsF.mappingAttributeContext( "ef_construction" ) + .failure( "Invalid value. Expected '5', actual is '6'" ); + + reportChecker = indexOptionsF; + } + + setupAndValidateExpectingFailure( index, reportChecker, operation ); } private void setupAndValidateExpectingFailure(StubMappedIndex index, FailureReportChecker failureReportChecker, diff --git a/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/dialect/ElasticsearchTestDialect.java b/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/dialect/ElasticsearchTestDialect.java index 22d0b020616..5a62a5ea35d 100644 --- a/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/dialect/ElasticsearchTestDialect.java +++ b/util/internal/integrationtest/backend/elasticsearch/src/main/java/org/hibernate/search/util/impl/integrationtest/backend/elasticsearch/dialect/ElasticsearchTestDialect.java @@ -120,17 +120,11 @@ public Map vectorFieldNames() { case ELASTIC: map.put( "dimension", "dims" ); map.put( "element_type", "element_type" ); - map.put( "similarity", "similarity" ); - map.put( "m", "index_options.m" ); - map.put( "ef_construction", "index_options.ef_construction" ); break; case OPENSEARCH: case AMAZON_OPENSEARCH_SERVERLESS: map.put( "dimension", "dimension" ); map.put( "element_type", "data_type" ); - map.put( "similarity", "method.space_type" ); - map.put( "m", "method.parameters.m" ); - map.put( "ef_construction", "method.parameters.ef_construction" ); break; default: throw new IllegalStateException( "Unknown distribution" );