From 394a57d531dfd94147e42a0784e06f6164c4224b Mon Sep 17 00:00:00 2001 From: Deepak Garg Date: Thu, 6 Feb 2025 20:09:30 +0530 Subject: [PATCH] feat(search): include timestamp for entity metadata change --- .../models/SearchableFieldSpecExtractor.java | 4 +++- .../models/annotation/SearchableAnnotation.java | 12 +++++++++++- .../elasticsearch/indexbuilder/MappingsBuilder.java | 11 +++++++++++ .../transformer/SearchDocumentTransformer.java | 11 +++++++++++ .../BusinessAttributeAssociation.pdl | 5 +++++ .../com/linkedin/common/GlossaryTermAssociation.pdl | 4 +++- .../com/linkedin/schema/EditableSchemaFieldInfo.pdl | 4 +++- 7 files changed, 47 insertions(+), 4 deletions(-) diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java index 0c5e1a4c31598f..f56df5e8c577be 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java @@ -176,7 +176,9 @@ private void extractSearchableAnnotation( annotation.getNumValuesFieldName(), annotation.getWeightsPerFieldValue(), annotation.getFieldNameAliases(), - annotation.isIncludeQueryEmptyAggregation()); + annotation.isIncludeQueryEmptyAggregation(), + annotation.isIncludeSystemModifiedAt(), + annotation.getSystemModifiedAtFieldName()); } } log.debug("Searchable annotation for field: {} : {}", schemaPathSpec, annotation); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java index f15dbb61d70511..87fa1e31fb4120 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java @@ -60,6 +60,10 @@ public class SearchableAnnotation { // only adds to query time not mapping boolean includeQueryEmptyAggregation; + boolean includeSystemModifiedAt; + + Optional systemModifiedAtFieldName; + public enum FieldType { KEYWORD, TEXT, @@ -125,6 +129,10 @@ public static SearchableAnnotation fromPegasusAnnotationObject( final List fieldNameAliases = getFieldNameAliases(map); final FieldType resolvedFieldType = getFieldType(fieldType, schemaDataType); + final Optional includeSystemModifiedAt = + AnnotationUtils.getField(map, "includeSystemModifiedAt", Boolean.class); + final Optional systemModifiedAtFieldName = + AnnotationUtils.getField(map, "systemModifiedAtFieldName", String.class); return new SearchableAnnotation( fieldName.orElse(schemaFieldName), resolvedFieldType, @@ -139,7 +147,9 @@ public static SearchableAnnotation fromPegasusAnnotationObject( numValuesFieldName, weightsPerFieldValueMap.orElse(ImmutableMap.of()), fieldNameAliases, - includeQueryEmptyAggregation.orElse(false)); + includeQueryEmptyAggregation.orElse(false), + includeSystemModifiedAt.orElse(false), + systemModifiedAtFieldName); } private static FieldType getFieldType( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index 47bf80bde6cf7b..1775039c695ff0 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -277,6 +277,17 @@ private static Map getMappingsForField( .getNumValuesFieldName() .ifPresent( fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.LONG_FIELD_TYPE))); + + final String fieldName = searchableFieldSpec.getSearchableAnnotation().getFieldName(); + if (searchableFieldSpec.getSearchableAnnotation().isIncludeSystemModifiedAt()) { + String modifiedAtFieldName = + searchableFieldSpec + .getSearchableAnnotation() + .getSystemModifiedAtFieldName() + .orElse(String.format("%sSystemModifiedAt", fieldName)); + mappings.put(modifiedAtFieldName, ImmutableMap.of(TYPE, ESUtils.DATE_FIELD_TYPE)); + } + mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec)); return mappings; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index 7a60b89d0127cc..fd0e516eaa0325 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -255,6 +255,17 @@ public void setSearchableValue( return; } + if (fieldSpec.getSearchableAnnotation().isIncludeSystemModifiedAt()) { + String modifiedAtFieldName = + fieldSpec + .getSearchableAnnotation() + .getSystemModifiedAtFieldName() + .orElse(String.format("%sSystemModifiedAt", fieldName)); + searchDocument.set( + modifiedAtFieldName, + JsonNodeFactory.instance.numberNode((Long) System.currentTimeMillis())); + } + if (isArray || (valueType == DataSchema.Type.MAP && !OBJECT_FIELD_TYPES.contains(fieldType))) { if (fieldType == FieldType.BROWSE_PATH_V2) { String browsePathV2Value = getBrowsePathV2Value(fieldValues); diff --git a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl index 5422864185f141..d1f741d6262cfe 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl @@ -5,5 +5,10 @@ record BusinessAttributeAssociation { /** * Urn of the applied businessAttribute */ + @Searchable = { + "fieldName": "schemaFieldBusinessAttribute", + "includeSystemModifiedAt": true, + "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt" + } businessAttributeUrn: BusinessAttributeUrn } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl index 58423ccc2228db..aab584d5887468 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl @@ -16,7 +16,9 @@ record GlossaryTermAssociation { "fieldType": "URN", "hasValuesFieldName": "hasGlossaryTerms", "addToFilters": true, - "filterNameOverride": "Glossary Term" + "filterNameOverride": "Glossary Term", + "includeSystemModifiedAt": true, + "systemModifiedAtFieldName": "termsModifiedAt" } urn: GlossaryTermUrn diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl index 048c2dcd9f58f6..f23ef082c59088 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl @@ -67,7 +67,9 @@ record EditableSchemaFieldInfo { "/terms/*/urn": { "fieldName": "editedFieldGlossaryTerms", "fieldType": "URN", - "boostScore": 0.5 + "boostScore": 0.5, + "includeSystemModifiedAt": true, + "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt" }, "/terms/*/attribution/time": { "fieldName": "editedFieldTermAttributionDates",