diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java index 0c5e1a4c31598f..f56df5e8c577be 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java @@ -176,7 +176,9 @@ private void extractSearchableAnnotation( annotation.getNumValuesFieldName(), annotation.getWeightsPerFieldValue(), annotation.getFieldNameAliases(), - annotation.isIncludeQueryEmptyAggregation()); + annotation.isIncludeQueryEmptyAggregation(), + annotation.isIncludeSystemModifiedAt(), + annotation.getSystemModifiedAtFieldName()); } } log.debug("Searchable annotation for field: {} : {}", schemaPathSpec, annotation); diff --git a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java index f15dbb61d70511..87fa1e31fb4120 100644 --- a/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java +++ b/entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java @@ -60,6 +60,10 @@ public class SearchableAnnotation { // only adds to query time not mapping boolean includeQueryEmptyAggregation; + boolean includeSystemModifiedAt; + + Optional systemModifiedAtFieldName; + public enum FieldType { KEYWORD, TEXT, @@ -125,6 +129,10 @@ public static SearchableAnnotation fromPegasusAnnotationObject( final List fieldNameAliases = getFieldNameAliases(map); final FieldType resolvedFieldType = getFieldType(fieldType, schemaDataType); + final Optional includeSystemModifiedAt = + AnnotationUtils.getField(map, "includeSystemModifiedAt", Boolean.class); + final Optional systemModifiedAtFieldName = + AnnotationUtils.getField(map, "systemModifiedAtFieldName", String.class); return new SearchableAnnotation( fieldName.orElse(schemaFieldName), resolvedFieldType, @@ -139,7 +147,9 @@ public static SearchableAnnotation fromPegasusAnnotationObject( numValuesFieldName, weightsPerFieldValueMap.orElse(ImmutableMap.of()), fieldNameAliases, - includeQueryEmptyAggregation.orElse(false)); + includeQueryEmptyAggregation.orElse(false), + includeSystemModifiedAt.orElse(false), + systemModifiedAtFieldName); } private static FieldType getFieldType( diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java index 47bf80bde6cf7b..9b50f4da54d591 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java @@ -277,6 +277,12 @@ private static Map getMappingsForField( .getNumValuesFieldName() .ifPresent( fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.LONG_FIELD_TYPE))); + + if (ESUtils.getSystemModifiedAtFieldName(searchableFieldSpec).isPresent()) { + String modifiedAtFieldName = ESUtils.getSystemModifiedAtFieldName(searchableFieldSpec).get(); + mappings.put(modifiedAtFieldName, ImmutableMap.of(TYPE, ESUtils.DATE_FIELD_TYPE)); + } + mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec)); return mappings; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java index 7a60b89d0127cc..0e0ea7e7dce1a0 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java @@ -31,6 +31,8 @@ import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType; import com.linkedin.metadata.models.extractor.FieldExtractor; import com.linkedin.metadata.models.registry.EntityRegistry; +import com.linkedin.metadata.search.utils.ESUtils; +import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.r2.RemoteInvocationException; import com.linkedin.structured.StructuredProperties; import com.linkedin.structured.StructuredPropertyDefinition; @@ -92,7 +94,9 @@ public Optional transformSnapshot( final ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); searchDocument.put("urn", snapshot.data().get("urn").toString()); extractedSearchableFields.forEach( - (key, value) -> setSearchableValue(key, value, searchDocument, forDelete)); + (key, value) -> + setSearchableValue( + key, value, searchDocument, forDelete, AuditStampUtils.createDefaultAuditStamp())); extractedSearchScoreFields.forEach( (key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete)); return Optional.of(searchDocument.toString()); @@ -149,7 +153,8 @@ public Optional transformAspect( final @Nonnull Urn urn, final @Nullable RecordTemplate aspect, final @Nonnull AspectSpec aspectSpec, - final Boolean forDelete) + final Boolean forDelete, + final AuditStamp mclCreateAuditStamp) throws RemoteInvocationException, URISyntaxException { final Map> extractedSearchableFields = FieldExtractor.extractFields(aspect, aspectSpec.getSearchableFieldSpecs(), maxValueLength); @@ -168,10 +173,12 @@ public Optional transformAspect( searchDocument.put("urn", urn.toString()); extractedSearchableFields.forEach( - (key, values) -> setSearchableValue(key, values, searchDocument, forDelete)); + (key, values) -> + setSearchableValue(key, values, searchDocument, forDelete, mclCreateAuditStamp)); extractedSearchRefFields.forEach( (key, values) -> - setSearchableRefValue(opContext, key, values, searchDocument, forDelete)); + setSearchableRefValue( + opContext, key, values, searchDocument, forDelete, mclCreateAuditStamp)); extractedSearchScoreFields.forEach( (key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete)); result = Optional.of(searchDocument); @@ -190,7 +197,8 @@ public void setSearchableValue( final SearchableFieldSpec fieldSpec, final List fieldValues, final ObjectNode searchDocument, - final Boolean forDelete) { + final Boolean forDelete, + final AuditStamp mclCreatedAuditStamp) { DataSchema.Type valueType = fieldSpec.getPegasusSchema().getType(); Optional firstValue = fieldValues.stream().findFirst(); boolean isArray = fieldSpec.isArray(); @@ -255,6 +263,13 @@ public void setSearchableValue( return; } + if (ESUtils.getSystemModifiedAtFieldName(fieldSpec).isPresent()) { + String modifiedAtFieldName = ESUtils.getSystemModifiedAtFieldName(fieldSpec).get(); + searchDocument.set( + modifiedAtFieldName, + JsonNodeFactory.instance.numberNode((Long) mclCreatedAuditStamp.getTime())); + } + if (isArray || (valueType == DataSchema.Type.MAP && !OBJECT_FIELD_TYPES.contains(fieldType))) { if (fieldType == FieldType.BROWSE_PATH_V2) { String browsePathV2Value = getBrowsePathV2Value(fieldValues); @@ -525,7 +540,8 @@ public void setSearchableRefValue( final SearchableRefFieldSpec searchableRefFieldSpec, final List fieldValues, final ObjectNode searchDocument, - final Boolean forDelete) { + final Boolean forDelete, + final AuditStamp mclCreatedAuditStamp) { String fieldName = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName(); FieldType fieldType = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldType(); boolean isArray = searchableRefFieldSpec.isArray(); @@ -540,11 +556,13 @@ public void setSearchableRefValue( fieldValues .subList(0, Math.min(fieldValues.size(), maxArrayLength)) .forEach( - value -> getNodeForRef(opContext, depth, value, fieldType).ifPresent(arrayNode::add)); + value -> + getNodeForRef(opContext, depth, value, fieldType, mclCreatedAuditStamp) + .ifPresent(arrayNode::add)); searchDocument.set(fieldName, arrayNode); } else if (!fieldValues.isEmpty()) { String finalFieldName = fieldName; - getNodeForRef(opContext, depth, fieldValues.get(0), fieldType) + getNodeForRef(opContext, depth, fieldValues.get(0), fieldType, mclCreatedAuditStamp) .ifPresent(node -> searchDocument.set(finalFieldName, node)); } else { searchDocument.set(fieldName, JsonNodeFactory.instance.nullNode()); @@ -555,7 +573,8 @@ private Optional getNodeForRef( @Nonnull OperationContext opContext, final int depth, final Object fieldValue, - final FieldType fieldType) { + final FieldType fieldType, + final AuditStamp auditStamp) { EntityRegistry entityRegistry = opContext.getEntityRegistry(); AspectRetriever aspectRetriever = opContext.getAspectRetriever(); @@ -598,7 +617,7 @@ private Optional getNodeForRef( SearchableFieldSpec spec = entry.getKey(); List value = entry.getValue(); if (!value.isEmpty()) { - setSearchableValue(spec, value, resultNode, false); + setSearchableValue(spec, value, resultNode, false, auditStamp); } } @@ -624,7 +643,8 @@ private Optional getNodeForRef( opContext, newDepth, val, - spec.getSearchableRefAnnotation().getFieldType()) + spec.getSearchableRefAnnotation().getFieldType(), + auditStamp) .ifPresent(arrayNode::add)); resultNode.set(fieldName, arrayNode); } else { @@ -633,7 +653,8 @@ private Optional getNodeForRef( opContext, newDepth, value.get(0), - spec.getSearchableRefAnnotation().getFieldType()); + spec.getSearchableRefAnnotation().getFieldType(), + auditStamp); if (node.isPresent()) { resultNode.set(fieldName, node.get()); } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java index 8f252f6440656e..94f53902542f87 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java @@ -907,4 +907,15 @@ public static BoolQueryBuilder buildFilterNonLatestEntities( queryFilterRewriteChain); return QueryBuilders.boolQuery().should(isLatest).should(isNotVersioned).minimumShouldMatch(1); } + + public static Optional getSystemModifiedAtFieldName( + @Nonnull SearchableFieldSpec searchableFieldSpec) { + final String fieldName = searchableFieldSpec.getSearchableAnnotation().getFieldName(); + return searchableFieldSpec.getSearchableAnnotation().isIncludeSystemModifiedAt() + ? searchableFieldSpec + .getSearchableAnnotation() + .getSystemModifiedAtFieldName() + .or(() -> Optional.of(String.format("%sSystemModifiedAt", fieldName))) + : Optional.empty(); + } } diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java index c410d2ccefdf24..3b158e7d7f25bc 100644 --- a/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java @@ -8,6 +8,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; import com.linkedin.common.Status; import com.linkedin.common.UrnArray; import com.linkedin.common.urn.Urn; @@ -293,7 +294,8 @@ private void handleNonSystemMetadataDeleteChangeEvent( specPair.getFirst().getName(), specPair.getSecond(), event.getRecordTemplate(), - isDeletingKey); + isDeletingKey, + event.getAuditStamp()); } } @@ -325,7 +327,7 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev try { searchDocument = searchDocumentTransformer - .transformAspect(opContext, urn, aspect, aspectSpec, false) + .transformAspect(opContext, urn, aspect, aspectSpec, false, event.getAuditStamp()) .map( objectNode -> withSystemCreated( @@ -356,7 +358,7 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev try { previousSearchDocument = searchDocumentTransformer.transformAspect( - opContext, urn, previousAspect, aspectSpec, false); + opContext, urn, previousAspect, aspectSpec, false, event.getAuditStamp()); } catch (Exception e) { log.error( "Error in getting documents from previous aspect state for urn: {} for aspect {}, continuing without diffing.", @@ -445,7 +447,8 @@ private void deleteSearchData( String entityName, AspectSpec aspectSpec, @Nullable RecordTemplate aspect, - Boolean isKeyAspect) { + Boolean isKeyAspect, + AuditStamp auditStamp) { String docId; try { docId = URLEncoder.encode(urn.toString(), "UTF-8"); @@ -463,7 +466,7 @@ private void deleteSearchData( try { searchDocument = searchDocumentTransformer - .transformAspect(opContext, urn, aspect, aspectSpec, true) + .transformAspect(opContext, urn, aspect, aspectSpec, true, auditStamp) .map(Objects::toString); // TODO } catch (Exception e) { log.error( diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java index 2ba03665068fbf..b67fd15ddfd607 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java @@ -170,7 +170,9 @@ public void testGetDefaultAggregationsHasFields() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchConfiguration config = new SearchConfiguration(); config.setMaxTermBucketSize(25); @@ -203,7 +205,9 @@ public void testGetDefaultAggregationsFields() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchConfiguration config = new SearchConfiguration(); config.setMaxTermBucketSize(25); @@ -235,7 +239,9 @@ public void testGetSpecificAggregationsHasFields() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchableAnnotation annotation2 = new SearchableAnnotation( @@ -252,7 +258,9 @@ public void testGetSpecificAggregationsHasFields() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchConfiguration config = new SearchConfiguration(); config.setMaxTermBucketSize(25); @@ -462,7 +470,9 @@ public void testAggregateOverFieldsAndStructProp() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchableAnnotation annotation2 = new SearchableAnnotation( @@ -479,7 +489,9 @@ public void testAggregateOverFieldsAndStructProp() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchConfiguration config = new SearchConfiguration(); config.setMaxTermBucketSize(25); @@ -532,7 +544,9 @@ public void testAggregateOverFieldsAndStructPropV1() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchableAnnotation annotation2 = new SearchableAnnotation( @@ -549,7 +563,9 @@ public void testAggregateOverFieldsAndStructPropV1() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - false); + false, + false, + Optional.empty()); SearchConfiguration config = new SearchConfiguration(); config.setMaxTermBucketSize(25); @@ -606,7 +622,9 @@ public void testMissingAggregation() { Optional.empty(), Collections.emptyMap(), Collections.emptyList(), - true); + true, + false, + Optional.empty()); SearchConfiguration config = new SearchConfiguration(); config.setMaxTermBucketSize(25); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java index 374a69ee9a5536..ac3d980b3bb364 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java @@ -454,7 +454,9 @@ public void testGetStandardFields() { Optional.empty(), Map.of(), List.of(), - false), + false, + false, + Optional.empty()), mock(DataSchema.class)), new SearchableFieldSpec( mock(PathSpec.class), @@ -472,7 +474,9 @@ public void testGetStandardFields() { Optional.empty(), Map.of(), List.of(), - false), + false, + false, + Optional.empty()), mock(DataSchema.class)), new SearchableFieldSpec( mock(PathSpec.class), @@ -490,7 +494,9 @@ public void testGetStandardFields() { Optional.empty(), Map.of(), List.of(), - false), + false, + false, + Optional.empty()), mock(DataSchema.class)))); fieldConfigs = diff --git a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java index 5a4fb39bd50e96..eff0e60a36c892 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformerTest.java @@ -30,6 +30,7 @@ import com.linkedin.metadata.models.registry.ConfigEntityRegistry; import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.search.query.request.TestSearchFieldConfig; +import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.r2.RemoteInvocationException; import io.datahubproject.metadata.context.OperationContext; import io.datahubproject.metadata.context.RetrieverContext; @@ -255,7 +256,12 @@ public void testSetSearchableRefValue() throws URISyntaxException, RemoteInvocat .build()); searchDocumentTransformer.setSearchableRefValue( - opContext, searchableRefFieldSpec, urnList, searchDocument, false); + opContext, + searchableRefFieldSpec, + urnList, + searchDocument, + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(searchDocument.has("refEntityUrns")); assertEquals(searchDocument.get("refEntityUrns").size(), 3); assertTrue(searchDocument.get("refEntityUrns").has("urn")); @@ -282,7 +288,12 @@ public void testSetSearchableRefValue_WithNonURNField() throws URISyntaxExceptio SearchableRefFieldSpec searchableRefFieldSpecText = TEST_ENTITY_REGISTRY.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(1); searchDocumentTransformer.setSearchableRefValue( - opContext, searchableRefFieldSpecText, urnList, searchDocument, false); + opContext, + searchableRefFieldSpecText, + urnList, + searchDocument, + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(searchDocument.isEmpty()); } @@ -315,7 +326,12 @@ public void testSetSearchableRefValue_RuntimeException() SearchableRefFieldSpec searchableRefFieldSpec = TEST_ENTITY_REGISTRY.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); searchDocumentTransformer.setSearchableRefValue( - opContext, searchableRefFieldSpec, urnList, searchDocument, false); + opContext, + searchableRefFieldSpec, + urnList, + searchDocument, + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(searchDocument.isEmpty()); } @@ -354,7 +370,12 @@ public void testSetSearchableRefValue_RuntimeException_URNExist() SearchableRefFieldSpec searchableRefFieldSpec = TEST_ENTITY_REGISTRY.getEntitySpec("testRefEntity").getSearchableRefFieldSpecs().get(0); searchDocumentTransformer.setSearchableRefValue( - opContext, searchableRefFieldSpec, urnList, searchDocument, false); + opContext, + searchableRefFieldSpec, + urnList, + searchDocument, + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(searchDocument.has("refEntityUrns")); assertEquals(searchDocument.get("refEntityUrns").size(), 1); assertTrue(searchDocument.get("refEntityUrns").has("urn")); @@ -387,7 +408,12 @@ void testSetSearchableRefValue_WithInvalidURN() ObjectNode searchDocument = JsonNodeFactory.instance.objectNode(); searchDocumentTransformer.setSearchableRefValue( - opContext, searchableRefFieldSpec, urnList, searchDocument, false); + opContext, + searchableRefFieldSpec, + urnList, + searchDocument, + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(searchDocument.has("refEntityUrns")); assertTrue(searchDocument.get("refEntityUrns").getNodeType().equals(JsonNodeType.NULL)); } @@ -406,7 +432,8 @@ public void testEmptyDescription() throws RemoteInvocationException, URISyntaxEx ENTITY_REGISTRY .getEntitySpec(DATASET_ENTITY_NAME) .getAspectSpec(EDITABLE_DATASET_PROPERTIES_ASPECT_NAME), - false); + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(transformed.isPresent()); assertEquals(transformed.get().get("urn").asText(), entityUrn); @@ -422,7 +449,8 @@ public void testEmptyDescription() throws RemoteInvocationException, URISyntaxEx ENTITY_REGISTRY .getEntitySpec(DATASET_ENTITY_NAME) .getAspectSpec(DATASET_PROPERTIES_ASPECT_NAME), - false); + false, + AuditStampUtils.createDefaultAuditStamp()); assertTrue(transformed.isPresent()); assertEquals(transformed.get().get("urn").asText(), entityUrn); diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java index 43f8cc0ef191d6..e8f72206117cc7 100644 --- a/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/UpdateIndicesServiceTest.java @@ -17,6 +17,7 @@ import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; import com.linkedin.metadata.systemmetadata.SystemMetadataService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; +import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.SystemMetadataUtils; import com.linkedin.mxe.MetadataChangeLog; import io.datahubproject.metadata.context.OperationContext; @@ -66,7 +67,7 @@ public void testContainerHandleDeleteEvent() throws Exception { event.setAspectName(CONTAINER_ASPECT_NAME); event.setEntityType(urn.getEntityType()); event.setSystemMetadata(SystemMetadataUtils.createDefaultSystemMetadata()); - + event.setCreated(AuditStampUtils.createDefaultAuditStamp()); // Execute Delete updateIndicesService.handleChangeEvent(operationContext, event); @@ -78,7 +79,8 @@ public void testContainerHandleDeleteEvent() throws Exception { eq(urn), nullable(RecordTemplate.class), eq(aspectSpec), - eq(true)); + eq(true), + eq(event.getCreated())); verify(updateGraphIndicesService).handleChangeEvent(operationContext, event); } } diff --git a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl index 5422864185f141..f94c07eb62d0ec 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/businessattribute/BusinessAttributeAssociation.pdl @@ -5,5 +5,11 @@ record BusinessAttributeAssociation { /** * Urn of the applied businessAttribute */ + @Searchable = { + "fieldName": "schemaFieldBusinessAttribute", + "queryByDefault": false, + "includeSystemModifiedAt": true, + "systemModifiedAtFieldName": "schemaFieldBusinessAttributeModifiedAt" + } businessAttributeUrn: BusinessAttributeUrn } \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl index 58423ccc2228db..aab584d5887468 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/common/GlossaryTermAssociation.pdl @@ -16,7 +16,9 @@ record GlossaryTermAssociation { "fieldType": "URN", "hasValuesFieldName": "hasGlossaryTerms", "addToFilters": true, - "filterNameOverride": "Glossary Term" + "filterNameOverride": "Glossary Term", + "includeSystemModifiedAt": true, + "systemModifiedAtFieldName": "termsModifiedAt" } urn: GlossaryTermUrn diff --git a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl index 048c2dcd9f58f6..f23ef082c59088 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/schema/EditableSchemaFieldInfo.pdl @@ -67,7 +67,9 @@ record EditableSchemaFieldInfo { "/terms/*/urn": { "fieldName": "editedFieldGlossaryTerms", "fieldType": "URN", - "boostScore": 0.5 + "boostScore": 0.5, + "includeSystemModifiedAt": true, + "systemModifiedAtFieldName": "schemaFieldTermsModifiedAt" }, "/terms/*/attribution/time": { "fieldName": "editedFieldTermAttributionDates", diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java index 04d73895802a8d..85a9a02bd0d29f 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestPoliciesStep.java @@ -20,6 +20,7 @@ import com.linkedin.metadata.query.ListUrnsResult; import com.linkedin.metadata.search.EntitySearchService; import com.linkedin.metadata.search.transformer.SearchDocumentTransformer; +import com.linkedin.metadata.utils.AuditStampUtils; import com.linkedin.metadata.utils.EntityKeyUtils; import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.mxe.GenericAspect; @@ -175,7 +176,8 @@ private void insertPolicyDocument( entityResponse.getUrn(), new DataHubPolicyInfo(aspect.getValue().data()), aspectSpec, - false) + false, + AuditStampUtils.createDefaultAuditStamp()) .map(Objects::toString); } catch (Exception e) { log.error(