Skip to content

Commit 2d762f0

Browse files
deepgarg-visadeepgarg-visa
authored
feat(search): include timestamp for entity metadata change (#12567)
1 parent 7326bb9 commit 2d762f0

File tree

14 files changed

+162
-43
lines changed

14 files changed

+162
-43
lines changed

‎entity-registry/src/main/java/com/linkedin/metadata/models/SearchableFieldSpecExtractor.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,9 @@ private void extractSearchableAnnotation(
176176
annotation.getNumValuesFieldName(),
177177
annotation.getWeightsPerFieldValue(),
178178
annotation.getFieldNameAliases(),
179-
annotation.isIncludeQueryEmptyAggregation());
179+
annotation.isIncludeQueryEmptyAggregation(),
180+
annotation.isIncludeSystemModifiedAt(),
181+
annotation.getSystemModifiedAtFieldName());
180182
}
181183
}
182184
log.debug("Searchable annotation for field: {} : {}", schemaPathSpec, annotation);

‎entity-registry/src/main/java/com/linkedin/metadata/models/annotation/SearchableAnnotation.java

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,10 @@ public class SearchableAnnotation {
6060
// only adds to query time not mapping
6161
boolean includeQueryEmptyAggregation;
6262

63+
boolean includeSystemModifiedAt;
64+
65+
Optional<String> systemModifiedAtFieldName;
66+
6367
public enum FieldType {
6468
KEYWORD,
6569
TEXT,
@@ -125,6 +129,10 @@ public static SearchableAnnotation fromPegasusAnnotationObject(
125129
final List<String> fieldNameAliases = getFieldNameAliases(map);
126130

127131
final FieldType resolvedFieldType = getFieldType(fieldType, schemaDataType);
132+
final Optional<Boolean> includeSystemModifiedAt =
133+
AnnotationUtils.getField(map, "includeSystemModifiedAt", Boolean.class);
134+
final Optional<String> systemModifiedAtFieldName =
135+
AnnotationUtils.getField(map, "systemModifiedAtFieldName", String.class);
128136
return new SearchableAnnotation(
129137
fieldName.orElse(schemaFieldName),
130138
resolvedFieldType,
@@ -139,7 +147,9 @@ public static SearchableAnnotation fromPegasusAnnotationObject(
139147
numValuesFieldName,
140148
weightsPerFieldValueMap.orElse(ImmutableMap.of()),
141149
fieldNameAliases,
142-
includeQueryEmptyAggregation.orElse(false));
150+
includeQueryEmptyAggregation.orElse(false),
151+
includeSystemModifiedAt.orElse(false),
152+
systemModifiedAtFieldName);
143153
}
144154

145155
private static FieldType getFieldType(

‎metadata-io/src/main/java/com/linkedin/metadata/search/elasticsearch/indexbuilder/MappingsBuilder.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,12 @@ private static Map<String, Object> getMappingsForField(
277277
.getNumValuesFieldName()
278278
.ifPresent(
279279
fieldName -> mappings.put(fieldName, ImmutableMap.of(TYPE, ESUtils.LONG_FIELD_TYPE)));
280+
281+
if (ESUtils.getSystemModifiedAtFieldName(searchableFieldSpec).isPresent()) {
282+
String modifiedAtFieldName = ESUtils.getSystemModifiedAtFieldName(searchableFieldSpec).get();
283+
mappings.put(modifiedAtFieldName, ImmutableMap.of(TYPE, ESUtils.DATE_FIELD_TYPE));
284+
}
285+
280286
mappings.putAll(getMappingsForFieldNameAliases(searchableFieldSpec));
281287

282288
return mappings;

‎metadata-io/src/main/java/com/linkedin/metadata/search/transformer/SearchDocumentTransformer.java

Lines changed: 33 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
import com.linkedin.metadata.models.annotation.SearchableAnnotation.FieldType;
3232
import com.linkedin.metadata.models.extractor.FieldExtractor;
3333
import com.linkedin.metadata.models.registry.EntityRegistry;
34+
import com.linkedin.metadata.search.utils.ESUtils;
35+
import com.linkedin.metadata.utils.AuditStampUtils;
3436
import com.linkedin.r2.RemoteInvocationException;
3537
import com.linkedin.structured.StructuredProperties;
3638
import com.linkedin.structured.StructuredPropertyDefinition;
@@ -92,7 +94,9 @@ public Optional<String> transformSnapshot(
9294
final ObjectNode searchDocument = JsonNodeFactory.instance.objectNode();
9395
searchDocument.put("urn", snapshot.data().get("urn").toString());
9496
extractedSearchableFields.forEach(
95-
(key, value) -> setSearchableValue(key, value, searchDocument, forDelete));
97+
(key, value) ->
98+
setSearchableValue(
99+
key, value, searchDocument, forDelete, AuditStampUtils.createDefaultAuditStamp()));
96100
extractedSearchScoreFields.forEach(
97101
(key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete));
98102
return Optional.of(searchDocument.toString());
@@ -149,7 +153,8 @@ public Optional<ObjectNode> transformAspect(
149153
final @Nonnull Urn urn,
150154
final @Nullable RecordTemplate aspect,
151155
final @Nonnull AspectSpec aspectSpec,
152-
final Boolean forDelete)
156+
final Boolean forDelete,
157+
final AuditStamp mclCreateAuditStamp)
153158
throws RemoteInvocationException, URISyntaxException {
154159
final Map<SearchableFieldSpec, List<Object>> extractedSearchableFields =
155160
FieldExtractor.extractFields(aspect, aspectSpec.getSearchableFieldSpecs(), maxValueLength);
@@ -168,10 +173,12 @@ public Optional<ObjectNode> transformAspect(
168173
searchDocument.put("urn", urn.toString());
169174

170175
extractedSearchableFields.forEach(
171-
(key, values) -> setSearchableValue(key, values, searchDocument, forDelete));
176+
(key, values) ->
177+
setSearchableValue(key, values, searchDocument, forDelete, mclCreateAuditStamp));
172178
extractedSearchRefFields.forEach(
173179
(key, values) ->
174-
setSearchableRefValue(opContext, key, values, searchDocument, forDelete));
180+
setSearchableRefValue(
181+
opContext, key, values, searchDocument, forDelete, mclCreateAuditStamp));
175182
extractedSearchScoreFields.forEach(
176183
(key, values) -> setSearchScoreValue(key, values, searchDocument, forDelete));
177184
result = Optional.of(searchDocument);
@@ -190,7 +197,8 @@ public void setSearchableValue(
190197
final SearchableFieldSpec fieldSpec,
191198
final List<Object> fieldValues,
192199
final ObjectNode searchDocument,
193-
final Boolean forDelete) {
200+
final Boolean forDelete,
201+
final AuditStamp mclCreatedAuditStamp) {
194202
DataSchema.Type valueType = fieldSpec.getPegasusSchema().getType();
195203
Optional<Object> firstValue = fieldValues.stream().findFirst();
196204
boolean isArray = fieldSpec.isArray();
@@ -255,6 +263,13 @@ public void setSearchableValue(
255263
return;
256264
}
257265

266+
if (ESUtils.getSystemModifiedAtFieldName(fieldSpec).isPresent()) {
267+
String modifiedAtFieldName = ESUtils.getSystemModifiedAtFieldName(fieldSpec).get();
268+
searchDocument.set(
269+
modifiedAtFieldName,
270+
JsonNodeFactory.instance.numberNode((Long) mclCreatedAuditStamp.getTime()));
271+
}
272+
258273
if (isArray || (valueType == DataSchema.Type.MAP && !OBJECT_FIELD_TYPES.contains(fieldType))) {
259274
if (fieldType == FieldType.BROWSE_PATH_V2) {
260275
String browsePathV2Value = getBrowsePathV2Value(fieldValues);
@@ -525,7 +540,8 @@ public void setSearchableRefValue(
525540
final SearchableRefFieldSpec searchableRefFieldSpec,
526541
final List<Object> fieldValues,
527542
final ObjectNode searchDocument,
528-
final Boolean forDelete) {
543+
final Boolean forDelete,
544+
final AuditStamp mclCreatedAuditStamp) {
529545
String fieldName = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldName();
530546
FieldType fieldType = searchableRefFieldSpec.getSearchableRefAnnotation().getFieldType();
531547
boolean isArray = searchableRefFieldSpec.isArray();
@@ -540,11 +556,13 @@ public void setSearchableRefValue(
540556
fieldValues
541557
.subList(0, Math.min(fieldValues.size(), maxArrayLength))
542558
.forEach(
543-
value -> getNodeForRef(opContext, depth, value, fieldType).ifPresent(arrayNode::add));
559+
value ->
560+
getNodeForRef(opContext, depth, value, fieldType, mclCreatedAuditStamp)
561+
.ifPresent(arrayNode::add));
544562
searchDocument.set(fieldName, arrayNode);
545563
} else if (!fieldValues.isEmpty()) {
546564
String finalFieldName = fieldName;
547-
getNodeForRef(opContext, depth, fieldValues.get(0), fieldType)
565+
getNodeForRef(opContext, depth, fieldValues.get(0), fieldType, mclCreatedAuditStamp)
548566
.ifPresent(node -> searchDocument.set(finalFieldName, node));
549567
} else {
550568
searchDocument.set(fieldName, JsonNodeFactory.instance.nullNode());
@@ -555,7 +573,8 @@ private Optional<JsonNode> getNodeForRef(
555573
@Nonnull OperationContext opContext,
556574
final int depth,
557575
final Object fieldValue,
558-
final FieldType fieldType) {
576+
final FieldType fieldType,
577+
final AuditStamp auditStamp) {
559578
EntityRegistry entityRegistry = opContext.getEntityRegistry();
560579
AspectRetriever aspectRetriever = opContext.getAspectRetriever();
561580

@@ -598,7 +617,7 @@ private Optional<JsonNode> getNodeForRef(
598617
SearchableFieldSpec spec = entry.getKey();
599618
List<Object> value = entry.getValue();
600619
if (!value.isEmpty()) {
601-
setSearchableValue(spec, value, resultNode, false);
620+
setSearchableValue(spec, value, resultNode, false, auditStamp);
602621
}
603622
}
604623

@@ -624,7 +643,8 @@ private Optional<JsonNode> getNodeForRef(
624643
opContext,
625644
newDepth,
626645
val,
627-
spec.getSearchableRefAnnotation().getFieldType())
646+
spec.getSearchableRefAnnotation().getFieldType(),
647+
auditStamp)
628648
.ifPresent(arrayNode::add));
629649
resultNode.set(fieldName, arrayNode);
630650
} else {
@@ -633,7 +653,8 @@ private Optional<JsonNode> getNodeForRef(
633653
opContext,
634654
newDepth,
635655
value.get(0),
636-
spec.getSearchableRefAnnotation().getFieldType());
656+
spec.getSearchableRefAnnotation().getFieldType(),
657+
auditStamp);
637658
if (node.isPresent()) {
638659
resultNode.set(fieldName, node.get());
639660
}

‎metadata-io/src/main/java/com/linkedin/metadata/search/utils/ESUtils.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -907,4 +907,15 @@ public static BoolQueryBuilder buildFilterNonLatestEntities(
907907
queryFilterRewriteChain);
908908
return QueryBuilders.boolQuery().should(isLatest).should(isNotVersioned).minimumShouldMatch(1);
909909
}
910+
911+
public static Optional<String> getSystemModifiedAtFieldName(
912+
@Nonnull SearchableFieldSpec searchableFieldSpec) {
913+
final String fieldName = searchableFieldSpec.getSearchableAnnotation().getFieldName();
914+
return searchableFieldSpec.getSearchableAnnotation().isIncludeSystemModifiedAt()
915+
? searchableFieldSpec
916+
.getSearchableAnnotation()
917+
.getSystemModifiedAtFieldName()
918+
.or(() -> Optional.of(String.format("%sSystemModifiedAt", fieldName)))
919+
: Optional.empty();
920+
}
910921
}

‎metadata-io/src/main/java/com/linkedin/metadata/service/UpdateIndicesService.java

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import com.fasterxml.jackson.databind.node.ObjectNode;
99
import com.google.common.annotations.VisibleForTesting;
1010
import com.google.common.collect.ImmutableSet;
11+
import com.linkedin.common.AuditStamp;
1112
import com.linkedin.common.Status;
1213
import com.linkedin.common.UrnArray;
1314
import com.linkedin.common.urn.Urn;
@@ -293,7 +294,8 @@ private void handleNonSystemMetadataDeleteChangeEvent(
293294
specPair.getFirst().getName(),
294295
specPair.getSecond(),
295296
event.getRecordTemplate(),
296-
isDeletingKey);
297+
isDeletingKey,
298+
event.getAuditStamp());
297299
}
298300
}
299301

@@ -325,7 +327,7 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev
325327
try {
326328
searchDocument =
327329
searchDocumentTransformer
328-
.transformAspect(opContext, urn, aspect, aspectSpec, false)
330+
.transformAspect(opContext, urn, aspect, aspectSpec, false, event.getAuditStamp())
329331
.map(
330332
objectNode ->
331333
withSystemCreated(
@@ -356,7 +358,7 @@ private void updateSearchService(@Nonnull OperationContext opContext, MCLItem ev
356358
try {
357359
previousSearchDocument =
358360
searchDocumentTransformer.transformAspect(
359-
opContext, urn, previousAspect, aspectSpec, false);
361+
opContext, urn, previousAspect, aspectSpec, false, event.getAuditStamp());
360362
} catch (Exception e) {
361363
log.error(
362364
"Error in getting documents from previous aspect state for urn: {} for aspect {}, continuing without diffing.",
@@ -445,7 +447,8 @@ private void deleteSearchData(
445447
String entityName,
446448
AspectSpec aspectSpec,
447449
@Nullable RecordTemplate aspect,
448-
Boolean isKeyAspect) {
450+
Boolean isKeyAspect,
451+
AuditStamp auditStamp) {
449452
String docId;
450453
try {
451454
docId = URLEncoder.encode(urn.toString(), "UTF-8");
@@ -463,7 +466,7 @@ private void deleteSearchData(
463466
try {
464467
searchDocument =
465468
searchDocumentTransformer
466-
.transformAspect(opContext, urn, aspect, aspectSpec, true)
469+
.transformAspect(opContext, urn, aspect, aspectSpec, true, auditStamp)
467470
.map(Objects::toString); // TODO
468471
} catch (Exception e) {
469472
log.error(

‎metadata-io/src/test/java/com/linkedin/metadata/search/query/request/AggregationQueryBuilderTest.java

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -170,7 +170,9 @@ public void testGetDefaultAggregationsHasFields() {
170170
Optional.empty(),
171171
Collections.emptyMap(),
172172
Collections.emptyList(),
173-
false);
173+
false,
174+
false,
175+
Optional.empty());
174176

175177
SearchConfiguration config = new SearchConfiguration();
176178
config.setMaxTermBucketSize(25);
@@ -203,7 +205,9 @@ public void testGetDefaultAggregationsFields() {
203205
Optional.empty(),
204206
Collections.emptyMap(),
205207
Collections.emptyList(),
206-
false);
208+
false,
209+
false,
210+
Optional.empty());
207211

208212
SearchConfiguration config = new SearchConfiguration();
209213
config.setMaxTermBucketSize(25);
@@ -235,7 +239,9 @@ public void testGetSpecificAggregationsHasFields() {
235239
Optional.empty(),
236240
Collections.emptyMap(),
237241
Collections.emptyList(),
238-
false);
242+
false,
243+
false,
244+
Optional.empty());
239245

240246
SearchableAnnotation annotation2 =
241247
new SearchableAnnotation(
@@ -252,7 +258,9 @@ public void testGetSpecificAggregationsHasFields() {
252258
Optional.empty(),
253259
Collections.emptyMap(),
254260
Collections.emptyList(),
255-
false);
261+
false,
262+
false,
263+
Optional.empty());
256264

257265
SearchConfiguration config = new SearchConfiguration();
258266
config.setMaxTermBucketSize(25);
@@ -462,7 +470,9 @@ public void testAggregateOverFieldsAndStructProp() {
462470
Optional.empty(),
463471
Collections.emptyMap(),
464472
Collections.emptyList(),
465-
false);
473+
false,
474+
false,
475+
Optional.empty());
466476

467477
SearchableAnnotation annotation2 =
468478
new SearchableAnnotation(
@@ -479,7 +489,9 @@ public void testAggregateOverFieldsAndStructProp() {
479489
Optional.empty(),
480490
Collections.emptyMap(),
481491
Collections.emptyList(),
482-
false);
492+
false,
493+
false,
494+
Optional.empty());
483495

484496
SearchConfiguration config = new SearchConfiguration();
485497
config.setMaxTermBucketSize(25);
@@ -532,7 +544,9 @@ public void testAggregateOverFieldsAndStructPropV1() {
532544
Optional.empty(),
533545
Collections.emptyMap(),
534546
Collections.emptyList(),
535-
false);
547+
false,
548+
false,
549+
Optional.empty());
536550

537551
SearchableAnnotation annotation2 =
538552
new SearchableAnnotation(
@@ -549,7 +563,9 @@ public void testAggregateOverFieldsAndStructPropV1() {
549563
Optional.empty(),
550564
Collections.emptyMap(),
551565
Collections.emptyList(),
552-
false);
566+
false,
567+
false,
568+
Optional.empty());
553569

554570
SearchConfiguration config = new SearchConfiguration();
555571
config.setMaxTermBucketSize(25);
@@ -606,7 +622,9 @@ public void testMissingAggregation() {
606622
Optional.empty(),
607623
Collections.emptyMap(),
608624
Collections.emptyList(),
609-
true);
625+
true,
626+
false,
627+
Optional.empty());
610628

611629
SearchConfiguration config = new SearchConfiguration();
612630
config.setMaxTermBucketSize(25);

‎metadata-io/src/test/java/com/linkedin/metadata/search/query/request/SearchQueryBuilderTest.java

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -454,7 +454,9 @@ public void testGetStandardFields() {
454454
Optional.empty(),
455455
Map.of(),
456456
List.of(),
457-
false),
457+
false,
458+
false,
459+
Optional.empty()),
458460
mock(DataSchema.class)),
459461
new SearchableFieldSpec(
460462
mock(PathSpec.class),
@@ -472,7 +474,9 @@ public void testGetStandardFields() {
472474
Optional.empty(),
473475
Map.of(),
474476
List.of(),
475-
false),
477+
false,
478+
false,
479+
Optional.empty()),
476480
mock(DataSchema.class)),
477481
new SearchableFieldSpec(
478482
mock(PathSpec.class),
@@ -490,7 +494,9 @@ public void testGetStandardFields() {
490494
Optional.empty(),
491495
Map.of(),
492496
List.of(),
493-
false),
497+
false,
498+
false,
499+
Optional.empty()),
494500
mock(DataSchema.class))));
495501

496502
fieldConfigs =

0 commit comments

Comments
 (0)