From 3c9a2d51e7ca20091b12efbdfd35ea0ed0b9b556 Mon Sep 17 00:00:00 2001 From: Andrei Bordak Date: Thu, 6 Feb 2025 14:47:07 +0400 Subject: [PATCH 1/8] MODINVSTOR-1342: Add "deleted" field to Instance schema --- ramls/instance.json | 4 ++++ .../handlers/MarcInstanceSharingHandlerImpl.java | 1 + .../folio/inventory/domain/instances/Instance.java | 14 ++++++++++++++ .../org/folio/inventory/support/InstanceUtil.java | 1 + .../MarcInstanceSharingHandlerImplTest.java | 2 ++ .../handlers/actions/util/InstanceUtilTest.java | 2 ++ 6 files changed, 24 insertions(+) diff --git a/ramls/instance.json b/ramls/instance.json index 5ce12a25e..45a1a7ae2 100644 --- a/ramls/instance.json +++ b/ramls/instance.json @@ -427,6 +427,10 @@ "type": "boolean", "description": "Records the fact that the record should not be displayed in a discovery system" }, + "deleted": { + "type": "boolean", + "description": "Indicates whether the record was marked for deletion" + }, "statisticalCodeIds": { "type": "array", "description": "List of statistical code IDs", diff --git a/src/main/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImpl.java b/src/main/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImpl.java index 491f713dc..5914c4a44 100644 --- a/src/main/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImpl.java +++ b/src/main/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImpl.java @@ -121,6 +121,7 @@ private Future updateSuppressFromDiscoveryFlagIfNeeded(Instance target private Instance populateTargetInstanceWithNonMarcControlledFields(Instance targetInstance, Instance sourceInstance) { targetInstance.setStaffSuppress(sourceInstance.getStaffSuppress()); targetInstance.setDiscoverySuppress(sourceInstance.getDiscoverySuppress()); + targetInstance.setDeleted(sourceInstance.getDeleted()); targetInstance.setCatalogedDate(sourceInstance.getCatalogedDate()); targetInstance.setStatusId(sourceInstance.getStatusId()); targetInstance.setStatisticalCodeIds(sourceInstance.getStatisticalCodeIds()); diff --git a/src/main/java/org/folio/inventory/domain/instances/Instance.java b/src/main/java/org/folio/inventory/domain/instances/Instance.java index 58387d528..5370411b6 100644 --- a/src/main/java/org/folio/inventory/domain/instances/Instance.java +++ b/src/main/java/org/folio/inventory/domain/instances/Instance.java @@ -63,6 +63,7 @@ public class Instance { public static final String PREVIOUSLY_HELD_KEY = "previouslyHeld"; public static final String STAFF_SUPPRESS_KEY = "staffSuppress"; public static final String DISCOVERY_SUPPRESS_KEY = "discoverySuppress"; + public static final String DELETED_KEY = "deleted"; public static final String STATISTICAL_CODE_IDS_KEY = "statisticalCodeIds"; public static final String ADMININSTRATIVE_NOTES_KEY = "administrativeNotes"; public static final String SOURCE_RECORD_FORMAT_KEY = "sourceRecordFormat"; @@ -109,6 +110,7 @@ public class Instance { private Boolean previouslyHeld; private Boolean staffSuppress; private Boolean discoverySuppress; + private Boolean deleted; private List statisticalCodeIds = new ArrayList<>(); private String sourceRecordFormat; private String statusId; @@ -182,6 +184,7 @@ public static Instance fromJson(JsonObject instanceJson) { .setPreviouslyHeld(instanceJson.getBoolean(PREVIOUSLY_HELD_KEY, false)) .setStaffSuppress(instanceJson.getBoolean(STAFF_SUPPRESS_KEY, false)) .setDiscoverySuppress(instanceJson.getBoolean(DISCOVERY_SUPPRESS_KEY, false)) + .setDeleted(instanceJson.getBoolean(DELETED_KEY, false)) .setStatisticalCodeIds(toListOfStrings(instanceJson.getJsonArray(STATISTICAL_CODE_IDS_KEY))) .setSourceRecordFormat(instanceJson.getString(SOURCE_RECORD_FORMAT_KEY)) .setStatusId(instanceJson.getString(STATUS_ID_KEY)) @@ -230,6 +233,7 @@ public JsonObject getJsonForStorage() { json.put(PREVIOUSLY_HELD_KEY, previouslyHeld); json.put(STAFF_SUPPRESS_KEY, staffSuppress); json.put(DISCOVERY_SUPPRESS_KEY, discoverySuppress); + json.put(DELETED_KEY, deleted); json.put(STATISTICAL_CODE_IDS_KEY, statisticalCodeIds); if (sourceRecordFormat != null) json.put(SOURCE_RECORD_FORMAT_KEY, sourceRecordFormat); json.put(STATUS_ID_KEY, statusId); @@ -282,6 +286,7 @@ public JsonObject getJsonForResponse(WebContext context) { putIfNotNull(json, PREVIOUSLY_HELD_KEY, getPreviouslyHeld()); putIfNotNull(json, STAFF_SUPPRESS_KEY, getStaffSuppress()); putIfNotNull(json, DISCOVERY_SUPPRESS_KEY, getDiscoverySuppress()); + putIfNotNull(json, DELETED_KEY, getDeleted()); putIfNotNull(json, STATISTICAL_CODE_IDS_KEY, getStatisticalCodeIds()); putIfNotNull(json, SOURCE_RECORD_FORMAT_KEY, getSourceRecordFormat()); putIfNotNull(json, STATUS_ID_KEY, getStatusId()); @@ -516,6 +521,11 @@ public Instance setDiscoverySuppress(Boolean discoverySuppress) { return this; } + public Instance setDeleted(Boolean deleted) { + this.deleted = deleted; + return this; + } + public Instance setStatisticalCodeIds(List statisticalCodeIds) { this.statisticalCodeIds = statisticalCodeIds; return this; @@ -693,6 +703,10 @@ public Boolean getDiscoverySuppress() { return discoverySuppress; } + public Boolean getDeleted() { + return deleted; + } + public List getStatisticalCodeIds() { return statisticalCodeIds; } diff --git a/src/main/java/org/folio/inventory/support/InstanceUtil.java b/src/main/java/org/folio/inventory/support/InstanceUtil.java index f1b06a4c9..c1bb6dd42 100644 --- a/src/main/java/org/folio/inventory/support/InstanceUtil.java +++ b/src/main/java/org/folio/inventory/support/InstanceUtil.java @@ -52,6 +52,7 @@ public static Instance mergeFieldsWhichAreNotControlled(Instance existing, org.f .withVersion(asIntegerOrNull(existing.getVersion())) .withDiscoverySuppress(existing.getDiscoverySuppress()) .withStaffSuppress(existing.getStaffSuppress()) + .withDeleted(existing.getDeleted()) .withPreviouslyHeld(existing.getPreviouslyHeld()) .withCatalogedDate(existing.getCatalogedDate()) .withStatusId(existing.getStatusId()) diff --git a/src/test/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImplTest.java b/src/test/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImplTest.java index 3c527e0c5..ba9850674 100644 --- a/src/test/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImplTest.java +++ b/src/test/java/org/folio/inventory/consortium/handlers/MarcInstanceSharingHandlerImplTest.java @@ -611,6 +611,7 @@ public void shouldPopulateTargetInstanceWithNonMarcControlledFields(TestContext Instance localSourceInstance = new Instance(instanceId, "1", "001", "MARC", "testTitle", UUID.randomUUID().toString()) .setDiscoverySuppress(Boolean.TRUE) .setStaffSuppress(Boolean.TRUE) + .setDeleted(Boolean.TRUE) .setCatalogedDate("1970-01-01") .setStatusId(UUID.randomUUID().toString()) .setStatisticalCodeIds(List.of(UUID.randomUUID().toString())) @@ -654,6 +655,7 @@ public void shouldPopulateTargetInstanceWithNonMarcControlledFields(TestContext testContext.assertEquals(targetInstanceHrid, targetInstanceWithNonMarcData.getHrid()); testContext.assertEquals(localSourceInstance.getDiscoverySuppress(), targetInstanceWithNonMarcData.getDiscoverySuppress()); testContext.assertEquals(localSourceInstance.getStaffSuppress(), targetInstanceWithNonMarcData.getStaffSuppress()); + testContext.assertEquals(localSourceInstance.getDeleted(), targetInstanceWithNonMarcData.getDeleted()); testContext.assertEquals(localSourceInstance.getCatalogedDate(), targetInstanceWithNonMarcData.getCatalogedDate()); testContext.assertEquals(localSourceInstance.getStatusId(), targetInstanceWithNonMarcData.getStatusId()); testContext.assertEquals(localSourceInstance.getStatisticalCodeIds(), targetInstanceWithNonMarcData.getStatisticalCodeIds()); diff --git a/src/test/java/org/folio/inventory/dataimport/handlers/actions/util/InstanceUtilTest.java b/src/test/java/org/folio/inventory/dataimport/handlers/actions/util/InstanceUtilTest.java index 2d15d9c62..25b412b98 100644 --- a/src/test/java/org/folio/inventory/dataimport/handlers/actions/util/InstanceUtilTest.java +++ b/src/test/java/org/folio/inventory/dataimport/handlers/actions/util/InstanceUtilTest.java @@ -65,6 +65,7 @@ public void shouldMergeInstances() { existing.setStatisticalCodeIds(statisticalCodeIds); existing.setDiscoverySuppress(true); existing.setStaffSuppress(true); + existing.setDeleted(true); existing.setPreviouslyHeld(true); existing.setCatalogedDate(""); existing.setStatusId("30773a27-b485-4dab-aeb6-b8c04fa3cb26"); @@ -85,6 +86,7 @@ public void shouldMergeInstances() { assertEquals(statisticalCodeIds, instance.getStatisticalCodeIds()); assertTrue(instance.getDiscoverySuppress()); assertTrue(instance.getStaffSuppress()); + assertTrue(instance.getDeleted()); assertTrue(instance.getPreviouslyHeld()); assertEquals("", instance.getCatalogedDate()); assertEquals("30773a27-b485-4dab-aeb6-b8c04fa3cb26", instance.getStatusId()); From 3822c50e9d6be654daab19a731c53a448424f1db Mon Sep 17 00:00:00 2001 From: Andrei Bordak Date: Thu, 6 Feb 2025 14:51:37 +0400 Subject: [PATCH 2/8] MODINVSTOR-1342: Updated NEWS.md --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 0dc606bb0..1ab93b617 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ * Fix handling optimistic locking behavior for instance update when consuming Marc Bib update event [MODINV-1125](https://folio-org.atlassian.net/browse/MODINV-1125) * Replace usage of deprecated instance-storage-batch API [MODINV-1101](https://folio-org.atlassian.net/browse/MODINV-1101Л) * Update the snapshot status from PROCESSING_FINISHED to COMMITTED in the InstanceIngressEventHandler [MODINV-1161](https://folio-org.atlassian.net/browse/MODINV-1161) +* Add "deleted" field to Instance schema [MODINVSTOR-1342](https://folio-org.atlassian.net/browse/MODINVSTOR-1342) ## 21.0.0 2024-10-29 * Existing "035" field is not retained the original position in imported record [MODINV-1049](https://folio-org.atlassian.net/browse/MODINV-1049) From e04c5e013c0a3933059b995012815765cf6880aa Mon Sep 17 00:00:00 2001 From: Andrei Bordak Date: Wed, 12 Feb 2025 10:58:25 +0400 Subject: [PATCH 3/8] MODINVSTOR-1138: Adjust mark-deleted endpoint behavior to set the "deleted" markers --- src/main/java/org/folio/inventory/resources/Instances.java | 1 + src/test/java/api/InstancesApiExamples.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/org/folio/inventory/resources/Instances.java b/src/main/java/org/folio/inventory/resources/Instances.java index 1aac7e013..bb158f3f3 100644 --- a/src/main/java/org/folio/inventory/resources/Instances.java +++ b/src/main/java/org/folio/inventory/resources/Instances.java @@ -343,6 +343,7 @@ private void softDelete(RoutingContext routingContext) { private void updateVisibility(Instance instance, RoutingContext routingContext, InstanceCollection instanceCollection) { instance.setDiscoverySuppress(true); instance.setStaffSuppress(true); + instance.setDeleted(true); instanceCollection.update(instance, v -> { log.info("staffSuppress and discoverySuppress properties are set to true for instance {}", instance.getId()); diff --git a/src/test/java/api/InstancesApiExamples.java b/src/test/java/api/InstancesApiExamples.java index 457ab9894..f0626651e 100644 --- a/src/test/java/api/InstancesApiExamples.java +++ b/src/test/java/api/InstancesApiExamples.java @@ -781,6 +781,7 @@ public void canSoftDeleteInstance() { assertTrue(getResponse.getJson().getBoolean("staffSuppress")); assertTrue(getResponse.getJson().getBoolean("discoverySuppress")); + assertTrue(getResponse.getJson().getBoolean("deleted")); Response getDeletedSourceRecordResponse = sourceRecordStorageClient.getById(instanceId); assertEquals(getDeletedSourceRecordResponse.getStatusCode(), HttpStatus.HTTP_NOT_FOUND.toInt()); From 7ac0db26e02361851b50f3fe19bc9041976d40f9 Mon Sep 17 00:00:00 2001 From: Andrei Bordak Date: Wed, 12 Feb 2025 11:20:05 +0400 Subject: [PATCH 4/8] MODINVSTOR-1138: Adjust log message --- src/main/java/org/folio/inventory/resources/Instances.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/folio/inventory/resources/Instances.java b/src/main/java/org/folio/inventory/resources/Instances.java index bb158f3f3..64c4087c6 100644 --- a/src/main/java/org/folio/inventory/resources/Instances.java +++ b/src/main/java/org/folio/inventory/resources/Instances.java @@ -345,7 +345,7 @@ private void updateVisibility(Instance instance, RoutingContext routingContext, instance.setStaffSuppress(true); instance.setDeleted(true); instanceCollection.update(instance, v -> { - log.info("staffSuppress and discoverySuppress properties are set to true for instance {}", + log.info("staffSuppress, discoverySuppress and deleted properties are set to true for instance {}", instance.getId()); noContent(routingContext.response()); }, From 9e10ebebeeef03e33c605c69743f94d659ab19d0 Mon Sep 17 00:00:00 2001 From: Andrei Bordak Date: Wed, 12 Feb 2025 15:18:47 +0400 Subject: [PATCH 5/8] MODINVSTOR-1342: Update versions in ModuleDescriptor --- descriptors/ModuleDescriptor-template.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/descriptors/ModuleDescriptor-template.json b/descriptors/ModuleDescriptor-template.json index 757e0b278..e8da85893 100644 --- a/descriptors/ModuleDescriptor-template.json +++ b/descriptors/ModuleDescriptor-template.json @@ -4,7 +4,7 @@ "provides": [ { "id": "inventory", - "version": "14.0", + "version": "14.1", "handlers": [ { "methods": ["GET"], @@ -619,7 +619,7 @@ }, { "id": "instance-storage", - "version": "11.0" + "version": "11.1" }, { "id": "instance-storage-batch-sync", From 74f1084e1a55a07e6339043aa71e15fcb11f5cc5 Mon Sep 17 00:00:00 2001 From: Andrei Bordak Date: Fri, 14 Feb 2025 14:23:13 +0400 Subject: [PATCH 6/8] MODINVSTOR-1138: NEWS.md update --- NEWS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEWS.md b/NEWS.md index 0dc606bb0..0bc097ac3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -5,6 +5,7 @@ * Fix handling optimistic locking behavior for instance update when consuming Marc Bib update event [MODINV-1125](https://folio-org.atlassian.net/browse/MODINV-1125) * Replace usage of deprecated instance-storage-batch API [MODINV-1101](https://folio-org.atlassian.net/browse/MODINV-1101Л) * Update the snapshot status from PROCESSING_FINISHED to COMMITTED in the InstanceIngressEventHandler [MODINV-1161](https://folio-org.atlassian.net/browse/MODINV-1161) +* Adjust /mark-deleted endpoint behavior to set the "deleted" flag [MODINV-1138](https://folio-org.atlassian.net/browse/MODINV-1138) ## 21.0.0 2024-10-29 * Existing "035" field is not retained the original position in imported record [MODINV-1049](https://folio-org.atlassian.net/browse/MODINV-1049) From e56b8edb7d28ac1e017b61fa943131f9ef9ff425 Mon Sep 17 00:00:00 2001 From: Ruslan Lavrov <47384893+RuslanLavrov@users.noreply.github.com> Date: Mon, 17 Feb 2025 01:30:24 +0200 Subject: [PATCH 7/8] MODINV-1114 - Implement marc bib submatch (#804) * Added support of processing marc-bib multi match result by marc-bib sub-match profile * Updated test to verify deduplication logic of CreateInstanceEventHandler properly --- NEWS.md | 3 +- .../AbstractMarcMatchEventHandler.java | 299 +++++++++++++----- .../MarcBibliographicMatchEventHandler.java | 16 +- .../CreateInstanceEventHandlerTest.java | 17 +- ...arcBibliographicMatchEventHandlerTest.java | 169 +++++++++- 5 files changed, 398 insertions(+), 106 deletions(-) diff --git a/NEWS.md b/NEWS.md index a64ebfa8b..2a35a64fe 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,4 +1,4 @@ -## 21.1.0-SNAPSHOT 2024-xx-xx +## 21.1.0-SNAPSHOT 2025-xx-xx * Provide consistent handling with concurrency two or more Marc Bib Update events for the same bib record [MODINV-1100](https://folio-org.atlassian.net/browse/MODINV-1100) * Enable system user for data-import processes [MODINV-1115](https://folio-org.atlassian.net/browse/MODINV-1115) * Missing x-okapi-user-id header in communications with inventory-storage [MODINV-1134](https://folio-org.atlassian.net/browse/MODINV-1134) @@ -7,6 +7,7 @@ * Update the snapshot status from PROCESSING_FINISHED to COMMITTED in the InstanceIngressEventHandler [MODINV-1161](https://folio-org.atlassian.net/browse/MODINV-1161) * Add "deleted" field to Instance schema [MODINVSTOR-1342](https://folio-org.atlassian.net/browse/MODINVSTOR-1342) * Adjust /mark-deleted endpoint behavior to set the "deleted" flag [MODINV-1138](https://folio-org.atlassian.net/browse/MODINV-1138) +* Implement marc bib submatch [MODINV-1114](https://issues.folio.org/browse/MODINV-1114) ## 21.0.0 2024-10-29 * Existing "035" field is not retained the original position in imported record [MODINV-1049](https://folio-org.atlassian.net/browse/MODINV-1049) diff --git a/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java b/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java index f9ee4401f..79cf2050d 100644 --- a/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java +++ b/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java @@ -3,6 +3,7 @@ import io.vertx.core.Future; import io.vertx.core.http.HttpClient; import io.vertx.core.json.Json; +import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -30,6 +31,7 @@ import org.folio.rest.jaxrs.model.Filter; import org.folio.rest.jaxrs.model.ProfileSnapshotWrapper; import org.folio.rest.jaxrs.model.Qualifier; +import org.folio.rest.jaxrs.model.ReactToType; import org.folio.rest.jaxrs.model.Record; import org.folio.rest.jaxrs.model.RecordIdentifiersDto; import org.folio.rest.jaxrs.model.RecordMatchingDto; @@ -45,9 +47,11 @@ import java.util.stream.Stream; import static java.lang.String.format; +import static java.util.Objects.nonNull; +import static org.apache.commons.collections.CollectionUtils.isNotEmpty; import static org.apache.http.HttpStatus.SC_OK; -import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.PAYLOAD_USER_ID; import static org.folio.processing.value.Value.ValueType.MISSING; +import static org.folio.rest.jaxrs.model.Filter.ComparisonPartType; import static org.folio.rest.jaxrs.model.MatchExpression.DataValueType.VALUE_FROM_RECORD; import static org.folio.rest.jaxrs.model.ProfileType.MATCH_PROFILE; @@ -65,6 +69,10 @@ public abstract class AbstractMarcMatchEventHandler implements EventHandler { private static final String USER_ID_HEADER = "userId"; private static final int EXPECTED_MATCH_EXPRESSION_FIELDS_NUMBER = 4; private static final String DEFAULT_RECORDS_IDENTIFIERS_LIMIT = "5000"; + private static final String FIELD_999 = "999"; + private static final String INDICATOR_F = "f"; + private static final String SUBFIELD_I = "i"; + private static final String SUBFIELD_S = "s"; protected final ConsortiumService consortiumService; private final DataImportEventTypes matchedEventType; @@ -95,9 +103,8 @@ public CompletableFuture handle(DataImportEventPayload p LOG.warn("handle:: {}", PAYLOAD_HAS_NO_DATA_MESSAGE); return CompletableFuture.failedFuture(new EventProcessingException(PAYLOAD_HAS_NO_DATA_MESSAGE)); } - String userId = context.get(USER_ID_HEADER); payload.getEventsChain().add(payload.getEventType()); - payload.setAdditionalProperty(USER_ID_HEADER, userId); + payload.setAdditionalProperty(USER_ID_HEADER, context.get(USER_ID_HEADER)); String recordAsString = context.get(getMarcType()); MatchDetail matchDetail = retrieveMatchDetail(payload); @@ -114,16 +121,11 @@ public CompletableFuture handle(DataImportEventPayload p return CompletableFuture.completedFuture(payload); } - RecordMatchingDto recordMatchingDto = buildRecordsMatchingRequest(matchDetail, value); - return retrieveMarcRecords(recordMatchingDto, payload.getTenant(), userId, payload) - .compose(localMatchedRecords -> { - if (isMatchingOnCentralTenantRequired()) { - return matchCentralTenantIfNeededAndCombineWithLocalMatchedRecords(recordMatchingDto, payload, localMatchedRecords); - } - return Future.succeededFuture(localMatchedRecords.stream().toList()); - }) - .compose(recordList -> ensureRelatedEntities(recordList, payload).map(recordList)) - .compose(recordList -> processSucceededResult(recordList, payload)) + RecordMatchingDto recordMatchingDto = buildRecordsMatchingRequest(matchDetail, value, payload); + return createRecordsMatchingContext(payload) + .compose(recordsMatchingContext -> matchRecords(recordMatchingDto, recordsMatchingContext, payload)) + .compose(recordOptional -> ensureRelatedEntities(recordOptional, payload).map(recordOptional)) + .compose(recordOptional -> processSucceededResult(recordOptional, payload)) .onFailure(e -> LOG.warn("handle:: Failed to process event for MARC record matching, jobExecutionId: '{}'", payload.getJobExecutionId(), e)) .toCompletionStage().toCompletableFuture(); } catch (Exception e) { @@ -141,8 +143,10 @@ public CompletableFuture handle(DataImportEventPayload p protected abstract String getMultiMatchResultKey(); + protected abstract boolean canSubMatchProfileProcessMultiMatchResult(MatchProfile matchProfile); + @SuppressWarnings("squid:S1172") - protected Future ensureRelatedEntities(List records, DataImportEventPayload eventPayload) { + protected Future ensureRelatedEntities(Optional recordOptional, DataImportEventPayload eventPayload) { return Future.succeededFuture(); } @@ -184,7 +188,7 @@ private boolean isValidMatchDetail(MatchDetail matchDetail) { return false; } - private RecordMatchingDto buildRecordsMatchingRequest(MatchDetail matchDetail, Value value) { + private RecordMatchingDto buildRecordsMatchingRequest(MatchDetail matchDetail, Value value, DataImportEventPayload payload) { List matchDetailFields = matchDetail.getExistingMatchExpression().getFields(); String field = matchDetailFields.get(0).getValue(); String ind1 = matchDetailFields.get(1).getValue(); @@ -199,50 +203,125 @@ private RecordMatchingDto buildRecordsMatchingRequest(MatchDetail matchDetail, V Qualifier qualifier = matchDetail.getExistingMatchExpression().getQualifier(); Filter.Qualifier qualifierFilterType = null; - Filter.ComparisonPartType comparisonPartType = null; + ComparisonPartType comparisonPartType = null; String qualifierValue = null; if (qualifier != null) { qualifierValue = qualifier.getQualifierValue(); - qualifierFilterType = - qualifier.getQualifierType() != null ? - Filter.Qualifier.valueOf(qualifier.getQualifierType().toString()) : null; - comparisonPartType = - qualifier.getComparisonPart() != null ? - Filter.ComparisonPartType.valueOf(qualifier.getComparisonPart().toString()) : null; + qualifierFilterType = qualifier.getQualifierType() != null + ? Filter.Qualifier.valueOf(qualifier.getQualifierType().toString()) : null; + comparisonPartType = qualifier.getComparisonPart() != null + ? ComparisonPartType.valueOf(qualifier.getComparisonPart().toString()) : null; } - return new RecordMatchingDto() + Filter matchCriteriaFilter = new Filter() + .withValues(values) + .withField(field) + .withIndicator1(ind1) + .withIndicator2(ind2) + .withSubfield(subfield) + .withQualifier(qualifierFilterType) + .withQualifierValue(qualifierValue) + .withComparisonPartType(comparisonPartType); + + RecordMatchingDto recordMatchingDto = new RecordMatchingDto() .withRecordType(getMatchedRecordType()) - .withFilters(List.of(new Filter() - .withValues(values) - .withField(field) - .withIndicator1(ind1) - .withIndicator2(ind2) - .withSubfield(subfield) - .withQualifier(qualifierFilterType) - .withQualifierValue(qualifierValue) - .withComparisonPartType(comparisonPartType))) .withReturnTotalRecordsCount(true); + recordMatchingDto.getFilters().add(matchCriteriaFilter); + + buildFilterBasedOnPreviousMatchResult(payload).ifPresent(filter -> { + recordMatchingDto.setLogicalOperator(RecordMatchingDto.LogicalOperator.AND); + recordMatchingDto.getFilters().add(filter); + }); + + return recordMatchingDto; } - private Future> retrieveMarcRecords(RecordMatchingDto recordMatchingDto, String tenantId, String userId, - DataImportEventPayload payload) { - SourceStorageRecordsClient sourceStorageRecordsClient = - new SourceStorageRecordsClientWrapper(payload.getOkapiUrl(), tenantId, payload.getToken(), userId, httpClient); + private Optional buildFilterBasedOnPreviousMatchResult(DataImportEventPayload payload) { + if (containsMultiMatchResult(payload)) { + return Optional.of(buildFilterForMultiMatchResult(payload)); + } else if (StringUtils.isNotEmpty(payload.getContext().get(getMatchedMarcKey()))) { + return Optional.of(buildFilterBasedOnPreviouslyMatchedRecord(payload)); + } + return Optional.empty(); + } - return getAllMatchedRecordsIdentifiers(recordMatchingDto, payload, sourceStorageRecordsClient) - .compose(recordsIdentifiersCollection -> { - if (recordsIdentifiersCollection.getIdentifiers().size() > 1) { - populatePayloadWithExternalIdentifiers(recordsIdentifiersCollection, payload); - return Future.succeededFuture(Optional.empty()); - } else if (recordsIdentifiersCollection.getIdentifiers().size() == 1) { - return getRecordById(recordsIdentifiersCollection.getIdentifiers().get(0), sourceStorageRecordsClient, payload); - } - LOG.info("retrieveMarcRecords:: {}, jobExecutionId: '{}', tenantId: '{}'", - RECORDS_NOT_FOUND_MESSAGE, payload.getJobExecutionId(), tenantId); - return Future.succeededFuture(Optional.empty()); + private Filter buildFilterForMultiMatchResult(DataImportEventPayload payload) { + List ids = new JsonArray(payload.getContext().get(getMultiMatchResultKey())) + .stream() + .map(String.class::cast) + .toList(); + + payload.getContext().remove(getMultiMatchResultKey()); + return buildFilter(ids, FIELD_999, INDICATOR_F, INDICATOR_F, SUBFIELD_I); + } + + private Filter buildFilterBasedOnPreviouslyMatchedRecord(DataImportEventPayload payload) { + Record previouslyMatchedRecord = Json.decodeValue(payload.getContext().get(getMatchedMarcKey()), Record.class); + String matchedId = previouslyMatchedRecord.getMatchedId(); + return buildFilter(List.of(matchedId), FIELD_999, INDICATOR_F, INDICATOR_F, SUBFIELD_S); + } + + private Filter buildFilter(List values, String field, String ind1, String ind2, String subfield) { + return new Filter() + .withValues(values) + .withField(field) + .withIndicator1(ind1) + .withIndicator2(ind2) + .withSubfield(subfield); + } + + private Future createRecordsMatchingContext(DataImportEventPayload payload) { + String userId = payload.getContext().get(USER_ID_HEADER); + RecordsMatchingContext recordsMatchingContext = new RecordsMatchingContext(); + recordsMatchingContext.setLocalTenantRecordsClient(new SourceStorageRecordsClientWrapper( + payload.getOkapiUrl(), payload.getTenant(), payload.getToken(), userId, httpClient)); + + if (!isMatchingOnCentralTenantRequired()) { + return Future.succeededFuture(recordsMatchingContext); + } + + Context context = EventHandlingUtil.constructContext(payload.getTenant(), payload.getToken(), + payload.getOkapiUrl(), userId); + + return consortiumService.getConsortiumConfiguration(context).map(consortiumConfigurationOptional -> { + consortiumConfigurationOptional.ifPresent(consortiumConfiguration -> { + recordsMatchingContext.setCentralTenantId(consortiumConfiguration.getCentralTenantId()); + recordsMatchingContext.setCentralTenantRecordsClient(new SourceStorageRecordsClientWrapper( + payload.getOkapiUrl(), consortiumConfiguration.getCentralTenantId(), payload.getToken(), userId, httpClient)); }); + return recordsMatchingContext; + }); + } + + private Future> matchRecords(RecordMatchingDto recordMatchingDto, + RecordsMatchingContext recordsMatchingContext, + DataImportEventPayload payload) { + SourceStorageRecordsClient localTenantStorageRecordsClient = recordsMatchingContext.getLocalTenantRecordsClient(); + + Future localMatchingFuture = + getAllMatchedRecordsIdentifiers(recordMatchingDto, payload, localTenantStorageRecordsClient); + Future centralMatchingFuture = + getMatchedRecordsIdentifiersOnCentralTenantIfNeeded(recordMatchingDto, recordsMatchingContext, payload); + + return Future.all(localMatchingFuture, centralMatchingFuture).compose(v -> { + RecordsIdentifiersCollection localMatchingRes = localMatchingFuture.result(); + RecordsIdentifiersCollection centralMatchingRes = centralMatchingFuture.result(); + + if (localMatchingRes.getTotalRecords() == 1 && centralMatchingRes.getIdentifiers().isEmpty()) { + return getRecordById(localMatchingRes.getIdentifiers().get(0), localTenantStorageRecordsClient, payload).map(Optional::of); + } else if (localMatchingRes.getIdentifiers().isEmpty() && centralMatchingRes.getTotalRecords() == 1) { + payload.getContext().put(CENTRAL_TENANT_ID_KEY, recordsMatchingContext.getCentralTenantId()); + return getRecordById(centralMatchingRes.getIdentifiers().get(0), recordsMatchingContext.getCentralTenantRecordsClient(), payload).map(Optional::of); + } else if (localMatchingRes.getIdentifiers().isEmpty() && centralMatchingRes.getIdentifiers().isEmpty()) { + LOG.info("matchRecords:: {}, jobExecutionId: '{}', tenantId: '{}'", + RECORDS_NOT_FOUND_MESSAGE, payload.getJobExecutionId(), payload.getTenant()); + return Future.succeededFuture(Optional.empty()); + } else { + populatePayloadWithExternalIdentifiers(localMatchingRes, centralMatchingRes, payload); + return Future.succeededFuture(Optional.empty()); + } + }); } private Future getAllMatchedRecordsIdentifiers(RecordMatchingDto recordMatchingDto, @@ -289,13 +368,13 @@ private Future getMatchedRecordsIdentifiers(Record }); } - private Future> getRecordById(RecordIdentifiersDto recordIdentifiersDto, SourceStorageRecordsClient sourceStorageRecordsClient, - DataImportEventPayload payload) { + private Future getRecordById(RecordIdentifiersDto recordIdentifiersDto, SourceStorageRecordsClient sourceStorageRecordsClient, + DataImportEventPayload payload) { String recordId = recordIdentifiersDto.getRecordId(); return sourceStorageRecordsClient.getSourceStorageRecordsById(recordId) .compose(response -> { if (response.statusCode() == SC_OK) { - return Future.succeededFuture(Optional.of(response.bodyAsJson(Record.class))); + return Future.succeededFuture(response.bodyAsJson(Record.class)); } String msg = format("Failed to retrieve record by id: '%s', responseStatus: '%s', body: '%s', jobExecutionId: '%s', tenant: '%s'", recordId, response.statusCode(), response.bodyAsString(), payload.getJobExecutionId(), payload.getTenant()); @@ -310,37 +389,32 @@ private Future> getRecordById(RecordIdentifiersDto recordIdenti * These external identifiers represent multiple match result returned by this handler * and can be used and deleted during further matching processing by other match handlers. * - * @param recordsIdentifiersCollection identifiers collection to extract external identifiers - * @param payload event payload to populate + * @param localRecordsIdentifiersCollection local tenant identifiers collection to extract external identifiers + * @param centralRecordsIdentifiersCollection central tenant identifiers collection to extract external identifiers + * @param payload event payload to populate */ - private void populatePayloadWithExternalIdentifiers(RecordsIdentifiersCollection recordsIdentifiersCollection, + private void populatePayloadWithExternalIdentifiers(RecordsIdentifiersCollection localRecordsIdentifiersCollection, + RecordsIdentifiersCollection centralRecordsIdentifiersCollection, DataImportEventPayload payload) { - List externalEntityIds = recordsIdentifiersCollection.getIdentifiers() - .stream() + List externalEntityIds = Stream.concat( + localRecordsIdentifiersCollection.getIdentifiers().stream(), + centralRecordsIdentifiersCollection.getIdentifiers().stream()) .map(RecordIdentifiersDto::getExternalId) .toList(); payload.getContext().put(getMultiMatchResultKey(), Json.encode(externalEntityIds)); } - private Future> matchCentralTenantIfNeededAndCombineWithLocalMatchedRecords(RecordMatchingDto recordMatchingDto, DataImportEventPayload payload, - Optional localMatchedRecord) { - Context context = EventHandlingUtil.constructContext(payload.getTenant(), payload.getToken(), payload.getOkapiUrl(), - payload.getContext().get(PAYLOAD_USER_ID)); - return consortiumService.getConsortiumConfiguration(context) - .compose(consortiumConfigurationOptional -> { - if (consortiumConfigurationOptional.isPresent() && !consortiumConfigurationOptional.get().getCentralTenantId().equals(payload.getTenant())) { - LOG.debug("matchCentralTenantIfNeededAndCombineWithLocalMatchedRecords:: Matching on centralTenant with id: {}", - consortiumConfigurationOptional.get().getCentralTenantId()); - - return retrieveMarcRecords(recordMatchingDto, consortiumConfigurationOptional.get().getCentralTenantId(), context.getUserId(), payload) - .map(centralRecordOptional -> { - centralRecordOptional.ifPresent(r -> payload.getContext().put(CENTRAL_TENANT_ID_KEY, consortiumConfigurationOptional.get().getCentralTenantId())); - return Stream.concat(localMatchedRecord.stream(), centralRecordOptional.stream()).toList(); - }); - } - return Future.succeededFuture(localMatchedRecord.stream().toList()); - }); + private Future getMatchedRecordsIdentifiersOnCentralTenantIfNeeded(RecordMatchingDto recordMatchingDto, + RecordsMatchingContext recordsMatchingContext, + DataImportEventPayload payload) { + if (isMatchingOnCentralTenantRequired() && nonNull(recordsMatchingContext.getCentralTenantId()) + && !recordsMatchingContext.getCentralTenantId().equals(payload.getTenant())) { + LOG.debug("getMatchedRecordsIdentifiersOnCentralTenantIfNeeded:: Matching on centralTenant with id: {}", + recordsMatchingContext.getCentralTenantId()); + return getAllMatchedRecordsIdentifiers(recordMatchingDto, payload, recordsMatchingContext.getCentralTenantRecordsClient()); + } + return Future.succeededFuture(new RecordsIdentifiersCollection()); } @Override @@ -362,28 +436,20 @@ private boolean isEligibleMatchProfile(MatchProfile matchProfile) { * Prepares {@link DataImportEventPayload} for the further processing * based on the number of specified records in {@code records} list * - * @param records records retrieved during matching processing + * @param recordOptional matched record retrieved during matching processing * @param payload event payload to prepare * @return Future of {@link DataImportEventPayload} with result of matching */ - private Future processSucceededResult(List records, DataImportEventPayload payload) { - if (records.size() == 1) { + private Future processSucceededResult(Optional recordOptional, DataImportEventPayload payload) { + if (recordOptional.isPresent()) { payload.setEventType(matchedEventType.toString()); - payload.getContext().put(getMatchedMarcKey(), Json.encode(records.get(0))); + payload.getContext().put(getMatchedMarcKey(), Json.encode(recordOptional.get())); LOG.debug("processSucceededResult:: Matched 1 record for jobExecutionId: '{}', tenantId: '{}'", payload.getJobExecutionId(), payload.getTenant()); return Future.succeededFuture(payload); } - if (records.size() > 1) { - LOG.warn("processSucceededResult:: Matched multiple records, jobExecutionId: '{}', tenantId: '{}'", - payload.getJobExecutionId(), payload.getTenant()); - return Future.failedFuture(new MatchingException(FOUND_MULTIPLE_RECORDS_ERROR_MESSAGE)); - } - if (payload.getContext().containsKey(getMultiMatchResultKey())) { - LOG.info("processSucceededResult:: Multiple records were found which match criteria, jobExecutionId: '{}', tenantId: '{}'", - payload.getJobExecutionId(), payload.getTenant()); - payload.setEventType(matchedEventType.toString()); - return Future.succeededFuture(payload); + if (containsMultiMatchResult(payload)) { + return handlePayloadWithMultiMatchResult(payload); } LOG.info("processSucceededResult:: {}, jobExecutionId: '{}', tenantId: '{}'", RECORDS_NOT_FOUND_MESSAGE, payload.getJobExecutionId(), payload.getTenant()); @@ -394,4 +460,63 @@ private String getMatchedMarcKey() { return format(MATCH_RESULT_KEY_PREFIX, getMarcType()); } + private boolean containsMultiMatchResult(DataImportEventPayload payload) { + return payload.getContext().containsKey(getMultiMatchResultKey()); + } + + private Future handlePayloadWithMultiMatchResult(DataImportEventPayload payload) { + if (canNextProfileProcessMultiMatchResult(payload)) { + LOG.info("handlePayloadWithMultiMatchResult:: Multiple records were found which match criteria, jobExecutionId: '{}', tenantId: '{}'", + payload.getJobExecutionId(), payload.getTenant()); + payload.setEventType(matchedEventType.toString()); + return Future.succeededFuture(payload); + } + + LOG.warn("handlePayloadWithMultiMatchResult:: Matched multiple records, jobExecutionId: '{}', tenantId: '{}'", + payload.getJobExecutionId(), payload.getTenant()); + return Future.failedFuture(new MatchingException(FOUND_MULTIPLE_RECORDS_ERROR_MESSAGE)); + } + + private boolean canNextProfileProcessMultiMatchResult(DataImportEventPayload eventPayload) { + List childProfiles = eventPayload.getCurrentNode().getChildSnapshotWrappers(); + if (isNotEmpty(childProfiles) && ReactToType.MATCH.equals(childProfiles.get(0).getReactTo()) + && MATCH_PROFILE.equals(childProfiles.get(0).getContentType())) { + MatchProfile nextMatchProfile = JsonObject.mapFrom(childProfiles.get(0).getContent()).mapTo(MatchProfile.class); + return canSubMatchProfileProcessMultiMatchResult(nextMatchProfile); + } + return false; + } + + private static class RecordsMatchingContext { + + private SourceStorageRecordsClient localTenantRecordsClient; + private SourceStorageRecordsClient centralTenantRecordsClient; + private String centralTenantId; + + public SourceStorageRecordsClient getLocalTenantRecordsClient() { + return localTenantRecordsClient; + } + + public SourceStorageRecordsClient getCentralTenantRecordsClient() { + return centralTenantRecordsClient; + } + + public String getCentralTenantId() { + return centralTenantId; + } + + public void setLocalTenantRecordsClient(SourceStorageRecordsClient sourceStorageRecordsClient) { + this.localTenantRecordsClient = sourceStorageRecordsClient; + } + + public void setCentralTenantRecordsClient(SourceStorageRecordsClient sourceStorageRecordsClient) { + this.centralTenantRecordsClient = sourceStorageRecordsClient; + } + + public void setCentralTenantId(String centralTenantId) { + this.centralTenantId = centralTenantId; + } + + } + } diff --git a/src/main/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandler.java b/src/main/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandler.java index 07e8d4664..ec9a1f49c 100644 --- a/src/main/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandler.java +++ b/src/main/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandler.java @@ -6,6 +6,7 @@ import io.vertx.core.json.Json; import org.folio.DataImportEventPayload; import org.folio.HoldingsRecord; +import org.folio.MatchProfile; import org.folio.inventory.common.Context; import org.folio.inventory.common.api.request.PagingParameters; import org.folio.inventory.consortium.services.ConsortiumService; @@ -15,11 +16,13 @@ import org.folio.inventory.domain.HoldingsRecordCollection; import org.folio.inventory.domain.instances.InstanceCollection; import org.folio.inventory.storage.Storage; +import org.folio.rest.jaxrs.model.EntityType; import org.folio.rest.jaxrs.model.Record; import org.folio.rest.jaxrs.model.RecordMatchingDto; import java.io.UnsupportedEncodingException; import java.util.List; +import java.util.Optional; import static java.lang.String.format; import static org.apache.commons.lang3.StringUtils.isBlank; @@ -63,9 +66,16 @@ protected String getMultiMatchResultKey() { } @Override - protected Future ensureRelatedEntities(List records, DataImportEventPayload eventPayload) { - if (records.size() == 1) { - Record matchedRecord = records.get(0); + protected boolean canSubMatchProfileProcessMultiMatchResult(MatchProfile matchProfile) { + return matchProfile.getExistingRecordType() == EntityType.MARC_BIBLIOGRAPHIC + || matchProfile.getExistingRecordType() == EntityType.INSTANCE + || matchProfile.getExistingRecordType() == EntityType.HOLDINGS; + } + + @Override + protected Future ensureRelatedEntities(Optional recordOptional, DataImportEventPayload eventPayload) { + if (recordOptional.isPresent()) { + Record matchedRecord = recordOptional.get(); String instanceId = ParsedRecordUtil.getAdditionalSubfieldValue(matchedRecord.getParsedRecord(), AdditionalSubfields.I); String matchedRecordTenantId = getTenant(eventPayload); Context context = EventHandlingUtil.constructContext(matchedRecordTenantId, eventPayload.getToken(), eventPayload.getOkapiUrl(), diff --git a/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java b/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java index 85d8bd7d9..fc3c24e53 100644 --- a/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java +++ b/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java @@ -37,6 +37,7 @@ import org.folio.inventory.storage.Storage; import org.folio.inventory.support.http.client.OkapiHttpClient; import org.folio.inventory.support.http.client.Response; +import org.folio.kafka.exception.DuplicateEventException; import org.folio.processing.mapping.MappingManager; import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters; import org.folio.processing.mapping.mapper.reader.Reader; @@ -53,6 +54,7 @@ import org.folio.rest.jaxrs.model.ParsedRecord; import org.folio.rest.jaxrs.model.ProfileSnapshotWrapper; import org.folio.rest.jaxrs.model.Record; +import org.hamcrest.junit.internal.ThrowableCauseMatcher; import org.junit.Assert; import org.junit.Before; import org.junit.Rule; @@ -79,6 +81,7 @@ import java.util.function.Consumer; import static com.github.tomakehurst.wiremock.client.WireMock.get; +import static java.lang.String.format; import static java.util.concurrent.CompletableFuture.completedStage; import static org.apache.http.HttpStatus.SC_CREATED; import static org.folio.ActionProfile.FolioRecord.INSTANCE; @@ -90,16 +93,18 @@ import static org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil.PAYLOAD_USER_ID; import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_005; import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.dateTime005Formatter; -import static org.folio.inventory.dataimport.util.DataImportConstants.UNIQUE_ID_ERROR_MESSAGE; +import static org.folio.inventory.dataimport.util.DataImportConstants.ALREADY_EXISTS_ERROR_MSG; import static org.folio.rest.jaxrs.model.ProfileType.ACTION_PROFILE; import static org.folio.rest.jaxrs.model.ProfileType.JOB_PROFILE; import static org.folio.rest.jaxrs.model.ProfileType.MAPPING_PROFILE; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.argThat; @@ -945,9 +950,8 @@ public void shouldNotProcessEventWhenRecordToInstanceFutureFails() throws Execut future.get(5, TimeUnit.SECONDS); } - - @Test(expected = Exception.class) - public void shouldNotProcessEventEvenIfDuplicatedInventoryStorageErrorExists() throws InterruptedException, ExecutionException, TimeoutException { + @Test() + public void shouldNotProcessEventEvenIfDuplicatedInventoryStorageErrorExists() { Reader fakeReader = Mockito.mock(Reader.class); String instanceTypeId = "fe19bae4-da28-472b-be90-d442e2428ead"; @@ -965,7 +969,7 @@ public void shouldNotProcessEventEvenIfDuplicatedInventoryStorageErrorExists() t when(instanceIdStorageService.store(any(), any(), any())).thenReturn(Future.succeededFuture(recordToInstance)); doAnswer(invocationOnMock -> { Consumer failureHandler = invocationOnMock.getArgument(2); - failureHandler.accept(new Failure(UNIQUE_ID_ERROR_MESSAGE, 400)); + failureHandler.accept(new Failure(format(ALREADY_EXISTS_ERROR_MSG, instanceId), 400)); return null; }).when(instanceRecordCollection).add(any(), any(), any()); @@ -989,7 +993,8 @@ public void shouldNotProcessEventEvenIfDuplicatedInventoryStorageErrorExists() t CompletableFuture future = createInstanceEventHandler.handle(dataImportEventPayload); - future.get(5, TimeUnit.SECONDS); + ExecutionException actualException = assertThrows(ExecutionException.class, () -> future.get(5, TimeUnit.SECONDS)); + assertThat(actualException, ThrowableCauseMatcher.hasCause(instanceOf(DuplicateEventException.class))); } @Test(expected = Exception.class) diff --git a/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java b/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java index 08fcea17f..fe0d326b2 100644 --- a/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java +++ b/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java @@ -6,6 +6,7 @@ import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.github.tomakehurst.wiremock.matching.RegexPattern; import com.github.tomakehurst.wiremock.matching.UrlPathPattern; +import com.github.tomakehurst.wiremock.verification.LoggedRequest; import io.vertx.core.Future; import io.vertx.core.Vertx; import io.vertx.core.json.Json; @@ -36,6 +37,7 @@ import org.folio.rest.jaxrs.model.MatchExpression; import org.folio.rest.jaxrs.model.ProfileSnapshotWrapper; import org.folio.rest.jaxrs.model.RecordIdentifiersDto; +import org.folio.rest.jaxrs.model.RecordMatchingDto; import org.folio.rest.jaxrs.model.RecordsIdentifiersCollection; import org.hamcrest.Matchers; import org.junit.After; @@ -53,6 +55,7 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; +import java.util.stream.Stream; import static com.github.tomakehurst.wiremock.client.WireMock.equalTo; import static com.github.tomakehurst.wiremock.client.WireMock.get; @@ -74,6 +77,7 @@ import static org.folio.rest.jaxrs.model.EntityType.MARC_BIBLIOGRAPHIC; import static org.folio.rest.jaxrs.model.MatchExpression.DataValueType.VALUE_FROM_RECORD; import static org.folio.rest.jaxrs.model.ProfileType.MATCH_PROFILE; +import static org.folio.rest.jaxrs.model.ReactToType.MATCH; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.junit.MatcherAssert.assertThat; import static org.junit.Assert.assertFalse; @@ -148,6 +152,7 @@ public class MarcBibliographicMatchEventHandlerTest { @Before public void setUp() throws UnsupportedEncodingException { WireMock.reset(); + System.clearProperty(RECORDS_IDENTIFIERS_FETCH_LIMIT_PARAM); this.closeable = MockitoAnnotations.openMocks(this); Instance existingInstance = Instance.fromJson(new JsonObject().put("id", UUID.randomUUID().toString())); @@ -229,7 +234,7 @@ public void shouldNotMatchMarcBibIfNoRecordsMatchingCriteria(TestContext context } @Test - public void shouldPopulatePayloadWithInstancesIdsOfMatchedRecordsIfMultipleRecordsMatchCriteriaOnTenant(TestContext context) { + public void shouldPopulatePayloadWithInstancesIdsOfMatchedRecordsIfMultipleRecordsMatchCriteriaAndNextProfileIsEligibleForMultiMatchResult(TestContext context) { Async async = context.async(); List instanceIds = List.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); List recordsIdentifiers = instanceIds.stream() @@ -246,7 +251,7 @@ public void shouldPopulatePayloadWithInstancesIdsOfMatchedRecordsIfMultipleRecor when(consortiumService.getConsortiumConfiguration(any(Context.class))) .thenReturn(Future.succeededFuture(Optional.empty())); - DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + DataImportEventPayload eventPayload = createEventPayloadWithSubmatchProfile(TENANT_ID); CompletableFuture future = matchMarcBibEventHandler.handle(eventPayload); @@ -284,7 +289,7 @@ public void shouldRequestRecordsIdentifiersMultipleTimesIfMultipleRecordsMatchCr .withTotalRecords(totalRecordsIdentifiers))))); assertThat(totalRecordsIdentifiers, Matchers.greaterThan(recordsIdentifiersLimit)); - DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + DataImportEventPayload eventPayload = createEventPayloadWithSubmatchProfile(TENANT_ID); MarcBibliographicMatchEventHandler eventHandler = new MarcBibliographicMatchEventHandler(consortiumService, vertx.createHttpClient(), mockedStorage); @@ -305,6 +310,35 @@ public void shouldRequestRecordsIdentifiersMultipleTimesIfMultipleRecordsMatchCr })); } + @Test + public void shouldReturnFailedFutureIfMultipleRecordsMatchCriteriaAndNextProfileNotEligibleForMultiMatchResult(TestContext context) { + Async async = context.async(); + List recordsIdentifiers = Stream.iterate(0, i -> i < 2, i -> ++i) + .map(v -> new RecordIdentifiersDto() + .withRecordId(UUID.randomUUID().toString()) + .withExternalId(UUID.randomUUID().toString())) + .toList(); + + WireMock.stubFor(post(RECORDS_MATCHING_PATH) + .willReturn(WireMock.ok().withBody(Json.encode(new RecordsIdentifiersCollection() + .withIdentifiers(recordsIdentifiers) + .withTotalRecords(recordsIdentifiers.size()))))); + + when(consortiumService.getConsortiumConfiguration(any(Context.class))) + .thenReturn(Future.succeededFuture(Optional.empty())); + + DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + context.assertTrue(eventPayload.getCurrentNode().getChildSnapshotWrappers().isEmpty()); + + CompletableFuture future = matchMarcBibEventHandler.handle(eventPayload); + + future.whenComplete((payload, throwable) -> { + context.assertNotNull(throwable); + context.assertEquals(DI_SRS_MARC_BIB_RECORD_NOT_MATCHED.value(), eventPayload.getEventType()); + async.complete(); + }); + } + @Test public void shouldNotMatchMarcBibIfIncomingRecordHasNoSpecifiedIncomingField(TestContext context) { Async async = context.async(); @@ -454,25 +488,42 @@ public void shouldSearchRecordAtCentralTenantOnlyOnceIfCurrentTenantIsCentralTen } @Test - public void shouldReturnFailedFutureIfMatchedRecordAtLocalTenantAndMatchedAtCentralTenant(TestContext context) { + public void shouldPopulatePayloadWithInstancesIdsOfMatchedRecordsIfRecordMatchesCriteriaInLocalAndCentralTenant(TestContext context) { Async async = context.async(); + String localTenantInstanceId = UUID.randomUUID().toString(); + String centralTenantInstanceId = UUID.randomUUID().toString(); + WireMock.stubFor(post(RECORDS_MATCHING_PATH) + .withHeader(XOkapiHeaders.TENANT.toLowerCase(), equalTo(TENANT_ID)) + .willReturn(WireMock.ok().withBody(Json.encode(new RecordsIdentifiersCollection() + .withIdentifiers(List.of( + new RecordIdentifiersDto() + .withRecordId(expectedMatchedRecord.getId()) + .withExternalId(localTenantInstanceId))) + .withTotalRecords(1))))); + + WireMock.stubFor(post(RECORDS_MATCHING_PATH) + .withHeader(XOkapiHeaders.TENANT.toLowerCase(), equalTo(CENTRAL_TENANT_ID)) .willReturn(WireMock.ok().withBody(Json.encode(new RecordsIdentifiersCollection() .withIdentifiers(List.of(new RecordIdentifiersDto() .withRecordId(expectedMatchedRecord.getId()) - .withExternalId(UUID.randomUUID().toString()))) + .withExternalId(centralTenantInstanceId))) .withTotalRecords(1))))); WireMock.stubFor(get(new UrlPathPattern(new RegexPattern(SOURCE_STORAGE_RECORDS_PATH_REGEX), true)) .willReturn(WireMock.ok().withBody(Json.encodePrettily(expectedMatchedRecord)))); - DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + DataImportEventPayload eventPayload = createEventPayloadWithSubmatchProfile(TENANT_ID); CompletableFuture future = matchMarcBibEventHandler.handle(eventPayload); - future.whenComplete((res, throwable) -> { - context.assertNotNull(throwable); - context.assertEquals(DI_SRS_MARC_BIB_RECORD_NOT_MATCHED.value(), eventPayload.getEventType()); + future.whenComplete((payload, throwable) -> { + context.assertNull(throwable); + context.assertEquals(DI_SRS_MARC_BIB_RECORD_MATCHED.value(), eventPayload.getEventType()); + context.assertNotNull(payload.getContext().get(INSTANCES_IDS_KEY)); + assertThat(new JsonArray(payload.getContext().get(INSTANCES_IDS_KEY)), + containsInAnyOrder(localTenantInstanceId, centralTenantInstanceId)); + context.assertNull(payload.getContext().get(MATCHED_MARC_BIB_KEY)); async.complete(); }); } @@ -557,6 +608,7 @@ public void shouldLoadOnlyInstanceFromCentralTenantIfMatchedRecordAtCentralTenan .withTotalRecords(1))))); WireMock.stubFor(get(new UrlPathPattern(new RegexPattern(SOURCE_STORAGE_RECORDS_PATH_REGEX), true)) + .withHeader(XOkapiHeaders.TENANT.toLowerCase(), equalTo(CENTRAL_TENANT_ID)) .willReturn(WireMock.ok().withBody(Json.encodePrettily(expectedMatchedRecord)))); DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); @@ -791,6 +843,86 @@ public void shouldNotSetHoldingIfMultipleHoldingsWereFoundForMatchedRecord(TestC }); } + @Test + public void shouldProcessPayloadContainingInstanceIdsRepresentingMultiMatchResultAndMatchMarcBib(TestContext context) { + Async async = context.async(); + when(consortiumService.getConsortiumConfiguration(any(Context.class))) + .thenReturn(Future.succeededFuture(Optional.empty())); + + JsonArray instancesIds = JsonArray.of(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + + WireMock.stubFor(post(RECORDS_MATCHING_PATH).willReturn(WireMock.ok() + .withBody(Json.encode(new RecordsIdentifiersCollection() + .withIdentifiers(List.of(new RecordIdentifiersDto() + .withRecordId(expectedMatchedRecord.getId()) + .withExternalId(UUID.randomUUID().toString()))) + .withTotalRecords(1))))); + + WireMock.stubFor(get(new UrlPathPattern(new RegexPattern(SOURCE_STORAGE_RECORDS_PATH_REGEX), true)) + .willReturn(WireMock.ok().withBody(Json.encodePrettily(expectedMatchedRecord)))); + + DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + eventPayload.getContext().put(INSTANCES_IDS_KEY, instancesIds.encode()); + + CompletableFuture future = matchMarcBibEventHandler.handle(eventPayload); + + future.whenComplete((payload, throwable) -> context.verify(v -> { + context.assertNull(throwable); + context.assertEquals(1, eventPayload.getEventsChain().size()); + context.assertEquals(DI_SRS_MARC_BIB_RECORD_MATCHED.value(), eventPayload.getEventType()); + context.assertNull(payload.getContext().get(INSTANCES_IDS_KEY)); + context.assertNotNull(payload.getContext().get(MATCHED_MARC_BIB_KEY)); + Record actualRecord = Json.decodeValue(payload.getContext().get(MATCHED_MARC_BIB_KEY), Record.class); + context.assertEquals(expectedMatchedRecord.getId(), actualRecord.getId()); + context.assertNotNull(payload.getContext().get(INSTANCE.value())); + WireMock.verify(1, postRequestedFor(urlEqualTo(RECORDS_MATCHING_PATH))); + List requests = WireMock.findAll(postRequestedFor(urlEqualTo(RECORDS_MATCHING_PATH))); + RecordMatchingDto matchingDto = Json.decodeValue(requests.get(0).getBodyAsString(), RecordMatchingDto.class); + context.assertEquals(2, matchingDto.getFilters().size()); + assertThat(matchingDto.getFilters().get(1).getValues(), containsInAnyOrder(instancesIds.getList().toArray())); + async.complete(); + })); + } + + @Test + public void shouldMatchRecordAndTakeIntoAccountPreviouslyMatchedRecord(TestContext context) { + Async async = context.async(); + Record previouslyMatchedRecord = new Record().withMatchedId(UUID.randomUUID().toString()); + when(consortiumService.getConsortiumConfiguration(any(Context.class))) + .thenReturn(Future.succeededFuture(Optional.empty())); + + WireMock.stubFor(post(RECORDS_MATCHING_PATH).willReturn(WireMock.ok() + .withBody(Json.encode(new RecordsIdentifiersCollection() + .withIdentifiers(List.of(new RecordIdentifiersDto() + .withRecordId(expectedMatchedRecord.getId()) + .withExternalId(UUID.randomUUID().toString()))) + .withTotalRecords(1))))); + + WireMock.stubFor(get(new UrlPathPattern(new RegexPattern(SOURCE_STORAGE_RECORDS_PATH_REGEX), true)) + .willReturn(WireMock.ok().withBody(Json.encodePrettily(expectedMatchedRecord)))); + + DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + eventPayload.getContext().put(MATCHED_MARC_BIB_KEY, Json.encode(previouslyMatchedRecord)); + + CompletableFuture future = matchMarcBibEventHandler.handle(eventPayload); + + future.whenComplete((payload, throwable) -> context.verify(v -> { + context.assertNull(throwable); + context.assertEquals(1, eventPayload.getEventsChain().size()); + context.assertEquals(DI_SRS_MARC_BIB_RECORD_MATCHED.value(), eventPayload.getEventType()); + context.assertNotNull(payload.getContext().get(MATCHED_MARC_BIB_KEY)); + Record actualRecord = Json.decodeValue(payload.getContext().get(MATCHED_MARC_BIB_KEY), Record.class); + context.assertEquals(expectedMatchedRecord.getId(), actualRecord.getId()); + context.assertNotNull(payload.getContext().get(INSTANCE.value())); + WireMock.verify(1, postRequestedFor(urlEqualTo(RECORDS_MATCHING_PATH))); + List requests = WireMock.findAll(postRequestedFor(urlEqualTo(RECORDS_MATCHING_PATH))); + RecordMatchingDto matchingRequest = Json.decodeValue(requests.get(0).getBodyAsString(), RecordMatchingDto.class); + context.assertEquals(2, matchingRequest.getFilters().size()); + context.assertTrue(matchingRequest.getFilters().get(1).getValues().contains(previouslyMatchedRecord.getMatchedId())); + async.complete(); + })); + } + @Test public void shouldReturnTrueIfHandlerIsEligibleForEventPayload() { DataImportEventPayload eventPayload = new DataImportEventPayload() @@ -817,6 +949,21 @@ public void shouldReturnFalseIfHandlerIsNotEligibleForPayload() { } private DataImportEventPayload createEventPayload(String tenantId) { + return createEventPayload(tenantId, null); + } + + private DataImportEventPayload createEventPayloadWithSubmatchProfile(String tenantId) { + ProfileSnapshotWrapper subMatchProfileWrapper = new ProfileSnapshotWrapper() + .withProfileId(matchProfile.getId()) + .withContentType(MATCH_PROFILE) + .withOrder(0) + .withReactTo(MATCH) + .withContent(JsonObject.mapFrom(matchProfile).getMap()); + + return createEventPayload(tenantId, subMatchProfileWrapper); + } + + private DataImportEventPayload createEventPayload(String tenantId, ProfileSnapshotWrapper nextProfileWrapper) { Record record = new Record() .withParsedRecord(new ParsedRecord().withContent(PARSED_CONTENT)); @@ -825,6 +972,10 @@ private DataImportEventPayload createEventPayload(String tenantId) { .withContentType(MATCH_PROFILE) .withContent(JsonObject.mapFrom(matchProfile).getMap()); + if (nextProfileWrapper != null) { + matchProfileWrapper.getChildSnapshotWrappers().add(nextProfileWrapper); + } + return new DataImportEventPayload() .withEventType(DI_INCOMING_MARC_BIB_RECORD_PARSED.value()) .withJobExecutionId(UUID.randomUUID().toString()) From abe01d1836615d65ff4b44e83de4e5312543b9cb Mon Sep 17 00:00:00 2001 From: Ruslan Lavrov <47384893+RuslanLavrov@users.noreply.github.com> Date: Fri, 21 Feb 2025 14:06:06 +0200 Subject: [PATCH 8/8] FAT-18739 - Fix check for case if no records matching criteria on non-consortium tenant (#816) --- .../AbstractMarcMatchEventHandler.java | 2 +- ...arcBibliographicMatchEventHandlerTest.java | 28 ++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java b/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java index 79cf2050d..1282e0ac8 100644 --- a/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java +++ b/src/main/java/org/folio/inventory/dataimport/handlers/matching/AbstractMarcMatchEventHandler.java @@ -414,7 +414,7 @@ private Future getMatchedRecordsIdentifiersOnCentr recordsMatchingContext.getCentralTenantId()); return getAllMatchedRecordsIdentifiers(recordMatchingDto, payload, recordsMatchingContext.getCentralTenantRecordsClient()); } - return Future.succeededFuture(new RecordsIdentifiersCollection()); + return Future.succeededFuture(new RecordsIdentifiersCollection().withTotalRecords(0)); } @Override diff --git a/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java b/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java index fe0d326b2..e918de5ff 100644 --- a/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java +++ b/src/test/java/org/folio/inventory/dataimport/handlers/matching/MarcBibliographicMatchEventHandlerTest.java @@ -211,7 +211,7 @@ public void shouldMatchMarcBibAtNonConsortiumTenant(TestContext context) { } @Test - public void shouldNotMatchMarcBibIfNoRecordsMatchingCriteria(TestContext context) { + public void shouldNotMatchMarcBibIfNoRecordsMatchingCriteriaAtLocalAndCentralTenants(TestContext context) { Async async = context.async(); WireMock.stubFor(post(RECORDS_MATCHING_PATH).willReturn(WireMock.ok() .withBody(Json.encode(new RecordsIdentifiersCollection() @@ -233,6 +233,32 @@ public void shouldNotMatchMarcBibIfNoRecordsMatchingCriteria(TestContext context }); } + @Test + public void shouldNotMatchMarcBibIfNoRecordsMatchingCriteriaAtNonConsortiumTenant(TestContext context) { + Async async = context.async(); + when(consortiumService.getConsortiumConfiguration(any(Context.class))) + .thenReturn(Future.succeededFuture(Optional.empty())); + + WireMock.stubFor(post(RECORDS_MATCHING_PATH).willReturn(WireMock.ok() + .withBody(Json.encode(new RecordsIdentifiersCollection() + .withIdentifiers(List.of()) + .withTotalRecords(0))))); + + DataImportEventPayload eventPayload = createEventPayload(TENANT_ID); + + CompletableFuture future = matchMarcBibEventHandler.handle(eventPayload); + + future.whenComplete((payload, throwable) -> { + context.assertNull(throwable); + context.assertEquals(DI_SRS_MARC_BIB_RECORD_NOT_MATCHED.value(), payload.getEventType()); + context.assertNull(payload.getContext().get(MATCHED_MARC_BIB_KEY)); + context.assertEquals(1, payload.getEventsChain().size()); + context.assertNull(payload.getContext().get(INSTANCE.value())); + context.assertNull(payload.getContext().get(HOLDINGS.value())); + async.complete(); + }); + } + @Test public void shouldPopulatePayloadWithInstancesIdsOfMatchedRecordsIfMultipleRecordsMatchCriteriaAndNextProfileIsEligibleForMultiMatchResult(TestContext context) { Async async = context.async();