Merge pull request #4 from cardano-foundation/netsuite-filter-close-t…

…o-source feat: netsuite client filtering transactions at the source.
cardano-foundation · Jul 25, 2024 · 546f897 · 546f897
2 parents 9b79c95 + 77ff25e
commit 546f897
Show file tree

Hide file tree

Showing 14 changed files with 343 additions and 67 deletions.
diff --git a/...ofoundation/lob/app/accounting_reporting_core/domain/core/SystemExtractionParameters.java b/...ofoundation/lob/app/accounting_reporting_core/domain/core/SystemExtractionParameters.java
@@ -7,7 +7,7 @@
 import java.time.YearMonth;
 
 @Getter
-@Builder
+@Builder(toBuilder = true)
 @NoArgsConstructor
 @AllArgsConstructor
 @ToString

diff --git a/...anofoundation/lob/app/accounting_reporting_core/domain/core/UserExtractionParameters.java b/...anofoundation/lob/app/accounting_reporting_core/domain/core/UserExtractionParameters.java
@@ -8,7 +8,7 @@
 import java.util.List;
 
 @Getter
-@Builder
+@Builder(toBuilder = true)
 @NoArgsConstructor
 @AllArgsConstructor
 @ToString

diff --git a/...foundation/lob/app/accounting_reporting_core/domain/event/TransactionBatchChunkEvent.java b/...foundation/lob/app/accounting_reporting_core/domain/event/TransactionBatchChunkEvent.java
@@ -3,6 +3,7 @@
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;
+import lombok.NoArgsConstructor;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.SystemExtractionParameters;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Transaction;
 import org.jmolecules.event.annotation.DomainEvent;
@@ -14,6 +15,7 @@
 @Builder
 @Getter
 @DomainEvent
+@NoArgsConstructor
 public class TransactionBatchChunkEvent {
 
     private String batchId;

diff --git a/...dation/lob/app/accounting_reporting_core/resource/ExperimentalAccountingCoreResource.java b/...dation/lob/app/accounting_reporting_core/resource/ExperimentalAccountingCoreResource.java
@@ -1,6 +1,5 @@
 package org.cardanofoundation.lob.app.accounting_reporting_core.resource;
 
-
 import jakarta.annotation.PostConstruct;
 import jakarta.validation.Valid;
 import lombok.RequiredArgsConstructor;

diff --git a/...app/accounting_reporting_core/service/business_rules/items/ProjectConversionTaskItem.java b/...app/accounting_reporting_core/service/business_rules/items/ProjectConversionTaskItem.java
@@ -1,6 +1,7 @@
 package org.cardanofoundation.lob.app.accounting_reporting_core.service.business_rules.items;
 
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.TransactionEntity;
 import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.Violation;
@@ -13,6 +14,7 @@
 import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.ViolationCode.PROJECT_DATA_NOT_FOUND;
 
 @RequiredArgsConstructor
+@Slf4j
 public class ProjectConversionTaskItem implements PipelineTaskItem {
 
     private final OrganisationPublicApiIF organisationPublicApi;
@@ -31,8 +33,12 @@ public void run(TransactionEntity tx) {
             val organisationId = tx.getOrganisation().getId();
             val customerCode = projectM.orElseThrow().getCustomerCode();
 
+            log.info("Looking for project mapping for organisationId:{}, customerCode:{}", organisationId, customerCode);
+
             val projectMappingM = organisationPublicApi.findProject(organisationId, customerCode);
 
+            log.info("Project mapping found: {}", projectMappingM);
+
             if (projectMappingM.isEmpty()) {
                 val v = Violation.builder()
                         .code(PROJECT_DATA_NOT_FOUND)

diff --git a/...ava/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/client/NetSuiteClient.java b/...ava/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/client/NetSuiteClient.java
@@ -3,18 +3,22 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import io.vavr.control.Either;
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 import org.scribe.model.OAuthRequest;
 import org.scribe.model.Response;
 import org.scribe.model.Token;
 import org.scribe.oauth.OAuthService;
+import org.springframework.web.util.UriComponentsBuilder;
 import org.zalando.problem.Problem;
 import org.zalando.problem.Status;
 
+import java.time.LocalDate;
 import java.util.Optional;
 
+import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
 import static org.scribe.model.Verb.GET;
 
 @Slf4j
@@ -25,20 +29,17 @@ public class NetSuiteClient {
 
     private final ObjectMapper objectMapper;
 
-    private final String url;
+    @Getter
+    private final String baseUrl;
 
     private final String realm;
 
     private final String token;
 
     private final String tokenSecret;
 
-    public final String netsuiteUrl() {
-        return url;
-    }
-
-    public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines() {
-        val response = callForTransactionLinesData();
+    public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(LocalDate extractionFrom, LocalDate extractionTo) {
+        val response = callForTransactionLinesData(extractionFrom, extractionTo);
 
         if (response.isSuccessful()) {
             log.info("Netsuite response success...customerCode:{}, message:{}", response.getCode(), response.getMessage());
@@ -49,6 +50,7 @@ public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(
                 if (bodyJsonTree.has("error")) {
                     val error = bodyJsonTree.get("error").asInt();
                     val text = bodyJsonTree.get("text").asText();
+                    log.error("Error api error:{}, message:{}", error, text);
 
                     if (error == 105) {
                         log.warn("No data to read from NetSuite API...");
@@ -58,7 +60,7 @@ public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(
 
                     return Either.left(Problem.builder()
                             .withStatus(Status.valueOf(response.getCode()))
-                            .withTitle("NetSuite API error")
+                            .withTitle("NETSUITE_API_ERROR")
                             .withDetail(String.format("Error customerCode: %d, message: %s", error, text))
                             .build());
                 }
@@ -69,22 +71,29 @@ public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(
 
                 return Either.left(Problem.builder()
                         .withStatus(Status.valueOf(response.getCode()))
-                        .withTitle("NetSuite API error")
+                        .withTitle("NETSUITE_API_ERROR")
                         .withDetail(e.getMessage())
                         .build());
             }
         }
 
         return Either.left(Problem.builder()
                 .withStatus(Status.valueOf(response.getCode()))
-                .withTitle("NetSuite API error")
+                .withTitle("NETSUITE_API_ERROR")
                 .withDetail(response.getBody())
                 .build());
     }
 
-    private Response callForTransactionLinesData() {
+    private Response callForTransactionLinesData(LocalDate from, LocalDate to) {
         log.info("Retrieving data from NetSuite...");
-        log.info("url: {}", url);
+        log.info("base url: {}", baseUrl);
+
+        val builder = UriComponentsBuilder.fromHttpUrl(baseUrl)
+                .queryParam("trandate:within", isoFormatDates(from, to));
+
+        val url = builder.toUriString();
+
+        log.info("call url: {}", url);
 
         val request = new OAuthRequest(GET, url);
         request.setRealm(realm);
@@ -95,4 +104,8 @@ private Response callForTransactionLinesData() {
         return request.send();
     }
 
+    private String isoFormatDates(LocalDate from, LocalDate to) {
+        return String.format("%s,%s", ISO_LOCAL_DATE.format(from), ISO_LOCAL_DATE.format(to));
+    }
+
 }
diff --git a/...ndation/lob/app/netsuite_altavia_erp_adapter/domain/core/TransactionDataSearchResult.java b/...ndation/lob/app/netsuite_altavia_erp_adapter/domain/core/TransactionDataSearchResult.java
@@ -1,7 +1,10 @@
 package org.cardanofoundation.lob.app.netsuite_altavia_erp_adapter.domain.core;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+
 import java.util.List;
 
+@JsonIgnoreProperties(ignoreUnknown = true)
 public record TransactionDataSearchResult(List<TxLine> lines,
                                           boolean more) {
 }
diff --git a/...n/java/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/domain/core/TxLine.java b/...n/java/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/domain/core/TxLine.java
@@ -3,107 +3,118 @@
 import com.fasterxml.jackson.annotation.JsonFormat;
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
-import com.fasterxml.jackson.databind.annotation.JsonSerialize;
-import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer;
-import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateSerializer;
 import jakarta.validation.constraints.*;
 
 import javax.annotation.Nullable;
 import java.math.BigDecimal;
 import java.time.LocalDate;
+import java.time.LocalDateTime;
+
+import static com.fasterxml.jackson.annotation.JsonFormat.Shape.STRING;
+import static org.cardanofoundation.lob.app.netsuite_altavia_erp_adapter.util.Dates.ISO_8601_FORMAT_QUASI;
 
 // https://docs.google.com/spreadsheets/d/1iGo1t2bLuWSONOYo6kG9uXSzt7laCrM8gluKkx8tmn0/edit#gid=501685631
 @JsonIgnoreProperties(ignoreUnknown = true)
 public record TxLine(
 
-        @JsonProperty("Line ID")
+        @JsonProperty("line")
         @PositiveOrZero
+        @NotNull
         Integer lineID,
 
-        @JsonProperty("Subsidiary (no hierarchy)")
+        @JsonProperty("subsidiarynohierarchy")
         @PositiveOrZero
+        @NotNull
         Long subsidiary,
 
-        @JsonProperty("Type")
+        @JsonProperty("type")
+        @NotNull
         String type,
 
-        @JsonProperty("Date")
-        @JsonDeserialize(using = LocalDateDeserializer.class)
-        @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy")
+        @JsonProperty("trandate")
+        @JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
+        @NotNull
         LocalDate date,
 
-        @JsonProperty("ID")
-        String id,
+        @JsonProperty("vendor.entityid")
+        @Nullable
+        String counterPartyId,
 
-        @JsonProperty("Company Name")
-        String companyName,
+        @JsonProperty("vendor.companyname")
+        String counterPartyName,
 
-        @JsonProperty("Tax Item")
+        @JsonProperty("taxcode")
         String taxItem,
 
-        @JsonProperty("Cost Center (no hierarchy)")
         @Nullable
+        @JsonProperty("departmentnohierarchy")
         String costCenter,
 
-        @JsonProperty("Transaction Number")
+        @JsonProperty("transactionnumber")
         @NotBlank
         String transactionNumber,
 
-        @JsonProperty("Document Number")
+        @JsonProperty("tranid")
         @Nullable
         String documentNumber,
 
-        @JsonProperty("Number")
+        @JsonProperty("account.number")
         String number,
 
-        @JsonProperty("Name")
+        @JsonProperty("account.name")
         @Nullable
         String name,
 
-        @JsonProperty("Project (no hierarchy)")
+        @JsonProperty("classnohierarchy")
         String project,
 
-        @JsonProperty("Account (Main)")
+        @JsonProperty("accountmain")
         String accountMain,
 
-        @JsonProperty("Memo (Main)")
-        String memo,
-
-        @JsonProperty("Currency")
+        @JsonProperty("currency")
         @NotNull
         String currency,
 
-        @JsonProperty("Symbol")
+        @JsonProperty("Currency.symbol")
         @NotNull
         String currencySymbol,
 
         // this is accounting period date
-        @JsonProperty("End Date")
-        @JsonSerialize(using = LocalDateSerializer.class)
-        @JsonDeserialize(using = LocalDateDeserializer.class)
-        @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy")
+        @JsonProperty("enddate")
+        @JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
         LocalDate endDate,
 
+        @JsonProperty("datecreated")
+        @JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
+        @NotNull
+        LocalDateTime dateCreated,
+
+        @JsonProperty("lastmodifieddate")
+        @JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
         @NotNull
-        @JsonProperty("Exchange Rate")
+        LocalDateTime lastModifiedDate,
+
+        @NotNull
+        @JsonProperty("exchangerate")
         @Positive
         BigDecimal exchangeRate,
 
         @DecimalMin(value = "0.0")
         @Nullable
-        @JsonProperty("Amount (Debit) (Foreign Currency)") BigDecimal amountDebitForeignCurrency,
+        @JsonProperty("debitfxamount") BigDecimal amountDebitForeignCurrency,
 
         @DecimalMin(value = "0.0")
         @Nullable
-        @JsonProperty("Amount (Credit) (Foreign Currency)") BigDecimal amountCreditForeignCurrency,
+        @JsonProperty("creditfxamount") BigDecimal amountCreditForeignCurrency,
 
         @DecimalMin(value = "0.0")
         @Nullable
-        @JsonProperty("Amount (Debit)") BigDecimal amountDebit,
+        @JsonProperty("debitamount") BigDecimal amountDebit,
 
         @DecimalMin(value = "0.0")
         @Nullable
-        @JsonProperty("Amount (Credit)") BigDecimal amountCredit
+        @JsonProperty("creditamount") BigDecimal amountCredit
+
+) {
 
-) { }
+}
diff --git a/...on/lob/app/netsuite_altavia_erp_adapter/service/ExtractionParametersFilteringService.java b/...on/lob/app/netsuite_altavia_erp_adapter/service/ExtractionParametersFilteringService.java
@@ -18,33 +18,36 @@ public Set<Transaction> applyExtractionParameters(UserExtractionParameters userE
                                                       SystemExtractionParameters systemExtractionParameters,
                                                       Set<Transaction> txs) {
         return txs.stream()
+                // this is for sanity reasons since actually we should be filtering out the transactions in the adapter layer and more specifically while making an HTTP call via NetSuiteClient
                 .filter(tx -> {
                     val txAccountingPeriod = tx.getAccountingPeriod();
 
-                    return txAccountingPeriod.equals(systemExtractionParameters.getAccountPeriodFrom()) || txAccountingPeriod.isAfter(systemExtractionParameters.getAccountPeriodFrom())
+                    return (txAccountingPeriod.equals(systemExtractionParameters.getAccountPeriodFrom()) || txAccountingPeriod.isAfter(systemExtractionParameters.getAccountPeriodFrom()))
                             &&
                             (txAccountingPeriod.equals(systemExtractionParameters.getAccountPeriodTo()) || txAccountingPeriod.isBefore(systemExtractionParameters.getAccountPeriodTo()));
                 })
                 .filter(tx -> userExtractionParameters.getOrganisationId().equals(tx.getOrganisation().getId()))
+                // this is for sanity reasons since actually we should be filtering out the transactions in the adapter layer and more specifically while making an HTTP call via NetSuiteClient
                 .filter(tx -> {
                     val from = userExtractionParameters.getFrom();
 
-                    return tx.getEntryDate().isEqual(from) ||  tx.getEntryDate().isAfter(from);
+                    return tx.getEntryDate().isEqual(from) || tx.getEntryDate().isAfter(from);
                 })
+                // this is for sanity reasons since actually we should be filtering out the transactions in the adapter layer and more specifically while making an HTTP call via NetSuiteClient
                 .filter(tx -> {
-                    val to = userExtractionParameters.getFrom();
+                    val to = userExtractionParameters.getTo();
 
-                    return tx.getEntryDate().isEqual(to) ||  tx.getEntryDate().isAfter(to);
+                    return tx.getEntryDate().isEqual(to) || tx.getEntryDate().isBefore(to);
                 })
                 .filter(tx -> {
                     val txTypes = userExtractionParameters.getTransactionTypes();
 
                     return txTypes.isEmpty() || txTypes.contains(tx.getTransactionType());
                 })
                 .filter(tx -> {
-                    val transactionNumber = userExtractionParameters.getTransactionNumbers();
+                    val transactionNumbers = userExtractionParameters.getTransactionNumbers();
 
-                    return transactionNumber.isEmpty() || transactionNumber.contains(tx.getInternalTransactionNumber());
+                    return transactionNumbers.isEmpty() || transactionNumbers.contains(tx.getInternalTransactionNumber());
                 })
                 .collect(Collectors.toSet());
     }