Skip to content

Commit 546f897

Browse files
matiwinnetoumatiwinnetou
authored
Merge pull request #4 from cardano-foundation/netsuite-filter-close-to-source
feat: netsuite client filtering transactions at the source.
2 parents 9b79c95 + 77ff25e commit 546f897

File tree

14 files changed

+343
-67
lines changed

14 files changed

+343
-67
lines changed

‎accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/domain/core/SystemExtractionParameters.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
import java.time.YearMonth;
88

99
@Getter
10-
@Builder
10+
@Builder(toBuilder = true)
1111
@NoArgsConstructor
1212
@AllArgsConstructor
1313
@ToString

‎accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/domain/core/UserExtractionParameters.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import java.util.List;
99

1010
@Getter
11-
@Builder
11+
@Builder(toBuilder = true)
1212
@NoArgsConstructor
1313
@AllArgsConstructor
1414
@ToString

‎accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/domain/event/TransactionBatchChunkEvent.java

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import lombok.AllArgsConstructor;
44
import lombok.Builder;
55
import lombok.Getter;
6+
import lombok.NoArgsConstructor;
67
import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.SystemExtractionParameters;
78
import org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.Transaction;
89
import org.jmolecules.event.annotation.DomainEvent;
@@ -14,6 +15,7 @@
1415
@Builder
1516
@Getter
1617
@DomainEvent
18+
@NoArgsConstructor
1719
public class TransactionBatchChunkEvent {
1820

1921
private String batchId;

‎accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/resource/ExperimentalAccountingCoreResource.java

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
package org.cardanofoundation.lob.app.accounting_reporting_core.resource;
22

3-
43
import jakarta.annotation.PostConstruct;
54
import jakarta.validation.Valid;
65
import lombok.RequiredArgsConstructor;

‎accounting_reporting_core/src/main/java/org/cardanofoundation/lob/app/accounting_reporting_core/service/business_rules/items/ProjectConversionTaskItem.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package org.cardanofoundation.lob.app.accounting_reporting_core.service.business_rules.items;
22

33
import lombok.RequiredArgsConstructor;
4+
import lombok.extern.slf4j.Slf4j;
45
import lombok.val;
56
import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.TransactionEntity;
67
import org.cardanofoundation.lob.app.accounting_reporting_core.domain.entity.Violation;
@@ -13,6 +14,7 @@
1314
import static org.cardanofoundation.lob.app.accounting_reporting_core.domain.core.ViolationCode.PROJECT_DATA_NOT_FOUND;
1415

1516
@RequiredArgsConstructor
17+
@Slf4j
1618
public class ProjectConversionTaskItem implements PipelineTaskItem {
1719

1820
private final OrganisationPublicApiIF organisationPublicApi;
@@ -31,8 +33,12 @@ public void run(TransactionEntity tx) {
3133
val organisationId = tx.getOrganisation().getId();
3234
val customerCode = projectM.orElseThrow().getCustomerCode();
3335

36+
log.info("Looking for project mapping for organisationId:{}, customerCode:{}", organisationId, customerCode);
37+
3438
val projectMappingM = organisationPublicApi.findProject(organisationId, customerCode);
3539

40+
log.info("Project mapping found: {}", projectMappingM);
41+
3642
if (projectMappingM.isEmpty()) {
3743
val v = Violation.builder()
3844
.code(PROJECT_DATA_NOT_FOUND)

‎netsuite_altavia_erp_adapter/src/main/java/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/client/NetSuiteClient.java

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,22 @@
33
import com.fasterxml.jackson.core.JsonProcessingException;
44
import com.fasterxml.jackson.databind.ObjectMapper;
55
import io.vavr.control.Either;
6+
import lombok.Getter;
67
import lombok.RequiredArgsConstructor;
78
import lombok.extern.slf4j.Slf4j;
89
import lombok.val;
910
import org.scribe.model.OAuthRequest;
1011
import org.scribe.model.Response;
1112
import org.scribe.model.Token;
1213
import org.scribe.oauth.OAuthService;
14+
import org.springframework.web.util.UriComponentsBuilder;
1315
import org.zalando.problem.Problem;
1416
import org.zalando.problem.Status;
1517

18+
import java.time.LocalDate;
1619
import java.util.Optional;
1720

21+
import static java.time.format.DateTimeFormatter.ISO_LOCAL_DATE;
1822
import static org.scribe.model.Verb.GET;
1923

2024
@Slf4j
@@ -25,20 +29,17 @@ public class NetSuiteClient {
2529

2630
private final ObjectMapper objectMapper;
2731

28-
private final String url;
32+
@Getter
33+
private final String baseUrl;
2934

3035
private final String realm;
3136

3237
private final String token;
3338

3439
private final String tokenSecret;
3540

36-
public final String netsuiteUrl() {
37-
return url;
38-
}
39-
40-
public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines() {
41-
val response = callForTransactionLinesData();
41+
public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(LocalDate extractionFrom, LocalDate extractionTo) {
42+
val response = callForTransactionLinesData(extractionFrom, extractionTo);
4243

4344
if (response.isSuccessful()) {
4445
log.info("Netsuite response success...customerCode:{}, message:{}", response.getCode(), response.getMessage());
@@ -49,6 +50,7 @@ public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(
4950
if (bodyJsonTree.has("error")) {
5051
val error = bodyJsonTree.get("error").asInt();
5152
val text = bodyJsonTree.get("text").asText();
53+
log.error("Error api error:{}, message:{}", error, text);
5254

5355
if (error == 105) {
5456
log.warn("No data to read from NetSuite API...");
@@ -58,7 +60,7 @@ public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(
5860

5961
return Either.left(Problem.builder()
6062
.withStatus(Status.valueOf(response.getCode()))
61-
.withTitle("NetSuite API error")
63+
.withTitle("NETSUITE_API_ERROR")
6264
.withDetail(String.format("Error customerCode: %d, message: %s", error, text))
6365
.build());
6466
}
@@ -69,22 +71,29 @@ public Either<Problem, Optional<String>> retrieveLatestNetsuiteTransactionLines(
6971

7072
return Either.left(Problem.builder()
7173
.withStatus(Status.valueOf(response.getCode()))
72-
.withTitle("NetSuite API error")
74+
.withTitle("NETSUITE_API_ERROR")
7375
.withDetail(e.getMessage())
7476
.build());
7577
}
7678
}
7779

7880
return Either.left(Problem.builder()
7981
.withStatus(Status.valueOf(response.getCode()))
80-
.withTitle("NetSuite API error")
82+
.withTitle("NETSUITE_API_ERROR")
8183
.withDetail(response.getBody())
8284
.build());
8385
}
8486

85-
private Response callForTransactionLinesData() {
87+
private Response callForTransactionLinesData(LocalDate from, LocalDate to) {
8688
log.info("Retrieving data from NetSuite...");
87-
log.info("url: {}", url);
89+
log.info("base url: {}", baseUrl);
90+
91+
val builder = UriComponentsBuilder.fromHttpUrl(baseUrl)
92+
.queryParam("trandate:within", isoFormatDates(from, to));
93+
94+
val url = builder.toUriString();
95+
96+
log.info("call url: {}", url);
8897

8998
val request = new OAuthRequest(GET, url);
9099
request.setRealm(realm);
@@ -95,4 +104,8 @@ private Response callForTransactionLinesData() {
95104
return request.send();
96105
}
97106

107+
private String isoFormatDates(LocalDate from, LocalDate to) {
108+
return String.format("%s,%s", ISO_LOCAL_DATE.format(from), ISO_LOCAL_DATE.format(to));
109+
}
110+
98111
}

‎netsuite_altavia_erp_adapter/src/main/java/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/domain/core/TransactionDataSearchResult.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,10 @@
11
package org.cardanofoundation.lob.app.netsuite_altavia_erp_adapter.domain.core;
22

3+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
4+
35
import java.util.List;
46

7+
@JsonIgnoreProperties(ignoreUnknown = true)
58
public record TransactionDataSearchResult(List<TxLine> lines,
69
boolean more) {
710
}

‎netsuite_altavia_erp_adapter/src/main/java/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/domain/core/TxLine.java

Lines changed: 48 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -3,107 +3,118 @@
33
import com.fasterxml.jackson.annotation.JsonFormat;
44
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
55
import com.fasterxml.jackson.annotation.JsonProperty;
6-
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
7-
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
8-
import com.fasterxml.jackson.datatype.jsr310.deser.LocalDateDeserializer;
9-
import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateSerializer;
106
import jakarta.validation.constraints.*;
117

128
import javax.annotation.Nullable;
139
import java.math.BigDecimal;
1410
import java.time.LocalDate;
11+
import java.time.LocalDateTime;
12+
13+
import static com.fasterxml.jackson.annotation.JsonFormat.Shape.STRING;
14+
import static org.cardanofoundation.lob.app.netsuite_altavia_erp_adapter.util.Dates.ISO_8601_FORMAT_QUASI;
1515

1616
// https://docs.google.com/spreadsheets/d/1iGo1t2bLuWSONOYo6kG9uXSzt7laCrM8gluKkx8tmn0/edit#gid=501685631
1717
@JsonIgnoreProperties(ignoreUnknown = true)
1818
public record TxLine(
1919

20-
@JsonProperty("Line ID")
20+
@JsonProperty("line")
2121
@PositiveOrZero
22+
@NotNull
2223
Integer lineID,
2324

24-
@JsonProperty("Subsidiary (no hierarchy)")
25+
@JsonProperty("subsidiarynohierarchy")
2526
@PositiveOrZero
27+
@NotNull
2628
Long subsidiary,
2729

28-
@JsonProperty("Type")
30+
@JsonProperty("type")
31+
@NotNull
2932
String type,
3033

31-
@JsonProperty("Date")
32-
@JsonDeserialize(using = LocalDateDeserializer.class)
33-
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy")
34+
@JsonProperty("trandate")
35+
@JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
36+
@NotNull
3437
LocalDate date,
3538

36-
@JsonProperty("ID")
37-
String id,
39+
@JsonProperty("vendor.entityid")
40+
@Nullable
41+
String counterPartyId,
3842

39-
@JsonProperty("Company Name")
40-
String companyName,
43+
@JsonProperty("vendor.companyname")
44+
String counterPartyName,
4145

42-
@JsonProperty("Tax Item")
46+
@JsonProperty("taxcode")
4347
String taxItem,
4448

45-
@JsonProperty("Cost Center (no hierarchy)")
4649
@Nullable
50+
@JsonProperty("departmentnohierarchy")
4751
String costCenter,
4852

49-
@JsonProperty("Transaction Number")
53+
@JsonProperty("transactionnumber")
5054
@NotBlank
5155
String transactionNumber,
5256

53-
@JsonProperty("Document Number")
57+
@JsonProperty("tranid")
5458
@Nullable
5559
String documentNumber,
5660

57-
@JsonProperty("Number")
61+
@JsonProperty("account.number")
5862
String number,
5963

60-
@JsonProperty("Name")
64+
@JsonProperty("account.name")
6165
@Nullable
6266
String name,
6367

64-
@JsonProperty("Project (no hierarchy)")
68+
@JsonProperty("classnohierarchy")
6569
String project,
6670

67-
@JsonProperty("Account (Main)")
71+
@JsonProperty("accountmain")
6872
String accountMain,
6973

70-
@JsonProperty("Memo (Main)")
71-
String memo,
72-
73-
@JsonProperty("Currency")
74+
@JsonProperty("currency")
7475
@NotNull
7576
String currency,
7677

77-
@JsonProperty("Symbol")
78+
@JsonProperty("Currency.symbol")
7879
@NotNull
7980
String currencySymbol,
8081

8182
// this is accounting period date
82-
@JsonProperty("End Date")
83-
@JsonSerialize(using = LocalDateSerializer.class)
84-
@JsonDeserialize(using = LocalDateDeserializer.class)
85-
@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "dd/MM/yyyy")
83+
@JsonProperty("enddate")
84+
@JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
8685
LocalDate endDate,
8786

87+
@JsonProperty("datecreated")
88+
@JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
89+
@NotNull
90+
LocalDateTime dateCreated,
91+
92+
@JsonProperty("lastmodifieddate")
93+
@JsonFormat(shape = STRING, pattern = ISO_8601_FORMAT_QUASI)
8894
@NotNull
89-
@JsonProperty("Exchange Rate")
95+
LocalDateTime lastModifiedDate,
96+
97+
@NotNull
98+
@JsonProperty("exchangerate")
9099
@Positive
91100
BigDecimal exchangeRate,
92101

93102
@DecimalMin(value = "0.0")
94103
@Nullable
95-
@JsonProperty("Amount (Debit) (Foreign Currency)") BigDecimal amountDebitForeignCurrency,
104+
@JsonProperty("debitfxamount") BigDecimal amountDebitForeignCurrency,
96105

97106
@DecimalMin(value = "0.0")
98107
@Nullable
99-
@JsonProperty("Amount (Credit) (Foreign Currency)") BigDecimal amountCreditForeignCurrency,
108+
@JsonProperty("creditfxamount") BigDecimal amountCreditForeignCurrency,
100109

101110
@DecimalMin(value = "0.0")
102111
@Nullable
103-
@JsonProperty("Amount (Debit)") BigDecimal amountDebit,
112+
@JsonProperty("debitamount") BigDecimal amountDebit,
104113

105114
@DecimalMin(value = "0.0")
106115
@Nullable
107-
@JsonProperty("Amount (Credit)") BigDecimal amountCredit
116+
@JsonProperty("creditamount") BigDecimal amountCredit
117+
118+
) {
108119

109-
) { }
120+
}

‎netsuite_altavia_erp_adapter/src/main/java/org/cardanofoundation/lob/app/netsuite_altavia_erp_adapter/service/ExtractionParametersFilteringService.java

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,33 +18,36 @@ public Set<Transaction> applyExtractionParameters(UserExtractionParameters userE
1818
SystemExtractionParameters systemExtractionParameters,
1919
Set<Transaction> txs) {
2020
return txs.stream()
21+
// this is for sanity reasons since actually we should be filtering out the transactions in the adapter layer and more specifically while making an HTTP call via NetSuiteClient
2122
.filter(tx -> {
2223
val txAccountingPeriod = tx.getAccountingPeriod();
2324

24-
return txAccountingPeriod.equals(systemExtractionParameters.getAccountPeriodFrom()) || txAccountingPeriod.isAfter(systemExtractionParameters.getAccountPeriodFrom())
25+
return (txAccountingPeriod.equals(systemExtractionParameters.getAccountPeriodFrom()) || txAccountingPeriod.isAfter(systemExtractionParameters.getAccountPeriodFrom()))
2526
&&
2627
(txAccountingPeriod.equals(systemExtractionParameters.getAccountPeriodTo()) || txAccountingPeriod.isBefore(systemExtractionParameters.getAccountPeriodTo()));
2728
})
2829
.filter(tx -> userExtractionParameters.getOrganisationId().equals(tx.getOrganisation().getId()))
30+
// this is for sanity reasons since actually we should be filtering out the transactions in the adapter layer and more specifically while making an HTTP call via NetSuiteClient
2931
.filter(tx -> {
3032
val from = userExtractionParameters.getFrom();
3133

32-
return tx.getEntryDate().isEqual(from) || tx.getEntryDate().isAfter(from);
34+
return tx.getEntryDate().isEqual(from) || tx.getEntryDate().isAfter(from);
3335
})
36+
// this is for sanity reasons since actually we should be filtering out the transactions in the adapter layer and more specifically while making an HTTP call via NetSuiteClient
3437
.filter(tx -> {
35-
val to = userExtractionParameters.getFrom();
38+
val to = userExtractionParameters.getTo();
3639

37-
return tx.getEntryDate().isEqual(to) || tx.getEntryDate().isAfter(to);
40+
return tx.getEntryDate().isEqual(to) || tx.getEntryDate().isBefore(to);
3841
})
3942
.filter(tx -> {
4043
val txTypes = userExtractionParameters.getTransactionTypes();
4144

4245
return txTypes.isEmpty() || txTypes.contains(tx.getTransactionType());
4346
})
4447
.filter(tx -> {
45-
val transactionNumber = userExtractionParameters.getTransactionNumbers();
48+
val transactionNumbers = userExtractionParameters.getTransactionNumbers();
4649

47-
return transactionNumber.isEmpty() || transactionNumber.contains(tx.getInternalTransactionNumber());
50+
return transactionNumbers.isEmpty() || transactionNumbers.contains(tx.getInternalTransactionNumber());
4851
})
4952
.collect(Collectors.toSet());
5053
}

0 commit comments

Comments
 (0)