Feat/ra 16 implement mempool (#105)

* feat: implemented /mempool * feat: implemented /mempool * feat: removed unused mapper * chore: added snapshot version of yaci-store * chore: added snapshot version of yaci-store * refactor according to github comments * refactor according to github comments * chore: changed snapshot, activated parallel processing for yaci-store * chore: adding configurability for socket and node db * chore: db path configurable * chore: adjusted path for node syncing * doc: added doc for restoring a snapshot * chore: added configs and adjusted env files * added configs * chore: yaci version back to rc1 and added volume to docker-compose * Changed mempool configurability to spring profiles. * chore: changed error code of exception to avoid overlaps
cardano-foundation · Apr 4, 2024 · 37582c8 · 37582c8
1 parent 4872065
commit 37582c8
Show file tree

Hide file tree

Showing 44 changed files with 16,467 additions and 15,022 deletions.
diff --git a/.env.IntegrationTest b/.env.IntegrationTest
@@ -1,24 +1,31 @@
+LOG=INFO
 NETWORK="devkit"  # devkit, mainnet, testnet, preprod
 PROTOCOL_MAGIC=42
 
 #common env
 DB_ADMIN_USER_NAME="rosetta_db_admin"
 DB_ADMIN_USER_SECRET="weakpwd#123_d"
 
+# Postgres variables
 DB_IMAGE_NAME="postgres"
 DB_IMAGE_TAG="latest"
 DB_NAME="rosetta-java"
 DB_HOST="db" # Service name in docker-compose or local db
 DB_PORT="5432"
 DB_SCHEMA=${NETWORK}
+DB_PATH="data"
 MAXIMUM_POOL_SIZE=80
 JDBC_BATCH_SIZE=1000
 SCHEMA=${NETWORK}
+
+# Cardano Node variables
 CARDANO_NODE_HOST="yaci-cli" # Service name in docker-compose or local cardano node
 CARDANO_NODE_PORT="9999" # Uncomment if you are using local cardano node
 CARDANO_NODE_VERSION="8.9.0"
 NODE_SUBMIT_API_PORT=8090
-LOG=INFO
+CARDANO_NODE_SOCKET="./node-ipc"
+CARDANO_NODE_DB="node/db"
+CARDANO_CONFIG="./config/${NETWORK}"
 
 #api env
 API_SPRING_PROFILES_ACTIVE_API="dev"
@@ -41,8 +48,8 @@ API_SPRING_PROFILES_ACTIVE_YACI_INDEXER="postgres" # database profiles: h2, post
 INDEXER_NODE_PORT=3001
 
 #devkit env
-yaci_store_enabled=true
-ogmios_enabled=true
+DEVKIT_ENABLED=true
+DEVKIT_URL=yaci-cli
 
 #######################################################
 # Ports

diff --git a/.env.docker-compose b/.env.docker-compose
@@ -1,24 +1,29 @@
+LOG=INFO
 NETWORK="preprod"  # devkit, mainnet, testnet, preprod
 PROTOCOL_MAGIC=1
 
 #common env
 DB_ADMIN_USER_NAME="rosetta_db_admin"
 DB_ADMIN_USER_SECRET="weakpwd#123_d"
 
+# Postgres variables
 DB_IMAGE_NAME="postgres"
 DB_IMAGE_TAG="14.11-bullseye"
 DB_NAME="rosetta-java-preprod"
 DB_HOST="db" # Service name in docker-compose or local db
 DB_PORT="5432"
 DB_SCHEMA=${NETWORK}
+DB_PATH="data"
 MAXIMUM_POOL_SIZE=80
 JDBC_BATCH_SIZE=1000
 SCHEMA=${NETWORK}
+
+# Cardano Node variables
 CARDANO_NODE_HOST="cardano-node" # Service name in docker-compose or local cardano node
 CARDANO_NODE_PORT="3001" # Uncomment if you are using local cardano node
 CARDANO_NODE_VERSION="8.9.0"
 NODE_SUBMIT_API_PORT=8090
-LOG=INFO
+CARDANO_NODE_SOCKET="./node-ipc"
 
 #api env
 API_SPRING_PROFILES_ACTIVE_API="dev"
@@ -33,9 +38,10 @@ ROSETTA_VERSION=1.4.13
 TOPOLOGY_FILEPATH=/config/${NETWORK}/topology.json
 GENESIS_SHELLEY_PATH=/config/${NETWORK}/shelley-genesis.json
 GENESIS_BYRON_PATH=/config/${NETWORK}/byron-genesis.json
-
+API_NODE_SOCKET_PATH=./node/node.socket
 PRINT_EXCEPTION=true
 
 #api env
 API_SPRING_PROFILES_ACTIVE_YACI_INDEXER="postgres" # database profiles: h2, postgres
-INDEXER_NODE_PORT=3001
+INDEXER_NODE_PORT=3001
+MEMPOOL_ENABLED=true
diff --git a/.env.h2 b/.env.h2
@@ -1,6 +1,6 @@
 #Use envfile plugin for IDEA
 #https://plugins.jetbrains.com/plugin/7861-envfile
-
+LOG=INFO
 API_SPRING_PROFILES_ACTIVE=h2
 
 NETWORK=devkit  # devkit, mainnet, testnet, preprod
@@ -23,7 +23,7 @@ CARDANO_NODE_HOST=localhost # Service name in docker-compose or local cardano no
 CARDANO_NODE_PORT=3001 # Uncomment if you are using local cardano node
 CARDANO_NODE_VERSION=8.9.0
 NODE_SUBMIT_API_PORT=8090
-LOG=INFO
+CARDANO_NODE_SOCKET="./node-ipc"
 
 #api env
 API_SPRING_PROFILES_ACTIVE_API=dev

diff --git a/.envMainnet b/.envMainnet
@@ -0,0 +1,46 @@
+LOG=INFO
+NETWORK=mainnet
+PROTOCOL_MAGIC=764824073
+
+#common env
+DB_ADMIN_USER_NAME="rosetta_db_admin"
+DB_ADMIN_USER_SECRET="weakpwd#123_d"
+
+DB_IMAGE_NAME="postgres"
+DB_IMAGE_TAG="latest"
+DB_NAME="rosetta-java-preprod"
+DB_HOST="db" # Service name in docker-compose or local db
+DB_PORT="5433"
+DB_SCHEMA=${NETWORK}
+DB_PATH="./data"
+
+MAXIMUM_POOL_SIZE=80
+JDBC_BATCH_SIZE=1000
+SCHEMA=${NETWORK}
+CARDANO_NODE_HOST="cardano-node"
+CARDANO_NODE_PORT=3002
+CARDANO_NODE_VERSION="8.9.0"
+NODE_SUBMIT_API_PORT=8090
+CARDANO_NODE_SOCKET="./nodeMN"
+CARDANO_NODE_DB="./nodeMN/db"
+CARDANO_CONFIG="./config/${NETWORK}"
+
+#api env
+API_SPRING_PROFILES_ACTIVE_API="dev"
+API_EXPOSED_PORT="8088"
+API_BIND_PORT="8088"
+TRANSACTION_TTL=3000
+
+DB_CONNECTION_PARAMS_PROVIDER_TYPE="ENVIRONMENT"
+DB_DRIVER_CLASS_NAME="org.postgresql.Driver"
+
+ROSETTA_VERSION=1.4.13
+TOPOLOGY_FILEPATH=/config/${NETWORK}/topology.json
+GENESIS_SHELLEY_PATH=/config/${NETWORK}/shelley-genesis.json
+GENESIS_BYRON_PATH=/config/${NETWORK}/byron-genesis.json
+API_NODE_SOCKET_PATH=./node/node.socket
+PRINT_EXCEPTION=true
+
+#api env
+API_SPRING_PROFILES_ACTIVE_YACI_INDEXER="postgres" # node profiles: devkit  || database profiles: h2, postgres
+INDEXER_NODE_PORT=3002
diff --git a/.envPreprod b/.envPreprod
@@ -1,30 +1,35 @@
+LOG=INFO
+NETWORK=preprod
+PROTOCOL_MAGIC=1
+
 #common env
 DB_ADMIN_USER_NAME="rosetta_db_admin"
 DB_ADMIN_USER_SECRET="weakpwd#123_d"
-SERVICE_USER_NAME="rosetta_db_service_user"
-SERVICE_USER_SECRET="weakerpwd#123_d"
 
 DB_IMAGE_NAME="postgres"
 DB_IMAGE_TAG="latest"
 DB_NAME="rosetta-java-preprod"
 DB_HOST="db" # Service name in docker-compose or local db
 DB_PORT="5432"
-DB_SCHEMA="preprod"
+DB_SCHEMA=${NETWORK}
+DB_PATH="./data"
+
 MAXIMUM_POOL_SIZE=80
 JDBC_BATCH_SIZE=1000
-SCHEMA=preprod
+SCHEMA=${NETWORK}
+CARDANO_NODE_HOST="cardano-node"
+CARDANO_NODE_PORT=3001
 CARDANO_NODE_VERSION="8.9.0"
 NODE_SUBMIT_API_PORT=8090
-CARDANO_NODE_HOST="cardano-node"
-LOG=INFO
+CARDANO_NODE_SOCKET="./node"
+CARDANO_NODE_DB="./node/db"
+CARDANO_CONFIG="./config/${NETWORK}"
 
 #api env
 API_SPRING_PROFILES_ACTIVE_API="dev"
 API_EXPOSED_PORT="8081"
 API_BIND_PORT="8081"
-PROTOCOL_MAGIC=1
 TRANSACTION_TTL=3000
-NETWORK=preprod
 
 DB_CONNECTION_PARAMS_PROVIDER_TYPE="ENVIRONMENT"
 DB_DRIVER_CLASS_NAME="org.postgresql.Driver"
@@ -33,7 +38,7 @@ ROSETTA_VERSION=1.4.13
 TOPOLOGY_FILEPATH=/config/${NETWORK}/topology.json
 GENESIS_SHELLEY_PATH=/config/${NETWORK}/shelley-genesis.json
 GENESIS_BYRON_PATH=/config/${NETWORK}/byron-genesis.json
-NODE_SOCKET_PATH=./node/node.socket
+API_NODE_SOCKET_PATH=./node/node.socket
 PRINT_EXCEPTION=true
 
 #api env

diff --git a/README.md b/README.md
@@ -14,12 +14,12 @@ to fetch the data from the node.
 - [x] Integration test setup
 - API calls
   - Data API
-    - [x] /network/list
-    - [x] /network/status
-    - [x] /network/options
+    - [x] /network/*
     - [x] /block/*
-    - [ ] /mempool/*
     - [x] /account/*
+    - /mempool
+      - [x] /mempool
+      - [ ] /mempool/transaction
   - Construction API
     - [x] /construction/derive
     - [x] /construction/preprocess
@@ -62,6 +62,22 @@ to fetch the data from the node.
   - Import the environment `./postmanTests/Rosetta-java-env.postman_environment.json`
   - Run the collection
 
+
+### Restore a snapshot
+**TBD for yaci-store**
+
+A node snapshot can be downloaded from [here](https://csnapshots.io/). Download the snapshot and place the files within the `CARDANO_NODE_DB` Path.
+
+For mainnet the following command can be used to restore the snapshot:
+```bash
+curl -o - https://downloads.csnapshots.io/mainnet/$(curl -s https://downloads.csnapshots.io/mainnet/mainnet-db-snapshot.json| jq -r .[].file_name ) | lz4 -c -d - | tar -x -C ${CARDANO_NODE_DB}
+```
+
+### Mempool Monitoring
+Mempool monitoring can be activated when adding the spring profile `mempool` to the rosetta api service.
+It will be turned off by default. Since it is only working for nodes, which participate in the network and available within P2P from other nodes. 
+So the Node itself needs extra configuration to be able to query mempool transactions.
+
 ### Environment variables
 <details>
 <summary>Full list of environment variables and standard values</summary>

diff --git a/api/src/main/java/org/cardanofoundation/rosetta/RosettaApiApplication.java b/api/src/main/java/org/cardanofoundation/rosetta/RosettaApiApplication.java
@@ -10,6 +10,7 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Profile;
 import org.springframework.core.Ordered;
+import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.web.filter.ForwardedHeaderFilter;
 import io.swagger.v3.oas.annotations.OpenAPIDefinition;
 import io.swagger.v3.oas.annotations.info.Info;
@@ -26,6 +27,7 @@
     "org.cardanofoundation.rosetta.api.network.model.entity",
     "org.cardanofoundation.rosetta.api.common.model.entity"})
 @OpenAPIDefinition(info = @Info(title = "APIs", version = "1.0", description = "Rosetta APIs v1.0"))
+@EnableScheduling
 public class RosettaApiApplication {
 
   public static void main(String[] args) {

diff --git a/...n/java/org/cardanofoundation/rosetta/api/mempool/controller/MempoolApiImplementation.java b/...n/java/org/cardanofoundation/rosetta/api/mempool/controller/MempoolApiImplementation.java
@@ -0,0 +1,37 @@
+package org.cardanofoundation.rosetta.api.mempool.controller;
+
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.NotImplementedException;
+import org.cardanofoundation.rosetta.api.mempool.service.MempoolService;
+import org.openapitools.client.api.MempoolApi;
+import org.openapitools.client.model.MempoolResponse;
+import org.openapitools.client.model.MempoolTransactionRequest;
+import org.openapitools.client.model.MempoolTransactionResponse;
+import org.openapitools.client.model.NetworkRequest;
+import org.openapitools.client.model.TransactionIdentifier;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Profile;
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+@RequiredArgsConstructor
+@Profile("mempool")
+public class MempoolApiImplementation implements MempoolApi {
+
+  private final MempoolService mempoolService;
+
+  @Override
+  public ResponseEntity<MempoolResponse> mempool(NetworkRequest networkRequest) {
+    MempoolResponse mempoolResponse = MempoolResponse.builder()
+        .transactionIdentifiers(mempoolService.getCurrentTransactionIdentifiers(networkRequest.getNetworkIdentifier().getNetwork()))
+        .build();
+    return ResponseEntity.ok(mempoolResponse);
+  }
+
+  @Override
+  public ResponseEntity<MempoolTransactionResponse> mempoolTransaction(
+      MempoolTransactionRequest mempoolTransactionRequest) {
+    throw new NotImplementedException("Not implemented yet");
+  }
+}
diff --git a/api/src/main/java/org/cardanofoundation/rosetta/api/mempool/service/MempoolService.java b/api/src/main/java/org/cardanofoundation/rosetta/api/mempool/service/MempoolService.java
@@ -0,0 +1,12 @@
+package org.cardanofoundation.rosetta.api.mempool.service;
+
+import java.util.List;
+import org.openapitools.client.model.Transaction;
+import org.openapitools.client.model.TransactionIdentifier;
+import org.springframework.context.annotation.Profile;
+
+@Profile("mempool")
+public interface MempoolService {
+
+  List<TransactionIdentifier> getCurrentTransactionIdentifiers(String network);
+}