Save tx at transaction submit eventHandler for nightfall-client

.github/workflows/on-pull-request-master.yml

@@ -895,3 +895,48 @@ jobs:
         with:
           name: periodic-payment-test-logs
           path: ./periodic-payment-test.log
+
+  test-client-sync:
+    env:
+      CONFIRMATIONS: 1
+      NF_SERVICES_TO_START: blockchain,client,deployer,mongodb,optimist,rabbitmq,worker
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '16.17.0'
+
+      - name: Start Containers
+        run: |
+          ./bin/setup-nightfall
+          ./bin/start-nightfall -g -d &> test-client-sync.log &disown
+      - name: Wait for images to be ready
+        uses: Wandalen/[email protected]
+        with:
+          command: |
+            docker wait nightfall_3_deployer_1
+          attempt_limit: 100
+          attempt_delay: 20000
+
+      - name: Debug logs - after image builds
+        if: always()
+        run: cat test-client-sync.log
+
+      - name: Run integration test
+        run: |
+          npm run test-client-sync
+      - name: Debug logs - after integration test run
+        if: always()
+        run: cat test-client-sync.log
+
+      - name: If integration test failed, shutdown the Containers
+        if: failure()
+        run: npm run nightfall-down
+
+      - name: If integration test failed, upload logs files as artifacts
+        if: failure()
+        uses: actions/upload-artifact@master
+        with:
+          name: test-client-sync-logs
+          path: ./test-client-sync.log

cli/lib/nf3.mjs

@@ -677,7 +677,7 @@ class Nf3 {
             this.shieldContractAddress,
             0,
           );
-          resolve(receipt);
+          resolve({ ...receipt, transactionHashL2: res.data.transaction.transactionHash });
         } catch (err) {
           reject(err);
         }
@@ -740,7 +740,7 @@ class Nf3 {
               this.shieldContractAddress,
               0,
             );
-            resolve(receipt);
+            resolve({ ...receipt, transactionHashL2: res.data.transaction.transactionHash });
           } catch (err) {
             reject(err);
           }

nightfall-client/src/event-handlers/block-proposed.mjs

@@ -14,8 +14,9 @@ import {
   setSiblingInfo,
   countCircuitTransactions,
   isTransactionHashBelongCircuit,
+  deleteNonNullifiedCommitments,
 } from '../services/commitment-storage.mjs';
-import getProposeBlockCalldata from '../services/process-calldata.mjs';
+import { getProposeBlockCalldata } from '../services/process-calldata.mjs';
 import { zkpPrivateKeys, nullifierKeys } from '../services/keys.mjs';
 import {
   getTreeByBlockNumberL2,
@@ -26,6 +27,8 @@ import {
   getNumberOfL2Blocks,
   getTransactionByTransactionHash,
   updateTransaction,
+  findDuplicateTransactions,
+  deleteTransactionsByTransactionHashes,
 } from '../services/database.mjs';
 import { decryptCommitment } from '../services/commitment-sync.mjs';
 import { syncState } from '../services/state-sync.mjs';
@@ -81,6 +84,7 @@ async function blockProposedEventHandler(data, syncing) {
 
   const dbUpdates = transactions.map(async transaction => {
     let saveTxToDb = false;
+    let duplicateTransactions = []; // duplicate tx holding same commitments or nullifiers
 
     // filter out non zero commitments and nullifiers
     const nonZeroCommitments = transaction.commitments.filter(c => c !== ZERO);
@@ -143,6 +147,12 @@ async function blockProposedEventHandler(data, syncing) {
           } else logger.warn(`Duplicate transaction in Proposed Block has been dropped`);
         } else throw new Error(err);
       }
+
+      duplicateTransactions = await findDuplicateTransactions(
+        nonZeroCommitments,
+        nonZeroNullifiers,
+        [transaction.transactionHash],
+      );
     }
 
     return Promise.all([
@@ -154,6 +164,13 @@ async function blockProposedEventHandler(data, syncing) {
         data.blockNumber,
         data.transactionHash,
       ),
+      deleteTransactionsByTransactionHashes([...duplicateTransactions.map(t => t.transactionHash)]),
+      deleteNonNullifiedCommitments([
+        ...duplicateTransactions
+          .map(t => t.commitments)
+          .flat()
+          .filter(c => c !== ZERO),
+      ]),
     ]);
   });
 

nightfall-client/src/event-handlers/index.mjs

@@ -2,15 +2,18 @@ import { startEventQueue } from './subscribe.mjs';
 import blockProposedEventHandler from './block-proposed.mjs';
 import rollbackEventHandler from './rollback.mjs';
 import removeBlockProposedEventHandler from './chain-reorg.mjs';
+import transactionSubmittedEventHandler from './transaction-submitted.mjs';
 
 const eventHandlers = {
   BlockProposed: blockProposedEventHandler,
+  TransactionSubmitted: transactionSubmittedEventHandler,
   Rollback: rollbackEventHandler,
   removers: {
     BlockProposed: removeBlockProposedEventHandler,
   },
   priority: {
     BlockProposed: 0,
+    TransactionSubmitted: 1,
     Rollback: 0,
   },
 };

nightfall-client/src/event-handlers/transaction-submitted.mjs

@@ -0,0 +1,63 @@
+import logger from 'common-files/utils/logger.mjs';
+import constants from 'common-files/constants/index.mjs';
+import { getTransactionSubmittedCalldata } from '../services/process-calldata.mjs';
+import { countCommitments, countNullifiers } from '../services/commitment-storage.mjs';
+import { saveTransaction } from '../services/database.mjs';
+
+const { ZERO } = constants;
+
+async function doesAnyOfCommitmentsExistInDB(commitments) {
+  const count = await countCommitments(commitments);
+  return Boolean(count);
+}
+
+async function doesAnyOfNullifiersExistInDB(nullifiers) {
+  const count = await countNullifiers(nullifiers);
+  return Boolean(count);
+}
+
+/**
+ * This handler runs whenever a new transaction is submitted to the blockchain
+ */
+async function transactionSubmittedEventHandler(eventParams) {
+  const { offchain = false, ...data } = eventParams;
+  let saveTxInDb = false;
+
+  const transaction = await getTransactionSubmittedCalldata(data);
+  transaction.blockNumber = data.blockNumber;
+  transaction.transactionHashL1 = data.transactionHash;
+
+  // logic: if any of non zero commitment in transaction alraedy exist in db
+  // That means this transaction belong to user using this nightfall-client
+  // Hence, proceed and save tx in db.
+  // Note: for deposit we store commitment in transaction submit event handler,
+  // similarly for transfer we store change commitment in transaction submit event handler.
+
+  // filter out non zero commitments and nullifiers
+  const nonZeroCommitments = transaction.commitments.filter(c => c !== ZERO);
+  const nonZeroNullifiers = transaction.nullifiers.filter(n => n !== ZERO);
+
+  if (await doesAnyOfCommitmentsExistInDB(nonZeroCommitments)) {
+    saveTxInDb = true;
+  } else if (doesAnyOfNullifiersExistInDB(nonZeroNullifiers)) {
+    saveTxInDb = true;
+  }
+
+  if (saveTxInDb) {
+    await saveTransaction({ ...transaction }).catch(err =>
+      logger.error({
+        msg: 'error while saving transaction in transactionSubmittedEventHandler',
+        err,
+      }),
+    );
+  }
+
+  logger.info({
+    msg: 'Client Transaction Handler - New transaction received.',
+    transaction,
+    offchain,
+    saveTxInDb,
+  });
+}
+
+export default transactionSubmittedEventHandler;

nightfall-client/src/routes/commitment.mjs

@@ -15,10 +15,12 @@ import {
   getWalletPendingSpentBalance,
   getCommitments,
   getCommitmentsByCompressedZkpPublicKeyList,
-  insertCommitmentsAndResync,
+  insertCommitments,
   getCommitmentsByCircuitHash,
   getCommitmentsDepositedRollbacked,
 } from '../services/commitment-storage.mjs';
+import { syncState } from '../services/state-sync.mjs';
+import { getAllTransactions } from '../services/database.mjs';
 
 const router = express.Router();
 
@@ -84,7 +86,8 @@ router.get('/commitments', async (req, res, next) => {
 router.post('/save', async (req, res, next) => {
   const listOfCommitments = req.body;
   try {
-    const response = await insertCommitmentsAndResync(listOfCommitments);
+    const response = await insertCommitments(listOfCommitments);
+    await syncState(); // Sycronize from beggining
     res.json(response);
   } catch (err) {
     next(err);
@@ -147,4 +150,13 @@ router.get('/commitmentsRollbacked', async (req, res, next) => {
   }
 });
 
+router.get('/transactions', async (req, res, next) => {
+  try {
+    const transactions = await getAllTransactions();
+    res.json({ transactions });
+  } catch (err) {
+    next(err);
+  }
+});
+
 export default router;

nightfall-client/src/services/commitment-storage.mjs

@@ -16,7 +16,6 @@ import {
   getTransactionByTransactionHash,
   getTransactionHashSiblingInfo,
 } from './database.mjs';
-import { syncState } from './state-sync.mjs';
 
 const { MONGO_URL, COMMITMENTS_DB, COMMITMENTS_COLLECTION } = config;
 const { generalise } = gen;
@@ -976,13 +975,13 @@ export async function findUsableCommitmentsMutex(
 
 /**
  *
- * @function insertCommitmentsAndResync save a list of commitments in the database
+ * @function insertCommitments save a list of commitments in the database
  * @param {[]} listOfCommitments a list of commitments to be saved in the database
  * @throws if all the commitments in the list already exists in the database
  * throw an error
  * @returns return a success message.
  */
-export async function insertCommitmentsAndResync(listOfCommitments) {
+export async function insertCommitments(listOfCommitments) {
   const connection = await mongo.connection(MONGO_URL);
   const db = connection.db(COMMITMENTS_DB);
 
@@ -1005,10 +1004,6 @@ export async function insertCommitmentsAndResync(listOfCommitments) {
   if (onlyNewCommitments.length > 0) {
     // 4. Insert all
     await db.collection(COMMITMENTS_COLLECTION).insertMany(onlyNewCommitments);
-
-    // 5. Sycronize from beggining
-    await syncState();
-
     return { successMessage: 'Commitments have been saved successfully!' };
   }
 
@@ -1072,3 +1067,11 @@ export async function getCommitmentsDepositedRollbacked(compressedZkpPublicKey)
 
   return db.collection(COMMITMENTS_COLLECTION).find(query).toArray();
 }
+
+// function to delete non nullified commitments
+export async function deleteNonNullifiedCommitments(commitments) {
+  const connection = await mongo.connection(MONGO_URL);
+  const query = { _id: { $in: commitments }, isNullifiedOnChain: -1 };
+  const db = connection.db(COMMITMENTS_DB);
+  return db.collection(COMMITMENTS_COLLECTION).deleteMany(query);
+}

nightfall-client/src/services/database.mjs

@@ -304,3 +304,18 @@ export async function getTransactionsByTransactionHashesByL2Block(transactionHas
   );
   return transactions;
 }
+
+/**
+ * Function to find duplicate transactions for an array of commitments or nullifiers
+ * this function is used in blockProposedEventHandler
+ */
+export async function findDuplicateTransactions(commitments, nullifiers, transactionHashes = []) {
+  const connection = await mongo.connection(MONGO_URL);
+  const db = connection.db(COMMITMENTS_DB);
+  const query = {
+    $or: [{ commitments: { $in: commitments } }, { nullifiers: { $in: nullifiers } }],
+    transactionHash: { $nin: transactionHashes },
+    blockNumberL2: { $eq: -1 },
+  };
+  return db.collection(TRANSACTIONS_COLLECTION).find(query).toArray();
+}

nightfall-client/src/services/process-calldata.mjs

@@ -10,7 +10,7 @@ import { unpackBlockInfo } from 'common-files/utils/block-utils.mjs';
 
 const { SIGNATURES } = config;
 
-async function getProposeBlockCalldata(eventData) {
+export async function getProposeBlockCalldata(eventData) {
   const web3 = Web3.connection();
   const { transactionHash } = eventData;
   const tx = await web3.eth.getTransaction(transactionHash);
@@ -76,4 +76,47 @@ async function getProposeBlockCalldata(eventData) {
   return { transactions, block };
 }
 
-export default getProposeBlockCalldata;
+export async function getTransactionSubmittedCalldata(eventData) {
+  const web3 = Web3.connection();
+  const { transactionHash } = eventData;
+  const tx = await web3.eth.getTransaction(transactionHash);
+  // Remove the '0x' and function signature to recove rhte abi bytecode
+  const abiBytecode = `0x${tx.input.slice(10)}`;
+  const transactionData = web3.eth.abi.decodeParameter(SIGNATURES.SUBMIT_TRANSACTION, abiBytecode);
+  const [
+    packedTransactionInfo,
+    historicRootBlockNumberL2Packed,
+    tokenId,
+    ercAddress,
+    recipientAddress,
+    commitments,
+    nullifiers,
+    compressedSecrets,
+    proof,
+  ] = transactionData;
+
+  const { value, fee, circuitHash, tokenType } =
+    Transaction.unpackTransactionInfo(packedTransactionInfo);
+
+  const historicRootBlockNumberL2 = Transaction.unpackHistoricRoot(
+    nullifiers.length,
+    historicRootBlockNumberL2Packed,
+  );
+
+  const transaction = {
+    value,
+    fee,
+    circuitHash,
+    tokenType,
+    historicRootBlockNumberL2,
+    tokenId,
+    ercAddress,
+    recipientAddress,
+    commitments,
+    nullifiers,
+    compressedSecrets,
+    proof,
+  };
+  transaction.transactionHash = Transaction.calcHash(transaction);
+  return transaction;
+}