fix: the projection now always stores blocks when it reaches tip - k

Author: AngelCastilloB

packages/cardano-services/src/Projection/prepareTypeormProjection.ts

@@ -31,6 +31,7 @@ import {
   storeUtxo,
   willStoreAddresses,
   willStoreAssets,
+  willStoreBlockData,
   willStoreHandleMetadata,
   willStoreHandles,
   willStoreNftMetadata,
@@ -371,6 +372,7 @@ export const prepareTypeormProjection = (
     .map((store) => Object.entries(storeOperators).find(([_, operator]) => store === operator)!)
     .map(([storeName]) => willStore[storeName as keyof StoreOperators])
     .filter(isNotNil);
+
   return {
     __debug: {
       entities: selectedEntities.map((Entity) => keyOf(entities, Entity)),
@@ -383,8 +385,10 @@ export const prepareTypeormProjection = (
     mappers: selectedMappers,
     stores: selectedStores,
     willStore: <T extends ProjectionEvent>(evt: T) =>
+      evt.eventType === ChainSyncEventType.RollBackward ||
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      evt.eventType === ChainSyncEventType.RollBackward || willStoreCheckers.some((check) => check(evt as any))
+      willStoreCheckers.some((check) => check(evt as any)) ||
+      willStoreBlockData(evt)
   };
 };
 

packages/cardano-services/test/Projection/createTypeormProjection.test.ts

@@ -6,6 +6,7 @@ import {
   TokensEntity,
   createDataSource
 } from '@cardano-sdk/projection-typeorm';
+import { Cardano, ChainSyncEventType } from '@cardano-sdk/core';
 import { ChainSyncDataSet, chainSyncData, logger } from '@cardano-sdk/util-dev';
 import { ProjectionName, createTypeormProjection, prepareTypeormProjection } from '../../src';
 import { lastValueFrom } from 'rxjs';
@@ -57,6 +58,12 @@ describe('createTypeormProjection', () => {
       .filter((evt) => (evt as any).block && (evt as any).block.body.length === 0)
       .map((evt) => (evt as any).block.header.hash);
 
+    // Make sure our empty blocks are outside the stability window.
+    for (const event of data.allEvents) {
+      if (event.eventType === ChainSyncEventType.RollForward && emptyBlocksHashes.includes(event.block.header.hash))
+        event.tip.blockNo = Cardano.BlockNo(Number.MAX_SAFE_INTEGER);
+    }
+
     const projection$ = createTypeormProjection({
       blocksBufferLength: 10,
       cardanoNode: data.cardanoNode,

packages/projection-typeorm/src/TypeormStabilityWindowBuffer.ts

@@ -21,6 +21,16 @@ export interface TypeormStabilityWindowBufferProps extends WithLogger {
   reconnectionConfig: ReconnectionConfig;
 }
 
+export const willStoreBlockData = ({
+  genesisParameters,
+  block,
+  tip
+}: {
+  genesisParameters: Cardano.CompactGenesis;
+  block: Cardano.Block;
+  tip: Cardano.Tip;
+}) => block.header.blockNo >= tip.blockNo - genesisParameters.securityParameter;
+
 export class TypeormStabilityWindowBuffer implements StabilityWindowBuffer {
   readonly #queryRunner$: Observable<QueryRunner>;
   readonly #retryBackoffConfig: RetryBackoffConfig;

packages/projection-typeorm/test/TypeormStabilityWindowBuffer.test.ts

@@ -3,7 +3,8 @@ import {
   BlockEntity,
   TypeormStabilityWindowBuffer,
   WithTypeormContext,
-  createObservableConnection
+  createObservableConnection,
+  willStoreBlockData
 } from '../src';
 import { Cardano, ChainSyncEventType } from '@cardano-sdk/core';
 import { DataSource, NoConnectionForRepositoryError, QueryRunner, Repository } from 'typeorm';
@@ -23,6 +24,12 @@ const createBlockDataEntity = (block: Cardano.Block, blockEntity: BlockEntity):
   data: block
 });
 
+const createCustomEvent = (secParameter: number, blockNo: number, tipBlockNo: number) => ({
+  block: { header: { blockNo } } as Cardano.Block,
+  genesisParameters: { securityParameter: secParameter } as Cardano.CompactGenesis,
+  tip: { blockNo: tipBlockNo } as Cardano.Tip
+});
+
 describe('TypeormStabilityWindowBuffer', () => {
   const entities = [BlockEntity, BlockDataEntity];
   const securityParameter = 50;
@@ -170,4 +177,14 @@ describe('TypeormStabilityWindowBuffer', () => {
       });
     });
   });
+
+  describe('willStoreBlockData', () => {
+    it('returns false when block is outside stability window', () => {
+      expect(willStoreBlockData(createCustomEvent(10, 1, 500))).toBe(false);
+    });
+
+    it('returns true when block is within stability window', () => {
+      expect(willStoreBlockData(createCustomEvent(10, 490, 500))).toBe(true);
+    });
+  });
 });

yarn-project.nix

@@ -1420,7 +1420,6 @@ cacheEntries = {
 "fs-minipass@npm:3.0.0" = { filename = "fs-minipass-npm-3.0.0-3692c14b65-b72e9fe426.zip"; sha512 = "b72e9fe426e39f05b35bf237c8218b7ab3f68a65f325725ad7b4e431ff5a10725946fc62883b78446c07515ab938d25fdde3d08fb5ac8693f7f9eb9990da21f0"; };
 "fs.realpath@npm:1.0.0" = { filename = "fs.realpath-npm-1.0.0-c8f05d8126-99ddea01a7.zip"; sha512 = "99ddea01a7e75aa276c250a04eedeffe5662bce66c65c07164ad6264f9de18fb21be9433ead460e54cff20e31721c811f4fb5d70591799df5f85dce6d6746fd0"; };
 "fsevents@npm:2.3.2" = { filename = "fsevents-npm-2.3.2-a881d6ac9f-97ade64e75.zip"; sha512 = "97ade64e75091afee5265e6956cb72ba34db7819b4c3e94c431d4be2b19b8bb7a2d4116da417950c3425f17c8fe693d25e20212cac583ac1521ad066b77ae31f"; };
-"fsevents@patch:fsevents@npm%3A2.3.2#~builtin<compat/fsevents>::version=2.3.2&hash=18f3a7" = { filename = "fsevents-patch-3340e2eb10-8.zip"; sha512 = "edbd0fd80be379c14409605f77e52fdc78a119e17f875e8b90a220c3e5b29e54a1477c21d91fd30b957ea4866406dc3ff87b61432d2840ff8866b309e5866140"; };
 "ftp@npm:0.3.10" = { filename = "ftp-npm-0.3.10-348fb9ac23-ddd313c1d4.zip"; sha512 = "ddd313c1d44eb7429f3a7d77a0155dc8fe86a4c64dca58f395632333ce4b4e74c61413c6e0ef66ea3f3d32d905952fbb6d028c7117d522f793eb1fa282e17357"; };
 "function-bind@npm:1.1.1" = { filename = "function-bind-npm-1.1.1-b56b322ae9-b32fbaebb3.zip"; sha512 = "b32fbaebb3f8ec4969f033073b43f5c8befbb58f1a79e12f1d7490358150359ebd92f49e72ff0144f65f2c48ea2a605bff2d07965f548f6474fd8efd95bf361a"; };
 "function.prototype.name@npm:1.1.5" = { filename = "function.prototype.name-npm-1.1.5-e776a642bb-acd21d733a.zip"; sha512 = "acd21d733a9b649c2c442f067567743214af5fa248dbeee69d8278ce7df3329ea5abac572be9f7470b4ec1cd4d8f1040e3c5caccf98ebf2bf861a0deab735c27"; };