rename keepMetadaNames to whitelistMetadataNames

Author: Cmdv

cardano-chain-gen/test/Test/Cardano/Db/Mock/Config.hs

@@ -268,7 +268,8 @@ mkSyncNodeParams staticDir mutableDir CommandLineArgs {..} = do
       , enpHasShelley = True
       , enpHasMultiAssets = claHasMultiAssets
       , enpHasMetadata = claHasMetadata
-      , enpKeepMetadataNames = []
+      , enpWhitelistMetadataNames = []
+      , enpWhitelistMAPolicies = []
       , enpHasPlutusExtra = True
       , enpHasGov = True
       , enpHasOffChainPoolData = True

cardano-db-sync/app/cardano-db-sync.hs

@@ -88,7 +88,8 @@ pRunDbSyncNode =
     <*> pHasShelley
     <*> pHasMultiAssets
     <*> pHasMetadata
-    <*> pKeepTxMetadata
+    <*> pWhiteListTxMetadata
+    <*> pWhiteListMAPolicies
     <*> pHasPlutusExtra
     <*> pHasGov
     <*> pHasOffChainPoolData
@@ -230,20 +231,28 @@ pSlotNo =
           <> Opt.metavar "WORD"
       )
 
-pKeepTxMetadata :: Parser [Word64]
-pKeepTxMetadata =
+pWhiteListTxMetadata :: Parser [Word64]
+pWhiteListTxMetadata =
   Opt.option
     (parseCommaSeparated <$> Opt.str)
-    ( Opt.long "keep-tx-metadata"
+    ( Opt.long "whitelist-tx-metadata"
         <> Opt.value []
         <> Opt.help "Insert a specific set of tx metadata, based on the tx metadata key names"
     )
-  where
-    parseCommaSeparated :: String -> [Word64]
-    parseCommaSeparated str =
-      case traverse readMaybe (splitOn "," str) of
-        Just values -> values
-        Nothing -> error "Failed to parse comma-separated values"
+
+pWhiteListMAPolicies :: Parser [Word64]
+pWhiteListMAPolicies =
+  Opt.option
+    (parseCommaSeparated <$> Opt.str)
+    ( Opt.long "whitelist-multi-asset-policy"
+        <> Opt.help "Only insert a specific sellected list of multi-assets, based on the multi-asset's policy name"
+    )
+
+parseCommaSeparated :: String -> [Word64]
+parseCommaSeparated str =
+  case traverse readMaybe (splitOn "," str) of
+    Just values -> values
+    Nothing -> error "Failed to parse comma-separated values"
 
 pHasInOut :: Parser Bool
 pHasInOut =

cardano-db-sync/src/Cardano/DbSync.hs

@@ -233,10 +233,13 @@ extractSyncOptions snp aop =
     , snapshotEveryLagging = enpSnEveryLagging snp
     }
   where
-    maybeKeepMNames =
-      if null (enpKeepMetadataNames snp)
+    maybeWhitelistMDNames = whitelistToMaybe (enpWhitelistMetadataNames snp)
+    maybeWhitelistMAPolicies = whitelistToMaybe (enpWhitelistMAPolicies snp)
+
+    whitelistToMaybe wList =
+      if null wList
         then Strict.Nothing
-        else Strict.Just (enpKeepMetadataNames snp)
+        else Strict.Just wList
 
     iopts
       | enpOnlyGov snp = onlyGovInsertOptions useLedger
@@ -251,7 +254,8 @@ extractSyncOptions snp aop =
             , ioRewards = True
             , ioMultiAssets = enpHasMultiAssets snp
             , ioMetadata = enpHasMetadata snp
-            , ioKeepMetadataNames = maybeKeepMNames
+            , ioWhitelistMetadataNames = maybeWhitelistMDNames
+            , ioWhitelistMAPolicies = maybeWhitelistMAPolicies
             , ioPlutusExtra = enpHasPlutusExtra snp
             , ioOffChainPoolData = enpHasOffChainPoolData snp
             , ioGov = enpHasGov snp

cardano-db-sync/src/Cardano/DbSync/Api.hs

@@ -208,7 +208,8 @@ fullInsertOptions useLedger =
     , ioRewards = True
     , ioMultiAssets = True
     , ioMetadata = True
-    , ioKeepMetadataNames = Strict.Nothing
+    , ioWhitelistMetadataNames = Strict.Nothing
+    , ioWhitelistMAPolicies = Strict.Nothing
     , ioPlutusExtra = True
     , ioOffChainPoolData = True
     , ioGov = True
@@ -223,7 +224,8 @@ onlyUTxOInsertOptions =
     , ioRewards = False
     , ioMultiAssets = True
     , ioMetadata = False
-    , ioKeepMetadataNames = Strict.Nothing
+    , ioWhitelistMetadataNames = Strict.Nothing
+    , ioWhitelistMAPolicies = Strict.Nothing
     , ioPlutusExtra = False
     , ioOffChainPoolData = False
     , ioGov = False
@@ -241,7 +243,8 @@ disableAllInsertOptions useLedger =
     , ioRewards = False
     , ioMultiAssets = False
     , ioMetadata = False
-    , ioKeepMetadataNames = Strict.Nothing
+    , ioWhitelistMetadataNames = Strict.Nothing
+    , ioWhitelistMAPolicies = Strict.Nothing
     , ioPlutusExtra = False
     , ioOffChainPoolData = False
     , ioGov = False

cardano-db-sync/src/Cardano/DbSync/Api/Ledger.hs

@@ -167,7 +167,7 @@ prepareTxOut syncEnv txCache (TxIn txHash (TxIx index), txOut) = do
   let txHashByteString = Generic.safeHashToByteString $ unTxId txHash
   let genTxOut = fromTxOut index txOut
   txId <- queryTxIdWithCache txCache txHashByteString
-  Insert.prepareTxOut trce cache iopts (txId, txHashByteString) genTxOut
+  Insert.prepareTxOut syncEnv trce cache iopts (txId, txHashByteString) genTxOut
   where
     trce = getTrace syncEnv
     cache = envCache syncEnv

cardano-db-sync/src/Cardano/DbSync/Api/Types.hs

@@ -78,7 +78,8 @@ data InsertOptions = InsertOptions
   , ioRewards :: !Bool
   , ioMultiAssets :: !Bool
   , ioMetadata :: !Bool
-  , ioKeepMetadataNames :: Strict.Maybe [Word64]
+  , ioWhitelistMetadataNames :: Strict.Maybe [Word64]
+  , ioWhitelistMAPolicies :: Strict.Maybe [Word64]
   , ioPlutusExtra :: !Bool
   , ioOffChainPoolData :: !Bool
   , ioGov :: !Bool

cardano-db-sync/src/Cardano/DbSync/Config/Types.hs

@@ -72,7 +72,8 @@ data SyncNodeParams = SyncNodeParams
   , enpHasShelley :: !Bool
   , enpHasMultiAssets :: !Bool
   , enpHasMetadata :: !Bool
-  , enpKeepMetadataNames :: ![Word64]
+  , enpWhitelistMetadataNames :: ![Word64]
+  , enpWhitelistMAPolicies :: ![Word64]
   , enpHasPlutusExtra :: !Bool
   , enpHasGov :: !Bool
   , enpHasOffChainPoolData :: !Bool

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs

@@ -320,7 +320,7 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped
 
   if not (Generic.txValidContract tx)
     then do
-      !txOutsGrouped <- mapM (prepareTxOut tracer cache iopts (txId, txHash)) (Generic.txOutputs tx)
+      !txOutsGrouped <- mapM (prepareTxOut syncEnv tracer cache iopts (txId, txHash)) (Generic.txOutputs tx)
 
       let !txIns = map (prepareTxIn txId Map.empty) resolvedInputs
       -- There is a custom semigroup instance for BlockGroupedData which uses addition for the values `fees` and `outSum`.
@@ -329,7 +329,7 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped
     else do
       -- The following operations only happen if the script passes stage 2 validation (or the tx has
       -- no script).
-      !txOutsGrouped <- mapM (prepareTxOut tracer cache iopts (txId, txHash)) (Generic.txOutputs tx)
+      !txOutsGrouped <- mapM (prepareTxOut syncEnv tracer cache iopts (txId, txHash)) (Generic.txOutputs tx)
 
       !redeemers <-
         Map.fromList
@@ -365,7 +365,7 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped
 
       maTxMint <-
         whenFalseMempty (ioMetadata iopts) $
-          prepareMaTxMint tracer cache txId $
+          prepareMaTxMint syncEnv tracer cache txId $
             Generic.txMint tx
 
       when (ioPlutusExtra iopts) $
@@ -389,13 +389,14 @@ insertTx syncEnv isMember blkId epochNo slotNo applyResult blockIndex tx grouped
 
 prepareTxOut ::
   (MonadBaseControl IO m, MonadIO m) =>
+  SyncEnv ->
   Trace IO Text ->
   Cache ->
   InsertOptions ->
   (DB.TxId, ByteString) ->
   Generic.TxOut ->
   ExceptT SyncNodeError (ReaderT SqlBackend m) (ExtendedTxOut, [MissingMaTxOut])
-prepareTxOut tracer cache iopts (txId, txHash) (Generic.TxOut index addr addrRaw value maMap mScript dt) = do
+prepareTxOut syncEnv tracer cache iopts (txId, txHash) (Generic.TxOut index addr addrRaw value maMap mScript dt) = do
   mSaId <- lift $ insertStakeAddressRefIfMissing tracer cache addr
   mDatumId <-
     whenFalseEmpty (ioPlutusExtra iopts) Nothing $
@@ -420,7 +421,7 @@ prepareTxOut tracer cache iopts (txId, txHash) (Generic.TxOut index addr addrRaw
           , DB.txOutReferenceScriptId = mScriptId
           }
   let !eutxo = ExtendedTxOut txHash txOut
-  !maTxOuts <- whenFalseMempty (ioMultiAssets iopts) $ prepareMaTxOuts tracer cache maMap
+  !maTxOuts <- whenFalseMempty (ioMultiAssets iopts) $ prepareMaTxOuts syncEnv tracer cache maMap
   pure (eutxo, maTxOuts)
   where
     hasScript :: Bool
@@ -1220,7 +1221,7 @@ prepareTxMetadata tracer txId inOpts mmetadata = do
       (Word64, TxMetadataValue) ->
       ExceptT SyncNodeError (ReaderT SqlBackend m) (Maybe DB.TxMetadata)
     prepare (key, md) = do
-      case ioKeepMetadataNames inOpts of
+      case ioWhitelistMetadataNames inOpts of
         Strict.Just metadataNames -> do
           let isMatchingKey = key `elem` metadataNames
           if isMatchingKey
@@ -1327,12 +1328,14 @@ insertEpochParam _tracer blkId (EpochNo epoch) params nonce = do
 
 prepareMaTxMint ::
   (MonadBaseControl IO m, MonadIO m) =>
+  SyncEnv ->
   Trace IO Text ->
   Cache ->
   DB.TxId ->
   MultiAsset StandardCrypto ->
   ExceptT SyncNodeError (ReaderT SqlBackend m) [DB.MaTxMint]
-prepareMaTxMint _tracer cache txId (MultiAsset mintMap) =
+prepareMaTxMint syncEnv _tracer cache txId (MultiAsset mintMap) =
+  -- TODO: VINCE HERE
   concatMapM (lift . prepareOuter) $ Map.toList mintMap
   where
     prepareOuter ::
@@ -1348,7 +1351,7 @@ prepareMaTxMint _tracer cache txId (MultiAsset mintMap) =
       (AssetName, Integer) ->
       ReaderT SqlBackend m DB.MaTxMint
     prepareInner policy (aname, amount) = do
-      maId <- insertMultiAsset cache policy aname
+      maId <- insertMultiAsset syncEnv cache policy aname
       pure $
         DB.MaTxMint
           { DB.maTxMintIdent = maId
@@ -1358,11 +1361,12 @@ prepareMaTxMint _tracer cache txId (MultiAsset mintMap) =
 
 prepareMaTxOuts ::
   (MonadBaseControl IO m, MonadIO m) =>
+  SyncEnv ->
   Trace IO Text ->
   Cache ->
   Map (PolicyID StandardCrypto) (Map AssetName Integer) ->
   ExceptT SyncNodeError (ReaderT SqlBackend m) [MissingMaTxOut]
-prepareMaTxOuts _tracer cache maMap =
+prepareMaTxOuts syncEnv _tracer cache maMap =
   concatMapM (lift . prepareOuter) $ Map.toList maMap
   where
     prepareOuter ::
@@ -1378,7 +1382,7 @@ prepareMaTxOuts _tracer cache maMap =
       (AssetName, Integer) ->
       ReaderT SqlBackend m MissingMaTxOut
     prepareInner policy (aname, amount) = do
-      maId <- insertMultiAsset cache policy aname
+      maId <- insertMultiAsset syncEnv cache policy aname
       pure $
         MissingMaTxOut
           { mmtoIdent = maId
@@ -1387,11 +1391,12 @@ prepareMaTxOuts _tracer cache maMap =
 
 insertMultiAsset ::
   (MonadBaseControl IO m, MonadIO m) =>
+  SyncEnv ->
   Cache ->
   PolicyID StandardCrypto ->
   AssetName ->
   ReaderT SqlBackend m DB.MultiAssetId
-insertMultiAsset cache policy aName = do
+insertMultiAsset _syncEnv cache policy aName = do
   mId <- queryMAWithCache cache policy aName
   case mId of
     Right maId -> pure maId

doc/configuration.md

@@ -155,8 +155,7 @@ Some field are left empty when using this flag, like
 
 Until the ledger state migration happens any restart requires reusing the `--bootstrap-tx-out` flag. After it's completed the flag can be omitted on restarts.
 
-### --keep-tx-metadata
+### --whitelist-tx-metadata
 
-This flag was introduced in v.13.2.0.0 as all postgres field with the type jsonb were removed to improve insertion performance.
-If they are required and you have database queries against jsonb then activate this flag to re-introduce the type jsonb.
-You can pass multiple values to the flag eg: `--keep-tx-metadata 1,2,3` make sure you are using commas between each key.
+To help improved database insert thoughput, user can chose to filter specific tx metadata they would like to keep and insert, ignore everything else.
+You can pass multiple values to the flag eg: `--whitelist-tx-metadata 1,2,3` make sure you are using commas between each key.