update docmentation for db-tool for address table schema

Cmdv · Cmdv · commit b1341142449b · 2024-09-27T13:11:32.000+01:00
diff --git a/cardano-db/app/gen-schema-docs.hs b/cardano-db/app/gen-schema-docs.hs
@@ -1,6 +1,8 @@
 {-# LANGUAGE OverloadedStrings #-}
 
 import Cardano.Db (schemaDocs)
+import Cardano.Db.Schema.Core.TxOut (schemaDocsTxOutCore)
+import Cardano.Db.Schema.Variant.TxOut (schemaDocsTxOutVariant)
 import Data.Text (Text)
 import qualified Data.Text as Text
 import qualified Data.Text.IO as Text
@@ -67,10 +69,18 @@ docHeader branchName =
     ]
 
 docBody :: Text
-docBody =
-  Text.replace "ID:" "Id:"
-    . Text.replace "#" "###"
-    $ render markdownTableRenderer schemaDocs
+docBody = do
+  coreDocBody <> variantDivider <> variantDocBody
+  where
+    coreDocBody = cleanUp $ render markdownTableRenderer (schemaDocs <> schemaDocsTxOutCore)
+    variantDocBody = cleanUp $ render markdownTableRenderer schemaDocsTxOutVariant
+    cleanUp = Text.replace "ID:" "Id:" . Text.replace "#" "###"
+    variantDivider =
+      mconcat
+        [ "# Variant Schema\n\n"
+        , "When using the `use_address_table` [configuration](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#tx-out), the `tx_out` table is split into two tables: `tx_out` and `address`.\n"
+        , "Bellow are the table documentation for this variaton. \n\n"
+        ]
 
 readGitBranch :: IO Text
 readGitBranch = do
diff --git a/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs b/cardano-db/src/Cardano/Db/Operations/Other/ConsumedTxOut.hs
@@ -101,17 +101,19 @@ runExtraMigrations trce txOutTableType blockNoDiff pcm = do
   ems <- queryAllExtraMigrations
   isTxOutNull <- queryTxOutIsNull txOutTableType
   let migrationValues = processMigrationValues ems pcm
+      isTxOutVariant = isTxOutVariantAddress txOutTableType
+      isTxOutAddressSet = isTxOutAddressPreviouslySet migrationValues
 
-  -- can only run "use_address_table" on a non populated database
-  when (isTxOutVariantAddress txOutTableType && not isTxOutNull) $
+  -- can only run "use_address_table" on a non populated database but don't throw if the migration was previously set
+  when (isTxOutVariant && not isTxOutNull && not isTxOutAddressSet) $
     throw $
-      DBExtraMigration "runExtraMigrations: The use the config 'tx_out.use_address_table' can only be caried out on a non populated database."
+      DBExtraMigration "runExtraMigrations: The use of the config 'tx_out.use_address_table' can only be caried out on a non populated database."
   -- Make sure the config "use_address_table" is there if the migration wasn't previously set in the past
-  when (not (isTxOutVariantAddress txOutTableType) && isTxOutAddressPreviouslySet migrationValues) $
+  when (not isTxOutVariant && isTxOutAddressSet) $
     throw $
       DBExtraMigration "runExtraMigrations: The configuration option 'tx_out.use_address_table' was previously set and the database updated. Unfortunately reverting this isn't possible."
   -- Has the user given txout address config && the migration wasn't previously set
-  when (isTxOutVariantAddress txOutTableType && not (isTxOutAddressPreviouslySet migrationValues)) $ do
+  when (isTxOutVariant && not isTxOutAddressSet) $ do
     updateTxOutAndCreateAddress
     insertExtraMigration TxOutAddressPreviouslySet
   -- first check if pruneTxOut flag is missing and it has previously been used
@@ -239,10 +241,8 @@ migrateTxOut ::
   ReaderT SqlBackend m ()
 migrateTxOut trce txOutTableType mMvs = do
   whenJust mMvs $ \mvs -> do
-    liftIO $ logInfo trce $ "migrateTxOut: previously set: " <> textShow (not (isTxOutAddressPreviouslySet mvs))
-    liftIO $ logInfo trce $ "migrateTxOut: pcmConsumedTxOut: " <> textShow (pcmConsumedTxOut (pruneConsumeMigration mvs))
     when (pcmConsumedTxOut (pruneConsumeMigration mvs) && not (isTxOutAddressPreviouslySet mvs)) $ do
-      liftIO $ logInfo trce "migrateTxOut: addeding consumed-by-id Index"
+      liftIO $ logInfo trce "migrateTxOut: adding consumed-by-id Index"
       void createConsumedIndexTxOut
     when (pcmPruneTxOut (pruneConsumeMigration mvs)) $ do
       liftIO $ logInfo trce "migrateTxOut: adding prune contraint on tx_out table"
@@ -259,12 +259,6 @@ migrateNextPageTxOut mTrce txOutTableType offst = do
     migrateNextPageTxOut mTrce txOutTableType $!
       (offst + pageSize)
 
--- TODO: cmdv put into tools or something
-migrateTxOutDbTool :: (MonadIO m, MonadBaseControl IO m) => TxOutTableType -> ReaderT SqlBackend m ()
-migrateTxOutDbTool txOutTableType = do
-  _ <- createConsumedIndexTxOut
-  migrateNextPageTxOut Nothing txOutTableType 0
-
 --------------------------------------------------------------------------------------------------
 -- Delete + Update
 --------------------------------------------------------------------------------------------------
@@ -458,7 +452,7 @@ updateTxOutAndCreateAddress = do
       "CREATE INDEX IF NOT EXISTS idx_address_payment_cred ON address(payment_cred);"
 
     createIndexRawQuery =
-      "CREATE INDEX IF NOT EXISTS idx_address_raw ON address(raw);"
+      "CREATE INDEX IF NOT EXISTS idx_address_raw ON address USING HASH (raw);"
 
     exceptHandler :: SqlError -> ReaderT SqlBackend m a
     exceptHandler e =
@@ -490,6 +484,11 @@ deleteConsumedBeforeTx trce txOutTableType txId = do
 --------------------------------------------------------------------------------------------------
 -- Helpers
 --------------------------------------------------------------------------------------------------
+migrateTxOutDbTool :: (MonadIO m, MonadBaseControl IO m) => TxOutTableType -> ReaderT SqlBackend m ()
+migrateTxOutDbTool txOutTableType = do
+  _ <- createConsumedIndexTxOut
+  migrateNextPageTxOut Nothing txOutTableType 0
+
 findMaxTxInId :: forall m. MonadIO m => Word64 -> ReaderT SqlBackend m (Either Text TxId)
 findMaxTxInId blockNoDiff = do
   mBlockHeight <- queryBlockHeight
diff --git a/cardano-db/src/Cardano/Db/Schema/Core/TxOut.hs b/cardano-db/src/Cardano/Db/Schema/Core/TxOut.hs
@@ -30,7 +30,7 @@ import Database.Persist.TH
 share
   [ mkPersist sqlSettings
   , mkMigrate "migrateCoreTxOutCardanoDb"
-  , mkEntityDefList "entityDefs"
+  , mkEntityDefList "entityDefsTxOutCore"
   , deriveShowFields
   ]
   [persistLowerCase|
@@ -62,13 +62,14 @@ share
 
 |]
 
-schemaDocs :: [EntityDef]
-schemaDocs =
-  document entityDefs $ do
+schemaDocsTxOutCore :: [EntityDef]
+schemaDocsTxOutCore =
+  document entityDefsTxOutCore $ do
     TxOut --^ do
       "A table for transaction outputs."
       TxOutAddress # "The human readable encoding of the output address. Will be Base58 for Byron era addresses and Bech32 for Shelley era."
       TxOutAddressHasScript # "Flag which shows if this address is locked by a script."
+      TxOutConsumedByTxId # "The Tx table index of the transaction that consumes this transaction output. Not populated by default, can be activated via tx-out configs."
       TxOutDataHash # "The hash of the transaction output datum. (NULL for Txs without scripts)."
       TxOutIndex # "The index of this transaction output with the transaction."
       TxOutInlineDatumId # "The inline datum of the output, if it has one. New in v13."
diff --git a/cardano-db/src/Cardano/Db/Schema/Variant/TxOut.hs b/cardano-db/src/Cardano/Db/Schema/Variant/TxOut.hs
@@ -30,7 +30,7 @@ import Database.Persist.TH
 share
   [ mkPersist sqlSettings
   , mkMigrate "migrateVariantAddressCardanoDb"
-  , mkEntityDefList "entityDefs"
+  , mkEntityDefList "entityDefsTxOutVariant"
   , deriveShowFields
   ]
   [persistLowerCase|
@@ -65,12 +65,13 @@ share
     deriving Show
 |]
 
-schemaDocs :: [EntityDef]
-schemaDocs =
-  document entityDefs $ do
+schemaDocsTxOutVariant :: [EntityDef]
+schemaDocsTxOutVariant =
+  document entityDefsTxOutVariant $ do
     TxOut --^ do
       "A table for transaction outputs."
       TxOutAddressId # "The human readable encoding of the output address. It is Base58 for Byron era addresses and Bech32 for Shelley era."
+      TxOutConsumedByTxId # "The Tx table index of the transaction that consumes this transaction output. Not populated by default, can be activated via tx-out configs."
       TxOutDataHash # "The hash of the transaction output datum. (NULL for Txs without scripts)."
       TxOutIndex # "The index of this transaction output with the transaction."
       TxOutInlineDatumId # "The inline datum of the output, if it has one. New in v13."
diff --git a/doc/configuration.md b/doc/configuration.md
@@ -280,9 +280,7 @@ can be changed.
 
  * Type: `boolean`
  
-This value defaults to false. 
-<!-- TODO: Need to add more of a description as to what it does -->
-
+When using `consumed` configuration `tx_in` will not be populated. That behaviour can be overridden by setting this value to `true`.
 
 ### Address Table
 
diff --git a/doc/schema-management.md b/doc/schema-management.md
@@ -15,13 +15,11 @@ indexes during syncing slows down db-sync and so they are added later. Index
 creation is idempotent and the `schema_version.stage_tree` field is ignored.
 These files cannot be modified but they can be extended, in case users want to
 introduce their own indexes from the begining.
-- `stage 4`: introduces all the other indexes. By default these are the indexes
-that were created by previous db-sync versions. This stage is executed when
-db-sync has reached 30mins before the tip of the chain. It is advised to increase
-the `maintenance_work_mem` from Postgres config to 0.5GB - 1GB to speed this
-process (default is 64MB). Also use the default (2) or higher
-`max_parallel_maintenance_workers`. These files can be modified or extended
-by users.
+- `stage 4`: introduces all the other indexes. By default these are the indexes that were created by previous db-sync versions. This stage is executed when
+db-sync has reached 30mins before the tip of the chain. It is advised to increase the `maintenance_work_mem` from Postgres config to 0.5GB - 1GB to speed this
+process (default is 64MB). 
+Also use the default (2) or higher `max_parallel_maintenance_workers`. These files can be modified or extended
+by users. 
 
 All of the schema migrations in these three stages are written to be idempotent (so that they
 "know" if they have already been applied).
@@ -34,6 +32,11 @@ where the `1` denotes "stage 1" of the SQL migration, the `0000` is the migratio
 last number is the date. Listing the directory containing the schema and sorting the list will
 order them in the correct order for applying to the database.
 
+Since the introduction of `use_address_table` [config](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#tx-out). The file `migration-4-001-*` when indexing will not be ran when the this configuration is active.
+
+There is also a flag you can use in cardano-db-tool `--use-tx-out-address` which handles the alternate variation of the schema, one might be using.
+
+
 ## Creating a Migration
 
 Whenever the Haskell schema definition in `Cardano.Db.Schema` is updated, a schema migration will need to be migrated.
@@ -52,6 +55,12 @@ cabal run cardano-db-tool -- create-migration --mdir schema/
 ```
 This will generate a migration if one is needed. 
 
+There is an alternate way to create/run a migration when using the `use_txout_address` configuration as previously mentioned, this uses a different variation of the schema.
+To do this, you simply add the flag `--use-tx-out-address` like so:
+```
+PGPASSFILE=config/pgpass-mainnet cabal run cardano-db-tool -- create-migration --use-tx-out-address --mdir schema/
+```
+
 Once this has completed it's good practice to rebuild `cardano-db-sync` due to how it caches schema files when built, this can be done using the following documentation [Build and Install](./installing.md#build-and-install)
 
 **Note:**  For extra reassurance one can run the test suite to check that the new migration hasn't broken any tests:
diff --git a/doc/schema.md b/doc/schema.md
@@ -1,5 +1,8 @@
 # Schema Documentation for cardano-db-sync
 
+Schema version: 13.5.0.2 (from branch **1333-new-AddressDetail-table** which may not accurately reflect the version number)
+**Note:** This file is auto-generated from the documentation in cardano-db/src/Cardano/Db/Schema/BaseSchema.hs by the command `cabal run -- gen-schema-docs doc/schema.md`. This document should only be updated during the release process and updated on the release branch.
+
 ### `schema_version`
 
 The version of the database schema. Schema versioning is split into three stages as detailed below. This table should only ever have a single row.
@@ -122,26 +125,6 @@ A table of unique stake addresses. Can be an actual address or a script hash.  T
 | `view` | string | The Bech32 encoded version of the stake address. |
 | `script_hash` | hash28type | The script hash, in case this address is locked by a script. |
 
-### `tx_out`
-
-A table for transaction outputs.
-
-* Primary Id: `id`
-
-| Column name | Type | Description |
-|-|-|-|
-| `id` | integer (64) |  |
-| `tx_id` | integer (64) | The Tx table index of the transaction that contains this transaction output. |
-| `index` | txindex | The index of this transaction output with the transaction. |
-| `address` | string | The human readable encoding of the output address. Will be Base58 for Byron era addresses and Bech32 for Shelley era. |
-| `address_has_script` | boolean | Flag which shows if this address is locked by a script. |
-| `payment_cred` | hash28type | The payment credential part of the Shelley address. (NULL for Byron addresses). For a script-locked address, this is the script hash. |
-| `stake_address_id` | integer (64) | The StakeAddress table index for the stake address part of the Shelley address. (NULL for Byron addresses). |
-| `value` | lovelace | The output value (in Lovelace) of the transaction output. |
-| `data_hash` | hash32type | The hash of the transaction output datum. (NULL for Txs without scripts). |
-| `inline_datum_id` | integer (64) | The inline datum of the output, if it has one. New in v13. |
-| `reference_script_id` | integer (64) | The reference script of the output, if it has one. New in v13. |
-
 ### `collateral_tx_out`
 
 A table for transaction collateral outputs. New in v13.
@@ -545,19 +528,6 @@ A table containing Multi-Asset mint events.
 | `quantity` | int65type | The amount of the Multi Asset to mint (can be negative to "burn" assets). |
 | `tx_id` | integer (64) | The Tx table index for the transaction that contains this minting event. |
 
-### `ma_tx_out`
-
-A table containing Multi-Asset transaction outputs.
-
-* Primary Id: `id`
-
-| Column name | Type | Description |
-|-|-|-|
-| `id` | integer (64) |  |
-| `ident` | integer (64) | The MultiAsset table index specifying the asset. |
-| `quantity` | word64type | The Multi Asset transaction output amount (denominated in the Multi Asset). |
-| `tx_out_id` | integer (64) | The TxOut table index for the transaction that this Multi Asset transaction output. |
-
 ### `redeemer`
 
 A table containing redeemers. A redeemer is provided for all items that are validated by a script.
@@ -1196,4 +1166,89 @@ A table containing pools that have been delisted.
 | `id` | integer (64) |  |
 | `hash_raw` | hash28type | The pool hash |
 
+### `tx_out`
+
+A table for transaction outputs.
+
+* Primary Id: `id`
+
+| Column name | Type | Description |
+|-|-|-|
+| `id` | integer (64) |  |
+| `address` | string | The human readable encoding of the output address. Will be Base58 for Byron era addresses and Bech32 for Shelley era. |
+| `address_has_script` | boolean | Flag which shows if this address is locked by a script. |
+| `data_hash` | hash32type | The hash of the transaction output datum. (NULL for Txs without scripts). |
+| `consumed_by_tx_id` | integer (64) | The Tx table index of the transaction that consumes this transaction output. Not populated by default, can be activated via tx-out configs. |
+| `index` | txindex | The index of this transaction output with the transaction. |
+| `inline_datum_id` | integer (64) | The inline datum of the output, if it has one. New in v13. |
+| `payment_cred` | hash28type | The payment credential part of the Shelley address. (NULL for Byron addresses). For a script-locked address, this is the script hash. |
+| `reference_script_id` | integer (64) | The reference script of the output, if it has one. New in v13. |
+| `stake_address_id` | integer (64) | The StakeAddress table index for the stake address part of the Shelley address. (NULL for Byron addresses). |
+| `tx_id` | integer (64) | The Tx table index of the transaction that contains this transaction output. |
+| `value` | lovelace | The output value (in Lovelace) of the transaction output. |
+
+### `ma_tx_out`
+
+A table containing Multi-Asset transaction outputs.
+
+* Primary Id: `id`
+
+| Column name | Type | Description |
+|-|-|-|
+| `id` | integer (64) |  |
+| `ident` | integer (64) | The MultiAsset table index specifying the asset. |
+| `quantity` | word64type | The Multi Asset transaction output amount (denominated in the Multi Asset). |
+| `tx_out_id` | integer (64) | The TxOut table index for the transaction that this Multi Asset transaction output. |
+
+# Variant Schema
+
+When using the `use_address_table` [configuration](https://github.com/IntersectMBO/cardano-db-sync/blob/master/doc/configuration.md#tx-out), the `tx_out` table is split into two tables: `tx_out` and `address`.
+Bellow are the table documentation for this variaton. 
+
+### `tx_out`
+
+A table for transaction outputs.
+
+* Primary Id: `id`
+
+| Column name | Type | Description |
+|-|-|-|
+| `id` | integer (64) |  |
+| `address_id` | integer (64) | The human readable encoding of the output address. It is Base58 for Byron era addresses and Bech32 for Shelley era. |
+| `consumed_by_tx_id` | integer (64) | The Tx table index of the transaction that consumes this transaction output. Not populated by default, can be activated via tx-out configs. |
+| `data_hash` | hash32type | The hash of the transaction output datum. (NULL for Txs without scripts). |
+| `index` | txindex | The index of this transaction output with the transaction. |
+| `inline_datum_id` | integer (64) | The inline datum of the output, if it has one. New in v13. |
+| `reference_script_id` | integer (64) | The reference script of the output, if it has one. New in v13. |
+| `tx_id` | integer (64) | The Tx table index of the transaction that contains this transaction output. |
+| `value` | lovelace | The output value (in Lovelace) of the transaction output. |
+
+### `address`
+
+A table for addresses that appear in outputs.
+
+* Primary Id: `id`
+
+| Column name | Type | Description |
+|-|-|-|
+| `id` | integer (64) |  |
+| `address` | string | The human readable encoding of the output address. Will be Base58 for Byron era addresses and Bech32 for Shelley era. |
+| `raw` | blob | The raw binary address. |
+| `has_script` | boolean | Flag which shows if this address is locked by a script. |
+| `payment_cred` | hash28type | The payment credential part of the Shelley address. (NULL for Byron addresses). For a script-locked address, this is the script hash. |
+| `stake_address_id` | integer (64) | The StakeAddress table index for the stake address part of the Shelley address. (NULL for Byron addresses). |
+
+### `ma_tx_out`
+
+A table containing Multi-Asset transaction outputs.
+
+* Primary Id: `id`
+
+| Column name | Type | Description |
+|-|-|-|
+| `id` | integer (64) |  |
+| `ident` | integer (64) | The MultiAsset table index specifying the asset. |
+| `quantity` | word64type | The Multi Asset transaction output amount (denominated in the Multi Asset). |
+| `tx_out_id` | integer (64) | The TxOut table index for the transaction that this Multi Asset transaction output. |
+
 
diff --git a/flake.nix b/flake.nix
@@ -202,7 +202,7 @@
                 packages.cardano-db.package.extraSrcFiles =
                   ["../config/pgpass-testnet"];
                 packages.cardano-db.components.tests.schema-rollback.extraSrcFiles =
-                  [ "src/Cardano/Db/Schema.hs" "src/Cardano/Db/Operations/Delete.hs" ];
+                  [ "src/Cardano/Db/BaseSchema.hs" "src/Cardano/Db/Operations/Delete.hs" ];
                 packages.cardano-db.components.tests.test-db.extraSrcFiles =
                   [ "../config/pgpass-mainnet" ];
                 packages.cardano-chain-gen.package.extraSrcFiles =
diff --git a/schema/migration-4-0001-20200702.sql b/schema/migration-4-0001-20200702.sql
@@ -4,7 +4,7 @@
 --
 
 CREATE INDEX IF NOT EXISTS idx_tx_out_payment_cred ON tx_out(payment_cred);
-CREATE INDEX IF NOT EXISTS idx_tx_out_stake_address_id ON tx_out(stake_address_id) ;
+CREATE INDEX IF NOT EXISTS idx_tx_out_stake_address_id ON tx_out(stake_address_id);
 
 -- Left here for reference, it's removed to speed up restoring from a snapshot as this index is very slow to create.
 -- CREATE INDEX IF NOT EXISTS idx_tx_out_address ON tx_out USING hash (address);
