Skip to content

Parallelize #1960

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 13 commits into
base: master
Choose a base branch
from
Draft

Parallelize #1960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
cc6ffbc
Add Epoch Stake Thread
kderme Mar 18, 2025
e7e0f4b
Add Stake thread
kderme Mar 21, 2025
ed8a586
Regroup modules
kderme Mar 21, 2025
064aaf1
Add Resolve Inputs thread
kderme Mar 23, 2025
256d0f5
Start using explocit Ids for Tx and Block
kderme Apr 4, 2025
6bc6098
Parallelise preparing the txs
kderme Apr 12, 2025
02aaaca
Parallelize ledger application
kderme Apr 12, 2025
c38499b
Force calculation of rewards
kderme Apr 24, 2025
7209d87
Add rewards thread
kderme Apr 24, 2025
6de7ea8
Fixes
kderme Apr 26, 2025
04576b6
Fix generated txIds
kderme Apr 27, 2025
3373fee
Add MA thread and dedicated connections for datum and scripts
kderme Apr 30, 2025
b4934be
Commit secondary transactions before the main one
kderme Apr 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 13 additions & 2 deletions cardano-db-sync/cardano-db-sync.cabal
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ library
Cardano.DbSync.Api
Cardano.DbSync.Api.Ledger
Cardano.DbSync.Api.Types
Cardano.DbSync.Ledger.Async
Cardano.DbSync.Config
Cardano.DbSync.Config.Alonzo
Cardano.DbSync.Config.Byron
Expand All @@ -53,7 +54,6 @@ library
Cardano.DbSync.Config.Node
Cardano.DbSync.Config.Shelley
Cardano.DbSync.Config.Types
Cardano.DbSync.Database
Cardano.DbSync.DbAction
Cardano.DbSync.Error

Expand Down Expand Up @@ -109,12 +109,14 @@ library

Cardano.DbSync.Metrics

Cardano.DbSync.Block
Cardano.DbSync.Cache
Cardano.DbSync.Cache.Epoch
Cardano.DbSync.Cache.FIFO
Cardano.DbSync.Cache.LRU
Cardano.DbSync.Cache.Stake
Cardano.DbSync.Cache.Types
Cardano.DbSync.Default
Cardano.DbSync.Cache.Util
Cardano.DbSync.Epoch

Cardano.DbSync.Rollback
Expand All @@ -130,6 +132,13 @@ library
Cardano.DbSync.LocalStateQuery
Cardano.DbSync.StateQuery
Cardano.DbSync.Sync
Cardano.DbSync.Threads.Database
Cardano.DbSync.Threads.EpochStake
Cardano.DbSync.Threads.Ledger
Cardano.DbSync.Threads.MultiAsset
Cardano.DbSync.Threads.Rewards
Cardano.DbSync.Threads.Stake
Cardano.DbSync.Threads.TxInResolve
Cardano.DbSync.Tracing.ToObjectOrphans
Cardano.DbSync.Types

Expand Down Expand Up @@ -209,6 +218,8 @@ library
, prometheus
, psqueues
, random-shuffle
, resourcet
, resource-pool
, scientific
, serialise
, small-steps
Expand Down
27 changes: 17 additions & 10 deletions cardano-db-sync/src/Cardano/DbSync.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,18 +29,21 @@ import qualified Cardano.Crypto as Crypto
import qualified Cardano.Db as DB
import qualified Cardano.Db as Db
import Cardano.DbSync.Api
import Cardano.DbSync.Api.Types (InsertOptions (..), RunMigration, SyncEnv (..), SyncOptions (..), envLedgerEnv)
import Cardano.DbSync.Api.Types
import Cardano.DbSync.Config (configureLogging)
import Cardano.DbSync.Config.Cardano
import Cardano.DbSync.Config.Types
import Cardano.DbSync.Database
import Cardano.DbSync.DbAction
import Cardano.DbSync.Era
import Cardano.DbSync.Error
import Cardano.DbSync.Ledger.State
import Cardano.DbSync.OffChain (runFetchOffChainPoolThread, runFetchOffChainVoteThread)
import Cardano.DbSync.Rollback (unsafeRollback)
import Cardano.DbSync.Sync (runSyncNodeClient)
import Cardano.DbSync.Threads.Database
import Cardano.DbSync.Threads.EpochStake
import Cardano.DbSync.Threads.Ledger
import Cardano.DbSync.Threads.Stake
import Cardano.DbSync.Tracing.ToObjectOrphans ()
import Cardano.DbSync.Types
import Cardano.DbSync.Util.Constraint (queryIsJsonbInSchema)
Expand All @@ -51,7 +54,7 @@ import Control.Monad.Extra (whenJust)
import qualified Data.Strict.Maybe as Strict
import qualified Data.Text as Text
import Data.Version (showVersion)
import Database.Persist.Postgresql (ConnectionString, withPostgresqlConn)
import Database.Persist.Postgresql (ConnectionString)
import qualified Ouroboros.Consensus.HardFork.Simple as HardFork
import Ouroboros.Network.NodeToClient (IOManager, withIOManager)
import Paths_cardano_db_sync (version)
Expand Down Expand Up @@ -163,20 +166,18 @@ runSyncNode metricsSetters trce iomgr dbConnString runMigrationFnc syncNodeConfi
logInfo trce $ "Using shelley genesis file from: " <> (show . unGenesisFile $ dncShelleyGenesisFile syncNodeConfigFromFile)
logInfo trce $ "Using alonzo genesis file from: " <> (show . unGenesisFile $ dncAlonzoGenesisFile syncNodeConfigFromFile)

let useLedger = shouldUseLedger (sioLedger $ dncInsertOptions syncNodeConfigFromFile)

Db.runIohkLogging trce $
withPostgresqlConn dbConnString $
\backend -> liftIO $ do
withDBSyncConnections dbConnString $
\backends -> liftIO $ do
runOrThrowIO $ runExceptT $ do
genCfg <- readCardanoGenesisConfig syncNodeConfigFromFile
isJsonbInSchema <- queryIsJsonbInSchema backend
isJsonbInSchema <- queryIsJsonbInSchema (mainBackend backends)
logProtocolMagicId trce $ genesisProtocolMagicId genCfg
syncEnv <-
ExceptT $
mkSyncEnvFromConfig
trce
backend
backends
dbConnString
syncOptions
genCfg
Expand All @@ -196,7 +197,7 @@ runSyncNode metricsSetters trce iomgr dbConnString runMigrationFnc syncNodeConfi
liftIO $ runExtraMigrationsMaybe syncEnv
unless useLedger $ liftIO $ do
logInfo trce "Migrating to a no ledger schema"
Db.noLedgerMigrations backend trce
Db.noLedgerMigrations (mainBackend backends) trce
insertValidateGenesisDist syncEnv (dncNetworkName syncNodeConfigFromFile) genCfg (useShelleyInit syncNodeConfigFromFile)

-- communication channel between datalayer thread and chainsync-client thread
Expand All @@ -206,6 +207,9 @@ runSyncNode metricsSetters trce iomgr dbConnString runMigrationFnc syncNodeConfi
id
[ runDbThread syncEnv metricsSetters threadChannels
, runSyncNodeClient metricsSetters syncEnv iomgr trce threadChannels (enpSocketPath syncNodeParams)
, runLedgerThread syncEnv
, runEpochStakeThread syncEnv
, runStakeThread syncEnv
, runFetchOffChainPoolThread syncEnv
, runFetchOffChainVoteThread syncEnv
, runLedgerStateWriteThread (getTrace syncEnv) (envLedgerEnv syncEnv)
Expand All @@ -219,6 +223,8 @@ runSyncNode metricsSetters trce iomgr dbConnString runMigrationFnc syncNodeConfi
removeJsonbFromSchemaConfig = ioRemoveJsonbFromSchema $ soptInsertOptions syncOptions
maybeLedgerDir = enpMaybeLedgerStateDir syncNodeParams

useLedger = shouldUseLedger (sioLedger $ dncInsertOptions syncNodeConfigFromFile)

logProtocolMagicId :: Trace IO Text -> Crypto.ProtocolMagicId -> ExceptT SyncNodeError IO ()
logProtocolMagicId tracer pm =
liftIO
Expand All @@ -237,6 +243,7 @@ extractSyncOptions snp aop snc =
not isTxOutConsumedBootstrap'
&& ioInOut iopts
&& not (enpEpochDisabled snp || not (enpHasCache snp))
&& False
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I assume this is for testing

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes I have it in the TODO list of the pr

, soptAbortOnInvalid = aop
, soptCache = enpHasCache snp
, soptPruneConsumeMigration =
Expand Down
Loading
Loading