diff --git a/cmd/conf/init.go b/cmd/conf/init.go index 97b31958d5..c490013bf6 100644 --- a/cmd/conf/init.go +++ b/cmd/conf/init.go @@ -13,61 +13,63 @@ import ( ) type InitConfig struct { - Force bool `koanf:"force"` - Url string `koanf:"url"` - Latest string `koanf:"latest"` - LatestBase string `koanf:"latest-base"` - ValidateChecksum bool `koanf:"validate-checksum"` - DownloadPath string `koanf:"download-path"` - DownloadPoll time.Duration `koanf:"download-poll"` - DevInit bool `koanf:"dev-init"` - DevInitAddress string `koanf:"dev-init-address"` - DevMaxCodeSize uint64 `koanf:"dev-max-code-size"` - DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` - Empty bool `koanf:"empty"` - ImportWasm bool `koanf:"import-wasm"` - AccountsPerSync uint `koanf:"accounts-per-sync"` - ImportFile string `koanf:"import-file"` - GenesisJsonFile string `koanf:"genesis-json-file"` - ThenQuit bool `koanf:"then-quit"` - Prune string `koanf:"prune"` - PruneBloomSize uint64 `koanf:"prune-bloom-size"` - PruneThreads int `koanf:"prune-threads"` - PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` - RecreateMissingStateFrom uint64 `koanf:"recreate-missing-state-from"` - RebuildLocalWasm string `koanf:"rebuild-local-wasm"` - ReorgToBatch int64 `koanf:"reorg-to-batch"` - ReorgToMessageBatch int64 `koanf:"reorg-to-message-batch"` - ReorgToBlockBatch int64 `koanf:"reorg-to-block-batch"` + Force bool `koanf:"force"` + Url string `koanf:"url"` + Latest string `koanf:"latest"` + LatestBase string `koanf:"latest-base"` + ValidateChecksum bool `koanf:"validate-checksum"` + DownloadPath string `koanf:"download-path"` + DownloadPoll time.Duration `koanf:"download-poll"` + DevInit bool `koanf:"dev-init"` + DevInitAddress string `koanf:"dev-init-address"` + DevMaxCodeSize uint64 `koanf:"dev-max-code-size"` + DevInitBlockNum uint64 `koanf:"dev-init-blocknum"` + Empty bool `koanf:"empty"` + ImportWasm bool `koanf:"import-wasm"` + AccountsPerSync uint `koanf:"accounts-per-sync"` + ImportFile string `koanf:"import-file"` + GenesisJsonFile string `koanf:"genesis-json-file"` + ThenQuit bool `koanf:"then-quit"` + Prune string `koanf:"prune"` + PruneParallelStorageTraversal bool `koanf:"prune-parallel-storage-traversal"` + PruneBloomSize uint64 `koanf:"prune-bloom-size"` + PruneThreads int `koanf:"prune-threads"` + PruneTrieCleanCache int `koanf:"prune-trie-clean-cache"` + RecreateMissingStateFrom uint64 `koanf:"recreate-missing-state-from"` + RebuildLocalWasm string `koanf:"rebuild-local-wasm"` + ReorgToBatch int64 `koanf:"reorg-to-batch"` + ReorgToMessageBatch int64 `koanf:"reorg-to-message-batch"` + ReorgToBlockBatch int64 `koanf:"reorg-to-block-batch"` } var InitConfigDefault = InitConfig{ - Force: false, - Url: "", - Latest: "", - LatestBase: "https://snapshot.arbitrum.foundation/", - ValidateChecksum: true, - DownloadPath: "/tmp/", - DownloadPoll: time.Minute, - DevInit: false, - DevInitAddress: "", - DevMaxCodeSize: 0, - DevInitBlockNum: 0, - Empty: false, - ImportWasm: false, - ImportFile: "", - GenesisJsonFile: "", - AccountsPerSync: 100000, - ThenQuit: false, - Prune: "", - PruneBloomSize: 2048, - PruneThreads: runtime.NumCPU(), - PruneTrieCleanCache: 600, - RecreateMissingStateFrom: 0, // 0 = disabled - RebuildLocalWasm: "auto", - ReorgToBatch: -1, - ReorgToMessageBatch: -1, - ReorgToBlockBatch: -1, + Force: false, + Url: "", + Latest: "", + LatestBase: "https://snapshot.arbitrum.foundation/", + ValidateChecksum: true, + DownloadPath: "/tmp/", + DownloadPoll: time.Minute, + DevInit: false, + DevInitAddress: "", + DevMaxCodeSize: 0, + DevInitBlockNum: 0, + Empty: false, + ImportWasm: false, + ImportFile: "", + GenesisJsonFile: "", + AccountsPerSync: 100000, + ThenQuit: false, + Prune: "", + PruneParallelStorageTraversal: false, + PruneBloomSize: 2048, + PruneThreads: runtime.NumCPU(), + PruneTrieCleanCache: 600, + RecreateMissingStateFrom: 0, // 0 = disabled + RebuildLocalWasm: "auto", + ReorgToBatch: -1, + ReorgToMessageBatch: -1, + ReorgToBlockBatch: -1, } func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { @@ -89,6 +91,7 @@ func InitConfigAddOptions(prefix string, f *pflag.FlagSet) { f.String(prefix+".genesis-json-file", InitConfigDefault.GenesisJsonFile, "path for genesis json file") f.Uint(prefix+".accounts-per-sync", InitConfigDefault.AccountsPerSync, "during init - sync database every X accounts. Lower value for low-memory systems. 0 disables.") f.String(prefix+".prune", InitConfigDefault.Prune, "pruning for a given use: \"full\" for full nodes serving RPC requests, or \"validator\" for validators") + f.Bool(prefix+".prune-parallel-storage-traversal", InitConfigDefault.PruneParallelStorageTraversal, "if true: use parallel pruning per account") f.Uint64(prefix+".prune-bloom-size", InitConfigDefault.PruneBloomSize, "the amount of memory in megabytes to use for the pruning bloom filter (higher values prune better)") f.Int(prefix+".prune-threads", InitConfigDefault.PruneThreads, "the number of threads to use when pruning") f.Int(prefix+".prune-trie-clean-cache", InitConfigDefault.PruneTrieCleanCache, "amount of memory in megabytes to cache unchanged state trie nodes with when traversing state database during pruning") diff --git a/cmd/pruning/pruning.go b/cmd/pruning/pruning.go index e89c79bc89..0d65ee496f 100644 --- a/cmd/pruning/pruning.go +++ b/cmd/pruning/pruning.go @@ -248,7 +248,7 @@ func PruneChainDb(ctx context.Context, chainDb ethdb.Database, stack *node.Node, return fmt.Errorf("failed to find root to retain for pruning: %w", err) } - pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize, Threads: initConfig.PruneThreads, CleanCacheSize: initConfig.PruneTrieCleanCache}) + pruner, err := pruner.NewPruner(chainDb, pruner.Config{Datadir: stack.InstanceDir(), BloomSize: initConfig.PruneBloomSize, Threads: initConfig.PruneThreads, CleanCacheSize: initConfig.PruneTrieCleanCache, ParallelStorageTraversal: initConfig.PruneParallelStorageTraversal}) if err != nil { return err } diff --git a/go-ethereum b/go-ethereum index 88b0e9d292..a0f1ea5727 160000 --- a/go-ethereum +++ b/go-ethereum @@ -1 +1 @@ -Subproject commit 88b0e9d2921f070760842b4bafcf2eecff20da63 +Subproject commit a0f1ea572742f8d10223001da05be4c4f4e28007 diff --git a/system_tests/pruning_test.go b/system_tests/pruning_test.go index f49ed8ddcf..4236a4d567 100644 --- a/system_tests/pruning_test.go +++ b/system_tests/pruning_test.go @@ -32,6 +32,14 @@ func countStateEntries(db ethdb.Iteratee) int { } func TestPruning(t *testing.T) { + testPruning(t, false) +} + +func TestPruningPruneParallelStorageTraversal(t *testing.T) { + testPruning(t, true) +} + +func testPruning(t *testing.T, pruneParallelStorageTraversal bool) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -91,6 +99,7 @@ func TestPruning(t *testing.T) { initConfig := conf.InitConfigDefault initConfig.Prune = "full" + initConfig.PruneParallelStorageTraversal = pruneParallelStorageTraversal coreCacheConfig := gethexec.DefaultCacheConfigFor(stack, &builder.execConfig.Caching) persistentConfig := conf.PersistentConfigDefault err = pruning.PruneChainDb(ctx, chainDb, stack, &initConfig, coreCacheConfig, &persistentConfig, builder.L1.Client, *builder.L2.ConsensusNode.DeployInfo, false)