From e9c0eea535180c4011259709558d5cda36dd6c85 Mon Sep 17 00:00:00 2001 From: Quentin Mc Gaw Date: Wed, 12 Feb 2025 12:38:19 +0100 Subject: [PATCH] chore(core/rawdb): `InspectDatabase` uses libevm - Using libevm `InspectDatabase` - Define options passed to libevm `InspectDatabase` --- core/rawdb/database.go | 201 +++++++++++------------------------------ go.mod | 2 +- 2 files changed, 52 insertions(+), 151 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 1bf30508c5..3db553ff2d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -31,15 +31,14 @@ import ( "fmt" "os" "path/filepath" - "time" "github.com/ava-labs/libevm/common" + ethrawdb "github.com/ava-labs/libevm/core/rawdb" "github.com/ava-labs/libevm/ethdb" "github.com/ava-labs/libevm/ethdb/leveldb" "github.com/ava-labs/libevm/ethdb/memorydb" "github.com/ava-labs/libevm/ethdb/pebble" "github.com/ava-labs/libevm/log" - "github.com/olekukonko/tablewriter" ) // nofreezedb is a database wrapper that disables freezer data retrievals. @@ -275,159 +274,61 @@ func (s *stat) Count() string { // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { - it := db.NewIterator(keyPrefix, keyStart) - defer it.Release() - var ( - count int64 - start = time.Now() - logged = time.Now() - - // Key-value store statistics - headers stat - bodies stat - receipts stat - numHashPairings stat - hashNumPairings stat - legacyTries stat - stateLookups stat - accountTries stat - storageTries stat - codes stat - txLookups stat - accountSnaps stat - storageSnaps stat - preimages stat - bloomBits stat - cliqueSnaps stat - - // State sync statistics - codeToFetch stat - syncProgress stat - syncSegments stat - syncPerformed stat - - // Les statistic - chtTrieNodes stat - bloomTrieNodes stat - - // Meta- and unaccounted data - metadata stat - unaccounted stat - - // Totals - total common.StorageSize + codeToFetch ethrawdb.DatabaseStat + syncPerformed ethrawdb.DatabaseStat + syncProgress ethrawdb.DatabaseStat + syncSegments ethrawdb.DatabaseStat ) - // Inspect key-value database first. - for it.Next() { - var ( - key = it.Key() - size = common.StorageSize(len(key) + len(it.Value())) - ) - total += size - switch { - case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): - headers.Add(size) - case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): - bodies.Add(size) - case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): - receipts.Add(size) - case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): - numHashPairings.Add(size) - case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): - hashNumPairings.Add(size) - case IsLegacyTrieNode(key, it.Value()): - legacyTries.Add(size) - case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: - stateLookups.Add(size) - case IsAccountTrieNode(key): - accountTries.Add(size) - case IsStorageTrieNode(key): - storageTries.Add(size) - case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: - codes.Add(size) - case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): - txLookups.Add(size) - case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): - accountSnaps.Add(size) - case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): - storageSnaps.Add(size) - case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): - preimages.Add(size) - case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): - metadata.Add(size) - case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): - bloomBits.Add(size) - case bytes.HasPrefix(key, BloomBitsIndexPrefix): - bloomBits.Add(size) - case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength: - syncProgress.Add(size) - case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength: - syncSegments.Add(size) - case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength: - codeToFetch.Add(size) - case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength: - syncPerformed.Add(size) - default: - var accounted bool - for _, meta := range [][]byte{ - databaseVersionKey, headHeaderKey, headBlockKey, - snapshotRootKey, snapshotBlockHashKey, snapshotGeneratorKey, - uncleanShutdownKey, syncRootKey, txIndexTailKey, - persistentStateIDKey, trieJournalKey, - } { - if bytes.Equal(key, meta) { - metadata.Add(size) - accounted = true - break - } + + options := []ethrawdb.InspectDatabaseOption{ + ethrawdb.WithDatabaseMetadataKeys(func(key []byte) bool { + return bytes.Equal(key, snapshotBlockHashKey) || + bytes.Equal(key, syncRootKey) + }), + ethrawdb.WithDatabaseStatRecorder(func(key []byte, size common.StorageSize) bool { + switch { + case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength: + syncSegments.Add(size) + return true + case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength: + syncProgress.Add(size) + return true + case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength: + codeToFetch.Add(size) + return true + case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength: + syncPerformed.Add(size) + return true + default: + return false } - if !accounted { - unaccounted.Add(size) + }), + ethrawdb.WithDatabaseStatsTransformer(func(rows [][]string) [][]string { + newRows := make([][]string, 0, len(rows)) + for _, row := range rows { + database := row[0] + category := row[1] + switch { + case database == "Key-Value store" && category == "Difficulties", + database == "Key-Value store" && category == "Beacon sync headers", + database == "Ancient store (Chain)": + continue + } + newRows = append(newRows, row) } - } - count++ - if count%1000 == 0 && time.Since(logged) > 8*time.Second { - log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) - logged = time.Now() - } - } - // Display the database statistic. - stats := [][]string{ - {"Key-Value store", "Headers", headers.Size(), headers.Count()}, - {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, - {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, - {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, - {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, - {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, - {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, - {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, - {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, - {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, - {"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, - {"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, - {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, - {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, - {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, - {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, - {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, - {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, - {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, - {"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()}, - {"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()}, - {"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()}, - {"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()}, - } - table := tablewriter.NewWriter(os.Stdout) - table.SetHeader([]string{"Database", "Category", "Size", "Items"}) - table.SetFooter([]string{"", "Total", total.String(), " "}) - table.AppendBulk(stats) - table.Render() - - if unaccounted.size > 0 { - log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) + + return append( + newRows, + []string{"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()}, + []string{"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()}, + []string{"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()}, + []string{"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()}, + ) + }), } - return nil + + return ethrawdb.InspectDatabase(db, keyPrefix, keyStart, options...) } // ClearPrefix removes all keys in db that begin with prefix and match an diff --git a/go.mod b/go.mod index 3da197357e..40b78d2cb8 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,6 @@ require ( github.com/holiman/uint256 v1.2.4 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.17 - github.com/olekukonko/tablewriter v0.0.5 github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model v0.3.0 github.com/spf13/cast v1.5.0 @@ -91,6 +90,7 @@ require ( github.com/mitchellh/pointerstructure v1.2.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect