Skip to content

Commit

Permalink
chore(core/rawdb): InspectDatabase uses libevm
Browse files Browse the repository at this point in the history 
- Using libevm `InspectDatabase`
- Define options passed to libevm `InspectDatabase`
  • Loading branch information
@qdm12
qdm12 committed Feb 12, 2025
1 parent ab19679 commit 1d24024
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 151 deletions.
201 changes: 51 additions & 150 deletions core/rawdb/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,14 @@ import (
"fmt"
"os"
"path/filepath"
"time"

"github.com/ava-labs/libevm/common"
ethrawdb "github.com/ava-labs/libevm/core/rawdb"
"github.com/ava-labs/libevm/ethdb"
"github.com/ava-labs/libevm/ethdb/leveldb"
"github.com/ava-labs/libevm/ethdb/memorydb"
"github.com/ava-labs/libevm/ethdb/pebble"
"github.com/ava-labs/libevm/log"
"github.com/olekukonko/tablewriter"
)

// nofreezedb is a database wrapper that disables freezer data retrievals.
Expand Down Expand Up @@ -275,159 +274,61 @@ func (s *stat) Count() string {
// InspectDatabase traverses the entire database and checks the size
// of all different categories of data.
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
it := db.NewIterator(keyPrefix, keyStart)
defer it.Release()

var (
count int64
start = time.Now()
logged = time.Now()

// Key-value store statistics
headers stat
bodies stat
receipts stat
numHashPairings stat
hashNumPairings stat
legacyTries stat
stateLookups stat
accountTries stat
storageTries stat
codes stat
txLookups stat
accountSnaps stat
storageSnaps stat
preimages stat
bloomBits stat
cliqueSnaps stat

// State sync statistics
codeToFetch stat
syncProgress stat
syncSegments stat
syncPerformed stat

// Les statistic
chtTrieNodes stat
bloomTrieNodes stat

// Meta- and unaccounted data
metadata stat
unaccounted stat

// Totals
total common.StorageSize
codeToFetch ethrawdb.DatabaseStat
syncPerformed ethrawdb.DatabaseStat
syncProgress ethrawdb.DatabaseStat
syncSegments ethrawdb.DatabaseStat
)
// Inspect key-value database first.
for it.Next() {
var (
key = it.Key()
size = common.StorageSize(len(key) + len(it.Value()))
)
total += size
switch {
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
headers.Add(size)
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
bodies.Add(size)
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
receipts.Add(size)
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
numHashPairings.Add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
hashNumPairings.Add(size)
case IsLegacyTrieNode(key, it.Value()):
legacyTries.Add(size)
case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
stateLookups.Add(size)
case IsAccountTrieNode(key):
accountTries.Add(size)
case IsStorageTrieNode(key):
storageTries.Add(size)
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
codes.Add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
txLookups.Add(size)
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
accountSnaps.Add(size)
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
storageSnaps.Add(size)
case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength):
preimages.Add(size)
case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength):
metadata.Add(size)
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
bloomBits.Add(size)
case bytes.HasPrefix(key, BloomBitsIndexPrefix):
bloomBits.Add(size)
case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength:
syncProgress.Add(size)
case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength:
syncSegments.Add(size)
case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength:
codeToFetch.Add(size)
case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength:
syncPerformed.Add(size)
default:
var accounted bool
for _, meta := range [][]byte{
databaseVersionKey, headHeaderKey, headBlockKey,
snapshotRootKey, snapshotBlockHashKey, snapshotGeneratorKey,
uncleanShutdownKey, syncRootKey, txIndexTailKey,
persistentStateIDKey, trieJournalKey,
} {
if bytes.Equal(key, meta) {
metadata.Add(size)
accounted = true
break
}

options := []ethrawdb.InspectDatabaseOption{
ethrawdb.WithDatabaseMetadataKeys(func(key []byte) bool {
return bytes.Equal(key, snapshotBlockHashKey) ||
bytes.Equal(key, syncRootKey)
}),
ethrawdb.WithDatabaseStatRecorder(func(key []byte, size common.StorageSize) bool {
switch {
case bytes.HasPrefix(key, syncSegmentsPrefix) && len(key) == syncSegmentsKeyLength:
syncSegments.Add(size)
return true
case bytes.HasPrefix(key, syncStorageTriesPrefix) && len(key) == syncStorageTriesKeyLength:
syncProgress.Add(size)
return true
case bytes.HasPrefix(key, CodeToFetchPrefix) && len(key) == codeToFetchKeyLength:
codeToFetch.Add(size)
return true
case bytes.HasPrefix(key, syncPerformedPrefix) && len(key) == syncPerformedKeyLength:
syncPerformed.Add(size)
return true
default:
return false
}
if !accounted {
unaccounted.Add(size)
}),
ethrawdb.WithDatabaseStatsTransformer(func(rows [][]string) [][]string {
newRows := make([][]string, 0, len(rows))
for _, row := range rows {
database := row[0]
category := row[1]
switch {
case database == "Key-Value store" && category == "Difficulties",
database == "Key-Value store" && category == "Beacon sync headers",
database == "Ancient store (Chain)":
continue
}
newRows = append(newRows, row)
}
}
count++
if count%1000 == 0 && time.Since(logged) > 8*time.Second {
log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
logged = time.Now()
}
}
// Display the database statistic.
stats := [][]string{
{"Key-Value store", "Headers", headers.Size(), headers.Count()},
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
{"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()},
{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()},
{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
{"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()},
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
{"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()},
{"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()},
{"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()},
{"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()},
}
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Database", "Category", "Size", "Items"})
table.SetFooter([]string{"", "Total", total.String(), " "})
table.AppendBulk(stats)
table.Render()

if unaccounted.size > 0 {
log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)

return append(
newRows,
[]string{"State sync", "Trie segments", syncSegments.Size(), syncSegments.Count()},
[]string{"State sync", "Storage tries to fetch", syncProgress.Size(), syncProgress.Count()},
[]string{"State sync", "Code to fetch", codeToFetch.Size(), codeToFetch.Count()},
[]string{"State sync", "Block numbers synced to", syncPerformed.Size(), syncPerformed.Count()},
)
}),
}
return nil

return ethrawdb.InspectDatabase(db, keyPrefix, keyStart, options...)
}

// ClearPrefix removes all keys in db that begin with prefix and match an
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,6 @@ require (
github.com/holiman/uint256 v1.2.4
github.com/mattn/go-colorable v0.1.13
github.com/mattn/go-isatty v0.0.17
github.com/olekukonko/tablewriter v0.0.5
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/client_model v0.3.0
github.com/shirou/gopsutil v3.21.11+incompatible
Expand Down Expand Up @@ -93,6 +92,7 @@ require (
github.com/mmcloughlin/addchain v0.4.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/nbutton23/zxcvbn-go v0.0.0-20180912185939-ae427f1e4c1d // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/pkg/errors v0.9.1 // indirect
Expand Down

0 comments on commit 1d24024

Please sign in to comment.