From 29eea8d7309b15a77eeafb0be52a9c6303c82fa2 Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Fri, 12 Jan 2024 09:34:02 -0800 Subject: [PATCH 1/2] remove ethdb --- accounts/abi/bind/backends/simulated.go | 2 +- cmd/evm/internal/t8ntool/execution.go | 2 +- core/bench_test.go | 2 +- core/blockchain.go | 2 +- core/blockchain_snapshot_test.go | 2 +- core/blockchain_test.go | 2 +- core/bloom_indexer.go | 2 +- core/chain_indexer.go | 2 +- core/chain_makers.go | 2 +- core/genesis.go | 2 +- core/genesis_test.go | 2 +- core/headerchain.go | 2 +- core/rawdb/accessors_chain.go | 2 +- core/rawdb/accessors_indexes.go | 2 +- core/rawdb/accessors_indexes_test.go | 2 +- core/rawdb/accessors_metadata.go | 2 +- core/rawdb/accessors_snapshot.go | 2 +- core/rawdb/accessors_state.go | 2 +- core/rawdb/accessors_state_sync.go | 2 +- core/rawdb/accessors_trie.go | 2 +- core/rawdb/chain_iterator.go | 2 +- core/rawdb/database.go | 83 ++- core/rawdb/databases_64bit.go | 4 +- core/rawdb/databases_non64bit.go | 2 +- core/rawdb/freezer.go | 21 + .../rawdb/freezer_table.go | 30 +- core/rawdb/key_length_iterator.go | 2 +- core/rawdb/table.go | 83 ++- core/rawdb/table_test.go | 2 +- core/state/database.go | 2 +- core/state/pruner/pruner.go | 2 +- core/state/snapshot/conversion.go | 2 +- core/state/snapshot/difflayer_test.go | 2 +- core/state/snapshot/disklayer.go | 2 +- core/state/snapshot/disklayer_test.go | 2 +- core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 2 +- core/state/snapshot/iterator.go | 2 +- core/state/snapshot/journal.go | 2 +- core/state/snapshot/snapshot.go | 2 +- core/state/snapshot/utils.go | 2 +- core/state/snapshot/wipe.go | 2 +- core/state/snapshot/wipe_test.go | 2 +- core/state/state_test.go | 2 +- core/state/sync_test.go | 2 +- core/state_manager.go | 2 +- core/test_blockchain.go | 2 +- eth/api_backend.go | 2 +- eth/backend.go | 2 +- eth/filters/bench_test.go | 2 +- eth/filters/filter_system.go | 2 +- eth/filters/filter_system_test.go | 2 +- eth/gasprice/gasprice_test.go | 2 +- eth/tracers/api.go | 2 +- eth/tracers/api_test.go | 2 +- ethdb/batch.go | 84 --- ethdb/database.go | 100 --- ethdb/dbtest/testsuite.go | 479 ------------- ethdb/iterator.go | 71 -- ethdb/leveldb/leveldb.go | 542 --------------- ethdb/leveldb/leveldb_test.go | 62 -- ethdb/memorydb/memorydb.go | 333 ---------- ethdb/pebble/pebble.go | 629 ------------------ ethdb/pebble/pebble_test.go | 68 -- ethdb/snapshot.go | 41 -- go.mod | 4 +- internal/ethapi/api_test.go | 2 +- internal/ethapi/backend.go | 2 +- internal/shutdowncheck/shutdown_tracker.go | 2 +- plugin/evm/block_test.go | 5 +- plugin/evm/database.go | 13 +- plugin/evm/network_handler.go | 2 +- plugin/evm/syncervm_client.go | 2 +- plugin/evm/syncervm_test.go | 2 +- plugin/evm/vm.go | 5 +- scripts/geth-allowed-packages.txt | 4 + sync/client/client.go | 6 +- sync/client/client_test.go | 8 +- sync/handlers/block_request_test.go | 6 +- sync/handlers/code_request.go | 2 +- sync/handlers/code_request_test.go | 2 +- sync/handlers/leafs_request.go | 4 +- sync/handlers/leafs_request_test.go | 7 +- sync/statesync/code_syncer.go | 3 +- sync/statesync/code_syncer_test.go | 4 +- sync/statesync/state_syncer.go | 2 +- sync/statesync/sync_helpers.go | 2 +- sync/statesync/sync_test.go | 59 +- sync/statesync/test_sync.go | 2 +- sync/statesync/trie_queue.go | 2 +- sync/statesync/trie_segments.go | 2 +- sync/statesync/trie_sync_tasks.go | 2 +- sync/syncutils/iterators.go | 2 +- tests/state_test_util.go | 2 +- trie/database_test.go | 2 +- trie/database_wrap.go | 2 +- trie/iterator_test.go | 8 +- trie/preimages.go | 2 +- trie/proof.go | 2 +- trie/proof_test.go | 2 +- trie/sync_test.go | 2 +- trie/trie_test.go | 3 +- trie/triedb/hashdb/database.go | 2 +- warp/backend.go | 2 +- 104 files changed, 342 insertions(+), 2575 deletions(-) create mode 100644 core/rawdb/freezer.go rename ethdb/memorydb/memorydb_test.go => core/rawdb/freezer_table.go (53%) delete mode 100644 ethdb/batch.go delete mode 100644 ethdb/database.go delete mode 100644 ethdb/dbtest/testsuite.go delete mode 100644 ethdb/iterator.go delete mode 100644 ethdb/leveldb/leveldb.go delete mode 100644 ethdb/leveldb/leveldb_test.go delete mode 100644 ethdb/memorydb/memorydb.go delete mode 100644 ethdb/pebble/pebble.go delete mode 100644 ethdb/pebble/pebble_test.go delete mode 100644 ethdb/snapshot.go diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index b2439c9775..ceea225f26 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -47,13 +47,13 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/filters" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/interfaces" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" ) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 17596fab42..0f82378ba4 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -36,12 +36,12 @@ import ( "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" diff --git a/core/bench_test.go b/core/bench_test.go index e6597dabfe..c0c9e8d8a0 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -36,11 +36,11 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" ) func BenchmarkInsertChain_empty_memdb(b *testing.B) { diff --git a/core/blockchain.go b/core/blockchain.go index ec02b24073..b2de7a2dc1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -46,13 +46,13 @@ import ( "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/internal/version" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/blockchain_snapshot_test.go b/core/blockchain_snapshot_test.go index b4deac33a2..14083555f8 100644 --- a/core/blockchain_snapshot_test.go +++ b/core/blockchain_snapshot_test.go @@ -42,9 +42,9 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // snapshotTestBasic wraps the common testing fields in the snapshot tests. diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 2c2be768a8..d76b422425 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -17,10 +17,10 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/tracers/logger" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/fsnotify/fsnotify" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go index 1531741f8d..a1f5b3f5e9 100644 --- a/core/bloom_indexer.go +++ b/core/bloom_indexer.go @@ -23,9 +23,9 @@ import ( "github.com/ava-labs/subnet-evm/core/bloombits" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/ethdb" ) const ( diff --git a/core/chain_indexer.go b/core/chain_indexer.go index 9a6606905b..2034619f75 100644 --- a/core/chain_indexer.go +++ b/core/chain_indexer.go @@ -36,8 +36,8 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/chain_makers.go b/core/chain_makers.go index f097939aa6..2231e413ae 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -38,10 +38,10 @@ import ( "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // BlockGen creates blocks for testing. diff --git a/core/genesis.go b/core/genesis.go index 9597eb4dad..05726faab8 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -39,13 +39,13 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/genesis_test.go b/core/genesis_test.go index ef75ac410d..ea751f7f90 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -37,7 +37,6 @@ import ( "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/precompile/allowlist" "github.com/ava-labs/subnet-evm/precompile/contracts/deployerallowlist" @@ -45,6 +44,7 @@ import ( "github.com/ava-labs/subnet-evm/utils" "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) diff --git a/core/headerchain.go b/core/headerchain.go index 405b869090..ae84e03e40 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -36,10 +36,10 @@ import ( "github.com/ava-labs/subnet-evm/consensus" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/ethdb" ) const ( diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 6f2119f908..7005019485 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -33,9 +33,9 @@ import ( "math/big" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index da3d65d190..5d705c4b4d 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -31,9 +31,9 @@ import ( "math/big" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index bb987c946c..6ee68178f2 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -23,8 +23,8 @@ import ( "testing" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index d42a10e296..8267fea4ec 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -30,9 +30,9 @@ import ( "encoding/json" "time" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/rawdb/accessors_snapshot.go b/core/rawdb/accessors_snapshot.go index f87098b62a..5ddad71900 100644 --- a/core/rawdb/accessors_snapshot.go +++ b/core/rawdb/accessors_snapshot.go @@ -27,8 +27,8 @@ package rawdb import ( - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index 5c831a0a89..1cb712d3e4 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -27,8 +27,8 @@ package rawdb import ( - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/rawdb/accessors_state_sync.go b/core/rawdb/accessors_state_sync.go index c75cb75a2d..32fb8013b3 100644 --- a/core/rawdb/accessors_state_sync.go +++ b/core/rawdb/accessors_state_sync.go @@ -7,8 +7,8 @@ import ( "encoding/binary" "github.com/ava-labs/avalanchego/utils/wrappers" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 28bd4ab8bb..a5e3a517e2 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -30,9 +30,9 @@ import ( "fmt" "sync" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "golang.org/x/crypto/sha3" ) diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 7940927097..ee3f6b3978 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -32,9 +32,9 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 24768d321f..45f6a584f3 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -34,10 +34,10 @@ import ( "path/filepath" "time" - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/leveldb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/leveldb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/olekukonko/tablewriter" ) @@ -47,6 +47,83 @@ type nofreezedb struct { ethdb.KeyValueStore } +// HasAncient returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) HasAncient(kind string, number uint64) (bool, error) { + return false, errNotSupported +} + +// Ancient returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { + return nil, errNotSupported +} + +// AncientRange returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) AncientRange(kind string, start, max, maxByteSize uint64) ([][]byte, error) { + return nil, errNotSupported +} + +// Ancients returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Ancients() (uint64, error) { + return 0, errNotSupported +} + +// Tail returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Tail() (uint64, error) { + return 0, errNotSupported +} + +// AncientSize returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) AncientSize(kind string) (uint64, error) { + return 0, errNotSupported +} + +// ModifyAncients is not supported. +func (db *nofreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, error) { + return 0, errNotSupported +} + +// TruncateHead returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) TruncateHead(items uint64) error { + return errNotSupported +} + +// TruncateTail returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) TruncateTail(items uint64) error { + return errNotSupported +} + +// Sync returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) Sync() error { + return errNotSupported +} + +func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { + // Unlike other ancient-related methods, this method does not return + // errNotSupported when invoked. + // The reason for this is that the caller might want to do several things: + // 1. Check if something is in freezer, + // 2. If not, check leveldb. + // + // This will work, since the ancient-checks inside 'fn' will return errors, + // and the leveldb work will continue. + // + // If we instead were to return errNotSupported here, then the caller would + // have to explicitly check for that, having an extra clause to do the + // non-ancient operations. + return fn(db) +} + +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (db *nofreezedb) MigrateTable(kind string, convert convertLegacyFn) error { + return errNotSupported +} + +// AncientDatadir returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) AncientDatadir() (string, error) { + return "", errNotSupported +} + // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { diff --git a/core/rawdb/databases_64bit.go b/core/rawdb/databases_64bit.go index 1f1e56477e..73bfeb2083 100644 --- a/core/rawdb/databases_64bit.go +++ b/core/rawdb/databases_64bit.go @@ -19,8 +19,8 @@ package rawdb import ( - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/pebble" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" ) // Pebble is unsuported on 32bit architecture diff --git a/core/rawdb/databases_non64bit.go b/core/rawdb/databases_non64bit.go index a658ed14ed..65ad802a6f 100644 --- a/core/rawdb/databases_non64bit.go +++ b/core/rawdb/databases_non64bit.go @@ -31,7 +31,7 @@ package rawdb import ( "errors" - "github.com/ava-labs/subnet-evm/ethdb" + "github.com/ethereum/go-ethereum/ethdb" ) // Pebble is unsuported on 32bit architecture diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go new file mode 100644 index 0000000000..622cbb3ff9 --- /dev/null +++ b/core/rawdb/freezer.go @@ -0,0 +1,21 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +// convertLegacyFn takes a raw freezer entry in an older format and +// returns it in the new format. +type convertLegacyFn = func([]byte) ([]byte, error) diff --git a/ethdb/memorydb/memorydb_test.go b/core/rawdb/freezer_table.go similarity index 53% rename from ethdb/memorydb/memorydb_test.go rename to core/rawdb/freezer_table.go index 971328920a..bc999be25b 100644 --- a/ethdb/memorydb/memorydb_test.go +++ b/core/rawdb/freezer_table.go @@ -1,14 +1,4 @@ -// (c) 2020-2021, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2018 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -24,19 +14,11 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package memorydb +package rawdb -import ( - "testing" +import "errors" - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/dbtest" +var ( + // errNotSupported is returned if the database doesn't support the required operation. + errNotSupported = errors.New("this operation is not supported") ) - -func TestMemoryDB(t *testing.T) { - t.Run("DatabaseSuite", func(t *testing.T) { - dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore { - return New() - }) - }) -} diff --git a/core/rawdb/key_length_iterator.go b/core/rawdb/key_length_iterator.go index 57f6547fb7..fe95d719f0 100644 --- a/core/rawdb/key_length_iterator.go +++ b/core/rawdb/key_length_iterator.go @@ -26,7 +26,7 @@ package rawdb -import "github.com/ava-labs/subnet-evm/ethdb" +import "github.com/ethereum/go-ethereum/ethdb" // KeyLengthIterator is a wrapper for a database iterator that ensures only key-value pairs // with a specific key length will be returned. diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 3a12c72c50..9563f8cc5f 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -27,7 +27,7 @@ package rawdb import ( - "github.com/ava-labs/subnet-evm/ethdb" + "github.com/ethereum/go-ethereum/ethdb" ) // table is a wrapper around a database that prefixes each key access with a pre- @@ -60,6 +60,80 @@ func (t *table) Get(key []byte) ([]byte, error) { return t.db.Get(append([]byte(t.prefix), key...)) } +// HasAncient is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) HasAncient(kind string, number uint64) (bool, error) { + return t.db.HasAncient(kind, number) +} + +// Ancient is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Ancient(kind string, number uint64) ([]byte, error) { + return t.db.Ancient(kind, number) +} + +// AncientRange is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { + return t.db.AncientRange(kind, start, count, maxBytes) +} + +// Ancients is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Ancients() (uint64, error) { + return t.db.Ancients() +} + +// Tail is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Tail() (uint64, error) { + return t.db.Tail() +} + +// AncientSize is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) AncientSize(kind string) (uint64, error) { + return t.db.AncientSize(kind) +} + +// ModifyAncients runs an ancient write operation on the underlying database. +func (t *table) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) { + return t.db.ModifyAncients(fn) +} + +func (t *table) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { + return t.db.ReadAncients(fn) +} + +// TruncateHead is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) TruncateHead(items uint64) error { + return t.db.TruncateHead(items) +} + +// TruncateTail is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) TruncateTail(items uint64) error { + return t.db.TruncateTail(items) +} + +// Sync is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) Sync() error { + return t.db.Sync() +} + +// MigrateTable processes the entries in a given table in sequence +// converting them to a new format if they're of an old format. +func (t *table) MigrateTable(kind string, convert convertLegacyFn) error { + return t.db.MigrateTable(kind, convert) +} + +// AncientDatadir returns the ancient datadir of the underlying database. +func (t *table) AncientDatadir() (string, error) { + return t.db.AncientDatadir() +} + // Put inserts the given value into the database at a prefixed version of the // provided key. func (t *table) Put(key []byte, value []byte) error { @@ -136,6 +210,13 @@ func (t *table) NewBatchWithSize(size int) ethdb.Batch { return &tableBatch{t.db.NewBatchWithSize(size), t.prefix} } +// NewSnapshot creates a database snapshot based on the current state. +// The created snapshot will not be affected by all following mutations +// happened on the database. +func (t *table) NewSnapshot() (ethdb.Snapshot, error) { + return t.db.NewSnapshot() +} + // tableBatch is a wrapper around a database batch that prefixes each key access // with a pre-configured string. type tableBatch struct { diff --git a/core/rawdb/table_test.go b/core/rawdb/table_test.go index 6b5b4607fe..9cb913c2a1 100644 --- a/core/rawdb/table_test.go +++ b/core/rawdb/table_test.go @@ -30,7 +30,7 @@ import ( "bytes" "testing" - "github.com/ava-labs/subnet-evm/ethdb" + "github.com/ethereum/go-ethereum/ethdb" ) func TestTableDatabase(t *testing.T) { testTableDatabase(t, "prefix") } diff --git a/core/state/database.go b/core/state/database.go index 8e646f73c4..f550a53a75 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -32,11 +32,11 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/ethdb" ) const ( diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 73cdc8d19b..84981a6d4a 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -40,9 +40,9 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/state/snapshot/conversion.go b/core/state/snapshot/conversion.go index 80ac41cd08..c731af79b0 100644 --- a/core/state/snapshot/conversion.go +++ b/core/state/snapshot/conversion.go @@ -38,9 +38,9 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/state/snapshot/difflayer_test.go b/core/state/snapshot/difflayer_test.go index 84358c59e3..063d4435e5 100644 --- a/core/state/snapshot/difflayer_test.go +++ b/core/state/snapshot/difflayer_test.go @@ -32,10 +32,10 @@ import ( "math/rand" "testing" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func copyDestructs(destructs map[common.Hash]struct{}) map[common.Hash]struct{} { diff --git a/core/state/snapshot/disklayer.go b/core/state/snapshot/disklayer.go index b2aca51793..7467ba8c9c 100644 --- a/core/state/snapshot/disklayer.go +++ b/core/state/snapshot/disklayer.go @@ -32,10 +32,10 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/state/snapshot/disklayer_test.go b/core/state/snapshot/disklayer_test.go index 9f5c2a55d0..c1fb2f5a36 100644 --- a/core/state/snapshot/disklayer_test.go +++ b/core/state/snapshot/disklayer_test.go @@ -31,8 +31,8 @@ import ( "testing" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 1db02dd03b..bf288d924e 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -35,11 +35,11 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index b8137b012d..7b2c6d7874 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -35,10 +35,10 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index 23e702ad90..f75fee23c9 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -32,8 +32,8 @@ import ( "sort" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // Iterator is an iterator to step over all the accounts or the specific diff --git a/core/state/snapshot/journal.go b/core/state/snapshot/journal.go index d7eea75751..30d9882bbe 100644 --- a/core/state/snapshot/journal.go +++ b/core/state/snapshot/journal.go @@ -33,9 +33,9 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/core/state/snapshot/snapshot.go b/core/state/snapshot/snapshot.go index 70a5f674bb..0d9707d029 100644 --- a/core/state/snapshot/snapshot.go +++ b/core/state/snapshot/snapshot.go @@ -35,11 +35,11 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/state/snapshot/utils.go b/core/state/snapshot/utils.go index e653773089..6131ae4c20 100644 --- a/core/state/snapshot/utils.go +++ b/core/state/snapshot/utils.go @@ -32,8 +32,8 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/state/snapshot/wipe.go b/core/state/snapshot/wipe.go index e7d3564d3a..9de121a244 100644 --- a/core/state/snapshot/wipe.go +++ b/core/state/snapshot/wipe.go @@ -31,8 +31,8 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/core/state/snapshot/wipe_test.go b/core/state/snapshot/wipe_test.go index 647d1b2e9e..c1080a1465 100644 --- a/core/state/snapshot/wipe_test.go +++ b/core/state/snapshot/wipe_test.go @@ -31,8 +31,8 @@ import ( "testing" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) // Tests that given a database with random data content, all parts of a snapshot diff --git a/core/state/state_test.go b/core/state/state_test.go index abeeda4d4b..96d9570cff 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -34,10 +34,10 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" ) type stateTest struct { diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 7edaa3c03e..5a83a9ac91 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -21,9 +21,9 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" ) // testAccount is the data associated with an account used by the state tests. diff --git a/core/state_manager.go b/core/state_manager.go index 87a04f41c7..77b5c5e739 100644 --- a/core/state_manager.go +++ b/core/state_manager.go @@ -32,8 +32,8 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) func init() { diff --git a/core/test_blockchain.go b/core/test_blockchain.go index 4853695e3c..fb40c918a6 100644 --- a/core/test_blockchain.go +++ b/core/test_blockchain.go @@ -14,7 +14,6 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/precompile/allowlist" "github.com/ava-labs/subnet-evm/precompile/contracts/deployerallowlist" @@ -22,6 +21,7 @@ import ( "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/stretchr/testify/assert" ) diff --git a/eth/api_backend.go b/eth/api_backend.go index 43ee3d96bb..f87c256803 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -44,10 +44,10 @@ import ( "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/gasprice" "github.com/ava-labs/subnet-evm/eth/tracers" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" ) diff --git a/eth/backend.go b/eth/backend.go index 85b9aca042..d2d8e255a5 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -48,7 +48,6 @@ import ( "github.com/ava-labs/subnet-evm/eth/filters" "github.com/ava-labs/subnet-evm/eth/gasprice" "github.com/ava-labs/subnet-evm/eth/tracers" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/internal/ethapi" "github.com/ava-labs/subnet-evm/internal/shutdowncheck" "github.com/ava-labs/subnet-evm/miner" @@ -56,6 +55,7 @@ import ( "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" ) diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 7ba3381f2f..2a23705313 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -35,9 +35,9 @@ import ( "github.com/ava-labs/subnet-evm/core/bloombits" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/bitutil" + "github.com/ethereum/go-ethereum/ethdb" "github.com/stretchr/testify/require" ) diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index fef5363fac..865766d29d 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -39,11 +39,11 @@ import ( "github.com/ava-labs/subnet-evm/core/bloombits" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/interfaces" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" ) diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 5b9d7ad079..6f49eabb27 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -43,12 +43,12 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/interfaces" "github.com/ava-labs/subnet-evm/internal/ethapi" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/stretchr/testify/require" ) diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 53a898faa0..af128903ad 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -417,7 +417,7 @@ func TestSuggestGasPriceAfterFeeConfigUpdate(t *testing.T) { // issue the block with tx that changes the fee genesis := backend.chain.Genesis() engine := backend.chain.Engine() - db := backend.chain.StateCache().DiskDB() + db := rawdb.NewDatabase(backend.chain.StateCache().DiskDB()) blocks, _, err := core.GenerateChain(&chainConfig, genesis, engine, db, 1, 0, func(i int, b *core.BlockGen) { b.SetCoinbase(common.Address{1}) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index abc442ab64..904ce12160 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -44,12 +44,12 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/tracers/logger" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/internal/ethapi" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index 52a705c564..ff140293b6 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -47,13 +47,13 @@ import ( "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/eth/tracers/logger" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/internal/ethapi" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" ) var ( diff --git a/ethdb/batch.go b/ethdb/batch.go deleted file mode 100644 index be4f52d8c1..0000000000 --- a/ethdb/batch.go +++ /dev/null @@ -1,84 +0,0 @@ -// (c) 2020-2021, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethdb - -// IdealBatchSize defines the size of the data batches should ideally add in one -// write. -const IdealBatchSize = 100 * 1024 - -// Batch is a write-only database that commits changes to its host database -// when Write is called. A batch cannot be used concurrently. -type Batch interface { - KeyValueWriter - - // ValueSize retrieves the amount of data queued up for writing. - ValueSize() int - - // Write flushes any accumulated data to disk. - Write() error - - // Reset resets the batch for reuse. - Reset() - - // Replay replays the batch contents. - Replay(w KeyValueWriter) error -} - -// Batcher wraps the NewBatch method of a backing data store. -type Batcher interface { - // NewBatch creates a write-only database that buffers changes to its host db - // until a final write is called. - NewBatch() Batch - - // NewBatchWithSize creates a write-only database batch with pre-allocated buffer. - NewBatchWithSize(size int) Batch -} - -// HookedBatch wraps an arbitrary batch where each operation may be hooked into -// to monitor from black box code. -type HookedBatch struct { - Batch - - OnPut func(key []byte, value []byte) // Callback if a key is inserted - OnDelete func(key []byte) // Callback if a key is deleted -} - -// Put inserts the given value into the key-value data store. -func (b HookedBatch) Put(key []byte, value []byte) error { - if b.OnPut != nil { - b.OnPut(key, value) - } - return b.Batch.Put(key, value) -} - -// Delete removes the key from the key-value data store. -func (b HookedBatch) Delete(key []byte) error { - if b.OnDelete != nil { - b.OnDelete(key) - } - return b.Batch.Delete(key) -} diff --git a/ethdb/database.go b/ethdb/database.go deleted file mode 100644 index 2cdac31c06..0000000000 --- a/ethdb/database.go +++ /dev/null @@ -1,100 +0,0 @@ -// (c) 2020-2021, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package ethdb defines the interfaces for an Ethereum data store. -package ethdb - -import "io" - -// KeyValueReader wraps the Has and Get method of a backing data store. -type KeyValueReader interface { - // Has retrieves if a key is present in the key-value data store. - Has(key []byte) (bool, error) - - // Get retrieves the given key if it's present in the key-value data store. - Get(key []byte) ([]byte, error) -} - -// KeyValueWriter wraps the Put method of a backing data store. -type KeyValueWriter interface { - // Put inserts the given value into the key-value data store. - Put(key []byte, value []byte) error - - // Delete removes the key from the key-value data store. - Delete(key []byte) error -} - -// Stater wraps the Stat method of a backing data store. -type Stater interface { - // Stat returns a particular internal stat of the database. - Stat(property string) (string, error) -} - -// Compacter wraps the Compact method of a backing data store. -type Compacter interface { - // Compact flattens the underlying data store for the given key range. In essence, - // deleted and overwritten versions are discarded, and the data is rearranged to - // reduce the cost of operations needed to access them. - // - // A nil start is treated as a key before all keys in the data store; a nil limit - // is treated as a key after all keys in the data store. If both is nil then it - // will compact entire data store. - Compact(start []byte, limit []byte) error -} - -// KeyValueStore contains all the methods required to allow handling different -// key-value data stores backing the high level database. -type KeyValueStore interface { - KeyValueReader - KeyValueWriter - Batcher - Iteratee - Stater - Compacter - io.Closer -} - -// Reader contains the methods required to read data from key-value storage. -type Reader interface { - KeyValueReader -} - -// Writer contains the methods required to write data to key-value storage. -type Writer interface { - KeyValueWriter -} - -// Database contains all the methods required by the high level database to not -// only access the key-value data store but also the chain freezer. -type Database interface { - Reader - Writer - Batcher - Iteratee - Stater - Compacter - io.Closer -} diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go deleted file mode 100644 index 81224c736c..0000000000 --- a/ethdb/dbtest/testsuite.go +++ /dev/null @@ -1,479 +0,0 @@ -// (c) 2020-2021, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package dbtest - -import ( - "bytes" - "crypto/rand" - "reflect" - "sort" - "testing" - - "github.com/ava-labs/subnet-evm/ethdb" -) - -// TestDatabaseSuite runs a suite of tests against a KeyValueStore database -// implementation. -func TestDatabaseSuite(t *testing.T, New func() ethdb.KeyValueStore) { - t.Run("Iterator", func(t *testing.T) { - tests := []struct { - content map[string]string - prefix string - start string - order []string - }{ - // Empty databases should be iterable - {map[string]string{}, "", "", nil}, - {map[string]string{}, "non-existent-prefix", "", nil}, - - // Single-item databases should be iterable - {map[string]string{"key": "val"}, "", "", []string{"key"}}, - {map[string]string{"key": "val"}, "k", "", []string{"key"}}, - {map[string]string{"key": "val"}, "l", "", nil}, - - // Multi-item databases should be fully iterable - { - map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "", "", - []string{"k1", "k2", "k3", "k4", "k5"}, - }, - { - map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "k", "", - []string{"k1", "k2", "k3", "k4", "k5"}, - }, - { - map[string]string{"k1": "v1", "k5": "v5", "k2": "v2", "k4": "v4", "k3": "v3"}, - "l", "", - nil, - }, - // Multi-item databases should be prefix-iterable - { - map[string]string{ - "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", - "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", - }, - "ka", "", - []string{"ka1", "ka2", "ka3", "ka4", "ka5"}, - }, - { - map[string]string{ - "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", - "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", - }, - "kc", "", - nil, - }, - // Multi-item databases should be prefix-iterable with start position - { - map[string]string{ - "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", - "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", - }, - "ka", "3", - []string{"ka3", "ka4", "ka5"}, - }, - { - map[string]string{ - "ka1": "va1", "ka5": "va5", "ka2": "va2", "ka4": "va4", "ka3": "va3", - "kb1": "vb1", "kb5": "vb5", "kb2": "vb2", "kb4": "vb4", "kb3": "vb3", - }, - "ka", "8", - nil, - }, - } - for i, tt := range tests { - // Create the key-value data store - db := New() - for key, val := range tt.content { - if err := db.Put([]byte(key), []byte(val)); err != nil { - t.Fatalf("test %d: failed to insert item %s:%s into database: %v", i, key, val, err) - } - } - // Iterate over the database with the given configs and verify the results - it, idx := db.NewIterator([]byte(tt.prefix), []byte(tt.start)), 0 - for it.Next() { - if len(tt.order) <= idx { - t.Errorf("test %d: prefix=%q more items than expected: checking idx=%d (key %q), expecting len=%d", i, tt.prefix, idx, it.Key(), len(tt.order)) - break - } - if !bytes.Equal(it.Key(), []byte(tt.order[idx])) { - t.Errorf("test %d: item %d: key mismatch: have %s, want %s", i, idx, string(it.Key()), tt.order[idx]) - } - if !bytes.Equal(it.Value(), []byte(tt.content[tt.order[idx]])) { - t.Errorf("test %d: item %d: value mismatch: have %s, want %s", i, idx, string(it.Value()), tt.content[tt.order[idx]]) - } - idx++ - } - if err := it.Error(); err != nil { - t.Errorf("test %d: iteration failed: %v", i, err) - } - if idx != len(tt.order) { - t.Errorf("test %d: iteration terminated prematurely: have %d, want %d", i, idx, len(tt.order)) - } - db.Close() - } - }) - - t.Run("IteratorWith", func(t *testing.T) { - db := New() - defer db.Close() - - keys := []string{"1", "2", "3", "4", "6", "10", "11", "12", "20", "21", "22"} - sort.Strings(keys) // 1, 10, 11, etc - - for _, k := range keys { - if err := db.Put([]byte(k), nil); err != nil { - t.Fatal(err) - } - } - - { - it := db.NewIterator(nil, nil) - got, want := iterateKeys(it), keys - if err := it.Error(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("Iterator: got: %s; want: %s", got, want) - } - } - - { - it := db.NewIterator([]byte("1"), nil) - got, want := iterateKeys(it), []string{"1", "10", "11", "12"} - if err := it.Error(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("IteratorWith(1,nil): got: %s; want: %s", got, want) - } - } - - { - it := db.NewIterator([]byte("5"), nil) - got, want := iterateKeys(it), []string{} - if err := it.Error(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("IteratorWith(5,nil): got: %s; want: %s", got, want) - } - } - - { - it := db.NewIterator(nil, []byte("2")) - got, want := iterateKeys(it), []string{"2", "20", "21", "22", "3", "4", "6"} - if err := it.Error(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("IteratorWith(nil,2): got: %s; want: %s", got, want) - } - } - - { - it := db.NewIterator(nil, []byte("5")) - got, want := iterateKeys(it), []string{"6"} - if err := it.Error(); err != nil { - t.Fatal(err) - } - if !reflect.DeepEqual(got, want) { - t.Errorf("IteratorWith(nil,5): got: %s; want: %s", got, want) - } - } - }) - - t.Run("KeyValueOperations", func(t *testing.T) { - db := New() - defer db.Close() - - key := []byte("foo") - - if got, err := db.Has(key); err != nil { - t.Error(err) - } else if got { - t.Errorf("wrong value: %t", got) - } - - value := []byte("hello world") - if err := db.Put(key, value); err != nil { - t.Error(err) - } - - if got, err := db.Has(key); err != nil { - t.Error(err) - } else if !got { - t.Errorf("wrong value: %t", got) - } - - if got, err := db.Get(key); err != nil { - t.Error(err) - } else if !bytes.Equal(got, value) { - t.Errorf("wrong value: %q", got) - } - - if err := db.Delete(key); err != nil { - t.Error(err) - } - - if got, err := db.Has(key); err != nil { - t.Error(err) - } else if got { - t.Errorf("wrong value: %t", got) - } - }) - - t.Run("Batch", func(t *testing.T) { - db := New() - defer db.Close() - - b := db.NewBatch() - for _, k := range []string{"1", "2", "3", "4"} { - if err := b.Put([]byte(k), nil); err != nil { - t.Fatal(err) - } - } - - if has, err := db.Has([]byte("1")); err != nil { - t.Fatal(err) - } else if has { - t.Error("db contains element before batch write") - } - - if err := b.Write(); err != nil { - t.Fatal(err) - } - - { - it := db.NewIterator(nil, nil) - if got, want := iterateKeys(it), []string{"1", "2", "3", "4"}; !reflect.DeepEqual(got, want) { - t.Errorf("got: %s; want: %s", got, want) - } - } - - b.Reset() - - // Mix writes and deletes in batch - b.Put([]byte("5"), nil) - b.Delete([]byte("1")) - b.Put([]byte("6"), nil) - b.Delete([]byte("3")) - b.Put([]byte("3"), nil) - - if err := b.Write(); err != nil { - t.Fatal(err) - } - - { - it := db.NewIterator(nil, nil) - if got, want := iterateKeys(it), []string{"2", "3", "4", "5", "6"}; !reflect.DeepEqual(got, want) { - t.Errorf("got: %s; want: %s", got, want) - } - } - }) - - t.Run("BatchReplay", func(t *testing.T) { - db := New() - defer db.Close() - - want := []string{"1", "2", "3", "4"} - b := db.NewBatch() - for _, k := range want { - if err := b.Put([]byte(k), nil); err != nil { - t.Fatal(err) - } - } - - b2 := db.NewBatch() - if err := b.Replay(b2); err != nil { - t.Fatal(err) - } - - if err := b2.Replay(db); err != nil { - t.Fatal(err) - } - - it := db.NewIterator(nil, nil) - if got := iterateKeys(it); !reflect.DeepEqual(got, want) { - t.Errorf("got: %s; want: %s", got, want) - } - }) - - t.Run("OperatonsAfterClose", func(t *testing.T) { - db := New() - db.Put([]byte("key"), []byte("value")) - db.Close() - if _, err := db.Get([]byte("key")); err == nil { - t.Fatalf("expected error on Get after Close") - } - if _, err := db.Has([]byte("key")); err == nil { - t.Fatalf("expected error on Get after Close") - } - if err := db.Put([]byte("key2"), []byte("value2")); err == nil { - t.Fatalf("expected error on Put after Close") - } - if err := db.Delete([]byte("key")); err == nil { - t.Fatalf("expected error on Delete after Close") - } - - b := db.NewBatch() - if err := b.Put([]byte("batchkey"), []byte("batchval")); err != nil { - t.Fatalf("expected no error on batch.Put after Close, got %v", err) - } - if err := b.Write(); err == nil { - t.Fatalf("expected error on batch.Write after Close") - } - }) -} - -// BenchDatabaseSuite runs a suite of benchmarks against a KeyValueStore database -// implementation. -func BenchDatabaseSuite(b *testing.B, New func() ethdb.KeyValueStore) { - var ( - keys, vals = makeDataset(1_000_000, 32, 32, false) - sKeys, sVals = makeDataset(1_000_000, 32, 32, true) - ) - // Run benchmarks sequentially - b.Run("Write", func(b *testing.B) { - benchWrite := func(b *testing.B, keys, vals [][]byte) { - b.ResetTimer() - b.ReportAllocs() - - db := New() - defer db.Close() - - for i := 0; i < len(keys); i++ { - db.Put(keys[i], vals[i]) - } - } - b.Run("WriteSorted", func(b *testing.B) { - benchWrite(b, sKeys, sVals) - }) - b.Run("WriteRandom", func(b *testing.B) { - benchWrite(b, keys, vals) - }) - }) - b.Run("Read", func(b *testing.B) { - benchRead := func(b *testing.B, keys, vals [][]byte) { - db := New() - defer db.Close() - - for i := 0; i < len(keys); i++ { - db.Put(keys[i], vals[i]) - } - b.ResetTimer() - b.ReportAllocs() - - for i := 0; i < len(keys); i++ { - db.Get(keys[i]) - } - } - b.Run("ReadSorted", func(b *testing.B) { - benchRead(b, sKeys, sVals) - }) - b.Run("ReadRandom", func(b *testing.B) { - benchRead(b, keys, vals) - }) - }) - b.Run("Iteration", func(b *testing.B) { - benchIteration := func(b *testing.B, keys, vals [][]byte) { - db := New() - defer db.Close() - - for i := 0; i < len(keys); i++ { - db.Put(keys[i], vals[i]) - } - b.ResetTimer() - b.ReportAllocs() - - it := db.NewIterator(nil, nil) - for it.Next() { - } - it.Release() - } - b.Run("IterationSorted", func(b *testing.B) { - benchIteration(b, sKeys, sVals) - }) - b.Run("IterationRandom", func(b *testing.B) { - benchIteration(b, keys, vals) - }) - }) - b.Run("BatchWrite", func(b *testing.B) { - benchBatchWrite := func(b *testing.B, keys, vals [][]byte) { - b.ResetTimer() - b.ReportAllocs() - - db := New() - defer db.Close() - - batch := db.NewBatch() - for i := 0; i < len(keys); i++ { - batch.Put(keys[i], vals[i]) - } - batch.Write() - } - b.Run("BenchWriteSorted", func(b *testing.B) { - benchBatchWrite(b, sKeys, sVals) - }) - b.Run("BenchWriteRandom", func(b *testing.B) { - benchBatchWrite(b, keys, vals) - }) - }) -} - -func iterateKeys(it ethdb.Iterator) []string { - keys := []string{} - for it.Next() { - keys = append(keys, string(it.Key())) - } - sort.Strings(keys) - it.Release() - return keys -} - -// randomHash generates a random blob of data and returns it as a hash. -func randBytes(len int) []byte { - buf := make([]byte, len) - if n, err := rand.Read(buf); n != len || err != nil { - panic(err) - } - return buf -} - -func makeDataset(size, ksize, vsize int, order bool) ([][]byte, [][]byte) { - var keys [][]byte - var vals [][]byte - for i := 0; i < size; i += 1 { - keys = append(keys, randBytes(ksize)) - vals = append(vals, randBytes(vsize)) - } - if order { - sort.Slice(keys, func(i, j int) bool { return bytes.Compare(keys[i], keys[j]) < 0 }) - } - return keys, vals -} diff --git a/ethdb/iterator.go b/ethdb/iterator.go deleted file mode 100644 index 54b10838b6..0000000000 --- a/ethdb/iterator.go +++ /dev/null @@ -1,71 +0,0 @@ -// (c) 2020-2021, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethdb - -// Iterator iterates over a database's key/value pairs in ascending key order. -// -// When it encounters an error any seek will return false and will yield no key/ -// value pairs. The error can be queried by calling the Error method. Calling -// Release is still necessary. -// -// An iterator must be released after use, but it is not necessary to read an -// iterator until exhaustion. An iterator is not safe for concurrent use, but it -// is safe to use multiple iterators concurrently. -type Iterator interface { - // Next moves the iterator to the next key/value pair. It returns whether the - // iterator is exhausted. - Next() bool - - // Error returns any accumulated error. Exhausting all the key/value pairs - // is not considered to be an error. - Error() error - - // Key returns the key of the current key/value pair, or nil if done. The caller - // should not modify the contents of the returned slice, and its contents may - // change on the next call to Next. - Key() []byte - - // Value returns the value of the current key/value pair, or nil if done. The - // caller should not modify the contents of the returned slice, and its contents - // may change on the next call to Next. - Value() []byte - - // Release releases associated resources. Release should always succeed and can - // be called multiple times without causing error. - Release() -} - -// Iteratee wraps the NewIterator methods of a backing data store. -type Iteratee interface { - // NewIterator creates a binary-alphabetical iterator over a subset - // of database content with a particular key prefix, starting at a particular - // initial key (or after, if it does not exist). - // - // Note: This method assumes that the prefix is NOT part of the start, so there's - // no need for the caller to prepend the prefix to the start - NewIterator(prefix []byte, start []byte) Iterator -} diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go deleted file mode 100644 index 3ffe3f3b2b..0000000000 --- a/ethdb/leveldb/leveldb.go +++ /dev/null @@ -1,542 +0,0 @@ -// (c) 2021-2022, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !js -// +build !js - -// Package leveldb implements the key-value database layer based on LevelDB. -package leveldb - -import ( - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/metrics" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -const ( - // degradationWarnInterval specifies how often warning should be printed if the - // leveldb database cannot keep up with requested writes. - degradationWarnInterval = time.Minute - - // minCache is the minimum amount of memory in megabytes to allocate to leveldb - // read and write caching, split half and half. - minCache = 16 - - // minHandles is the minimum number of files handles to allocate to the open - // database files. - minHandles = 16 - - // metricsGatheringInterval specifies the interval to retrieve leveldb database - // compaction, io and pause stats to report to the user. - metricsGatheringInterval = 3 * time.Second -) - -// Database is a persistent key-value store. Apart from basic data storage -// functionality it also supports batch writes and iterating over the keyspace in -// binary-alphabetical order. -type Database struct { - fn string // filename for reporting - db *leveldb.DB // LevelDB instance - - compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter metrics.Meter // Meter for measuring the data read during compaction - compWriteMeter metrics.Meter // Meter for measuring the data written during compaction - writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction - writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction - diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database - diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read - diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written - memCompGauge metrics.Gauge // Gauge for tracking the number of memory compaction - level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0 - nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level - seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt - manualMemAllocGauge metrics.Gauge // Gauge to track the amount of memory that has been manually allocated (not a part of runtime/GC) - - quitLock sync.Mutex // Mutex protecting the quit channel access - quitChan chan chan error // Quit channel to stop the metrics collection before closing the database - - log log.Logger // Contextual logger tracking the database path -} - -// New returns a wrapped LevelDB object. The namespace is the prefix that the -// metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { - return NewCustom(file, namespace, func(options *opt.Options) { - // Ensure we have some minimal caching and file guarantees - if cache < minCache { - cache = minCache - } - if handles < minHandles { - handles = minHandles - } - // Set default options - options.OpenFilesCacheCapacity = handles - options.BlockCacheCapacity = cache / 2 * opt.MiB - options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally - if readonly { - options.ReadOnly = true - } - }) -} - -// NewCustom returns a wrapped LevelDB object. The namespace is the prefix that the -// metrics reporting should use for surfacing internal stats. -// The customize function allows the caller to modify the leveldb options. -func NewCustom(file string, namespace string, customize func(options *opt.Options)) (*Database, error) { - options := configureOptions(customize) - logger := log.New("database", file) - usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2 - logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()} - if options.ReadOnly { - logCtx = append(logCtx, "readonly", "true") - } - logger.Info("Allocated cache and file handles", logCtx...) - - // Open the db and recover any potential corruptions - db, err := leveldb.OpenFile(file, options) - if _, corrupted := err.(*errors.ErrCorrupted); corrupted { - db, err = leveldb.RecoverFile(file, nil) - } - if err != nil { - return nil, err - } - // Assemble the wrapper with all the registered metrics - ldb := &Database{ - fn: file, - db: db, - log: logger, - quitChan: make(chan chan error), - } - ldb.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil) - ldb.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil) - ldb.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil) - ldb.diskSizeGauge = metrics.NewRegisteredGauge(namespace+"disk/size", nil) - ldb.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil) - ldb.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil) - ldb.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil) - ldb.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil) - ldb.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil) - ldb.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil) - ldb.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil) - ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) - ldb.manualMemAllocGauge = metrics.NewRegisteredGauge(namespace+"memory/manualalloc", nil) - - // Start up the metrics gathering and return - go ldb.meter(metricsGatheringInterval) - return ldb, nil -} - -// configureOptions sets some default options, then runs the provided setter. -func configureOptions(customizeFn func(*opt.Options)) *opt.Options { - // Set default options - options := &opt.Options{ - Filter: filter.NewBloomFilter(10), - DisableSeeksCompaction: true, - } - // Allow caller to make custom modifications to the options - if customizeFn != nil { - customizeFn(options) - } - return options -} - -// Close stops the metrics collection, flushes any pending data to disk and closes -// all io accesses to the underlying key-value store. -func (db *Database) Close() error { - db.quitLock.Lock() - defer db.quitLock.Unlock() - - if db.quitChan != nil { - errc := make(chan error) - db.quitChan <- errc - if err := <-errc; err != nil { - db.log.Error("Metrics collection failed", "err", err) - } - db.quitChan = nil - } - return db.db.Close() -} - -// Has retrieves if a key is present in the key-value store. -func (db *Database) Has(key []byte) (bool, error) { - return db.db.Has(key, nil) -} - -// Get retrieves the given key if it's present in the key-value store. -func (db *Database) Get(key []byte) ([]byte, error) { - dat, err := db.db.Get(key, nil) - if err != nil { - return nil, err - } - return dat, nil -} - -// Put inserts the given value into the key-value store. -func (db *Database) Put(key []byte, value []byte) error { - return db.db.Put(key, value, nil) -} - -// Delete removes the key from the key-value store. -func (db *Database) Delete(key []byte) error { - return db.db.Delete(key, nil) -} - -// NewBatch creates a write-only key-value store that buffers changes to its host -// database until a final write is called. -func (db *Database) NewBatch() ethdb.Batch { - return &batch{ - db: db.db, - b: new(leveldb.Batch), - } -} - -// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. -func (db *Database) NewBatchWithSize(size int) ethdb.Batch { - return &batch{ - db: db.db, - b: leveldb.MakeBatch(size), - } -} - -// NewIterator creates a binary-alphabetical iterator over a subset -// of database content with a particular key prefix, starting at a particular -// initial key (or after, if it does not exist). -func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - return db.db.NewIterator(bytesPrefixRange(prefix, start), nil) -} - -// Stat returns a particular internal stat of the database. -func (db *Database) Stat(property string) (string, error) { - return db.db.GetProperty(property) -} - -// Compact flattens the underlying data store for the given key range. In essence, -// deleted and overwritten versions are discarded, and the data is rearranged to -// reduce the cost of operations needed to access them. -// -// A nil start is treated as a key before all keys in the data store; a nil limit -// is treated as a key after all keys in the data store. If both is nil then it -// will compact entire data store. -func (db *Database) Compact(start []byte, limit []byte) error { - return db.db.CompactRange(util.Range{Start: start, Limit: limit}) -} - -// Path returns the path to the database directory. -func (db *Database) Path() string { - return db.fn -} - -// meter periodically retrieves internal leveldb counters and reports them to -// the metrics subsystem. -// -// This is how a LevelDB stats table looks like (currently): -// -// Compactions -// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) -// -------+------------+---------------+---------------+---------------+--------------- -// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 -// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 -// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 -// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 -// -// This is how the write delay look like (currently): -// DelayN:5 Delay:406.604657ms Paused: false -// -// This is how the iostats look like (currently): -// Read(MB):3895.04860 Write(MB):3654.64712 -func (db *Database) meter(refresh time.Duration) { - // Create the counters to store current and previous compaction values - compactions := make([][]float64, 2) - for i := 0; i < 2; i++ { - compactions[i] = make([]float64, 4) - } - // Create storage for iostats. - var iostats [2]float64 - - // Create storage and warning log tracer for write delay. - var ( - delaystats [2]int64 - lastWritePaused time.Time - ) - - var ( - errc chan error - merr error - ) - - timer := time.NewTimer(refresh) - defer timer.Stop() - - // Iterate ad infinitum and collect the stats - for i := 1; errc == nil && merr == nil; i++ { - // Retrieve the database stats - stats, err := db.db.GetProperty("leveldb.stats") - if err != nil { - db.log.Error("Failed to read database stats", "err", err) - merr = err - continue - } - // Find the compaction table, skip the header - lines := strings.Split(stats, "\n") - for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { - lines = lines[1:] - } - if len(lines) <= 3 { - db.log.Error("Compaction leveldbTable not found") - merr = errors.New("compaction leveldbTable not found") - continue - } - lines = lines[3:] - - // Iterate over all the leveldbTable rows, and accumulate the entries - for j := 0; j < len(compactions[i%2]); j++ { - compactions[i%2][j] = 0 - } - for _, line := range lines { - parts := strings.Split(line, "|") - if len(parts) != 6 { - break - } - for idx, counter := range parts[2:] { - value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) - if err != nil { - db.log.Error("Compaction entry parsing failed", "err", err) - merr = err - continue - } - compactions[i%2][idx] += value - } - } - // Update all the requested meters - if db.diskSizeGauge != nil { - db.diskSizeGauge.Update(int64(compactions[i%2][0] * 1024 * 1024)) - } - if db.compTimeMeter != nil { - db.compTimeMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1000 * 1000 * 1000)) - } - if db.compReadMeter != nil { - db.compReadMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) - } - if db.compWriteMeter != nil { - db.compWriteMeter.Mark(int64((compactions[i%2][3] - compactions[(i-1)%2][3]) * 1024 * 1024)) - } - // Retrieve the write delay statistic - writedelay, err := db.db.GetProperty("leveldb.writedelay") - if err != nil { - db.log.Error("Failed to read database write delay statistic", "err", err) - merr = err - continue - } - var ( - delayN int64 - delayDuration string - duration time.Duration - paused bool - ) - if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { - db.log.Error("Write delay statistic not found") - merr = err - continue - } - duration, err = time.ParseDuration(delayDuration) - if err != nil { - db.log.Error("Failed to parse delay duration", "err", err) - merr = err - continue - } - if db.writeDelayNMeter != nil { - db.writeDelayNMeter.Mark(delayN - delaystats[0]) - } - if db.writeDelayMeter != nil { - db.writeDelayMeter.Mark(duration.Nanoseconds() - delaystats[1]) - } - // If a warning that db is performing compaction has been displayed, any subsequent - // warnings will be withheld for one minute not to overwhelm the user. - if paused && delayN-delaystats[0] == 0 && duration.Nanoseconds()-delaystats[1] == 0 && - time.Now().After(lastWritePaused.Add(degradationWarnInterval)) { - db.log.Warn("Database compacting, degraded performance") - lastWritePaused = time.Now() - } - delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() - - // Retrieve the database iostats. - ioStats, err := db.db.GetProperty("leveldb.iostats") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - merr = err - continue - } - var nRead, nWrite float64 - parts := strings.Split(ioStats, " ") - if len(parts) < 2 { - db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) - continue - } - if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { - db.log.Error("Bad syntax of read entry", "entry", parts[0]) - merr = err - continue - } - if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { - db.log.Error("Bad syntax of write entry", "entry", parts[1]) - merr = err - continue - } - if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) - } - if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) - } - iostats[0], iostats[1] = nRead, nWrite - - compCount, err := db.db.GetProperty("leveldb.compcount") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - merr = err - continue - } - - var ( - memComp uint32 - level0Comp uint32 - nonLevel0Comp uint32 - seekComp uint32 - ) - if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil { - db.log.Error("Compaction count statistic not found") - merr = err - continue - } - db.memCompGauge.Update(int64(memComp)) - db.level0CompGauge.Update(int64(level0Comp)) - db.nonlevel0CompGauge.Update(int64(nonLevel0Comp)) - db.seekCompGauge.Update(int64(seekComp)) - - // Sleep a bit, then repeat the stats collection - select { - case errc = <-db.quitChan: - // Quit requesting, stop hammering the database - case <-timer.C: - timer.Reset(refresh) - // Timeout, gather a new set of stats - } - } - - if errc == nil { - errc = <-db.quitChan - } - errc <- merr -} - -// batch is a write-only leveldb batch that commits changes to its host database -// when Write is called. A batch cannot be used concurrently. -type batch struct { - db *leveldb.DB - b *leveldb.Batch - size int -} - -// Put inserts the given value into the batch for later committing. -func (b *batch) Put(key, value []byte) error { - b.b.Put(key, value) - b.size += len(value) - return nil -} - -// Delete inserts the a key removal into the batch for later committing. -func (b *batch) Delete(key []byte) error { - b.b.Delete(key) - b.size += len(key) - return nil -} - -// ValueSize retrieves the amount of data queued up for writing. -func (b *batch) ValueSize() int { - return b.size -} - -// Write flushes any accumulated data to disk. -func (b *batch) Write() error { - return b.db.Write(b.b, nil) -} - -// Reset resets the batch for reuse. -func (b *batch) Reset() { - b.b.Reset() - b.size = 0 -} - -// Replay replays the batch contents. -func (b *batch) Replay(w ethdb.KeyValueWriter) error { - return b.b.Replay(&replayer{writer: w}) -} - -// replayer is a small wrapper to implement the correct replay methods. -type replayer struct { - writer ethdb.KeyValueWriter - failure error -} - -// Put inserts the given value into the key-value data store. -func (r *replayer) Put(key, value []byte) { - // If the replay already failed, stop executing ops - if r.failure != nil { - return - } - r.failure = r.writer.Put(key, value) -} - -// Delete removes the key from the key-value data store. -func (r *replayer) Delete(key []byte) { - // If the replay already failed, stop executing ops - if r.failure != nil { - return - } - r.failure = r.writer.Delete(key) -} - -// bytesPrefixRange returns key range that satisfy -// - the given prefix, and -// - the given seek position -func bytesPrefixRange(prefix, start []byte) *util.Range { - r := util.BytesPrefix(prefix) - r.Start = append(r.Start, start...) - return r -} diff --git a/ethdb/leveldb/leveldb_test.go b/ethdb/leveldb/leveldb_test.go deleted file mode 100644 index 877112fd76..0000000000 --- a/ethdb/leveldb/leveldb_test.go +++ /dev/null @@ -1,62 +0,0 @@ -// (c) 2021-2022, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package leveldb - -import ( - "testing" - - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/dbtest" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -func TestLevelDB(t *testing.T) { - t.Run("DatabaseSuite", func(t *testing.T) { - dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore { - db, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - t.Fatal(err) - } - return &Database{ - db: db, - } - }) - }) -} - -func BenchmarkLevelDB(b *testing.B) { - dbtest.BenchDatabaseSuite(b, func() ethdb.KeyValueStore { - db, err := leveldb.Open(storage.NewMemStorage(), nil) - if err != nil { - b.Fatal(err) - } - return &Database{ - db: db, - } - }) -} diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go deleted file mode 100644 index d7d14a547b..0000000000 --- a/ethdb/memorydb/memorydb.go +++ /dev/null @@ -1,333 +0,0 @@ -// (c) 2020-2021, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package memorydb implements the key-value database layer based on memory maps. -package memorydb - -import ( - "errors" - "sort" - "strings" - "sync" - - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ethereum/go-ethereum/common" -) - -var ( - // errMemorydbClosed is returned if a memory database was already closed at the - // invocation of a data access operation. - errMemorydbClosed = errors.New("database closed") - - // errMemorydbNotFound is returned if a key is requested that is not found in - // the provided memory database. - errMemorydbNotFound = errors.New("not found") -) - -// Database is an ephemeral key-value store. Apart from basic data storage -// functionality it also supports batch writes and iterating over the keyspace in -// binary-alphabetical order. -type Database struct { - db map[string][]byte - lock sync.RWMutex -} - -// New returns a wrapped map with all the required database interface methods -// implemented. -func New() *Database { - return &Database{ - db: make(map[string][]byte), - } -} - -// NewWithCap returns a wrapped map pre-allocated to the provided capacity with -// all the required database interface methods implemented. -func NewWithCap(size int) *Database { - return &Database{ - db: make(map[string][]byte, size), - } -} - -// Close deallocates the internal map and ensures any consecutive data access op -// fails with an error. -func (db *Database) Close() error { - db.lock.Lock() - defer db.lock.Unlock() - - db.db = nil - return nil -} - -// Has retrieves if a key is present in the key-value store. -func (db *Database) Has(key []byte) (bool, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - if db.db == nil { - return false, errMemorydbClosed - } - _, ok := db.db[string(key)] - return ok, nil -} - -// Get retrieves the given key if it's present in the key-value store. -func (db *Database) Get(key []byte) ([]byte, error) { - db.lock.RLock() - defer db.lock.RUnlock() - - if db.db == nil { - return nil, errMemorydbClosed - } - if entry, ok := db.db[string(key)]; ok { - return common.CopyBytes(entry), nil - } - return nil, errMemorydbNotFound -} - -// Put inserts the given value into the key-value store. -func (db *Database) Put(key []byte, value []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - if db.db == nil { - return errMemorydbClosed - } - db.db[string(key)] = common.CopyBytes(value) - return nil -} - -// Delete removes the key from the key-value store. -func (db *Database) Delete(key []byte) error { - db.lock.Lock() - defer db.lock.Unlock() - - if db.db == nil { - return errMemorydbClosed - } - delete(db.db, string(key)) - return nil -} - -// NewBatch creates a write-only key-value store that buffers changes to its host -// database until a final write is called. -func (db *Database) NewBatch() ethdb.Batch { - return &batch{ - db: db, - } -} - -// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. -func (db *Database) NewBatchWithSize(size int) ethdb.Batch { - return &batch{ - db: db, - } -} - -// NewIterator creates a binary-alphabetical iterator over a subset -// of database content with a particular key prefix, starting at a particular -// initial key (or after, if it does not exist). -func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - db.lock.RLock() - defer db.lock.RUnlock() - - var ( - pr = string(prefix) - st = string(append(prefix, start...)) - keys = make([]string, 0, len(db.db)) - values = make([][]byte, 0, len(db.db)) - ) - // Collect the keys from the memory database corresponding to the given prefix - // and start - for key := range db.db { - if !strings.HasPrefix(key, pr) { - continue - } - if key >= st { - keys = append(keys, key) - } - } - // Sort the items and retrieve the associated values - sort.Strings(keys) - for _, key := range keys { - values = append(values, db.db[key]) - } - return &iterator{ - index: -1, - keys: keys, - values: values, - } -} - -// Stat returns a particular internal stat of the database. -func (db *Database) Stat(property string) (string, error) { - return "", errors.New("unknown property") -} - -// Compact is not supported on a memory database, but there's no need either as -// a memory database doesn't waste space anyway. -func (db *Database) Compact(start []byte, limit []byte) error { - return nil -} - -// Len returns the number of entries currently present in the memory database. -// -// Note, this method is only used for testing (i.e. not public in general) and -// does not have explicit checks for closed-ness to allow simpler testing code. -func (db *Database) Len() int { - db.lock.RLock() - defer db.lock.RUnlock() - - return len(db.db) -} - -// keyvalue is a key-value tuple tagged with a deletion field to allow creating -// memory-database write batches. -type keyvalue struct { - key []byte - value []byte - delete bool -} - -// batch is a write-only memory batch that commits changes to its host -// database when Write is called. A batch cannot be used concurrently. -type batch struct { - db *Database - writes []keyvalue - size int -} - -// Put inserts the given value into the batch for later committing. -func (b *batch) Put(key, value []byte) error { - b.writes = append(b.writes, keyvalue{common.CopyBytes(key), common.CopyBytes(value), false}) - b.size += len(value) - return nil -} - -// Delete inserts the a key removal into the batch for later committing. -func (b *batch) Delete(key []byte) error { - b.writes = append(b.writes, keyvalue{common.CopyBytes(key), nil, true}) - b.size += len(key) - return nil -} - -// ValueSize retrieves the amount of data queued up for writing. -func (b *batch) ValueSize() int { - return b.size -} - -// Write flushes any accumulated data to the memory database. -func (b *batch) Write() error { - b.db.lock.Lock() - defer b.db.lock.Unlock() - - if b.db.db == nil { - return errMemorydbClosed - } - for _, keyvalue := range b.writes { - if keyvalue.delete { - delete(b.db.db, string(keyvalue.key)) - continue - } - b.db.db[string(keyvalue.key)] = keyvalue.value - } - return nil -} - -// Reset resets the batch for reuse. -func (b *batch) Reset() { - b.writes = b.writes[:0] - b.size = 0 -} - -// Replay replays the batch contents. -func (b *batch) Replay(w ethdb.KeyValueWriter) error { - for _, keyvalue := range b.writes { - if keyvalue.delete { - if err := w.Delete(keyvalue.key); err != nil { - return err - } - continue - } - if err := w.Put(keyvalue.key, keyvalue.value); err != nil { - return err - } - } - return nil -} - -// iterator can walk over the (potentially partial) keyspace of a memory key -// value store. Internally it is a deep copy of the entire iterated state, -// sorted by keys. -type iterator struct { - index int - keys []string - values [][]byte -} - -// Next moves the iterator to the next key/value pair. It returns whether the -// iterator is exhausted. -func (it *iterator) Next() bool { - // Short circuit if iterator is already exhausted in the forward direction. - if it.index >= len(it.keys) { - return false - } - it.index += 1 - return it.index < len(it.keys) -} - -// Error returns any accumulated error. Exhausting all the key/value pairs -// is not considered to be an error. A memory iterator cannot encounter errors. -func (it *iterator) Error() error { - return nil -} - -// Key returns the key of the current key/value pair, or nil if done. The caller -// should not modify the contents of the returned slice, and its contents may -// change on the next call to Next. -func (it *iterator) Key() []byte { - // Short circuit if iterator is not in a valid position - if it.index < 0 || it.index >= len(it.keys) { - return nil - } - return []byte(it.keys[it.index]) -} - -// Value returns the value of the current key/value pair, or nil if done. The -// caller should not modify the contents of the returned slice, and its contents -// may change on the next call to Next. -func (it *iterator) Value() []byte { - // Short circuit if iterator is not in a valid position - if it.index < 0 || it.index >= len(it.keys) { - return nil - } - return it.values[it.index] -} - -// Release releases associated resources. Release should always succeed and can -// be called multiple times without causing error. -func (it *iterator) Release() { - it.index, it.keys, it.values = -1, nil, nil -} diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go deleted file mode 100644 index 5e06e63e94..0000000000 --- a/ethdb/pebble/pebble.go +++ /dev/null @@ -1,629 +0,0 @@ -// (c) 2023, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build (arm64 || amd64) && !openbsd - -// Package pebble implements the key-value database layer based on pebble. -package pebble - -import ( - "bytes" - "fmt" - "runtime" - "sync" - "sync/atomic" - "time" - - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/metrics" - "github.com/cockroachdb/pebble" - "github.com/cockroachdb/pebble/bloom" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/log" -) - -const ( - // minCache is the minimum amount of memory in megabytes to allocate to pebble - // read and write caching, split half and half. - minCache = 16 - - // minHandles is the minimum number of files handles to allocate to the open - // database files. - minHandles = 16 - - // metricsGatheringInterval specifies the interval to retrieve pebble database - // compaction, io and pause stats to report to the user. - metricsGatheringInterval = 3 * time.Second -) - -// Database is a persistent key-value store based on the pebble storage engine. -// Apart from basic data storage functionality it also supports batch writes and -// iterating over the keyspace in binary-alphabetical order. -type Database struct { - fn string // filename for reporting - db *pebble.DB // Underlying pebble storage engine - - compTimeMeter metrics.Meter // Meter for measuring the total time spent in database compaction - compReadMeter metrics.Meter // Meter for measuring the data read during compaction - compWriteMeter metrics.Meter // Meter for measuring the data written during compaction - writeDelayNMeter metrics.Meter // Meter for measuring the write delay number due to database compaction - writeDelayMeter metrics.Meter // Meter for measuring the write delay duration due to database compaction - diskSizeGauge metrics.Gauge // Gauge for tracking the size of all the levels in the database - diskReadMeter metrics.Meter // Meter for measuring the effective amount of data read - diskWriteMeter metrics.Meter // Meter for measuring the effective amount of data written - memCompGauge metrics.Gauge // Gauge for tracking the number of memory compaction - level0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in level0 - nonlevel0CompGauge metrics.Gauge // Gauge for tracking the number of table compaction in non0 level - seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt - manualMemAllocGauge metrics.Gauge // Gauge for tracking amount of non-managed memory currently allocated - - quitLock sync.RWMutex // Mutex protecting the quit channel and the closed flag - quitChan chan chan error // Quit channel to stop the metrics collection before closing the database - closed bool // keep track of whether we're Closed - - log log.Logger // Contextual logger tracking the database path - - activeComp int // Current number of active compactions - compStartTime time.Time // The start time of the earliest currently-active compaction - compTime atomic.Int64 // Total time spent in compaction in ns - level0Comp atomic.Uint32 // Total number of level-zero compactions - nonLevel0Comp atomic.Uint32 // Total number of non level-zero compactions - writeDelayStartTime time.Time // The start time of the latest write stall - writeDelayCount atomic.Int64 // Total number of write stall counts - writeDelayTime atomic.Int64 // Total time spent in write stalls -} - -func (d *Database) onCompactionBegin(info pebble.CompactionInfo) { - if d.activeComp == 0 { - d.compStartTime = time.Now() - } - l0 := info.Input[0] - if l0.Level == 0 { - d.level0Comp.Add(1) - } else { - d.nonLevel0Comp.Add(1) - } - d.activeComp++ -} - -func (d *Database) onCompactionEnd(info pebble.CompactionInfo) { - if d.activeComp == 1 { - d.compTime.Add(int64(time.Since(d.compStartTime))) - } else if d.activeComp == 0 { - panic("should not happen") - } - d.activeComp-- -} - -func (d *Database) onWriteStallBegin(b pebble.WriteStallBeginInfo) { - d.writeDelayStartTime = time.Now() -} - -func (d *Database) onWriteStallEnd() { - d.writeDelayTime.Add(int64(time.Since(d.writeDelayStartTime))) -} - -// New returns a wrapped pebble DB object. The namespace is the prefix that the -// metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { - // Ensure we have some minimal caching and file guarantees - if cache < minCache { - cache = minCache - } - if handles < minHandles { - handles = minHandles - } - logger := log.New("database", file) - logger.Info("Allocated cache and file handles", "cache", common.StorageSize(cache*1024*1024), "handles", handles) - - // The max memtable size is limited by the uint32 offsets stored in - // internal/arenaskl.node, DeferredBatchOp, and flushableBatchEntry. - // Taken from https://github.com/cockroachdb/pebble/blob/master/open.go#L38 - maxMemTableSize := 4<<30 - 1 // Capped by 4 GB - - // Two memory tables is configured which is identical to leveldb, - // including a frozen memory table and another live one. - memTableLimit := 2 - memTableSize := cache * 1024 * 1024 / 2 / memTableLimit - if memTableSize > maxMemTableSize { - memTableSize = maxMemTableSize - } - db := &Database{ - fn: file, - log: logger, - quitChan: make(chan chan error), - } - opt := &pebble.Options{ - // Pebble has a single combined cache area and the write - // buffers are taken from this too. Assign all available - // memory allowance for cache. - Cache: pebble.NewCache(int64(cache * 1024 * 1024)), - MaxOpenFiles: handles, - - // The size of memory table(as well as the write buffer). - // Note, there may have more than two memory tables in the system. - MemTableSize: memTableSize, - - // MemTableStopWritesThreshold places a hard limit on the size - // of the existent MemTables(including the frozen one). - // Note, this must be the number of tables not the size of all memtables - // according to https://github.com/cockroachdb/pebble/blob/master/options.go#L738-L742 - // and to https://github.com/cockroachdb/pebble/blob/master/db.go#L1892-L1903. - MemTableStopWritesThreshold: memTableLimit, - - // The default compaction concurrency(1 thread), - // Here use all available CPUs for faster compaction. - MaxConcurrentCompactions: func() int { return runtime.NumCPU() }, - - // Per-level options. Options for at least one level must be specified. The - // options for the last level are used for all subsequent levels. - Levels: []pebble.LevelOptions{ - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - {TargetFileSize: 2 * 1024 * 1024, FilterPolicy: bloom.FilterPolicy(10)}, - }, - ReadOnly: readonly, - EventListener: &pebble.EventListener{ - CompactionBegin: db.onCompactionBegin, - CompactionEnd: db.onCompactionEnd, - WriteStallBegin: db.onWriteStallBegin, - WriteStallEnd: db.onWriteStallEnd, - }, - } - // Disable seek compaction explicitly. Check https://github.com/ethereum/go-ethereum/pull/20130 - // for more details. - opt.Experimental.ReadSamplingMultiplier = -1 - - // Open the db and recover any potential corruptions - innerDB, err := pebble.Open(file, opt) - if err != nil { - return nil, err - } - db.db = innerDB - - db.compTimeMeter = metrics.NewRegisteredMeter(namespace+"compact/time", nil) - db.compReadMeter = metrics.NewRegisteredMeter(namespace+"compact/input", nil) - db.compWriteMeter = metrics.NewRegisteredMeter(namespace+"compact/output", nil) - db.diskSizeGauge = metrics.NewRegisteredGauge(namespace+"disk/size", nil) - db.diskReadMeter = metrics.NewRegisteredMeter(namespace+"disk/read", nil) - db.diskWriteMeter = metrics.NewRegisteredMeter(namespace+"disk/write", nil) - db.writeDelayMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/duration", nil) - db.writeDelayNMeter = metrics.NewRegisteredMeter(namespace+"compact/writedelay/counter", nil) - db.memCompGauge = metrics.NewRegisteredGauge(namespace+"compact/memory", nil) - db.level0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/level0", nil) - db.nonlevel0CompGauge = metrics.NewRegisteredGauge(namespace+"compact/nonlevel0", nil) - db.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) - db.manualMemAllocGauge = metrics.NewRegisteredGauge(namespace+"memory/manualalloc", nil) - - // Start up the metrics gathering and return - go db.meter(metricsGatheringInterval) - return db, nil -} - -// Close stops the metrics collection, flushes any pending data to disk and closes -// all io accesses to the underlying key-value store. -func (d *Database) Close() error { - d.quitLock.Lock() - defer d.quitLock.Unlock() - // Allow double closing, simplifies things - if d.closed { - return nil - } - d.closed = true - if d.quitChan != nil { - errc := make(chan error) - d.quitChan <- errc - if err := <-errc; err != nil { - d.log.Error("Metrics collection failed", "err", err) - } - d.quitChan = nil - } - return d.db.Close() -} - -// Has retrieves if a key is present in the key-value store. -func (d *Database) Has(key []byte) (bool, error) { - d.quitLock.RLock() - defer d.quitLock.RUnlock() - if d.closed { - return false, pebble.ErrClosed - } - _, closer, err := d.db.Get(key) - if err == pebble.ErrNotFound { - return false, nil - } else if err != nil { - return false, err - } - closer.Close() - return true, nil -} - -// Get retrieves the given key if it's present in the key-value store. -func (d *Database) Get(key []byte) ([]byte, error) { - d.quitLock.RLock() - defer d.quitLock.RUnlock() - if d.closed { - return nil, pebble.ErrClosed - } - dat, closer, err := d.db.Get(key) - if err != nil { - return nil, err - } - ret := make([]byte, len(dat)) - copy(ret, dat) - closer.Close() - return ret, nil -} - -// Put inserts the given value into the key-value store. -func (d *Database) Put(key []byte, value []byte) error { - d.quitLock.RLock() - defer d.quitLock.RUnlock() - if d.closed { - return pebble.ErrClosed - } - return d.db.Set(key, value, pebble.NoSync) -} - -// Delete removes the key from the key-value store. -func (d *Database) Delete(key []byte) error { - d.quitLock.RLock() - defer d.quitLock.RUnlock() - if d.closed { - return pebble.ErrClosed - } - return d.db.Delete(key, nil) -} - -// NewBatch creates a write-only key-value store that buffers changes to its host -// database until a final write is called. -func (d *Database) NewBatch() ethdb.Batch { - return &batch{ - b: d.db.NewBatch(), - db: d, - } -} - -// NewBatchWithSize creates a write-only database batch with pre-allocated buffer. -// It's not supported by pebble, but pebble has better memory allocation strategy -// which turns out a lot faster than leveldb. It's performant enough to construct -// batch object without any pre-allocated space. -func (d *Database) NewBatchWithSize(_ int) ethdb.Batch { - return &batch{ - b: d.db.NewBatch(), - db: d, - } -} - -// snapshot wraps a pebble snapshot for implementing the Snapshot interface. -type snapshot struct { - db *pebble.Snapshot -} - -// NewSnapshot creates a database snapshot based on the current state. -// The created snapshot will not be affected by all following mutations -// happened on the database. -// Note don't forget to release the snapshot once it's used up, otherwise -// the stale data will never be cleaned up by the underlying compactor. -func (d *Database) NewSnapshot() (ethdb.Snapshot, error) { - snap := d.db.NewSnapshot() - return &snapshot{db: snap}, nil -} - -// Has retrieves if a key is present in the snapshot backing by a key-value -// data store. -func (snap *snapshot) Has(key []byte) (bool, error) { - _, closer, err := snap.db.Get(key) - if err != nil { - if err != pebble.ErrNotFound { - return false, err - } else { - return false, nil - } - } - closer.Close() - return true, nil -} - -// Get retrieves the given key if it's present in the snapshot backing by -// key-value data store. -func (snap *snapshot) Get(key []byte) ([]byte, error) { - dat, closer, err := snap.db.Get(key) - if err != nil { - return nil, err - } - ret := make([]byte, len(dat)) - copy(ret, dat) - closer.Close() - return ret, nil -} - -// Release releases associated resources. Release should always succeed and can -// be called multiple times without causing error. -func (snap *snapshot) Release() { - snap.db.Close() -} - -// upperBound returns the upper bound for the given prefix -func upperBound(prefix []byte) (limit []byte) { - for i := len(prefix) - 1; i >= 0; i-- { - c := prefix[i] - if c == 0xff { - continue - } - limit = make([]byte, i+1) - copy(limit, prefix) - limit[i] = c + 1 - break - } - return limit -} - -// Stat returns a particular internal stat of the database. -func (d *Database) Stat(property string) (string, error) { - return "", nil -} - -// Compact flattens the underlying data store for the given key range. In essence, -// deleted and overwritten versions are discarded, and the data is rearranged to -// reduce the cost of operations needed to access them. -// -// A nil start is treated as a key before all keys in the data store; a nil limit -// is treated as a key after all keys in the data store. If both is nil then it -// will compact entire data store. -func (d *Database) Compact(start []byte, limit []byte) error { - // There is no special flag to represent the end of key range - // in pebble(nil in leveldb). Use an ugly hack to construct a - // large key to represent it. - // Note any prefixed database entry will be smaller than this - // flag, as for trie nodes we need the 32 byte 0xff because - // there might be a shared prefix starting with a number of - // 0xff-s, so 32 ensures than only a hash collision could touch it. - // https://github.com/cockroachdb/pebble/issues/2359#issuecomment-1443995833 - if limit == nil { - limit = bytes.Repeat([]byte{0xff}, 32) - } - return d.db.Compact(start, limit, true) // Parallelization is preferred -} - -// Path returns the path to the database directory. -func (d *Database) Path() string { - return d.fn -} - -// meter periodically retrieves internal pebble counters and reports them to -// the metrics subsystem. -func (d *Database) meter(refresh time.Duration) { - var errc chan error - timer := time.NewTimer(refresh) - defer timer.Stop() - - // Create storage and warning log tracer for write delay. - var ( - compTimes [2]int64 - writeDelayTimes [2]int64 - writeDelayCounts [2]int64 - compWrites [2]int64 - compReads [2]int64 - - nWrites [2]int64 - ) - - // Iterate ad infinitum and collect the stats - for i := 1; errc == nil; i++ { - var ( - compWrite int64 - compRead int64 - nWrite int64 - - metrics = d.db.Metrics() - compTime = d.compTime.Load() - writeDelayCount = d.writeDelayCount.Load() - writeDelayTime = d.writeDelayTime.Load() - nonLevel0CompCount = int64(d.nonLevel0Comp.Load()) - level0CompCount = int64(d.level0Comp.Load()) - ) - writeDelayTimes[i%2] = writeDelayTime - writeDelayCounts[i%2] = writeDelayCount - compTimes[i%2] = compTime - - for _, levelMetrics := range metrics.Levels { - nWrite += int64(levelMetrics.BytesCompacted) - nWrite += int64(levelMetrics.BytesFlushed) - compWrite += int64(levelMetrics.BytesCompacted) - compRead += int64(levelMetrics.BytesRead) - } - - nWrite += int64(metrics.WAL.BytesWritten) - - compWrites[i%2] = compWrite - compReads[i%2] = compRead - nWrites[i%2] = nWrite - - if d.writeDelayNMeter != nil { - d.writeDelayNMeter.Mark(writeDelayCounts[i%2] - writeDelayCounts[(i-1)%2]) - } - if d.writeDelayMeter != nil { - d.writeDelayMeter.Mark(writeDelayTimes[i%2] - writeDelayTimes[(i-1)%2]) - } - if d.compTimeMeter != nil { - d.compTimeMeter.Mark(compTimes[i%2] - compTimes[(i-1)%2]) - } - if d.compReadMeter != nil { - d.compReadMeter.Mark(compReads[i%2] - compReads[(i-1)%2]) - } - if d.compWriteMeter != nil { - d.compWriteMeter.Mark(compWrites[i%2] - compWrites[(i-1)%2]) - } - if d.diskSizeGauge != nil { - d.diskSizeGauge.Update(int64(metrics.DiskSpaceUsage())) - } - if d.diskReadMeter != nil { - d.diskReadMeter.Mark(0) // pebble doesn't track non-compaction reads - } - if d.diskWriteMeter != nil { - d.diskWriteMeter.Mark(nWrites[i%2] - nWrites[(i-1)%2]) - } - // See https://github.com/cockroachdb/pebble/pull/1628#pullrequestreview-1026664054 - manuallyAllocated := metrics.BlockCache.Size + int64(metrics.MemTable.Size) + int64(metrics.MemTable.ZombieSize) - d.manualMemAllocGauge.Update(manuallyAllocated) - d.memCompGauge.Update(metrics.Flush.Count) - d.nonlevel0CompGauge.Update(nonLevel0CompCount) - d.level0CompGauge.Update(level0CompCount) - d.seekCompGauge.Update(metrics.Compact.ReadCount) - - // Sleep a bit, then repeat the stats collection - select { - case errc = <-d.quitChan: - // Quit requesting, stop hammering the database - case <-timer.C: - timer.Reset(refresh) - // Timeout, gather a new set of stats - } - } - errc <- nil -} - -// batch is a write-only batch that commits changes to its host database -// when Write is called. A batch cannot be used concurrently. -type batch struct { - b *pebble.Batch - db *Database - size int -} - -// Put inserts the given value into the batch for later committing. -func (b *batch) Put(key, value []byte) error { - b.b.Set(key, value, nil) - b.size += len(key) + len(value) - return nil -} - -// Delete inserts the a key removal into the batch for later committing. -func (b *batch) Delete(key []byte) error { - b.b.Delete(key, nil) - b.size += len(key) - return nil -} - -// ValueSize retrieves the amount of data queued up for writing. -func (b *batch) ValueSize() int { - return b.size -} - -// Write flushes any accumulated data to disk. -func (b *batch) Write() error { - b.db.quitLock.RLock() - defer b.db.quitLock.RUnlock() - if b.db.closed { - return pebble.ErrClosed - } - return b.b.Commit(pebble.NoSync) -} - -// Reset resets the batch for reuse. -func (b *batch) Reset() { - b.b.Reset() - b.size = 0 -} - -// Replay replays the batch contents. -func (b *batch) Replay(w ethdb.KeyValueWriter) error { - reader := b.b.Reader() - for { - kind, k, v, ok := reader.Next() - if !ok { - break - } - // The (k,v) slices might be overwritten if the batch is reset/reused, - // and the receiver should copy them if they are to be retained long-term. - if kind == pebble.InternalKeyKindSet { - w.Put(k, v) - } else if kind == pebble.InternalKeyKindDelete { - w.Delete(k) - } else { - return fmt.Errorf("unhandled operation, keytype: %v", kind) - } - } - return nil -} - -// pebbleIterator is a wrapper of underlying iterator in storage engine. -// The purpose of this structure is to implement the missing APIs. -type pebbleIterator struct { - iter *pebble.Iterator - moved bool -} - -// NewIterator creates a binary-alphabetical iterator over a subset -// of database content with a particular key prefix, starting at a particular -// initial key (or after, if it does not exist). -func (d *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - iter := d.db.NewIter(&pebble.IterOptions{ - LowerBound: append(prefix, start...), - UpperBound: upperBound(prefix), - }) - iter.First() - return &pebbleIterator{iter: iter, moved: true} -} - -// Next moves the iterator to the next key/value pair. It returns whether the -// iterator is exhausted. -func (iter *pebbleIterator) Next() bool { - if iter.moved { - iter.moved = false - return iter.iter.Valid() - } - return iter.iter.Next() -} - -// Error returns any accumulated error. Exhausting all the key/value pairs -// is not considered to be an error. -func (iter *pebbleIterator) Error() error { - return iter.iter.Error() -} - -// Key returns the key of the current key/value pair, or nil if done. The caller -// should not modify the contents of the returned slice, and its contents may -// change on the next call to Next. -func (iter *pebbleIterator) Key() []byte { - return iter.iter.Key() -} - -// Value returns the value of the current key/value pair, or nil if done. The -// caller should not modify the contents of the returned slice, and its contents -// may change on the next call to Next. -func (iter *pebbleIterator) Value() []byte { - return iter.iter.Value() -} - -// Release releases associated resources. Release should always succeed and can -// be called multiple times without causing error. -func (iter *pebbleIterator) Release() { iter.iter.Close() } diff --git a/ethdb/pebble/pebble_test.go b/ethdb/pebble/pebble_test.go deleted file mode 100644 index 1f3e4f9315..0000000000 --- a/ethdb/pebble/pebble_test.go +++ /dev/null @@ -1,68 +0,0 @@ -// (c) 2023, Ava Labs, Inc. -// -// This file is a derived work, based on the go-ethereum library whose original -// notices appear below. -// -// It is distributed under a license compatible with the licensing terms of the -// original code from which it is derived. -// -// Much love to the original authors for their work. -// ********** -// Copyright 2023 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build (arm64 || amd64) && !openbsd - -package pebble - -import ( - "testing" - - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/dbtest" - "github.com/cockroachdb/pebble" - "github.com/cockroachdb/pebble/vfs" -) - -func TestPebbleDB(t *testing.T) { - t.Run("DatabaseSuite", func(t *testing.T) { - dbtest.TestDatabaseSuite(t, func() ethdb.KeyValueStore { - db, err := pebble.Open("", &pebble.Options{ - FS: vfs.NewMem(), - }) - if err != nil { - t.Fatal(err) - } - return &Database{ - db: db, - } - }) - }) -} - -func BenchmarkPebbleDB(b *testing.B) { - dbtest.BenchDatabaseSuite(b, func() ethdb.KeyValueStore { - db, err := pebble.Open("", &pebble.Options{ - FS: vfs.NewMem(), - }) - if err != nil { - b.Fatal(err) - } - return &Database{ - db: db, - } - }) -} diff --git a/ethdb/snapshot.go b/ethdb/snapshot.go deleted file mode 100644 index 03b7794a77..0000000000 --- a/ethdb/snapshot.go +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package ethdb - -type Snapshot interface { - // Has retrieves if a key is present in the snapshot backing by a key-value - // data store. - Has(key []byte) (bool, error) - - // Get retrieves the given key if it's present in the snapshot backing by - // key-value data store. - Get(key []byte) ([]byte, error) - - // Release releases associated resources. Release should always succeed and can - // be called multiple times without causing error. - Release() -} - -// Snapshotter wraps the Snapshot method of a backing data store. -type Snapshotter interface { - // NewSnapshot creates a database snapshot based on the current state. - // The created snapshot will not be affected by all following mutations - // happened on the database. - // Note don't forget to release the snapshot once it's used up, otherwise - // the stale data will never be cleaned up by the underlying compactor. - NewSnapshot() (Snapshot, error) -} diff --git a/go.mod b/go.mod index d04b55474e..f7746814aa 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/ava-labs/avalanche-network-runner v1.7.4-0.20231127162258-2f3ceed8ae4b github.com/ava-labs/avalanchego v1.10.17 github.com/cespare/cp v0.1.0 - github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 github.com/docker/docker v1.6.2 @@ -39,7 +38,6 @@ require ( github.com/spf13/viper v1.12.0 github.com/status-im/keycard-go v0.2.0 github.com/stretchr/testify v1.8.4 - github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa go.uber.org/goleak v1.2.1 @@ -66,6 +64,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/errors v1.9.1 // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect github.com/cockroachdb/redact v1.1.3 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.10.0 // indirect @@ -130,6 +129,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/subosito/gotenv v1.3.0 // indirect github.com/supranational/blst v0.3.11 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a // indirect github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tklauser/numcpus v0.2.2 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 5f170b86ed..dd59dd89e3 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -49,13 +49,13 @@ import ( "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "golang.org/x/crypto/sha3" ) diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 832f939cee..cf27d124fb 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -40,10 +40,10 @@ import ( "github.com/ava-labs/subnet-evm/core/state" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" ) diff --git a/internal/shutdowncheck/shutdown_tracker.go b/internal/shutdowncheck/shutdown_tracker.go index afb5f059a0..25613ef784 100644 --- a/internal/shutdowncheck/shutdown_tracker.go +++ b/internal/shutdowncheck/shutdown_tracker.go @@ -30,8 +30,8 @@ import ( "time" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/plugin/evm/block_test.go b/plugin/evm/block_test.go index d00193c76b..90c56138ef 100644 --- a/plugin/evm/block_test.go +++ b/plugin/evm/block_test.go @@ -7,7 +7,6 @@ import ( "math/big" "testing" - "github.com/ava-labs/avalanchego/database/memdb" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/params" @@ -23,9 +22,9 @@ func TestHandlePrecompileAccept(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - db := memdb.New() + db := rawdb.NewMemoryDatabase() vm := &VM{ - chaindb: Database{db}, + chaindb: db, chainConfig: params.TestChainConfig, } diff --git a/plugin/evm/database.go b/plugin/evm/database.go index 6f26b9ac0b..479c995ba3 100644 --- a/plugin/evm/database.go +++ b/plugin/evm/database.go @@ -4,12 +4,17 @@ package evm import ( - "github.com/ava-labs/subnet-evm/ethdb" + "errors" "github.com/ava-labs/avalanchego/database" + "github.com/ethereum/go-ethereum/ethdb" ) -var _ ethdb.Database = &Database{} +var ( + _ ethdb.KeyValueStore = &Database{} + + ErrSnapshotNotSupported = errors.New("snapshot is not supported") +) // Database implements ethdb.Database type Database struct{ database.Database } @@ -24,6 +29,10 @@ func (db Database) NewBatch() ethdb.Batch { return Batch{db.Database.NewBatch()} // TODO: propagate size through avalanchego Database interface func (db Database) NewBatchWithSize(size int) ethdb.Batch { return Batch{db.Database.NewBatch()} } +func (db Database) NewSnapshot() (ethdb.Snapshot, error) { + return nil, ErrSnapshotNotSupported +} + // NewIterator implements ethdb.Database // // Note: This method assumes that the prefix is NOT part of the start, so there's diff --git a/plugin/evm/network_handler.go b/plugin/evm/network_handler.go index 21bbf83d7e..cf31035358 100644 --- a/plugin/evm/network_handler.go +++ b/plugin/evm/network_handler.go @@ -8,7 +8,6 @@ import ( "github.com/ava-labs/avalanchego/codec" "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/plugin/evm/message" syncHandlers "github.com/ava-labs/subnet-evm/sync/handlers" @@ -16,6 +15,7 @@ import ( "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/warp" warpHandlers "github.com/ava-labs/subnet-evm/warp/handlers" + "github.com/ethereum/go-ethereum/ethdb" ) var _ message.RequestHandler = &networkHandler{} diff --git a/plugin/evm/syncervm_client.go b/plugin/evm/syncervm_client.go index 2a3f4c3cbf..bec4897341 100644 --- a/plugin/evm/syncervm_client.go +++ b/plugin/evm/syncervm_client.go @@ -18,12 +18,12 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/eth" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/plugin/evm/message" syncclient "github.com/ava-labs/subnet-evm/sync/client" "github.com/ava-labs/subnet-evm/sync/statesync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/plugin/evm/syncervm_test.go b/plugin/evm/syncervm_test.go index a51b3e22e1..8c6bc77ec6 100644 --- a/plugin/evm/syncervm_test.go +++ b/plugin/evm/syncervm_test.go @@ -30,7 +30,6 @@ import ( "github.com/ava-labs/subnet-evm/core" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/predicate" @@ -38,6 +37,7 @@ import ( "github.com/ava-labs/subnet-evm/sync/statesync" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/plugin/evm/vm.go b/plugin/evm/vm.go index c3666dbb70..f795982539 100644 --- a/plugin/evm/vm.go +++ b/plugin/evm/vm.go @@ -57,6 +57,7 @@ import ( _ "github.com/ava-labs/subnet-evm/precompile/registry" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -202,7 +203,7 @@ type VM struct { metadataDB database.Database // [chaindb] is the database supplied to the Ethereum backend - chaindb Database + chaindb ethdb.Database // [acceptedBlockDB] is the database to store the last accepted // block. @@ -299,7 +300,7 @@ func (vm *VM) Initialize( vm.shutdownChan = make(chan struct{}, 1) // Use NewNested rather than New so that the structure of the database // remains the same regardless of the provided baseDB type. - vm.chaindb = Database{prefixdb.NewNested(ethDBPrefix, db)} + vm.chaindb = rawdb.NewDatabase(Database{prefixdb.NewNested(ethDBPrefix, db)}) vm.db = versiondb.New(db) vm.acceptedBlockDB = prefixdb.New(acceptedPrefix, vm.db) vm.metadataDB = prefixdb.New(metadataPrefix, vm.db) diff --git a/scripts/geth-allowed-packages.txt b/scripts/geth-allowed-packages.txt index b697ede5cf..beb384e49a 100644 --- a/scripts/geth-allowed-packages.txt +++ b/scripts/geth-allowed-packages.txt @@ -11,6 +11,10 @@ "github.com/ethereum/go-ethereum/crypto/blake2b" "github.com/ethereum/go-ethereum/crypto/bls12381" "github.com/ethereum/go-ethereum/crypto/bn256" +"github.com/ethereum/go-ethereum/ethdb" +"github.com/ethereum/go-ethereum/ethdb/leveldb" +"github.com/ethereum/go-ethereum/ethdb/memorydb" +"github.com/ethereum/go-ethereum/ethdb/pebble" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" diff --git a/sync/client/client.go b/sync/client/client.go index 5168aa8dda..c6aa5b1e5d 100644 --- a/sync/client/client.go +++ b/sync/client/client.go @@ -13,7 +13,6 @@ import ( "github.com/ava-labs/avalanchego/ids" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/sync/client/stats" @@ -24,11 +23,12 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/peer" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/trie" + "github.com/ethereum/go-ethereum/ethdb" ) const ( @@ -151,7 +151,7 @@ func parseLeafsResponse(codec codec.Manager, reqIntf message.Request, data []byt // Populate proof when ProofVals are present in the response. Its ok to pass it as nil to the trie.VerifyRangeProof // function as it will assert that all the leaves belonging to the specified root are present. if len(leafsResponse.ProofVals) > 0 { - proof = memorydb.New() + proof = rawdb.NewMemoryDatabase() defer proof.Close() for _, proofVal := range leafsResponse.ProofVals { proofKey := crypto.Keccak256(proofVal) diff --git a/sync/client/client_test.go b/sync/client/client_test.go index 3c907b4a42..251e6bb276 100644 --- a/sync/client/client_test.go +++ b/sync/client/client_test.go @@ -17,8 +17,8 @@ import ( "github.com/ava-labs/subnet-evm/consensus/dummy" "github.com/ava-labs/subnet-evm/core" + "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/plugin/evm/message" clientstats "github.com/ava-labs/subnet-evm/sync/client/stats" @@ -141,7 +141,7 @@ func TestGetBlocks(t *testing.T) { var gspec = &core.Genesis{ Config: params.TestChainConfig, } - memdb := memorydb.New() + memdb := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(memdb) engine := dummy.NewETHFaker() numBlocks := 110 @@ -409,7 +409,7 @@ func TestGetLeafs(t *testing.T) { const leafsLimit = 1024 - trieDB := trie.NewDatabase(memorydb.New()) + trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase()) largeTrieRoot, largeTrieKeys, _ := trie.GenerateTrie(t, trieDB, 100_000, common.HashLength) smallTrieRoot, _, _ := trie.GenerateTrie(t, trieDB, leafsLimit, common.HashLength) @@ -780,7 +780,7 @@ func TestGetLeafs(t *testing.T) { func TestGetLeafsRetries(t *testing.T) { rand.Seed(1) - trieDB := trie.NewDatabase(memorydb.New()) + trieDB := trie.NewDatabase(rawdb.NewMemoryDatabase()) root, _, _ := trie.GenerateTrie(t, trieDB, 100_000, common.HashLength) handler := handlers.NewLeafsRequestHandler(trieDB, nil, message.Codec, handlerstats.NewNoopHandlerStats()) diff --git a/sync/handlers/block_request_test.go b/sync/handlers/block_request_test.go index c3886ce78a..241eb6d31a 100644 --- a/sync/handlers/block_request_test.go +++ b/sync/handlers/block_request_test.go @@ -10,8 +10,8 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/subnet-evm/consensus/dummy" "github.com/ava-labs/subnet-evm/core" + "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/sync/handlers/stats" @@ -24,7 +24,7 @@ func TestBlockRequestHandler(t *testing.T) { var gspec = &core.Genesis{ Config: params.TestChainConfig, } - memdb := memorydb.New() + memdb := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(memdb) engine := dummy.NewETHFaker() blocks, _, err := core.GenerateChain(params.TestChainConfig, genesis, engine, memdb, 96, 0, func(i int, b *core.BlockGen) {}) @@ -146,7 +146,7 @@ func TestBlockRequestHandlerCtxExpires(t *testing.T) { var gspec = &core.Genesis{ Config: params.TestChainConfig, } - memdb := memorydb.New() + memdb := rawdb.NewMemoryDatabase() genesis := gspec.MustCommit(memdb) engine := dummy.NewETHFaker() blocks, _, err := core.GenerateChain(params.TestChainConfig, genesis, engine, memdb, 11, 0, func(i int, b *core.BlockGen) {}) diff --git a/sync/handlers/code_request.go b/sync/handlers/code_request.go index 08162b85af..cbe29b3b94 100644 --- a/sync/handlers/code_request.go +++ b/sync/handlers/code_request.go @@ -11,10 +11,10 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/sync/handlers/stats" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/sync/handlers/code_request_test.go b/sync/handlers/code_request_test.go index a7094742e1..31112f636b 100644 --- a/sync/handlers/code_request_test.go +++ b/sync/handlers/code_request_test.go @@ -12,11 +12,11 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/sync/handlers/stats" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/stretchr/testify/assert" ) diff --git a/sync/handlers/leafs_request.go b/sync/handlers/leafs_request.go index c20f00d85e..c543696db5 100644 --- a/sync/handlers/leafs_request.go +++ b/sync/handlers/leafs_request.go @@ -14,14 +14,14 @@ import ( "github.com/ava-labs/avalanchego/utils/math" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/sync/handlers/stats" "github.com/ava-labs/subnet-evm/sync/syncutils" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" ) diff --git a/sync/handlers/leafs_request_test.go b/sync/handlers/leafs_request_test.go index 6d374c10cd..860fbfdc21 100644 --- a/sync/handlers/leafs_request_test.go +++ b/sync/handlers/leafs_request_test.go @@ -13,13 +13,12 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/plugin/evm/message" "github.com/ava-labs/subnet-evm/sync/handlers/stats" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" ) @@ -27,7 +26,7 @@ import ( func TestLeafsRequestHandler_OnLeafsRequest(t *testing.T) { rand.Seed(1) mockHandlerStats := &stats.MockHandlerStats{} - memdb := memorydb.New() + memdb := rawdb.NewMemoryDatabase() trieDB := trie.NewDatabase(memdb) corruptedTrieRoot, _, _ := trie.GenerateTrie(t, trieDB, 100, common.HashLength) @@ -686,7 +685,7 @@ func assertRangeProofIsValid(t *testing.T, request *message.LeafsRequest, respon var proof ethdb.Database if len(response.ProofVals) > 0 { - proof = memorydb.New() + proof = rawdb.NewMemoryDatabase() defer proof.Close() for _, proofVal := range response.ProofVals { proofKey := crypto.Keccak256(proofVal) diff --git a/sync/statesync/code_syncer.go b/sync/statesync/code_syncer.go index f31a6d7645..296d958c2c 100644 --- a/sync/statesync/code_syncer.go +++ b/sync/statesync/code_syncer.go @@ -12,11 +12,10 @@ import ( "github.com/ava-labs/avalanchego/ids" "github.com/ava-labs/avalanchego/utils/set" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/plugin/evm/message" statesyncclient "github.com/ava-labs/subnet-evm/sync/client" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) const ( diff --git a/sync/statesync/code_syncer_test.go b/sync/statesync/code_syncer_test.go index 84b4909768..dbc9b7969a 100644 --- a/sync/statesync/code_syncer_test.go +++ b/sync/statesync/code_syncer_test.go @@ -10,13 +10,13 @@ import ( "github.com/ava-labs/avalanchego/utils" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/plugin/evm/message" statesyncclient "github.com/ava-labs/subnet-evm/sync/client" "github.com/ava-labs/subnet-evm/sync/handlers" handlerstats "github.com/ava-labs/subnet-evm/sync/handlers/stats" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/stretchr/testify/assert" ) @@ -44,7 +44,7 @@ func testCodeSyncer(t *testing.T, test codeSyncerTest) { mockClient := statesyncclient.NewMockClient(message.Codec, nil, codeRequestHandler, nil) mockClient.GetCodeIntercept = test.getCodeIntercept - clientDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() codeSyncer := newCodeSyncer(CodeSyncerConfig{ MaxOutstandingCodeHashes: DefaultMaxOutstandingCodeHashes, diff --git a/sync/statesync/state_syncer.go b/sync/statesync/state_syncer.go index ef38f14c36..9c646148d3 100644 --- a/sync/statesync/state_syncer.go +++ b/sync/statesync/state_syncer.go @@ -9,10 +9,10 @@ import ( "sync" "github.com/ava-labs/subnet-evm/core/state/snapshot" - "github.com/ava-labs/subnet-evm/ethdb" syncclient "github.com/ava-labs/subnet-evm/sync/client" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "golang.org/x/sync/errgroup" ) diff --git a/sync/statesync/sync_helpers.go b/sync/statesync/sync_helpers.go index d0d2ca81b0..c11844cf55 100644 --- a/sync/statesync/sync_helpers.go +++ b/sync/statesync/sync_helpers.go @@ -7,9 +7,9 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // writeAccountSnapshot stores the account represented by [acc] to the snapshot at [accHash], using diff --git a/sync/statesync/sync_test.go b/sync/statesync/sync_test.go index d9a31aa9ee..e8e04195ed 100644 --- a/sync/statesync/sync_test.go +++ b/sync/statesync/sync_test.go @@ -16,8 +16,6 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/plugin/evm/message" statesyncclient "github.com/ava-labs/subnet-evm/sync/client" "github.com/ava-labs/subnet-evm/sync/handlers" @@ -25,6 +23,7 @@ import ( "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" ) @@ -119,15 +118,15 @@ func TestSimpleSyncCases(t *testing.T) { tests := map[string]syncTest{ "accounts": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, nil) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, }, "accounts with code": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, index int, account types.StateAccount) types.StateAccount { if index%3 == 0 { @@ -143,20 +142,20 @@ func TestSimpleSyncCases(t *testing.T) { } return account }) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, }, "accounts with code and storage": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root := fillAccountsWithStorage(t, serverDB, serverTrieDB, common.Hash{}, numAccounts) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, }, "accounts with storage": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccounts, func(t *testing.T, i int, account types.StateAccount) types.StateAccount { if i%5 == 0 { @@ -165,23 +164,23 @@ func TestSimpleSyncCases(t *testing.T) { return account }) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, }, "accounts with overlapping storage": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, common.Hash{}, numAccounts, 3) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, }, "failed to fetch leafs": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root, _ := trie.FillAccounts(t, serverTrieDB, common.Hash{}, numAccountsSmall, nil) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, GetLeafsIntercept: func(_ message.LeafsRequest, _ message.LeafsResponse) (message.LeafsResponse, error) { return message.LeafsResponse{}, clientErr @@ -190,10 +189,10 @@ func TestSimpleSyncCases(t *testing.T) { }, "failed to fetch code": { prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root := fillAccountsWithStorage(t, serverDB, serverTrieDB, common.Hash{}, numAccountsSmall) - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, GetCodeIntercept: func(_ []common.Hash, _ [][]byte) ([][]byte, error) { return nil, clientErr @@ -210,7 +209,7 @@ func TestSimpleSyncCases(t *testing.T) { } func TestCancelSync(t *testing.T) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) // Create trie with 2000 accounts (more than one leaf request) root := fillAccountsWithStorage(t, serverDB, serverTrieDB, common.Hash{}, 2000) @@ -219,7 +218,7 @@ func TestCancelSync(t *testing.T) { testSync(t, syncTest{ ctx: ctx, prepareForTest: func(t *testing.T) (ethdb.Database, ethdb.Database, *trie.Database, common.Hash) { - return memorydb.New(), serverDB, serverTrieDB, root + return rawdb.NewMemoryDatabase(), serverDB, serverTrieDB, root }, expectedError: context.Canceled, GetLeafsIntercept: func(_ message.LeafsRequest, lr message.LeafsResponse) (message.LeafsResponse, error) { @@ -251,10 +250,10 @@ func (i *interruptLeafsIntercept) getLeafsIntercept(request message.LeafsRequest } func TestResumeSyncAccountsTrieInterrupted(t *testing.T) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, common.Hash{}, 2000, 3) - clientDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() intercept := &interruptLeafsIntercept{ root: root, interruptAfter: 1, @@ -277,7 +276,7 @@ func TestResumeSyncAccountsTrieInterrupted(t *testing.T) { } func TestResumeSyncLargeStorageTrieInterrupted(t *testing.T) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) largeStorageRoot, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) @@ -288,7 +287,7 @@ func TestResumeSyncLargeStorageTrieInterrupted(t *testing.T) { } return account }) - clientDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() intercept := &interruptLeafsIntercept{ root: largeStorageRoot, interruptAfter: 1, @@ -309,7 +308,7 @@ func TestResumeSyncLargeStorageTrieInterrupted(t *testing.T) { } func TestResumeSyncToNewRootAfterLargeStorageTrieInterrupted(t *testing.T) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) largeStorageRoot1, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) @@ -327,7 +326,7 @@ func TestResumeSyncToNewRootAfterLargeStorageTrieInterrupted(t *testing.T) { } return account }) - clientDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() intercept := &interruptLeafsIntercept{ root: largeStorageRoot1, interruptAfter: 1, @@ -350,7 +349,7 @@ func TestResumeSyncToNewRootAfterLargeStorageTrieInterrupted(t *testing.T) { } func TestResumeSyncLargeStorageTrieWithConsecutiveDuplicatesInterrupted(t *testing.T) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) largeStorageRoot, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) @@ -361,7 +360,7 @@ func TestResumeSyncLargeStorageTrieWithConsecutiveDuplicatesInterrupted(t *testi } return account }) - clientDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() intercept := &interruptLeafsIntercept{ root: largeStorageRoot, interruptAfter: 1, @@ -382,7 +381,7 @@ func TestResumeSyncLargeStorageTrieWithConsecutiveDuplicatesInterrupted(t *testi } func TestResumeSyncLargeStorageTrieWithSpreadOutDuplicatesInterrupted(t *testing.T) { - serverDB := memorydb.New() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) largeStorageRoot, _, _ := trie.GenerateTrie(t, serverTrieDB, 2000, common.HashLength) @@ -392,7 +391,7 @@ func TestResumeSyncLargeStorageTrieWithSpreadOutDuplicatesInterrupted(t *testing } return account }) - clientDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() intercept := &interruptLeafsIntercept{ root: largeStorageRoot, interruptAfter: 1, @@ -487,8 +486,8 @@ func TestResyncNewRootAfterDeletes(t *testing.T) { func testSyncerSyncsToNewRoot(t *testing.T, deleteBetweenSyncs func(*testing.T, common.Hash, ethdb.Database)) { rand.Seed(1) - clientDB := memorydb.New() - serverDB := memorydb.New() + clientDB := rawdb.NewMemoryDatabase() + serverDB := rawdb.NewMemoryDatabase() serverTrieDB := trie.NewDatabase(serverDB) root1, _ := FillAccountsWithOverlappingStorage(t, serverTrieDB, common.Hash{}, 1000, 3) diff --git a/sync/statesync/test_sync.go b/sync/statesync/test_sync.go index 42646ed59f..2d43589851 100644 --- a/sync/statesync/test_sync.go +++ b/sync/statesync/test_sync.go @@ -12,10 +12,10 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/assert" ) diff --git a/sync/statesync/trie_queue.go b/sync/statesync/trie_queue.go index 15415c5606..b8de049003 100644 --- a/sync/statesync/trie_queue.go +++ b/sync/statesync/trie_queue.go @@ -5,8 +5,8 @@ package statesync import ( "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // trieQueue persists storage trie roots with their associated diff --git a/sync/statesync/trie_segments.go b/sync/statesync/trie_segments.go index b0abed4bd9..e8a70d4697 100644 --- a/sync/statesync/trie_segments.go +++ b/sync/statesync/trie_segments.go @@ -12,11 +12,11 @@ import ( "github.com/ava-labs/avalanchego/utils/wrappers" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" syncclient "github.com/ava-labs/subnet-evm/sync/client" "github.com/ava-labs/subnet-evm/trie" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/sync/statesync/trie_sync_tasks.go b/sync/statesync/trie_sync_tasks.go index 04bdd2e047..a734550e1e 100644 --- a/sync/statesync/trie_sync_tasks.go +++ b/sync/statesync/trie_sync_tasks.go @@ -8,10 +8,10 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/sync/syncutils" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/sync/syncutils/iterators.go b/sync/syncutils/iterators.go index 8f026fa317..c546cccd37 100644 --- a/sync/syncutils/iterators.go +++ b/sync/syncutils/iterators.go @@ -5,7 +5,7 @@ package syncutils import ( "github.com/ava-labs/subnet-evm/core/state/snapshot" - "github.com/ava-labs/subnet-evm/ethdb" + "github.com/ethereum/go-ethereum/ethdb" ) var ( diff --git a/tests/state_test_util.go b/tests/state_test_util.go index d3cdd8358a..9b44009172 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -41,13 +41,13 @@ import ( "github.com/ava-labs/subnet-evm/core/state/snapshot" "github.com/ava-labs/subnet-evm/core/types" "github.com/ava-labs/subnet-evm/core/vm" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/trie" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) diff --git a/trie/database_test.go b/trie/database_test.go index 9818580021..19394b55fe 100644 --- a/trie/database_test.go +++ b/trie/database_test.go @@ -28,8 +28,8 @@ package trie import ( "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" + "github.com/ethereum/go-ethereum/ethdb" ) // newTestDatabase initializes the trie database with specified scheme. diff --git a/trie/database_wrap.go b/trie/database_wrap.go index 004d33e1eb..dab26df692 100644 --- a/trie/database_wrap.go +++ b/trie/database_wrap.go @@ -21,11 +21,11 @@ import ( "runtime" "time" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie/triedb/hashdb" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ava-labs/subnet-evm/utils" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 0883d9e280..4e8f956d28 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -35,11 +35,11 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) func TestEmptyIterator(t *testing.T) { @@ -573,6 +573,10 @@ func (l *loggingDb) NewIterator(prefix []byte, start []byte) ethdb.Iterator { return l.backend.NewIterator(prefix, start) } +func (l *loggingDb) NewSnapshot() (ethdb.Snapshot, error) { + return l.backend.NewSnapshot() +} + func (l *loggingDb) Stat(property string) (string, error) { return l.backend.Stat(property) } diff --git a/trie/preimages.go b/trie/preimages.go index 0efa8be5f9..e8977e2a57 100644 --- a/trie/preimages.go +++ b/trie/preimages.go @@ -30,8 +30,8 @@ import ( "sync" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) const defaultPreimagesLimit = 4 * 1024 * 1024 // 4 MB diff --git a/trie/proof.go b/trie/proof.go index e92045a921..a90d76bb15 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -31,8 +31,8 @@ import ( "errors" "fmt" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) diff --git a/trie/proof_test.go b/trie/proof_test.go index 03a2d6a5a0..b62668810c 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -36,9 +36,9 @@ import ( "testing" "github.com/ava-labs/subnet-evm/core/rawdb" - "github.com/ava-labs/subnet-evm/ethdb/memorydb" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb/memorydb" ) // Prng is a pseudo random number generator seeded by strong randomness. diff --git a/trie/sync_test.go b/trie/sync_test.go index a20f5e3306..fcf4863a02 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -31,9 +31,9 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" ) // makeTestTrie create a sample test trie to test node-wise reconstruction. diff --git a/trie/trie_test.go b/trie/trie_test.go index 2c08690769..f986f8128a 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -40,11 +40,11 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/require" "golang.org/x/crypto/sha3" @@ -800,6 +800,7 @@ func (s *spongeDb) Get(key []byte) ([]byte, error) { return nil, error func (s *spongeDb) Delete(key []byte) error { panic("implement me") } func (s *spongeDb) NewBatch() ethdb.Batch { return &spongeBatch{s} } func (s *spongeDb) NewBatchWithSize(size int) ethdb.Batch { return &spongeBatch{s} } +func (s *spongeDb) NewSnapshot() (ethdb.Snapshot, error) { panic("implement me") } func (s *spongeDb) Stat(property string) (string, error) { panic("implement me") } func (s *spongeDb) Compact(start []byte, limit []byte) error { panic("implement me") } func (s *spongeDb) Close() error { return nil } diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index 6a88372adc..72c758519f 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -34,10 +34,10 @@ import ( "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/ethdb" "github.com/ava-labs/subnet-evm/metrics" "github.com/ava-labs/subnet-evm/trie/trienode" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" ) diff --git a/warp/backend.go b/warp/backend.go index 8c8a0aa330..acdeecbe9d 100644 --- a/warp/backend.go +++ b/warp/backend.go @@ -16,7 +16,7 @@ import ( "github.com/ava-labs/avalanchego/utils/crypto/bls" avalancheWarp "github.com/ava-labs/avalanchego/vms/platformvm/warp" "github.com/ava-labs/avalanchego/vms/platformvm/warp/payload" - "github.com/ava-labs/subnet-evm/ethdb" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) From e3958925ee040350f2ebe27d1520b39c6e6949e2 Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Fri, 5 Jan 2024 11:40:11 -0800 Subject: [PATCH 2/2] refactor: move AllowUnfinalizedQueries out of vm.Config --- accounts/abi/bind/backends/simulated.go | 4 ++-- core/vm/interpreter.go | 3 --- eth/api_backend.go | 19 ++++++++++++------- eth/backend.go | 2 +- eth/filters/api.go | 8 ++++---- eth/filters/filter.go | 2 +- eth/filters/filter_system.go | 3 +-- eth/filters/filter_system_test.go | 5 ++--- plugin/evm/vm_test.go | 8 ++++---- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index ceea225f26..103dfa8c5e 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -894,8 +894,8 @@ func (fb *filterBackend) SubscribeAcceptedTransactionEvent(ch chan<- core.NewTxs return fb.bc.SubscribeAcceptedTransactionEvent(ch) } -func (fb *filterBackend) GetVMConfig() *vm.Config { - return fb.bc.GetVMConfig() +func (fb *filterBackend) IsAllowUnfinalizedQueries() bool { + return false } func (fb *filterBackend) LastAcceptedBlock() *types.Block { diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 1752669d0d..ef119ba26a 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -40,9 +40,6 @@ type Config struct { NoBaseFee bool // Forces the EIP-1559 baseFee to 0 (needed for 0 price calls) EnablePreimageRecording bool // Enables recording of SHA3/keccak preimages ExtraEips []int // Additional EIPS that are to be enabled - - // AllowUnfinalizedQueries allow unfinalized queries - AllowUnfinalizedQueries bool } // ScopeContext contains the things that are per-call, such as stack and memory, diff --git a/eth/api_backend.go b/eth/api_backend.go index f87c256803..c6804fb000 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -58,6 +58,7 @@ type EthAPIBackend struct { extRPCEnabled bool allowUnprotectedTxs bool allowUnprotectedTxHashes map[common.Hash]struct{} // Invariant: read-only after creation. + allowUnfinalizedQueries bool eth *Ethereum gpo *gasprice.Oracle } @@ -67,8 +68,12 @@ func (b *EthAPIBackend) ChainConfig() *params.ChainConfig { return b.eth.blockchain.Config() } -func (b *EthAPIBackend) GetVMConfig() *vm.Config { - return b.eth.blockchain.GetVMConfig() +func (b *EthAPIBackend) IsAllowUnfinalizedQueries() bool { + return b.allowUnfinalizedQueries +} + +func (b *EthAPIBackend) SetAllowUnfinalizedQueries(allow bool) { + b.allowUnfinalizedQueries = allow } func (b *EthAPIBackend) CurrentBlock() *types.Header { @@ -90,7 +95,7 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb return acceptedBlock.Header(), nil } - if !b.GetVMConfig().AllowUnfinalizedQueries && acceptedBlock != nil { + if !b.IsAllowUnfinalizedQueries() && acceptedBlock != nil { if number.Int64() > acceptedBlock.Number().Int64() { return nil, ErrUnfinalizedData } @@ -118,7 +123,7 @@ func (b *EthAPIBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*ty } acceptedBlock := b.eth.LastAcceptedBlock() - if !b.GetVMConfig().AllowUnfinalizedQueries && acceptedBlock != nil { + if !b.IsAllowUnfinalizedQueries() && acceptedBlock != nil { if header.Number.Cmp(acceptedBlock.Number()) > 0 { return nil, ErrUnfinalizedData } @@ -154,7 +159,7 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe return acceptedBlock, nil } - if !b.GetVMConfig().AllowUnfinalizedQueries && acceptedBlock != nil { + if !b.IsAllowUnfinalizedQueries() && acceptedBlock != nil { if number.Int64() > acceptedBlock.Number().Int64() { return nil, ErrUnfinalizedData } @@ -179,7 +184,7 @@ func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*typ } acceptedBlock := b.eth.LastAcceptedBlock() - if !b.GetVMConfig().AllowUnfinalizedQueries && acceptedBlock != nil { + if !b.IsAllowUnfinalizedQueries() && acceptedBlock != nil { if number.Cmp(acceptedBlock.Number()) > 0 { return nil, ErrUnfinalizedData } @@ -350,7 +355,7 @@ func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) // expectations with clients (expect an empty response when a transaction // does not exist). acceptedBlock := b.eth.LastAcceptedBlock() - if !b.GetVMConfig().AllowUnfinalizedQueries && acceptedBlock != nil && tx != nil { + if !b.IsAllowUnfinalizedQueries() && acceptedBlock != nil && tx != nil { if blockNumber > acceptedBlock.NumberU64() { return nil, common.Hash{}, 0, 0, nil } diff --git a/eth/backend.go b/eth/backend.go index d2d8e255a5..77606b6589 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -181,7 +181,6 @@ func New( var ( vmConfig = vm.Config{ EnablePreimageRecording: config.EnablePreimageRecording, - AllowUnfinalizedQueries: config.AllowUnfinalizedQueries, } cacheConfig = &core.CacheConfig{ TrieCleanLimit: config.TrieCleanCache, @@ -241,6 +240,7 @@ func New( extRPCEnabled: stack.Config().ExtRPCEnabled(), allowUnprotectedTxs: config.AllowUnprotectedTxs, allowUnprotectedTxHashes: allowUnprotectedTxHashes, + allowUnfinalizedQueries: config.AllowUnfinalizedQueries, eth: eth, } if config.AllowUnprotectedTxs { diff --git a/eth/filters/api.go b/eth/filters/api.go index 911439935d..a4412cd975 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -245,7 +245,7 @@ func (api *FilterAPI) NewBlockFilter() rpc.ID { headerSub *Subscription ) - if api.sys.backend.GetVMConfig().AllowUnfinalizedQueries { + if api.sys.backend.IsAllowUnfinalizedQueries() { headerSub = api.events.SubscribeNewHeads(headers) } else { headerSub = api.events.SubscribeAcceptedHeads(headers) @@ -291,7 +291,7 @@ func (api *FilterAPI) NewHeads(ctx context.Context) (*rpc.Subscription, error) { headersSub event.Subscription ) - if api.sys.backend.GetVMConfig().AllowUnfinalizedQueries { + if api.sys.backend.IsAllowUnfinalizedQueries() { headersSub = api.events.SubscribeNewHeads(headers) } else { headersSub = api.events.SubscribeAcceptedHeads(headers) @@ -328,7 +328,7 @@ func (api *FilterAPI) Logs(ctx context.Context, crit FilterCriteria) (*rpc.Subsc err error ) - if api.sys.backend.GetVMConfig().AllowUnfinalizedQueries { + if api.sys.backend.IsAllowUnfinalizedQueries() { logsSub, err = api.events.SubscribeLogs(interfaces.FilterQuery(crit), matchedLogs) if err != nil { return nil, err @@ -383,7 +383,7 @@ func (api *FilterAPI) NewFilter(crit FilterCriteria) (rpc.ID, error) { err error ) - if api.sys.backend.GetVMConfig().AllowUnfinalizedQueries { + if api.sys.backend.IsAllowUnfinalizedQueries() { logsSub, err = api.events.SubscribeLogs(interfaces.FilterQuery(crit), logs) if err != nil { return rpc.ID(""), err diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 0bcd0a7b5f..63a3fd3d04 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -54,7 +54,7 @@ type Filter struct { // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) (*Filter, error) { - allowUnfinalizedQueries := sys.backend.GetVMConfig().AllowUnfinalizedQueries + allowUnfinalizedQueries := sys.backend.IsAllowUnfinalizedQueries() acceptedBlock := sys.backend.LastAcceptedBlock() // Flatten the address and topic filter clauses into a single bloombits filter diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 865766d29d..4ebd672d47 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -38,7 +38,6 @@ import ( "github.com/ava-labs/subnet-evm/core" "github.com/ava-labs/subnet-evm/core/bloombits" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/interfaces" "github.com/ava-labs/subnet-evm/params" "github.com/ava-labs/subnet-evm/rpc" @@ -85,7 +84,7 @@ type Backend interface { ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) // Added to the backend interface to support limiting of logs requests - GetVMConfig() *vm.Config + IsAllowUnfinalizedQueries() bool LastAcceptedBlock() *types.Block GetMaxBlocksPerRequest() int64 } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 6f49eabb27..962cfc6967 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -42,7 +42,6 @@ import ( "github.com/ava-labs/subnet-evm/core/bloombits" "github.com/ava-labs/subnet-evm/core/rawdb" "github.com/ava-labs/subnet-evm/core/types" - "github.com/ava-labs/subnet-evm/core/vm" "github.com/ava-labs/subnet-evm/interfaces" "github.com/ava-labs/subnet-evm/internal/ethapi" "github.com/ava-labs/subnet-evm/params" @@ -78,8 +77,8 @@ func (b *testBackend) ChainDb() ethdb.Database { return b.db } -func (b *testBackend) GetVMConfig() *vm.Config { - return &vm.Config{AllowUnfinalizedQueries: true} +func (b *testBackend) IsAllowUnfinalizedQueries() bool { + return true } func (b *testBackend) GetMaxBlocksPerRequest() int64 { diff --git a/plugin/evm/vm_test.go b/plugin/evm/vm_test.go index 0f2c4571c0..c5620c7f92 100644 --- a/plugin/evm/vm_test.go +++ b/plugin/evm/vm_test.go @@ -1070,7 +1070,7 @@ func TestNonCanonicalAccept(t *testing.T) { t.Fatal(err) } - vm1.blockChain.GetVMConfig().AllowUnfinalizedQueries = true + vm1.eth.APIBackend.SetAllowUnfinalizedQueries(true) blkBHeight := vm1BlkB.Height() blkBHash := vm1BlkB.(*chain.BlockWrapper).Block.(*Block).ethBlock.Hash() @@ -1241,7 +1241,7 @@ func TestStickyPreference(t *testing.T) { t.Fatal(err) } - vm1.blockChain.GetVMConfig().AllowUnfinalizedQueries = true + vm1.eth.APIBackend.SetAllowUnfinalizedQueries(true) blkBHeight := vm1BlkB.Height() blkBHash := vm1BlkB.(*chain.BlockWrapper).Block.(*Block).ethBlock.Hash() @@ -1940,7 +1940,7 @@ func TestLastAcceptedBlockNumberAllow(t *testing.T) { blkHeight := blk.Height() blkHash := blk.(*chain.BlockWrapper).Block.(*Block).ethBlock.Hash() - vm.blockChain.GetVMConfig().AllowUnfinalizedQueries = true + vm.eth.APIBackend.SetAllowUnfinalizedQueries(true) ctx := context.Background() b, err := vm.eth.APIBackend.BlockByNumber(ctx, rpc.BlockNumber(blkHeight)) @@ -1951,7 +1951,7 @@ func TestLastAcceptedBlockNumberAllow(t *testing.T) { t.Fatalf("expected block at %d to have hash %s but got %s", blkHeight, blkHash.Hex(), b.Hash().Hex()) } - vm.blockChain.GetVMConfig().AllowUnfinalizedQueries = false + vm.eth.APIBackend.SetAllowUnfinalizedQueries(false) _, err = vm.eth.APIBackend.BlockByNumber(ctx, rpc.BlockNumber(blkHeight)) if !errors.Is(err, eth.ErrUnfinalizedData) {