Merge branch 'master' into merge-v1.14.5

Author: amsanghi

arbitrum/apibackend.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
+	"net/url"
 	"strconv"
 	"strings"
 	"time"
@@ -51,6 +52,22 @@ type APIBackend struct {
 	sync           SyncProgressBackend
 }
 
+type errorFilteredFallbackClient struct {
+	impl types.FallbackClient
+	url  string
+}
+
+func (c *errorFilteredFallbackClient) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
+	err := c.impl.CallContext(ctx, result, method, args...)
+	if err != nil && strings.Contains(err.Error(), c.url) {
+		// avoids leaking the URL in the error message.
+		// URL can contain sensitive information such as API keys.
+		log.Warn("fallback client error", "error", err)
+		return errors.New("Failed to call fallback API")
+	}
+	return err
+}
+
 type timeoutFallbackClient struct {
 	impl    types.FallbackClient
 	timeout time.Duration
@@ -77,12 +94,22 @@ func CreateFallbackClient(fallbackClientUrl string, fallbackClientTimeout time.D
 		types.SetFallbackError(strings.Join(fields, ":"), int(errNumber))
 		return nil, nil
 	}
+
 	var fallbackClient types.FallbackClient
 	var err error
 	fallbackClient, err = rpc.Dial(fallbackClientUrl)
 	if err != nil {
 		return nil, fmt.Errorf("failed creating fallback connection: %w", err)
 	}
+	url, err := url.Parse(fallbackClientUrl)
+	if err != nil {
+		return nil, fmt.Errorf("failed parsing fallback URL: %w", err)
+	}
+	fallbackClient = &errorFilteredFallbackClient{
+		impl: fallbackClient,
+		url:  url.String(),
+	}
+
 	if fallbackClientTimeout != 0 {
 		fallbackClient = &timeoutFallbackClient{
 			impl:    fallbackClient,
@@ -96,6 +123,7 @@ type SyncProgressBackend interface {
 	SyncProgressMap() map[string]interface{}
 	SafeBlockNumber(ctx context.Context) (uint64, error)
 	FinalizedBlockNumber(ctx context.Context) (uint64, error)
+	BlockMetadataByNumber(blockNum uint64) (common.BlockMetadata, error)
 }
 
 func createRegisterAPIBackend(backend *Backend, filterConfig filters.Config, fallbackClientUrl string, fallbackClientTimeout time.Duration) (*filters.FilterSystem, error) {
@@ -460,6 +488,10 @@ func (a *APIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.
 	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
+func (a *APIBackend) BlockMetadataByNumber(blockNum uint64) (common.BlockMetadata, error) {
+	return a.sync.BlockMetadataByNumber(blockNum)
+}
+
 func StateAndHeaderFromHeader(ctx context.Context, chainDb ethdb.Database, bc *core.BlockChain, maxRecreateStateDepth int64, header *types.Header, err error) (*state.StateDB, *types.Header, error) {
 	if err != nil {
 		return nil, header, err

arbitrum/recordingdb.go

@@ -194,7 +194,7 @@ func (r *RecordingDatabase) StateFor(header *types.Header) (*state.StateDB, erro
 	r.mutex.Lock()
 	defer r.mutex.Unlock()
 
-	sdb, err := state.NewDeterministic(header.Root, r.db)
+	sdb, err := state.NewRecording(header.Root, r.db)
 	if err == nil {
 		r.referenceRootLockHeld(header.Root)
 	}
@@ -267,7 +267,7 @@ func (r *RecordingDatabase) PrepareRecording(ctx context.Context, lastBlockHeade
 	if lastBlockHeader != nil {
 		prevRoot = lastBlockHeader.Root
 	}
-	recordingStateDb, err := state.NewDeterministic(prevRoot, recordingStateDatabase)
+	recordingStateDb, err := state.NewRecording(prevRoot, recordingStateDatabase)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("failed to create recordingStateDb: %w", err)
 	}

common/types.go

@@ -486,3 +486,24 @@ func (b PrettyBytes) TerminalString() string {
 	}
 	return fmt.Sprintf("%#x...%x (%dB)", b[:3], b[len(b)-3:], len(b))
 }
+
+type BlockMetadata []byte
+
+// IsTxTimeboosted given a tx's index in the block returns whether the tx was timeboosted
+// or not. The first byte of blockMetadata byte array is reserved to indicate the version,
+// starting from the second byte, (N)th bit would represent if (N)th tx is timeboosted or not, 1 means yes and 0 means no
+// blockMetadata[index / 8 + 1] & (1 << (index % 8)) != 0; where index = (N - 1), implies whether (N)th tx in a block is timeboosted
+// note that number of txs in a block will always lag behind (len(blockMetadata) - 1) * 8 but it wont lag more than a value of 7
+func (b BlockMetadata) IsTxTimeboosted(txIndex int) (bool, error) {
+	if len(b) == 0 {
+		return false, errors.New("blockMetadata is not set")
+	}
+	if txIndex < 0 {
+		return false, fmt.Errorf("invalid transaction index- %d, should be positive", txIndex)
+	}
+	maxTxCount := (len(b) - 1) * 8
+	if txIndex >= maxTxCount {
+		return false, nil
+	}
+	return b[1+(txIndex/8)]&(1<<(txIndex%8)) != 0, nil
+}

core/blockchain.go

@@ -78,6 +78,9 @@ var (
 
 	triedbCommitTimer = metrics.NewRegisteredResettingTimer("chain/triedb/commits", nil)
 
+	triedbSizeGauge   = metrics.NewRegisteredGauge("chain/triedb/size", nil)
+	triedbGCProcGauge = metrics.NewRegisteredGauge("chain/triedb/gcproc", nil)
+
 	blockInsertTimer     = metrics.NewRegisteredResettingTimer("chain/inserts", nil)
 	blockValidationTimer = metrics.NewRegisteredResettingTimer("chain/validation", nil)
 	blockExecutionTimer  = metrics.NewRegisteredResettingTimer("chain/execution", nil)
@@ -1654,6 +1657,11 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 			bc.triedb.Dereference(prevEntry.Root)
 		}
 	}
+
+	_, dirtyNodesBufferedSize, _ := bc.triedb.Size()
+	triedbSizeGauge.Update(int64(dirtyNodesBufferedSize))
+	triedbGCProcGauge.Update(int64(bc.gcproc))
+
 	return nil
 }
 

core/blockchain_arbitrum.go

@@ -21,12 +21,50 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
+func (bc *BlockChain) FlushTrieDB(capLimit common.StorageSize) error {
+	if bc.triedb.Scheme() == rawdb.PathScheme {
+		return nil
+	}
+
+	if !bc.chainmu.TryLock() {
+		return errChainStopped
+	}
+	defer bc.chainmu.Unlock()
+
+	if !bc.triegc.Empty() {
+		_, triegcBlockNumber := bc.triegc.Peek()
+		blockNumber := uint64(-triegcBlockNumber)
+
+		header := bc.GetHeaderByNumber(blockNumber)
+		if header == nil {
+			log.Warn("Reorg in progress, trie commit postponed")
+		} else {
+			err := bc.triedb.Commit(header.Root, true)
+			if err != nil {
+				return err
+			}
+
+			bc.gcproc = 0
+			bc.lastWrite = blockNumber
+		}
+	}
+
+	err := bc.triedb.Cap(capLimit)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // WriteBlockAndSetHeadWithTime also counts processTime, which will cause intermittent TrieDirty cache writes
 func (bc *BlockChain) WriteBlockAndSetHeadWithTime(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool, processTime time.Duration) (status WriteStatus, err error) {
 	if !bc.chainmu.TryLock() {

core/state/snapshot/disklayer.go

@@ -76,30 +76,28 @@ func (dl *diskLayer) Stale() bool {
 
 // Account directly retrieves the account associated with a particular hash in
 // the snapshot slim data format.
-func (dl *diskLayer) Account(hash common.Hash) (*types.SlimAccount, error) {
-	data, err := dl.AccountRLP(hash)
+func (dl *diskLayer) account(hash common.Hash, evenIfStale bool) (*types.SlimAccount, error) {
+	data, err := dl.accountRLP(hash, evenIfStale)
 	if err != nil {
 		return nil, err
 	}
 	if len(data) == 0 { // can be both nil and []byte{}
 		return nil, nil
 	}
 	account := new(types.SlimAccount)
-	if err := rlp.DecodeBytes(data, account); err != nil {
-		panic(err)
-	}
-	return account, nil
+	err = rlp.DecodeBytes(data, account)
+	return account, err
 }
 
 // AccountRLP directly retrieves the account RLP associated with a particular
 // hash in the snapshot slim data format.
-func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) {
+func (dl *diskLayer) accountRLP(hash common.Hash, evenIfStale bool) ([]byte, error) {
 	dl.lock.RLock()
 	defer dl.lock.RUnlock()
 
 	// If the layer was flattened into, consider it invalid (any live reference to
 	// the original should be marked as unusable).
-	if dl.stale {
+	if dl.stale && !evenIfStale {
 		return nil, ErrSnapshotStale
 	}
 	// If the layer is being generated, ensure the requested hash has already been
@@ -121,7 +119,9 @@ func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	dl.cache.Set(hash[:], blob)
+	if !dl.stale {
+		dl.cache.Set(hash[:], blob)
+	}
 
 	snapshotCleanAccountMissMeter.Mark(1)
 	if n := len(blob); n > 0 {
@@ -132,6 +132,14 @@ func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) {
 	return blob, nil
 }
 
+func (dl *diskLayer) Account(hash common.Hash) (*types.SlimAccount, error) {
+	return dl.account(hash, false)
+}
+
+func (dl *diskLayer) AccountRLP(hash common.Hash) ([]byte, error) {
+	return dl.accountRLP(hash, false)
+}
+
 // Storage directly retrieves the storage data associated with a particular hash,
 // within a particular account.
 func (dl *diskLayer) Storage(accountHash, storageHash common.Hash) ([]byte, error) {

core/state/snapshot/snapshot.go

@@ -564,10 +564,23 @@ func diffToDisk(bottom *diffLayer) *diskLayer {
 		if base.genMarker != nil && bytes.Compare(hash[:], base.genMarker) > 0 {
 			continue
 		}
-		// Remove all storage slots
+
+		oldAccount, err := base.account(hash, true)
+
+		// Delete the account snapshot
 		rawdb.DeleteAccountSnapshot(batch, hash)
 		base.cache.Set(hash[:], nil)
 
+		// Try to check if there's any associated storage to delete from the snapshot
+		if err != nil {
+			log.Warn("Failed to get destructed account from snapshot", "err", err)
+			// Fall through to deleting storage in case this account has any
+		} else if oldAccount == nil || oldAccount.Root == nil {
+			// There's no storage associated with this account to delete
+			continue
+		}
+
+		// Remove all storage slots
 		it := rawdb.IterateStorageSnapshots(base.diskdb, hash)
 		for it.Next() {
 			key := it.Key()

core/state/statedb.go

@@ -170,6 +170,7 @@ type StateDB struct {
 	StorageDeleted atomic.Int64
 
 	deterministic bool
+	recording     bool
 }
 
 // New creates a new state from a given trie.
@@ -949,6 +950,21 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash {
 		usedAddrs    [][]byte
 		deletedAddrs []common.Address
 	)
+	// When recording, it's important to handle deletions before other mutations
+	// to ensure that access the most state possible in the database. If, instead,
+	// the updates were applied before the deletions, it might be possible to
+	// record an update for an address that would then be subsequently deleted.
+	if s.recording {
+		for addr, op := range s.mutations {
+			if op.applied {
+				continue
+			}
+			if op.isDelete() {
+				op.applied = true
+				s.deleteStateObject(addr)
+			}
+		}
+	}
 	for addr, op := range s.mutations {
 		if op.applied {
 			continue

core/state/statedb_arbitrum.go

@@ -160,6 +160,16 @@ func NewDeterministic(root common.Hash, db Database) (*StateDB, error) {
 	return sdb, nil
 }
 
+func NewRecording(root common.Hash, db Database) (*StateDB, error) {
+	sdb, err := New(root, db, nil)
+	if err != nil {
+		return nil, err
+	}
+	sdb.deterministic = true
+	sdb.recording = true
+	return sdb, nil
+}
+
 func (s *StateDB) Deterministic() bool {
 	return s.deterministic
 }

eth/api_backend.go

@@ -182,6 +182,10 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r
 	return nil, errors.New("invalid arguments; neither block nor hash specified")
 }
 
+func (b *EthAPIBackend) BlockMetadataByNumber(blockNum uint64) (common.BlockMetadata, error) {
+	return nil, nil
+}
+
 func (b *EthAPIBackend) Pending() (*types.Block, types.Receipts, *state.StateDB) {
 	return b.eth.miner.Pending()
 }

eth/tracers/native/call_flat.go

@@ -195,6 +195,9 @@ func (t *flatCallTracer) OnExit(depth int, output []byte, gasUsed uint64, err er
 	if t.config.IncludePrecompiles {
 		return
 	}
+	if len(t.tracer.callstack[len(t.tracer.callstack)-1].Calls) == 0 {
+		return
+	}
 	var (
 		// call has been nested in parent
 		parent = t.tracer.callstack[len(t.tracer.callstack)-1]

internal/ethapi/api.go

@@ -1308,7 +1308,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr
 	call := args.ToMessage(header.BaseFee, gasCap, header, state, core.MessageGasEstimationMode)
 
 	// Arbitrum: raise the gas cap to ignore L1 costs so that it's compute-only
-	{
+	if gasCap > 0 {
 		postingGas, err := core.RPCPostingGasHook(call, header, state)
 		if err != nil {
 			return 0, err
@@ -1988,6 +1988,17 @@ func marshalReceipt(ctx context.Context, receipt *types.Receipt, blockHash commo
 				fields["l1BlockNumber"] = hexutil.Uint64(arbTx.L1BlockNumber)
 			}
 		}
+
+		blockMetadata, err := backend.BlockMetadataByNumber(blockNumber)
+		if err != nil {
+			return nil, err
+		}
+		if blockMetadata != nil {
+			fields["timeboosted"], err = blockMetadata.IsTxTimeboosted(txIndex)
+			if err != nil {
+				log.Error("Error checking if a tx was timeboosted", "txIndex", txIndex, "txHash", tx.Hash(), "err", err)
+			}
+		}
 	}
 	return fields, nil
 }

internal/ethapi/api_test.go

@@ -527,6 +527,9 @@ func (b testBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.
 	}
 	panic("unknown type rpc.BlockNumberOrHash")
 }
+func (b testBackend) BlockMetadataByNumber(blockNum uint64) (common.BlockMetadata, error) {
+	return nil, nil
+}
 func (b testBackend) GetBody(ctx context.Context, hash common.Hash, number rpc.BlockNumber) (*types.Body, error) {
 	return b.chain.GetBlock(hash, uint64(number.Int64())).Body(), nil
 }

internal/ethapi/backend.go

@@ -67,6 +67,7 @@ type Backend interface {
 	BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error)
 	BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error)
 	BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error)
+	BlockMetadataByNumber(blockNum uint64) (common.BlockMetadata, error)
 	StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error)
 	StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error)
 	Pending() (*types.Block, types.Receipts, *state.StateDB)