diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 3e52933a900d..76bfd22a2367 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -123,6 +123,11 @@ type BlobAndProofV1 struct { Proof hexutil.Bytes `json:"proof"` } +type BlobAndProofV2 struct { + Blob hexutil.Bytes `json:"blob"` + CellProofs []hexutil.Bytes `json:"proofs"` +} + // JSON type overrides for ExecutionPayloadEnvelope. type executionPayloadEnvelopeMarshaling struct { BlockValue *hexutil.Big @@ -331,7 +336,9 @@ func BlockToExecutableData(block *types.Block, fees *big.Int, sidecars []*types. for j := range sidecar.Blobs { bundle.Blobs = append(bundle.Blobs, hexutil.Bytes(sidecar.Blobs[j][:])) bundle.Commitments = append(bundle.Commitments, hexutil.Bytes(sidecar.Commitments[j][:])) - bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(sidecar.Proofs[j][:])) + } + for _, proof := range sidecar.Proofs { + bundle.Proofs = append(bundle.Proofs, hexutil.Bytes(proof[:])) } } diff --git a/beacon/engine/types_test.go b/beacon/engine/types_test.go new file mode 100644 index 000000000000..ff376dfd2e7d --- /dev/null +++ b/beacon/engine/types_test.go @@ -0,0 +1,56 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package engine + +import ( + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" +) + +func TestBlobs(t *testing.T) { + var ( + emptyBlob = new(kzg4844.Blob) + emptyBlobCommit, _ = kzg4844.BlobToCommitment(emptyBlob) + emptyBlobProof, _ = kzg4844.ComputeBlobProof(emptyBlob, emptyBlobCommit) + emptyCellProof, _ = kzg4844.ComputeCellProofs(emptyBlob) + ) + header := types.Header{} + block := types.NewBlock(&header, &types.Body{}, nil, nil) + + sidecarWithoutCellProofs := &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{*emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: []kzg4844.Proof{emptyBlobProof}, + } + env := BlockToExecutableData(block, common.Big0, []*types.BlobTxSidecar{sidecarWithoutCellProofs}, nil) + if len(env.BlobsBundle.Proofs) != 1 { + t.Fatalf("Expect 1 proof in blobs bundle, got %v", len(env.BlobsBundle.Proofs)) + } + + sidecarWithCellProofs := &types.BlobTxSidecar{ + Blobs: []kzg4844.Blob{*emptyBlob}, + Commitments: []kzg4844.Commitment{emptyBlobCommit}, + Proofs: emptyCellProof, + } + env = BlockToExecutableData(block, common.Big0, []*types.BlobTxSidecar{sidecarWithCellProofs}, nil) + if len(env.BlobsBundle.Proofs) != 128 { + t.Fatalf("Expect 128 proofs in blobs bundle, got %v", len(env.BlobsBundle.Proofs)) + } +} diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 2602d821040a..99f723078606 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -1302,27 +1301,13 @@ func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { } } -// GetBlobs returns a number of blobs are proofs for the given versioned hashes. +// GetBlobs returns a number of blobs and proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. -func (p *BlobPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) { - // Create a map of the blob hash to indices for faster fills - var ( - blobs = make([]*kzg4844.Blob, len(vhashes)) - proofs = make([]*kzg4844.Proof, len(vhashes)) - ) - index := make(map[common.Hash]int) - for i, vhash := range vhashes { - index[vhash] = i - } - // Iterate over the blob hashes, pulling transactions that fill it. Take care - // to also fill anything else the transaction might include (probably will). - for i, vhash := range vhashes { - // If already filled by a previous fetch, skip - if blobs[i] != nil { - continue - } - // Unfilled, retrieve the datastore item (in a short lock) +func (p *BlobPool) GetBlobs(vhashes []common.Hash) []*types.BlobTxSidecar { + sidecars := make([]*types.BlobTxSidecar, len(vhashes)) + for idx, vhash := range vhashes { + // Retrieve the datastore item (in a short lock) p.lock.RLock() id, exists := p.lookup.storeidOfBlob(vhash) if !exists { @@ -1342,16 +1327,24 @@ func (p *BlobPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844. log.Error("Blobs corrupted for traced transaction", "id", id, "err", err) continue } - // Fill anything requested, not just the current versioned hash - sidecar := item.BlobTxSidecar() - for j, blobhash := range item.BlobHashes() { - if idx, ok := index[blobhash]; ok { - blobs[idx] = &sidecar.Blobs[j] - proofs[idx] = &sidecar.Proofs[j] - } + sidecars[idx] = item.BlobTxSidecar() + } + return sidecars +} + +// AvailableBlobs returns the number of blobs that are available in the subpool. +func (p *BlobPool) AvailableBlobs(vhashes []common.Hash) int { + available := 0 + for _, vhash := range vhashes { + // Retrieve the datastore item (in a short lock) + p.lock.RLock() + _, exists := p.lookup.storeidOfBlob(vhash) + p.lock.RUnlock() + if exists { + available++ } } - return blobs, proofs + return available } // Add inserts a set of blob transactions into the pool if they pass validation (both diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index 12b64bf67488..f8d3f38412c5 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -417,8 +417,23 @@ func verifyBlobRetrievals(t *testing.T, pool *BlobPool) { for i := range testBlobVHashes { copy(hashes[i][:], testBlobVHashes[i][:]) } - blobs, proofs := pool.GetBlobs(hashes) - + sidecars := pool.GetBlobs(hashes) + var blobs []*kzg4844.Blob + var proofs []*kzg4844.Proof + for idx, sidecar := range sidecars { + if sidecar == nil { + blobs = append(blobs, nil) + proofs = append(proofs, nil) + continue + } + blobHashes := sidecar.BlobHashes() + for i, hash := range blobHashes { + if hash == hashes[idx] { + blobs = append(blobs, &sidecar.Blobs[i]) + proofs = append(proofs, &sidecar.Proofs[i]) + } + } + } // Cross validate what we received vs what we wanted if len(blobs) != len(hashes) || len(proofs) != len(hashes) { t.Errorf("retrieved blobs/proofs size mismatch: have %d/%d, want %d", len(blobs), len(proofs), len(hashes)) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 0223b456e659..1de714742660 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -35,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -1063,12 +1062,6 @@ func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { } } -// GetBlobs is not supported by the legacy transaction pool, it is just here to -// implement the txpool.SubPool interface. -func (pool *LegacyPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) { - return nil, nil -} - // Has returns an indicator whether txpool has a transaction cached with the // given hash. func (pool *LegacyPool) Has(hash common.Hash) bool { diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 8cfc14f16407..ed4a42940d5e 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -23,7 +23,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/holiman/uint256" ) @@ -133,11 +132,6 @@ type SubPool interface { // given transaction hash. GetMetadata(hash common.Hash) *TxMetadata - // GetBlobs returns a number of blobs are proofs for the given versioned hashes. - // This is a utility method for the engine API, enabling consensus clients to - // retrieve blobs from the pools directly instead of the network. - GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) - // ValidateTxBasics checks whether a transaction is valid according to the consensus // rules, but does not check state-dependent validation such as sufficient balance. // This check is meant as a static check which can be performed without holding the diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index cc8f74c1b8e5..b5470cd7fc0c 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -26,7 +26,6 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" @@ -308,22 +307,6 @@ func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata { return nil } -// GetBlobs returns a number of blobs are proofs for the given versioned hashes. -// This is a utility method for the engine API, enabling consensus clients to -// retrieve blobs from the pools directly instead of the network. -func (p *TxPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) { - for _, subpool := range p.subpools { - // It's an ugly to assume that only one pool will be capable of returning - // anything meaningful for this call, but anythingh else requires merging - // partial responses and that's too annoying to do until we get a second - // blobpool (probably never). - if blobs, proofs := subpool.GetBlobs(vhashes); blobs != nil { - return blobs, proofs - } - } - return nil, nil -} - // Add enqueues a batch of transactions into the pool if they are valid. Due // to the large transaction churn, add may postpone fully integrating the tx // to a later point to batch multiple ones together. diff --git a/core/txpool/validation.go b/core/txpool/validation.go index 720d0d3b722c..fef1a99f7ba1 100644 --- a/core/txpool/validation.go +++ b/core/txpool/validation.go @@ -138,28 +138,7 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return fmt.Errorf("%w: gas tip cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.GasTipCap(), opts.MinTip) } if tx.Type() == types.BlobTxType { - // Ensure the blob fee cap satisfies the minimum blob gas price - if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { - return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) - } - sidecar := tx.BlobTxSidecar() - if sidecar == nil { - return errors.New("missing sidecar in blob transaction") - } - // Ensure the number of items in the blob transaction and various side - // data match up before doing any expensive validations - hashes := tx.BlobHashes() - if len(hashes) == 0 { - return errors.New("blobless blob transaction") - } - maxBlobs := eip4844.MaxBlobsPerBlock(opts.Config, head.Time) - if len(hashes) > maxBlobs { - return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), maxBlobs) - } - // Ensure commitments, proofs and hashes are valid - if err := validateBlobSidecar(hashes, sidecar); err != nil { - return err - } + return validateBlobTx(tx, head, opts) } if tx.Type() == types.SetCodeTxType { if len(tx.SetCodeAuthorizations()) == 0 { @@ -169,18 +148,46 @@ func ValidateTransaction(tx *types.Transaction, head *types.Header, signer types return nil } -func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) error { +// validateBlobTx implements the blob-transaction specific validations. +func validateBlobTx(tx *types.Transaction, head *types.Header, opts *ValidationOptions) error { + sidecar := tx.BlobTxSidecar() + if sidecar == nil { + return errors.New("missing sidecar in blob transaction") + } + // Ensure the blob fee cap satisfies the minimum blob gas price + if tx.BlobGasFeeCapIntCmp(blobTxMinBlobGasPrice) < 0 { + return fmt.Errorf("%w: blob fee cap %v, minimum needed %v", ErrTxGasPriceTooLow, tx.BlobGasFeeCap(), blobTxMinBlobGasPrice) + } + // Ensure the number of items in the blob transaction and various side + // data match up before doing any expensive validations + hashes := tx.BlobHashes() + if len(hashes) == 0 { + return errors.New("blobless blob transaction") + } + maxBlobs := eip4844.MaxBlobsPerBlock(opts.Config, head.Time) + if len(hashes) > maxBlobs { + return fmt.Errorf("too many blobs in transaction: have %d, permitted %d", len(hashes), maxBlobs) + } if len(sidecar.Blobs) != len(hashes) { return fmt.Errorf("invalid number of %d blobs compared to %d blob hashes", len(sidecar.Blobs), len(hashes)) } - if len(sidecar.Proofs) != len(hashes) { - return fmt.Errorf("invalid number of %d blob proofs compared to %d blob hashes", len(sidecar.Proofs), len(hashes)) - } if err := sidecar.ValidateBlobCommitmentHashes(hashes); err != nil { return err } - // Blob commitments match with the hashes in the transaction, verify the - // blobs themselves via KZG + // Fork-specific sidecar checks, including proof verification. + if opts.Config.IsOsaka(head.Number, head.Time) { + return validateBlobSidecarOsaka(sidecar, hashes) + } + return validateBlobSidecarLegacy(sidecar, hashes) +} + +func validateBlobSidecarLegacy(sidecar *types.BlobTxSidecar, hashes []common.Hash) error { + if sidecar.Version != 0 { + return fmt.Errorf("invalid sidecar version pre-osaka: %v", sidecar.Version) + } + if len(sidecar.Proofs) != len(hashes) { + return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)) + } for i := range sidecar.Blobs { if err := kzg4844.VerifyBlobProof(&sidecar.Blobs[i], sidecar.Commitments[i], sidecar.Proofs[i]); err != nil { return fmt.Errorf("invalid blob %d: %v", i, err) @@ -189,6 +196,16 @@ func validateBlobSidecar(hashes []common.Hash, sidecar *types.BlobTxSidecar) err return nil } +func validateBlobSidecarOsaka(sidecar *types.BlobTxSidecar, hashes []common.Hash) error { + if sidecar.Version != 1 { + return fmt.Errorf("invalid sidecar version post-osaka: %v", sidecar.Version) + } + if len(sidecar.Proofs) != len(hashes)*kzg4844.CellProofsPerBlob { + return fmt.Errorf("invalid number of %d blob proofs expected %d", len(sidecar.Proofs), len(hashes)*kzg4844.CellProofsPerBlob) + } + return kzg4844.VerifyCellProofs(sidecar.Blobs, sidecar.Commitments, sidecar.Proofs) +} + // ValidationOptionsWithState define certain differences between stateful transaction // validation across the different pools without having to duplicate those checks. type ValidationOptionsWithState struct { diff --git a/core/types/tx_blob.go b/core/types/tx_blob.go index 9b1d53958fed..93a76a28b73d 100644 --- a/core/types/tx_blob.go +++ b/core/types/tx_blob.go @@ -19,9 +19,12 @@ package types import ( "bytes" "crypto/sha256" + "errors" "fmt" "math/big" + "slices" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/params" @@ -55,6 +58,7 @@ type BlobTx struct { // BlobTxSidecar contains the blobs of a blob transaction. type BlobTxSidecar struct { + Version byte // Version Blobs []kzg4844.Blob // Blobs needed by the blob pool Commitments []kzg4844.Commitment // Commitments needed by the blob pool Proofs []kzg4844.Proof // Proofs needed by the blob pool @@ -70,6 +74,20 @@ func (sc *BlobTxSidecar) BlobHashes() []common.Hash { return h } +// CellProofsAt returns the cell proofs for blob with index idx. +func (sc *BlobTxSidecar) CellProofsAt(idx int) []kzg4844.Proof { + var cellProofs []kzg4844.Proof + for i := range kzg4844.CellProofsPerBlob { + index := idx*kzg4844.CellProofsPerBlob + i + if index > len(sc.Proofs) { + return nil + } + proof := sc.Proofs[index] + cellProofs = append(cellProofs, proof) + } + return cellProofs +} + // encodedSize computes the RLP size of the sidecar elements. This does NOT return the // encoded size of the BlobTxSidecar, it's just a helper for tx.Size(). func (sc *BlobTxSidecar) encodedSize() uint64 { @@ -102,14 +120,55 @@ func (sc *BlobTxSidecar) ValidateBlobCommitmentHashes(hashes []common.Hash) erro return nil } -// blobTxWithBlobs is used for encoding of transactions when blobs are present. -type blobTxWithBlobs struct { +// blobTxWithBlobs represents blob tx with its corresponding sidecar. +// This is an interface because sidecars are versioned. +type blobTxWithBlobs interface { + tx() *BlobTx + assign(*BlobTxSidecar) error +} + +type blobTxWithBlobsV0 struct { + BlobTx *BlobTx + Blobs []kzg4844.Blob + Commitments []kzg4844.Commitment + Proofs []kzg4844.Proof +} + +type blobTxWithBlobsV1 struct { BlobTx *BlobTx + Version byte Blobs []kzg4844.Blob Commitments []kzg4844.Commitment Proofs []kzg4844.Proof } +func (btx *blobTxWithBlobsV0) tx() *BlobTx { + return btx.BlobTx +} + +func (btx *blobTxWithBlobsV0) assign(sc *BlobTxSidecar) error { + sc.Version = 0 + sc.Blobs = btx.Blobs + sc.Commitments = btx.Commitments + sc.Proofs = btx.Proofs + return nil +} + +func (btx *blobTxWithBlobsV1) tx() *BlobTx { + return btx.BlobTx +} + +func (btx *blobTxWithBlobsV1) assign(sc *BlobTxSidecar) error { + if btx.Version != 1 { + return fmt.Errorf("unsupported blob tx version %d", btx.Version) + } + sc.Version = 1 + sc.Blobs = btx.Blobs + sc.Commitments = btx.Commitments + sc.Proofs = btx.Proofs + return nil +} + // copy creates a deep copy of the transaction data and initializes all fields. func (tx *BlobTx) copy() TxData { cpy := &BlobTx{ @@ -158,9 +217,9 @@ func (tx *BlobTx) copy() TxData { } if tx.Sidecar != nil { cpy.Sidecar = &BlobTxSidecar{ - Blobs: append([]kzg4844.Blob(nil), tx.Sidecar.Blobs...), - Commitments: append([]kzg4844.Commitment(nil), tx.Sidecar.Commitments...), - Proofs: append([]kzg4844.Proof(nil), tx.Sidecar.Proofs...), + Blobs: slices.Clone(tx.Sidecar.Blobs), + Commitments: slices.Clone(tx.Sidecar.Commitments), + Proofs: slices.Clone(tx.Sidecar.Proofs), } } return cpy @@ -215,48 +274,98 @@ func (tx *BlobTx) withSidecar(sideCar *BlobTxSidecar) *BlobTx { } func (tx *BlobTx) encode(b *bytes.Buffer) error { - if tx.Sidecar == nil { + switch { + case tx.Sidecar == nil: return rlp.Encode(b, tx) + + case tx.Sidecar.Version == 0: + return rlp.Encode(b, &blobTxWithBlobsV0{ + BlobTx: tx, + Blobs: tx.Sidecar.Blobs, + Commitments: tx.Sidecar.Commitments, + Proofs: tx.Sidecar.Proofs, + }) + + case tx.Sidecar.Version == 1: + return rlp.Encode(b, &blobTxWithBlobsV1{ + BlobTx: tx, + Version: tx.Sidecar.Version, + Blobs: tx.Sidecar.Blobs, + Commitments: tx.Sidecar.Commitments, + Proofs: tx.Sidecar.Proofs, + }) + + default: + return errors.New("unsupported sidecar version") } - inner := &blobTxWithBlobs{ - BlobTx: tx, - Blobs: tx.Sidecar.Blobs, - Commitments: tx.Sidecar.Commitments, - Proofs: tx.Sidecar.Proofs, - } - return rlp.Encode(b, inner) } func (tx *BlobTx) decode(input []byte) error { - // Here we need to support two formats: the network protocol encoding of the tx (with - // blobs) or the canonical encoding without blobs. + // Here we need to support two outer formats: the network protocol encoding of the tx + // (with blobs) or the canonical encoding without blobs. // - // The two encodings can be distinguished by checking whether the first element of the - // input list is itself a list. + // The canonical encoding is just a list of fields: + // + // [chainID, nonce, ...] + // + // The network encoding is a list where the first element is the tx in the canonical encoding, + // and the remaining elements are the 'sidecar': + // + // [[chainID, nonce, ...], ...] + // + // The two outer encodings can be distinguished by checking whether the first element + // of the input list is itself a list. If it's the canonical encoding, the first + // element is the chainID, which is a number. - outerList, _, err := rlp.SplitList(input) + firstElem, _, err := rlp.SplitList(input) if err != nil { return err } - firstElemKind, _, _, err := rlp.Split(outerList) + firstElemKind, _, secondElem, err := rlp.Split(firstElem) if err != nil { return err } - if firstElemKind != rlp.List { + // Blob tx without blobs. return rlp.DecodeBytes(input, tx) } - // It's a tx with blobs. - var inner blobTxWithBlobs - if err := rlp.DecodeBytes(input, &inner); err != nil { + + // Now we know it's the network encoding with the blob sidecar. Here we again need to + // support multiple encodings: legacy sidecars (v0) with a blob proof, and versioned + // sidecars. + // + // The legacy encoding is: + // + // [tx, blobs, commitments, proofs] + // + // The versioned encoding is: + // + // [tx, version, blobs, ...] + // + // We can tell the two apart by checking whether the second element is the version byte. + // For legacy sidecar the second element is a list of blobs. + + secondElemKind, _, _, err := rlp.Split(secondElem) + if err != nil { return err } - *tx = *inner.BlobTx - tx.Sidecar = &BlobTxSidecar{ - Blobs: inner.Blobs, - Commitments: inner.Commitments, - Proofs: inner.Proofs, + var payload blobTxWithBlobs + if secondElemKind == rlp.List { + // No version byte: blob sidecar v0. + payload = new(blobTxWithBlobsV0) + } else { + // It has a version byte. Decode as v1, version is checked by assign() + payload = new(blobTxWithBlobsV1) + } + if err := rlp.DecodeBytes(input, payload); err != nil { + return err + } + sc := new(BlobTxSidecar) + if err := payload.assign(sc); err != nil { + return err } + *tx = *payload.tx() + tx.Sidecar = sc return nil } diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index fd2e2991fec0..a002affda47f 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -34,6 +34,8 @@ var ( blobT = reflect.TypeOf(Blob{}) commitmentT = reflect.TypeOf(Commitment{}) proofT = reflect.TypeOf(Proof{}) + + CellProofsPerBlob = 128 ) // Blob represents a 4844 data blob. @@ -84,6 +86,10 @@ type Claim [32]byte // useCKZG controls whether the cryptography should use the Go or C backend. var useCKZG atomic.Bool +func init() { + UseCKZG(true) +} + // UseCKZG can be called to switch the default Go implementation of KZG to the C // library if for some reason the user wishes to do so (e.g. consensus bug in one // or the other). @@ -149,6 +155,16 @@ func VerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { return gokzgVerifyBlobProof(blob, commitment, proof) } +// VerifyCellProofs verifies a batch of proofs corresponding to the blobs and commitments. +// Expects length of blobs and commitments to be equal. +// Expects length of proofs be 128 * length of blobs. +func VerifyCellProofs(blobs []Blob, commitments []Commitment, proofs []Proof) error { + if useCKZG.Load() { + return ckzgVerifyCellProofBatch(blobs, commitments, proofs) + } + return gokzgVerifyCellProofBatch(blobs, commitments, proofs) +} + // ComputeCellProofs returns the KZG cell proofs that are used to verify the blob against // the commitment. // diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index 49a7046fe0d7..b215b19928be 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -149,3 +149,44 @@ func ckzgComputeCellProofs(blob *Blob) ([]Proof, error) { } return p, nil } + +// ckzgVerifyCellProofs verifies that the blob data corresponds to the provided commitment. +func ckzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, cellProofs []Proof) error { + ckzgIniter.Do(ckzgInit) + var ( + proofs = make([]ckzg4844.Bytes48, len(cellProofs)) + commits = make([]ckzg4844.Bytes48, 0, len(cellProofs)) + cellIndices = make([]uint64, 0, len(cellProofs)) + cells = make([]ckzg4844.Cell, 0, len(cellProofs)) + ) + // Copy over the cell proofs + for i, proof := range cellProofs { + proofs[i] = (ckzg4844.Bytes48)(proof) + } + // Blow up the commitments to be the same length as the proofs + for _, commitment := range commitments { + for range gokzg4844.CellsPerExtBlob { + commits = append(commits, (ckzg4844.Bytes48)(commitment)) + } + } + // Compute the cells and cell indices + for i := range blobs { + cellsI, err := ckzg4844.ComputeCells((*ckzg4844.Blob)(&blobs[i])) + if err != nil { + return err + } + cells = append(cells, cellsI[:]...) + for idx := range len(cellsI) { + cellIndices = append(cellIndices, uint64(idx)) + } + } + + valid, err := ckzg4844.VerifyCellKZGProofBatch(commits, cellIndices, cells, proofs) + if err != nil { + return err + } + if !valid { + return errors.New("invalid proof") + } + return nil +} diff --git a/crypto/kzg4844/kzg4844_ckzg_nocgo.go b/crypto/kzg4844/kzg4844_ckzg_nocgo.go index 6f4bd3b8236f..7c552e9a18bb 100644 --- a/crypto/kzg4844/kzg4844_ckzg_nocgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_nocgo.go @@ -61,6 +61,11 @@ func ckzgVerifyBlobProof(blob *Blob, commitment Commitment, proof Proof) error { panic("unsupported platform") } +// ckzgVerifyCellProofBatch verifies that the blob data corresponds to the provided commitment. +func ckzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, proof []Proof) error { + panic("unsupported platform") +} + // ckzgComputeCellProofs returns the KZG cell proofs that are used to verify the blob against // the commitment. // diff --git a/crypto/kzg4844/kzg4844_gokzg.go b/crypto/kzg4844/kzg4844_gokzg.go index 46a38a89136d..82ec8379d41e 100644 --- a/crypto/kzg4844/kzg4844_gokzg.go +++ b/crypto/kzg4844/kzg4844_gokzg.go @@ -114,3 +114,37 @@ func gokzgComputeCellProofs(blob *Blob) ([]Proof, error) { } return p, nil } + +// gokzgVerifyCellProofs verifies that the blob data corresponds to the provided commitment. +func gokzgVerifyCellProofBatch(blobs []Blob, commitments []Commitment, cellProofs []Proof) error { + gokzgIniter.Do(gokzgInit) + + var ( + proofs = make([]gokzg4844.KZGProof, len(cellProofs)) + commits = make([]gokzg4844.KZGCommitment, 0, len(cellProofs)) + cellIndices = make([]uint64, 0, len(cellProofs)) + cells = make([]*gokzg4844.Cell, 0, len(cellProofs)) + ) + // Copy over the cell proofs + for i, proof := range cellProofs { + proofs[i] = gokzg4844.KZGProof(proof) + } + // Blow up the commitments to be the same length as the proofs + for _, commitment := range commitments { + for range gokzg4844.CellsPerExtBlob { + commits = append(commits, gokzg4844.KZGCommitment(commitment)) + } + } + // Compute the cell and cell indices + for i := range blobs { + cellsI, err := context.ComputeCells((*gokzg4844.Blob)(&blobs[i]), 2) + if err != nil { + return err + } + cells = append(cells, cellsI[:]...) + for idx := range len(cellsI) { + cellIndices = append(cellIndices, uint64(idx)) + } + } + return context.VerifyCellKZGProofBatch(commits, cellIndices, cells[:], proofs) +} diff --git a/crypto/kzg4844/kzg4844_test.go b/crypto/kzg4844/kzg4844_test.go index 7fa261e523e9..7e73efd85089 100644 --- a/crypto/kzg4844/kzg4844_test.go +++ b/crypto/kzg4844/kzg4844_test.go @@ -193,3 +193,40 @@ func benchmarkVerifyBlobProof(b *testing.B, ckzg bool) { VerifyBlobProof(blob, commitment, proof) } } + +func TestCKZGCells(t *testing.T) { testKZGCells(t, true) } +func TestGoKZGCells(t *testing.T) { testKZGCells(t, false) } +func testKZGCells(t *testing.T, ckzg bool) { + if ckzg && !ckzgAvailable { + t.Skip("CKZG unavailable in this test build") + } + defer func(old bool) { useCKZG.Store(old) }(useCKZG.Load()) + useCKZG.Store(ckzg) + + blob1 := randBlob() + blob2 := randBlob() + + commitment1, err := BlobToCommitment(blob1) + if err != nil { + t.Fatalf("failed to create KZG commitment from blob: %v", err) + } + commitment2, err := BlobToCommitment(blob2) + if err != nil { + t.Fatalf("failed to create KZG commitment from blob: %v", err) + } + + proofs1, err := ComputeCellProofs(blob1) + if err != nil { + t.Fatalf("failed to create KZG proof at point: %v", err) + } + + proofs2, err := ComputeCellProofs(blob2) + if err != nil { + t.Fatalf("failed to create KZG proof at point: %v", err) + } + proofs := append(proofs1, proofs2...) + blobs := []Blob{*blob1, *blob2} + if err := VerifyCellProofs(blobs, []Commitment{commitment1, commitment2}, proofs); err != nil { + t.Fatalf("failed to verify KZG proof at point: %v", err) + } +} diff --git a/eth/backend.go b/eth/backend.go index 7f9e45edeae8..9288033eb463 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -92,6 +92,7 @@ type Ethereum struct { // core protocol objects config *ethconfig.Config txPool *txpool.TxPool + blobTxPool *blobpool.BlobPool localTxTracker *locals.TxTracker blockchain *core.BlockChain @@ -395,6 +396,7 @@ func (s *Ethereum) Miner() *miner.Miner { return s.miner } func (s *Ethereum) AccountManager() *accounts.Manager { return s.accountManager } func (s *Ethereum) BlockChain() *core.BlockChain { return s.blockchain } func (s *Ethereum) TxPool() *txpool.TxPool { return s.txPool } +func (s *Ethereum) BlobTxPool() *blobpool.BlobPool { return s.blobTxPool } func (s *Ethereum) Engine() consensus.Engine { return s.engine } func (s *Ethereum) ChainDb() ethdb.Database { return s.chainDb } func (s *Ethereum) IsListening() bool { return true } // Always listening diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 40dae3f6f241..cbbdbbb36190 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -18,6 +18,7 @@ package catalyst import ( + "crypto/sha256" "errors" "fmt" "strconv" @@ -30,10 +31,12 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/version" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -92,7 +95,9 @@ var caps = []string{ "engine_getPayloadV2", "engine_getPayloadV3", "engine_getPayloadV4", + "engine_getPayloadV5", "engine_getBlobsV1", + "engine_getBlobsV2", "engine_newPayloadV1", "engine_newPayloadV2", "engine_newPayloadV3", @@ -112,6 +117,17 @@ var caps = []string{ "engine_getClientVersionV1", } +var ( + // Number of blobs requested via getBlobsV2 + getBlobsRequestedCounter = metrics.NewRegisteredCounter("engine/getblobs/requested", nil) + // Number of blobs requested via getBlobsV2 that are present in the blobpool + getBlobsAvailableCounter = metrics.NewRegisteredCounter("engine/getblobs/available", nil) + // Number of times getBlobsV2 responded with “hit” + getBlobsV2RequestHit = metrics.NewRegisteredCounter("engine/getblobs/hit", nil) + // Number of times getBlobsV2 responded with “miss” + getBlobsV2RequestMiss = metrics.NewRegisteredCounter("engine/getblobs/miss", nil) +) + type ConsensusAPI struct { eth *eth.Ethereum @@ -229,7 +245,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, pa return engine.STATUS_INVALID, attributesErr("missing withdrawals") case params.BeaconRoot == nil: return engine.STATUS_INVALID, attributesErr("missing beacon root") - case !api.checkFork(params.Timestamp, forks.Cancun, forks.Prague): + case !api.checkFork(params.Timestamp, forks.Cancun, forks.Prague, forks.Osaka): return engine.STATUS_INVALID, unsupportedForkErr("fcuV3 must only be called for cancun or prague payloads") } } @@ -450,6 +466,14 @@ func (api *ConsensusAPI) GetPayloadV4(payloadID engine.PayloadID) (*engine.Execu return api.getPayload(payloadID, false) } +// GetPayloadV5 returns a cached payload by id. +func (api *ConsensusAPI) GetPayloadV5(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + if !payloadID.Is(engine.PayloadV3) { + return nil, engine.UnsupportedFork + } + return api.getPayload(payloadID, false) +} + func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID, full bool) (*engine.ExecutionPayloadEnvelope, error) { log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) data := api.localBlocks.get(payloadID, full) @@ -464,14 +488,87 @@ func (api *ConsensusAPI) GetBlobsV1(hashes []common.Hash) ([]*engine.BlobAndProo if len(hashes) > 128 { return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) } - res := make([]*engine.BlobAndProofV1, len(hashes)) + var ( + res = make([]*engine.BlobAndProofV1, len(hashes)) + hasher = sha256.New() + index = make(map[common.Hash]int) + sidecars = api.eth.BlobTxPool().GetBlobs(hashes) + ) + + for i, hash := range hashes { + index[hash] = i + } + for i, sidecar := range sidecars { + if res[i] != nil || sidecar == nil { + // already filled + continue + } + for cIdx, commitment := range sidecar.Commitments { + computed := kzg4844.CalcBlobHashV1(hasher, &commitment) + if idx, ok := index[computed]; ok { + res[idx] = &engine.BlobAndProofV1{ + Blob: sidecar.Blobs[cIdx][:], + Proof: sidecar.Proofs[cIdx][:], + } + } + } + } + return res, nil +} + +// GetBlobsV2 returns a blob from the transaction pool. +func (api *ConsensusAPI) GetBlobsV2(hashes []common.Hash) ([]*engine.BlobAndProofV2, error) { + if len(hashes) > 128 { + return nil, engine.TooLargeRequest.With(fmt.Errorf("requested blob count too large: %v", len(hashes))) + } + + available := api.eth.BlobTxPool().AvailableBlobs(hashes) + getBlobsRequestedCounter.Inc(int64(len(hashes))) + getBlobsAvailableCounter.Inc(int64(available)) + // Optimization: check first if all blobs are available, if not, return empty response + if available != len(hashes) { + getBlobsV2RequestMiss.Inc(1) + return nil, nil + } + getBlobsV2RequestHit.Inc(1) - blobs, proofs := api.eth.TxPool().GetBlobs(hashes) - for i := 0; i < len(blobs); i++ { - if blobs[i] != nil { - res[i] = &engine.BlobAndProofV1{ - Blob: (*blobs[i])[:], - Proof: (*proofs[i])[:], + // pull up the blob hashes + var ( + res = make([]*engine.BlobAndProofV2, len(hashes)) + index = make(map[common.Hash][]int) + sidecars = api.eth.BlobTxPool().GetBlobs(hashes) + ) + + for i, hash := range hashes { + index[hash] = append(index[hash], i) + } + for i, sidecar := range sidecars { + if res[i] != nil { + // already filled + continue + } + if sidecar == nil { + // not found, return empty response + return nil, nil + } + if sidecar.Version != 1 { + log.Info("GetBlobs queried V0 transaction: index %v, blobhashes %v", index, sidecar.BlobHashes()) + return nil, nil + } + blobHashes := sidecar.BlobHashes() + for bIdx, hash := range blobHashes { + if idxes, ok := index[hash]; ok { + proofs := sidecar.CellProofsAt(bIdx) + var cellProofs []hexutil.Bytes + for _, proof := range proofs { + cellProofs = append(cellProofs, proof[:]) + } + for _, idx := range idxes { + res[idx] = &engine.BlobAndProofV2{ + Blob: sidecar.Blobs[bIdx][:], + CellProofs: cellProofs, + } + } } } } @@ -544,7 +641,7 @@ func (api *ConsensusAPI) NewPayloadV4(params engine.ExecutableData, versionedHas return invalidStatus, paramsErr("nil beaconRoot post-cancun") case executionRequests == nil: return invalidStatus, paramsErr("nil executionRequests post-prague") - case !api.checkFork(params.Timestamp, forks.Prague): + case !api.checkFork(params.Timestamp, forks.Prague, forks.Osaka): return invalidStatus, unsupportedForkErr("newPayloadV3 must only be called for cancun payloads") } requests := convertRequests(executionRequests) diff --git a/miner/worker.go b/miner/worker.go index d80cb8913baf..198745ad2785 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" @@ -390,6 +391,23 @@ func (miner *Miner) commitTransactions(env *environment, plainTxs, blobTxs *tran continue } + // Make sure all transactions after osaka have cell proofs + if miner.chainConfig.IsOsaka(env.header.Number, env.header.Time) { + if sidecar := tx.BlobTxSidecar(); sidecar != nil { + if sidecar.Version == 0 { + log.Info("Including blob tx with v0 sidecar, recomputing proofs", "hash", ltx.Hash) + sidecar.Proofs = make([]kzg4844.Proof, 0, len(sidecar.Blobs)*kzg4844.CellProofsPerBlob) + for _, blob := range sidecar.Blobs { + cellProofs, err := kzg4844.ComputeCellProofs(&blob) + if err != nil { + panic(err) + } + sidecar.Proofs = append(sidecar.Proofs, cellProofs...) + } + } + } + } + // Error may be ignored here. The error has already been checked // during transaction acceptance in the transaction pool. from, _ := types.Sender(env.signer, tx)