Bedrock: use Engine API authentication (ethereum-optimism#2591)

* feat: new --l2.jwt-secret flag to authenticate engine API comms with l2, remove l2.eth flag, update e2e tests

* feat: update bedrock ops with jwt secret flag

Co-authored-by: Matthew Slipper <[email protected]>

Author: protolambda

op-e2e/geth.go

@@ -6,8 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"math/big"
-	"strconv"
-	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
@@ -98,36 +96,30 @@ func initL1Geth(cfg *SystemConfig, wallet *hdwallet.Wallet, genesis *core.Genesi
 	}
 	nodeConfig := &node.Config{
 		Name:        "l1-geth",
-		WSHost:      cfg.L1WsAddr,
-		WSPort:      cfg.L1WsPort,
+		WSHost:      "127.0.0.1",
+		WSPort:      0,
 		WSModules:   []string{"debug", "admin", "eth", "txpool", "net", "rpc", "web3", "personal", "engine"},
 		HTTPModules: []string{"debug", "admin", "eth", "txpool", "net", "rpc", "web3", "personal", "engine"},
 	}
 
 	return createGethNode(false, nodeConfig, ethConfig, []*ecdsa.PrivateKey{pk})
 }
 
-func initL2Geth(name, addr string, l2ChainID *big.Int, genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) {
+// init a geth node.
+func initL2Geth(name string, l2ChainID *big.Int, genesis *core.Genesis, jwtPath string) (*node.Node, *eth.Ethereum, error) {
 	ethConfig := &ethconfig.Config{
 		NetworkId: l2ChainID.Uint64(),
 		Genesis:   genesis,
 	}
-	// Parsing ws://127.0.0.1:9091 for "127.0.0.1" and "9091"
-	s := strings.Split(addr, ":")
-	_, host, ok := strings.Cut(s[1], "//")
-	if !ok {
-		return nil, nil, fmt.Errorf("could not find ws host in %s", addr)
-	}
-	port, err := strconv.ParseInt(s[2], 10, 32)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to parse port from address: %w", err)
-	}
 	nodeConfig := &node.Config{
 		Name:        fmt.Sprintf("l2-geth-%v", name),
-		WSHost:      host,
-		WSPort:      int(port),
+		WSHost:      "127.0.0.1",
+		WSPort:      0,
+		AuthAddr:    "127.0.0.1",
+		AuthPort:    0,
 		WSModules:   []string{"debug", "admin", "eth", "txpool", "net", "rpc", "web3", "personal", "engine"},
 		HTTPModules: []string{"debug", "admin", "eth", "txpool", "net", "rpc", "web3", "personal", "engine"},
+		JWTSecret:   jwtPath,
 	}
 	return createGethNode(true, nodeConfig, ethConfig, nil)
 }

op-e2e/setup.go

@@ -28,6 +28,7 @@ import (
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	hdwallet "github.com/miguelmota/go-ethereum-hdwallet"
 )
@@ -75,10 +76,12 @@ type SystemConfig struct {
 	L2OOCfg    L2OOContractConfig
 	DepositCFG DepositContractConfig
 
-	L1WsAddr     string
-	L1WsPort     int
-	L1ChainID    *big.Int
-	L2ChainID    *big.Int
+	L1ChainID *big.Int
+	L2ChainID *big.Int
+
+	JWTFilePath string
+	JWTSecret   [32]byte
+
 	Nodes        map[string]*rollupNode.Config // Per node config. Don't use populate rollup.Config
 	Loggers      map[string]log.Logger
 	RollupConfig rollup.Config // Shared rollup configs
@@ -293,8 +296,9 @@ func (cfg SystemConfig) start() (*System, error) {
 	}
 	sys.nodes["l1"] = l1Node
 	sys.backends["l1"] = l1Backend
-	for name, l2Cfg := range cfg.Nodes {
-		node, backend, err := initL2Geth(name, l2Cfg.L2EngineAddrs[0], cfg.L2ChainID, l2Genesis)
+
+	for name := range cfg.Nodes {
+		node, backend, err := initL2Geth(name, cfg.L2ChainID, l2Genesis, cfg.JWTFilePath)
 		if err != nil {
 			return nil, err
 		}
@@ -324,14 +328,28 @@ func (cfg SystemConfig) start() (*System, error) {
 		}
 	}
 
+	// Configure connections to L1 and L2 for rollup nodes.
+	// TODO: refactor testing to use in-process rpc connections instead of websockets.
+	for name, rollupCfg := range cfg.Nodes {
+		rollupCfg.L1 = &rollupNode.L1EndpointConfig{
+			L1NodeAddr: l1Node.WSEndpoint(),
+			L1TrustRPC: false,
+		}
+		rollupCfg.L2s = &rollupNode.L2EndpointsConfig{
+			L2EngineAddrs:      []string{sys.nodes[name].WSAuthEndpoint()},
+			L2EngineJWTSecrets: [][32]byte{cfg.JWTSecret},
+		}
+	}
+
 	// Geth Clients
 	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 	defer cancel()
-	l1Client, err := ethclient.DialContext(ctx, fmt.Sprintf("ws://%s:%d", cfg.L1WsAddr, cfg.L1WsPort))
+	l1Srv, err := l1Node.RPCHandler()
 	if err != nil {
 		didErrAfterStart = true
 		return nil, err
 	}
+	l1Client := ethclient.NewClient(rpc.DialInProc(l1Srv))
 	sys.Clients["l1"] = l1Client
 	for name, node := range sys.nodes {
 		client, err := ethclient.DialContext(ctx, node.WSEndpoint())
@@ -512,8 +530,8 @@ func (cfg SystemConfig) start() (*System, error) {
 
 	// L2Output Submitter
 	sys.l2OutputSubmitter, err = l2os.NewL2OutputSubmitter(l2os.Config{
-		L1EthRpc:                  "ws://127.0.0.1:9090",
-		L2EthRpc:                  sys.cfg.Nodes["sequencer"].L2NodeAddr,
+		L1EthRpc:                  sys.nodes["l1"].WSEndpoint(),
+		L2EthRpc:                  sys.nodes["sequencer"].WSEndpoint(),
 		RollupRpc:                 rollupEndpoint,
 		L2OOAddress:               sys.L2OOContractAddr.String(),
 		PollInterval:              50 * time.Millisecond,
@@ -544,8 +562,8 @@ func (cfg SystemConfig) start() (*System, error) {
 
 	// Batch Submitter
 	sys.batchSubmitter, err = bss.NewBatchSubmitter(bss.Config{
-		L1EthRpc:                   "ws://127.0.0.1:9090",
-		L2EthRpc:                   sys.cfg.Nodes["sequencer"].L2NodeAddr,
+		L1EthRpc:                   sys.nodes["l1"].WSEndpoint(),
+		L2EthRpc:                   sys.nodes["sequencer"].WSEndpoint(),
 		RollupRpc:                  rollupEndpoint,
 		MinL1TxSize:                1,
 		MaxL1TxSize:                120000,

op-e2e/system_test.go

@@ -5,6 +5,8 @@ import (
 	"flag"
 	"fmt"
 	"math/big"
+	"os"
+	"path"
 	"testing"
 	"time"
 
@@ -24,6 +26,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi/bind"
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethclient"
@@ -59,8 +62,18 @@ const (
 
 var (
 	batchInboxAddress = common.Address{0xff, 0x02}
+	testingJWTSecret  = [32]byte{123}
 )
 
+func writeDefaultJWT(t *testing.T) string {
+	// Sadly the geth node config cannot load JWT secret from memory, it has to be a file
+	jwtPath := path.Join(t.TempDir(), "jwt_secret")
+	if err := os.WriteFile(jwtPath, []byte(hexutil.Encode(testingJWTSecret[:])), 0600); err != nil {
+		t.Fatalf("failed to prepare jwt file for geth: %v", err)
+	}
+	return jwtPath
+}
+
 func defaultSystemConfig(t *testing.T) SystemConfig {
 	return SystemConfig{
 		Mnemonic: "squirrel green gallery layer logic title habit chase clog actress language enrich body plate fun pledge gap abuse mansion define either blast alien witness",
@@ -86,23 +99,14 @@ func defaultSystemConfig(t *testing.T) SystemConfig {
 		CliqueSignerDerivationPath: cliqueSignerHDPath,
 		L1InfoPredeployAddress:     derive.L1InfoPredeployAddr,
 		L1BlockTime:                2,
-		L1WsAddr:                   "127.0.0.1",
-		L1WsPort:                   9090,
 		L1ChainID:                  big.NewInt(900),
 		L2ChainID:                  big.NewInt(901),
+		JWTFilePath:                writeDefaultJWT(t),
+		JWTSecret:                  testingJWTSecret,
 		Nodes: map[string]*rollupNode.Config{
-			"verifier": {
-				L1NodeAddr:    "ws://127.0.0.1:9090",
-				L2EngineAddrs: []string{"ws://127.0.0.1:9091"},
-				L2NodeAddr:    "ws://127.0.0.1:9091",
-				L1TrustRPC:    false,
-			},
+			"verifier": {},
 			"sequencer": {
-				L1NodeAddr:    "ws://127.0.0.1:9090",
-				L2EngineAddrs: []string{"ws://127.0.0.1:9092"},
-				L2NodeAddr:    "ws://127.0.0.1:9092",
-				L1TrustRPC:    false,
-				Sequencer:     true,
+				Sequencer: true,
 				// Submitter PrivKey is set in system start for rollup nodes where sequencer = true
 				RPC: node.RPCConfig{
 					ListenAddr: "127.0.0.1",
@@ -786,7 +790,7 @@ func TestWithdrawals(t *testing.T) {
 	header, err = l2Seq.HeaderByNumber(ctx, blockNumber)
 	require.Nil(t, err)
 
-	rpc, err := rpc.Dial(cfg.Nodes["sequencer"].L2NodeAddr)
+	rpc, err := rpc.Dial(sys.nodes["sequencer"].WSEndpoint())
 	require.Nil(t, err)
 	l2client := withdrawals.NewClient(rpc)
 

op-node/cmd/main.go

@@ -63,14 +63,16 @@ func main() {
 
 func RollupNodeMain(ctx *cli.Context) error {
 	log.Info("Initializing Rollup Node")
-	cfg, err := opnode.NewConfig(ctx)
+	logCfg, err := opnode.NewLogConfig(ctx)
 	if err != nil {
-		log.Error("Unable to create the rollup node config", "error", err)
+		log.Error("Unable to create the log config", "error", err)
 		return err
 	}
-	logCfg, err := opnode.NewLogConfig(ctx)
+	log := logCfg.NewLogger()
+
+	cfg, err := opnode.NewConfig(ctx, log)
 	if err != nil {
-		log.Error("Unable to create the log config", "error", err)
+		log.Error("Unable to create the rollup node config", "error", err)
 		return err
 	}
 	snapshotLog, err := opnode.NewSnapshotLogger(ctx)
@@ -79,7 +81,7 @@ func RollupNodeMain(ctx *cli.Context) error {
 		return err
 	}
 
-	n, err := node.New(context.Background(), cfg, logCfg.NewLogger(), snapshotLog, VersionWithMeta)
+	n, err := node.New(context.Background(), cfg, log, snapshotLog, VersionWithMeta)
 	if err != nil {
 		log.Error("Unable to create the rollup node", "error", err)
 		return err

op-node/flags/flags.go

@@ -31,12 +31,6 @@ var (
 		Required: true,
 		EnvVar:   prefixEnvVar("ROLLUP_CONFIG"),
 	}
-	L2EthNodeAddr = cli.StringFlag{
-		Name:     "l2.eth",
-		Usage:    "Address of L2 User JSON-RPC endpoint to use (eth namespace required)",
-		Required: true,
-		EnvVar:   prefixEnvVar("L2_ETH_RPC"),
-	}
 	RPCListenAddr = cli.StringFlag{
 		Name:     "rpc.addr",
 		Usage:    "RPC listening address",
@@ -56,7 +50,14 @@ var (
 		Usage:  "Trust the L1 RPC, sync faster at risk of malicious/buggy RPC providing bad or inconsistent L1 data",
 		EnvVar: prefixEnvVar("L1_TRUST_RPC"),
 	}
-
+	L2EngineJWTSecret = cli.StringSliceFlag{
+		Name: "l2.jwt-secret",
+		Usage: "Paths to JWT secret keys, one per L2 endpoint, in the same order as the provided l2 addresses. " +
+			"Keys are 32 bytes, hex encoded in a file. A new key per endpoint will be generated if left empty.",
+		Required: false,
+		Value:    &cli.StringSlice{},
+		EnvVar:   prefixEnvVar("L2_ENGINE_AUTH"),
+	}
 	SequencingEnabledFlag = cli.BoolFlag{
 		Name:   "sequencing.enabled",
 		Usage:  "enable sequencing",
@@ -92,13 +93,13 @@ var requiredFlags = []cli.Flag{
 	L1NodeAddr,
 	L2EngineAddrs,
 	RollupConfig,
-	L2EthNodeAddr,
 	RPCListenAddr,
 	RPCListenPort,
 }
 
 var optionalFlags = append([]cli.Flag{
 	L1TrustRPC,
+	L2EngineJWTSecret,
 	SequencingEnabledFlag,
 	LogLevelFlag,
 	LogFormatFlag,