EspressoSystems
diff --git a/‎.github/buildspec.yml
+4 b/‎.github/buildspec.yml
+4
diff --git a/‎.github/manifest-buildspec.yml
+57 b/‎.github/manifest-buildspec.yml
+57
diff --git a/‎.github/workflows/ci.yml
+4 b/‎.github/workflows/ci.yml
+4
diff --git a/‎.github/workflows/codeql-analysis.yml
+3 b/‎.github/workflows/codeql-analysis.yml
+3
diff --git a/‎CODEOWNERS
+1-1 b/‎CODEOWNERS
+1-1
diff --git a/‎Makefile
-5 b/‎Makefile
-5
diff --git a/‎arbnode/batch_poster.go
+95-53 b/‎arbnode/batch_poster.go
+95-53
diff --git a/‎arbnode/node.go
+5-3 b/‎arbnode/node.go
+5-3
@@ -1,5 +1,9 @@
 version: 0.2
 
+env:
+  exported-variables:
+    - IMAGE_TAG
+
 phases:
   pre_build:
     commands:
 
@@ -0,0 +1,57 @@
+version: 0.2
+
+phases:
+  pre_build:
+    commands:
+      - COMMIT_HASH=$(git rev-parse --short=7 HEAD || echo "latest")
+      - VERSION_TAG=$(git tag --points-at HEAD | sed '/-/!s/$/_/' | sort -rV | sed 's/_$//' | head -n 1 | grep ^ || git show -s --pretty=%D | sed 's/, /\n/g' | grep -v '^origin/' |grep -v '^grafted\|HEAD\|master\|main$' || echo "dev")
+      - NITRO_VERSION=${VERSION_TAG}-${COMMIT_HASH}
+      - IMAGE_TAG=${NITRO_VERSION}
+
+      # Log IMAGE_TAG environment variable
+      - echo "Using IMAGE_TAG environment variable $IMAGE_TAG"
+
+      # Login to ECR
+      - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $REPOSITORY_URI
+      # Login to DockerHub if credentials provided
+      - |
+        if [ -n "$DOCKERHUB_USERNAME" ] && [ -n "$DOCKERHUB_PASSWORD" ]; then
+          echo "$DOCKERHUB_PASSWORD" | docker login --username "$DOCKERHUB_USERNAME" --password-stdin
+        fi
+      # Enable experimental features while preserving auth
+      - mkdir -p $HOME/.docker
+      - |
+        if [ -f "$HOME/.docker/config.json" ]; then
+          # Add experimental flag to existing config
+          cat $HOME/.docker/config.json | jq '. + {"experimental":"enabled"}' > $HOME/.docker/config.json.tmp
+          mv $HOME/.docker/config.json.tmp $HOME/.docker/config.json
+        else
+          # Create new config with experimental flag
+          echo '{"experimental":"enabled"}' > $HOME/.docker/config.json
+        fi
+
+  build:
+    commands:
+      # Regular node image
+      - docker manifest create $REPOSITORY_URI:$IMAGE_TAG $REPOSITORY_URI:$IMAGE_TAG-amd64 $REPOSITORY_URI:$IMAGE_TAG-arm64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG $REPOSITORY_URI:$IMAGE_TAG-amd64 --arch amd64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG $REPOSITORY_URI:$IMAGE_TAG-arm64 --arch arm64
+      - docker manifest push $REPOSITORY_URI:$IMAGE_TAG
+
+      # Slim variant
+      - docker manifest create $REPOSITORY_URI:$IMAGE_TAG-slim $REPOSITORY_URI:$IMAGE_TAG-slim-amd64 $REPOSITORY_URI:$IMAGE_TAG-slim-arm64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG-slim $REPOSITORY_URI:$IMAGE_TAG-slim-amd64 --arch amd64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG-slim $REPOSITORY_URI:$IMAGE_TAG-slim-arm64 --arch arm64
+      - docker manifest push $REPOSITORY_URI:$IMAGE_TAG-slim
+
+      # Dev variant
+      - docker manifest create $REPOSITORY_URI:$IMAGE_TAG-dev $REPOSITORY_URI:$IMAGE_TAG-dev-amd64 $REPOSITORY_URI:$IMAGE_TAG-dev-arm64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG-dev $REPOSITORY_URI:$IMAGE_TAG-dev-amd64 --arch amd64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG-dev $REPOSITORY_URI:$IMAGE_TAG-dev-arm64 --arch arm64
+      - docker manifest push $REPOSITORY_URI:$IMAGE_TAG-dev
+
+      # Validator variant
+      - docker manifest create $REPOSITORY_URI:$IMAGE_TAG-validator $REPOSITORY_URI:$IMAGE_TAG-validator-amd64 $REPOSITORY_URI:$IMAGE_TAG-validator-arm64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG-validator $REPOSITORY_URI:$IMAGE_TAG-validator-amd64 --arch amd64
+      - docker manifest annotate $REPOSITORY_URI:$IMAGE_TAG-validator $REPOSITORY_URI:$IMAGE_TAG-validator-arm64 --arch arm64
+      - docker manifest push $REPOSITORY_URI:$IMAGE_TAG-validator
@@ -127,6 +127,9 @@ jobs:
 
       - name: Install Foundry
         uses: foundry-rs/foundry-toolchain@v1
+        
+      - name: Install cbindgen
+        run: cargo install --force cbindgen
 
       - name: Install cbindgen
         run: cargo install --force cbindgen
@@ -241,6 +244,7 @@ jobs:
         run: |
           make build-prover-bin
           target/bin/prover target/machines/latest/machine.wavm.br -b --json-inputs="${{ github.workspace }}/target/TestProgramStorage/block_inputs.json"
+      
       - name: run jit prover on block input json
         if: matrix.test-mode == 'defaults'
         run: |
 
@@ -86,6 +86,9 @@ jobs:
     - name: Install cbindgen
       run: cargo install --force cbindgen
 
+    - name: Install cbindgen
+      run: cargo install --force cbindgen
+
     - name: Cache Rust Build Products
       uses: actions/cache@v3
       with:
 
@@ -2,4 +2,4 @@
 # later match takes precedence, they will be requested for review when someone
 # opens a pull request.
 
-*      @ImJeremyHe @nomaxg @alysiahuggins @zacshowa @Sneh1999 @sveitser @jbearer
+*      @ImJeremyHe @nomaxg @alysiahuggins @zacshowa @Sneh1999 @sveitser @jbearer @jjeangal
@@ -596,11 +596,6 @@ $(stylus_test_hostio-test_wasm): $(stylus_test_hostio-test_src)
 	./scripts/remove_reference_types.sh $@
 	@touch -c $@ # cargo might decide to not rebuild the binary
 
-$(stylus_test_hostio-test_wasm): $(stylus_test_hostio-test_src)
-	$(cargo_nightly) --manifest-path $< --release --config $(stylus_cargo)
-	./scripts/remove_reference_types.sh $@
-	@touch -c $@ # cargo might decide to not rebuild the binary
-
 contracts/test/prover/proofs/float%.json: $(arbitrator_cases)/float%.wasm $(prover_bin) $(output_latest)/soft-float.wasm
 	$(prover_bin) $< -l $(output_latest)/soft-float.wasm -o $@ -b --allow-hostapi --require-success
 
 
@@ -34,7 +34,6 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/offchainlabs/bold/solgen/go/bridgegen"
-
 	"github.com/offchainlabs/nitro/arbnode/dataposter"
 	"github.com/offchainlabs/nitro/arbnode/dataposter/storage"
 	"github.com/offchainlabs/nitro/arbnode/redislock"
@@ -143,7 +142,8 @@ const (
 )
 
 type BatchPosterDangerousConfig struct {
-	AllowPostingFirstBatchWhenSequencerMessageCountMismatch bool `koanf:"allow-posting-first-batch-when-sequencer-message-count-mismatch"`
+	AllowPostingFirstBatchWhenSequencerMessageCountMismatch bool   `koanf:"allow-posting-first-batch-when-sequencer-message-count-mismatch"`
+	FixedGasLimit                                           uint64 `koanf:"fixed-gas-limit"`
 }
 
 type BatchPosterConfig struct {
@@ -229,6 +229,7 @@ type BatchPosterConfigFetcher func() *BatchPosterConfig
 
 func DangerousBatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
 	f.Bool(prefix+".allow-posting-first-batch-when-sequencer-message-count-mismatch", DefaultBatchPosterConfig.Dangerous.AllowPostingFirstBatchWhenSequencerMessageCountMismatch, "allow posting the first batch even if sequence number doesn't match chain (useful after force-inclusion)")
+	f.Uint64(prefix+".fixed-gas-limit", DefaultBatchPosterConfig.Dangerous.FixedGasLimit, "use this gas limit for batch posting instead of estimating it")
 }
 
 func BatchPosterConfigAddOptions(prefix string, f *pflag.FlagSet) {
@@ -1182,73 +1183,85 @@ type estimateGasParams struct {
 	BlobHashes   []common.Hash    `json:"blobVersionedHashes,omitempty"`
 }
 
+type OverrideAccount struct {
+	StateDiff map[common.Hash]common.Hash `json:"stateDiff"`
+}
+
+type StateOverride map[common.Address]OverrideAccount
+
 func estimateGas(client rpc.ClientInterface, ctx context.Context, params estimateGasParams) (uint64, error) {
 	var gas hexutil.Uint64
 	err := client.CallContext(ctx, &gas, "eth_estimateGas", params)
 	return uint64(gas), err
 }
 
-func (b *BatchPoster) estimateGas(
+func (b *BatchPoster) estimateGasSimple(
 	ctx context.Context,
-	sequencerMessage []byte,
-	delayedMessages uint64,
 	realData []byte,
 	realBlobs []kzg4844.Blob,
-	realNonce uint64,
 	realAccessList types.AccessList,
-	delayProof *bridgegen.DelayProof,
 ) (uint64, error) {
 
 	config := b.config()
 	rpcClient := b.l1Reader.Client()
 	rawRpcClient := rpcClient.Client()
-	useNormalEstimation := b.dataPoster.MaxMempoolTransactions() == 1
-	if !useNormalEstimation {
-		// Check if we can use normal estimation anyways because we're at the latest nonce
-		latestNonce, err := rpcClient.NonceAt(ctx, b.dataPoster.Sender(), nil)
-		if err != nil {
-			return 0, err
-		}
-		useNormalEstimation = latestNonce == realNonce
-	}
 	latestHeader, err := rpcClient.HeaderByNumber(ctx, nil)
 	if err != nil {
 		return 0, err
 	}
 	maxFeePerGas := arbmath.BigMulByUBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips)
-	if useNormalEstimation {
-		_, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs)
-		if err != nil {
-			return 0, fmt.Errorf("failed to compute real blob commitments: %w", err)
-		}
-		// If we're at the latest nonce, we can skip the special future tx estimate stuff
-		gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
-			From:         b.dataPoster.Sender(),
-			To:           &b.seqInboxAddr,
-			Data:         realData,
-			MaxFeePerGas: (*hexutil.Big)(maxFeePerGas),
-			BlobHashes:   realBlobHashes,
-			AccessList:   realAccessList,
-		})
-		if err != nil {
-			return 0, fmt.Errorf("%w: %w", ErrNormalGasEstimationFailed, err)
-		}
-		return gas + config.ExtraBatchGas, nil
+	_, realBlobHashes, err := blobs.ComputeCommitmentsAndHashes(realBlobs)
+	if err != nil {
+		return 0, fmt.Errorf("failed to compute real blob commitments: %w", err)
+	}
+	// If we're at the latest nonce, we can skip the special future tx estimate stuff
+	gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
+		From:         b.dataPoster.Sender(),
+		To:           &b.seqInboxAddr,
+		Data:         realData,
+		MaxFeePerGas: (*hexutil.Big)(maxFeePerGas),
+		BlobHashes:   realBlobHashes,
+		AccessList:   realAccessList,
+	})
+	if err != nil {
+		return 0, fmt.Errorf("%w: %w", ErrNormalGasEstimationFailed, err)
+	}
+	return gas + config.ExtraBatchGas, nil
+}
+
+// This estimates gas for a batch with future nonce
+// a prev. batch is already pending in the parent chain's mempool
+func (b *BatchPoster) estimateGasForFutureTx(
+	ctx context.Context,
+	sequencerMessage []byte,
+	delayedMessagesBefore uint64,
+	delayedMessagesAfter uint64,
+	realAccessList types.AccessList,
+	usingBlobs bool,
+	delayProof *bridgegen.DelayProof,
+) (uint64, error) {
+	config := b.config()
+	rpcClient := b.l1Reader.Client()
+	rawRpcClient := rpcClient.Client()
+	latestHeader, err := rpcClient.HeaderByNumber(ctx, nil)
+	if err != nil {
+		return 0, err
 	}
+	maxFeePerGas := arbmath.BigMulByUBips(latestHeader.BaseFee, config.GasEstimateBaseFeeMultipleBips)
 
 	// Here we set seqNum to MaxUint256, and prevMsgNum to 0, because it disables the smart contracts' consistency checks.
 	// However, we set nextMsgNum to 1 because it is necessary for a correct estimation for the final to be non-zero.
 	// Because we're likely estimating against older state, this might not be the actual next message,
 	// but the gas used should be the same.
-	data, kzgBlobs, err := b.encodeAddBatch(abi.MaxUint256, 0, 1, sequencerMessage, delayedMessages, len(realBlobs) > 0, delayProof)
+	data, kzgBlobs, err := b.encodeAddBatch(abi.MaxUint256, 0, 1, sequencerMessage, delayedMessagesAfter, usingBlobs, delayProof)
 	if err != nil {
 		return 0, err
 	}
 	_, blobHashes, err := blobs.ComputeCommitmentsAndHashes(kzgBlobs)
 	if err != nil {
 		return 0, fmt.Errorf("failed to compute blob commitments: %w", err)
 	}
-	gas, err := estimateGas(rawRpcClient, ctx, estimateGasParams{
+	gasParams := estimateGasParams{
 		From:         b.dataPoster.Sender(),
 		To:           &b.seqInboxAddr,
 		Data:         data,
@@ -1257,7 +1270,22 @@ func (b *BatchPoster) estimateGas(
 		// This isn't perfect because we're probably estimating the batch at a different sequence number,
 		// but it should overestimate rather than underestimate which is fine.
 		AccessList: realAccessList,
-	})
+	}
+	// slot 0 in the SequencerInbox smart contract holds totalDelayedMessagesRead -
+	// This is the number of delayed messages that sequencer knows were processed
+	// SequencerInbox checks this value to make sure delayed inbox isn't going backward,
+	// And it makes it know if a delayProof is needed
+	// Both are required for successful batch posting
+	stateOverride := StateOverride{
+		b.seqInboxAddr: {
+			StateDiff: map[common.Hash]common.Hash{
+				// slot 0
+				{}: common.Hash(arbmath.Uint64ToU256Bytes(delayedMessagesBefore)),
+			},
+		},
+	}
+	var gas hexutil.Uint64
+	err = rawRpcClient.CallContext(ctx, &gas, "eth_estimateGas", gasParams, rpc.PendingBlockNumber, stateOverride)
 	if err != nil {
 		sequencerMessageHeader := sequencerMessage
 		if len(sequencerMessageHeader) > 33 {
@@ -1266,13 +1294,14 @@ func (b *BatchPoster) estimateGas(
 		log.Warn(
 			"error estimating gas for batch",
 			"err", err,
-			"delayedMessages", delayedMessages,
+			"delayedMessagesBefore", delayedMessagesBefore,
+			"delayedMessagesAfter", delayedMessagesAfter,
 			"sequencerMessageHeader", hex.EncodeToString(sequencerMessageHeader),
 			"sequencerMessageLen", len(sequencerMessage),
 		)
 		return 0, fmt.Errorf("error estimating gas for batch: %w", err)
 	}
-	return gas + config.ExtraBatchGas, nil
+	return uint64(gas) + config.ExtraBatchGas, nil
 }
 
 const ethPosBlockTime = 12 * time.Second
@@ -1359,12 +1388,6 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 		return false, nil
 	}
 
-	lastPotentialMsg, err := b.streamer.GetMessage(msgCount - 1)
-	if err != nil {
-
-		return false, err
-	}
-
 	config := b.config()
 	forcePostBatch := config.MaxDelay <= 0
 
@@ -1651,15 +1674,34 @@ func (b *BatchPoster) maybePostSequencerBatch(ctx context.Context) (bool, error)
 		return false, fmt.Errorf("produced %v blobs for batch but a block can only hold %v (compressed batch was %v bytes long)", len(kzgBlobs), params.MaxBlobGasPerBlock/params.BlobTxBlobGasPerBlob, len(sequencerMsg))
 	}
 	accessList := b.accessList(batchPosition.NextSeqNum, b.building.segments.delayedMsg)
+	var gasLimit uint64
+	if b.config().Dangerous.FixedGasLimit != 0 {
+		gasLimit = b.config().Dangerous.FixedGasLimit
+	} else {
+		useSimpleEstimation := b.dataPoster.MaxMempoolTransactions() == 1
+		if !useSimpleEstimation {
+			// Check if we can use normal estimation anyways because we're at the latest nonce
+			latestNonce, err := b.l1Reader.Client().NonceAt(ctx, b.dataPoster.Sender(), nil)
+			if err != nil {
+				return false, err
+			}
+			useSimpleEstimation = latestNonce == nonce
+		}
 
-	// On restart, we may be trying to estimate gas for a batch whose successor has
-	// already made it into pending state, if not latest state.
-	// In that case, we might get a revert with `DelayedBackwards()`.
-	// To avoid that, we artificially increase the delayed messages to `lastPotentialMsg.DelayedMessagesRead`.
-	// In theory, this might reduce gas usage, but only by a factor that's already
-	// accounted for in `config.ExtraBatchGas`, as that same factor can appear if a user
-	// posts a new delayed message that we didn't see while gas estimating.
-	gasLimit, err := b.estimateGas(ctx, sequencerMsg, lastPotentialMsg.DelayedMessagesRead, data, kzgBlobs, nonce, accessList, delayProof)
+		if useSimpleEstimation {
+			gasLimit, err = b.estimateGasSimple(ctx, data, kzgBlobs, accessList)
+		} else {
+			// When there are previous batches queued up in the dataPoster, we override the delayed message count in the sequencer inbox
+			// so it accepts the corresponding delay proof. Otherwise, the gas estimation would revert.
+			var delayedMsgBefore uint64
+			if b.building.firstDelayedMsg != nil {
+				delayedMsgBefore = b.building.firstDelayedMsg.DelayedMessagesRead - 1
+			} else if b.building.firstNonDelayedMsg != nil {
+				delayedMsgBefore = b.building.firstNonDelayedMsg.DelayedMessagesRead
+			}
+			gasLimit, err = b.estimateGasForFutureTx(ctx, sequencerMsg, delayedMsgBefore, b.building.segments.delayedMsg, accessList, len(kzgBlobs) > 0, delayProof)
+		}
+	}
 	if err != nil {
 		return false, err
 	}
 
@@ -1082,9 +1082,6 @@ func (n *Node) StopAndWait() {
 	if n.MessagePruner != nil && n.MessagePruner.Started() {
 		n.MessagePruner.StopAndWait()
 	}
-	if n.BroadcastServer != nil && n.BroadcastServer.Started() {
-		n.BroadcastServer.StopAndWait()
-	}
 	if n.BroadcastClients != nil {
 		n.BroadcastClients.StopAndWait()
 	}
@@ -1106,6 +1103,11 @@ func (n *Node) StopAndWait() {
 	if n.TxStreamer.Started() {
 		n.TxStreamer.StopAndWait()
 	}
+	// n.BroadcastServer is stopped after txStreamer and inboxReader because if done before it would lead to a deadlock, as the threads from these two components
+	// attempt to Broadcast i.e send feedMessage to clientManager's broadcastChan when there wont be any reader to read it as n.BroadcastServer would've been stopped
+	if n.BroadcastServer != nil && n.BroadcastServer.Started() {
+		n.BroadcastServer.StopAndWait()
+	}
 	if n.SeqCoordinator != nil && n.SeqCoordinator.Started() {
 		// Just stops the redis client (most other stuff was stopped earlier)
 		n.SeqCoordinator.StopAndWait()