Use Espresso Network Go (#586)

* initial commit

* Use espresso network go library directly

* lint

* fix dockerfile

* fix dockerfile

* debug and free up space in the runner

* fix ci

* use tar file instead of git submodules

* remove espresso network go from dockerfile

* fix dockerfile

* dockerfile update

* fix makefile

* fix tests

* fix arch

* fix triple code

* address comments and fix dockerfile

* fix dockerfile again

* fix dockerfile again-1

* add makefile as well in docker build

* make path relative

* use workspace

* fix makefile

* revert to using curl

* lint

* add curl

Author: Sneh1999

.github/workflows/ci.yml

@@ -17,7 +17,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  free-disk-space:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
   test:
+    needs: free-disk-space
     name: Go Tests
     runs-on: ubuntu-latest
 

.github/workflows/create-asana-attachment.yaml

@@ -11,7 +11,15 @@ on:
       - develop
 
 jobs:
+  free-disk-space:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
   docker:
+    needs: free-disk-space
     name: Docker build
     runs-on: ubuntu-latest
     services:

.github/workflows/docker.yml

@@ -27,7 +27,15 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  free-disk-space:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: true
   docker_build:
+    needs: free-disk-space
     strategy:
       matrix:
         platform: [linux/amd64, linux/arm64]

.github/workflows/espresso-docker.yml

@@ -24,4 +24,6 @@ system_tests/test-data/*
 system_tests/testdata/*
 arbos/testdata/*
 
-/.pre-commit-config.yaml
+/.pre-commit-config.yaml
+espresso-network-go
+*.tar.gz

.gitignore

@@ -40,3 +40,4 @@
 	path = bold
 	url = https://github.com/EspressoSystems/bold.git
 	branch = integration
+

.gitmodules

@@ -56,6 +56,10 @@ COPY arbitrator/prover arbitrator/prover
 COPY arbitrator/wasm-libraries arbitrator/wasm-libraries
 COPY arbitrator/tools/wasmer arbitrator/tools/wasmer
 COPY brotli brotli
+ARG ESPRESSO_NETWORK_GO_VER=0.0.36
+ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz .
+RUN tar -xzf v${ESPRESSO_NETWORK_GO_VER}.tar.gz && \
+    mv espresso-network-go-${ESPRESSO_NETWORK_GO_VER} espresso-network-go
 COPY scripts/build-brotli.sh scripts/
 COPY scripts/remove_reference_types.sh scripts/
 COPY --from=brotli-wasm-export / target/
@@ -90,6 +94,10 @@ COPY ./safe-smart-account ./safe-smart-account
 COPY ./solgen/gen.go ./solgen/
 COPY ./fastcache ./fastcache
 COPY ./go-ethereum ./go-ethereum
+ARG ESPRESSO_NETWORK_GO_VER=0.0.36
+ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz .
+RUN tar -xzf v${ESPRESSO_NETWORK_GO_VER}.tar.gz && \
+    mv espresso-network-go-${ESPRESSO_NETWORK_GO_VER} espresso-network-go
 COPY scripts/remove_reference_types.sh scripts/
 COPY --from=brotli-wasm-export / target/
 COPY --from=contracts-builder workspace/contracts/build/contracts/src/precompiles/ contracts/build/contracts/src/precompiles/
@@ -114,7 +122,6 @@ COPY arbitrator/wasm-libraries arbitrator/wasm-libraries
 COPY arbitrator/jit arbitrator/jit
 COPY arbitrator/stylus arbitrator/stylus
 COPY arbitrator/tools/wasmer arbitrator/tools/wasmer
-COPY espressocrypto espressocrypto
 COPY --from=brotli-wasm-export / target/
 COPY scripts/build-brotli.sh scripts/
 COPY brotli brotli
@@ -125,9 +132,11 @@ RUN apt-get update && \
     apt-get install -y \
     libssl-dev \
     pkg-config \
+    curl \
     perl \
     perl-modules-5.36 \
     libfindbin-libs-perl
+
 RUN NITRO_BUILD_IGNORE_TIMESTAMPS=1 make build-espresso-crypto-lib
 
 FROM scratch AS prover-header-export
@@ -255,6 +264,10 @@ COPY go.mod go.sum ./
 COPY go-ethereum/go.mod go-ethereum/go.sum go-ethereum/
 COPY fastcache/go.mod fastcache/go.sum fastcache/
 COPY bold/go.mod bold/go.sum bold/
+ARG ESPRESSO_NETWORK_GO_VER=0.0.36
+ADD https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$ESPRESSO_NETWORK_GO_VER.tar.gz .
+RUN tar -xzf v${ESPRESSO_NETWORK_GO_VER}.tar.gz && \
+    mv espresso-network-go-${ESPRESSO_NETWORK_GO_VER} espresso-network-go
 RUN go mod download
 COPY . ./
 COPY --from=contracts-builder workspace/contracts/build/ contracts/build/
@@ -281,7 +294,7 @@ ENTRYPOINT [ "/usr/local/bin/fuzz.bash", "FuzzStateTransition", "--binary-path",
 
 FROM debian:bookworm-slim AS nitro-node-slim
 WORKDIR /home/user
-COPY --from=node-builder /workspace/target/lib/libespresso_crypto_helper.so /usr/local/lib/
+COPY --from=node-builder /workspace/target/lib/libespresso_crypto_helper-*.so /usr/local/lib/
 RUN ldconfig
 COPY --from=node-builder /workspace/target/bin/nitro /usr/local/bin/
 COPY --from=node-builder /workspace/target/bin/relay /usr/local/bin/

Dockerfile

@@ -32,20 +32,23 @@ ifneq ($(origin GOLANG_LDFLAGS),undefined)
 endif
 
 UNAME_S := $(shell uname -s)
+UNAME_M := $(shell uname -m)
+
 
 # In Mac OSX, there are a lot of warnings emitted if these environment variables aren't set.
 ifeq ($(UNAME_S), Darwin)
   export MACOSX_DEPLOYMENT_TARGET := $(shell sw_vers -productVersion)
   export CGO_LDFLAGS := -Wl,-no_warn_duplicate_libraries
 endif
 
+
 precompile_names = AddressTable Aggregator BLS Debug FunctionTable GasInfo Info osTest Owner RetryableTx Statistics Sys
 precompiles = $(patsubst %,./solgen/generated/%.go, $(precompile_names))
 
 output_root=target
 output_latest=$(output_root)/machines/latest
 
-repo_dirs = arbos arbcompress arbnode arbutil arbstate cmd das espressocrypto precompiles solgen system_tests util validator wavmio
+repo_dirs = arbos arbcompress arbnode arbutil arbstate cmd das  precompiles solgen system_tests util validator wavmio
 go_source.go = $(wildcard $(patsubst %,%/*.go, $(repo_dirs)) $(patsubst %,%/*/*.go, $(repo_dirs)))
 go_source.s  = $(wildcard $(patsubst %,%/*.s, $(repo_dirs)) $(patsubst %,%/*/*.s, $(repo_dirs)))
 go_source = $(go_source.go) $(go_source.s)
@@ -154,27 +157,76 @@ stylus_test_hostio-test_src       = $(call get_stylus_test_rust,hostio-test)
 
 stylus_test_wasms = $(stylus_test_keccak_wasm) $(stylus_test_keccak-100_wasm) $(stylus_test_fallible_wasm) $(stylus_test_storage_wasm) $(stylus_test_multicall_wasm) $(stylus_test_log_wasm) $(stylus_test_create_wasm) $(stylus_test_math_wasm) $(stylus_test_sdk-storage_wasm) $(stylus_test_erc20_wasm) $(stylus_test_read-return-data_wasm) $(stylus_test_evm-data_wasm) $(stylus_test_hostio-test_wasm) $(stylus_test_bfs:.b=.wasm)
 stylus_benchmarks = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(stylus_test_wasms)
+CBROTLI_WASM_BUILD_ARGS ?=-d
+
+
+ESPRESSO_NETWORK_GO_VER ?= 0.0.36
+ESPRESSO_TAR = espresso-network-go-$(ESPRESSO_NETWORK_GO_VER).tar.gz
+ESPRESSO_URL = https://github.com/EspressoSystems/espresso-network-go/archive/refs/tags/v$(ESPRESSO_NETWORK_GO_VER).tar.gz
+ESPRESSO_DIR = espresso-network-go
 
-espresso_crypto_dir = ./espressocrypto/lib/espresso-crypto-helper
+# Download the tarball
+$(ESPRESSO_TAR):
+	curl -L -o $@ $(ESPRESSO_URL)
+
+# Extract into target directory (strip the top-level folder)
+$(ESPRESSO_DIR): $(ESPRESSO_TAR)
+	@echo "Extracting $(ESPRESSO_TAR) into $(ESPRESSO_DIR)/..."
+	rm -rf $(ESPRESSO_DIR)
+	mkdir -p $(ESPRESSO_DIR)
+	tar -xzf $(ESPRESSO_TAR) --strip-components=1 -C $(ESPRESSO_DIR)
+
+espresso_crypto_dir = $(ESPRESSO_DIR)/verification/rust
 espresso_crypto_files = $(wildcard $(espresso_crypto_dir)/*.toml $(espresso_crypto_dir)/src/*.rs)
 espresso_crypto_lib = $(output_root)/lib/libespresso_crypto_helper
 espresso_crypto_filename = libespresso_crypto_helper.so
-ifeq ($(UNAME_S), Darwin)
+espresso_target_lib = $(ESPRESSO_DIR)/target/lib
+
+
+# Normalize architecture names
+ifeq ($(UNAME_M),arm64)
+    # Apple Silicon reports as arm64, but Rust uses aarch64
+    DETECTED_ARCH := aarch64
+else
+    DETECTED_ARCH := $(UNAME_M)
+endif
+
+# Determine target triple
+ifeq ($(DETECTED_ARCH),aarch64)
+    ifeq ($(UNAME_S),Darwin)
+        TRIPLE := aarch64-apple-darwin
+    else
+        TRIPLE := aarch64-unknown-linux-gnu
+    endif
+else ifeq ($(DETECTED_ARCH),x86_64)
+    ifeq ($(UNAME_S),Darwin)
+        TRIPLE := x86_64-apple-darwin
+    else
+        TRIPLE := x86_64-unknown-linux-gnu
+    endif
+else
+    $(error Architecture $(DETECTED_ARCH) is not supported)
+endif
+
+# Set library extension based on OS
+ifeq ($(UNAME_S),Darwin)
+    LIB_EXT := dylib
 	espresso_crypto_filename = libespresso_crypto_helper.dylib
 else
+    LIB_EXT := so
 	export LD_LIBRARY_PATH := $(shell pwd)/target/lib:$LD_LIBRARY_PATH
 endif
 
-CBROTLI_WASM_BUILD_ARGS ?=-d
 
 # user targets
 .PHONY: build-espresso-crypto-lib
-build-espresso-crypto-lib: $(espresso_crypto_lib)
-
-$(espresso_crypto_lib): $(DEP_PREDICATE) $(espresso_crypto_files)
+build-espresso-crypto-lib: $(ESPRESSO_DIR)
 	mkdir -p `dirname $(espresso_crypto_lib)`
 	cargo build --release --manifest-path $(espresso_crypto_dir)/Cargo.toml
-	install $(espresso_crypto_dir)/target/release/$(espresso_crypto_filename) $(output_root)/lib/$(espresso_crypto_filename)
+	mkdir -p $(espresso_target_lib)
+	install $(espresso_crypto_dir)/target/release/libespresso_crypto_helper.$(LIB_EXT) \
+		$(espresso_target_lib)/libespresso_crypto_helper-$(TRIPLE).$(LIB_EXT)
+	install $(espresso_crypto_dir)/target/release/$(espresso_crypto_filename) $(output_root)/lib/libespresso_crypto_helper-$(TRIPLE).$(LIB_EXT)
 
 .PHONY: push
 push: lint test-go .make/fmt
@@ -308,8 +360,9 @@ clean:
 	@rm -rf contracts/build contracts/cache solgen/go/
 	@rm -f .make/*
 	rm -rf brotli/buildfiles
-	cargo clean --manifest-path $(espresso_crypto_dir)/Cargo.toml
-
+	@rm -f $(output_root)/lib/$(espresso_crypto_filename)
+	rm -f $(ESPRESSO_TAR)
+	rm -rf $(ESPRESSO_DIR)
 # Ensure lib64 is a symlink to lib
 	mkdir -p $(output_root)/lib
 	ln -s lib $(output_root)/lib64

Makefile

@@ -186,7 +186,6 @@ type BatchPosterConfig struct {
 	// Espresso specific flags
 	LightClientAddress          string        `koanf:"light-client-address"`
 	HotShotUrl                  string        `koanf:"hotshot-url"`
-	FallBackUrl                 string        `koanf:"fall-back-url"`
 	UseEscapeHatch              bool          `koanf:"use-escape-hatch"`
 	EspressoTxnsPollingInterval time.Duration `koanf:"espresso-txns-polling-interval"`
 	ResubmitEspressoTxDeadline  time.Duration `koanf:"resubmit-espresso-tx-deadline"`
@@ -388,7 +387,7 @@ func NewBatchPoster(ctx context.Context, opts *BatchPosterOpts) (*BatchPoster, e
 	lightClientAddr := opts.Config().LightClientAddress
 
 	if hotShotUrl != "" {
-		hotShotClient := hotshotClient.NewClient(hotShotUrl, opts.Config().FallBackUrl)
+		hotShotClient := hotshotClient.NewClient(hotShotUrl)
 		opts.Streamer.espressoClient = hotShotClient
 	}
 

arbnode/batch_poster.go

@@ -23,6 +23,7 @@ import (
 	lightclient "github.com/EspressoSystems/espresso-network-go/light-client"
 	tagged_base64 "github.com/EspressoSystems/espresso-network-go/tagged-base64"
 	espressoTypes "github.com/EspressoSystems/espresso-network-go/types"
+	espressoVerification "github.com/EspressoSystems/espresso-network-go/verification"
 	"github.com/ccoveille/go-safecast"
 	flag "github.com/spf13/pflag"
 
@@ -37,7 +38,6 @@ import (
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/broadcaster"
 	m "github.com/offchainlabs/nitro/broadcaster/message"
-	"github.com/offchainlabs/nitro/espressocrypto"
 	"github.com/offchainlabs/nitro/execution"
 	"github.com/offchainlabs/nitro/staker"
 	"github.com/offchainlabs/nitro/util"
@@ -1452,10 +1452,10 @@ func (s *TransactionStreamer) checkSubmittedTransactionForFinality(ctx context.C
 
 	blockMerkleTreeRoot := nextHeader.Header.GetBlockMerkleTreeRoot()
 
-	ok := espressocrypto.VerifyMerkleProof(proof.Proof, jsonHeader, *blockMerkleTreeRoot, snapshot.Root)
-	if !ok {
-		log.Info("error validating merkle proof", "root", snapshot.Root, "height", height)
-		return fmt.Errorf("error validating merkle proof (height: %d, snapshot height: %d)", height, snapshot.Height)
+	ok, err := espressoVerification.VerifyMerkleProof(proof.Proof, jsonHeader, *blockMerkleTreeRoot, snapshot.Root)
+	if err != nil || !ok {
+		log.Error("error validating merkle proof", "root", snapshot.Root, "height", height, "err", err)
+		return fmt.Errorf("error validating merkle proof (height: %d, snapshot height: %d): %w", height, snapshot.Height, err)
 	}
 
 	// Verify the namespace proof
@@ -1464,7 +1464,7 @@ func (s *TransactionStreamer) checkSubmittedTransactionForFinality(ctx context.C
 		return fmt.Errorf("failed to fetch the transactions in block (height: %d): %w", height, err)
 	}
 
-	namespaceOk := espressocrypto.VerifyNamespace(
+	namespaceOk, err := espressoVerification.VerifyNamespace(
 		s.chainConfig.ChainID.Uint64(),
 		resp.Proof,
 		*header.Header.GetPayloadCommitment(),
@@ -1473,8 +1473,9 @@ func (s *TransactionStreamer) checkSubmittedTransactionForFinality(ctx context.C
 		resp.VidCommon,
 	)
 
-	if !namespaceOk {
-		return fmt.Errorf("error validating namespace proof (height: %d)", height)
+	if err != nil || !namespaceOk {
+		log.Error("error validating namespace proof", "root", snapshot.Root, "height", height, "err", err)
+		return fmt.Errorf("error validating namespace proof (height: %d): %w", height, err)
 	}
 
 	submittedPayload := firstSubmitted.Payload