update main to v0.0.4 (#126)

* code refactor (#115)

* wip

* refactor eq poly

* wip

* fix tests

* wip

* wip

* minor refactoring

* wip

* simplify proof

* more refactoring

* wip

* wip

* more edits

* wip

* bench polynomials

* wip

* improving polynomial

* fmt

* ping specific version of openmpi

* fix ci

* commit cargo lock

* try fix

* try again

* Update ci.yml

* Update ci.yml

* clean up

* lint

* Zz/fix poseidon witnesses (#118)

* fix poseidon witnesses

* wip

* fix all binaries

* add e2e CI

* use 7950x3d

* fix circuit

* update ci

* mpi chunk size

* clippy

* update branch for scripts (#120)

* Mpi chunk size (#121)

* mpi chunk size

* clippy

* fix a correctness problem

* fix a problem in mpi chunk size

* Fix expander-exec (#122)

* fix expander-exec

* fix

* Update readme.md

---------

Co-authored-by: zhenfei <[email protected]>

* fix mpi (#124)

* fix mpi

* add tests in e2e

* fix

* optimize expander eq

* minor

---------

Signed-off-by: Zhiyong Fang <[email protected]>
Co-authored-by: Zhiyong Fang <[email protected]>
Co-authored-by: mcfx <[email protected]>

Author: 3 people

.github/workflows/benchmarks.yml

@@ -29,7 +29,7 @@ jobs:
       run: cargo install cargo-criterion
     - name: Install mpi
       run: python3 ./scripts/install.py
-    - name: setup
+    - name: Setup
       run: cargo run --bin=dev-setup --release
     - name: Run benchmark
       run: RUSTFLAGS="-C target-cpu=native" cargo criterion --message-format=json > benchmark_results.json

.github/workflows/ci.yml

@@ -34,17 +34,21 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+        with:
+          # The prefix cache key, this can be changed to start a new cache manually.
+          prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
       - name: Set RUSTFLAGS for AVX
         if: matrix.feature != ''
-        run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV
+        run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV  
       - name: Build and Test
         run: |
           python3 ./scripts/install.py
+          mpiexec --version
           cargo run --bin=dev-setup --release
           cargo build --all-features --release
           cargo test --all-features --release --workspace
 
-  benchmark:
+  gkr-e2e:
     name: Benchmark (${{ matrix.os }}${{ matrix.feature != '' && format(', {0}', matrix.feature) || '' }}, ${{ matrix.field }})
     runs-on: ${{ matrix.os }}
     strategy:
@@ -78,6 +82,9 @@ jobs:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
       - uses: Swatinem/rust-cache@v2
+        with:
+          # The prefix cache key, this can be changed to start a new cache manually.
+          prefix-key: "mpi-v5.0.5" # update me if brew formula changes to a new version
       - name: Set RUSTFLAGS for AVX
         if: matrix.feature != ''
         run: echo "RUSTFLAGS=$RUSTFLAGS -C target-feature=+${{ matrix.feature }}" >> $GITHUB_ENV

.github/workflows/nightly_e2e.yml

@@ -0,0 +1,68 @@
+name: E2E Tests
+
+on:
+  schedule:
+    # This will run the workflow every day at 2:00 AM UTC
+    - cron: '0 2 * * *'
+  
+  pull_request:
+    branches:
+    # not to be run on DEV branch to save resources
+      - main
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  build-and-test:
+    runs-on: 7950x3d
+    
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Rust
+      uses: dtolnay/rust-toolchain@stable
+    - uses: Swatinem/rust-cache@v2
+    
+    - name: Set up Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: '^1.16'
+
+    - name: Run tests
+      run: |
+        mkdir -p tmp/data
+        cd tmp
+
+        # ECC part
+        git clone https://github.com/PolyhedraZK/ExpanderCompilerCollection.git
+        cd ExpanderCompilerCollection 
+        git switch dev
+        cargo test --release
+        cp expander_compiler/*.txt ../data
+
+        go run ecgo/examples/poseidon_m31/main.go
+        cp *.txt ../data
+
+        cd ../
+
+        # Expander part
+        git clone https://github.com/PolyhedraZK/Expander.git
+        cd Expander
+        git switch dev
+
+        mkdir data
+        mv ../data/*.txt data/
+        ls -l data
+
+        # Run local tests
+        RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f gf2ext128 -t 16
+        RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f m31ext3 -t 16
+        RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s keccak -f fr -t 16
+        RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" cargo +nightly run --release --bin=gkr -- -s poseidon -f m31ext3 -t 16
+
+        # # Run MPI tests
+        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f gf2ext128
+        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f m31ext3
+        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s keccak -f fr
+        # RUSTFLAGS="-C target-cpu=native -C target-feature=+avx512f" mpiexec -n 2 cargo +nightly run --release --bin=gkr-mpi -- -s poseidon -f m31ext3

.gitignore

@@ -5,7 +5,7 @@ target/
 
 # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
 # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
-Cargo.lock
+# Cargo.lock
 
 # These are backup files generated by rustfmt
 **/*.rs.bk
@@ -30,6 +30,7 @@ flamegraph.svg
 output.txt
 .DS_Store
 *.log
+*.txt
 
 # circuit files
 /data/*
@@ -39,4 +40,4 @@ output.txt
 /data
 
 # vscode settings
-.vscode/
+.vscode/