Stacktrie getproof (privacy-scaling-explorations#1779)

### Description

- Fix in the stack trie `GetProof` method
- Added description for `TestTransactions`

### Issue Link


privacy-scaling-explorations#1752

### Type of change

- [x ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] This change requires a documentation update
- [ ] Refactor (no updates to logic)

### How Has This Been Tested?

`TestTransactions` and `TestGetProof` in
`zkevm-circuits/geth-utils/gethutil/mpt/witness/gen_witness_transactions_test.go`.

Author: miha-stopar

.github/workflows/geth-utils.yml

@@ -30,7 +30,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
 
     - name: Format
       uses: Jerome1337/[email protected]

.github/workflows/integration.yml

@@ -70,19 +70,9 @@ jobs:
         with:
           override: false
       - name: Setup golang
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: ~1.19
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: ~1.21
       - name: Cargo cache
         uses: actions/cache@v3
         with:

.github/workflows/lints.yml

@@ -38,16 +38,10 @@ jobs:
         with:
           components: rustfmt, clippy
           override: false
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
+      - name: Setup golang
+        uses: actions/setup-go@v4
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: lint-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: ~1.21
       - name: Cargo cache
         uses: actions/cache@v3
         with:

.github/workflows/main-tests.yml

@@ -46,19 +46,9 @@ jobs:
         with:
           override: false
       - name: Setup golang
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
-          go-version: ~1.19
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: ~1.21
       - name: Cargo cache
         uses: actions/cache@v3
         with:
@@ -104,19 +94,9 @@ jobs:
         with:
           override: false
       - name: Setup golang
-        uses: actions/setup-go@v3
-        with:
-          go-version: ~1.19
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
+        uses: actions/setup-go@v4
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: ~1.21
       - name: Cargo cache
         uses: actions/cache@v3
         with:
@@ -155,16 +135,10 @@ jobs:
           override: false
       - name: Add target
         run: rustup target add x86_64-unknown-linux-gnu
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
+      - name: Setup golang
+        uses: actions/setup-go@v4
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          go-version: ~1.21
       - name: Cargo cache
         uses: actions/cache@v3
         with:

.github/workflows/test-features.yml

@@ -36,16 +36,10 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           override: false
-
-      # Go cache for building geth-utils
-      - name: Go cache
-        uses: actions/cache@v3
+      - name: Setup golang
+        uses: actions/setup-go@v4
         with:
-          path: |
-            ~/.cache/go-build
-            ~/go/pkg/mod
-          key: ${{ github.workflow }}-${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-
+          go-version: ~1.21
       - name: Cargo cache
         uses: actions/cache@v3
         with:

geth-utils/gethutil/mpt/trie/stacktrie.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"io"
 	"main/gethutil/mpt/types"
+	"slices"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -230,9 +231,6 @@ func (st *StackTrie) getDiffIndex(key []byte) int {
 // Helper function to that inserts a (key, value) pair into
 // the trie.
 func (st *StackTrie) insert(key, value []byte) {
-	if key[0] == 1 && key[1] == 0 {
-		fmt.Println("d")
-	}
 	switch st.nodeType {
 	case branchNode: /* Branch */
 		idx := int(key[st.keyOffset])
@@ -659,7 +657,6 @@ func (st *StackTrie) UpdateAndGetProofs(db ethdb.KeyValueReader, list types.Deri
 
 func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, error) {
 	k := KeybytesToHex(key)
-	i := 0
 
 	if st.nodeType == emptyNode {
 		return [][]byte{}, nil
@@ -678,13 +675,11 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, er
 	}
 
 	var proof [][]byte
-
 	var nodes []*StackTrie
-
 	c := st
 	isHashed := false
 
-	for i < len(k) {
+	for i := 0; i < len(k); i++ {
 		if c.nodeType == extNode {
 			nodes = append(nodes, c)
 			c = st.children[0]
@@ -701,33 +696,19 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, er
 			isHashed = true
 			c_rlp, error := db.Get(c.val)
 			if error != nil {
-				fmt.Println(error)
 				panic(error)
 			}
-			fmt.Println(c_rlp)
 
 			proof = append(proof, c_rlp)
+			branchChild := st.getNodeFromBranchRLP(c_rlp, k[i])
 
-			for i < len(k)-1 {
-				node := st.getNodeFromBranchRLP(c_rlp, k[i])
-				i += 1
-				fmt.Println(node)
-
-				if len(node) == 1 && node[0] == 128 { // no child at this position
-					break
-				}
-
-				c_rlp, error = db.Get(node)
-				if error != nil {
-					fmt.Println(error)
-					panic(error)
-				}
-				fmt.Println(c_rlp)
-
-				proof = append(proof, c_rlp)
+			// branchChild is of length 1 when there is no child at this position in the branch
+			// (`branchChild = [128]` in this case), but it is also of length 1 when `c_rlp` is a leaf.
+			if len(branchChild) == 1 {
+				// no child at this position - 128 is RLP encoding for nil object
+				break
 			}
-
-			break
+			c.val = branchChild
 		}
 	}
 
@@ -740,7 +721,6 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, er
 		lNodes := len(nodes)
 		for i := lNodes - 1; i >= 0; i-- {
 			node := nodes[i]
-			fmt.Println(node)
 
 			if node.nodeType == leafNode {
 				rlp, error := db.Get(node.val)
@@ -760,11 +740,9 @@ func (st *StackTrie) GetProof(db ethdb.KeyValueReader, key []byte) ([][]byte, er
 			}
 
 		}
-	}
-
-	fmt.Println("----------")
-	for i := 0; i < len(proof); i++ {
-		fmt.Println(proof[i])
+		// The proof is now reversed (only for non-hashed),
+		// let's reverse it back to have the leaf at the bottom:
+		slices.Reverse(proof)
 	}
 
 	return proof, nil

geth-utils/gethutil/mpt/witness/gen_witness_transactions_test.go

@@ -15,8 +15,54 @@ import (
 	"github.com/ethereum/go-ethereum/rlp"
 )
 
-func TestTransactions(t *testing.T) {
-	t.Skip("failing test")
+/*
+TestNonHashedTransactionsStackTrieGetProof inserts 70 transactions into a stacktrie.
+For each of the 70 modifications of the trie it asks for a proof - GetProof is called before
+and after the modification. The transactions in the trie are not hashed and thus GetProof
+does not require to query a database to get the preimages.
+
+When the first transaction is added, a leaf is added to the trie
+(it has index 1 which is used as a key when inserting into a trie, the key is changed by
+calling KeybytesToHex(key) in TryUpdate to [0, 1, 16]).
+
+When the second transaction is added (it has index 2, when inserting into a trie,
+it gets changed to [0, 2, 16]), the extension node E is created with nibble 0 (note that
+both leaves/transactions have the first nibble 0) and the underlying branch B with children
+at positions 1 and 2 (second nibble of the two leaves).
+
+At this point, the proof for the second transaction is (proofC when index = 2):
+[226 16 160 212 52 159 164 192 63 244 122 229 5 208 58 20 16 54 62 169 98 62 238 163 88 174 155 252 118 132 91 148 62 122 23]
+[248 81 128 160 32 244 75 78 180 11 251 73 229 254 70 16 254 170 54 254 200 97 231 24 180 34 220 244 153 76 1 194 23 63 64 224 160 46 141 2 37 188 204 110 232 46 31 230 82 226 213 98 71 18 241 37 90 213 167 221 58 33 36 249 248 180 207 235 47 128 128 128 128 128 128 128 128 128 128 128 128 128 128]
+[248 200 2 131 4 147 224 131 1 226 65 148 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 4 184 100 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 37 160 163 65 77 46 20 3 175 162 34 86 182 41 225 43 90 92 158 249 153 67 193 148 178 63 8 81 26 169 176 56 242 78 160 21 37 82 209 254 5 113 171 235 198 244 52 17 233 162 51 76 151 85 224 28 101 146 160 197 216 253 237 240 187 19 184]
+
+Note that the first proof element is an extension node with nibble 0 = 16 - 16 (see
+the second byte). The third byte (32 = 160 - 128) denotes the length of the subsequent stream
+which represents the hash of the underlying branch.
+The second proof element is the underlying branch. The second byte (81) denotes the length
+of the subsequent RLP stream. The third byte (128) denotes the nil element at position 0.
+Then there are two children at positions 1 (32 244 75...) and 2 (46 141 2...). Note that
+these two 32-long value are the hashes of the two leaves/transactions in the branch.
+The bytes 128 at the end of the branch RLP represents nil objects at positions 3 - 15.
+The last proof element is the second transaction, for example the third nibble (2)
+represents the index of the transaction.
+
+When further 13 transactions are added, the branch B gets populated at positions 3 - 15
+(the position 0 remains nil).
+
+When the 16-th transaction is added (it has index 16, it gets changed to [1, 0, 16]),
+the extension node E is turned into branch B1 which has children at positions 0 and 1.
+At position 0 it has a branch B (which used to be the underlying branch of E1) and
+at position 1 it has a leaf that represents the 16-transaction.
+
+At this point, the proof for the second transaction is (proofC when index = 16):
+[248 81 160 204 44 112 166 132 56 23 211 247 164 233 113 161 247 117 64 34 142 106 19 106 151 213 163 170 185 19 10 144 231 85 229 160 23 243 146 56 210 93 132 177 170 102 160 150 91 57 192 254 50 122 118 157 138 67 46 2 247 8 89 216 105 197 213 36 128 128 128 128 128 128 128 128 128 128 128 128 128 128 128]
+[248 203 16 131 4 147 224 131 1 226 65 148 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 131 1 0 0 184 100 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 38 160 4 18 182 163 225 56 150 243 120 135 140 57 183 97 55 103 25 62 11 37 106 178 135 7 12 58 133 148 112 183 105 66 160 106 208 180 0 175 152 228 224 143 226 84 16 188 253 79 140 234 80 104 143 32 229 83 105 24 251 62 24 122 66 11 59]
+
+The first proof element is a branch with children at position 0 (branch B) and 1 (newly added leaf).
+The second element is the 16-th transaction. For example, the third byte (16) represents
+the transaction index.
+*/
+func TestNonHashedTransactionsStackTrieGetProof(t *testing.T) {
 	txs := make([]*types.Transaction, 70)
 	key, _ := crypto.GenerateKey()
 	signer := types.LatestSigner(params.TestChainConfig)
@@ -41,16 +87,26 @@ func TestTransactions(t *testing.T) {
 	fmt.Println("===")
 }
 
-// No update for each step, just final proof.
-func TestGetProof(t *testing.T) {
-	txs := make([]*types.Transaction, 70)
+/*
+TestHashedTransactionsStackTrieGetProof inserts 2 transactions into a stacktrie,
+the trie is then hashed (DeriveSha call).
+The proof is asked for one of the two transactions. The transactions in the trie are hashed and thus
+GetProof requires to query a database to get the preimages.
+*/
+func TestHashedTransactionsStackTrieGetProof(t *testing.T) {
+	txs := make([]*types.Transaction, 2)
 	key, _ := crypto.GenerateKey()
 	signer := types.LatestSigner(params.TestChainConfig)
 
 	for i := range txs {
 		amount := math.BigPow(2, int64(i))
 		price := big.NewInt(300000)
 		data := make([]byte, 100)
+		data[3] = 3
+		data[4] = 3
+		data[5] = 3
+		data[6] = 3
+		data[7] = 3
 		tx := types.NewTransaction(uint64(i), common.Address{}, amount, 123457, price, data)
 		signedTx, err := types.SignTx(tx, signer, key)
 		if err != nil {

geth-utils/go.mod

@@ -1,10 +1,56 @@
 module main
 
-go 1.16
+go 1.21
 
 require github.com/ethereum/go-ethereum v1.11.5
 
 require golang.org/x/crypto v0.1.0
 
+require (
+	github.com/DataDog/zstd v1.5.2 // indirect
+	github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect
+	github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/btcsuite/btcd/btcec/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cockroachdb/errors v1.9.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
+	github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 // indirect
+	github.com/cockroachdb/redact v1.1.3 // indirect
+	github.com/deckarep/golang-set/v2 v2.1.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect
+	github.com/getsentry/sentry-go v0.18.0 // indirect
+	github.com/go-ole/go-ole v1.2.1 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/gofrs/flock v0.8.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
+	github.com/holiman/uint256 v1.2.0 // indirect
+	github.com/klauspost/compress v1.15.15 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/mattn/go-runewidth v0.0.9 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/prometheus/client_golang v1.14.0 // indirect
+	github.com/prometheus/client_model v0.3.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
+	github.com/prometheus/procfs v0.9.0 // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
+	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
+	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
+	github.com/tklauser/go-sysconf v0.3.5 // indirect
+	github.com/tklauser/numcpus v0.2.2 // indirect
+	golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
+)
+
 // Uncomment for debugging
 // replace github.com/ethereum/go-ethereum => ../../go-ethereum