Contract storage layout verification improvements (#2129)

* divide generate and verify tasks

* fix state var validation, using only required files

* fix tests

* add flattening of nested structs

* add storage parser by batches

* refactor structs flattening

* fix build

* remove contract to address rendering

* add dependency imports

* fix contracts references

* use solidity-parser

* migrate previous validations

* add dependencies parsing

* add standalone storage validations

* update dump

* add contract for address replacement

* add import directives with correct aliases

* revert synthetix storage dump

* fix imports duplication

* add new dump format

* update tests

* refactor dump to json

* add enum compatibility

* fix array values on dump

* add storage slot sizes

* add storage slot sizes

* remove storage size

* update yarn.lock

* fix struct size

* refactor tasks

* add layout print

* Add simple validations

* add storage mutation validations

* add variable modifications checks

* add verification errors

* fix CI

* update and execute verification script

* fix dependencies lint errors

* fix lint errors

* remove empty dumps

* fix yarn.lock

* fix yarn dedupe

* add comments

* add comments

* fix storage dump diff check

* build is working much better

still some issues tho

* successful build

* fix

* fix subtask spec

* fix yarn lock

* fix

* fixem

* try storage dump ci work

* move hre back

* fix check test

* update commands

* build everything, not just ts

* more fixes

* fix local quirk

* fix

* ignore whitespace

* add check compile step

* prettier got in the way

* fix more

* add copy storage job to ensure its in sync before commit

* small fix to the copy storage pre commit hook

* use git merge base

---------

Co-authored-by: Daniel Beal <[email protected]>
Co-authored-by: dbeal <[email protected]>

Author: mjlescano

.circleci/config.yml

@@ -346,9 +346,9 @@ jobs:
             # if there are no tests, terminate execution after this step
             if [ -s "/tmp/tests.txt" ]; then
               echo cat "/tmp/tests.txt"
-              cat "/tmp/tests.txt" 
+              cat "/tmp/tests.txt"
             else
-              echo "SKIP: No tests found" 
+              echo "SKIP: No tests found"
               circleci-agent step halt
             fi
 
@@ -428,8 +428,12 @@ jobs:
       - install-foundry
       - yarn-install
       - run: yarn build:ts
-      - run: yarn storage:verify
+      - run: yarn storage:dump
       - run: yarn check:storage
+      # pull the storage dumps from main so that we can compare against them
+      - run: for f in $(find . -name 'storage.dump.json'); do git checkout $(git merge-base HEAD FETCH_HEAD) -- $f || true; done
+      # todo: next step may be removable/optimizable, but it only takes like 20 seconds
+      - run: yarn storage:verify
 
   lint:
     docker:

.gitignore

@@ -18,6 +18,7 @@ typechain-types
 lerna-debug.log
 docs/
 junit
+storage.new.dump.json
 
 # Yarn v3
 .pnp.*

.prettierignore

@@ -27,6 +27,8 @@ lerna.json
 **/tests/.latest.json
 **/lib
 **/out
+storage.dump.json
+storage.new.dump.json
 
 utils/docgen/deployments/
 utils/docgen/abis/

.solhintignore

@@ -1,4 +1,5 @@
 \*.dump.sol
+\*.dump.json
 artifacts/
 node_modules/
 **/contracts/generated
@@ -17,8 +18,6 @@ utils/core-modules/contracts/modules/UpgradeModule.sol
 utils/core-contracts/contracts/ownership/Ownable.sol
 utils/core-contracts/contracts/ownership/OwnableStorage.sol
 
-storage.dump.sol
-
 # Immutable InitialProxy
 markets/legacy-market/contracts/InitialModuleBundle.sol
 markets/perps-market/contracts/modules/CoreModule.sol

.yarn/cache/@types-lodash.memoize-npm-4.1.9-57da73777b-d11efe6049.zip

@@ -8,10 +8,11 @@
   "scripts": {
     "clean": "hardhat clean",
     "build": "yarn build:contracts",
-    "storage:verify": "hardhat compile --force && hardhat storage:verify",
-    "build:contracts": "hardhat compile --force && CANNON_REGISTRY_PRIORITY=local hardhat cannon:build",
     "build-testable": "CANNON_REGISTRY_PRIORITY=local hardhat cannon:build cannonfile.test.toml",
-    "check:storage": "git diff --exit-code storage.dump.sol",
+    "build:contracts": "hardhat compile --force && yarn storage:dump && CANNON_REGISTRY_PRIORITY=local hardhat cannon:build",
+    "storage:dump": "hardhat storage:dump --output storage.new.dump.json",
+    "check:storage": "diff -uw storage.dump.json storage.new.dump.json",
+    "storage:verify": " hardhat storage:verify",
     "test": "CANNON_REGISTRY_PRIORITY=local hardhat test",
     "coverage": "hardhat coverage --network hardhat",
     "compile-contracts": "hardhat compile",

.yarn/cache/cli-table3-npm-0.6.5-c3f24f9c39-8dca71256f.zip

@@ -0,0 +1,66 @@
+{
+  "contracts/ArbGasPriceOracle.sol:ArbGasPriceOracle": {
+    "name": "ArbGasPriceOracle",
+    "kind": "contract",
+    "structs": {
+      "RuntimeParams": [
+        {
+          "type": "uint256",
+          "name": "l1SettleGasUnits",
+          "size": 32,
+          "slot": "0",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "l2SettleGasUnits",
+          "size": 32,
+          "slot": "1",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "l1FlagGasUnits",
+          "size": 32,
+          "slot": "2",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "l2FlagGasUnits",
+          "size": 32,
+          "slot": "3",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "l1LiquidateGasUnits",
+          "size": 32,
+          "slot": "4",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "l2LiquidateGasUnits",
+          "size": 32,
+          "slot": "5",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "numberOfUpdatedFeeds",
+          "size": 32,
+          "slot": "6",
+          "offset": 0
+        },
+        {
+          "type": "uint256",
+          "name": "executionKind",
+          "size": 32,
+          "slot": "7",
+          "offset": 0
+        }
+      ]
+    }
+  }
+}

auxiliary/ArbitrumGasPriceOracle/package.json

@@ -10,10 +10,11 @@
     "coverage": "hardhat coverage --network hardhat",
     "clean": "hardhat clean",
     "build": "yarn build:contracts",
-    "storage:verify": "hardhat compile --force && hardhat storage:verify",
-    "build:contracts": "hardhat compile --force && CANNON_REGISTRY_PRIORITY=local hardhat cannon:build",
+    "build:contracts": "hardhat compile --force && yarn storage:dump && CANNON_REGISTRY_PRIORITY=local hardhat cannon:build",
+    "storage:dump": "hardhat storage:dump --output storage.new.dump.json",
+    "storage:verify": " hardhat storage:verify",
     "build-testable": "CANNON_REGISTRY_PRIORITY=local hardhat cannon:build cannonfile.test.toml",
-    "check:storage": "git diff --exit-code storage.dump.sol",
+    "check:storage": "diff -uw storage.dump.json storage.new.dump.json",
     "compile-contracts": "hardhat compile",
     "size-contracts": "hardhat compile && hardhat size-contracts",
     "publish-contracts": "cannon publish buyback-snx:$(node -p 'require(`./package.json`).version') --chain-id 13370 --quiet --tags $(node -p '/^\\d+\\.\\d+\\.\\d+$/.test(require(`./package.json`).version) ? `latest` : `dev`')",

auxiliary/ArbitrumGasPriceOracle/storage.dump.json

@@ -0,0 +1 @@
+{}