From 2dc25bc1db67a7b68a545ad4174a54efdce53e8f Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Wed, 13 Nov 2024 10:14:40 +0000 Subject: [PATCH 1/9] ci: use k8 self-hosted runners --- .github/workflows/lints.yml | 70 ++++++++++++++++----------- .github/workflows/scheduled_lints.yml | 33 +++++++++---- .github/workflows/tests.yml | 31 ++++++++++-- Makefile.toml | 5 +- engine-tests/src/lib.rs | 1 + engine/src/engine.rs | 2 +- scripts/ci-deps/dind.sh | 5 ++ scripts/ci-deps/native.sh | 11 +++++ 8 files changed, 111 insertions(+), 47 deletions(-) create mode 100755 scripts/ci-deps/dind.sh create mode 100755 scripts/ci-deps/native.sh diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml index 064e64b38..702379b87 100644 --- a/.github/workflows/lints.yml +++ b/.github/workflows/lints.yml @@ -8,49 +8,61 @@ on: name: Lints jobs: - fmt: - name: Format - runs-on: [ self-hosted, light ] + rust: + name: Run ${{ matrix.command }} + runs-on: k8s-infrastructure-native + container: ubuntu:20.04 + strategy: + fail-fast: false + matrix: + command: [ check-fmt, clippy, udeps ] steps: - name: Potential broken submodules fix run: | git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - name: Clone the repository uses: actions/checkout@v4 - - name: Run cargo fmt - run: cargo make check-fmt - clippy: - name: Clippy - runs-on: [ self-hosted, heavy ] - steps: - - name: Potential broken submodules fix - run: | - git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - - name: Clone the repository - uses: actions/checkout@v4 - - run: cargo make build-contracts - - name: Run Contract cargo clippy - run: cargo make clippy - udeps: - name: Udeps - runs-on: [ self-hosted, heavy ] - steps: - - name: Potential broken submodules fix - run: | - git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - - name: Clone the repository - uses: actions/checkout@v4 - - name: Run udeps - run: cargo make udeps + - name: Install dependencies + run: scripts/ci-deps/native.sh + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + toolchain: stable,nightly + override: false + - name: Install cargo-make + run: cargo +stable make -V || cargo +stable install cargo-make + - name: Run ${{ matrix.command }} + run: cargo make ${{ matrix.command }} + contracts: name: Contracts - runs-on: [ self-hosted, light ] + runs-on: k8s-infrastructure-native + container: ubuntu:20.04 steps: - name: Potential broken submodules fix run: | git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - name: Clone the repository uses: actions/checkout@v4 + - name: Install dependencies + run: scripts/ci-deps/native.sh + - name: Setup Node and cache + uses: actions/setup-node@v4 + with: + node-version: 18 + cache: yarn + cache-dependency-path: | + etc/eth-contracts/yarn.lock + etc/tests/uniswap/yarn.lock + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + toolchain: stable + override: false + - name: Install cargo-make + run: cargo +stable make -V || cargo +stable install cargo-make - name: Run yarn lint run: cargo make check-contracts - name: Check committed EvmErc20.bin diff --git a/.github/workflows/scheduled_lints.yml b/.github/workflows/scheduled_lints.yml index c596fafb8..5ca529bba 100644 --- a/.github/workflows/scheduled_lints.yml +++ b/.github/workflows/scheduled_lints.yml @@ -2,6 +2,7 @@ on: schedule: - cron: '0 9 * * 1-5' + name: Scheduled checks jobs: tests: @@ -15,13 +16,10 @@ jobs: - name: Potential broken submodules fix run: | git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - - name: Install dependencies - run: | - curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - - echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list - sudo apt update && sudo apt install -y yarn build-essential pkg-config libclang-dev libssl-dev - name: Clone the repository uses: actions/checkout@v4 + - name: Install dependencies + run: scripts/ci-deps/dind.sh - name: Install Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 with: @@ -41,11 +39,11 @@ jobs: - name: Setup Node and cache uses: actions/setup-node@v4 with: - node-version: 16 - cache: 'yarn' + node-version: 18 + cache: yarn cache-dependency-path: | - etc/eth-contracts - etc/tests/uniswap + etc/eth-contracts/yarn.lock + etc/tests/uniswap/yarn.lock - name: Install cargo-make run: cargo +stable make -V || cargo +stable install cargo-make - name: Build actual neard-sandbox @@ -62,13 +60,28 @@ jobs: checks: name: Run checks - runs-on: [ self-hosted, heavy ] + runs-on: k8s-infrastructure-native + container: ubuntu:20.04 steps: - name: Potential broken submodules fix run: | git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - name: Clone the repository uses: actions/checkout@v4 + - name: Install dependencies + run: scripts/ci-deps/native.sh + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + toolchain: stable,nightly + override: false + - name: Install cargo-make + run: cargo +stable make -V || cargo +stable install cargo-make + - name: Setup Node and cache + uses: actions/setup-node@v4 + with: + node-version: 18 - run: cargo make check - uses: 8398a7/action-slack@v3 if: failure() diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 54f2f36e3..98ccd81d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ name: Tests jobs: test: name: Test suite ${{ matrix.profile }} - runs-on: github-hosted-heavy-runner + runs-on: k8s-infrastructure-dind strategy: fail-fast: false matrix: @@ -21,6 +21,14 @@ jobs: git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - name: Clone the repository uses: actions/checkout@v4 + - name: Install dependencies + run: scripts/ci-deps/dind.sh + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + toolchain: stable + override: false - name: Cargo Cache uses: actions/cache@v4 with: @@ -35,10 +43,10 @@ jobs: uses: actions/setup-node@v4 with: node-version: 18 - cache: 'yarn' + cache: yarn cache-dependency-path: | - etc/eth-contracts - etc/tests/uniswap + etc/eth-contracts/yarn.lock + etc/tests/uniswap/yarn.lock - name: Install dependencies run: cargo +stable make -V || cargo +stable install cargo-make - name: Build main contract @@ -58,7 +66,8 @@ jobs: test_modexp: name: Test modexp suite ${{ matrix.profile }} - runs-on: github-hosted-heavy-runner + runs-on: k8s-infrastructure-native + container: ubuntu:20.04 strategy: fail-fast: false matrix: @@ -69,6 +78,14 @@ jobs: git checkout -f $(git -c user.name=x -c user.email=x@x commit-tree $(git hash-object -t tree /dev/null) < /dev/null) || : - name: Clone the repository uses: actions/checkout@v4 + - name: Install dependencies + run: scripts/ci-deps/native.sh + - name: Install Rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + toolchain: stable + override: false - name: Cargo Cache uses: actions/cache@v4 with: @@ -81,6 +98,10 @@ jobs: key: ${{ matrix.profile }}-cargo-modexp-test - name: Install dependencies run: cargo +stable make -V || cargo +stable install cargo-make + - name: Setup Node and cache + uses: actions/setup-node@v4 + with: + node-version: 18 - name: Test ${{ matrix.profile }} bench-modexp run: cargo make --profile ${{ matrix.profile }} bench-modexp diff --git a/Makefile.toml b/Makefile.toml index a399a8ee6..ad1632194 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -72,8 +72,9 @@ args = [ [tasks.udeps] category = "Check" -env = { "CARGO_MAKE_RUST_CHANNEL" = "nightly", "CARGO_MAKE_CRATE_INSTALLATION_LOCKED" = "true" } -install_crate = { crate_name = "cargo-udeps", binary = "cargo", min_version = "0.1.41", test_arg = ["udeps", "-h"], force = true } +env = { "CARGO_MAKE_CRATE_INSTALLATION_LOCKED" = "true" } +toolchain = "nightly" +install_crate = { crate_name = "cargo-udeps", binary = "cargo", min_version = "0.1.52", test_arg = ["udeps", "-h"], force = true } command = "${CARGO}" args = [ "udeps", diff --git a/engine-tests/src/lib.rs b/engine-tests/src/lib.rs index 62c9e62dc..8b5c87e56 100644 --- a/engine-tests/src/lib.rs +++ b/engine-tests/src/lib.rs @@ -1,3 +1,4 @@ +#![allow(stable_features)] #![feature(lazy_cell)] #![deny(clippy::pedantic, clippy::nursery)] #![allow(clippy::unreadable_literal, clippy::module_name_repetitions)] diff --git a/engine/src/engine.rs b/engine/src/engine.rs index b6e32b0ff..ff2289f7e 100644 --- a/engine/src/engine.rs +++ b/engine/src/engine.rs @@ -393,7 +393,7 @@ impl<'env, I: IO + Copy, E: Env, H: ReadOnlyPromiseHandler> StackExecutorParams< ) -> executor::stack::StackExecutor< 'static, 'a, - executor::stack::MemoryStackState>, + executor::stack::MemoryStackState<'a, 'static, Engine<'env, I, E, M>>, Precompiles<'env, I, E, H>, > { let metadata = executor::stack::StackSubstateMetadata::new(self.gas_limit, CONFIG); diff --git a/scripts/ci-deps/dind.sh b/scripts/ci-deps/dind.sh new file mode 100755 index 000000000..d0e853148 --- /dev/null +++ b/scripts/ci-deps/dind.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list +sudo apt update && sudo apt install -y yarn build-essential pkg-config libclang-dev libssl-dev diff --git a/scripts/ci-deps/native.sh b/scripts/ci-deps/native.sh new file mode 100755 index 000000000..d74b916b0 --- /dev/null +++ b/scripts/ci-deps/native.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +export DEBIAN_FRONTEND=noninteractive + +apt update +apt install -y build-essential pkg-config libclang-dev libssl-dev gnupg curl git +curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - +echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list +apt update +apt install -y yarn +yarn install From abb9c79982c6aee9e35de2c7fe1d23a3cebaa377 Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Wed, 13 Nov 2024 18:09:37 +0000 Subject: [PATCH 2/9] temp: check cache --- .github/workflows/tests.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98ccd81d2..82cee5dd1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,6 +39,11 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.profile }}-cargo-test + - name: Check cache + run: | + ls -la target/ + ls -la target/liquidity-protocol + ls -la target/limit-order-protocol - name: Setup Node and cache uses: actions/setup-node@v4 with: From 79d49cb5de8525f0b5d62b4ecaec1bd3b55b853f Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Wed, 13 Nov 2024 18:28:01 +0000 Subject: [PATCH 3/9] temp: check cache #2 --- .github/workflows/tests.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 82cee5dd1..1ba4e5128 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,8 +38,11 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ + target/liquidity-protocol + target/limit-order-protocol key: ${{ matrix.profile }}-cargo-test - name: Check cache + continue-on-error: true run: | ls -la target/ ls -la target/liquidity-protocol @@ -68,6 +71,12 @@ jobs: esac - name: Tests ${{ matrix.profile }} run: cargo make --profile ${{ matrix.profile }} test-flow + - name: Check cache after tests + continue-on-error: true + run: | + ls -la target/ + ls -la target/liquidity-protocol + ls -la target/limit-order-protocol test_modexp: name: Test modexp suite ${{ matrix.profile }} From 6dae85aacf8a0cf0fbf29b9152a7d7b9d029e433 Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Wed, 13 Nov 2024 19:58:06 +0000 Subject: [PATCH 4/9] temp: check cache #2 --- .github/workflows/tests.yml | 2 -- engine-tests/src/utils/one_inch/mod.rs | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1ba4e5128..f9d4ea6f2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,8 +38,6 @@ jobs: ~/.cargo/registry/cache/ ~/.cargo/git/db/ target/ - target/liquidity-protocol - target/limit-order-protocol key: ${{ matrix.profile }}-cargo-test - name: Check cache continue-on-error: true diff --git a/engine-tests/src/utils/one_inch/mod.rs b/engine-tests/src/utils/one_inch/mod.rs index fdd5ec757..e472325a0 100644 --- a/engine-tests/src/utils/one_inch/mod.rs +++ b/engine-tests/src/utils/one_inch/mod.rs @@ -14,6 +14,8 @@ pub static LIMIT_ORDER_PROTOCOL_PATH: LazyLock = fn download_and_compile_solidity_sources(repo_name: &str) -> PathBuf { let sources_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() .join("target") .join(repo_name); // Contracts not already present, so download and compile them (but only once, even From 2a7a3ea87616e6dbd7028876d5525fd7c6bcc264 Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Wed, 13 Nov 2024 20:10:00 +0000 Subject: [PATCH 5/9] temp: remove check cache --- .github/workflows/tests.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f9d4ea6f2..98ccd81d2 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,12 +39,6 @@ jobs: ~/.cargo/git/db/ target/ key: ${{ matrix.profile }}-cargo-test - - name: Check cache - continue-on-error: true - run: | - ls -la target/ - ls -la target/liquidity-protocol - ls -la target/limit-order-protocol - name: Setup Node and cache uses: actions/setup-node@v4 with: @@ -69,12 +63,6 @@ jobs: esac - name: Tests ${{ matrix.profile }} run: cargo make --profile ${{ matrix.profile }} test-flow - - name: Check cache after tests - continue-on-error: true - run: | - ls -la target/ - ls -la target/liquidity-protocol - ls -la target/limit-order-protocol test_modexp: name: Test modexp suite ${{ matrix.profile }} From 6a2d5a7412562917128efbd39d67ded8cb98454e Mon Sep 17 00:00:00 2001 From: Oleksandr Anyshchenko Date: Thu, 14 Nov 2024 12:54:09 +0000 Subject: [PATCH 6/9] ci: increase percent --- engine-tests/src/tests/promise_results_precompile.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine-tests/src/tests/promise_results_precompile.rs b/engine-tests/src/tests/promise_results_precompile.rs index 8b79f3d82..e423f359b 100644 --- a/engine-tests/src/tests/promise_results_precompile.rs +++ b/engine-tests/src/tests/promise_results_precompile.rs @@ -101,7 +101,7 @@ fn test_promise_result_gas_cost() { assert!( utils::within_x_percent( - 5, + 10, base_cost.as_u64(), costs::PROMISE_RESULT_BASE_COST.as_u64(), ), From 37d575d4d10bd2b71412f25a7abf3c911356edf0 Mon Sep 17 00:00:00 2001 From: kogeler Date: Fri, 22 Nov 2024 10:17:20 +0200 Subject: [PATCH 7/9] migrate to new k8s runners --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 98ccd81d2..ebc0289ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ name: Tests jobs: test: name: Test suite ${{ matrix.profile }} - runs-on: k8s-infrastructure-dind + runs-on: selfhosted-heavy strategy: fail-fast: false matrix: @@ -66,7 +66,7 @@ jobs: test_modexp: name: Test modexp suite ${{ matrix.profile }} - runs-on: k8s-infrastructure-native + runs-on: selfhosted-heavy container: ubuntu:20.04 strategy: fail-fast: false From 9c03c2bb70544ad1811ab4ac84f4e0d9944bc857 Mon Sep 17 00:00:00 2001 From: kogeler Date: Fri, 22 Nov 2024 10:24:53 +0200 Subject: [PATCH 8/9] check runners --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ebc0289ee..522db8365 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,7 @@ on: branches: - master - develop + - ci/aleksuss/k8-runners pull_request: name: Tests From b3255d944231f5fe28959472d5cd0cf31fc426b5 Mon Sep 17 00:00:00 2001 From: kogeler Date: Fri, 22 Nov 2024 10:43:36 +0200 Subject: [PATCH 9/9] change k8s runners --- .github/workflows/lints.yml | 4 ++-- .github/workflows/scheduled_lints.yml | 4 ++-- .github/workflows/tests.yml | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lints.yml b/.github/workflows/lints.yml index 702379b87..c056930c8 100644 --- a/.github/workflows/lints.yml +++ b/.github/workflows/lints.yml @@ -10,7 +10,7 @@ name: Lints jobs: rust: name: Run ${{ matrix.command }} - runs-on: k8s-infrastructure-native + runs-on: selfhosted container: ubuntu:20.04 strategy: fail-fast: false @@ -37,7 +37,7 @@ jobs: contracts: name: Contracts - runs-on: k8s-infrastructure-native + runs-on: selfhosted container: ubuntu:20.04 steps: - name: Potential broken submodules fix diff --git a/.github/workflows/scheduled_lints.yml b/.github/workflows/scheduled_lints.yml index 5a5f6cb6c..88e9433a0 100644 --- a/.github/workflows/scheduled_lints.yml +++ b/.github/workflows/scheduled_lints.yml @@ -7,7 +7,7 @@ name: Scheduled checks jobs: tests: name: Run tests - runs-on: k8s-infrastructure-dind + runs-on: selfhosted strategy: fail-fast: false matrix: @@ -60,7 +60,7 @@ jobs: checks: name: Run checks - runs-on: k8s-infrastructure-native + runs-on: selfhosted container: ubuntu:20.04 steps: - name: Potential broken submodules fix diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 522db8365..ebc0289ee 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,7 +4,6 @@ on: branches: - master - develop - - ci/aleksuss/k8-runners pull_request: name: Tests