From 57b392234738af89621df6912d8e27eee651c5ea Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Fri, 21 Feb 2025 18:03:43 +0100 Subject: [PATCH 1/3] [ci] Simplify tmpnet monitoring action Incorporate the following into the action to simplify usage: - Configure network shutdown delay to ensure a final metrics scrape - Save tmpnet dir as a github artifact --- .../run-monitored-tmpnet-cmd/action.yml | 8 +++++++ .github/workflows/ci.yml | 22 +++++-------------- tests/fixture/e2e/flags.go | 14 ++++++++---- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/actions/run-monitored-tmpnet-cmd/action.yml b/.github/actions/run-monitored-tmpnet-cmd/action.yml index 3a3bc911efbc..a1e85a2836bf 100644 --- a/.github/actions/run-monitored-tmpnet-cmd/action.yml +++ b/.github/actions/run-monitored-tmpnet-cmd/action.yml @@ -7,6 +7,8 @@ inputs: required: true filter_by_owner: default: '' + artifact_name: + default: 'tmpnet-data' prometheus_username: required: true prometheus_password: @@ -65,9 +67,15 @@ runs: shell: bash run: ${{ inputs.run }} env: + TMPNET_DELAY_NETWORK_SHUTDOWN: true # Ensure shutdown waits for a final metrics scrape GH_REPO: ${{ inputs.repository_owner }}/${{ inputs.repository_name }} GH_WORKFLOW: ${{ inputs.workflow }} GH_RUN_ID: ${{ inputs.run_id }} GH_RUN_NUMBER: ${{ inputs.run_number }} GH_RUN_ATTEMPT: ${{ inputs.run_attempt }} GH_JOB_ID: ${{ inputs.job }} + - name: Upload tmpnet network dir + uses: ./.github/actions/upload-tmpnet-artifact + if: always() + with: + name: ${{ inputs.artifact_name }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d0f6eaa9044..0fd9a786a44a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,17 +62,13 @@ jobs: - name: Run e2e tests uses: ./.github/actions/run-monitored-tmpnet-cmd with: - run: E2E_SERIAL=1 ./scripts/tests.e2e.sh --delay-network-shutdown + run: E2E_SERIAL=1 ./scripts/tests.e2e.sh + artifact_name: e2e-tmpnet-data filter_by_owner: avalanchego-e2e prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} loki_username: ${{ secrets.LOKI_ID || '' }} loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - - name: Upload tmpnet network dir - uses: ./.github/actions/upload-tmpnet-artifact - if: always() - with: - name: e2e-tmpnet-data e2e_existing_network: runs-on: ubuntu-latest steps: @@ -88,16 +84,12 @@ jobs: - name: Run e2e tests with existing network uses: ./.github/actions/run-monitored-tmpnet-cmd with: - run: E2E_SERIAL=1 ./scripts/tests.e2e.existing.sh --delay-network-shutdown + run: E2E_SERIAL=1 ./scripts/tests.e2e.existing.sh + artifact_name: e2e-existing-network-tmpnet-data prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} loki_username: ${{ secrets.LOKI_ID || '' }} loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - - name: Upload tmpnet network dir - uses: ./.github/actions/upload-tmpnet-artifact - if: always() - with: - name: e2e-existing-network-tmpnet-data Upgrade: runs-on: ubuntu-latest steps: @@ -114,15 +106,11 @@ jobs: uses: ./.github/actions/run-monitored-tmpnet-cmd with: run: ./scripts/tests.upgrade.sh + artifact_name: upgrade-tmpnet-data prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} loki_username: ${{ secrets.LOKI_ID || '' }} loki_password: ${{ secrets.LOKI_PASSWORD || '' }} - - name: Upload tmpnet network dir - uses: ./.github/actions/upload-tmpnet-artifact - if: always() - with: - name: upgrade-tmpnet-data Lint: runs-on: ubuntu-latest steps: diff --git a/tests/fixture/e2e/flags.go b/tests/fixture/e2e/flags.go index 629db51ce6f0..ef15d43e3ec7 100644 --- a/tests/fixture/e2e/flags.go +++ b/tests/fixture/e2e/flags.go @@ -10,12 +10,18 @@ import ( "path/filepath" "time" + "github.com/spf13/cast" + "github.com/ava-labs/avalanchego/tests/fixture/tmpnet" ) -// Ensure that this value takes into account the scrape_interval -// defined in scripts/run_prometheus.sh. -const networkShutdownDelay = 12 * time.Second +const ( + // Ensure that this value takes into account the scrape_interval + // defined in scripts/run_prometheus.sh. + networkShutdownDelay = 12 * time.Second + + delayNetworkShutdownEnvName = "TMPNET_DELAY_NETWORK_SHUTDOWN" +) type FlagVars struct { avalancheGoExecPath string @@ -143,7 +149,7 @@ func RegisterFlags() *FlagVars { flag.BoolVar( &vars.delayNetworkShutdown, "delay-network-shutdown", - false, + cast.ToBool(GetEnvWithDefault(delayNetworkShutdownEnvName, "false")), "[optional] whether to delay network shutdown to allow a final metrics scrape.", ) flag.BoolVar( From 027e739641bb740ef8418557cd688c470abb14fd Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sat, 22 Feb 2025 15:58:03 +0100 Subject: [PATCH 2/3] fixup: Reduce simplify the use of the monitoring action --- .github/actions/run-monitored-tmpnet-cmd/action.yml | 13 ++++++++----- .github/workflows/ci.yml | 12 +++++++----- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/actions/run-monitored-tmpnet-cmd/action.yml b/.github/actions/run-monitored-tmpnet-cmd/action.yml index a1e85a2836bf..873ae29fe731 100644 --- a/.github/actions/run-monitored-tmpnet-cmd/action.yml +++ b/.github/actions/run-monitored-tmpnet-cmd/action.yml @@ -3,12 +3,15 @@ description: 'Run the provided command in an environment configured to monitor t inputs: run: - description: "the bash command to run" + description: "the bash script to run e.g. ./scripts/my-script.sh" required: true + run_env: + description: 'a string containing env vars for the command e.g. "MY_VAR1=foo MY_VAR2=bar"' + default: '' filter_by_owner: default: '' - artifact_name: - default: 'tmpnet-data' + artifact_prefix: + default: '' prometheus_username: required: true prometheus_password: @@ -65,7 +68,7 @@ runs: FILTER_BY_OWNER: ${{ inputs.filter_by_owner }} - name: Run command shell: bash - run: ${{ inputs.run }} + run: ${{ inputs.run_env }} ${{ inputs.run }} env: TMPNET_DELAY_NETWORK_SHUTDOWN: true # Ensure shutdown waits for a final metrics scrape GH_REPO: ${{ inputs.repository_owner }}/${{ inputs.repository_name }} @@ -78,4 +81,4 @@ runs: uses: ./.github/actions/upload-tmpnet-artifact if: always() with: - name: ${{ inputs.artifact_name }} + name: ${{ inputs.artifact_prefix }}-tmpnet-data diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fd9a786a44a..a0cf85d94804 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,8 +62,9 @@ jobs: - name: Run e2e tests uses: ./.github/actions/run-monitored-tmpnet-cmd with: - run: E2E_SERIAL=1 ./scripts/tests.e2e.sh - artifact_name: e2e-tmpnet-data + run: ./scripts/tests.e2e.sh + run_env: E2E_SERIAL=1 + artifact_prefix: e2e filter_by_owner: avalanchego-e2e prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} @@ -84,8 +85,9 @@ jobs: - name: Run e2e tests with existing network uses: ./.github/actions/run-monitored-tmpnet-cmd with: - run: E2E_SERIAL=1 ./scripts/tests.e2e.existing.sh - artifact_name: e2e-existing-network-tmpnet-data + run: ./scripts/tests.e2e.existing.sh + run_env: E2E_SERIAL=1 + artifact_prefix: e2e-existing-network prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} loki_username: ${{ secrets.LOKI_ID || '' }} @@ -106,7 +108,7 @@ jobs: uses: ./.github/actions/run-monitored-tmpnet-cmd with: run: ./scripts/tests.upgrade.sh - artifact_name: upgrade-tmpnet-data + artifact_prefix: upgrade prometheus_username: ${{ secrets.PROMETHEUS_ID || '' }} prometheus_password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} loki_username: ${{ secrets.LOKI_ID || '' }} From 21115558444091fdf8c3d025d20ea1d2649f85a2 Mon Sep 17 00:00:00 2001 From: Maru Newby Date: Sat, 22 Feb 2025 15:59:26 +0100 Subject: [PATCH 3/3] fixup: Simplify nix installation with a custom action --- .github/actions/install-nix/action.yml | 17 +++++++++++++++++ .../actions/run-monitored-tmpnet-cmd/action.yml | 3 +++ .github/workflows/ci.yml | 17 +---------------- 3 files changed, 21 insertions(+), 16 deletions(-) create mode 100644 .github/actions/install-nix/action.yml diff --git a/.github/actions/install-nix/action.yml b/.github/actions/install-nix/action.yml new file mode 100644 index 000000000000..49028a397dbb --- /dev/null +++ b/.github/actions/install-nix/action.yml @@ -0,0 +1,17 @@ +name: 'Install nix' +description: 'Install nix and populate the store for the repo flake' + +inputs: + github_token: + description: "github token to authenticate with to avoid being rate-limited" + default: ${{ github.token }} + required: false + +runs: + using: composite + steps: + - uses: cachix/install-nix-action@v30 + with: + github_access_token: ${{ inputs.github_token }} + - run: nix develop --command echo "dependencies installed" + shell: bash diff --git a/.github/actions/run-monitored-tmpnet-cmd/action.yml b/.github/actions/run-monitored-tmpnet-cmd/action.yml index 873ae29fe731..d701a55ab3fc 100644 --- a/.github/actions/run-monitored-tmpnet-cmd/action.yml +++ b/.github/actions/run-monitored-tmpnet-cmd/action.yml @@ -41,6 +41,9 @@ inputs: runs: using: composite steps: + # Ensure promtail and prometheus are available + - name: Install nix + uses: ./.github/actions/install-nix - name: Start prometheus # Only run for the original repo; a forked repo won't have access to the monitoring credentials if: (inputs.prometheus_username != '') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0cf85d94804..2eb054755c8c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -51,11 +51,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project - - uses: cachix/install-nix-action@v30 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - # TODO(marun) Maybe figure out how to use `nix build` somehow i.e. make the default shell double as the default package - - run: nix develop --command echo "dependencies installed" - name: Build AvalancheGo Binary shell: bash run: ./scripts/build.sh -r @@ -75,10 +70,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project - - uses: cachix/install-nix-action@v30 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - run: nix develop --command echo "dependencies installed" - name: Build AvalancheGo Binary shell: bash run: ./scripts/build.sh -r @@ -97,10 +88,6 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project - - uses: cachix/install-nix-action@v30 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - run: nix develop --command echo "dependencies installed" - name: Build AvalancheGo Binary shell: bash run: ./scripts/build.sh @@ -233,9 +220,7 @@ jobs: steps: - uses: actions/checkout@v4 - uses: ./.github/actions/setup-go-for-project - - uses: cachix/install-nix-action@v30 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} + - uses: ./.github/actions/install-nix - run: nix develop --command echo "dependencies installed" - name: Run e2e tests shell: bash