e2e-test.yml

name: e2e-test

on:
  schedule:
    # Every weekday (Mon–Fri) at 02:00 UTC
    - cron: "0 2 * * 1-5"
  workflow_dispatch:
    inputs:
      test_filters:
        description: "Only execute these (space-separated) tests. Can be used in
          conjunction with skip_tests"
        required: false
      skip_tests:
        description:
          "Space-separated list of tests to skip. Can be used in conjunction
          with test_filters"
        required: false
      delay_deprovision_on_failure:
        description: "Delay teardown by 1 hour on failure so you can SSH in to debug"
        type: boolean
        default: false
      instance_type:
        description: "Exoscale compute instance type"
        type: choice
        options:
          - standard.medium
          - standard.large

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always
  TEST_CRATE_DIR: nym-vpn-core/crates/test
  DEFAULT_INSTANCE_TYPE: standard.medium

jobs:
  build:
    name: Build test binaries
    runs-on: arc-linux-latest
    steps:
      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          fetch-depth: 1

      - name: Dotenv Action
        uses: xom9ikk/dotenv@v2.4.0

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_VERSION }}
          targets: x86_64-unknown-linux-gnu

      - name: Install Zig
        uses: mlugg/setup-zig@v1
        with:
          version: 0.13.0

      - name: Install cargo-zigbuild
        run: cargo install --locked cargo-zigbuild

      - name: Install system dependencies
        run: |
          sudo apt-get update && sudo apt-get install -y \
            gcc make pkg-config \
            libdbus-1-dev libmnl-dev libnftnl-dev \
            libssl-dev libpcap-dev

      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          version: "21.12"
          repo-token: ${{ secrets.GITHUB_TOKEN }}

      - name: Restore cargo registry cache
        uses: actions/cache/restore@v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
          key: ${{ runner.os }}-cargo-${{ env.RUST_VERSION }}-${{ hashFiles('nym-vpn-core/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-cargo-${{ env.RUST_VERSION }}-

      - name: Restore cargo build cache
        uses: actions/cache/restore@v5
        with:
          path: nym-vpn-core/target/
          key: ${{ runner.os }}-cargo-build-${{ env.RUST_VERSION }}-${{ hashFiles('nym-vpn-core/Cargo.lock') }}-${{ hashFiles('nym-vpn-core/**/*.rs') }}
          restore-keys: |
            ${{ runner.os }}-cargo-build-${{ env.RUST_VERSION }}-${{ hashFiles('nym-vpn-core/Cargo.lock') }}-
            ${{ runner.os }}-cargo-build-${{ env.RUST_VERSION }}-

      - name: Install Go
        uses: actions/setup-go@v6
        with:
          go-version: ${{ env.GOLANG_VERSION}}
          cache-dependency-path: wireguard/libwg/go.sum

      - name: Build wireguard-go
        run: ./wireguard/build-wireguard-go.sh

      - name: Build nym-vpnc, nym-vpnd and nym-socks5-proxy
        env:
          CFLAGS_x86_64_unknown_linux_gnu: -I/usr/include/x86_64-linux-gnu
          BINDGEN_EXTRA_CLANG_ARGS_x86_64_unknown_linux_gnu: -I/usr/include/x86_64-linux-gnu
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -L /usr/lib/x86_64-linux-gnu
        run: cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 -p nym-vpnc -p nym-vpnd -p nym-socks5-proxy
        working-directory: nym-vpn-core

      - name: Build test-manager, test-runner
        env:
          CFLAGS_x86_64_unknown_linux_gnu: -I/usr/include/x86_64-linux-gnu
          BINDGEN_EXTRA_CLANG_ARGS_x86_64_unknown_linux_gnu: -I/usr/include/x86_64-linux-gnu
          CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_RUSTFLAGS: -L /usr/lib/x86_64-linux-gnu
        run: |
          cargo zigbuild --release --target x86_64-unknown-linux-gnu.2.36 \
            -p test-manager -p test-runner
        working-directory: ${{ env.TEST_CRATE_DIR }}

      - name: Save cargo registry cache
        if: always()
        uses: actions/cache/save@v5
        with:
          path: |
            ~/.cargo/bin/
            ~/.cargo/registry/index/
            ~/.cargo/registry/cache/
            ~/.cargo/git/db/
          key: ${{ runner.os }}-cargo-${{ env.RUST_VERSION }}-${{ hashFiles('nym-vpn-core/Cargo.lock') }}

      - name: Save cargo build cache
        if: always()
        uses: actions/cache/save@v5
        with:
          path: nym-vpn-core/target/
          key: ${{ runner.os }}-cargo-build-${{ env.RUST_VERSION }}-${{ hashFiles('nym-vpn-core/Cargo.lock') }}-${{ hashFiles('nym-vpn-core/**/*.rs') }}

      - name: Save Go modules cache
        if: always()
        uses: actions/cache/save@v5
        with:
          path: |
            ~/go/pkg/mod
            ~/.cache/go-build
          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}

      - name: Collect binaries
        run: |
          mkdir -p /tmp/dist
          cp nym-vpn-core/target/x86_64-unknown-linux-gnu/release/nym-vpnc /tmp/dist/
          cp nym-vpn-core/target/x86_64-unknown-linux-gnu/release/nym-vpnd /tmp/dist/
          cp nym-vpn-core/target/x86_64-unknown-linux-gnu/release/nym-socks5-proxy /tmp/dist/
          cp nym-vpn-core/target/x86_64-unknown-linux-gnu/release/test-manager /tmp/dist/
          cp nym-vpn-core/target/x86_64-unknown-linux-gnu/release/test-runner /tmp/dist/

      - name: Upload binaries
        uses: actions/upload-artifact@v7
        with:
          name: test-binaries
          path: /tmp/dist/
          retention-days: 3

  e2e-test:
    name: Run E2E tests
    needs: build
    runs-on: arc-linux-latest
    env:
      SSH_KEY_PATH: /tmp/exoscale_key
      SSH_OPTS: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o
        ConnectTimeout=10 -o ServerAliveInterval=30 -o ServerAliveCountMax=3"
      VM_USER: ubuntu
      VM_DIST_DIR: /home/ubuntu/dist
      TF_DIR: nym-vpn-core/crates/test/ci
    steps:
      - name: Resolve instance type
        id: instance-type
        run: |
          TYPE="${{ inputs.instance_type || env.DEFAULT_INSTANCE_TYPE }}"
          echo "value=$TYPE" >> "$GITHUB_OUTPUT"

      - name: Checkout repo
        uses: actions/checkout@v6
        with:
          fetch-depth: 1

      - name: Download binaries
        uses: actions/download-artifact@v8
        with:
          name: test-binaries
          path: /tmp/dist

      - name: Make binaries executable
        run: chmod +x /tmp/dist/*

      - name: Setup SSH key
        run: |
          mkdir -p ~/.ssh
          echo "${{ secrets.EXOSCALE_SSH_PRIVATE_KEY }}" > ${{ env.SSH_KEY_PATH }}
          chmod 600 ${{ env.SSH_KEY_PATH }}

      - name: Setup Terraform
        uses: hashicorp/setup-terraform@v4
        with:
          terraform_wrapper: false

      - name: Terraform init
        working-directory: ${{ env.TF_DIR }}
        run: terraform init

      - name: Terraform apply
        working-directory: ${{ env.TF_DIR }}
        env:
          TF_VAR_exoscale_api_key: ${{ secrets.EXOSCALE_API_KEY }}
          TF_VAR_exoscale_api_secret: ${{ secrets.EXOSCALE_API_SECRET }}
          TF_VAR_data_volume_id: ${{ secrets.EXOSCALE_DATA_VOLUME_ID }}
          TF_VAR_template_id: ${{ secrets.EXOSCALE_TEMPLATE_ID }}
          TF_VAR_instance_type: ${{ steps.instance-type.outputs.value }}
          TF_VAR_run_id: ${{ github.run_id }}
        run: terraform apply -auto-approve

      - name: Get VM IP
        id: vm-ip
        working-directory: ${{ env.TF_DIR }}
        run: echo "ip=$(terraform output -raw instance_ip)" >> "$GITHUB_OUTPUT"

      - name: Wait for SSH
        env:
          VM_IP: ${{ steps.vm-ip.outputs.ip }}
        run: |
          echo "Waiting for VM SSH to become available..."
          for i in $(seq 1 30); do
            if ssh ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
              ${{ env.VM_USER }}@${VM_IP} "echo ok" 2>/dev/null; then
              echo "SSH is ready"
              exit 0
            fi
            echo "Attempt $i/30 — waiting 10s..."
            sleep 10
          done
          echo "ERROR: SSH never became available after 30 attempts"
          exit 1

      - name: SCP binaries and test script to VM
        env:
          VM_IP: ${{ steps.vm-ip.outputs.ip }}
        run: |
          ssh ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
            ${{ env.VM_USER }}@${VM_IP} "mkdir -p ${{ env.VM_DIST_DIR }}"

          scp ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
            /tmp/dist/* ${{ env.VM_USER }}@${VM_IP}:${{ env.VM_DIST_DIR }}/

          scp ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
            ${{ env.TEST_CRATE_DIR }}/test-manager/test.sh \
            ${{ env.VM_USER }}@${VM_IP}:${{ env.VM_DIST_DIR }}/test.sh

      - name: Configure test VM
        env:
          VM_IP: ${{ steps.vm-ip.outputs.ip }}
          TEST_HARNESS_MNEMONIC: ${{ secrets.TEST_HARNESS_MAINNET_MNEMONIC }}
        run: |
          ssh ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
            ${{ env.VM_USER }}@${VM_IP} \
            "NYM_TEST_QCOW_IMAGE=/mnt/data/fresh_debian12_cli.qcow2 \
             NYM_TEST_VM_CONFIG=debian12_cli \
             TEST_HARNESS_MNEMONIC='${TEST_HARNESS_MNEMONIC}' \
             TEST_DIST_DIR=${{ env.VM_DIST_DIR }} \
             bash ${{ env.VM_DIST_DIR }}/test.sh configure"

      - name: Run E2E tests
        timeout-minutes: 45
        env:
          VM_IP: ${{ steps.vm-ip.outputs.ip }}
          TEST_HARNESS_MNEMONIC: ${{ secrets.TEST_HARNESS_MAINNET_MNEMONIC }}
          TEST_FILTERS: ${{ inputs.test_filters }}
          SKIP_TESTS: ${{ inputs.skip_tests }}
        # escaping metacharacters so they become literal - to avoid shell injection in GH job input fields
        run: |
          SAFE_FILTERS=$(printf '%q' "$TEST_FILTERS")
          SAFE_SKIP=$(printf '%q' "$SKIP_TESTS")

          ssh ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
            ${{ env.VM_USER }}@${VM_IP} \
            "NYM_TEST_QCOW_IMAGE=/mnt/data/fresh_debian12_cli.qcow2 \
             NYM_TEST_VM_CONFIG=debian12_cli \
             TEST_HARNESS_MNEMONIC='${TEST_HARNESS_MNEMONIC}' \
             TEST_DIST_DIR=${{ env.VM_DIST_DIR }} \
             TEST_REPORT=/tmp/test-report.txt \
             TEST_FILTERS=${SAFE_FILTERS} \
             SKIP_TESTS=${SAFE_SKIP} \
             bash ${{ env.VM_DIST_DIR }}/test.sh run-tests"

      - name: Retrieve test report
        if: always()
        env:
          VM_IP: ${{ steps.vm-ip.outputs.ip }}
        run: |
          scp ${{ env.SSH_OPTS }} -i ${{ env.SSH_KEY_PATH }} \
            ${{ env.VM_USER }}@${VM_IP}:/tmp/test-report.txt /tmp/test-report.txt 2>/dev/null || \
            echo "No test report found"

      - name: Format test report
        if: always()
        run: |
          if [ -f /tmp/test-report.txt ]; then
            /tmp/dist/test-manager format-test-reports /tmp/test-report.txt > /tmp/test-report.html
            cat /tmp/test-report.html >> "$GITHUB_STEP_SUMMARY"
          else
            echo "No test report file to format"
          fi

      - name: Upload test report
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: test-report
          path: |
            /tmp/test-report.txt
            /tmp/test-report.html
          if-no-files-found: warn
          # repo maximum
          retention-days: 20

      - name: Delay deprovision for debugging
        if: >-
          always() && inputs.delay_deprovision_on_failure == true &&
          steps.vm-ip.outputs.ip && job.status == 'failure'
        env:
          VM_IP: ${{ steps.vm-ip.outputs.ip }}
        run: |
          echo "Tests failed, keeping the VM around for 1 hour for debugging..."
          echo "You can SSH into the VM at ${{ env.VM_USER }}@${VM_IP}"
          echo "If you want to trigger VM deprovision earlier, cancel this workflow"
          sleep 3600

      - name: Terraform destroy
        if: always()
        working-directory: ${{ env.TF_DIR }}
        env:
          TF_VAR_exoscale_api_key: ${{ secrets.EXOSCALE_API_KEY }}
          TF_VAR_exoscale_api_secret: ${{ secrets.EXOSCALE_API_SECRET }}
          TF_VAR_data_volume_id: ${{ secrets.EXOSCALE_DATA_VOLUME_ID }}
          TF_VAR_template_id: ${{ secrets.EXOSCALE_TEMPLATE_ID }}
          TF_VAR_instance_type: ${{ steps.instance-type.outputs.value }}
          TF_VAR_run_id: ${{ github.run_id }}
        run: terraform destroy -auto-approve

      - name: Cleanup SSH key
        if: always()
        run: rm -f ${{ env.SSH_KEY_PATH }}