tests-ipsec-upgrade.yaml

name: Cilium IPsec upgrade (ci-ipsec-upgrade)

# Any change in triggers needs to be reflected in the concurrency group.
on:
  workflow_dispatch:
    inputs:
      PR-number:
        description: "Pull request number."
        required: true
      context-ref:
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
        required: true
      SHA:
        description: "SHA under test (head of the PR branch)."
        required: true
      extra-args:
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
        required: false
        default: '{}'
  push:
    branches:
      - 'renovate/main-**'
  # Run every 6 hours
  schedule:
    - cron:  '0 5/6 * * *'

# By specifying the access of one of the scopes, all of those that are not
# specified are set to 'none'.
permissions:
  # To read actions state with catchpoint/workflow-telemetry-action
  actions: read
  # To be able to access the repository with actions/checkout
  contents: read
  # To allow retrieving information from the PR API
  pull-requests: read
  # To be able to set commit status
  statuses: write

concurrency:
  # Structure:
  # - Workflow name
  # - Event type
  # - A unique identifier depending on event type:
  #   - schedule: SHA
  #   - workflow_dispatch: PR number
  #
  # This structure ensures a unique concurrency group name is generated for each
  # type of testing, such that re-runs will cancel the previous run.
  group: |
    ${{ github.workflow }}
    ${{ github.event_name }}
    ${{
      (github.event_name == 'push' && github.sha) ||
      (github.event_name == 'schedule' && github.sha) ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
    }}
  cancel-in-progress: true

env:
  cilium_cli_ci_version:

jobs:
  echo-inputs:
    if: ${{ github.event_name == 'workflow_dispatch' }}
    name: Echo Workflow Dispatch Inputs
    runs-on: ubuntu-22.04
    steps:
      - name: Echo Workflow Dispatch Inputs
        run: |
          echo '${{ tojson(inputs) }}'

  commit-status-start:
    name: Commit Status Start
    runs-on: ubuntu-latest
    steps:
      - name: Set initial commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}

  setup-and-test:
    runs-on: ${{ vars.GH_RUNNER_EXTRA_POWER }}
    name: 'Setup & Test'
    env:
      job_name: 'Setup & Test'
    strategy:
      fail-fast: false
      max-parallel: 16
      matrix:
        config: ['5.4', '5.10', '6.1', '5.15', 'bpf-next']
        mode: ['minor', 'patch']
        include:
          # Define three config sets
          - config: '5.4'
            # renovate: datasource=docker depName=quay.io/lvh-images/kind
            kernel: '5.4-20240710.064909'
            kube-proxy: 'iptables'
            kpr: 'false'
            tunnel: 'disabled'
            encryption: 'ipsec'

          - config: '5.10'
            # renovate: datasource=docker depName=quay.io/lvh-images/kind
            kernel: '5.10-20240710.064909'
            kube-proxy: 'iptables'
            kpr: 'false'
            tunnel: 'disabled'
            encryption: 'ipsec'
            endpoint-routes: 'true'

          - config: '5.15'
            # renovate: datasource=docker depName=quay.io/lvh-images/kind
            kernel: '5.15-20240710.064909'
            kube-proxy: 'iptables'
            kpr: 'false'
            tunnel: 'vxlan'
            encryption: 'ipsec'
            encryption-overlay: 'true'
            endpoint-routes: 'false'

          - config: '6.1'
            # renovate: datasource=docker depName=quay.io/lvh-images/kind
            kernel: '6.1-20240710.064909'
            kube-proxy: 'iptables'
            kpr: 'false'
            tunnel: 'vxlan'
            encryption: 'ipsec'
            endpoint-routes: 'false'

          - config: 'bpf-next'
            # renovate: datasource=docker depName=quay.io/lvh-images/kind
            kernel: 'bpf-next-20240711.013133'
            kube-proxy: 'iptables'
            kpr: 'false'
            tunnel: 'vxlan'
            encryption: 'ipsec'
            endpoint-routes: 'true'

          # Add names to matrix combinations of {config, mode}
          - config: '5.4'
            mode: 'minor'
            name: '1'

          - config: '5.10'
            mode: 'minor'
            name: '2'

          - config: '6.1'
            mode: 'minor'
            name: '3'

          - config: 'bpf-next'
            mode: 'minor'
            name: '4'

          - config: '5.4'
            mode: 'patch'
            name: '5'

          - config: '5.10'
            mode: 'patch'
            name: '6'

          - config: '6.1'
            mode: 'patch'
            name: '7'

          - config: 'bpf-next'
            mode: 'patch'
            name: '8'

          - config: '5.15'
            mode: 'minor'
            name: '9'

          - config: '5.15'
            mode: 'patch'
            name: '10'

    timeout-minutes: 70
    steps:
      - name: Collect Workflow Telemetry
        uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
        with:
          comment_on_pr: false

      - name: Checkout context ref (trusted)
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          # We keep the credentials here, to make sure we're able to run
          # "git fetch" in print-downgrade-version.sh in a few steps below.
          # We'll call it again to remove the credentials before pulling the
          # untrusted branch from the PR. We remain in a trusted context while
          # credentials persist.
          # This remains faster than downloading the full project history to
          # make tags available to print-downgrade-version.sh.
          persist-credentials: true

      - name: Cleanup Disk space in runner
        uses: ./.github/actions/disk-cleanup

      - name: Set Environment Variables
        uses: ./.github/actions/set-env-variables

      - name: Set up job variables
        id: vars
        run: |
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
            SHA="${{ inputs.SHA }}"
          else
            SHA="${{ github.sha }}"
          fi
          echo sha=${SHA} >> $GITHUB_OUTPUT
          if [ "${{ matrix.mode }}" = "minor" ]; then
            CILIUM_DOWNGRADE_VERSION=$(contrib/scripts/print-downgrade-version.sh stable)
            IMAGE_TAG=${CILIUM_DOWNGRADE_VERSION}
          else
            # Upgrade from / downgrade to patch release.
            # In some cases we expect to fail to get the version number, do not
            # fail the workflow in such case. This is typically the case on
            # main branch where we don't have preceeding patch releases.
            CILIUM_DOWNGRADE_VERSION=$(contrib/scripts/print-downgrade-version.sh patch || true)
            # Pass an empty tag to the cilium-config action to fall back to the
            # default release image, without crafting an image path with the
            # "-ci" suffix
            IMAGE_TAG=''
          fi
          echo "CILIUM_DOWNGRADE_VERSION: ${CILIUM_DOWNGRADE_VERSION}"
          echo "IMAGE_TAG: ${IMAGE_TAG}"
          if [ -z "${CILIUM_DOWNGRADE_VERSION}" ]; then
            echo "::notice::No CILIUM_DOWNGRADE_VERSION returned; skipping remaining steps"
          fi
          echo downgrade_version=${CILIUM_DOWNGRADE_VERSION} >> $GITHUB_OUTPUT
          echo image_tag=${IMAGE_TAG} >> $GITHUB_OUTPUT

      - name: Checkout pull request branch (NOT TRUSTED)
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false

      - name: Check we effectively removed Git credentials
        shell: bash
        run: |
          # For private repositories requiring authentication, check that we
          # can no longer fetch from the repository.
          if ! curl -L \
            -H "Accept: application/vnd.github+json" \
            -H "X-GitHub-Api-Version: 2022-11-28" \
            "https://api.github.com/repos/${{ github.repository }}" | \
            tee /dev/stderr | \
            jq --exit-status '.private == false'; then
            echo 'Checking whether "git fetch" succeeds'
            if git fetch origin HEAD; then
              echo "::error::Git credentials not removed, aborting now."
              false
            fi
          fi

      - name: Derive stable Cilium installation config
        id: cilium-stable-config
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/cilium-config
        with:
          image-tag: ${{ steps.vars.outputs.image_tag }}
          chart-dir: './untrusted/cilium-downgrade/install/kubernetes/cilium'
          tunnel: ${{ matrix.tunnel }}
          endpoint-routes: ${{ matrix.endpoint-routes }}
          ipv6: ${{ matrix.ipv6 }}
          kpr: ${{ matrix.kpr }}
          lb-mode: ${{ matrix.lb-mode }}
          lb-acceleration: ${{ matrix.lb-acceleration }}
          encryption: ${{ matrix.encryption }}
          encryption-node: ${{ matrix.encryption-node }}
          encryption-overlay: ${{ matrix.encryption-overlay }}
          egress-gateway: ${{ matrix.egress-gateway }}
          host-fw: ${{ matrix.host-fw }}
          mutual-auth: false
          misc: 'bpfClockProbe=false,cni.uninstall=false'

      - name: Derive newest Cilium installation config
        id: cilium-newest-config
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/cilium-config
        with:
          image-tag: ${{ steps.vars.outputs.sha }}
          chart-dir: './untrusted/cilium-newest/install/kubernetes/cilium'
          tunnel: ${{ matrix.tunnel }}
          endpoint-routes: ${{ matrix.endpoint-routes }}
          ipv6: ${{ matrix.ipv6 }}
          kpr: ${{ matrix.kpr }}
          lb-mode: ${{ matrix.lb-mode }}
          lb-acceleration: ${{ matrix.lb-acceleration }}
          encryption: ${{ matrix.encryption }}
          encryption-node: ${{ matrix.encryption-node }}
          encryption-overlay: ${{ matrix.encryption-overlay }}
          egress-gateway: ${{ matrix.egress-gateway }}
          host-fw: ${{ matrix.host-fw }}
          mutual-auth: false
          misc: 'bpfClockProbe=false,cni.uninstall=false'

      - name: Set Kind params
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        id: kind-params
        shell: bash
        run: |
          IP_FAM="dual"
          if [ "${{ matrix.ipv6 }}" == "false" ]; then
            IP_FAM="ipv4"
          fi
          echo params="\"\" 3 \"\" \"\" ${{ matrix.kube-proxy }} $IP_FAM" >> $GITHUB_OUTPUT

      - name: Provision K8s on LVH VM
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/lvh-kind
        with:
          test-name: e2e-conformance
          kernel: ${{ matrix.kernel }}
          kind-params: "${{ steps.kind-params.outputs.params }}"
          kind-image: ${{ env.KIND_K8S_IMAGE }}

      - name: Install Cilium CLI
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: cilium/cilium-cli@62bd4511031211b50a4623870955a5ad27b43e3b # v0.16.16
        with:
          skip-build: ${{ env.CILIUM_CLI_SKIP_BUILD }}
          image-repo: ${{ env.CILIUM_CLI_IMAGE_REPO }}
          image-tag: ${{ steps.vars.outputs.sha }}

      # Warning: since this is a privileged workflow, subsequent workflow job
      # steps must take care not to execute untrusted code.
      - name: Checkout pull request branch (NOT TRUSTED)
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ steps.vars.outputs.sha }}
          persist-credentials: false
          path: untrusted/cilium-newest
          sparse-checkout: |
            install/kubernetes/cilium

      - name: Checkout ${{ steps.vars.outputs.downgrade_version }} branch to get the Helm chart
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ steps.vars.outputs.downgrade_version }}
          persist-credentials: false
          path: untrusted/cilium-downgrade
          sparse-checkout: |
            install/kubernetes/cilium

      - name: Wait for images to be available
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        timeout-minutes: 30
        shell: bash
        run: |
          for image in cilium-ci operator-generic-ci hubble-relay-ci ; do
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
          done

      - name: Install Cilium ${{ steps.vars.outputs.downgrade_version }} (${{ matrix.name }})
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        shell: bash
        run: |
          kubectl patch node kind-worker3 --type=json -p='[{"op":"add","path":"/metadata/labels/cilium.io~1no-schedule","value":"true"}]'
          kubectl create -n kube-system secret generic cilium-ipsec-keys \
              --from-literal=keys="3+ rfc4106(gcm(aes)) $(echo $(dd if=/dev/urandom count=20 bs=1 2> /dev/null | xxd -p -c 64)) 128"

          mkdir -p cilium-junits

          cilium install \
            ${{ steps.cilium-stable-config.outputs.config }}

          cilium status --wait
          kubectl get pods --all-namespaces -o wide
          kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg status


      - name: Setup conn-disrupt-test before upgrading (${{ matrix.name }})
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/conn-disrupt-test-setup

      - name: Upgrade Cilium (${{ matrix.name }})
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        shell: bash
        run: |
          cilium upgrade \
            ${{ steps.cilium-newest-config.outputs.config }}

          cilium status --wait
          kubectl get pods --all-namespaces -o wide
          kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg status

      - name: Run tests after upgrading (${{ matrix.name }})
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/conn-disrupt-test-check
        with:
          job-name: cilium-upgrade-${{ matrix.name }}
          full-test: 'true'

      - name: Setup conn-disrupt-test before downgrading
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/conn-disrupt-test-setup

      - name: Downgrade Cilium to ${{ steps.vars.outputs.downgrade_version }} (${{ matrix.name }})
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        shell: bash
        run: |
          cilium upgrade \
            ${{ steps.cilium-stable-config.outputs.config }}

          cilium status --wait
          kubectl get pods --all-namespaces -o wide
          kubectl -n kube-system exec daemonset/cilium -c cilium-agent -- cilium-dbg status

      - name: Check conn-disrupt-test after downgrading
        if: ${{ steps.vars.outputs.downgrade_version != '' }}
        uses: ./.github/actions/conn-disrupt-test-check
        with:
          job-name: cilium-downgrade-${{ matrix.name }}
          full-test: 'true'

      - name: Fetch artifacts
        if: ${{ steps.vars.outputs.downgrade_version != '' && !success() }}
        shell: bash
        run: |
          kubectl get pods --all-namespaces -o wide
          cilium status
          mkdir -p cilium-sysdumps
          cilium sysdump --output-filename cilium-sysdump-${{ matrix.name }}-final

      - name: Upload artifacts
        if: ${{ steps.vars.outputs.downgrade_version != '' && !success() }}
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: cilium-sysdumps-${{ matrix.config }}-${{ matrix.mode }}
          path: cilium-sysdump-*.zip

      - name: Upload JUnits [junit]
        if: ${{ steps.vars.outputs.downgrade_version != '' && always() }}
        uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: cilium-junits-${{ matrix.config }}-${{ matrix.mode }}
          path: cilium-junits/*.xml

      - name: Publish Test Results As GitHub Summary
        if: ${{ steps.vars.outputs.downgrade_version != '' && always() }}
        uses: aanm/junit2md@332ebf0fddd34e91b03a832cfafaa826306558f9 # v0.0.3
        with:
          junit-directory: "cilium-junits"

  merge-upload:
    if: ${{ always() }}
    name: Merge and Upload Artifacts
    runs-on: ubuntu-latest
    needs: setup-and-test
    steps:
      - name: Merge Sysdumps
        if: ${{ needs.setup-and-test.result == 'failure' }}
        uses: actions/upload-artifact/merge@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: cilium-sysdumps
          pattern: cilium-sysdumps-*
          retention-days: 5
          delete-merged: true
        continue-on-error: true
      - name: Merge JUnits
        uses: actions/upload-artifact/merge@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
        with:
          name: cilium-junits
          pattern: cilium-junits-*
          retention-days: 5
          delete-merged: true

  commit-status-final:
    if: ${{ always() }}
    name: Commit Status Final
    needs: setup-and-test
    runs-on: ubuntu-latest
    steps:
      - name: Set final commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}
          status: ${{ needs.setup-and-test.result }}