conformance-ingress.yaml

name: Conformance Ingress (ci-ingress)

# Any change in triggers needs to be reflected in the concurrency group.
on:
  workflow_dispatch:
    inputs:
      PR-number:
        description: "Pull request number."
        required: true
      context-ref:
        description: "Context in which the workflow runs. If PR is from a fork, will be the PR target branch (general case). If PR is NOT from a fork, will be the PR branch itself (this allows committers to test changes to workflows directly from PRs)."
        required: true
      SHA:
        description: "SHA under test (head of the PR branch)."
        required: true
      extra-args:
        description: "[JSON object] Arbitrary arguments passed from the trigger comment via regex capture group. Parse with 'fromJson(inputs.extra-args).argName' in workflow."
        required: false
        default: '{}'
  push:
    branches:
      - main
      - ft/main/**
      - 'renovate/main-**'
    paths-ignore:
      - 'Documentation/**'
      - 'test/**'

# By specifying the access of one of the scopes, all of those that are not
# specified are set to 'none'.
permissions:
  # To read actions state with catchpoint/workflow-telemetry-action
  actions: read
  # To be able to access the repository with actions/checkout
  contents: read
  # To allow retrieving information from the PR API
  pull-requests: read
  # To be able to set commit status
  statuses: write

concurrency:
  # Structure:
  # - Workflow name
  # - Event type
  # - A unique identifier depending on event type:
  #   - schedule: SHA
  #   - workflow_dispatch: PR number
  #
  # This structure ensures a unique concurrency group name is generated for each
  # type of testing, such that re-runs will cancel the previous run.
  group: |
    ${{ github.workflow }}
    ${{ github.event_name }}
    ${{
      (github.event_name == 'push' && github.sha) ||
      (github.event_name == 'workflow_dispatch' && github.event.inputs.PR-number)
    }}
  cancel-in-progress: true

env:
  kind_config: .github/kind-config.yaml
  timeout: 5m

jobs:
  echo-inputs:
    if: ${{ github.event_name == 'workflow_dispatch' }}
    name: Echo Workflow Dispatch Inputs
    runs-on: ubuntu-22.04
    steps:
      - name: Echo Workflow Dispatch Inputs
        run: |
          echo '${{ tojson(inputs) }}'

  commit-status-start:
    name: Commit Status Start
    runs-on: ubuntu-latest
    steps:
      - name: Set initial commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
        with:
          sha: ${{ inputs.SHA || github.sha }}

  ingress-conformance-test:
    name: Ingress Conformance Test
    runs-on: ubuntu-latest
    timeout-minutes: 120
    strategy:
      fail-fast: false
      matrix:
        include:
        - name: Without_XDP
          kube-proxy-replacement: true
          enable-node-port: false
          bpf-lb-acceleration: disabled
          loadbalancer-mode: dedicated
          default-ingress-controller: false
        - name: With_XDP
          kube-proxy-replacement: true
          enable-node-port: false
          bpf-lb-acceleration: native
          loadbalancer-mode: dedicated
          default-ingress-controller: false
        - name: With_Shared_LB
          kube-proxy-replacement: true
          enable-node-port: false
          bpf-lb-acceleration: disabled
          loadbalancer-mode: shared
          default-ingress-controller: false
        - name: With_Default_Ingress_Controller
          kube-proxy-replacement: true
          enable-node-port: false
          bpf-lb-acceleration: disabled
          loadbalancer-mode: dedicated
          default-ingress-controller: true
        - name: Without_KPR
          kube-proxy-replacement: false
          enable-node-port: true
          bpf-lb-acceleration: disabled
          loadbalancer-mode: dedicated
          default-ingress-controller: false

    steps:
      - name: Collect Workflow Telemetry
        uses: catchpoint/workflow-telemetry-action@94c3c3d9567a0205de6da68a76c428ce4e769af1 # v2.0.0
        with:
          comment_on_pr: false

      - name: Checkout context ref (trusted)
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ inputs.context-ref || github.sha }}
          persist-credentials: false

      - name: Set Environment Variables
        uses: ./.github/actions/set-env-variables

      - name: Get Cilium's default values
        id: default_vars
        uses: ./.github/actions/helm-default
        with:
          image-tag: ${{ inputs.SHA }}
          chart-dir: ./untrusted/install/kubernetes/cilium

      - name: Set image tag
        id: vars
        run: |
          echo sha=${{ steps.default_vars.outputs.sha }} >> $GITHUB_OUTPUT

          CILIUM_INSTALL_DEFAULTS="${{ steps.default_vars.outputs.cilium_install_defaults }} \
            --helm-set kubeProxyReplacement=${{ matrix.kube-proxy-replacement }} \
            --helm-set nodePort.enabled=${{ matrix.enable-node-port }} \
            --helm-set=ingressController.enabled=true \
            --helm-set=ingressController.loadbalancerMode=${{ matrix.loadbalancer-mode }} \
            --helm-set=ingressController.default=${{ matrix.default-ingress-controller }} \
            --helm-set=extraConfig.bpf-lb-acceleration=${{ matrix.bpf-lb-acceleration }} \
            --helm-set=l2announcements.enabled=true"

          echo cilium_install_defaults=${CILIUM_INSTALL_DEFAULTS} >> $GITHUB_OUTPUT

      # Warning: since this is a privileged workflow, subsequent workflow job
      # steps must take care not to execute untrusted code.
      - name: Checkout pull request branch (NOT TRUSTED)
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          ref: ${{ steps.vars.outputs.sha }}
          persist-credentials: false
          path: untrusted
          sparse-checkout: |
            install/kubernetes/cilium
            examples

      - name: Create kind cluster
        uses: helm/kind-action@0025e74a8c7512023d06dc019c617aa3cf561fde # v1.10.0
        with:
          version: ${{ env.KIND_VERSION }}
          node_image: ${{ env.KIND_K8S_IMAGE }}
          kubectl_version: ${{ env.KIND_K8S_VERSION }}
          config: ${{ env.kind_config }}
          wait: 0 # The control-plane never becomes ready, since no CNI is present

      - name: Install Cilium CLI
        uses: cilium/cilium-cli@1afb3ff7eb6ace8ab5f8d4d844afe02e2018bb4e # v0.16.13
        with:
          skip-build: ${{ env.CILIUM_CLI_SKIP_BUILD }}
          image-repo: ${{ env.CILIUM_CLI_IMAGE_REPO }}
          image-tag: ${{ env.CILIUM_CLI_VERSION }}

      - name: Checkout ingress-controller-conformance
        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
        with:
          # Use the forked repo with retry mechanism
          # Please refer to https://github.com/kubernetes-sigs/ingress-controller-conformance/pull/101 for more details.
          repository: cilium/ingress-controller-conformance
          path: ingress-controller-conformance
          ref: 6a193b3f73d8b1201a818bb7c8f204059b064857
          persist-credentials: false

      - name: Install Ingress conformance test tool
        timeout-minutes: 10
        run: |
          cd ingress-controller-conformance
          make build

      - name: Wait for images to be available
        timeout-minutes: 30
        shell: bash
        run: |
          for image in cilium-ci operator-generic-ci ; do
            until docker manifest inspect quay.io/${{ env.QUAY_ORGANIZATION_DEV }}/$image:${{ steps.vars.outputs.sha }} &> /dev/null; do sleep 45s; done
          done

      - name: Install Cilium
        id: install-cilium
        run: |
          cilium install ${{ steps.vars.outputs.cilium_install_defaults }}

      - name: Wait for Cilium to be ready
        run: |
          cilium status --wait
          kubectl get pods -n kube-system

      - name: Install Cilium LB IPPool and L2 Announcement Policy
        timeout-minutes: 10
        run: |
          KIND_NET_CIDR=$(docker network inspect kind -f '{{json .IPAM.Config}}' | jq -r '.[] | select(.Subnet | test("^[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+")) | .Subnet')
          echo "KIND_NET_CIDR: $KIND_NET_CIDR"
          LB_CIDR=$(echo ${KIND_NET_CIDR} | sed "s@0.0/16@255.200/28@")
          echo "LB_CIDR: $LB_CIDR"
          echo "Deploying LB-IPAM Pool..."
          cat << EOF > pool.yaml
          apiVersion: "cilium.io/v2alpha1"
          kind: CiliumLoadBalancerIPPool
          metadata:
            name: "pool"
          spec:
            blocks:
              - cidr: "$LB_CIDR"
          EOF
          cat pool.yaml
          kubectl apply -f pool.yaml
          
          echo "Deploying L2-Announcement Policy..."
          cat << 'EOF' > l2policy.yaml
          apiVersion: "cilium.io/v2alpha1"
          kind: CiliumL2AnnouncementPolicy
          metadata:
            name: l2policy
          spec:
            loadBalancerIPs: true
            interfaces:
              - eth0
            nodeSelector:
              matchExpressions:
                - key: node-role.kubernetes.io/control-plane
                  operator: DoesNotExist
          EOF
          cat l2policy.yaml
          kubectl apply -f l2policy.yaml

      - name: Create sample workload
        timeout-minutes: 5
        run: |
          kubectl apply -n default -f https://raw.githubusercontent.com/istio/istio/release-1.11/samples/bookinfo/platform/kube/bookinfo.yaml
          if [ ${{ matrix.default-ingress-controller }} = "true" ]; then
            # remove ingressClassName line from basic-ingress.yaml
            sed -i '/ingressClassName/d' untrusted/examples/kubernetes/servicemesh/basic-ingress.yaml
            kubectl apply -n default -f untrusted/examples/kubernetes/servicemesh/basic-ingress.yaml
            kubectl wait -n default --for=condition=Ready --all pod --timeout=${{ env.timeout }}
          fi

          kubectl apply -n default -f untrusted/examples/kubernetes/servicemesh/basic-ingress.yaml
          kubectl wait -n default --for=condition=Ready --all pod --timeout=${{ env.timeout }}

      - name: Run Sanity check (external)
        timeout-minutes: 5
        run: |
          lb=$(kubectl get ingress basic-ingress -o jsonpath='{.status.loadBalancer.ingress[0].ip}')     
          curl -s -v --connect-timeout 5 --max-time 20 --retry 3 --retry-all-errors --retry-delay 5 --fail -- http://"$lb"

          # By now the service should be up, no need to do the manual retries for the second request
          curl -s -v --connect-timeout 5 --max-time 20 --retry 3 --fail -- http://"$lb"/details/1

      - name: Run Sanity check (internal to NodePort)
        if: ${{ matrix.kube-proxy-replacement == 'true' }}
        timeout-minutes: 5
        run: |
          if [ ${{ matrix.loadbalancer-mode }} = "dedicated" ]; then
            node_port=$(kubectl get svc cilium-ingress-basic-ingress -o jsonpath='{.spec.ports[?(@.port==80)].nodePort}')
          else
            node_port=$(kubectl get -n kube-system svc cilium-ingress -o jsonpath='{.spec.ports[?(@.port==80)].nodePort}')
          fi
          docker exec -i chart-testing-control-plane curl -s -v --connect-timeout 5 --max-time 20 --retry 3 --fail http://localhost:$node_port/details/1 

      - name: Run Sanity check (headless service)
        timeout-minutes: 5
        run: |
          BACKEND_IP=$(kubectl get pod -l app=details -o jsonpath="{.items[*].status.podIP}")
          cat << EOF > ingress-with-headless-service.yaml
          apiVersion: v1
          kind: Endpoints
          metadata:
            name: details-headless
          subsets:
          - addresses:
            - ip: ${BACKEND_IP}
            ports:
            - name: http
              port: 9080
              protocol: TCP
          ---
          apiVersion: v1
          kind: Service
          metadata:
            name: details-headless
          spec:
            ports:
            - name: http
              port: 9080
              protocol: TCP
              targetPort: 9080
            clusterIP: None
            ipFamilies:
            - IPv4
            ipFamilyPolicy: SingleStack
          ---
          apiVersion: discovery.k8s.io/v1
          kind: EndpointSlice
          metadata:
            name: details-headless-endpoint-slice
            labels:
              kubernetes.io/service-name: details-headless-endpoint-slice
          addressType: IPv4
          endpoints:
            - addresses:
                - ${BACKEND_IP}
          ports:
            - name: http
              port: 9080
              protocol: TCP
          ---
          apiVersion: v1
          kind: Service
          metadata:
            name: details-headless-endpoint-slice
          spec:
            ports:
              - name: http
                port: 9082
                protocol: TCP
                targetPort: 9080
            clusterIP: None
            ipFamilies:
              - IPv4
            ipFamilyPolicy: SingleStack
          ---
          apiVersion: networking.k8s.io/v1
          kind: Ingress
          metadata:
            name: basic-ingress-headless
          spec:
            ingressClassName: cilium
            rules:
            - http:
                paths:
                - backend:
                    service:
                      name: details-headless
                      port:
                        number: 9080
                  path: /details/1
                  pathType: Prefix
                - backend:
                    service:
                      name: details-headless-endpoint-slice
                      port:
                        number: 9082
                  path: /details/2
                  pathType: Prefix
          EOF
          kubectl apply -n default -f ingress-with-headless-service.yaml
          until [ -n "$(kubectl get ingress basic-ingress-headless -o jsonpath='{.status.loadBalancer.ingress[0].ip}')" ]; do
            sleep 3
          done
          lb=$(kubectl get ingress basic-ingress-headless -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
          curl -s -v --connect-timeout 2 --max-time 20 --retry 3 --retry-all-errors --retry-delay 3 --fail -- http://"$lb"/details/1      
          curl -s -v --connect-timeout 2 --max-time 20 --retry 3 --retry-all-errors --retry-delay 3 --fail -- http://"$lb"/details/2

      - name: Cleanup Sanity check
        timeout-minutes: 5
        run: |
          # Clean up after sanity check to avoid any conflicts with the conformance test
          kubectl delete -n default -f untrusted/examples/kubernetes/servicemesh/basic-ingress.yaml
          kubectl delete -n default -f ingress-with-headless-service.yaml
          kubectl delete -n default -f https://raw.githubusercontent.com/istio/istio/release-1.11/samples/bookinfo/platform/kube/bookinfo.yaml
          kubectl wait ingress basic-ingress --for=delete
          kubectl wait ingress basic-ingress-headless --for=delete

      - name: Run Ingress conformance test
        timeout-minutes: 30
        run: |
          cd ingress-controller-conformance
          ./ingress-controller-conformance \
            -ingress-class cilium \
            -wait-time-for-ingress-status 60s \
            -wait-time-for-ready 60s \
            -http-client-timeout 60s \
            -enable-http-debug \
            -stop-on-failure

      - name: Post-test information gathering
        if: ${{ !success() && steps.install-cilium.outcome != 'skipped' }}
        run: |
          kubectl get pods --all-namespaces -o wide
          cilium status
          cilium sysdump --output-filename cilium-sysdump-out-${{ matrix.name }}
        shell: bash {0} # Disable default fail-fast behaviour so that all commands run independently

      - name: Upload artifacts
        if: ${{ !success() }}
        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
        with:
          name: cilium-sysdump-out-${{ matrix.name }}
          path: cilium-sysdump-out-*.zip
          retention-days: 5

  commit-status-final:
    if: ${{ always() }}
    name: Commit Status Final
    needs: ingress-conformance-test
    runs-on: ubuntu-latest
    steps:
      - name: Set final commit status
        uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1  
        with:
          sha: ${{ inputs.SHA || github.sha }}
          status: ${{ needs.ingress-conformance-test.result }}