From 87603001fdb2e37377039fa897df2569e1d694c2 Mon Sep 17 00:00:00 2001 From: Sam Heilbron Date: Tue, 3 Dec 2024 10:32:21 -0700 Subject: [PATCH] .github: add v1.18.x to the nightly tests (#10429) --- .github/workflows/docs-gen.yaml | 4 +- .github/workflows/nightly-tests.yaml | 180 ++++++++++++++---- .../workflows/trivy-analysis-scheduled.yaml | 2 +- Makefile | 2 +- .../1-19-main-branch-cleanup.yaml | 7 + docs/Makefile | 2 +- 6 files changed, 150 insertions(+), 47 deletions(-) create mode 100644 changelog/v1.19.0-beta1/1-19-main-branch-cleanup.yaml diff --git a/.github/workflows/docs-gen.yaml b/.github/workflows/docs-gen.yaml index ac4d3b82cf3..6036b36ec05 100644 --- a/.github/workflows/docs-gen.yaml +++ b/.github/workflows/docs-gen.yaml @@ -2,15 +2,15 @@ name: Docs env: SLACK_DEBUG_TESTING: false # when set to "true", send notifications to #slack-integration-testing. Otherwise, post to #edge-team-bots - MIN_SCANNED_VERSION: 'v1.14.0' # ⚠️ you should also change trivy-analysis-scheduled.yaml ⚠️ + MIN_SCANNED_VERSION: 'v1.15.0' # ⚠️ you should also change trivy-analysis-scheduled.yaml ⚠️ on: push: branches: - 'main' + - 'v1.18.x' - 'v1.17.x' - 'v1.16.x' - 'v1.15.x' - - 'v1.14.x' pull_request: types: [opened, synchronize, reopened, ready_for_review] branches: diff --git a/.github/workflows/nightly-tests.yaml b/.github/workflows/nightly-tests.yaml index 5de8a6418f0..a929eea07ce 100644 --- a/.github/workflows/nightly-tests.yaml +++ b/.github/workflows/nightly-tests.yaml @@ -12,9 +12,9 @@ on: # Day of the week ([0,6] with 0=Sunday) schedule: - cron: "0 5 * * *" # every day @ 05:00 UTC, run tests against latest main - - cron: "0 6 * * 1" # monday @ 06:00 UTC, run expanded tests against v1.17.x - - cron: "0 7 * * 1" # monday @ 07:00 UTC, run expanded tests against v1.16.x - - cron: "0 8 * * 1" # monday @ 08:00 UTC, run expanded tests against v1.15.x + - cron: "0 6 * * 1" # monday @ 06:00 UTC, run expanded tests against v1.18.x + - cron: "0 7 * * 1" # monday @ 07:00 UTC, run expanded tests against v1.17.x + - cron: "0 8 * * 1" # monday @ 08:00 UTC, run expanded tests against v1.16.x workflow_dispatch: inputs: branch: @@ -22,9 +22,9 @@ on: type: choice options: - main + - v1.18.x - v1.17.x - v1.16.x - - v1.15.x - workflow_initiating_branch run-regression: description: "Run regression tests" @@ -177,9 +177,71 @@ jobs: matrix-label: ${{ matrix.version-files.label }} # Reminder: when setting up the job next release branch, copy from "end_to_end_tests_main" not the previous release job as configuration may have changed + end_to_end_tests_18: + name: End-to-End (branch=v1.18.x, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} ) + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }} + runs-on: ubuntu-22.04 + timeout-minutes: 180 + strategy: + # Since we are running these on a schedule, there is no value in failing fast + # In fact, we want to ensure that all tests run, so that we have a clearer picture of which tests are prone to flaking + fail-fast: false + matrix: + test: + # When running the tests at night, there is no value in splitting the tests across multiple clusters and running them in parallel. + # As a result, we increase the threshold for the tests, since they all run serially on a single cluster + - cluster-name: 'cluster-one' + go-test-args: '-v -timeout=150m' + # Specifying an empty regex means all tests will be run. + go-test-run-regex: "" + # In our nightly tests, we run the suite of tests using the lower and upper ends of versions that we claim to support + # The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/ + version-files: + - label: 'min' + file: './.github/workflows/.env/nightly-tests/min_versions.env' + - label: 'max' + file: './.github/workflows/.env/nightly-tests/max_versions.env' + + steps: + - uses: actions/checkout@v4 + with: + ref: main + # The dotenv action is used to load key-value pairs from files. + # In this case, the file is specified in the matrix and will contain the versions of the tools to use + - name: Dotenv Action + uses: falti/dotenv-action@v1.1.4 + id: dotenv + with: + path: ${{ matrix.version-files.file }} + log-variables: true + - name: Prep Go Runner + uses: ./.github/workflows/composite-actions/prep-go-runner + # Set up the KinD cluster that the tests will use + - id: setup-kind-cluster + name: Setup KinD Cluster + uses: ./.github/workflows/composite-actions/setup-kind-cluster + with: + cluster-name: ${{ matrix.test.cluster-name }} + kind-node-version: ${{ steps.dotenv.outputs.node_version }} + kind-version: ${{ steps.dotenv.outputs.kind_version }} + kubectl-version: ${{ steps.dotenv.outputs.kubectl_version }} + helm-version: ${{ steps.dotenv.outputs.helm_version }} + istio-version: ${{ steps.dotenv.outputs.istio_version }} + k8sgateway-api-version: ${{ steps.dotenv.outputs.k8sgateway_api_version }} + # Run the tests + - id: run-tests + name: Run Kubernetes e2e Tests + uses: ./.github/workflows/composite-actions/kubernetes-e2e-tests + with: + cluster-name: ${{ matrix.test.cluster-name }} + test-args: ${{ matrix.test.go-test-args }} + run-regex: ${{ matrix.test.go-test-run-regex }} + istio-version: ${{ steps.dotenv.outputs.istio_version }} + matrix-label: ${{ matrix.version-files.label }} + end_to_end_tests_17: name: End-to-End (branch=v1.17.x, cluster=${{ matrix.test.cluster-name }}, version=${{ matrix.version-files.label }} ) - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-kubernetes-end-to-end && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 150 strategy: @@ -286,9 +348,39 @@ jobs: ref: main - uses: ./.github/workflows/composite-actions/regression-tests + regression_tests_18: + name: v1.18.x regression tests + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }} + runs-on: ubuntu-22.04 + timeout-minutes: 60 + # Previously, there was an environment variable, RELEASED_VERSION="LATEST" set. This made use of some internal code: + # https://github.com/solo-io/gloo/blob/main/test/kube2e/util.go#L229-L241 + # which modified our testing process to pull the latest beta release. + # + # NOW, however, running this job is the same as normal CI. (building a local chart, then using it) + strategy: + fail-fast: false + matrix: + # TODO: + # As part of the end_to_end_tests_main job, we added support for importing versions from a .env file + # We should extend the support/usage of those .env files to these other jobs. + # The tests are currently in flux, and some of these regression tests are being migrated, so we decided + # to limit the scope (and potentially unnecessary work) for now + kube-e2e-test-type: [ 'gateway', 'gloo', 'upgrade' ] + kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' }, + { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' } ] + image-variant: + - standard + steps: + - uses: actions/checkout@v4 + with: + ref: v1.18.x + - uses: ./.github/workflows/composite-actions/regression-tests + + regression_tests_17: name: v1.17.x regression tests - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 60 strategy: @@ -306,7 +398,7 @@ jobs: regression_tests_16: name: v1.16.x regression tests - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.16.x') || github.event.schedule == '0 7 * * 1' }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.16.x') || github.event.schedule == '0 8 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 60 strategy: @@ -321,23 +413,6 @@ jobs: ref: v1.16.x - uses: ./.github/workflows/composite-actions/regression-tests - regression_tests_15: - name: v1.15.x regression tests - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-regression && inputs.branch == 'v1.15.x') || github.event.schedule == '0 8 * * 1' }} - runs-on: ubuntu-22.04 - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - kube-e2e-test-type: ['gateway', 'gloo', 'ingress', 'helm', 'gloomtls', 'glooctl', 'upgrade'] - kube-version: [ { node: 'v1.23.13@sha256:ef453bb7c79f0e3caba88d2067d4196f427794086a7d0df8df4f019d5e336b61', kubectl: 'v1.23.17', kind: 'v0.17.0', helm: 'v3.11.2' }, - { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' } ] - steps: - - uses: actions/checkout@v4 - with: - ref: v1.15.x - - uses: ./.github/workflows/composite-actions/regression-tests - performance_tests_on_demand: name: on demand performance tests if: ${{ github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'workflow_initiating_branch' }} @@ -362,39 +437,39 @@ jobs: - uses: ./.github/workflows/composite-actions/prep-go-runner - uses: ./.github/workflows/composite-actions/performance-tests - performance_tests_17: - name: v1.17.x performance tests - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }} + performance_tests_18: + name: v1.18.x performance tests + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 with: - ref: v1.17.x + ref: v1.18.x - uses: ./.github/workflows/composite-actions/prep-go-runner - uses: ./.github/workflows/composite-actions/performance-tests - performance_tests_16: - name: v1.16.x performance tests - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.16.x') || github.event.schedule == '0 7 * * 1' }} + performance_tests_17: + name: v1.17.x performance tests + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 with: - ref: v1.16.x + ref: v1.17.x - uses: ./.github/workflows/composite-actions/prep-go-runner - uses: ./.github/workflows/composite-actions/performance-tests - performance_tests_15: - name: v1.15.x performance tests - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.15.x') || github.event.schedule == '0 8 * * 1' }} + performance_tests_16: + name: v1.16.x performance tests + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-performance && inputs.branch == 'v1.16.x') || github.event.schedule == '0 8 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 60 steps: - uses: actions/checkout@v4 with: - ref: v1.15.x + ref: v1.16.x - uses: ./.github/workflows/composite-actions/prep-go-runner - uses: ./.github/workflows/composite-actions/performance-tests @@ -416,9 +491,28 @@ jobs: ref: main - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests + kube_gateway_api_conformance_tests_18: + name: Conformance (branch=v1.18.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} ) + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.18.x') || github.event.schedule == '0 6 * * 1' }} + runs-on: ubuntu-22.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + kube-version: [ { node: 'v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72', kubectl: 'v1.27.3', kind: 'v0.20.0', helm: 'v3.13.2' }, + { node: 'v1.31.0@sha256:53df588e04085fd41ae12de0c3fe4c72f7013bba32a20e7325357a1ac94ba865', kubectl: 'v1.31.0', kind: 'v0.24.0', helm: 'v3.14.4' }] + image-variant: + - standard + steps: + - uses: actions/checkout@v4 + with: + ref: v1.18.x + - uses: ./.github/workflows/composite-actions/kube-gateway-api-conformance-tests + + kube_gateway_api_conformance_tests_17: name: Conformance (branch=v1.17.x, type=Kubernetes Gateway API, version=${{matrix.kube-version.node}} ) - if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 6 * * 1' }} + if: ${{ (github.event_name == 'workflow_dispatch' && inputs.run-conformance && inputs.branch == 'v1.17.x') || github.event.schedule == '0 7 * * 1' }} runs-on: ubuntu-22.04 timeout-minutes: 60 strategy: @@ -440,16 +534,18 @@ jobs: if: ${{ always() }} needs: - end_to_end_tests_main + - end_to_end_tests_18 - end_to_end_tests_17 - regression_tests_main + - regression_tests_18 - regression_tests_17 - regression_tests_16 - - regression_tests_15 - performance_tests_main + - performance_tests_18 - performance_tests_17 - performance_tests_16 - - performance_tests_15 - kube_gateway_api_conformance_tests_main + - kube_gateway_api_conformance_tests_18 - kube_gateway_api_conformance_tests_17 - end_to_end_tests_on_demand - regression_tests_on_demand @@ -470,13 +566,13 @@ jobs: branch="main" elif [[ ${{github.event.schedule == '0 6 * * 1'}} = true ]]; then trigger="Gloo OSS weeklies" - branch="v1.17.x" + branch="v1.18.x" elif [[ ${{github.event.schedule == '0 7 * * 1'}} = true ]]; then trigger="Gloo OSS weeklies" - branch="v1.16.x" + branch="v1.17.x" elif [[ ${{github.event.schedule == '0 8 * * 1'}} = true ]]; then trigger="Gloo OSS nightlies" - branch="v1.15.x" + branch="v1.16.x" fi preamble="$trigger ($branch)" echo "Setting PREAMBLE as $preamble" diff --git a/.github/workflows/trivy-analysis-scheduled.yaml b/.github/workflows/trivy-analysis-scheduled.yaml index abf7dc8e85f..ddc82ee246c 100644 --- a/.github/workflows/trivy-analysis-scheduled.yaml +++ b/.github/workflows/trivy-analysis-scheduled.yaml @@ -38,7 +38,7 @@ jobs: SCAN_DIR: _output/scans IMAGE_REGISTRY: quay.io/solo-io # ON_LTS_UPDATE - bump version - MIN_SCANNED_VERSION: 'v1.14.0' # ⚠️ you should also change docs-gen.yaml ⚠️ + MIN_SCANNED_VERSION: 'v1.15.0' # ⚠️ you should also change docs-gen.yaml ⚠️ run: | mkdir -p $SCAN_DIR make run-security-scan diff --git a/Makefile b/Makefile index 416e5ed06fb..1652e9a610a 100644 --- a/Makefile +++ b/Makefile @@ -1250,7 +1250,7 @@ SCAN_DIR ?= $(OUTPUT_DIR)/scans SCAN_BUCKET ?= solo-gloo-security-scans # The minimum version to scan with trivy # ON_LTS_UPDATE - bump version -MIN_SCANNED_VERSION ?= v1.14.0 +MIN_SCANNED_VERSION ?= v1.15.0 .PHONY: run-security-scans run-security-scan: diff --git a/changelog/v1.19.0-beta1/1-19-main-branch-cleanup.yaml b/changelog/v1.19.0-beta1/1-19-main-branch-cleanup.yaml new file mode 100644 index 00000000000..861623d5f23 --- /dev/null +++ b/changelog/v1.19.0-beta1/1-19-main-branch-cleanup.yaml @@ -0,0 +1,7 @@ +changelog: + - type: NON_USER_FACING + description: >- + Update the main branch nightly tests after cutting the 1.18.x branch + + skipCI-kube-tests:true + skipCI-docs-build:true diff --git a/docs/Makefile b/docs/Makefile index 0a01a7b1bd2..824fbd6794d 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -11,7 +11,7 @@ HUGO_VERSION := 0.81.0 SOLO_HUGO_THEME_REVISION := v0.0.29 # The minimum version to maintain in our public docs -MIN_SCANNED_VERSION ?= v1.13.0 +MIN_SCANNED_VERSION ?= v1.15.0 #---------------------------------------------------------------------------------- # Docs