Merge branch 'stackhpc/2024.1' into add-gitlab-kayobe-automation

jackhodgkiss · web-flow · commit ffaf5d73b810 · 2025-01-27T11:41:53.000Z
diff --git a/.github/path-filters.yml b/.github/path-filters.yml
@@ -11,6 +11,7 @@ aio: &aio
   - '.github/workflows/stackhpc-pull-request.yml'
   - 'etc/kayobe/*.yml'
   - 'etc/kayobe/ansible/configure-aio-resources.yml'
+  - 'etc/kayobe/ansible/stackhpc-openstack-tests.yml'
   - 'etc/kayobe/ansible/growroot.yml'
   - 'etc/kayobe/ansible/requirements.yml'
   - 'etc/kayobe/ansible/scripts/aio-init.sh'
diff --git a/.github/workflows/stackhpc-all-in-one.yml b/.github/workflows/stackhpc-all-in-one.yml
@@ -453,17 +453,17 @@ jobs:
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
-      - name: StackHPC OpenStack tests
-        id: stackhpc-openstack-tests
+      - name: StackHPC Cloud tests
+        id: stackhpc-cloud-tests
         continue-on-error: true
         run: |
-          mkdir -p sot-results
+          mkdir -p sct-results
           docker run -t --rm \
             -v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-            -v $(pwd)/sot-results:/stack/sot-results \
+            -v $(pwd)/sct-results:/stack/sct-results \
             -e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
             $KAYOBE_IMAGE \
-            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-openstack-tests.yml' -e sot_version=${{ inputs.stackhpc_cloud_tests_version }}
+            /stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-cloud-tests.yml' -e sot_version=${{ inputs.stackhpc_cloud_tests_version }}
         env:
           KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
 
@@ -488,19 +488,19 @@ jobs:
           path: |
             diagnostics/
             tempest-artifacts/
-            sot-results/
-        if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
+            sct-results/
+        if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-cloud-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
 
       - name: Fail if any Tempest tests failed
         run: |
           test $(wc -l < tempest-artifacts/failed-tests) -lt 1
 
-      - name: Fail if any StackHPC OpenStack tests failed
+      - name: Fail if any StackHPC Cloud tests failed
         run: |
-          echo "Some StackHPC OpenStack tests failed."
-          echo "See HTML results artifact (sot-results) for details."
+          echo "Some StackHPC Cloud tests failed."
+          echo "See HTML results artifact (sct-results) for details."
           exit 1
-        if: steps.stackhpc-openstack-tests.outcome == 'failure'
+        if: steps.stackhpc-cloud-tests.outcome == 'failure'
 
       - name: Destroy
         run: terraform destroy -auto-approve
diff --git a/doc/source/contributor/testing-ci-automation.rst b/doc/source/contributor/testing-ci-automation.rst
@@ -122,8 +122,8 @@ The workflow performs the following high-level steps:
    subnets, routers, etc.).
 #. If this is an upgrade job, upgrade the cloud under test to the target
    release.
-#. Run Tempest and `StackHPC OpenStack Tests
-   <https://github.com/stackhpc/stackhpc-openstack-tests>`_ to test the cloud.
+#. Run Tempest and `StackHPC Cloud Tests
+   <https://github.com/stackhpc/stackhpc-cloud-tests>`_ to test the cloud.
 #. Collect diagnostic information.
 #. Upload results as an artifact.
 #. Destroy the VM using Terraform.
@@ -345,12 +345,12 @@ The workflow performs the following high-level steps:
    previous OpenStack release is deployed.
 #. Register test resources in the cloud under test (images, flavors, networks,
    subnets, routers, etc.).
-#. Run Tempest and `StackHPC OpenStack Tests
-   <https://github.com/stackhpc/stackhpc-openstack-tests>`__ to test the cloud.
+#. Run Tempest and `StackHPC Cloud Tests
+   <https://github.com/stackhpc/stackhpc-cloud-tests>`__ to test the cloud.
 #. If this is an upgrade job, upgrade the cloud under test to the target
    release.
-#. Run Tempest and `StackHPC OpenStack Tests
-   <https://github.com/stackhpc/stackhpc-openstack-tests>`__ to test the cloud.
+#. Run Tempest and `StackHPC Cloud Tests
+   <https://github.com/stackhpc/stackhpc-cloud-tests>`__ to test the cloud.
 #. Collect diagnostic information.
 #. Upload results as an artifact.
 #. Destroy the VMs using Terraform.
diff --git a/etc/kayobe/ansible/pulp-sync-publish-promote.yml b/etc/kayobe/ansible/pulp-sync-publish-promote.yml
@@ -0,0 +1,16 @@
+---
+- name: "Import pulp-repo-sync"
+  import_playbook: pulp-repo-sync.yml
+
+- name: "Import pulp-repo-publish"
+  import_playbook: pulp-repo-publish.yml
+
+- name: "Import pulp-repo-promote-production"
+  import_playbook: pulp-repo-promote-production.yml
+  when: repo_promote_production | default(true)
+
+- name: "Import pulp-container-sync"
+  import_playbook: pulp-container-sync.yml
+
+- name: "Import pulp-container-publish"
+  import_playbook: pulp-container-publish.yml
diff --git a/etc/kayobe/ansible/stackhpc-cloud-tests.yml b/etc/kayobe/ansible/stackhpc-cloud-tests.yml
@@ -1,33 +1,33 @@
 ---
-- name: Run StackHPC OpenStack tests
+- name: Run StackHPC Cloud tests
   hosts: tempest_runner
   tags:
-    - stackhpc-openstack-tests
+    - stackhpc-cloud-tests
   vars:
-    sot_venv: "{{ virtualenv_path }}/sot-venv"
-    sot_repo: https://github.com/stackhpc/stackhpc-cloud-tests
-    sot_version: main
-    sot_timeout: 30
-    results_path_local: "{{ lookup('env', 'HOME') }}/sot-results"
+    sct_venv: "{{ virtualenv_path }}/sct-venv"
+    sct_repo: https://github.com/stackhpc/stackhpc-cloud-tests
+    sct_version: main
+    sct_timeout: 30
+    results_path_local: "{{ lookup('env', 'HOME') }}/sct-results"
   tasks:
-    - name: Stackhpc OpenStack tests
+    - name: Stackhpc Cloud tests
       block:
         - name: Create a temporary directory for tests repo
           ansible.builtin.tempfile:
             state: directory
-            suffix: sot-repo
+            suffix: sct-repo
           register: repo_tmpdir
 
         - name: Create a temporary directory for results
           ansible.builtin.tempfile:
             state: directory
-            suffix: sot-results
+            suffix: sct-results
           register: results_tmpdir
 
-        - name: Clone the StackHPC OpenStack tests repository
+        - name: Clone the StackHPC Cloud tests repository
           ansible.builtin.git:
-            repo: "{{ sot_repo }}"
-            version: "{{ sot_version }}"
+            repo: "{{ sct_repo }}"
+            version: "{{ sct_version }}"
             dest: "{{ repo_tmpdir.path }}"
             depth: 1
             single_branch: true
@@ -36,7 +36,7 @@
           ansible.builtin.pip:
             name: "{{ item.name }}"
             state: latest
-            virtualenv: "{{ sot_venv }}"
+            virtualenv: "{{ sct_venv }}"
             virtualenv_command: python3 -m venv
           with_items:
             - { name: pip }
@@ -48,47 +48,48 @@
               - "{{ repo_tmpdir.path }}"
               - pytest-html
               - pytest-timeout
-            virtualenv: "{{ sot_venv }}"
+            virtualenv: "{{ sct_venv }}"
 
         - name: Ensure Python requirements file packages are installed
           ansible.builtin.pip:
             requirements: "{{ repo_tmpdir.path }}/requirements.txt"
-            virtualenv: "{{ sot_venv }}"
+            virtualenv: "{{ sct_venv }}"
 
         - name: Include Kolla Ansible passwords
           ansible.builtin.include_vars:
             file: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
             name: kolla_passwords
 
-        - name: Run StackHPC OpenStack tests
+        - name: Run StackHPC Cloud tests
           ansible.builtin.command:
             cmd: >
-              {{ sot_venv }}/bin/py.test
-              --html={{ results_tmpdir.path }}/stackhpc-openstack-tests.html
+              {{ sct_venv }}/bin/py.test
+              --html={{ results_tmpdir.path }}/stackhpc-cloud-tests.html
               --self-contained-html
               --pyargs stackhpc_cloud_tests
-              --timeout {{ sot_timeout }}
+              --timeout {{ sct_timeout }}
+              -rfEx
               -vv
           environment:
-            OPENSEARCH_HOSTS: "{{ sot_opensearch_hosts }}"
-            OPENSEARCH_PORT: "{{ sot_opensearch_port }}"
-            OPENSEARCH_TLS: "{{ sot_opensearch_tls }}"
-            PROMETHEUS_URL: "{{ sot_prometheus_url }}"
-            PROMETHEUS_USERNAME: "{{ sot_prometheus_username }}"
-            PROMETHEUS_PASSWORD: "{{ sot_prometheus_password }}"
+            OPENSEARCH_HOSTS: "{{ sct_opensearch_hosts }}"
+            OPENSEARCH_PORT: "{{ sct_opensearch_port }}"
+            OPENSEARCH_TLS: "{{ sct_opensearch_tls }}"
+            PROMETHEUS_URL: "{{ sct_prometheus_url }}"
+            PROMETHEUS_USERNAME: "{{ sct_prometheus_username }}"
+            PROMETHEUS_PASSWORD: "{{ sct_prometheus_password }}"
           vars:
             kolla_external_scheme: "{{ 'https' if kolla_enable_tls_external | bool else 'http' }}"
             kolla_internal_scheme: "{{ 'https' if kolla_enable_tls_internal | bool else 'http' }}"
-            sot_opensearch_hosts: "{{ kolla_internal_fqdn }}"
-            sot_opensearch_port: 9200
-            sot_opensearch_tls: false
-            sot_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
-            sot_prometheus_username: admin
-            sot_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
+            sct_opensearch_hosts: "{{ kolla_internal_fqdn }}"
+            sct_opensearch_port: 9200
+            sct_opensearch_tls: false
+            sct_prometheus_url: "{{ kolla_internal_scheme }}://{{ kolla_internal_fqdn }}:9091"
+            sct_prometheus_username: admin
+            sct_prometheus_password: "{{ kolla_passwords.prometheus_password }}"
       always:
         - name: Fetch results
           ansible.builtin.fetch:
-            src: "{{ results_tmpdir.path }}/stackhpc-openstack-tests.html"
+            src: "{{ results_tmpdir.path }}/stackhpc-cloud-tests.html"
             dest: "{{ results_path_local }}/"
             flat: true
 
diff --git a/etc/kayobe/environments/ci-builder/stackhpc-ci.yml b/etc/kayobe/environments/ci-builder/stackhpc-ci.yml
@@ -22,8 +22,9 @@ kolla_enable_letsencrypt: true
 kolla_enable_magnum: true
 kolla_enable_manila: true
 kolla_enable_mariabackup: true
-kolla_enable_neutron_sriov: true
+kolla_enable_neutron_bgp_dragent: true
 kolla_enable_neutron_mlnx: true
+kolla_enable_neutron_sriov: true
 kolla_enable_octavia: true
 kolla_enable_opensearch: true
 kolla_enable_prometheus: true
diff --git a/etc/kayobe/ipa.yml b/etc/kayobe/ipa.yml
@@ -126,7 +126,7 @@ image_download_unredirected_headers:
 #ipa_collectors_default:
 
 # List of additional inspection collectors to run.
-ipa_collectors_extra: "{{ ['dmi-decode', 'extra-hardware', 'numa-topology'] if ipa_build_images else [] }}"
+ipa_collectors_extra: "{{ ['dmi-decode', 'extra-hardware', 'numa-topology'] if ipa_build_images or stackhpc_ipa_image_overcloud_enabled else [] }}"
 
 # List of inspection collectors to run.
 #ipa_collectors:
diff --git a/etc/kayobe/kolla-image-tags.yml b/etc/kayobe/kolla-image-tags.yml
@@ -6,3 +6,6 @@ kolla_image_tags:
   openstack:
     rocky-9: 2024.1-rocky-9-20241218T141751
     ubuntu-jammy: 2024.1-ubuntu-jammy-20241218T141809
+  magnum:
+    rocky-9: 2024.1-rocky-9-20250102T094625
+    ubuntu-jammy: 2024.1-ubuntu-jammy-20250102T094625
diff --git a/etc/kayobe/kolla.yml b/etc/kayobe/kolla.yml
@@ -342,12 +342,6 @@ kolla_build_blocks:
         && grafana-cli plugins install grafana-opensearch-datasource
   ironic_inspector_header: |
     ADD additions-archive /
-  magnum_base_footer: |
-    RUN curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | head -n -1 | bash
-    {% raw %}
-    {% set magnum_capi_packages = ['magnum-capi-helm==1.1.0'] %}
-    RUN {{ macros.install_pip(magnum_capi_packages | customizable("pip_packages")) }}
-    {% endraw %}
 
 # Dict mapping image customization variable names to their values.
 # Each variable takes the form:
@@ -360,6 +354,9 @@ kolla_build_customizations_common:
     - /additions/*
   ironic_inspector_pip_packages_append:
     - /additions/*
+  magnum_base_pip_packages_override:
+    - /magnum[osprofiler]
+    - magnum-capi-helm==1.2.0
   neutron_server_packages_append:
     - python3-libvirt
     - python3-ethtool
diff --git a/etc/kayobe/kolla/globals.yml b/etc/kayobe/kolla/globals.yml
@@ -22,13 +22,6 @@ kolla_image_tags:
 # Variables defining which tag to use for each container's image.
 {{ lookup('pipe', 'python3 ' ~ kayobe_config_path ~ '/../../tools/kolla-images.py list-tag-vars') }}
 
-# FIXME: Pin to Antelope Magnum until it is working again upstream
-kayobe_image_tags:
-  magnum:
-    rocky: 2023.1-rocky-9-20240821T102442
-    ubuntu: 2023.1-ubuntu-jammy-20240821T102442
-
-magnum_tag: "{% raw %}{{ kayobe_image_tags['magnum'][kolla_base_distro] }}{% endraw %}"
 #############################################################################
 # Monitoring and alerting related settings
 
diff --git a/releasenotes/notes/enable-build-of-neutron-dragent-14897d9744c7de1f.yaml b/releasenotes/notes/enable-build-of-neutron-dragent-14897d9744c7de1f.yaml
@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Enable building of the ``Neutron BGP Dragent`` container image.
diff --git a/releasenotes/notes/fix-ipa-collectors-f7afc852d5005282.yaml b/releasenotes/notes/fix-ipa-collectors-f7afc852d5005282.yaml
@@ -0,0 +1,5 @@
+---
+fixes:
+  - |
+    Restores the use of extra IPA collectors (including ``extra-hardware``)
+    when using StackHPC IPA images.
diff --git a/releasenotes/notes/pulp-sync-publish-promote-playbook-b638be7c77a649dd.yaml b/releasenotes/notes/pulp-sync-publish-promote-playbook-b638be7c77a649dd.yaml
@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Added a new playbook `pulp_sync_publish_promote` that can be used to sync,
+    publish and promote all repositories in a single step, as well as sync and
+    publish container repos. If you do not want to promote repos then run with
+    ``-e repo_promote_production=false``.
diff --git a/releasenotes/notes/update-magnum-containers-8814b1425ec45fc0.yaml b/releasenotes/notes/update-magnum-containers-8814b1425ec45fc0.yaml
@@ -0,0 +1,5 @@
+fixes:
+  - |
+    Updates Magnum to a proper Caracal release version following critical bug
+    upstream merged & backported. Also, updates CAPI Helm driver version to the
+    latest (1.2.0).
