diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c89c43c918..c7ca5d4635 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,15 +1,20 @@ # Default code owners -* @pablintino @lewisdenny +* @pablintino @lewisdenny @evallesp @bshewale @frenzyfriday # Adoption roles/adoption_osp_deploy @openstack-k8s-operators/adoption-core-reviewers +# BGP +roles/ci_gen_kustomize_values/templates/bgp @openstack-k8s-operators/bgp +roles/ci_gen_kustomize_values/templates/bgp_dt01 @openstack-k8s-operators/bgp + # Compliance roles/compliance @openstack-k8s-operators/security # DCN roles/ci_dcn_site @openstack-k8s-operators/dcn playbooks/dcn.yml @openstack-k8s-operators/dcn +roles/ci_gen_kustomize_values/templates/dcn @openstack-k8s-operators/dcn # Must gather roles/os_must_gather @openstack-k8s-operators/must-gather @@ -18,7 +23,11 @@ roles/os_must_gather @openstack-k8s-operators/must-gather roles/idrac_configuration @openstack-k8s-operators/nfv roles/switch_config @openstack-k8s-operators/nfv roles/tofu @eshulman2 @openstack-k8s-operators/nfv - +roles/ci_gen_kustomize_values/templates/nfv-ovs-dpdk-sriov-hci @openstack-k8s-operators/nfv +roles/ci_gen_kustomize_values/templates/ovs-dpdk @openstack-k8s-operators/nfv +roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov @openstack-k8s-operators/nfv +roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets @openstack-k8s-operators/nfv +roles/ci_gen_kustomize_values/templates/sriov @openstack-k8s-operators/nfv # Polarion roles/polarion @tosky @ciecierski @jparoly @jirimacku @@ -28,6 +37,7 @@ roles/reportportal @jirimacku @dsariel @sdatko # Shiftstack roles/shiftstack @rlobillo @eurijon +roles/ci_gen_kustomize_values/templates/shiftstack @rlobillo @eurijon # Storage roles/cifmw_block_device @openstack-k8s-operators/storage @@ -42,6 +52,9 @@ roles/nat64_appliance @hjensas roles/ci_lvms_storage @fultonj @fmount @abays roles/ci_local_storage @fultonj @fmount @abays +# Uni +roles/ci_gen_kustomize_values/templates/uni* @openstack-k8s-operators/ciops + # Update roles/update @openstack-k8s-operators/updates diff --git a/.github/workflows/auto-draft.yml b/.github/workflows/auto-draft.yml deleted file mode 100644 index 2c2730d849..0000000000 --- a/.github/workflows/auto-draft.yml +++ /dev/null @@ -1,35 +0,0 @@ -# Marks all newly opened and reopened pull requests as drafts -# and adds a helpful comment to the PR - -# Run only on opened and reopened pull request events -name: Set to draft status when PR is [re]opened -on: - pull_request_target: - types: [opened, reopened] - -jobs: - mark-as-draft: - runs-on: ubuntu-latest - permissions: # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions - contents: write # Required for setting draft state - pull-requests: write # Required for posting comment - steps: - # Use the gh cli tool to set the PR into draft state if opened without it - # and add a helpful comment - - name: Set to draft and add comment - continue-on-error: true # We should not block if this action fails for some reason - if: | - github.event.pull_request.draft == false && - github.actor != 'dependabot[bot]' && - github.actor != 'renovate[bot]' - shell: bash - run: | - if ! [[ "$TITLE" =~ ^nit:.* ]]; then - gh pr ready "$URL" --undo - printf "$MESSAGE" | gh pr comment "$URL" -F - - fi - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - URL: ${{ github.event.pull_request.html_url }} - TITLE: ${{ github.event.pull_request.title }} - MESSAGE: "Thanks for the PR! :heart:\nI'm marking it as a draft, once your happy with it merging and the PR is passing CI, click the \"Ready for review\" button below." diff --git a/OWNERS b/OWNERS index 419226ed75..2a0a8a60c4 100644 --- a/OWNERS +++ b/OWNERS @@ -1,12 +1,12 @@ approvers: - abays + - bshewale - cescgina + - evallesp + - frenzyfriday - fultonj - lewisdenny - pablintino - - raukadah - - rlandy - - viroel reviewers: - adrianfusco diff --git a/ci/playbooks/kuttl/run-kuttl-tests.yml b/ci/playbooks/kuttl/run-kuttl-tests.yml index 97f781b622..e874ed9f2e 100644 --- a/ci/playbooks/kuttl/run-kuttl-tests.yml +++ b/ci/playbooks/kuttl/run-kuttl-tests.yml @@ -52,6 +52,8 @@ ansible.builtin.include_role: name: 'install_yamls_makes' tasks_from: 'make_{{ operator }}_kuttl.yml' + apply: + ignore_errors: true # noqa: ignore-errors - name: 'Get resource status after {{ operator }}_kuttl run' environment: @@ -61,3 +63,41 @@ {{ item }} > {{ cifmw_artifacts_basedir }}/logs/cmd_after_{{ operator }}_kuttl.log loop: "{{ commands_after_kuttl_run }}" ignore_errors: true + +# Some of the xml files may not be the JUnitXML results, but getting the exact file +# name (parsing the KUTTL configuration) may be too cumbersome. +- name: Find the generated JUnitXML files + ansible.builtin.find: + paths: "{{ cifmw_installyamls_repos }}" + file_type: file + patterns: '*.xml' + register: "_cifmw_kuttl_xml_files" + +- name: Copy the log results of {{ operator }} + vars: + _kuttl_test_result_dir: "{{ cifmw_artifacts_basedir }}/tests/kuttl_{{ operator }}" + block: + - name: Create the test results directory + ansible.builtin.file: + path: "{{ _kuttl_test_result_dir }}" + state: directory + mode: '0755' + + - name: Copy the generated test results to the test results directory + ansible.builtin.copy: + src: "{{ item.path }}" + dest: "{{ _kuttl_test_result_dir }}" + mode: '0644' + loop: "{{ _cifmw_kuttl_xml_files.files }}" + ignore_errors: true # noqa: ignore-errors + +# Note: the variable checked here is set by the specific autogenerated task +# for the autogenerated install_yamls_makes role. +- name: Fail when any tests failed + vars: + _kuttl_curr_operator_result: "{{ lookup('vars', 'make_' + operator + '_kuttl_status')|default(1) }}" + ansible.builtin.assert: + that: + - _kuttl_curr_operator_result.rc == 0 + success_msg: "KUTTL tests for {{ operator }} passed" + fail_msg: "KUTTL tests for {{ operator }} failed" diff --git a/ci/playbooks/multinode-autohold.yml b/ci/playbooks/multinode-autohold.yml index db5a6ecc4b..cce54b32bf 100644 --- a/ci/playbooks/multinode-autohold.yml +++ b/ci/playbooks/multinode-autohold.yml @@ -20,10 +20,16 @@ block: - name: Fetch existing autoholds from zuul vars: + _zuul_host: >- + {{ + zuul.executor.hostname + if (zuul.executor.hostname is ansible.utils.ip) else + (zuul.executor.hostname | split('.'))[1:] | join('.') + }} _zuul_api_url: >- {{ [ - ('https://'+ (zuul.executor.hostname | split('.'))[1:] | join('.')), + ('https://'+ _zuul_host), 'zuul', 'api', 'tenant', @@ -32,7 +38,7 @@ ] | join('/') }} ansible.builtin.uri: - url: "{{ _zuul_api_url }}" + url: "{{ zuul_autohold_endpoint | default(_zuul_api_url) }}" method: GET headers: Content-Type: "application/json" diff --git a/common-requirements.txt b/common-requirements.txt index 68e6bc2003..b612f7a926 100644 --- a/common-requirements.txt +++ b/common-requirements.txt @@ -1,9 +1,7 @@ -ansible-core>=2.14.0 -oauthlib>=3.2.0 # k8s lib that requires manual upgrade -kubernetes -kubernetes-validate -openstacksdk -jsonschema>=4.20.0 - -# Allows to unpin cryptography -pyOpenSSL>=22.1.0 +ansible-core==2.15.13 +oauthlib==3.2.2 +kubernetes==31.0.0 +kubernetes-validate==1.31.0 +openstacksdk==4.1.0 +jsonschema==4.23.0 +pyOpenSSL==24.2.1 diff --git a/docs/dictionary/en-custom.txt b/docs/dictionary/en-custom.txt index 783c236922..c4751c2a6d 100644 --- a/docs/dictionary/en-custom.txt +++ b/docs/dictionary/en-custom.txt @@ -1,6 +1,7 @@ aaabbcc abcdefghij addr +afuscoar alertmanager ansible ansibleee @@ -105,6 +106,8 @@ ctl ctlplane ctrl ctx +cve +customizations dashboard dataplane dataplanedeployments @@ -164,6 +167,7 @@ fbqufbqkfbzxrja fci fedoraproject fil +filesystem fips firewalld flbxutz @@ -173,6 +177,7 @@ freefonts frmo fsid fultonj +fusco fwcybtb gapped genericcloud @@ -295,6 +300,7 @@ manpage mawxlihjizaogicbjyxbzig mawxlihjizcbwb maxdepth +mcs mellanox metallb metalsmith @@ -302,6 +308,7 @@ mgmt mins minsizegigabytes mlnx +mls modprobe mountpoints mtcylje @@ -463,7 +470,11 @@ scansettingbinding scap scp sdk +selevel selinux +serole +setype +seuser sha shiftstack shiftstackclient @@ -472,6 +483,7 @@ sizepercent skbg skiplist specificities +spnego spxzvbhvtzxmsihbyb src sshkey diff --git a/docs/source/baremetal/01_baremetal_hosts_data.md b/docs/source/baremetal/01_baremetal_hosts_data.md index 2999ac7cf2..ba8e25cc89 100644 --- a/docs/source/baremetal/01_baremetal_hosts_data.md +++ b/docs/source/baremetal/01_baremetal_hosts_data.md @@ -53,6 +53,8 @@ cifmw_baremetal_hosts: nmstate: # interfaces: # Sample nmstate state snippet # - name: enp6s0f0.161 + # Optional: (string) app label + label: openstack compute-1: # Another BM host diff --git a/docs/source/reproducers/03-zuul.md b/docs/source/reproducers/03-zuul.md index 69d9bdf7e0..acda886c63 100644 --- a/docs/source/reproducers/03-zuul.md +++ b/docs/source/reproducers/03-zuul.md @@ -1,5 +1,9 @@ # Zuul job +~~~{warning} +This feature isn't tested nor used by the CI Framework team. This feature will be supported and rewritten in our new version of the tool. +~~~ + ~~~{tip} It is strongly advised to run this reproducer against a dedicated hypervisor with enough resources. The current configuration will require close to 40G diff --git a/galaxy.yml b/galaxy.yml index bd43289486..e7b205fb48 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -71,20 +71,20 @@ tags: # L(specifiers,https://python-semanticversion.readthedocs.io/en/latest/#requirement-specification). Multiple version # range specifiers can be set and are separated by ',' dependencies: - 'git+https://github.com/containers/ansible-podman-collections': 'main' - 'git+https://github.com/ansible-collections/community.general': 'main' - 'git+https://github.com/ansible-collections/ansible.posix': 'main' - 'git+https://github.com/ansible-collections/ansible.utils': 'main' - 'git+https://github.com/ansible-collections/community.libvirt': 'main' - 'git+https://github.com/ansible-collections/community.crypto': 'main' - 'git+https://github.com/ansible-collections/kubernetes.core': 'main' - 'git+https://github.com/ansible-collections/ansible.netcommon': 'main' - 'git+https://github.com/openstack/ansible-config_template': 'master' - 'git+https://github.com/ansible-collections/junipernetworks.junos': 'main' - 'git+https://github.com/ansible-collections/cisco.ios': 'main' - 'git+https://github.com/ansible-collections/mellanox.onyx': 'master' - 'git+https://github.com/openshift/community.okd': 'main' - 'git+https://github.com/ovirt/ovirt-ansible-collection': 'master' + 'git+https://github.com/containers/ansible-podman-collections': '1.16.2' + 'git+https://github.com/ansible-collections/community.general': '10.0.1' + 'git+https://github.com/ansible-collections/ansible.posix': '1.6.2' + 'git+https://github.com/ansible-collections/ansible.utils': 'v5.1.2' + 'git+https://github.com/ansible-collections/community.libvirt': '1.3.0' + 'git+https://github.com/ansible-collections/community.crypto': '2.22.3' + 'git+https://github.com/ansible-collections/kubernetes.core': '5.0.0' + 'git+https://github.com/ansible-collections/ansible.netcommon': 'v7.1.0' + 'git+https://github.com/openstack/ansible-config_template': '2.1.1' + 'git+https://github.com/ansible-collections/junipernetworks.junos': 'v9.1.0' + 'git+https://github.com/ansible-collections/cisco.ios': 'v9.0.3' + 'git+https://github.com/ansible-collections/mellanox.onyx': 'master' # Unmaintained, no tags + 'git+https://github.com/openshift/community.okd': '4.0.0' + 'git+https://github.com/ovirt/ovirt-ansible-collection': '3.2.0-1' # The URL of the originating SCM repository repository: https://github.com/openstack-k8s-operators/ci-framework diff --git a/hooks/playbooks/adoption_ironic_post_oc.yml b/hooks/playbooks/adoption_ironic_post_oc.yml new file mode 100644 index 0000000000..aa286ff2c6 --- /dev/null +++ b/hooks/playbooks/adoption_ironic_post_oc.yml @@ -0,0 +1,174 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: OSP 17 - Ironic post overcloud + hosts: "{{ cifmw_target_host | default('localhost') }}" + tasks: + - name: Gather ansible_user_dir from OSP nodes + delegate_to: "{{ item }}" + ansible.builtin.setup: + gather_subset: + - user_dir + loop: + - controller-0 + - osp-undercloud-0 + - osp-controller-0 + - osp-controller-1 + - osp-controller-2 + + - name: Install ironic-python-agent images package + block: + - name: Install the ironic-python-agent images package + become: true + delegate_to: "{{ item }}" + ansible.builtin.dnf: + name: rhosp-director-images-ipa-x86_64 + state: present + loop: + - osp-undercloud-0 + - osp-controller-0 + - osp-controller-1 + - osp-controller-2 + + - name: Create ironic-python-agent directory + delegate_to: "{{ item }}" + ansible.builtin.file: + state: directory + path: "{{ ansible_user_dir }}/ironic-python-agent" + loop: + - osp-undercloud-0 + - osp-controller-0 + - osp-controller-1 + - osp-controller-2 + + - name: Extract ironic-python-images from tar + delegate_to: "{{ item }}" + ansible.builtin.unarchive: + src: /usr/share/rhosp-director-images/ironic-python-agent-latest.tar + dest: "{{ ansible_user_dir }}/ironic-python-agent" + remote_src: true + loop: + - osp-undercloud-0 + - osp-controller-0 + - osp-controller-1 + - osp-controller-2 + + - name: Copy ironic-python-agent to /var/lib/ironic/httpboot + become: true + block: + - name: Copy ironic-python-agent kernel + delegate_to: "{{ item }}" + ansible.builtin.copy: + src: "{{ ansible_user_dir }}/ironic-python-agent/ironic-python-agent.kernel" + dest: /var/lib/ironic/httpboot/agent.kernel + remote_src: true + loop: + - osp-controller-0 + - osp-controller-1 + - osp-controller-2 + + - name: Copy ironic-python-agent ramdisk + delegate_to: "{{ item }}" + ansible.builtin.copy: + src: "{{ ansible_user_dir }}/ironic-python-agent/ironic-python-agent.initramfs" + dest: /var/lib/ironic/httpboot/agent.ramdisk + remote_src: true + loop: + - osp-controller-0 + - osp-controller-1 + - osp-controller-2 + + - name: Glance images - ironic-python-agent + delegate_to: osp-undercloud-0 + block: + - name: Upload ironic-python-agent kernel to glance + environment: + OS_CLOUD: "{{ cifmw_adoption_osp_deploy_scenario.stacks[0].stackname }}" + ansible.builtin.shell: + cmd: >- + openstack image show deploy-kernel &>/dev/null || \ + openstack image create deploy-kernel \ + --public \ + --container-format aki \ + --disk-format aki \ + --file ironic-python-agent.kernel + args: + chdir: "{{ ansible_user_dir }}/ironic-python-agent" + + - name: Upload ironic-python-agent ramdisk to glance + environment: + OS_CLOUD: "{{ cifmw_adoption_osp_deploy_scenario.stacks[0].stackname }}" + ansible.builtin.shell: + cmd: >- + openstack image show deploy-ramdisk &>/dev/null || \ + openstack image create deploy-ramdisk \ + --public \ + --container-format ari \ + --disk-format ari \ + --file ironic-python-agent.initramfs + args: + chdir: "{{ ansible_user_dir }}/ironic-python-agent" + + - name: Create ironic provisioning network + delegate_to: osp-undercloud-0 + environment: + OS_CLOUD: "{{ cifmw_adoption_osp_deploy_scenario.stacks[0].stackname }}" + block: + - name: Create network + ansible.builtin.shell: + cmd: >- + openstack network show provisioning &>/dev/null || \ + openstack network create provisioning \ + --share \ + --provider-physical-network baremetal \ + --provider-network-type flat + + - name: Create subnet + ansible.builtin.shell: + cmd: >- + openstack subnet show provisioning-subnet &>/dev/null || \ + openstack subnet create provisioning-subnet \ + --network provisioning \ + --subnet-range 172.20.1.0/24 \ + --gateway 172.20.1.1 \ + --allocation-pool start=172.20.1.100,end=172.20.1.199 + + - name: Slurp ironic_nodes.yaml from controller-0 + delegate_to: controller-0 + register: _ironic_nodes_slurp + ansible.builtin.slurp: + src: "{{ ansible_user_dir }}/ci-framework-data/parameters/ironic_nodes.yaml" + + - name: Enroll ironic nodes + delegate_to: osp-undercloud-0 + block: + - name: Ensure directory exists + ansible.builtin.file: + state: directory + path: "{{ ansible_user_dir }}/ci-framework-data/parameters" + + - name: Write ironic_nodes.yaml on osp-unercloud-o + ansible.builtin.copy: + content: "{{ _ironic_nodes_slurp.content | b64decode }}" + dest: "{{ ansible_user_dir }}/ci-framework-data/parameters/ironic_nodes.yaml" + + - name: Run baremetal create command to enroll the nodes in the Ironic service + environment: + OS_CLOUD: "{{ cifmw_adoption_osp_deploy_scenario.stacks[0].stackname }}" + ansible.builtin.command: >- + openstack baremetal create ironic_nodes.yaml + args: + chdir: "{{ ansible_user_dir }}/ci-framework-data/parameters/" diff --git a/hooks/playbooks/control_plane_horizon.yml b/hooks/playbooks/control_plane_horizon.yml new file mode 100644 index 0000000000..010e1eace7 --- /dev/null +++ b/hooks/playbooks/control_plane_horizon.yml @@ -0,0 +1,28 @@ +--- +- name: Kustomize ControlPlane for horizon service + hosts: "{{ cifmw_target_hook_host | default('localhost') }}" + gather_facts: false + tasks: + - name: Ensure the kustomizations dir exists + ansible.builtin.file: + path: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane" + state: directory + mode: "0755" + + - name: Create kustomize yaml to enable Horizon + ansible.builtin.copy: + dest: "{{ cifmw_basedir }}/artifacts/manifests/kustomizations/controlplane/80-horizon-kustomization.yaml" + content: |- + apiVersion: kustomize.config.k8s.io/v1beta1 + kind: Kustomization + namespace: {{ namespace }} + patches: + - target: + kind: OpenStackControlPlane + patch: |- + - op: add + path: /spec/horizon/enabled + value: true + - op: add + path: /spec/horizon/template/memcachedInstance + value: memcached diff --git a/plugins/modules/url_request.py b/plugins/modules/url_request.py new file mode 100644 index 0000000000..5e761f5faa --- /dev/null +++ b/plugins/modules/url_request.py @@ -0,0 +1,216 @@ +#!/usr/bin/python + +# Copyright: (c) 2025, Pablo Rodriguez +# Apache License Version 2.0 (see LICENSE) + +__metaclass__ = type + +DOCUMENTATION = r""" +--- +module: url_request +short_description: Downloads/fetches the content of a SPNEGO secured URL +extends_documentation_fragment: + - files + +description: +- Downloads/fetches the content of a SPNEGO secured URL +- A kerberos ticket should be already issued + +author: + - Adrian Fusco (@afuscoar) + - Pablo Rodriguez (@pablintino) + +options: + url: + description: + - The URL to retrieve the content from + required: True + type: str + verify_ssl: + description: + - Enables/disables using TLS to reach the URL + required: False + type: bool + default: true + dest: + description: + - Path to the destination file/dir where the content should be downloaded + - If not provided the content won't be written into disk + required: False + type: str + +""" + +EXAMPLES = r""" +- name: Get some content + url_request: + url: "http://someurl.local/resource" + dest: "{{ ansible_user_dir }}/content.raw" + mode: "0644" + register: _fetched_content + +- name: Show base64 content + debug: + msg: "{{ _fetched_content.response_b64 }}" +""" + +RETURN = r""" +status_code: + description: HTTP response code + type: int + returned: returned request +content_type: + description: HTTP response Content-Type header content + type: str + returned: returned request +headers: + description: HTTP response headers + type: dict + returned: returned request +response_b64: + description: Returned content base64 encoded + type: str + returned: successful request +response_json: + description: Returned content as a dict + type: str + returned: successful request that returned application/json +path: + description: Written file path + type: str + returned: successful request +""" + +import base64 +import os.path +import re + +from ansible.module_utils.basic import AnsibleModule + + +try: + from requests import get, head + + python_requests_installed = True +except ImportError: + python_requests_installed = False +try: + from requests_kerberos import HTTPKerberosAuth, OPTIONAL + + python_requests_kerberos_installed = True +except ImportError: + python_requests_kerberos_installed = False + +try: + from requests_gssapi import HTTPSPNEGOAuth, OPTIONAL + + python_requests_gssapi_installed = True +except ImportError: + python_requests_gssapi_installed = False + + +def _get_auth_module(module, url, verify_ssl): + # If the auth modules are loaded there's no need to check if the url is secured beforehand + if python_requests_gssapi_installed: + return HTTPSPNEGOAuth(mutual_authentication=OPTIONAL) + elif python_requests_kerberos_installed: + return HTTPKerberosAuth(mutual_authentication=OPTIONAL) + + response = head(url=url, verify=verify_ssl, allow_redirects=True, timeout=30) + # If the response in a 401 or 403 we need to authenticate + if response.status_code in [401, 403]: + # Kerberos module not present, fail + module.fail_json( + msg="Neither requests_gssapi or requests_kerberos are installed and the url requires authentication" + ) + + +def main(): + module_args = { + "url": {"type": "str", "required": True}, + "verify_ssl": {"type": "bool", "default": True}, + "dest": {"type": "str", "required": False}, + } + + result = { + "changed": False, + } + + module = AnsibleModule( + argument_spec=module_args, supports_check_mode=False, add_file_common_args=True + ) + + if not python_requests_installed: + module.fail_json(msg="requests required for this module.") + + url = module.params["url"] + verify_ssl = module.params["verify_ssl"] + + try: + response = get( + url=url, + auth=_get_auth_module(module, url, verify_ssl), + verify=verify_ssl, + allow_redirects=True, + ) + + result["status_code"] = response.status_code + result["headers"] = dict(response.headers) + result["content_type"] = response.headers.get("Content-Type", None) + + if response.status_code < 200 or response.status_code >= 300: + module.fail_json( + msg=f"Error fetching the information {response.status_code}: {response.text}" + ) + + result["response_b64"] = base64.b64encode(response.content) + if "application/json" in result["content_type"]: + try: + result["response_json"] = response.json() + except ValueError as e: + module.fail_json(msg=f"Error with the JSON response: {str(e)}") + + if "dest" in module.params: + dest = module.params["dest"] + if ( + os.path.exists(dest) + and os.path.isdir(dest) + and "content-disposition" in response.headers + ): + # Destination is a directory but the filename is available in Content-Disposition + filename = re.findall( + "filename=(.+)", response.headers["content-disposition"] + ) + dest = filename[0] if filename else None + elif os.path.exists(dest) and os.path.isdir(dest): + # Destination is a directory but we cannot guess the filename from Content-Disposition + dest = None + + if not dest: + # Reached if dest points to a directory and: + # - Content-Disposition not available + # - Cannot extract the filename part from the Content-Disposition header + module.fail_json( + msg="Destination points to a directory and the filename cannot be retrieved from the response" + ) + + exists = os.path.exists(dest) + original_sha1 = module.sha1(dest) if exists else None + with open(dest, mode="wb") as file: + file.write(response.content) + file_args = module.load_file_common_arguments(module.params, path=dest) + result["changed"] = ( + (not exists) + or (module.sha1(dest) != original_sha1) + or module.set_fs_attributes_if_different(file_args, result["changed"]) + ) + result["path"] = dest + + except Exception as e: + module.fail_json(msg=f"Error fetching the information: {str(e)}") + + module.exit_json(**result) + + +if __name__ == "__main__": + main() diff --git a/requirements.yml b/requirements.yml index 995e244dd0..22fece6f58 100644 --- a/requirements.yml +++ b/requirements.yml @@ -17,29 +17,42 @@ collections: - name: https://github.com/ansible-collections/ansible.posix type: git + version: "1.6.2" - name: https://github.com/ansible-collections/ansible.utils type: git + version: "v5.1.2" - name: https://github.com/ansible-collections/community.general type: git + version: "10.0.1" - name: https://github.com/ansible-collections/community.crypto type: git + version: "2.22.3" - name: https://github.com/containers/ansible-podman-collections type: git + version: "1.16.2" - name: https://github.com/ansible-collections/community.libvirt type: git + version: "1.3.0" - name: https://github.com/ansible-collections/kubernetes.core type: git + version: "5.0.0" - name: https://github.com/ansible-collections/ansible.netcommon type: git + version: "v7.1.0" - name: https://github.com/openstack/ansible-config_template type: git + version: "2.1.1" - name: https://github.com/ansible-collections/junipernetworks.junos type: git + version: "v9.1.0" - name: https://github.com/ansible-collections/cisco.ios type: git + version: "v9.0.3" - name: https://github.com/ansible-collections/mellanox.onyx type: git - name: https://github.com/openshift/community.okd type: git + version: "4.0.0" - name: https://github.com/ovirt/ovirt-ansible-collection type: git + version: "3.2.0-1" diff --git a/roles/build_openstack_packages/defaults/main.yml b/roles/build_openstack_packages/defaults/main.yml index 683c79cb63..e2f11e2cbe 100644 --- a/roles/build_openstack_packages/defaults/main.yml +++ b/roles/build_openstack_packages/defaults/main.yml @@ -69,7 +69,7 @@ cifmw_bop_change_list: [] cifmw_bop_release_mapping: master: master - antelope: stable/2023.1 + antelope: unmaintained/2023.1 cifmw_bop_skipped_projects: - testproject diff --git a/roles/ci_gen_kustomize_values/README.md b/roles/ci_gen_kustomize_values/README.md index 4fd81bcb4f..6c23440cdf 100644 --- a/roles/ci_gen_kustomize_values/README.md +++ b/roles/ci_gen_kustomize_values/README.md @@ -35,8 +35,8 @@ with a message. * `ci_gen_kustomize_fetch_ocp_state`: (Boolean) If true it enables generating CI templates based on the OCP state. Defaults to `true`. * `cifmw_ci_gen_kustomize_values_storage_class_prefix`: (String) Prefix for `storageClass` in generated values.yaml files. Defaults to `"lvms-"` only if `cifmw_use_lvms` is True, otherwise it defaults to `""`. The prefix is prepended to the `cifmw_ci_gen_kustomize_values_storage_class`. It is not recommended to override this value, instead set `cifmw_use_lvms` True or False. * `cifmw_ci_gen_kustomize_values_storage_class`: (String) Value for `storageClass` in generated values.yaml files. Defaults to `"lvms-local-storage"` only if `cifmw_use_lvms` is True, otherwise it defaults to `"local-storage"`. -* `cifmw_ci_gen_kustomize_values_remove_keys_expressions`: (List) Remove keys matching the regular expressions from source ConfigMaps (values.yaml). - Defaults to `["^nodes$", "^node(_[0-9]+)?$"]`. +* `cifmw_ci_gen_kustomize_values_remove_keys_expressions*`: (List) Remove keys matching the regular expressions from source ConfigMaps (values.yaml). + Defaults to `["^nodes$", "^node(_[0-9]+)?$"]`. Accepts passing additional expressions by passing variables that matches `cifmw_ci_gen_kustomize_values_remove_keys_expressions.+`. ### Specific parameters for edpm-values This configMap needs some more parameters in order to properly override the `architecture` provided one. diff --git a/roles/ci_gen_kustomize_values/tasks/generate_snippets.yml b/roles/ci_gen_kustomize_values/tasks/generate_snippets.yml index fe6d535786..b5afcbb243 100644 --- a/roles/ci_gen_kustomize_values/tasks/generate_snippets.yml +++ b/roles/ci_gen_kustomize_values/tasks/generate_snippets.yml @@ -69,6 +69,22 @@ cifmw_ci_gen_kustomize_values_snippets_dir_prefix ~ _datatype) | path_join }} + _cifmw_gen_kustomize_values_reject_extra_expressions: >- + {{ + hostvars[inventory_hostname] | + dict2items | + selectattr("key", "match", "^cifmw_ci_gen_kustomize_values_remove_keys_expressions.+") | + sort(attribute='key') | + map(attribute="value") | + list + }} + _cifmw_gen_kustomize_values_reject_expressions: >- + {{ + ( + _cifmw_gen_kustomize_values_reject_extra_expressions + + cifmw_ci_gen_kustomize_values_remove_keys_expressions + ) | flatten | unique + }} ansible.builtin.set_fact: values_datatype: "{{ _datatype }}" snippet_datadir: "{{ _dest_dir }}" @@ -81,7 +97,7 @@ {{ _config_map_content | ansible.utils.remove_keys( - target=cifmw_ci_gen_kustomize_values_remove_keys_expressions, + target=_cifmw_gen_kustomize_values_reject_expressions, matching_parameter='regex') }} cacheable: false diff --git a/roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets/edpm-nodeset-values/values.yaml.j2 b/roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets/edpm-nodeset-values/values.yaml.j2 new file mode 100644 index 0000000000..6ae842b92b --- /dev/null +++ b/roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets/edpm-nodeset-values/values.yaml.j2 @@ -0,0 +1,58 @@ +--- +# source: ovs-dpdk-sriov-2nodesets/edpm-nodeset-values/values.yaml.j2 +{% set instances_names = [] %} +{% set _original_nodeset = (original_content.data | default({})).nodeset | default({}) %} +{% set _original_nodes = _original_nodeset.nodes | default({}) %} +{% set _original_services = _original_nodeset['services'] | default([]) %} +{% if cifmw_baremetal_hosts | default([]) | length > 0 %} +{% for _inst in cifmw_baremetal_hosts.keys() %} +{% if (('label' in cifmw_baremetal_hosts[_inst]) and + (cifmw_baremetal_hosts[_inst]['label'] == 'openstack')) %} +{% set _ = instances_names.append(_inst) %} +{% endif %} +{% endfor %} +{% else %} +# Needed for verification gate +{% set _vm_type = (_original_nodes.keys() | first).split('-')[1] %} +{% for _inst in cifmw_networking_env_definition.instances.keys() %} +{% if _inst.startswith(_vm_type) %} +{% set _ = instances_names.append(_inst) %} +{% endif %} +{% endfor %} +{% endif %} +data: + ssh_keys: + authorized: {{ cifmw_ci_gen_kustomize_values_ssh_authorizedkeys | b64encode }} + private: {{ cifmw_ci_gen_kustomize_values_ssh_private_key | b64encode }} + public: {{ cifmw_ci_gen_kustomize_values_ssh_public_key | b64encode }} + nova: + migration: + ssh_keys: + private: {{ cifmw_ci_gen_kustomize_values_migration_priv_key | b64encode }} + public: {{ cifmw_ci_gen_kustomize_values_migration_pub_key | b64encode }} + nodeset: + ansible: + ansibleVars: + edpm_fips_mode: "{{ 'enabled' if cifmw_fips_enabled|default(false)|bool else 'check' }}" + timesync_ntp_servers: + - hostname: "{{ cifmw_ci_gen_kustomize_values_ntp_srv | default('pool.ntp.org') }}" +{% if cifmw_ci_gen_kustomize_values_sshd_ranges | default([]) | length > 0 %} + edpm_sshd_allowed_ranges: +{% for range in cifmw_ci_gen_kustomize_values_sshd_ranges %} + - "{{ range }}" +{% endfor %} +{% endif %} + nodes: +{% for instance in instances_names %} + edpm-{{ instance }}: + hostName: {{ instance }} +{% endfor %} + +{% if ('repo-setup' not in (_original_nodeset['services'] | default([]))) and + ('repo-setup' in ci_gen_kustomize_edpm_nodeset_predeployed_services) %} + services: + - "repo-setup" +{% for svc in _original_services %} + - "{{ svc }}" +{% endfor %} +{% endif %} diff --git a/roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets/edpm-nodeset2-values/values.yaml.j2 b/roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets/edpm-nodeset2-values/values.yaml.j2 new file mode 100644 index 0000000000..6e2109bc50 --- /dev/null +++ b/roles/ci_gen_kustomize_values/templates/ovs-dpdk-sriov-2nodesets/edpm-nodeset2-values/values.yaml.j2 @@ -0,0 +1,58 @@ +--- +# source: ovs-dpdk-sriov-2nodesets/edpm-nodeset2-values/values.yaml.j2 +{% set instances_names = [] %} +{% set _original_nodeset = (original_content.data | default({})).nodeset | default({}) %} +{% set _original_nodes = _original_nodeset.nodes | default({}) %} +{% set _original_services = _original_nodeset['services'] | default([]) %} +{% if cifmw_baremetal_hosts | default([]) | length > 0 %} +{% for _inst in cifmw_baremetal_hosts.keys() %} +{% if (('label' in cifmw_baremetal_hosts[_inst]) and + (cifmw_baremetal_hosts[_inst]['label'] == 'openstack-2')) %} +{% set _ = instances_names.append(_inst) %} +{% endif %} +{% endfor %} +{% else %} +# Needed for verification gate +{% set _vm_type = (_original_nodes.keys() | first).split('-')[1] %} +{% for _inst in cifmw_networking_env_definition.instances.keys() %} +{% if _inst.startswith(_vm_type) %} +{% set _ = instances_names.append(_inst) %} +{% endif %} +{% endfor %} +{% endif %} +data: + ssh_keys: + authorized: {{ cifmw_ci_gen_kustomize_values_ssh_authorizedkeys | b64encode }} + private: {{ cifmw_ci_gen_kustomize_values_ssh_private_key | b64encode }} + public: {{ cifmw_ci_gen_kustomize_values_ssh_public_key | b64encode }} + nova: + migration: + ssh_keys: + private: {{ cifmw_ci_gen_kustomize_values_migration_priv_key | b64encode }} + public: {{ cifmw_ci_gen_kustomize_values_migration_pub_key | b64encode }} + nodeset: + ansible: + ansibleVars: + edpm_fips_mode: "{{ 'enabled' if cifmw_fips_enabled|default(false)|bool else 'check' }}" + timesync_ntp_servers: + - hostname: "{{ cifmw_ci_gen_kustomize_values_ntp_srv | default('pool.ntp.org') }}" +{% if cifmw_ci_gen_kustomize_values_sshd_ranges | default([]) | length > 0 %} + edpm_sshd_allowed_ranges: +{% for range in cifmw_ci_gen_kustomize_values_sshd_ranges %} + - "{{ range }}" +{% endfor %} +{% endif %} + nodes: +{% for instance in instances_names %} + edpm-{{ instance }}: + hostName: {{ instance }} +{% endfor %} + +{% if ('repo-setup' not in (_original_nodeset['services'] | default([]))) and + ('repo-setup' in ci_gen_kustomize_edpm_nodeset_predeployed_services) %} + services: + - "repo-setup" +{% for svc in _original_services %} + - "{{ svc }}" +{% endfor %} +{% endif %} diff --git a/roles/ci_setup/tasks/epel.yml b/roles/ci_setup/tasks/epel.yml index eb7d992bae..e8622faed1 100644 --- a/roles/ci_setup/tasks/epel.yml +++ b/roles/ci_setup/tasks/epel.yml @@ -4,6 +4,16 @@ name: ci_setup tasks_from: repos.yml +# Some roles like repo_setup may have altered the RPM installed files +# without removing the RPM itself. Ensuring it's not installed before +# enforcing its installation will make the EPEL rpm install consistent +# with the actul state of the repository files. +- name: Ensure EPEL is not already installed + become: true + ansible.builtin.dnf: + name: epel-release + state: absent + - name: Install EPEL become: true ansible.builtin.dnf: diff --git a/roles/cifmw_cephadm/defaults/main.yml b/roles/cifmw_cephadm/defaults/main.yml index 3eb487c13b..f1334fdec3 100644 --- a/roles/cifmw_cephadm/defaults/main.yml +++ b/roles/cifmw_cephadm/defaults/main.yml @@ -147,3 +147,5 @@ cifmw_cephadm_repository_override: false cifmw_cephadm_version: "squid" # bug in cephadm: if you skip "prepare host" it will fail cifmw_cephadm_prepare_host: false +cifmw_cephadm_wait_install_retries: 8 +cifmw_cephadm_wait_install_delay: 15 diff --git a/roles/cifmw_cephadm/tasks/ceph_upgrade.yml b/roles/cifmw_cephadm/tasks/ceph_upgrade.yml index 31b8f33872..a2982cdf10 100644 --- a/roles/cifmw_cephadm/tasks/ceph_upgrade.yml +++ b/roles/cifmw_cephadm/tasks/ceph_upgrade.yml @@ -18,12 +18,15 @@ ansible.builtin.set_fact: ceph_health: "{{ ceph.stdout | from_json }}" -- name: Fail if health is HEALTH_WARN || HEALTH_ERR +- name: Log ceph health for debug purposes + ansible.builtin.debug: + var: ceph_health + +- name: Fail if health is HEALTH_ERR ansible.builtin.fail: msg: Ceph is in {{ ceph_health.health.status }} state. when: - - ceph_health.health.status == 'HEALTH_WARN' or - ceph_health.health.status == 'HEALTH_ERR' + - ceph_health.health.status == 'HEALTH_ERR' - name: Build container image target ansible.builtin.set_fact: diff --git a/roles/cifmw_cephadm/tasks/install_cephadm.yml b/roles/cifmw_cephadm/tasks/install_cephadm.yml index 4c932f2823..d334a725ee 100644 --- a/roles/cifmw_cephadm/tasks/install_cephadm.yml +++ b/roles/cifmw_cephadm/tasks/install_cephadm.yml @@ -30,6 +30,10 @@ ansible.builtin.dnf: name: cephadm state: present + register: task_result + retries: "{{ cifmw_cephadm_wait_install_retries }}" + delay: "{{ cifmw_cephadm_wait_install_delay }}" + until: task_result is success - name: Stat cephadm file ansible.builtin.stat: diff --git a/roles/copy_container/files/copy-quay/go.mod b/roles/copy_container/files/copy-quay/go.mod index 081886f0ce..ba5a076fed 100644 --- a/roles/copy_container/files/copy-quay/go.mod +++ b/roles/copy_container/files/copy-quay/go.mod @@ -106,15 +106,15 @@ require ( go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/sdk v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 // indirect - golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sync v0.6.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.18.1-0.20240311201521-78fbdeb61842 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect google.golang.org/grpc v1.61.1 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/roles/copy_container/files/copy-quay/go.sum b/roles/copy_container/files/copy-quay/go.sum index 0850fc428f..d31c7ff591 100644 --- a/roles/copy_container/files/copy-quay/go.sum +++ b/roles/copy_container/files/copy-quay/go.sum @@ -388,8 +388,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= @@ -400,8 +400,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -417,8 +417,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -430,8 +430,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -452,15 +452,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -468,8 +468,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -485,8 +486,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.18.1-0.20240311201521-78fbdeb61842 h1:No0LMXYFkp3j4oEsPdtY8LUQz33gu79Rm9DE+izMeGQ= -golang.org/x/tools v0.18.1-0.20240311201521-78fbdeb61842/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/roles/deploy_bmh/template/bmh.yml.j2 b/roles/deploy_bmh/template/bmh.yml.j2 index 3c9cd7f4f1..587917ef04 100644 --- a/roles/deploy_bmh/template/bmh.yml.j2 +++ b/roles/deploy_bmh/template/bmh.yml.j2 @@ -9,7 +9,7 @@ metadata: inspect.metal3.io: "disabled" {% endif %} labels: - app: openstack + app: {{ node_data['label'] | default("openstack") }} workload: {{ node_name.split('-')[0] }} spec: bmc: diff --git a/roles/edpm_prepare/tasks/main.yml b/roles/edpm_prepare/tasks/main.yml index 88a7c4b356..9029dab6f9 100644 --- a/roles/edpm_prepare/tasks/main.yml +++ b/roles/edpm_prepare/tasks/main.yml @@ -119,6 +119,34 @@ --namespace={{ cifmw_install_yamls_defaults['OPERATOR_NAMESPACE'] }} --for=jsonpath='{.status.phase}'=Complete --timeout=20m +- name: Check if the OpenStack initialization CRD exists + when: + - not cifmw_edpm_prepare_dry_run + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + kind: CustomResourceDefinition + name: openstacks.operator.openstack.org + register: _cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out + tags: + - control-plane + +- name: OpenStack meta-operator initialization, if necessary + when: + - not cifmw_edpm_prepare_dry_run + - _cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out is defined + - _cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out.resources is defined + - (_cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out.resources | length) > 0 + vars: + make_openstack_init_env: "{{ cifmw_edpm_prepare_common_env }}" + make_openstack_init_dryrun: "{{ cifmw_edpm_prepare_dry_run }}" + ansible.builtin.include_role: + name: 'install_yamls_makes' + tasks_from: 'make_openstack_init' + tags: + - control-plane + # Note(chkumar): Keeping set_openstack_containers role # till we migrate this task to update_containers role - name: Update OpenStack Services containers Env diff --git a/roles/env_op_images/tasks/main.yml b/roles/env_op_images/tasks/main.yml index cefe27810d..17b852d580 100644 --- a/roles/env_op_images/tasks/main.yml +++ b/roles/env_op_images/tasks/main.yml @@ -131,7 +131,7 @@ cifmw_openstack_operator_images_content | combine( { - item.metadata.labels['openstack.org/operator-name'] | upper ~ '_OP_IMG': item.status.containerStatuses[1].imageID + item.metadata.labels['openstack.org/operator-name'] | upper ~ '_OP_IMG': (item.status.containerStatuses | last).imageID } ) }} diff --git a/roles/kustomize_deploy/defaults/main.yml b/roles/kustomize_deploy/defaults/main.yml index 1d14a72461..eefd888a7f 100644 --- a/roles/kustomize_deploy/defaults/main.yml +++ b/roles/kustomize_deploy/defaults/main.yml @@ -108,6 +108,24 @@ cifmw_kustomize_deploy_nmstate_dest_file: >- ] | path_join }} +# openstack init resource +cifmw_kustomize_deploy_openstack_source_files: >- + {{ + [ + cifmw_kustomize_deploy_architecture_repo_dest_dir, + cifmw_kustomize_deploy_architecture_examples_common_path, + 'openstack' + ] | path_join + }} + +cifmw_kustomize_deploy_openstack_dest_file: >- + {{ + [ + cifmw_kustomize_deploy_kustomizations_dest_dir, + 'openstack.yaml' + ] | path_join + }} + # controlsplane resources cifmw_kustomize_deploy_nncp_source_files: >- diff --git a/roles/kustomize_deploy/tasks/install_operators.yml b/roles/kustomize_deploy/tasks/install_operators.yml index 8be1a7500f..eca39c58a2 100644 --- a/roles/kustomize_deploy/tasks/install_operators.yml +++ b/roles/kustomize_deploy/tasks/install_operators.yml @@ -107,7 +107,7 @@ - _cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | length == 1 - (_cifmw_kustomize_deploy_olm_osp_operator_sub_out.resources | first)['status']['installPlanRef'] is defined - - name: Wait for the openstack operators InstallPlan to be finish + - name: Wait for the openstack operators InstallPlan to be finished vars: _install_plan: >- {{ @@ -319,8 +319,69 @@ status: "True" wait_timeout: 300 -- name: Wait until OpenStack operators are deployed and ready - when: not cifmw_kustomize_deploy_generate_crs_only +- name: Check if the OpenStack initialization CRD exists + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + kind: CustomResourceDefinition + name: openstacks.operator.openstack.org + register: _cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out + +- name: Handle OpenStack initialization, if necessary + when: (_cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out.resources | length) > 0 + block: + - name: Generate OpenStack initialization kustomization file + ansible.builtin.copy: + content: >- + {{ + lookup( + 'kubernetes.core.kustomize', + dir=cifmw_kustomize_deploy_openstack_source_files + ) + }} + dest: "{{ cifmw_kustomize_deploy_openstack_dest_file }}" + mode: "0644" + + - name: Openstack initialization resources + when: not cifmw_kustomize_deploy_generate_crs_only + block: + - name: Apply the kustomized OpenStack initialization CRs + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit) }}" + context: "{{ cifmw_openshift_context | default(omit) }}" + state: present + wait: true + src: "{{ cifmw_kustomize_deploy_openstack_dest_file }}" + check_mode: >- + {{ + cifmw_kustomize_deploy_check_mode | + default(false, true) + }} + + - name: Wait until OpenStack operators are deployed and ready (new install paradigm) + when: not cifmw_kustomize_deploy_generate_crs_only + kubernetes.core.k8s_info: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + kind: Openstack + namespace: openstack-operators + name: openstack + wait: true + wait_condition: + type: Ready + status: "True" + wait_timeout: 600 + check_mode: >- + {{ + cifmw_kustomize_deploy_check_mode | + default(false, true) + }} + +- name: Wait until OpenStack operators are deployed and ready (old install paradigm) + when: + - not cifmw_kustomize_deploy_generate_crs_only + - (_cifmw_kustomize_deploy_olm_osp_operator_openstack_crd_out.resources | length) == 0 kubernetes.core.k8s_info: kubeconfig: "{{ cifmw_openshift_kubeconfig }}" kind: Deployment diff --git a/roles/libvirt_manager/tasks/manage_vms.yml b/roles/libvirt_manager/tasks/manage_vms.yml index 98a3f354f2..6564e69f14 100644 --- a/roles/libvirt_manager/tasks/manage_vms.yml +++ b/roles/libvirt_manager/tasks/manage_vms.yml @@ -64,7 +64,7 @@ cat ~/.ssh/authorized_keys | ssh -v core@{{ vm_con_name }} "cat >> ~/.ssh/authorized_keys" retries: 5 - delay: 10 + delay: 60 register: _ssh_access until: _ssh_access.rc == 0 diff --git a/roles/libvirt_manager/templates/all-inventory.yml.j2 b/roles/libvirt_manager/templates/all-inventory.yml.j2 index 9b9d8499a7..04190a6364 100644 --- a/roles/libvirt_manager/templates/all-inventory.yml.j2 +++ b/roles/libvirt_manager/templates/all-inventory.yml.j2 @@ -20,6 +20,20 @@ virtual machines. ansible_ssh_common_args: "-o StrictHostKeyChecking=no -J {{ ansible_user | default(ansible_user_id) }}@{{ _hostname }}" {% endif %} {% endfor %} +hypervisors: + hosts: +{% set _hypervisors = ( + ( _cifmw_libvirt_manager_layout.vms.values() | selectattr('target', 'defined') | map(attribute='target') | list) + + [inventory_hostname] ) | unique %} +{% for _hypervisor in _hypervisors %} + {{ _hypervisor }}: +{% if hostvars[_hypervisor]['ansible_host'] is defined %} +{% set _hostname = hostvars[_hypervisor]['ansible_host'] %} + ansible_host: "{{ _hostname }}" + ansible_ssh_private_key_file: "~/.ssh/ssh_{{ _hostname }}" + vars: +{% endif %} +{% endfor %} localhosts: hosts: localhost: diff --git a/roles/libvirt_manager/templates/inventory.yml.j2 b/roles/libvirt_manager/templates/inventory.yml.j2 index 861155f5b7..17509fe351 100644 --- a/roles/libvirt_manager/templates/inventory.yml.j2 +++ b/roles/libvirt_manager/templates/inventory.yml.j2 @@ -14,4 +14,5 @@ {% else %} ansible_ssh_private_key_file: ~/.ssh/id_cifw {% endif %} + cifmw_hypervisor_host: "{{ _cifmw_libvirt_manager_layout.vms[host.key].target | default(inventory_hostname) }}" {% endfor %} diff --git a/roles/polarion/tasks/main.yml b/roles/polarion/tasks/main.yml index a82d365002..d097e6fd0f 100644 --- a/roles/polarion/tasks/main.yml +++ b/roles/polarion/tasks/main.yml @@ -131,7 +131,7 @@ --testrun-title={{ cifmw_polarion_testrun_title }}-{{ loop_idx }} --xml-file={{ item.path }} --update_testcases={{ cifmw_polarion_update_testcases | default(false) }} - --custom-fields {{ _custom_fields_string }} + --custom-fields {{ _custom_fields_string | quote }} {{ cifmw_polarion_jump_extra_vars | default ('') }} loop: "{{ merged_xml_files.files }}" loop_control: diff --git a/roles/repo_setup/README.md b/roles/repo_setup/README.md index f6dcc73d0f..46dc74eb86 100644 --- a/roles/repo_setup/README.md +++ b/roles/repo_setup/README.md @@ -30,6 +30,7 @@ using `cifmw_repo_setup_src` role default var. * `cifmw_repo_setup_rhos_release_rpm`: (String) URL to rhos-release RPM. * `cifmw_repo_setup_rhos_release_args`: (String) Parameters to pass down to `rhos-release`. * `cifmw_repo_setup_rhos_release_gpg_check`: (Bool) Skips the gpg check during rhos-release rpm installation. Defaults to `True`. +* `cifmw_repo_setup_rhos_release_path`: (String) The path where the rhos-release rpm is downloaded. Defaults to `{{ cifmw_repo_setup_basedir }}/rhos-release`. ## Notes diff --git a/roles/repo_setup/defaults/main.yml b/roles/repo_setup/defaults/main.yml index 177f5df200..4cddb08773 100644 --- a/roles/repo_setup/defaults/main.yml +++ b/roles/repo_setup/defaults/main.yml @@ -40,3 +40,4 @@ cifmw_repo_setup_component_promotion_tag: component-ci-testing # cifmw_repo_setup_rhos_release_args: cifmw_repo_setup_enable_rhos_release: false cifmw_repo_setup_rhos_release_gpg_check: true +cifmw_repo_setup_rhos_release_path: "{{ cifmw_repo_setup_basedir }}/rhos-release" diff --git a/roles/repo_setup/library/url_request.py b/roles/repo_setup/library/url_request.py new file mode 120000 index 0000000000..9a1a7bca8a --- /dev/null +++ b/roles/repo_setup/library/url_request.py @@ -0,0 +1 @@ +../../../plugins/modules/url_request.py \ No newline at end of file diff --git a/roles/repo_setup/tasks/rhos_release.yml b/roles/repo_setup/tasks/rhos_release.yml index 81bb7638be..6d1f9ab80d 100644 --- a/roles/repo_setup/tasks/rhos_release.yml +++ b/roles/repo_setup/tasks/rhos_release.yml @@ -1,8 +1,29 @@ --- +- name: Download RHOS Release if the given rpm is a URL + when: cifmw_repo_setup_rhos_release_rpm is url + block: + - name: Create download directory + ansible.builtin.file: + path: "{{ cifmw_repo_setup_rhos_release_path }}" + state: directory + mode: "0755" + + - name: Download the RPM + url_request: + url: "{{ cifmw_repo_setup_rhos_release_rpm }}" + dest: "{{ cifmw_repo_setup_rhos_release_path }}/rhos-release.rpm" + mode: "0644" + register: _cifmw_repo_setup_url_get + - name: Install RHOS Release tool become: true ansible.builtin.package: - name: "{{ cifmw_repo_setup_rhos_release_rpm }}" + name: >- + {{ + cifmw_repo_setup_rhos_release_rpm + if cifmw_repo_setup_rhos_release_rpm is not url + else _cifmw_repo_setup_url_get.path + }} state: present disable_gpg_check: "{{ cifmw_repo_setup_rhos_release_gpg_check | bool }}" diff --git a/roles/reproducer/tasks/configure_controller.yml b/roles/reproducer/tasks/configure_controller.yml index cc86bc3db6..71ad3e3958 100644 --- a/roles/reproducer/tasks/configure_controller.yml +++ b/roles/reproducer/tasks/configure_controller.yml @@ -179,6 +179,7 @@ marker: "## {mark} {{ _ssh_host }}" block: |- Host {{ _ssh_host }} {{ hostvars[host]['inventory_hostname'] }} + Hostname {{ _ssh_host }} IdentityFile ~/.ssh/ssh_{{ _ssh_host }} StrictHostKeyChecking no UserKnownHostsFile /dev/null diff --git a/roles/shiftstack/README.md b/roles/shiftstack/README.md index dea394b598..88fa0bbe7c 100644 --- a/roles/shiftstack/README.md +++ b/roles/shiftstack/README.md @@ -5,21 +5,26 @@ Role for triggering Openshift on Openstack QA automation (installation and tests * `cifmw_shiftstack_ansible_command_logs_dir`: (*string*) Directory name for the ansible command module output. Defaults to `"{{ cifmw_shiftstack_basedir }}/ansible_command_logs"`. * `cifmw_shiftstack_artifacts_dir`: (*string*) Directory name for the role artifacts. Defaults to `"{{ cifmw_shiftstack_basedir }}/artifacts"`. * `cifmw_shiftstack_basedir`: (*string*) Base directory for the role artifacts and logs. Defaults to `{{ cifmw_basedir }}/tests/shiftstack` (which defaults to `~/ci-framework-data/tests/shiftstack`. +* `cifmw_shiftstack_client_incluster_secret_manifest`: (*string*) The manifest file for creating the secret that will hold the RHOSO kubeconfig. Defaults to `{{ cifmw_shiftstack_client_pod_name }}_incluster_secret.yml`. +* `cifmw_shiftstack_client_incluster_secret_name:`: (*string*) The secret name that will hold the RHOSO kubeconfig. Defaults to `incluster-kubeconfig`. * `cifmw_shiftstack_client_pod_name`: (*string*) Pod name for the pod running the Openshift installer and tests. Defaults to `shiftstackclient`. * `cifmw_shiftstack_client_pod_manifest`: (*string*) The file name for the shiftstackclient pod manifest. Defaults to `"{{ cifmw_shiftstack_client_pod_name }}_pod.yml"`. * `cifmw_shiftstack_client_pod_namespace`: (*string*) The namespace where the `cifmw_shiftstack_client_pod_name` will be deployed. Defaults to `openstack`. * `cifmw_shiftstack_client_pod_image`: (*string*) The image for the container running in the `cifmw_shiftstack_client_pod_name` pod. Defaults to `quay.io/shiftstack-qe/shiftstack-client:latest`. * `cifmw_shiftstack_client_pvc_manifest`: (*string*) The file name for the shiftstackclient pvc manifest. Defaults to `"{{ cifmw_shiftstack_client_pod_name }}_pvc.yml"`. * `cifmw_shiftstack_cluster_name`: (*string*) The Openshift cluster name. Defaults to `ostest`. +* `cifmw_shiftstack_hypervisor`: (*string*) The hypervisor where RHOSO is deployed. Defaults to `"{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}"`. +* `cifmw_shiftstack_exclude_artifacts_regex`: (*string*) Regex that will be passed on `oc rsync` command as `--exclude` param, so the role does not gather the artifacts matching it. * `cifmw_shiftstack_installation_dir`: (*string*) Directory to place installation files. Defaults to `"{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/installation"`. * `cifmw_shiftstack_manifests_dir`: (*string*) Directory name for the role generated Openshift manifests. Defaults to `"{{ cifmw_shiftstack_basedir }}/manifests"`. * `cifmw_shiftstack_project_name`: (*string*) The Openstack project name. Defaults to `shiftstack`. +* `cifmw_shiftstack_proxy`: (*string*) The proxy url that should be used to reach the underlying OCP. Defaults to omit. * `cifmw_shiftstack_qa_gerrithub_change`: (*string*) The gerrithub change to fetch from the `cifmw_shiftstack_qa_repo` repository (i.e. 'refs/changes/29/1188429/50)'. Defaults to ''. * `cifmw_shiftstack_qa_repo`: (*string*) The repository containing the Openshift on Openstack QA automation. Defaults to `https://review.gerrithub.io/shiftstack/shiftstack-qa`. * `cifmw_shiftstack_run_playbook`: (*string*) The playbook to be run from the `cifmw_shiftstack_qa_repo` repository. Defaults to `ocp_testing.yaml`. * `cifmw_shiftstack_sc`: (*string*) The storage class to be used for PVC for the shiftstackclient pod. Defaults to `local-storage`. * `cifmw_shiftstack_shiftstackclient_artifacts_dir`: (*string*) The artifacts directory path for the shiftstackclient pod. Defaults to `/home/cloud-admin/artifacts`. -* `cifmw_shiftstack_proxy`: (*string*) The proxy url that should be used to reach the underlying OCP. Defaults to omit. +* `cifmw_shiftstack_shiftstackclient_incluster_kubeconfig_dir`: (*string*) The directory path in shiftstackclient pod the will hold the RHOSO kubeconfig. Defaults to `/home/cloud-admin/incluster-kubeconfig`. ## Examples The role is imported in the test playbook, i.e. when: diff --git a/roles/shiftstack/defaults/main.yml b/roles/shiftstack/defaults/main.yml index 3956c8a84d..1fc9401d18 100644 --- a/roles/shiftstack/defaults/main.yml +++ b/roles/shiftstack/defaults/main.yml @@ -20,12 +20,16 @@ cifmw_shiftstack_ansible_command_logs_dir: "{{ cifmw_shiftstack_basedir }}/ansible_command_logs" cifmw_shiftstack_artifacts_dir: "{{ cifmw_shiftstack_basedir }}/artifacts" cifmw_shiftstack_basedir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}/tests/shiftstack" +cifmw_shiftstack_client_incluster_secret_manifest: "{{ cifmw_shiftstack_client_pod_name }}_incluster_secret.yml" +cifmw_shiftstack_client_incluster_secret_name: "incluster-kubeconfig" cifmw_shiftstack_client_pod_image: "quay.io/shiftstack-qe/shiftstack-client:latest" cifmw_shiftstack_client_pod_manifest: "{{ cifmw_shiftstack_client_pod_name }}_pod.yml" cifmw_shiftstack_client_pod_name: "shiftstackclient-{{ cifmw_shiftstack_project_name }}" cifmw_shiftstack_client_pod_namespace: "openstack" cifmw_shiftstack_client_pvc_manifest: "{{ cifmw_shiftstack_client_pod_name }}_pvc.yml" cifmw_shiftstack_cluster_name: "ostest" +cifmw_shiftstack_exclude_artifacts_regex: "openshift-install" +cifmw_shiftstack_hypervisor: "{{ hostvars[hostvars['controller-0']['cifmw_hypervisor_host'] | default ('')]['ansible_host'] | default('') }}" cifmw_shiftstack_installation_dir: "{{ cifmw_shiftstack_basedir }}/installation" cifmw_shiftstack_manifests_dir: "{{ cifmw_shiftstack_basedir }}/manifests" cifmw_shiftstack_project_name: "shiftstack" @@ -35,3 +39,4 @@ cifmw_shiftstack_run_playbook: "ocp_testing.yaml" cifmw_shiftstack_sc: "local-storage" cifmw_shiftstack_shiftstackclient_artifacts_dir: "/home/cloud-admin/artifacts" cifmw_shiftstack_shiftstackclient_installation_dir: "{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/installation" +cifmw_shiftstack_shiftstackclient_incluster_kubeconfig_dir: "/home/cloud-admin/incluster-kubeconfig" diff --git a/roles/shiftstack/molecule/default/converge.yml b/roles/shiftstack/molecule/default/converge.yml index ad49e1169d..fb30de5414 100644 --- a/roles/shiftstack/molecule/default/converge.yml +++ b/roles/shiftstack/molecule/default/converge.yml @@ -22,6 +22,7 @@ cifmw_path: "{{ ansible_user_dir }}/.crc/bin:{{ ansible_user_dir }}/.crc/bin/oc:{{ ansible_user_dir }}/bin:{{ ansible_env.PATH }}" cifmw_openshift_kubeconfig: "{{ ansible_user_dir }}/.crc/machines/crc/kubeconfig" cifmw_shiftstack_run_playbook: cifmw-gate.yaml + cifmw_shiftstack_exclude_artifacts_regex: '' cifmw_run_test_shiftstack_testconfig: - cifmw-gate.yaml - cifmw-gate.yaml # The purpose of this repeated test config is to check the test config loop logic diff --git a/roles/shiftstack/tasks/cleanup.yml b/roles/shiftstack/tasks/cleanup.yml index ad5bb073f4..131f2968a8 100644 --- a/roles/shiftstack/tasks/cleanup.yml +++ b/roles/shiftstack/tasks/cleanup.yml @@ -18,6 +18,16 @@ ansible.builtin.debug: msg: "TODO: to be added once it's supported in shiftstack-qa automation" +- name: Delete the secret '{{ cifmw_shiftstack_client_incluster_secret_name }}' + kubernetes.core.k8s: + state: absent + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_version: v1 + kind: Secret + namespace: openstack + name: "{{ cifmw_shiftstack_client_incluster_secret_name }}" + proxy: "{{ cifmw_shiftstack_proxy | default(omit) }}" + - name: Delete the pod '{{ cifmw_shiftstack_client_pod_name }}' kubernetes.core.k8s: state: absent diff --git a/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml b/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml index 36fb5209f0..37f5d60707 100644 --- a/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml +++ b/roles/shiftstack/tasks/deploy_shiftstackclient_pod.yml @@ -14,6 +14,23 @@ # License for the specific language governing permissions and limitations # under the License. +- name: Encode kubeconfig file to Base64 + ansible.builtin.set_fact: + incluster_kubeconfig_base64: "{{ lookup('ansible.builtin.file', cifmw_openshift_kubeconfig ) | b64encode }}" + +- name: Render the incluster kubeconfig secret manifest + ansible.builtin.template: + src: shiftstackclient_incluster_kubeconfig_secret.yml.j2 + dest: "{{ (cifmw_shiftstack_manifests_dir, cifmw_shiftstack_client_incluster_secret_manifest) | path_join }}" + mode: "0644" + +- name: Apply the manifest for the incluster kubeconfig secret + kubernetes.core.k8s: + state: present + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + src: "{{ (cifmw_shiftstack_manifests_dir, cifmw_shiftstack_client_incluster_secret_manifest) | path_join }}" + proxy: "{{ cifmw_shiftstack_proxy | default(omit) }}" + - name: Render the pvc manifest ansible.builtin.template: src: templates/shiftstackclient_pvc.yml.j2 diff --git a/roles/shiftstack/tasks/pre_test_shiftstack.yml b/roles/shiftstack/tasks/pre_test_shiftstack.yml index 001ced3049..a285016b9a 100644 --- a/roles/shiftstack/tasks/pre_test_shiftstack.yml +++ b/roles/shiftstack/tasks/pre_test_shiftstack.yml @@ -21,6 +21,7 @@ - name: Remove the shiftstackclient pod if exists kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" state: absent api_version: v1 kind: Pod diff --git a/roles/shiftstack/tasks/test_config.yml b/roles/shiftstack/tasks/test_config.yml index c6d697d32d..02e7cf746a 100644 --- a/roles/shiftstack/tasks/test_config.yml +++ b/roles/shiftstack/tasks/test_config.yml @@ -31,8 +31,14 @@ namespace: "{{ cifmw_shiftstack_client_pod_namespace }}" pod_name: "{{ cifmw_shiftstack_client_pod_name }}" command: >- + source .bashrc && cd shiftstack-qa && - ansible-navigator run playbooks/{{ cifmw_shiftstack_run_playbook }} -e @jobs_definitions/{{ testconfig }} -e ocp_cluster_name={{ cifmw_shiftstack_cluster_name }} -e user_cloud={{ cifmw_shiftstack_project_name }} + ansible-navigator run playbooks/{{ cifmw_shiftstack_run_playbook }} + -e @jobs_definitions/{{ testconfig }} + -e ocp_cluster_name={{ cifmw_shiftstack_cluster_name }} + -e user_cloud={{ cifmw_shiftstack_project_name }} + -e hypervisor={{ cifmw_shiftstack_hypervisor }} + -e rhoso_kubeconfig={{ cifmw_shiftstack_shiftstackclient_incluster_kubeconfig_dir }} ansible.builtin.include_tasks: exec_command_in_pod.yml rescue: @@ -48,12 +54,19 @@ always: - name: "Copy the artifacts from the pod '{{ cifmw_shiftstack_client_pod_name }}'" + vars: + _exclude_options: >- + {{ + '--exclude=' ~ cifmw_shiftstack_exclude_artifacts_regex + if cifmw_shiftstack_exclude_artifacts_regex | default('') != '' + else '' + }} environment: PATH: "{{ cifmw_path }}" KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" ansible.builtin.command: cmd: > - oc rsync -n {{ cifmw_shiftstack_client_pod_namespace }} + oc rsync -n {{ cifmw_shiftstack_client_pod_namespace }} {{ _exclude_options }} {{ cifmw_shiftstack_client_pod_name }}:{{ cifmw_shiftstack_shiftstackclient_artifacts_dir }}/ {{ testconfig_artifacts_dir }}/ failed_when: false diff --git a/roles/shiftstack/templates/shiftstackclient_incluster_kubeconfig_secret.yml.j2 b/roles/shiftstack/templates/shiftstackclient_incluster_kubeconfig_secret.yml.j2 new file mode 100644 index 0000000000..9e4e2b83fa --- /dev/null +++ b/roles/shiftstack/templates/shiftstackclient_incluster_kubeconfig_secret.yml.j2 @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: "{{ cifmw_shiftstack_client_incluster_secret_name }}" + namespace: openstack +type: Opaque +data: + kubeconfig: {{ incluster_kubeconfig_base64 }} diff --git a/roles/shiftstack/templates/shiftstackclient_pod.yml.j2 b/roles/shiftstack/templates/shiftstackclient_pod.yml.j2 index 682db1204a..1ba3f474dc 100644 --- a/roles/shiftstack/templates/shiftstackclient_pod.yml.j2 +++ b/roles/shiftstack/templates/shiftstackclient_pod.yml.j2 @@ -30,6 +30,9 @@ spec: readOnly: true - name: installation-volume mountPath: {{ cifmw_shiftstack_shiftstackclient_installation_dir }} + - name: {{ cifmw_shiftstack_client_incluster_secret_name }}-volume + mountPath: {{ cifmw_shiftstack_shiftstackclient_incluster_kubeconfig_dir }} + readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true preemptionPolicy: PreemptLowerPriority @@ -48,6 +51,9 @@ spec: operator: Exists tolerationSeconds: 300 volumes: + - name: {{ cifmw_shiftstack_client_incluster_secret_name }}-volume + secret: + secretName: {{ cifmw_shiftstack_client_incluster_secret_name }} - configMap: defaultMode: 420 name: openstack-config diff --git a/roles/test_operator/README.md b/roles/test_operator/README.md index 0ca3b0898f..4f86894a2d 100644 --- a/roles/test_operator/README.md +++ b/roles/test_operator/README.md @@ -24,12 +24,15 @@ Execute tests via the [test-operator](https://openstack-k8s-operators.github.io/ * `cifmw_test_operator_delete_logs_pod`: (Boolean) Delete tempest log pod created by the role at the end of the testing. Default value: `false`. * `cifmw_test_operator_privileged`: (Boolean) Spawn the test pods with `allowPrivilegedEscalation: true` and default linux capabilities. This is required for certain test-operator functionalities to work properly (e.g.: `extraRPMs`, certain set of tobiko tests). Default value: `true` * `cifmw_test_operator_selinux_level`: (String) Specify SELinux level that should be used for pods spawned with the test-operator. Note, that `cifmw_test_operator_privileged: true` must be set when this parameter has non-empty value. Default value: `s0:c478,c978` +* `cifmw_test_operator_default_registry`: (String) Default registry for all the supported test frameworks (tempest, tobiko, horizontest and ansibletest) to pull their container images. It can be overridden at test framework level. Defaults to `quay.io` +* `cifmw_test_operator_default_namespace`: (String) Default registry's namespace for all the supported test frameworks (tempest, tobiko, horizontest and ansibletest) to pull their container images. It can be overridden at test framework level. Defaults to `podified-antelope-centos9` +* `cifmw_test_operator_default_image_tag`: (String) Default tag for all the supported test frameworks (tempest, tobiko, horizontest and ansibletest) to pull their container images. It can be overridden at test framework level. Defaults to `current-podified` * `cifmw_test_operator_stages`: (List) List of dictionaries defining the stages that should be used in the test operator role. List items options are: * `name`: (String) The name of the stage. The name must be unique. * `type`: (String) The framework name you would like to call, currently the options are: tempest, ansibletest, horizontest, tobiko. * `test_vars_file`: (String) Path to the file used for testing, this file should contain the testing params for this stage. Only parameters specific for the controller can be used (Tempest, Ansibletest, Horizontest and Tobiko). * `test_vars`: (String) Testing parameters for this specific stage if a `test_vars` is used the specified parameters would override the ones in the `test_vars_file`. Only parameters specific for the controller can be used (Tempest, Ansibletest, Horizontest and Tobiko). - > Important note! Only variables with the following structure can be used to override inside a stage: `cifmw_test_operator_[test-operator CR name]_[parameter name]` the only exception to that is `cifmw_test_operator_concurrency` which can not be override inside a stage. + > Important note! Only variables with the following structure can be used to override inside a stage: `cifmw_test_operator_[test-operator CR name]_[parameter name]`. For example, these variables cannot be overridden per stage: `cifmw_test_operator_concurrency`, `cifmw_test_operator_default_registry`, `cifmw_test_operator_default_namespace`, `cifmw_test_operator_default_image_tag`. * `pre_test_stage_hooks`: (List) List of pre hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format). * `post_test_stage_hooks`: (List) List of post hooks to run as described [hooks README](https://github.com/openstack-k8s-operators/ci-framework/tree/main/roles/run_hook#hooks-expected-format). Default value: @@ -39,13 +42,14 @@ cifmw_test_operator_stages: type: tempest ``` + ## Tempest specific parameters * `cifmw_test_operator_tempest_name`: (String) Value used in the `Tempest.Metadata.Name` field. The value specifies the name of some resources spawned by the test-operator role. Default value: `tempest-tests` -* `cifmw_test_operator_tempest_registry`: (String) The registry where to pull tempest container. Default value: `quay.io` -* `cifmw_test_operator_tempest_namespace`: (String) Registry's namespace where to pull tempest container. Default value: `podified-antelope-centos9` +* `cifmw_test_operator_tempest_registry`: (String) The registry where to pull tempest container. Default value: `{{ cifmw_test_operator_default_registry }}` +* `cifmw_test_operator_tempest_namespace`: (String) Registry's namespace where to pull tempest container. Default value: `{{ cifmw_test_operator_default_namespace }}` * `cifmw_test_operator_tempest_container`: (String) Name of the tempest container. Default value: `openstack-tempest` * `cifmw_test_operator_tempest_image`: (String) Tempest image to be used. Default value: `{{ cifmw_test_operator_tempest_registry }}/{{ cifmw_test_operator_tempest_namespace }}/{{ cifmw_test_operator_tempest_container }}` -* `cifmw_test_operator_tempest_image_tag`: (String) Tag for the `cifmw_test_operator_tempest_image`. Default value: `current-podified` +* `cifmw_test_operator_tempest_image_tag`: (String) Tag for the `cifmw_test_operator_tempest_image`. Default value: `{{ cifmw_test_operator_default_image_tag }}` * `cifmw_test_operator_tempest_include_list`: (String) List of tests to be executed. Setting this will not use the `list_allowed` plugin. Default value: `''` * `cifmw_test_operator_tempest_exclude_list`: (String) List of tests to be skipped. Setting this will not use the `list_skipped` plugin. Default value: `''` * `cifmw_test_operator_tempest_expected_failures_list`: (String) List of tests for which failures will be ignored. Default value: `''` @@ -61,6 +65,7 @@ cifmw_test_operator_stages: * `cifmw_test_operator_tempest_extra_rpms`: (List) . A list of URLs that point to RPMs that should be installed before the execution of tempest. Note that this parameter has no effect when `cifmw_test_operator_tempest_external_plugin` is used. Default value: `[]` * `cifmw_test_operator_tempest_extra_configmaps_mounts`: (List) A list of configmaps that should be mounted into the tempest test pods. Default value: `[]` * `cifmw_test_operator_tempest_debug`: (Bool) Run Tempest in debug mode, it keeps the operator pod sleeping infinity (it must only set to `true`only for debugging purposes). Default value: `false` +* `cifmw_test_operator_tempest_resources`: (Dict) A dictionary that specifies resources (cpu, memory) for the test pods. When untouched it clears the default values set on the test-operator side. This means that the tempest test pods run with unspecified resource limits. Default value: `{requests: {}, limits: {}}` * `cifmw_tempest_tempestconf_config`: Deprecated, please use `cifmw_test_operator_tempest_tempestconf_config` instead * `cifmw_test_operator_tempest_tempestconf_config`: (Dict) This parameter can be used to customize the execution of the `discover-tempest-config` run. Please consult the test-operator documentation. For example, to pass a custom configuration for `tempest.conf`, use the `overrides` section: ``` @@ -97,16 +102,17 @@ Default value: {} ## Tobiko specific parameters * `cifmw_test_operator_tobiko_name`: (String) Value used in the `Tobiko.Metadata.Name` field. The value specifies the name of some resources spawned by the test-operator role. Default value: `tobiko-tests` -* `cifmw_test_operator_tobiko_registry`: (String) The registry where to pull tobiko container. Default value: `quay.io` -* `cifmw_test_operator_tobiko_namespace`: (String) Registry's namespace where to pull tobiko container. Default value: `podified-antelope-centos9` +* `cifmw_test_operator_tobiko_registry`: (String) The registry where to pull tobiko container. Default value: `{{ cifmw_test_operator_default_registry }}` +* `cifmw_test_operator_tobiko_namespace`: (String) Registry's namespace where to pull tobiko container. Default value: `{{ cifmw_test_operator_default_namespace }}` * `cifmw_test_operator_tobiko_container`: (String) Name of the tobiko container. Default value: `openstack-tobiko` * `cifmw_test_operator_tobiko_image`: (String) Tobiko image to be used. Default value: `{{ cifmw_test_operator_tobiko_registry }}/{{ cifmw_test_operator_tobiko_namespace }}/{{ cifmw_test_operator_tobiko_container }}` -* `cifmw_test_operator_tobiko_image_tag`: (String) Tag for the `cifmw_test_operator_tobiko_image`. Default value: `current-podified` +* `cifmw_test_operator_tobiko_image_tag`: (String) Tag for the `cifmw_test_operator_tobiko_image`. Default value: `{{ cifmw_test_operator_default_image_tag }}` * `cifmw_test_operator_tobiko_testenv`: (String) Executed tobiko testenv. See tobiko `tox.ini` file for further details. Some allowed values: scenario, sanity, faults, neutron, octavia, py3, etc. Default value: `scenario` * `cifmw_test_operator_tobiko_version`: (String) Tobiko version to install. It could refer to a branch (master, osp-16.2), a tag (0.6.x, 0.7.x) or an sha-1. Default value: `master` * `cifmw_test_operator_tobiko_pytest_addopts`: (String) `PYTEST_ADDOPTS` env variable with input pytest args. Example: `-m --maxfail --skipregex `. Defaults to `null`. In case of `null` value, `PYTEST_ADDOPTS` is not set (tobiko tests are executed without any extra pytest options). * `cifmw_test_operator_tobiko_prevent_create`: (Boolean) Sets the value of the env variable `TOBIKO_PREVENT_CREATE` that specifies whether tobiko scenario tests create new resources or expect that those resource had been created before. Default to `null`. In case of `null` value, `TOBIKO_PREVENT_CREATE` is not set (tobiko tests create new resources). * `cifmw_test_operator_tobiko_num_processes`: (Integer) Sets the value of the env variable `TOX_NUM_PROCESSES` that is used to run pytest with `--numprocesses $TOX_NUM_PROCESSES`. Defaults to `null`. In case of `null` value, `TOX_NUM_PROCESSES` is not set (tobiko internally uses the value `auto`, see pytest documentation about the `--numprocesses` option). +* `cifmw_test_operator_tobiko_advanced_image_url`: (String) Tobiko will download images from this URL that will be used to create advance VM instances. By default, the provided image will include all the customizations required by the tobiko tests. Defaults to `https://softwarefactory-project.io/ubuntu-minimal-customized-enp3s0`. * `cifmw_test_operator_tobiko_kubeconfig_secret`: (String) Name of the Openshift Secret required to use Openshift Client from the Tobiko pod. Default value: `tobiko-secret` * `cifmw_test_operator_tobiko_override_conf`: (Dict) Overrides the default configuration from `cifmw_test_operator_tobiko_default_conf` that is used to generate the tobiko.conf file. Default value: empty dictionary * `cifmw_test_operator_tobiko_ssh_keytype`: (String) Type of ssh key that tobiko will use to connect to the VM instances it creates. Defaults to `cifmw_ssh_keytype` which default to `ecdsa`. @@ -114,6 +120,7 @@ Default value: {} * `cifmw_test_operator_tobiko_debug`: (Bool) Run Tobiko in debug mode, it keeps the operator pod sleeping infinity (it must only set to `true`only for debugging purposes). Default value: `false` * `cifmw_test_operator_tobiko_network_attachments`: (List) List of network attachment definitions to attach to the tobiko pods spawned by test-operator. Default value: `[]`. * `cifmw_test_operator_tobiko_workflow`: (List) Definition of a Tobiko workflow that consists of multiple steps. Each step can contain all values from Spec section of [Tobiko CR](https://openstack-k8s-operators.github.io/test-operator/crds.html#tobiko-custom-resource). +* `cifmw_test_operator_tobiko_resources`: (Dict) A dictionary that specifies resources (cpu, memory) for the test pods. When kept untouched it defaults to the resource limits specified on the test-operator side. Default value: `{}` * `cifmw_test_operator_tobiko_config`: (Dict) Definition of Tobiko CRD instance that is passed to the test-operator (see [the test-operator documentation](https://openstack-k8s-operators.github.io/test-operator/crds.html#tobiko-custom-resource)). Default value: ``` apiVersion: test.openstack.org/v1beta1 @@ -142,11 +149,11 @@ Default value: {} ## AnsibleTest specific parameters * `cifmw_test_operator_ansibletest_name`: (String) Value used in the `Ansibletest.Metadata.Name` field. The value specifies the name of some resources spawned by the test-operator role. Default value: `ansibletest` -* `cifmw_test_operator_ansibletest_registry`: (String) The registry where to pull ansibletests container. Default value: `quay.io` -* `cifmw_test_operator_ansibletest_namespace`: (String) Registry's namespace where to pull ansibletests container. Default value:podified-antelope-centos9 +* `cifmw_test_operator_ansibletest_registry`: (String) The registry where to pull ansibletests container. Default value: `{{ cifmw_test_operator_default_registry }}` +* `cifmw_test_operator_ansibletest_namespace`: (String) Registry's namespace where to pull ansibletests container. Default value: `{{ cifmw_test_operator_default_namespace }}` * `cifmw_test_operator_ansibletest_container`: (String) Name of the ansibletests container. Default value: `openstack-ansible-tests` * `cifmw_test_operator_ansibletest_image`: (String) Ansibletests image to be used. Default value: `{{ cifmw_test_operator_ansibletest_registry }}/{{ cifmw_test_operator_ansibletest_namespace }}/{{ cifmw_test_operator_ansibletest_container }}` -* `cifmw_test_operator_ansibletest_image_tag`: (String) Ansibletests image to be used. Default value: `current-podified` +* `cifmw_test_operator_ansibletest_image_tag`: (String) Ansibletests image to be used. Default value: `{{ cifmw_test_operator_default_image_tag }}` * `cifmw_test_operator_ansibletest_compute_ssh_key_secret_name`: (String) The name of the k8s secret that contains an ssh key for computes. Default value: `dataplane-ansible-ssh-private-key-secret` * `cifmw_test_operator_ansibletest_workload_ssh_key_secret_name`: (String) The name of the k8s secret that contains an ssh key for the ansible workload. Default value: `""` * `cifmw_test_operator_ansibletest_ansible_git_repo`: (String) Git repo to clone into container. Default value: `""` @@ -160,6 +167,7 @@ Default value: {} * `cifmw_test_operator_ansibletest_debug`: (Bool) Run ansible playbook with -vvvv. Default value: `false` * `cifmw_test_operator_ansibletest_workflow`: (List) A parameter that contains a workflow definition. Default value: `[]` * `cifmw_test_operator_ansibletest_extra_configmaps_mounts`: (List) Extra configmaps for mounting in the pod. Default value: `[]` +* `cifmw_test_operator_ansibletest_resources`: (Dict) A dictionary that specifies resources (cpu, memory) for the test pods. When kept untouched it defaults to the resource limits specified on the test-operator side. Default value: `{}` * `cifmw_test_operator_ansibletest_config`: Definition of AnsibleTest CRD instance that is passed to the test-operator (see [the test-operator documentation](https://openstack-k8s-operators.github.io/test-operator/crds.html)). Default value: ``` apiVersion: test.openstack.org/v1beta1 @@ -187,11 +195,11 @@ Default value: {} ## Horizontest specific parameters * `cifmw_test_operator_horizontest_name`: (String) Value used in the `Horizontest.Metadata.Name` field. The value specifies the name of some resources spawned by the test-operator role. Default value: `horizontest-tests` -* `cifmw_test_operator_horizontest_registry`: (String) The registry where to pull horizontest container. Default value: `quay.io` -* `cifmw_test_operator_horizontest_namespace`: (String) Registry's namespace where to pull horizontest container. Default value: `podified-antelope-centos9` +* `cifmw_test_operator_horizontest_registry`: (String) The registry where to pull horizontest container. Default value: `{{ cifmw_test_operator_default_registry }}` +* `cifmw_test_operator_horizontest_namespace`: (String) Registry's namespace where to pull horizontest container. Default value: `{{ cifmw_test_operator_default_namespace }}` * `cifmw_test_operator_horizontest_container`: (String) Name of the horizontest container. Default value: `openstack-horizontest` * `cifmw_test_operator_horizontest_image`: (String) Horizontest image to be used. Default value: `{{ cifmw_test_operator_horizontest_registry }}/{{ cifmw_test_operator_horizontest_namespace }}/{{ cifmw_test_operator_horizontest_container }}` -* `cifmw_test_operator_horizontest_image_tag`: (String) Tag for the `cifmw_test_operator_horizontest_image`. Default value: `current-podified` +* `cifmw_test_operator_horizontest_image_tag`: (String) Tag for the `cifmw_test_operator_horizontest_image`. Default value: `{{ cifmw_test_operator_default_image_tag }}` * `cifmw_test_operator_horizontest_admin_username`: (String) OpenStack admin credentials. Default value: `admin` * `cifmw_test_operator_horizontest_admin_password`: (String) OpenStack admin credentials. Default value: `12345678` * `cifmw_test_operator_horizontest_dashboard_url`: (String) The URL of the Horizon dashboard. Default value: `https://horizon-openstack.apps.ocp.openstack.lab/` @@ -205,6 +213,8 @@ Default value: {} * `cifmw_test_operator_horizontest_flavor_name`: (String) The name of the OpenStack flavor to create for Horizon tests. Default value: `m1.tiny` * `cifmw_test_operator_horizontest_logs_directory_name`: (String) The name of the directory to store test logs. Default value: `horizon` * `cifmw_test_operator_horizontest_horizon_test_dir`: (String) The directory path for Horizon tests. Default value: `/var/lib/horizontest` +* `cifmw_test_operator_horizontest_resources`: (Dict) A dictionary that specifies resources (cpu, memory) for the test pods. When kept untouched it defaults to the resource limits specified on the test-operator side. Default value: `{}` +* `cifmw_test_operator_horizontest_debug`: (Bool) Run HorizonTest in debug mode, it keeps the operator pod sleeping infinitely (it must only set to `true` only for debugging purposes). Default value: `false` * `cifmw_test_operator_horizontest_config`: (Dict) Definition of HorizonTest CR instance that is passed to the test-operator (see [the test-operator documentation](https://openstack-k8s-operators.github.io/test-operator/crds.html#horizontest-custom-resource)). Default value: ``` apiVersion: test.openstack.org/v1beta1 @@ -226,6 +236,7 @@ Default value: {} password: "{{ cifmw_test_operator_horizontest_password }}" flavorName: "{{ cifmw_test_operator_horizontest_flavor_name }}" logsDirectoryName: "{{ cifmw_test_operator_horizontest_logs_directory_name }}" + debug: "{{ cifmw_test_operator_horizontest_debug }}" horizonTestDir: "{{ cifmw_test_operator_horizontest_horizon_test_dir }}" ``` diff --git a/roles/test_operator/defaults/main.yml b/roles/test_operator/defaults/main.yml index cac69a585b..4863f1b928 100644 --- a/roles/test_operator/defaults/main.yml +++ b/roles/test_operator/defaults/main.yml @@ -41,14 +41,18 @@ cifmw_test_operator_storage_class: "{{ cifmw_test_operator_storage_class_prefix cifmw_test_operator_delete_logs_pod: false cifmw_test_operator_privileged: true cifmw_test_operator_selinux_level: "s0:c478,c978" +# default test framework registry, namespace and tag can be overridden per test framework (tempest, tobiko, horizontest and ansibletest) +cifmw_test_operator_default_registry: quay.io +cifmw_test_operator_default_namespace: podified-antelope-centos9 +cifmw_test_operator_default_image_tag: current-podified # Section 2: tempest parameters - used when run_test_fw is 'tempest' cifmw_test_operator_tempest_name: "tempest-tests" -cifmw_test_operator_tempest_registry: quay.io -cifmw_test_operator_tempest_namespace: podified-antelope-centos9 +cifmw_test_operator_tempest_registry: "{{ cifmw_test_operator_default_registry }}" +cifmw_test_operator_tempest_namespace: "{{ cifmw_test_operator_default_namespace }}" cifmw_test_operator_tempest_container: openstack-tempest-all cifmw_test_operator_tempest_image: "{{ stage_vars_dict.cifmw_test_operator_tempest_registry }}/{{ stage_vars_dict.cifmw_test_operator_tempest_namespace }}/{{ stage_vars_dict.cifmw_test_operator_tempest_container }}" -cifmw_test_operator_tempest_image_tag: current-podified +cifmw_test_operator_tempest_image_tag: "{{ cifmw_test_operator_default_image_tag }}" cifmw_test_operator_tempest_network_attachments: [] cifmw_test_operator_tempest_tests_include_override_scenario: false cifmw_test_operator_tempest_tests_exclude_override_scenario: false @@ -56,6 +60,13 @@ cifmw_test_operator_tempest_workflow: [] cifmw_test_operator_tempest_cleanup: false cifmw_test_operator_tempest_tempestconf_config: "{{ cifmw_tempest_tempestconf_config }}" +# TODO: The default value of this parameter should be changed to {} once this fix +# for tempest reaches the upstream build of the openstack-tempest-all image: +# https://review.opendev.org/c/openstack/tempest/+/934980 +cifmw_test_operator_tempest_resources: + requests: {} + limits: {} + # Enabling SRBAC by default, in jobs where this does not make sense should be turned off explicitly # # auth.tempest_roles is set to an empty value because otherwise @@ -119,6 +130,7 @@ cifmw_test_operator_tempest_config: tolerations: "{{ cifmw_test_operator_tolerations | default(omit) }}" nodeSelector: "{{ cifmw_test_operator_node_selector | default(omit) }}" extraConfigmapsMounts: "{{ stage_vars_dict.cifmw_test_operator_tempest_extra_configmaps_mounts | default(omit) }}" + resources: "{{ stage_vars_dict.cifmw_test_operator_tempest_resources }}" tempestRun: includeList: | {{ stage_vars_dict.cifmw_test_operator_tempest_include_list | default('') }} @@ -137,16 +149,17 @@ cifmw_test_operator_tempest_config: # Section 3: tobiko parameters - used when run_test_fw is 'tobiko' cifmw_test_operator_tobiko_name: "tobiko-tests" -cifmw_test_operator_tobiko_registry: quay.io -cifmw_test_operator_tobiko_namespace: podified-antelope-centos9 +cifmw_test_operator_tobiko_registry: "{{ cifmw_test_operator_default_registry }}" +cifmw_test_operator_tobiko_namespace: "{{ cifmw_test_operator_default_namespace }}" cifmw_test_operator_tobiko_container: openstack-tobiko cifmw_test_operator_tobiko_image: "{{ stage_vars_dict.cifmw_test_operator_tobiko_registry }}/{{ stage_vars_dict.cifmw_test_operator_tobiko_namespace }}/{{ stage_vars_dict.cifmw_test_operator_tobiko_container }}" -cifmw_test_operator_tobiko_image_tag: current-podified +cifmw_test_operator_tobiko_image_tag: "{{ cifmw_test_operator_default_image_tag }}" cifmw_test_operator_tobiko_testenv: scenario cifmw_test_operator_tobiko_version: master cifmw_test_operator_tobiko_pytest_addopts: null cifmw_test_operator_tobiko_prevent_create: null cifmw_test_operator_tobiko_num_processes: null +cifmw_test_operator_tobiko_advanced_image_url: "https://softwarefactory-project.io/ubuntu-minimal-customized-enp3s0" cifmw_test_operator_tobiko_override_conf: {} cifmw_test_operator_tobiko_kubeconfig_secret: tobiko-secret cifmw_test_operator_tobiko_ssh_keytype: "{{ cifmw_ssh_keytype | default('ecdsa') }}" @@ -154,6 +167,7 @@ cifmw_test_operator_tobiko_ssh_keysize: "{{ cifmw_ssh_keysize | default(521) }}" cifmw_test_operator_tobiko_debug: false cifmw_test_operator_tobiko_network_attachments: [] cifmw_test_operator_tobiko_workflow: [] +cifmw_test_operator_tobiko_resources: {} cifmw_test_operator_tobiko_config: apiVersion: test.openstack.org/v1beta1 kind: Tobiko @@ -173,6 +187,7 @@ cifmw_test_operator_tobiko_config: nodeSelector: "{{ cifmw_test_operator_node_selector | default(omit) }}" debug: "{{ stage_vars_dict.cifmw_test_operator_tobiko_debug }}" networkAttachments: "{{ stage_vars_dict.cifmw_test_operator_tobiko_network_attachments }}" + resources: "{{ stage_vars_dict.cifmw_test_operator_tobiko_resources }}" # preventCreate: preventCreate is generated by the test_operator role based on the value of stage_vars_dict.cifmw_test_operator_tobiko_prevent_create # numProcesses: numProcesses is generated by the test_operator role based on the value of stage_vars_dict.cifmw_test_operator_tobiko_num_processes # privateKey: privateKey is automatically by the test_operator role @@ -182,11 +197,11 @@ cifmw_test_operator_tobiko_config: # Section 4: ansibletest parameters - used when run_test_fw is 'ansibletest' cifmw_test_operator_ansibletest_name: "ansibletest" -cifmw_test_operator_ansibletest_registry: quay.io -cifmw_test_operator_ansibletest_namespace: podified-antelope-centos9 +cifmw_test_operator_ansibletest_registry: "{{ cifmw_test_operator_default_registry }}" +cifmw_test_operator_ansibletest_namespace: "{{ cifmw_test_operator_default_namespace }}" cifmw_test_operator_ansibletest_container: openstack-ansible-tests cifmw_test_operator_ansibletest_image: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_registry }}/{{ stage_vars_dict.cifmw_test_operator_ansibletest_namespace }}/{{ stage_vars_dict.cifmw_test_operator_ansibletest_container }}" -cifmw_test_operator_ansibletest_image_tag: current-podified +cifmw_test_operator_ansibletest_image_tag: "{{ cifmw_test_operator_default_image_tag }}" cifmw_test_operator_ansibletest_compute_ssh_key_secret_name: "dataplane-ansible-ssh-private-key-secret" cifmw_test_operator_ansibletest_workload_ssh_key_secret_name: "" cifmw_test_operator_ansibletest_ansible_git_repo: "" @@ -200,6 +215,7 @@ cifmw_test_operator_ansibletest_openstack_config_secret: "openstack-config-secre cifmw_test_operator_ansibletest_debug: false cifmw_test_operator_ansibletest_workflow: [] cifmw_test_operator_ansibletest_extra_configmaps_mounts: [] +cifmw_test_operator_ansibletest_resources: {} cifmw_test_operator_ansibletest_config: apiVersion: test.openstack.org/v1beta1 kind: AnsibleTest @@ -224,14 +240,15 @@ cifmw_test_operator_ansibletest_config: openStackConfigSecret: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_openstack_config_secret }}" workflow: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_workflow }}" debug: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_debug }}" + resources: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_resources }}" # Section 5: horizontest parameters - used when run_test_fw is 'horizontest' cifmw_test_operator_horizontest_name: "horizontest-tests" -cifmw_test_operator_horizontest_registry: quay.io -cifmw_test_operator_horizontest_namespace: podified-antelope-centos9 +cifmw_test_operator_horizontest_registry: "{{ cifmw_test_operator_default_registry }}" +cifmw_test_operator_horizontest_namespace: "{{ cifmw_test_operator_default_namespace }}" cifmw_test_operator_horizontest_container: openstack-horizontest cifmw_test_operator_horizontest_image: "{{ stage_vars_dict.cifmw_test_operator_horizontest_registry }}/{{ stage_vars_dict.cifmw_test_operator_horizontest_namespace }}/{{ stage_vars_dict.cifmw_test_operator_horizontest_container }}" -cifmw_test_operator_horizontest_image_tag: current-podified +cifmw_test_operator_horizontest_image_tag: "{{ cifmw_test_operator_default_image_tag }}" cifmw_test_operator_horizontest_admin_username: admin cifmw_test_operator_horizontest_admin_password: "12345678" cifmw_test_operator_horizontest_dashboard_url: "https://horizon-openstack.apps.ocp.openstack.lab/" @@ -244,7 +261,9 @@ cifmw_test_operator_horizontest_user: "horizontest" cifmw_test_operator_horizontest_password: "horizontest" cifmw_test_operator_horizontest_flavor_name: "m1.tiny" cifmw_test_operator_horizontest_logs_directory_name: "horizon" +cifmw_test_operator_horizontest_debug: false cifmw_test_operator_horizontest_horizon_test_dir: "/var/lib/horizontest" +cifmw_test_operator_horizontest_resources: {} cifmw_test_operator_horizontest_config: apiVersion: test.openstack.org/v1beta1 kind: HorizonTest @@ -268,4 +287,6 @@ cifmw_test_operator_horizontest_config: password: "{{ stage_vars_dict.cifmw_test_operator_horizontest_password }}" flavorName: "{{ stage_vars_dict.cifmw_test_operator_horizontest_flavor_name }}" logsDirectoryName: "{{ stage_vars_dict.cifmw_test_operator_horizontest_logs_directory_name }}" + debug: "{{ stage_vars_dict.cifmw_test_operator_horizontest_debug }}" horizonTestDir: "{{ stage_vars_dict.cifmw_test_operator_horizontest_horizon_test_dir }}" + resources: "{{ stage_vars_dict.cifmw_test_operator_horizontest_resources }}" diff --git a/roles/test_operator/tasks/run-test-operator-job.yml b/roles/test_operator/tasks/run-test-operator-job.yml index 2299b250b3..887bcd809c 100644 --- a/roles/test_operator/tasks/run-test-operator-job.yml +++ b/roles/test_operator/tasks/run-test-operator-job.yml @@ -37,35 +37,35 @@ definition: "{{ test_operator_cr }}" when: not cifmw_test_operator_dry_run | bool -- name: Wait for the last job to be Completed - {{ run_test_fw }} +- name: Wait for the last Pod to be Completed - {{ run_test_fw }} kubernetes.core.k8s_info: kubeconfig: "{{ cifmw_openshift_kubeconfig }}" api_key: "{{ cifmw_openshift_token | default(omit) }}" context: "{{ cifmw_openshift_context | default(omit) }}" namespace: "{{ cifmw_test_operator_namespace }}" - kind: Job + kind: Pod label_selectors: - "workflowStep={{ [(test_operator_workflow | length) - 1, 0] | max }}" - - "instanceName={{ test_operator_job_name }}" + - "instanceName={{ test_operator_instance_name }}" retries: "{{ (cifmw_test_operator_timeout / 10) | round | int }}" delay: 10 until: > - testjob.resources[0].status.succeeded | default(0) | int >= 1 or - testjob.resources[0].status.failed | default(0) | int >= 1 + testpod.resources[0].status.phase | default(omit) == "Succeeded" or + testpod.resources[0].status.phase | default(omit) == "Failed" ignore_errors: true - register: testjob + register: testpod when: not cifmw_test_operator_dry_run | bool - name: Check whether timed out - {{ run_test_fw }} ansible.builtin.set_fact: - testjob_timed_out: >- - {{ testjob.attempts == (cifmw_test_operator_timeout / 10) | round | int }} + testpod_timed_out: >- + {{ testpod.attempts == (cifmw_test_operator_timeout / 10) | round | int }} when: not cifmw_test_operator_dry_run | bool - name: Collect logs when: - not cifmw_test_operator_dry_run | bool - - not testjob_timed_out + - not testpod_timed_out block: - name: Reset volumes and volume_mounts to an empty list ansible.builtin.set_fact: @@ -79,7 +79,7 @@ context: "{{ cifmw_openshift_context | default(omit)}}" kind: PersistentVolumeClaim label_selectors: - - "instanceName={{ test_operator_job_name }}" + - "instanceName={{ test_operator_instance_name }}" register: logsPVCs - name: Set up volume mounts and volumes for all PVCs @@ -88,7 +88,7 @@ {{ (volume_mounts | default([])) + [{ 'name': "logs-volume-" ~ index, - 'mountPath': "/mnt/logs-{{ test_operator_job_name }}-step-" ~ index + 'mountPath': "/mnt/logs-{{ test_operator_instance_name }}-step-" ~ index }] }} volumes: > @@ -115,7 +115,7 @@ apiVersion: v1 kind: Pod metadata: - name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_job_name }}" + name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}" namespace: "{{ cifmw_test_operator_namespace }}" spec: containers: @@ -134,7 +134,7 @@ context: "{{ cifmw_openshift_context | default(omit) }}" namespace: "{{ cifmw_test_operator_namespace }}" kind: Pod - name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_job_name }}" + name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}" wait: true register: logs_pod until: logs_pod.resources[0].status.phase == "Running" @@ -146,10 +146,10 @@ KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" PATH: "{{ cifmw_path }}" vars: - pod_path: mnt/logs-{{ test_operator_job_name }}-step-{{ index }} + pod_path: mnt/logs-{{ test_operator_instance_name }}-step-{{ index }} ansible.builtin.shell: > oc cp -n {{ cifmw_test_operator_namespace }} - openstack/test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_job_name }}:{{ pod_path }} + openstack/test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}:{{ pod_path }} {{ cifmw_test_operator_artifacts_basedir }} loop: "{{ logsPVCs.resources }}" loop_control: @@ -174,7 +174,7 @@ namespace: "{{ cifmw_test_operator_namespace }}" kind: Pod label_selectors: - - "instanceName={{ test_operator_job_name }}" + - "instanceName={{ test_operator_instance_name }}" when: not cifmw_test_operator_dry_run | bool - name: Get status from test pods @@ -225,7 +225,7 @@ kind: "{{ test_operator_kind_name }}" state: absent api_version: test.openstack.org/v1beta1 - name: "{{ test_operator_job_name }}" + name: "{{ test_operator_instance_name }}" namespace: "{{ cifmw_test_operator_namespace }}" wait: true wait_timeout: 600 @@ -251,7 +251,7 @@ kind: Pod state: absent api_version: v1 - name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_job_name }}" + name: "test-operator-logs-pod-{{ run_test_fw }}-{{ test_operator_instance_name }}" namespace: "{{ cifmw_test_operator_namespace }}" wait: true wait_timeout: 600 diff --git a/roles/test_operator/tasks/runners/ansibletest_runner.yml b/roles/test_operator/tasks/runners/ansibletest_runner.yml index 80b0285bfa..0763a034c3 100644 --- a/roles/test_operator/tasks/runners/ansibletest_runner.yml +++ b/roles/test_operator/tasks/runners/ansibletest_runner.yml @@ -3,7 +3,7 @@ vars: run_test_fw: ansibletest test_operator_config: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_config }}" - test_operator_job_name: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_name }}-{{ _stage_vars.name }}" + test_operator_instance_name: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_name }}-{{ _stage_vars.name }}" test_operator_kind_name: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_kind_name }}" test_operator_crd_name: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_crd_name }}" test_operator_workflow: "{{ stage_vars_dict.cifmw_test_operator_ansibletest_workflow }}" diff --git a/roles/test_operator/tasks/runners/horizontest_runner.yml b/roles/test_operator/tasks/runners/horizontest_runner.yml index dd772617b3..63c5121d8a 100644 --- a/roles/test_operator/tasks/runners/horizontest_runner.yml +++ b/roles/test_operator/tasks/runners/horizontest_runner.yml @@ -3,7 +3,7 @@ vars: run_test_fw: horizontest test_operator_config: "{{ stage_vars_dict.cifmw_test_operator_horizontest_config }}" - test_operator_job_name: "{{ stage_vars_dict.cifmw_test_operator_horizontest_name }}-{{ _stage_vars.name }}" + test_operator_instance_name: "{{ stage_vars_dict.cifmw_test_operator_horizontest_name }}-{{ _stage_vars.name }}" test_operator_kind_name: "{{ stage_vars_dict.cifmw_test_operator_horizontest_kind_name }}" test_operator_crd_name: "{{ stage_vars_dict.cifmw_test_operator_horizontest_crd_name }}" test_operator_workflow: [] diff --git a/roles/test_operator/tasks/runners/tempest_runner.yml b/roles/test_operator/tasks/runners/tempest_runner.yml index 876f0a2c3a..1385b0fa9a 100644 --- a/roles/test_operator/tasks/runners/tempest_runner.yml +++ b/roles/test_operator/tasks/runners/tempest_runner.yml @@ -3,7 +3,7 @@ vars: run_test_fw: tempest test_operator_config: "{{ stage_vars_dict.cifmw_test_operator_tempest_config }}" - test_operator_job_name: "{{ stage_vars_dict.cifmw_test_operator_tempest_name }}-{{ _stage_vars.name }}" + test_operator_instance_name: "{{ stage_vars_dict.cifmw_test_operator_tempest_name }}-{{ _stage_vars.name }}" test_operator_kind_name: "{{ stage_vars_dict.cifmw_test_operator_tempest_kind_name }}" test_operator_crd_name: "{{ stage_vars_dict.cifmw_test_operator_tempest_crd_name }}" test_operator_workflow: "{{ stage_vars_dict.cifmw_test_operator_tempest_workflow }}" diff --git a/roles/test_operator/tasks/runners/tobiko_runner.yml b/roles/test_operator/tasks/runners/tobiko_runner.yml index 92ea903615..da663290be 100644 --- a/roles/test_operator/tasks/runners/tobiko_runner.yml +++ b/roles/test_operator/tasks/runners/tobiko_runner.yml @@ -4,7 +4,7 @@ vars: run_test_fw: tobiko test_operator_config: "{{ stage_vars_dict.cifmw_test_operator_tobiko_config }}" - test_operator_job_name: "{{ stage_vars_dict.cifmw_test_operator_tobiko_name }}-{{ _stage_vars.name }}" + test_operator_instance_name: "{{ stage_vars_dict.cifmw_test_operator_tobiko_name }}-{{ _stage_vars.name }}" test_operator_kind_name: "{{ stage_vars_dict.cifmw_test_operator_tobiko_kind_name }}" test_operator_crd_name: "{{ stage_vars_dict.cifmw_test_operator_tobiko_crd_name }}" test_operator_workflow: "{{ stage_vars_dict.cifmw_test_operator_tobiko_workflow }}" diff --git a/roles/test_operator/tasks/tempest-tests.yml b/roles/test_operator/tasks/tempest-tests.yml index 5aa7539bef..a7fc5ef2c1 100644 --- a/roles/test_operator/tasks/tempest-tests.yml +++ b/roles/test_operator/tasks/tempest-tests.yml @@ -29,7 +29,7 @@ tempest_list_allowed: yaml_file: "{{ cifmw_test_operator_artifacts_basedir }}/list_allowed.yml" groups: "{{ cifmw_test_operator_default_groups }}" - job: "{{ cifmw_test_operator_job_name | default(omit) }}" + job: "{{ cifmw_test_operator_instance_name | default(omit) }}" register: list_allowed diff --git a/roles/test_operator/vars/main.yml b/roles/test_operator/vars/main.yml index aac84f627f..20fb5d3537 100644 --- a/roles/test_operator/vars/main.yml +++ b/roles/test_operator/vars/main.yml @@ -35,7 +35,8 @@ cifmw_test_operator_tobiko_default_conf: test_runner_timeout: 14400.0 ubuntu: interface_name: enp3s0 - image_file: /usr/local/share/ubuntu-minimal + customized_image_provided: "True" + image_url: "{{ cifmw_test_operator_tobiko_advanced_image_url }}" keystone: interface: public manila: diff --git a/roles/update/README.md b/roles/update/README.md index 73fac13004..a42b8c5e9e 100644 --- a/roles/update/README.md +++ b/roles/update/README.md @@ -12,6 +12,9 @@ Role to run update * `cifmw_update_ping_loss_second` : (Integer) Number of seconds that the ping test is allowed to fail. Default to `0`. Note that 1 packet loss is always accepted to avoid false positive. * `cifmw_update_ping_loss_percent` : (Integer) Maximum percentage of ping loss accepted. Default to `0`. Only relevant when `cifmw_update_ping_loss_second` is not 0. * `cifmw_update_control_plane_check`: (Boolean) Activate a continuous control plane testing. Default to `False` +* `cifmw_update_ctl_plane_max_cons_fail`: (Integer) For continuous control plane testing, maximum number of consecutive failures allowed. Default to 2. +* `cifmw_update_ctl_plane_max_fail`: (Integer) For continuous control plane testing, maximum number of failures allowed. Default to 3. +* `cifmw_update_ctl_plane_max_tries`: (Integer) For continuous control plane testing, number of retries allowed to stop and destroy the last vm created. Each retry is 5 seconds apart. Default to 84, so 7 minutes. * `cifmw_update_openstackclient_pod_timeout`: (Integer) Maximum number of seconds to wait for the openstackclient Pod to be available during control plane testing, as it is being restarted during update. Default to `10` seconds. ## Examples diff --git a/roles/update/defaults/main.yml b/roles/update/defaults/main.yml index 527c6361f2..4d72c61a14 100644 --- a/roles/update/defaults/main.yml +++ b/roles/update/defaults/main.yml @@ -47,3 +47,6 @@ cifmw_update_ping_loss_percent: 0 # Control plane Testing cifmw_update_control_plane_check: false cifmw_update_openstackclient_pod_timeout: 10 # in seconds. +cifmw_update_ctl_plane_max_cons_fail: 2 +cifmw_update_ctl_plane_max_fail: 3 +cifmw_update_ctl_plane_max_tries: 84 diff --git a/roles/update/tasks/main.yml b/roles/update/tasks/main.yml index 6fbd74e4a3..93987dd205 100644 --- a/roles/update/tasks/main.yml +++ b/roles/update/tasks/main.yml @@ -72,5 +72,5 @@ when: - cifmw_update_control_plane_check | bool - not cifmw_update_run_dryrun | bool - ansible.builtin.shell: | + ansible.builtin.command: | {{ cifmw_update_artifacts_basedir }}/control_plane_test_stop.sh diff --git a/roles/update/templates/control_plane_test_stop.sh.j2 b/roles/update/templates/control_plane_test_stop.sh.j2 index 9eb3648c2a..47d203e18a 100644 --- a/roles/update/templates/control_plane_test_stop.sh.j2 +++ b/roles/update/templates/control_plane_test_stop.sh.j2 @@ -4,10 +4,10 @@ # Get the pid, kill it and wait for the end of the last run. set -eu -MAX_CONS_FAIL=${1:-2} -MAX_FAIL=${2:-3} +MAX_CONS_FAIL=${1:-{{ cifmw_update_ctl_plane_max_cons_fail }}} +MAX_FAIL=${2:-{{ cifmw_update_ctl_plane_max_fail }}} BASE_DIR="${3:-{{ cifmw_update_artifacts_basedir }}}" -STOP_MAX_TRIES=${4:-60} # 5 seconds x MAX_TRIES = 5 min by default +STOP_MAX_TRIES=${4:-{{ cifmw_update_ctl_plane_max_tries }}} # times 5 seconds pid_file="${BASE_DIR}/control-plane-test.pid" diff --git a/scenarios/centos-9/horizon.yml b/scenarios/centos-9/horizon.yml new file mode 100644 index 0000000000..83c427507c --- /dev/null +++ b/scenarios/centos-9/horizon.yml @@ -0,0 +1,5 @@ +--- +pre_deploy: + - name: 80 Kustomize OpenStack CR + type: playbook + source: control_plane_horizon.yml diff --git a/scenarios/reproducers/dt-nfv-ovs-dpdk-sriov-2nodesets.yml b/scenarios/reproducers/dt-nfv-ovs-dpdk-sriov-2nodesets.yml new file mode 100644 index 0000000000..f491145b4c --- /dev/null +++ b/scenarios/reproducers/dt-nfv-ovs-dpdk-sriov-2nodesets.yml @@ -0,0 +1,117 @@ +--- +cifmw_architecture_scenario: "ovs-dpdk-sriov-2nodesets" + +# Automation section. Most of those parameters will be passed to the +# controller-0 as-is and be consumed by the `deploy-va.sh` script. +# Please note, all paths are on the controller-0, meaning managed by the +# Framework. Please do not edit them! +_arch_repo: "{{ cifmw_architecture_repo | default('/home/zuul/src/github.com/openstack-k8s-operators/architecture') }}" + +# HERE if you want to override kustomization, you can uncomment this parameter +# and push the data structure you want to apply. +# cifmw_architecture_user_kustomize: +# stage_0: +# 'network-values': +# data: +# starwars: Obiwan + +# HERE, if you want to stop the deployment loop at any stage, you can uncomment +# the following parameter and update the value to match the stage you want to +# reach. Known stages are: +# pre_kustomize_stage_INDEX +# pre_apply_stage_INDEX +# post_apply_stage_INDEX +# +# cifmw_deploy_architecture_stopper: + +cifmw_libvirt_manager_net_prefix_add: false +cifmw_libvirt_manager_fixed_networks: + - ocpbm + - ocppr + - osp_external + - osp_trunk + +cifmw_libvirt_manager_configuration: + networks: + ocpbm: | + + ocpbm + + + + ocppr: | + + ocppr + + + + osp_external: | + + osp_external + + + + osp_trunk: | + + osp_trunk + + + + vms: + controller: + uefi: "{{ cifmw_use_uefi }}" + root_part_id: "{{ cifmw_root_partition_id }}" + image_url: "{{ cifmw_discovered_image_url }}" + sha256_image_name: "{{ cifmw_discovered_hash }}" + image_local_dir: "{{ cifmw_basedir }}/images/" + disk_file_name: "base-os.qcow2" + disksize: 50 + memory: 8 + cpus: 4 + nets: + - ocpbm + - osp_trunk + ocp: + amount: 3 + uefi: true + root_part_id: 4 + admin_user: core + image_local_dir: "{{ cifmw_basedir }}/images/" + disk_file_name: "ocp_master" + disksize: "100" + extra_disks_num: 3 + extra_disks_size: "50G" + cpus: 10 + memory: 32 + nets: + - ocppr + - ocpbm + - osp_trunk + - osp_external + +# Note: with that extra_network_names "osp_trunk", we instruct +# devscripts role to create a new network, and associate it to +# the OCP nodes. This one is a "private network", and will hold +# the VLANs used for network isolation. + +# Please create a custom env file to provide: +# cifmw_devscripts_ci_token: +# cifmw_devscripts_pull_secret: + +# Baremetal host configuration +cifmw_config_bmh: true + +# BMH are deployed in a differnt NS than the secret OSP BMO +# references in each BMH. Metal3 requires the referenced +# secrets to be in the same NS or be allowed to access them +cifmw_openshift_setup_metal3_watch_all_ns: true + +# Use EDPM image for computes +cifmw_update_containers_edpm_image_url: "{{ cifmw_update_containers_registry }}/{{ cifmw_update_containers_org }}/edpm-hardened-uefi:{{ cifmw_update_containers_tag }}" + +# Set Logical Volume Manager Storage by default for local storage +cifmw_use_lvms: true +cifmw_lvms_disk_list: + - /dev/vda + - /dev/vdb + - /dev/vdc diff --git a/test-requirements.txt b/test-requirements.txt index dbfece1ca0..f69ccac045 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,28 +1,28 @@ -ansible-core>=2.14.0 +ansible-core==2.15.13 # Molecule Deps -molecule>=5.1.0,<6.0.0 -molecule-plugins[podman] -ruamel.yaml -ansi2html -dogpile.cache>=0.9.2 -jmespath # required by devscripts role -netaddr # required by libvirt_manager role -dnspython # required by community.general.dig +molecule==5.1.0 +molecule-plugins[podman]==23.5.0 +ruamel.yaml==0.18.6 +ansi2html==1.9.2 +dogpile.cache==1.3.3 +jmespath==1.0.1 # required by devscripts role +netaddr==1.3.0 # required by libvirt_manager role +dnspython==2.7.0 # required by community.general.dig # UT Deps -pytest -pytest-cov -pytest-html -pytest-testinfra -pytest-xdist -mock +pytest==8.3.3 +pytest-cov==6.0.0 +pytest-html==4.1.1 +pytest-testinfra==10.1.1 +pytest-xdist==3.6.1 +mock==5.1.0 # CI Deps -pre-commit # MIT -yamllint -pyspelling -mkdocs-pymdownx-material-extras +pre-commit==4.0.1 +yamllint==1.35.1 +pyspelling==2.10 +mkdocs-pymdownx-material-extras==2.6 # Common requirements -ansi2txt +ansi2txt==0.2.0 diff --git a/tests/sanity/ignore.txt b/tests/sanity/ignore.txt index d746bf2521..a8db927abb 100644 --- a/tests/sanity/ignore.txt +++ b/tests/sanity/ignore.txt @@ -3,3 +3,4 @@ plugins/modules/generate_make_tasks.py validate-modules:missing-gplv3-license # plugins/modules/tempest_list_allowed.py validate-modules:missing-gplv3-license # ignore license check plugins/modules/tempest_list_skipped.py validate-modules:missing-gplv3-license # ignore license check plugins/modules/cephx_key.py validate-modules:missing-gplv3-license # ignore license check +plugins/modules/url_request.py validate-modules:missing-gplv3-license # ignore license check diff --git a/zuul.d/edpm_multinode.yaml b/zuul.d/edpm_multinode.yaml index 5438fbcb38..5eadecd4b2 100644 --- a/zuul.d/edpm_multinode.yaml +++ b/zuul.d/edpm_multinode.yaml @@ -316,6 +316,7 @@ vars: cifmw_extras: - '@scenarios/centos-9/multinode-ci.yml' + - '@scenarios/centos-9/horizon.yml' run: - ci/playbooks/edpm/run.yml