Add support for Sushy Emulator backed virtual baremetal

This patch enables Redfish BMC control of VMs deployed via the reproducer
layout.

Sushy Emulator is deployed onto the Ansible controller as a Podman Pod
and access' the hypervisor's libvirt socket via SSH.

- `cifmw_use_sushy_emulator` parameter is added to control the
deployment of Sushy Emulator in the reproducer and set to true.
- `cifmw_use_vbmc` parameter is added to control the
deployment of Virtual BMC in the reproducer and behavoir changed to
default to false.
- sushy_emulator role now creates and uses it's own ssh keypair to
to communicate with the hypervisor
- `baremetal-info.yml` creation has now been extracted into it's own
task file and enhanced to support adding new baremetal hosts entries
resulting from scaling or additional hosts for baremetal instances.
- Sushy Emulator verification has now moved to it's own task file and
tests both Sushy Emulator connection and connection to each host.
- sushy_emulator role now uses the `cifmw_libvirt_manager_uuids` fact
to filter vms rather than using an additional `virsh` call. This means
`libvirt-client` can be removed from the packages list to install.
- Add `cifmw_sushy_redfish_bmc_protocol` param to control the bmc protocol Sushy Emulator uses.

Jira: OSPRH-6497

docs/source/usage/01_usage.md

@@ -32,6 +32,9 @@ are shared among multiple roles:
 - `cifmw_use_devscripts`: (Bool) toggle OpenShift deploying using devscripts role.
 - `cifmw_openshift_crio_stats`: (Bool) toggle collecting cri-o stats in CRC deployment.
 - `cifmw_deploy_edpm`: (Bool) toggle deploying EDPM. Default to false.
+- `cifmw_use_vbmc`: (Bool) Toggle VirtualBMC usage. Defaults to `false`.
+- `cifmw_use_sushy_emulator`: (Bool) Toggle Sushy Emulator usage. Defaults to `true`.
+- `cifmw_sushy_redfish_bmc_protocol`: (String) The RedFish BMC protocol you would like to use with Sushy Emulator, options are `redfish` or `redfish-virtualmedia`. Defaults to `redfish-virtualmedia`
 - `cifmw_config_nmstate`: (Bool) toggle NMstate networking deployment. Default to false.
 - `cifmw_config_bmh`: (Bool) toggle Metal3 BareMetalHost CRs deployment. Default to false.
 - `cifmw_config_certmanager`: (Bool) toggle cert-manager deployment. Default to false.

roles/libvirt_manager/tasks/create_networks.yml

@@ -100,3 +100,16 @@
   loop: "{{ networks }}"
   loop_control:
     label: "{{ item.name }}"
+
+- name: Extract public IP address from network bridge
+  block:
+    - name: Extract IP address from network bridges
+      ansible.builtin.include_tasks:
+        file: network_bridge_info_gen.yml
+      loop: "{{ networks }}"
+      loop_control:
+        label: "{{ item.name }}"
+  rescue:
+    - name: Clear error if IP address isn't defined
+      ansible.builtin.meta: clear_host_errors
+      when: networkxml.msg == 'Xpath /network/ip does not reference a node!'

roles/libvirt_manager/tasks/create_vms.yml

@@ -192,6 +192,7 @@
 
 - name: Create VBMC entity
   when:
+    - cifmw_use_vbmc | default(false) | bool
     - _vbmc_available is defined
     - _vbmc_available | bool
   vars:

roles/libvirt_manager/tasks/deploy_layout.yml

@@ -21,21 +21,24 @@
   loop_control:
     label: "{{ item.key }}"
 
+# todo(Lewis): We need to deploy VirtualBMC on the controller rather than hypervisor
 # Deploy VBMC only on the hypervisor running OpenShift services,
 # being OCP cluster or single CRC.
 - name: Deploy VirtualBMC service container
   tags:
     - bootstrap
     - bootstrap_layout
   when:
+    - cifmw_use_vbmc | default(false) | bool
     - (_layout.vms.crc.target is defined and
        _layout.vms.crc.target == inventory_hostname) or
       (_layout.vms.crc is defined and
        _layout.vms.crc.target is undefined) or
       (_layout.vms.ocp.target is defined and
        _layout.vms.ocp.target == inventory_hostname) or
       (_layout.vms.ocp is defined and
-       _layout.vms.ocp.target is undefined)
+       _layout.vms.ocp.target is undefined) or
+      (_layout.vms.crc is undefined and layout.vms.ocp is undefined)
   block:
     - name: Deploy virtualbmc
       ansible.builtin.include_role:
@@ -302,6 +305,7 @@
 
 - name: Refresh and dump vbmc hosts
   when:
+    - cifmw_use_vbmc | default(false) | bool
     - _vbmc_host is defined
     - _vbmc_host == inventory_hostname
   block:

roles/libvirt_manager/tasks/network_bridge_info_gen.yml

@@ -0,0 +1,10 @@
+- name: Extract IP address from network bridges
+  community.general.xml:
+    xmlstring: "{{ item.xml }}"
+    xpath: /network/ip
+    content: attribute
+  register: networkxml
+
+- name: Set network_bridge_info fact with network and address
+  ansible.builtin.set_fact:
+    network_bridge_info: "{{ network_bridge_info | default({}) | combine( {item.name: networkxml.matches[0].ip.address} ) }}"

roles/reproducer/defaults/main.yml

@@ -37,3 +37,10 @@ cifmw_reproducer_supported_hypervisor_os:
     minimum_version: 9
   RedHat:
     minimum_version: 9.3
+cifmw_reproducer_controller_basedir: >-
+  {{
+    (
+      '/home/zuul',
+      'ci-framework-data',
+      ) | path_join
+  }}

roles/reproducer/tasks/configure_controller.yml

@@ -29,6 +29,7 @@
       loop:
         - parameters
         - artifacts
+
     - name: Install custom CA if needed
       ansible.builtin.import_role:
         name: install_ca
@@ -107,53 +108,33 @@
         dest: "{{ _ctl_reproducer_basedir }}/parameters/interfaces-info.yml"
         content: "{{ cifmw_libvirt_manager_mac_map | to_nice_yaml }}"
 
-    # Here, we update the existing cifmw_virtualbmc_known_hosts fact
-    # to inject networking information such as provisioning MAC (if any),
-    # and fixed Boot mode
-    - name: Generate IPMI information
+    # Deploy Sushy Emulator on the Ansible controlelr,
+    - name: Deploy Sushy Emulator service container
+      tags:
+        - bootstrap
+        - bootstrap_layout
+      when: cifmw_use_sushy_emulator | default(true) | bool
       block:
-        - name: Convert VBMC list into a dict for better usage
+        - name: Deploy Sushy Emulator
           vars:
-            keys: "{{ item.keys() | difference(['Domain name']) }}"
-            vals: "{{ keys | map('extract', item) | list }}"
-            value: "{{ dict(keys | map('lower') | zip(vals)) }}"
-            _host: "{{ item['Domain name'] | regex_replace('^cifmw-', '') }}"
-            _nics: >-
-              {{
-                cifmw_libvirt_manager_mac_map[_host]
-              }}
-            _uefi: >-
-              {% set _type = _host | regex_replace('-[0-9]+$', '') -%}
-              {{ _layout.vms[_type].uefi | default(false) | bool }}
-            _boot_mode: "{{ _uefi | ternary('UEFI', 'legacy') }}"
-            ## TODO: add sushy driver address once we get sushy in
-            ## TODO: create new parameter to allow chosing which connection
-            #        to expose in the generated file
-            _connections:
-              ipmi: "ipmi://{{ value.address }}:{{ value.port }}"
+            cifmw_sushy_emulator_hypervisor_target: "{{ cifmw_target_host | default('localhost') }}"
+            cifmw_sushy_emulator_install_type: podman
+            cifmw_sushy_emulator_hypervisor_target_connection_ip: "{{ network_bridge_info['cifmw-public'] }}"
+          ansible.builtin.include_role:
+            name: sushy_emulator
+
+        - name: Let the project know we have Sushy Emulator available
           ansible.builtin.set_fact:
-            _ipmi_dict: >-
-              {{
-                _ipmi_dict | default({}) |
-                combine({_host: value}, recursive=true) |
-                combine({_host: {
-                                 'boot_mode': _boot_mode,
-                                 'nics': _nics,
-                                 'connection': _connections['ipmi']
-                                }
-                        }, recursive=true)
-              }}
-            cacheable: false
-          loop: "{{ cifmw_virtualbmc_known_hosts }}"
-
-        - name: Output IPMI data in a file
-          vars:
-            _content:
-              cifmw_baremetal_hosts: "{{ _ipmi_dict }}"
-          ansible.builtin.copy:
-            dest: "{{ _ctl_reproducer_basedir }}/parameters/baremetal-info.yml"
-            content: "{{ _content | to_nice_yaml }}"
-            mode: "0644"
+            _sushy_emulator_available: true
+
+    - name: Generate baremetal-info fact
+      ansible.builtin.import_tasks: generate_bm_info.yml
+
+    - name: Verify connection to baremetal VMs via Sushy Emulator
+      when: _sushy_emulator_available
+      ansible.builtin.include_role:
+        name: sushy_emulator
+        tasks_from: verify.yml
 
     - name: Inject other Hypervisor SSH keys
       when:

roles/reproducer/tasks/generate_bm_info.yml

@@ -0,0 +1,118 @@
+---
+- name: Ensure directories exist
+  ansible.builtin.file:
+    path: "{{ cifmw_reproducer_controller_basedir }}/{{ item }}"
+    state: directory
+    mode: "0755"
+  loop:
+    - parameters
+    - artifacts
+
+- name: Convert VBMC list into a dict for better usage
+  when:
+    - cifmw_use_vbmc | default(false) | bool
+    - cifmw_virtualbmc_known_hosts is defined
+  vars:
+    keys: "{{ item.keys() | difference(['Domain name']) }}"
+    vals: "{{ keys | map('extract', item) | list }}"
+    value: "{{ dict(keys | map('lower') | zip(vals)) }}"
+    _host: "{{ item['Domain name'] | regex_replace('^cifmw-', '') }}"
+  ansible.builtin.set_fact:
+    _vbmc_info_dict: >-
+      {{
+        _vbmc_info_dict | default({}) |
+        combine({_host: value}, recursive=true)
+      }}
+    cacheable: false
+  loop: "{{ cifmw_virtualbmc_known_hosts }}"
+
+- name: Check if baremetal-info.yml exists
+  register: _stat_baremetal_info_file
+  ansible.builtin.stat:
+    path: "{{ cifmw_reproducer_controller_basedir }}/parameters/baremetal-info.yml"
+
+- name: Slurp current baremetal-info file into _oringal_cifmw_baremetal_hosts var
+  when: _stat_baremetal_info_file.stat.exists
+  block:
+    - name: Get content of baremetal-info file
+      ansible.builtin.slurp:
+        src: "{{ cifmw_reproducer_controller_basedir }}/parameters/baremetal-info.yml"
+      register: _baremetal_info_file
+
+    - name: Interpret remote file content as yaml
+      vars:
+        _yaml: "{{ _baremetal_info_file.content | b64decode | from_yaml }}"
+      ansible.builtin.set_fact:
+        _oringal_cifmw_baremetal_hosts: "{{ _yaml.cifmw_baremetal_hosts }}"
+
+- name: Load cifmw_libvirt_manager_mac_map fact from file
+  vars:
+    _file: "{{ cifmw_reproducer_controller_basedir }}/artifacts/interfaces-info.yml"
+  when:
+    - cifmw_libvirt_manager_mac_map is not defined
+  block:
+    - name: "Slurp content of: {{ _file }}"
+      ansible.builtin.slurp:
+        src: "{{ _file }}"
+      register: _slurped_file
+
+    - name: "Set cifmw_libvirt_manager_mac_map fact from {{ _file }}"
+      vars:
+        _yaml: "{{ _slurped_file.content | b64decode | from_yaml }}"
+      ansible.builtin.set_fact:
+        cifmw_libvirt_manager_mac_map: "{{ _yaml }}"
+
+
+# In this task we combine information from _layout, cifmw_libvirt_manager_mac_map
+# and cifmw_libvirt_manager_uuids to create the libvirt_manager_bm_info_data fact.
+# to create the `baremetal-info.yml` file which is used as a source of truth for
+# for baremetal nodes. If `baremetal-info.yml` already exists we combine
+# `_oringal_cifmw_baremetal_hosts` with the updated hosts and override
+# `baremetal-info.yml`
+- name: Generate libvirt_manager_bm_info_data fact
+  when: item.key is match('cifmw-')
+  vars:
+    _host: "{{ item.key | replace('cifmw-', '') }}"
+    _uefi: >-
+      {% set _type = _host | regex_replace('-[0-9]+$', '') -%}
+      {{ _layout.vms[_type].uefi | default(false) | bool }}
+    _data: |
+      "{{ _host }}":
+        boot_mode: "{{ _uefi | ternary('UEFI', 'legacy') }}"
+        uuid: "{{ item.value }}"
+      {% if cifmw_use_vbmc | default(false) | bool %}
+        address: "{{ _vbmc_info_dict[_host].address }}"
+        connection: "ipmi://{{ _vbmc_info_dict[_host].address }}:{{ _vbmc_info_dict[_host].port }}"
+        password: "{{ _vbmc_info_dict[_host].password }}"
+        port: {{ _vbmc_info_dict[_host].port | int }}
+        username: "{{ _vbmc_info_dict[_host].username }}"
+      {% elif cifmw_use_sushy_emulator | default(true) | bool %}
+      {% if item.value in _cifmw_sushy_emulator_instances | default([]) %}
+        connection: "{{ cifmw_sushy_redfish_bmc_protocol | default('redfish-virtualmedia') ~ '+' ~ _sushy_url }}/redfish/v1/Systems/{{ item.value }}"
+        username: "{{ cifmw_redfish_username | default('admin') }}"
+        password: "{{ cifmw_redfish_password | default('password') }}"
+      {% endif %}
+      {% endif %}
+    _with_nics: >-
+      {{
+        _data | from_yaml |
+        combine({_host: {'nics': cifmw_libvirt_manager_mac_map[_host]}},
+                recursive=true)
+      }}
+  ansible.builtin.set_fact:
+    libvirt_manager_bm_info_data: >-
+      {{
+        libvirt_manager_bm_info_data | default({}) |
+        combine((_with_nics | from_yaml), recursive=true) |
+        combine((_oringal_cifmw_baremetal_hosts | default({}) | from_yaml), recursive=true)
+      }}
+  loop: "{{ cifmw_libvirt_manager_uuids | dict2items }}"
+
+- name: Output baremetal info file
+  vars:
+    _content:
+      cifmw_baremetal_hosts: "{{ libvirt_manager_bm_info_data }}"
+  ansible.builtin.copy:
+    dest: "{{ cifmw_reproducer_controller_basedir }}/parameters/baremetal-info.yml"
+    content: "{{ _content | to_nice_yaml }}"
+    mode: "0644"

roles/sushy_emulator/README.md

@@ -6,33 +6,45 @@ Sushi Emulator is a virtual Redfish BMC, this role supports both the OpenStack a
 
 Baremetal instances must be configured prior to running this role, for the Libvirt driver, the `libvirt_manager` role can be used.
 
+This role supports installing via a Podman pod onto the controller node or a pod in OCP or CRC.
+
 ## Privilege escalation
 
 Required to installed required packages.
 
 ## Parameters
 
+* `cifmw_sushy_emulator_basedir`: (String) Base directory. Defaults to `{{ ansible_user_dir ~ '/ci-framework-data' }}`
+* `cifmw_sushy_emulator_container_name`: (String) Name of Podman container created. Defaults to `cifmw-sushy_emulator`
 * `cifmw_sushy_emulator_driver`: (String) Select between `openstack` and `libvirt` sushy emulator driver. Defaults to `libvirt`
-* `cifmw_sushy_emulator_sshkey_path`: (String) Path of SSH key used by sushy emulator to talk to libvirt socket. Defaults to `"{{ ansible_user_dir }}/.ssh/id_cifw"`
-* `cifmw_sushy_emulator_libvirt_user`: (String) Username used by Sushi Emulator to connect to Libvirt socket. Defaults to `zuul`
-* `cifmw_sushy_emulator_listen_ip`: (String) IP Sushi Emulator service listens on. Defaults to `0.0.0.0`
 * `cifmw_sushy_emulator_driver_openstack_client_config_file`: (String) Path to OpenStack config file, used by OpenStack Sushi Emulator driver. Defaults to `/etc/openstack/clouds.yaml`
 * `cifmw_sushy_emulator_driver_openstack_cloud`: (String) Cloud key reference in OpenStack config file. Defaults to `None`
+* `cifmw_sushy_emulator_hypervisor_target` (String) Hostname of the Libvirt hypervisor to connect to.
+* `cifmw_sushy_emulator_hypervisor_target_connection_ip` (String) IP Address used to connect to hypervisor, optional override for `cifmw_sushy_emulator_hypervisor_target` when specific address is required.
+* `cifmw_sushy_emulator_install_type`: (String) Install type can either be `ocp` or `podman` and dictates where Sushi Emulator will be installed. Defaults to `ocp`
+* `cifmw_sushy_emulator_image`: (String) Container image used when deploying Sushi Emulator container and pod. Defaults to `quay.io/metal3-io/sushy-tools:latest`
+* `cifmw_sushy_emulator_libvirt_uri`: (String) Internal URI to access qemu daemon. Defaults to `qemu+ssh://{{ cifmw_sushy_emulator_libvirt_user }}@{{ cifmw_sushy_emulator_hypervisor_target_connection_ip | default(cifmw_sushy_emulator_hypervisor_target) }}/system?no_tty=1`
+* `cifmw_sushy_emulator_libvirt_user`: (String) Username used by Sushi Emulator to connect to Libvirt socket. Defaults to `zuul`
+* `cifmw_sushy_emulator_listen_ip`: (String) IP address Sushy Emulator listens on. Defaults to `0.0.0.0`
 * `cifmw_sushy_emulator_namespace`: (String) Namespace Sushi Emulator is deployed into when using the `ocp` install method. Defaults to `sushy-emulator`
-* `cifmw_sushy_emulator_redfish_username`: (String) Redfish username. Defaults to `admin`
-* `cifmw_sushy_emulator_redfish_password`: (String) Redfish password. Defaults to `password`
+* `cifmw_sushy_emulator_parameters_file`:  (string) Path of the file which contains (v)bmc parameters. Defaults to `cifmw_sushy_emulator_basedir + parameters + baremetal-info.yml`
+* `cifmw_sushy_emulator_redfish_username`: (String) Redfish username. Defaults to `{{ cifmw_redfish_username | default('admin') }}`
+* `cifmw_sushy_emulator_redfish_password`: (String) Redfish password. Defaults to `{{ cifmw_redfish_password | default('password') }}`
 * `cifmw_sushy_emulator_resource_directory`: (String) Path where resource files will be written. Defaults to `"{{ (ansible_user_dir, 'ci-framework-data', 'artifacts', 'sushy_emulator') | path_join }}"`
-* `cifmw_sushy_emulator_image`: (String) Container image used when deploying Sushi Emulator container and pod. Defaults to `quay.io/metal3-io/sushy-tools:latest`
-* `cifmw_sushy_emulator_instance_node_name_prefix`: (String) String used to find instances created for baremetal use. Defaults to `cifmw-`
-* `cifmw_sushy_emulator_container_name`: (String) Name of Podman container created. Defaults to `cifmw-sushy_emulator`
-* `cifmw_sushy_emulator_install_type`: (String) Install type can either be `ocp` or `podman` and dictates where Sushi Emulator will be installed. Defaults to `ocp`
+* `cifmw_sushy_emulator_sshkey_path`: (String) Path of SSH key used by sushy emulator to talk to libvirt socket. Defaults to `"{{ ansible_user_dir }}/.ssh/sushy_emulator-key"`
+* `cifmw_sushy_emulator_sshkey_type`: (String) Type of SSH keypair. Defaults to `{{ cifmw_ssh_keytype | default('ecdsa') }}`.
+* `cifmw_sushy_emulator_sshkey_size`: (Integer) Size of the SSH keypair. Defaults to `{{ cifmw_ssh_keysize | default(521) }}`.
+* `cifmw_sushy_emulator_vm_prefix_filter`: (String) Prefix string used to filter instances created for baremetal use. Defaults to `.*`
 
 ## Examples
 
 ```yaml
 - hosts: all
   vars:
-    cifmw_baremetal_hypervisor_target: baremetal-hypervisor
+    cifmw_sushy_emulator_hypervisor_target: "{{ cifmw_target_host | default('localhost') }}"
+    cifmw_sushy_emulator_install_type: podman
+    cifmw_sushy_emulator_hypervisor_target_connection_ip: "{{ network_bridge_info['cifmw-public'] }}"
+    cifmw_sushy_emulator_vm_prefix_filter: cifmw-compute
   tasks:
     - name: Deploy and configure sushy-emulator
       ansible.builtin.import_role:

roles/sushy_emulator/defaults/main.yml

@@ -18,17 +18,33 @@
 # All variables intended for modification should be placed in this file.
 # All variables within this role should have a prefix of "cifmw_sushy_emulator"
 
+cifmw_sushy_emulator_basedir: "{{ ansible_user_dir ~ '/ci-framework-data' }}"
+cifmw_sushy_emulator_container_name: "cifmw-sushy_emulator"
 cifmw_sushy_emulator_driver: libvirt
-cifmw_sushy_emulator_sshkey_path: "{{ ansible_user_dir }}/.ssh/id_cifw"
-cifmw_sushy_emulator_libvirt_user: zuul
-cifmw_sushy_emulator_listen_ip: 0.0.0.0
 cifmw_sushy_emulator_driver_openstack_client_config_file: /etc/openstack/clouds.yaml
 cifmw_sushy_emulator_driver_openstack_cloud: None
-cifmw_sushy_emulator_namespace: sushy-emulator
-cifmw_sushy_emulator_redfish_username: admin
-cifmw_sushy_emulator_redfish_password: password
-cifmw_sushy_emulator_resource_directory: "{{ (ansible_user_dir, 'ci-framework-data', 'artifacts', 'sushy_emulator') | path_join }}"
-cifmw_sushy_emulator_image: quay.io/metal3-io/sushy-tools:latest
-cifmw_sushy_emulator_instance_node_name_prefix: cifmw-
-cifmw_sushy_emulator_container_name: "cifmw-sushy_emulator"
 cifmw_sushy_emulator_install_type: 'ocp'
+cifmw_sushy_emulator_image: quay.io/metal3-io/sushy-tools:latest
+cifmw_sushy_emulator_libvirt_uri: >-
+  qemu+ssh://
+  {{- cifmw_sushy_emulator_libvirt_user -}}
+  @
+  {{- cifmw_sushy_emulator_hypervisor_target_connection_ip | default(hostvars[cifmw_sushy_emulator_hypervisor_target].ansible_host) -}}
+  /system?no_tty=1
+cifmw_sushy_emulator_libvirt_user: zuul
+cifmw_sushy_emulator_listen_ip: 0.0.0.0
+cifmw_sushy_emulator_namespace: sushy-emulator
+cifmw_sushy_emulator_parameters_file: >-
+  {{
+    [
+      cifmw_sushy_emulator_basedir,
+      'parameters',
+      'baremetal-info.yml'
+    ] | path_join
+  }}
+cifmw_sushy_emulator_redfish_username: "{{ cifmw_redfish_username | default('admin') }}"
+cifmw_sushy_emulator_redfish_password: "{{ cifmw_redfish_password | default('password') }}"
+cifmw_sushy_emulator_resource_directory: "{{ (cifmw_sushy_emulator_basedir, 'artifacts', 'sushy_emulator') | path_join }}"
+cifmw_sushy_emulator_sshkey_path: "{{ ansible_user_dir }}/.ssh/sushy_emulator-key"
+cifmw_sushy_emulator_sshkey_type: "{{ cifmw_ssh_keytype | default('ecdsa') }}"
+cifmw_sushy_emulator_sshkey_size: "{{ cifmw_ssh_keysize | default(521) }}"