From 6405a2943a2495821fe639724d832ad3567dfc94 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Thu, 28 Sep 2023 17:16:39 +0200 Subject: [PATCH 1/3] Gather cinder service information This patch adds the gatthering of information from the Cinder service, such as: - Services status - Backend Pool information - Volume types - QoS definitions - Volum transfer status - Volume summary --- collection-scripts/gather_services_status | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/collection-scripts/gather_services_status b/collection-scripts/gather_services_status index a40d1d5..8ff8daa 100755 --- a/collection-scripts/gather_services_status +++ b/collection-scripts/gather_services_status @@ -22,6 +22,9 @@ get_status() { "neutron") get_neutron_status ;; + "cinder") + get_cinder_status + ;; *) ;; esac } @@ -58,6 +61,19 @@ get_neutron_status() { ${BASH_ALIASES[os]} security group list > "$NEUTRON_PATH"/security_group_list } +# Cinder service gathering - services, vol types, qos, transfers, pools, +get_cinder_status() { + local CINDER_PATH="$BASE_COLLECTION_PATH/ctlplane/cinder" + mkdir -p "$CINDER_PATH" + ${BASH_ALIASES[os]} volume service list > "$CINDER_PATH"/service_list + ${BASH_ALIASES[os]} volume type list --long > "$CINDER_PATH"/type_list + ${BASH_ALIASES[os]} volume qos list > "$CINDER_PATH"/qos_list + ${BASH_ALIASES[os]} volume transfer request list --all-project > "$CINDER_PATH"/transfer_list + ${BASH_ALIASES[os]} --os-volume-api-version 3.12 volume summary --all-projects > "$CINDER_PATH"/total_volumes_list + # Add --fit once we have https://review.opendev.org/c/openstack/python-openstackclient/+/895971 + ${BASH_ALIASES[os]} volume backend pool list --long > "$CINDER_PATH"/pool_list +} + # first we gather generic status of the openstack ctlplane # then we process the existing services (if an associated # function has been defined) From 3848725e614f8d276baa98f72efdb9d5d725a4d2 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Fri, 29 Sep 2023 12:54:47 +0200 Subject: [PATCH 2/3] Trigger Guru Meditation Reports Most OpenStack services have the possibility of having Guru Meditation Reports: https://github.com/openstack/oslo.reports These reports are presented in the logs, so we have to trigger them before we gather the logs. This patch adds the mechanims to triger those reports in whichever way each service has configured: Via signal or via file. The example provided is for the cinder service, which is configured to use files because the services are running in the host PID namespace, so we don't know the PID of the service. --- collection-scripts/gather_run | 3 ++ collection-scripts/gather_trigger_gmr | 54 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100755 collection-scripts/gather_trigger_gmr diff --git a/collection-scripts/gather_run b/collection-scripts/gather_run index abf334a..823b234 100755 --- a/collection-scripts/gather_run +++ b/collection-scripts/gather_run @@ -20,6 +20,9 @@ source "${DIR_NAME}/common.sh" # if exist so we can gather resources about tests expand_ns "kuttl" +# Trigger Guru Meditation Reports so we then have then in the service logs +/usr/bin/gather_trigger_gmr + for NS in "${DEFAULT_NAMESPACES[@]}"; do # get Services Config (CM) /usr/bin/gather_services_cm "$NS" diff --git a/collection-scripts/gather_trigger_gmr b/collection-scripts/gather_trigger_gmr new file mode 100755 index 0000000..75753a9 --- /dev/null +++ b/collection-scripts/gather_trigger_gmr @@ -0,0 +1,54 @@ +#!/bin/bash +# +# Trigger Guru Meditation Reports in the different services to have them in the +# logs: https://github.com/openstack/oslo.reports +# +# Some services use signals and others use touching a file to trigger the +# reporting. + +DIR_NAME=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +# shellcheck disable=SC1091 +source "${DIR_NAME}/common.sh" + +oc="/usr/bin/oc -n openstack " +oce="$oc exec " + + +trigger_gmr() { + service="$1" + case "${service}" in + "cinder") + cinder_trigger_gmr + ;; + *) ;; + esac + +} + + +cinder_trigger_gmr() { + # https://docs.openstack.org/cinder/latest/contributor/gmr.html + echo "Trigger GMR for Cinder services" + + # Get pod name and type of service for cinder pods + svcs=`$oc get pod -l service=cinder -o=custom-columns=N:.metadata.name,T:metadata.labels.component --no-headers` + + # Cinder uses files to trigger GMR because volume and backup share the PID + # with the host so we don't know what PID the service has + while read -r line; do + podname="${line% *}" + svctype="${line#* }" + $oce $podname -c $svctype -- touch /etc/cinder + done <<< "$svcs" +} + + +# get the list of existing ctlplane services (once) and +# filter the whole list processing only services with an +# associated function +services=`$oce openstackclient -- openstack service list -c Name -f value` +for svc in "${OSP_SERVICES[@]}"; do + [[ "${services[*]}" =~ ${svc} ]] && trigger_gmr "$svc" +done + +exit 0 From 1df7586276e1f041baf151468e0e63e884abde15 Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Tue, 3 Oct 2023 18:35:14 +0200 Subject: [PATCH 3/3] Update readme with component location This patch adds a small section to the README.md with the locations that the different openstack components need to look into to ensure that they are gathering the information they want from the control plane in the customer cases. --- README.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/README.md b/README.md index eb36dcc..c0ab023 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,26 @@ sh-5.1# gather sh-5.1# rm /tmp/rm-to-finish-gathering ``` +### Component specifics + +Besides the generic OpenShift etcd objects that must-gather scripts are +currently gathering there's also component specific code that gather other +information, so when adding or improving a component we should look into: + +- `collection-scripts/common.sh`: Services are in the `OSP_SERVICES` variable + to indicate that the script must gather its config maps, secrets, additional + information, trigger Guru Mediation Reports, etc. + +- `collection-scripts/gather_services_status`: Runs OpenStack commands to + gather additional information. For example for Cinder it gather state of the + services, volume types, qos specs, volume transfer requests, summary of + existing volumes, pool information, etc. + +- `collection-scripts/gather_trigger_gmr`: Triggers Guru Meditation Reports on + the services so they are present in the logs when those are gathered + afterwards. + + ## Example (optional) Build and push the must-gather image to a registry: