Running all gather scripts in background

.ci-operator.yaml

@@ -0,0 +1,4 @@
+build_root_image:
+  name: release
+  namespace: openshift
+  tag: rhel-8-release-golang-1.20-openshift-4.14

.github/workflows/odf-must-gather-ci.yaml

@@ -18,6 +18,8 @@ jobs:
 
       - name: Run ShellCheck
         uses: ludeeus/action-shellcheck@master
+        with:
+          scandir: './collection-scripts'
         env:
           SHELLCHECK_OPTS: -e SC2181
 
@@ -34,4 +36,5 @@ jobs:
           check_filenames: true
           check_hidden: true
           ignore_words_list: xdescribe,contails,shouldnot
+          path: collection-scripts
 

Dockerfile

@@ -0,0 +1,20 @@
+# Builder stage
+FROM registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.20-openshift-4.14 AS builder
+WORKDIR /go/src/github.com/red-hat-storage/odf-must-gather
+
+COPY . .
+ENV GO_PACKAGE github.com/red-hat-storage/odf-must-gather
+
+# Prod stage
+FROM registry.ci.openshift.org/ocp/4.14:cli
+
+RUN mkdir -p /templates
+COPY --from=builder /go/src/github.com/red-hat-storage/odf-must-gather/collection-scripts/* /usr/bin/
+COPY --from=builder /go/src/github.com/red-hat-storage/odf-must-gather/templates/* /templates
+
+# We do not need it as of now
+# jq is not preinstalled on openshift/origin-cli either
+# Removing this step makes local development easier.
+# RUN yum install --setopt=tsflags=nodocs -y jq && yum clean all && rm -rf /var/cache/yum/*
+
+ENTRYPOINT /usr/bin/gather

Makefile

@@ -1,18 +1,51 @@
-IMAGE_REGISTRY ?= "quay.io"
-REGISTRY_NAMESPACE ?= "ocs-dev"
-IMAGE_TAG ?= "latest"
-MUST_GATHER_IMAGE_NAME ?= "ocs-must-gather"
+all: images
+.PHONY: all local docker-rancher-build docker-engine-build
 
-MUST_GATHER_IMAGE_NAME ?= $(IMAGE_REGISTRY)/$(REGISTRY_NAMESPACE)/$(MUST_GATHER_IMAGE_NAME):$(IMAGE_TAG)
+# Include the library makefile
+include $(addprefix ./vendor/github.com/openshift/build-machinery-go/make/, \
+	golang.mk \
+	targets/openshift/images.mk \
+)
 
-OCS_MUST_GATHER_DIR ?= "${OCS_MUST_GATHER_DIR:-ocs-must-gather}"
-OCP_MUST_GATHER_DIR ?= "${OCP_MUST_GATHER_DIR:-ocp-must-gather}"
+IMAGE_REGISTRY :=registry.svc.ci.openshift.org
+IMG_TAG ?= odf-mg
 
-PLATFORM ?= "docker"
+# This will call a macro called "build-image" which will generate image specific targets based on the parameters:
+# $0 - macro name
+# $1 - target name
+# $2 - image ref
+# $3 - Dockerfile path
+# $4 - context directory for image build
+$(call build-image,odf-must-gather,$(IMAGE_REGISTRY)/ocp/4.14:odf-must-gather, ./Dockerfile,.)
+
+$(call verify-golang-versions,Dockerfile)
+
+docker-rancher-build:
+	docker context use rancher-desktop
+	docker build . -t $(IMG_TAG) --platform=linux/amd64
+
+docker-engine-build:
+	docker build . -t $(IMG_TAG) --platform=linux/amd64
+
+local:
+ifndef ODF_MG_TOKEN
+	@echo "ERROR: The ODF_MG_TOKEN environment variable is not set."
+	@exit 1
+endif
+
+	@echo "Logging in to the cluster with token: $(ODF_MG_TOKEN)"
+	@oc login --token=$(ODF_MG_TOKEN) --server=https://api.ci.l2s4.p1.openshiftapps.com:6443 1>/dev/null
+
+	@echo "Logging into the CI image registry..."
+	@oc registry login --registry registry.ci.openshift.org &>/dev/null
+
+	@echo "Will be using $(IMG_TAG) as the image tag, you can change this using IMG_TAG env var."
+
+	@echo "Building the image..."
+
+ifdef RANCHER
+	@make docker-rancher-build
+else
+	@make docker-engine-build
+endif
 
-.PHONY: \
-	odf-must-gather 
-
-odf-must-gather:
-	@echo "Building the ocs-must-gather image"
-	${PLATFORM} build -f must-gather/Dockerfile -t ${MUST_GATHER_IMAGE_NAME} must-gather/

OWNERS

@@ -9,6 +9,7 @@ approvers:
   - nb-ohad
   - yati1998
   - iamniting
+  - black-dragon74
 # review == this code is good /lgtm
 reviewers:
   - jarrpa
@@ -20,4 +21,5 @@ reviewers:
   - nb-ohad
   - yati1998
   - iamniting
+  - black-dragon74
 

README.md

@@ -6,7 +6,7 @@ that expands its capabilities to gather Openshift Container Storage for informat
 
 ### Usage
 ```sh
-oc adm must-gather --image=quay.io/rhceph-dev/odf4-odf-must-gather-rhel9:latest-4.13
+oc adm must-gather --image=registry.redhat.io/odf4/odf-must-gather-rhel9:v4.13
 ```
 
 The command above will create a local directory with a dump of the ocs state.
@@ -43,6 +43,21 @@ You will get a dump of:
 In order to get data about other parts of the cluster (not specific to OCS) you should
 run `oc adm must-gather` (without passing a custom image). Run `oc adm must-gather -h` to see more options.
 
+### Building locally for testing
+You need to be autheticated against the OpenShift registry CI, registry.ci.openshift.org in order to be able to pull the images in the `Dockerfile`.
+
+Before proceeding make sure you have `oc` (OpenShift cli) installed.
+
+Follow these steps to authenticate:
+
+- Navigate to the OpenShift console for `app.ci` cluster located [here](https://console-openshift-console.apps.ci.l2s4.p1.openshiftapps.com/).
+- Login using SSO
+- Go to your profile name in the top right and select "Copy login command".
+- Copy the displayed TOKEN
+- Run `ODF_MG_TOKEN=YOUR_TOKEN_HERE make local`. 
+
+See [here](https://github.com/red-hat-storage/odf-must-gather/pull/76) for more options on building locally.
+
 ### How to Contribute
 
 - Refer and follow the standards mentioned in [ODF-MUST-GATHER How to Contribute](./CONTRIBUTING.md)

must-gather/collection-scripts/gather → collection-scripts/gather

@@ -4,7 +4,7 @@ export BASE_COLLECTION_PATH="must-gather"
 mkdir -p ${BASE_COLLECTION_PATH}
 
 # Command line argument
-SINCE_TIME=$1
+export SINCE_TIME=$1
 
 # Source the utils
 # shellcheck disable=SC1091
@@ -19,9 +19,10 @@ dbglog "collection started at: ${START_TIME}"
 pre-install.sh ${BASE_COLLECTION_PATH}
 
 # Call other gather scripts
-gather_namespaced_resources ${BASE_COLLECTION_PATH} "${SINCE_TIME}"
-gather_clusterscoped_resources ${BASE_COLLECTION_PATH} "${SINCE_TIME}"
-gather_noobaa_resources ${BASE_COLLECTION_PATH} "${SINCE_TIME}"
+gather_namespaced_resources ${BASE_COLLECTION_PATH} &
+gather_clusterscoped_resources ${BASE_COLLECTION_PATH} &
+gather_noobaa_resources ${BASE_COLLECTION_PATH} &
+gather_odf_client ${BASE_COLLECTION_PATH} &
 
 namespace=$(oc get deploy --all-namespaces -o go-template --template='{{range .items}}{{if .metadata.labels}}{{printf "%s %v" .metadata.namespace (index .metadata.labels "olm.owner")}} {{printf "\n"}}{{end}}{{end}}' | grep ocs-operator | awk '{print $1}' | uniq)
 storageClusterPresent=$(oc get storagecluster -n "${namespace}" -o go-template='{{range .items}}{{.metadata.name}}{{"\n"}}{{end}}')
@@ -30,16 +31,19 @@ reconcileStrategy=$(oc get storagecluster -n "${namespace}" -o go-template='{{ra
 
 if [ "${externalStorageClusterPresent}" = "true" ]; then
     dbglog "Collecting limited ceph logs since external cluster is present"
-    gather_common_ceph_resources "${BASE_COLLECTION_PATH}" "${SINCE_TIME}"
+    gather_common_ceph_resources "${BASE_COLLECTION_PATH}" &
 elif [ -z "${storageClusterPresent}" ]; then
     dbglog "Skipping ceph collection as Storage Cluster is not present"
 elif [ "${reconcileStrategy}" = "standalone" ]; then
     dbglog "Skipping ceph collection as this is a MCG only cluster"
 else
     dbglog "Collecting entire ceph logs"
-    gather_ceph_resources ${BASE_COLLECTION_PATH} "${SINCE_TIME}"
+    gather_ceph_resources ${BASE_COLLECTION_PATH} &
 fi
 
+# Wait until gather scripts finish
+wait
+
 # Call post-uninstall.sh
 post-uninstall.sh
 

must-gather/collection-scripts/gather_ceph_resources → collection-scripts/gather_ceph_resources

@@ -4,9 +4,6 @@
 # If it is not defined, use PWD instead
 BASE_COLLECTION_PATH=${BASE_COLLECTION_PATH:-"$(pwd)"}
 
-# Expect time option as an argument
-SINCE_TIME=$2
-
 gather_common_ceph_resources "${BASE_COLLECTION_PATH}" "${SINCE_TIME}"
 CEPH_GATHER_DBGLOG="${BASE_COLLECTION_PATH}"/gather-ceph-debug.log
 CEPH_COLLECTION_PATH="${BASE_COLLECTION_PATH}/ceph"

must-gather/collection-scripts/gather_clusterscoped_resources → collection-scripts/gather_clusterscoped_resources

@@ -4,9 +4,6 @@
 # If it is not defined, use PWD instead
 BASE_COLLECTION_PATH=${BASE_COLLECTION_PATH:-"$(pwd)"}
 
-# Expect time option as an argument
-SINCE_TIME=$2
-
 # Command List
 commands_get=()
 

must-gather/collection-scripts/gather_common_ceph_resources → collection-scripts/gather_common_ceph_resources

@@ -4,8 +4,6 @@
 # If it is not defined, use PWD instead
 BASE_COLLECTION_PATH=${BASE_COLLECTION_PATH:-"$(pwd)"}
 
-# Expect time option as an argument
-SINCE_TIME=$2
 CEPH_COLLECTION_PATH="${BASE_COLLECTION_PATH}/ceph"
 
 # common Ceph resources

must-gather/collection-scripts/gather_namespaced_resources → collection-scripts/gather_namespaced_resources

@@ -4,9 +4,6 @@
 # If it is not defined, use PWD instead
 BASE_COLLECTION_PATH=${BASE_COLLECTION_PATH:-"$(pwd)"}
 
-# Expect time option as an argument
-SINCE_TIME=$2
-
 # Make a global variable for namespace
 INSTALL_NAMESPACES=$(oc get storagecluster -A --no-headers | awk '{print $1}')
 PRODUCT_NAMESPACE=$(oc get managedFusionOffering -A --no-headers | awk '{print $1}')
@@ -44,14 +41,14 @@ commands_get+=("subscription")
 commands_get+=("csv")
 commands_get+=("catalogsource")
 commands_get+=("installplan")
-commands_get+=("events")
 commands_get+=("all -o wide")
 commands_get+=("role")
 commands_get+=("hpa")
 commands_get+=("secrets")
 commands_get+=("configmap")
 commands_get+=("rolebinding")
 commands_get+=("storageconsumer")
+commands_get+=("storageprofiles")
 commands_get+=("storageclassrequest")
 commands_get+=("alertmanager")
 commands_get+=("alertmanagerconfig")
@@ -65,6 +62,7 @@ commands_desc+=("hpa")
 commands_desc+=("subscription")
 commands_desc+=("storagecluster")
 commands_desc+=("storageconsumer")
+commands_desc+=("storageprofiles")
 commands_desc+=("storageclassrequest")
 commands_desc+=("alertmanager")
 commands_desc+=("prometheus")
@@ -112,9 +110,18 @@ for INSTALL_NAMESPACE in $PRODUCT_NAMESPACE $INSTALL_NAMESPACES $MANAGED_FUSION_
 done
 
 # NOTE: This is a temporary fix for collecting the storagecluster as we are not able to collect the storagecluster using the inspect command
-{ oc get storageclusters -n "${INSTALL_NAMESPACE}" -o yaml; } >"$BASE_COLLECTION_PATH/namespaces/${INSTALL_NAMESPACE}/oc_output/storagecluster.yaml" 2>&1
-{ oc get storagesystem -n "${INSTALL_NAMESPACE}" -o yaml; } >"$BASE_COLLECTION_PATH/namespaces/${INSTALL_NAMESPACE}/oc_output/storagesystem.yaml" 2>&1
-{ oc get storageconsumer -n "${INSTALL_NAMESPACE}" -o yaml; } >"$BASE_COLLECTION_PATH/namespaces/${INSTALL_NAMESPACE}/oc_output/storageconsumer.yaml" 2>&1
+{ oc get storageclusters -n "${INSTALL_NAMESPACE}" -o yaml; } >"${BASE_COLLECTION_PATH}/namespaces/${INSTALL_NAMESPACE}/oc_output/storagecluster.yaml" 2>&1
+{ oc get storagesystem -n "${INSTALL_NAMESPACE}" -o yaml; } >"${BASE_COLLECTION_PATH}/namespaces/${INSTALL_NAMESPACE}/oc_output/storagesystem.yaml" 2>&1
+{ oc get storageconsumer -n "${INSTALL_NAMESPACE}" -o yaml; } >"${BASE_COLLECTION_PATH}/namespaces/${INSTALL_NAMESPACE}/oc_output/storageconsumer.yaml" 2>&1
+
+# Collect oc get events with sorted timestamp
+dbglog "collecting output of oc get events with sorted timestamp"
+{ oc get event -o custom-columns="LAST SEEN:{lastTimestamp},FIRST SEEN:{firstTimestamp},COUNT:{count},NAME:{metadata.name},KIND:{involvedObject.kind},SUBOBJECT:{involvedObject.fieldPath},TYPE:{type},REASON:{reason},SOURCE:{source.component},MESSAGE:{message}" --sort-by=lastTimestamp -n openshift-storage; } >"${BASE_COLLECTION_PATH}/namespaces/${INSTALL_NAMESPACE}/oc_output/events_get" 2>&1
+events=$(timeout 60 oc get events -n openshift-storage -o custom-columns="NAME:{metadata.name}" --sort-by=lastTimestamp --no-headers)
+for event in "${events[@]}"; do
+     { oc describe events "${event}" -n openshift-storage >> "${BASE_COLLECTION_PATH}/namespaces/${INSTALL_NAMESPACE}/oc_output/events_desc"; }
+     { oc get events "${event}" -n openshift-storage -o yaml >> "${BASE_COLLECTION_PATH}/namespaces/${INSTALL_NAMESPACE}/oc_output/events_yaml"; }
+done
 
 # Create the dir for data from all namespaces
 mkdir -p "${BASE_COLLECTION_PATH}/namespaces/all/"

must-gather/collection-scripts/gather_noobaa_resources → collection-scripts/gather_noobaa_resources

@@ -4,9 +4,6 @@
 # If it is not defined, use PWD instead
 BASE_COLLECTION_PATH=${BASE_COLLECTION_PATH:-"$(pwd)"}
 
-# Expect time option as an argument
-SINCE_TIME=$2
-
 # Make a global variable for namespace
 INSTALL_NAMESPACE=openshift-storage
 
@@ -71,6 +68,7 @@ NOOBAA_POSTGRES_LABEL='noobaa-db in (postgres)'
 for pod in $(oc -n "${ns}" get pods -l "${NOOBAA_POSTGRES_LABEL}" | grep -v NAME | awk '{print $1}'); do
     dbglog "collecting noobaa db pod logs from ${ns}"
     { timeout 120 oc -n "${ns}" logs --all-containers "${pod}" &>"${LOG_DIR}"/"${pod}".log; } >>"${BASE_COLLECTION_PATH}"/gather-debug.log 2>&1
+    { timeout 120 oc -n "${ns}" cp "${pod}":/var/lib/pgsql/data/userdata/log "${LOG_DIR}"/"${pod}"/pg_log; } >>"${BASE_COLLECTION_PATH}"/gather-debug.log 2>&1
 done
 
 # Add important notifications to a notifications.txt file at the root of the noobaa collection path

go.mod

@@ -0,0 +1,5 @@
+module github.com/red-hat-storage/odf-must-gather
+
+go 1.20
+
+require github.com/openshift/build-machinery-go v0.0.0-20230824093055-6a18da01283c

go.sum

@@ -0,0 +1,2 @@
+github.com/openshift/build-machinery-go v0.0.0-20230824093055-6a18da01283c h1:H5k87xq6hGgR1YCF/8hLv3j5jWd64Eh3ZhqF9WUJ15Q=
+github.com/openshift/build-machinery-go v0.0.0-20230824093055-6a18da01283c/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=