Detect and report invalid predefined lists

pr_testing/_helper_functions.sh

@@ -196,3 +196,45 @@ function get_gpu_matrix_args() {
   OPTS=$(PYTHONPATH=${PR_TESTING_DIR}/.. ${CMSBOT_PYTHON_CMD} -c 'from RelValArgs import GPU_RELVALS_FLAGS;print(GPU_RELVALS_FLAGS)')
   echo ${OPTS}
 }
+
+function check_invalid_wf_lists () {
+  local CMD_OPTS="$1"
+  local WKFS=$(sed -n 's/.*-l \([^ ]*\).*/\1/p' <<< "$1")
+  local DUMP_BADLIST=${2:-true}
+  local WFLISTS_CNT WFS_CNT BADLIST_CNT
+
+  rm -f "$WORKSPACE/bad-workflow-lists.txt"
+
+  # Count workflow lists vs numeric workflows
+  WFLISTS_CNT=$(echo "$WKFS" | tr ',' '\n' | grep -Ev "^[1-9][0-9]*(\.[0-9]+)?$" | wc -l)
+  WFS_CNT=$(echo "$WKFS" | tr ',' '\n' | wc -l)
+
+  # Capture both output and exit code of runTheMatrix.py
+  local RUN_OUTPUT
+  if ! RUN_OUTPUT=$(runTheMatrix.py ${CMD_OPTS} -n 2>&1); then
+    echo "ERROR : runTheMatrix returned non-zero exit code"
+    return 1
+  fi
+
+  # Extract bad workflow lists directly from the output
+  if (( WFLISTS_CNT > 0 )); then
+    local BADLIST=$(grep "is not a possible selected entry" <<< "$RUN_OUTPUT" | awk '{print $1}')
+    BADLIST_CNT=$(echo $BADLIST | wc -w)
+    if (( BADLIST_CNT > 0 )); then
+      if [[ "$DUMP_BADLIST" == "true" ]]; then
+        echo " - $(echo $BADLIST | tr ' ' ',')"
+      fi
+      if (( WFLISTS_CNT != WFS_CNT )); then
+        echo "WARNING : some workflow lists were not recognized"
+      else
+        if (( BADLIST_CNT == WFS_CNT )); then
+          echo "ERROR : all workflow lists were not recognized, and no additional workflows were requested"
+          return 1
+        else
+          echo "WARNING : none of the workflow lists were recognized, only running explicitly requested workflows"
+        fi
+      fi
+    fi
+  fi
+  return 0
+}

pr_testing/run-pr-relvals.sh

@@ -104,6 +104,12 @@ else
   echo "no errors in the RelVals!!"
   echo "MATRIX${UC_TEST_FLAVOR}_TESTS;OK,Matrix ${UC_TEST_FLAVOR} Tests Outputs,See Logs,runTheMatrix${UC_TEST_FLAVOR}-results" >> ${RESULTS_DIR}/$(get_status_file_name relval "$TEST_FLAVOR")
 
+  if [[ -s "$WORKSPACE/bad-workflow-lists.txt" ]]; then
+    echo "### Invalid workflow lists" >  ${RESULTS_DIR}/0a-bad-workflows-report.res
+    echo "The following workflow names were not recognized and not ran:" >> ${RESULTS_DIR}/0a-bad-workflows-report.res
+    cat "$WORKSPACE/bad-workflow-lists.txt" >> ${RESULTS_DIR}/0a-bad-workflows-report.res
+  fi
+
   if $DO_COMPARISON ; then
     echo "COMPARISON${UC_TEST_FLAVOR};QUEUED,Comparison ${UC_TEST_FLAVOR} with the baseline,See results,See results" >> ${RESULTS_DIR}/comparison${UC_TEST_FLAVOR}.txt
     TRIGGER_COMPARISON_FILE=$WORKSPACE/'comparison.properties'

pr_testing/test_multiple_prs.sh

@@ -351,6 +351,8 @@ if $DO_COMPARISON ; then
     echo "TEST_FLAVOR="                  >> run-baseline-${BUILD_ID}-01.default
     echo "REAL_ARCH=${RELVAL_REAL_ARCH}" >> run-baseline-${BUILD_ID}-01.default
     echo "PRODUCTION_RELEASE=true"       >> run-baseline-${BUILD_ID}-01.default
+    # -- REMOVE NEXT LINE BEFORE MERGING --
+    echo "CMS_BOT_BRANCH=run-ib-pr-matrix-list" >> run-baseline-${BUILD_ID}-01.default
     WF_LIST=$(get_pr_baseline_worklflow)
     [ "${WF_LIST}" = "" ] || WF_LIST="-l ${WF_LIST}"
     echo "WORKFLOWS=-s ${WF_LIST}" >> run-baseline-${BUILD_ID}-01.default

run-ib-pr-matrix.sh

@@ -26,12 +26,18 @@ if [ "${CHECK_WORKFLOWS}" = "true" ] ; then
     ALL_WFS=$(runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' | tr '\n' ',' | sed 's|,$||')
     WORKFLOWS=$(echo "${WORKFLOWS}" | sed "s|all|${ALL_WFS}|")
   fi
-  runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} ${WORKFLOWS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' > $WORKSPACE/req.wfs
+  rm -f $WORKSPACE/req.wfs
+  if  ! check_invalid_wf_lists "${OPTS} ${MATRIX_ARGS} ${WORKFLOWS}" false ; then
+    touch $WORKSPACE/req.wfs
+  else
+    runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} ${WORKFLOWS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' > $WORKSPACE/req.wfs
+  fi
   for wf in $(cat $WORKSPACE/req.wfs) ; do
-    [ $(echo " $REL_WFS " | grep " $wf "  | wc -l) -eq 0 ] || continue
-    WFS="${wf},${WFS}"
+      [ $(echo " $REL_WFS " | grep " $wf "  | wc -l) -eq 0 ] || continue
+      WFS="${wf},${WFS}"
   done
   WFS=$(echo ${WFS} | sed 's|,$||')
+
   if [ "${WFS}" = "" ] ; then
     mv ${WORKSPACE}/workflows-${BUILD_ID}.log ${WORKSPACE}/workflows-${BUILD_ID}.done
     send_jenkins_artifacts ${WORKSPACE}/workflows-${BUILD_ID}.done ${ARTIFACT_DIR}/workflows-${BUILD_ID}.done
@@ -74,10 +80,17 @@ pushd "$WORKSPACE/matrix-results"
   else
     CMD_OPTS="${CMD_OPTS} ${EXTRA_MATRIX_COMMAND_ARGS}"
   fi
+
+  # Check what workflows will be ran (without any --command option)
+  if ! check_invalid_wf_lists "${MATRIX_ARGS}" ; then
+    exit 1
+  fi
+
   [ "${CMD_OPTS}" != "" ] && MATRIX_ARGS="${MATRIX_ARGS} --command ' ${CMD_OPTS}'"
   if [ "X$CMS_SITE_OVERRIDE" == "X" ]; then
     CMS_SITE_OVERRIDE="local"
   fi
+
   eval CMS_PATH=/cvmfs/cms-ib.cern.ch SITECONFIG_PATH=/cvmfs/cms-ib.cern.ch/SITECONF/$CMS_SITE_OVERRIDE runTheMatrix.py -j ${NJOBS} ${MATRIX_ARGS} 2>&1 | tee -a matrixTests.${BUILD_ID}.log
   mv runall-report-step123-.log runall-report-step123-.${BUILD_ID}.log
   find . -name DQM*.root | sort | sed 's|^./||' > wf_mapping.${BUILD_ID}.txt