diff --git a/pr_testing/_helper_functions.sh b/pr_testing/_helper_functions.sh index f330971cb0a0..48825382c624 100755 --- a/pr_testing/_helper_functions.sh +++ b/pr_testing/_helper_functions.sh @@ -196,3 +196,45 @@ function get_gpu_matrix_args() { OPTS=$(PYTHONPATH=${PR_TESTING_DIR}/.. ${CMSBOT_PYTHON_CMD} -c 'from RelValArgs import GPU_RELVALS_FLAGS;print(GPU_RELVALS_FLAGS)') echo ${OPTS} } + +function check_invalid_wf_lists () { + local CMD_OPTS="$1" + local WKFS=$(sed -n 's/.*-l \([^ ]*\).*/\1/p' <<< "$1") + local DUMP_BADLIST=${2:-true} + local WFLISTS_CNT WFS_CNT BADLIST_CNT + + rm -f "$WORKSPACE/bad-workflow-lists.txt" + + # Count workflow lists vs numeric workflows + WFLISTS_CNT=$(echo "$WKFS" | tr ',' '\n' | grep -Ev "^[1-9][0-9]*(\.[0-9]+)?$" | wc -l) + WFS_CNT=$(echo "$WKFS" | tr ',' '\n' | wc -l) + + # Capture both output and exit code of runTheMatrix.py + local RUN_OUTPUT + if ! RUN_OUTPUT=$(runTheMatrix.py ${CMD_OPTS} -n 2>&1); then + echo "ERROR : runTheMatrix returned non-zero exit code" + return 1 + fi + + # Extract bad workflow lists directly from the output + if (( WFLISTS_CNT > 0 )); then + local BADLIST=$(grep "is not a possible selected entry" <<< "$RUN_OUTPUT" | awk '{print $1}') + BADLIST_CNT=$(echo $BADLIST | wc -w) + if (( BADLIST_CNT > 0 )); then + if [[ "$DUMP_BADLIST" == "true" ]]; then + echo " - $(echo $BADLIST | tr ' ' ',')" + fi + if (( WFLISTS_CNT != WFS_CNT )); then + echo "WARNING : some workflow lists were not recognized" + else + if (( BADLIST_CNT == WFS_CNT )); then + echo "ERROR : all workflow lists were not recognized, and no additional workflows were requested" + return 1 + else + echo "WARNING : none of the workflow lists were recognized, only running explicitly requested workflows" + fi + fi + fi + fi + return 0 +} diff --git a/pr_testing/run-pr-relvals.sh b/pr_testing/run-pr-relvals.sh index d3b6ca86ab73..11e5f69a7f5f 100755 --- a/pr_testing/run-pr-relvals.sh +++ b/pr_testing/run-pr-relvals.sh @@ -104,6 +104,12 @@ else echo "no errors in the RelVals!!" echo "MATRIX${UC_TEST_FLAVOR}_TESTS;OK,Matrix ${UC_TEST_FLAVOR} Tests Outputs,See Logs,runTheMatrix${UC_TEST_FLAVOR}-results" >> ${RESULTS_DIR}/$(get_status_file_name relval "$TEST_FLAVOR") + if [[ -s "$WORKSPACE/bad-workflow-lists.txt" ]]; then + echo "### Invalid workflow lists" > ${RESULTS_DIR}/0a-bad-workflows-report.res + echo "The following workflow names were not recognized and not ran:" >> ${RESULTS_DIR}/0a-bad-workflows-report.res + cat "$WORKSPACE/bad-workflow-lists.txt" >> ${RESULTS_DIR}/0a-bad-workflows-report.res + fi + if $DO_COMPARISON ; then echo "COMPARISON${UC_TEST_FLAVOR};QUEUED,Comparison ${UC_TEST_FLAVOR} with the baseline,See results,See results" >> ${RESULTS_DIR}/comparison${UC_TEST_FLAVOR}.txt TRIGGER_COMPARISON_FILE=$WORKSPACE/'comparison.properties' diff --git a/pr_testing/test_multiple_prs.sh b/pr_testing/test_multiple_prs.sh index 510c5936bc92..162174a0d61f 100755 --- a/pr_testing/test_multiple_prs.sh +++ b/pr_testing/test_multiple_prs.sh @@ -351,6 +351,8 @@ if $DO_COMPARISON ; then echo "TEST_FLAVOR=" >> run-baseline-${BUILD_ID}-01.default echo "REAL_ARCH=${RELVAL_REAL_ARCH}" >> run-baseline-${BUILD_ID}-01.default echo "PRODUCTION_RELEASE=true" >> run-baseline-${BUILD_ID}-01.default + # -- REMOVE NEXT LINE BEFORE MERGING -- + echo "CMS_BOT_BRANCH=run-ib-pr-matrix-list" >> run-baseline-${BUILD_ID}-01.default WF_LIST=$(get_pr_baseline_worklflow) [ "${WF_LIST}" = "" ] || WF_LIST="-l ${WF_LIST}" echo "WORKFLOWS=-s ${WF_LIST}" >> run-baseline-${BUILD_ID}-01.default diff --git a/run-ib-pr-matrix.sh b/run-ib-pr-matrix.sh index a228185f6918..7dce0ec8170e 100755 --- a/run-ib-pr-matrix.sh +++ b/run-ib-pr-matrix.sh @@ -26,12 +26,18 @@ if [ "${CHECK_WORKFLOWS}" = "true" ] ; then ALL_WFS=$(runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' | tr '\n' ',' | sed 's|,$||') WORKFLOWS=$(echo "${WORKFLOWS}" | sed "s|all|${ALL_WFS}|") fi - runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} ${WORKFLOWS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' > $WORKSPACE/req.wfs + rm -f $WORKSPACE/req.wfs + if ! check_invalid_wf_lists "${OPTS} ${MATRIX_ARGS} ${WORKFLOWS}" false ; then + touch $WORKSPACE/req.wfs + else + runTheMatrix.py -n ${OPTS} ${MATRIX_ARGS} ${WORKFLOWS} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' > $WORKSPACE/req.wfs + fi for wf in $(cat $WORKSPACE/req.wfs) ; do - [ $(echo " $REL_WFS " | grep " $wf " | wc -l) -eq 0 ] || continue - WFS="${wf},${WFS}" + [ $(echo " $REL_WFS " | grep " $wf " | wc -l) -eq 0 ] || continue + WFS="${wf},${WFS}" done WFS=$(echo ${WFS} | sed 's|,$||') + if [ "${WFS}" = "" ] ; then mv ${WORKSPACE}/workflows-${BUILD_ID}.log ${WORKSPACE}/workflows-${BUILD_ID}.done send_jenkins_artifacts ${WORKSPACE}/workflows-${BUILD_ID}.done ${ARTIFACT_DIR}/workflows-${BUILD_ID}.done @@ -74,10 +80,17 @@ pushd "$WORKSPACE/matrix-results" else CMD_OPTS="${CMD_OPTS} ${EXTRA_MATRIX_COMMAND_ARGS}" fi + + # Check what workflows will be ran (without any --command option) + if ! check_invalid_wf_lists "${MATRIX_ARGS}" ; then + exit 1 + fi + [ "${CMD_OPTS}" != "" ] && MATRIX_ARGS="${MATRIX_ARGS} --command ' ${CMD_OPTS}'" if [ "X$CMS_SITE_OVERRIDE" == "X" ]; then CMS_SITE_OVERRIDE="local" fi + eval CMS_PATH=/cvmfs/cms-ib.cern.ch SITECONFIG_PATH=/cvmfs/cms-ib.cern.ch/SITECONF/$CMS_SITE_OVERRIDE runTheMatrix.py -j ${NJOBS} ${MATRIX_ARGS} 2>&1 | tee -a matrixTests.${BUILD_ID}.log mv runall-report-step123-.log runall-report-step123-.${BUILD_ID}.log find . -name DQM*.root | sort | sed 's|^./||' > wf_mapping.${BUILD_ID}.txt