Run PR gpu utests/relvals on both CUDA and ROCm GPUs

Author: iarspider

cleanup-cmssdt

@@ -26,7 +26,7 @@ done
 
 RUNS_TO_KEEP=1
 #Keep $JOBS_TO_KEEP of these with RUNS_TO_KEEP
-for type in "" CUDA ROCM $(PYTHONPATH=${CMS_BOT_DIR} python3 -c 'from process_pr import EXTRA_RELVALS_TESTS;print(" ".join([x.upper().replace("-","_") for x in EXTRA_RELVALS_TESTS if x]))') ; do
+for type in "" $(cat "gpu_flavors.txt" | tr '[:lower:]' '[:upper:]' | tr '\n' ' ') $(PYTHONPATH=${CMS_BOT_DIR} python3 -c 'from process_pr import EXTRA_RELVALS_TESTS;print(" ".join([x.upper().replace("-","_") for x in EXTRA_RELVALS_TESTS if x]))') ; do
   dir=baseLineComparisons${type}
   [ -d ${JENKINS_ARTIFACTS}/$dir ] || continue
   DIRS_PROCESSED="${DIRS_PROCESSED} ${dir}"

cmssw-pr-test-config

@@ -1,4 +1,6 @@
 #!/bin/bash
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
+readarray -t ALL_GPU_TYPES < ${SCRIPTPATH}/gpu_flavors.txt
 CMSSW_VER=$CMSSW_VERSION
 [ "${CMSSW_VER}" != "" ] || CMSSW_VER=${RELEASE_FORMAT}
 CMSSW_MAJOR=0
@@ -20,6 +22,12 @@ elif [ "$CMSSW_VER" -ge 1300 ] ; then
 else
   PR_TEST_MATRIX_EXTRAS_GPU=11634.586,11634.587
 fi
+if [ X"$PR_TEST_MATRIX_EXTRAS_GPU" != X"" ]; then
+  for GPU_T in ${ALL_GPU_TYPES[@]} ; do
+    GPU_T_UC=$(echo ${GPU_T} | tr '[a-z]' '[A-Z]')
+    eval "PR_TEST_MATRIX_EXTRAS_${GPU_T_UC}=$PR_TEST_MATRIX_EXTRAS_GPU"
+  done
+fi
 PR_TEST_MATRIX_EXTRAS_PROFILING=29834.21,13034.21
 PR_TEST_MATRIX_EXTRAS_HIGH_STATS=35034.0
 PR_TEST_MATRIX_EXTRAS_NANO=all

gpu_flavors.txt

@@ -0,0 +1,2 @@
+cuda
+rocm

pr_testing/_helper_functions.sh

@@ -79,7 +79,7 @@ function prepare_upload_results (){
     else
       mkdir -p upload
     fi
-    for f in external_checks git-recent-commits.json cmssw.tar.gz unitTests gpuUnitTests dasqueries testsResults build-logs clang-logs runTheMatrix*-results llvm-analysis *.log *.html *.txt *.js DQMTestsResults valgrindResults-* cfg-viewerResults igprof-results-data git-merge-result git-log-recent-commits addOnTests codeRules dupDict material-budget cmsset_default; do
+    for f in external_checks git-recent-commits.json cmssw.tar.gz unitTests *UnitTests dasqueries testsResults build-logs clang-logs runTheMatrix*-results llvm-analysis *.log *.html *.txt *.js DQMTestsResults valgrindResults-* cfg-viewerResults igprof-results-data git-merge-result git-log-recent-commits addOnTests codeRules dupDict material-budget cmsset_default; do
       [ -e $f ] && mv $f upload/$f
     done
     if [ -e upload/renderPRTests.js ] ; then mkdir -p upload/js && mv upload/renderPRTests.js upload/js/ ; fi
@@ -120,3 +120,16 @@ function prepare_upload_results (){
     mkdir -p ${RESULTS_DIR}
   popd
 }
+
+function is_in_array() {
+    local value="$1"
+    shift
+    local array=("$@")
+
+    for item in "${array[@]}"; do
+        if [[ "$item" == "$value" ]]; then
+            return 0  # Found match
+        fi
+    done
+    return 1  # No match
+}

pr_testing/run-pr-comparisons

@@ -264,7 +264,7 @@ for WF in ${WORKFLOWS_TO_COMPARE//,/ }; do
 done
 
 echo $WFS_WITH_DAS_INCONSISTENCY >> $WORKSPACE/$DAS_NON_CONSISTENT_WFS_FILE
-echo "COMPARISON${TEST_FLAVOR_STR};RUNNING,Comparison with the ${UC_TEST_FLAVOR} baseline,See results,See results" >> ${RESULTS_FILE}
+echo "COMPARISON${TEST_FLAVOR_STR};RUNNING,Comparison with the ${UC_TEST_FLAVOR} baseline,See results,/SDT/jenkins-artifacts/${COMP_UPLOAD_DIR}/" >> ${RESULTS_FILE}
 if [ "$DRY_RUN" = "" ] ; then
   send_jenkins_artifacts ${RESULTS_FILE} ${PR_BASELINE_JOBDIR}/testsResults/comparison${UC_TEST_FLAVOR}.txt
 fi

pr_testing/run-pr-relvals.sh

@@ -1,5 +1,7 @@
 #!/bin/bash -ex
 source $(dirname $0)/setup-pr-test-env.sh
+readarray -t ALL_GPU_TYPES < ${CMS_BOT_DIR}/gpu_flavors.txt
+
 GH_CONTEXT="relvals"
 GH_COMP_CONTEXT="comparison"
 UC_TEST_FLAVOR=$(echo ${TEST_FLAVOR} | tr '[a-z]' '[A-Z]')
@@ -17,10 +19,12 @@ echo "${MATRIX_ARGS}"  | tr ';' '\n' | while IFS= read -r args; do
   if [ $(echo "${args}" | sed 's|.*-l ||;s| .*||' | tr ',' '\n' | grep '^all$' | wc -l) -gt 0 ] ; then
     OPTS=""
     case "${TEST_FLAVOR}" in
-      gpu ) OPTS="-w gpu" ;;
       high_stats ) ;;
       nano ) OPTS="-w nano" ;;
-      * ) ;;
+      * ) if is_in_array "${TEST_FLAVOR}" "${ALL_GPU_TYPES[@]}" ; then
+            OPTS="-w gpu"
+          fi
+          ;;
     esac
     ALL_WFS=$(runTheMatrix.py -n ${OPTS} ${args} | grep -v ' workflows ' | grep '^[1-9][0-9]*\(.[0-9][0-9]*\|\)\s' | sed 's| .*||' | tr '\n' ',' | sed 's|,$||')
     args=$(echo "${args}" | sed "s|all|${ALL_WFS}|")

pr_testing/run-pr-unittests.sh

@@ -10,57 +10,59 @@ cd $WORKSPACE/${CMSSW_VERSION}
 CMSSW_PKG_COUNT=$(ls -d $LOCALRT/src/*/* | wc -l)
 REPORT_OPTS="--report-url ${PR_RESULT_URL} $NO_POST"
 
-rm -f ${RESULTS_DIR}/unittestGPU.txt
-mark_commit_status_all_prs 'unittests/gpu' 'pending' -u "${BUILD_URL}" -d "Running tests" || true
+rm -f ${RESULTS_DIR}/unittest${TEST_FLAVOR}.txt
+mark_commit_status_all_prs "unittests/${TEST_FLAVOR}" 'pending' -u "${BUILD_URL}" -d "Running tests" || true
 echo '--------------------------------------'
-mkdir -p $WORKSPACE/gpuUnitTests
+mkdir -p $WORKSPACE/${TEST_FLAVOR}UnitTests
 let UT_TIMEOUT=7200+${CMSSW_PKG_COUNT}*20
-UTESTS_CMD="USER_UNIT_TESTS=cuda timeout ${UT_TIMEOUT} scram b -v -k -j ${NCPU}  unittests "
+gpu_t_lc=$(echo ${TEST_FLAVOR} | tr '[A-Z]' '[a-z]')
+UTESTS_CMD="USER_UNIT_TESTS=${gpu_t_lc} timeout ${UT_TIMEOUT} scram b -v -k -j ${NCPU}  unittests "
 echo "LD_LIBRARY_PATH: ${LD_LIBRARY_PATH}"
 scram build echo_LD_LIBRARY_PATH || true
 scram build -r echo_CXX || true
 cms_major=$(echo ${CMSSW_IB} | cut -d_ -f2)
 cms_minor=$(echo ${CMSSW_IB} | cut -d_ -f3)
 cms_ver="$(echo 00${cms_major} | sed -E 's|^.*(..)$|\1|')$(echo 00${cms_minor} | sed -E 's|^.*(..)$|\1|')"
-echo $UTESTS_CMD > $WORKSPACE/gpuUnitTests/log.txt
-(eval $UTESTS_CMD && echo 'ALL_OK') > $WORKSPACE/gpuUnitTests/log.txt 2>&1 || true
+echo $UTESTS_CMD > $WORKSPACE/${TEST_FLAVOR}UnitTests/log.txt
+(eval $UTESTS_CMD && echo 'ALL_OK') > $WORKSPACE/${TEST_FLAVOR}UnitTests/log.txt 2>&1 || true
 echo 'END OF UNIT TESTS'
 echo '--------------------------------------'
 
-TEST_ERRORS=$(grep -ai 'had errors\|recipe for target' $WORKSPACE/gpuUnitTests/log.txt | sed "s|'||g;s|.*recipe for target *||;s|.*unittests_|---> test |;s| failed$| timeout|" || true)
-TEST_ERRORS=`grep -ai "had errors" $WORKSPACE/gpuUnitTests/log.txt` || true
-GENERAL_ERRORS=`grep -a "ALL_OK" $WORKSPACE/gpuUnitTests/log.txt` || true
+TEST_ERRORS=$(grep -ai 'had errors\|recipe for target' $WORKSPACE/${TEST_FLAVOR}UnitTests/log.txt | sed "s|'||g;s|.*recipe for target *||;s|.*unittests_|---> test |;s| failed$| timeout|" || true)
+TEST_ERRORS=`grep -ai "had errors" $WORKSPACE/${TEST_FLAVOR}UnitTests/log.txt` || true
+GENERAL_ERRORS=`grep -a "ALL_OK" $WORKSPACE/${TEST_FLAVOR}UnitTests/log.txt` || true
 
+TEST_FLAVOR_UC=$(echo $TEST_FLAVOR | tr '[:lower:]' '[:upper:]')
 if [ "X$TEST_ERRORS" != "X" -o "X$GENERAL_ERRORS" = "X" ]; then
-  echo "Errors in the gpu unit tests"
-  echo 'GPU_UNIT_TEST_RESULTS;ERROR,GPU Unit Tests,See Log,gpuUnitTests' >> ${RESULTS_DIR}/unittestGPU.txt
+  echo "Errors in the ${TEST_FLAVOR} unit tests"
+  echo "${TEST_FLAVOR_UC}_UNIT_TEST_RESULTS;ERROR,Unit Tests ${TEST_FLAVOR_UC},See Log,${TEST_FLAVOR}UnitTests" >> ${RESULTS_DIR}/unittest${TEST_FLAVOR}.txt
   ALL_OK=false
   UNIT_TESTS_OK=false
-  $CMS_BOT_DIR/report-pull-request-results PARSE_GPU_UNIT_TESTS_FAIL -f $WORKSPACE/gpuUnitTests/log.txt --report-file ${RESULTS_DIR}/14-unittestGPU-report.res ${REPORT_OPTS}
-  echo "GpuUnitTests" > ${RESULTS_DIR}/14-failed.res
+  $CMS_BOT_DIR/report-pull-request-results PARSE_${TEST_FLAVOR_UC}_UNIT_TESTS_FAIL -f $WORKSPACE/${TEST_FLAVOR}UnitTests/log.txt --report-file ${RESULTS_DIR}/14-unittest${TEST_FLAVOR}-report.res ${REPORT_OPTS}
+  echo "${TEST_FLAVOR}UnitTests" > ${RESULTS_DIR}/14-${TEST_FLAVOR}-failed.res
 else
-  echo 'GPU_UNIT_TEST_RESULTS;OK,GPU Unit Tests,See Log,gpuUnitTests' >> ${RESULTS_DIR}/unittestGPU.txt
+  echo "${TEST_FLAVOR_UC}_UNIT_TEST_RESULTS;OK,Unit Tests ${TEST_FLAVOR_UC},See Log,${TEST_FLAVOR}UnitTests" >> ${RESULTS_DIR}/unittest${TEST_FLAVOR}.txt
 fi
-echo "<html><head></head><body>" > $WORKSPACE/gpuUnitTests/success.html
-cp $WORKSPACE/gpuUnitTests/success.html $WORKSPACE/gpuUnitTests/failed.html
+echo "<html><head></head><body>" > $WORKSPACE/${TEST_FLAVOR}UnitTests/success.html
+cp $WORKSPACE/${TEST_FLAVOR}UnitTests/success.html $WORKSPACE/${TEST_FLAVOR}UnitTests/failed.html
 UT_ERR=false
 utlog="testing.log"
 for t in $(find $WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/src -name ${utlog} -type f | sed "s|$WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/||;s|/${utlog}$||") ; do
-  mkdir -p $WORKSPACE/gpuUnitTests/${t}
-  mv $WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/${t}/${utlog} $WORKSPACE/gpuUnitTests/${t}/
-  if [ $(grep -a '^\-\-\-> test  *[^ ]*  *succeeded$' $WORKSPACE/gpuUnitTests/${t}/${utlog} | wc -l) -gt 0 ] ; then
-    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/gpuUnitTests/success.html
+  mkdir -p $WORKSPACE/${TEST_FLAVOR}UnitTests/${t}
+  mv $WORKSPACE/$CMSSW_IB/tmp/${SCRAM_ARCH}/${t}/${utlog} $WORKSPACE/${TEST_FLAVOR}UnitTests/${t}/
+  if [ $(grep -a '^\-\-\-> test  *[^ ]*  *succeeded$' $WORKSPACE/${TEST_FLAVOR}UnitTests/${t}/${utlog} | wc -l) -gt 0 ] ; then
+    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/${TEST_FLAVOR}UnitTests/success.html
   else
-    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/gpuUnitTests/failed.html
+    echo "<a href='${t}/${utlog}'>${t}</a><br/>" >> $WORKSPACE/${TEST_FLAVOR}UnitTests/failed.html
     UT_ERR=true
   fi
 done
-if ! $UT_ERR ; then echo "No unit test failed" >> $WORKSPACE/gpuUnitTests/failed.html ; fi
-echo "</body></html>" >> $WORKSPACE/gpuUnitTests/success.html
-echo "</body></html>" >> $WORKSPACE/gpuUnitTests/failed.html
+if ! $UT_ERR ; then echo "No unit test failed" >> $WORKSPACE/${TEST_FLAVOR}UnitTests/failed.html ; fi
+echo "</body></html>" >> $WORKSPACE/${TEST_FLAVOR}UnitTests/success.html
+echo "</body></html>" >> $WORKSPACE/${TEST_FLAVOR}UnitTests/failed.html
 prepare_upload_results
 if $UNIT_TESTS_OK ; then
-  mark_commit_status_all_prs 'unittests/gpu' 'success' -u "${BUILD_URL}" -d "Passed"
+  mark_commit_status_all_prs "unittests/${TEST_FLAVOR}" 'success' -u "${BUILD_URL}" -d "Passed"
 else
-  mark_commit_status_all_prs 'unittests/gpu' 'error' -u "${BUILD_URL}" -d "Some unit tests were failed."
+  mark_commit_status_all_prs "unittests/${TEST_FLAVOR}" 'error' -u "${BUILD_URL}" -d "Some unit tests were failed."
 fi

pr_testing/test_multiple_prs.sh

@@ -79,20 +79,21 @@ function process_changed_files() {
   sort -u "$directlyChangedFiles" $WORKSPACE/indirectly-changed-files.txt > "$allChangedFiles"
 }
 
+SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"  # Absolute path to script
+CMS_BOT_DIR=$(dirname ${SCRIPTPATH})  # To get CMS_BOT dir path
+
 # Constants
 echo LD_LIBRARY_PATH=${LD_LIBRARY_PATH} || true
 ls ${LD_LIBRARY_PATH} || true
-SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"  # Absolute path to script
-CMS_BOT_DIR=$(dirname ${SCRIPTPATH})  # To get CMS_BOT dir path
 export SCRAM_PREFIX_PATH=${CMS_BOT_DIR}/das-utils
 source ${CMS_BOT_DIR}/cmsrep.sh
 CACHED=${WORKSPACE}/CACHED            # Where cached PR metada etc are kept
 PR_TESTING_DIR=${CMS_BOT_DIR}/pr_testing
 COMMON=${CMS_BOT_DIR}/common
 CONFIG_MAP=$CMS_BOT_DIR/config.map
 [ "${USE_IB_TAG}" != "true" ] && export USE_IB_TAG=false
-[ "${EXTRA_RELVALS_TESTS}" = "" ] && EXTRA_RELVALS_TESTS="GPU THREADING HIGH_STATS NANO"
-EXTRA_RELVALS_TESTS=$(echo ${EXTRA_RELVALS_TESTS} | tr ' ' '\n' | grep -v THREADING | tr '\n' ' ')
+[ "${EXTRA_RELVALS_TESTS}" = "" ] && EXTRA_RELVALS_TESTS="THREADING HIGH_STATS NANO $(echo ${ALL_GPU_TYPES[@]} | tr '[a-z]' '[A-Z]')"
+EXTRA_RELVALS_TESTS=$(echo ${EXTRA_RELVALS_TESTS} | tr ' ' '\n' | grep -v THREADING | grep -v GPU | tr '\n' ' ')
 # ---
 # doc: Input variable
 # PULL_REQUESTS   # "cms-sw/cmsdist#4488,cms-sw/cmsdist#4480,cms-sw/cmsdist#4479,cms-sw/root#116"
@@ -163,6 +164,20 @@ if [ $(echo "${CONFIG_LINE}" | grep "PROD_ARCH=1" | wc -l) -gt 0 ] ; then
   fi
 fi
 
+readarray -t ALL_GPU_TYPES < ${CMS_BOT_DIR}/gpu_flavors.txt
+
+declare -a ENABLE_GPU_FLAVORS
+for ex_type in ${ENABLE_BOT_TESTS} ; do
+  ex_type_lc=$(echo $ex_type | tr '[A-Z]' '[a-z]')
+  if is_in_array "$ex_type_lc" "${ALL_GPU_TYPES[@]}" ; then
+    ENABLE_GPU_FLAVORS+=( $ex_type )
+    VAR_NAME="MATRIX_EXTRAS_${ex_type}"
+    if [ -z "${!VAR_NAME}" ]; then
+      eval "$VAR_NAME=${MATRIX_EXTRAS_GPU}"
+    fi
+  fi
+done
+
 # ----------
 # -- MAIN --
 # ----------
@@ -1324,9 +1339,8 @@ if [ "X$BUILD_OK" = Xtrue -a "$RUN_TESTS" = "true" ]; then
       done
     fi
   fi
-  if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^GPU$' | wc -l) -gt 0 -a X"${DISABLE_GPU_TESTS}" != X"true" ] ; then
+  if [ ${#ENABLE_GPU_FLAVORS[@]} -ne 0 -a X"${DISABLE_GPU_TESTS}" != X"true" ] ; then
     DO_GPU_TESTS=true
-    mark_commit_status_all_prs 'unittests/gpu' 'pending' -u "${BUILD_URL}" -d "Waiting for tests to start"
   fi
   if [ $(echo ${ENABLE_BOT_TESTS} | tr ',' ' ' | tr ' ' '\n' | grep '^HLT_P2_TIMING$' | wc -l) -gt 0 ] ; then
     if [ $(echo ${ARCHITECTURE}   | grep "_amd64_" | wc -l) -gt 0 ] ; then
@@ -1499,7 +1513,12 @@ if [ "X$DO_ADDON_TESTS" = Xtrue ]; then
 fi
 
 if [ "X$DO_GPU_TESTS" = Xtrue ]; then
-  cp $WORKSPACE/test-env.txt $WORKSPACE/run-unittests.prop
+  for GPU_T in ${ENABLE_GPU_FLAVORS[@]}; do
+    GPU_T_LC=$(echo $GPU_T | tr '[A-Z]' '[a-z]')
+    cp $WORKSPACE/test-env.txt $WORKSPACE/run-unittests-${GPU_T_LC}.prop
+    echo "TEST_FLAVOR=${GPU_T_LC}" >> $WORKSPACE/run-unittests-${GPU_T_LC}.prop
+    mark_commit_status_all_prs "unittests/${GPU_T_LC}" 'pending' -u "${BUILD_URL}" -d "Waiting for tests to start"
+  done
 fi
 
 if ${BUILD_EXTERNAL} ; then
@@ -1510,7 +1529,7 @@ fi
 
 if [ "${DO_PROFILING}" = "true" ]  ; then
   PROFILING_WORKFLOWS=$($CMS_BOT_DIR/cmssw-pr-test-config _PROFILING | tr ',' ' ')
-  for wf in ${PROFILING_WORKFLOWS};do
+  for wf in ${PROFILING_WORKFLOWS}; do
     cp $WORKSPACE/test-env.txt $WORKSPACE/run-profiling-$wf.prop
     echo "PROFILING_WORKFLOWS=${wf}" >> $WORKSPACE/run-profiling-$wf.prop
   done
@@ -1525,3 +1544,4 @@ if [ "${DO_HLT_P2_INTEGRATION}" = "true" ] ;  then
 fi
 
 rm -f $WORKSPACE/test-env.txt
+

process_pr.py

@@ -143,7 +143,10 @@ def format(s, **kwds):
 REGEX_IGNORE_FILE_COUNT = r"\+file-count"
 TEST_WAIT_GAP = 720
 ALL_CHECK_FUNCTIONS = None
-EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"]
+ALL_GPU_FLAVORS = [
+    x.strip() for x in open(join(dirname(__file__), "gpu_flavors.txt"), "r").read().splitlines()
+]
+EXTRA_RELVALS_TESTS = ["threading", "gpu", "high-stats", "nano"] + ALL_GPU_FLAVORS
 EXTRA_RELVALS_TESTS_OPTS = "_" + "|_".join(EXTRA_RELVALS_TESTS)
 EXTRA_TESTS = (
     "|".join(EXTRA_RELVALS_TESTS)
@@ -168,7 +171,7 @@ def format(s, **kwds):
     "disable_poison": ["true|false", "DISABLE_POISON"],
     "use_ib_tag": ["true|false", "USE_IB_TAG"],
     "baseline": ["self|default", "USE_BASELINE"],
-    "set_env": ["[A-Z][A-Z0-9_]+(\s*,\s*[A-Z][A-Z0-9_]+|)*", "CMSBOT_SET_ENV"],
+    "set_env": [r"[A-Z][A-Z0-9_]+(\s*,\s*[A-Z][A-Z0-9_]+|)*", "CMSBOT_SET_ENV"],
     "skip_test(s|)": [format(r"(%(tests)s)(\s*,\s*(%(tests)s))*", tests=SKIP_TESTS), "SKIP_TESTS"],
     "dry_run": ["true|false", "DRY_RUN"],
     "jenkins_(slave|node)": [JENKINS_NODES, "RUN_ON_SLAVE"],
@@ -1315,7 +1318,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
         elif re.match(REGEX_EX_ENABLE_TESTS, first_line, re.I):
             comment_emoji = "-1"
             if valid_commenter:
-                enable_tests, ignore = check_enable_bot_tests(first_line.split(" ", 1)[-1])
+                enable_tests, _ = check_enable_bot_tests(first_line.split(" ", 1)[-1])
                 comment_emoji = "+1"
         elif re.match(r"^allow\s+@([^ ]+)\s+test\s+rights$", first_line, re.I):
             comment_emoji = "-1"
@@ -1375,6 +1378,7 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                 first_line, comment_lines, repository
             )
             if test_params_m:
+                # Error in parameters
                 test_params_msg = str(comment.id) + ":" + test_params_m
                 test_params_comment = comment
                 continue
@@ -1589,6 +1593,21 @@ def process_pr(repo_config, gh, repo, issue, dryRun, cmsbuild_user=None, force=F
                         set_comment_emoji_cache(dryRun, bot_cache, comment, repository)
 
     # end of parsing comments section
+
+    # Extract enabled GPU flavors and remove them from enable_tests
+    new_enable_tests = []
+    enabled_gpu_flavors = set()
+    for test in enable_tests.split():
+        if test == "GPU":
+            enabled_gpu_flavors.update([x.upper() for x in ALL_GPU_FLAVORS])
+        elif test.lower() in ALL_GPU_FLAVORS:
+            enabled_gpu_flavors.add(test)
+        else:
+            new_enable_tests.append(test)
+
+    new_enable_tests.extend(list(enabled_gpu_flavors))
+    enable_tests = " ".join(new_enable_tests)
+
     # Check if it needs to be automatically closed.
     if mustClose:
         if issue.state == "open":

report-pull-request-results.py

@@ -411,7 +411,7 @@ def read_unit_tests_file(unit_tests_file):
     send_message_pr(message)
 
 
-def read_gpu_tests_file(unit_tests_file):
+def read_gpu_tests_file(unit_tests_file, gpu_flavor="GPU"):
     errors_found = ""
     err_cnt = 0
     for line in openlog(unit_tests_file):
@@ -423,8 +423,8 @@ def read_gpu_tests_file(unit_tests_file):
                 continue
             errors_found += line
     message = (
-        "\n## GPU Unit Tests\n\nI found %s errors in the following unit tests:\n\n<pre>%s</pre>"
-        % (err_cnt, errors_found)
+        "\n## %s Unit Tests\n\nI found %s errors in the following unit tests:\n\n<pre>%s</pre>"
+        % (gpu_flavor, err_cnt, errors_found)
     )
     send_message_pr(message)
 
@@ -606,6 +606,8 @@ def complain_missing_param(param_name):
 GITLOG_FILE_BASE_URL = "%s/git-recent-commits.json" % options.report_url
 GIT_CMS_MERGE_TOPIC_BASE_URL = "%s/git-merge-result" % options.report_url
 
+ACTION = ACTION.upper()
+
 if ACTION == "GET_BASE_MESSAGE":
     get_base_message()
 elif ACTION == "PARSE_UNIT_TESTS_FAIL":
@@ -630,6 +632,10 @@ def complain_missing_param(param_name):
     read_material_budget_log_file(options.unit_tests_file)
 elif ACTION == "MERGE_COMMITS":
     add_to_report(get_recent_merges_message())
+elif ACTION == "PARSE_CUDA_UNIT_TESTS_FAIL":
+    read_gpu_tests_file(options.unit_tests_file, "CUDA")
+elif ACTION == "PARSE_ROCM_UNIT_TESTS_FAIL":
+    read_gpu_tests_file(options.unit_tests_file, "ROCm")
 elif ACTION == "PARSE_GPU_UNIT_TESTS_FAIL":
     read_gpu_tests_file(options.unit_tests_file)
 else: