Fix error log parsing in compute build CI (#3301)

This PR adds a new feature to capture and report which compute builds
failed when running CI.

Resolves #3296

Author: DavidHuber-NOAA

sorc/build_compute.sh

@@ -100,13 +100,29 @@ while [[ "${finished}" == "false" ]]; do
       finished=true
    elif [[ "${state}" == "RUNNING" ]]; then
       finished=false
-   elif [[ "${state}" == "DEAD" ]]; then
-      echo "FATAL ERROR: ${BASH_SOURCE[0]} one or more builds failed!"
-      # TODO add capability to determine which build(s) failed
-      exit 2
    else
       echo "FATAL ERROR: ${BASH_SOURCE[0]} rocoto failed with state '${state}'"
-      exit 3
+      rm -f logs/error.logs
+      # Determine which build(s) failed
+      stat_out="$(rocotostat -w "${build_xml}" -d "${build_db}")"
+      echo "${stat_out}" > rocotostat.out
+      line_number=0
+      while read -r line; do
+         (( line_number += 1 ))
+         # Skip the first two lines (header)
+         if [[ ${line_number} -lt 3 ]]; then
+            continue
+         fi
+
+         if [[ "${line}" =~ "DEAD" || "${line}" =~ "UNKNOWN" ||
+               "${line}" =~ "UNAVAILABLE" || "${line}" =~ "FAIL" ]]; then
+            job=$(echo "${line}" | awk '{ print $2 }')
+            log_file="logs/${job}.log"
+            echo "${log_file}" >> logs/error.logs
+            echo "Rocoto reported that the build failed for ${job}"
+         fi  
+      done < rocotostat.out
+      exit 2
    fi
 done
 

workflow/build_opts.yaml

@@ -23,72 +23,60 @@ systems:
 build:
   gfs_model:
     command: "./build_ufs.sh -e gfs_model.x -j 12"
-    log: "build_ufs_gfs.log"
     cores: 12
     walltime: "00:30:00"
 
   gfs_ww3prepost:
     command: "./build_ww3prepost.sh -j 4"
-    log: "build_ww3prepost_gfs.log"
     cores: 4
     walltime: "00:10:00"
 
   gefs_model:
     command: "./build_ufs.sh -w -e gefs_model.x -j 12"
-    log: "build_ufs_gefs.log"
     cores: 12
     walltime: "00:30:00"
 
   gefs_ww3_prepost:
     command: "./build_ww3prepost.sh -w -j 4"
-    log: "build_ww3prepost_gefs.log"
     cores: 4
     walltime: "00:10:00"
 
   sfs_model:
     command: "./build_ufs.sh -y -e sfs_model.x -j 12"
-    log: "build_ufs_sfs.log"
     cores: 12
     walltime: "00:30:00"
 
   upp:
     command: "./build_upp.sh -j 8"
-    log: "build_upp.log"
     cores: 8
     walltime: "00:10:00"
 
   gsi_enkf:
     command: "./build_gsi_enkf.sh -j 8"
-    log: "build_gsi_enkf.log"
     cores: 8
     walltime: "00:15:00"
 
   gsi_monitor:
     command: "./build_gsi_monitor.sh -j 4"
-    log: "build_gsi_monitor.log"
     cores: 4
     walltime: "00:10:00"
 
   gsi_utils:
     command: "./build_gsi_utils.sh -j 6"
-    log: "build_gsi_utils.log"
     cores: 6
     walltime: "00:10:00"
 
   ufs_utils:
     command: "./build_ufs_utils.sh -j 8"
-    log: "build_ufs_utils.log"
     cores: 8
     walltime: "00:10:00"
 
   gfs_utils:
     command: "./build_gfs_utils.sh -j 6"
-    log: "build_gfs_utils.log"
     cores: 6
     walltime: "00:10:00"
 
   gdas:
     command: "./build_gdas.sh -j 24"
-    log: "build_gdas.log"
     cores: 24
     walltime: "01:30:00"