diff --git a/sorc/build_compute.sh b/sorc/build_compute.sh index 794b4fa350..0a56c1ba13 100755 --- a/sorc/build_compute.sh +++ b/sorc/build_compute.sh @@ -100,13 +100,29 @@ while [[ "${finished}" == "false" ]]; do finished=true elif [[ "${state}" == "RUNNING" ]]; then finished=false - elif [[ "${state}" == "DEAD" ]]; then - echo "FATAL ERROR: ${BASH_SOURCE[0]} one or more builds failed!" - # TODO add capability to determine which build(s) failed - exit 2 else echo "FATAL ERROR: ${BASH_SOURCE[0]} rocoto failed with state '${state}'" - exit 3 + rm -f logs/error.logs + # Determine which build(s) failed + stat_out="$(rocotostat -w "${build_xml}" -d "${build_db}")" + echo "${stat_out}" > rocotostat.out + line_number=0 + while read -r line; do + (( line_number += 1 )) + # Skip the first two lines (header) + if [[ ${line_number} -lt 3 ]]; then + continue + fi + + if [[ "${line}" =~ "DEAD" || "${line}" =~ "UNKNOWN" || + "${line}" =~ "UNAVAILABLE" || "${line}" =~ "FAIL" ]]; then + job=$(echo "${line}" | awk '{ print $2 }') + log_file="logs/${job}.log" + echo "${log_file}" >> logs/error.logs + echo "Rocoto reported that the build failed for ${job}" + fi + done < rocotostat.out + exit 2 fi done diff --git a/workflow/build_opts.yaml b/workflow/build_opts.yaml index f2074c7cb0..c17432d4a2 100644 --- a/workflow/build_opts.yaml +++ b/workflow/build_opts.yaml @@ -23,72 +23,60 @@ systems: build: gfs_model: command: "./build_ufs.sh -e gfs_model.x -j 12" - log: "build_ufs_gfs.log" cores: 12 walltime: "00:30:00" gfs_ww3prepost: command: "./build_ww3prepost.sh -j 4" - log: "build_ww3prepost_gfs.log" cores: 4 walltime: "00:10:00" gefs_model: command: "./build_ufs.sh -w -e gefs_model.x -j 12" - log: "build_ufs_gefs.log" cores: 12 walltime: "00:30:00" gefs_ww3_prepost: command: "./build_ww3prepost.sh -w -j 4" - log: "build_ww3prepost_gefs.log" cores: 4 walltime: "00:10:00" sfs_model: command: "./build_ufs.sh -y -e sfs_model.x -j 12" - log: "build_ufs_sfs.log" cores: 12 walltime: "00:30:00" upp: command: "./build_upp.sh -j 8" - log: "build_upp.log" cores: 8 walltime: "00:10:00" gsi_enkf: command: "./build_gsi_enkf.sh -j 8" - log: "build_gsi_enkf.log" cores: 8 walltime: "00:15:00" gsi_monitor: command: "./build_gsi_monitor.sh -j 4" - log: "build_gsi_monitor.log" cores: 4 walltime: "00:10:00" gsi_utils: command: "./build_gsi_utils.sh -j 6" - log: "build_gsi_utils.log" cores: 6 walltime: "00:10:00" ufs_utils: command: "./build_ufs_utils.sh -j 8" - log: "build_ufs_utils.log" cores: 8 walltime: "00:10:00" gfs_utils: command: "./build_gfs_utils.sh -j 6" - log: "build_gfs_utils.log" cores: 6 walltime: "00:10:00" gdas: command: "./build_gdas.sh -j 24" - log: "build_gdas.log" cores: 24 walltime: "01:30:00"