Skip to content

Commit 834ce31

Browse files
WalterKolczynski-NOAAWalterKolczynski-NOAA
authored
Refactor gempak jobs for new COM and style (#2374)
Updates the gempak jobs to fit the new COM structure while also refactoring them some to improve the style. Despite these technical changes, the overall structure is left unchanged for most scripts, though some have been rewritten to make the needed changes easier. Some of these scripts had already been updated some in the original COM refactor and thus needed fewer updates. Style updates includes converting all gempak scripts to bash, making them shellcheck compliant, and removing trailing whitespace. Further refactoring to improve maintainability will be needed in the future (see #2341, #2342, #2343, #2348). The GFS gif scripts were identical except the forecast hour, so they are collapsed down into two: one for f000 and one for other forecast hours. The gempak executables have short path limits. To get around this without having the gempak module recompiled, target directories (mostly relevant for the gempak meta jobs) are symlinked into the working directory to drasticly reduce the path lengths. Part of this update includes replacing existing MPMD calls with the new standard `ush/run_mpmd.sh` script. A new function, `wait_for_file()`, is introduced to standardize waiting for a file to be available. Gempak forecast hours are often hard-coded within scripts. In addition to issues with maintainability, this causes problems for shorter forecasts, such as we typically run for testing purposes. For now, we simply check the values against the forecast length and reduce if necessary. Future work (#2348) will be needed to remove these hard-coded values with variables set in the config file (or just use update gempak products to match standard output time variables). One-degree gempak files have been updated to include `1p00` in the filename. Several gempak job dependencies are corrected. Fake gempak data for external models is being staged on tier-1 machines to allow testing. **Output has not been verified.** Future PRs will likely be needed to bring full functionality online. Resolves #2158 Resolves #2152 Resolves #2151 Resolves #2249 Resolves #2247 Refs #2157 Refs #2348
1 parent 20635b0 commit 834ce31

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+3559
-6467
lines changed

‎.shellcheckrc

+3
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,6 @@ disable=SC1091
1414

1515
# Disable -p -m only applies to deepest directory
1616
disable=SC2174
17+
18+
# Disable warning of functions in test statements
19+
disable=SC2310

‎gempak/fix/datatype.tbl

+7-7
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,10 @@ LTNG $OBS/ltng YYYYMMDDHH.ltng CAT_MSC SCAT_N
102102
!
103103
CLIMO $GEMPAK/climo climate_MM.mos CAT_NIL SCAT_NIL 1 -1 -1
104104
!
105-
GFS $MODEL/gfs gfs_YYYYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
106-
F-GFS $COMIN gfs_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
107-
F-GFSP $COMIN gfs_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
108-
F-GFSHPC $HPCGFS gfs_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
105+
GFS $MODEL/gfs gfs_1p00_YYYYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
106+
F-GFS $COMIN gfs_1p00_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
107+
F-GFSP $COMIN gfs_1p00_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
108+
F-GFSHPC $HPCGFS gfs_1p00_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
109109
GFSEXT $MODEL/ens gfs.YYYYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
110110
GFS1 $MODEL/ens gfs1.YYYYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
111111
GFS2 $MODEL/ens gfs2.YYYYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
@@ -156,9 +156,9 @@ F-NAMP20 $COMIN nam20_YYYYMMDDHHfFFF CAT_GRD SCAT_F
156156
F-NAMP44 $COMIN nam44_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
157157
F-THREATS $COMIN ${NEST}_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
158158
F-NAMHPC $HPCNAM nam_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
159-
GDAS $MODEL/gdas gdas_YYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
160-
F-GDAS $COMIN gdas_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
161-
F-GFS $COMIN gfs_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
159+
GDAS $MODEL/gdas gdas_1p00_YYYYMMDDHH CAT_GRD SCAT_FCT -1 -1 -1
160+
F-GDAS $COMIN gdas_1p00_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
161+
F-GFS $COMIN gfs_1p00_YYYYMMDDHHfFFF CAT_GRD SCAT_FCT -1 -1 -1
162162
F-HWRF $COMIN hwrfp_YYYYMMDDHHfFFF_* CAT_GRD SCAT_FCT -1 -1 -1
163163
F-HWRFN $COMIN hwrfn_YYYYMMDDHHfFFF_* CAT_GRD SCAT_FCT -1 -1 -1
164164
F-GHM $COMIN ghmg_YYYYMMDDHHfFFF_* CAT_GRD SCAT_FCT -1 -1 -1

‎gempak/ush/gdas_ecmwf_meta_ver.sh

+45-100
Original file line numberDiff line numberDiff line change
@@ -1,75 +1,39 @@
1-
#!/bin/sh
2-
#
3-
# Metafile Script : gdas_ecmwf_meta_ver
1+
#! /usr/bin/env bash
42
#
53
# Creates a loop comparing the 6 hr gdas fcst to the pervious 7 days
64
# of ecmwf fcsts
75
#
8-
# Log :
9-
# J. Carr/HPC 3/2001 New metafile for verification of ecmwf.
10-
# J. Carr/HPC 5/2001 Added a mn variable for a/b side dbnet root variable.
11-
# M. Klein/HPC 11/2004 Changed verification grid from fnl to gdas
12-
# M. Klein/HPC 2/2005 Changed location of working directory to /ptmp
13-
# M. Klein/HPC 11/2006 Modify to run in production.
14-
#
15-
16-
#cd $DATA
176

18-
set -xa
19-
20-
if [ $cyc -ne "06" ] ; then
21-
exit
22-
fi
7+
source "${HOMEgfs}/ush/preamble.sh"
238

24-
export pgm=gdplot2_nc;. prep_step; startmsg
9+
export pgm=gdplot2_nc;. prep_step
2510

26-
cyc=12
11+
cyc2=12
2712
device="nc | ecmwfver.meta"
28-
PDY2=$(echo ${PDY} | cut -c3-)
2913

3014
#
3115
# Copy in datatype table to define gdfile type
3216
#
33-
cp ${HOMEgfs}/gempak/fix/datatype.tbl datatype.tbl
17+
18+
cp "${HOMEgfs}/gempak/fix/datatype.tbl" datatype.tbl
3419
export err=$?
35-
if [[ $err -ne 0 ]] ; then
36-
echo " File datatype.tbl does not exist."
37-
exit $err
20+
if (( err != 0 )) ; then
21+
echo "FATAL ERROR: File datatype.tbl does not exist."
22+
exit "${err}"
3823
fi
3924

40-
#
41-
# DEFINE YESTERDAY
42-
date1=$($NDATE -24 ${PDY}${cyc} | cut -c -8)
43-
sdate1=$(echo ${date1} | cut -c 3-)
44-
# DEFINE 2 DAYS AGO
45-
date2=$($NDATE -48 ${PDY}${cyc} | cut -c -8)
46-
sdate2=$(echo ${date2} | cut -c 3-)
47-
# DEFINE 3 DAYS AGO
48-
date3=$($NDATE -72 ${PDY}${cyc} | cut -c -8)
49-
sdate3=$(echo ${date3} | cut -c 3-)
50-
# DEFINE 4 DAYS AGO
51-
date4=$($NDATE -96 ${PDY}${cyc} | cut -c -8)
52-
sdate4=$(echo ${date4} | cut -c 3-)
53-
# DEFINE 5 DAYS AGO
54-
date5=$($NDATE -120 ${PDY}${cyc} | cut -c -8)
55-
sdate5=$(echo ${date5} | cut -c 3-)
56-
# DEFINE 6 DAYS AGO
57-
date6=$($NDATE -144 ${PDY}${cyc} | cut -c -8)
58-
sdate6=$(echo ${date6} | cut -c 3-)
59-
# DEFINE 7 DAYS AGO
60-
date7=$($NDATE -168 ${PDY}${cyc} | cut -c -8)
61-
sdate7=$(echo ${date7} | cut -c 3-)
62-
63-
vergrid="F-GDAS | ${PDY2}/0600"
25+
export COMIN="gdas.${PDY}${cyc}"
26+
if [[ ! -L ${COMIN} ]]; then
27+
ln -sf "${COM_ATMOS_GEMPAK_1p00}" "${COMIN}"
28+
fi
29+
vergrid="F-GDAS | ${PDY:2}/0600"
6430
fcsthr="0600f006"
6531

6632
# GENERATING THE METAFILES.
6733
areas="SAM NAM"
68-
verdays="${date1} ${date2} ${date3} ${date4} ${date5} ${date6} ${date7}"
6934

70-
for area in $areas
71-
do
72-
if [ $area == "NAM" ] ; then
35+
for area in ${areas}; do
36+
if [[ "${area}" == "NAM" ]] ; then
7337
garea="5.1;-124.6;49.6;-11.9"
7438
proj="STR/90.0;-95.0;0.0"
7539
latlon="0"
@@ -80,37 +44,18 @@ for area in $areas
8044
latlon="1/10/1/2/10;10"
8145
run=" "
8246
fi
83-
for verday in $verdays
84-
do
85-
verddate=$(echo ${verday} | cut -c 3-)
86-
if [ ${verday} -eq ${date1} ] ; then
87-
dgdattim=f024
88-
sdatenum=$sdate1
89-
elif [ ${verday} -eq ${date2} ] ; then
90-
dgdattim=f048
91-
sdatenum=$sdate2
92-
elif [ ${verday} -eq ${date3} ] ; then
93-
dgdattim=f072
94-
sdatenum=$sdate3
95-
elif [ ${verday} -eq ${date4} ] ; then
96-
dgdattim=f096
97-
sdatenum=$sdate4
98-
elif [ ${verday} -eq ${date5} ] ; then
99-
dgdattim=f120
100-
sdatenum=$sdate5
101-
elif [ ${verday} -eq ${date6} ] ; then
102-
dgdattim=f144
103-
sdatenum=$sdate6
104-
elif [ ${verday} -eq ${date7} ] ; then
105-
dgdattim=f168
106-
sdatenum=$sdate7
47+
for (( fhr=24; fhr<=168; fhr+=24 )); do
48+
dgdattim=$(printf "f%03d" "${fhr}")
49+
sdatenum=$(date --utc +%y%m%d -d "${PDY} ${cyc2} - ${fhr} hours")
50+
51+
if [[ ! -L "ecmwf.20${sdatenum}" ]]; then
52+
ln -sf "${COMINecmwf}/ecmwf.20${sdatenum}/gempak" "ecmwf.20${sdatenum}"
10753
fi
108-
# JY grid="$COMROOT/nawips/${envir}/ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
109-
grid="${COMINecmwf}.20${sdatenum}/gempak/ecmwf_glob_20${sdatenum}12"
54+
gdfile="ecmwf.20${sdatenum}/ecmwf_glob_20${sdatenum}12"
11055

111-
# 500 MB HEIGHT METAFILE
56+
# 500 MB HEIGHT METAFILE
11257

113-
$GEMEXE/gdplot2_nc << EOFplt
58+
"${GEMEXE}/gdplot2_nc" << EOFplt
11459
\$MAPFIL = mepowo.gsf
11560
PROJ = ${proj}
11661
GAREA = ${garea}
@@ -134,7 +79,7 @@ line = 6/1/3
13479
title = 6/-2/~ GDAS 500 MB HGT (6-HR FCST)|~${area} 500 HGT DF
13580
r
13681
137-
gdfile = ${grid}
82+
gdfile = ${gdfile}
13883
gdattim = ${dgdattim}
13984
line = 5/1/3
14085
contur = 4
@@ -157,15 +102,15 @@ clear = yes
157102
latlon = ${latlon}
158103
r
159104
160-
gdfile = ${grid}
105+
gdfile = ${gdfile}
161106
gdattim = ${dgdattim}
162107
line = 5/1/3
163108
contur = 4
164109
title = 5/-1/~ ECMWF PMSL
165110
clear = no
166111
r
167112
168-
PROJ =
113+
PROJ =
169114
GAREA = bwus
170115
gdfile = ${vergrid}
171116
gdattim = ${fcsthr}
@@ -181,7 +126,7 @@ clear = yes
181126
latlon = ${latlon}
182127
${run}
183128
184-
gdfile = ${grid}
129+
gdfile = ${gdfile}
185130
gdattim = ${dgdattim}
186131
line = 5/1/3
187132
contur = 4
@@ -195,28 +140,28 @@ EOFplt
195140
done
196141
done
197142

198-
export err=$?;err_chk
143+
export err=$?
144+
199145
#####################################################
200146
# GEMPAK DOES NOT ALWAYS HAVE A NON ZERO RETURN CODE
201147
# WHEN IT CAN NOT PRODUCE THE DESIRED GRID. CHECK
202148
# FOR THIS CASE HERE.
203149
#####################################################
204-
ls -l ecmwfver.meta
205-
export err=$?;export pgm="GEMPAK CHECK FILE";err_chk
206-
207-
if [ $SENDCOM = "YES" ] ; then
208-
mkdir -p -m 775 ${COMOUTecmwf}.${PDY}/meta
209-
mv ecmwfver.meta ${COMOUTecmwf}.${PDY}/meta/ecmwfver_${PDY}_${cyc}
210-
export err=$?
211-
if [[ $err -ne 0 ]] ; then
212-
echo " File ecmwfver.meta does not exist."
213-
exit $err
214-
fi
150+
if (( err != 0 )) || [[ ! -s ecmwfver.meta ]]; then
151+
echo "FATAL ERROR: Failed to create ecmwf meta file"
152+
exit "${err}"
153+
fi
215154

216-
if [ $SENDDBN = "YES" ] ; then
217-
${DBNROOT}/bin/dbn_alert MODEL ECMWFVER_HPCMETAFILE $job \
218-
${COMOUTecmwf}.${PDY}/meta/ecmwfver_${PDY}_${cyc}
219-
fi
155+
mv ecmwfver.meta "${COM_ATMOS_GEMPAK_META}/ecmwfver_${PDY}_${cyc2}"
156+
export err=$?
157+
if (( err != 0 )) ; then
158+
echo "FATAL ERROR: Failed to move meta file to ${COM_ATMOS_GEMPAK_META}/ecmwfver_${PDY}_${cyc2}"
159+
exit "${err}"
160+
fi
161+
162+
if [[ "${SENDDBN}" == "YES" ]] ; then
163+
"${DBNROOT}/bin/dbn_alert" MODEL ECMWFVER_HPCMETAFILE "${job}" \
164+
"${COM_ATMOS_GEMPAK_META}/ecmwfver_${PDY}_${cyc2}"
220165
fi
221166

222167
exit

0 commit comments

Comments
 (0)