Fix cleanup script and handling of input 'model' (NOAA-EMC#56)

1. Fix the `om_cleanup.sh` script to correctly move plot files to ptmp
space.
2. Remove `model` from the `driver/runObsMon.sh`. Make the `-y|--yaml`
argument required and extract the `model` value from the yaml file.
3. Remove `model` from the minimization plot specification in
parm/gfs/gfs_plot.yaml. Modify `ush/splitPlotYaml.py` and
`plotObsMon.py` to correctly handle the changed minimization inputs in
the plot yaml file.

All changes have been tested on wcoss2.

Closes NOAA-EMC#55

driver/runObsMon.sh

@@ -10,21 +10,17 @@
 #  usage
 #--------------------------------------------------------------------
 function usage {
-  echo "Usage:  runObsMon.sh -p|--pdate pdate -m|--model, [-y|--yaml]"
+  echo "Usage:  runObsMon.sh -p|--pdate pdate -y|--yaml"
   echo "            -p | --pdate 	cycle time to be processed, format yyyymmddhh."
   echo "              			If unspecified the last available date will be processed."
-  echo "            -m | --model	model or experiment name (i.e. gfs, exp1, etc.)"
   echo "            -y | --yaml 	yaml plot file, with full or relative path."
-  echo "                                If no yaml file is specified the default is "
-  echo "                                parm/[model]/[model]_plot.yaml"
   echo " "
 }
 
 
 echo begin runObsMon.sh
 
 nargs=$#
-echo nargs: $nargs
 if [[ ${nargs} -lt 4 || ${nargs} -gt 6 ]]; then
    usage
    exit 1
@@ -36,7 +32,6 @@ fi
 #
 
 pdate=""
-model=""
 yaml_file=""
 
 while [[ $# -ge 1 ]]
@@ -49,10 +44,6 @@ do
          pdate="$2"
          shift # past argument
       ;;
-      -m|--model)
-         model="$2"
-         shift # past argument
-      ;;
       -y|--yaml)
          yaml_file="$2"
          shift # past argument
@@ -63,17 +54,19 @@ do
 done
 
 echo pdate: $pdate
-echo model: $model
-
-if [ -n "${yaml_file}" ]; then 
-   if  [ ! -e ${yaml_file} ]; then
-      echo "ERROR:  input yaml file ${yaml_file} not found"
-      exit 1 
-   fi
-   yaml_file=`realpath ${yaml_file}`
-fi
 echo yaml_file:  $yaml_file
 
+if  [ ! -e ${yaml_file} ]; then
+   echo "ERROR:  input yaml file ${yaml_file} not found"
+   exit 1 
+fi
+yaml_file=`realpath ${yaml_file}`
+
+#-------------------------------
+# find model name in $yaml_file
+#
+model=`grep -m1 "model:" $yaml_file | head -1 | gawk '{print $NF}'`
+
 export PDY=`echo ${pdate}|cut -c1-8`
 export cyc=`echo ${pdate}|cut -c9-10`
 export NET=obsmon

parm/gfs/gfs_plots.yaml

@@ -624,19 +624,18 @@ satellites:
             run: gdas
 
 minimization:
-  - model: gfs
-    plot_list:
-      - plot: min summary
-        run: gdas
-        times: 28
+  plot_list:
+    - plot: min summary
+      run: gdas
+      times: 28
 
-      - plot: min gnorm four cycle
-        run: gdas
-        times: 28
+    - plot: min gnorm four cycle
+      run: gdas
+      times: 28
 
-      - plot: min gnorm one cycle
-        run: gdas
-        times: 28
+    - plot: min gnorm one cycle
+      run: gdas
+      times: 28
 
 observations:
   - obstype: ps

scripts/exobsmon_plot.sh

@@ -65,7 +65,6 @@ if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then
             plotjob_id=`echo ${plotjob_id} | gawk '{ print $4 }'`
             ${SUB} --account ${ACCOUNT} -n 1 -o ${logfile_clnup} -D . -J "OM_cleanup" --time=0:10:00 \
                    -p ${SERVICE_PARTITION} --dependency=afterok:${plotjob_id} ${USHobsmon}/om_cleanup.sh
-
          ;;
 
 	 wcoss2)  
@@ -80,10 +79,10 @@ if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then
             # submit cleanup job to run after plot job
  	    ${SUB} -q $JOB_QUEUE -A $ACCOUNT -o ${logfile_clnup} -e ${logfile_clnup} \
   	        -v "DATA=${DATA}, KEEPDATA=${KEEPDATA}, NET=${NET}, DATAROOT=${DATAROOT}, \
- 	    COMOUTplots=${COMOUTplots}, DATA=${DATA}, MACHINE_ID=${MACHINE_ID}" \
+                    COMOUTplots=${COMOUTplots}, DATA=${DATA}, MACHINE_ID=${MACHINE_ID}" \
                 -l select=1:mem=500mb,walltime=1:00:00 -W depend=afterok:${plotjob_id} -N "OM_cleanup" ${USHobsmon}/om_cleanup.sh
-
          ;;     
+
       esac
    fi
 fi

ush/om_cleanup.sh

@@ -1,13 +1,23 @@
 #!/bin/bash
 
+echo "NET: $NET"
+echo "RUN: $RUN"
+echo "DATA: $DATA"
+echo "COMOUTplots: $COMOUTplots"
+
 # ---------------------------------------------
 # Sync image files with $COMOUTplots directory
 # 
-img_dirs=`ls -d ${DATA}/*_plots/`
+img_dirs=`ls -d ${DATA}/*/*plots/`
+
 for dir in $img_dirs; do
-   echo "syncing ${dir} and ${COMOUTplots}/${dir}"
+
    base_name=$(basename ${dir})
-   rsync -a ${dir} ${COMOUTplots}/${base_name} 
+   destination=${COMOUTplots}/${base_name}
+   echo "syncing ${dir} and ${destination}"
+   echo ""
+
+   rsync -a ${dir} ${destination} 
 done
 
 # ---------------------------------------------

ush/plotObsMon.py

@@ -215,31 +215,28 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
         instrument = None
         obstype = None
 
-        for min in mon_dict.get('minimization'):
-            model = min.get('model')
+        for plot in mon_dict.get('minimization').get('plot_list'):
+            config = loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
+                                data_location, model)
 
-            for plot in min.get('plot_list'):
-                config = loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
-                                    data_location, model)
+            plot_template = f"{config['PLOT_TEMPLATE']}.yaml"
+            plot_yaml = f"{config['MODEL']}_{config['RUN']}_{plot_template}"
 
-                plot_template = f"{config['PLOT_TEMPLATE']}.yaml"
-                plot_yaml = f"{config['MODEL']}_{config['RUN']}_{plot_template}"
+            parm = os.environ.get('PARMobsmon', '../parm')
+            parm_location = os.path.join(parm, 'templates')
+            plot_template = os.path.join(parm_location, plot_template)
 
-                parm = os.environ.get('PARMobsmon', '../parm')
-                parm_location = os.path.join(parm, 'templates')
-                plot_template = os.path.join(parm_location, plot_template)
+            # cd to unique directory based on plot_yaml file
+            plot_dir = os.path.join(workdir, plot_yaml.split('.')[0])
+            os.makedirs(plot_dir)
+            os.chdir(plot_dir)
 
-                # cd to unique directory based on plot_yaml file
-                plot_dir = os.path.join(workdir, plot_yaml.split('.')[0])
-                os.makedirs(plot_dir)
-                os.chdir(plot_dir)
-
-                config['DATA'] = plot_dir
-                genYaml(plot_template, plot_yaml, config)
+            config['DATA'] = plot_dir
+            genYaml(plot_template, plot_yaml, config)
 
-                plotData = OM_data(data_location, config, plot_yaml, logger)
-                eva(plot_yaml)
-                os.remove(plot_yaml)
+            plotData = OM_data(data_location, config, plot_yaml, logger)
+            eva(plot_yaml)
+            os.remove(plot_yaml)
 
     if 'observations' in mon_dict.keys():
         satname = None

ush/splitPlotYaml.py

@@ -174,9 +174,9 @@ def check_plotlist(type, logger, plotlist, sat=None, instr=None, obstype=None):
         fname = f'OM_PLOT_minimization.yaml'
         mm = md.get('minimization')
 
-        for pl in mm[0]['plot_list']:
+        for pl in mm['plot_list']:
             if not check_plotlist('min', logger, pl):
-                mm[0]['plot_list'].remove(pl)
+                mm['plot_list'].remove(pl)
 
         file = open(fname, "w")
         yaml.dump(md, file)