Avoid attempting to make plots which have an incomplete specification (NOAA-EMC#50)

Avoid attempting to make any plots which have an incomplete
specification in the top-level yaml plot file. This is in response to
problems Andrew found in testing. Absent these changes such incomplete
specifications cause a catastrophic failure of all remaining plots. With
these changes incomplete specifications are skipped over and an
identifying warning message is produced.

Closes NOAA-EMC#49

Author: EdwardSafford-NOAA

ush/plotObsMon.py

@@ -9,11 +9,15 @@
 from re import sub
 import yaml
 from om_data import OM_data
+import datetime as dt
 
 from wxflow import parse_j2yaml, save_as_yaml
 from wxflow import add_to_datetime, to_timedelta, to_datetime
 from eva.eva_driver import eva
 from eva.utilities.logger import Logger
+from datetime import timedelta
+
+# --------------------------------------------------------------------------------------------
 
 
 def genYaml(input_yaml, output_yaml, config):
@@ -181,7 +185,6 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
                 for plot in inst.get('plot_list'):
                     config = loadConfig(satname, instrument, obstype, plot, cycle_tm,
                                         cycle_interval, data_location, model, chan_dict)
-
                     plot_template = f"{config['PLOT_TEMPLATE']}.yaml"
                     plot_yaml = f"{config['SENSOR']}_{config['SAT']}_{plot_template}"
 
@@ -195,11 +198,17 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
                     os.chdir(plot_dir)
 
                     config['DATA'] = plot_dir
-                    genYaml(plot_template, plot_yaml, config)
+                    try:
+                        genYaml(plot_template, plot_yaml, config)
+
+                        plotData = OM_data(data_location, config, plot_yaml, logger)
+                        eva(plot_yaml)
+                        os.remove(plot_yaml)
 
-                    plotData = OM_data(data_location, config, plot_yaml, logger)
-                    eva(plot_yaml)
-                    os.remove(plot_yaml)
+                    except Exception as e:
+                        logger.info(f'Warning: unable to run genYaml() with plot_yaml file ' +
+                                    f'{plot_yaml} error: {e}')
+                        continue
 
     if 'minimization' in mon_dict.keys():
         satname = None

ush/splitPlotYaml.py

@@ -29,6 +29,61 @@ def removeKey(d, keys):
     return r
 
 
+def check_plotlist(type, logger, plotlist, sat=None, instr=None, obstype=None):
+    """
+    Parse plotlist for missing required elements.
+
+    Parameters:
+        type (str): type of plot
+        logger (Logger):  logger for output warnings
+        plotlist (list):  list of requested plot specifications
+        sat (str): satellite name (default None)
+        instr (str); instrument name (default None)
+        obstype (str): observation type namea (default None)
+
+    Return:
+        True if no required elements are missing from plotlist, else False
+    """
+
+    sat_reqs = {'rad': ['plot', 'times', 'channels', 'component', 'run'],
+                'ozn': ['plot', 'times', 'levels', 'component', 'run']}
+    min_reqs = ['plot', 'times', 'run']
+    obs_reqs = ['plot', 'times', 'datatypes', 'run']
+
+    rtn_val = True
+    missing = []
+    match type:
+        case 'sat':
+            if 'rad' in plotlist.get('plot'):
+                reqs = sat_reqs.get('rad')
+            else:
+                reqs = sat_reqs.get('ozn')
+
+            for val in reqs:
+                if val not in plotlist:
+                    missing.append(val)
+                    plot_info = (f" {instr} {sat} {plotlist.get('plot')}")
+
+        case 'min':
+            for val in min_reqs:
+                if val not in plotlist:
+                    missing.append(val)
+                    plot_info = (f" {plotlist.get('plot')}")
+
+        case 'obs':
+            for val in obs_reqs:
+                if val not in plotlist:
+                    missing.append(val)
+                    plot_info = (f" {obstype}, {plotlist.get('plot')}")
+
+    if len(missing) > 0:
+        logger.info(f'WARNING:  YAML for plot {plot_info}  is missing  {missing}')
+        logger.info(f'WARNING:  Requested plot will be SKIPPED.')
+        rtn_val = False
+
+    return (rtn_val)
+
+
 if __name__ == "__main__":
     """
     splitPlotYaml
@@ -79,6 +134,9 @@ def removeKey(d, keys):
             for inst in sat.get('instruments'):
                 iname = inst.get('name')
                 plist = inst.get('plot_list')
+                for pl in inst.get('plot_list'):
+                    if not check_plotlist('sat', logger, pl, sat=satname, instr=iname):
+                        continue
 
                 # --------------------------------------------------------------------
                 # For instruments with a large number of channels split the plot_list
@@ -114,12 +172,26 @@ def removeKey(d, keys):
     if 'minimization' in mon_dict.keys():
         md = removeKey(mon_dict, ['satellites', 'observations'])
         fname = f'OM_PLOT_minimization.yaml'
+        mm = md.get('minimization')
+
+        for pl in mm[0]['plot_list']:
+            if not check_plotlist('min', logger, pl):
+                mm[0]['plot_list'].remove(pl)
+
         file = open(fname, "w")
         yaml.dump(md, file)
         file.close()
 
     if 'observations' in mon_dict.keys():
         od = removeKey(mon_dict, ['satellites', 'minimization'])
+        obs = od.get('observations')
+
+        for ob in obs:
+            obstype = ob.get('obstype')
+            for pl in ob.get('plot_list'):
+                if not check_plotlist('obs', logger, pl, obstype=obstype):
+                    ob.get('plot_list').remove(pl)
+
         fname = f'OM_PLOT_observations.yaml'
         file = open(fname, "w")
         yaml.dump(od, file)