Merge branch 'feature/gdas-validation' into feature/gdas-validation-c…

…onvps2
NOAA-EMC · Dec 18, 2023 · fd5bd3f · fd5bd3f
2 parents 3e68c1d + c2d9edb
commit fd5bd3f
Show file tree

Hide file tree

Showing 27 changed files with 1,056 additions and 129 deletions.
diff --git a/parm/aero/hofx/hofx_nomodel.yaml b/parm/aero/hofx/hofx_nomodel.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
-window length: $(AERO_WINDOW_LENGTH)
+time window:
+  begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
+  length: $(AERO_WINDOW_LENGTH)
 geometry:
   fms initialization:
     namelist filename: $(DATA)/fv3jedi/fmsmpp.nml

diff --git a/parm/aero/variational/3dvar_fgat_gfs_aero.yaml b/parm/aero/variational/3dvar_fgat_gfs_aero.yaml
@@ -1,7 +1,8 @@
 cost function:
   cost type: 3D-FGAT
-  window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
-  window length: $(AERO_WINDOW_LENGTH)
+  time window:
+    begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
+    length: $(AERO_WINDOW_LENGTH)
   analysis variables: &3dvars
                       [mass_fraction_of_sulfate_in_air,
                        mass_fraction_of_hydrophobic_black_carbon_in_air,

diff --git a/parm/aero/variational/3dvar_gfs_aero.yaml b/parm/aero/variational/3dvar_gfs_aero.yaml
@@ -1,7 +1,8 @@
 cost function:
   cost type: 3D-Var
-  window begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
-  window length: $(AERO_WINDOW_LENGTH)
+  time window:
+    begin: '{{ AERO_WINDOW_BEGIN | to_isotime }}'
+    length: $(AERO_WINDOW_LENGTH)
   analysis variables: &3dvars
                       [mass_fraction_of_sulfate_in_air,
                        mass_fraction_of_hydrophobic_black_carbon_in_air,

diff --git a/parm/atm/hofx/hofx4d.yaml b/parm/atm/hofx/hofx4d.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-window length: $(ATM_WINDOW_LENGTH)
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  length: $(ATM_WINDOW_LENGTH)
 forecast length: $(ATM_WINDOW_LENGTH)
 geometry:
   fms initialization:

diff --git a/parm/atm/hofx/hofx_nomodel.yaml b/parm/atm/hofx/hofx_nomodel.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-window length: $(ATM_WINDOW_LENGTH)
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  length: $(ATM_WINDOW_LENGTH)
 geometry:
   fms initialization:
     namelist filename: $(DATA)/fv3jedi/fmsmpp.nml

diff --git a/parm/atm/hofx/hofx_ufotest.yaml b/parm/atm/hofx/hofx_ufotest.yaml
@@ -1,3 +1,4 @@
-window begin: '{{ATM_WINDOW_BEGIN}}'
-window end: '{{ATM_WINDOW_END}}'
+time window:
+  begin: '{{ATM_WINDOW_BEGIN}}'
+  end: '{{ATM_WINDOW_END}}'
 observations: !INC ${OBS_LIST}
diff --git a/parm/atm/lgetkf/lgetkf.yaml b/parm/atm/lgetkf/lgetkf.yaml
@@ -10,8 +10,9 @@ geometry:
   npy: $(npy_ges)
   npz: $(npz_ges)
   field metadata override: ./fv3jedi/fv3jedi_fieldmetadata_restart.yaml
-window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-window length: $(ATM_WINDOW_LENGTH)
+time window:
+  begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+  length: $(ATM_WINDOW_LENGTH)
 increment variables: [ua,va,DZ,delp,t,ps,sphum,ice_wat,liq_wat,o3mr]
 
 background:

diff --git a/parm/atm/variational/3dvar_dripcg.yaml b/parm/atm/variational/3dvar_dripcg.yaml
@@ -1,8 +1,9 @@
 cost function:
   cost type: 3D-Var
-  window begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
-  window length: $(ATM_WINDOW_LENGTH)
-  window shift: true
+  time window:
+    begin: '{{ ATM_WINDOW_BEGIN | to_isotime }}'
+    length: $(ATM_WINDOW_LENGTH)
+    bound to include: begin 
   analysis variables: &3dvars [ua,va,t,ps,sphum,ice_wat,liq_wat,o3mr]
   geometry:
     fms initialization:

diff --git a/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json b/parm/ioda/bufr2ioda/bufr2ioda_adpupa_prepbufr.json
@@ -1,11 +1,11 @@
 {
-    "data_format": "prepbufr",
-    "data_type": "prepbufr",
-    "cycle_type": "{{ RUN }}",
-    "cycle_datetime"   : "{{ current_cycle | to_YMDH }}",
-    "dump_directory": "{{ DMPDIR }}",
-    "ioda_directory": "{{ COM_OBS }}",
-    "subsets": [ "ADPUPA" ],
-    "data_provider": "U.S. NOAA",
-    "data_description": "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS",
+  "data_format"     : "prepbufr",
+  "data_type"       : "ADPUPA",
+  "cycle_type"      : "{{ RUN }}",
+  "cycle_datetime"  : "{{ current_cycle | to_YMDH }}",
+  "dump_directory"  : "{{ DMPDIR }}",
+  "ioda_directory"  : "{{ COM_OBS }}",
+  "subsets"         : [ "ADPUPA" ],
+  "data_provider"   : "U.S. NOAA",
+  "data_description": "UPPER-AIR (RAOB, PIBAL, RECCO, DROPS) REPORTS"
 }
diff --git a/parm/land/hofx/hofx_nomodel.yaml b/parm/land/hofx/hofx_nomodel.yaml
@@ -1,5 +1,6 @@
-window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
-window length: $(LAND_WINDOW_LENGTH)
+time window:
+  begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
+  length: $(LAND_WINDOW_LENGTH)
 geometry:
   fms initialization:
     namelist filename: $(DATA)/fv3jedi/fmsmpp.nml

diff --git a/parm/land/letkfoi/letkfoi.yaml b/parm/land/letkfoi/letkfoi.yaml
@@ -19,8 +19,9 @@ geometry:
       datapath: $(FIXgfs)/orog/${CASE}/
       filename_orog: $(CASE)_oro_data.nc
 
-window begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
-window length: $(LAND_WINDOW_LENGTH)
+time window:
+  begin: '{{ LAND_WINDOW_BEGIN | to_isotime }}'
+  length: $(LAND_WINDOW_LENGTH)
 
 background:
    datetime: '{{ current_cycle | to_isotime }}'

diff --git a/parm/soca/variational/3dvarfgat.yaml b/parm/soca/variational/3dvarfgat.yaml
@@ -1,7 +1,8 @@
 cost function:
   cost type: 3D-FGAT
-  window begin: '{{ATM_WINDOW_BEGIN}}'
-  window length: $(ATM_WINDOW_LENGTH)
+  time window:
+    begin: '{{ATM_WINDOW_BEGIN}}'
+    length: $(ATM_WINDOW_LENGTH)
   analysis variables: &soca_ana_vars  [cicen, hicen, hsnon, socn, tocn, uocn, vocn, ssh]
 
   geometry:

diff --git a/scripts/exglobal_prep_ocean_obs.py b/scripts/exglobal_prep_ocean_obs.py
@@ -8,6 +8,7 @@
 import subprocess
 from wxflow import YAMLFile, save_as_yaml
 
+# TODO (AFE) figure out why logger is not logging
 # set up logger
 logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', level=logging.INFO, datefmt='%Y-%m-%d %H:%M:%S')
 
@@ -24,30 +25,52 @@
 OCNOBS2IODAEXEC = os.getenv('OCNOBS2IODAEXEC')
 
 OBS_YAML = os.getenv('OBS_YAML')
+# this will fail with FileNotFoundError if all yaml files in OBS_YAML are not
+# present in OBS_YAML_DIR
 obsConfig = YAMLFile(OBS_YAML)
 
 OBSPROC_YAML = os.getenv('OBSPROC_YAML')
 obsprocConfig = YAMLFile(OBSPROC_YAML)
 
 # TODO (AFE): needs more error handling (missing sources, missing files)
-for observer in obsConfig['observers']:
+try:
+    # For each of the observation sources (observers) specificed in the OBS_YAML...
+    for observer in obsConfig['observers']:
 
-    obsSpaceName = observer['obs space']['name']
-    print(f"obsSpaceName: {obsSpaceName}")
+        try:
+            obsSpaceName = observer['obs space']['name']
+            print(f"obsSpaceName: {obsSpaceName}")
+        except KeyError:
+            print(f"observer: {observer}")
+            print("WARNING: Ill-formed observer yaml file, skipping")
+            continue  # to next observer
 
-    for observation in obsprocConfig['observations']:
+# ...look through the observations in OBSPROC_YAML...
+        for observation in obsprocConfig['observations']:
 
-        obsprocSpace = observation['obs space']
-        obsprocSpaceName = obsprocSpace['name']
+            obsprocSpace = observation['obs space']
+            obsprocSpaceName = obsprocSpace['name']
 
-        if obsprocSpaceName == obsSpaceName:
+# ...for a matching name, and process the observation source
+            if obsprocSpaceName == obsSpaceName:
 
-            matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace)
-            obsprocSpace['input files'] = matchingFiles
-            obsprocSpace['window begin'] = windowBegin
-            obsprocSpace['window end'] = windowEnd
+                print(f"obsprocSpaceName: {obsSpaceName}")
 
-            iodaYamlFilename = obsprocSpaceName + '2ioda.yaml'
-            save_as_yaml(obsprocSpace, iodaYamlFilename)
+                # fetch the obs files from DMPDIR to RUNDIR
+                matchingFiles = prep_marine_obs.obs_fetch(obsprocSpace)
 
-            subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True)
+                if not matchingFiles:
+                    print("WARNING: No files found for obs source , skipping")
+                    break  # to next observation source in OBS_YAML
+
+                obsprocSpace['input files'] = matchingFiles
+                obsprocSpace['window begin'] = windowBegin
+                obsprocSpace['window end'] = windowEnd
+
+                iodaYamlFilename = obsprocSpaceName + '2ioda.yaml'
+                save_as_yaml(obsprocSpace, iodaYamlFilename)
+
+                subprocess.run([OCNOBS2IODAEXEC, iodaYamlFilename], check=True)
+except TypeError:
+    print("CRITICAL: Ill-formed OBS_YAML file, exiting")
+    raise
diff --git a/test/land/letkfoi_land.yaml b/test/land/letkfoi_land.yaml
@@ -164,5 +164,6 @@ observations:
 output increment:
   filename_sfcd: xainc.sfc_data.nc
   filetype: fms restart
-window begin: '2021-03-23T15:00:00Z'
-window length: PT6H
+time window:
+  begin: '2021-03-23T15:00:00Z'
+  length: PT6H