Accessing GSI observing system records for active and available chann…

…els (#273) * add new task * adding task to generate sat channel record yamls * stash changes * stash * ramblings * cloning of geos * changes for obs system recs * adding tasks and workflow changes * add get_active_channels * updating tasks * parse active and available channels * workflow working with amsua_n19 * eva observations fix * stash * some clean up * coding norms * fix yaml * norms * pandas * sat_db_utils fix * fix eva_obs * requested changes * coding norms * fix test * fix observing sys test * task question test fix * code test fix * adding new tasks to hofx and geosadas * Undo Dan's bad suggestions * task q bug * add more protection from when component does not need obs sys records * code norm * fix bugs * minor bug in time --------- Co-authored-by: danholdaway <[email protected]>
GEOS-ESM · Nov 15, 2023 · 90129ec · 90129ec
1 parent f0bee5d
commit 90129ec
Show file tree

Hide file tree

Showing 30 changed files with 948 additions and 293 deletions.
diff --git a/requirements-github.txt b/requirements-github.txt
@@ -2,7 +2,7 @@ click==8.1.5
 jinja2==3.1.2
 pyyaml==6.0
 pycodestyle==2.10.0
-#pandas==1.4.0
+pandas==1.4.0
 isodate==0.6.1
 f90nml==1.4.3
 questionary==1.10.0

diff --git a/src/swell/__init__.py b/src/swell/__init__.py
@@ -9,4 +9,4 @@
 repo_directory = os.path.dirname(__file__)
 
 # Set the version for swell
-__version__ = '1.7.1'
+__version__ = '1.8.0'
diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml
@@ -191,9 +191,7 @@ obs post filters:
         channels: *amsua_n19_available_channels
         use passive_bc: true
         sensor: *Sensor_ID
-        use_flag: [-1, -1, -1,  1,  1,
-                    1, -1, -1,  1,  1,
-                    1,  1,  1,  1, -1]
+        use_flag: &amsua_n19_use_flag {{amsua_n19_active_channels}}
     maxvalue: 1.0e-12
     action:
       name: reject
@@ -207,9 +205,8 @@ obs post filters:
       channels: *amsua_n19_available_channels
       options:
         channels: *amsua_n19_available_channels
-        use_flag: [-1, -1, -1,  1,  1,
-                    1, -1, -1,  1,  1,
-                    1,  1,  1,  1, -1]
+#       use passive_bc: true
+        use_flag: *amsua_n19_use_flag
     minvalue: 1.0e-12
     action:
       name: reject
diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml
@@ -184,6 +184,12 @@ observations:
   - omi_aura
   - ompsnm_npp
 
+observing_system_records_gsi_path:
+  default_value: None
+
+observing_system_records_path:
+  default_value: None
+
 path_to_ensemble:
   default_value: /discover/nobackup/drholdaw/SwellTestData/letk/ensemble/Y%Y/M%m/D%d/H%H/geos*%Y%m%d_%H%M%Sz.nc4
 

diff --git a/src/swell/suites/geosadas/flow.cylc b/src/swell/suites/geosadas/flow.cylc
@@ -31,9 +31,15 @@
 
             # Stage JEDI static files
             CloneJedi => StageJedi
+
+            # Clone geos ana for generating observing system records
+            CloneGeosAna
         """
 
         T00 = """
+            # Generate satellite channel records
+            CloneGeosAna[^] => GenerateObservingSystemRecords
+
             # Get and convert bias correction coefficients
             GetGsiBc => GsiBcToIoda
 
@@ -44,6 +50,7 @@
             GetGeosAdasBackground
 
             # Run Jedi variational executable
+            GenerateObservingSystemRecords => RunJediVariationalExecutable
             BuildJediByLinking[^]  => RunJediVariationalExecutable
             StageJedi[^] => RunJediVariationalExecutable
             GsiBcToIoda => RunJediVariationalExecutable
@@ -69,6 +76,12 @@
 
     # Tasks
     # -----
+    [[CloneGeosAna]]
+        script = "swell task CloneGeosAna $config"
+
+    [[GenerateObservingSystemRecords]]
+        script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere"
+
     [[CloneJedi]]
         script = "swell task CloneJedi $config"
 

diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc
@@ -38,15 +38,22 @@
             {% for model_component in model_components %}
             # Stage JEDI static files
             CloneJedi => StageJedi-{{model_component}}
+
+            # Clone geos ana for generating observing system records
+            CloneGeosAna-{{model_component}}
             {% endfor %}
         """
 
         {% for cycle_time in cycle_times %}
         {{cycle_time.cycle_time}} = """
         {% for model_component in model_components %}
         {% if cycle_time[model_component] %}
+
             # Task triggers for: {{model_component}}
             # ------------------
+            # Generate satellite channel records
+            CloneGeosAna-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}}
+
             # Get background
             GetBackground-{{model_component}}
 
@@ -62,6 +69,7 @@
             StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}}
             GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}}
             GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}}
+            GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}}
 
             # EvaObservations
             RunJediHofxExecutable-{{model_component}} => EvaObservations-{{model_component}}
@@ -115,6 +123,13 @@
             {% endif %}
 
     {% for model_component in model_components %}
+
+    [[CloneGeosAna-{{model_component}}]]
+        script = "swell task CloneGeosAna $config -m {{model_component}}"
+
+    [[GenerateObservingSystemRecords-{{model_component}}]]
+        script = "swell task GenerateObservingSystemRecords $config -d $datetime -m {{model_component}}"
+
     [[StageJedi-{{model_component}}]]
         script = "swell task StageJedi $config -m {{model_component}}"
 

diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc
@@ -34,11 +34,17 @@
 
             # If not able to link to build create the build
             BuildJediByLinking:fail? => BuildJedi
+
+            # Clone geos ana for generating observing system records
+            CloneGeosAna
         """
 
         {% for cycle_time in cycle_times %}
         {{cycle_time.cycle_time}} = """
 
+            # Generate satellite channel records
+            CloneGeosAna[^] => GenerateObservingSystemRecords
+
             # Convert bias correction to ioda
             GetGsiBc
             GetGsiBc => GsiBcToIoda
@@ -52,6 +58,7 @@
             GetGeovals
 
             # Run Jedi hofx executable
+            GenerateObservingSystemRecords => RunJediUfoTestsExecutable
             GsiNcdiagToIoda => RunJediUfoTestsExecutable
             GsiBcToIoda => RunJediUfoTestsExecutable
             GetGeovals => RunJediUfoTestsExecutable
@@ -101,6 +108,12 @@
             --partition={{scheduling["BuildJedi"]["partition"]}}
             {% endif %}
 
+    [[CloneGeosAna]]
+        script = "swell task CloneGeosAna $config -m geos_atmosphere"
+
+    [[GenerateObservingSystemRecords]]
+        script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere"
+
     [[ GetGsiBc ]]
         script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere"
 

diff --git a/src/swell/tasks/base/task_base.py b/src/swell/tasks/base/task_base.py
@@ -88,7 +88,8 @@ def __init__(self, config_input, datetime_input, model, task_name):
         # Add JEDI config rendering helper
         # --------------------------------
         self.jedi_rendering = JediConfigRendering(self.logger, self.__experiment_root__,
-                                                  self.__experiment_id__, cycle_dir, self.__model__)
+                                                  self.__experiment_id__, cycle_dir,
+                                                  self.__datetime__, self.__model__)
 
         # Add GEOS utils
         # --------------

diff --git a/src/swell/tasks/clone_geos_ana.py b/src/swell/tasks/clone_geos_ana.py
@@ -0,0 +1,48 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+import os
+from swell.tasks.base.task_base import taskBase
+from swell.utilities.build import link_path
+
+# --------------------------------------------------------------------------------------------------
+
+
+class CloneGeosAna(taskBase):
+
+    def execute(self):
+
+        """
+        Generate the satellite channel record from GEOSadas files
+        """
+
+        # This task should only execute for geos_atmosphere
+        # -------------------------------------------------
+        if self.get_model() != 'geos_atmosphere':
+            self.logger.info('Skipping GenerateObservingSystemRecords for: ' + self.get_model())
+            return
+
+        # Parse config
+        # ------------
+        path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path()
+
+        # If observing_system_records_gsi_path is None, clone GEOSana_GridComp repo to experiment
+        # directory
+        if path_to_geosana_gridcomp == 'None':
+            # Clone GEOSana_GridComp develop repo to experiment directory
+            os.system('git clone https://github.com/GEOS-ESM/GEOSana_GridComp.git '
+                      + os.path.join(self.experiment_path(), 'GEOSana_GridComp'))
+        else:
+            # Link the source code directory
+            link_path(self.config.observing_system_records_gsi_path(),
+                      os.path.join(self.experiment_path(), 'GEOSana_GridComp'))
+
+
+# ----------------------------------------------------------------------------------------------
diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py
@@ -81,10 +81,13 @@ def execute(self):
                              445, 552, 573, 906, 1121, 1194, 1427, 1585],
             }
 
-        # Loop over observations and create dictionaries
-        # ----------------------------------------------
+        # Loop over observations
+        # -------------------
         eva_dicts = []  # Empty list of dictionaries
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         for observation in self.config.observations():
 
             # Load the observation dictionary

diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py
@@ -0,0 +1,50 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+import os
+
+from swell.tasks.base.task_base import taskBase
+from swell.utilities.observing_system_records import ObservingSystemRecords
+
+# --------------------------------------------------------------------------------------------------
+
+
+class GenerateObservingSystemRecords(taskBase):
+
+    def execute(self):
+
+        """
+        Generate the observing system channel records from GEOSadas files
+        """
+
+        # This task should only execute for geos_atmosphere
+        # -------------------------------------------------
+        if self.get_model() != 'geos_atmosphere':
+            self.logger.info('Skipping GenerateObservingSystemRecords for: ' + self.get_model())
+            return
+
+        # Parse GSI records and save channel selection yamls
+        # --------------------------------------------------
+        observations = self.config.observations()
+        observing_system_records_path = self.config.observing_system_records_path(None)
+        if observing_system_records_path == 'None':
+            cycle_dir = self.cycle_dir()
+            observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records')
+
+        path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path()
+        if path_to_geosana_gridcomp == 'None':
+            path_to_geosana_gridcomp = os.path.join(self.experiment_path(), 'GEOSana_GridComp')
+        path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp',
+                                           'GSI_GridComp', 'mksi', 'sidb')
+        sat_records = ObservingSystemRecords()
+        sat_records.parse_records(path_to_gsi_records)
+        sat_records.save_yamls(observing_system_records_path, observations)
+
+# ----------------------------------------------------------------------------------------------
diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py
@@ -43,6 +43,9 @@ def execute(self):
         crtm_coeff_dir = self.config.crtm_coeff_dir(None)
         window_offset = self.config.window_offset()
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         # Get window begin time
         window_begin = self.da_window_params.window_begin(window_offset)
         background_time = self.da_window_params.background_time(window_offset,

diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py
@@ -47,6 +47,10 @@ def execute(self):
         sensors = []
         sensors_satbias = []
         sensors_tlapse = []
+
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         for observation in observations:
 
             print('observation', observation)

diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py
@@ -37,6 +37,9 @@ def execute(self):
         jedi_forecast_model = self.config.jedi_forecast_model(None)
         generate_yaml_and_exit = self.config.generate_yaml_and_exit(False)
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         # Compute data assimilation window parameters
         background_time = self.da_window_params.background_time(window_offset,
                                                                 background_time_offset)

diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py
@@ -37,6 +37,9 @@ def execute(self):
         jedi_forecast_model = self.config.jedi_forecast_model(None)
         generate_yaml_and_exit = self.config.generate_yaml_and_exit(False)
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         # Compute data assimilation window parameters
         background_time = self.da_window_params.background_time(window_offset,
                                                                 background_time_offset)

diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py
@@ -39,6 +39,9 @@ def execute(self):
         single_observations = self.config.single_observations()
         generate_yaml_and_exit = self.config.generate_yaml_and_exit(False)
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         # Compute data assimilation window parameters
         window_begin = self.da_window_params.window_begin(window_offset)
         window_begin_iso = self.da_window_params.window_begin_iso(window_offset)

diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py
@@ -38,6 +38,9 @@ def execute(self):
         jedi_forecast_model = self.config.jedi_forecast_model(None)
         generate_yaml_and_exit = self.config.generate_yaml_and_exit(False)
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         npx_proc = self.config.npx_proc(None)
         npy_proc = self.config.npy_proc(None)
 

diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py
@@ -29,6 +29,9 @@ def execute(self):
         observations = self.config.observations()
         window_offset = self.config.window_offset()
 
+        # Set the observing system records path
+        self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None))
+
         # Get window beginning
         window_begin = self.da_window_params.window_begin(window_offset)
         background_time = self.da_window_params.background_time(window_offset,