Make x0050 default experiment for geos_atmosphere (#501)

* fix tier1 overrides * aircraft is two separate entries now * codestyle * arrange defaults and overrides for tier1 tests * make ufo_testing work for x0050 * deactivate ocean for hofx temporarily * c91 as tier1 test, adapt to geos experiment folder style * hofx has to use x0050 data * add geovals for ufo_testing
GEOS-ESM · Feb 11, 2025 · fe3ad63 · fe3ad63
1 parent e1996c1
commit fe3ad63
Show file tree

Hide file tree

Showing 12 changed files with 90 additions and 67 deletions.
diff --git a/...well/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft_temperature.yaml b/...well/configuration/jedi/interfaces/geos_atmosphere/observations/aircraft_temperature.yaml
@@ -15,8 +15,8 @@ obs space:
   simulated variables: [airTemperature]
 
 obs bias:
-  input file: '{{cycle_dir}}/aircraft.{{background_time}}.acftbias'
-  output file: '{{cycle_dir}}/aircraft.{{window_begin}}.acftbias'
+  input file: '{{cycle_dir}}/aircraft_temperature.{{background_time}}.acftbias'
+  output file: '{{cycle_dir}}/aircraft_temperature.{{window_begin}}.acftbias'
   bc by record: true
   variational bc:
     predictors:
@@ -33,12 +33,12 @@ obs bias:
     step size: 1.0e-4
     largest analysis variance: 10000.0
     prior:
-      input file: '{{cycle_dir}}/aircraft.{{background_time}}.acftbias_cov'
+      input file: '{{cycle_dir}}/aircraft_temperature.{{background_time}}.acftbias_cov'
       inflation:
         ratio: 1.005
         ratio for small dataset: 2.0
 # RTODLING: this needs attention: we must implement writer for cov parameters
-#   output file: '{{cycle_dir}}/aircraft.{{window_begin}}.acftbias_cov'
+#   output file: '{{cycle_dir}}/aircraft_temperature.{{window_begin}}.acftbias_cov'
 
 obs operator:
   name: Composite

diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/task_questions.yaml
@@ -43,7 +43,10 @@ background_error_model:
   - GSIbec
 
 background_experiment:
-  default_value: x0048
+  default_value: x0050
+  options:
+  - x0048
+  - x0050
 
 background_frequency:
   default_value: PT1H
@@ -82,7 +85,10 @@ ensemble_num_members:
   options: None
 
 geovals_experiment:
-  default_value: x0048v3-geovals
+  default_value: x0050-geovals
+  options:
+  - x0048v3-geovals
+  - x0050-geovals
 
 geovals_provider:
   default_value: ncdiag
@@ -110,6 +116,7 @@ horizontal_resolution:
   default_value: '361'
   options:
   - '91'
+  - '181'
   - '361'
 
 jedi_forecast_model:
@@ -154,7 +161,10 @@ number_of_iterations:
   - 5
 
 obs_experiment:
-  default_value: x0048v3
+  default_value: x0050
+  options:
+  - x0048v3
+  - x0050
 
 obs_provider:
   default_value: ncdiag

diff --git a/src/swell/suites/hofx/flow.cylc b/src/swell/suites/hofx/flow.cylc
@@ -51,8 +51,8 @@
             # Generate satellite channel records
             CloneGeosMksi-{{model_component}}[^] => GenerateObservingSystemRecords-{{model_component}}
 
-            # Get background
-            GetBackground-{{model_component}}
+            # Get background, provide a way to get background directly from GEOS experiment
+            GetBackgroundGeosExperiment-{{model_component}} :fail? => GetBackground-{{model_component}}
 
             # Get observations
             GetObservations-{{model_component}}
@@ -64,7 +64,7 @@
             BuildJediByLinking[^]? | BuildJedi[^]  => RunJediHofxExecutable-{{model_component}}
             CloneJedi[^] => StageJediCycle-{{model_component}}
             StageJediCycle-{{model_component}} => RunJediHofxExecutable-{{model_component}}
-            GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}}
+            GetBackgroundGeosExperiment-{{model_component}}? | GetBackground-{{model_component}} => RunJediHofxExecutable-{{model_component}}
             GetObservations-{{model_component}} => RunJediHofxExecutable-{{model_component}}
             GenerateObservingSystemRecords-{{model_component}} => RunJediHofxExecutable-{{model_component}}
 
@@ -124,9 +124,12 @@
     [[StageJediCycle-{{model_component}}]]
         script = "swell task StageJedi $config -d $datetime -m {{model_component}}"
 
-    [[ GetBackground-{{model_component}} ]]
+    [[GetBackground-{{model_component}}]]
         script = "swell task GetBackground $config -d $datetime -m {{model_component}}"
 
+    [[GetBackgroundGeosExperiment-{{model_component}} ]]
+        script = "swell task GetBackgroundGeosExperiment $config -d $datetime -m {{model_component}}"
+
     [[GetObservations-{{model_component}}]]
         script = "swell task GetObservations $config -d $datetime -m {{model_component}}"
 

diff --git a/src/swell/suites/suite_questions.yaml b/src/swell/suites/suite_questions.yaml
@@ -66,7 +66,7 @@ experiment_root:
 
 start_cycle_point:
   ask_question: True
-  default_value: '2021-12-12T00:00:00Z'
+  default_value: '2023-10-10T00:00:00Z'
   prompt: What is the time of the first cycle (middle of the window)?
   suites:
   - 3dfgat_atmos
@@ -83,7 +83,7 @@ start_cycle_point:
 
 final_cycle_point:
   ask_question: True
-  default_value: '2021-12-12T06:00:00Z'
+  default_value: '2023-10-10T06:00:00Z'
   prompt: What is the time of the final cycle (middle of the window)?
   suites:
   - 3dfgat_atmos

diff --git a/src/swell/tasks/get_background_geos_experiment.py b/src/swell/tasks/get_background_geos_experiment.py
@@ -47,6 +47,7 @@ def execute(self):
         background_experiment = self.config.background_experiment()
         geos_x_background_directory = self.config.geos_x_background_directory()
         background_time_offset = self.config.background_time_offset()
+        horizontal_resolution = self.config.horizontal_resolution()
 
         # Since this is an optional task, check if the geos_x_background_directory is
         # set to /dev/null, if so fail the task
@@ -77,6 +78,7 @@ def execute(self):
         # -------------------------------------
         bkgr_tar_file = f'{background_experiment}.bkgcrst.{bkgr_exp_start_geos}.tar'
         bkgr_tar = os.path.join(geos_x_background_directory,
+                                horizontal_resolution,
                                 background_experiment,
                                 'rs',
                                 bkgr_exp_start_dto.strftime('Y%Y'),

diff --git a/src/swell/tasks/get_gsi_ncdiag.py b/src/swell/tasks/get_gsi_ncdiag.py
@@ -28,12 +28,12 @@ def execute(self) -> None:
         # Replace gsi_diag_path datetime string with the actual datetime
         # --------------------------------------------------------------
         cycle_time_dto = self.cycle_time_dto()
-        gsi_diag_path = cycle_time_dto.strftime(gsi_diag_path)
 
         # Get list of ncdiags to test with
         # --------------------------------
-        gsi_diag_path_files_pattern = os.path.join(gsi_diag_path, '*ges*.nc*')
-        gsi_diag_path_files = glob.glob(gsi_diag_path_files_pattern)
+        gsi_sat_diag_path = os.path.join(gsi_diag_path, '*ges*%Y%m%d_%Hz*nc*')
+        gsi_sat_diag_path = cycle_time_dto.strftime(gsi_sat_diag_path)
+        gsi_diag_path_files = glob.glob(gsi_sat_diag_path)
 
         # Get cycle dir and create if needed
         # ----------------------------------
@@ -43,7 +43,7 @@ def execute(self) -> None:
         # Assert that some files were found
         self.logger.assert_abort(len(gsi_diag_path_files) != 0 is not None, f'No ncdiag ' +
                                  f'files found in the source directory ' +
-                                 f'\'{gsi_diag_path_files_pattern}\'')
+                                 f'\'{gsi_diag_path}\'')
 
         # Copy all the files into the cycle directory
         # -------------------------------------------

diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py
@@ -229,28 +229,28 @@ def execute(self) -> None:
 
             # This will skip the fetch if we are cycling VarBC
             if bias_file_type != 'null':
-               if bias_file_type != 'null' and fetch_required:
-                   self.logger.info(f'Processing bias file {target_bccoef}')
-                   fetch(date=background_time,
-                         target_file=target_bccoef,
-                         provider='gsi',
-                         obs_type=observation,
-                         type='bc',
-                         experiment=obs_experiment,
-                         file_type=bias_file_type)
-
-                   self.logger.info(f'Processing bias file {target_bccovr}')
-                   fetch(date=background_time,
-                         target_file=target_bccovr,
-                         provider='gsi',
-                         obs_type=observation,
-                         type='bc',
-                         experiment=obs_experiment,
-                         file_type=bias_file_type+'_cov')
-
-               # Change permission
-               os.chmod(target_bccoef, 0o644)
-               os.chmod(target_bccovr, 0o644)
+                if bias_file_type != 'null' and fetch_required:
+                    self.logger.info(f'Processing bias file {target_bccoef}')
+                    fetch(date=background_time,
+                          target_file=target_bccoef,
+                          provider='gsi',
+                          obs_type=observation,
+                          type='bc',
+                          experiment=obs_experiment,
+                          file_type=bias_file_type)
+
+                    self.logger.info(f'Processing bias file {target_bccovr}')
+                    fetch(date=background_time,
+                          target_file=target_bccovr,
+                          provider='gsi',
+                          obs_type=observation,
+                          type='bc',
+                          experiment=obs_experiment,
+                          file_type=bias_file_type+'_cov')
+
+                # Change permission
+                os.chmod(target_bccoef, 0o644)
+                os.chmod(target_bccovr, 0o644)
 
             # Skip time lapse part for aircraft observations
             # ----------------------------------------------

diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml
@@ -369,10 +369,10 @@ geos_x_background_directory:
   ask_question: true
   default_value: /dev/null/
   models:
-  - geos_atmosphere
+  - all
   options:
   - /dev/null/
-  - /discover/nobackup/projects/gmao/dadev/rtodling/archive/541/Milan
+  - /discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x
   prompt: What is the path to the GEOS X-backgrounds directory?
   tasks:
   - GetBackgroundGeosExperiment
@@ -464,6 +464,7 @@ horizontal_resolution:
   - GenerateBClimatology
   - GenerateBClimatologyByLinking
   - GetBackground
+  - GetBackgroundGeosExperiment
   - RunJediEnsembleMeanVariance
   - RunJediFgatExecutable
   - RunJediHofxEnsembleExecutable

diff --git a/src/swell/test/suite_tests/3dfgat_atmos-tier1.yaml b/src/swell/test/suite_tests/3dfgat_atmos-tier1.yaml
@@ -1,6 +1,8 @@
+start_cycle_point: '2023-10-10T00:00:00Z'
+final_cycle_point: '2023-10-10T06:00:00Z'
 jedi_build_method: use_existing
 model_components: ['geos_atmosphere']
-runahead_limit: 'P3'
+runahead_limit: 'P2'
 models:
   geos_atmosphere:
     cycle_times:
@@ -9,6 +11,7 @@ models:
     - T12
     - T18
     horizontal_resolution: '91'
+    geos_x_background_directory: /discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x
     window_type: 4D
     observations:
     - aircraft_temperature

diff --git a/src/swell/test/suite_tests/3dvar_atmos-tier1.yaml b/src/swell/test/suite_tests/3dvar_atmos-tier1.yaml
@@ -1,6 +1,6 @@
-start_cycle_point: '2021-12-12T00:00:00Z'
-final_cycle_point: '2021-12-12T06:00:00Z'
-runahead_limit: 'P3'
+start_cycle_point: '2023-10-10T00:00:00Z'
+final_cycle_point: '2023-10-10T06:00:00Z'
+runahead_limit: 'P2'
 jedi_build_method: use_existing
 model_components: ['geos_atmosphere']
 models:
@@ -10,12 +10,12 @@ models:
     - T06
     - T12
     - T18
+    geos_x_background_directory: /discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x
     window_length: PT6H
     window_offset: PT3H
     window_type: 3D
     horizontal_resolution: '91'
     vertical_resolution: '72'
-    total_processors: 6
     observations:
     - aircraft_temperature
     - aircraft_wind

diff --git a/src/swell/test/suite_tests/hofx-tier1.yaml b/src/swell/test/suite_tests/hofx-tier1.yaml
@@ -1,9 +1,11 @@
 jedi_build_method: use_existing
 save_geovals: true
-model_components: ['geos_atmosphere', 'geos_ocean']
+# model_components: ['geos_atmosphere', 'geos_ocean']
+model_components: ['geos_atmosphere']
 models:
   geos_atmosphere:
     horizontal_resolution: '91'
+    geos_x_background_directory: /discover/nobackup/projects/gmao/dadev/rtodling/archive/Restarts/JEDI/541x
     npx_proc: 2
     npy_proc: 2
     observations:
@@ -42,19 +44,19 @@ models:
     - sondes
     - ssmis_f17
     clean_patterns: []
-  geos_ocean:
-    obs_experiment: s2s_v1
-    obs_provider: ['odas', 'gdas_marine']
-    horizontal_resolution: 72x36
-    vertical_resolution: '50'
-    observations:
-    - adt_cryosat2n
-    - adt_jason3
-    - adt_saral
-    - adt_sentinel3a
-    - adt_sentinel3b
-    - insitu_profile_argo
-    - sst_ostia
-    - sss_smos
-    - sss_smapv5
-    - temp_profile_xbt
+  # geos_ocean:
+  #   obs_experiment: s2s_v1
+  #   obs_provider: ['odas', 'gdas_marine']
+  #   horizontal_resolution: 72x36
+  #   vertical_resolution: '50'
+  #   observations:
+  #   - adt_cryosat2n
+  #   - adt_jason3
+  #   - adt_saral
+  #   - adt_sentinel3a
+  #   - adt_sentinel3b
+  #   - insitu_profile_argo
+  #   - sst_ostia
+  #   - sss_smos
+  #   - sss_smapv5
+  #   - temp_profile_xbt
diff --git a/src/swell/test/suite_tests/ufo_testing-tier1.yaml b/src/swell/test/suite_tests/ufo_testing-tier1.yaml
@@ -1,4 +1,4 @@
-final_cycle_point: '2021-12-12T00:00:00Z'
+final_cycle_point: '2023-10-10T00:00:00Z'
 jedi_build_method: use_existing
 bundles: REMOVE
 model_components: ['geos_atmosphere']
@@ -15,12 +15,12 @@ models:
     - amsua_metop-b
     - amsua_metop-c
     - amsua_n15
-    - amsua_n18
+    # - amsua_n18
     - amsua_n19
     - atms_n20
     - atms_npp
     - avhrr3_metop-b
-    - avhrr3_n18
+    # - avhrr3_n18
     - avhrr3_n19
     - cris-fsr_n20
     - cris-fsr_npp
@@ -55,4 +55,6 @@ models:
     - gsi_ncdiags/aircraft/*.nc4
     - gsi_ncdiags/aircraft
     - gsi_ncdiags
-    path_to_gsi_nc_diags: /discover/nobackup/projects/gmao/advda/SwellTestData/ufo_testing/ncdiagv2/%Y%m%d%H
+    # path_to_gsi_nc_diags: /discover/nobackup/projects/gmao/advda/SwellTestData/ufo_testing/ncdiagv2/%Y%m%d%H
+    path_to_gsi_bc_coefficients: /discover/nobackup/projects/gmao/dadev/rtodling/archive/541/Milan/x0050/ana/Y%Y/M%m/*bias*%Y%m%d_%Hz.txt
+    path_to_gsi_nc_diags: /discover/nobackup/projects/gmao/dadev/rtodling/archive/541/Milan/x0050/obs/Y%Y/M%m/D%d/H%H/