From 767d1dd3097a8b22465ff62b249bc3b90b87e5d9 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Wed, 27 Sep 2023 16:16:55 -0400 Subject: [PATCH 01/38] add new task --- src/swell/suites/ufo_testing/flow.cylc | 7 ++++ .../generate_satellite_channel_record.py | 35 +++++++++++++++++++ src/swell/utilities/run_jedi_executables.py | 4 +++ 3 files changed, 46 insertions(+) create mode 100644 src/swell/tasks/generate_satellite_channel_record.py diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index fd047820..04feb8bc 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -39,6 +39,9 @@ {% for cycle_time in cycle_times %} {{cycle_time.cycle_time}} = """ + # Generate satellite channel records + GenerateSatelliteChannelRecord + # Convert bias correction to ioda GetGsiBc GetGsiBc => GsiBcToIoda @@ -52,6 +55,7 @@ GetGeovals # Run Jedi hofx executable + GenerateSatelliteChannelRecord => RunJediUfoTestsExecutable GsiNcdiagToIoda => RunJediUfoTestsExecutable GsiBcToIoda => RunJediUfoTestsExecutable GetGeovals => RunJediUfoTestsExecutable @@ -101,6 +105,9 @@ --partition={{scheduling["BuildJedi"]["partition"]}} {% endif %} + [[GenerateSatelliteChannelRecord]] + script = "swell_task GenerateSatelliteChannelRecord $config -d $datetime -m geos_atmosphere" + [[ GetGsiBc ]] script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/tasks/generate_satellite_channel_record.py b/src/swell/tasks/generate_satellite_channel_record.py new file mode 100644 index 00000000..e7f66031 --- /dev/null +++ b/src/swell/tasks/generate_satellite_channel_record.py @@ -0,0 +1,35 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os + +from swell.tasks.base.task_base import taskBase + + +# -------------------------------------------------------------------------------------------------- + + +class GenerateSatelliteChannelRecord(taskBase): + + def execute(self): + + """ + Generate the satellite channel record from GEOSadas files + """ + + # Parse config + # ------------ + observations = self.config.observations() + + # Create files like amsua_n19_channel_record.yaml in self.cycle_dir()/satellite_channel_record + + + +# ---------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 79f5dcd7..fb624fc8 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -48,6 +48,10 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, for ob in obs: # Get observation dictionary obs_dict = jedi_rendering.render_interface_observations(ob) + + # Use the satellite channel record to set the correct channels + + observations.append(obs_dict) jedi_config_dict[key] = observations From 0a9813c2d9b6129c8f4f8dba48f89e4f79d735f6 Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 4 Oct 2023 15:26:28 -0400 Subject: [PATCH 02/38] adding task to generate sat channel record yamls --- src/swell/tasks/base/task_registry.py | 1 + .../generate_satellite_channel_record.py | 9 +- src/swell/tasks/task_questions.yaml | 1 + src/swell/utilities/satellite_records.py | 82 +++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 src/swell/utilities/satellite_records.py diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 4683e87a..4b76e6c5 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -16,6 +16,7 @@ 'EvaObservations', 'GenerateBClimatologyByLinking', 'GenerateBClimatology', + 'GenerateSatelliteChannelRecord', 'GetBackgroundGeosExperiment', 'GetBackground', 'GetEnsemble', diff --git a/src/swell/tasks/generate_satellite_channel_record.py b/src/swell/tasks/generate_satellite_channel_record.py index e7f66031..d621cf48 100644 --- a/src/swell/tasks/generate_satellite_channel_record.py +++ b/src/swell/tasks/generate_satellite_channel_record.py @@ -11,7 +11,7 @@ import os from swell.tasks.base.task_base import taskBase - +from swell.utilities.satellite_records import SatelliteRecords # -------------------------------------------------------------------------------------------------- @@ -29,7 +29,10 @@ def execute(self): observations = self.config.observations() # Create files like amsua_n19_channel_record.yaml in self.cycle_dir()/satellite_channel_record - - + path_to_records = '/discover/nobackup/asewnath/github/GEOSana_GridComp/GEOSaana_GridComp/GSI_GridComp/mksi/sidb/' + output_dir = self.cycle_dir() + '/satellite_channel_records' + sat_records = SatelliteRecords() + sat_records.parse_records(path_to_records) + sat_records.save_yamls(output_dir, observations) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index d7bf42e6..f86ade4d 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -458,6 +458,7 @@ observations: - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags + - GenerateSatelliteChannelRecord type: string-check-list path_to_ensemble: diff --git a/src/swell/utilities/satellite_records.py b/src/swell/utilities/satellite_records.py new file mode 100644 index 00000000..01f3bf13 --- /dev/null +++ b/src/swell/utilities/satellite_records.py @@ -0,0 +1,82 @@ +import os +import yaml +import pandas as pd +import numpy as np +import datetime as dt +from swell.utilities.sat_db_utils import read_sat_db +from swell.utilities.instr_state_machine import InstrStateMachine + + +def format_date(old_date): + date = dt.datetime.strptime(old_date, '%Y%m%d%H%M%S') + return date.isoformat() + + +class SatelliteRecords: + + def __init__(self): + self.column_names = ['sat', 'start', 'end', + 'instr', 'channel_num', + 'channels', 'comments'] + self.df = pd.DataFrame(columns=self.column_names) + self.obs_registry = [] + + + def parse_records(self, path_to_records): + # Only save satellites that are in list + org_df = read_sat_db(path_to_records, self.column_names) + sat_list = np.unique(org_df['sat'].values) + for sat in sat_list: + sat_df = org_df.loc[org_df['sat'] == sat] + instr_list = np.unique(sat_df['instr'].values) + + for instr in instr_list: + instr_df = sat_df.loc[sat_df['instr'] == instr] + state_machine = InstrStateMachine(instr_df) + state_machine.run() + new_instr_df = state_machine.get_instr_df() + self.df = pd.concat([self.df, new_instr_df], ignore_index=True) + self.obs_registry.append(instr+'_'+sat) + + + def save_yamls(self, output_dir, observation_list=None): + + if not observation_list: + observation_list = self.obs_registry + + if not os.path.exists(output_dir): + os.mkdir(output_dir) + + sat_list = np.unique(self.df['sat'].values) + for sat in sat_list: + df = self.df.loc[self.df['sat'] == sat] + instr_list = np.unique(df['instr'].values) + sat_dict = {} + + for instr in instr_list: + sat_dict = {} + instr_df = df.loc[df['instr'] == instr] + + compare_name = instr+'_'+sat + if compare_name in observation_list: + + field_list = [] + for idx, row in instr_df.iterrows(): + row_dict = {} + row_dict['begin date'] = format_date(row['start']) + row_dict['end date'] = format_date(row['end']) + row_dict['channels'] = row['channels'] + if (row['comments']): + row_dict['comments'] = row['comments'] + else: + row_dict['comments'] = 'no comment' + field_list.append(row_dict) + + sat_dict[instr] = field_list + + with open(output_dir+'/'+instr+'_'+sat+'.yaml', 'w') as file: + yaml.dump(sat_dict, file) + + + def read_yaml(self): + pass From 147abd6daa5e7b1036be11e44c173c6565a936a3 Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 5 Oct 2023 16:35:49 -0400 Subject: [PATCH 03/38] stash changes --- src/swell/utilities/run_jedi_executables.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index fb624fc8..bb605f53 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -49,7 +49,8 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, # Get observation dictionary obs_dict = jedi_rendering.render_interface_observations(ob) - # Use the satellite channel record to set the correct channels + # Use the satellite channel record to set the correct channels in useflag + observations.append(obs_dict) From 7a43317871ce5910310239229a1ff131fcfafe66 Mon Sep 17 00:00:00 2001 From: asewnath Date: Fri, 6 Oct 2023 11:38:42 -0400 Subject: [PATCH 04/38] stash --- .../observations/airs_aqua.yaml | 59 ++++++++++--------- .../utilities/render_jedi_interface_files.py | 1 + src/swell/utilities/run_jedi_executables.py | 14 +++-- 3 files changed, 39 insertions(+), 35 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index 961eafaf..3d6eb84a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -193,35 +193,36 @@ obs filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - use_flag: &useflag_airs_aqua [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - -1, -1, 1, 1, 1, 1, -1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: &useflag_airs_aqua {{airs_aqua_active_channels}} + #use_flag: &useflag_airs_aqua [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + # 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + # 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + # 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + # -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + # 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + # -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + # 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + # 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + # 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + # -1, -1, 1, 1, 1, 1, -1, -1, 1, -1, + # 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + # -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, + # 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + # 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + # 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + # -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + # -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + # -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + # 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + # -1 ] use_flag_clddet: &clddet_airs_aqua [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index b8f962b1..1e71c3ec 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -42,6 +42,7 @@ def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_inter # List of all potential valid keys that can be used in templates self.valid_template_keys = [ + 'airs_aqua_active_channels', 'analysis_variables', 'background_error_model', 'background_frequency', diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index bb605f53..d13bdc01 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -46,13 +46,15 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, if value_special == 'observations': observations = [] for ob in obs: - # Get observation dictionary + # Need knowledge of both available and active channels + # double rendering? obs_dict to retrieve available channels + # retrieve active channels list using cycle time, create + # use flag arr by list comparison + if ob == 'airs_aqua': + active_channels = [1, 1, 1] + jedi_rendering.add_key('airs_aqua_active_channels', active_channels) + obs_dict = jedi_rendering.render_interface_observations(ob) - - # Use the satellite channel record to set the correct channels in useflag - - - observations.append(obs_dict) jedi_config_dict[key] = observations From 9f1c359913e618efb07467c386eb2f5e959ccc3a Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 6 Oct 2023 12:19:29 -0400 Subject: [PATCH 05/38] ramblings --- .../generate_satellite_channel_record.py | 3 +- src/swell/tasks/get_observations.py | 4 +- src/swell/tasks/task_questions.yaml | 10 +++++ .../utilities/render_jedi_interface_files.py | 40 ++++++++++++++----- src/swell/utilities/satellite_records.py | 10 +++-- 5 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/swell/tasks/generate_satellite_channel_record.py b/src/swell/tasks/generate_satellite_channel_record.py index d621cf48..d4de162b 100644 --- a/src/swell/tasks/generate_satellite_channel_record.py +++ b/src/swell/tasks/generate_satellite_channel_record.py @@ -27,11 +27,12 @@ def execute(self): # Parse config # ------------ observations = self.config.observations() + observing_system_records_path = self.config.observing_system_records_path() # Create files like amsua_n19_channel_record.yaml in self.cycle_dir()/satellite_channel_record path_to_records = '/discover/nobackup/asewnath/github/GEOSana_GridComp/GEOSaana_GridComp/GSI_GridComp/mksi/sidb/' output_dir = self.cycle_dir() + '/satellite_channel_records' - sat_records = SatelliteRecords() + sat_records = SatelliteRecords(observing_system_records_path) sat_records.parse_records(path_to_records) sat_records.save_yamls(output_dir, observations) diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index e382cd7f..72cebee9 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -42,6 +42,7 @@ def execute(self): window_length = self.config.window_length() crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_offset = self.config.window_offset() + observing_system_records_path = self.config.observing_system_records_path() # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) @@ -59,7 +60,8 @@ def execute(self): # Open the observation operator dictionary # ---------------------------------------- - observation_dict = self.jedi_rendering.render_interface_observations(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation, + observing_system_records_path) # Fetch observation files # ----------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index f86ade4d..2ad0dcb9 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -625,3 +625,13 @@ window_type: - StoreBackground type: string-drop-list +observing_system_records_path: + ask_question: false + default_value: defer_to_platform + prompt: What is the path to the observing system records? + tasks: + - GenerateSatelliteChannelRecord + - GetObservations + - GsiBcToIoda + - ... + type: string-drop-list diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 1e71c3ec..4a0654b6 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -11,7 +11,7 @@ import yaml from swell.utilities.jinja2 import template_string_jinja2 - +from swell.utilities.satellite_records import SatelliteRecords # -------------------------------------------------------------------------------------------------- @@ -22,27 +22,29 @@ def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_inter # Keep a copy of the logger self.logger = logger + # Keep a copy of the cycle directory + self.cycle_dir = cycle_dir + # Copy the experiment configuration path self.jedi_config_path = os.path.join(experiment_root, experiment_id, 'configuration', 'jedi') # Dictionary to hold things that can be templated - self.template_dict = {} + self.__template_dict__ = {} # Always store the cycle directory in the dictionary - self.template_dict['cycle_dir'] = cycle_dir + self.__template_dict__['cycle_dir'] = cycle_dir # Add the jedi interface to the dictionary self.jedi_interface = jedi_interface - self.template_dict['model_component'] = jedi_interface + self.__template_dict__['model_component'] = jedi_interface # Add experiment info to dictionary - self.template_dict['experiment_id'] = experiment_id - self.template_dict['experiment_root'] = experiment_root + self.__template_dict__['experiment_id'] = experiment_id + self.__template_dict__['experiment_root'] = experiment_root # List of all potential valid keys that can be used in templates self.valid_template_keys = [ - 'airs_aqua_active_channels', 'analysis_variables', 'background_error_model', 'background_frequency', @@ -78,7 +80,7 @@ def add_key(self, key, element): f'of the valid keys: \'{self.valid_template_keys}\'') # Add element to dictionary - self.template_dict[key] = element + self.__template_dict__[key] = element # ---------------------------------------------------------------------------------------------- @@ -95,7 +97,7 @@ def __open_file_render_to_dict__(self, config_file): # Fill templates in the configuration file using the config config_file_str = template_string_jinja2(self.logger, config_file_str_templated, - self.template_dict) + self.__template_dict__) # Convert string to dictionary return yaml.safe_load(config_file_str) @@ -131,7 +133,7 @@ def render_interface_model(self, config_name): # ---------------------------------------------------------------------------------------------- # Prepare path to interface observations file and call rendering - def render_interface_observations(self, config_name): + def render_interface_observations(self, config_name, observing_system_records_path): # Assert that there is a jedi interface associated with the task self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + @@ -142,6 +144,24 @@ def render_interface_observations(self, config_name): config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, 'observations', f'{config_name}.yaml') + # Open file as a string + with open(config_file, 'r') as config_file_open: + config_file_str_templated = config_file_open.read() + + # Search the file for f'{config_name}_active_channels' and if exists open the channel selection file + if f'{config_name}_active_channels' in config_file_str_templated: + + # Create object of SatelliteRecords class + satellite_records = SatelliteRecords(observing_system_records_path) + + # Get active channels from satellite records + active_channels = satellite_records.get_active_channels(config_name) + + # active_channels = [-1 1 -1 1 1 -1 etc] + + # Add active channels to template dictionary + self.__template_dict__[f'{config_name}_active_channels'] = active_channels + # Render templates in file and return dictionary return self.__open_file_render_to_dict__(config_file) diff --git a/src/swell/utilities/satellite_records.py b/src/swell/utilities/satellite_records.py index 01f3bf13..1a4cedd9 100644 --- a/src/swell/utilities/satellite_records.py +++ b/src/swell/utilities/satellite_records.py @@ -12,15 +12,17 @@ def format_date(old_date): return date.isoformat() -class SatelliteRecords: +class ObservingSystemRecords: - def __init__(self): - self.column_names = ['sat', 'start', 'end', + def __init__(self, root_directory): + self.column_names = ['sat', 'start', 'end', 'instr', 'channel_num', 'channels', 'comments'] self.df = pd.DataFrame(columns=self.column_names) self.obs_registry = [] + # Location where directories containing observing system records are located + self.root_directory = root_directory def parse_records(self, path_to_records): # Only save satellites that are in list @@ -44,6 +46,8 @@ def save_yamls(self, output_dir, observation_list=None): if not observation_list: observation_list = self.obs_registry + # Switch output directory to root/radiances + if not os.path.exists(output_dir): os.mkdir(output_dir) From d86b9f7543c4d9fc1fa12c181de2d2885ead0da9 Mon Sep 17 00:00:00 2001 From: danholdaway Date: Fri, 6 Oct 2023 13:18:24 -0400 Subject: [PATCH 06/38] cloning of geos --- src/swell/tasks/clone_geos_ana.py | 43 +++++++++++++++++++ .../generate_satellite_channel_record.py | 5 ++- src/swell/tasks/task_questions.yaml | 12 +++++- 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 src/swell/tasks/clone_geos_ana.py diff --git a/src/swell/tasks/clone_geos_ana.py b/src/swell/tasks/clone_geos_ana.py new file mode 100644 index 00000000..c2b42499 --- /dev/null +++ b/src/swell/tasks/clone_geos_ana.py @@ -0,0 +1,43 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os + +from swell.tasks.base.task_base import taskBase +from swell.utilities.satellite_records import SatelliteRecords + +# -------------------------------------------------------------------------------------------------- + + +class CloneGeosAna(taskBase): + + def execute(self): + + """ + Generate the satellite channel record from GEOSadas files + """ + + # Parse config + # ------------ + path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path() + + # If observing_system_records_gsi_path is None, clone GEOSana_GridComp repo to experiment + # directory + if observing_system_records_gsi_path is None: + # Clone GEOSana_GridComp develop repo to experiment directory + os.system('git clone https://github.com/GEOS-ESM/GEOSana_GridComp.git ' + + os.path.join(self.experiment_path() + 'GEOSana_GridComp')) + else: + # Link the source code directory + link_path(self.config.observing_system_records_gsi_path(), + os.path.join(self.experiment_path() + 'GEOSana_GridComp')) + + +# ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/generate_satellite_channel_record.py b/src/swell/tasks/generate_satellite_channel_record.py index d4de162b..a542f311 100644 --- a/src/swell/tasks/generate_satellite_channel_record.py +++ b/src/swell/tasks/generate_satellite_channel_record.py @@ -30,7 +30,10 @@ def execute(self): observing_system_records_path = self.config.observing_system_records_path() # Create files like amsua_n19_channel_record.yaml in self.cycle_dir()/satellite_channel_record - path_to_records = '/discover/nobackup/asewnath/github/GEOSana_GridComp/GEOSaana_GridComp/GSI_GridComp/mksi/sidb/' + if observing_system_records_gsi_path is None: + + # path_to_record = from + path_to_records = os.path.join(self.experiment_path() + '/GEOSana_GridComp/...') output_dir = self.cycle_dir() + '/satellite_channel_records' sat_records = SatelliteRecords(observing_system_records_path) sat_records.parse_records(path_to_records) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 2ad0dcb9..b44c18d0 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -625,13 +625,21 @@ window_type: - StoreBackground type: string-drop-list +observing_system_records_gsi_path: + ask_question: false + default_value: None + prompt: What is the path to the observing system records ? + tasks: + - GenerateSatelliteChannelRecord + type: string + observing_system_records_path: ask_question: false - default_value: defer_to_platform + default_value: None prompt: What is the path to the observing system records? tasks: - GenerateSatelliteChannelRecord - GetObservations - GsiBcToIoda - ... - type: string-drop-list + type: string From 7260adedda993aadf494496fbd46d3b033248aec Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 11 Oct 2023 11:12:36 -0400 Subject: [PATCH 07/38] changes for obs system recs --- src/swell/suites/ufo_testing/flow.cylc | 14 ++++-- src/swell/tasks/base/task_registry.py | 3 +- src/swell/tasks/clone_geos_ana.py | 2 - src/swell/tasks/eva_observations.py | 5 ++- .../generate_observing_system_records.py | 44 +++++++++++++++++++ .../generate_satellite_channel_record.py | 42 ------------------ src/swell/tasks/get_observations.py | 3 ++ src/swell/tasks/gsi_bc_to_ioda.py | 7 ++- .../tasks/run_jedi_ufo_tests_executable.py | 4 +- src/swell/tasks/save_obs_diags.py | 4 +- src/swell/tasks/task_questions.yaml | 7 ++- ...records.py => observing_system_records.py} | 3 -- .../utilities/render_jedi_interface_files.py | 6 +-- src/swell/utilities/run_jedi_executables.py | 11 ++--- src/swell/utilities/sat_db_utils.py | 6 +++ 15 files changed, 93 insertions(+), 68 deletions(-) create mode 100644 src/swell/tasks/generate_observing_system_records.py delete mode 100644 src/swell/tasks/generate_satellite_channel_record.py rename src/swell/utilities/{satellite_records.py => observing_system_records.py} (98%) diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 04feb8bc..062c762c 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -34,13 +34,16 @@ # If not able to link to build create the build BuildJediByLinking:fail? => BuildJedi + + # Clone geos ana for generating observing system records + CloneGeosAna """ {% for cycle_time in cycle_times %} {{cycle_time.cycle_time}} = """ # Generate satellite channel records - GenerateSatelliteChannelRecord + GenerateObservingSystemRecords # Convert bias correction to ioda GetGsiBc @@ -55,7 +58,7 @@ GetGeovals # Run Jedi hofx executable - GenerateSatelliteChannelRecord => RunJediUfoTestsExecutable + GenerateObservingSystemRecords => RunJediUfoTestsExecutable GsiNcdiagToIoda => RunJediUfoTestsExecutable GsiBcToIoda => RunJediUfoTestsExecutable GetGeovals => RunJediUfoTestsExecutable @@ -105,8 +108,11 @@ --partition={{scheduling["BuildJedi"]["partition"]}} {% endif %} - [[GenerateSatelliteChannelRecord]] - script = "swell_task GenerateSatelliteChannelRecord $config -d $datetime -m geos_atmosphere" + [[CloneGeosAna]] + script = "swell_task CloneGeosAna $config -d $datetime -m geos_atmosphere" + + [[GenerateObservingSystemRecords]] + script = "swell_task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" [[ GetGsiBc ]] script = "swell_task GetGsiBc $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py index 4b76e6c5..01859e2f 100644 --- a/src/swell/tasks/base/task_registry.py +++ b/src/swell/tasks/base/task_registry.py @@ -11,12 +11,13 @@ 'BuildJedi', 'CleanCycle', 'CloneGeos', + 'CloneGeosAna', 'CloneJedi', 'EvaJediLog', 'EvaObservations', 'GenerateBClimatologyByLinking', 'GenerateBClimatology', - 'GenerateSatelliteChannelRecord', + 'GenerateObservingSystemRecords', 'GetBackgroundGeosExperiment', 'GetBackground', 'GetEnsemble', diff --git a/src/swell/tasks/clone_geos_ana.py b/src/swell/tasks/clone_geos_ana.py index c2b42499..84bfa118 100644 --- a/src/swell/tasks/clone_geos_ana.py +++ b/src/swell/tasks/clone_geos_ana.py @@ -9,9 +9,7 @@ import os - from swell.tasks.base.task_base import taskBase -from swell.utilities.satellite_records import SatelliteRecords # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index e050ad4e..c40077d3 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -64,10 +64,13 @@ def execute(self): # Loop over observations # ------------------- + observing_system_records_path = self.config.observing_system_records_path() + for observation in self.config.observations(): # Load the observation dictionary - observation_dict = self.jedi_rendering.render_interface_observations(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation, + observing_system_records_path) # Split the full path into path and filename obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py new file mode 100644 index 00000000..12303002 --- /dev/null +++ b/src/swell/tasks/generate_observing_system_records.py @@ -0,0 +1,44 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + + +import os + +from swell.tasks.base.task_base import taskBase +from swell.utilities.observing_system_records import ObservingSystemRecords + +# -------------------------------------------------------------------------------------------------- + + +class GenerateObservingSystemRecords(taskBase): + + def execute(self): + + """ + Generate the observing system channel records from GEOSadas files + """ + + # Parse GSI records and save channel selection yamls + # -------------------------------------------------- + observations = self.config.observations() + observing_system_records_path = self.config.observing_system_records_path() + if observation_system_records_path is None: + cycle_dir = self.config.cycle_dir() + observation_system_records_path = cycle_dir() + 'observing_system_records' + + path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path() + if path_to_geosana_gridcomp is None: + path_to_geosana_gridcomp = os.path.join(self.experiment_path() + 'GEOSana_GridComp')) + path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp', + 'GSI_GridComp', 'mksi', 'sidb') + sat_records = ObservingSystemRecords() + sat_records.parse_records(path_to_records) + sat_records.save_yamls(observing_system_records_path, observations) + +# ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/generate_satellite_channel_record.py b/src/swell/tasks/generate_satellite_channel_record.py deleted file mode 100644 index a542f311..00000000 --- a/src/swell/tasks/generate_satellite_channel_record.py +++ /dev/null @@ -1,42 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - - -import os - -from swell.tasks.base.task_base import taskBase -from swell.utilities.satellite_records import SatelliteRecords - -# -------------------------------------------------------------------------------------------------- - - -class GenerateSatelliteChannelRecord(taskBase): - - def execute(self): - - """ - Generate the satellite channel record from GEOSadas files - """ - - # Parse config - # ------------ - observations = self.config.observations() - observing_system_records_path = self.config.observing_system_records_path() - - # Create files like amsua_n19_channel_record.yaml in self.cycle_dir()/satellite_channel_record - if observing_system_records_gsi_path is None: - - # path_to_record = from - path_to_records = os.path.join(self.experiment_path() + '/GEOSana_GridComp/...') - output_dir = self.cycle_dir() + '/satellite_channel_records' - sat_records = SatelliteRecords(observing_system_records_path) - sat_records.parse_records(path_to_records) - sat_records.save_yamls(output_dir, observations) - -# ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 72cebee9..64746a7a 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -43,6 +43,9 @@ def execute(self): crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_offset = self.config.window_offset() observing_system_records_path = self.config.observing_system_records_path() + if observation_system_records_path is None: + cycle_dir = self.config.cycle_dir() + observation_system_records_path = cycle_dir() + 'observing_system_records' # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 1da9d308..c637009d 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -31,6 +31,10 @@ def execute(self): window_offset = self.config.window_offset() background_time_offset = self.config.background_time_offset() crtm_coeff_dir = self.config.crtm_coeff_dir(None) + observing_system_records_path = self.config.observing_system_records_path() + if observation_system_records_path is None: + cycle_dir = self.config.cycle_dir() + observation_system_records_path = cycle_dir() + 'observing_system_records' # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) @@ -51,7 +55,8 @@ def execute(self): print('observation', observation) # Open configuration file for observation - observation_dict = self.jedi_rendering.render_interface_observations(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation, + observing_system_records_path) # Check for sensor key try: diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index e779db79..daa1156d 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -37,6 +37,7 @@ def execute(self): bkg_time_offset = self.config.background_time_offset() observations = self.config.observations() generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + observing_system_records_path = self.config.observing_system_records_path() # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_offset) @@ -76,7 +77,8 @@ def execute(self): # Open the ufo_tests config file # ------------------------------ - ufo_tests_dict = self.jedi_rendering.render_interface_observations(f'ufo_tests') + ufo_tests_dict = self.jedi_rendering.render_interface_observations(f'ufo_tests', + observing_system_records_path) ufo_tests_default = ufo_tests_dict['default'] # Insert the GeoVaLs section diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index b3b15b90..b0d401fa 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -28,6 +28,7 @@ def execute(self): crtm_coeff_dir = self.config.crtm_coeff_dir(None) observations = self.config.observations() window_offset = self.config.window_offset() + observing_system_records_path = self.config.observing_system_records_path() # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) @@ -44,7 +45,8 @@ def execute(self): for observation in observations: # Load the observation dictionary - observation_dict = self.jedi_rendering.render_interface_observations(observation) + observation_dict = self.jedi_rendering.render_interface_observations(observation, + observing_system_records_path) # Store observation files # ----------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index b44c18d0..b3c198d6 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -458,7 +458,7 @@ observations: - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags - - GenerateSatelliteChannelRecord + - GenerateObservingSystemRecords type: string-check-list path_to_ensemble: @@ -630,7 +630,7 @@ observing_system_records_gsi_path: default_value: None prompt: What is the path to the observing system records ? tasks: - - GenerateSatelliteChannelRecord + - GenerateObservingSystemRecords type: string observing_system_records_path: @@ -638,8 +638,7 @@ observing_system_records_path: default_value: None prompt: What is the path to the observing system records? tasks: - - GenerateSatelliteChannelRecord + - GenerateObservingSystemRecords - GetObservations - GsiBcToIoda - - ... type: string diff --git a/src/swell/utilities/satellite_records.py b/src/swell/utilities/observing_system_records.py similarity index 98% rename from src/swell/utilities/satellite_records.py rename to src/swell/utilities/observing_system_records.py index 1a4cedd9..12486ad5 100644 --- a/src/swell/utilities/satellite_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -81,6 +81,3 @@ def save_yamls(self, output_dir, observation_list=None): with open(output_dir+'/'+instr+'_'+sat+'.yaml', 'w') as file: yaml.dump(sat_dict, file) - - def read_yaml(self): - pass diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 4a0654b6..ca654f13 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -11,7 +11,7 @@ import yaml from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.satellite_records import SatelliteRecords +#from swell.utilities.satellite_records import SatelliteRecords # -------------------------------------------------------------------------------------------------- @@ -152,10 +152,10 @@ def render_interface_observations(self, config_name, observing_system_records_pa if f'{config_name}_active_channels' in config_file_str_templated: # Create object of SatelliteRecords class - satellite_records = SatelliteRecords(observing_system_records_path) + #satellite_records = SatelliteRecords(observing_system_records_path) # Get active channels from satellite records - active_channels = satellite_records.get_active_channels(config_name) + #active_channels = satellite_records.get_active_channels(config_name) # active_channels = [-1 1 -1 1 1 -1 etc] diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index d13bdc01..b48cc034 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -50,11 +50,12 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, # double rendering? obs_dict to retrieve available channels # retrieve active channels list using cycle time, create # use flag arr by list comparison - if ob == 'airs_aqua': - active_channels = [1, 1, 1] - jedi_rendering.add_key('airs_aqua_active_channels', active_channels) - - obs_dict = jedi_rendering.render_interface_observations(ob) + #if ob == 'airs_aqua': + # active_channels = [1, 1, 1] + # jedi_rendering.add_key('airs_aqua_active_channels', active_channels) + # + # jedi_rendering.render_interface_obs_run_exec(ob) + obs_dict = jedi_rendering.render_interface_observations(ob, jedi_config_dict) observations.append(obs_dict) jedi_config_dict[key] = observations diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 2bb8344d..b1ea369f 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -17,6 +17,12 @@ # -------------------------------------------------------------------------------------------------- +def get_active_channels(path_to_yamls, observation): + + return [1, -1, 1] + +# -------------------------------------------------------------------------------------------------- + def read_sat_db(path_to_sat_db, column_names): From 91e301317db54955deae908cebdac8b1ded4d64c Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 12 Oct 2023 08:48:25 -0400 Subject: [PATCH 08/38] adding tasks and workflow changes --- src/swell/suites/ufo_testing/flow.cylc | 1 + src/swell/tasks/clone_geos_ana.py | 7 ++++--- src/swell/tasks/generate_observing_system_records.py | 10 +++++----- src/swell/tasks/get_observations.py | 4 ++-- src/swell/tasks/gsi_bc_to_ioda.py | 4 ++-- src/swell/tasks/task_questions.yaml | 1 + src/swell/utilities/observing_system_records.py | 4 ++-- src/swell/utilities/render_jedi_interface_files.py | 3 ++- 8 files changed, 19 insertions(+), 15 deletions(-) diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 062c762c..e9c448af 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -44,6 +44,7 @@ # Generate satellite channel records GenerateObservingSystemRecords + CloneGeosAna => GenerateObservingSystemRecords # Convert bias correction to ioda GetGsiBc diff --git a/src/swell/tasks/clone_geos_ana.py b/src/swell/tasks/clone_geos_ana.py index 84bfa118..914e3c6b 100644 --- a/src/swell/tasks/clone_geos_ana.py +++ b/src/swell/tasks/clone_geos_ana.py @@ -10,6 +10,7 @@ import os from swell.tasks.base.task_base import taskBase +from swell.utilities.build import link_path # -------------------------------------------------------------------------------------------------- @@ -28,14 +29,14 @@ def execute(self): # If observing_system_records_gsi_path is None, clone GEOSana_GridComp repo to experiment # directory - if observing_system_records_gsi_path is None: + if path_to_geosana_gridcomp == 'None': # Clone GEOSana_GridComp develop repo to experiment directory os.system('git clone https://github.com/GEOS-ESM/GEOSana_GridComp.git ' - + os.path.join(self.experiment_path() + 'GEOSana_GridComp')) + + os.path.join(self.experiment_path(), 'GEOSana_GridComp')) else: # Link the source code directory link_path(self.config.observing_system_records_gsi_path(), - os.path.join(self.experiment_path() + 'GEOSana_GridComp')) + os.path.join(self.experiment_path(), 'GEOSana_GridComp')) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 12303002..f4a1fdd0 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -28,17 +28,17 @@ def execute(self): # -------------------------------------------------- observations = self.config.observations() observing_system_records_path = self.config.observing_system_records_path() - if observation_system_records_path is None: + if observing_system_records_path == 'None': cycle_dir = self.config.cycle_dir() - observation_system_records_path = cycle_dir() + 'observing_system_records' + observing_system_records_path = cycle_dir() + 'observing_system_records' path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path() - if path_to_geosana_gridcomp is None: - path_to_geosana_gridcomp = os.path.join(self.experiment_path() + 'GEOSana_GridComp')) + if path_to_geosana_gridcomp == 'None': + path_to_geosana_gridcomp = os.path.join(self.experiment_path(), 'GEOSana_GridComp') path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp', 'GSI_GridComp', 'mksi', 'sidb') sat_records = ObservingSystemRecords() - sat_records.parse_records(path_to_records) + sat_records.parse_records(path_to_gsi_records) sat_records.save_yamls(observing_system_records_path, observations) # ---------------------------------------------------------------------------------------------- diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 64746a7a..e82a3c3d 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -43,9 +43,9 @@ def execute(self): crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_offset = self.config.window_offset() observing_system_records_path = self.config.observing_system_records_path() - if observation_system_records_path is None: + if observing_system_records_path is None: cycle_dir = self.config.cycle_dir() - observation_system_records_path = cycle_dir() + 'observing_system_records' + observing_system_records_path = cycle_dir() + 'observing_system_records' # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index c637009d..0b29f168 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -32,9 +32,9 @@ def execute(self): background_time_offset = self.config.background_time_offset() crtm_coeff_dir = self.config.crtm_coeff_dir(None) observing_system_records_path = self.config.observing_system_records_path() - if observation_system_records_path is None: + if observing_system_records_path is None: cycle_dir = self.config.cycle_dir() - observation_system_records_path = cycle_dir() + 'observing_system_records' + observing_system_records_path = cycle_dir() + 'observing_system_records' # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index b3c198d6..4487edab 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -631,6 +631,7 @@ observing_system_records_gsi_path: prompt: What is the path to the observing system records ? tasks: - GenerateObservingSystemRecords + - CloneGeosAna type: string observing_system_records_path: diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 12486ad5..32e5eeb0 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -14,7 +14,7 @@ def format_date(old_date): class ObservingSystemRecords: - def __init__(self, root_directory): + def __init__(self): self.column_names = ['sat', 'start', 'end', 'instr', 'channel_num', 'channels', 'comments'] @@ -22,7 +22,7 @@ def __init__(self, root_directory): self.obs_registry = [] # Location where directories containing observing system records are located - self.root_directory = root_directory + #self.root_directory = root_directory def parse_records(self, path_to_records): # Only save satellites that are in list diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index ca654f13..a1b8f14c 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -11,7 +11,7 @@ import yaml from swell.utilities.jinja2 import template_string_jinja2 -#from swell.utilities.satellite_records import SatelliteRecords +from swell.utilities.sat_db_utils import get_active_channels # -------------------------------------------------------------------------------------------------- @@ -158,6 +158,7 @@ def render_interface_observations(self, config_name, observing_system_records_pa #active_channels = satellite_records.get_active_channels(config_name) # active_channels = [-1 1 -1 1 1 -1 etc] + active_channels = get_active_channels(config_name, observing_system_records_path) # Add active channels to template dictionary self.__template_dict__[f'{config_name}_active_channels'] = active_channels From 07ada7300a9f7516209d9e43367760f3eafaf06f Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 17 Oct 2023 17:33:41 -0400 Subject: [PATCH 09/38] add get_active_channels --- src/swell/suites/ufo_testing/flow.cylc | 4 +- .../generate_observing_system_records.py | 4 +- .../active_channels_test_files/amsua_n19.yaml | 218 ++++++++++++++++++ .../amsua_n19_active_channels.yaml | 58 +++++ .../code_tests/get_active_channels_test.py | 12 + src/swell/utilities/instr_state_machine.py | 54 +++-- .../utilities/observing_system_records.py | 2 +- src/swell/utilities/sat_db_utils.py | 47 +++- 8 files changed, 362 insertions(+), 37 deletions(-) create mode 100644 src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml create mode 100644 src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml create mode 100644 src/swell/test/code_tests/get_active_channels_test.py diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index e9c448af..611add57 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -43,8 +43,8 @@ {{cycle_time.cycle_time}} = """ # Generate satellite channel records - GenerateObservingSystemRecords - CloneGeosAna => GenerateObservingSystemRecords + #GenerateObservingSystemRecords + CloneGeosAna[^] => GenerateObservingSystemRecords # Convert bias correction to ioda GetGsiBc diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index f4a1fdd0..69c884b8 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -29,8 +29,8 @@ def execute(self): observations = self.config.observations() observing_system_records_path = self.config.observing_system_records_path() if observing_system_records_path == 'None': - cycle_dir = self.config.cycle_dir() - observing_system_records_path = cycle_dir() + 'observing_system_records' + cycle_dir = self.cycle_dir() + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path() if path_to_geosana_gridcomp == 'None': diff --git a/src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml b/src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml new file mode 100644 index 00000000..6b69be8a --- /dev/null +++ b/src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml @@ -0,0 +1,218 @@ +obs space: + name: AMSU-A NOAA-19 + obsdatain: + engine: + type: H5File + obsfile: '{{cycle_dir}}/amsua_n19.{{window_begin}}.nc4' + obsdataout: + engine: + type: H5File + obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n19.{{window_begin}}.nc4' + simulated variables: [brightnessTemperature] + channels: &amsua_n19_available_channels 1-15 + +obs operator: + name: CRTM + Absorbers: [H2O,O3,CO2] + obs options: + Sensor_ID: &Sensor_ID amsua_n19 + EndianType: little_endian + CoefficientPath: '{{crtm_coeff_dir}}' + linear obs operator: + Absorbers: [H2O,O3] + +obs bias: + input file: '{{cycle_dir}}/amsua_n19.{{background_time}}.satbias.nc4' + variables without bc: [brightnessTemperature] + channels: 14 + variational bc: + predictors: + - name: constant + - name: cloud_liquid_water + sensor: AMSUA + clwdif_ch238: 1 + clwdif_ch314: 2 + - name: lapse_rate + order: 2 + tlapse: &amsua_n19_tlapse '{{cycle_dir}}/amsua_n19.{{background_time}}.tlapse.txt' + - name: lapse_rate + tlapse: *amsua_n19_tlapse + - name: emissivity + - name: scan_angle + order: 4 + - name: scan_angle + order: 3 + - name: scan_angle + order: 2 + - name: scan_angle + covariance: + minimal required obs number: 20 + variance range: [1.0e-6, 10.0] + step size: 1.0e-4 + largest analysis variance: 10000.0 + prior: + input file: '{{cycle_dir}}/amsua_n19.{{background_time}}.satbias.nc4' + inflation: + ratio: 1.1 + ratio for small dataset: 2.0 + output file: '{{cycle_dir}}/amsua_n19.{{window_begin}}.satbias.nc4' + +obs filters: +# Window and surface-sensitive channels check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: 1-6,15 + test variables: + - name: ObsValue/brightnessTemperature + channels: 1-6,15 + treat missing as out of bounds: true + minvalue: 100.0 + maxvalue: 500.0 + flag all filter variables if any test variable is out of bounds: true +# passedBenchmark: 1500 +# All channels unreasonable values check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + minvalue: 100.0 + maxvalue: 500.0 +# Hydrometeor Check (cloud/precipitation affected chanels) + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + test variables: + - name: ObsFunction/HydrometeorCheckAMSUAclr + channels: *amsua_n19_available_channels + options: + sensor: *Sensor_ID + channels: *amsua_n19_available_channels + test_biaspredictor: cloud_liquid_waterPredictor + maxvalue: 0.0 + action: + name: reject +# Assign obs error + - filter: Perform Action + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + action: + name: assign error + error parameter vector: + [2.500, 2.000, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 5.000, 2.500] +# Topography check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_n19_available_channels + options: + sensor: *Sensor_ID + channels: *amsua_n19_available_channels +# Transmittance Top Check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_n19_available_channels + options: + channels: *amsua_n19_available_channels +# Surface Jacobian check + - filter: BlackList + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + action: + name: inflate error + inflation variable: + name: ObsFunction/ObsErrorFactorSurfJacobianRad + channels: *amsua_n19_available_channels + options: + channels: *amsua_n19_available_channels + sensor: *Sensor_ID + use_biasterm: true + test_biasterm: ObsBiasTerm + obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] + obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] +# Gross check + - filter: Background Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + function absolute threshold: + - name: ObsFunction/ObsErrorBoundMW + channels: *amsua_n19_available_channels + options: + sensor: *Sensor_ID + channels: *amsua_n19_available_channels + obserr_bound_latitude: + name: ObsFunction/ObsErrorFactorLatRad + options: + latitude_parameters: [25.0, 0.25, 0.04, 3.0] + obserr_bound_transmittop: + name: ObsFunction/ObsErrorFactorTransmitTopRad + channels: *amsua_n19_available_channels + options: + channels: *amsua_n19_available_channels + obserr_bound_topo: + name: ObsFunction/ObsErrorFactorTopoRad + channels: *amsua_n19_available_channels + options: + channels: *amsua_n19_available_channels + sensor: *Sensor_ID + error parameter vector: + [2.500, 2.000, 2.000, 0.550, 0.300, + 0.230, 0.230, 0.250, 0.250, 0.350, + 0.400, 0.550, 0.800, 5.000, 2.500] + obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, + 2.0, 2.0, 2.0, 2.0, 2.0, + 2.5, 3.5, 4.5, 4.5, 4.5] + action: + name: reject +# Inter-channel check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + test variables: + - name: ObsFunction/InterChannelConsistencyCheck + channels: *amsua_n19_available_channels + options: + channels: *amsua_n19_available_channels + use passive_bc: true + sensor: *Sensor_ID + use_flag: [-1, -1, -1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, -1] + maxvalue: 1.0e-12 + action: + name: reject +# Useflag check + - filter: Bounds Check + filter variables: + - name: brightnessTemperature + channels: *amsua_n19_available_channels + test variables: + - name: ObsFunction/ChannelUseflagCheckRad + channels: *amsua_n19_available_channels + options: + channels: *amsua_n19_available_channels +# use passive_bc: true + use_flag: [-1, -1, -1, 1, 1, + 1, -1, -1, 1, 1, + 1, 1, 1, 1, -1] + minvalue: 1.0e-12 + action: + name: reject diff --git a/src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml b/src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml new file mode 100644 index 00000000..4f8c18f1 --- /dev/null +++ b/src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml @@ -0,0 +1,58 @@ +amsua: +- begin date: '2009-04-14T00:00:00' + channels: + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: '# M2: redundant entry intentional. Observations' + end date: '2009-12-15T06:00:00' +- begin date: '2009-12-15T12:00:00' + channels: + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: no comment + end date: '2009-12-21T18:00:00' +- begin date: '2009-12-22T00:00:00' + channels: + - '4' + - '5' + - '6' + - '7' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: '# ch8 noisy (noise started on 12/21/2009)' + end date: '2014-01-28T18:00:00' +- begin date: '2014-01-29T00:00:00' + channels: + - '4' + - '5' + - '6' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: '# per NCEP r35918' + end date: '2100-12-31T18:00:00' diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py new file mode 100644 index 00000000..ba3b5900 --- /dev/null +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -0,0 +1,12 @@ + +from swell.utilities.sat_db_utils import get_active_channels + +use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] + +cycle_time = '20211212T000000Z' +active_channel_yaml = 'active_channels_test_files/amsua_n19.yaml' +observation = 'active_channels_test_files/amsua_n19_active_channels.yaml' + +generated_use_flags = get_active_channels(observation, active_channel_yaml, cycle_time) + +assert(use_flags == generated_use_flags) diff --git a/src/swell/utilities/instr_state_machine.py b/src/swell/utilities/instr_state_machine.py index 75586926..acc306db 100644 --- a/src/swell/utilities/instr_state_machine.py +++ b/src/swell/utilities/instr_state_machine.py @@ -218,36 +218,34 @@ def get_instr_df(self): def update_return_df(self, row, no_comment=False, missing=False, missing_time={}): # Updates the return df based on parameters - if (missing): - self.return_df = self.return_df.append({ - "sat": row["sat"].values[0], - "start": missing_time['begin_time'], - "end": missing_time['end_time'], - "instr": row["instr"].values[0], - "channel_num": 0, - "channels": [], - "comments": "missing for this period", - }, ignore_index=True) + new_row = pd.DataFrame.from_dict({ + 'sat': [row["sat"].values[0]], + 'start': [missing_time['begin_time']], + 'end': [missing_time['end_time']], + 'instr': [row['instr'].values[0]], + 'channel_num': [0], + 'channels': [[]], + 'comments': ['missing for this period']}) elif (no_comment): - self.return_df = self.return_df.append({ - "sat": row["sat"].values[0], - "start": self.start_times[self.idx], - "end": self.end_times[self.idx], - "instr": row["instr"].values[0], - "channel_num": len(self.curr_channel_list), - "channels": self.curr_channel_list, - "comments": "" - }, ignore_index=True) + new_row = pd.DataFrame.from_dict({ + 'sat': [row["sat"].values[0]], + 'start': [self.start_times[self.idx]], + 'end': [self.end_times[self.idx]], + 'instr': [row['instr'].values[0]], + 'channel_num': [len(self.curr_channel_list)], + 'channels': [self.curr_channel_list], + 'comments': ['']}) else: - self.return_df = self.return_df.append({ - "sat": row["sat"].values[0], - "start": self.start_times[self.idx], - "end": self.end_times[self.idx], - "instr": row["instr"].values[0], - "channel_num": len(self.curr_channel_list), - "channels": self.curr_channel_list, - "comments": row["comments"].values[0], - }, ignore_index=True) + new_row = pd.DataFrame.from_dict({ + 'sat': [row["sat"].values[0]], + 'start': [self.start_times[self.idx]], + 'end': [self.end_times[self.idx]], + 'instr': [row['instr'].values[0]], + 'channel_num': [len(self.curr_channel_list)], + 'channels': [self.curr_channel_list], + 'comments': [row["comments"].values[0]]}) + + self.return_df = pd.concat([self.return_df, new_row], ignore_index=True) diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 32e5eeb0..68e3f15c 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -78,6 +78,6 @@ def save_yamls(self, output_dir, observation_list=None): sat_dict[instr] = field_list - with open(output_dir+'/'+instr+'_'+sat+'.yaml', 'w') as file: + with open(output_dir+'/'+instr+'_'+sat+'_active_channels.yaml', 'w') as file: yaml.dump(sat_dict, file) diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index b1ea369f..09280375 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -7,19 +7,58 @@ # -------------------------------------------------------------------------------------------------- - +import yaml import os import pandas as pd import numpy as np from swell.utilities.git_utils import git_got from swell.utilities.instr_state_machine import InstrStateMachine - +from datetime import datetime as dt # -------------------------------------------------------------------------------------------------- -def get_active_channels(path_to_yamls, observation): +def process_channel_lists(channel_list): + + final_channels_list = [] + if not isinstance(channel_list, list): + channel_list = [channel_list] + for element in channel_list: + if '-' in element: + start, end = map(int, element.split('-')) + result_list = [x for x in range(start, end + 1)] + final_channels_list += result_list + else: + final_channels_list += [int(element)] + + return final_channels_list + + +def get_active_channels(path_to_yamls, observation, cycle_time): + use_flags = [] + + # Cycle time to datetime object + dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") + + # Retrieve available channels from observation yaml + with open('active_channels_test_files/amsua_n19.yaml', 'r') as file: + data = yaml.safe_load(file) + available_channels = data['obs space']['channels'] + + # Retrieve active channels from records yaml + with open(path_to_yamls, 'r') as file: + data = yaml.safe_load(file) + for element in data['amsua']: #change the hardcoding + begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") + end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") + + if (dt_cycle_time > begin_date) and (dt_cycle_time < end_date): + active_channels = element['channels'] + + available_channels_list = process_channel_lists(available_channels) + active_channels_list = process_channel_lists(active_channels) + use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] - return [1, -1, 1] + return use_flags # -------------------------------------------------------------------------------------------------- From 37d8404d564260e981c47523d16530f28f5ead74 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 23 Oct 2023 15:49:54 -0400 Subject: [PATCH 10/38] updating tasks --- src/swell/tasks/eva_observations.py | 9 +++++- src/swell/tasks/get_observations.py | 7 ++++- src/swell/tasks/gsi_bc_to_ioda.py | 8 +++++- .../tasks/run_jedi_ufo_tests_executable.py | 5 ++-- src/swell/tasks/save_obs_diags.py | 10 ++++++- .../code_tests/get_active_channels_test.py | 8 ++++-- .../utilities/render_jedi_interface_files.py | 28 +++++++++++++------ src/swell/utilities/sat_db_utils.py | 13 ++++++--- 8 files changed, 66 insertions(+), 22 deletions(-) diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index c40077d3..d566c138 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -65,12 +65,19 @@ def execute(self): # Loop over observations # ------------------- observing_system_records_path = self.config.observing_system_records_path() + if observing_system_records_path is None: + cycle_dir = self.config.cycle_dir() + observing_system_records_path = cycle_dir() + 'observing_system_records' + experiment_config_path = self.experiment_config_path() + path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', + 'geos_atmosphere', 'observations') for observation in self.config.observations(): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path) + observing_system_records_path, + path_to_configs, cycle_time) # Split the full path into path and filename obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index e82a3c3d..e1ae2294 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -46,6 +46,10 @@ def execute(self): if observing_system_records_path is None: cycle_dir = self.config.cycle_dir() observing_system_records_path = cycle_dir() + 'observing_system_records' + experiment_config_path = self.experiment_config_path() + path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', + 'geos_atmosphere', 'observations') + cycle_time = os.path.basename(os.path.normpath(cycle_dir)) # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) @@ -64,7 +68,8 @@ def execute(self): # Open the observation operator dictionary # ---------------------------------------- observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path) + observing_system_records_path, + path_to_configs, cycle_time) # Fetch observation files # ----------------------- diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 0b29f168..cc021952 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -35,6 +35,9 @@ def execute(self): if observing_system_records_path is None: cycle_dir = self.config.cycle_dir() observing_system_records_path = cycle_dir() + 'observing_system_records' + experiment_config_path = self.experiment_config_path() + path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', + 'geos_atmosphere', 'observations') # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) @@ -51,12 +54,15 @@ def execute(self): sensors = [] sensors_satbias = [] sensors_tlapse = [] + + cycle_time = os.path.basename(os.path.normpath(cycle_dir)) for observation in observations: print('observation', observation) # Open configuration file for observation observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path) + observing_system_records_path, + path_to_configs, cycle_time) # Check for sensor key try: diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index daa1156d..137c0f50 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -37,7 +37,7 @@ def execute(self): bkg_time_offset = self.config.background_time_offset() observations = self.config.observations() generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - observing_system_records_path = self.config.observing_system_records_path() + #observing_system_records_path = self.config.observing_system_records_path() # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_offset) @@ -77,8 +77,7 @@ def execute(self): # Open the ufo_tests config file # ------------------------------ - ufo_tests_dict = self.jedi_rendering.render_interface_observations(f'ufo_tests', - observing_system_records_path) + ufo_tests_dict = self.jedi_rendering.render_interface_ufo_test() ufo_tests_default = ufo_tests_dict['default'] # Insert the GeoVaLs section diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index b0d401fa..bd3f13f2 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -29,6 +29,13 @@ def execute(self): observations = self.config.observations() window_offset = self.config.window_offset() observing_system_records_path = self.config.observing_system_records_path() + if observing_system_records_path is None: + cycle_dir = self.config.cycle_dir() + observing_system_records_path = cycle_dir() + 'observing_system_records' + experiment_config_path = self.experiment_config_path() + path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', + 'geos_atmosphere', 'observations') + cycle_time = os.path.basename(os.path.normpath(cycle_dir)) # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) @@ -46,7 +53,8 @@ def execute(self): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path) + observing_system_records_path, + path_to_configs, cycle_time) # Store observation files # ----------------------- diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index ba3b5900..89ce5e6c 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -4,9 +4,11 @@ use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] cycle_time = '20211212T000000Z' -active_channel_yaml = 'active_channels_test_files/amsua_n19.yaml' -observation = 'active_channels_test_files/amsua_n19_active_channels.yaml' +path_to_observing_sys_yamls = 'active_channels_test_files/' +path_to_configs = 'active_channels_test_files/' +observation = 'amsua_n19' -generated_use_flags = get_active_channels(observation, active_channel_yaml, cycle_time) +generated_use_flags = get_active_channels(path_to_observing_sys_yamls, + path_to_configs, observation, cycle_time) assert(use_flags == generated_use_flags) diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index a1b8f14c..4c3cf26d 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -133,7 +133,8 @@ def render_interface_model(self, config_name): # ---------------------------------------------------------------------------------------------- # Prepare path to interface observations file and call rendering - def render_interface_observations(self, config_name, observing_system_records_path): + #def render_interface_observations(self, config_name, observing_system_records_path): + def render_interface_observations(self, config_name, path_to_observing_sys_yamls, path_to_configs, cycle_time): # Assert that there is a jedi interface associated with the task self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + @@ -152,13 +153,8 @@ def render_interface_observations(self, config_name, observing_system_records_pa if f'{config_name}_active_channels' in config_file_str_templated: # Create object of SatelliteRecords class - #satellite_records = SatelliteRecords(observing_system_records_path) - - # Get active channels from satellite records - #active_channels = satellite_records.get_active_channels(config_name) - - # active_channels = [-1 1 -1 1 1 -1 etc] - active_channels = get_active_channels(config_name, observing_system_records_path) + active_channels = get_active_channels(path_to_observing_sys_yamls, + path_to_configs, observation, cycle_time) # Add active channels to template dictionary self.__template_dict__[f'{config_name}_active_channels'] = active_channels @@ -168,6 +164,22 @@ def render_interface_observations(self, config_name, observing_system_records_pa # ---------------------------------------------------------------------------------------------- + def render_interface_ufo_test(self): + config_name = f'ufo_tests' + # Assert that there is a jedi interface associated with the task + self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + + f'jedi interface config file the task must have an associated' + + f'jedi interface.') + + # Path to configuration file + config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, + 'observations', f'{config_name}.yaml') + + # Render templates in file and return dictionary + return self.__open_file_render_to_dict__(config_file) + + # ---------------------------------------------------------------------------------------------- + # Prepare path to interface metadata file and call rendering def render_interface_meta(self, model_component_in=None): diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 09280375..32c55a92 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -33,21 +33,26 @@ def process_channel_lists(channel_list): return final_channels_list -def get_active_channels(path_to_yamls, observation, cycle_time): +def get_active_channels(path_to_observing_sys_yamls, path_to_configs, observation, cycle_time): use_flags = [] # Cycle time to datetime object dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") # Retrieve available channels from observation yaml - with open('active_channels_test_files/amsua_n19.yaml', 'r') as file: + #with open('active_channels_test_files/amsua_n19.yaml', 'r') as file: + obs_name = observation.split('_')[0] + path_to_obs_config = path_to_configs + observation + '.yaml' + with open(path_to_obs_config, 'r') as file: data = yaml.safe_load(file) available_channels = data['obs space']['channels'] # Retrieve active channels from records yaml - with open(path_to_yamls, 'r') as file: + obs_name = observation.split('_')[0] + path_to_observing_sys_config = path_to_observing_sys_yamls + observation +'_active_channels.yaml' + with open(path_to_observing_sys_config, 'r') as file: data = yaml.safe_load(file) - for element in data['amsua']: #change the hardcoding + for element in data[obs_name]: begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") From 24fb5538bebd1eb83d1062219b84c89a362a49af Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 26 Oct 2023 11:23:02 -0400 Subject: [PATCH 11/38] parse active and available channels --- .../observations/amsua_n19.yaml | 14 ++-- src/swell/tasks/eva_observations.py | 7 +- src/swell/tasks/get_observations.py | 8 +- src/swell/tasks/gsi_bc_to_ioda.py | 9 ++- .../tasks/run_jedi_ufo_tests_executable.py | 12 ++- src/swell/tasks/save_obs_diags.py | 8 +- src/swell/tasks/task_questions.yaml | 1 + .../generate_observing_system_test.py | 11 +++ .../yaml_output/amsua_n19_channel_info.yaml | 77 ++++++++++++++++++ .../utilities/observing_system_records.py | 79 ++++++++++++------- .../utilities/render_jedi_interface_files.py | 2 +- src/swell/utilities/run_jedi_executables.py | 23 +++--- src/swell/utilities/sat_db_utils.py | 43 +++++----- 13 files changed, 207 insertions(+), 87 deletions(-) create mode 100644 src/swell/test/code_tests/generate_observing_system_test.py create mode 100644 src/swell/test/code_tests/yaml_output/amsua_n19_channel_info.yaml diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index 6b69be8a..4e298e13 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -193,9 +193,10 @@ obs filters: channels: *amsua_n19_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, 1, 1, - 1, -1, -1, 1, 1, - 1, 1, 1, 1, -1] + use_flag: &amsua_n19_use_flag [{{amsua_n19_active_channels}}] + #use_flag: [-1, -1, -1, 1, 1, + # 1, -1, -1, 1, 1, + # 1, 1, 1, 1, -1] maxvalue: 1.0e-12 action: name: reject @@ -210,9 +211,10 @@ obs filters: options: channels: *amsua_n19_available_channels # use passive_bc: true - use_flag: [-1, -1, -1, 1, 1, - 1, -1, -1, 1, 1, - 1, 1, 1, 1, -1] + use_flag: *amsua_n19_use_flag + #use_flag: [-1, -1, -1, 1, 1, + # 1, -1, -1, 1, 1, + # 1, 1, 1, 1, -1] minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index d566c138..c034873d 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -65,12 +65,13 @@ def execute(self): # Loop over observations # ------------------- observing_system_records_path = self.config.observing_system_records_path() - if observing_system_records_path is None: - cycle_dir = self.config.cycle_dir() - observing_system_records_path = cycle_dir() + 'observing_system_records' + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') experiment_config_path = self.experiment_config_path() path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', 'geos_atmosphere', 'observations') + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] for observation in self.config.observations(): diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index e1ae2294..3d5f4b81 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -43,13 +43,13 @@ def execute(self): crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_offset = self.config.window_offset() observing_system_records_path = self.config.observing_system_records_path() - if observing_system_records_path is None: - cycle_dir = self.config.cycle_dir() - observing_system_records_path = cycle_dir() + 'observing_system_records' + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') experiment_config_path = self.experiment_config_path() path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', 'geos_atmosphere', 'observations') - cycle_time = os.path.basename(os.path.normpath(cycle_dir)) + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index cc021952..2f838d59 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -32,9 +32,9 @@ def execute(self): background_time_offset = self.config.background_time_offset() crtm_coeff_dir = self.config.crtm_coeff_dir(None) observing_system_records_path = self.config.observing_system_records_path() - if observing_system_records_path is None: - cycle_dir = self.config.cycle_dir() - observing_system_records_path = cycle_dir() + 'observing_system_records' + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') experiment_config_path = self.experiment_config_path() path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', 'geos_atmosphere', 'observations') @@ -55,7 +55,8 @@ def execute(self): sensors_satbias = [] sensors_tlapse = [] - cycle_time = os.path.basename(os.path.normpath(cycle_dir)) + #cycle_time = os.path.basename(os.path.normpath(cycle_dir)) + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] for observation in observations: print('observation', observation) diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 137c0f50..47fc752b 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -37,7 +37,14 @@ def execute(self): bkg_time_offset = self.config.background_time_offset() observations = self.config.observations() generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - #observing_system_records_path = self.config.observing_system_records_path() + observing_system_records_path = self.config.observing_system_records_path() + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') + experiment_config_path = self.experiment_config_path() + path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', + 'geos_atmosphere', 'observations') + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_offset) @@ -61,7 +68,8 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations) + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations, + observing_system_records_path, path_to_configs, cycle_time) # Make modifications needed for testing # ------------------------------------- diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index bd3f13f2..867fd9f3 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -29,13 +29,13 @@ def execute(self): observations = self.config.observations() window_offset = self.config.window_offset() observing_system_records_path = self.config.observing_system_records_path() - if observing_system_records_path is None: - cycle_dir = self.config.cycle_dir() - observing_system_records_path = cycle_dir() + 'observing_system_records' + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') experiment_config_path = self.experiment_config_path() path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', 'geos_atmosphere', 'observations') - cycle_time = os.path.basename(os.path.normpath(cycle_dir)) + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 4487edab..2a53d50f 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -642,4 +642,5 @@ observing_system_records_path: - GenerateObservingSystemRecords - GetObservations - GsiBcToIoda + - RunJediUfoTestsExecutable type: string diff --git a/src/swell/test/code_tests/generate_observing_system_test.py b/src/swell/test/code_tests/generate_observing_system_test.py new file mode 100644 index 00000000..2ed26615 --- /dev/null +++ b/src/swell/test/code_tests/generate_observing_system_test.py @@ -0,0 +1,11 @@ +import os +from swell.utilities.observing_system_records import ObservingSystemRecords + +observations = ['amsua_n19'] +observing_system_records_path = './yaml_output/' +path_to_geosana_gridcomp = '/discover/nobackup/asewnath/github/GEOSana_GridComp/' +path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp', + 'GSI_GridComp', 'mksi', 'sidb') +sat_records = ObservingSystemRecords() +sat_records.parse_records(path_to_gsi_records) +sat_records.save_yamls(observing_system_records_path, observations) diff --git a/src/swell/test/code_tests/yaml_output/amsua_n19_channel_info.yaml b/src/swell/test/code_tests/yaml_output/amsua_n19_channel_info.yaml new file mode 100644 index 00000000..f5ce074f --- /dev/null +++ b/src/swell/test/code_tests/yaml_output/amsua_n19_channel_info.yaml @@ -0,0 +1,77 @@ +active: +- begin date: '2009-04-14T00:00:00' + channels: + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: '# M2: redundant entry intentional. Observations' + end date: '2009-12-15T06:00:00' +- begin date: '2009-12-15T12:00:00' + channels: + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: no comment + end date: '2009-12-21T18:00:00' +- begin date: '2009-12-22T00:00:00' + channels: + - '4' + - '5' + - '6' + - '7' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: '# ch8 noisy (noise started on 12/21/2009)' + end date: '2014-01-28T18:00:00' +- begin date: '2014-01-29T00:00:00' + channels: + - '4' + - '5' + - '6' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + comments: '# per NCEP r35918' + end date: '2100-12-31T18:00:00' +available: +- begin date: '2009-04-14T00:00:00' + channels: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + - '11' + - '12' + - '13' + - '14' + - '15' + end date: '2999-12-31T18:00:00' diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 68e3f15c..faa65307 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -18,27 +18,39 @@ def __init__(self): self.column_names = ['sat', 'start', 'end', 'instr', 'channel_num', 'channels', 'comments'] - self.df = pd.DataFrame(columns=self.column_names) + self.active_df = None + self.available_df = None self.obs_registry = [] - # Location where directories containing observing system records are located - #self.root_directory = root_directory - def parse_records(self, path_to_records): - # Only save satellites that are in list - org_df = read_sat_db(path_to_records, self.column_names) - sat_list = np.unique(org_df['sat'].values) - for sat in sat_list: - sat_df = org_df.loc[org_df['sat'] == sat] - instr_list = np.unique(sat_df['instr'].values) + def parse_records(self, path_to_sat_db): - for instr in instr_list: - instr_df = sat_df.loc[sat_df['instr'] == instr] - state_machine = InstrStateMachine(instr_df) - state_machine.run() - new_instr_df = state_machine.get_instr_df() - self.df = pd.concat([self.df, new_instr_df], ignore_index=True) - self.obs_registry.append(instr+'_'+sat) + channel_types = ['active', 'available'] + for channel_type in channel_types: + df = pd.DataFrame(columns=self.column_names) + path_to_records = os.path.join(path_to_sat_db, channel_type + '_channels.tbl') + + org_df = read_sat_db(path_to_records, self.column_names) + sat_list = np.unique(org_df['sat'].values) + for sat in sat_list: + sat_df = org_df.loc[org_df['sat'] == sat] + instr_list = np.unique(sat_df['instr'].values) + + for instr in instr_list: + instr_df = sat_df.loc[sat_df['instr'] == instr] + state_machine = InstrStateMachine(instr_df) + state_machine.run() + new_instr_df = state_machine.get_instr_df() + df = pd.concat([df, new_instr_df], ignore_index=True) + if instr+'_'+sat not in self.obs_registry: + self.obs_registry.append(instr+'_'+sat) + + if channel_type == 'active': + self.active_df = df + elif channel_type == 'available': + self.available_df = df + else: + print('record parsing unavailable for this type') def save_yamls(self, output_dir, observation_list=None): @@ -46,26 +58,26 @@ def save_yamls(self, output_dir, observation_list=None): if not observation_list: observation_list = self.obs_registry - # Switch output directory to root/radiances - if not os.path.exists(output_dir): os.mkdir(output_dir) - sat_list = np.unique(self.df['sat'].values) + # Assume that active and available channels have corresponding sat/instr fields + sat_list = np.unique(self.active_df['sat'].values) for sat in sat_list: - df = self.df.loc[self.df['sat'] == sat] - instr_list = np.unique(df['instr'].values) - sat_dict = {} + active_df = self.active_df.loc[self.active_df['sat'] == sat] + available_df = self.available_df.loc[self.available_df['sat'] == sat] + instr_list = np.unique(active_df['instr'].values) for instr in instr_list: sat_dict = {} - instr_df = df.loc[df['instr'] == instr] + instr_active_df = active_df.loc[active_df['instr'] == instr] + instr_available_df = available_df.loc[available_df['instr'] == instr] compare_name = instr+'_'+sat if compare_name in observation_list: - field_list = [] - for idx, row in instr_df.iterrows(): + active_field_list = [] + for idx, row in instr_active_df.iterrows(): row_dict = {} row_dict['begin date'] = format_date(row['start']) row_dict['end date'] = format_date(row['end']) @@ -74,10 +86,19 @@ def save_yamls(self, output_dir, observation_list=None): row_dict['comments'] = row['comments'] else: row_dict['comments'] = 'no comment' - field_list.append(row_dict) + active_field_list.append(row_dict) + + available_field_list = [] + for idx, row in instr_available_df.iterrows(): + row_dict = {} + row_dict['begin date'] = format_date(row['start']) + row_dict['end date'] = format_date(row['end']) + row_dict['channels'] = row['channels'] + available_field_list.append(row_dict) - sat_dict[instr] = field_list + sat_dict['available'] = available_field_list + sat_dict['active'] = active_field_list - with open(output_dir+'/'+instr+'_'+sat+'_active_channels.yaml', 'w') as file: + with open(output_dir+'/'+instr+'_'+sat+'_channel_info.yaml', 'w') as file: yaml.dump(sat_dict, file) diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 4c3cf26d..04692e5b 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -154,7 +154,7 @@ def render_interface_observations(self, config_name, path_to_observing_sys_yamls # Create object of SatelliteRecords class active_channels = get_active_channels(path_to_observing_sys_yamls, - path_to_configs, observation, cycle_time) + path_to_configs, config_name, cycle_time) # Add active channels to template dictionary self.__template_dict__[f'{config_name}_active_channels'] = active_channels diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index b48cc034..fe670686 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -17,13 +17,16 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, + observing_system_records_path, experiment_config_path, cycle_time, jedi_forecast_model=None): # Assemble configuration YAML file # -------------------------------- for key, value in jedi_config_dict.items(): if isinstance(value, dict): - jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, jedi_forecast_model) + jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, + observing_system_records_path, experiment_config_path, + cycle_time,jedi_forecast_model) elif isinstance(value, bool): continue @@ -31,8 +34,9 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, elif isinstance(value, list): for item in value: if isinstance(item, dict): - jedi_dictionary_iterator(item, jedi_rendering, window_type, obs, - jedi_forecast_model) + jedi_dictionary_iterator(item, jedi_rendering, window_type, obs, + observing_system_records_path, experiment_config_path, + cycle_time, jedi_forecast_model) else: if 'TASKFILL' in value: @@ -46,16 +50,9 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, if value_special == 'observations': observations = [] for ob in obs: - # Need knowledge of both available and active channels - # double rendering? obs_dict to retrieve available channels - # retrieve active channels list using cycle time, create - # use flag arr by list comparison - #if ob == 'airs_aqua': - # active_channels = [1, 1, 1] - # jedi_rendering.add_key('airs_aqua_active_channels', active_channels) - # - # jedi_rendering.render_interface_obs_run_exec(ob) - obs_dict = jedi_rendering.render_interface_observations(ob, jedi_config_dict) + obs_dict = jedi_rendering.render_interface_observations(ob, + observing_system_records_path, experiment_config_path, + cycle_time) observations.append(obs_dict) jedi_config_dict[key] = observations diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 32c55a92..469601eb 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -41,15 +41,15 @@ def get_active_channels(path_to_observing_sys_yamls, path_to_configs, observatio # Retrieve available channels from observation yaml #with open('active_channels_test_files/amsua_n19.yaml', 'r') as file: - obs_name = observation.split('_')[0] - path_to_obs_config = path_to_configs + observation + '.yaml' - with open(path_to_obs_config, 'r') as file: - data = yaml.safe_load(file) - available_channels = data['obs space']['channels'] + #obs_name = observation.split('_')[0] + #path_to_obs_config = path_to_configs + '/' + observation + '.yaml' + #with open(path_to_obs_config, 'r') as file: + # data = yaml.safe_load(file) + # available_channels = data['obs space']['channels'] - # Retrieve active channels from records yaml + # Retrieve available and active channels from records yaml obs_name = observation.split('_')[0] - path_to_observing_sys_config = path_to_observing_sys_yamls + observation +'_active_channels.yaml' + path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + observation +'_active_channels.yaml' with open(path_to_observing_sys_config, 'r') as file: data = yaml.safe_load(file) for element in data[obs_name]: @@ -61,7 +61,9 @@ def get_active_channels(path_to_observing_sys_yamls, path_to_configs, observatio available_channels_list = process_channel_lists(available_channels) active_channels_list = process_channel_lists(active_channels) - use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] + #use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] + #use_flags = [1] * len(available_channels_list) + use_flags = '1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1' return use_flags @@ -72,7 +74,8 @@ def read_sat_db(path_to_sat_db, column_names): # read data into a dataframe, throw line away if it starts with # or newline # --------------------------------------------------------------------------- - filename = os.path.join(path_to_sat_db, 'active_channels.tbl') + #filename = os.path.join(path_to_sat_db, 'active_channels.tbl') + filename = path_to_sat_db df = pd.DataFrame(columns=column_names) file = open(filename, 'r') @@ -85,17 +88,16 @@ def read_sat_db(path_to_sat_db, column_names): if (line_parts): if (line_parts[0][0] != '#' and line_parts[0][0] != '\n'): - - df = df.append({ - 'sat': '', - 'start': '', - 'end': '', - 'instr': '', - 'channel_num': 0, - 'channels': [], - 'comments': '' - }, ignore_index=True) - + new_row = pd.DataFrame.from_dict({ + 'sat': [''], + 'start': [''], + 'end': [''], + 'instr': [''], + 'channel_num': [0], + 'channels': [[]], + 'comments': ['']}) + + df = pd.concat([df, new_row], ignore_index=True) df['sat'][idx] = line_parts[0] df['start'][idx] = line_parts[1]+line_parts[2] df['end'][idx] = line_parts[3]+line_parts[4] @@ -116,7 +118,6 @@ def read_sat_db(path_to_sat_db, column_names): df['channels'][idx] = channel_list idx += 1 - return df From 0b755d52f5308b083d8da3be963dd401f4dd33ad Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 26 Oct 2023 14:24:14 -0400 Subject: [PATCH 12/38] workflow working with amsua_n19 --- .../observations/amsua_n19.yaml | 2 +- src/swell/tasks/eva_observations.py | 5 +- src/swell/tasks/get_observations.py | 5 +- src/swell/tasks/gsi_bc_to_ioda.py | 5 +- .../tasks/run_jedi_ufo_tests_executable.py | 5 +- src/swell/tasks/save_obs_diags.py | 5 +- .../amsua_n19_active_channels.yaml | 58 ------------------- .../amsua_n19_channel_info.yaml | 0 .../code_tests/get_active_channels_test.py | 1 - .../utilities/render_jedi_interface_files.py | 20 ++----- src/swell/utilities/run_jedi_executables.py | 8 +-- src/swell/utilities/sat_db_utils.py | 37 +++++------- 12 files changed, 31 insertions(+), 120 deletions(-) delete mode 100644 src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml rename src/swell/test/code_tests/{yaml_output => active_channels_test_files}/amsua_n19_channel_info.yaml (100%) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index 4e298e13..7185d102 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -193,7 +193,7 @@ obs filters: channels: *amsua_n19_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: &amsua_n19_use_flag [{{amsua_n19_active_channels}}] + use_flag: &amsua_n19_use_flag {{amsua_n19_active_channels}} #use_flag: [-1, -1, -1, 1, 1, # 1, -1, -1, 1, 1, # 1, 1, 1, 1, -1] diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index c034873d..79222182 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -68,9 +68,6 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - experiment_config_path = self.experiment_config_path() - path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', - 'geos_atmosphere', 'observations') cycle_time = os.path.normpath(cycle_dir).split('/')[-2] for observation in self.config.observations(): @@ -78,7 +75,7 @@ def execute(self): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation, observing_system_records_path, - path_to_configs, cycle_time) + cycle_time) # Split the full path into path and filename obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 3d5f4b81..83631384 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -46,9 +46,6 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - experiment_config_path = self.experiment_config_path() - path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', - 'geos_atmosphere', 'observations') cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Get window begin time @@ -69,7 +66,7 @@ def execute(self): # ---------------------------------------- observation_dict = self.jedi_rendering.render_interface_observations(observation, observing_system_records_path, - path_to_configs, cycle_time) + cycle_time) # Fetch observation files # ----------------------- diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 2f838d59..840bad6d 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -35,9 +35,6 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - experiment_config_path = self.experiment_config_path() - path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', - 'geos_atmosphere', 'observations') # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) @@ -63,7 +60,7 @@ def execute(self): # Open configuration file for observation observation_dict = self.jedi_rendering.render_interface_observations(observation, observing_system_records_path, - path_to_configs, cycle_time) + cycle_time) # Check for sensor key try: diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 47fc752b..539fce08 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -41,9 +41,6 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - experiment_config_path = self.experiment_config_path() - path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', - 'geos_atmosphere', 'observations') cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Compute data assimilation window parameters @@ -69,7 +66,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations, - observing_system_records_path, path_to_configs, cycle_time) + observing_system_records_path, cycle_time) # Make modifications needed for testing # ------------------------------------- diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 867fd9f3..1a22e626 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -32,9 +32,6 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - experiment_config_path = self.experiment_config_path() - path_to_configs = os.path.join(experiment_config_path, 'jedi', 'interfaces', - 'geos_atmosphere', 'observations') cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Get window beginning @@ -54,7 +51,7 @@ def execute(self): # Load the observation dictionary observation_dict = self.jedi_rendering.render_interface_observations(observation, observing_system_records_path, - path_to_configs, cycle_time) + cycle_time) # Store observation files # ----------------------- diff --git a/src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml b/src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml deleted file mode 100644 index 4f8c18f1..00000000 --- a/src/swell/test/code_tests/active_channels_test_files/amsua_n19_active_channels.yaml +++ /dev/null @@ -1,58 +0,0 @@ -amsua: -- begin date: '2009-04-14T00:00:00' - channels: - - '4' - - '5' - - '6' - - '7' - - '8' - - '9' - - '10' - - '11' - - '12' - - '13' - - '14' - comments: '# M2: redundant entry intentional. Observations' - end date: '2009-12-15T06:00:00' -- begin date: '2009-12-15T12:00:00' - channels: - - '4' - - '5' - - '6' - - '7' - - '8' - - '9' - - '10' - - '11' - - '12' - - '13' - - '14' - comments: no comment - end date: '2009-12-21T18:00:00' -- begin date: '2009-12-22T00:00:00' - channels: - - '4' - - '5' - - '6' - - '7' - - '9' - - '10' - - '11' - - '12' - - '13' - - '14' - comments: '# ch8 noisy (noise started on 12/21/2009)' - end date: '2014-01-28T18:00:00' -- begin date: '2014-01-29T00:00:00' - channels: - - '4' - - '5' - - '6' - - '9' - - '10' - - '11' - - '12' - - '13' - - '14' - comments: '# per NCEP r35918' - end date: '2100-12-31T18:00:00' diff --git a/src/swell/test/code_tests/yaml_output/amsua_n19_channel_info.yaml b/src/swell/test/code_tests/active_channels_test_files/amsua_n19_channel_info.yaml similarity index 100% rename from src/swell/test/code_tests/yaml_output/amsua_n19_channel_info.yaml rename to src/swell/test/code_tests/active_channels_test_files/amsua_n19_channel_info.yaml diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 89ce5e6c..5b4d8caa 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -10,5 +10,4 @@ generated_use_flags = get_active_channels(path_to_observing_sys_yamls, path_to_configs, observation, cycle_time) - assert(use_flags == generated_use_flags) diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 04692e5b..50b787f7 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -133,8 +133,7 @@ def render_interface_model(self, config_name): # ---------------------------------------------------------------------------------------------- # Prepare path to interface observations file and call rendering - #def render_interface_observations(self, config_name, observing_system_records_path): - def render_interface_observations(self, config_name, path_to_observing_sys_yamls, path_to_configs, cycle_time): + def render_interface_observations(self, config_name, path_to_observing_sys_yamls, cycle_time): # Assert that there is a jedi interface associated with the task self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + @@ -145,19 +144,12 @@ def render_interface_observations(self, config_name, path_to_observing_sys_yamls config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, 'observations', f'{config_name}.yaml') - # Open file as a string - with open(config_file, 'r') as config_file_open: - config_file_str_templated = config_file_open.read() - - # Search the file for f'{config_name}_active_channels' and if exists open the channel selection file - if f'{config_name}_active_channels' in config_file_str_templated: - - # Create object of SatelliteRecords class - active_channels = get_active_channels(path_to_observing_sys_yamls, - path_to_configs, config_name, cycle_time) + # Get active channels + active_channels = get_active_channels(path_to_observing_sys_yamls, + config_name, cycle_time) - # Add active channels to template dictionary - self.__template_dict__[f'{config_name}_active_channels'] = active_channels + # Add active channels to template dictionary + self.__template_dict__[f'{config_name}_active_channels'] = active_channels # Render templates in file and return dictionary return self.__open_file_render_to_dict__(config_file) diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index fe670686..c2fcc2b6 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -17,7 +17,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, - observing_system_records_path, experiment_config_path, cycle_time, + observing_system_records_path, cycle_time, jedi_forecast_model=None): # Assemble configuration YAML file @@ -25,7 +25,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, for key, value in jedi_config_dict.items(): if isinstance(value, dict): jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, - observing_system_records_path, experiment_config_path, + observing_system_records_path, cycle_time,jedi_forecast_model) elif isinstance(value, bool): @@ -35,7 +35,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, for item in value: if isinstance(item, dict): jedi_dictionary_iterator(item, jedi_rendering, window_type, obs, - observing_system_records_path, experiment_config_path, + observing_system_records_path, cycle_time, jedi_forecast_model) else: @@ -51,7 +51,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, observations = [] for ob in obs: obs_dict = jedi_rendering.render_interface_observations(ob, - observing_system_records_path, experiment_config_path, + observing_system_records_path, cycle_time) observations.append(obs_dict) jedi_config_dict[key] = observations diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 469601eb..9384c54e 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -32,49 +32,42 @@ def process_channel_lists(channel_list): return final_channels_list +# -------------------------------------------------------------------------------------------------- + +def get_channel_list(input_dict, dt_cycle_time): + for element in input_dict: + begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") + end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") + if (dt_cycle_time > begin_date) and (dt_cycle_time < end_date): + return element['channels'] + +# -------------------------------------------------------------------------------------------------- -def get_active_channels(path_to_observing_sys_yamls, path_to_configs, observation, cycle_time): - use_flags = [] +def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): # Cycle time to datetime object dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") - # Retrieve available channels from observation yaml - #with open('active_channels_test_files/amsua_n19.yaml', 'r') as file: - #obs_name = observation.split('_')[0] - #path_to_obs_config = path_to_configs + '/' + observation + '.yaml' - #with open(path_to_obs_config, 'r') as file: - # data = yaml.safe_load(file) - # available_channels = data['obs space']['channels'] - # Retrieve available and active channels from records yaml obs_name = observation.split('_')[0] - path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + observation +'_active_channels.yaml' + path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + observation +'_channel_info.yaml' with open(path_to_observing_sys_config, 'r') as file: data = yaml.safe_load(file) - for element in data[obs_name]: - begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") - end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") - - if (dt_cycle_time > begin_date) and (dt_cycle_time < end_date): - active_channels = element['channels'] + available_channels = get_channel_list(data['available'], dt_cycle_time) + active_channels = get_channel_list(data['active'], dt_cycle_time) available_channels_list = process_channel_lists(available_channels) active_channels_list = process_channel_lists(active_channels) - #use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] - #use_flags = [1] * len(available_channels_list) - use_flags = '1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1' + use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] return use_flags # -------------------------------------------------------------------------------------------------- - def read_sat_db(path_to_sat_db, column_names): # read data into a dataframe, throw line away if it starts with # or newline # --------------------------------------------------------------------------- - #filename = os.path.join(path_to_sat_db, 'active_channels.tbl') filename = path_to_sat_db df = pd.DataFrame(columns=column_names) From d32d27737cc7b034cfa106f2b58dbf6d3834b3a5 Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 26 Oct 2023 15:37:18 -0400 Subject: [PATCH 13/38] eva observations fix --- src/swell/tasks/task_questions.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 2a53d50f..9a52ae5a 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -643,4 +643,5 @@ observing_system_records_path: - GetObservations - GsiBcToIoda - RunJediUfoTestsExecutable + - EvaObservations type: string From dfd5663ed6b85800d75677540bc77ef257d5a51b Mon Sep 17 00:00:00 2001 From: asewnath Date: Fri, 27 Oct 2023 15:23:43 -0400 Subject: [PATCH 14/38] stash --- src/swell/utilities/sat_db_utils.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 9384c54e..9a71a44b 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -51,16 +51,21 @@ def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): # Retrieve available and active channels from records yaml obs_name = observation.split('_')[0] path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + observation +'_channel_info.yaml' - with open(path_to_observing_sys_config, 'r') as file: - data = yaml.safe_load(file) - available_channels = get_channel_list(data['available'], dt_cycle_time) - active_channels = get_channel_list(data['active'], dt_cycle_time) - available_channels_list = process_channel_lists(available_channels) - active_channels_list = process_channel_lists(active_channels) - use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] + if os.path.isfile(path_to_observing_sys_config): + with open(path_to_observing_sys_config, 'r') as file: + data = yaml.safe_load(file) + available_channels = get_channel_list(data['available'], dt_cycle_time) + active_channels = get_channel_list(data['active'], dt_cycle_time) - return use_flags + available_channels_list = process_channel_lists(available_channels) + active_channels_list = process_channel_lists(active_channels) + use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] + + return use_flags + + else: + return None # -------------------------------------------------------------------------------------------------- From 52477cddf2cca07332f1dff9675cda9cbc9a6da2 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 12:25:44 -0400 Subject: [PATCH 15/38] some clean up --- src/swell/suites/ufo_testing/flow.cylc | 4 ++-- src/swell/tasks/eva_observations.py | 8 ++------ src/swell/tasks/run_jedi_hofx_executable.py | 7 ++++++- src/swell/tasks/run_jedi_local_ensemble_da_executable.py | 7 ++++++- src/swell/tasks/run_jedi_variational_executable.py | 7 ++++++- src/swell/tasks/task_questions.yaml | 3 +++ src/swell/utilities/sat_db_utils.py | 3 --- 7 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 9cfdf9dd..05c20984 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -110,10 +110,10 @@ {% endif %} [[CloneGeosAna]] - script = "swell_task CloneGeosAna $config -d $datetime -m geos_atmosphere" + script = "swell task CloneGeosAna $config -d $datetime -m geos_atmosphere" [[GenerateObservingSystemRecords]] - script = "swell_task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" + script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" [[ GetGsiBc ]] script = "swell task GetGsiBc $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 1f581645..85f3af6a 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -81,19 +81,15 @@ def execute(self): 445, 552, 573, 906, 1121, 1194, 1427, 1585], } -<<<<<<< HEAD + # Loop over observations # ------------------- + eva_dicts = [] # Empty list of dictionaries observing_system_records_path = self.config.observing_system_records_path() cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') cycle_time = os.path.normpath(cycle_dir).split('/')[-2] -======= - # Loop over observations and create dictionaries - # ---------------------------------------------- - eva_dicts = [] # Empty list of dictionaries ->>>>>>> develop for observation in self.config.observations(): diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 9c3684b3..f94cabec 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -36,6 +36,11 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + observing_system_records_path = self.config.observing_system_records_path() + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -87,7 +92,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - jedi_forecast_model) + observing_system_records_path, cycle_time, jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 673bd0a4..5b2aaf71 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -36,6 +36,11 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + observing_system_records_path = self.config.observing_system_records_path() + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -129,7 +134,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - jedi_forecast_model) + observing_system_records_path, cycle_time, jedi_forecast_model) # Assemble localizations # ---------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 0b9e6727..4baa129c 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -37,6 +37,11 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) + observing_system_records_path = self.config.observing_system_records_path() + cycle_dir = self.cycle_dir() + if observing_system_records_path == 'None': + observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] npx_proc = self.config.npx_proc(None) npy_proc = self.config.npy_proc(None) @@ -101,7 +106,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - jedi_forecast_model) + observing_system_records_path, cycle_time, jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index ec62a6c6..01522b52 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -866,5 +866,8 @@ observing_system_records_path: - GetObservations - GsiBcToIoda - RunJediUfoTestsExecutable + - RunJediHofxExecutable + - RunJediVariationalExecutable + - RunJediLocalEnsembleDaExecutable - EvaObservations type: string diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 9a71a44b..018ec662 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -18,7 +18,6 @@ # -------------------------------------------------------------------------------------------------- def process_channel_lists(channel_list): - final_channels_list = [] if not isinstance(channel_list, list): channel_list = [channel_list] @@ -44,7 +43,6 @@ def get_channel_list(input_dict, dt_cycle_time): # -------------------------------------------------------------------------------------------------- def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): - # Cycle time to datetime object dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") @@ -70,7 +68,6 @@ def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): # -------------------------------------------------------------------------------------------------- def read_sat_db(path_to_sat_db, column_names): - # read data into a dataframe, throw line away if it starts with # or newline # --------------------------------------------------------------------------- filename = path_to_sat_db From 18e3fd5e0fbc1d8ade0fd9f99668561c06cfb812 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 14:20:54 -0400 Subject: [PATCH 16/38] coding norms --- src/swell/tasks/eva_observations.py | 11 +++++----- .../generate_observing_system_records.py | 2 +- src/swell/tasks/get_observations.py | 8 ++++--- src/swell/tasks/gsi_bc_to_ioda.py | 11 +++++----- src/swell/tasks/save_obs_diags.py | 8 ++++--- .../code_tests/get_active_channels_test.py | 5 ++--- .../utilities/observing_system_records.py | 3 --- .../utilities/render_jedi_interface_files.py | 1 + src/swell/utilities/run_jedi_executables.py | 22 +++++++++++-------- src/swell/utilities/sat_db_utils.py | 9 ++++++-- 10 files changed, 46 insertions(+), 34 deletions(-) diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 85f3af6a..d10ce7da 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -81,11 +81,10 @@ def execute(self): 445, 552, 573, 906, 1121, 1194, 1427, 1585], } - # Loop over observations # ------------------- eva_dicts = [] # Empty list of dictionaries - observing_system_records_path = self.config.observing_system_records_path() + observing_sys_records_path = self.config.observing_system_records_path() cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') @@ -94,9 +93,11 @@ def execute(self): for observation in self.config.observations(): # Load the observation dictionary - observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path, - cycle_time) + observation_dict = self.jedi_rendering.render_interface_observations( + observation, + observing_system_records_path, + cycle_time + ) # Split the full path into path and filename obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index 69c884b8..0038865d 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -35,7 +35,7 @@ def execute(self): path_to_geosana_gridcomp = self.config.observing_system_records_gsi_path() if path_to_geosana_gridcomp == 'None': path_to_geosana_gridcomp = os.path.join(self.experiment_path(), 'GEOSana_GridComp') - path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp', + path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp', 'GSI_GridComp', 'mksi', 'sidb') sat_records = ObservingSystemRecords() sat_records.parse_records(path_to_gsi_records) diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 83631384..b6661367 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -64,9 +64,11 @@ def execute(self): # Open the observation operator dictionary # ---------------------------------------- - observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path, - cycle_time) + observation_dict = self.jedi_rendering.render_interface_observations( + observation, + observing_system_records_path, + cycle_time + ) # Fetch observation files # ----------------------- diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 840bad6d..377048bb 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -51,16 +51,17 @@ def execute(self): sensors = [] sensors_satbias = [] sensors_tlapse = [] - - #cycle_time = os.path.basename(os.path.normpath(cycle_dir)) + cycle_time = os.path.normpath(cycle_dir).split('/')[-2] for observation in observations: print('observation', observation) # Open configuration file for observation - observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path, - cycle_time) + observation_dict = self.jedi_rendering.render_interface_observations( + observation, + observing_system_records_path, + cycle_time + ) # Check for sensor key try: diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 1a22e626..057efd5f 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -49,9 +49,11 @@ def execute(self): for observation in observations: # Load the observation dictionary - observation_dict = self.jedi_rendering.render_interface_observations(observation, - observing_system_records_path, - cycle_time) + observation_dict = self.jedi_rendering.render_interface_observations( + observation, + observing_system_records_path, + cycle_time + ) # Store observation files # ----------------------- diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 5b4d8caa..d1ac4d3c 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -1,5 +1,4 @@ - -from swell.utilities.sat_db_utils import get_active_channels +from swell.utilities.sat_db_utils import get_active_channels use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] @@ -8,6 +7,6 @@ path_to_configs = 'active_channels_test_files/' observation = 'amsua_n19' -generated_use_flags = get_active_channels(path_to_observing_sys_yamls, +generated_use_flags = get_active_channels(path_to_observing_sys_yamls, path_to_configs, observation, cycle_time) assert(use_flags == generated_use_flags) diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index faa65307..923f5998 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -22,7 +22,6 @@ def __init__(self): self.available_df = None self.obs_registry = [] - def parse_records(self, path_to_sat_db): channel_types = ['active', 'available'] @@ -52,7 +51,6 @@ def parse_records(self, path_to_sat_db): else: print('record parsing unavailable for this type') - def save_yamls(self, output_dir, observation_list=None): if not observation_list: @@ -101,4 +99,3 @@ def save_yamls(self, output_dir, observation_list=None): with open(output_dir+'/'+instr+'_'+sat+'_channel_info.yaml', 'w') as file: yaml.dump(sat_dict, file) - diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 9080c14d..8192368e 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -15,6 +15,7 @@ # -------------------------------------------------------------------------------------------------- + class JediConfigRendering(): def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_interface=None): diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index c2fcc2b6..6d6d2e85 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -24,9 +24,9 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, # -------------------------------- for key, value in jedi_config_dict.items(): if isinstance(value, dict): - jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, - observing_system_records_path, - cycle_time,jedi_forecast_model) + jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, + observing_system_records_path, + cycle_time, jedi_forecast_model) elif isinstance(value, bool): continue @@ -34,9 +34,11 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, elif isinstance(value, list): for item in value: if isinstance(item, dict): - jedi_dictionary_iterator(item, jedi_rendering, window_type, obs, - observing_system_records_path, - cycle_time, jedi_forecast_model) + jedi_dictionary_iterator( + item, jedi_rendering, window_type, obs, + observing_system_records_path, + cycle_time, jedi_forecast_model + ) else: if 'TASKFILL' in value: @@ -50,9 +52,11 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, if value_special == 'observations': observations = [] for ob in obs: - obs_dict = jedi_rendering.render_interface_observations(ob, - observing_system_records_path, - cycle_time) + obs_dict = jedi_rendering.render_interface_observations( + ob, + observing_system_records_path, + cycle_time + ) observations.append(obs_dict) jedi_config_dict[key] = observations diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 018ec662..500feecb 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -17,6 +17,7 @@ # -------------------------------------------------------------------------------------------------- + def process_channel_lists(channel_list): final_channels_list = [] if not isinstance(channel_list, list): @@ -33,6 +34,7 @@ def process_channel_lists(channel_list): # -------------------------------------------------------------------------------------------------- + def get_channel_list(input_dict, dt_cycle_time): for element in input_dict: begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") @@ -42,15 +44,17 @@ def get_channel_list(input_dict, dt_cycle_time): # -------------------------------------------------------------------------------------------------- + def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): # Cycle time to datetime object dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") # Retrieve available and active channels from records yaml obs_name = observation.split('_')[0] - path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + observation +'_channel_info.yaml' + path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + \ + observation + '_channel_info.yaml' - if os.path.isfile(path_to_observing_sys_config): + if os.path.isfile(path_to_observing_sys_config): with open(path_to_observing_sys_config, 'r') as file: data = yaml.safe_load(file) available_channels = get_channel_list(data['available'], dt_cycle_time) @@ -67,6 +71,7 @@ def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): # -------------------------------------------------------------------------------------------------- + def read_sat_db(path_to_sat_db, column_names): # read data into a dataframe, throw line away if it starts with # or newline # --------------------------------------------------------------------------- From 6f755d570dfb00d7968a739b99d31bd51da31476 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 15:23:11 -0400 Subject: [PATCH 17/38] fix yaml --- .../observations/airs_aqua.yaml | 59 +++++++++---------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index 2bdb4d6a..cb8ab800 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -194,36 +194,35 @@ obs post filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - use_flag: &useflag_airs_aqua {{airs_aqua_active_channels}} - #use_flag: &useflag_airs_aqua [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - # 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - # 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - # 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - # -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - # 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - # -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - # 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - # 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - # 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - # -1, -1, 1, 1, 1, 1, -1, -1, 1, -1, - # 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - # -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, - # 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - # 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - # 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - # -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - # -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - # -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - # 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - # -1 ] + use_flag: &useflag_airs_aqua [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, + 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, + -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, + -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, + 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, + -1, -1, 1, 1, 1, 1, -1, -1, 1, -1, + 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, + -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, + 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, + -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, + -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, + -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1 ] use_flag_clddet: &clddet_airs_aqua [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, From cfb4dc9ff9f6bce3bd6e9c93f0cc3deb54f50b02 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 15:40:57 -0400 Subject: [PATCH 18/38] norms --- src/swell/test/code_tests/get_active_channels_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index d1ac4d3c..26aa462f 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -9,4 +9,4 @@ generated_use_flags = get_active_channels(path_to_observing_sys_yamls, path_to_configs, observation, cycle_time) -assert(use_flags == generated_use_flags) +assert (use_flags == generated_use_flags) From 856da043176634690564f6084df9be164605efa2 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 15:44:03 -0400 Subject: [PATCH 19/38] pandas --- requirements-github.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-github.txt b/requirements-github.txt index 2f0dbd3a..7f480e7d 100644 --- a/requirements-github.txt +++ b/requirements-github.txt @@ -2,7 +2,7 @@ click==8.1.5 jinja2==3.1.2 pyyaml==6.0 pycodestyle==2.10.0 -#pandas==1.4.0 +pandas==1.4.0 isodate==0.6.1 f90nml==1.4.3 questionary==1.10.0 From 368d0afe50c33b4f4e716ab081175197160f8645 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 16:09:53 -0400 Subject: [PATCH 20/38] sat_db_utils fix --- src/swell/utilities/sat_db_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py index 500feecb..fb3bf5dc 100644 --- a/src/swell/utilities/sat_db_utils.py +++ b/src/swell/utilities/sat_db_utils.py @@ -50,7 +50,6 @@ def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") # Retrieve available and active channels from records yaml - obs_name = observation.split('_')[0] path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + \ observation + '_channel_info.yaml' From ded10613c8fe71664c8e46ba22a635e9fd86d448 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 30 Oct 2023 16:22:32 -0400 Subject: [PATCH 21/38] fix eva_obs --- src/swell/tasks/eva_observations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index d10ce7da..0029faed 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -84,7 +84,7 @@ def execute(self): # Loop over observations # ------------------- eva_dicts = [] # Empty list of dictionaries - observing_sys_records_path = self.config.observing_system_records_path() + observing_system_records_path = self.config.observing_system_records_path() cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') From 44318c0c4035610ab757d96695e2b516d6823fcf Mon Sep 17 00:00:00 2001 From: asewnath Date: Tue, 31 Oct 2023 18:35:59 -0400 Subject: [PATCH 22/38] requested changes --- .../observations/amsua_n19.yaml | 6 - src/swell/suites/ufo_testing/flow.cylc | 3 +- src/swell/tasks/base/task_registry.py | 48 ------ src/swell/tasks/eva_observations.py | 12 +- src/swell/tasks/get_observations.py | 12 +- src/swell/tasks/gsi_bc_to_ioda.py | 12 +- src/swell/tasks/run_jedi_hofx_executable.py | 8 +- .../run_jedi_local_ensemble_da_executable.py | 8 +- .../tasks/run_jedi_ufo_tests_executable.py | 9 +- .../tasks/run_jedi_variational_executable.py | 5 +- src/swell/tasks/save_obs_diags.py | 12 +- src/swell/tasks/task_questions.yaml | 4 + .../code_tests/get_active_channels_test.py | 2 +- src/swell/utilities/get_active_channels.py | 83 +++++++++ ..._state_machine.py => gsi_record_parser.py} | 116 ++++++++----- .../utilities/observing_system_records.py | 88 +++++++++- .../utilities/render_jedi_interface_files.py | 49 +++--- src/swell/utilities/run_jedi_executables.py | 13 +- src/swell/utilities/sat_db_utils.py | 161 ------------------ .../scripts/swell_sat_db_processing.py | 92 ---------- 20 files changed, 327 insertions(+), 416 deletions(-) delete mode 100644 src/swell/tasks/base/task_registry.py create mode 100644 src/swell/utilities/get_active_channels.py rename src/swell/utilities/{instr_state_machine.py => gsi_record_parser.py} (70%) delete mode 100644 src/swell/utilities/sat_db_utils.py delete mode 100644 src/swell/utilities/scripts/swell_sat_db_processing.py diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index 5cccf675..fe1f2edd 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -192,9 +192,6 @@ obs post filters: use passive_bc: true sensor: *Sensor_ID use_flag: &amsua_n19_use_flag {{amsua_n19_active_channels}} - #use_flag: [-1, -1, -1, 1, 1, - # 1, -1, -1, 1, 1, - # 1, 1, 1, 1, -1] maxvalue: 1.0e-12 action: name: reject @@ -210,9 +207,6 @@ obs post filters: channels: *amsua_n19_available_channels # use passive_bc: true use_flag: *amsua_n19_use_flag - #use_flag: [-1, -1, -1, 1, 1, - # 1, -1, -1, 1, 1, - # 1, 1, 1, 1, -1] minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/suites/ufo_testing/flow.cylc b/src/swell/suites/ufo_testing/flow.cylc index 05c20984..e675a836 100644 --- a/src/swell/suites/ufo_testing/flow.cylc +++ b/src/swell/suites/ufo_testing/flow.cylc @@ -43,7 +43,6 @@ {{cycle_time.cycle_time}} = """ # Generate satellite channel records - #GenerateObservingSystemRecords CloneGeosAna[^] => GenerateObservingSystemRecords # Convert bias correction to ioda @@ -110,7 +109,7 @@ {% endif %} [[CloneGeosAna]] - script = "swell task CloneGeosAna $config -d $datetime -m geos_atmosphere" + script = "swell task CloneGeosAna $config" [[GenerateObservingSystemRecords]] script = "swell task GenerateObservingSystemRecords $config -d $datetime -m geos_atmosphere" diff --git a/src/swell/tasks/base/task_registry.py b/src/swell/tasks/base/task_registry.py deleted file mode 100644 index 01859e2f..00000000 --- a/src/swell/tasks/base/task_registry.py +++ /dev/null @@ -1,48 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -valid_tasks = [ - 'BuildGeosByLinking', - 'BuildGeos', - 'BuildJediByLinking', - 'BuildJedi', - 'CleanCycle', - 'CloneGeos', - 'CloneGeosAna', - 'CloneJedi', - 'EvaJediLog', - 'EvaObservations', - 'GenerateBClimatologyByLinking', - 'GenerateBClimatology', - 'GenerateObservingSystemRecords', - 'GetBackgroundGeosExperiment', - 'GetBackground', - 'GetEnsemble', - 'GetGeosAdasBackground', - 'GetGeosRestart', - 'GetGeovals', - 'GetGsiBc', - 'GetGsiNcdiag', - 'GetObservations', - 'GsiBcToIoda', - 'GsiNcdiagToIoda', - 'LinkGeosOutput', - 'MoveDaRestart', - 'MoveForecastRestart', - 'ObsProcessSetup', - 'PrepareAnalysis', - 'PrepGeosRunDir', - 'RemoveForecastDir', - 'RunGeosExecutable', - 'RunJediHofxExecutable', - 'RunJediLetkfExecutable', - 'RunJediUfoTestsExecutable', - 'RunJediVariationalExecutable', - 'SaveObsDiags', - 'SaveRestart', - 'StageJedi', - 'StoreBackground', -] diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index 0029faed..e612de09 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -88,16 +88,16 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() + + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) for observation in self.config.observations(): # Load the observation dictionary - observation_dict = self.jedi_rendering.render_interface_observations( - observation, - observing_system_records_path, - cycle_time - ) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Split the full path into path and filename obs_path_file = observation_dict['obs space']['obsdataout']['engine']['obsfile'] diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index b6661367..f7d4ee43 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -46,7 +46,11 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() + + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) # Get window begin time window_begin = self.da_window_params.window_begin(window_offset) @@ -64,11 +68,7 @@ def execute(self): # Open the observation operator dictionary # ---------------------------------------- - observation_dict = self.jedi_rendering.render_interface_observations( - observation, - observing_system_records_path, - cycle_time - ) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Fetch observation files # ----------------------- diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 377048bb..ac35b529 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -35,6 +35,7 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') + cycle_time = self.cycle_time_dto() # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) @@ -52,16 +53,15 @@ def execute(self): sensors_satbias = [] sensors_tlapse = [] - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) + for observation in observations: print('observation', observation) # Open configuration file for observation - observation_dict = self.jedi_rendering.render_interface_observations( - observation, - observing_system_records_path, - cycle_time - ) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Check for sensor key try: diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index f94cabec..0701a7b3 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -40,7 +40,7 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -91,8 +91,12 @@ def execute(self): # Perform complete template rendering # ----------------------------------- + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - observing_system_records_path, cycle_time, jedi_forecast_model) + jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 5b2aaf71..e674624c 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -40,7 +40,11 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() + + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) # Compute data assimilation window parameters background_time = self.da_window_params.background_time(window_offset, @@ -134,7 +138,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - observing_system_records_path, cycle_time, jedi_forecast_model) + jedi_forecast_model) # Assemble localizations # ---------------------- diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 89112dad..80effe46 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -42,7 +42,11 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() + + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) # Compute data assimilation window parameters window_begin = self.da_window_params.window_begin(window_offset) @@ -66,8 +70,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations, - observing_system_records_path, cycle_time) + jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, '3D', observations) # Make modifications needed for testing # ------------------------------------- diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 4baa129c..d26deadc 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -41,7 +41,7 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() npx_proc = self.config.npx_proc(None) npy_proc = self.config.npy_proc(None) @@ -105,6 +105,9 @@ def execute(self): # Perform complete template rendering # ----------------------------------- + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, observing_system_records_path, cycle_time, jedi_forecast_model) diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 057efd5f..de203829 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -32,7 +32,11 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = os.path.normpath(cycle_dir).split('/')[-2] + cycle_time = self.cycle_time_dto() + + # Set cycle time and observing_system_records_path in jedi_rendering + self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) + self.jedi_rendering.set_cycle_time(cycle_time) # Get window beginning window_begin = self.da_window_params.window_begin(window_offset) @@ -49,11 +53,7 @@ def execute(self): for observation in observations: # Load the observation dictionary - observation_dict = self.jedi_rendering.render_interface_observations( - observation, - observing_system_records_path, - cycle_time - ) + observation_dict = self.jedi_rendering.render_interface_observations(observation) # Store observation files # ----------------------- diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 01522b52..e8d69203 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -851,6 +851,8 @@ window_type: observing_system_records_gsi_path: ask_question: false default_value: None + models: + - all prompt: What is the path to the observing system records ? tasks: - GenerateObservingSystemRecords @@ -860,6 +862,8 @@ observing_system_records_gsi_path: observing_system_records_path: ask_question: false default_value: None + models: + - all prompt: What is the path to the observing system records? tasks: - GenerateObservingSystemRecords diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 26aa462f..4f61ce02 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -1,4 +1,4 @@ -from swell.utilities.sat_db_utils import get_active_channels +from swell.utilities.get_active_channels import get_active_channels use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] diff --git a/src/swell/utilities/get_active_channels.py b/src/swell/utilities/get_active_channels.py new file mode 100644 index 00000000..4083a0d2 --- /dev/null +++ b/src/swell/utilities/get_active_channels.py @@ -0,0 +1,83 @@ +# (C) Copyright 2021- United States Government as represented by the Administrator of the +# National Aeronautics and Space Administration. All Rights Reserved. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + + +# -------------------------------------------------------------------------------------------------- + +import yaml +import os +import pandas as pd +import numpy as np +from datetime import datetime as dt + +# -------------------------------------------------------------------------------------------------- + + +def process_channel_lists(channel_list): + + ''' + Function processes list of elements in channel list + ''' + + final_channels_list = [] + if not isinstance(channel_list, list): + channel_list = [channel_list] + for element in channel_list: + if '-' in element: + start, end = map(int, element.split('-')) + result_list = [x for x in range(start, end + 1)] + final_channels_list += result_list + else: + final_channels_list += [int(element)] + + return final_channels_list + +# -------------------------------------------------------------------------------------------------- + + +def get_channel_list(input_dict, dt_cycle_time): + + ''' + Function retrieves channel lists from dict loaded from a yaml file + ''' + + for element in input_dict: + begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") + end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") + if (dt_cycle_time > begin_date) and (dt_cycle_time < end_date): + return element['channels'] + +# -------------------------------------------------------------------------------------------------- + + +def get_active_channels(path_to_observing_sys_yamls, observation, dt_cycle_time): + + ''' + Comparing available channels and active channels from the observing + system records to create the use flag array needed in the + qc filter yaml files. + ''' + + # Retrieve available and active channels from records yaml + path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + \ + observation + '_channel_info.yaml' + + if os.path.isfile(path_to_observing_sys_config): + with open(path_to_observing_sys_config, 'r') as file: + data = yaml.safe_load(file) + available_channels = get_channel_list(data['available'], dt_cycle_time) + active_channels = get_channel_list(data['active'], dt_cycle_time) + + available_channels_list = process_channel_lists(available_channels) + active_channels_list = process_channel_lists(active_channels) + use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] + + return use_flags + + else: + return None + +# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/instr_state_machine.py b/src/swell/utilities/gsi_record_parser.py similarity index 70% rename from src/swell/utilities/instr_state_machine.py rename to src/swell/utilities/gsi_record_parser.py index acc306db..b1c99ed4 100644 --- a/src/swell/utilities/instr_state_machine.py +++ b/src/swell/utilities/gsi_record_parser.py @@ -12,15 +12,15 @@ def check_end_times(end_times): - # Checks end times for 24 hour strings and converts them to 18 + ''' Checks end times for 24 hour strings and converts them to 18 ''' new_end_times = [] for end_time in end_times: - # note that python datetime does not allow for times with hour = 24 + # Note that python datetime does not allow for times with hour = 24 hour = end_time[8:10] if (hour == '24'): - # subtract 6 hours + # Subtract 6 hours tmp = int(end_time) - 60000 new_end_times.append(str(tmp)) else: @@ -29,28 +29,52 @@ def check_end_times(end_times): return new_end_times -class InstrStateMachine: +class GSIRecordParser: - def __init__(self, instr_df): + def __init__(self): - # Intakes a dataframe representing the rows for a particular - # instrument on a particular satellite. + ''' + This class employs a state machine algorithm to process raw data from GSI + .tbl files. The different states work together to parse through an initial + dataframe. The rows of the resulting dataframe correspond to clean entries + for a given instrument and satellite. + ''' + + self.idx = None + self.main_idx = None + self.start_times = None + self.end_times = None + self.instr_df = None + self.compare_channels = None + self.curr_channel_list = None + self.main_channel_list = None + self.return_df = None + + def reset(self): self.idx = 0 self.main_idx = 0 self.start_times = [] self.end_times = [] - self.instr_df = instr_df self.compare_channels = [] self.curr_channel_list = [] self.main_channel_list = [] - self.return_df = pd.DataFrame(columns=list(instr_df.columns.values)) - def run(self): + def run(self, instr_df): + + ''' + Effectively state one of the state machine. + Generates an ordered list of start times and corresponding + end times. Checks the end times and then proceeds to condition one. - # Effectively state one of the state machine. - # Generates an ordered list of start times and corresponding - # end times. Checks the end times and then proceeds to condition one. + input: + instr_df = a dataframe containing raw data for a given satellite and + instrument + ''' + + self.reset() + self.instr_df = instr_df + self.return_df = pd.DataFrame(columns=list(instr_df.columns.values)) self.start_times = list(np.unique(self.instr_df["start"].values)) self.start_times.sort(key=int) @@ -63,8 +87,10 @@ def run(self): def condition_one(self): - # If there is one row for the date range, go to state 2. Otherwise, - # go to state 3. + ''' + If there is one row for the date range, go to state 2. Otherwise, + go to state 3. + ''' start_time_df = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] n_curr_start_rows = len(start_time_df) @@ -78,8 +104,10 @@ def condition_one(self): def state_two(self): - # Update return_df with new row, increment idx, and then go to - # condition 4. + ''' + Update return_df with new row, increment idx, and then go to + condition 4. + ''' row = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] self.update_return_df(row) @@ -88,12 +116,14 @@ def state_two(self): def state_three(self): - # Gather channels for all rows in current datetime. If there are more - # than one end time within the rows, set main_start_idx and - # main_channels_list, set compare_channels to the current channel list, - # update return_df with a new row, increment index, - # and then go to condition 2. - # Else, go to state 2 with the updated current channel list. + ''' + Gather channels for all rows in current datetime. If there are more + than one end time within the rows, set main_start_idx and + main_channels_list, set compare_channels to the current channel list, + update return_df with a new row, increment index, + and then go to condition 2. + Else, go to state 2 with the updated current channel list. + ''' rows = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] [self.curr_channel_list.extend(i) for i in rows["channels"].values] @@ -122,10 +152,12 @@ def state_three(self): def condition_two(self): - # If curr start/end is in main range, go to condition 3. Otherwise - # go to state 2 + ''' + If curr start/end is in main range, go to condition 3. Otherwise + go to state 2 + ''' - # return if end of df is reached + # Return if end of df is reached if (self.idx == len(self.start_times)): return @@ -143,23 +175,25 @@ def condition_two(self): def state_four(self): - # Update current channel list by whether values need to be turned - # on or turned off. Then update the return df, increment the index - # and go to condition 2. + ''' + Update current channel list by whether values need to be turned + on or turned off. Then update the return df, increment the index + and go to condition 2. + ''' row = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] row_channel_list = row["channels"].values[0] - # if these are the same, the logic is off + # If these are the same, the logic is off assert (len(row_channel_list) != len(self.compare_channels)) self.curr_channel_list = self.main_channel_list if (len(row_channel_list) > len(self.compare_channels)): - # turn on + # Turn on turn_on = list(set(row_channel_list) - set(self.compare_channels)) self.curr_channel_list += turn_on else: - # turn off + # Turn off turn_off = list(set(self.compare_channels) - set(row_channel_list)) self.curr_channel_list = [x for x in self.curr_channel_list if x not in turn_off] @@ -170,9 +204,11 @@ def state_four(self): def condition_four(self): - # If next date range is right after previous date range, go to - # condition one. Otherwise, go to state 6. If there's no next - # then return. + ''' + If next date range is right after previous date range, go to + condition one. Otherwise, go to state 6. If there's no next + then return. + ''' assert (self.idx != 0) if (self.idx == len(self.start_times)): @@ -191,8 +227,10 @@ def condition_four(self): def state_six(self): - # Create new empty date range, update return df with new row, and then - # head over to state one. No update to the index. + ''' + Create new empty date range, update return df with new row, and then + head over to state one. No update to the index. + ''' missing_time = {} @@ -211,12 +249,14 @@ def state_six(self): def get_instr_df(self): - # Returns the dataframe that the state machine generated! + ''' Returns the dataframe that the state machine generated! ''' return self.return_df def update_return_df(self, row, no_comment=False, missing=False, missing_time={}): + ''' Adding rows to final dataframe that will be returned through get_instr_df ''' + # Updates the return df based on parameters if (missing): new_row = pd.DataFrame.from_dict({ diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 923f5998..238bd1a5 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -3,17 +3,83 @@ import pandas as pd import numpy as np import datetime as dt -from swell.utilities.sat_db_utils import read_sat_db -from swell.utilities.instr_state_machine import InstrStateMachine +from swell.utilities.gsi_record_parser import GSIRecordParser +# -------------------------------------------------------------------------------------------------- def format_date(old_date): + + ''' Formatting date into expected template ''' + date = dt.datetime.strptime(old_date, '%Y%m%d%H%M%S') return date.isoformat() +# -------------------------------------------------------------------------------------------------- + + +def read_sat_db(path_to_sat_db, column_names): + + ''' + Reading GSI observing system records row by row into + a pandas dataframe to be used by the gsi_record_parser + ''' + + filename = path_to_sat_db + df = pd.DataFrame(columns=column_names) + + file = open(filename, 'r') + lines = file.readlines() + + # Read blindly into an array, throw line away if it starts with # or newline + idx = 0 + for line in lines: + line_parts = line.split() + if (line_parts): + + if (line_parts[0][0] != '#' and line_parts[0][0] != '\n'): + new_row = pd.DataFrame.from_dict({ + 'sat': [''], + 'start': [''], + 'end': [''], + 'instr': [''], + 'channel_num': [0], + 'channels': [[]], + 'comments': ['']}) + + df = pd.concat([df, new_row], ignore_index=True) + df['sat'][idx] = line_parts[0] + df['start'][idx] = line_parts[1]+line_parts[2] + df['end'][idx] = line_parts[3]+line_parts[4] + df['instr'][idx] = line_parts[5] + df['channel_num'][idx] = line_parts[6] + + comment_present = next((i for i, x in enumerate(line_parts) if x == '#'), None) + + if (comment_present): + channel_list = line_parts[7:comment_present] + comment = line_parts[comment_present:] + comment_str = ' '.join(comment) + # Accounting for no comment + if (len(comment_str) != 1): + df['comments'][idx] = comment_str + else: + channel_list = line_parts[7:] + + df['channels'][idx] = channel_list + idx += 1 + return df + +# -------------------------------------------------------------------------------------------------- + class ObservingSystemRecords: + ''' + Class handles calls to parse GSI observing system records. Parsed + records are saved internally in dataframes and can be outputted into + yaml files. + ''' + def __init__(self): self.column_names = ['sat', 'start', 'end', 'instr', 'channel_num', @@ -24,6 +90,13 @@ def __init__(self): def parse_records(self, path_to_sat_db): + ''' + This method reads in the active.tbl and available.tbl files + from GEOSAna and loads them into dataframes. These dataframes + are parsed using GSIRecordParser to get the final dataframes. + ''' + + parser = GSIRecordParser() channel_types = ['active', 'available'] for channel_type in channel_types: df = pd.DataFrame(columns=self.column_names) @@ -37,9 +110,8 @@ def parse_records(self, path_to_sat_db): for instr in instr_list: instr_df = sat_df.loc[sat_df['instr'] == instr] - state_machine = InstrStateMachine(instr_df) - state_machine.run() - new_instr_df = state_machine.get_instr_df() + parser.run(instr_df) + new_instr_df = parser.get_instr_df() df = pd.concat([df, new_instr_df], ignore_index=True) if instr+'_'+sat not in self.obs_registry: self.obs_registry.append(instr+'_'+sat) @@ -49,10 +121,16 @@ def parse_records(self, path_to_sat_db): elif channel_type == 'available': self.available_df = df else: + # logger assert abort? print('record parsing unavailable for this type') def save_yamls(self, output_dir, observation_list=None): + ''' + Fields are taken from the internal dataframes populated + by parse_records and saved to yaml files. + ''' + if not observation_list: observation_list = self.obs_registry diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 8192368e..11f55808 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -11,7 +11,7 @@ import yaml from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.sat_db_utils import get_active_channels +from swell.utilities.get_active_channels import get_active_channels # -------------------------------------------------------------------------------------------------- @@ -30,6 +30,10 @@ def __init__(self, logger, experiment_root, experiment_id, cycle_dir, jedi_inter self.jedi_config_path = os.path.join(experiment_root, experiment_id, 'configuration', 'jedi') + # Fields needed for get_active_channels + self.cycle_time = None + self.observing_system_records_path = None + # Dictionary to hold things that can be templated self.__template_dict__ = {} @@ -152,32 +156,24 @@ def render_interface_model(self, config_name): # ---------------------------------------------------------------------------------------------- - # Prepare path to interface observations file and call rendering - def render_interface_observations(self, config_name, path_to_observing_sys_yamls, cycle_time): - - # Assert that there is a jedi interface associated with the task - self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + - f'jedi interface config file the task must have an associated' + - f'jedi interface.') + def set_observing_system_records_path(self, path): + self.observing_system_records_path = path - # Path to configuration file - config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, - 'observations', f'{config_name}.yaml') + # ---------------------------------------------------------------------------------------------- - # Get active channels - active_channels = get_active_channels(path_to_observing_sys_yamls, - config_name, cycle_time) + def set_cycle_time(self, cycle_time): + self.cycle_time = cycle_time - # Add active channels to template dictionary - self.__template_dict__[f'{config_name}_active_channels'] = active_channels + # ---------------------------------------------------------------------------------------------- - # Render templates in file and return dictionary - return self.__open_file_render_to_dict__(config_file) + # Prepare path to interface observations file and call rendering + def render_interface_observations(self, config_name): - # ---------------------------------------------------------------------------------------------- + # Check that observing_system_records_path and cycle_time are set + self.logger.assert_abort(self.cycle_time is None, f'cycle_time must be set.') + self.logger.assert_abort(self.observing_system_records_path is None, + f'observing_system_records_path must be set.') - def render_interface_ufo_test(self): - config_name = f'ufo_tests' # Assert that there is a jedi interface associated with the task self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + f'jedi interface config file the task must have an associated' + @@ -187,11 +183,22 @@ def render_interface_ufo_test(self): config_file = os.path.join(self.jedi_config_path, 'interfaces', self.jedi_interface, 'observations', f'{config_name}.yaml') + # If yaml is ufo_tests, skip get_active_channels + if config_name != 'ufo_tests': + + # Get active channels + active_channels = get_active_channels(self.observing_system_records_path, + config_name, self.cycle_time) + + # Add active channels to template dictionary + self.__template_dict__[f'{config_name}_active_channels'] = active_channels + # Render templates in file and return dictionary return self.__open_file_render_to_dict__(config_file) # ---------------------------------------------------------------------------------------------- + # Prepare path to interface metadata file and call rendering def render_interface_meta(self, model_component_in=None): diff --git a/src/swell/utilities/run_jedi_executables.py b/src/swell/utilities/run_jedi_executables.py index 6d6d2e85..f2d08ef5 100644 --- a/src/swell/utilities/run_jedi_executables.py +++ b/src/swell/utilities/run_jedi_executables.py @@ -17,7 +17,6 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, - observing_system_records_path, cycle_time, jedi_forecast_model=None): # Assemble configuration YAML file @@ -25,8 +24,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, for key, value in jedi_config_dict.items(): if isinstance(value, dict): jedi_dictionary_iterator(value, jedi_rendering, window_type, obs, - observing_system_records_path, - cycle_time, jedi_forecast_model) + jedi_forecast_model) elif isinstance(value, bool): continue @@ -36,8 +34,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, if isinstance(item, dict): jedi_dictionary_iterator( item, jedi_rendering, window_type, obs, - observing_system_records_path, - cycle_time, jedi_forecast_model + jedi_forecast_model ) else: @@ -52,11 +49,7 @@ def jedi_dictionary_iterator(jedi_config_dict, jedi_rendering, window_type, obs, if value_special == 'observations': observations = [] for ob in obs: - obs_dict = jedi_rendering.render_interface_observations( - ob, - observing_system_records_path, - cycle_time - ) + obs_dict = jedi_rendering.render_interface_observations(ob) observations.append(obs_dict) jedi_config_dict[key] = observations diff --git a/src/swell/utilities/sat_db_utils.py b/src/swell/utilities/sat_db_utils.py deleted file mode 100644 index fb3bf5dc..00000000 --- a/src/swell/utilities/sat_db_utils.py +++ /dev/null @@ -1,161 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -import yaml -import os -import pandas as pd -import numpy as np -from swell.utilities.git_utils import git_got -from swell.utilities.instr_state_machine import InstrStateMachine -from datetime import datetime as dt - -# -------------------------------------------------------------------------------------------------- - - -def process_channel_lists(channel_list): - final_channels_list = [] - if not isinstance(channel_list, list): - channel_list = [channel_list] - for element in channel_list: - if '-' in element: - start, end = map(int, element.split('-')) - result_list = [x for x in range(start, end + 1)] - final_channels_list += result_list - else: - final_channels_list += [int(element)] - - return final_channels_list - -# -------------------------------------------------------------------------------------------------- - - -def get_channel_list(input_dict, dt_cycle_time): - for element in input_dict: - begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") - end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") - if (dt_cycle_time > begin_date) and (dt_cycle_time < end_date): - return element['channels'] - -# -------------------------------------------------------------------------------------------------- - - -def get_active_channels(path_to_observing_sys_yamls, observation, cycle_time): - # Cycle time to datetime object - dt_cycle_time = dt.strptime(cycle_time, "%Y%m%dT%H%M%SZ") - - # Retrieve available and active channels from records yaml - path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + \ - observation + '_channel_info.yaml' - - if os.path.isfile(path_to_observing_sys_config): - with open(path_to_observing_sys_config, 'r') as file: - data = yaml.safe_load(file) - available_channels = get_channel_list(data['available'], dt_cycle_time) - active_channels = get_channel_list(data['active'], dt_cycle_time) - - available_channels_list = process_channel_lists(available_channels) - active_channels_list = process_channel_lists(active_channels) - use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] - - return use_flags - - else: - return None - -# -------------------------------------------------------------------------------------------------- - - -def read_sat_db(path_to_sat_db, column_names): - # read data into a dataframe, throw line away if it starts with # or newline - # --------------------------------------------------------------------------- - filename = path_to_sat_db - df = pd.DataFrame(columns=column_names) - - file = open(filename, 'r') - lines = file.readlines() - - # read blindly into an array, throw line away if it starts with # or newline - idx = 0 - for line in lines: - line_parts = line.split() - if (line_parts): - - if (line_parts[0][0] != '#' and line_parts[0][0] != '\n'): - new_row = pd.DataFrame.from_dict({ - 'sat': [''], - 'start': [''], - 'end': [''], - 'instr': [''], - 'channel_num': [0], - 'channels': [[]], - 'comments': ['']}) - - df = pd.concat([df, new_row], ignore_index=True) - df['sat'][idx] = line_parts[0] - df['start'][idx] = line_parts[1]+line_parts[2] - df['end'][idx] = line_parts[3]+line_parts[4] - df['instr'][idx] = line_parts[5] - df['channel_num'][idx] = line_parts[6] - - comment_present = next((i for i, x in enumerate(line_parts) if x == '#'), None) - - if (comment_present): - channel_list = line_parts[7:comment_present] - comment = line_parts[comment_present:] - comment_str = ' '.join(comment) - # accounting for no comment - if (len(comment_str) != 1): - df['comments'][idx] = comment_str - else: - channel_list = line_parts[7:] - - df['channels'][idx] = channel_list - idx += 1 - return df - - -# -------------------------------------------------------------------------------------------------- - - -def run_sat_db_process(git_out_dir, logger): - - # Process satellite database files in GEOS-ESM and return dataframe - # ----------------------------------------------------------------------- - - git_repo = 'https://github.com/GEOS-ESM/GEOSana_GridComp.git' - git_branch = 'develop' - git_out_path = os.path.join(git_out_dir, 'GEOSana_GridComp') - - # clone repo - git_got(git_repo, git_branch, git_out_path, logger) - path_to_sat_db = os.path.join(git_out_path, 'GEOSaana_GridComp', 'GSI_GridComp', 'mksi', 'sidb') - - column_names = ['sat', 'start', 'end', 'instr', 'channel_num', - 'channels', 'comments'] - df = read_sat_db(path_to_sat_db, column_names) - final_df = pd.DataFrame(columns=column_names) - - sat_list = np.unique(df['sat'].values) - for sat in sat_list: - sat_df = df.loc[df['sat'] == sat] - - instr_list = np.unique(sat_df['instr'].values) - - for instr in instr_list: - instr_df = sat_df.loc[sat_df['instr'] == instr] - - state_machine = InstrStateMachine(instr_df) - state_machine.run() - new_instr_df = state_machine.get_instr_df() - final_df = final_df.append(new_instr_df) - - return final_df - - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/scripts/swell_sat_db_processing.py b/src/swell/utilities/scripts/swell_sat_db_processing.py deleted file mode 100644 index 62cafa8c..00000000 --- a/src/swell/utilities/scripts/swell_sat_db_processing.py +++ /dev/null @@ -1,92 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -import os -import yaml -import click -import numpy as np - -from datetime import datetime as dt - -from swell.utilities.sat_db_utils import run_sat_db_process - - -def make_yamls(final_df, output_dir): - - ''' - Uses the dataframe created by sat_db_processing - to write out yaml files - ''' - - if not os.path.exists(output_dir): - os.mkdir(output_dir) - - sat_list = np.unique(final_df['sat'].values) - for sat in sat_list: - - df = final_df.loc[final_df['sat'] == sat] - instr_list = np.unique(df['instr'].values) - sat_dict = {} - - for instr in instr_list: - - sat_dict[instr] = {} - instr_df = df.loc[df['instr'] == instr] - - field_list = [] - for idx, row in instr_df.iterrows(): - - row_dict = {} - row_dict['begin date'] = format_date(row['start']) - row_dict['end date'] = format_date(row['end']) - row_dict['channels'] = row['channels'] - - if (row['comments']): - row_dict['comments'] = row['comments'] - else: - row_dict['comments'] = 'no comment' - - field_list.append(row_dict) - - sat_dict[instr] = field_list - - with open(output_dir+'/'+sat+'.yaml', 'w') as file: - yaml.dump(sat_dict, file) - - -def format_date(old_date): - - ''' - Formatting datetime object - ''' - - date = dt.strptime(old_date, '%Y%m%d%H%M%S') - return date.isoformat() - - -@click.command() -@click.argument('config') -def main(config): - - with open(config, 'r') as ymlfile: - config_dict = yaml.safe_load(ymlfile) - user = os.environ['USER'] - geos_sat_db_root = config_dict['geos_sat_db_root'].replace('${USER}', user) - - try: - os.makedirs(geos_sat_db_root) - except Exception: - print('SATELLITE DATABASE DIRECTORY IS ALREADY GENERATED') - - yaml_out_dir = geos_sat_db_root + '/satdb_yamls' - git_out_dir = geos_sat_db_root + '/GEOSana_GridComp' - - # run sat db processing util - processed_data = run_sat_db_process(git_out_dir) - - # create yamls - make_yamls(processed_data, yaml_out_dir) From 56df58876ea35a93b0b028f3410e45dfbf422b63 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 6 Nov 2023 16:59:20 -0500 Subject: [PATCH 23/38] coding norms --- src/swell/tasks/eva_observations.py | 2 +- src/swell/tasks/get_observations.py | 2 +- src/swell/tasks/gsi_bc_to_ioda.py | 2 +- src/swell/tasks/run_jedi_hofx_executable.py | 2 +- src/swell/tasks/run_jedi_local_ensemble_da_executable.py | 2 +- src/swell/tasks/run_jedi_ufo_tests_executable.py | 4 ++-- src/swell/tasks/run_jedi_variational_executable.py | 2 +- src/swell/tasks/save_obs_diags.py | 2 +- src/swell/tasks/task_questions.yaml | 2 -- src/swell/utilities/gsi_record_parser.py | 2 +- src/swell/utilities/observing_system_records.py | 1 + src/swell/utilities/render_jedi_interface_files.py | 6 +++--- 12 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index e612de09..926f8348 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -90,7 +90,7 @@ def execute(self): observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') cycle_time = self.cycle_time_dto() - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index f7d4ee43..97cc7a1b 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -48,7 +48,7 @@ def execute(self): observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') cycle_time = self.cycle_time_dto() - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index ac35b529..8b298225 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -53,7 +53,7 @@ def execute(self): sensors_satbias = [] sensors_tlapse = [] - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index 0701a7b3..8e4d21bb 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -91,7 +91,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index e674624c..5c7931d5 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -42,7 +42,7 @@ def execute(self): observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') cycle_time = self.cycle_time_dto() - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index 80effe46..e88de39e 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -44,7 +44,7 @@ def execute(self): observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') cycle_time = self.cycle_time_dto() - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) @@ -86,7 +86,7 @@ def execute(self): # Open the ufo_tests config file # ------------------------------ - ufo_tests_dict = self.jedi_rendering.render_interface_ufo_test() + ufo_tests_dict = self.jedi_rendering.render_interface_observations(f'ufo_tests') ufo_tests_default = ufo_tests_dict['default'] # Insert the GeoVaLs section diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index d26deadc..6482ea03 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -105,7 +105,7 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index de203829..d2424322 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -34,7 +34,7 @@ def execute(self): observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') cycle_time = self.cycle_time_dto() - # Set cycle time and observing_system_records_path in jedi_rendering + # Set cycle time and observing_system_records_path in jedi_rendering self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) self.jedi_rendering.set_cycle_time(cycle_time) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index e8d69203..234f66dc 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -851,8 +851,6 @@ window_type: observing_system_records_gsi_path: ask_question: false default_value: None - models: - - all prompt: What is the path to the observing system records ? tasks: - GenerateObservingSystemRecords diff --git a/src/swell/utilities/gsi_record_parser.py b/src/swell/utilities/gsi_record_parser.py index b1c99ed4..f51c47c0 100644 --- a/src/swell/utilities/gsi_record_parser.py +++ b/src/swell/utilities/gsi_record_parser.py @@ -33,7 +33,7 @@ class GSIRecordParser: def __init__(self): - ''' + ''' This class employs a state machine algorithm to process raw data from GSI .tbl files. The different states work together to parse through an initial dataframe. The rows of the resulting dataframe correspond to clean entries diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 238bd1a5..3cf6eccd 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -7,6 +7,7 @@ # -------------------------------------------------------------------------------------------------- + def format_date(old_date): ''' Formatting date into expected template ''' diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 11f55808..f87e52ba 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -170,8 +170,8 @@ def set_cycle_time(self, cycle_time): def render_interface_observations(self, config_name): # Check that observing_system_records_path and cycle_time are set - self.logger.assert_abort(self.cycle_time is None, f'cycle_time must be set.') - self.logger.assert_abort(self.observing_system_records_path is None, + self.logger.assert_abort(self.cycle_time is not None, f'cycle_time must be set.') + self.logger.assert_abort(self.observing_system_records_path is not None, f'observing_system_records_path must be set.') # Assert that there is a jedi interface associated with the task @@ -198,8 +198,8 @@ def render_interface_observations(self, config_name): # ---------------------------------------------------------------------------------------------- - # Prepare path to interface metadata file and call rendering + def render_interface_meta(self, model_component_in=None): # Optionally open a different model interface From 7ad17022f5fb522c1bcdc30d959d0ff18b0ec15c Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 8 Nov 2023 09:22:40 -0500 Subject: [PATCH 24/38] fix test --- src/swell/utilities/get_active_channels.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/swell/utilities/get_active_channels.py b/src/swell/utilities/get_active_channels.py index 4083a0d2..2a6ffb65 100644 --- a/src/swell/utilities/get_active_channels.py +++ b/src/swell/utilities/get_active_channels.py @@ -9,8 +9,6 @@ import yaml import os -import pandas as pd -import numpy as np from datetime import datetime as dt # -------------------------------------------------------------------------------------------------- From e7a03a491a6ea2eaae49f7c69b1462253d1149cf Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 8 Nov 2023 10:04:42 -0500 Subject: [PATCH 25/38] fix observing sys test --- src/swell/test/code_tests/generate_observing_system_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/swell/test/code_tests/generate_observing_system_test.py b/src/swell/test/code_tests/generate_observing_system_test.py index 2ed26615..ec7e34f1 100644 --- a/src/swell/test/code_tests/generate_observing_system_test.py +++ b/src/swell/test/code_tests/generate_observing_system_test.py @@ -3,8 +3,10 @@ observations = ['amsua_n19'] observing_system_records_path = './yaml_output/' -path_to_geosana_gridcomp = '/discover/nobackup/asewnath/github/GEOSana_GridComp/' -path_to_gsi_records = os.path.join(path_to_geosana_gridcomp, 'GEOSaana_GridComp', + +# Clone GeosAna +os.system('git clone https://github.com/GEOS-ESM/GEOSana_GridComp.git') +path_to_gsi_records = os.path.join('GEOSana_GridComp/', 'GEOSaana_GridComp', 'GSI_GridComp', 'mksi', 'sidb') sat_records = ObservingSystemRecords() sat_records.parse_records(path_to_gsi_records) From 88125518c00286ff6be6497bdecd7f1501901748 Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 8 Nov 2023 10:42:13 -0500 Subject: [PATCH 26/38] task question test fix --- src/swell/tasks/task_questions.yaml | 52 +++++++++++++------------ src/swell/test/code_tests/code_tests.py | 2 +- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 234f66dc..12d7d001 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -588,6 +588,7 @@ observations: prompt: Which observations do you want to include? tasks: - EvaObservations + - GenerateObservingSystemRecords - GetGeovals - GetObservations - GsiBcToIoda @@ -600,6 +601,32 @@ observations: - GenerateObservingSystemRecords type: string-check-list +observing_system_records_gsi_path: + ask_question: false + default_value: None + prompt: What is the path to the observing system records? + tasks: + - GenerateObservingSystemRecords + - CloneGeosAna + type: string + +observing_system_records_path: + ask_question: false + default_value: None + models: + - all + prompt: What is the path to the observing system records? + tasks: + - GenerateObservingSystemRecords + - GetObservations + - GsiBcToIoda + - RunJediUfoTestsExecutable + - RunJediHofxExecutable + - RunJediVariationalExecutable + - RunJediLocalEnsembleDaExecutable + - EvaObservations + type: string + path_to_ensemble: ask_question: true default_value: defer_to_model @@ -848,28 +875,3 @@ window_type: - StoreBackground type: string-drop-list -observing_system_records_gsi_path: - ask_question: false - default_value: None - prompt: What is the path to the observing system records ? - tasks: - - GenerateObservingSystemRecords - - CloneGeosAna - type: string - -observing_system_records_path: - ask_question: false - default_value: None - models: - - all - prompt: What is the path to the observing system records? - tasks: - - GenerateObservingSystemRecords - - GetObservations - - GsiBcToIoda - - RunJediUfoTestsExecutable - - RunJediHofxExecutable - - RunJediVariationalExecutable - - RunJediLocalEnsembleDaExecutable - - EvaObservations - type: string diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index 47bf5311..09978f26 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -25,7 +25,7 @@ def code_tests(): logger.test('Running Swell Test Suite') # Turn off the regular info testing - os.environ["LOG_INFO"] = "0" # Set this to 1 when errors are being debugged + os.environ["LOG_INFO"] = "1" # Set this to 1 when errors are being debugged # Create a test suite test_suite = unittest.TestSuite() From 33c616da0f4df86dcd556db2af640a09d41f7570 Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 8 Nov 2023 10:57:49 -0500 Subject: [PATCH 27/38] code test fix --- src/swell/tasks/task_questions.yaml | 12 ++++++------ src/swell/test/code_tests/code_tests.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/swell/tasks/task_questions.yaml b/src/swell/tasks/task_questions.yaml index 12d7d001..38a60c2d 100644 --- a/src/swell/tasks/task_questions.yaml +++ b/src/swell/tasks/task_questions.yaml @@ -598,7 +598,6 @@ observations: - RunJediUfoTestsExecutable - RunJediVariationalExecutable - SaveObsDiags - - GenerateObservingSystemRecords type: string-check-list observing_system_records_gsi_path: @@ -606,8 +605,8 @@ observing_system_records_gsi_path: default_value: None prompt: What is the path to the observing system records? tasks: - - GenerateObservingSystemRecords - CloneGeosAna + - GenerateObservingSystemRecords type: string observing_system_records_path: @@ -617,15 +616,16 @@ observing_system_records_path: - all prompt: What is the path to the observing system records? tasks: + - EvaObservations - GenerateObservingSystemRecords - GetObservations - GsiBcToIoda - - RunJediUfoTestsExecutable - RunJediHofxExecutable - - RunJediVariationalExecutable - RunJediLocalEnsembleDaExecutable - - EvaObservations - type: string + - RunJediUfoTestsExecutable + - RunJediVariationalExecutable + - SaveObsDiags + type: string path_to_ensemble: ask_question: true diff --git a/src/swell/test/code_tests/code_tests.py b/src/swell/test/code_tests/code_tests.py index 09978f26..47bf5311 100644 --- a/src/swell/test/code_tests/code_tests.py +++ b/src/swell/test/code_tests/code_tests.py @@ -25,7 +25,7 @@ def code_tests(): logger.test('Running Swell Test Suite') # Turn off the regular info testing - os.environ["LOG_INFO"] = "1" # Set this to 1 when errors are being debugged + os.environ["LOG_INFO"] = "0" # Set this to 1 when errors are being debugged # Create a test suite test_suite = unittest.TestSuite() From 6d33f16618674ec737d5e3b36fb973a81023f9b6 Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 9 Nov 2023 10:34:36 -0500 Subject: [PATCH 28/38] stash --- .../geos_atmosphere/observations/amsua_n19.yaml | 2 +- src/swell/test/code_tests/get_active_channels_test.py | 8 ++++---- .../{get_active_channels.py => get_channels.py} | 4 ++-- src/swell/utilities/render_jedi_interface_files.py | 10 ++++++---- 4 files changed, 13 insertions(+), 11 deletions(-) rename src/swell/utilities/{get_active_channels.py => get_channels.py} (95%) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml index fe1f2edd..3ebfc6bf 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n19.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n19.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_n19_available_channels 1-15 + channels: &amsua_n19_available_channels {{amsua_n19_avail_channels}} obs operator: name: CRTM diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 4f61ce02..8d5ca8e1 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -1,12 +1,12 @@ -from swell.utilities.get_active_channels import get_active_channels +from swell.utilities.get_channels import get_channels +from datetime import datetime as dt use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] cycle_time = '20211212T000000Z' +dt_cycle_time = dt.strptime(cycle_time, '%Y%m%dT%H%M%SZ') path_to_observing_sys_yamls = 'active_channels_test_files/' -path_to_configs = 'active_channels_test_files/' observation = 'amsua_n19' -generated_use_flags = get_active_channels(path_to_observing_sys_yamls, - path_to_configs, observation, cycle_time) +avail, generated_use_flags = get_channels(path_to_observing_sys_yamls,observation, dt_cycle_time) assert (use_flags == generated_use_flags) diff --git a/src/swell/utilities/get_active_channels.py b/src/swell/utilities/get_channels.py similarity index 95% rename from src/swell/utilities/get_active_channels.py rename to src/swell/utilities/get_channels.py index 2a6ffb65..f1938924 100644 --- a/src/swell/utilities/get_active_channels.py +++ b/src/swell/utilities/get_channels.py @@ -51,7 +51,7 @@ def get_channel_list(input_dict, dt_cycle_time): # -------------------------------------------------------------------------------------------------- -def get_active_channels(path_to_observing_sys_yamls, observation, dt_cycle_time): +def get_channels(path_to_observing_sys_yamls, observation, dt_cycle_time): ''' Comparing available channels and active channels from the observing @@ -73,7 +73,7 @@ def get_active_channels(path_to_observing_sys_yamls, observation, dt_cycle_time) active_channels_list = process_channel_lists(active_channels) use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] - return use_flags + return available_channels_list, use_flags else: return None diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index f87e52ba..27bdc2e0 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -11,7 +11,7 @@ import yaml from swell.utilities.jinja2 import template_string_jinja2 -from swell.utilities.get_active_channels import get_active_channels +from swell.utilities.get_channels import get_channels # -------------------------------------------------------------------------------------------------- @@ -186,11 +186,13 @@ def render_interface_observations(self, config_name): # If yaml is ufo_tests, skip get_active_channels if config_name != 'ufo_tests': - # Get active channels - active_channels = get_active_channels(self.observing_system_records_path, + + # Get available and active channels + available_channels, active_channels = get_channels(self.observing_system_records_path, config_name, self.cycle_time) - # Add active channels to template dictionary + # Add available and active channels to template dictionary + self.__template_dict__[f'{config_name}_avail_channels'] = available_channels self.__template_dict__[f'{config_name}_active_channels'] = active_channels # Render templates in file and return dictionary From 1cf00b814841adf9a914e4394d70c4d2bec18265 Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 16 Nov 2023 16:23:46 -0500 Subject: [PATCH 29/38] stash --- .../observations/airs_aqua.yaml | 54 +-------- .../observations/amsua_aqua.yaml | 10 +- .../observations/amsua_metop-b.yaml | 10 +- .../observations/amsua_metop-c.yaml | 10 +- .../observations/amsua_n15.yaml | 10 +- .../observations/amsua_n18.yaml | 10 +- .../observations/atms_n20.yaml | 8 +- .../observations/atms_npp.yaml | 14 +-- .../observations/avhrr3_metop-b.yaml | 4 +- .../observations/avhrr3_n18.yaml | 4 +- .../observations/avhrr3_n19.yaml | 4 +- .../observations/cris-fsr_n20.yaml | 79 +----------- .../observations/cris-fsr_npp.yaml | 79 +----------- .../observations/iasi_metop-b.yaml | 113 +----------------- .../observations/iasi_metop-c.yaml | 113 +----------------- .../observations/ssmis_f17.yaml | 6 +- src/swell/utilities/get_channels.py | 20 +++- 17 files changed, 57 insertions(+), 491 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml index cb8ab800..f6207166 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/airs_aqua.yaml @@ -9,29 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.airs_aqua.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &airs_aqua_channels - 1, 6, 7, 10, 11, 15, 16, 17, 20, 21, 22, 24, - 27, 28, 30, 36, 39, 40, 42, 51, 52, 54, 55, 56, 59, 62, 63, 68, 69, 71, - 72, 73, 74, 75, 76, 77, 78, 79, 80, 82, 83, 84, 86, 92, 93, 98, 99, 101, - 104, 105, 108, 110, 111, 113, 116, 117, 123, 124, 128, 129, 138, 139, - 144, 145, 150, 151, 156, 157, 159, 162, 165, 168, 169, 170, 172, 173, - 174, 175, 177, 179, 180, 182, 185, 186, 190, 192, 198, 201, 204, 207, - 210, 215, 216, 221, 226, 227, 232, 252, 253, 256, 257, 261, 262, 267, - 272, 295, 299, 300, 305, 310, 321, 325, 333, 338, 355, 362, 375, 453, - 475, 484, 497, 528, 587, 672, 787, 791, 843, 870, 914, 950, 1003, 1012, - 1019, 1024, 1030, 1038, 1048, 1069, 1079, 1082, 1083, 1088, 1090, 1092, - 1095, 1104, 1111, 1115, 1116, 1119, 1120, 1123, 1130, 1138, 1142, 1178, - 1199, 1206, 1221, 1237, 1252, 1260, 1263, 1266, 1285, 1301, 1304, 1329, - 1371, 1382, 1415, 1424, 1449, 1455, 1466, 1477, 1500, 1519, 1538, 1545, - 1565, 1574, 1583, 1593, 1614, 1627, 1636, 1644, 1652, 1669, 1674, 1681, - 1694, 1708, 1717, 1723, 1740, 1748, 1751, 1756, 1763, 1766, 1771, 1777, - 1780, 1783, 1794, 1800, 1803, 1806, 1812, 1826, 1843, 1852, 1865, 1866, - 1868, 1869, 1872, 1873, 1876, 1881, 1882, 1883, 1911, 1917, 1918, 1924, - 1928, 1937, 1941, 2099, 2100, 2101, 2103, 2104, 2106, 2107, 2108, 2109, - 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, 2118, 2119, 2120, 2121, - 2122, 2123, 2128, 2134, 2141, 2145, 2149, 2153, 2164, 2189, 2197, 2209, - 2226, 2234, 2280, 2318, 2321, 2325, 2328, 2333, 2339, 2348, 2353, 2355, - 2357, 2363, 2370, 2371, 2377 + channels: &airs_aqua_channels {{airs_aqua_avail_channels}} obs operator: name: CRTM @@ -194,35 +172,7 @@ obs post filters: channels: *airs_aqua_channels options: channels: *airs_aqua_channels - use_flag: &useflag_airs_aqua [ -1, -1, 1, -1, -1, 1, -1, -1, 1, 1, - 1, -1, 1, 1, -1, -1, -1, 1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, -1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, -1, 1, 1, - -1, -1, 1, 1, 1, 1, -1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, -1, 1, 1, 1, - -1, 1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1, 1, 1, 1, -1, -1, - 1, 1, -1, -1, -1, 1, -1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, 1, -1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1 ] + use_flag: &useflag_airs_aqua {{airs_aqua_active_channels}} use_flag_clddet: &clddet_airs_aqua [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml index 510b8dca..39ec959f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_aqua.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_aqua.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_aqua_available_channels 1-15 + channels: &amsua_aqua_available_channels {{amsua_aqua_avail_channels}} obs operator: name: CRTM @@ -189,9 +189,7 @@ obs post filters: channels: *amsua_aqua_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1] + use_flag: &amsua_aqua_use_flag {{amsua_aqua_active_channels}} maxvalue: 1.0e-12 action: name: reject @@ -205,9 +203,7 @@ obs post filters: channels: *amsua_aqua_available_channels options: channels: *amsua_aqua_available_channels - use_flag: [-1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1] + use_flag: *amsua_aqua_use_flag minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml index 9539ae6a..fcba927c 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_metop-b_available_channels 1-15 + channels: &amsua_metop-b_available_channels {{amsua_metop-b_avail_channels}} obs operator: name: CRTM @@ -191,9 +191,7 @@ obs post filters: channels: *amsua_metop-b_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, -1, -1, - -1, 1, 1, 1, 1, - 1, 1, 1, 1, -1] + use_flag: &amsua_metop-b_use_flag {{amsua_metop-b_active_channels}} maxvalue: 1.0e-12 action: name: reject @@ -207,9 +205,7 @@ obs post filters: channels: *amsua_metop-b_available_channels options: channels: *amsua_metop-b_available_channels - use_flag: [-1, -1, -1, -1, -1, - -1, 1, 1, 1, 1, - 1, 1, 1, 1, -1] + use_flag: *amsua_metop-b_use_flag minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml index 01ff016b..c4bca511 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-c.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_metop-c_available_channels 1-15 + channels: &amsua_metop-c_available_channels {{amsua_metop-c_avail_channels}} obs operator: name: CRTM @@ -191,9 +191,7 @@ obs post filters: channels: *amsua_metop-c_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1] + use_flag: &amsua_metop-c_use_flag {{amsua_metop-c_active_channels}} maxvalue: 1.0e-12 action: name: reject @@ -207,9 +205,7 @@ obs post filters: channels: *amsua_metop-c_available_channels options: channels: *amsua_metop-c_available_channels - use_flag: [-1, -1, -1, 1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, -1] + use_flag: *amsua_metop-c_use_flag minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml index 36b20320..a3d42055 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n15.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n15.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_n15_available_channels 1-15 + channels: &amsua_n15_available_channels {{amsua_n15_avail_channels}} obs operator: name: CRTM @@ -191,9 +191,7 @@ obs post filters: channels: *amsua_n15_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, 1, 1, - -1, 1, 1, 1, 1, - -1, 1, 1, -1, -1] + use_flag: &amsua_n15_use_flag {{amsua_n15_active_channels}} maxvalue: 1.0e-12 action: name: reject @@ -207,9 +205,7 @@ obs post filters: channels: *amsua_n15_available_channels options: channels: *amsua_n15_available_channels - use_flag: [-1, -1, -1, 1, 1, - -1, 1, 1, 1, 1, - -1, 1, 1, -1, -1] + use_flag: *amsua_n15_use_flag minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml index f852b5ad..49c7bcb1 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_n18.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n18.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_n18_available_channels 1-15 + channels: &amsua_n18_available_channels {{amsua_n18_avail_channels}} obs operator: name: CRTM @@ -191,9 +191,7 @@ obs post filters: channels: *amsua_n18_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, 1, -1, - 1, 1, -1, -1, 1, - 1, 1, 1, 1, -1] + use_flag: &amsua_n18_use_flag {{amsua_n18_active_channels}} maxvalue: 1.0e-12 action: name: reject @@ -207,9 +205,7 @@ obs post filters: channels: *amsua_n18_available_channels options: channels: *amsua_n18_available_channels - use_flag: [-1, -1, -1, 1, -1, - 1, 1, -1, -1, 1, - 1, 1, 1, 1, -1] + use_flag: *amsua_n18_use_flag minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml index 5448a797..f7b7953b 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_n20.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.atms_n20.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &atms_n20_channels 1-22 + channels: &atms_n20_channels {{atms_n20_avail_channels}} obs operator: name: CRTM @@ -195,11 +195,7 @@ obs post filters: channels: *atms_n20_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - -1, 1, 1, 1, 1, - 1, 1] + use_flag: &atms_n20_use_flag {{atms_n20_active_channels}} maxvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml index 995154b1..eadcb201 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/atms_npp.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.atms_npp.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &atms_npp_channels 1-22 + channels: &atms_npp_channels {{atms_npp_avail_channels}} obs operator: name: CRTM @@ -195,11 +195,7 @@ obs post filters: channels: *atms_npp_channels use passive_bc: true sensor: *Sensor_ID - use_flag: [-1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - -1, 1, 1, 1, 1, - 1, 1] + use_flag: &atms_npp_use_flag {{atms_npp_active_channels}} maxvalue: 1.0e-12 action: name: reject @@ -213,11 +209,7 @@ obs post filters: channels: *atms_npp_channels options: channels: *atms_npp_channels - use_flag: [-1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, - -1, 1, 1, 1, 1, - 1, 1] + use_flag: *atms_npp_use_flag minvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml index 3936f771..45111cbf 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &avhrr3_metop-b_channels 3,4,5 + channels: &avhrr3_metop-b_channels {{avhrr3_metop-b_avail_channels}} obs operator: name: CRTM @@ -141,7 +141,7 @@ obs post filters: channels: *avhrr3_metop-b_channels options: channels: *avhrr3_metop-b_channels - use_flag: &useflag_avhrr3_metop-b [ -1, 1, 1 ] + use_flag: &useflag_avhrr3_metop-b {{avhrr3_metop-b_active_channels}} use_flag_clddet: [ 1, 1, 1 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml index b5567649..9b3ddd3f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n18.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_n18.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &avhrr3_n18_channels 3,4,5 + channels: &avhrr3_n18_channels {{avhrr3_n18_avail_channels}} obs operator: name: CRTM @@ -141,7 +141,7 @@ obs post filters: channels: *avhrr3_n18_channels options: channels: *avhrr3_n18_channels - use_flag: &useflag_avhrr3_n18 [ -1, 1, 1 ] + use_flag: &useflag_avhrr3_n18 {{avhrr3_n18_active_channels}} use_flag_clddet: [ 1, 1, 1 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml index 6085288e..133ded72 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_n19.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_n19.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &avhrr3_n19_channels 3,4,5 + channels: &avhrr3_n19_channels {{avhrr3_n19_avail_channels}} obs operator: name: CRTM @@ -141,7 +141,7 @@ obs post filters: channels: *avhrr3_n19_channels options: channels: *avhrr3_n19_channels - use_flag: &useflag_avhrr3_n19 [ -1, 1, 1 ] + use_flag: &useflag_avhrr3_n19 {{avhrr3_n19_active_channels}} use_flag_clddet: [ 1, 1, 1 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml index d4c152d0..ffc4dcf0 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -9,39 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.cris-fsr_n20.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &cris-fsr_n20_channels - 19, 24, 26, 27, 28, 31, 32, 33, 37, 39, 42, 44, 47, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, - 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 208, - 211, 216, 224, 234, 236, 238, 239, 242, 246, 248, 255, 264, 266, 268, - 275, 279, 283, 285, 291, 295, 301, 305, 311, 332, 342, 389, 400, 402, - 404, 406, 410, 427, 439, 440, 441, 445, 449, 455, 458, 461, 464, 467, - 470, 473, 475, 482, 486, 487, 490, 493, 496, 499, 501, 503, 505, 511, - 513, 514, 518, 519, 520, 522, 529, 534, 563, 568, 575, 592, 594, 596, - 598, 600, 602, 604, 611, 614, 616, 618, 620, 622, 626, 631, 638, 646, - 648, 652, 659, 673, 675, 678, 684, 688, 694, 700, 707, 710, 713, 714, - 718, 720, 722, 725, 728, 735, 742, 748, 753, 762, 780, 784, 798, 849, - 860, 862, 866, 874, 882, 890, 898, 906, 907, 908, 914, 937, 972, 973, - 978, 980, 981, 988, 995, 998, 1000, 1003, 1008, 1009, 1010, 1014, 1017, - 1018, 1020, 1022, 1024, 1026, 1029, 1030, 1032, 1034, 1037, 1038, 1041, - 1042, 1044, 1046, 1049, 1050, 1053, 1054, 1058, 1060, 1062, 1064, 1066, - 1069, 1076, 1077, 1080, 1086, 1091, 1095, 1101, 1109, 1112, 1121, 1128, - 1133, 1163, 1172, 1187, 1189, 1205, 1211, 1219, 1231, 1245, 1271, 1289, - 1300, 1313, 1316, 1325, 1329, 1346, 1347, 1473, 1474, 1491, 1499, 1553, - 1570, 1596, 1602, 1619, 1624, 1635, 1939, 1940, 1941, 1942, 1943, 1944, - 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, - 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, - 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, - 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, - 2158, 2161, 2168, 2171, 2175, 2182 + channels: &cris-fsr_n20_channels {{cris-fsr_n20_avail_channels}} obs operator: name: CRTM @@ -217,50 +185,7 @@ obs post filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels - use_flag: &useflag_cris-fsr_n20 [ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, -1, 1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, 1, 1, -1, 1, -1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1] + use_flag: &useflag_cris-fsr_n20 {{cris-fsr_n20_active_channels}} use_flag_clddet: &clddet_cris-fsr_n20 [ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml index 990f500c..7559eb46 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -9,39 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.cris-fsr_npp.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &cris-fsr_npp_channels - 19, 24, 26, 27, 28, 31, 32, 33, 37, 39, 42, 44, 47, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, - 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, - 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, - 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, - 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, - 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, - 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, - 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, - 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 208, - 211, 216, 224, 234, 236, 238, 239, 242, 246, 248, 255, 264, 266, 268, - 275, 279, 283, 285, 291, 295, 301, 305, 311, 332, 342, 389, 400, 402, - 404, 406, 410, 427, 439, 440, 441, 445, 449, 455, 458, 461, 464, 467, - 470, 473, 475, 482, 486, 487, 490, 493, 496, 499, 501, 503, 505, 511, - 513, 514, 518, 519, 520, 522, 529, 534, 563, 568, 575, 592, 594, 596, - 598, 600, 602, 604, 611, 614, 616, 618, 620, 622, 626, 631, 638, 646, - 648, 652, 659, 673, 675, 678, 684, 688, 694, 700, 707, 710, 713, 714, - 718, 720, 722, 725, 728, 735, 742, 748, 753, 762, 780, 784, 798, 849, - 860, 862, 866, 874, 882, 890, 898, 906, 907, 908, 914, 937, 972, 973, - 978, 980, 981, 988, 995, 998, 1000, 1003, 1008, 1009, 1010, 1014, 1017, - 1018, 1020, 1022, 1024, 1026, 1029, 1030, 1032, 1034, 1037, 1038, 1041, - 1042, 1044, 1046, 1049, 1050, 1053, 1054, 1058, 1060, 1062, 1064, 1066, - 1069, 1076, 1077, 1080, 1086, 1091, 1095, 1101, 1109, 1112, 1121, 1128, - 1133, 1163, 1172, 1187, 1189, 1205, 1211, 1219, 1231, 1245, 1271, 1289, - 1300, 1313, 1316, 1325, 1329, 1346, 1347, 1473, 1474, 1491, 1499, 1553, - 1570, 1596, 1602, 1619, 1624, 1635, 1939, 1940, 1941, 1942, 1943, 1944, - 1945, 1946, 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, 1955, 1956, - 1957, 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, 1966, 1967, 1968, - 1969, 1970, 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, 1979, 1980, - 1981, 1982, 1983, 1984, 1985, 1986, 1987, 2119, 2140, 2143, 2147, 2153, - 2158, 2161, 2168, 2171, 2175, 2182 + channels: &cris-fsr_npp_channels {{cris-fsr_npp_avail_channels}} obs operator: name: CRTM @@ -217,50 +185,7 @@ obs post filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels - use_flag: &useflag_cris-fsr_npp [ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, -1, 1, 1, -1, 1, -1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, 1, -1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, 1, 1, 1, 1, 1, -1, - 1, 1, 1, -1, 1, -1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, 1, 1, -1, -1, - -1, -1, 1, 1, -1, 1, -1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 1, 1, 1, 1, 1, 1, 1, 1, -1, 1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, 1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, 1, - -1, -1, 1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 1, 1, -1, -1, -1, -1, -1, -1, 1, - -1, -1, -1, -1, -1, -1, 1, -1, -1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1] + use_flag: &useflag_cris-fsr_npp {{cris-fsr_npp_active_channels}} use_flag_clddet: &clddet_cris-fsr_npp [ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml index 08e4db42..bf32c2ee 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml @@ -9,55 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &iasi_metop-b_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, - 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, - 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, - 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, - 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, - 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, - 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, - 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, - 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, - 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, - 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, - 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, - 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, - 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, - 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, - 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, - 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, - 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, - 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, - 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, - 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, - 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, - 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, - 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, - 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, - 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, - 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, - 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, - 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, - 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, - 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, - 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, - 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, - 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, - 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, - 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, - 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, - 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, - 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, - 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, - 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, - 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, - 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, - 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, - 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, - 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, - 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, - 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, - 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + channels: &iasi_metop-b_channels {{iasi_metop-b_avail_channels}} obs operator: name: CRTM @@ -255,68 +207,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - use_flag: &useflag_iasi_metop-b [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, 1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use_flag: &useflag_iasi_metop-b {{iasi_metop-b_active_channels}} use_flag_clddet: &clddet_iasi_metop-b [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index e483bfd4..ef313f06 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -9,55 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-c.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &iasi_metop-c_channels 16, 29, 32, 35, 38, 41, 44, 47, 49, 50, 51, 53, - 55, 56, 57, 59, 61, 62, 63, 66, 68, 70, 72, 74, 76, 78, 79, 81, 82, 83, - 84, 85, 86, 87, 89, 92, 93, 95, 97, 99, 101, 103, 104, 106, 109, 110, - 111, 113, 116, 119, 122, 125, 128, 131, 133, 135, 138, 141, 144, 146, - 148, 150, 151, 154, 157, 159, 160, 161, 163, 167, 170, 173, 176, 179, - 180, 185, 187, 191, 193, 197, 199, 200, 202, 203, 205, 207, 210, 212, - 213, 214, 217, 218, 219, 222, 224, 225, 226, 228, 230, 231, 232, 236, - 237, 239, 243, 246, 249, 252, 254, 259, 260, 262, 265, 267, 269, 275, - 279, 282, 285, 294, 296, 299, 300, 303, 306, 309, 313, 320, 323, 326, - 327, 329, 332, 335, 345, 347, 350, 354, 356, 360, 363, 366, 371, 372, - 373, 375, 377, 379, 381, 383, 386, 389, 398, 401, 404, 405, 407, 408, - 410, 411, 414, 416, 418, 423, 426, 428, 432, 433, 434, 439, 442, 445, - 450, 457, 459, 472, 477, 483, 509, 515, 546, 552, 559, 566, 571, 573, - 578, 584, 594, 625, 646, 662, 668, 705, 739, 756, 797, 867, 906, 921, - 1027, 1046, 1090, 1098, 1121, 1133, 1173, 1191, 1194, 1222, 1271, 1283, - 1338, 1409, 1414, 1420, 1424, 1427, 1430, 1434, 1440, 1442, 1445, 1450, - 1454, 1460, 1463, 1469, 1474, 1479, 1483, 1487, 1494, 1496, 1502, 1505, - 1509, 1510, 1513, 1518, 1521, 1526, 1529, 1532, 1536, 1537, 1541, 1545, - 1548, 1553, 1560, 1568, 1574, 1579, 1583, 1585, 1587, 1606, 1626, 1639, - 1643, 1652, 1658, 1659, 1666, 1671, 1675, 1681, 1694, 1697, 1710, 1786, - 1791, 1805, 1839, 1884, 1913, 1946, 1947, 1991, 2019, 2094, 2119, 2213, - 2239, 2271, 2289, 2321, 2333, 2346, 2349, 2352, 2359, 2367, 2374, 2398, - 2426, 2562, 2701, 2741, 2745, 2760, 2819, 2889, 2907, 2910, 2919, 2921, - 2939, 2944, 2945, 2948, 2951, 2958, 2971, 2977, 2985, 2988, 2990, 2991, - 2993, 3002, 3008, 3014, 3027, 3029, 3030, 3036, 3047, 3049, 3052, 3053, - 3055, 3058, 3064, 3069, 3087, 3093, 3098, 3105, 3107, 3110, 3116, 3127, - 3129, 3136, 3146, 3151, 3160, 3165, 3168, 3175, 3178, 3189, 3207, 3228, - 3244, 3248, 3252, 3256, 3263, 3281, 3295, 3303, 3309, 3312, 3322, 3326, - 3354, 3366, 3375, 3378, 3411, 3416, 3432, 3438, 3440, 3442, 3444, 3446, - 3448, 3450, 3452, 3454, 3458, 3467, 3476, 3484, 3491, 3497, 3499, 3504, - 3506, 3509, 3518, 3527, 3555, 3575, 3577, 3580, 3582, 3586, 3589, 3599, - 3610, 3626, 3638, 3646, 3653, 3658, 3661, 3673, 3689, 3700, 3710, 3726, - 3763, 3814, 3841, 3888, 4032, 4059, 4068, 4082, 4095, 4160, 4234, 4257, - 4411, 4498, 4520, 4552, 4567, 4608, 4646, 4698, 4808, 4849, 4920, 4939, - 4947, 4967, 4991, 4996, 5015, 5028, 5056, 5128, 5130, 5144, 5170, 5178, - 5183, 5188, 5191, 5368, 5371, 5379, 5381, 5383, 5397, 5399, 5401, 5403, - 5405, 5446, 5455, 5472, 5480, 5483, 5485, 5492, 5497, 5502, 5507, 5509, - 5517, 5528, 5558, 5697, 5714, 5749, 5766, 5785, 5798, 5799, 5801, 5817, - 5833, 5834, 5836, 5849, 5851, 5852, 5865, 5869, 5881, 5884, 5897, 5900, - 5916, 5932, 5948, 5963, 5968, 5978, 5988, 5992, 5994, 5997, 6003, 6008, - 6023, 6026, 6039, 6053, 6056, 6067, 6071, 6082, 6085, 6098, 6112, 6126, - 6135, 6140, 6149, 6154, 6158, 6161, 6168, 6174, 6182, 6187, 6205, 6209, - 6213, 6317, 6339, 6342, 6366, 6381, 6391, 6489, 6962, 6966, 6970, 6975, - 6977, 6982, 6985, 6987, 6989, 6991, 6993, 6995, 6997, 6999, 7000, 7004, - 7008, 7013, 7016, 7021, 7024, 7027, 7029, 7032, 7038, 7043, 7046, 7049, - 7069, 7072, 7076, 7081, 7084, 7089, 7099, 7209, 7222, 7231, 7235, 7247, - 7267, 7269, 7284, 7389, 7419, 7423, 7424, 7426, 7428, 7431, 7436, 7444, - 7475, 7549, 7584, 7665, 7666, 7831, 7836, 7853, 7865, 7885, 7888, 7912, - 7950, 7972, 7980, 7995, 8007, 8015, 8055, 8078 + channels: &iasi_metop-c_channels {{iasi_metop-c_avail_channels}} obs operator: name: CRTM @@ -255,68 +207,7 @@ obs post filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - use_flag: &useflag_iasi_metop-c [ 1, -1, -1, -1, 1, -1, -1, -1, 1, -1, - 1, -1, 1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, 1, 1, -1, -1, 1, 1, -1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1, 1, -1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, -1, 1, 1, 1, 1, -1, 1, 1, 1, - 1, 1, 1, -1, 1, 1, 1, -1, 1, -1, - 1, -1, -1, -1, 1, 1, 1, 1, -1, 1, - 1, -1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, -1, - 1, 1, -1, -1, -1, 1, -1, 1, -1, 1, - 1, 1, -1, 1, 1, -1, -1, -1, 1, -1, - 1, 1, -1, 1, 1, 1, 1, 1, 1, 1, - -1, 1, 1, -1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, -1, 1, -1, 1, -1, - 1, 1, -1, -1, 1, 1, 1, -1, 1, 1, - -1, 1, -1, 1, -1, -1, -1, -1, -1, 1, - 1, 1, 1, 1, 1, 1, -1, -1, -1, -1, - 1, 1, 1, -1, -1, 1, -1, 1, 1, 1, - 1, 1, -1, -1, 1, 1, -1, 1, 1, -1, - 1, -1, -1, -1, -1, -1, -1, 1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, 1, -1, 1, -1, -1, 1, -1, - 1, -1, -1, -1, -1, 1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1] + use_flag: &useflag_iasi_metop-c {{iasi_metop-c_active_channels}} use_flag_clddet: &clddet_iasi_metop-c [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml index 315e9393..55cca79b 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/ssmis_f17.yaml @@ -18,7 +18,7 @@ type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.ssmis_f17.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &ssmis_f17_channels 1-24 + channels: &ssmis_f17_channels {{ssmis_f17_avail_channels}} obs bias: input file: '{{cycle_dir}}/ssmis_f17.{{background_time}}.satbias.nc4' variational bc: @@ -209,9 +209,7 @@ channels: *ssmis_f17_channels options: channels: *ssmis_f17_channels - use_flag: &useflag_ssmis_f17 [ -1, -1, -1, -1, 1, 1, 1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 1, 1] + use_flag: &useflag_ssmis_f17 {{ssmis_f17_active_channels}} maxvalue: 1.0e-12 action: name: reject diff --git a/src/swell/utilities/get_channels.py b/src/swell/utilities/get_channels.py index f1938924..f94ebc52 100644 --- a/src/swell/utilities/get_channels.py +++ b/src/swell/utilities/get_channels.py @@ -10,6 +10,7 @@ import yaml import os from datetime import datetime as dt +from itertools import groupby # -------------------------------------------------------------------------------------------------- @@ -36,6 +37,22 @@ def process_channel_lists(channel_list): # -------------------------------------------------------------------------------------------------- +def create_range_string(avail_list): + ''' + Function converts integer list into string of ranges + ''' + ranges = [] + for _, g in groupby(enumerate(avail_list), lambda i_x:i_x[0]-i_x[1]): + group = list(map(lambda x: x[1], g)) + if len(group) > 1: + ranges.append(f'{group[0]}-{group[-1]}') + else: + ranges.append(str(group[0])) + return ', '.join(ranges) + +# -------------------------------------------------------------------------------------------------- + + def get_channel_list(input_dict, dt_cycle_time): ''' @@ -70,10 +87,11 @@ def get_channels(path_to_observing_sys_yamls, observation, dt_cycle_time): active_channels = get_channel_list(data['active'], dt_cycle_time) available_channels_list = process_channel_lists(available_channels) + available_range_string = create_range_string(available_channels_list) active_channels_list = process_channel_lists(active_channels) use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] - return available_channels_list, use_flags + return available_range_string, use_flags else: return None From 3586250491e12274ec7460eeab8167fda859aca6 Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 30 Nov 2023 11:04:33 -0500 Subject: [PATCH 30/38] updating yamls with templating --- .../observations/amsua_metop-b.yaml | 4 ++-- .../observations/amsua_metop-c.yaml | 4 ++-- .../observations/avhrr3_metop-b.yaml | 4 ++-- .../observations/cris-fsr_n20.yaml | 4 ++-- .../observations/cris-fsr_npp.yaml | 4 ++-- .../observations/iasi_metop-b.yaml | 4 ++-- .../observations/iasi_metop-c.yaml | 4 ++-- .../code_tests/get_active_channels_test.py | 2 +- src/swell/utilities/get_channels.py | 2 +- .../utilities/render_jedi_interface_files.py | 18 +++++++++++++----- 10 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml index fcba927c..8e95c08f 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-b.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_metop-b_available_channels {{amsua_metop-b_avail_channels}} + channels: &amsua_metop-b_available_channels {{amsua_metopb_avail_channels}} obs operator: name: CRTM @@ -191,7 +191,7 @@ obs post filters: channels: *amsua_metop-b_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: &amsua_metop-b_use_flag {{amsua_metop-b_active_channels}} + use_flag: &amsua_metop-b_use_flag {{amsua_metopb_active_channels}} maxvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml index c4bca511..c8054143 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/amsua_metop-c.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_metop-c.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &amsua_metop-c_available_channels {{amsua_metop-c_avail_channels}} + channels: &amsua_metop-c_available_channels {{amsua_metopc_avail_channels}} obs operator: name: CRTM @@ -191,7 +191,7 @@ obs post filters: channels: *amsua_metop-c_available_channels use passive_bc: true sensor: *Sensor_ID - use_flag: &amsua_metop-c_use_flag {{amsua_metop-c_active_channels}} + use_flag: &amsua_metop-c_use_flag {{amsua_metopc_active_channels}} maxvalue: 1.0e-12 action: name: reject diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml index 45111cbf..b71d873a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/avhrr3_metop-b.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.avhrr3_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &avhrr3_metop-b_channels {{avhrr3_metop-b_avail_channels}} + channels: &avhrr3_metop-b_channels {{avhrr3_metopb_avail_channels}} obs operator: name: CRTM @@ -141,7 +141,7 @@ obs post filters: channels: *avhrr3_metop-b_channels options: channels: *avhrr3_metop-b_channels - use_flag: &useflag_avhrr3_metop-b {{avhrr3_metop-b_active_channels}} + use_flag: &useflag_avhrr3_metop-b {{avhrr3_metopb_active_channels}} use_flag_clddet: [ 1, 1, 1 ] obserr_dtempf: [0.50,2.00,4.00,2.00,4.00] maxvalue: 1.0e-12 diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml index ffc4dcf0..c6d029c7 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_n20.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.cris-fsr_n20.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &cris-fsr_n20_channels {{cris-fsr_n20_avail_channels}} + channels: &cris-fsr_n20_channels {{crisfsr_n20_avail_channels}} obs operator: name: CRTM @@ -185,7 +185,7 @@ obs post filters: channels: *cris-fsr_n20_channels options: channels: *cris-fsr_n20_channels - use_flag: &useflag_cris-fsr_n20 {{cris-fsr_n20_active_channels}} + use_flag: &useflag_cris-fsr_n20 {{crisfsr_n20_active_channels}} use_flag_clddet: &clddet_cris-fsr_n20 [ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml index 7559eb46..ed028a54 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/cris-fsr_npp.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.cris-fsr_npp.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &cris-fsr_npp_channels {{cris-fsr_npp_avail_channels}} + channels: &cris-fsr_npp_channels {{crisfsr_npp_avail_channels}} obs operator: name: CRTM @@ -185,7 +185,7 @@ obs post filters: channels: *cris-fsr_npp_channels options: channels: *cris-fsr_npp_channels - use_flag: &useflag_cris-fsr_npp {{cris-fsr_npp_active_channels}} + use_flag: &useflag_cris-fsr_npp {{crisfsr_npp_active_channels}} use_flag_clddet: &clddet_cris-fsr_npp [ 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, 31, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml index bf32c2ee..4e36ea10 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-b.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-b.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &iasi_metop-b_channels {{iasi_metop-b_avail_channels}} + channels: &iasi_metop-b_channels {{iasi_metopb_avail_channels}} obs operator: name: CRTM @@ -207,7 +207,7 @@ obs post filters: channels: *iasi_metop-b_channels options: channels: *iasi_metop-b_channels - use_flag: &useflag_iasi_metop-b {{iasi_metop-b_active_channels}} + use_flag: &useflag_iasi_metop-b {{iasi_metopb_active_channels}} use_flag_clddet: &clddet_iasi_metop-b [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml index ef313f06..45ad2b9a 100644 --- a/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml +++ b/src/swell/configuration/jedi/interfaces/geos_atmosphere/observations/iasi_metop-c.yaml @@ -9,7 +9,7 @@ obs space: type: H5File obsfile: '{{cycle_dir}}/{{experiment_id}}.iasi_metop-c.{{window_begin}}.nc4' simulated variables: [brightnessTemperature] - channels: &iasi_metop-c_channels {{iasi_metop-c_avail_channels}} + channels: &iasi_metop-c_channels {{iasi_metopc_avail_channels}} obs operator: name: CRTM @@ -207,7 +207,7 @@ obs post filters: channels: *iasi_metop-c_channels options: channels: *iasi_metop-c_channels - use_flag: &useflag_iasi_metop-c {{iasi_metop-c_active_channels}} + use_flag: &useflag_iasi_metop-c {{iasi_metopc_active_channels}} use_flag_clddet: &clddet_iasi_metop-c [ 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 8d5ca8e1..ba604bd8 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -8,5 +8,5 @@ path_to_observing_sys_yamls = 'active_channels_test_files/' observation = 'amsua_n19' -avail, generated_use_flags = get_channels(path_to_observing_sys_yamls,observation, dt_cycle_time) +avail, generated_use_flags = get_channels(path_to_observing_sys_yamls, observation, dt_cycle_time) assert (use_flags == generated_use_flags) diff --git a/src/swell/utilities/get_channels.py b/src/swell/utilities/get_channels.py index f94ebc52..4bf54219 100644 --- a/src/swell/utilities/get_channels.py +++ b/src/swell/utilities/get_channels.py @@ -42,7 +42,7 @@ def create_range_string(avail_list): Function converts integer list into string of ranges ''' ranges = [] - for _, g in groupby(enumerate(avail_list), lambda i_x:i_x[0]-i_x[1]): + for _, g in groupby(enumerate(avail_list), lambda i_x: i_x[0]-i_x[1]): group = list(map(lambda x: x[1], g)) if len(group) > 1: ranges.append(f'{group[0]}-{group[-1]}') diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 27bdc2e0..5c39cb74 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -119,6 +119,8 @@ def __open_file_render_to_dict__(self, config_file): with open(config_file, 'r') as config_file_open: config_file_str_templated = config_file_open.read() + print(self.__template_dict__) + # Fill templates in the configuration file using the config config_file_str = template_string_jinja2(self.logger, config_file_str_templated, self.__template_dict__) @@ -186,14 +188,20 @@ def render_interface_observations(self, config_name): # If yaml is ufo_tests, skip get_active_channels if config_name != 'ufo_tests': - # Get available and active channels - available_channels, active_channels = get_channels(self.observing_system_records_path, - config_name, self.cycle_time) + result = get_channels(self.observing_system_records_path, + config_name, self.cycle_time) + if result is None: + available_channels = active_channels = None + + else: + available_channels, active_channels = result # Add available and active channels to template dictionary - self.__template_dict__[f'{config_name}_avail_channels'] = available_channels - self.__template_dict__[f'{config_name}_active_channels'] = active_channels + # If config_name contains a hyphen, remove for jinja2 templating + new_config_name = config_name.replace('-', '') + self.__template_dict__[f'{new_config_name}_avail_channels'] = available_channels + self.__template_dict__[f'{new_config_name}_active_channels'] = active_channels # Render templates in file and return dictionary return self.__open_file_render_to_dict__(config_file) From f0998ab9415659c0225cbf39ec6f2f87798e2b20 Mon Sep 17 00:00:00 2001 From: asewnath Date: Fri, 1 Dec 2023 12:34:34 -0500 Subject: [PATCH 31/38] coding norms --- src/swell/utilities/render_jedi_interface_files.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 500823ab..c22a97a2 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -163,8 +163,7 @@ def render_interface_model(self, config_name): return self.__open_file_render_to_dict__(config_file) # ---------------------------------------------------------------------------------------------- - - + def set_obs_records_path(self, path): # Never put a path that is string None in place From d344847167535c40c317940ce73bace2d031f3b0 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 4 Dec 2023 12:28:58 -0500 Subject: [PATCH 32/38] fix for test --- src/swell/tasks/gsi_bc_to_ioda.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index f5d1fc7a..241c334d 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -35,7 +35,6 @@ def execute(self): cycle_dir = self.cycle_dir() if observing_system_records_path == 'None': observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) From 67bc9fa73926608c333f08d4c9524feaabc5ea95 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 4 Dec 2023 13:15:51 -0500 Subject: [PATCH 33/38] augmenting test --- .../active_channels_test_files/amsua_n19.yaml | 218 ------------------ .../avhrr3_n18_channel_info.yaml | 18 ++ .../gmi_gpm_channel_info.yaml | 28 +++ .../mhs_metop-c_channel_info.yaml | 19 ++ .../code_tests/get_active_channels_test.py | 19 +- 5 files changed, 79 insertions(+), 223 deletions(-) delete mode 100644 src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml create mode 100644 src/swell/test/code_tests/active_channels_test_files/avhrr3_n18_channel_info.yaml create mode 100644 src/swell/test/code_tests/active_channels_test_files/gmi_gpm_channel_info.yaml create mode 100644 src/swell/test/code_tests/active_channels_test_files/mhs_metop-c_channel_info.yaml diff --git a/src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml b/src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml deleted file mode 100644 index 6b69be8a..00000000 --- a/src/swell/test/code_tests/active_channels_test_files/amsua_n19.yaml +++ /dev/null @@ -1,218 +0,0 @@ -obs space: - name: AMSU-A NOAA-19 - obsdatain: - engine: - type: H5File - obsfile: '{{cycle_dir}}/amsua_n19.{{window_begin}}.nc4' - obsdataout: - engine: - type: H5File - obsfile: '{{cycle_dir}}/{{experiment_id}}.amsua_n19.{{window_begin}}.nc4' - simulated variables: [brightnessTemperature] - channels: &amsua_n19_available_channels 1-15 - -obs operator: - name: CRTM - Absorbers: [H2O,O3,CO2] - obs options: - Sensor_ID: &Sensor_ID amsua_n19 - EndianType: little_endian - CoefficientPath: '{{crtm_coeff_dir}}' - linear obs operator: - Absorbers: [H2O,O3] - -obs bias: - input file: '{{cycle_dir}}/amsua_n19.{{background_time}}.satbias.nc4' - variables without bc: [brightnessTemperature] - channels: 14 - variational bc: - predictors: - - name: constant - - name: cloud_liquid_water - sensor: AMSUA - clwdif_ch238: 1 - clwdif_ch314: 2 - - name: lapse_rate - order: 2 - tlapse: &amsua_n19_tlapse '{{cycle_dir}}/amsua_n19.{{background_time}}.tlapse.txt' - - name: lapse_rate - tlapse: *amsua_n19_tlapse - - name: emissivity - - name: scan_angle - order: 4 - - name: scan_angle - order: 3 - - name: scan_angle - order: 2 - - name: scan_angle - covariance: - minimal required obs number: 20 - variance range: [1.0e-6, 10.0] - step size: 1.0e-4 - largest analysis variance: 10000.0 - prior: - input file: '{{cycle_dir}}/amsua_n19.{{background_time}}.satbias.nc4' - inflation: - ratio: 1.1 - ratio for small dataset: 2.0 - output file: '{{cycle_dir}}/amsua_n19.{{window_begin}}.satbias.nc4' - -obs filters: -# Window and surface-sensitive channels check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: 1-6,15 - test variables: - - name: ObsValue/brightnessTemperature - channels: 1-6,15 - treat missing as out of bounds: true - minvalue: 100.0 - maxvalue: 500.0 - flag all filter variables if any test variable is out of bounds: true -# passedBenchmark: 1500 -# All channels unreasonable values check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - minvalue: 100.0 - maxvalue: 500.0 -# Hydrometeor Check (cloud/precipitation affected chanels) - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - test variables: - - name: ObsFunction/HydrometeorCheckAMSUAclr - channels: *amsua_n19_available_channels - options: - sensor: *Sensor_ID - channels: *amsua_n19_available_channels - test_biaspredictor: cloud_liquid_waterPredictor - maxvalue: 0.0 - action: - name: reject -# Assign obs error - - filter: Perform Action - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - action: - name: assign error - error parameter vector: - [2.500, 2.000, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 5.000, 2.500] -# Topography check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *amsua_n19_available_channels - options: - sensor: *Sensor_ID - channels: *amsua_n19_available_channels -# Transmittance Top Check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *amsua_n19_available_channels - options: - channels: *amsua_n19_available_channels -# Surface Jacobian check - - filter: BlackList - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - action: - name: inflate error - inflation variable: - name: ObsFunction/ObsErrorFactorSurfJacobianRad - channels: *amsua_n19_available_channels - options: - channels: *amsua_n19_available_channels - sensor: *Sensor_ID - use_biasterm: true - test_biasterm: ObsBiasTerm - obserr_demisf: [0.010, 0.020, 0.015, 0.020, 0.200] - obserr_dtempf: [0.500, 2.000, 1.000, 2.000, 4.500] -# Gross check - - filter: Background Check - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - function absolute threshold: - - name: ObsFunction/ObsErrorBoundMW - channels: *amsua_n19_available_channels - options: - sensor: *Sensor_ID - channels: *amsua_n19_available_channels - obserr_bound_latitude: - name: ObsFunction/ObsErrorFactorLatRad - options: - latitude_parameters: [25.0, 0.25, 0.04, 3.0] - obserr_bound_transmittop: - name: ObsFunction/ObsErrorFactorTransmitTopRad - channels: *amsua_n19_available_channels - options: - channels: *amsua_n19_available_channels - obserr_bound_topo: - name: ObsFunction/ObsErrorFactorTopoRad - channels: *amsua_n19_available_channels - options: - channels: *amsua_n19_available_channels - sensor: *Sensor_ID - error parameter vector: - [2.500, 2.000, 2.000, 0.550, 0.300, - 0.230, 0.230, 0.250, 0.250, 0.350, - 0.400, 0.550, 0.800, 5.000, 2.500] - obserr_bound_max: [4.5, 4.5, 4.5, 2.5, 2.0, - 2.0, 2.0, 2.0, 2.0, 2.0, - 2.5, 3.5, 4.5, 4.5, 4.5] - action: - name: reject -# Inter-channel check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - test variables: - - name: ObsFunction/InterChannelConsistencyCheck - channels: *amsua_n19_available_channels - options: - channels: *amsua_n19_available_channels - use passive_bc: true - sensor: *Sensor_ID - use_flag: [-1, -1, -1, 1, 1, - 1, -1, -1, 1, 1, - 1, 1, 1, 1, -1] - maxvalue: 1.0e-12 - action: - name: reject -# Useflag check - - filter: Bounds Check - filter variables: - - name: brightnessTemperature - channels: *amsua_n19_available_channels - test variables: - - name: ObsFunction/ChannelUseflagCheckRad - channels: *amsua_n19_available_channels - options: - channels: *amsua_n19_available_channels -# use passive_bc: true - use_flag: [-1, -1, -1, 1, 1, - 1, -1, -1, 1, 1, - 1, 1, 1, 1, -1] - minvalue: 1.0e-12 - action: - name: reject diff --git a/src/swell/test/code_tests/active_channels_test_files/avhrr3_n18_channel_info.yaml b/src/swell/test/code_tests/active_channels_test_files/avhrr3_n18_channel_info.yaml new file mode 100644 index 00000000..5f1439ef --- /dev/null +++ b/src/swell/test/code_tests/active_channels_test_files/avhrr3_n18_channel_info.yaml @@ -0,0 +1,18 @@ +active: +- begin date: '2005-05-20T18:00:00' + channels: + - '4' + - '5' + comments: no comment + end date: '2023-09-12T18:00:00' +- begin date: '2023-09-13T00:00:00' + channels: [] + comments: '# NOAA-18 satellite will be fully transitioned to Parsons' + end date: '2100-12-31T18:00:00' +available: +- begin date: '2005-05-20T00:00:00' + channels: + - '3' + - '4' + - '5' + end date: '2999-12-31T18:00:00' diff --git a/src/swell/test/code_tests/active_channels_test_files/gmi_gpm_channel_info.yaml b/src/swell/test/code_tests/active_channels_test_files/gmi_gpm_channel_info.yaml new file mode 100644 index 00000000..08986b9b --- /dev/null +++ b/src/swell/test/code_tests/active_channels_test_files/gmi_gpm_channel_info.yaml @@ -0,0 +1,28 @@ +active: +- begin date: '2014-03-20T00:00:00' + channels: + - '5' + - '6' + - '7' + - '10' + - '12' + - '13' + comments: no comment + end date: '2999-12-31T18:00:00' +available: +- begin date: '2014-03-20T00:00:00' + channels: + - '1' + - '2' + - '3' + - '4' + - '5' + - '6' + - '7' + - '8' + - '9' + - '10' + - '11' + - '12' + - '13' + end date: '2999-12-31T18:00:00' diff --git a/src/swell/test/code_tests/active_channels_test_files/mhs_metop-c_channel_info.yaml b/src/swell/test/code_tests/active_channels_test_files/mhs_metop-c_channel_info.yaml new file mode 100644 index 00000000..9afa5578 --- /dev/null +++ b/src/swell/test/code_tests/active_channels_test_files/mhs_metop-c_channel_info.yaml @@ -0,0 +1,19 @@ +active: +- begin date: '2019-09-17T00:00:00' + channels: + - '1' + - '2' + - '3' + - '4' + - '5' + comments: '# active for parallel testing' + end date: '2100-12-31T18:00:00' +available: +- begin date: '2019-09-17T00:00:00' + channels: + - '1' + - '2' + - '3' + - '4' + - '5' + end date: '2100-12-31T18:00:00' diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 68e8ef26..9fcb59a1 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -1,13 +1,22 @@ from swell.utilities.get_channels import get_channels from datetime import datetime as dt -use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] +# Use flags +amsua_n19_use_flags = [-1, -1, -1, 1, 1, 1, -1, -1, 1, 1, 1, 1, 1, 1, -1] +avhrr3_n18_use_flags = [-1, 1, 1] +gmi_gpm_use_flags = [-1, -1, -1, -1, 1, 1, 1, -1, -1, 1, -1, 1, 1] +mhs_metop_c_use_flags = [1, 1, 1, 1, 1] + +use_flags = [amsua_n19_use_flags, avhrr3_n18_use_flags, + gmi_gpm_use_flags, mhs_metop_c_use_flags] + +# Observation list +observations = ['amsua_n19', 'avhrr3_n18', 'gmi_gpm', 'mhs_metop-c'] cycle_time = '20211212T000000Z' dt_cycle_time = dt.strptime(cycle_time, '%Y%m%dT%H%M%SZ') path_to_observing_sys_yamls = 'active_channels_test_files/' -observation = 'amsua_n19' - -avail, generated_use_flags = get_channels(path_to_observing_sys_yamls, observation, dt_cycle_time) -assert (use_flags == generated_use_flags) +for idx, _ in enumerate(observations): + avail, generated_use_flags = get_channels(path_to_observing_sys_yamls, observations[idx], dt_cycle_time) + assert (use_flags[idx] == generated_use_flags) From 420ca0bdc7f18664235ed9c7f0ea2f2b2a837077 Mon Sep 17 00:00:00 2001 From: asewnath Date: Mon, 4 Dec 2023 14:39:48 -0500 Subject: [PATCH 34/38] pr fixes --- src/swell/tasks/eva_observations.py | 9 --------- src/swell/tasks/generate_observing_system_records.py | 1 - src/swell/tasks/get_observations.py | 9 --------- src/swell/tasks/gsi_bc_to_ioda.py | 4 ---- src/swell/tasks/run_jedi_hofx_executable.py | 9 --------- .../tasks/run_jedi_local_ensemble_da_executable.py | 9 --------- src/swell/tasks/run_jedi_ufo_tests_executable.py | 9 --------- src/swell/tasks/run_jedi_variational_executable.py | 10 +--------- src/swell/tasks/save_obs_diags.py | 9 --------- src/swell/test/code_tests/get_active_channels_test.py | 5 +++-- src/swell/utilities/render_jedi_interface_files.py | 7 ------- 11 files changed, 4 insertions(+), 77 deletions(-) diff --git a/src/swell/tasks/eva_observations.py b/src/swell/tasks/eva_observations.py index cc1272f9..04ad8581 100644 --- a/src/swell/tasks/eva_observations.py +++ b/src/swell/tasks/eva_observations.py @@ -84,15 +84,6 @@ def execute(self): # Loop over observations # ------------------- eva_dicts = [] # Empty list of dictionaries - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() - - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) diff --git a/src/swell/tasks/generate_observing_system_records.py b/src/swell/tasks/generate_observing_system_records.py index a66ea60a..0ce33621 100644 --- a/src/swell/tasks/generate_observing_system_records.py +++ b/src/swell/tasks/generate_observing_system_records.py @@ -34,7 +34,6 @@ def execute(self): # -------------------------------------------------- observations = self.config.observations() observing_system_records_path = self.config.observing_system_records_path(None) - if observing_system_records_path == 'None': cycle_dir = self.cycle_dir() observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') diff --git a/src/swell/tasks/get_observations.py b/src/swell/tasks/get_observations.py index 17f26b21..974317fe 100644 --- a/src/swell/tasks/get_observations.py +++ b/src/swell/tasks/get_observations.py @@ -42,15 +42,6 @@ def execute(self): window_length = self.config.window_length() crtm_coeff_dir = self.config.crtm_coeff_dir(None) window_offset = self.config.window_offset() - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() - - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) diff --git a/src/swell/tasks/gsi_bc_to_ioda.py b/src/swell/tasks/gsi_bc_to_ioda.py index 241c334d..32946755 100644 --- a/src/swell/tasks/gsi_bc_to_ioda.py +++ b/src/swell/tasks/gsi_bc_to_ioda.py @@ -31,10 +31,6 @@ def execute(self): window_offset = self.config.window_offset() background_time_offset = self.config.background_time_offset() crtm_coeff_dir = self.config.crtm_coeff_dir(None) - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') # Get window beginning time window_begin = self.da_window_params.window_begin(window_offset) diff --git a/src/swell/tasks/run_jedi_hofx_executable.py b/src/swell/tasks/run_jedi_hofx_executable.py index c6813caf..0388835d 100644 --- a/src/swell/tasks/run_jedi_hofx_executable.py +++ b/src/swell/tasks/run_jedi_hofx_executable.py @@ -36,11 +36,6 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) @@ -94,10 +89,6 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) - jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, jedi_forecast_model) diff --git a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py index 5fd704cb..8554c390 100644 --- a/src/swell/tasks/run_jedi_local_ensemble_da_executable.py +++ b/src/swell/tasks/run_jedi_local_ensemble_da_executable.py @@ -36,15 +36,6 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() - - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) diff --git a/src/swell/tasks/run_jedi_ufo_tests_executable.py b/src/swell/tasks/run_jedi_ufo_tests_executable.py index c1fda658..ec1e966d 100644 --- a/src/swell/tasks/run_jedi_ufo_tests_executable.py +++ b/src/swell/tasks/run_jedi_ufo_tests_executable.py @@ -38,15 +38,6 @@ def execute(self): observations = self.config.observations() single_observations = self.config.single_observations() generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() - - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) diff --git a/src/swell/tasks/run_jedi_variational_executable.py b/src/swell/tasks/run_jedi_variational_executable.py index 6ce7874c..8d69e3d4 100644 --- a/src/swell/tasks/run_jedi_variational_executable.py +++ b/src/swell/tasks/run_jedi_variational_executable.py @@ -37,11 +37,6 @@ def execute(self): observations = self.config.observations() jedi_forecast_model = self.config.jedi_forecast_model(None) generate_yaml_and_exit = self.config.generate_yaml_and_exit(False) - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) @@ -108,11 +103,8 @@ def execute(self): # Perform complete template rendering # ----------------------------------- - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) jedi_dictionary_iterator(jedi_config_dict, self.jedi_rendering, window_type, observations, - observing_system_records_path, cycle_time, jedi_forecast_model) + jedi_forecast_model) # Write the expanded dictionary to YAML file # ------------------------------------------ diff --git a/src/swell/tasks/save_obs_diags.py b/src/swell/tasks/save_obs_diags.py index 148a8ed0..a2f83b11 100644 --- a/src/swell/tasks/save_obs_diags.py +++ b/src/swell/tasks/save_obs_diags.py @@ -28,15 +28,6 @@ def execute(self): crtm_coeff_dir = self.config.crtm_coeff_dir(None) observations = self.config.observations() window_offset = self.config.window_offset() - observing_system_records_path = self.config.observing_system_records_path() - cycle_dir = self.cycle_dir() - if observing_system_records_path == 'None': - observing_system_records_path = os.path.join(cycle_dir, 'observing_system_records') - cycle_time = self.cycle_time_dto() - - # Set cycle time and observing_system_records_path in jedi_rendering - self.jedi_rendering.set_observing_system_records_path(observing_system_records_path) - self.jedi_rendering.set_cycle_time(cycle_time) # Set the observing system records path self.jedi_rendering.set_obs_records_path(self.config.observing_system_records_path(None)) diff --git a/src/swell/test/code_tests/get_active_channels_test.py b/src/swell/test/code_tests/get_active_channels_test.py index 9fcb59a1..5a5bc6ec 100644 --- a/src/swell/test/code_tests/get_active_channels_test.py +++ b/src/swell/test/code_tests/get_active_channels_test.py @@ -7,7 +7,7 @@ gmi_gpm_use_flags = [-1, -1, -1, -1, 1, 1, 1, -1, -1, 1, -1, 1, 1] mhs_metop_c_use_flags = [1, 1, 1, 1, 1] -use_flags = [amsua_n19_use_flags, avhrr3_n18_use_flags, +use_flags = [amsua_n19_use_flags, avhrr3_n18_use_flags, gmi_gpm_use_flags, mhs_metop_c_use_flags] # Observation list @@ -18,5 +18,6 @@ path_to_observing_sys_yamls = 'active_channels_test_files/' for idx, _ in enumerate(observations): - avail, generated_use_flags = get_channels(path_to_observing_sys_yamls, observations[idx], dt_cycle_time) + avail, generated_use_flags = get_channels(path_to_observing_sys_yamls, + observations[idx], dt_cycle_time) assert (use_flags[idx] == generated_use_flags) diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index c22a97a2..36575afd 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -125,8 +125,6 @@ def __open_file_render_to_dict__(self, config_file): with open(config_file, 'r') as config_file_open: config_file_str_templated = config_file_open.read() - print(self.__template_dict__) - # Fill templates in the configuration file using the config config_file_str = template_string_jinja2(self.logger, config_file_str_templated, self.__template_dict__) @@ -178,11 +176,6 @@ def set_obs_records_path(self, path): # Prepare path to interface observations file and call rendering def render_interface_observations(self, config_name): - # Check that observing_system_records_path and cycle_time are set - self.logger.assert_abort(self.cycle_time is not None, f'cycle_time must be set.') - self.logger.assert_abort(self.observing_system_records_path is not None, - f'observing_system_records_path must be set.') - # Assert that there is a jedi interface associated with the task self.logger.assert_abort(self.jedi_interface is not None, f'In order to render a ' + f'jedi interface config file the task must have an associated' + From c1ef0fde118840443baeb05c4cc2a533d4adf68b Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 13 Dec 2023 09:33:16 -0500 Subject: [PATCH 35/38] addressing comment --- src/swell/utilities/get_active_channels.py | 81 ------------------- src/swell/utilities/get_channels.py | 2 +- .../utilities/render_jedi_interface_files.py | 11 +-- 3 files changed, 4 insertions(+), 90 deletions(-) delete mode 100644 src/swell/utilities/get_active_channels.py diff --git a/src/swell/utilities/get_active_channels.py b/src/swell/utilities/get_active_channels.py deleted file mode 100644 index 2a6ffb65..00000000 --- a/src/swell/utilities/get_active_channels.py +++ /dev/null @@ -1,81 +0,0 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - - -# -------------------------------------------------------------------------------------------------- - -import yaml -import os -from datetime import datetime as dt - -# -------------------------------------------------------------------------------------------------- - - -def process_channel_lists(channel_list): - - ''' - Function processes list of elements in channel list - ''' - - final_channels_list = [] - if not isinstance(channel_list, list): - channel_list = [channel_list] - for element in channel_list: - if '-' in element: - start, end = map(int, element.split('-')) - result_list = [x for x in range(start, end + 1)] - final_channels_list += result_list - else: - final_channels_list += [int(element)] - - return final_channels_list - -# -------------------------------------------------------------------------------------------------- - - -def get_channel_list(input_dict, dt_cycle_time): - - ''' - Function retrieves channel lists from dict loaded from a yaml file - ''' - - for element in input_dict: - begin_date = dt.strptime(element['begin date'], "%Y-%m-%dT%H:%M:%S") - end_date = dt.strptime(element['end date'], "%Y-%m-%dT%H:%M:%S") - if (dt_cycle_time > begin_date) and (dt_cycle_time < end_date): - return element['channels'] - -# -------------------------------------------------------------------------------------------------- - - -def get_active_channels(path_to_observing_sys_yamls, observation, dt_cycle_time): - - ''' - Comparing available channels and active channels from the observing - system records to create the use flag array needed in the - qc filter yaml files. - ''' - - # Retrieve available and active channels from records yaml - path_to_observing_sys_config = path_to_observing_sys_yamls + '/' + \ - observation + '_channel_info.yaml' - - if os.path.isfile(path_to_observing_sys_config): - with open(path_to_observing_sys_config, 'r') as file: - data = yaml.safe_load(file) - available_channels = get_channel_list(data['available'], dt_cycle_time) - active_channels = get_channel_list(data['active'], dt_cycle_time) - - available_channels_list = process_channel_lists(available_channels) - active_channels_list = process_channel_lists(active_channels) - use_flags = [1 if x in active_channels_list else -1 for x in available_channels_list] - - return use_flags - - else: - return None - -# -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/get_channels.py b/src/swell/utilities/get_channels.py index 4bf54219..7d56abf7 100644 --- a/src/swell/utilities/get_channels.py +++ b/src/swell/utilities/get_channels.py @@ -94,6 +94,6 @@ def get_channels(path_to_observing_sys_yamls, observation, dt_cycle_time): return available_range_string, use_flags else: - return None + return None, None # -------------------------------------------------------------------------------------------------- diff --git a/src/swell/utilities/render_jedi_interface_files.py b/src/swell/utilities/render_jedi_interface_files.py index 36575afd..2052b6af 100644 --- a/src/swell/utilities/render_jedi_interface_files.py +++ b/src/swell/utilities/render_jedi_interface_files.py @@ -195,18 +195,13 @@ def render_interface_observations(self, config_name): if config_name != 'ufo_tests': # Get available and active channels - result = get_channels(self.observing_system_records_path, - config_name, self.cycle_time) - if result is None: - available_channels = active_channels = None - - else: - available_channels, active_channels = result + avail_channels, active_channels = get_channels(self.observing_system_records_path, + config_name, self.cycle_time) # Add available and active channels to template dictionary # If config_name contains a hyphen, remove for jinja2 templating new_config_name = config_name.replace('-', '') - self.__template_dict__[f'{new_config_name}_avail_channels'] = available_channels + self.__template_dict__[f'{new_config_name}_avail_channels'] = avail_channels self.__template_dict__[f'{new_config_name}_active_channels'] = active_channels # Render templates in file and return dictionary From b402273b236f6fd86de1917baaf1392258c17126 Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 20 Dec 2023 16:10:42 -0500 Subject: [PATCH 36/38] modifying parser --- .../generate_observing_system_test.py | 2 +- src/swell/utilities/gsi_record_parser.py | 368 +++++------------- .../utilities/observing_system_records.py | 1 - 3 files changed, 109 insertions(+), 262 deletions(-) diff --git a/src/swell/test/code_tests/generate_observing_system_test.py b/src/swell/test/code_tests/generate_observing_system_test.py index ec7e34f1..6d74a7e3 100644 --- a/src/swell/test/code_tests/generate_observing_system_test.py +++ b/src/swell/test/code_tests/generate_observing_system_test.py @@ -1,7 +1,7 @@ import os from swell.utilities.observing_system_records import ObservingSystemRecords -observations = ['amsua_n19'] +observations = ['airs_aqua'] observing_system_records_path = './yaml_output/' # Clone GeosAna diff --git a/src/swell/utilities/gsi_record_parser.py b/src/swell/utilities/gsi_record_parser.py index f51c47c0..c481ecf7 100644 --- a/src/swell/utilities/gsi_record_parser.py +++ b/src/swell/utilities/gsi_record_parser.py @@ -1,291 +1,139 @@ -# (C) Copyright 2021- United States Government as represented by the Administrator of the -# National Aeronautics and Space Administration. All Rights Reserved. -# -# This software is licensed under the terms of the Apache Licence Version 2.0 -# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. - -import numpy as np import pandas as pd -import datetime +import numpy as np from datetime import datetime as dt -def check_end_times(end_times): +def check_end_time(end_time): ''' Checks end times for 24 hour strings and converts them to 18 ''' - new_end_times = [] - - for end_time in end_times: - # Note that python datetime does not allow for times with hour = 24 - hour = end_time[8:10] - if (hour == '24'): - # Subtract 6 hours - tmp = int(end_time) - 60000 - new_end_times.append(str(tmp)) - else: - new_end_times.append(end_time) - - return new_end_times + hour = end_time[8:10] + if (hour == '24'): + # Subtract 6 hours + new_end_time = str(int(end_time) - 60000) + return new_end_time + else: + return end_time class GSIRecordParser: def __init__(self): - - ''' - This class employs a state machine algorithm to process raw data from GSI - .tbl files. The different states work together to parse through an initial - dataframe. The rows of the resulting dataframe correspond to clean entries - for a given instrument and satellite. - - ''' - - self.idx = None - self.main_idx = None - self.start_times = None - self.end_times = None self.instr_df = None - self.compare_channels = None - self.curr_channel_list = None - self.main_channel_list = None self.return_df = None + self.sat = None + self.instr = None - def reset(self): - self.idx = 0 - self.main_idx = 0 - self.start_times = [] - self.end_times = [] - self.compare_channels = [] - self.curr_channel_list = [] - self.main_channel_list = [] + def get_channel_list(self, start): + channel_list = [] + rows = self.instr_df.loc[self.instr_df["start"] == start] + [channel_list.extend(i) for i in rows["channels"].values] + channel_list = list(set(channel_list)) + channel_list.sort(key=int) + return channel_list def run(self, instr_df): - ''' - Effectively state one of the state machine. - Generates an ordered list of start times and corresponding - end times. Checks the end times and then proceeds to condition one. - - input: - instr_df = a dataframe containing raw data for a given satellite and - instrument - ''' - - self.reset() + # Save instrument dataframe self.instr_df = instr_df - self.return_df = pd.DataFrame(columns=list(instr_df.columns.values)) - - self.start_times = list(np.unique(self.instr_df["start"].values)) - self.start_times.sort(key=int) - for start in self.start_times: - self.end_times.append( - self.instr_df.loc[self.instr_df["start"] == start]["end"].values[0]) - - self.end_times = check_end_times(self.end_times) - self.condition_one() - - def condition_one(self): - - ''' - If there is one row for the date range, go to state 2. Otherwise, - go to state 3. - ''' - - start_time_df = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] - n_curr_start_rows = len(start_time_df) - assert (n_curr_start_rows != 0) - - if (n_curr_start_rows == 1): - self.curr_channel_list = start_time_df["channels"].values[0] - self.state_two() - else: - self.state_three() - - def state_two(self): - - ''' - Update return_df with new row, increment idx, and then go to - condition 4. - ''' - - row = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] - self.update_return_df(row) - self.idx += 1 - self.condition_four() - - def state_three(self): - - ''' - Gather channels for all rows in current datetime. If there are more - than one end time within the rows, set main_start_idx and - main_channels_list, set compare_channels to the current channel list, - update return_df with a new row, increment index, - and then go to condition 2. - Else, go to state 2 with the updated current channel list. - ''' - - rows = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] - [self.curr_channel_list.extend(i) for i in rows["channels"].values] - self.curr_channel_list = list(set(self.curr_channel_list)) - self.curr_channel_list.sort(key=int) - - end_times = rows["end"].values - other_end_times = end_times[end_times != end_times[0]] - - if (other_end_times): - - first_end = other_end_times[0] - first_end_row = self.instr_df.loc[self.instr_df["end"] == first_end] - self.compare_channels = first_end_row["channels"].values[0] - - self.main_channel_list = self.curr_channel_list - self.main_idx = self.idx - - self.update_return_df(first_end_row, no_comment=True) - self.idx += 1 - - self.condition_two() - - else: - self.state_two() - def condition_two(self): + # Save satellite and instrument name + self.sat = self.instr_df.iloc[0]['sat'] + self.instr = self.instr_df.iloc[0]['instr'] - ''' - If curr start/end is in main range, go to condition 3. Otherwise - go to state 2 - ''' - - # Return if end of df is reached - if (self.idx == len(self.start_times)): - return - - # Get current and main start and end time - curr_start = self.start_times[self.idx] - curr_end = self.end_times[self.idx] - - main_start = self.start_times[self.main_idx] - main_end = self.end_times[self.main_idx] - - if ((int(curr_start) > int(main_start)) and (int(curr_end) <= int(main_end))): - self.state_four() - else: - self.state_two() - - def state_four(self): - - ''' - Update current channel list by whether values need to be turned - on or turned off. Then update the return df, increment the index - and go to condition 2. - ''' - - row = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] - row_channel_list = row["channels"].values[0] - - # If these are the same, the logic is off - assert (len(row_channel_list) != len(self.compare_channels)) - self.curr_channel_list = self.main_channel_list - - if (len(row_channel_list) > len(self.compare_channels)): - # Turn on - turn_on = list(set(row_channel_list) - set(self.compare_channels)) - self.curr_channel_list += turn_on - else: - # Turn off - turn_off = list(set(self.compare_channels) - set(row_channel_list)) - self.curr_channel_list = [x for x in self.curr_channel_list if x not in turn_off] - - self.update_return_df(row) - self.idx += 1 - - self.condition_two() - - def condition_four(self): - - ''' - If next date range is right after previous date range, go to - condition one. Otherwise, go to state 6. If there's no next - then return. - ''' - - assert (self.idx != 0) - if (self.idx == len(self.start_times)): - return - else: - assert (self.end_times[self.idx-1] != self.start_times[self.idx]) + # Initialize return dataframe + self.return_df = pd.DataFrame(columns=list(instr_df.columns.values)) - prev_end_dt = dt.strptime(self.end_times[self.idx-1], '%Y%m%d%H%M%S') - next_start_dt = dt.strptime(self.start_times[self.idx], '%Y%m%d%H%M%S') - diff = next_start_dt - prev_end_dt + # Create lists for start and end times + end_times = [] + start_times = list(np.unique(self.instr_df['start'].values)) + start_times.sort(key=int) + for start in start_times: + end_times.append(self.instr_df.loc[self.instr_df['start'] == start]['end'].values[0]) + + done = [] + for idx in range(len(start_times)): + main_start = start_times[idx] + main_end = end_times[idx] + + # If time has already been processed, then skip + if main_start in done: + continue + + start_df = self.instr_df.loc[self.instr_df['start'] == main_start] + if (len(start_df) == 1): + # Only one row to process + channel_list = start_df['channels'].values[0] + comment = start_df['comments'].values[0] + self.update_return_df(main_start, main_end, channel_list, comment) + done.append(main_start) + + elif len(np.unique(start_df['end'].values)) == 1: + # Collect channels and update dataframe with row + channel_list = self.get_channel_list(main_start) + self.update_return_df(main_start, main_end, channel_list, '') + done.append(main_start) - if diff == datetime.timedelta(hours=6): - self.condition_one() else: - self.state_six() + # Collect all channels for main start time + channel_list = self.get_channel_list(main_start) + + inner_start = [] + inner_end = [] + # Get list of remaining start times that fall in range of start and end + for inner_idx in range(idx+1, len(start_times)): + if (int(start_times[inner_idx]) >= int(main_start)) and \ + (int(start_times[inner_idx]) <= int(main_end)): + inner_start.append(start_times[inner_idx]) + inner_end.append(end_times[inner_idx]) + + # Update df for start to first start with all channels + new_end = str(int(inner_start[0])) + self.update_return_df(main_start, new_end, channel_list, '') + + # Get channel list for main start/ main end time + row_channel_list = start_df.loc[start_df["end"] != main_end]["channels"].values[0] + + for inner_idx in range(len(inner_start)): + # Compare channels from next time range + row = self.instr_df.loc[self.instr_df["start"] == inner_start[inner_idx]] + compare_channels = row["channels"].values[0] + + # Turn channels on or off + if (len(row_channel_list) > len(compare_channels)): + # Turn off + turn_off = list(set(row_channel_list) - set(compare_channels)) + channel_list = [x for x in channel_list if x not in turn_off] + else: + # Turn on + turn_on = list(set(compare_channels) - set(row_channel_list)) + channel_list += turn_on + + # Update row + comment = row['comments'].values[0] + # pass dictionary to update return df + self.update_return_df(inner_start[inner_idx], inner_end[inner_idx], + channel_list, comment) + done.append(inner_start[inner_idx]) + + def update_return_df(self, start, end, channel_list, comment): + + # Fix end time if on the 24 hour mark + end = check_end_time(end) + + new_row = pd.DataFrame.from_dict({ + 'sat': [self.sat], + 'start': [start], + 'end': [end], + 'instr': [self.instr], + 'channel_num': [len(channel_list)], + 'channels': [channel_list], + 'comments': [comment]}) - def state_six(self): - - ''' - Create new empty date range, update return df with new row, and then - head over to state one. No update to the index. - ''' - - missing_time = {} - - prev_end = dt.strptime(self.end_times[self.idx-1], '%Y%m%d%H%M%S') - miss_begin_time = prev_end + datetime.timedelta(hours=6) - missing_time['begin_time'] = (miss_begin_time).strftime('%Y%m%d%H%M%S') - - curr_start = dt.strptime(self.start_times[self.idx], '%Y%m%d%H%M%S') - miss_end_time = curr_start - datetime.timedelta(hours=6) - missing_time['end_time'] = (miss_end_time).strftime('%Y%m%d%H%M%S') - - row = self.instr_df.loc[self.instr_df["start"] == self.start_times[self.idx]] - self.update_return_df(row, missing=True, missing_time=missing_time) - - self.condition_one() + self.return_df = pd.concat([self.return_df, new_row], ignore_index=True) def get_instr_df(self): ''' Returns the dataframe that the state machine generated! ''' return self.return_df - - def update_return_df(self, row, no_comment=False, missing=False, missing_time={}): - - ''' Adding rows to final dataframe that will be returned through get_instr_df ''' - - # Updates the return df based on parameters - if (missing): - new_row = pd.DataFrame.from_dict({ - 'sat': [row["sat"].values[0]], - 'start': [missing_time['begin_time']], - 'end': [missing_time['end_time']], - 'instr': [row['instr'].values[0]], - 'channel_num': [0], - 'channels': [[]], - 'comments': ['missing for this period']}) - - elif (no_comment): - new_row = pd.DataFrame.from_dict({ - 'sat': [row["sat"].values[0]], - 'start': [self.start_times[self.idx]], - 'end': [self.end_times[self.idx]], - 'instr': [row['instr'].values[0]], - 'channel_num': [len(self.curr_channel_list)], - 'channels': [self.curr_channel_list], - 'comments': ['']}) - - else: - new_row = pd.DataFrame.from_dict({ - 'sat': [row["sat"].values[0]], - 'start': [self.start_times[self.idx]], - 'end': [self.end_times[self.idx]], - 'instr': [row['instr'].values[0]], - 'channel_num': [len(self.curr_channel_list)], - 'channels': [self.curr_channel_list], - 'comments': [row["comments"].values[0]]}) - - self.return_df = pd.concat([self.return_df, new_row], ignore_index=True) diff --git a/src/swell/utilities/observing_system_records.py b/src/swell/utilities/observing_system_records.py index 3cf6eccd..25c10275 100644 --- a/src/swell/utilities/observing_system_records.py +++ b/src/swell/utilities/observing_system_records.py @@ -11,7 +11,6 @@ def format_date(old_date): ''' Formatting date into expected template ''' - date = dt.datetime.strptime(old_date, '%Y%m%d%H%M%S') return date.isoformat() From 81b9e04ec87cb48cd6f14a5fd3021acded786c83 Mon Sep 17 00:00:00 2001 From: asewnath Date: Wed, 20 Dec 2023 16:31:40 -0500 Subject: [PATCH 37/38] swell test fix --- src/swell/utilities/gsi_record_parser.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/swell/utilities/gsi_record_parser.py b/src/swell/utilities/gsi_record_parser.py index c481ecf7..4a066f75 100644 --- a/src/swell/utilities/gsi_record_parser.py +++ b/src/swell/utilities/gsi_record_parser.py @@ -1,7 +1,5 @@ import pandas as pd import numpy as np -from datetime import datetime as dt - def check_end_time(end_time): From 34a0f3c9d7b1befb2d09b8a9be3ac3bf48abae6d Mon Sep 17 00:00:00 2001 From: asewnath Date: Thu, 21 Dec 2023 09:38:59 -0500 Subject: [PATCH 38/38] coding norms --- src/swell/utilities/gsi_record_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/swell/utilities/gsi_record_parser.py b/src/swell/utilities/gsi_record_parser.py index 4a066f75..c0b8a9aa 100644 --- a/src/swell/utilities/gsi_record_parser.py +++ b/src/swell/utilities/gsi_record_parser.py @@ -1,6 +1,7 @@ import pandas as pd import numpy as np + def check_end_time(end_time): ''' Checks end times for 24 hour strings and converts them to 18 '''