Add task: CloneGmaoPerllib. Allow use of acquire and acquire_obsys fo…

…r convert_bufr suite

src/swell/deployment/platforms/nccs_discover_sles15/task_questions.yaml

@@ -25,6 +25,9 @@ geos_experiment_directory:
 geos_restarts_directory:
   default_value: restarts_20210601_030000
 
+gmao_perllib_path:
+  default_value: /discover/nobackup/projects/gmao/advda/perllib_opt/GMAO_perllib/g1.0.1/
+
 r2d2_local_path:
   default_value: /discover/nobackup/${USER}/R2D2DataStore/Local
 

src/swell/suites/convert_bufr/flow.cylc

@@ -0,0 +1,97 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+# --------------------------------------------------------------------------------------------------
+
+# Cylc suite for executing geos_atmosphere ObsFilters tests
+
+# --------------------------------------------------------------------------------------------------
+
+[scheduler]
+    UTC mode = True
+    allow implicit tasks = False
+
+# --------------------------------------------------------------------------------------------------
+
+[scheduling]
+
+    initial cycle point = {{start_cycle_point}}
+    final cycle point = {{final_cycle_point}}
+    runahead limit = {{runahead_limit}}
+
+    [[graph]]
+        R1 = """
+            # Triggers for non cycle time dependent tasks
+            # -------------------------------------------
+            # Clone JEDI source code
+            CloneJedi
+
+            # Clone GMAO_perllib
+            CloneGMAOPerllib
+
+            # Build JEDI source code by linking
+            CloneJedi => BuildJediByLinking?
+
+            # If not able to link to build create the build
+            BuildJediByLinking:fail? => BuildJedi
+        """
+
+        {% for cycle_time in cycle_times %}
+        {{cycle_time.cycle_time}} = """
+
+            # Convert BUFR to ioda
+            GetBufr
+            GetBufr => BufrToIoda
+            BuildJediByLinking[^]? | BuildJedi[^]  => BufrToIoda
+
+            # Clean up
+            BufrToIoda => CleanCycle
+        """
+        {% endfor %}
+
+# --------------------------------------------------------------------------------------------------
+
+[runtime]
+
+    # Task defaults
+    # -------------
+    [[root]]
+        pre-script = "source $CYLC_SUITE_DEF_PATH/modules"
+
+        [[[environment]]]
+            datetime = $CYLC_TASK_CYCLE_POINT
+            config   = $CYLC_SUITE_DEF_PATH/experiment.yaml
+
+    # Tasks
+    # -----
+    [[CloneJedi]]
+        script = "swell task CloneJedi $config"
+
+    [[CloneGMAOPerllib]]
+	script = "swell task CloneGmaoPerllib $config"
+
+    [[BuildJediByLinking]]
+        script = "swell task BuildJediByLinking $config"
+
+    [[BuildJedi]]
+        script = "swell task BuildJedi $config"
+        platform = {{platform}}
+        execution time limit = {{scheduling["BuildJedi"]["execution_time_limit"]}}
+        [[[directives]]]
+        {%- for key, value in scheduling["BuildJedi"]["directives"]["all"].items() %}
+            --{{key}} = {{value}}
+        {%- endfor %}
+
+    [[ GetBufr ]]
+        script = "swell task GetBufr $config -d $datetime -m geos_atmosphere"
+
+    [[ BufrToIoda ]]
+        script = "swell task BufrToIoda $config -d $datetime -m geos_atmosphere"
+
+    [[CleanCycle]]
+        script = "swell task CleanCycle $config -d $datetime -m geos_atmosphere"
+
+# --------------------------------------------------------------------------------------------------

src/swell/suites/suite_questions.yaml

@@ -10,6 +10,7 @@ cycle_times:
   - 3dvar_atmos
   - 3dvar_cycle
   - 3dvar
+  - convert_bufr
   - convert_ncdiags
   - forecast_geos
   - hofx
@@ -74,6 +75,7 @@ start_cycle_point:
   - 3dvar_atmos
   - 3dvar_cycle
   - 3dvar
+  - convert_bufr
   - convert_ncdiags
   - forecast_geos
   - hofx
@@ -91,6 +93,7 @@ final_cycle_point:
   - 3dvar_atmos
   - 3dvar_cycle
   - 3dvar
+  - convert_bufr
   - convert_ncdiags
   - forecast_geos
   - hofx
@@ -123,6 +126,7 @@ model_components:
   - 3dvar_atmos
   - 3dvar_cycle
   - 3dvar
+  - convert_bufr
   - convert_ncdiags
   - hofx
   - localensembleda
@@ -140,6 +144,7 @@ runahead_limit:
   - 3dvar_atmos
   - 3dvar
   - 3dvar_cycle
+  - convert_bufr
   - convert_ncdiags
   - hofx
   - localensembleda

src/swell/tasks/clone_gmao_perllib.py

@@ -0,0 +1,76 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+import os
+import subprocess
+
+from swell.tasks.base.task_base import taskBase
+
+# --------------------------------------------------------------------------------------------------
+
+
+class CloneGmaoPerllib(taskBase):
+
+    def execute(self) -> None:
+
+        # Get experiment directory where GMAO_perllib will be cloned/linked
+        swell_exp_path = self.experiment_path()
+        experiment_perllib_path = os.path.join(swell_exp_path, 'GMAO_perllib')
+
+        # Get the existing location of GMAO_perllib
+        existing_perllib_path = self.config.gmao_perllib_path(None)
+        gmao_perllib_tag = self.config.gmao_perllib_tag(None)
+
+        # Set the default tag to g1.0.1 if not specified
+        if gmao_perllib_tag is None:
+            gmao_perllib_tag = 'g1.0.1'
+            self.logger.info("The target branch for gmao_perllib, 'gmao_perllib_branch', is unspecified in the experiment config file, " +
+                            f"defaulting to {gmao_perllib_tag}")
+
+        # Don't clone or link if GMAO_perllib already exists in experiment directory
+        if os.path.exists(experiment_perllib_path):
+            self.logger.info(f"{experiment_perllib_path} exists, defaulting to this installation")
+
+        # Clone from github if existing perllib is unspecified
+        elif existing_perllib_path is None:
+            self.logger.info("The path to GMAO_perllib, 'gmao_perllib_path'" +
+                                "has not been specified in experiment.yaml. CloneGMAOPerllib will now attempt to clone" +
+                             " it from Github, which will fail on compute nodes with no internet access.")
+            subprocess.run(f"git clone -b {gmao_perllib_tag} https://github.com/GEOS-ESM/GMAO_perllib.git "
+                           + os.path.join(experiment_path()))
+
+        # Link to existing GMAO_perllib
+        elif os.path.exists(existing_perllib_path):
+
+            # Check that the tag of the existing repository matches the intended tag
+            existing_tag = self.get_tag(existing_perllib_path)
+            if not existing_tag or existing_tag != gmao_perllib_tag:
+                self.logger.debug("The git tag of the existing GMAO_perllib repository, {existing_perllib_path}, does not match " +
+                               "the specified tag {gmao_perllib_tag}. Please check the directory and make sure the installation is correct.")
+
+            # Link the existing directory to the experiment directory
+            os.symlink(existing_perllib_path, experiment_perllib_path)
+
+        # Check to make sure the path contains acquire and acquire_obsys
+        if len(list(set(['acquire', 'acquire_obsys']) & set(os.listdir(experiment_perllib_path)))) < 2:
+            self.logger.abort(f"{experiment_perllib_path} does not contain acquire and acquire_obsys")
+
+    # --------------------------------------------------------------------------------------------------
+
+    def get_tag(self, path):
+        try:
+            prc = subprocess.run(['git', 'describe'], capture_output=True, text=True, check=True)
+            return prc.stdout.strip()
+        except subprocess.CalledProcessError:
+            return None
+
+
+
+# --------------------------------------------------------------------------------------------------

src/swell/tasks/task_questions.yaml

@@ -398,6 +398,22 @@ geovals_provider:
   - GetGeovals
   type: string
 
+gmao_perllib_path:
+  ask_question: false
+  default_value: defer_to_platform
+  prompt: What is the path to where the GMAO_perllib executables, acquire_obsys and acquire, are located?
+  tasks:
+  - CloneGmaoPerllib
+  type: string
+
+gmao_perllib_tag:
+  ask_question: false
+  default_value: g1.0.1
+  prompt: What is the target tag for GMAO_perllib?
+  tasks:
+  - CloneGmaoPerllib
+  type: string
+
 gradient_norm_reduction:
   ask_question: false
   default_value: defer_to_model
@@ -734,6 +750,7 @@ observations:
   options: defer_to_model
   prompt: Which observations do you want to include?
   tasks:
+  - BufrToIoda
   - EvaObservations
   - GenerateObservingSystemRecords
   - GetGeovals
@@ -794,6 +811,14 @@ observing_system_records_path:
   - SaveObsDiags
   type: string
 
+path_to_bufr:
+  ask_question: true
+  default_value: None
+  prompt: What is the path to the BUFR files to convert?
+  tasks:
+  - GetBufr
+  type: string
+
 path_to_ensemble:
   ask_question: true
   default_value: defer_to_model
@@ -846,6 +871,7 @@ produce_geovals:
   prompt: When running the ncdiag to ioda converted do you want to produce GeoVaLs
     files?
   tasks:
+  - BufrToIoda
   - GsiNcdiagToIoda
   type: boolean
 
@@ -882,6 +908,7 @@ single_observations:
   - false
   prompt: Is it a single-observation test?
   tasks:
+  - BufrToIoda
   - GsiNcdiagToIoda
   - RunJediUfoTestsExecutable
   type: boolean
@@ -1060,6 +1087,7 @@ window_offset:
   - all
   prompt: What is the duration between the middle of the window and the beginning?
   tasks:
+  - BufrToIoda
   - EvaIncrement
   - EvaObservations
   - GenerateBClimatology

src/swell/tasks/use_acquire.py

@@ -0,0 +1,50 @@
+# (C) Copyright 2021- United States Government as represented by the Administrator of the
+# National Aeronautics and Space Administration. All Rights Reserved.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+
+# --------------------------------------------------------------------------------------------------
+
+
+import os
+import subprocess
+
+from swell.tasks.base.task_base import taskBase
+
+# --------------------------------------------------------------------------------------------------
+
+
+class UseAcquire(taskBase):
+
+    def execute(self) -> None:
+
+        swell_exp_path = self.experiment_path()
+        perl_executable_path = os.path.join(swell_exp_path, 'GMAO_perllib')
+
+        env_dict = {"PATH": os.environ.get('PATH') + ":" +perl_executable_path}
+
+        subprocess.run(['acquire_obsys'], env=env_dict)
+
+        """
+        Translated c-shell to python:
+        satbfrdb = os.path.join(swell_exp_path, "GEOS_mksi", "ObsClass")
+        my_exp = "/gpfsm/dnb05/projes/p139/rtodling/TEST/GETBUFR/"
+
+        work = os.path.join(my_exp, "work")
+        spool = os.path.join(my_exp, "spool")
+
+        obs_class = "disc_airs_bufr,disc_amsua_bufr,gmao_amsr2_bufr,gmao_gmi_bufr,mls_nrt_nc,ncep_1bamua_bufr,ncep_acftpfl_bufr,ncep_atms_bufr,ncep_aura_omi_bufr,ncep_avcsam_bufr,ncep_avcspm_bufr,ncep_crisfsr_bufr,ncep_goesfv_bufr,ncep_gpsro_bufr,ncep_mhs_bufr,ncep_mtiasi_bufr,ncep_prep_bufr,ncep_satwnd_bufr,ncep_ssmis_bufr,ncep_tcvitals,npp_ompsnm_bufr,r21c_npp_ompslp_nc,m2scr_n21_ompslp_nc,gmao_mlst_bufr"
+
+        nymd = "20231010"
+        nhms = "120000"
+
+        nfreq = "060000"
+        nstep = "1"
+
+        subprocess.run(["acquire_obsys", "-v", "-d", work, "-s", spool, "strict", "-ssh", nymd, nhms, nfreq, nstep, obs_class])
+
+        """
+
+# --------------------------------------------------------------------------------------------------

src/swell/test/suite_tests/convert_bufr-tier1.yaml

@@ -0,0 +1,20 @@
+start_cycle_point: '2021-12-12T00:00:00Z'
+final_cycle_point: '2021-12-12T06:00:00Z'
+jedi_build_method: use_existing
+bundles: REMOVE
+model_components: ['geos_atmosphere']
+models:
+  geos_atmosphere:
+    cycle_times:
+    - T00
+    - T06
+    clean_patterns:
+    - gsi_bcs/*.nc4
+    - gsi_bcs/*.txt
+    - gsi_bcs/*.yaml
+    - gsi_bcs
+    - gsi_ncdiags/*.nc4
+    - gsi_ncdiags/aircraft/*.nc4
+    - gsi_ncdiags/aircraft
+    - gsi_ncdiags
+    path_to_gsi_nc_diags: /discover/nobackup/projects/gmao/advda/SwellTestData/ufo_testing/ncdiagv2/%Y%m%d%H