v1.1.0

.github/workflows/validate_yaml_files.yml

@@ -15,7 +15,7 @@ jobs:
   validate-YAML:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3.3.0
+      - uses: actions/checkout@v4
       - id: yaml-lint
         name: yaml-lint
         uses: ibiqlik/action-yamllint@v3
@@ -24,7 +24,7 @@ jobs:
           format: colored
           config_file: .yamllint.yml
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         if: always()
         with:
           name: yamllint-logfile

CHANGELOG.md

@@ -8,6 +8,23 @@
 
 ### Changed
 
+## [v1.1.0] 2025-07-02
+
+### Added
+
+- create an Applications folder for GAAS_App scripts that do the NNR pre-processing for the DAS
+- ability to train on the AE linear fit to spectral AOD
+- use TQV and TO3 as inputs
+
+### Fixed
+
+- Minor fix associated with installation location of software.
+
+### Changed
+
+- updated GMAOpyobs to latest v1.0.5
+- Update to ESMA_env v4.8.2 (Fixes for RHEL8 GMAO Machines)
+
 ## [v1.0.0] 2023-06-07
 
 ### Added

components.yaml

@@ -5,7 +5,7 @@ AeroML:
 env:
   local: ./env@
   remote: ../ESMA_env.git
-  tag: v4.8.0
+  tag: v4.8.2
   develop: main
 
 cmake:
@@ -22,5 +22,5 @@ ecbuild:
 GMAOpyobs:
   local: ./src/Shared/GMAO_Shared/GMAO_pyobs@
   remote: ../GMAOpyobs.git
-  branch: fix/pcastell/no-types-module
+  tag: v1.0.5
   develop: develop

src/Applications/CMakeLists.txt

@@ -0,0 +1 @@
+add_subdirectory (GAAS_App)

src/Applications/GAAS_App/CMakeLists.txt

@@ -0,0 +1,17 @@
+esma_set_this()
+
+set(pythonscripts modis_l2a.py mxd04_l2a.py avhrr_l2a.py patmosx_l2a.py)
+install(PROGRAMS ${pythonscripts}  DESTINATION bin)
+
+install(
+   FILES mxd04_nnr.py avhrr_nnr.py
+   DESTINATION lib/Python
+   )
+
+
+file(GLOB pcf_files *.pcf)
+
+install (
+   FILES ${pcf_files}
+   DESTINATION etc
+   )

src/Applications/GAAS_App/aeronet_all.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""
+Splits AERONET into synoptic chunks.
+"""
+
+import os
+import sys
+
+from datetime  import date, datetime, timedelta
+
+from pyobs.aeronet import AERONET_L2, granules
+
+
+#---------------------------------------------------------------------
+def makethis_dir(path):
+    """Creates the relevant directory if necessary."""
+    if path != '':
+        rc = os.system('mkdir -p '+path)
+        if rc:
+            raise IOError("could not create directory "+path)
+
+if __name__ == "__main__":
+
+    RootDir = '/nobackup/AERONET/MERRA-2'
+    ods_blank = '/nobackup/NNR/Misc/blank.ods'
+
+    oneday = timedelta(seconds=24*60*60)
+    onehour = timedelta(seconds=60*60)
+
+    if len(sys.argv)<2:
+        print("Usage:")
+        print("        aeronet4_all.py  year1 [year2]")
+        sys.exit(1)
+    else:
+        y1 = sys.argv[1]
+        if len(sys.argv)>2:
+            y2 = sys.argv[2]
+        else:
+            y2 = y1
+        Years = list(range(int(y1),int(y2)+1))
+
+    # Loop over years
+    # ---------------
+    for year in Years:
+        tyme0 = datetime(year,1,1)
+        toy = datetime(year,12,31) - tyme0
+        ndays = 1 + int(toy.total_seconds()/(24*60*60))
+        for doy in range(1,ndays+1):
+
+            today = tyme0 + (doy-1) * oneday
+
+            print('Day: ', today)
+
+            # Read AERONET for this day
+            # -------------------------
+            Files = granules(today,bracket='left')
+            a = AERONET_L2(Files,Verbose=True)
+
+            # Output directories
+            # ------------------
+            dirL2 = RootDir + '/Level2/ODS/Y%d/M%02d'%(today.year,today.month)
+            dirL3 = RootDir + '/Level3/Y%d/M%02d'%(today.year,today.month)
+
+            for h in range(0,24,3):
+
+                tyme = tyme0 + (doy-1) * oneday + h * onehour
+
+                # Make sure directories exist
+                # ---------------------------
+                makethis_dir(dirL2)
+                makethis_dir(dirL3)
+
+                # Write & compress the files
+                # --------------------------
+                fnL2, nobs = a.writeODS(tyme,dir=dirL2)                
+                fnL3 = a.writeGridded(tyme,dir=dirL3)
+
+            # Compress daily file
+            # -------------------
+            if a.nobs>0:
+                if os.system("n4zip "+fnL2+" > /dev/null"):
+                    warnings.warn('cannot compress output ODS file <%s>'%fnL2)
+                if os.system("n4zip "+fnL3+" > /dev/null"):
+                    warnings.warn('cannot compress output NC4 file <%s>'%fnL3)
+            else:
+                os.system("touch %s.empty"%fnL2)

src/Applications/GAAS_App/avhrr_all.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""
+Splits AVHRR into synoptic chunks.
+"""
+
+import os
+import sys
+
+from datetime  import date, datetime, timedelta
+
+if __name__ == "__main__":
+
+    oneday = timedelta(seconds=24*60*60)
+    onehour = timedelta(seconds=60*60)
+
+    if len(sys.argv)<2:
+        print("Usage:")
+        print("        avhrr_all.py  year1 [year2]")
+        sys.exit(1)
+    else:
+        y1 = sys.argv[1]
+        if len(sys.argv)>2:
+            y2 = sys.argv[2]
+        else:
+            y2 = y1
+        Years = list(range(int(y1),int(y2)+1))
+
+    # Loop over years
+    # ---------------
+    for year in Years:
+        tyme0 = datetime(year,1,1)
+        toy = datetime(year,12,31) - tyme0
+        ndays = 1 + int(toy.total_seconds()/(24*60*60))
+        for doy in range(1,ndays+1):
+
+            for h in range(0,24,3):
+
+                tyme = tyme0 + (doy-1) * oneday + h * onehour
+
+                nymd = tyme.year*10000 + tyme.month*100   + tyme.day
+                nhms = tyme.hour*10000 + tyme.minute*100  + tyme.second
+
+                cmd = 'python avhrr_l2a.py -v asc %d %d'%(nymd,nhms)
+
+                print(cmd)
+                os.system(cmd)

src/Applications/GAAS_App/avhrr_l2a.pcf

@@ -0,0 +1,25 @@
+#AVHRR_L2A  processing
+
+AVHRR_L2A_VERBOSE         = YES
+
+AVHRR_L2A_EXPID           = ${EXPID}
+AVHRR_L2A_ORBITS          = asc,des
+
+#--AVHRR_L2A_L2_DIR       = /archive/input/dao_ops/obs/reanalysis/patmosx/Synoptic
+#--AVHRR_L2A_OUT_DIR      = ./Results/obs/Level%lev/%prod/Y%y4/M%m2
+AVHRR_L2A_L2_DIR          = ${FVWORK}
+AVHRR_L2A_OUT_DIR         = ${FVWORK}
+
+AVHRR_L2A_OVERWRITE       = YES
+AVHRR_L2A_OUT_TEMPLATE    = '%s.%prod_L%leva.%orb.%y4%m2%d2_%h2%n2z.%ext'
+AVHRR_L2A_RESOLUTION      = e
+
+AVHRR_L2A_WIND_FILE       = ${EXPID}.gaas_bkg.sfc.%y4%m2%d2_%h2z.nc4
+AVHRR_L2A_TPW_FILE        = ${EXPID}.gaas_bkg.sfc.%y4%m2%d2_%h2z.nc4
+AVHRR_L2A_AOD_FILE        = ${EXPID}.gaas_bkg.sfc.%y4%m2%d2_%h2z.nc4
+
+AVHRR_L2A_NN_FILE         = ExtData/g5chem/x/NN/nnr_001.avhrr_Tau.net
+AVHRR_L2A_BLANK_ODS       = ExtData/g5chem/x/blank_syn8.ods
+AVHRR_L2A_COXMUNK_LUT     = ExtData/g5chem/x/avhrr.cox-munk_lut.npz
+
+#END AVHRR_L2A  processing

src/Applications/GAAS_App/avhrr_l2a.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+# -W ignore::DeprecationWarning
+
+"""
+
+  Simple wrapper script to parse Prep config file and create ODS with NNR AVHRR retrievals.
+
+  October 2013
+  arlindo.dasilva@nasa.gov
+"""
+
+from os       import system
+from optparse import OptionParser
+from MAPL     import Config
+
+if __name__ == "__main__":
+
+#   Parse command line options
+#   --------------------------
+    parser = OptionParser(usage="Usage: %prog prep_config_file nymd nhms",
+                          version='avhrr_l2a-1.0.0' )
+    parser.add_option("-n", "--dryrun",
+                      action="store_true", dest="dryrun",
+                      help="Dry run.")
+
+    (options, args) = parser.parse_args()
+
+    if len(args) == 3:
+        prep_config, nymd, nhms = args
+    else:
+        parser.error("must have 3 arguments: prep_config_filename nymd nhms")
+
+    # Parse prep config
+    # -----------------
+    cf = Config(prep_config,delim=' = ')
+
+    Options =     " --expid=" + cf('AVHRR_L2A_EXPID')        + \
+                 " --l2_dir=" + cf('AVHRR_L2A_L2_DIR')       + \
+                    " --res=" + cf('AVHRR_L2A_RESOLUTION')   + \
+                   "  --dir=" + cf('AVHRR_L2A_OUT_DIR')      + \
+                  " --fname=" + cf('AVHRR_L2A_OUT_TEMPLATE') + \
+                    " --net=" + cf('AVHRR_L2A_NN_FILE')      + \
+                   " --wind=" + cf('AVHRR_L2A_WIND_FILE')    + \
+                    " --tpw=" + cf('AVHRR_L2A_TPW_FILE')     + \
+                    " --aod=" + cf('AVHRR_L2A_AOD_FILE')     + \
+              " --blank_ods=" + cf('AVHRR_L2A_BLANK_ODS')    + \
+                " --coxmunk=" + cf('AVHRR_L2A_COXMUNK_LUT') 
+
+    if cf('AVHRR_L2A_OVERWRITE').upper() == 'YES': Options += " --force"
+    if   cf('AVHRR_L2A_VERBOSE').upper() == 'YES': Options += " -v"
+
+    # Generate products
+    # -----------------
+    i = 0
+    for orbit in cf('AVHRR_L2A_ORBITS').split(','):
+        cmd = "patmosx_l2a.py %s %s %s %s"%(Options,orbit,nymd,nhms)
+        print(cmd)
+        if not options.dryrun:
+            if system(cmd):
+                raise ValueError("patmosx_l2a.py failed for %s on %s %s"%(orbit,nymd,nhms))
+
+        i += 1
+
+