Merge pull request #310 from juanchossn/dev

Solve Issue #307 (bug on ECMWF retrievals) + documentation udpate + bug BRDF

Main.py

@@ -22,7 +22,7 @@
 from Source.Controller import Controller
 from Source.ConfigFile import ConfigFile
 from Source.ConfigWindow import ConfigWindow
-import Source.GetAnc_credentials as credentials
+from Source.GetAnc_credentials import GetAnc_credentials
 from Source.SeaBASSHeader import SeaBASSHeader
 from Source.SeaBASSHeaderWindow import SeaBASSHeaderWindow
 from Source.Utilities import Utilities
@@ -783,8 +783,8 @@ def __init__(
         os.environ["HYPERINSPACE_CMD"] = "TRUE" # Must be a string
 
         # Pop up credential windows if credentials not stored...
-        credentials.credentialsWindow('NASA_Earth_Data')
-        credentials.credentialsWindow('ECMWF_ADS')
+        GetAnc_credentials.credentialsWindow('NASA_Earth_Data')
+        GetAnc_credentials.credentialsWindow('ECMWF_ADS')
 
         Command(configFilePath, inputFile, multiLevel, outputDirectory, level, ancFile)
     else:

README_ancillary.md

@@ -0,0 +1,90 @@
+# Ancillary Sources Used in HyperCP: GMAO/MERRA-2 reanalysis and ECMWF/CAMS forecast
+
+HyperCP uses ancillary information such as wind speed, aerosol optical depth, salinity, and sea surface temperature 
+during the following steps:
+
+- L1BQC processing: QC screening uses wind speed to filter the data for minimizing glint contamination
+- L2 processing: ancillary information is also required to calculate the [L2 Sky/Sunglint Correction](README_configuration/#l2-sky-sunglint-correction-rho-and-nir-correction) 
+  - Mobley (1999) rho factors require wind speed
+  - Zhang et al. 2017 rho factors require wind speed, aerosol optical depth, salinity, and sea surface temperature
+
+Since most field collections of above water radiometry are missing some or all of these ancillary parameters (though 
+they can be input in the Ancillary file, if available), an embedded function allows the user to download model data from
+two possible sources (to be selected by the user):
+
+- NASA [Global Modeling and Assimilation Office (GMAO)](https://gmao.gsfc.nasa.gov/) data are 
+ requested to the NASA EARTHDATA server. GMAO data are generated by the as hourly, 
+ global [MERRA2](https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/) HDF files at 0.5 deg (latitude) by 0.625 deg (longitude) resolution.
+ Two files will be downloaded for each hour of data processed (total ~8.3 MB for one hour of field data) and stored in
+ /Data/Anc. Global ancillary data files from GMAO will be reused, so it is not recommended to clear this directory unless 
+ updated models are being released by GMAO.
+
+        Note: MERRA2 files downloaded prior to March 15, 2022 can be deleted as the file name format has changed
+- ECMWF [CAMS global atmospheric composition forecasts](https://ads.atmosphere.copernicus.eu/datasets/cams-global-atmospheric-composition-forecasts?tab=overview)
+data are requested to ECMWF via the [ADS API](https://confluence.ecmwf.int/display/CKB/Atmosphere+Data+Store+%28ADS%29+documentation).
+One netCDF file at 0.4 degree (lat/lon) is downladed for each hour of processed data (totalling ~ 45K for one hour of
+- filed data) and stored in /Data/Anc. Ancillary data files are downloaded only at the 0.4 x 0.4 pixel
+cotaining the lat/lon reported for the field acquisition. 
+
+## Obtain the credentials
+
+When either of the two sources is selected in the L1B section of the [Configuration Window](#README_configuration), a 
+window will pop-up to insert the required credentials. Please note, that the ancillary source may be selected by default
+for a given setup, meaning that the window may pop-up before opening the configuration window for edition.
+
+### GMAO MERRA2 credentials
+
+A pop-up window like this will show up:
+
+<left><img src="Data/Img/EARTHDATA_ACCESS_popup.png" alt="banner"></left>
+
+Access to GMAO MERRA2 data requires a user login and password, which can be obtained for free
+[here](https://urs.earthdata.nasa.gov/):
+
+<left><img src="Data/Img/EARTHDATA_ACCESS.png" alt="banner"></left>
+
+The login and password will be stored under ```~/.netrc``` (or ```~/_netrc``` for Windows users).
+
+### ECMWF CAMS GACF credentials
+
+A pop-up window like this will show up:
+
+<left><img src="Data/Img/ECMWF_popup.png" alt="banner"></left>
+
+First, you must go [here](https://ads.atmosphere.copernicus.eu/) and register for free:
+
+<left><img src="Data/Img/ECMWF_register0.png" alt="banner"></left>
+
+Once registered and logged in your URL and KEY API credentials should show [here](https://ads.atmosphere.copernicus.eu/how-to-api):
+
+<left><img src="Data/Img/ECMWF_api_key.png" alt="banner"></left>
+
+Please insert URL and KEY in the pop-up window. Please note:
+
+- The API URL and KEY that you need to insert in the HyperCP pop-up window are these last ones and are different from your ECMWF credentials!
+- This process is agnostic to your operative system (disregard the options shown by ECMWF with respect to this)
+
+Finally *don't forget* to accept the Terms and Conditions scrolling all the way down in the 
+[CAMS GACF Download tab](https://ads.atmosphere.copernicus.eu/datasets/cams-global-atmospheric-composition-forecasts?tab=download):
+
+<left><img src="Data/Img/ECMWF_terms.png" alt="banner"></left>
+
+## When should I use MERRA2 and when CAMS?
+
+We *do not* offer a comparison in terms of quality and precision of the two ancillary sources. The user is left
+to perform their own independent investigation.
+
+- Global MERRA2 hourly ancillary model data are not available until the calendar month following the model date.
+This may lead to a 401 error if you are trying to acquire MERRA2 data within the month being processed. In this situation
+the user should switch to using the ECMWF model or the default fall-back values based on best estimates.
+
+- On the other hand, even if ECMWF's CAMS-GACF dataset is a forecast source, temporary downfalls of the ECMWF API may occur.
+If your field acquisitions are older than a calendar month, and you are experiencing slow downloads, please consider switching to MERRA-2.
+
+- CAMS data are not available for field acquisitions done before 2015.
+
+## What is the fallback in HyperCP in case ancillary information is missing?
+
+These ancillary data from models will be incorporated if field data are not available in the Ancillary file provided
+in the Main window. If field data and model data are both inaccessible for any reason, the system will use the Default
+values (i.e., Wind Speed, AOD, Salinity, and SST) provided in the L1BQC Configuration setup.

README_configuration.md

@@ -6,7 +6,7 @@ Launch the configuration module and GUI (ConfigWindow.py) from the Main window b
  Some cruises (e.g. moving between significantly different water types) may also require multiple configurations to
  obtain the highest quality ocean color products at Level 2. Sharp gradients in environmental conditions could also
  warrant multiple configurations for the same cruise (e.g. sharp changes in air temperature may effect how data
- deglitching is parameterized, as described [below]).
+ deglitching is parameterized, as described [below](README_deglitching.md)).
 
 The configuration window looks like this:
 
@@ -190,31 +190,14 @@ when applying calibration to the dark current corrected radiometry, the offsets
 (see ProSoftUserManual7.7 11.1.1.5 Eqns 5-6) presuming light and dark factory cals are equivalent (which they
 historically have been from Satlantic/Sea-Bird).
 
-Use of the Mobley (1999) and Zhang et al. 2017 glint corrections require wind data, and Zhang (2017) also requires
-aerosol optical depth, salinity, and sea surface temperature. L1BQC processing also uses wind speed to filter the data
-for minimizing glint contamination. Since most field collections of above water radiometry are missing some or all of
-these ancillary parameters (though they can be input in the Ancillary file, if available), an embedded function allows
-the user to download model data from the NASA EARTHDATA server. These data are generated by the NASA Global Modeling and
- Assimilation Office (GMAO) as hourly, global 'MERRA2' HDF files at 0.5 deg (latitude) by 0.625 deg (longitude)
- resolution (https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/). Two files will be downloaded for each hour of data
- processed (total ~8.3 MB for one hour of field data) and stored in /Data/Anc. Global ancillary data files from GMAO
- will be reused, so it is not recommended to clear this directory unless updated models are being released by GMAO.
- *(Note: MERRA2 files downloaded prior to March 15, 2022 can be deleted as the file name format has changed.)*
- Details for how these data are applied to above water radiometry are given below.
-
-Access to GMAO MERRA2 data requires a user login and password, which can be obtained for free
-[here](https://www.earthdata.nasa.gov/eosdis/science-system-description/eosdis-components/earthdata-login). A link to register is also provided in the Configuration window at L1BQC. When the user selects ```Download Ancillary Models```, pop-up windows will allow the user to enter a login and
-password. Once this has been done once, canceling the login pop-up dialog will force the program to use the current
-configuration (i.e. it is only necessary to re-enter the password if it has changed.)
-
-*Note: Global MERRA2 hourly ancillary model data are not available until the calendar month following the model date.* 
-This may lead to a 401 error if you are trying to acquire MERRA2 data within the month being processed. In this situation
-the user should switch to using the ECMWF model or the default fall-back values based on best estimates.
-
-These ancillary data from models will be incorporated if field data are not available in the Ancillary file provided
-in the Main window. If field data and model data are both inaccessible for any reason, the system will use the Default
-values (i.e., Wind Speed, AOD, Salinity, and SST) provided in the L1BQC Configuration setup here.
+### L1B: Ingestion of ancillary information for posterior processing
 
+At Level 1B, ancillary information will be queried from either [GMAO's MERRA](https://gmao.gsfc.nasa.gov/reanalysis/merra-2/)
+or [ECMWF's CAMS GACF](https://ads.atmosphere.copernicus.eu/) reanalysis/forecast model databases and used in
+posterior processing. To know more about these sources, how they are used in HyperCP and how to obtain the required
+access credentials, read [here](README_ancillary.md).
+
+### L1B: Calibration regimes
 Three calibration/characterization regimes are available:
 
 **Factory:**
@@ -386,14 +369,17 @@ Remote sensing reflectance is then calculated as
 
 $$
 \displaystyle
-Rrs = \frac{L_{t} - \rho_{sky}.L_{i}}{E_{s}}
+Rrs = \frac{L_{t} - \rho_{sky}.L_{i}}{E_{s}} = \frac{L_{w}}{E_{s}},
 $$
 
+where $L_{w}$ is the water leaving radiance.
+
 (e.g. Mobley 1999, Mueller et al. 2003, Ruddick et al. 2006)).
 Normalized water leaving radiance (nLw) is calculated as $Rrs.F0$, where F0 is the top of atmosphere incident radiation
 adjusted for the Earth-Sun distance on the day sampled. This is now estimated using the Coddington et al. (2021) TSIS-1
 hybrid model.
 
+
 Uncertainties in $L_{i}$, $L_{t}$, $E_{s}$ ($u(L_{i})$, $u(L_{t})$, $u(E_{s})$ ) are derived as the quadrature sum of uncertainties associated with standard error (i.e., variability among samples) and instrument uncertainties based on laboratory characterization of a specific instrument or class of instruments. Uncertainty in the skylight reflectance factor, $u(\rho_{sky})$, was historically estimated as +/- 0.003 from Ruddick et al. 2006 Appendix 2 (intended for clear skies), but in HyperCP v1.2+ is estimated using Monte Carlo iterations perturbing the input solar-sensor geometries and environmental conditions over normal distributions around the current measurement in addition to differences between multiple models for $\rho_{sky}$ (i.e., Mobley 1999 and Zhang et al. 2017).
 
 Combined absolute standard uncertainty ( $u_{c}$) in $L_{w}$ ($u_{c}(L_{w})$ ) is estimated from $u(L_{i})$, $u(L_{t})$, and $u(\rho_{sky})$ with the Law of Propagation of Uncertainties (LPU) assuming random, uncorrelated error. LPU defines combined standard uncertainty, $u_{c}$ as:
@@ -456,7 +442,18 @@ Negative reflectances may be removed as follows: any spectrum with any negative
 is removed from the record entirely. Negative reflectances outside of this range (e.g. noisy data deeper in the NIR) are
 set to 0.
 
-TODO: describe BRDF correction
+### BRDF correction
+
+A correction factor for bi-directional effects (often called "BRDF correction") must be applied to "convert" the measured 
+remote-sensing reflectance (or equivalently, the nomralised water-leaving radiance) from the given 
+illumination-observation geometry to the "normalised" geometry (Sun at zenith, water-leaving radiance acquired at nadir). 
+
+HyperCP supports two BRDF schemes:
+- [Morel et al. 2002](https://opg.optica.org/ao/abstract.cfm?uri=ao-41-30-6289) (the so-called "Chlorophyll-based" approach), and
+- [Lee et al. 2011](https://opg.optica.org/ao/fulltext.cfm?uri=ao-50-19-3155&id=219080) (the so-called "IOP-based" apprach)
+
+The Python module to calculate the BRDF scheme was developed as part of 
+[this](https://www.eumetsat.int/brdf-correction-s3-olci-water-reflectance-products) Copernicus Programme - EUMETSAT study.
 
 ### L2 Products
 

Source/ConfigWindow.py

@@ -11,7 +11,7 @@
 from Source.AnomalyDetection import AnomAnalWindow
 from Source.SeaBASSHeader import SeaBASSHeader
 from Source.SeaBASSHeaderWindow import SeaBASSHeaderWindow
-from Source import GetAnc_credentials as credentials
+from Source.GetAnc_credentials import GetAnc_credentials
 from Source.OCproductsWindow import OCproductsWindow
 
 
@@ -478,14 +478,14 @@ def initUI(self):
         if int(ConfigFile.settings["bL1bGetAnc"]) == 1:
             self.l1bGetAncCheckBox1.setChecked(True)
             self.l1bGetAncCheckBox2.setChecked(False)
-            credentials.credentialsWindow('NASA_Earth_Data')
+            GetAnc_credentials.credentialsWindow('NASA_Earth_Data')
             self.l1bGetAncUntickIfNoCredentials('NASA_Earth_Data')
 
         # Case: ECMWF ADS (tick box before window pops up)
         if int(ConfigFile.settings["bL1bGetAnc"]) == 2:
             self.l1bGetAncCheckBox1.setChecked(False)
             self.l1bGetAncCheckBox2.setChecked(True)
-            credentials.credentialsWindow('ECMWF_ADS')
+            GetAnc_credentials.credentialsWindow('ECMWF_ADS')
             self.l1bGetAncUntickIfNoCredentials('ECMWF_ADS')
 
         # Case: NO ancillary selected (disable Zhang before config window pops-up)
@@ -1669,7 +1669,7 @@ def l1bGetAncUntickIfNoCredentials(self,ancillarySource):
         - If credentials are found set bL1bGetAnc corresp. to given ancillarySource and enable Zhang glint correction option
         - If not: set bL1bGetAnc = 0 (consequently, Zhang will be disabled after this function)
         '''
-        if credentials.credentials_stored(ancillarySource):
+        if GetAnc_credentials.credentials_stored(ancillarySource):
             if ancillarySource == 'NASA_Earth_Data':
                 ConfigFile.settings["bL1bGetAnc"] = 1
             elif ancillarySource == 'ECMWF_ADS':
@@ -1704,8 +1704,8 @@ def l1bGetAncResetButtonUpdate(self):
         # Erase pre-existing credentials, open pop-up window and untick resp. options if credentials not set...
         if ancillarySource:
             print('Reset %s credentials' % ancillarySource.replace('_', ' '))
-            credentials.erase_user_credentials(ancillarySource)
-            credentials.credentialsWindow(ancillarySource)
+            GetAnc_credentials.erase_user_credentials(ancillarySource)
+            GetAnc_credentials.credentialsWindow(ancillarySource)
             self.l1bGetAncUntickIfNoCredentials(ancillarySource)
 
         # NB: This is not the same as an "if not ancillarySource": bL1bGetAnc = 0 is set after "l1bGetAncUntickIfNoCredentials" is triggered.
@@ -1743,12 +1743,12 @@ def l1bGetAncCheckBoxUpdate(self,ancillarySource):
         if self.l1bGetAncCheckBox1.isChecked():
             print("ConfigWindow - l1bGetAncCheckBoxUpdate GMAO MERRA2")
             ConfigFile.settings["bL1bGetAnc"] = 1
-            credentials.credentialsWindow('NASA_Earth_Data')
+            GetAnc_credentials.credentialsWindow('NASA_Earth_Data')
             self.l1bGetAncUntickIfNoCredentials('NASA_Earth_Data')
         elif self.l1bGetAncCheckBox2.isChecked():
             print("ConfigWindow - l1bGetAncCheckBoxUpdate ECMWF CAMS")
             ConfigFile.settings["bL1bGetAnc"] = 2
-            credentials.credentialsWindow('ECMWF_ADS')
+            GetAnc_credentials.credentialsWindow('ECMWF_ADS')
             self.l1bGetAncUntickIfNoCredentials('ECMWF_ADS')
         else:
             ConfigFile.settings["bL1bGetAnc"] = 0