fixed unit conversion bug

Author: denebowdalo

configurations/test_forecast_raphael.conf

@@ -1,32 +1,28 @@
 [All]
-network = eea/eionet
-ghost_version = 1.5
-species = sconco3 #,sconco3,sconcco,pm10,pm2p5
-resolution = hourly # daily for interpolation, hourly/3hourly for download
-start_date = 20230101
-end_date = 20230201
+network = EEA,CHILE_SINCA,CNEMC,US_EPA_AQS,GHOST #,CHILE_SINCA,CNEMC,EEA,US_EPA_AQS,GHOST,CHILE_SINCA,CNEMC,EEA,US_EPA_AQS,GHOST
+ghost_version = 1.6
+species =  sconco3,sconcno2 #sconco3,sconcno2,sconcco #sconco3,sconco3,sconco3,sconco3,sconco3,sconcno2,sconcno2,sconcno2,sconcno2,sconcno2,sconcco,sconcco,sconcco,sconcco,sconcco #,pm10,pm2p5
+resolution = hourly # daily/hourly for interpolation, hourly/3hourly for download, hourly for report
+start_date = 20231201
+end_date = 20240301
+domain = global
 map_extent = -180, 180, -90, 90
-experiments = a59g-regional-000 # cams_reanalysis for EAC4 reanalysis; cams_forecast for forecast
+experiments = cams_forecast (camsf) # camsr for EAC4 reanalysis; camsf for forecast
+ensemble = default
 interp_n_neighbours = default
 interpolated = False # download non interpolated experiments
-forecast = day1,day4 # for interpolation: will interpolate all forecast data available (no matter what is set here)
-temporal_colocation = True
-spatial_colocation = True
+forecast = day1 # for interpolation: will interpolate all forecast data available (no matter what is set here)
+resampling = 3hourly # resample to this resolution after interpolation
+
+temporal_colocation = True # between exp and obs
+spatial_colocation = False # across species
 report_type = cams2_82_bis
 report_summary = True
 report_stations = False
-report_title = CAMS2_82_bis_2024_MAM
-report_filename = CAMS2_82_bis_2024_MAM
+report_title = CAMS2_82_bis_2024_DJF_EEA_JP
+report_filename = CAMS2_82_bis_2024_DJF_EEA_JP
+statistic_mode = Temporal|Spatial
 statistic_aggregation = Median
 periodic_statistic_mode = Independent
 periodic_statistic_aggregation = Median
-timeseries_statistic_aggregation = Median
-
-    [[Aggregation_temporal_spatial]]
-    statistic_mode = Temporal|Spatial
-
-    [[Aggregation_spatial_temporal]]
-    statistic_mode = Spatial|Temporal
-
-    [[Aggregation_Flattened]]
-    statistic_mode = Flattened
+timeseries_statistic_aggregation = Median

providentia/auxiliar.py

@@ -191,22 +191,22 @@ def get_standard_parameters_by_speci(speci, ghost_version):
             return standard_parameters[standard_parameter]
 
 
-def get_conversion_factor(initial_units, final_units, standard_parameter_speci):
-    """ Get conversion factor to convert from initial to final units
+def unit_conversion(initial_units, final_units, standard_parameter_speci):
+    """ Use unit_converter class to return conversion object from initial to final units.
 
     Parameters
     ----------
-    final_units : str
-        Input units
     initial_units : str
+        Input units
+    final_units : str
         Output units
     standard_parameter_speci : dict
         GHOST standard parameters dictionary
 
     Returns
     -------
-    float, str
-        Conversion factor or error
+    obj, str
+        Conversion object or error
     """
 
     # import unit converter
@@ -215,43 +215,43 @@ def get_conversion_factor(initial_units, final_units, standard_parameter_speci):
 
     # if units are unitless, then no need for conversion (i.e. conversion factor = 1.0)   
     if (final_units == 'unitless') or (final_units == '-') or (final_units == '1'):
-        return 1.0
+        return type('convert_units', (object,), {'conversion_factor':1.0, 'output_standard_units':'unitless'})
 
     # determine chemical formula of species 
     if 'chemical_formula_charge' in list(standard_parameter_speci.keys()):
         speci_chemical_formula = standard_parameter_speci['chemical_formula_charge']
     else:
         speci_chemical_formula = standard_parameter_speci['chemical_formula']
 
-    # get observational quantity for conversion
-    if ('units_quantity' in list(standard_parameter_speci.keys())) and (final_units == standard_parameter_speci['standard_units']):
-        final_quantity = standard_parameter_speci['units_quantity']
+    # get input (model) quantity for conversion
+    if ('units_quantity' in list(standard_parameter_speci.keys())) and (initial_units == standard_parameter_speci['standard_units']):
+        initial_quantity = standard_parameter_speci['units_quantity']
     else:
-        conv_obj = unit_converter.convert_units(final_units, final_units, 1, 
+        conv_obj = unit_converter.convert_units(initial_units, initial_units, 1, 
                                                 species=speci_chemical_formula)
-        final_quantity = conv_obj.output_quantity
+        initial_quantity = conv_obj.output_quantity
 
-    # get model quantity for conversion
+    # get output (observational) quantity for conversion
     if ('units_quantity' in list(standard_parameter_speci.keys())) and (final_units == standard_parameter_speci['standard_units']):
-        initial_quantity = standard_parameter_speci['units_quantity']
+        final_quantity = standard_parameter_speci['units_quantity']
     else:
-        conv_obj = unit_converter.convert_units(initial_units, initial_units, 1, 
+        conv_obj = unit_converter.convert_units(final_units, final_units, 1, 
                                                 species=speci_chemical_formula)
-        initial_quantity = conv_obj.output_quantity
+        final_quantity = conv_obj.output_quantity
 
     # unit converter module does not produce conversion factor for temperature, 
-    # but both observational and model units should be Kelvin (i.e. conversion factor = 1.0) 
-    # if model not in K then return error
+    # but both input and output units should be Kelvin (i.e. conversion factor = 1.0) 
+    # if input not in K then return error
     if final_quantity == 'temperature': 
         if initial_units == 'K':
-            return 1.0
+            return type('convert_units', (object,), {'conversion_factor':1.0, 'output_standard_units':'K'})
         else:
-            error = "Experiment units should be 'K', but are set as '{}'".format(initial_units)
+            error = "Error: Experiment units should be 'K', but are set as '{}'".format(initial_units)
             return error
 
     # initial and final quantities not equal (convert to observational units, 
     # standard_temperature=293.15, standard_pressure=1013.25)
-    if final_quantity != initial_quantity:     
+    if initial_quantity != final_quantity:     
 
         # convert units
         input_units = {'temperature': 'K', 
@@ -266,12 +266,57 @@ def get_conversion_factor(initial_units, final_units, standard_parameter_speci):
                                                 species=speci_chemical_formula, 
                                                 input_quantity=initial_quantity, 
                                                 output_quantity=final_quantity)
-        return conv_obj.conversion_factor
 
     # same quantity conversion
     else:
         conv_obj = unit_converter.convert_units(initial_units, final_units, 1.0, 
                                                 species=speci_chemical_formula, 
                                                 input_quantity=initial_quantity, 
                                                 output_quantity=final_quantity) 
-        return conv_obj.conversion_factor
+    
+    # return conversion object
+    return conv_obj
+
+
+def get_conversion_factor(initial_units, final_units, standard_parameter_speci):
+    """ Get conversion factor to convert from initial to final units.
+        Convenience wrapper for unit_conversion function.
+
+    Parameters
+    ----------
+    initial_units : str
+        Input units
+    final_units : str
+        Output units
+    standard_parameter_speci : dict
+        GHOST standard parameters dictionary
+
+    Returns
+    -------
+    float, str
+        Conversion factor or error
+    """
+
+    conv_obj = unit_conversion(initial_units, final_units, standard_parameter_speci)
+    return conv_obj.conversion_factor
+
+
+def get_standard_units(initial_units, standard_parameter_speci):
+    """ Get Standarsised units for given initial units.
+        Convenience wrapper for unit_conversion function.
+
+    Parameters
+    ----------
+    initial_units : str
+        Input units
+    standard_parameter_speci : dict
+        GHOST standard parameters dictionary
+
+    Returns
+    -------
+    str
+        Output standard units or error
+    """
+    
+    conv_obj = unit_conversion(initial_units, initial_units, standard_parameter_speci)
+    return conv_obj.output_standard_units

providentia/interpolation/experiment_interpolation.py

@@ -42,10 +42,6 @@
 # load the defined experiments paths and agrupations jsons
 interp_experiments = yaml.safe_load(open(join(PROVIDENTIA_ROOT, 'settings', 'interp_experiments.yaml')))
 
-# add unit-converter submodule to python load path
-sys.path.append(join(PROVIDENTIA_ROOT,'providentia','dependencies','unit-converter'))
-import unit_converter
-
 class ExperimentInterpolation(object):
     """ Class which handles interpolation of experiment data to surface observations. """
 

providentia/interpolation/experiment_interpolation_submission.py

@@ -891,7 +891,7 @@ def submit_job_multiprocessing(self):
         """Submit interpolation jobs using multiprocessing pool."""
 
         # set resource usage parameters for estimating safe number of pool workers 
-        self.per_worker_mem_gb=0.75
+        self.per_worker_mem_gb=2.0
         self.per_worker_cpu_fraction=1.0/self.n_cpus
         self.per_worker_swap_gb=0.1
         self.cpu_fraction_limit=0.75 

providentia/read.py

@@ -13,7 +13,7 @@
 import numpy as np
 import pandas as pd
 
-from providentia.auxiliar import CURRENT_PATH, join
+from providentia.auxiliar import CURRENT_PATH, join, get_standard_units, get_standard_parameters_by_speci
 from .fields_menus import (init_representativity, init_period, init_metadata,
                            update_representativity_fields, update_period_fields, update_metadata_fields,
                            representativity_conf, period_conf, metadata_conf)
@@ -280,24 +280,17 @@ def read_setup(self, operations, experiments_to_remove=None, experiments_to_read
             # update measurement units for all species (take standard units for each speci from parameter dictionary)
             # non-GHOST
             if not self.read_instance.reading_ghost:
-                # import unit converter
-                sys.path.insert(1, join(CURRENT_PATH, 'dependencies/unit-converter'))
-                import unit_converter
 
                 # convert non-GHOST units to standard format
                 nonghost_standard_units = {}
                 for speci in self.read_instance.nonghost_units.keys():
                     input_units = self.read_instance.nonghost_units[speci]
-                    if input_units not in ['-', 'unitless']:
-                        output_units = copy.deepcopy(input_units)
-                        if 'chemical_formula_charge' in list(self.read_instance.parameter_dictionary[speci].keys()):
-                            formula = self.read_instance.parameter_dictionary[speci]['chemical_formula_charge']
-                        else:
-                            formula = self.read_instance.parameter_dictionary[speci]['chemical_formula']
-                        conv_obj = unit_converter.convert_units(input_units, output_units, 1, species=formula)
-                        nonghost_standard_units[speci] = conv_obj.output_standard_units
-                    else:
-                        nonghost_standard_units[speci] = 'unitless'
+                    standard_parameter_speci = get_standard_parameters_by_speci(speci, self.read_instance.ghost_version)
+                    standard_input_units = get_standard_units(input_units, standard_parameter_speci)
+                    if 'Error:' in standard_input_units:
+                        self.read_instance.logger.error(standard_input_units)
+                        sys.exit(1)
+                    nonghost_standard_units[speci] = standard_input_units
                 self.read_instance.measurement_units = {speci.split('|')[1]:nonghost_standard_units[speci.split('|')[1]] 
                                                         for speci in self.read_instance.networkspecies}
             # GHOST