working on command line overwrite issue

Author: denebowdalo

configurations/test_forecast.conf

@@ -9,5 +9,5 @@ experiments = a59g-regional-000, a59j-regional-006
 temporal_colocation = True
 spatial_colocation = True
 statistic_mode = Spatial|Temporal
-#forecast = daily
+forecast = daily
 report_type = operational

providentia/canvas.py

@@ -69,9 +69,10 @@ def __init__(self, read_instance):
         self.plot_characteristics_templates = self.read_instance.plot_characteristics_templates
         self.plot_characteristics = {}
 
-        # add general plot characteristics to self
+        # add general plot characteristics to self (if not passed via command line)
         for k, val in self.plot_characteristics_templates['general'].items():
-            setattr(self, k, val)
+            if k not in self.read_instance.commandline_arguments:
+                setattr(self, k, val)
 
         # initialise some key vars
         self.filter_data = None

providentia/dashboard.py

@@ -59,6 +59,10 @@ def __init__(self, **kwargs):
         # allow access to methods of parent class QtWidgets.QWidget
         super(Dashboard, self).__init__()
 
+        # update self with command line arguments
+        self.commandline_arguments = copy.deepcopy(kwargs)
+        self.commandline_arguments['dashboard'] = True
+
         # load statistical yamls
         self.basic_stats = yaml.safe_load(open(join(PROVIDENTIA_ROOT, 'settings/basic_stats.yaml')))
         self.expbias_stats = yaml.safe_load(open(join(PROVIDENTIA_ROOT, 'settings/experiment_bias_stats.yaml')))
@@ -70,11 +74,9 @@ def __init__(self, **kwargs):
         self.delay = True
         self.delayed_warnings = []
 
-        kwargs['dashboard'] = True
-
         # initialise default configuration variables
         # modified by commandline arguments, if given
-        self.provconf = ProvConfiguration(self, **kwargs)
+        self.provconf = ProvConfiguration(self, **self.commandline_arguments)
 
         # update variables from config file (if available)
         self.from_conf = False
@@ -90,15 +92,15 @@ def __init__(self, **kwargs):
                     read_conf = True
 
             if read_conf:
-                if 'section' in kwargs:
+                if 'section' in self.commandline_arguments:
                     # config and section defined 
                     load_conf(self, fpath=self.config)
-                    if kwargs['section'] in self.all_sections:
+                    if self.commandline_arguments['section'] in self.all_sections:
                         self.from_conf = True
-                        self.current_config = self.sub_opts[kwargs['section']]
-                        self.section = kwargs['section']
+                        self.current_config = self.sub_opts[self.commandline_arguments['section']]
+                        self.section = self.commandline_arguments['section']
                     else:
-                        msg = 'Error: The section specified in the command line ({0}) does not exist.'.format(kwargs['section'])
+                        msg = 'Error: The section specified in the command line ({0}) does not exist.'.format(self.commandline_arguments['section'])
                         msg += '\nTip: For subsections, add the name of the parent section followed by an interpunct (·) '
                         msg += 'before the subsection name (e.g. SECTIONA·Spain). Available: {0}'.format(self.all_sections)
                         self.logger.error(msg)
@@ -142,10 +144,11 @@ def __init__(self, **kwargs):
         self.previous_calibration_factor = {}
         self.calibration_factor = {}
 
-        # update variables from defined config file
+        # update variables from defined config file (if not passed via command line)
         if self.current_config:
             for k, val in self.current_config.items():
-                setattr(self, k, self.provconf.parse_parameter(k, val))
+                if k not in self.commandline_arguments:
+                    setattr(self, k, self.provconf.parse_parameter(k, val))
 
         # now all variables have been parsed, check validity of those, throwing errors where necessary
         self.provconf.check_validity()
@@ -185,7 +188,7 @@ def __init__(self, **kwargs):
                 OrderedDict(sorted(self.expbias_stats.items(), key=lambda x: x[1]['order'])).keys()))
 
         # initialise UI
-        self.init_ui(**kwargs)
+        self.init_ui()
 
         # setup callback events upon resizing/moving of Providentia window
         self.resized.connect(self.get_geometry)
@@ -322,7 +325,7 @@ def update_qt_elements_geometry(self, plot_types='ALL', positions = [1,2,3,4,5],
 
                     break
 
-    def init_ui(self, **kwargs):
+    def init_ui(self):
         """ Initialise user interface. """
 
         self.logger.info("Starting Providentia dashboard...")

providentia/download.py

@@ -36,6 +36,9 @@
 class Download(object):
     def __init__(self, **kwargs):
 
+        # update self with command line arguments
+        self.commandline_arguments = copy.deepcopy(kwargs)
+
         # get providentia start time
         self.prov_start_time = time.time()
 
@@ -55,7 +58,7 @@ def __init__(self, **kwargs):
 
         # initialise default configuration variables
         # modified by commandline arguments, if given
-        self.provconf = ProvConfiguration(self, **kwargs)
+        self.provconf = ProvConfiguration(self, **self.commandline_arguments)
 
         self.logger.info("Starting Providentia download...")
 
@@ -72,12 +75,12 @@ def __init__(self, **kwargs):
                 load_conf(self, self.config)
                 self.from_conf = True
                 # get the section in case it was passed on the command line                
-                if 'section' in kwargs:
+                if 'section' in self.commandline_arguments:
                     # config and section defined 
-                    if kwargs['section'] in self.all_sections:
-                        self.sections = [kwargs['section']]
+                    if self.commandline_arguments['section'] in self.all_sections:
+                        self.sections = [self.commandline_arguments['section']]
                     else:
-                        msg = 'Error: The section specified in the command line ({0}) does not exist.'.format(kwargs['section'])
+                        msg = 'Error: The section specified in the command line ({0}) does not exist.'.format(self.commandline_arguments['section'])
                         msg += '\nTip: For subsections, add the name of the parent section followed by an interpunct (·) '
                         msg += 'before the subsection name (e.g. SECTIONA·Spain). Available: {0}'.format(self.all_sections)
                         self.logger.error(msg)
@@ -114,15 +117,15 @@ def __init__(self, **kwargs):
             # initialise boolean thath indicates whether remote machine changed 
             self.switched_remote = False
 
-    def run(self, **kwargs):
+    def run(self):
         for section_ind, section in enumerate(self.sections):
             # update for new section parameters
             self.section = section
             self.section_opts = self.sub_opts[self.section]
 
-            # update self with section variables
+            # update self with section variables (if not passed via command line)
             for k, val in self.section_opts.items():
-                if k not in kwargs:
+                if k not in self.commandline_arguments:
                     setattr(self, k, self.provconf.parse_parameter(k, val))
 
             # now all variables have been parsed, check validity of those, throwing errors where necessary
@@ -1501,4 +1504,4 @@ def main(**kwargs):
     """ Main function when running download function. """
 
     download = Download(**kwargs)
-    download.run(**kwargs)
+    download.run()

providentia/interpolation/experiment_interpolation_submission.py

@@ -30,7 +30,10 @@
 class SubmitInterpolation(object):
     """ Class that handles the interpolation submission. """
 
-    def __init__(self,**kwargs):
+    def __init__(self, **kwargs):
+
+        # update self with command line arguments
+        self.commandline_arguments = copy.deepcopy(kwargs)
 
         # start timer
         self.start = time.time()
@@ -43,7 +46,7 @@ def __init__(self,**kwargs):
         self.interpolation_log_dir = join(PROVIDENTIA_ROOT, 'logs/interpolation/interpolation_logs')
 
         # initialize commandline arguments, if given
-        provconf = ProvConfiguration(self, **kwargs)
+        provconf = ProvConfiguration(self, **self.commandline_arguments)
 
         print('\n')
 
@@ -67,8 +70,8 @@ def __init__(self,**kwargs):
             sys.exit(error)
 
         # get section args
-        if "section" in kwargs:
-            section = kwargs["section"]
+        if "section" in self.commandline_arguments:
+            section = self.commandline_arguments["section"]
             if section in self.parent_section_names:
                 self.current_config = self.sub_opts[section]
             else:
@@ -94,10 +97,11 @@ def __init__(self,**kwargs):
         # dictionary that stores utilized interpolation variables
         self.interpolation_variables = {}
 
-        # update variables from defined config file
+        # update variables from defined config file (if not passed via command line)
         if self.current_config:
             for k, val in sorted(self.current_config.items()):
-                setattr(self, k, provconf.parse_parameter(k, val))
+                if k not in self.commandline_arguments:
+                    setattr(self, k, provconf.parse_parameter(k, val))
 
         # now all variables have been parsed, check validity of those, throwing errors where necessary
         provconf.check_validity()

providentia/report.py

@@ -45,6 +45,9 @@ class Report:
 
     def __init__(self, **kwargs):
 
+        # update self with command line arguments
+        self.commandline_arguments = copy.deepcopy(kwargs)
+
         # load statistical yamls
         self.basic_stats = yaml.safe_load(open(join(PROVIDENTIA_ROOT, 'settings/basic_stats.yaml')))
         self.expbias_stats = yaml.safe_load(open(join(PROVIDENTIA_ROOT, 'settings/experiment_bias_stats.yaml')))
@@ -54,10 +57,7 @@ def __init__(self, **kwargs):
 
         # initialise default configuration variables
         # modified by commandline arguments, if given
-        provconf = ProvConfiguration(self, **kwargs)
-
-        # update self with command line arguments
-        self.commandline_arguments = copy.deepcopy(kwargs)
+        provconf = ProvConfiguration(self, **self.commandline_arguments)
 
         self.logger.info("Creating a Providentia Report...")
 
@@ -143,10 +143,11 @@ def __init__(self, **kwargs):
             # get GHOST version before reading configuration file section parameters
             current_ghost_version = self.ghost_version 
 
-            # update self with section variables
+            # update self with section variables (if not passed via command line)
             for k, val in self.section_opts.items():
-                setattr(self, k, provconf.parse_parameter(k, val))
-            
+                if k not in self.commandline_arguments:
+                    setattr(self, k, provconf.parse_parameter(k, val))
+
             # if first section or GHOST version has changed
             if (section_ind == 0) or (current_ghost_version != self.ghost_version):
                 generate_file_trees(self)
@@ -164,9 +165,10 @@ def __init__(self, **kwargs):
             # initialise Plotting class
             self.plotting = Plotting(read_instance=self, canvas_instance=self)
 
-            # add general plot characteristics to self
+            # add general plot characteristics to self (if not passed via command line)
             for k, val in self.plot_characteristics_templates['general'].items():
-                setattr(self, k, val)
+                if k not in self.commandline_arguments:
+                    setattr(self, k, val)
 
             # now all variables have been parsed, check validity of those, throwing errors where necessary
             provconf.check_validity()
@@ -842,34 +844,34 @@ def make_plots_per_subsection(self, summary_plots_to_make, station_plots_to_make
                 read_species = copy.deepcopy(self.species)
                 read_networkspecies = copy.deepcopy(self.networkspecies)
 
+                # if have forecast active then save current experiment variable in memory as may need to set it if not re-reading data 
+                if len(self.forecast) != 0:
+                    forecast_experiments = copy.deepcopy(self.experiments)  
+
                 # get subsection variables
                 self.subsection_opts = self.sub_opts[self.subsection]
 
                 # ensure all fixed section variables defined in subsection have same value as current section variables
                 self.subsection_opts = {k: (self.section_opts[k] if k in self.fixed_section_vars else val) 
                                         for (k, val) in self.subsection_opts.items()}
 
-                # if have forecast active then save current experiment variable in memory as may need to set it if not re-reading data 
-                if len(self.forecast) != 0:
-                    forecast_experiments = copy.deepcopy(self.experiments)  
-
                 # reinitialise default configuration variables
                 # modified by commandline arguments, if given
                 provconf = ProvConfiguration(self, **self.commandline_arguments)
 
-                # update subsection variables
+                # update subsection variables (if not passed via command line)
                 for k, val in self.subsection_opts.items():
-                    value = provconf.parse_parameter(k, val, deactivate_warning=True)
-                    # keep only the species that were available when we read the parent section data
-                    if k == 'species':
-                        value = [speci for speci in value if speci in read_species]
-                    elif k == 'networkspecies':
-                        value = [speci for speci in value if speci in read_networkspecies]
-                    setattr(self, k, value)
+                    if k not in self.commandline_arguments:
+                        value = provconf.parse_parameter(k, val, deactivate_warning=True)
+                        setattr(self, k, value)
 
                 # now all variables have been parsed, check validity of those, throwing errors where necessary
                 provconf.check_validity(deactivate_warning=True)
 
+                # keep only the species that were available when we read the parent section data
+                self.species = read_species
+                self.networkspecies = read_networkspecies
+
             # determine if need to re-read data (qa, flags, filter_species or calibration factor have changed)
             if (np.array_equal(self.qa, self.previous_qa) == False) or (
                 np.array_equal(self.flags, self.previous_flags) == False) or (

providentia/statistics.py

@@ -126,7 +126,7 @@ def get_selected_station_data(read_instance, canvas_instance, networkspecies,
                                                             order='F')
 
                 # aggregate across the time dimension, after this shape is (24, label, station, fct)
-                data_array_forecast_agg = aggregation(read_instance, read_instance, read_instance, read_instance, read_instance, read_instance, data_array_forecast_grouped_rs, read_instance.timeseries_statistic_aggregation, axis=-2)
+                data_array_forecast_agg = aggregation(data_array_forecast_grouped_rs, read_instance.timeseries_statistic_aggregation, axis=-2)
 
                 # move per hour dimension from first to second last
                 data_array_forecast_agg = data_array_forecast_agg.transpose(1, 2, 0, 3)