application_config.py

# Allen Institute Software License - This software license is the 2-clause BSD
# license plus a third clause that prohibits redistribution for commercial
# purposes without further permission.
#
# Copyright 2014-2015. Allen Institute. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# 3. Redistributions for commercial purposes are not permitted without the
# Allen Institute's written permission.
# For purposes of this license, commercial purposes is the incorporation of the
# Allen Institute's software into anything for which you will charge fees or
# other compensation. Contact terms@alleninstitute.org for commercial licensing
# opportunities.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
from allensdk.core.json_utilities import JsonComments
import argparse
import os
import io
import logging
import logging.config as lc
from pkg_resources import resource_filename  # @UnresolvedImport

try:
    from configparser import ConfigParser  # @UnresolvedImport
except:
    from ConfigParser import ConfigParser  # @UnresolvedImport


class ApplicationConfig(object):
    ''' Convenience class that handles of application configuration
    from environment variables, .conf files and the command line
    using Python standard libraries and formats.
    '''

    _log = logging.getLogger(__name__)
    _DEFAULT_LOG_CONFIG = os.getenv(
        'LOG_CFG', resource_filename(__name__, 'logging.conf'))

    def __init__(self,
                 defaults,
                 name="app",
                 halp="Run application.",
                 default_log_config=None):

        self.application_name = name
        self.help = halp
        self.debug_enabled = False

        if default_log_config is None:
            default_log_config = ApplicationConfig._DEFAULT_LOG_CONFIG
            lc.fileConfig(_DEFAULT_LOG_CONFIG)

        ApplicationConfig._log.info(
            "default log config: %s" % (default_log_config))

        self.defaults = {
            'config_file_path': {
                'default': "%s.conf" % (self.application_name),
                'help': 'configuration file path'
            },
            'log_config_path': {
                'default': default_log_config,
                'help': 'logging configuration path'
            }
        }

        self.defaults.update(defaults)

        logging.info("defaults: %s" % (self.defaults))

        self.argparser = self.create_argparser()

        for key, value in self.defaults.items():
            setattr(self, key, value['default'])

    def load(self, command_line_args, disable_existing_loggers=True):
        ''' Load application configuration options, first from the environment,
        then from the configuration file, then from the command line.

        Each stage of loading can override the previous stage.

        Parameters
        ----------
        command_line_args : dict
            Parameters passed to the application.
        disable_existing_loggers : boolean
            Reset the logging system or not.

        Returns
        -------
        fileConfig
            Configuration object with all levels applied
        '''
        # read and apply options from the environment
        self.apply_configuration_from_environment()

        # command line so we can find the config file.
        parsed_args = self.parse_command_line_args(command_line_args)

        try:
            # read and apply the configuration file options
            config_file_path = parsed_args.config_file_path

            if config_file_path:
                self.config_file_path = config_file_path

            self.apply_configuration_from_file(self.config_file_path)

            # apply the remaining command line options
            self.apply_configuration_from_command_line(parsed_args)
        except Exception as e:
            ApplicationConfig._log.error("Could not load configuration file: %s\n%s" %
                                         (parsed_args.config_file_path,
                                          e))
            raise

        if parsed_args.log_config_path:
            try:
                lc.fileConfig(self.log_config_path,
                              disable_existing_loggers=disable_existing_loggers)
            except:
                logging.error("Could not load log configuration file: %s" %
                              (parsed_args.log_config_path))
        else:
            # TODO: configure default logging
            pass

    def create_argparser(self):
        '''Initialization for the command-line parsing stage.

        An application specific prefix is applied to argument names.

        Parameters
        ----------
        prog : string
            Application specific prefix for argument names.
        description : string
            A brief 'help' description of the application.

        Returns
        -------
        argParse.ArgumentParser
            The initialized argument parser object.

        Notes
        -----
        Defaults are set at the first environment reading.
        Command line args only override them when present
        '''
        parser = argparse.ArgumentParser(prog=self.application_name,
                                         description=self.help)
        for key, value in self.defaults.items():
            if key == 'config_file_path':
                parser.add_argument(
                    "%s" % (key), default=None, help=value['help'])
            else:
                parser.add_argument("--%s" %
                                    (key), default=None, help=value['help'])

        return parser

    def parse_command_line_args(self, args):
        '''Simply call the internal argparser object.

        Parameters
        ----------
        args : array
            Parameters passed to the application.

        Returns
        -------
        Namespace
            Parsed paramenters.
        '''
        return self.argparser.parse_args(args)

    def apply_configuration_from_command_line(self, parsed_args):
        '''Read application configuration variables from the command line.

        Unassigned variables are left unchanged if previously assigned,
        set to their default values,
        or None if no default is specified at init time.
        Assigned variables will overwrite the previous value.

        see: https://docs.python.org/2/howto/argparse.html

        Parameters
        ----------
        parsed_args : dict
            the arguments as parsed from the command line.

        '''
        logging.info('command_line args: %s' % (parsed_args))

        for key in self.defaults:
            parsed_value = getattr(parsed_args, key)
            if parsed_value and getattr(self, key) is None:
                setattr(self, key, parsed_value)

    def apply_configuration_from_environment(self):
        '''Read application configuration variables from the environment.

        The variable names are upper case and have a
        prefix defined by the application.

        See: https://docs.python.org/2/library/os.html
        '''
        for key in self.defaults:
            environment_variable = "%s_%s" % (
                self.application_name.upper(), key.upper())
            environment_value = os.environ.get(environment_variable)
            if environment_value:
                setattr(self, key, environment_value)

    def from_json_file(self, json_path):
        '''Read an application configuration from a JSON format file.

        Parameters
        ----------
        json_path : string
            Path to the JSON file.

        Returns
        -------
        string
            An application configuration in INI format

        '''
        description = JsonComments.read_file(json_path)

        return self.to_config_string(description)

    def from_json_string(self, json_string):
        '''Read a configuration from a JSON format string.

        Parameters
        ----------
        json_string : string
            A JSON-formatted string containing an application configuration.

        Returns
        -------
        string
            An application configuration in INI format
        '''
        description = JsonComments.read_string(json_string)

        return self.to_config_string(description)

    def to_config_string(self, description):
        '''Create a configuration string from a dict.

        Parameters
        ----------
        description : dict
            Configuration options for an application.

        Returns
        -------
        string
            Equivalent configuration as an INI format string

        Notes
        -----
        The Python configparser library natively supports this functionality in Python 3.
        '''
        if 'biophys' not in description:
            bps_config_string = '[biophys]\n\n'
            return bps_config_string

        bps_config = description['biophys'][0]

        cfg_array = ['[biophys]']

        if 'log_config_path' in bps_config:
            cfg_array.append(str('log_config_path: %s' %
                                 bps_config['log_config_path']))

        if 'debug' in bps_config:
            cfg_array.append(str('debug: %s' % bps_config['debug']))

        if 'model_file' in bps_config:
            cfg_array.append(str('model_file: %s' %
                                 ','.join(bps_config['model_file'])))

        cfg_array.append("\n")

        bps_cfg_string = "\n".join(cfg_array)
        ApplicationConfig._log.info(bps_cfg_string)

        return bps_cfg_string

    def apply_configuration_from_file(self, config_file_path):
        ''' Read application configuration variables from a .conf file.

        Unassigned variables are set to their default values
        or None if no default is specified at init time.
        The variables are found in a section named by the application.

        Parameters
        ----------
        config_file_path : string
            path to to an INI (.conf) or JSON format application config file.

        Returns
        -------

        see: https://docs.python.org/2/library/configparser.html
        '''
        none_defaults = {}

        # defaults are set in environment
        # they are only overriden by the config file if present
        for key in self.defaults:
            none_defaults[key] = None

        logging.info("none_defaults: %s" % (none_defaults))

        config = None

        try:
            config = ConfigParser(defaults=none_defaults,
                                  allow_no_value=True)
        except:
            logging.warn(
                "This python installation does not support configuration defaults.")
            config = ConfigParser()

        if config_file_path.endswith('.json'):
            cfg_string = self.from_json_file(config_file_path)
            try:
                config.readfp(io.BytesIO(cfg_string))
            except (NameError, TypeError, AttributeError):
                # readfp was removed in Python 3.12
                config.read_string(cfg_string)
        else:
            config.read(config_file_path)

        for key in self.defaults:
            try:
                file_value = config.get(self.application_name, key)
                if file_value:
                    logging.info("setting %s to %s" % (key, file_value))
                    setattr(self, key, file_value)
            except:
                logging.info("Configuration option not specified: %s" %
                             (key))