New feature to allow the program environment to be loaded from an external file or program.

- This allows supervisord to be used in conjunction with any secrets
pattern using a root-only file or a program that can provide environment
variables that a program should have. It can be set globally in the
supervisord section, or per program in a program section. The new
options are environment_file or environment_loader. They are optional,
and errors in one of them will prevent startup. They can be set in the
[supervisord] section and then will be passed down to the programs, or
in the program definitions. The file/loader is checked right before the
calls to spawn() in order to avoid problems with calling a subprocess
after the child fork, and to allow restarts to reload those environment
values.
- Updated the docs for these new options
- Updated the tests to add a new test to check these new options.

Author: wynnw

docs/configuration.rst

@@ -464,6 +464,42 @@ follows.
 
   *Introduced*: 3.0
 
+``environment_file``
+
+  An absolute path to a file that contains a ``KEY=VAL`` entry on each line.
+  Lines that begin with a '#' character are ignored. Leading and trailing
+  whitespace are stripped off. Each valid ``KEY=VAL`` line will be placed
+  in the environment of all child processes. The VAL entries must not be quoted,
+  and interpolation is not supported for these values. The file must be readable
+  by supervisord, and may be only readable by the user supervisord runs as since
+  these values are loaded before any privileges are dropped for child processes.
+  All other behaviors of the ``environment`` values are followed. When this is
+  set in the supervisord section, it will be applied to all program sections unless
+  they explicitly set either ``environment_file`` or ``environment_loader``. Only one of
+  the program setting or the supervisord setting for environment_file is processed.
+
+  *Default*: no value
+
+  *Required*:  No.
+
+  *Introduced*: 4.2.3
+
+``environment_loader``
+
+  A shell command or an absolute path to a program that will be run by supervisord before launching
+  the child processes, and the stdout will be captured and parsed according to the rules for
+  ``environment_file``.  Only one of ``environment_file`` or ``environment_loader`` should be set, and
+  ``environment_file`` takes precedence. When this is set in the supervisord section,
+  it will be applied to all program sections unless they explicitly set either
+  ``environment_file`` or ``environment_loader``. Only one of the program setting or the
+  supervisord setting for environment_loader is processed.
+
+  *Default*: no value
+
+  *Required*:  No.
+
+  *Introduced*: 4.2.3
+
 ``identifier``
 
   The identifier string for this supervisor process, used by the RPC
@@ -1099,6 +1135,37 @@ where specified.
 
   *Introduced*: 3.0
 
+``environment_file``
+
+  An absolute path to a file that contains a ``KEY=VAL`` entry on each line.
+  Lines that begin with a '#' character are ignored. Leading and trailing
+  whitespace between the values are stripped off. Each valid ``KEY=VAL`` line will be placed
+  in the environment of all child processes. The VAL entries must not be quoted,
+  and interpolation is not supported for these values. The file must be readable
+  by supervisord, and may be only readable by the user supervisord runs as since
+  these values are loaded before any privileges are dropped for child processes.
+  All other behaviors of the ``environment`` values are followed.
+
+  *Default*: no value
+
+  *Required*:  No.
+
+  *Introduced*: 4.2.3
+
+``environment_loader``
+
+  A shell command or an absolute path to a program that will be by supervisord before launching
+  a child process, and the stdout will be captured and parsed according to the rules for
+  ``environment_file``. The program must be executable by supervisord. Only one of
+  ``environment_file`` or ``environment_loader`` should be set, and ``environment_file`` takes precedence.
+
+  *Default*: no values
+
+  *Required*:  No.
+
+  *Introduced*: 4.2.3
+
+
 ``directory``
 
   A file path representing a directory to which :program:`supervisord`

supervisor/options.py

@@ -656,6 +656,8 @@ def get(opt, default, **kwargs):
         environ_str = get('environment', '')
         environ_str = expand(environ_str, expansions, 'environment')
         section.environment = dict_of_key_value_pairs(environ_str)
+        section.environment_file = get('environment_file', None)
+        section.environment_loader = get('environment_loader', None)
 
         # extend expansions for global from [supervisord] environment definition
         for k, v in section.environment.items():
@@ -674,6 +676,13 @@ def get(opt, default, **kwargs):
                 env = section.environment.copy()
                 env.update(proc.environment)
                 proc.environment = env
+
+                # set the environment file/loader on the process configs but let them override it
+                if not proc.environment_file and not proc.environment_loader:
+                    if section.environment_file:
+                        proc.environment_file = section.environment_file
+                    elif section.environment_loader:
+                        proc.environment_loader = section.environment_loader
         section.server_configs = self.server_configs_from_parser(parser)
         section.profile_options = None
         return section
@@ -925,6 +934,8 @@ def get(section, opt, *args, **kwargs):
         numprocs = integer(get(section, 'numprocs', 1))
         numprocs_start = integer(get(section, 'numprocs_start', 0))
         environment_str = get(section, 'environment', '', do_expand=False)
+        environment_file = get(section, 'environment_file', '', do_expand=False)
+        environment_loader = get(section, 'environment_loader', '', do_expand=False)
         stdout_cmaxbytes = byte_size(get(section,'stdout_capture_maxbytes','0'))
         stdout_events = boolean(get(section, 'stdout_events_enabled','false'))
         stderr_cmaxbytes = byte_size(get(section,'stderr_capture_maxbytes','0'))
@@ -1057,6 +1068,8 @@ def get(section, opt, *args, **kwargs):
                 exitcodes=exitcodes,
                 redirect_stderr=redirect_stderr,
                 environment=environment,
+                environment_file=environment_file,
+                environment_loader=environment_loader,
                 serverurl=serverurl)
 
             programs.append(pconfig)
@@ -1875,7 +1888,7 @@ class ProcessConfig(Config):
         'stderr_events_enabled', 'stderr_syslog',
         'stopsignal', 'stopwaitsecs', 'stopasgroup', 'killasgroup',
         'exitcodes', 'redirect_stderr' ]
-    optional_param_names = [ 'environment', 'serverurl' ]
+    optional_param_names = [ 'environment', 'environment_file', 'environment_loader', 'serverurl' ]
 
     def __init__(self, options, **params):
         self.options = options
@@ -1939,6 +1952,57 @@ def make_dispatchers(self, proc):
             dispatchers[stdin_fd] = PInputDispatcher(proc, 'stdin', stdin_fd)
         return dispatchers, p
 
+    def load_external_environment_definition(self):
+        return self.load_external_environment_definition_for_config(self)
+
+    # this is separated out in order to make it easier to test
+    @classmethod
+    def load_external_environment_definition_for_config(cls, config):
+        # lazily load extra env vars before we drop privileges so that this can be used to load a secrets file
+        # or execute a program to get more env configuration. It doesn't have to be secrets, just config that
+        # needs to be separate from the supervisor config for whatever reason. The supervisor config interpolation
+        # is not supported here. The data format is just plain text, with one k=v value per line. Lines starting
+        # with '#' are ignored.
+        env = {}
+        envdata = None
+        if config.environment_file:
+            if os.path.exists(config.environment_file):
+                try:
+                    with open(config.environment_file, 'r') as f:
+                        envdata = f.read()
+
+                except Exception as e:
+                    raise ProcessException("environment_file read failure on %s: %s" % (config.environment_file, e))
+
+        elif config.environment_loader:
+            try:
+                from subprocess import check_output, CalledProcessError
+                kwargs = dict(shell=True)
+                if not PY2:
+                    kwargs['text'] = True
+
+                envdata = check_output(config.environment_loader, **kwargs)
+
+            except CalledProcessError as e:
+                raise ProcessException("environment_loader failure with %s: %d, %s" % (config.environment_loader, e.returncode, e.output))
+
+        if envdata:
+            extra_env = {}
+
+            for line in envdata.splitlines():
+                line = line.strip()
+                if line.startswith('#'):  # ignore comments
+                    continue
+
+                key, val = [s.strip() for s in line.split('=', 1)]
+                if key:
+                    extra_env[key.upper()] = val
+
+            if extra_env:
+                env.update(extra_env)
+
+        return env
+
 class EventListenerConfig(ProcessConfig):
     def make_dispatchers(self, proc):
         # always use_stderr=True for eventlisteners because mixing stderr

supervisor/process.py

@@ -217,6 +217,10 @@ def spawn(self):
 
         try:
             filename, argv = self.get_execv_args()
+
+            # check the environment_file/environment_loader options before we fork to simplify child process management
+            extra_env = self.config.load_external_environment_definition()
+
         except ProcessException as what:
             self.record_spawnerr(what.args[0])
             self._assertInState(ProcessStates.STARTING)
@@ -260,7 +264,7 @@ def spawn(self):
             return self._spawn_as_parent(pid)
 
         else:
-            return self._spawn_as_child(filename, argv)
+            return self._spawn_as_child(filename, argv, extra_env=extra_env)
 
     def _spawn_as_parent(self, pid):
         # Parent
@@ -284,7 +288,7 @@ def _prepare_child_fds(self):
         for i in range(3, options.minfds):
             options.close_fd(i)
 
-    def _spawn_as_child(self, filename, argv):
+    def _spawn_as_child(self, filename, argv, extra_env=None):
         options = self.config.options
         try:
             # prevent child from receiving signals sent to the
@@ -322,6 +326,9 @@ def _spawn_as_child(self, filename, argv):
             if self.config.environment is not None:
                 env.update(self.config.environment)
 
+            if extra_env:
+                env.update(extra_env)
+
             # change directory
             cwd = self.config.directory
             try:

supervisor/tests/base.py

@@ -520,7 +520,8 @@ def __init__(self, options, name, command, directory=None, umask=None,
                  stderr_syslog=False,
                  redirect_stderr=False,
                  stopsignal=None, stopwaitsecs=10, stopasgroup=False, killasgroup=False,
-                 exitcodes=(0,), environment=None, serverurl=None):
+                 exitcodes=(0,), environment=None, environment_file=None, environment_loader=None,
+                 serverurl=None):
         self.options = options
         self.name = name
         self.command = command
@@ -552,6 +553,8 @@ def __init__(self, options, name, command, directory=None, umask=None,
         self.killasgroup = killasgroup
         self.exitcodes = exitcodes
         self.environment = environment
+        self.environment_file = environment_file
+        self.environment_loader = environment_loader
         self.directory = directory
         self.umask = umask
         self.autochildlogs_created = False
@@ -582,6 +585,10 @@ def make_dispatchers(self, proc):
             dispatchers[stdin_fd] = DummyDispatcher(writable=True)
         return dispatchers, pipes
 
+    def load_external_environment_definition(self):
+        from supervisor.options import ProcessConfig
+        return ProcessConfig.load_external_environment_definition_for_config(self)
+
 def makeExecutable(file, substitutions=None):
     import os
     import sys