Allow configuration via environment variables

src/qlever/commands/config.py

@@ -0,0 +1,196 @@
+from __future__ import annotations
+
+import shlex
+import subprocess
+from pathlib import Path
+
+import qlever.globals
+from qlever.command import QleverCommand
+from qlever.envvars import Envvars
+from qlever.log import log
+from qlever.util import get_random_string
+
+
+class ConfigCommand(QleverCommand):
+    """
+    Class for executing the `config` command.
+    """
+
+    def __init__(self):
+        self.qleverfiles_path = Path(__file__).parent.parent / "Qleverfiles"
+        self.qleverfile_names = \
+            [p.name.split(".")[1]
+             for p in self.qleverfiles_path.glob("Qleverfile.*")]
+
+    def description(self) -> str:
+        return "Set up a Qleverfile or show the current configuration"
+
+    def should_have_qleverfile(self) -> bool:
+        return False
+
+    def relevant_qleverfile_arguments(self) -> dict[str: list[str]]:
+        return {}
+
+    def additional_arguments(self, subparser) -> None:
+        subparser.add_argument(
+                "--get-qleverfile", type=str,
+                choices=self.qleverfile_names,
+                help="Get one the many pre-configured Qleverfiles")
+        subparser.add_argument(
+                "--show-qleverfile", action="store_true",
+                default=False,
+                help="Show the configuration from the Qleverfile "
+                "(if it exists)")
+        subparser.add_argument(
+                "--show-envvars", action="store_true", default=False,
+                help="Show all existing environment variables of the form "
+                "QLEVER_SECTION_VARIABLE")
+        subparser.add_argument(
+                "--varname-width", type=int, default=25,
+                help="Width for variable names in the output")
+        subparser.add_argument(
+                "--set-envvars-from-qleverfile", action="store_true",
+                default=False,
+                help="Set the environment variables that correspond to the "
+                "Qleverfile configuration (for copying and pasting)")
+        subparser.add_argument(
+                "--unset-envvars", action="store_true", default=False,
+                help="Unset all environment variables of the form "
+                "QLEVER_SECTION_VARIABLE (for copying and pasting)")
+
+    def execute(self, args) -> bool:
+        # Show the configuration from the Qleverfile.
+        if args.show_qleverfile:
+            if qlever.globals.qleverfile_path is None:
+                log.error("No Qleverfile found")
+                return False
+            if qlever.globals.qleverfile_config is None:
+                log.error("Qleverfile found, but contains no configuration")
+                return False
+            self.show(f"Show the configuration from "
+                      f"{qlever.globals.qleverfile_path} (with any variables "
+                      f"on the right-hand side already substituted)",
+                      only_show=args.show)
+            if args.show:
+                return False
+            else:
+                print_empty_line_before_section = False
+                for section, varname_and_values in \
+                        qlever.globals.qleverfile_config.items():
+                    if section == "DEFAULT":
+                        continue
+                    if print_empty_line_before_section:
+                        log.info("")
+                    print_empty_line_before_section = True
+                    log.info(f"[{section}]")
+                    for varname, value in varname_and_values.items():
+                        log.info(f"{varname.upper():{args.varname_width}} = "
+                                 f"{value}")
+            return True
+
+        # Show all environment variables of the form QLEVER_SECTION_VARIABLE.
+        if args.show_envvars:
+            self.show("Show all environment variables of the form "
+                      "QLEVER_SECTION_VARIABLE", only_show=args.show)
+            if args.show:
+                return False
+            if qlever.globals.envvars_config is None:
+                log.info("No environment variables found")
+            else:
+                for section, varname_and_values in \
+                        qlever.globals.envvars_config.items():
+                    for varname, value in varname_and_values.items():
+                        var = Envvars.envvar_name(section, varname)
+                        log.info(f"{var:{args.varname_width+7}}"
+                                 f" = {shlex.quote(value)}")
+            return True
+
+        # Show the environment variables that correspond to the Qleverfile.
+        if args.set_envvars_from_qleverfile:
+            if qlever.globals.qleverfile_path is None:
+                log.error("No Qleverfile found")
+                return False
+            if qlever.globals.qleverfile_config is None:
+                log.error("Qleverfile found, but contains no configuration")
+                return False
+            self.show("Show the environment variables that correspond to the "
+                      "Qleverfile configuration (for copying and pasting)",
+                      only_show=args.show)
+            if args.show:
+                return False
+            else:
+                for section, varname_and_values in \
+                        qlever.globals.qleverfile_config.items():
+                    if section == "DEFAULT":
+                        continue
+                    for varname, value in varname_and_values.items():
+                        var = Envvars.envvar_name(section, varname)
+                        log.info(f"export {var}={shlex.quote(value)}")
+            return True
+
+        # Unset all environment variables of the form QLEVER_SECTION_VARIABLE.
+        # Note that this cannot be done in this script because it would not
+        # affect the shell calling this script. Instead, show the commands for
+        # unsetting the environment variables to copy and paste.
+        if args.unset_envvars:
+            self.show("Unset all environment variables of the form "
+                      "QLEVER_SECTION_VARIABLE (for copying and pasting, "
+                      "this command cannot affect the shell from which you "
+                      " are calling it)", only_show=args.show)
+            if args.show:
+                return False
+            if qlever.globals.envvars_config is None:
+                log.info("No environment variables found")
+            else:
+                envvar_names = []
+                for section, varname_and_values in \
+                        qlever.globals.envvars_config.items():
+                    for varname, value in varname_and_values.items():
+                        envvar_name = Envvars.envvar_name(section, varname)
+                        envvar_names.append(envvar_name)
+                log.info(f"unset {' '.join(envvar_names)}")
+            return True
+
+        # Get one of the pre-configured Qleverfiles.
+        if args.get_qleverfile:
+            config_name = args.get_qleverfile
+            preconfigured_qleverfile_path = \
+                    self.qleverfiles_path / f"Qleverfile.{config_name}"
+            random_string = get_random_string(12)
+            setup_config_cmd = (
+                    f"cat {preconfigured_qleverfile_path}"
+                    f" | sed -E 's/(^ACCESS_TOKEN.*)/\\1_{random_string}/'"
+                    f" > Qleverfile")
+            self.show(setup_config_cmd, only_show=args.show)
+            if args.show:
+                return False
+
+            # If there is already a Qleverfile in the current directory, exit.
+            existing_qleverfile_path = Path("Qleverfile")
+            if existing_qleverfile_path.exists():
+                log.error("`Qleverfile` already exists in current directory")
+                log.info("")
+                log.info("If you want to create a new Qleverfile using "
+                         "`qlever setup-config`, delete the existing "
+                         "Qleverfile first")
+                return False
+
+            # Copy the Qleverfile to the current directory.
+            try:
+                subprocess.run(setup_config_cmd, shell=True, check=True,
+                               stdin=subprocess.DEVNULL,
+                               stdout=subprocess.DEVNULL)
+            except Exception as e:
+                log.error(f"Could not copy \"{preconfigured_qleverfile_path}\""
+                          f" to current directory: {e}")
+                return False
+
+            # If we get here, everything went well.
+            log.info(f"Created Qleverfile for config \"{config_name}\""
+                     f" in current directory")
+            return True
+
+        # Calling `qlever config` without arguments is an error. Show the help.
+        log.error("`qlever config` requires at least one argument, "
+                  "see `qlever config --help`")
+        return False

src/qlever/config.py

@@ -10,6 +10,7 @@
 from termcolor import colored
 
 from qlever import command_objects, script_name
+from qlever.envvars import Envvars
 from qlever.log import log, log_levels
 from qlever.qleverfile import Qleverfile
 
@@ -40,17 +41,26 @@ class QleverConfig:
 
     def add_subparser_for_command(self, subparsers, command_name,
                                   command_object, all_qleverfile_args,
-                                  qleverfile_config=None):
+                                  qleverfile_config=None,
+                                  envvars_config=None):
         """
         Add subparser for the given command. Take the arguments from
         `command_object.relevant_qleverfile_arguments()` and report an error if
         one of them is not contained in `all_qleverfile_args`. Overwrite the
-        default values with the values from `qleverfile_config` if specified.
+        default values with the values from `qleverfile_config` or
+        `envvars_config` if they are given.
+
+        NOTE: For now, throw an exception if both `qleverfile_config` and
+        `envvars_config` are given. It would be easy to let one override the
+        other, but that might lead to unexpected behavior for a user. For
+        example, the user might write or create `Qleverfile` and is unaware
+        that they also some environment variables set.
         """
 
+        # All argument names for this command.
         arg_names = command_object.relevant_qleverfile_arguments()
 
-        # Helper function that shows a detailed error messahe when an argument
+        # Helper function that shows a detailed error message when an argument
         # from `relevant_qleverfile_arguments` is not contained in
         # `all_qleverfile_args`.
         def argument_error(prefix):
@@ -81,8 +91,8 @@ def argument_error(prefix):
                                    f"`{section}` not found")
                 args, kwargs = all_qleverfile_args[section][arg_name]
                 kwargs_copy = kwargs.copy()
-                # If `qleverfile_config` is given, add info about default
-                # values to the help string.
+                # If `qleverfile_config` is given, add the corresponding
+                # default values.
                 if qleverfile_config is not None:
                     default_value = kwargs.get("default", None)
                     qleverfile_value = qleverfile_config.get(
@@ -94,6 +104,21 @@ def argument_error(prefix):
                                                 f" {qleverfile_value}]")
                     else:
                         kwargs_copy["help"] += f" [default: {default_value}]"
+                # If `envvars_config` is given, add the corresponding default
+                # values.
+                if envvars_config is not None:
+                    default_value = kwargs.get("default", None)
+                    envvar_name = Envvars.envvar_name(section, arg_name)
+                    envvars_value = envvars_config[section].get(arg_name, None)
+                    if envvars_value is not None:
+                        kwargs_copy["default"] = envvars_value
+                        kwargs_copy["required"] = False
+                        kwargs_copy["help"] += (f" [default, from environment "
+                                                f"variable `{envvar_name}`: "
+                                                f"{envvars_value}]")
+                    else:
+                        kwargs_copy["help"] += f" [default: {default_value}]"
+                # Now add the argument to the subparser.
                 subparser.add_argument(*args, **kwargs_copy)
 
         # Additional arguments that are shared by all commands.
@@ -170,6 +195,24 @@ def add_qleverfile_option(parser):
                 exit(1)
         else:
             qleverfile_config = None
+            qleverfile_path = None
+
+        # Now also check if the user has set any environment variables.
+        envvars_config = Envvars.read()
+
+        # Check that at most one of `qleverfile_config` and `envvars_config` is
+        # not `None`, unless the command is `config` (which can be used to
+        # produce a `Qleverfile` or unset all environment variables).
+        if qleverfile_args.command != "config" \
+                and qleverfile_config is not None \
+                and envvars_config is not None:
+            raise ConfigException(
+                    "You both have a `Qleverfile` and environment variables "
+                    "of the QLEVER_SECTION_VARIABLE. This is not supported "
+                    "because it is bound to lead to unexpected behavior. "
+                    "Either remove the `Qleverfile` (just delete it), or the "
+                    "environment variables (use `qlever config "
+                    "--unset-envvars`).")
 
         # Now the regular parser with commands and a subparser for each
         # command. We have a dedicated class for each command. These classes
@@ -188,7 +231,7 @@ def add_qleverfile_option(parser):
         for command_name, command_object in command_objects.items():
             self.add_subparser_for_command(
                     subparsers, command_name, command_object,
-                    all_args, qleverfile_config)
+                    all_args, qleverfile_config, envvars_config)
 
         # Enable autocompletion for the commands and their options.
         #
@@ -204,13 +247,4 @@ def add_qleverfile_option(parser):
         # Parse the command line arguments.
         args = parser.parse_args()
 
-        # If the command says that we should have a Qleverfile, but we don't,
-        # issue a warning.
-        if command_objects[args.command].should_have_qleverfile():
-            if not qleverfile_exists:
-                log.warning(f"Invoking command `{args.command}` without a "
-                            "Qleverfile. You have to specify all required "
-                            "arguments on the command line. This is possible, "
-                            "but not recommended.")
-
-        return args
+        return args, qleverfile_path, qleverfile_config, envvars_config

src/qlever/envvars.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import os
+
+from qlever.qleverfile import Qleverfile
+
+
+class EnvvarsException(Exception):
+    pass
+
+
+class Envvars:
+    """
+    Class for parsing environment variables with analogous names to those in
+    the `Qleverfile` class, according to the schema `QLEVER_SECTION_VARIABLE`.
+    For example, variable `PORT` in section `server` corresponds to the
+    environment variable `QLEVER_SERVER_PORT`.
+    """
+
+    @staticmethod
+    def envvar_name(section: str, name: str) -> str:
+        """
+        For a given section and variable name, return the environment variable
+        name according to the schema described above.
+        """
+        return f"QLEVER_{section.upper()}_{name.upper()}"
+
+    @staticmethod
+    def read():
+        """
+        Check all environment variables that correspond to an entry in
+        `Qleverfile.all_arguments()` according to the schema described above,
+        and return a dictionary `config` with all the values found that way.
+        For example, for `QLEVER_SERVER_PORT=8000`, there would be an entry
+        `config['server']['port'] = 8000`.
+
+        NOTE: If no environment variables was found at all, the method will
+        return `None`. Otherwise, there will be an entry for each section, even
+        if it is empty.
+        """
+
+        all_args = Qleverfile.all_arguments()
+        config = {}
+        num_envvars_found = 0
+        for section, args in all_args.items():
+            config[section] = {}
+            for arg in args:
+                envvar = Envvars.envvar_name(section, arg)
+                if envvar in os.environ:
+                    config[section][arg] = os.environ[envvar]
+                    num_envvars_found += 1
+        return config if num_envvars_found > 0 else None

src/qlever/globals.py

@@ -0,0 +1,15 @@
+# Global variables for the full configuration, set in `qlever_main.py`.
+# For example, these are used by by `qlever config --show-qleverfile-config` or
+# `qlever config --show-envvars`.
+#
+# NOTE 1: Most commands do not (and should not) use these: the `args` passed to
+# the `execute` method of a command class is deliberately reduced to those
+# arguments that are relevant for the command.
+#
+# NOTE 2: If we would define these in `config.py`, which seems like the natural
+# place, we get a circular import error, because we need these in
+# `qlever/commands/config.py`, which would have to import `config.py`, which
+# imports `__init__.py`, which imports all the command modules.
+qleverfile_path = None
+qleverfile_config = None
+envvars_config = None