Merge pull request #289 from rjra2611/feature-set-algorithm-id-support

Feature add support to set algorithm Id using `extra-config`

lean/commands/backtest.py

@@ -23,8 +23,7 @@
 from lean.models.utils import DebuggingMethod
 from lean.models.logger import Option
 from lean.models.data_providers import QuantConnectDataProvider, all_data_providers
-from lean.models.addon_modules import all_addon_modules
-from lean.models.addon_modules.addon_module import AddonModule
+from lean.components.util.addon_modules_handler import build_and_configure_modules
 
 # The _migrate_* methods automatically update launch configurations for a given debugging method.
 #
@@ -323,7 +322,6 @@ def backtest(project: Path,
     Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
     """
     from datetime import datetime
-    addon_modules_to_build: List[AddonModule] = []
     logger = container.logger
     project_manager = container.project_manager
     algorithm_file = project_manager.find_algorithm_file(Path(project))
@@ -378,31 +376,26 @@ def backtest(project: Path,
     if not output.exists():
         output.mkdir(parents=True)
 
-    output_config_manager = container.output_config_manager
-    lean_config["algorithm-id"] = str(output_config_manager.get_backtest_id(output))
-
     # Set backtest name
     if backtest_name is not None and backtest_name != "":
         lean_config["backtest-name"] = backtest_name
 
     # Set extra config
+    given_algorithm_id = None
     for key, value in extra_config:
-        lean_config[key] = value
+        if key == "algorithm-id":
+            given_algorithm_id = int(value)
+        else:
+            lean_config[key] = value
+
+    output_config_manager = container.output_config_manager
+    lean_config["algorithm-id"] = str(output_config_manager.get_backtest_id(output, given_algorithm_id))
 
     if python_venv is not None and python_venv != "":
         lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}'
 
-    for given_module in addon_module:
-        found_module = next((module for module in all_addon_modules if module.get_name().lower() == given_module.lower()), None)
-        if found_module:
-            addon_modules_to_build.append(found_module)
-        else:
-            logger.error(f"Addon module '{given_module}' not found")
-
-    # build and configure addon modules
-    for module in addon_modules_to_build:
-        module.build(lean_config, logger).configure(lean_config, "backtesting")
-        module.ensure_module_installed(container.organization_manager.try_get_working_organization_id())
+    # Configure addon modules
+    build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, "backtesting")
 
     lean_runner = container.lean_runner
     lean_runner.run_lean(lean_config,

lean/commands/live/deploy.py

@@ -28,6 +28,7 @@
 from lean.components.util.live_utils import _get_configs_for_options, get_last_portfolio_cash_holdings, configure_initial_cash_balance, configure_initial_holdings,\
                                             _configure_initial_cash_interactively, _configure_initial_holdings_interactively
 from lean.models.data_providers import all_data_providers
+from lean.components.util.addon_modules_handler import build_and_configure_modules
 
 _environment_skeleton = {
     "live-mode": True,
@@ -266,6 +267,14 @@ def _get_default_value(key: str) -> Optional[Any]:
               default=False,
               help="Pull the LEAN engine image before starting live trading")
 @option("--show-secrets", is_flag=True, show_default=True, default=False, help="Show secrets as they are input")
+@option("--addon-module",
+              type=str,
+              multiple=True,
+              hidden=True)
+@option("--extra-config",
+              type=(str, str),
+              multiple=True,
+              hidden=True)
 @option("--no-update",
               is_flag=True,
               default=False,
@@ -284,6 +293,8 @@ def deploy(project: Path,
            live_holdings: Optional[str],
            update: bool,
            show_secrets: bool,
+           addon_module: Optional[List[str]],
+           extra_config: Optional[Tuple[str, str]],
            no_update: bool,
            **kwargs) -> None:
     """Start live trading a project locally using Docker.
@@ -381,9 +392,6 @@ def deploy(project: Path,
     if not output.exists():
         output.mkdir(parents=True)
 
-    output_config_manager = container.output_config_manager
-    lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output)}"
-
     if python_venv is not None and python_venv != "":
         lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}'
 
@@ -422,5 +430,19 @@ def deploy(project: Path,
     if str(engine_image) != DEFAULT_ENGINE_IMAGE:
         logger.warn(f'A custom engine image: "{engine_image}" is being used!')
 
+    # Set extra config
+    given_algorithm_id = None
+    for key, value in extra_config:
+        if key == "algorithm-id":
+            given_algorithm_id = int(value)
+        else:
+            lean_config[key] = value
+
+    output_config_manager = container.output_config_manager
+    lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output, given_algorithm_id)}"
+
+    # Configure addon modules
+    build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, environment_name)
+
     lean_runner = container.lean_runner
     lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach)

lean/components/config/output_config_manager.py

@@ -36,13 +36,14 @@ def get_output_config(self, output_directory: Path) -> Storage:
         """
         return Storage(str(output_directory / "config"))
 
-    def get_backtest_id(self, backtest_directory: Path) -> int:
+    def get_backtest_id(self, backtest_directory: Path, backtest_id: int = None) -> int:
         """Returns the id of a backtest.
 
         :param backtest_directory: the path to the backtest to retrieve the id of
+        :param backtest_id: the id that needs to be set in the config file
         :return: the id of the given backtest
         """
-        return self._get_id(backtest_directory, 1)
+        return self._get_id(backtest_directory, 1, backtest_id)
 
     def get_backtest_name(self, backtest_directory: Path) -> str:
         """Returns the name of a backtest.
@@ -96,13 +97,14 @@ def get_optimization_by_id(self, optimization_id: int, root_directory: Optional[
         """
         return self._get_by_id("Optimization", optimization_id, ["optimizations/*"], root_directory)
 
-    def get_live_deployment_id(self, live_deployment_directory: Path) -> int:
+    def get_live_deployment_id(self, live_deployment_directory: Path, live_deployment_id: int = None) -> int:
         """Returns the id of a live deployment.
 
         :param live_deployment_directory: the path to the live deployment to retrieve the id of
+        :param live_deployment_id: the id that needs to be set in the config file
         :return: the id of the given optimization
         """
-        return self._get_id(live_deployment_directory, 3)
+        return self._get_id(live_deployment_directory, 3, live_deployment_id)
 
     def get_live_deployment_by_id(self, live_deployment_id: int, root_directory: Optional[Path] = None) -> Path:
         """Finds the directory of a live deployment by its id.
@@ -145,9 +147,13 @@ def get_output_id(self, output_directory: Path) -> Optional[int]:
 
         return output_id
 
-    def _get_id(self, output_directory: Path, prefix: int) -> int:
+    def _get_id(self, output_directory: Path, prefix: int, id: int = None) -> int:
         config = self.get_output_config(output_directory)
 
+        if id is not None:
+            config.set("id", id)
+            return id
+
         if config.has("id"):
             return config.get("id")
 

lean/components/util/addon_modules_handler.py

@@ -0,0 +1,36 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Any, Dict, List
+from lean.models.addon_modules.addon_module import AddonModule
+from lean.models.addon_modules import all_addon_modules
+from lean.components.util.logger import Logger
+
+def build_and_configure_modules(modules: List[AddonModule], organization_id: str, lean_config: Dict[str, Any], logger: Logger, environment_name: str) -> Dict[str, Any]:
+    """Capitalizes the given word.
+
+    :param word: the word to capitalize
+    :return: the word with the first letter capitalized (any other uppercase characters are preserved)
+    """
+    for given_module in modules:
+        try:
+            found_module = next((module for module in all_addon_modules if module.get_name().lower() == given_module.lower()), None)
+            if found_module:
+                found_module.build(lean_config, logger).configure(lean_config, environment_name)
+                found_module.ensure_module_installed(organization_id)
+            else:
+                logger.error(f"Addon module '{given_module}' not found")
+        except Exception as e:
+            logger.error(f"Addon module '{given_module}' failed to configure: {e}")
+    return lean_config
+