From 8383e6f70f3618a3376c65fdfd7abeeb077e3712 Mon Sep 17 00:00:00 2001 From: Ronit Jain Date: Mon, 22 Aug 2022 21:22:19 +0530 Subject: [PATCH 1/3] check for uninstlled modules --- lean/commands/live/deploy.py | 51 ++++++++++++++++++++++----- lean/models/json_module.py | 3 ++ lean/models/lean_config_configurer.py | 5 ++- 3 files changed, 47 insertions(+), 12 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index f56cc91e..52d6bfa3 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -16,20 +16,22 @@ import time from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple import click from lean.click import LeanCommand, PathParameter, ensure_options from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds from lean.models.errors import MoreInfoError +from lean.models.lean_config_configurer import LeanConfigConfigurer from lean.models.logger import Option -from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput +from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json from lean.models.json_module import JsonModule from lean.commands.live.live import live from lean.models.data_providers import all_data_providers + _environment_skeleton = { "live-mode": True, "setup-handler": "QuantConnect.Lean.Engine.Setup.BrokerageSetupHandler", @@ -38,12 +40,13 @@ "real-time-handler": "QuantConnect.Lean.Engine.RealTime.LiveTradingRealTimeHandler" } -def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: str, lean_config_path: Path) -> None: - """Raises an error if any required properties are missing. + +def _get_configurable_modules_from_environment(lean_config: Dict[str, Any], environment_name: str) -> Tuple[LeanConfigConfigurer, List[LeanConfigConfigurer]]: + """Returns the configurable modules from the given environment. :param lean_config: the LEAN configuration that should be used :param environment_name: the name of the environment - :param lean_config_path: the path to the LEAN configuration file + :return: the configurable modules from the given environment """ environment = lean_config["environments"][environment_name] for key in ["live-mode-brokerage", "data-queue-handler"]: @@ -53,9 +56,32 @@ def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: brokerage = environment["live-mode-brokerage"] data_queue_handlers = environment["data-queue-handler"] - [brokerage_configurer] = [local_brokerage for local_brokerage in all_local_brokerages if local_brokerage.get_live_name(environment_name) == brokerage] data_feed_configurers = [local_data_feed for local_data_feed in all_local_data_feeds if local_data_feed.get_live_name(environment_name) in data_queue_handlers] + return brokerage_configurer, data_feed_configurers + + +def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, Any]) -> None: + """Raises an error if any of the given modules are not installed. + + :param modules: the modules to check + """ + for module in modules: + organization_id = module.get_organzation_id() + if organization_id is None or organization_id == "": + [organization_config] = module.get_config_from_type(OrganzationIdConfiguration) + organization_id = _get_non_interactive_organization_id(module, organization_config, user_kwargs) + module.ensure_module_installed(organization_id) + + +def _raise_for_missing_properties(lean_config: Dict[str, Any], environment_name: str, lean_config_path: Path) -> None: + """Raises an error if any required properties are missing. + + :param lean_config: the LEAN configuration that should be used + :param environment_name: the name of the environment + :param lean_config_path: the path to the LEAN configuration file + """ + brokerage_configurer, data_feed_configurers = _get_configurable_modules_from_environment(lean_config, environment_name) brokerage_properties = brokerage_configurer.get_required_properties() data_queue_handler_properties = [] for data_feed_configurer in data_feed_configurers: @@ -136,7 +162,7 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], environmen essential_properties_value = {config._id : config._value for config in brokerage.get_essential_configs()} data_feed.update_configs(essential_properties_value) container.logger().debug(f"interactive: essential_properties_value: {brokerage._id} {essential_properties_value}") - # now required properties can be fetched as per data/filter provider from esssential properties + # now required properties can be fetched as per data/filter provider from essential properties required_properties_value = {config._id : config._value for config in brokerage.get_required_configs([InternalInputUserInput])} data_feed.update_configs(required_properties_value) container.logger().debug(f"interactive: required_properties_value: {required_properties_value}") @@ -178,6 +204,10 @@ def _get_organization_id(given_input: Optional[str], label: str) -> str: container.logger().debug(f"deploy._get_organization_id: user selected organization id: {organization.id}") return organization.id +def _get_non_interactive_organization_id(module: LeanConfigConfigurer, + organization_config: OrganzationIdConfiguration, user_kwargs: Dict[str, Any]) -> str: + return _get_organization_id(user_kwargs[module._convert_lean_key_to_variable(organization_config._id)], module._id) + def _get_and_build_module(target_module_name: str, module_list: List[JsonModule], properties: Dict[str, Any]): [target_module] = [module for module in module_list if module.get_name() == target_module_name] # update essential properties from brokerage to datafeed @@ -187,12 +217,12 @@ def _get_and_build_module(target_module_name: str, module_list: List[JsonModule] essential_properties_value = {target_module._convert_variable_to_lean_key(prop) : properties[prop] for prop in essential_properties} target_module.update_configs(essential_properties_value) container.logger().debug(f"non-interactive: essential_properties_value with module {target_module_name}: {essential_properties_value}") - # now required properties can be fetched as per data/filter provider from esssential properties + # now required properties can be fetched as per data/filter provider from essential properties required_properties: List[str] = [] organization_info: Dict[str,str] = {} for config in target_module.get_required_configs([InternalInputUserInput]): if config.is_type_organization_id: - organization_info[config._id] = _get_organization_id(properties[target_module._convert_lean_key_to_variable(config._id)], target_module._id) + organization_info[config._id] = _get_non_interactive_organization_id(target_module, config, properties) properties[target_module._convert_lean_key_to_variable(config._id)] = organization_info[config._id] # skip organization id from ensure_options() because it is fetched using _get_organization_id() continue @@ -362,6 +392,9 @@ def deploy(project: Path, raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/lean-cli/live-trading") + env_brokerage, env_data_queue_handlers = _get_configurable_modules_from_environment(lean_config, environment_name) + _install_modules([env_brokerage] + env_data_queue_handlers, kwargs) + _raise_for_missing_properties(lean_config, environment_name, lean_config_manager.get_lean_config_path()) project_config_manager = container.project_config_manager() diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 2564f6c3..c146a3a1 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -89,6 +89,9 @@ def get_configurations_env_values_from_name(self, target_env: str) -> List[Dict[ env_config_values = env_config._env_and_values[target_env] return env_config_values + def get_config_from_type(self, config_type: Configuration) -> str: + return [copy.copy(config) for config in self._lean_configs if type(config) is config_type] + def get_organzation_id(self) -> str: [organization_id] = [ config._value for config in self._lean_configs if config.is_type_organization_id] diff --git a/lean/models/lean_config_configurer.py b/lean/models/lean_config_configurer.py index 4cbd29d5..a8a9e44b 100644 --- a/lean/models/lean_config_configurer.py +++ b/lean/models/lean_config_configurer.py @@ -39,7 +39,6 @@ def _configure_environment(self, lean_config: Dict[str, Any], environment_name: :param lean_config: the Lean configuration dict to write to :param environment_name: the name of the environment to update """ - self.ensure_module_installed() for environment_config in self.get_configurations_env_values_from_name(environment_name): environment_config_name = environment_config["name"] if self.__class__.__name__ == 'DataFeed': @@ -86,11 +85,11 @@ def configure_credentials(self, lean_config: Dict[str, Any]) -> None: container.logger().debug(f"LeanConfigConfigurer.ensure_module_installed(): _save_properties for module {self._id}: {self.get_required_properties()}") self._save_properties(lean_config, self.get_required_properties()) - def ensure_module_installed(self) -> None: + def ensure_module_installed(self, organization_id: str) -> None: if not self._is_module_installed and self._installs: container.logger().debug(f"LeanConfigConfigurer.ensure_module_installed(): installing module for module {self._id}: {self._product_id}") container.module_manager().install_module( - self._product_id, self.get_organzation_id()) + self._product_id, organization_id) self._is_module_installed = True def _get_default(cls, lean_config: Dict[str, Any], key: str) -> Optional[Any]: From 87d7d62393a6961d4117f45b75edaeec991f853f Mon Sep 17 00:00:00 2001 From: Ronit Jain Date: Mon, 22 Aug 2022 23:15:57 +0530 Subject: [PATCH 2/3] check if module can be installed --- lean/commands/live/deploy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 52d6bfa3..800b0d38 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -31,7 +31,6 @@ from lean.commands.live.live import live from lean.models.data_providers import all_data_providers - _environment_skeleton = { "live-mode": True, "setup-handler": "QuantConnect.Lean.Engine.Setup.BrokerageSetupHandler", @@ -67,6 +66,8 @@ def _install_modules(modules: List[LeanConfigConfigurer], user_kwargs: Dict[str, :param modules: the modules to check """ for module in modules: + if not module._installs: + continue organization_id = module.get_organzation_id() if organization_id is None or organization_id == "": [organization_config] = module.get_config_from_type(OrganzationIdConfiguration) From a19de3f4b7d8bcb05c8b1c2ce0332b05231db0c7 Mon Sep 17 00:00:00 2001 From: Ronit Jain Date: Mon, 22 Aug 2022 23:16:16 +0530 Subject: [PATCH 3/3] mock api client --- tests/commands/test_live.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 86c0ec97..8bd8cd86 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -75,6 +75,12 @@ def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"]) traceback.print_exception(*result.exc_info) @@ -134,6 +140,12 @@ def test_live_calls_lean_runner_with_default_output_directory() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"]) assert result.exit_code == 0 @@ -155,6 +167,12 @@ def test_live_calls_lean_runner_with_custom_output_directory() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", @@ -179,6 +197,12 @@ def test_live_calls_lean_runner_with_release_mode() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + result = CliRunner().invoke(lean, ["live", "CSharp Project", "--environment", "live-paper", "--release"]) assert result.exit_code == 0 @@ -203,6 +227,12 @@ def test_live_calls_lean_runner_with_detach() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", "--detach"]) assert result.exit_code == 0