From 56fae431c45c087f909cf8b08db058a92c7a5117 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Wed, 3 Apr 2024 15:28:31 -0400 Subject: [PATCH 01/10] feat: json module config option to be mounted to container --- lean/commands/backtest.py | 5 ++++- lean/components/docker/lean_runner.py | 23 ++++++++++++++++++++++- lean/models/configuration.py | 1 + lean/models/json_module.py | 3 +++ 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index b7ba0bc9..832fe228 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -352,12 +352,14 @@ def backtest(project: Path, data_provider_historical = "QuantConnect" organization_id = container.organization_manager.try_get_working_organization_id() + paths_to_mount = None if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, cli_data_downloaders, kwargs, logger, environment_name) data_provider.ensure_module_installed(organization_id) container.lean_config_manager.set_properties(data_provider.get_settings()) + paths_to_mount = data_provider.get_paths_to_mount() lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) @@ -406,4 +408,5 @@ def backtest(project: Path, debugging_method, release, detach, - loads(extra_docker_config)) + loads(extra_docker_config), + paths_to_mount) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 9a1e0507..e58e4e86 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -73,7 +73,8 @@ def run_lean(self, debugging_method: Optional[DebuggingMethod], release: bool, detach: bool, - extra_docker_config: Optional[Dict[str, Any]] = None) -> None: + extra_docker_config: Optional[Dict[str, Any]] = None, + paths_to_mount: Optional[Dict[str, str]] = None) -> None: """Runs the LEAN engine locally in Docker. Raises an error if something goes wrong. @@ -87,10 +88,27 @@ def run_lean(self, :param release: whether C# projects should be compiled in release configuration instead of debug :param detach: whether LEAN should run in a detached container :param extra_docker_config: additional docker configurations + :param paths_to_mount: additional paths to mount to the container """ self._logger.debug(f'LeanRunner().run_lean: lean_config: {lean_config}') project_dir = algorithm_file.parent + # Add additional paths to mount to the container + mounts = [] + if paths_to_mount is not None: + from docker.types import Mount + for key, pathStr in paths_to_mount.items(): + path = Path(pathStr).resolve() + target = f"/Files/{Path(path).name}" + + self._logger.info(f"Mounting {path} to {target}") + + mounts.append(Mount(target=target, + source=str(path), + type="bind", + read_only=True)) + lean_config["environments"][environment][key] = target + # The dict containing all options passed to `docker run` # See all available options at https://docker-py.readthedocs.io/en/stable/containers.html run_options = self.get_basic_docker_config(lean_config, @@ -104,6 +122,9 @@ def run_lean(self, # Add known additional run options from the extra docker config self.parse_extra_docker_config(run_options, extra_docker_config) + # Add the additional mounts + run_options["mounts"].extend(mounts) + # Set up PTVSD debugging if debugging_method == DebuggingMethod.PTVSD: run_options["ports"]["5678"] = "5678" diff --git a/lean/models/configuration.py b/lean/models/configuration.py index e4c6e77c..4489d734 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -104,6 +104,7 @@ def __init__(self, config_json_object): self._filter = Filter([]) self._input_default = config_json_object["input-default"] if "input-default" in config_json_object else None self._optional = config_json_object["optional"] if "optional" in config_json_object else False + self._should_mount = config_json_object["mount"] if "mount" in config_json_object else False def factory(config_json_object) -> 'Configuration': """Creates an instance of the child classes. diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 071daac6..7376ad98 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -228,6 +228,9 @@ def config_build(self, .join(missing_options)}""".strip()) return self + def get_paths_to_mount(self) -> Dict[str, str]: + return {config._id: config._value for config in self._lean_configs if config._should_mount} + def ensure_module_installed(self, organization_id: str) -> None: if not self._is_module_installed and self._installs: container.logger.debug(f"JsonModule.ensure_module_installed(): installing module {self}: {self._product_id}") From 36677647df2c5d5716a76dabdd63b26b7e181e3e Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 4 Apr 2024 16:36:33 -0400 Subject: [PATCH 02/10] add unit tests --- tests/commands/test_backtest.py | 167 ++++++++++++-------- tests/components/docker/test_lean_runner.py | 62 ++++++++ 2 files changed, 161 insertions(+), 68 deletions(-) diff --git a/tests/commands/test_backtest.py b/tests/commands/test_backtest.py index a59ea0f0..9a3a373d 100644 --- a/tests/commands/test_backtest.py +++ b/tests/commands/test_backtest.py @@ -24,6 +24,7 @@ from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container from lean.models.api import QCLanguage +from lean.models.json_module import JsonModule from lean.models.utils import DebuggingMethod from lean.models.docker import DockerImage from tests.conftest import initialize_container @@ -51,14 +52,15 @@ def test_backtest_calls_lean_runner_with_correct_algorithm_file() -> None: assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - mock.ANY, - ENGINE_IMAGE, - None, - False, - False, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + ENGINE_IMAGE, + None, + False, + False, + {}, + {}) def test_backtest_calls_lean_runner_with_default_output_directory() -> None: @@ -83,14 +85,15 @@ def test_backtest_calls_lean_runner_with_custom_output_directory() -> None: assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - Path.cwd() / "Python Project" / "custom", - ENGINE_IMAGE, - None, - False, - False, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + Path.cwd() / "Python Project" / "custom", + ENGINE_IMAGE, + None, + False, + False, + {}, + {}) def test_backtest_calls_lean_runner_with_release_mode() -> None: @@ -101,14 +104,15 @@ def test_backtest_calls_lean_runner_with_release_mode() -> None: assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("CSharp Project/Main.cs").resolve(), - mock.ANY, - ENGINE_IMAGE, - None, - True, - False, - {}) + "backtesting", + Path("CSharp Project/Main.cs").resolve(), + mock.ANY, + ENGINE_IMAGE, + None, + True, + False, + {}, + {}) def test_backtest_calls_lean_runner_with_detach() -> None: @@ -119,14 +123,15 @@ def test_backtest_calls_lean_runner_with_detach() -> None: assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - mock.ANY, - ENGINE_IMAGE, - None, - False, - True, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + ENGINE_IMAGE, + None, + False, + True, + {}, + {}) def test_backtest_aborts_when_project_does_not_exist() -> None: @@ -161,14 +166,15 @@ def test_backtest_forces_update_when_update_option_given() -> None: docker_manager.pull_image.assert_called_once_with(ENGINE_IMAGE) container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - mock.ANY, - ENGINE_IMAGE, - None, - False, - False, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + ENGINE_IMAGE, + None, + False, + False, + {}, + {}) def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> None: @@ -181,14 +187,15 @@ def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> Non assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - mock.ANY, - DockerImage(name="custom/lean", tag="123"), - None, - False, - False, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + DockerImage(name="custom/lean", tag="123"), + None, + False, + False, + {}, + {}) def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> None: @@ -201,19 +208,20 @@ def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> N assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - mock.ANY, - DockerImage(name="custom/lean", tag="456"), - None, - False, - False, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + DockerImage(name="custom/lean", tag="456"), + None, + False, + False, + {}, + {}) @pytest.mark.parametrize("python_venv", ["Custom-venv", - "/Custom-venv", - None]) + "/Custom-venv", + None]) def test_backtest_passes_custom_python_venv_to_lean_runner_when_given_as_option(python_venv: str) -> None: create_fake_lean_cli_directory() @@ -290,14 +298,15 @@ def test_backtest_passes_correct_debugging_method_to_lean_runner(value: str, deb assert result.exit_code == 0 container.lean_runner.run_lean.assert_called_once_with(mock.ANY, - "backtesting", - Path("Python Project/main.py").resolve(), - mock.ANY, - ENGINE_IMAGE, - debugging_method, - False, - False, - {}) + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + ENGINE_IMAGE, + debugging_method, + False, + False, + {}, + {}) def test_backtest_auto_updates_outdated_python_pycharm_debug_config() -> None: @@ -612,7 +621,8 @@ def test_backtest_passes_data_purchase_limit_to_lean_runner() -> None: } """) - result = CliRunner().invoke(lean, ["backtest", "Python Project", "--data-provider-historical", "QuantConnect", "--data-purchase-limit", "1000"]) + result = CliRunner().invoke(lean, ["backtest", "Python Project", "--data-provider-historical", "QuantConnect", + "--data-purchase-limit", "1000"]) assert result.exit_code == 0 @@ -684,4 +694,25 @@ def test_backtest_calls_lean_runner_with_extra_docker_config() -> None: "volumes": { "extra/path": {"bind": "/extra/path", "mode": "rw"} } - }) + }, + {}) + + +def test_backtest_calls_lean_runner_with_paths_to_mount() -> None: + create_fake_lean_cli_directory() + + with mock.patch.object(JsonModule, "get_paths_to_mount", return_value={"some-config": "/path/to/file.json"}): + result = CliRunner().invoke(lean, ["backtest", "Python Project", "--data-provider-historical", "QuantConnect"]) + + assert result.exit_code == 0 + + container.lean_runner.run_lean.assert_called_once_with(mock.ANY, + "backtesting", + Path("Python Project/main.py").resolve(), + mock.ANY, + ENGINE_IMAGE, + None, + False, + False, + {}, + {"some-config": "/path/to/file.json"}) diff --git a/tests/components/docker/test_lean_runner.py b/tests/components/docker/test_lean_runner.py index 1520b5f6..1a15c1d0 100644 --- a/tests/components/docker/test_lean_runner.py +++ b/tests/components/docker/test_lean_runner.py @@ -690,3 +690,65 @@ def test_run_lean_passes_extra_volumes() -> None: assert "extra/path" in volumes assert volumes["extra/path"]["bind"] == "/extra/bound/path" assert volumes["extra/path"]["mode"] == "rw" + + +def test_run_lean_mounts_additional_paths() -> None: + create_fake_lean_cli_directory() + + docker_manager = mock.Mock() + docker_manager.run_image.return_value = True + + lean_runner = create_lean_runner(docker_manager) + + lean_config = { + "transaction-log": "transaction-log.log", + "environments": { + "backtesting": {} + } + } + paths_to_mount = { + "file-to-mount-key": "../some/path/to/mount/file.json", + "directory-to-mount-key": "../some/path/to/mount" + } + + lean_runner.run_lean(lean_config, + "backtesting", + Path.cwd() / "Python Project" / "main.py", + Path.cwd() / "output", + ENGINE_IMAGE, + None, + False, + False, + {}, + paths_to_mount) + + docker_manager.run_image.assert_called_once() + _, kwargs = docker_manager.run_image.call_args + + assert "mounts" in kwargs + mounts = kwargs["mounts"] + + def source_to_target(path): + return f"/Files/{str(Path(path).name)}" + + def path_is_in_mounts(path): + return any([mount["Source"] == str(Path(path).resolve()) for mount in mounts]) + + def path_is_mounted_in_files(path): + return any([mount["Target"] == source_to_target(path) for mount in mounts]) + + assert all([path_is_in_mounts(path) and path_is_mounted_in_files(path) for path in paths_to_mount.values()]) + + # The target paths should have been added to the lean config + lean_config_path = next((mount["Source"] for mount in mounts if mount["Target"].endswith("config.json")), None) + assert lean_config_path is not None + + # Read temporal lean config + with open(lean_config_path, "r") as f: + import json + lean_config = json.load(f) + + backtesting_env = lean_config["environments"]["backtesting"] + + assert all([key in backtesting_env and backtesting_env[key] == source_to_target(value) + for key, value in paths_to_mount.items()]) From c5c9317572ed9ec964808a3fb2ba918282d44ef9 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 4 Apr 2024 16:39:21 -0400 Subject: [PATCH 03/10] Update readme --- README.md | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d72b7951..173f62a3 100644 --- a/README.md +++ b/README.md @@ -143,7 +143,7 @@ Options: -d, --detach Run the backtest in a detached Docker container and return immediately --debug [pycharm|ptvsd|vsdbg|rider|local-platform] Enable a certain debugging method (see --help for more information) - --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link] + --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link] Update the Lean configuration file to retrieve data from the given historical provider --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures] Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures] @@ -161,6 +161,8 @@ Options: --iqfeed-version TEXT The product version of your IQFeed developer account --iqfeed-host TEXT The IQFeed host address --polygon-api-key TEXT Your Polygon.io API Key + --factset-auth-config-file TEXT + The path to the FactSet authentication configuration file --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] Your IEX Cloud Price plan @@ -1081,7 +1083,7 @@ Options: The brokerage to use --data-provider-live [Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Advanced Trade|Binance|Zerodha|Samco|Terminal Link|Trading Technologies|Kraken|TDAmeritrade|IQFeed|Polygon|IEX|CoinApi|Custom data only|Bybit] The live data provider to use - --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local] + --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local] Update the Lean configuration file to retrieve data from the given historical provider --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -1194,6 +1196,8 @@ Options: --coinapi-api-key TEXT Your coinapi.io Api Key --coinapi-product [Free|Startup|Streamer|Professional|Enterprise] CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing) + --factset-auth-config-file TEXT + The path to the FactSet authentication configuration file --alpha-vantage-api-key TEXT Your Alpha Vantage Api Key --alpha-vantage-price-plan [Free|Plan30|Plan75|Plan150|Plan300|Plan600|Plan1200] Your Alpha Vantage Premium API Key plan @@ -1486,7 +1490,7 @@ Options: --parameter ... The 'parameter min max step' pairs configuring the parameters to optimize --constraint TEXT The 'statistic operator value' pairs configuring the constraints of the optimization - --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link] + --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link] Update the Lean configuration file to retrieve data from the given historical provider --download-data Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider-historical QuantConnect @@ -1517,6 +1521,8 @@ Options: --iqfeed-version TEXT The product version of your IQFeed developer account --iqfeed-host TEXT The IQFeed host address --polygon-api-key TEXT Your Polygon.io API Key + --factset-auth-config-file TEXT + The path to the FactSet authentication configuration file --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] Your IEX Cloud Price plan @@ -1641,7 +1647,7 @@ Usage: lean research [OPTIONS] PROJECT Options: --port INTEGER The port to run Jupyter Lab on (defaults to 8888) - --data-provider-historical [Binance|Kraken|IQFeed|Polygon|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link] + --data-provider-historical [Binance|Kraken|IQFeed|Polygon|FactSet|IEX|AlphaVantage|CoinApi|QuantConnect|Local|Terminal Link] Update the Lean configuration file to retrieve data from the given historical provider --binance-exchange-name [Binance|BinanceUS|Binance-USDM-Futures|Binance-COIN-Futures] Binance exchange name [Binance, BinanceUS, Binance-USDM-Futures, Binance-COIN-Futures] @@ -1659,6 +1665,8 @@ Options: --iqfeed-version TEXT The product version of your IQFeed developer account --iqfeed-host TEXT The IQFeed host address --polygon-api-key TEXT Your Polygon.io API Key + --factset-auth-config-file TEXT + The path to the FactSet authentication configuration file --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] Your IEX Cloud Price plan From b2f51adadc244afe6efbac956ee19a3ff85d2ebd Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 10:29:20 -0400 Subject: [PATCH 04/10] refactor: move mounting paths to more generic location --- lean/components/docker/lean_runner.py | 84 +++++++++++++-------------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index e58e4e86..88cdd7e3 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -93,22 +93,6 @@ def run_lean(self, self._logger.debug(f'LeanRunner().run_lean: lean_config: {lean_config}') project_dir = algorithm_file.parent - # Add additional paths to mount to the container - mounts = [] - if paths_to_mount is not None: - from docker.types import Mount - for key, pathStr in paths_to_mount.items(): - path = Path(pathStr).resolve() - target = f"/Files/{Path(path).name}" - - self._logger.info(f"Mounting {path} to {target}") - - mounts.append(Mount(target=target, - source=str(path), - type="bind", - read_only=True)) - lean_config["environments"][environment][key] = target - # The dict containing all options passed to `docker run` # See all available options at https://docker-py.readthedocs.io/en/stable/containers.html run_options = self.get_basic_docker_config(lean_config, @@ -117,14 +101,12 @@ def run_lean(self, debugging_method, release, detach, - image) + image, + paths_to_mount) # Add known additional run options from the extra docker config self.parse_extra_docker_config(run_options, extra_docker_config) - # Add the additional mounts - run_options["mounts"].extend(mounts) - # Set up PTVSD debugging if debugging_method == DebuggingMethod.PTVSD: run_options["ports"]["5678"] = "5678" @@ -199,7 +181,8 @@ def get_basic_docker_config(self, debugging_method: Optional[DebuggingMethod], release: bool, detach: bool, - image: DockerImage) -> Dict[str, Any]: + image: DockerImage, + paths_to_mount: Optional[Dict[str, str]] = None) -> Dict[str, Any]: """Creates a basic Docker config to run the engine with. This method constructs the parts of the Docker config that is the same for both the engine and the optimizer. @@ -212,6 +195,7 @@ def get_basic_docker_config(self, :param detach: whether LEAN should run in a detached container :param image: The docker image that will be used :return: the Docker configuration containing basic configuration to run Lean + :param paths_to_mount: additional paths to mount to the container """ from docker.types import Mount from uuid import uuid4 @@ -261,6 +245,9 @@ def get_basic_docker_config(self, "ports": docker_project_config.get("ports", {}) } + # mount the paths passed in + self.mount_paths(paths_to_mount, lean_config, run_options) + # mount the project and library directories self.mount_project_and_library_directories(project_dir, run_options) @@ -334,10 +321,7 @@ def get_basic_docker_config(self, # Create a C# project used to resolve the dependencies of the modules run_options["commands"].append("mkdir /ModulesProject") run_options["commands"].append("dotnet new sln -o /ModulesProject") - - framework_ver = self._docker_manager.get_image_label(image, 'target_framework', - DEFAULT_LEAN_DOTNET_FRAMEWORK) - run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") + run_options["commands"].append("dotnet new classlib -o /ModulesProject -f net6.0 --no-restore") run_options["commands"].append("rm /ModulesProject/Class1.cs") # Add all modules to the project, automatically resolving all dependencies @@ -353,8 +337,7 @@ def get_basic_docker_config(self, # Set up language-specific run options self.setup_language_specific_run_options(run_options, project_dir, algorithm_file, - set_up_common_csharp_options_called, release, - image) + set_up_common_csharp_options_called, release) # Save the final Lean config to a temporary file so we can mount it into the container config_path = self._temp_manager.create_temporary_directory() / "config.json" @@ -388,12 +371,11 @@ def get_basic_docker_config(self, return run_options - def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None: + def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) -> None: """Sets up Docker run options specific to Python projects. :param project_dir: the path to the project directory :param run_options: the dictionary to append run options to - :param image: the docker image that will be used """ from docker.types import Mount @@ -440,14 +422,10 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], "mode": "rw" } - python_version = self._docker_manager.get_image_label(image, 'python_version', - DEFAULT_LEAN_PYTHON_VERSION) - site_packages_path = DOCKER_PYTHON_SITE_PACKAGES_PATH.replace('{LEAN_PYTHON_VERSION}', python_version) - # Mount a volume to the user packages directory so we don't install packages every time site_packages_volume = self._docker_manager.create_site_packages_volume(requirements_txt) run_options["volumes"][site_packages_volume] = { - "bind": f"{site_packages_path}", + "bind": f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}", "mode": "rw" } @@ -458,7 +436,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], # We only need to do this if it hasn't already been done before for this site packages volume # To keep track of this we create a special file in the site packages directory after installation # If this file already exists we can skip pip install completely - marker_file = f"{site_packages_path}/pip-install-done" + marker_file = f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}/pip-install-done" run_options["commands"].extend([ f"! test -f {marker_file} && pip install --user --progress-bar off -r /requirements.txt", f"touch {marker_file}" @@ -486,14 +464,12 @@ def _concat_python_requirements(self, requirements_files: List[Path]) -> str: requirements = sorted(set(requirements)) return "\n".join(requirements) - def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool, - image: DockerImage) -> None: + def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool) -> None: """Sets up Docker run options specific to C# projects. :param project_dir: the path to the project directory :param run_options: the dictionary to append run options to :param release: whether C# projects should be compiled in release configuration instead of debug - :param image: the docker image that will be used """ compile_root = self._get_csharp_compile_root(project_dir) @@ -510,13 +486,11 @@ def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], for path in compile_root.rglob("*.csproj"): self._ensure_csproj_uses_correct_lean(compile_root, path, csproj_temp_dir, run_options) - framework_ver = self._docker_manager.get_image_label(image, 'target_framework', - DEFAULT_LEAN_DOTNET_FRAMEWORK) # Set up the MSBuild properties msbuild_properties = { "Configuration": "Release" if release else "Debug", "Platform": "AnyCPU", - "TargetFramework": framework_ver, + "TargetFramework": "net6.0", "OutputPath": "/Compile/bin", "GenerateAssemblyInfo": "false", "GenerateTargetFrameworkAttribute": "false", @@ -760,14 +734,14 @@ def _force_disk_provider_if_necessary(self, lean_config[config_key] = disk_provider def setup_language_specific_run_options(self, run_options, project_dir, algorithm_file, - set_up_common_csharp_options_called, release, image: DockerImage) -> None: + set_up_common_csharp_options_called, release) -> None: # Set up language-specific run options if algorithm_file.name.endswith(".py"): - self.set_up_python_options(project_dir, run_options, image) + self.set_up_python_options(project_dir, run_options) else: if not set_up_common_csharp_options_called: self.set_up_common_csharp_options(run_options) - self.set_up_csharp_options(project_dir, run_options, release, image) + self.set_up_csharp_options(project_dir, run_options, release) def format_error_before_logging(self, chunk: str): from lean.components.util import compiler @@ -803,6 +777,28 @@ def mount_project_and_library_directories(self, project_dir: Path, run_options: "mode": "rw" } + def mount_paths(self, paths_to_mount, lean_config, run_options): + mounts = [] + environment = {} + if "environment" in lean_config and "environments" in lean_config: + environment = lean_config["environments"][lean_config["environment"]] + if paths_to_mount is not None: + from docker.types import Mount + for key, pathStr in paths_to_mount.items(): + path = Path(pathStr).resolve() + target = f"/Files/{Path(path).name}" + + self._logger.info(f"Mounting {path} to {target}") + + mounts.append(Mount(target=target, + source=str(path), + type="bind", + read_only=True)) + environment[key] = target + + # Add the additional mounts to the container run uptions + run_options["mounts"].extend(mounts) + @staticmethod def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None: from docker.types import DeviceRequest From 8361b05afb0a08b67f0146667a74b2bf56604861 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 11:39:52 -0400 Subject: [PATCH 05/10] feature: support mounting json module config path in research, optimize and live --- lean/commands/live/deploy.py | 13 +++++++- lean/commands/optimize.py | 5 ++- lean/commands/research.py | 6 +++- lean/components/docker/lean_runner.py | 36 +++++++++++---------- tests/commands/test_live.py | 24 ++++++++++++++ tests/commands/test_optimize.py | 28 ++++++++++++++++ tests/commands/test_research.py | 26 ++++++++++++++- tests/components/docker/test_lean_runner.py | 1 + 8 files changed, 118 insertions(+), 21 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index ffc33473..25a28c26 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -282,9 +282,11 @@ def deploy(project: Path, environment_name=environment_name)) organization_id = container.organization_manager.try_get_working_organization_id() + paths_to_mount = {} for module in (data_provider_live_instances + [data_downloader_instances, brokerage_instance] + history_providers_instances): module.ensure_module_installed(organization_id) + paths_to_mount.update(module.get_paths_to_mount()) if not lean_config["environments"][environment_name]["live-mode"]: raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", @@ -362,4 +364,13 @@ def deploy(project: Path, raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts") lean_runner = container.lean_runner - lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config)) + lean_runner.run_lean(lean_config, + environment_name, + algorithm_file, + output, + engine_image, + None, + release, + detach, + loads(extra_docker_config), + paths_to_mount) diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 92611b2c..7df8065c 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -305,11 +305,14 @@ def optimize(project: Path, if download_data: data_provider_historical = "QuantConnect" + paths_to_mount = None + if data_provider_historical is not None: data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, cli_data_downloaders, kwargs, logger, environment_name) data_provider.ensure_module_installed(organization_id) container.lean_config_manager.set_properties(data_provider.get_settings()) + paths_to_mount = data_provider.get_paths_to_mount() lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) @@ -339,7 +342,7 @@ def optimize(project: Path, container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update) run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach, - engine_image) + engine_image, paths_to_mount) run_options["working_dir"] = "/Lean/Optimizer.Launcher/bin/Debug" run_options["commands"].append(f"dotnet QuantConnect.Optimizer.Launcher.dll{' --estimate' if estimate else ''}") diff --git a/lean/commands/research.py b/lean/commands/research.py index 42715584..6596e87c 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -113,12 +113,15 @@ def research(project: Path, if download_data: data_provider_historical = "QuantConnect" + paths_to_mount = None + if data_provider_historical is not None: organization_id = container.organization_manager.try_get_working_organization_id() data_provider = non_interactive_config_build_for_name(lean_config, data_provider_historical, cli_data_downloaders, kwargs, logger, environment_name) data_provider.ensure_module_installed(organization_id) container.lean_config_manager.set_properties(data_provider.get_settings()) + paths_to_mount = data_provider.get_paths_to_mount() lean_config_manager.configure_data_purchase_limit(lean_config, data_purchase_limit) lean_runner = container.lean_runner @@ -145,7 +148,8 @@ def research(project: Path, None, False, detach, - research_image) + research_image, + paths_to_mount) # Mount the config in the notebooks directory as well local_config_path = next(m["Source"] for m in run_options["mounts"] if m["Target"].endswith("config.json")) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 88cdd7e3..3b6d524a 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -778,26 +778,28 @@ def mount_project_and_library_directories(self, project_dir: Path, run_options: } def mount_paths(self, paths_to_mount, lean_config, run_options): - mounts = [] + if not paths_to_mount: + return + + from docker.types import Mount + environment = {} if "environment" in lean_config and "environments" in lean_config: environment = lean_config["environments"][lean_config["environment"]] - if paths_to_mount is not None: - from docker.types import Mount - for key, pathStr in paths_to_mount.items(): - path = Path(pathStr).resolve() - target = f"/Files/{Path(path).name}" - - self._logger.info(f"Mounting {path} to {target}") - - mounts.append(Mount(target=target, - source=str(path), - type="bind", - read_only=True)) - environment[key] = target - - # Add the additional mounts to the container run uptions - run_options["mounts"].extend(mounts) + + mounts = run_options["mounts"] + + for key, pathStr in paths_to_mount.items(): + path = Path(pathStr).resolve() + target = f"/Files/{Path(path).name}" + + self._logger.info(f"Mounting {path} to {target}") + + mounts.append(Mount(target=target, + source=str(path), + type="bind", + read_only=True)) + environment[key] = target @staticmethod def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None: diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index e9c78c9f..a82d858f 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -24,6 +24,7 @@ from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container from lean.models.docker import DockerImage +from lean.models.json_module import JsonModule from tests.test_helpers import create_fake_lean_cli_directory, reset_state_installed_modules from tests.conftest import initialize_container from click.testing import Result @@ -151,6 +152,29 @@ def test_live_calls_lean_runner_with_extra_docker_config() -> None: }) +def test_live_calls_lean_runner_with_paths_to_mount() -> None: + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + + with mock.patch.object(JsonModule, "get_paths_to_mount", return_value={"some-config": "/path/to/file.json"}): + result = CliRunner().invoke(lean, ["live", "Python Project", + "--environment", "live-paper", + "--data-provider-historical", "QuantConnect"]) + + assert result.exit_code == 0 + + container.lean_runner.run_lean.assert_called_once_with(mock.ANY, + "live-paper", + Path("Python Project/main.py").resolve(), + mock.ANY, + ENGINE_IMAGE, + None, + False, + False, + {}, + {"some-config": "/path/to/file.json"}) + + def test_live_aborts_when_environment_does_not_exist() -> None: create_fake_lean_cli_directory() diff --git a/tests/commands/test_optimize.py b/tests/commands/test_optimize.py index c85c22a6..34d67073 100644 --- a/tests/commands/test_optimize.py +++ b/tests/commands/test_optimize.py @@ -24,6 +24,7 @@ from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH from lean.container import container from lean.models.docker import DockerImage +from lean.models.json_module import JsonModule from lean.models.optimizer import (OptimizationConstraint, OptimizationExtremum, OptimizationParameter, OptimizationTarget) from tests.test_helpers import create_fake_lean_cli_directory @@ -816,3 +817,30 @@ def test_optimize_used_data_downloader_specified_with_data_provider_option() -> config = json.loads(Path(lean_config_filename).read_text(encoding="utf-8")) assert "data-downloader" in config['environments']['backtesting'] assert config['environments']['backtesting']["data-downloader"] == "QuantConnect.Lean.DataSource.Polygon.PolygonDataDownloader" + + +def test_optimize_runs_lean_container_with_paths_to_mount() -> None: + create_fake_lean_cli_directory() + + docker_manager = mock.MagicMock() + docker_manager.run_image.side_effect = run_image + container.initialize(docker_manager=docker_manager) + container.optimizer_config_manager = _get_optimizer_config_manager_mock() + + Storage(str(Path.cwd() / "Python Project" / "config.json")).set("parameters", {"param1": "1"}) + + with mock.patch.object(JsonModule, "get_paths_to_mount", return_value={"some-config": "/path/to/file.json"}): + result = CliRunner().invoke(lean, ["optimize", "Python Project", "--data-provider-historical", "QuantConnect"]) + + assert result.exit_code == 0 + + docker_manager.run_image.assert_called_once() + args, kwargs = docker_manager.run_image.call_args + + assert args[0] == ENGINE_IMAGE + + expected_source = str(Path("/path/to/file.json").resolve()) + mount = next((m for m in kwargs["mounts"] if m["Source"] == expected_source), None) + + assert mount is not None + assert mount["Target"] == "/Files/file.json" diff --git a/tests/commands/test_research.py b/tests/commands/test_research.py index a6a2281a..9dd57e80 100644 --- a/tests/commands/test_research.py +++ b/tests/commands/test_research.py @@ -21,6 +21,7 @@ from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH from lean.container import container from lean.models.docker import DockerImage +from lean.models.json_module import JsonModule from tests.test_helpers import create_fake_lean_cli_directory RESEARCH_IMAGE = DockerImage.parse(DEFAULT_RESEARCH_IMAGE) @@ -229,7 +230,7 @@ def test_research_runs_custom_image_when_given_as_option() -> None: assert args[0] == DockerImage(name="custom/research", tag="456") -def test_optimize_runs_lean_container_with_extra_docker_config() -> None: +def test_research_runs_lean_container_with_extra_docker_config() -> None: import docker.types create_fake_lean_cli_directory() @@ -256,3 +257,26 @@ def test_optimize_runs_lean_container_with_extra_docker_config() -> None: volumes = kwargs["volumes"] assert "extra/path" in volumes assert volumes["extra/path"] == {"bind": "/extra/path", "mode": "rw"} + + +def test_research_runs_lean_container_with_paths_to_mount() -> None: + create_fake_lean_cli_directory() + + docker_manager = mock.MagicMock() + container.initialize(docker_manager) + + with mock.patch.object(JsonModule, "get_paths_to_mount", return_value={"some-config": "/path/to/file.json"}): + result = CliRunner().invoke(lean, ["research", "Python Project", "--data-provider-historical", "QuantConnect"]) + + assert result.exit_code == 0 + + docker_manager.run_image.assert_called_once() + args, kwargs = docker_manager.run_image.call_args + + assert args[0] == RESEARCH_IMAGE + + expected_source = str(Path("/path/to/file.json").resolve()) + mount = next((m for m in kwargs["mounts"] if m["Source"] == expected_source), None) + + assert mount is not None + assert mount["Target"] == "/Files/file.json" diff --git a/tests/components/docker/test_lean_runner.py b/tests/components/docker/test_lean_runner.py index 1a15c1d0..ff2e7138 100644 --- a/tests/components/docker/test_lean_runner.py +++ b/tests/components/docker/test_lean_runner.py @@ -702,6 +702,7 @@ def test_run_lean_mounts_additional_paths() -> None: lean_config = { "transaction-log": "transaction-log.log", + "environment": "backtesting", "environments": { "backtesting": {} } From cd651ae13b45abeb5e2338d6b3b4ebaa99b5d697 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 12:13:11 -0400 Subject: [PATCH 06/10] minor fix --- lean/models/configuration.py | 1 - lean/models/json_module.py | 8 ++++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lean/models/configuration.py b/lean/models/configuration.py index 4489d734..e4c6e77c 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -104,7 +104,6 @@ def __init__(self, config_json_object): self._filter = Filter([]) self._input_default = config_json_object["input-default"] if "input-default" in config_json_object else None self._optional = config_json_object["optional"] if "optional" in config_json_object else False - self._should_mount = config_json_object["mount"] if "mount" in config_json_object else False def factory(config_json_object) -> 'Configuration': """Creates an instance of the child classes. diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 7376ad98..dca7b556 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -20,7 +20,8 @@ from lean.components.util.logger import Logger from lean.constants import MODULE_TYPE, MODULE_PLATFORM, MODULE_CLI_PLATFORM from lean.container import container -from lean.models.configuration import BrokerageEnvConfiguration, Configuration, InternalInputUserInput +from lean.models.configuration import BrokerageEnvConfiguration, Configuration, InternalInputUserInput, \ + PathParameterUserInput from copy import copy from abc import ABC @@ -229,7 +230,10 @@ def config_build(self, return self def get_paths_to_mount(self) -> Dict[str, str]: - return {config._id: config._value for config in self._lean_configs if config._should_mount} + return {config._id: config._value + for config in self._lean_configs + if (isinstance(config, PathParameterUserInput) + and self._check_if_config_passes_filters(config, all_for_platform_type=False))} def ensure_module_installed(self, organization_id: str) -> None: if not self._is_module_installed and self._installs: From 5407ee0c3fe04dc1717088eeb01533607b6e2ef1 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 13:00:50 -0400 Subject: [PATCH 07/10] minor unit tests fix --- tests/commands/test_live.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index a82d858f..191e5f28 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -115,6 +115,7 @@ def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: None, False, False, + {}, {}) @@ -149,7 +150,8 @@ def test_live_calls_lean_runner_with_extra_docker_config() -> None: "volumes": { "extra/path": {"bind": "/extra/path", "mode": "rw"} } - }) + }, + {}) def test_live_calls_lean_runner_with_paths_to_mount() -> None: @@ -248,6 +250,7 @@ def test_live_calls_lean_runner_with_release_mode() -> None: None, True, False, + {}, {}) @@ -268,6 +271,7 @@ def test_live_calls_lean_runner_with_detach() -> None: None, False, True, + {}, {}) @@ -491,6 +495,7 @@ def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: None, False, False, + {}, {}) @@ -594,6 +599,7 @@ def test_live_non_interactive_do_not_store_non_persistent_properties_in_lean_con None, False, False, + {}, {}) config = container.lean_config_manager.get_lean_config() @@ -636,6 +642,7 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s None, False, False, + {}, {}) @pytest.mark.parametrize("brokerage,data_feed1,data_feed2",[(brokerage, *data_feeds) for brokerage, data_feeds in @@ -676,6 +683,7 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given_with_multipl None, False, False, + {}, {}) @@ -833,6 +841,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_multiple_data_feed_s None, False, False, + {}, {}) @@ -853,6 +862,7 @@ def test_live_forces_update_when_update_option_given() -> None: None, False, False, + {}, {}) @@ -874,6 +884,7 @@ def test_live_passes_custom_image_to_lean_runner_when_set_in_config() -> None: None, False, False, + {}, {}) @@ -896,6 +907,7 @@ def test_live_passes_custom_image_to_lean_runner_when_given_as_option() -> None: None, False, False, + {}, {}) From 6391f9f04cd0ede7821ab2094ea32fdacc92c488 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 13:01:58 -0400 Subject: [PATCH 08/10] update readme --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 173f62a3..3f9ed307 100644 --- a/README.md +++ b/README.md @@ -161,7 +161,7 @@ Options: --iqfeed-version TEXT The product version of your IQFeed developer account --iqfeed-host TEXT The IQFeed host address --polygon-api-key TEXT Your Polygon.io API Key - --factset-auth-config-file TEXT + --factset-auth-config-file FILE The path to the FactSet authentication configuration file --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] @@ -1196,7 +1196,7 @@ Options: --coinapi-api-key TEXT Your coinapi.io Api Key --coinapi-product [Free|Startup|Streamer|Professional|Enterprise] CoinApi pricing plan (https://www.coinapi.io/market-data-api/pricing) - --factset-auth-config-file TEXT + --factset-auth-config-file FILE The path to the FactSet authentication configuration file --alpha-vantage-api-key TEXT Your Alpha Vantage Api Key --alpha-vantage-price-plan [Free|Plan30|Plan75|Plan150|Plan300|Plan600|Plan1200] @@ -1521,7 +1521,7 @@ Options: --iqfeed-version TEXT The product version of your IQFeed developer account --iqfeed-host TEXT The IQFeed host address --polygon-api-key TEXT Your Polygon.io API Key - --factset-auth-config-file TEXT + --factset-auth-config-file FILE The path to the FactSet authentication configuration file --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] @@ -1665,7 +1665,7 @@ Options: --iqfeed-version TEXT The product version of your IQFeed developer account --iqfeed-host TEXT The IQFeed host address --polygon-api-key TEXT Your Polygon.io API Key - --factset-auth-config-file TEXT + --factset-auth-config-file FILE The path to the FactSet authentication configuration file --iex-cloud-api-key TEXT Your iexcloud.io API token publishable key --iex-price-plan [Launch|Grow|Enterprise] From 37942364bfbfbaf70c1ae41f6d5b9f87ab76700f Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 14:23:15 -0400 Subject: [PATCH 09/10] fix: revert changes reverted from master by mistake --- lean/components/docker/lean_runner.py | 35 ++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 3b6d524a..648c6657 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -321,7 +321,10 @@ def get_basic_docker_config(self, # Create a C# project used to resolve the dependencies of the modules run_options["commands"].append("mkdir /ModulesProject") run_options["commands"].append("dotnet new sln -o /ModulesProject") - run_options["commands"].append("dotnet new classlib -o /ModulesProject -f net6.0 --no-restore") + + framework_ver = self._docker_manager.get_image_label(image, 'target_framework', + DEFAULT_LEAN_DOTNET_FRAMEWORK) + run_options["commands"].append(f"dotnet new classlib -o /ModulesProject -f {framework_ver} --no-restore") run_options["commands"].append("rm /ModulesProject/Class1.cs") # Add all modules to the project, automatically resolving all dependencies @@ -335,9 +338,10 @@ def get_basic_docker_config(self, run_options["commands"].append( "python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") - # Set up language-specific run options + # Set up language-specific run options self.setup_language_specific_run_options(run_options, project_dir, algorithm_file, - set_up_common_csharp_options_called, release) + set_up_common_csharp_options_called, release, + image) # Save the final Lean config to a temporary file so we can mount it into the container config_path = self._temp_manager.create_temporary_directory() / "config.json" @@ -371,11 +375,12 @@ def get_basic_docker_config(self, return run_options - def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) -> None: + def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any], image: DockerImage) -> None: """Sets up Docker run options specific to Python projects. :param project_dir: the path to the project directory :param run_options: the dictionary to append run options to + :param image: the docker image that will be used """ from docker.types import Mount @@ -422,10 +427,14 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) "mode": "rw" } + python_version = self._docker_manager.get_image_label(image, 'python_version', + DEFAULT_LEAN_PYTHON_VERSION) + site_packages_path = DOCKER_PYTHON_SITE_PACKAGES_PATH.replace('{LEAN_PYTHON_VERSION}', python_version) + # Mount a volume to the user packages directory so we don't install packages every time site_packages_volume = self._docker_manager.create_site_packages_volume(requirements_txt) run_options["volumes"][site_packages_volume] = { - "bind": f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}", + "bind": f"{site_packages_path}", "mode": "rw" } @@ -436,7 +445,7 @@ def set_up_python_options(self, project_dir: Path, run_options: Dict[str, Any]) # We only need to do this if it hasn't already been done before for this site packages volume # To keep track of this we create a special file in the site packages directory after installation # If this file already exists we can skip pip install completely - marker_file = f"{DOCKER_PYTHON_SITE_PACKAGES_PATH}/pip-install-done" + marker_file = f"{site_packages_path}/pip-install-done" run_options["commands"].extend([ f"! test -f {marker_file} && pip install --user --progress-bar off -r /requirements.txt", f"touch {marker_file}" @@ -464,12 +473,14 @@ def _concat_python_requirements(self, requirements_files: List[Path]) -> str: requirements = sorted(set(requirements)) return "\n".join(requirements) - def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool) -> None: + def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], release: bool, + image: DockerImage) -> None: """Sets up Docker run options specific to C# projects. :param project_dir: the path to the project directory :param run_options: the dictionary to append run options to :param release: whether C# projects should be compiled in release configuration instead of debug + :param image: the docker image that will be used """ compile_root = self._get_csharp_compile_root(project_dir) @@ -486,11 +497,13 @@ def set_up_csharp_options(self, project_dir: Path, run_options: Dict[str, Any], for path in compile_root.rglob("*.csproj"): self._ensure_csproj_uses_correct_lean(compile_root, path, csproj_temp_dir, run_options) + framework_ver = self._docker_manager.get_image_label(image, 'target_framework', + DEFAULT_LEAN_DOTNET_FRAMEWORK) # Set up the MSBuild properties msbuild_properties = { "Configuration": "Release" if release else "Debug", "Platform": "AnyCPU", - "TargetFramework": "net6.0", + "TargetFramework": framework_ver, "OutputPath": "/Compile/bin", "GenerateAssemblyInfo": "false", "GenerateTargetFrameworkAttribute": "false", @@ -734,14 +747,14 @@ def _force_disk_provider_if_necessary(self, lean_config[config_key] = disk_provider def setup_language_specific_run_options(self, run_options, project_dir, algorithm_file, - set_up_common_csharp_options_called, release) -> None: + set_up_common_csharp_options_called, release, image: DockerImage) -> None: # Set up language-specific run options if algorithm_file.name.endswith(".py"): - self.set_up_python_options(project_dir, run_options) + self.set_up_python_options(project_dir, run_options, image) else: if not set_up_common_csharp_options_called: self.set_up_common_csharp_options(run_options) - self.set_up_csharp_options(project_dir, run_options, release) + self.set_up_csharp_options(project_dir, run_options, release, image) def format_error_before_logging(self, chunk: str): from lean.components.util import compiler From c49d4823f59f9a35978b9f7b8dbb1f0db104372e Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 5 Apr 2024 14:24:16 -0400 Subject: [PATCH 10/10] minor change --- lean/components/docker/lean_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index 648c6657..4c247869 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -338,7 +338,7 @@ def get_basic_docker_config(self, run_options["commands"].append( "python /copy_csharp_dependencies.py /Compile/obj/ModulesProject/project.assets.json") - # Set up language-specific run options + # Set up language-specific run options self.setup_language_specific_run_options(run_options, project_dir, algorithm_file, set_up_common_csharp_options_called, release, image)