Skip to content

Commit

Permalink
Support extra docker configs (#360)
Browse files Browse the repository at this point in the history 
* Get additional docker configs in local deployment commands.

This accepts only device_requests for now.

* Update new docker extra config option documentation

* Support mounting volumes using extra docker configs

* Update readme

* Update readme
  • Loading branch information
@jhonabreul
jhonabreul authored Sep 19, 2023
1 parent 92e5987 commit d43621c
Show file tree
Hide file tree
Showing 11 changed files with 344 additions and 26 deletions.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,8 @@ Options:
--python-venv TEXT The path of the python virtual environment to be used
--update Pull the LEAN engine image before running the backtest
--backtest-name TEXT Backtest name
--extra-docker-config TEXT Extra docker configuration as a JSON string. For more information https://docker-
py.readthedocs.io/en/stable/containers.html
--no-update Use the local LEAN engine image instead of pulling the latest version
--lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json)
--verbose Enable debug logging
Expand Down Expand Up @@ -1098,6 +1100,8 @@ Options:
holdings
--update Pull the LEAN engine image before starting live trading
--show-secrets Show secrets as they are input
--extra-docker-config TEXT Extra docker configuration as a JSON string. For more information https://docker-
py.readthedocs.io/en/stable/containers.html
--no-update Use the local LEAN engine image instead of pulling the latest version
--lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json)
--verbose Enable debug logging
Expand Down Expand Up @@ -1383,6 +1387,8 @@ Options:
--estimate Estimate optimization runtime without running it
--max-concurrent-backtests INTEGER RANGE
Maximum number of concurrent backtests to run [x>=1]
--extra-docker-config TEXT Extra docker configuration as a JSON string. For more information https://docker-
py.readthedocs.io/en/stable/containers.html
--no-update Use the local LEAN engine image instead of pulling the latest version
--lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json)
--verbose Enable debug logging
Expand Down Expand Up @@ -1505,6 +1511,8 @@ Options:
--no-open Don't open the Jupyter Lab environment in the browser after starting it
--image TEXT The LEAN research image to use (defaults to quantconnect/research:latest)
--update Pull the LEAN research image before starting the research environment
--extra-docker-config TEXT Extra docker configuration as a JSON string. For more information https://docker-
py.readthedocs.io/en/stable/containers.html
--no-update Use the local LEAN research image instead of pulling the latest version
--lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json)
--verbose Enable debug logging
Expand Down
12 changes: 10 additions & 2 deletions lean/commands/backtest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.


from pathlib import Path
from typing import List, Optional, Tuple
from click import command, option, argument, Choice
Expand Down Expand Up @@ -289,6 +288,11 @@ def _select_organization() -> QCMinimalOrganization:
type=(str, str),
multiple=True,
hidden=True)
@option("--extra-docker-config",
type=str,
default="{}",
help="Extra docker configuration as a JSON string. "
"For more information https://docker-py.readthedocs.io/en/stable/containers.html")
@option("--no-update",
is_flag=True,
default=False,
Expand All @@ -307,6 +311,7 @@ def backtest(project: Path,
backtest_name: str,
addon_module: Optional[List[str]],
extra_config: Optional[Tuple[str, str]],
extra_docker_config: Optional[str],
no_update: bool,
**kwargs) -> None:
"""Backtest a project locally using Docker.
Expand All @@ -324,6 +329,8 @@ def backtest(project: Path,
Alternatively you can set the default engine image for all commands using `lean config set engine-image <image>`.
"""
from datetime import datetime
from json import loads

logger = container.logger
project_manager = container.project_manager
algorithm_file = project_manager.find_algorithm_file(Path(project))
Expand Down Expand Up @@ -407,4 +414,5 @@ def backtest(project: Path,
engine_image,
debugging_method,
release,
detach)
detach,
loads(extra_docker_config))
9 changes: 8 additions & 1 deletion lean/commands/live/deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,11 @@ def _get_default_value(key: str) -> Optional[Any]:
type=(str, str),
multiple=True,
hidden=True)
@option("--extra-docker-config",
type=str,
default="{}",
help="Extra docker configuration as a JSON string. "
"For more information https://docker-py.readthedocs.io/en/stable/containers.html")
@option("--no-update",
is_flag=True,
default=False,
Expand All @@ -275,6 +280,7 @@ def deploy(project: Path,
show_secrets: bool,
addon_module: Optional[List[str]],
extra_config: Optional[Tuple[str, str]],
extra_docker_config: Optional[str],
no_update: bool,
**kwargs) -> None:
"""Start live trading a project locally using Docker.
Expand All @@ -299,6 +305,7 @@ def deploy(project: Path,
"""
from copy import copy
from datetime import datetime
from json import loads
# Reset globals so we reload everything in between tests
global _cached_lean_config
_cached_lean_config = None
Expand Down Expand Up @@ -430,4 +437,4 @@ def deploy(project: Path,
raise RuntimeError(f"InteractiveBrokers is currently not supported for ARM hosts")

lean_runner = container.lean_runner
lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach)
lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach, loads(extra_docker_config))
10 changes: 10 additions & 0 deletions lean/commands/optimize.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from click import command, argument, option, Choice, IntRange

from lean.click import LeanCommand, PathParameter, ensure_options
from lean.components.docker.lean_runner import LeanRunner
from lean.constants import DEFAULT_ENGINE_IMAGE
from lean.container import container
from lean.models.api import QCParameter, QCBacktest
Expand Down Expand Up @@ -119,6 +120,11 @@ def get_filename_timestamp(path: Path) -> datetime:
type=(str, str),
multiple=True,
hidden=True)
@option("--extra-docker-config",
type=str,
default="{}",
help="Extra docker configuration as a JSON string. "
"For more information https://docker-py.readthedocs.io/en/stable/containers.html")
@option("--no-update",
is_flag=True,
default=False,
Expand All @@ -139,6 +145,7 @@ def optimize(project: Path,
max_concurrent_backtests: Optional[int],
addon_module: Optional[List[str]],
extra_config: Optional[Tuple[str, str]],
extra_docker_config: Optional[str],
no_update: bool) -> None:
"""Optimize a project's parameters locally using Docker.
Expand Down Expand Up @@ -308,6 +315,9 @@ def optimize(project: Path,
)
container.update_manager.pull_docker_image_if_necessary(engine_image, update, no_update)

# Add known additional run options from the extra docker config
LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))

project_manager.copy_code(algorithm_file.parent, output / "code")

success = container.docker_manager.run_image(engine_image, **run_options)
Expand Down
13 changes: 12 additions & 1 deletion lean/commands/research.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from typing import Optional, Tuple
from click import command, argument, option, Choice
from lean.click import LeanCommand, PathParameter
from lean.components.docker.lean_runner import LeanRunner
from lean.constants import DEFAULT_RESEARCH_IMAGE, LEAN_ROOT_PATH
from lean.container import container
from lean.models.data_providers import QuantConnectDataProvider, all_data_providers
Expand Down Expand Up @@ -65,6 +66,11 @@ def _check_docker_output(chunk: str, port: int) -> None:
type=(str, str),
multiple=True,
hidden=True)
@option("--extra-docker-config",
type=str,
default="{}",
help="Extra docker configuration as a JSON string. "
"For more information https://docker-py.readthedocs.io/en/stable/containers.html")
@option("--no-update",
is_flag=True,
default=False,
Expand All @@ -79,6 +85,7 @@ def research(project: Path,
image: Optional[str],
update: bool,
extra_config: Optional[Tuple[str, str]],
extra_docker_config: Optional[str],
no_update: bool,
**kwargs) -> None:
"""Run a Jupyter Lab environment locally using Docker.
Expand All @@ -89,6 +96,7 @@ def research(project: Path,
"""
from docker.types import Mount
from docker.errors import APIError
from json import loads

logger = container.logger

Expand Down Expand Up @@ -116,7 +124,7 @@ def research(project: Path,
# Set extra config
for key, value in extra_config:
lean_config[key] = value

run_options = lean_runner.get_basic_docker_config(lean_config,
algorithm_file,
temp_manager.create_temporary_directory(),
Expand Down Expand Up @@ -160,6 +168,9 @@ def research(project: Path,
# Run the script that starts Jupyter Lab when all set up has been done
run_options["commands"].append("./start.sh")

# Add known additional run options from the extra docker config
LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config))

project_config_manager = container.project_config_manager
cli_config_manager = container.cli_config_manager

Expand Down
24 changes: 23 additions & 1 deletion lean/components/docker/lean_runner.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

from pathlib import Path
from typing import Any, Dict, Optional, List

from lean.components.cloud.module_manager import ModuleManager
from lean.components.config.lean_config_manager import LeanConfigManager
from lean.components.config.output_config_manager import OutputConfigManager
Expand Down Expand Up @@ -70,7 +71,8 @@ def run_lean(self,
image: DockerImage,
debugging_method: Optional[DebuggingMethod],
release: bool,
detach: bool) -> None:
detach: bool,
extra_docker_config: Optional[Dict[str, Any]] = None) -> None:
"""Runs the LEAN engine locally in Docker.
Raises an error if something goes wrong.
Expand All @@ -83,6 +85,7 @@ def run_lean(self,
:param debugging_method: the debugging method if debugging needs to be enabled, None if not
:param release: whether C# projects should be compiled in release configuration instead of debug
:param detach: whether LEAN should run in a detached container
:param extra_docker_config: additional docker configurations
"""
project_dir = algorithm_file.parent

Expand All @@ -95,6 +98,9 @@ def run_lean(self,
release,
detach)

# Add known additional run options from the extra docker config
self.parse_extra_docker_config(run_options, extra_docker_config)

# Set up PTVSD debugging
if debugging_method == DebuggingMethod.PTVSD:
run_options["ports"]["5678"] = "5678"
Expand Down Expand Up @@ -762,3 +768,19 @@ def mount_project_and_library_directories(self, project_dir: Path, run_options:
"bind": "/Library",
"mode": "rw"
}

@staticmethod
def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: Optional[Dict[str, Any]]) -> None:
from docker.types import DeviceRequest
# Add known additional run options from the extra docker config.
# For now, only device_requests is supported
if extra_docker_config is not None:
if "device_requests" in extra_docker_config:
run_options["device_requests"] = [DeviceRequest(**device_request)
for device_request in extra_docker_config["device_requests"]]

if "volumes" in extra_docker_config:
volumes = run_options.get("volumes")
if not volumes:
volumes = run_options["volumes"] = {}
volumes.update(extra_docker_config["volumes"])
52 changes: 44 additions & 8 deletions tests/commands/test_backtest.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,8 @@ def test_backtest_calls_lean_runner_with_correct_algorithm_file() -> None:
ENGINE_IMAGE,
None,
False,
False)
False,
{})


def test_backtest_calls_lean_runner_with_default_output_directory() -> None:
Expand Down Expand Up @@ -88,7 +89,8 @@ def test_backtest_calls_lean_runner_with_custom_output_directory() -> None:
ENGINE_IMAGE,
None,
False,
False)
False,
{})


def test_backtest_calls_lean_runner_with_release_mode() -> None:
Expand All @@ -105,7 +107,8 @@ def test_backtest_calls_lean_runner_with_release_mode() -> None:
ENGINE_IMAGE,
None,
True,
False)
False,
{})


def test_backtest_calls_lean_runner_with_detach() -> None:
Expand All @@ -122,7 +125,8 @@ def test_backtest_calls_lean_runner_with_detach() -> None:
ENGINE_IMAGE,
None,
False,
True)
True,
{})


def test_backtest_aborts_when_project_does_not_exist() -> None:
Expand Down Expand Up @@ -163,7 +167,8 @@ def test_backtest_forces_update_when_update_option_given() -> None:
ENGINE_IMAGE,
None,
False,
False)
False,
{})


def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> None:
Expand All @@ -182,7 +187,8 @@ def test_backtest_passes_custom_image_to_lean_runner_when_set_in_config() -> Non
DockerImage(name="custom/lean", tag="123"),
None,
False,
False)
False,
{})


def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> None:
Expand All @@ -201,7 +207,8 @@ def test_backtest_passes_custom_image_to_lean_runner_when_given_as_option() -> N
DockerImage(name="custom/lean", tag="456"),
None,
False,
False)
False,
{})


@pytest.mark.parametrize("python_venv", ["Custom-venv",
Expand Down Expand Up @@ -289,7 +296,8 @@ def test_backtest_passes_correct_debugging_method_to_lean_runner(value: str, deb
ENGINE_IMAGE,
debugging_method,
False,
False)
False,
{})


def test_backtest_auto_updates_outdated_python_pycharm_debug_config() -> None:
Expand Down Expand Up @@ -649,3 +657,31 @@ def test_backtest_adds_python_libraries_path_to_lean_config() -> None:
expected_library_path = (Path("/") / library_path.relative_to(lean_cli_root_dir)).as_posix()

assert expected_library_path in lean_config.get('python-additional-paths')


def test_backtest_calls_lean_runner_with_extra_docker_config() -> None:
create_fake_lean_cli_directory()

result = CliRunner().invoke(lean, ["backtest", "Python Project",
"--extra-docker-config",
'{"device_requests": [{"count": -1, "capabilities": [["compute"]]}],'
'"volumes": {"extra/path": {"bind": "/extra/path", "mode": "rw"}}}'])

assert result.exit_code == 0

container.lean_runner.run_lean.assert_called_once_with(mock.ANY,
"backtesting",
Path("Python Project/main.py").resolve(),
mock.ANY,
ENGINE_IMAGE,
None,
False,
False,
{
"device_requests": [
{"count": -1, "capabilities": [["compute"]]}
],
"volumes": {
"extra/path": {"bind": "/extra/path", "mode": "rw"}
}
})
Loading

0 comments on commit d43621c

Please sign in to comment.