Skip to content

Commit

Permalink
Merge pull request #264 from jhonabreul/bug-256-lean-report-wrong-file
Browse files Browse the repository at this point in the history 
Fix backtest file name filter in for report generation
  • Loading branch information
@Martin-Molinero
Martin-Molinero authored Jan 11, 2023
2 parents 4d0cfcd + cb04a8e commit a127655
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 47 deletions.
14 changes: 11 additions & 3 deletions lean/commands/report.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
from lean.constants import DEFAULT_ENGINE_IMAGE, PROJECT_CONFIG_FILE_NAME
from lean.container import container
from lean.models.errors import MoreInfoError
from lean.components.util.live_utils import get_state_json
from lean.components.util.live_utils import get_latest_result_json_file


def _find_project_directory(backtest_file: Path) -> Optional[Path]:
Expand Down Expand Up @@ -108,8 +108,16 @@ def report(backtest_results: Optional[Path],
if report_destination.exists() and not overwrite:
raise RuntimeError(f"{report_destination} already exists, use --overwrite to overwrite it")

environment = "backtests"
output_config_manager = container.output_config_manager
output_directory = output_config_manager.get_latest_output_directory(environment)

if output_directory is None:
raise ValueError(f"No output {environment} directories were found. "
f"Make sure you run a backtest or live deployment first.")

if backtest_results is None:
backtest_results = get_state_json("backtests")
backtest_results = get_latest_result_json_file(output_directory)
if not backtest_results:
raise MoreInfoError(
"Could not find a recent backtest result file, please use the --backtest-results option",
Expand Down Expand Up @@ -175,7 +183,7 @@ def report(backtest_results: Optional[Path],
with config_path.open("w+", encoding="utf-8") as file:
dump(report_config, file)

backtest_id = container.output_config_manager.get_backtest_id(backtest_results.parent)
backtest_id = container.output_config_manager.get_backtest_id(output_directory)

lean_config_manager = container.lean_config_manager
data_dir = lean_config_manager.get_data_directory()
Expand Down
32 changes: 32 additions & 0 deletions lean/components/config/output_config_manager.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,38 @@ def get_live_deployment_by_id(self, live_deployment_id: int, root_directory: Opt
"""
return self._get_by_id("Live deployment", live_deployment_id, ["live/*"], root_directory)

def get_latest_output_directory(self, environment: str) -> Optional[Path]:
"""Finds the latest output directory for the given environment (live or backtests)
:param environment: The environment to find the latest output directory for (live or backtests)
:return: The path to the latest output directory for the given environment
:raises RuntimeError: If no output directory is found for the given environment
"""
output_json_files = sorted(Path.cwd().rglob(f"{environment}/*/*.json"),
key=lambda d: d.stat().st_mtime,
reverse=True)

if len(output_json_files) == 0:
return None

return output_json_files[0].parent

def get_output_id(self, output_directory: Path) -> Optional[int]:
"""Returns the id of an output, regardless of whether it is a backtest or a live deployment.
It will return None if the output directory does not contain any output with an existing id.
:param output_directory: the path to the output to retrieve the id of
:return: the id of the given output
"""
output_id = self._get_id(output_directory, 9)

if str(output_id)[0] == '9':
# no existing id was found
return None

return output_id

def _get_id(self, output_directory: Path, prefix: int) -> int:
config = self.get_output_config(output_directory)

Expand Down
29 changes: 15 additions & 14 deletions lean/components/util/live_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
from click import prompt, confirm

from pathlib import Path
from typing import Any, Dict, List
from typing import Any, Dict, List, Optional
from lean.components.api.api_client import APIClient
from lean.components.util.logger import Logger
from lean.models.brokerages.cloud import all_cloud_brokerages
Expand Down Expand Up @@ -67,7 +67,11 @@ def _get_last_portfolio(api_client: APIClient, project_id: str, project_name: Pa
last_state = api_client.get("live/read/portfolio", {"projectId": project_id})
previous_portfolio_state = last_state["portfolio"]
elif cloud_last_time < local_last_time:
previous_state_file = get_state_json("live")
from lean.container import container
output_directory = container.output_config_manager.get_latest_output_directory("live")
if not output_directory:
return None
previous_state_file = get_latest_result_json_file(output_directory)
if not previous_state_file:
return None
previous_portfolio_state = {x.lower(): y for x, y in loads(open(previous_state_file, "r", encoding="utf-8").read()).items()}
Expand Down Expand Up @@ -198,20 +202,17 @@ def configure_initial_holdings(logger: Logger, holdings_option: LiveInitialState
return _configure_initial_holdings_interactively(logger, holdings_option, previous_holdings)


def _filter_json_name_backtest(file: Path) -> bool:
return not file.name.endswith("-order-events.json") and not file.name.endswith("alpha-results.json")

def get_latest_result_json_file(output_directory: Path) -> Optional[Path]:
from lean.container import container

def _filter_json_name_live(file: Path) -> bool:
return file.name.replace("L-", "", 1).replace(".json", "").isdigit() # The json should have name like "L-1234567890.json"
output_config_manager = container.output_config_manager
output_id = output_config_manager.get_output_id(output_directory)

if output_id is None:
return None

def get_state_json(environment: str) -> str:
json_files = list(Path.cwd().rglob(f"{environment}/*/*.json"))
name_filter = _filter_json_name_backtest if environment == "backtests" else _filter_json_name_live
filtered_json_files = [f for f in json_files if name_filter(f)]

if len(filtered_json_files) == 0:
result_file = output_directory / f"{output_id}.json"
if not result_file.exists():
return None

return sorted(filtered_json_files, key=lambda f: f.stat().st_mtime, reverse=True)[0]
return result_file
67 changes: 37 additions & 30 deletions tests/commands/test_report.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,18 @@ def setup_backtest_results() -> None:
"""A pytest fixture which creates a backtest results file before every test."""
create_fake_lean_cli_directory()

results_path = Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "results.json"
results_dir = Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00"

results_id = 1459804915
results_path = results_dir / f"{results_id}.json"
results_path.parent.mkdir(parents=True, exist_ok=True)
with results_path.open("w+", encoding="utf-8") as file:
file.write("{}")

results_config_path = results_dir / "config"
with results_config_path.open("w+", encoding="utf-8") as file:
file.write(json.dumps({'id': results_id}))


def run_image(image: DockerImage, **kwargs) -> bool:
config_mount = [mount for mount in kwargs["mounts"] if mount["Target"] == "/Lean/Report/bin/Debug/config.json"][0]
Expand All @@ -61,7 +68,7 @@ def test_report_runs_lean_container() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -78,7 +85,7 @@ def test_report_runs_report_creator() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -100,7 +107,7 @@ def test_report_sets_container_name() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -117,7 +124,7 @@ def test_report_runs_detached_container() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--detach"])

assert result.exit_code == 0
Expand All @@ -135,7 +142,7 @@ def test_report_mounts_report_config() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -152,7 +159,7 @@ def test_report_mounts_data_directory() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -172,7 +179,7 @@ def test_report_mounts_output_directory() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -189,15 +196,15 @@ def test_report_mounts_given_backtest_data_source_file() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

docker_manager.run_image.assert_called_once()
args, kwargs = docker_manager.run_image.call_args

mount = [m for m in kwargs["mounts"] if m["Target"] == "/Lean/Report/bin/Debug/backtest-data-source-file.json"][0]
assert mount["Source"] == str(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "results.json")
assert mount["Source"] == str(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "1459804915.json")


def test_report_finds_latest_backtest_data_source_file_when_not_given() -> None:
Expand All @@ -213,15 +220,15 @@ def test_report_finds_latest_backtest_data_source_file_when_not_given() -> None:
args, kwargs = docker_manager.run_image.call_args

mount = [m for m in kwargs["mounts"] if m["Target"] == "/Lean/Report/bin/Debug/backtest-data-source-file.json"][0]
assert mount["Source"] == str(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "results.json")
assert mount["Source"] == str(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "1459804915.json")


def test_report_aborts_when_backtest_data_source_file_not_given_and_cannot_be_found() -> None:
docker_manager = mock.Mock()
docker_manager.run_image.side_effect = run_image
initialize_container(docker_manager_to_use=docker_manager)

(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "results.json").unlink()
(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "1459804915.json").unlink()

result = CliRunner().invoke(lean, ["report"])

Expand All @@ -237,17 +244,17 @@ def test_report_mounts_live_data_source_file_when_given() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--live-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

docker_manager.run_image.assert_called_once()
args, kwargs = docker_manager.run_image.call_args

mount = [m for m in kwargs["mounts"] if m["Target"] == "/Lean/Report/bin/Debug/live-data-source-file.json"][0]
assert mount["Source"] == str(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "results.json")
assert mount["Source"] == str(Path.cwd() / "Python Project" / "backtests" / "2020-01-01_00-00-00" / "1459804915.json")


def test_report_uses_project_directory_as_strategy_name_when_strategy_name_not_given() -> None:
Expand All @@ -257,7 +264,7 @@ def test_report_uses_project_directory_as_strategy_name_when_strategy_name_not_g

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -277,7 +284,7 @@ def test_report_uses_given_strategy_name() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--strategy-name", "My Strategy"])

assert result.exit_code == 0
Expand All @@ -300,7 +307,7 @@ def test_report_uses_description_from_config_when_strategy_description_not_given

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -320,7 +327,7 @@ def test_report_uses_given_strategy_description() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--strategy-description", "My strategy description"])

assert result.exit_code == 0
Expand All @@ -341,7 +348,7 @@ def test_report_uses_given_strategy_version() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--strategy-version", "1.2.3"])

assert result.exit_code == 0
Expand All @@ -360,10 +367,10 @@ def test_report_uses_given_blank_name_version_description_when_not_given_and_bac
docker_manager.run_image.side_effect = run_image
initialize_container(docker_manager_to_use=docker_manager)

with (Path.cwd() / "results.json").open("w+", encoding="utf-8") as file:
with (Path.cwd() / "1459804915.json").open("w+", encoding="utf-8") as file:
file.write("{}")

result = CliRunner().invoke(lean, ["report", "--backtest-results", "results.json"])
result = CliRunner().invoke(lean, ["report", "--backtest-results", "1459804915.json"])

assert result.exit_code == 0

Expand All @@ -385,7 +392,7 @@ def test_report_writes_to_report_html_when_no_report_destination_given() -> None

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -399,7 +406,7 @@ def test_report_writes_to_given_report_destination() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--report-destination", "path/to/report.html"])

assert result.exit_code == 0
Expand All @@ -419,7 +426,7 @@ def test_report_aborts_when_report_destination_already_exists() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--report-destination", "path/to/report.html"])

assert result.exit_code != 0
Expand All @@ -439,7 +446,7 @@ def test_report_overwrites_report_destination_when_overwrite_flag_given() -> Non

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--report-destination", "path/to/report.html",
"--overwrite"])

Expand All @@ -455,7 +462,7 @@ def test_report_aborts_when_run_image_fails() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code != 0

Expand All @@ -469,7 +476,7 @@ def test_report_forces_update_when_update_option_given() -> None:

result = CliRunner().invoke(lean,
["report",
"--backtest-results", "Python Project/backtests/2020-01-01_00-00-00/results.json",
"--backtest-results", "Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--update"])

assert result.exit_code == 0
Expand All @@ -487,7 +494,7 @@ def test_report_runs_custom_image_when_set_in_config() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json"])
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json"])

assert result.exit_code == 0

Expand All @@ -506,7 +513,7 @@ def test_report_runs_custom_image_when_given_as_option() -> None:

result = CliRunner().invoke(lean, ["report",
"--backtest-results",
"Python Project/backtests/2020-01-01_00-00-00/results.json",
"Python Project/backtests/2020-01-01_00-00-00/1459804915.json",
"--image", "custom/lean:456"])

assert result.exit_code == 0
Expand Down

0 comments on commit a127655

Please sign in to comment.