From ff590308b516a50ab3f264f01966889b577ad825 Mon Sep 17 00:00:00 2001
From: Roman Yavnikov <45608740+Romazes@users.noreply.github.com>
Date: Sat, 11 Jan 2025 01:12:40 +0200
Subject: [PATCH] Fix: backtest is aborted in the middle with half of report
 (#539)

* feat: compare backtest result when run it in cloud

* feat: use delay between is_backtest_done validation

* refactor: more performance in is_backtest_done

* refactor: more readable and reset counter in is_backtest_done

* Update cloud_runner.py

---------

Co-authored-by: Martin-Molinero <martin.molinero1@gmail.com>
---
 lean/components/cloud/cloud_runner.py | 40 ++++++++++++++++++++++++++-
 1 file changed, 39 insertions(+), 1 deletion(-)

diff --git a/lean/components/cloud/cloud_runner.py b/lean/components/cloud/cloud_runner.py
index cb980377..f42ddfc6 100644
--- a/lean/components/cloud/cloud_runner.py
+++ b/lean/components/cloud/cloud_runner.py
@@ -36,6 +36,44 @@ def __init__(self, logger: Logger, api_client: APIClient, task_manager: TaskMana
         self._logger = logger
         self._api_client = api_client
         self._task_manager = task_manager
+        self._mismatch_counter = 0
+
+    def is_backtest_done(self, backtest_data: QCBacktest, delay: float = 10.0):
+        """Checks if the backtest is complete.
+
+        :param backtest_data: The current state of the backtest.
+        :param delay: The delay in seconds between consecutive checks. Default is 60 seconds (1 minute).
+        :return: True if the backtest is complete and the state has changed, False otherwise.
+        """
+        try:
+            if backtest_data.error or backtest_data.stacktrace:
+                self._mismatch_counter = 0
+                return True
+
+            is_complete = backtest_data.is_complete()
+            self._logger.debug(f"[Backtest ID: {backtest_data.backtestId}] Completion status: {is_complete}")
+
+            if is_complete:
+                if backtest_data.totalPerformance:
+                    self._mismatch_counter = 0
+                    return True
+
+                if self._mismatch_counter >= 6:
+                    self._logger.error(f"[Backtest ID: {backtest_data.backtestId}] We could not retrieve "
+                                       f"the complete backtest results, please try again later.")
+                    self._mismatch_counter = 0
+                    return True
+
+                self._mismatch_counter += 1
+                self._logger.debug(f"[Backtest ID: {backtest_data.backtestId}] Incremented mismatch counter to "
+                                   f"{self._mismatch_counter}. Will re-check after {delay} seconds.")
+                import time
+                time.sleep(delay)
+
+            return False
+        except Exception as e:
+            self._logger.error(f"Error checking backtest completion status for ID {backtest_data.backtestId}: {e}")
+            raise
 
     def run_backtest(self, project: QCProject, name: str) -> QCBacktest:
         """Runs a backtest in the cloud.
@@ -53,7 +91,7 @@ def run_backtest(self, project: QCProject, name: str) -> QCBacktest:
         try:
             return self._task_manager.poll(
                 make_request=lambda: self._api_client.backtests.get(project.projectId, created_backtest.backtestId),
-                is_done=lambda data: data.is_complete(),
+                is_done=self.is_backtest_done,
                 get_progress=lambda data: data.progress
             )
         except KeyboardInterrupt as e: