Merge pull request #207 from jhonabreul/pull-command-performance-improvements

Pull command performance improvements

Author: Martin-Molinero

lean/commands/cloud/pull.py

@@ -31,16 +31,24 @@ def pull(project: Optional[str], pull_bootcamp: bool) -> None:
     """
     # Parse which projects need to be pulled
     project_id = None
+    project_name = None
     if project is not None:
         try:
             project_id = int(project)
         except ValueError:
-            pass
+            # We treat it as a name rather than an id
+            project_name = project
 
     api_client = container.api_client()
-    all_projects = api_client.projects.get_all()
-    project_manager = container.project_manager()
-    projects_to_pull = project_manager.get_projects_by_name_or_id(all_projects, project_id or project)
+    projects_to_pull = []
+    all_projects = None
+
+    if project_id is not None:
+        projects_to_pull.append(api_client.projects.get(project_id))
+    else:
+        all_projects = api_client.projects.get_all()
+        project_manager = container.project_manager()
+        projects_to_pull = project_manager.get_projects_by_name_or_id(all_projects, project_name)
 
     if project is None and not pull_bootcamp:
         projects_to_pull = [p for p in projects_to_pull if not p.name.startswith("Boot Camp/")]

lean/components/cloud/pull_manager.py

@@ -13,15 +13,15 @@
 
 import traceback
 from pathlib import Path
-from typing import List, Dict
+from typing import List, Optional, Tuple
 
 from lean.components.api.api_client import APIClient
 from lean.components.config.project_config_manager import ProjectConfigManager
 from lean.components.util.library_manager import LibraryManager
 from lean.components.util.logger import Logger
 from lean.components.util.platform_manager import PlatformManager
 from lean.components.util.project_manager import ProjectManager
-from lean.models.api import QCProject, QCLeanEnvironment, QCLanguage
+from lean.models.api import QCProject, QCLanguage
 from lean.models.utils import LeanLibraryReference
 
 
@@ -52,38 +52,54 @@ def __init__(self,
         self._platform_manager = platform_manager
         self._last_file = None
 
-    def pull_projects(self, projects_to_pull: List[QCProject], all_cloud_projects: List[QCProject]) -> None:
+    def _get_libraries(self, project: QCProject, seen_projects: List[int] = None) -> List[QCProject]:
+        if seen_projects is None:
+            seen_projects = [project.projectId]
+
+        libraries = []
+        for library_id in project.libraries:
+            if library_id in seen_projects:
+                continue
+            seen_projects.append(library_id)
+            library = self._api_client.projects.get(library_id)
+            libraries.append(library)
+            libraries.extend(self._get_libraries(library, seen_projects))
+
+        return libraries
+
+    def pull_projects(self, projects_to_pull: List[QCProject], all_cloud_projects: Optional[List[QCProject]]) -> None:
         """Pulls the given projects from the cloud to the local drive.
 
         This will also pull libraries referenced by the project.
 
         :param projects_to_pull: the cloud projects that need to be pulled
         :param all_cloud_projects: all the projects available in the cloud
         """
-        projects_to_pull.extend(self._project_manager.get_cloud_projects_libraries(all_cloud_projects,
-                                                                                   projects_to_pull))
+        if all_cloud_projects is not None:
+            projects_to_pull.extend(self._project_manager.get_cloud_projects_libraries(all_cloud_projects,
+                                                                                       projects_to_pull))
+        else:
+            for project in projects_to_pull:
+                projects_to_pull.extend(self._get_libraries(project, [p.projectId for p in projects_to_pull]))
+
         projects_to_pull = sorted(projects_to_pull, key=lambda p: p.name)
-        environments = self._api_client.lean.environments()
-        projects_not_pulled = []
-        project_paths = {}
+        projects_with_paths = []
 
         for index, project in enumerate(projects_to_pull, start=1):
             try:
                 self._logger.info(f"[{index}/{len(projects_to_pull)}] Pulling '{project.name}'")
-                project_paths[project.projectId] = self._pull_project(project, environments)
+                projects_with_paths.append((project, self._pull_project(project)))
             except Exception as ex:
-                projects_not_pulled.append(project)
                 self._logger.debug(traceback.format_exc().strip())
                 if self._last_file is not None:
                     self._logger.warn(
                         f"Cannot pull '{project.name}' (id {project.projectId}, failed on {self._last_file}): {ex}")
                 else:
                     self._logger.warn(f"Cannot pull '{project.name}' (id {project.projectId}): {ex}")
 
-        projects_to_update = [project for project in projects_to_pull if project not in projects_not_pulled]
-        self._update_local_library_references(projects_to_update, project_paths)
+        self._update_local_library_references(projects_with_paths)
 
-    def _pull_project(self, project: QCProject, environments: List[QCLeanEnvironment]) -> Path:
+    def _pull_project(self, project: QCProject) -> Path:
         """Pulls a single project from the cloud to the local drive.
 
         Raises an error with a descriptive message if the project cannot be pulled.
@@ -103,18 +119,13 @@ def _pull_project(self, project: QCProject, environments: List[QCLeanEnvironment
         project_config.set("parameters", {parameter.key: parameter.value for parameter in project.parameters})
         project_config.set("description", project.description)
         project_config.set("organization-id", project.organizationId)
+        project_config.set("python-venv", project.leanEnvironment)
 
         if not project.leanPinnedToMaster:
             project_config.set("lean-engine", project.leanVersionId)
         else:
             project_config.delete("lean-engine")
 
-        python_venv = next((env.path for env in environments if env.id == project.leanEnvironment), None)
-        if python_venv is not None:
-            project_config.set("python-venv", project.leanEnvironment)
-        else:
-            project_config.delete("python-venv")
-
         return local_project_path
 
     def _pull_files(self, project: QCProject, local_project_path: Path) -> None:
@@ -255,22 +266,19 @@ def _remove_local_library_references_from_project(self,
                                                                    Path.cwd() / library_reference.path,
                                                                    True)
 
-    def _update_local_library_references(self, projects: List[QCProject], paths: Dict[int, Path]) -> None:
-        for project in projects:
-            cloud_libraries_paths = [paths[library_id]
-                                     for library_id in project.libraries
-                                     for library in projects if library.projectId == library_id]
-
-            project_path = paths[project.projectId]
+    def _update_local_library_references(self, projects: List[Tuple[QCProject, Path]]) -> None:
+        for project, path in projects:
+            cloud_libraries_paths = [library_path for library, library_path in projects
+                                     if library.projectId in project.libraries]
 
             # Add cloud library references to local config
-            self._add_local_library_references_to_project(project_path, cloud_libraries_paths)
+            self._add_local_library_references_to_project(path, cloud_libraries_paths)
 
             # Remove library references locally if they were removed in the cloud
-            self._remove_local_library_references_from_project(project_path, cloud_libraries_paths)
+            self._remove_local_library_references_from_project(path, cloud_libraries_paths)
 
             # Restore the project to automatically enable local auto-complete
-            self._restore_project(project, project_path)
+            self._restore_project(project, path)
 
     def _restore_project(self, project: QCProject, project_dir: Path) -> None:
         if project.language != QCLanguage.CSharp:

tests/commands/cloud/test_pull.py

@@ -75,19 +75,20 @@ def test_cloud_pull_pulls_project_by_id() -> None:
                       create_api_project(3, "Project 3"),
                       create_api_project(4, "Boot Camp/Project 4"),
                       create_api_project(5, "Boot Camp/Project 5")]
+    project_to_pull = cloud_projects[0]
 
     api_client = mock.Mock()
-    api_client.projects.get_all.return_value = cloud_projects
+    api_client.projects.get.return_value = project_to_pull
     container.api_client.override(providers.Object(api_client))
 
     pull_manager = mock.Mock()
     container.pull_manager.override(providers.Object(pull_manager))
 
-    result = CliRunner().invoke(lean, ["cloud", "pull", "--project", "1"])
+    result = CliRunner().invoke(lean, ["cloud", "pull", "--project", project_to_pull.projectId])
 
     assert result.exit_code == 0
 
-    pull_manager.pull_projects.assert_called_once_with([cloud_projects[0]], cloud_projects)
+    pull_manager.pull_projects.assert_called_once_with([project_to_pull], None)
 
 
 def test_cloud_pull_pulls_project_by_name() -> None:

tests/components/util/test_pull_manager.py

@@ -113,21 +113,6 @@ def test_pull_manager_removes_lean_engine_from_config_when_lean_pinned_to_master
     _assert_pull_manager_removes_property_from_project_config("lean-engine", [cloud_project])
 
 
-def test_pull_manager_removes_python_venv_from_config_when_set_to_default() -> None:
-    create_fake_lean_cli_directory()
-
-    project_path = Path.cwd() / "Python Project"
-    config = Storage(str(project_path / "config.json"))
-    environments = create_lean_environments()
-    config.set("python-venv", next(env.path for env in environments if env.path is not None))
-
-    project_id = 1000
-    cloud_project = create_api_project(project_id, project_path.name)
-    cloud_project.leanPinnedToMaster = True
-
-    _assert_pull_manager_removes_property_from_project_config("python-venv", [cloud_project])
-
-
 def _make_cloud_projects_and_libraries(project_count: int,
                                        library_count: int) -> Tuple[List[QCProject], List[QCProject]]:
     cloud_projects = [create_api_project(i, f"Project {i}") for i in range(1, project_count + 1)]
@@ -328,6 +313,7 @@ def test_pull_projects_updates_lean_config() -> None:
     api_client.files.get_all = mock.MagicMock(return_value=[])
 
     project_config = mock.Mock()
+    project_config.get = mock.MagicMock(return_value=[])    # get("libraries")
 
     project_config_manager = mock.Mock()
     project_config_manager.get_project_config = mock.MagicMock(return_value=project_config)
@@ -337,7 +323,8 @@ def test_pull_projects_updates_lean_config() -> None:
     pull_manager = _create_pull_manager(api_client, project_config_manager, library_manager)
     pull_manager.pull_projects(cloud_projects, cloud_projects)
 
-    project_config.set.assert_called_with("organization-id", "123")
+    project_config.set.assert_has_calls([mock.call("python-venv", 1), mock.call("organization-id", "123")],
+                                        any_order=True)
 
 
 @pytest.mark.parametrize("test_platform, unsupported_character", [