Fixing tests, only 15 remaining.

Author: theypsilon

src/downloader/jobs/process_db_index_worker.py

@@ -30,7 +30,8 @@
 from downloader.jobs.errors import WrongDatabaseOptions
 from downloader.jobs.fetch_file_job import FetchFileJob
 from downloader.jobs.process_zip_index_job import ProcessZipIndexJob
-from downloader.path_package import PathPackage, PathType, RemovedCopy
+from downloader.path_package import PathPackage, PathType, RemovedCopy, PEXT_KIND_EXTERNAL, PATH_PACKAGE_KIND_STANDARD, \
+    PEXT_KIND_STANDARD, PATH_PACKAGE_KIND_PEXT
 from downloader.jobs.process_db_index_job import ProcessDbIndexJob
 from downloader.jobs.index import Index
 from downloader.jobs.validate_file_job import ValidateFileJob
@@ -225,30 +226,36 @@ def process_create_folder_packages(ctx: DownloaderWorkerContext, create_folder_p
         ctx.swallow_error(e)
 
     folder_copies_to_be_removed: List[Tuple[bool, str, str, PathType]] = []
-    parents: Dict[str, Dict[str, PathPackage]] = defaultdict(defaultdict)
     processing_folders: List[PathPackage] = []
+
+    parent_drives: Dict[str, set[str]] = defaultdict(set)
     parent_pkgs: Dict[str, PathPackage] = dict()
+    parents_to_add: List[PathPackage] = []
 
     for pkg in sorted(create_folder_pkgs, key=lambda x: len(x.rel_path)):
-        if pkg.is_pext_parent():
+        if pkg.kind != PATH_PACKAGE_KIND_PEXT:
+            processing_folders.append(pkg)
+        elif pkg.is_pext_parent():
             parent_pkgs[pkg.rel_path] = pkg
             continue
-
-        processing_folders.append(pkg)
-
-        if pkg.pext_props:
-            if pkg.pext_props.drive not in parents[pkg.pext_props.parent]:
-                p_pkg = pkg.pext_props.parent_pkg()
-                parents[pkg.pext_props.parent][pkg.pext_props.drive] = p_pkg
-                if pkg.pext_props.parent in parent_pkgs:
-                    p_pkg.description.update(parent_pkgs[pkg.pext_props.parent].description)
+        else:
+            processing_folders.append(pkg)
+
+            pkg_parent = pkg.pext_props.parent
+            if pkg_parent in parent_pkgs and pkg.drive not in parent_drives[pkg_parent]:
+                parent_drives[pkg_parent].add(pkg.drive)
+                parent_pkg = parent_pkgs[pkg_parent].clone()
+                parent_pkg.drive = pkg.drive
+                parent_pkg.pext_props.kind = pkg.pext_props.kind
+                parent_pkg.pext_props.drive = pkg.pext_props.drive
+                parent_pkg.pext_props.parent = ''
+                parents_to_add.append(parent_pkg)
 
         _maybe_add_copies_to_remove(ctx, folder_copies_to_be_removed, store, pkg.rel_path, pkg.pext_drive())
 
-    for parent_path, drives in parents.items():
-        for d, parent_pkg in drives.items():
-            processing_folders.append(parent_pkg)
-            _maybe_add_copies_to_remove(ctx, folder_copies_to_be_removed, store, parent_path, d)
+    for parent_pkg in parents_to_add:
+        processing_folders.append(parent_pkg)
+        _maybe_add_copies_to_remove(ctx, folder_copies_to_be_removed, store, parent_pkg.rel_path, parent_pkg.drive)
 
     ctx.logger.bench('add_processed_folders start: ', db_id, len(processing_folders))
     non_existing_folders = ctx.installation_report.add_processed_folders(processing_folders, db_id)

src/downloader/local_store_wrapper.py

@@ -161,14 +161,29 @@ def add_file_pkg(self, file_pkg: PathPackage):
         else:
             self.add_file(file_pkg.rel_path, file_pkg.description)
 
+    def remove_file_pkg(self, file_pkg: PathPackage):
+        if file_pkg.is_pext_external():
+            self.remove_external_file(file_pkg.drive, file_pkg.rel_path)
+        else:
+            self.remove_local_file(file_pkg.rel_path)
+
     def add_folder_pkg(self, folder_pkg: PathPackage):
         if folder_pkg.is_pext_external():
             self.add_external_folder(folder_pkg.pext_props.drive, folder_pkg.rel_path, folder_pkg.description)
         else:
             self.add_folder(folder_pkg.rel_path, folder_pkg.description)
 
-    def add_file(self, file, description):
-        self._add_entry('files', file, description)
+    def remove_folder_pkg(self, folder_pkg: PathPackage):
+        if folder_pkg.is_pext_external():
+            self.remove_external_folder(folder_pkg.drive, folder_pkg.rel_path)
+        else:
+            self.remove_local_folder(folder_pkg.rel_path)
+
+    def add_file(self, file_path, description):
+        if file_path in self._external_additions['files']:
+            for drive in self._external_additions['files'][file_path]:
+                self.remove_external_file(drive, file_path)
+        self._add_entry('files', file_path, description)
 
     def add_folder(self, folder, description):
         self._add_entry('folders', folder, description)
@@ -189,6 +204,13 @@ def add_external_folder(self, drive, folder_path, description):
         self._add_external_entry('folders', drive, folder_path, description)
 
     def add_external_file(self, drive, file_path, description):
+        if file_path in self._store['files']:
+            self.remove_file(file_path)
+        if file_path in self._external_additions['files']:
+            for d in self._external_additions['files'][file_path]:
+                if d == drive:
+                    continue
+                self.remove_external_file(d, file_path)
         self._add_external_entry('files', drive, file_path, description)
 
     def _add_external_entry(self, kind, drive, path, description):
@@ -262,7 +284,7 @@ def _clean_external_additions(self, kind, path):
         if changed: self._top_wrapper.mark_force_save()
 
     def _remove_entry(self, kind, path):
-        self._clean_external_additions(kind, path)
+        #self._clean_external_additions(kind, path)
         self._remove_local_entry(kind, path)
 
     def _remove_local_entry(self, kind, path):
@@ -405,12 +427,12 @@ def drop_removed_zips_from_store(self, db_zips):
         self.remove_zip_ids(removed_zip_ids)
 
     def save_filtered_zip_data(self, filtered_zip_data):
-        for drive in list(filtered_zip_data):
-            data = filtered_zip_data[drive]
+        for zip_id in list(filtered_zip_data):
+            data = filtered_zip_data[zip_id]
             empty_files = 'files' not in data or len(data['files']) == 0
             empty_folders = 'folders' not in data or len(data['folders']) == 0
             if empty_files and empty_folders:
-                filtered_zip_data.pop(drive)
+                filtered_zip_data.pop(zip_id)
 
         if len(filtered_zip_data):
             if 'filtered_zip_data' in self._store and equal_dicts(self._store['filtered_zip_data'], filtered_zip_data):

src/downloader/online_importer.py

@@ -127,7 +127,7 @@ def download_dbs_contents(self, db_pkgs: list[DbSectionPackage], full_resync: bo
             box.add_processed_files(job.non_duplicated_files, job.db.db_id)
             box.add_present_validated_files(job.present_validated_files)
             box.add_skipped_updated_files(job.skipped_updated_files, job.db.db_id)
-            box.add_removed_copies(job.removed_folders)
+            box.add_removed_copies(job.removed_folders, job.db.db_id)
             box.add_installed_folders(job.installed_folders)
             box.queue_directory_removal(job.directories_to_remove, job.db.db_id)
             box.queue_file_removal(job.files_to_remove, job.db.db_id)
@@ -145,7 +145,7 @@ def download_dbs_contents(self, db_pkgs: list[DbSectionPackage], full_resync: bo
             box.add_processed_files(job.non_duplicated_files, job.db.db_id)
             box.add_present_validated_files(job.present_validated_files)
             box.add_skipped_updated_files(job.skipped_updated_files, job.db.db_id)
-            box.add_removed_copies(job.removed_folders)
+            box.add_removed_copies(job.removed_folders, job.db.db_id)
             box.add_installed_folders(job.installed_folders)
             box.queue_directory_removal(job.directories_to_remove, job.db.db_id)
             box.queue_file_removal(job.files_to_remove, job.db.db_id)
@@ -278,97 +278,36 @@ def download_dbs_contents(self, db_pkgs: list[DbSectionPackage], full_resync: bo
                 stores[db_id].write_only().remove_folder(pkg.rel_path)
                 stores[db_id].write_only().remove_folder_from_zips(pkg.rel_path)
 
-        external_parents_by_db: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
-        parents_by_db: dict[str, set[str]] = defaultdict(set)
-        all_parents_by_db: dict[str, dict[str, set[str]]] = defaultdict(lambda: defaultdict(set))
-
-        def add_parent(el_pkg: PathPackage, db_id: str) -> None:
-            if el_pkg.pext_props is not None:
-                if el_pkg.is_pext_standard():
-                    parents_by_db[db_id].add(el_pkg.pext_props.parent)
-                else:
-                    external_parents_by_db[db_id][el_pkg.pext_props.parent].add(el_pkg.pext_props.drive)
-
         for file_path in box.installed_files():
             file = box.processed_file(file_path)
             if 'reboot' in file.pkg.description and file.pkg.description['reboot'] == True:
                 self._needs_reboot = True
-
-            add_parent(file.pkg, file.db_id)
-            all_parents_by_db[file.db_id][file.pkg.parent].add(file.pkg.drive)
-    
-        for file_path in box.installed_files():
-            file = box.processed_file(file_path)
-
-            for is_external, other_drive in stores[file.db_id].read_only().list_other_drives_for_file(file.pkg.rel_path, file.pkg.pext_drive()):
-                other_file = os.path.join(other_drive, file.pkg.rel_path)
-                if not self._worker_ctx.file_system.is_file(other_file):
-                    if is_external:
-                        stores[file.db_id].write_only().remove_external_file(other_drive, file.pkg.rel_path)
-                    else:
-                        stores[file.db_id].write_only().remove_local_file(file.pkg.rel_path)
-
             stores[file.db_id].write_only().add_file_pkg(file.pkg)
 
-            if file.pkg.pext_props is not None:
-                for other in file.pkg.pext_props.other_drives:
-                    if self._worker_ctx.file_system.is_file(os.path.join(other, file.pkg.rel_path)):
-                        stores[file.db_id].write_only().add_external_file(other, file.pkg.rel_path, file.pkg.description)
-
         for folder_path in sorted(box.installed_folders(), key=lambda x: len(x), reverse=True):
             for db_id, folder_pkg in report.processed_folder(folder_path).items():
-                if folder_pkg.is_pext_parent():
-                    continue
-
                 stores[db_id].write_only().add_folder_pkg(folder_pkg)
-                add_parent(folder_pkg, db_id)
-
-        for folder_path in sorted(box.installed_folders(), key=lambda x: len(x), reverse=True):
-            for db_id, folder_pkg in report.processed_folder(folder_path).items():
-                if not folder_pkg.is_pext_parent():
-                    continue
-
-                if folder_pkg.pext_props.parent in parents_by_db[db_id]:
-                    stores[db_id].write_only().add_folder(folder_pkg.rel_path, folder_pkg.description)
-                if folder_pkg.pext_props.parent in external_parents_by_db[db_id]:
-                    for drive in external_parents_by_db[db_id][folder_pkg.pext_props.parent]:
-                        stores[db_id].write_only().add_external_folder(drive, folder_pkg.rel_path, folder_pkg.description)
 
         for file_path in box.removed_files():
             file = box.processed_file(file_path)
-            stores[file.db_id].write_only().remove_file(file.pkg.rel_path)
+            stores[file.db_id].write_only().remove_file_pkg(file.pkg)
 
-        for is_external, el_path, drive, ty in box.removed_copies():
+        for db_id, is_external, el_path, drive, ty in box.removed_copies():
             if ty == PathType.FILE:
                 file = box.processed_file(el_path)
-                if is_external:
-                    stores[file.db_id].write_only().remove_external_file(drive, el_path)
-                else:
-                    stores[file.db_id].write_only().remove_local_file(el_path)
+                stores[file.db_id].write_only().remove_file_pkg(file.pkg)
 
             elif ty == PathType.FOLDER:
-                for db_id, folder_pkg in report.processed_folder(el_path).items():
-                    if is_external:
-                        stores[db_id].write_only().remove_external_folder(drive, el_path)
-                    else:
-                        stores[db_id].write_only().remove_local_folder(el_path)
+                if is_external:
+                    stores[db_id].write_only().remove_external_folder(el_path)
+                else:
+                    stores[db_id].write_only().remove_folder(el_path)
 
         for db_id, zip_id, zip_summary, zip_description in box.installed_zip_summary():
             stores[db_id].write_only().add_zip_summary(zip_id, zip_summary, zip_description)
 
-        filtered_zip_data = {}
-        for db_id, zip_id, files, folders in box.filtered_zip_data():
-            if db_id not in filtered_zip_data:
-                filtered_zip_data[db_id] = {}
-
-            if zip_id not in filtered_zip_data[db_id]:
-                filtered_zip_data[db_id][zip_id] = {'files': {}, 'folders': {}}
-
-            filtered_zip_data[db_id][zip_id]['files'].update(files)
-            filtered_zip_data[db_id][zip_id]['folders'].update(folders)
-
-        for db_id, filtered_zip_data_by_db in filtered_zip_data.items():
-            stores[db_id].write_only().save_filtered_zip_data(filtered_zip_data_by_db)
+        for db_id, filtered_zip_data in box.filtered_zip_data().items():
+            stores[db_id].write_only().save_filtered_zip_data(filtered_zip_data)
 
         for store in stores.values():
             store.write_only().cleanup_externals()
@@ -459,10 +398,10 @@ def __init__(self):
         self._full_partitions: dict[str, int] = dict()
         self._failed_db_options: list[WrongDatabaseOptions] = []
         self._removed_files: list[str] = []
-        self._removed_copies: list[RemovedCopy] = []
+        self._removed_copies: list[tuple[str, bool, str, str, PathType]] = []
         self._removed_zips: list[tuple[str, str]] = []
         self._skipped_updated_files: dict[str, list[str]] = dict()
-        self._filtered_zip_data: list[tuple[str, str, dict[str, Any], dict[str, Any]]] = []
+        self._filtered_zip_data: dict[str, dict[str, dict[str, dict[str, Any]]]] = defaultdict(dict)
         self._installed_zip_summary: list[tuple[str, str, StoreFragmentDrivePaths, dict[str, Any]]] = []
         self._installed_folders: set[str] = set()
         self._directories = dict()
@@ -534,9 +473,7 @@ def add_installed_db(self, db: DbEntity):
     def add_updated_db(self, db_id: str):
         self._updated_dbs.append(db_id)
     def add_filtered_zip_data(self, db_id: str, zip_id: str, filtered_data: FileFoldersHolder) -> None:
-        files, folders = filtered_data['files'], filtered_data['folders']
-        #if len(files) == 0 and len(folders) == 0: return
-        self._filtered_zip_data.append((db_id, zip_id, files, folders))
+        self._filtered_zip_data[db_id][zip_id] = filtered_data
     def add_failed_db_options(self, exception: WrongDatabaseOptions):
         self._failed_db_options.append(exception)
     def add_removed_file(self, file: str):
@@ -545,9 +482,9 @@ def add_removed_files(self, files: list[PathPackage]):
         if len(files) == 0: return
         for pkg in files:
             self._removed_files.append(pkg.rel_path)
-    def add_removed_copies(self, copies: list[RemovedCopy]):
+    def add_removed_copies(self, copies: list[RemovedCopy], db_id: str):
         if len(copies) == 0: return
-        self._removed_copies.extend(copies)
+        self._removed_copies.extend([(db_id, is_external, folder_path, other_drive, el_kind) for is_external, folder_path, other_drive, el_kind  in copies])
     def add_installed_folders(self, folders: list[PathPackage]):
         if len(folders) == 0: return
         for pkg in folders:

src/downloader/path_package.py

@@ -106,6 +106,16 @@ def is_pext_external_subfolder(self) -> bool:
     def is_pext_parent(self) -> bool:
         return self.pext_props is not None and self.pext_props.kind == PEXT_KIND_PARENT
 
+    def clone(self) -> 'PathPackage':
+        return PathPackage(
+            self.rel_path,
+            self.drive,
+            self.description,
+            self.ty,
+            self.kind,
+            None if self.pext_props is None else self.pext_props.clone(),
+        )
+
     def db_path(self) -> str:
         if self.kind == PATH_PACKAGE_KIND_PEXT:
             return '|' + self.rel_path
@@ -147,6 +157,15 @@ def __init__(
         self.other_drives = other_drives
         self.is_subfolder = is_subfolder
 
+    def clone(self) -> 'PextPathProps':
+        return PextPathProps(
+            self.kind,
+            self.parent,
+            self.drive,
+            self.other_drives,
+            self.is_subfolder,
+        )
+
     def parent_pkg(self) -> PathPackage:
         return PathPackage(self.parent, self.drive, {}, PATH_TYPE_FOLDER, PATH_PACKAGE_KIND_PEXT, PextPathProps(
             PEXT_KIND_PARENT, self.parent, self.drive, self.other_drives, False