Improved zip summary performance.

Author: theypsilon

src/downloader/jobs/open_zip_contents_worker.py

@@ -103,4 +103,4 @@ def operate_on(self, job: OpenZipContentsJob) -> WorkerResult:  # type: ignore[o
 
         logger.bench('OpenZipContentsWorker launching recovery process index...', job.db.db_id, job.zip_id)
 
-        return create_fetch_jobs(self._ctx, job.db.db_id, invalid_files, job.zip_description.get('base_files_url', job.db.base_files_url))
+        return create_fetch_jobs(self._ctx, job.db.db_id, invalid_files, [], job.zip_description.get('base_files_url', job.db.base_files_url)), None

src/downloader/jobs/process_db_main_worker.py

@@ -60,9 +60,15 @@ def operate_on(self, job: ProcessDbMainJob) -> WorkerResult:  # type: ignore[ove
             zip_jobs = []
             zip_job_tags = []
 
+            logger.bench('zip_summaries start: ', job.db.db_id)
+            zip_summaries = read_only_store.zip_summaries()
+            logger.bench('zip_summaries end: ', job.db.db_id)
+
             for zip_id, zip_description in job.db.zips.items():
                 #if zip_id != 'cheats_folder_psx': continue
-                zip_job, err = _make_zip_job(ZipJobContext(zip_id=zip_id, zip_description=zip_description, config=config, job=job), self._ctx.logger)
+                logger.bench('make zip job start: ', job.db.db_id, zip_id)
+                zip_job, err = _make_zip_job(zip_summaries.get(zip_id, None), ZipJobContext(zip_id=zip_id, zip_description=zip_description, config=config, job=job))
+                logger.bench('make zip job end: ', job.db.db_id, zip_id)
                 if err is not None:
                     self._ctx.swallow_error(err)
                     job.ignored_zips.append(zip_id)
@@ -98,16 +104,12 @@ def operate_on(self, job: ProcessDbMainJob) -> WorkerResult:  # type: ignore[ove
         return next_jobs, None
 
 
-def _make_zip_job(z: ZipJobContext, logger: Logger) -> Tuple[Job, Optional[Exception]]:
-    logger.bench('make zip job start: ', z.job.db.db_id, z.zip_id)
+def _make_zip_job(index: Optional, z: ZipJobContext) -> Tuple[Job, Optional[Exception]]:
     try:
         check_zip(z.zip_description, z.job.db.db_id, z.zip_id)
     except Exception as e:
         return NilJob(), e
     if 'summary_file' in z.zip_description:
-        logger.bench('zip_summary start: ', z.job.db.db_id, z.zip_id)
-        index = z.job.store.read_only().zip_summary(z.zip_id)
-        logger.bench('zip_summary end: ', z.job.db.db_id, z.zip_id)
         process_zip_job = None if index is None else _make_process_zip_job_from_ctx(z, zip_summary=index, has_new_zip_summary=False)
 
         # if there is a recent enough index in the store, use it
@@ -119,7 +121,6 @@ def _make_zip_job(z: ZipJobContext, logger: Logger) -> Tuple[Job, Optional[Excep
     else:
         job = _make_process_zip_job_from_ctx(z, zip_summary=z.zip_description['internal_summary'], has_new_zip_summary=True)
 
-    logger.bench('make zip job end: ', z.job.db.db_id, z.zip_id)
     return job, None
 
 

src/downloader/local_store_wrapper.py

@@ -546,58 +546,40 @@ def list_missing_files(self, db_files):
                     files.update(external['files'])
         return {f: d for f, d in files.items() if f not in db_files}
 
-    def zip_summary(self, zip_id) -> Optional[Dict[str, Any]]:
-        files = {}
-        folders = {}
-        for file_path, file_description in self._aggregated_summary['files'].items():
-            if 'zip_id' in file_description and file_description['zip_id'] == zip_id:
-                # @TODO: This if should not be necessary if we store all the zip information on the store
-                #        Explicit asking for games and cheats is a hack, as this should only be declared in
-                #        the database information. Remove ASAP
-                if file_path.startswith('games') or file_path.startswith('cheats'):
-                    files['|' + file_path] = file_description
-                else:
-                    files[file_path] = file_description
-
-        for folder_path, folder_description in self._aggregated_summary['folders'].items():
-            if 'zip_id' in folder_description and folder_description['zip_id'] == zip_id:
-                # @TODO: This if should not be necessary if we store all the zip information on the store
-                #        Explicit asking for games and cheats is a hack, as this should only be declared in
-                #        the database information. Remove ASAP
-                if folder_path.startswith('games') or folder_path.startswith('cheats'):
-                    folders['|' + folder_path] = folder_description
-                else:
-                    folders[folder_path] = folder_description
-
-        for zip_id, zip_description in self._store.get('filtered_zip_data', {}).items():
-            if zip_id != zip_id:
-                continue
+    def zip_summaries(self) -> dict[str, Any]:
+        grouped = defaultdict(lambda: {'files': {}, 'folders': {}})
+
+        # @TODO: This if startswith('games') should be removed when we store all the zip information on the store
+        #        Explicit asking for games is a hack, as this should only be declared in the database information. Remove ASAP
+
+        for fp, fd in self._store.get('files', {}).items():
+            if 'zip_id' not in fd: continue
+            grouped[fd['zip_id']]['files'][f'|{fp}' if fp.startswith('games') else fp] = fd
+
+        for dp, dd in self._store.get('folders', {}).items():
+            if 'zip_id' not in dd: continue
+            grouped[dd['zip_id']]['folders'][f'|{dp}' if dp.startswith('games') else dp] = dd
+
+        for summary in self._store.get('external', {}).values():
+            for fp, fd in summary.get('files', {}).items():
+                if 'zip_id' not in fd: continue
+                grouped[fd['zip_id']]['files'][f'|{fp}' if fp.startswith('games') else fp] = fd
+
+        for summary in self._store.get('external', {}).values():
+            for dp, dd in summary.get('folders', {}).items():
+                if 'zip_id' not in dd: continue
+                grouped[dd['zip_id']]['folders'][f'|{dp}' if dp.startswith('games') else dp] = dd
+
+        for zip_id, summary in self._store.get('filtered_zip_data', {}).items():
+            for fp, fd in summary.get('files', {}).items():
+                grouped[zip_id]['files'][f'|{fp}' if fp.startswith('games') else fp] = fd
+            for dp, dd in summary.get('folders', {}).items():
+                grouped[zip_id]['folders'][f'|{dp}' if dp.startswith('games') else dp] = dd
+
+        for zip_id, data in grouped.items():
+            data['hash'] = self._store.get('zips', {}).get(zip_id, {}).get('summary_file', {}).get('hash', NO_HASH_IN_STORE_CODE)
 
-            for file_path, file_description in zip_description['files'].items():
-                if 'zip_id' in file_description and file_description['zip_id'] == zip_id:
-                    # @TODO: This if should not be necessary if we store all the zip information on the store
-                    #        Explicit asking for games and cheats is a hack, as this should only be declared in
-                    #        the database information. Remove ASAP
-                    if file_path.startswith('games') or file_path.startswith('cheats'):
-                        files['|' + file_path] = file_description
-                    else:
-                        files[file_path] = file_description
-
-            for folder_path, folder_description in zip_description['folders'].items():
-                if 'zip_id' in folder_description and folder_description['zip_id'] == zip_id:
-                    # @TODO: This if should not be necessary if we store all the zip information on the store
-                    #        Explicit asking for games and cheats is a hack, as this should only be declared in
-                    #        the database information. Remove ASAP
-                    if folder_path.startswith('games') or folder_path.startswith('cheats'):
-                        folders['|' + folder_path] = folder_description
-                    else:
-                        folders[folder_path] = folder_description
-
-        if len(files) == 0 and len(folders) == 0:
-            return None
-
-        summary_hash = self._store.get('zips', {}).get(zip_id, {}).get('summary_file', {}).get('hash', NO_HASH_IN_STORE_CODE)
-        return {'files': files, 'folders': folders, 'hash': summary_hash}
+        return grouped
 
     @property
     def filtered_zip_data(self):