Accounting for startup time since the beggining, this adds 4 second to the app timer, and lets 1 second for unaccounted startup time. Previously we had 5 seconds unaccounted startup time. Fixed tests for the optimized store writing code.

Author: theypsilon

src/__main__.py

@@ -17,6 +17,9 @@
 # You can download the latest version of this tool from:
 # https://github.com/MiSTer-devel/Downloader_MiSTer
 
+import time
+start_time = time.time()
+
 from sys import exit
 
 try:
@@ -36,5 +39,5 @@
     default_commit = None
 
 if __name__ == '__main__':
-    exit_code = main(read_env(default_commit))
+    exit_code = main(read_env(default_commit), start_time)
     exit(exit_code)

src/build.sh

@@ -20,6 +20,9 @@ cp __main__.py "${TEMPDIR}/__main__.py"
 if [[ "${SKIP_REMOVALS:-false}" != "true" ]] ; then
   find "${TEMPDIR}" -type f -name '*.py' -exec perl -i -0pe 's/"""(.*?)"""/""/sg; s/^\s*#.*\n//mg; s/^\s*\n//mg' {} +
 fi
+#if which strip-hints 2>&1 > /dev/null ; then
+#  find "${TEMPDIR}" -type f -name '*.py' -exec strip-hints --inplace {} + 2> /dev/null
+#fi
 find "${TEMPDIR}" -type f ! -name '*.py' -exec rm -f {} +
 find "${TEMPDIR}" -type f -iname "*.py" -print0 | while IFS= read -r -d '' file ; do pin_metadata "${file}" ; done
 pushd "${TEMPDIR}" >/dev/null 2>&1

src/downloader/config_reader.py

@@ -19,9 +19,9 @@
 
 import configparser
 import re
-import time
 from pathlib import Path
-from typing import Optional, List, TypeVar, Union, Dict, Any, SupportsInt
+from typing import Optional, List, TypeVar, Union, SupportsInt
+
 
 from downloader.config import Environment, Config, default_config, InvalidConfigParameter, AllowReboot, \
     ConfigDatabaseSection, ConfigMisterSection, AllowDelete
@@ -30,13 +30,14 @@
     K_STORAGE_PRIORITY, K_ALLOW_DELETE, K_ALLOW_REBOOT, K_VERBOSE, K_UPDATE_LINUX, K_MINIMUM_SYSTEM_FREE_SPACE_MB, \
     K_MINIMUM_EXTERNAL_FREE_SPACE_MB, STORAGE_PRIORITY_OFF, STORAGE_PRIORITY_PREFER_SD, STORAGE_PRIORITY_PREFER_EXTERNAL
 from downloader.db_options import DbOptions, DbOptionsProps, DbOptionsValidationException
-from downloader.logger import Logger
+from downloader.logger import Logger, time_str
 
 
 class ConfigReader:
-    def __init__(self, logger: Logger, env: Environment):
+    def __init__(self, logger: Logger, env: Environment, start_time: float):
         self._logger = logger
         self._env = env
+        self._start_time = start_time
 
     def calculate_config_path(self, current_working_dir: str) -> str:
         if self._env['PC_LAUNCHER'] is not None:
@@ -75,12 +76,24 @@ def read_config(self, config_path: str) -> Config:
         result['debug'] = self._env['DEBUG'] == 'true'
         if result['debug']:
             result['verbose'] = True
+        if self._env['LOGLEVEL'] != '':
+            if 'info' in self._env['LOGLEVEL']:
+                result['verbose'] = False
+            if 'debug' in self._env['LOGLEVEL']:
+                result['verbose'] = True
+            if 'http' in self._env['LOGLEVEL']:
+                result['http_logging'] = True
+
+        if result['verbose']: self._logger.print(f'BENCH {time_str(self._start_time)}| Read config start.')
 
         if self._env['DEFAULT_BASE_PATH'] is not None:
             result['base_path'] = self._env['DEFAULT_BASE_PATH']
             result['base_system_path'] = self._env['DEFAULT_BASE_PATH']
 
         ini_config = self._load_ini_config(config_path)
+
+        if result['verbose']: self._logger.print(f'BENCH {time_str(self._start_time)}| Load ini done.')
+
         default_db = self._default_db_config()
 
         for section in ini_config.sections():
@@ -97,6 +110,8 @@ def read_config(self, config_path: str) -> Config:
             self._logger.print("Reading '%s' db section" % section)
             result['databases'][section_id] = self._parse_database_section(default_db, parser, section_id)
 
+        if result['verbose']: self._logger.print(f'BENCH {time_str(self._start_time)}| Read sections done.')
+
         if len(result['databases']) == 0:
             self._logger.print('Reading default db')
             self._add_default_database(ini_config, result)
@@ -115,7 +130,7 @@ def read_config(self, config_path: str) -> Config:
         result['fail_on_file_error'] = self._env['FAIL_ON_FILE_ERROR'] == 'true'
         result['commit'] = self._valid_max_length('COMMIT', self._env['COMMIT'], 50)
         result['default_db_id'] = self._valid_db_id(K_DEFAULT_DB_ID, self._env['DEFAULT_DB_ID'])
-        result['start_time'] = time.time()
+        result['start_time'] = self._start_time
         result['logfile'] = self._env['LOGFILE']
         result['config_path'] = Path(config_path)
 
@@ -149,15 +164,9 @@ def read_config(self, config_path: str) -> Config:
             result['logfile'] = str(launcher_path.with_suffix('.log'))
             result['curl_ssl'] = ''
 
-        if self._env['LOGLEVEL'] != '':
-            if 'info' in self._env['LOGLEVEL']:
-                result['verbose'] = False
-            if 'debug' in self._env['LOGLEVEL']:
-                result['verbose'] = True
-            if 'http' in self._env['LOGLEVEL']:
-                result['http_logging'] = True
-
         result['environment'] = self._env
+
+        if result['verbose']: self._logger.print(f'BENCH {time_str(self._start_time)}| Read config done.')
         return result
 
     @staticmethod

src/downloader/jobs/process_db_index_job.py

@@ -51,7 +51,7 @@ def retry_job(self): return None
     duplicated_files: List[str] = field(default_factory=list)
 
     installed_folders: List[PathPackage] = field(default_factory=list)
-    removed_folders: List[RemovedCopy] = field(default_factory=list)  #  @TODO: Why there is removed_folders AND directories_to_remove?
+    removed_folders: List[PathPackage] = field(default_factory=list)  #  @TODO: Why there is removed_folders AND directories_to_remove?
 
     directories_to_remove: List[PathPackage] = field(default_factory=list)
     files_to_remove: List[PathPackage] = field(default_factory=list)

src/downloader/jobs/process_db_index_worker.py

@@ -127,8 +127,8 @@ def create_packages_from_index(ctx: DownloaderWorkerContext, config: Config, sum
     List[_DeleteFolderPackage]
 ]:
     calculator = ctx.target_paths_calculator_factory.target_paths_calculator(config)
-    check_file_pkgs, remove_files_pkgs = _translate_items(ctx, calculator, summary.files, PathType.FILE, store.files)
-    create_folder_pkgs, delete_folder_pkgs = _translate_items(ctx, calculator, summary.folders, PathType.FOLDER, store.folders)
+    check_file_pkgs, remove_files_pkgs = _translate_items(ctx, calculator, summary.files, PathType.FILE, store.all_files())
+    create_folder_pkgs, delete_folder_pkgs = _translate_items(ctx, calculator, summary.folders, PathType.FOLDER, store.all_folders())
     return check_file_pkgs, remove_files_pkgs, create_folder_pkgs, delete_folder_pkgs
 
 def _translate_items(ctx: DownloaderWorkerContext, calculator: TargetPathsCalculator, items: Dict[str, Dict[str, Any]], path_type: PathType, stored: Dict[str, Dict[str, Any]]) -> Tuple[List[PathPackage], List[PathPackage]]:
@@ -216,7 +216,7 @@ def try_reserve_space(ctx: DownloaderWorkerContext, file_pkgs: Iterable[PathPack
         return full_partitions
 
 
-def process_create_folder_packages(ctx: DownloaderWorkerContext, create_folder_pkgs: List[PathPackage], db_id: str, db_folder_index: Dict[str, Any], store: ReadOnlyStoreAdapter) -> Tuple[List[RemovedCopy], List[PathPackage], List[str]]:
+def process_create_folder_packages(ctx: DownloaderWorkerContext, create_folder_pkgs: List[PathPackage], db_id: str, db_folder_index: Dict[str, Any], store: ReadOnlyStoreAdapter) -> Tuple[List[PathPackage], List[PathPackage], List[str]]:
     if len(create_folder_pkgs) == 0:
         return [], [], []
 
@@ -225,37 +225,52 @@ def process_create_folder_packages(ctx: DownloaderWorkerContext, create_folder_p
     except Exception as e:
         ctx.swallow_error(e)
 
-    folder_copies_to_be_removed: List[Tuple[bool, str, str, PathType]] = []
+    folder_copies_to_be_removed: List[PathPackage] = []
     processing_folders: List[PathPackage] = []
 
     parent_drives: Dict[str, set[str]] = defaultdict(set)
     parent_pkgs: Dict[str, PathPackage] = dict()
-    parents_to_add: List[PathPackage] = []
 
     for pkg in sorted(create_folder_pkgs, key=lambda x: len(x.rel_path)):
-        if pkg.kind != PATH_PACKAGE_KIND_PEXT:
-            processing_folders.append(pkg)
-        elif pkg.is_pext_parent():
+        if pkg.is_pext_parent():
             parent_pkgs[pkg.rel_path] = pkg
             continue
-        else:
-            processing_folders.append(pkg)
 
-            pkg_parent = pkg.pext_props.parent
-            if pkg_parent in parent_pkgs and pkg.drive not in parent_drives[pkg_parent]:
-                parent_drives[pkg_parent].add(pkg.drive)
-                parent_pkg = parent_pkgs[pkg_parent].clone()
-                parent_pkg.drive = pkg.drive
-                parent_pkg.pext_props.kind = pkg.pext_props.kind
-                parent_pkg.pext_props.drive = pkg.pext_props.drive
-                parent_pkg.pext_props.parent = ''
-                parents_to_add.append(parent_pkg)
+        processing_folders.append(pkg)
+        if pkg.kind != PATH_PACKAGE_KIND_PEXT:
+            continue
 
-        _maybe_add_copies_to_remove(ctx, folder_copies_to_be_removed, store, pkg.rel_path, pkg.pext_drive())
+        pkg_parent = pkg.pext_props.parent
+        if pkg_parent not in parent_pkgs or pkg.drive in parent_drives:
+            continue
 
-    for parent_pkg in parents_to_add:
+        parent_drives[pkg_parent].add(pkg.drive)
+        parent_pkg = parent_pkgs[pkg_parent].clone()
+        parent_pkg.drive = pkg.drive
+        parent_pkg.pext_props.kind = pkg.pext_props.kind
+        parent_pkg.pext_props.drive = pkg.pext_props.drive
+        parent_pkg.pext_props.parent = ''
         processing_folders.append(parent_pkg)
-        _maybe_add_copies_to_remove(ctx, folder_copies_to_be_removed, store, parent_pkg.rel_path, parent_pkg.drive)
+
+    for pkg in processing_folders:
+        folder_path, drive = pkg.rel_path, pkg.drive
+        for is_external, other_drive in store.list_other_drives_for_folder(pkg):
+            if ctx.file_system.is_folder(os.path.join(other_drive, folder_path)):
+                continue
+
+            if is_external:
+                removed_pkg = pkg.clone_as_pext()
+                removed_pkg.drive = other_drive
+                removed_pkg.pext_props.drive = other_drive
+                removed_pkg.pext_props.kind = PEXT_KIND_EXTERNAL
+            else:
+                removed_pkg = pkg.clone()
+                removed_pkg.drive = other_drive
+                if removed_pkg.pext_props is not None:
+                    removed_pkg.pext_props.kind = PEXT_KIND_STANDARD
+                    removed_pkg.pext_props.drive = other_drive
+
+            folder_copies_to_be_removed.append(removed_pkg)
 
     ctx.logger.bench('add_processed_folders start: ', db_id, len(processing_folders))
     non_existing_folders = ctx.installation_report.add_processed_folders(processing_folders, db_id)
@@ -273,15 +288,6 @@ def process_create_folder_packages(ctx: DownloaderWorkerContext, create_folder_p
     installed_folders = [f for f in processing_folders if f.db_path() in db_folder_index]
     return folder_copies_to_be_removed, installed_folders, errors
 
-def _maybe_add_copies_to_remove(ctx: DownloaderWorkerContext, copies: List[Tuple[bool, str, str, PathType]], store: ReadOnlyStoreAdapter, folder_path: str, drive: Optional[str]):
-    if store.folder_drive(folder_path) == drive: return
-    copies.extend([
-        (is_external, folder_path, other_drive, PathType.FOLDER)
-        for is_external, other_drive in store.list_other_drives_for_folder(folder_path, drive)
-        if not ctx.file_system.is_folder(os.path.join(other_drive, folder_path))
-    ])
-
-
 def create_fetch_jobs(ctx: DownloaderWorkerContext, db_id: str, non_existing_pkgs: List[_FetchFilePackage], need_update_pkgs: List[_FetchFilePackage], base_files_url: str) -> List[Job]:
     if len(non_existing_pkgs) == 0 and len(need_update_pkgs) == 0:
         return []

src/downloader/jobs/process_zip_index_job.py

@@ -57,7 +57,7 @@ def retry_job(self): return None
     duplicated_files: List[str] = field(default_factory=list)
 
     installed_folders: List[PathPackage] = field(default_factory=list)
-    removed_folders: List[RemovedCopy] = field(default_factory=list)  #  @TODO: Why there is removed_folders AND directories_to_remove?
+    removed_folders: List[PathPackage] = field(default_factory=list)  #  @TODO: Why there is removed_folders AND directories_to_remove?
 
     directories_to_remove: List[PathPackage] = field(default_factory=list)
     files_to_remove: List[PathPackage] = field(default_factory=list)

src/downloader/local_store_wrapper.py

@@ -20,9 +20,9 @@
 from downloader.jobs.index import Index
 from downloader.other import empty_store_without_base_path
 from typing import Any, Dict, Optional, Set, Tuple, List, TypedDict
-from collections import defaultdict
+from collections import defaultdict, ChainMap
 
-from downloader.path_package import PathPackage
+from downloader.path_package import PathPackage, PATH_PACKAGE_KIND_PEXT, PextPathProps, PEXT_KIND_EXTERNAL, PathType
 
 NO_HASH_IN_STORE_CODE = 'file_does_not_exist_so_cant_get_hash'
 
@@ -183,25 +183,25 @@ def add_file(self, file_path, description):
         if file_path in self._external_additions['files']:
             for drive in self._external_additions['files'][file_path]:
                 self.remove_external_file(drive, file_path)
-        self._add_entry('files', file_path, description)
+        self._add_entry('files', PathType.FILE, file_path, description)
 
     def add_folder(self, folder, description):
-        self._add_entry('folders', folder, description)
+        self._add_entry('folders', PathType.FOLDER, folder, description)
 
-    def _add_entry(self, kind, path, description):
+    def _add_entry(self, kind: str, ty: PathType, path: str, description: dict[str, Any]):
         self._clean_external_additions(kind, path)
 
         if 'zip_id' not in description and 'tags' in description:
             description.pop('tags')
 
-        if path in self._store[kind] and equal_dicts(self._store[kind][path], description):
+        if path in self._store[kind] and equal_descriptions(self._store[kind][path], description, ty):
             return
 
         self._store[kind][path] = description
         self._top_wrapper.mark_force_save()
 
     def add_external_folder(self, drive, folder_path, description):
-        self._add_external_entry('folders', drive, folder_path, description)
+        self._add_external_entry('folders', PathType.FOLDER, drive, folder_path, description)
 
     def add_external_file(self, drive, file_path, description):
         if file_path in self._store['files']:
@@ -211,20 +211,18 @@ def add_external_file(self, drive, file_path, description):
                 if d == drive:
                     continue
                 self.remove_external_file(d, file_path)
-        self._add_external_entry('files', drive, file_path, description)
+        self._add_external_entry('files', PathType.FILE, drive, file_path, description)
 
-    def _add_external_entry(self, kind, drive, path, description):
+    def _add_external_entry(self, kind: str, ty: PathType, drive: str, path: str, description: dict[str, Any]):
         external = self._external_by_drive(drive)
 
         if 'zip_id' not in description and 'tags' in description:
             description.pop('tags')
 
         entries = external[kind]
-        if path in entries and equal_dicts(entries[path], description):
+        if path in entries and equal_descriptions(entries[path], description, ty):
             return
 
-        #if path in self._store[kind]: del self._store[kind][path]
-
         entries[path] = description
         self._top_wrapper.mark_force_save()
 
@@ -605,10 +603,22 @@ def zips(self) -> Dict[str, Dict[str, Any]]:
     def files(self) -> Dict[str, Dict[str, Any]]:
         return self._store['files']
 
+    def all_files(self):
+        if not 'external' in self._store:
+            return self._store['files']
+
+        return ChainMap(self._store['files'], *[external['files'] for external in self._store['external'].values() if 'files' in external])
+
     @property
     def folders(self) -> Dict[str, Dict[str, Any]]:
         return self._store['folders']
 
+    def all_folders(self):
+        if not 'external' in self._store:
+            return self._store['folders']
+
+        return ChainMap(self._store['folders'], *[external['folders'] for external in self._store['external'].values() if 'folders' in external])
+
     @property
     def has_externals(self) -> bool:
         return 'external' in self._store
@@ -668,8 +678,8 @@ def list_other_drives_for_file(self, file_path: str, drive: Optional[str]) -> Li
 
         return result
 
-    def list_other_drives_for_folder(self, folder_path: str, drive: Optional[str]) -> List[Tuple[bool, str]]:
-        if drive is None: drive = self.base_path
+    def list_other_drives_for_folder(self, folder_pkg: PathPackage) -> List[Tuple[bool, str]]:
+        folder_path, drive = folder_pkg.rel_path, folder_pkg.drive
         if 'external' in self._store:
             result = [
                 (True, external_drive)
@@ -689,6 +699,24 @@ def has_no_files(self):
         return len(self._store['files']) == 0
 
 
+def equal_descriptions(lhs: dict[str, Any], b: dict[str, Any], ty: PathType) -> bool:
+    if ty == PathType.FOLDER: return equal_dicts_or_lhs_bigger(lhs, b)
+    else: return equal_dicts(lhs, b)
+
+def equal_dicts_or_lhs_bigger(lhs: dict[str, Any], b: dict[str, Any]) -> bool:
+    if len(b) > len(lhs):
+        return False
+
+    for key, value in lhs.items():
+        if key not in b:
+            continue
+
+        if not equal_values(value, b[key]):
+            return False
+
+    return True
+
+
 def equal_dicts(a, b):
     if len(a) != len(b):
         return False

src/downloader/logger.py

@@ -136,8 +136,7 @@ def bench(self, *args):
         if self._start_time is None:
             return
 
-        time_str = str(datetime.timedelta(seconds=time.time() - self._start_time))[0:-3]
-        bench_header = f'BENCH {time_str}| '
+        bench_header = f'BENCH {time_str(self._start_time)}| '
         self.print_logger.bench(bench_header, *args)
         self.file_logger.bench(bench_header, *args)
 
@@ -193,3 +192,6 @@ def debug(self, *args, sep='', end='\n', flush=True):
 
     def bench(self, *args):
         self._decorated_logger.bench(*args)
+
+def time_str(start_time: float) -> str:
+    return str(datetime.timedelta(seconds=time.time() - start_time))[0:-3]