Progress on cleaning store.

Author: theypsilon

docs/jobs_diagram.png

@@ -16,31 +16,37 @@
 # NON-ZIP NODES
 dot.node('START', 'START', shape='ellipse', fillcolor='black', fontcolor='white')
 dot.node('END', 'END', shape='ellipse', fillcolor='#77DD77')
-dot.node('A', 'FetchFileJob [db]')
-dot.node('C', 'OpenDbJob', fillcolor='#ffefd5')
+dot.node('ABORT', 'ABORT', shape='ellipse', fillcolor='#ff8f8f')
+dot.node('A', 'FetchDataJob [db]')
+dot.node('B', 'LoadLocalStoreJob', fillcolor='#ffefd5')
+dot.node('C', 'OpenDbJob', fillcolor='#B0E0E6')
 dot.node('D', 'ProcessDbMainJob', fillcolor='#B0E0E6')
-dot.node('E', 'ProcessDbIndexJob', fillcolor='#B0E0E6')
+dot.node('E', 'ProcessDbIndexJob', fillcolor='#B0E0E6:#ffefd5')
 dot.node('M', 'FetchFileJob [file]')
-dot.node('N', 'ValidateFileJob [file]', fillcolor='#ffefd5')
 
 # Example "legend" nodes
 dot.node('0', 'CPU', fillcolor='#B0E0E6')
 dot.node('1', 'File System', fillcolor='#ffefd5')
 dot.node('2', 'Network')
 
 dot.edge('START', 'A', label='1:N')
+dot.edge('START', 'B')
 
 dot.edge('A', 'C', weight='10')
-dot.edge('C', 'D', weight='10')
-dot.edge('D', 'E', weight='10')
+dot.edge('C', 'D', weight='10', label=' db + store')
+dot.edge('D', 'E', weight='10', label=' if no zips')
 
-dot.edge('E', 'M', label='1:N')
-dot.edge('M', 'N')
+dot.edge('E', 'M', label=' 1:N')
 
+dot.edge('B', 'C', style='dotted', constraint='true', label='wait\nstore', dir='back')
+
+dot.edge('B', 'B', label=' retry', style='dashed', constraint='false')
+dot.edge('A', 'A', label=' retry', style='dashed', constraint='false')
 dot.edge('C', 'A', label=' retry', style='dashed', constraint='true')
-dot.edge('N', 'M', label=' retry', style='dashed', constraint='false')
+dot.edge('M', 'M', label=' retry', style='dashed', constraint='false')
 
-dot.edge('N', 'END', weight='20', constraint='true')
+dot.edge('M', 'END', weight='20', constraint='true')
+dot.edge('B', 'ABORT', weight='20', constraint='true', label=' if always fails', style='dashed')
 
 dot.render('jobs_diagram', format='png', cleanup=True)
 
@@ -50,49 +56,50 @@
     c.node('3', 'Zip Feature', style='filled', fillcolor='#8f8fff', fontcolor='white', penwidth='0')
 
     c.node('O', 'WaitDbZipsJob', fillcolor='#B0E0E6')
-    c.node('F', 'ProcessZipIndexJob', fillcolor='#B0E0E6')
-    c.node('G', 'FetchFileJob [zip summary]')
-    c.node('H', 'ValidateFileJob [zip summary]', fillcolor='#ffefd5')
-    c.node('I', 'OpenZipSummaryJob', fillcolor='#ffefd5')
-    c.node('J', 'FetchFileJob [zip contents]')
-    c.node('K', 'ValidateFileJob [zip contents]', fillcolor='#ffefd5')
+    c.node('F', 'ProcessZipIndexJob', fillcolor='#B0E0E6:#ffefd5')
+    c.node('G', 'FetchDataJob [zip summary]')
+    c.node('I', 'OpenZipSummaryJob', fillcolor='#B0E0E6')
+    c.node('DOT0', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
+    c.node('J', 'FetchDataJob [zip contents]')
     c.node('L', 'OpenZipContentsJob', fillcolor='#ffefd5')
-    c.node('Q', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
+    c.node('DOT1', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
+    c.node('DOT2', '', shape='circle', fixedsize='true', width='0.01', fillcolor='#ff8f8f')
 #
 # EDGES
 #
-dot.edge('D', 'O', weight='10')
-dot.edge('O', 'E', weight='10')
-dot.edge('D', 'G', label='1:N')
+dot.edge('D', 'O', weight='10', label='if zips')
+dot.edge('O', 'E', weight='10', label='store w/ deselected\nzip indexes')
+dot.edge('D', 'G', label='1:N (missing zips)')
 
-dot.edge('F', 'J')
+dot.edge('F', 'J', label='if many file installs')
 dot.edge('I', 'F')
-dot.edge('G', 'H')
-dot.edge('H', 'I')
-dot.edge('J', 'K')
-dot.edge('K', 'L')
-dot.edge('L', 'M', label='1:N')
-dot.edge('F', 'Q', label='1:N', dir='none')
-dot.edge('Q', 'M')
+dot.edge('G', 'I')
+dot.edge('J', 'L')
+dot.edge('L', 'M', label='1:N\n(invalid files)')
+dot.edge('F', 'DOT1', dir='none')
+dot.edge('DOT1', 'DOT2', label='if just few\n file installs', dir='none')
+dot.edge('DOT2', 'M', label=' 1:N')
 #dot.edge('L', 'M', label='1:N')
 
 # “Wait” edges
-dot.edge('F', 'O', style='dotted', constraint='true', label='wait')
+dot.edge('F', 'O', style='dotted', constraint='true', label='wait\n zip indexes', dir='back')
 
 # Labeled “1:N” edges
-dot.edge('D', 'F', label='1:N', weight='5')
+dot.edge('D', 'F', label='1:N (stored zips)', weight='5')
 
 # Edges to END
 dot.edge('L', 'END', weight='20', constraint='true')
 
 # “Retry” edges
-dot.edge('H', 'G', label=' r', style='dashed', constraint='false')
-dot.edge('I', 'G', label=' r', style='dashed', constraint='false')
-dot.edge('K', 'J', label=' r', style='dashed', constraint='false')
-dot.edge('L', 'J', label=' r', style='dashed', constraint='false')
+dot.edge('G', 'G', label=' retry', style='dashed', constraint='false')
+dot.edge('I', 'G', label=' retry', style='dashed', constraint='false')
+dot.edge('J', 'J', label=' retry', style='dashed', constraint='false')
+dot.edge('L', 'J', label=' retry', style='dashed', constraint='false')
 
 # "Backup" edges
-dot.edge('G', 'F', label='backup if stored', style='dashed', constraint='false')
+dot.edge('G', 'DOT0', style='dashed', constraint='true', dir='none')
+dot.edge('I', 'DOT0', style='dashed', constraint='true', dir='none')
+dot.edge('DOT0', 'F', label='backup:\nstored summary', style='dashed', constraint='false')
 #dot.edge('H', 'I', label='b', style='dashed', constraint='false', dir='none')
 #dot.edge('I', 'F', label='b', style='dashed', constraint='false')
 #dot.edge('J', 'E', label='b', style='dashed', constraint='false')

docs/jobs_diagram_with_zip_feature.png

@@ -107,6 +107,7 @@ def _full_run_impl(self):
         local_store = self._local_repository.load_store()
 
         db_pkgs = [DbSectionPackage(db_id, section, local_store.store_by_id(db_id)) for db_id, section in sorted_db_sections(self._config)]
+        #db_pkgs = [db_pkg for db_pkg in db_pkgs  if db_pkg.db_id == 'distribution_mister']
 
         for relocation_package in self._base_path_relocator.relocating_base_paths(db_pkgs):
             self._base_path_relocator.relocate_non_system_files(relocation_package)

docs/jobs_flow_diagram.py

@@ -39,10 +39,9 @@ def operate_on(self, job: OpenDbJob) -> WorkerResult:  # type: ignore[override]
         return [ProcessDbMainJob(db=db, ini_description=ini_description, store=store, full_resync=full_resync).add_tag(make_db_tag(job.section))], None
 
     def _open_db(self, section: str, transfer: Any) -> DbEntity:
-        self._ctx.logger.bench('Loading database start: ', section)
+        self._ctx.logger.bench('OpenDbWorker Loading database: ', section)
         db_raw = self._ctx.file_system.load_dict_from_transfer(transfer)
-        self._ctx.logger.bench('Loading database end: ', section)
-        self._ctx.logger.bench('Validating database start: ', section)
+        self._ctx.logger.bench('OpenDbWorker Validating database: ', section)
         db_entity = DbEntity(db_raw, section)
-        self._ctx.logger.bench('Validating database end: ', section)
+        self._ctx.logger.bench('OpenDbWorker end: ', section)
         return db_entity

src/downloader/full_run_service.py

@@ -32,6 +32,7 @@
 class ProcessDbIndexJob(Job):
     type_id: int = field(init=False, default=JobSystem.get_job_type_id())
 
+    # Inputs
     db: DbEntity
     store: StoreWrapper
     ini_description: Dict[str, Any]