fix manifest error by adding checkpoint

Author: ptrebert

config/testing/params_refcon.yaml

@@ -1,7 +1,7 @@
 
 reference_container_store: "."
 reference_container_names: 
-  - test_v0
+  - test_v1
 use_reference_container: True
 
 

workflow/Snakefile

@@ -9,7 +9,6 @@ rule run_all:
         # add output of final rule(s) here
         # to trigger complete run
         [],
-        
 
 
 onsuccess:

workflow/rules/commons/02_pyutils.smk

@@ -244,7 +244,9 @@ def load_accounting_information(wildcards):
                 logerr(f"Size of accounting file list: {len(created_files)}")
         except FileNotFoundError:
             if VERBOSE:
-                logerr(f"Accounting file does not exist (yet): {account_file}")
+                warn_msg = f"Accounting file does not exist (yet): {account_file}\n"
+                warn_msg += "Please RERUN the workflow in DRY RUN MODE to create the file accounts!"
+                logerr(warn_msg)
     return sorted(created_files)
 
 
@@ -484,16 +486,35 @@ def collect_git_labels():
 # =======================================================
 
 
+def trigger_refcon_manifest_caching(wildcards):
+    """
+    This function merely triggers the checkpoint
+    to merge all reference containers caches into
+    one. This checkpoint is needed to get a
+    start-to-end run, otherwise "refcon_find_container"
+    would produce an error.
+    """
+    refcon_manifest_cache = str(
+        checkpoints.refcon_cache_manifests.get(**wildcards).output.cache
+    )
+    expected_path = DIR_PROC.joinpath(".cache", "refcon", "refcon_manifests.cache")
+    # following assert safeguard against future changes
+    assert pathlib.Path(refcon_manifest_cache).resolve() == expected_path.resolve()
+    return refcon_manifest_cache
+
+
 def refcon_find_container(manifest_cache, ref_filename):
 
     if not pathlib.Path(manifest_cache).is_file():
         if DRYRUN:
             return ""
         else:
-            raise FileNotFoundError(
-                f"Reference container manifest cache does not exist: {manifest_cache}"
-            )
-
+            if VERBOSE:
+                warn_msg = "Warning: reference container manifest cache "
+                warn_msg += "does not exist yet. Returning empty reference "
+                warn_msg += "container path."
+                logerr(warn_msg)
+            return ""
 
     manifests = pandas.read_csv(manifest_cache, sep="\t", header=0)
 

workflow/rules/commons/03_smkutils.smk

@@ -98,13 +98,33 @@ rule create_manifest:
         import collections
         import pandas
 
+        if not all([af.is_file() for af in ACCOUNTING_FILES.values()]):
+            err_msg = "Accounting files have not been created yet, cannot create workflow manifest.\n"
+            err_msg += "Please rerun the workflow twice in dry run mode:\n"
+            err_msg += "snakemake --dry-run (or: -n) [...other options...]"
+            logerr(err_msg)
+            raise RuntimeError(
+                "Cannot proceed with workflow execution w/o accouting files."
+            )
+
+        if len(input.manifest_files) == 0:
+            warn_msg = "No files recorded for inclusion in workflow manifest.\n"
+            warn_msg += "Are you sure you did not forget annotating rules with:\n"
+            warn_msg += "commons/02_pyutils.smk::register_input()\n"
+            warn_msg += "commons/02_pyutils.smk::register_result()\n"
+            logerr(warn_msg)
+
         records = collections.defaultdict(dict)
         for line in fileinput.input(ACCOUNTING_FILES.values(), mode="r"):
             path_id, path_record = process_accounting_record(line)
             records[path_id].update(path_record)
 
         df = pandas.DataFrame.from_records(list(records.values()))
-        df.sort_values(["file_category", "file_name"], ascending=True)
+        if df.empty:
+            logerr("Manifest DataFrame is empty - aborting")
+            raise RuntimeError("Manifest DataFrame is empty")
+
+        df.sort_values(["file_category", "file_name"], ascending=True, inplace=True)
         reordered_columns = [
             "file_name",
             "file_category",

workflow/rules/commons/05_refcon.smk

@@ -14,7 +14,6 @@ if USE_REFERENCE_CONTAINER:
         shell:
             "{input.sif} manifest > {output.manifest}"
 
-
     rule refcon_run_get_file:
         """
         Snakemake interacts with Singularity containers using "exec",
@@ -26,7 +25,7 @@ if USE_REFERENCE_CONTAINER:
         (i.e., treat them like a regular file)
         """
         input:
-            cache=DIR_PROC.joinpath(".cache", "refcon", "refcon_manifests.cache"),
+            cache=trigger_refcon_manifest_caching,
         output:
             DIR_GLOBAL_REF.joinpath("{filename}"),
         envmodules:
@@ -39,8 +38,7 @@ if USE_REFERENCE_CONTAINER:
         shell:
             "{params.refcon_path} get {wildcards.filename} {output}"
 
-
-    rule refcon_cache_manifests:
+    checkpoint refcon_cache_manifests:
         input:
             manifests=expand(
                 DIR_PROC.joinpath(".cache", "refcon", "{refcon_name}.manifest"),

workflow/snaketests.smk

@@ -31,6 +31,7 @@ rule create_test_file:
         # END OF RUN BLOCK
 
 
+
 rule test_log_functions:
     """
     Test pyutil logging functions
@@ -51,6 +52,7 @@ rule test_log_functions:
         # END OF RUN BLOCK
 
 
+
 rule test_find_script_success:
     input:
         expand(rules.test_log_functions.output, logtype=["err", "out"]),
@@ -69,6 +71,7 @@ rule test_find_script_success:
         # END OF RUN BLOCK
 
 
+
 rule test_find_script_fail:
     input:
         rules.test_find_script_success.output,
@@ -88,6 +91,7 @@ rule test_find_script_fail:
         # END OF RUN BLOCK
 
 
+
 rule test_rsync_f2d:
     input:
         rules.create_test_file.output,
@@ -106,6 +110,7 @@ rule test_rsync_f2d:
         # END OF RUN BLOCK
 
 
+
 rule test_rsync_f2f:
     input:
         rules.create_test_file.output,
@@ -118,6 +123,7 @@ rule test_rsync_f2f:
         # END OF RUN BLOCK
 
 
+
 rule test_rsync_fail:
     input:
         rules.create_test_file.output,
@@ -138,6 +144,7 @@ rule test_rsync_fail:
         # END OF RUN BLOCK
 
 
+
 rule test_git_labels:
     input:
         rules.test_rsync_f2d.output,
@@ -155,15 +162,18 @@ rule test_git_labels:
         # END OF RUN BLOCK
 
 
+
 if USE_REFERENCE_CONTAINER:
     CONTAINER_TEST_FILES = [
         DIR_GLOBAL_REF.joinpath("genome.fasta.fai"),
+        DIR_GLOBAL_REF.joinpath("exclusions.bed"),
+        DIR_GLOBAL_REF.joinpath("hg38_full.fasta.fai"),
         DIR_PROC.joinpath(".cache", "refcon", "refcon_manifests.cache"),
     ]
-    REGISTER_REFERENCE_FILE = CONTAINER_TEST_FILES[0]
+    REGISTER_REFERENCE_FILES = CONTAINER_TEST_FILES[:3]
 else:
     CONTAINER_TEST_FILES = []
-    REGISTER_REFERENCE_FILE = []
+    REGISTER_REFERENCE_FILES = []
 
 
 rule trigger_tests:
@@ -174,7 +184,7 @@ rule trigger_tests:
         DIR_RES.joinpath("testing", "all-ok.txt"),
     params:
         acc_out=lambda wildcards, output: register_result(output),
-        acc_ref=lambda wildcards, input: register_reference(REGISTER_REFERENCE_FILE),
+        acc_ref=lambda wildcards, input: register_reference(REGISTER_REFERENCE_FILES),
     run:
         with open(output[0], "w") as testfile:
             testfile.write("ok")