Add more wrangler utils for SMaHT

CHANGELOG.rst

@@ -3,6 +3,13 @@
 Change Log
 ==========
 
+3.11.0
+=====
+* Most MetaWorkflowRun creation functions now accept multiple files or filesets as input.
+* Added `archive_broad_crams_and_fastqs` function to `wrangler_utils` to archive Broad submissions more easily. 
+* Added `purge_fileset` function to `wrangler_utils` to delete a fileset and its associated files.
+* Added `remove_property` function to `wrangler_utils` to remove properties from items.
+
 3.10.0
 =====
 * Added functionality to rerun a MetaWorkflowRun with imported steps from a previous run

magma_smaht/commands/create_meta_workflow_run.py

@@ -357,7 +357,12 @@ def qc_long_read_bam(file_accessions, replace_qc, auth_env):
 @cli.command()
 @click.help_option("--help", "-h")
 @click.option(
-    "-f", "--fileset-accession", required=True, type=str, help="Fileset accession"
+    "-f",
+    "--fileset-accessions",
+    required=True,
+    multiple=True,
+    type=str,
+    help="Fileset accession(s)",
 )
 @click.option(
     "-e",
@@ -366,16 +371,23 @@ def qc_long_read_bam(file_accessions, replace_qc, auth_env):
     type=str,
     help="Name of environment in smaht-keys file",
 )
-def conversion_cram_to_fastq(fileset_accession, auth_env):
+def conversion_cram_to_fastq(fileset_accessions, auth_env):
     """Conversion MWFR for CRAM to FASTQ (paired-end)"""
     smaht_key = get_auth_key(auth_env)
-    mwfr_cram_to_fastq_paired_end(fileset_accession, smaht_key)
+    for fileset_accession in fileset_accessions:
+        print(f"Working on FileSet {fileset_accession}")
+        mwfr_cram_to_fastq_paired_end(fileset_accession, smaht_key)
 
 
 @cli.command()
 @click.help_option("--help", "-h")
 @click.option(
-    "-f", "--fileset-accession", required=True, type=str, help="Fileset accession"
+    "-f",
+    "--fileset-accessions",
+    required=True,
+    multiple=True,
+    type=str,
+    help="Fileset accession(s)",
 )
 @click.option(
     "-e",
@@ -384,10 +396,12 @@ def conversion_cram_to_fastq(fileset_accession, auth_env):
     type=str,
     help="Name of environment in smaht-keys file",
 )
-def conversion_bam_to_fastq(fileset_accession, auth_env):
+def conversion_bam_to_fastq(fileset_accessions, auth_env):
     """Conversion MWFR for BAM to FASTQ (paired-end)"""
     smaht_key = get_auth_key(auth_env)
-    mwfr_bam_to_fastq_paired_end(fileset_accession, smaht_key)
+    for fileset_accession in fileset_accessions:
+        print(f"Working on FileSet {fileset_accession}")
+        mwfr_bam_to_fastq_paired_end(fileset_accession, smaht_key)
 
 
 @cli.command()

magma_smaht/commands/wrangler_utils.py

@@ -87,10 +87,17 @@ def reset_mwfrs(mwfr_uuids, auth_env):
     type=str,
     help="Name of environment in smaht-keys file",
 )
-def reset_all_failed_mwfrs(auth_env):
+@click.option(
+    "--ignore-md5",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Ignore MD5 checksum runs",
+)
+def reset_all_failed_mwfrs(auth_env, ignore_md5):
     """Reset all failed MetaWorkflowRuns on the portal"""
     smaht_key = get_auth_key(auth_env)
-    wrangler_utils.reset_all_failed_mwfrs(smaht_key)
+    wrangler_utils.reset_all_failed_mwfrs(smaht_key, ignore_md5)
 
 
 @cli.command()
@@ -270,6 +277,42 @@ def archive_unaligned_reads(fileset_accessions, dry_run, auth_env):
         wrangler_utils.archive_unaligned_reads(f, dry_run, smaht_key)
 
 
+@cli.command()
+@click.help_option("--help", "-h")
+@click.option(
+    "-f",
+    "--fileset-accessions",
+    required=True,
+    type=str,
+    multiple=True,
+    help="Fileset accessions",
+)
+@click.option(
+    "-d",
+    "--dry-run",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Dry run",
+)
+@click.option(
+    "-e",
+    "--auth-env",
+    required=True,
+    type=str,
+    help="Name of environment in smaht-keys file",
+)
+def archive_broad_crams_and_fastqs(fileset_accessions, dry_run, auth_env):
+    """
+    Archive (submitted) CRAMs and the associated FASTQs from the conversion.
+    Every such file in the fileset will receive the s3_lifecycle_category=long_term_archive.
+    """
+    smaht_key = get_auth_key(auth_env)
+    for f in fileset_accessions:
+        print(f"Working on Fileset {f}")
+        wrangler_utils.archive_broad_crams_and_fastqs(f, dry_run, smaht_key)
+
+
 @cli.command()
 @click.help_option("--help", "-h")
 @click.option(
@@ -294,6 +337,64 @@ def sample_identity_check_status(num_files, auth_env):
     wrangler_utils.sample_identity_check_status(num_files, smaht_key)
 
 
+@cli.command()
+@click.help_option("--help", "-h")
+@click.option(
+    "-f",
+    "--fileset-accessions",
+    required=True,
+    type=str,
+    multiple=True,
+    help="Fileset accessions",
+)
+@click.option(
+    "-d",
+    "--dry-run",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Dry run",
+)
+@click.option(
+    "--delete-submitted-files",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Delete submitted files",
+)
+@click.option(
+    "--delete-fileset",
+    default=False,
+    is_flag=True,
+    show_default=True,
+    help="Delete fileset",
+)
+@click.option(
+    "-e",
+    "--auth-env",
+    required=True,
+    type=str,
+    help="Name of environment in smaht-keys file",
+)
+def purge_fileset(
+    fileset_accessions, dry_run, delete_submitted_files, delete_fileset, auth_env
+):
+    """
+    Delete all files in a fileset, delete associated MetaWorkflowRuns (incl. files)
+    and remove the fileset from the portal. Use with caution!
+    """
+    smaht_key = get_auth_key(auth_env)
+    for fileset_accession in fileset_accessions:
+        print(f"Working on Fileset {fileset_accession}")
+        wrangler_utils.purge_fileset(
+            fileset_accession,
+            dry_run,
+            delete_submitted_files,
+            delete_fileset,
+            smaht_key,
+        )
+
+
 @cli.command()
 @click.help_option("--help", "-h")
 @click.option(
@@ -330,5 +431,34 @@ def set_property(identifier, property_key, property_value, auth_env):
     wrangler_utils.set_property(identifier, property_key, property_value, smaht_key)
 
 
+@cli.command()
+@click.help_option("--help", "-h")
+@click.option(
+    "-i",
+    "--identifier",
+    required=True,
+    type=str,
+    help="Item UUID",
+)
+@click.option(
+    "-p",
+    "--property",
+    required=True,
+    type=str,
+    help="Item property",
+)
+@click.option(
+    "-e",
+    "--auth-env",
+    required=True,
+    type=str,
+    help="Name of environment in smaht-keys file",
+)
+def remove_property(identifier, property, auth_env):
+    """Remove item property by uuid."""
+    smaht_key = get_auth_key(auth_env)
+    wrangler_utils.remove_property(identifier, property, smaht_key)
+
+
 if __name__ == "__main__":
     cli()

magma_smaht/constants.py

@@ -32,7 +32,6 @@
 LENGTH_REQUIRED = "length_required"
 LIBRARY_ID = "library_id"
 GENOME_REFERENCE_STAR = "genome_reference_star"
-IS_STRANDED = "is_stranded"
 STRANDEDNESS = "strandedness"
 
 # Schema fields
@@ -47,6 +46,7 @@
 FILE_SETS = "file_sets"
 META_WORFLOW_RUN = "MetaWorkflowRun"
 ACCESSION = "accession"
+DISPLAY_TITLE = "display_title"
 ALIASES = "aliases"
 UPLOADED = "uploaded"
 DELETED = "deleted"

magma_smaht/create_metawfr.py

@@ -59,7 +59,6 @@
     LENGTH_REQUIRED,
     LIBRARY_ID,
     GENOME_REFERENCE_STAR,
-    IS_STRANDED,
     STRANDEDNESS,
     COMMON_FIELDS,
     UUID,
@@ -135,7 +134,6 @@ def mwfr_rnaseq_alignment(fileset_accession, sequence_length, smaht_key):
         strandedness_mapping = {FIRST_STRANDED: "rf", SECOND_STRANDED: "fr"}
         mwfr_input.extend(
             [
-                get_mwfr_parameter_input_arg(IS_STRANDED, "true"),
                 get_mwfr_parameter_input_arg(
                     STRANDEDNESS, strandedness_mapping[strand]
                 ),