Merge pull request #77 from nextstrain/properly-handle-tsvs

ingest: properly handle TSVs with `csvtk`
nextstrain · Dec 9, 2024 · aa3dd48 · aa3dd48
2 parents f55c6c2 + 241b1bb
commit aa3dd48
Show file tree

Hide file tree

Showing 3 changed files with 4 additions and 6 deletions.
diff --git a/ingest/rules/curate.smk b/ingest/rules/curate.smk
@@ -126,6 +126,6 @@ rule subset_metadata:
         metadata_fields=",".join(config["curate"]["metadata_columns"]),
     shell:
         """
-        tsv-select -H -f {params.metadata_fields} \
+        csvtk cut -t -f {params.metadata_fields} \
             {input.metadata} > {output.subset_metadata}
         """
diff --git a/ingest/rules/fetch_from_ncbi.smk b/ingest/rules/fetch_from_ncbi.smk
@@ -97,11 +97,9 @@ rule format_ncbi_dataset_report:
             --fields {params.ncbi_datasets_fields:q} \
             --elide-header \
             | csvtk fix-quotes -Ht \
-            | csvtk add-header -t -l -n {params.ncbi_datasets_fields:q} \
+            | csvtk add-header -t -n {params.ncbi_datasets_fields:q} \
             | csvtk rename -t -f accession -n accession_version \
-            | csvtk -t mutate -f accession_version -n accession -p "^(.+?)\." \
-            | csvtk del-quotes -t \
-            | tsv-select -H -f accession --rest last \
+            | csvtk -t mutate -f accession_version -n accession -p "^(.+?)\." --at 1 \
             > {output.ncbi_dataset_tsv}
         """
 

diff --git a/ingest/rules/nextclade.smk b/ingest/rules/nextclade.smk
@@ -77,7 +77,7 @@ rule nextclade_metadata:
             --id-column {params.nextclade_id_field:q} \
             --field-map {params.nextclade_field_map:q} \
             --output-metadata - \
-        | tsv-select --header --fields {params.nextclade_fields:q} \
+        | csvtk cut -t --fields {params.nextclade_fields:q} \
         > {output.nextclade_metadata:q}
         """