Merge pull request #19 from UMCUGenetics/release/v2.0.2

Release/v2.0.2

Author: BasMonkey

BAF/IGV.nf

@@ -6,7 +6,7 @@ process IGV {
     shell = ['/bin/bash', '-eo', 'pipefail']
 
     input:
-        tuple(output_name, path(vcf_files), path(vcf_idx_files))
+        tuple(val(output_name), path(vcf_files), path(vcf_idx_files))
 
     output:
         path("${output_name}_baf.igv", emit: BAF_IGV_files)

CheckQC/check_qc.py

@@ -6,9 +6,10 @@
 from pathlib import Path
 import re
 import sys
+import warnings
 
 # Third party libraries alphabetic order of main package.
-from pandas import DataFrame, merge, read_csv
+from pandas import concat, DataFrame, merge, read_csv
 import yaml
 
 
@@ -72,7 +73,8 @@ def check_required_keys_metrics(qc_settings):
 def select_metrics(filename, input_files):
     metrics = list(filter(re.compile(f".*{filename}").match, input_files))
     if not metrics:
-        raise ValueError(f"No input file provided with filename pattern {filename}")
+        warnings.warn(UserWarning(f"No input file provided with filename pattern {filename}"))
+        return None
     return metrics
 
 
@@ -84,8 +86,10 @@ def get_columns_to_report(qc_report_cols, qc_metric_cols, qc_col):
         raise TypeError(f"{qc_report_cols} not string, list or '@all'")
     elif not_existing_cols:
         raise ValueError(f"Some column names provided as report_cols do not exists: {not_existing_cols}")
-    qc_report_cols = list(map(lambda x: x.replace(qc_col, "qc_value"), qc_report_cols))  # rename qc_col with qc_value
-    qc_report_cols.insert(0, "qc_title")  # add column qc_title
+    # Rename qc_col with qc_value
+    qc_report_cols = list(map(lambda x: x.replace(qc_col, "qc_value"), qc_report_cols))
+    # Add column qc_title
+    qc_report_cols.insert(0, "qc_title")
     return qc_report_cols
 
 
@@ -128,16 +132,17 @@ def add_failed_samples_metric(qc_metric, failed_rows, report_cols, sample_cols):
         # A single qc metric could have multiple sample columns
         # If a qc check fails for a 'multiple sample check', each individual sample is flagged as "failed"
         for sample_col in sample_cols:
-            qc_metric_out = qc_metric_out.append(
+            qc_metric_out = concat([
+                qc_metric_out,
                 (
                     qc_metric
                     .rename(columns={sample_col: "sample"})
                     .loc[failed_rows, qc_metric_out.columns.to_list()]
                     .groupby(["sample", "qc_check", "qc_status"], dropna=False)
-                    .agg(lambda val: ';'.join(val.astype(str)))  # Or .agg(lambda val: val.to_list())
+                    .agg(lambda val: ';'.join(val.astype(str)))
                     .reset_index()
                 )
-            )
+            ])
         # Drop failed samples current metric
         for sample_col in sample_cols:
             drop_index = qc_metric[qc_metric[sample_col].isin(set(failed_samples))].index
@@ -149,13 +154,14 @@ def add_failed_samples_metric(qc_metric, failed_rows, report_cols, sample_cols):
 def add_passed_samples_metric(qc_metric, qc_metric_out, sample_cols):
     # Add passed samples to output
     for sample_col in sample_cols:
-        qc_metric_out = qc_metric_out.append(
+        qc_metric_out = concat([
+            qc_metric_out,
             (
                 qc_metric
                 .rename(columns={sample_col: "sample"})
                 .loc[:, qc_metric_out.columns]
             )
-        )
+        ])
     # In case 'multiple sample qc check',
     # output could contain duplicate rows for individual samples used in multiple comparisons.
     return qc_metric_out.sort_values(by=["qc_check", "qc_status"]).drop_duplicates(keep="first")
@@ -169,36 +175,67 @@ def create_and_write_output(qc_output, output_path, output_prefix):
     qc_output.to_csv(output_path + output_prefix + "_summary.csv", index=False, header=True)
 
 
+def read_and_judge_metrics(qc, metrics):
+    for qc_file in metrics:
+        qc_metric_raw = read_csv(qc_file, comment=qc.get("comment", None), delimiter="\t", quotechar='"')
+        report_cols = get_columns_to_report(qc["report_cols"], qc_metric_raw.columns.to_list(), qc["qc_col"])
+        qc_metric_edit = add_and_rename_columns(qc_metric_raw, qc["title"], qc["qc_col"], qc["operator"], qc["threshold"])
+        failed_rows = get_failed_rows(qc_metric_edit, "qc_value", qc["operator"], qc["threshold"])
+        qc_metric_subset, qc_metric_judged = add_failed_samples_metric(
+            qc_metric_edit, failed_rows, report_cols, qc["sample_cols"]
+            )
+        qc_metric_judged = add_passed_samples_metric(qc_metric_subset, qc_metric_judged, qc["sample_cols"])
+        # Rename columns
+        suffix = f"_{qc['title'].lower()}"
+        qc_judged_renamed = qc_metric_judged.add_suffix(suffix).rename(columns={f"sample{suffix}": "sample"})
+        # Concatenate/merge metric output
+        if "output" not in locals():  # First time
+            output = qc_judged_renamed
+        else:
+            is_duplicate_sample = False
+            # Check for duplicate sampleIDs before merge.
+            if any(qc_judged_renamed["sample"].isin(output["sample"])):
+                is_duplicate_sample = True
+            output = merge(output, qc_judged_renamed, on=output.columns.tolist(), how="outer")
+            if is_duplicate_sample:
+                dup_sampleIDs = output[output['sample'].duplicated()]['sample'].to_list()
+                # Duplicate sampleIDs with different column values
+                if output["sample"].nunique() != output.shape[0]:
+                    # Warning to parse all qc values / samples.
+                    msg = f"Different qc values for duplicated sample IDs in input: {dup_sampleIDs}"
+                # Duplicate sampleIDs same column values
+                else:
+                    msg = f"Sample IDs occur multiple times in input: {dup_sampleIDs}"
+                warnings.warn(UserWarning(msg))
+    return output
+
+
 def check_qc(input_files, settings, output_path, output_prefix):
     # A single qc metric file can be used multiple times, by defining a metric section for each check in the qc settings.
     qc_settings = read_yaml(settings)
     check_required_keys_metrics(qc_settings)
-    for qc in qc_settings["metrics"]:
-        check_allowed_operators(qc["operator"])
-        metrics = select_metrics(qc["filename"], input_files)
-        for qc_file in metrics:
-            qc_metric_raw = read_csv(qc_file, comment=qc.get("comment", None), delimiter="\t", quotechar='"')
-            report_cols = get_columns_to_report(qc["report_cols"], qc_metric_raw.columns.to_list(), qc["qc_col"])
-            qc_metric_edit = add_and_rename_columns(qc_metric_raw, qc["title"], qc["qc_col"], qc["operator"], qc["threshold"])
-            failed_rows = get_failed_rows(qc_metric_edit, "qc_value", qc["operator"], qc["threshold"])
-            qc_metric_subset, qc_metric_judged = add_failed_samples_metric(
-                qc_metric_edit, failed_rows, report_cols, qc["sample_cols"]
-            )
-            qc_metric_judged = add_passed_samples_metric(qc_metric_subset, qc_metric_judged, qc["sample_cols"])
-            # Rename columns
-            suffix = f"_{qc['title'].lower()}"
-            qc_judged_renamed = qc_metric_judged.add_suffix(suffix).rename(columns={f"sample{suffix}": "sample"})
-            # Concatenate/merge metric output
-            try:
-                output = merge(output, qc_judged_renamed, on="sample", how="outer")
-            except NameError:  # First time:
-                output = merge(
-                    DataFrame(qc_metric_judged['sample'], columns=["sample"]),
-                    qc_judged_renamed,
-                    on="sample",
-                    how="outer"
-                )
-    create_and_write_output(output, output_path, output_prefix)
+    duplicated_sample_file = []
+    for qc_metric_settings in qc_settings["metrics"]:
+        check_allowed_operators(qc_metric_settings["operator"])
+        metric_files = select_metrics(qc_metric_settings["filename"], input_files)
+        if not metric_files:
+            continue
+        # Join multiple metrices files into single table
+        metric_out = read_and_judge_metrics(qc_metric_settings, metric_files)
+        if any(metric_out.duplicated(subset="sample")):
+            duplicated_sample_file.append(qc_metric_settings["filename"])
+            continue
+        if "merged_out" not in locals():
+            merged_out = metric_out
+        else:
+            # Join all metrics output to single table.
+            merged_out = merge(merged_out, metric_out, on="sample", how="outer")
+
+    if "metric_out" not in locals():
+        raise ValueError("No input files found to match any qc metric pattern.")
+    if duplicated_sample_file:
+        raise ValueError(f"Duplicated samples with different values found in files matching {duplicated_sample_file}.")
+    create_and_write_output(merged_out, output_path, output_prefix)
 
 
 if __name__ == "__main__":

CheckQC/requirements.txt

@@ -1,10 +1,12 @@
-pandas==1.3.3
+pandas==2.1.4
 pytest==6.2.5
 pytest-cov==3.0.0
+pytest-datadir==1.5.0
 pytest-datafiles==2.0
+pytest-dataset==0.3.2
 pytest-flake8==1.0.7
 pytest-mock==3.8.2
 pytest-raises==0.11
 pytest-reqs==0.2.1
 pytest-unordered==0.5.2
-PyYAML==5.4.1
+PyYAML==6.0.1

CheckQC/test_check_qc.py

@@ -21,6 +21,11 @@ def setup_test_path(tmp_path_factory):
     return test_tmp_path
 
 
+@pytest.fixture(scope="class")
+def mock_settings(class_mocker, dataset_class):
+    return class_mocker.patch("check_qc.read_yaml", return_value=dataset_class["settings_single_metric"])
+
+
 class TestNonEmptyExistingPath():
     def test_existing_dir(self, setup_test_path):
         file_or_dir = check_qc.non_empty_existing_path(setup_test_path)
@@ -99,11 +104,12 @@ def test_select_metric(self, input_files, expected):
         assert metrics == expected
 
     def test_no_match(self):
-        with pytest.raises(ValueError) as match_error:
-            check_qc.select_metrics("test", ["fake1.txt", "fake2.txt"])
-        error_val = str(match_error.value)
-        assert "No input file provided with filename pattern" in error_val
-        assert "test" in error_val
+        with pytest.warns(UserWarning) as match_warning:
+            return_val = check_qc.select_metrics("test", ["fake1.txt", "fake2.txt"])
+        warn_msg = match_warning[0].message.args[0]
+        assert "No input file provided with filename pattern" in warn_msg
+        assert "test" in warn_msg
+        assert not return_val
 
 
 class TestGetColumnsToReport():
@@ -252,3 +258,93 @@ def test_create_and_write_output(self, setup_test_path, exp_summary, qc_output):
         out = read_csv(expected_output)
         assert "qc_summary" in out.columns.to_list()
         assert out["qc_summary"].values == exp_summary
+
+
+class TestGetOutputMetrics():
+    @pytest.mark.parametrize("data_in,nr_rows", [
+        # single sample
+        (["sample1_fake_check.txt"], 1),
+        # multiple single samples
+        (["sample1_fake_check.txt", "sample2_fake_check.txt"], 2),
+        # single multi samples
+        (["240101_fake_check.txt"], 2),
+        # multiple multi samples
+        (["240101_fake_check.txt", "240102_fake_check.txt"], 4),
+        # multi and single sample
+        (["sample1_fake_check.txt", "240101_fake_check.txt"], 3),
+    ])
+    def test_input_ok(self, data_in, nr_rows, dataset, datadir):
+        datadir_files = [f"{datadir}/{filename}" for filename in data_in]
+        # input1 = datadir / "sample1_fake_check.txt"
+        df_output = check_qc.read_and_judge_metrics(dataset["settings_single_metric"]["metrics"][0], datadir_files)
+        assert not df_output.empty
+        observed_cols = df_output.columns.to_list()
+        assert df_output.shape[0] == nr_rows  # shape results in tuple with no. rows and no. cols
+        assert len(observed_cols) == 5
+        assert observed_cols == ['sample', 'qc_check_fc', 'qc_status_fc', 'qc_msg_fc', 'qc_value_fc']
+
+    @pytest.mark.parametrize("data_in,nr_rows,exp_warn_msg", [
+        # single sample duplicate
+        (["sample1_fake_check.txt"]*2, 1, "Sample IDs occur multiple times in input:"),
+        # single multi samples duplicate
+        (["240101_fake_check.txt"]*2, 2, "Sample IDs occur multiple times in input:"),
+        # multiple multi samples, duplicate samples
+        (["240101_fake_check.txt", "240101_v2_fake_check.txt"], 4, "Different qc values for duplicated sample IDs in input:"),
+    ])
+    def test_input_warn(self, data_in, nr_rows, exp_warn_msg, dataset, datadir):
+        datadir_files = [f"{datadir}/{filename}" for filename in data_in]
+        # input1 = datadir / "sample1_fake_check.txt"
+        with pytest.warns(UserWarning) as match_warning:
+            df_output = check_qc.read_and_judge_metrics(dataset["settings_single_metric"]["metrics"][0], datadir_files)
+        warn_msg = match_warning[0].message.args[0]
+        assert exp_warn_msg in warn_msg
+        assert not df_output.empty
+        observed_cols = df_output.columns.to_list()
+        assert df_output.shape[0] == nr_rows  # Shape: tuple with no. rows and no. cols
+        assert len(observed_cols) == 5
+        assert observed_cols == ['sample', 'qc_check_fc', 'qc_status_fc', 'qc_msg_fc', 'qc_value_fc']
+
+
+class TestCheckQc():
+    @pytest.mark.parametrize("settings,data_in,exp_shape", [
+        # single metric, single sample input
+        ("settings_single_metric", ["sample1_fake_check.txt"], (1, 5)),
+        # two metrics, single sample input
+        ("settings_two_metrics", ["sample1_fake_check.txt"], (1, 9)),
+        # single metric, multiple samples input
+        ("settings_single_metric", ["240101_fake_check.txt"], (2, 5)),
+        ("settings_single_metric", ["240101_fake_check.txt", "240102_fake_check.txt"], (4, 5)),
+        # two metrics, multiple sample input
+        ("settings_two_metrics", ["240101_fake_check.txt", "240102_fake_check.txt"], (4, 9)),
+        # two metric, multi and single sample input
+        ("settings_two_metrics", ["sample1_fake_check.txt", "240101_fake_check.txt"], (3, 9)),
+    ])
+    def test_ok(self, settings, data_in, exp_shape, datadir, dataset, mocker, ):
+        datadir_files = [f"{datadir}/{filename}" for filename in data_in]
+        mocker.patch("check_qc.read_yaml", return_value=dataset[settings])
+        mock_write_output = mocker.patch("check_qc.create_and_write_output")
+        check_qc.check_qc(input_files=datadir_files, settings="", output_path="", output_prefix="")
+        mock_write_output.assert_called_once()
+        # Shape: tuple with no. rows and no. cols
+        assert mock_write_output.call_args[0][0].shape == exp_shape
+        mock_write_output.reset_mock()
+
+    def test_no_match_input_error(self, mocker, mock_settings):
+        mock_select_metrics = mocker.patch("check_qc.select_metrics", return_value=None)
+        mock_get_output = mocker.patch("check_qc.read_and_judge_metrics")
+        with pytest.raises(ValueError) as no_match_error:
+            check_qc.check_qc(input_files=[], settings="", output_path="", output_prefix="")
+        mock_select_metrics.assert_called_once()
+        assert not mock_get_output.called
+        assert "No input files found to match any qc metric pattern." == str(no_match_error.value)
+        mock_settings.reset_mock()
+
+    def test_duplicate_samples_error(self, datadir, mocker, mock_settings):
+        mock_pandas_merge = mocker.patch("pandas.merge")
+        with pytest.raises(ValueError) as duplicate_error:
+            check_qc.check_qc(input_files=[f"{datadir}/240101_fake_check.txt", f"{datadir}/240101_v2_fake_check.txt"],
+                              settings="", output_path="", output_prefix="")
+        assert "Duplicated samples with different values found in files matching" in str(duplicate_error.value)
+        assert "fake_check.txt" in str(duplicate_error.value)
+        assert not mock_pandas_merge.called
+        mock_settings.reset_mock()

CheckQC/test_check_qc/240101_fake_check.txt

@@ -0,0 +1,3 @@
+sample_id	value
+sample3	1
+sample4	1

CheckQC/test_check_qc/240101_v2_fake_check.txt

@@ -0,0 +1,3 @@
+sample_id	value
+sample3	0
+sample4	0

CheckQC/test_check_qc/240102_fake_check.txt

@@ -0,0 +1,3 @@
+sample_id	value
+sample5	1
+sample6	1

CheckQC/test_check_qc/sample1_fake_check.txt

@@ -0,0 +1,2 @@
+sample_id	value
+sample1	1

CheckQC/test_check_qc/sample2_fake_check.txt

@@ -0,0 +1,2 @@
+sample_id	value
+sample2	1

CheckQC/test_check_qc/settings_single_metric.yaml

@@ -0,0 +1,8 @@
+metrics:
+  - filename: ".*fake_check.txt$"
+    qc_col: "value"
+    threshold: 0
+    operator: ">"
+    report_cols: ["sample_id", "value"]
+    sample_cols: ["sample_id"]
+    title: "FC"

CheckQC/test_check_qc/settings_two_metrics.yaml

@@ -0,0 +1,15 @@
+metrics:
+  - filename: ".*fake_check.txt$"
+    qc_col: "value"
+    threshold: 0
+    operator: ">"
+    report_cols: ["sample_id", "value"]
+    sample_cols: ["sample_id"]
+    title: "FC"
+  - filename: ".*fake_check.txt$"
+    qc_col: "value"
+    threshold: 0
+    operator: ">"
+    report_cols: ["sample_id", "value"]
+    sample_cols: ["sample_id"]
+    title: "FC2"

ClarityEpp/SampleIndications.nf

@@ -10,7 +10,7 @@ process SampleIndications {
         val(sample_id)
 
     output:
-        tuple(sample_id, stdout)
+        tuple(val(sample_id), stdout)
 
     script:
         """

ExomeDepth/CallCNV.nf

@@ -6,7 +6,7 @@ process CallCNV {
     shell = ['/bin/bash', '-eo', 'pipefail']
 
     input:
-        tuple(analysis_id, sample_id, path(bam_file), path(bai_file))
+        tuple(val(analysis_id), val(sample_id), path(bam_file), path(bai_file))
 
     output:
         path("*.log", emit: ED_log)

ExomeDepth/GetRefset.nf

@@ -7,10 +7,10 @@ process GetRefset {
     cache = false
 
     input:
-        tuple(sample_id, path(bam_file))
+        tuple(val(sample_id), path(bam_file))
 
     output:
-        tuple(sample_id, stdout)
+        tuple(val(sample_id), stdout)
 
     script:
         """