Merge branch 'master' into jdaw/add-indel-encoding

Author: tijyojwad

requirements.txt

@@ -14,6 +14,7 @@
 # limitations under the License.
 #
 
+bgzip==0.3.3
 h5py==2.10.0
 nemo-toolkit==0.10.1
 numpy==1.18.3

setup.py

@@ -21,7 +21,7 @@
 
 import os
 
-from setuptools import find_packages, setup
+from setuptools import setup, find_packages
 
 
 def get_verified_absolute_path(path):
@@ -83,8 +83,7 @@ def get_installation_requirments(file_path):
           get_verified_absolute_path(
               os.path.join(current_dir, 'requirements.txt')))
       ],
-      packages=find_packages(where=current_dir,
-                             include=['variantworks']),
+      packages=find_packages(where=current_dir, include=["variantworks*"]),
       python_requires='>=3.7',
       long_description='Python libraries and utilities for manipulating '
                        'genomics data',

tests/conftest.py

@@ -0,0 +1,51 @@
+#
+# Copyright 2020 NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Share fixtures across multiple test fies."""
+
+import bgzip
+import os
+import pytest
+import subprocess
+import tempfile
+
+
+@pytest.fixture(scope='function')
+def get_created_vcf_tabix_files():
+    """Fixture for creating compressed vcf file and corresponding tabix file from bytes string.
+
+    Returns:
+        A function which creates these files
+    """
+    def created_vcf_tabix_files(vcf_content):
+        _, tmp_input_path = tempfile.mkstemp(prefix='vw_test_file_', suffix='.vcf.gz')
+        with open(tmp_input_path, 'wb') as raw_fd:
+            with bgzip.BGZipWriter(raw_fd) as fh:
+                fh.write(vcf_content)
+        tabix_cmd_response = subprocess.run(['tabix', '-p', 'vcf', raw_fd.name])
+        tabix_cmd_response.check_returncode()
+        files_path = (raw_fd.name, raw_fd.name + ".tbi")
+        created_files.append(files_path)
+        return files_path
+    created_files = list()
+    yield created_vcf_tabix_files
+    # cleanup
+    try:
+        for entry in created_files:
+            os.remove(entry[0])
+            os.remove(entry[1])
+    except OSError as err:
+        raise type(err)('Can not remove input files: {}, {}'.format(*entry)) from err

tests/data/vcf_file_mock.py

@@ -1,11 +1,25 @@
-"""Contains mocked file object inputs for tests."""
+#
+# Copyright 2020 NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
-import io
+"""Contains mocked file object inputs for tests."""
 
 
 def mock_file_input():
     """Return a string stream of an unfiltered vcf file content."""
-    return io.StringIO("""##fileformat=VCFv4.2
+    return b"""##fileformat=VCFv4.2
 ##FILTER=<ID=PASS,Description="All filters passed">
 ##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
 ##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
@@ -28,15 +42,15 @@ def mock_file_input():
 1	240090	.	T	A	50	.	DP=22;AF=0.0454545	GT:GQ	1:50
 1	240147	.	C	T	50	.	DP=13;AF=0.692308	GT:GQ	1:50
 1	240154	.	T	C	50	.	DP=13;AF=0.0769231	GT:GQ	1:50
-""")
+"""
 
 
 def mock_invalid_file_input():
     """Returns a string stream of a vcf file content which is supposed to raise a RuntimeError.
 
     More than one called sample
     """
-    return io.StringIO("""##fileformat=VCFv4.2
+    return b"""##fileformat=VCFv4.2
 ##FILTER=<ID=PASS,Description="All filters passed">
 ##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
 ##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
@@ -45,12 +59,12 @@ def mock_invalid_file_input():
 #CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO	FORMAT	CALLED  CALLED2
 1	240147	.	C	T	50	.	DP=13;AF=0.692308	GT:GQ	1:50    1/1:50
 1	240154	.	T	C	50	.	DP=13;AF=0.0769231	GT:GQ	1:50    0/1:50
-""")
+"""
 
 
-def mock_vcf_file_reader_input(dummy_file_path):
+def mock_small_filtered_file_input():
     """Return string stream of small filtered vcf content."""
-    return io.StringIO("""##fileformat=VCFv4.2
+    return b"""##fileformat=VCFv4.2
 ##FILTER=<ID=PASS,Description="All filters passed">
 ##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype Quality">
 ##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">
@@ -60,4 +74,4 @@ def mock_vcf_file_reader_input(dummy_file_path):
 1	139861	.	T	A	50	.	DP=15;AF=0.0666667	GT:GQ	0/1:50
 1	139976	.	G	A	50	.	DP=35;AF=0.0185714	GT:GQ	1/1:50
 1	240147	.	C	T	50	.	DP=13;AF=0.692308	GT:GQ	0/1:50
-""")
+"""

tests/test_vcf_writer.py

@@ -22,45 +22,36 @@
 from variantworks.types import VariantZygosity
 from variantworks.result_writer import VCFResultWriter
 
-from data.vcf_file_mock import mock_vcf_file_reader_input
+from data.vcf_file_mock import mock_small_filtered_file_input
 
 
-class MockPyVCFReader:
-    original_pyvcf_reader_init_function = vcf.Reader.__init__
-
-    @staticmethod
-    def new_vcf_reader_init(self, *args, **kargs):
-        if 'filename' not in kargs:  # Reader must be initiated using `filename`
-            raise RuntimeError('Please use `filename` to initiate vcf.Reader')
-        MockPyVCFReader.original_pyvcf_reader_init_function(
-            self, mock_vcf_file_reader_input(kargs['filename']))
-
-
-def test_vcf_outputting(monkeypatch):
+def test_vcf_outputting(get_created_vcf_tabix_files):
     """Write inference output into vcf files
     """
-    first_vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path1.gz", bam="temp.bam", is_fp=False)
-    second_vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path2.gz", bam="temp.bam", is_fp=False)
-    with monkeypatch.context() as mp:
-        mp.setattr(vcf.Reader, "__init__", MockPyVCFReader.new_vcf_reader_init)
-        vcf_loader = VCFReader([first_vcf_bam_tuple, second_vcf_bam_tuple])
+    first_vcf_file_path, first_tabix_file_path = get_created_vcf_tabix_files(mock_small_filtered_file_input())
+    second_vcf_file_path, second_tabix_file_path = get_created_vcf_tabix_files(mock_small_filtered_file_input())
+    first_vcf_bam_tuple = VCFReader.VcfBamPath(vcf=first_vcf_file_path, bam=first_tabix_file_path, is_fp=False)
+    second_vcf_bam_tuple = VCFReader.VcfBamPath(vcf=second_vcf_file_path, bam=second_tabix_file_path, is_fp=False)
+    vcf_loader = VCFReader([first_vcf_bam_tuple, second_vcf_bam_tuple])
+
     inferred_results = [VariantZygosity.HOMOZYGOUS, VariantZygosity.HOMOZYGOUS, VariantZygosity.HETEROZYGOUS,
                         VariantZygosity.HETEROZYGOUS, VariantZygosity.HOMOZYGOUS, VariantZygosity.HETEROZYGOUS]
     assert (len(inferred_results) == len(vcf_loader))
-    with monkeypatch.context() as mp:
-        mp.setattr(vcf.Reader, "__init__", MockPyVCFReader.new_vcf_reader_init)
-        result_writer = VCFResultWriter(vcf_loader, inferred_results)
-        result_writer.write_output()
+
+    result_writer = VCFResultWriter(vcf_loader, inferred_results)
+    result_writer.write_output()
+
     # Validate output files format and make sure the outputted genotype for each record matches to the network output
+    first_output_file_name = \
+        '{}_{}.{}'.format("inferred", "".join(os.path.basename(first_vcf_file_path).split('.')[0:-2]), 'vcf')
+    second_output_file_name = \
+        '{}_{}.{}'.format("inferred", "".join(os.path.basename(second_vcf_file_path).split('.')[0:-2]), 'vcf')
     i = 0
-    for f in ['inferred_path1.vcf', 'inferred_path2.vcf']:
+    for f in [first_output_file_name, second_output_file_name]:
         vcf_reader = vcf.Reader(filename=os.path.join(
             result_writer.output_location, f))
         for record in vcf_reader:
-            assert(record.samples[0]['GT'] ==
-                   result_writer.zygosity_to_vcf_genotype[inferred_results[i]])
+            assert(record.samples[0]['GT'] == result_writer.zygosity_to_vcf_genotype[inferred_results[i]])
             i += 1
     assert (i == 6)
     # Clean up files

tests/test_vcfio.py

@@ -15,93 +15,59 @@
 #
 
 import pytest
-import vcf
 
 from variantworks.io.vcfio import VCFReader
 from variantworks.types import VariantZygosity, Variant
 
 from data.vcf_file_mock import mock_file_input, mock_invalid_file_input
 
 
-class MockPyVCFReader:
-    original_pyvcf_reader_init_function = vcf.Reader.__init__
-
-    @staticmethod
-    def new_vcf_reader_init(self, *args, **kargs):
-        MockPyVCFReader.original_pyvcf_reader_init_function(
-            self, mock_file_input())
-
-    @staticmethod
-    def new_bad_vcf_reader_init(self, *args, **kargs):
-        MockPyVCFReader.original_pyvcf_reader_init_function(
-            self, mock_invalid_file_input())
-
-    @staticmethod
-    def get_vcf(mp, vcf_bam_list):
-        with mp.context() as m:
-            # Mock vcf.Reader.__init__() return value
-            m.setattr(vcf.Reader, "__init__",
-                      MockPyVCFReader.new_vcf_reader_init)
-            vcf_loader = VCFReader(vcf_bam_list)
-        return vcf_loader
-
-    @staticmethod
-    def get_invalid_vcf(mp, vcf_bam_list):
-        with mp.context() as m:
-            # Mock vcf.Reader.__init__() return value
-            m.setattr(vcf.Reader, "__init__",
-                      MockPyVCFReader.new_bad_vcf_reader_init)
-            vcf_loader = VCFReader(vcf_bam_list)
-        return vcf_loader
-
-
-def test_vcf_loader_snps(monkeypatch):
+def test_vcf_loader_snps(get_created_vcf_tabix_files):
     """Get all variants from mocked file stream, filter SNPs, multi allele & multi samples
     """
-    vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path.gz", bam="temp.bam", is_fp=False)
-    vcf_loader = MockPyVCFReader.get_vcf(monkeypatch, [vcf_bam_tuple])
+    vcf_file_path, tabix_file_path = get_created_vcf_tabix_files(mock_file_input())
+    vcf_bam_tuple = VCFReader.VcfBamPath(vcf=vcf_file_path, bam=tabix_file_path, is_fp=False)
+    vcf_loader = VCFReader([vcf_bam_tuple])
     assert(len(vcf_loader) == 15)
 
 
-def test_vcf_fetch_variant(monkeypatch):
+def test_vcf_fetch_variant(get_created_vcf_tabix_files):
     """Get first variant from mocked VCF file stream.
     """
-    vcf_bam_tuple = VCFReader.VcfBamPath(vcf="/dummy/path.gz", bam="temp.bam", is_fp=False)
-    vcf_loader = MockPyVCFReader.get_vcf(monkeypatch, [vcf_bam_tuple])
+    vcf_file_path, tabix_file_path = get_created_vcf_tabix_files(mock_file_input())
+    vcf_bam_tuple = VCFReader.VcfBamPath(vcf=vcf_file_path, bam=tabix_file_path, is_fp=False)
+    vcf_loader = VCFReader([vcf_bam_tuple])
     try:
         assert (type(vcf_loader[0]) == Variant)
     except IndexError:
         pytest.fail("Can not retrieve first element from VCFReader")
 
 
-def test_vcf_load_fp(monkeypatch):
+def test_vcf_load_fp(get_created_vcf_tabix_files):
     """Get first variant from false positive mocked VCF file stream and check zygosity.
     """
-    vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path.gz", bam="temp.bam", is_fp=True)
-    vcf_loader = MockPyVCFReader.get_vcf(monkeypatch, [vcf_bam_tuple])
+    vcf_file_path, tabix_file_path = get_created_vcf_tabix_files(mock_file_input())
+    vcf_bam_tuple = VCFReader.VcfBamPath(vcf=vcf_file_path, bam=tabix_file_path, is_fp=True)
+    vcf_loader = VCFReader([vcf_bam_tuple])
     for v in vcf_loader:
         assert(v.zygosity == VariantZygosity.NO_VARIANT)
 
 
-def test_vcf_load_variant_from_multiple_files(monkeypatch):
+def test_vcf_load_variant_from_multiple_files(get_created_vcf_tabix_files):
     """Get variants from multiple mocked VCF files.
     """
-    first_vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path.gz", bam="temp.bam", is_fp=False)
-    second_vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path.gz", bam="temp.bam", is_fp=False)
-    vcf_loader = MockPyVCFReader.get_vcf(monkeypatch, [first_vcf_bam_tuple])
-    vcf_loader_2x = MockPyVCFReader.get_vcf(
-        monkeypatch, [first_vcf_bam_tuple, second_vcf_bam_tuple])
+    vcf_file_path, tabix_file_path = get_created_vcf_tabix_files(mock_file_input())
+    first_vcf_bam_tuple = VCFReader.VcfBamPath(vcf=vcf_file_path, bam=tabix_file_path, is_fp=False)
+    second_vcf_bam_tuple = VCFReader.VcfBamPath(vcf=vcf_file_path, bam=tabix_file_path, is_fp=False)
+    vcf_loader = VCFReader([first_vcf_bam_tuple])
+    vcf_loader_2x = VCFReader([first_vcf_bam_tuple, second_vcf_bam_tuple])
     assert (2 * len(vcf_loader) == len(vcf_loader_2x))
 
 
-def test_load_vcf_content_with_wrong_format(monkeypatch):
+def test_load_vcf_content_with_wrong_format(get_created_vcf_tabix_files):
     """ parse vcf file with wrong format
     """
-    vcf_bam_tuple = VCFReader.VcfBamPath(
-        vcf="/dummy/path.gz", bam="temp.bam", is_fp=False)
+    vcf_file_path, tabix_file_path = get_created_vcf_tabix_files(mock_invalid_file_input())
+    vcf_bam_tuple = VCFReader.VcfBamPath(vcf=vcf_file_path, bam=tabix_file_path, is_fp=False)
     with pytest.raises(RuntimeError):
-        MockPyVCFReader.get_invalid_vcf(monkeypatch, [vcf_bam_tuple])
+        VCFReader([vcf_bam_tuple])

variantworks/__init__.py

@@ -0,0 +1,17 @@
+#
+# Copyright 2020 NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Init file for variantworks."""

variantworks/io/__init__.py

@@ -0,0 +1,17 @@
+#
+# Copyright 2020 NVIDIA CORPORATION.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Init file for I/O module."""