multi-stage decontamination. Update containers and comments

Pathogen-Genomics-Cymru · Aug 23, 2024 · 1699cb0 · 1699cb0
1 parent db40743
commit 1699cb0
Show file tree

Hide file tree

Showing 15 changed files with 829 additions and 34 deletions.
diff --git a/config/containers.config b/config/containers.config
@@ -1,15 +1,15 @@
 process { 
 
     withLabel:getversion {
-        container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.8"
+        container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.9-alpha"
     }
 
     withLabel:preprocessing {
-        container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.8.1"
+        container = "quay.io/pathogen-genomics-cymru/preprocessing:0.9.9-alpha"
     }
 
     withLabel:tbprofiler {
-        container = "quay.io/pathogen-genomics-cymru/tbprofiler:0.9.8.1"
+        container = "quay.io/pathogen-genomics-cymru/tbprofiler:0.9.9-alpha"
     }
 
     withName:downloadContamGenomes {
@@ -25,10 +25,10 @@ process {
     }
 
     withLabel:clockwork {
-        container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.8"
+        container = "quay.io/pathogen-genomics-cymru/clockwork:0.9.9-alpha"
     }
 
     withLabel:vcfpredict {
-        container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.8"
+        container = "quay.io/pathogen-genomics-cymru/vcfpredict:0.9.9-alpha"
     }
  }
diff --git a/docker/Dockerfile.clockwork-0.9.9-alpha b/docker/Dockerfile.clockwork-0.9.9-alpha
@@ -0,0 +1,119 @@
+FROM debian:buster
+
+LABEL maintainer="[email protected]" \
+about.summary="container for the clockwork workflow"
+
+
+ENV samtools_version=1.12 \
+htslib_version=1.12 \
+bcftools_version=1.12 \
+minimap2_version=2.17 \
+picard_version=2.18.16 \
+gramtools_version=8af53f6c8c0d72ef95223e89ab82119b717044f2 \
+vt_version=2187ff6347086e38f71bd9f8ca622cd7dcfbb40c \
+minos_version=0.11.0 \
+cortex_version=3a235272e4e0121be64527f01e73f9e066d378d3 \
+vcftools_version=0.1.15 \
+mccortex_version=97aba198d632ee98ac1aa496db33d1a7a8cb7e51 \
+stampy_version=1.0.32r3761 \
+python_version=3.6.5 \
+clockwork_version=2364dec4cbf25c844575e19e8fe0a319d10721b5
+
+ENV PACKAGES="procps curl git build-essential wget zlib1g-dev pkg-config jq r-base-core rsync autoconf libncurses-dev libbz2-dev liblzma-dev libcurl4-openssl-dev cmake tabix libvcflib-tools libssl-dev software-properties-common perl locales locales-all" \
+PYTHON="python2.7 python-dev"
+
+COPY bin/ /opt/bin/
+ENV PATH=/opt/bin:$PATH
+
+
+RUN apt-get update \
+&& apt-get install -y $PACKAGES $PYTHON \
+&& curl -fsSL https://www.python.org/ftp/python/${python_version}/Python-${python_version}.tgz | tar -xz \
+&& cd Python-${python_version} \
+&& ./configure --enable-optimizations \
+&& make altinstall \
+&& cd .. \
+&& ln -s /usr/local/bin/python3.6 /usr/local/bin/python3 \
+&& ln -s /usr/local/bin/pip3.6 /usr/local/bin/pip3 \
+&& pip3 install --upgrade pip \
+&& pip3 install 'cluster_vcf_records==0.13.1' pysam setuptools  awscli \
+&& apt-get update && apt-get install -y openjdk-11-jdk
+
+RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \
+&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \
+&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \
+&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \
+&& rm -r samtools-${samtools_version} \
+&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \
+&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \
+&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \
+&& rm -r bcftools-${bcftools_version}
+
+
+RUN curl -fsSL minimap2-${minimap2_version}.tar.gz https://github.com/lh3/minimap2/archive/v${minimap2_version}.tar.gz | tar -xz \
+&& cd minimap2-${minimap2_version} \
+&& make \
+&& chmod +x minimap2 \
+&& mv minimap2 /usr/local/bin \
+&& cd .. \
+&& rm -r minimap2-${minimap2_version} \
+&& wget https://github.com/broadinstitute/picard/releases/download/${picard_version}/picard.jar -O /usr/local/bin/picard.jar
+
+
+RUN git clone https://github.com/atks/vt.git vt-git \
+&& cd vt-git \
+&& git checkout ${vt_version} \
+&& make \
+&& cd .. \
+&& mv vt-git/vt /usr/local/bin \
+&& pip3 install tox "six>=1.14.0" \
+&& git clone https://github.com/iqbal-lab-org/gramtools \
+&& cd gramtools \
+&& git checkout ${gramtools_version} \
+&& pip3 install . \
+&& cd .. \
+&& pip3 install cython \
+&& pip3 install git+https://github.com/iqbal-lab-org/minos@v${minos_version}
+
+
+RUN git clone --recursive https://github.com/iqbal-lab/cortex.git \
+&& cd cortex \
+&& git checkout ${cortex_version} \
+&& bash install.sh \
+&& make NUM_COLS=1 cortex_var \
+&& make NUM_COLS=2 cortex_var \
+&& cd .. \
+&& mkdir bioinf-tools \
+&& cd bioinf-tools \
+&& curl -fsSL http://www.well.ox.ac.uk/~gerton/software/Stampy/stampy-${stampy_version}.tgz | tar -xz \
+&& make -C stampy-* \
+&& cp -s stampy-*/stampy.py . \
+&& curl -fsSL https://github.com/vcftools/vcftools/releases/download/v${vcftools_version}/vcftools-${vcftools_version}.tar.gz | tar -xz \
+&& cd vcftools-${vcftools_version} \
+&& ./configure --prefix $PWD/install \
+&& make && make install \
+&& ln -s src/perl/ . \
+&& cd .. \
+&& git clone --recursive https://github.com/mcveanlab/mccortex \
+&& cd mccortex \
+&& git checkout ${mccortex_version} \
+&& make all \
+&& cd .. \
+&& cp -s mccortex/bin/mccortex31 . \
+&& cd .. \
+&& git clone https://github.com/iqbal-lab-org/clockwork \
+&& cd clockwork \
+&& git checkout ${clockwork_version} \
+&& cd python \
+&& pip3 install . \
+&& chmod +x scripts/clockwork
+
+ENV CLOCKWORK_CORTEX_DIR=/cortex \
+PATH=${PATH}:/clockwork/python/scripts \
+PICARD_JAR=/usr/local/bin/picard.jar
+
+ENV LC_ALL en_US.UTF-8 \
+LANG en_US.UTF-8 \
+LANGUAGE en_US.UTF-8
+
+
diff --git a/docker/Dockerfile.preprocessing-0.9.9-alpha b/docker/Dockerfile.preprocessing-0.9.9-alpha
@@ -0,0 +1,132 @@
+FROM ubuntu:focal
+
+LABEL maintainer="[email protected]" \
+about.summary="container for the preprocessing workflow"
+
+ENV samtools_version=1.12 \
+bcftools_version=1.12 \
+htslib_version=1.12 \
+bedtools_version=2.29.2 \
+bowtie2_version=2.4.2 \
+fastp_version=0.20.1 \
+fastqc_version=0.11.9 \
+fqtools_version=2.3 \
+kraken2_version=2.1.1 \
+afanc_version=0.10.2 \
+mykrobe_version=0.12.1 \
+bwa_version=0.7.17 \
+mash_version=2.3 \
+fastani_version=1.33
+
+
+ENV PACKAGES="procps curl git wget build-essential zlib1g-dev libncurses-dev libz-dev libbz2-dev liblzma-dev libcurl4-openssl-dev libgsl-dev rsync unzip ncbi-blast+ pigz jq libtbb-dev openjdk-11-jre-headless autoconf r-base-core locales locales-all" \
+PYTHON="python3 python3-pip python3-dev" \
+PYTHON_PACKAGES="biopython awscli boto3"
+
+ENV PATH=${PATH}:/usr/local/bin/mccortex/bin:/usr/local/bin/bwa-${bwa_version}:/opt/edirect \
+LD_LIBRARY_PATH=/usr/local/lib
+
+RUN export DEBIAN_FRONTEND="noninteractive"
+
+COPY bin/ /opt/bin/
+ENV PATH=/opt/bin:$PATH
+
+RUN apt-get update \
+&& DEBIAN_FRONTEND="noninteractive" apt-get install -y $PACKAGES $PYTHON \
+&& pip3 install --upgrade pip \
+&& pip3 install $PYTHON_PACKAGES \
+&& ln -s /usr/bin/python3 /usr/bin/python
+
+RUN curl -fsSL https://github.com/samtools/samtools/archive/${samtools_version}.tar.gz | tar -xz \
+&& curl -fsSL https://github.com/samtools/htslib/releases/download/${htslib_version}/htslib-${htslib_version}.tar.bz2 | tar -xj \
+&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} \
+&& make -C samtools-${samtools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \
+&& rm -r samtools-${samtools_version} \
+&& curl -fsSL https://github.com/samtools/bcftools/archive/refs/tags/${bcftools_version}.tar.gz | tar -xz \
+&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} \
+&& make -C bcftools-${bcftools_version} -j HTSDIR=../htslib-${htslib_version} prefix=/usr/local install \
+&& rm -r bcftools-${bcftools_version}
+
+RUN curl -fsSL https://github.com/alastair-droop/fqtools/archive/v${fqtools_version}.tar.gz | tar -xz \
+&& mv htslib-${htslib_version} fqtools-${fqtools_version} \
+&& cd fqtools-${fqtools_version} \
+&& mv htslib-${htslib_version} htslib \
+&& cd htslib \
+&& autoreconf -i \
+&& ./configure \
+&& make \
+&& make install \
+&& cd .. \
+&& make \
+&& mv bin/* /usr/local/bin \
+&& chmod +x /usr/local/bin/fqtools \
+&& cd .. \
+&& rm -r fqtools-${fqtools_version}
+
+RUN curl -fsSL https://github.com/arq5x/bedtools2/releases/download/v${bedtools_version}/bedtools-${bedtools_version}.tar.gz | tar -xz \
+&& make -C bedtools2 \
+&& mv bedtools2/bin/* /usr/local/bin \
+&& rm -r bedtools2
+
+RUN curl -fsSL https://sourceforge.net/projects/bowtie-bio/files/bowtie2/${bowtie2_version}/bowtie2-${bowtie2_version}-source.zip -o bowtie2-${bowtie2_version}-source.zip \
+&& unzip bowtie2-${bowtie2_version}-source.zip \
+&& make -C bowtie2-${bowtie2_version} prefix=/usr/local install \
+&& rm -r bowtie2-${bowtie2_version} \
+&& rm bowtie2-${bowtie2_version}-source.zip
+
+RUN curl -fsSL https://github.com/OpenGene/fastp/archive/v${fastp_version}.tar.gz | tar -xz \
+&& cd fastp-${fastp_version} \
+&& make \
+&& make install \
+&& cd .. \
+&& rm -r fastp-${fastp_version}
+
+RUN wget https://www.bioinformatics.babraham.ac.uk/projects/fastqc/fastqc_v${fastqc_version}.zip \
+&& unzip fastqc_v${fastqc_version}.zip \
+&& chmod +x FastQC/fastqc \
+&& mv FastQC/* /usr/local/bin \
+&& rm fastqc_v${fastqc_version}.zip \
+&& rm -r FastQC
+
+RUN curl -fsSL https://github.com/DerrickWood/kraken2/archive/v${kraken2_version}.tar.gz | tar -xz \
+&& cd kraken2-${kraken2_version} \
+&& ./install_kraken2.sh /usr/local/bin \
+&& cd ..
+
+RUN curl -fsSL https://github.com/ArthurVM/Afanc/archive/refs/tags/v${afanc_version}-alpha.tar.gz | tar -xz \
+&& cd Afanc-${afanc_version}-alpha \
+&& pip3 install ./ \
+&& cd .. \
+&& curl -fsSL "https://github.com/marbl/Mash/releases/download/v${mash_version}/mash-Linux64-v${mash_version}.tar" | tar -x \
+&& mv mash-Linux64-v${mash_version}/mash /usr/local/bin \
+&& rm -r mash-Linux* \
+&& wget https://github.com/ParBLiSS/FastANI/releases/download/v${fastani_version}/fastANI-Linux64-v${fastani_version}.zip \
+&& unzip fastANI-Linux64-v${fastani_version}.zip \
+&& mv fastANI /usr/local/bin
+
+RUN sh -c "$(curl -fsSL https://ftp.ncbi.nlm.nih.gov/entrez/entrezdirect/install-edirect.sh)" \
+&& mkdir -p /opt/edirect \
+&& mv /root/edirect/* /opt/edirect
+
+RUN git clone --recursive -b geno_kmer_count https://github.com/phelimb/mccortex \
+&& make -C mccortex \
+&& mv mccortex /usr/local/bin \
+&& curl -fsSL mykrobe-${mykrobe_version}.tar.gz https://github.com/Mykrobe-tools/mykrobe/archive/v${mykrobe_version}.tar.gz | tar -xz \
+&& cd mykrobe-${mykrobe_version} \
+&& pip3 install requests \
+&& pip3 install . \
+&& ln -s /usr/local/bin/mccortex/bin/mccortex31 /usr/local/lib/python3.8/dist-packages/mykrobe/cortex/mccortex31 \
+&& mykrobe panels update_metadata \
+&& mykrobe panels update_species all \
+&& cd ..
+
+RUN curl -fsSL https://github.com/lh3/bwa/archive/v${bwa_version}.tar.gz | tar -C /usr/local/bin -xz \
+&& make -C /usr/local/bin/bwa-${bwa_version} \
+&& chmod +x /usr/local/bin/bwa-${bwa_version}/bwa
+
+RUN unset DEBIAN_FRONTEND
+
+ENV LC_ALL en_US.UTF-8 \
+LANG en_US.UTF-8 \
+LANGUAGE en_US.UTF-8
+
diff --git a/docker/Dockerfile.tbprofiler-0.9.9-alpha b/docker/Dockerfile.tbprofiler-0.9.9-alpha
@@ -0,0 +1,62 @@
+FROM ubuntu:focal
+
+#copy the reference genome to pre-compute our index
+COPY resources/tuberculosis.fasta /data/tuberculosis.fasta
+
+ENV tbdb_version=a5e1d48 \
+    tbprofiler_version=6.2.1 \
+    TMPDIR="."
+
+#USER root
+WORKDIR /
+ENV TMPDIR="/data"
+ARG TBPROFILER_VER="6.2.1"
+
+# this version is the shortened commit hash on the `master` branch here https://github.com/jodyphelan/tbdb/
+# commits are found on https://github.com/jodyphelan/tbdb/commits/master
+# this was the latest commit as of 2024-05-21
+ARG TBDB_VER="a5e1d48"
+
+# LABEL instructions tag the image with metadata that might be important to the user
+LABEL base.image="ubuntu:focal"
+LABEL dockerfile.version="0.9.8.1"
+LABEL software="tbprofiler"
+LABEL software.version="${TBPROFILER_VER}"
+LABEL description="The pipeline aligns reads to the H37Rv reference using bowtie2, BWA or minimap2 and then calls variants using bcftools. These variants are then compared to a drug-resistance database."
+LABEL maintainer3="Tom Whalley"
+LABEL maintainer3.email="[email protected]"
+
+#set env for root prefix
+ENV MAMBA_ROOT_PREFIX="/opt/conda"
+
+RUN apt-get update && apt-get install -y apt-utils wget bzip2 curl
+RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest| tar -xvj bin/micromamba \
+    && touch /root/.bashrc \
+    && ./bin/micromamba shell init -s bash -p /opt/conda  \
+    && grep -v '[ -z "\$PS1" ] && return' /root/.bashrc  > /opt/conda/bashrc   # this line has been modified \
+    && apt-get clean autoremove --yes \
+    && rm -rf /var/lib/{apt,dpkg,cache,log}
+
+
+# install tb-profiler via bioconda; install into 'base' conda env
+RUN micromamba install --yes --name base --channel conda-forge --channel bioconda  \
+    tb-profiler=${TBPROFILER_VER}
+
+RUN micromamba install --yes --name base --channel conda-forge --channel bioconda gatk4 
+RUN micromamba install --yes --name base --channel conda-forge --channel bioconda samtools 
+RUN micromamba install --yes --name base --channel conda-forge jq
+RUN micromamba clean --all --yes
+
+# hardcode 'base' env bin into PATH, so conda env does not have to be "activated" at run time
+ENV PATH="/opt/conda/bin:${PATH}"
+
+
+# Version of database can be confirmed at /opt/conda/share/tbprofiler/tbdb.version.json
+# can also run 'tb-profiler list_db' to find the same version info
+# In 5.0.1 updating_tbdb does not work with tb-profiler update_tbdb --commit ${TBDB_VER}
+
+WORKDIR /data
+
+#wants full path to reference
+RUN tb-profiler update_tbdb --match_ref /data/tuberculosis.fasta --commit ${TBDB_VER}
+
diff --git a/docker/Dockerfile.vcfpredict-0.9.9-alpha b/docker/Dockerfile.vcfpredict-0.9.9-alpha
@@ -0,0 +1,27 @@
+FROM ubuntu:20.04
+
+LABEL maintainer="[email protected]" \
+about.summary="container for the vcf predict workflow"
+
+#add run-vcf to container
+COPY bin/ /opt/bin/
+ENV PATH=/opt/bin:$PATH
+
+ENV PACKAGES="procps curl wget git build-essential libhdf5-dev libffi-dev r-base-core jq" \
+PYTHON="python3 python3-pip python3-dev"
+
+ENV vcfmix_version=d4693344bf612780723e39ce27c8ae3868f95417
+
+#apt updates
+RUN apt-get update \
+&& DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata \
+&& apt-get install -y $PACKAGES $PYTHON \
+&& apt-get install -y python3-packaging \
+&& git clone https://github.com/JeremyWesthead/VCFMIX.git \
+&& cd VCFMIX \
+&& git checkout ${vcfmix_version} \
+&& pip3 install recursive_diff \
+&& pip3 install awscli \
+&& pip3 install . \
+&& cp -r data /usr/local/lib/python3.8/dist-packages \
+&& cd ..
diff --git a/modules/clockworkModules.nf b/modules/clockworkModules.nf
@@ -11,8 +11,8 @@ process getRefFromJSON {
     val(do_we_align)
     val(sample_name)
 
-    when:
-    do_we_align =~ /NOW\_ALIGN\_TO\_REF\_${sample_name}/
+    //when:
+    //do_we_align =~ /NOW\_ALIGN\_TO\_REF\_${sample_name}/
 
     output:
     stdout
@@ -47,8 +47,8 @@ process alignToRef {
     tuple val(sample_name), path(fq1), path(fq2), path(software_json), path(species_json), val(doWeAlign)
     path(reference_path)
 
-    //when:
-    //doWeAlign =~ /NOW\_ALIGN\_TO\_REF\_${sample_name}/
+    when:
+    doWeAlign =~ /NOW\_ALIGN\_TO\_REF\_${sample_name}/
 
     output:
     tuple val(sample_name), path("${sample_name}_report.json"), path("${sample_name}.bam"), path(reference_path), stdout, emit: alignToRef_bam