Merge branch 'master' of https://github.com/cov-lineages/pangolin

Author: antunderwood

.dockerignore

@@ -0,0 +1,2 @@
+.git
+.dockerignore

.github/workflows/pangolin.yml

@@ -1,6 +1,8 @@
 name: pangolin
-
 on: [push, pull_request]
+defaults:
+  run:
+    shell: bash -l {0}
 
 jobs:
   run_pangolin_test:
@@ -18,12 +20,9 @@ jobs:
           channels: conda-forge,bioconda,defaults
           mamba-version: "*"
       - name: Install pangolin
-        shell: bash -l {0}
         run: pip install -e .
       - name: Check pangolin version
-        shell: bash -l {0}
         run: pangolin --version
       - name: Run pangolin with test data
-        shell: bash -l {0}
         run: pangolin pangolin/test/test_seqs.fasta 2>&1 | tee pangolin.log
-# add more tests here
+# add more tests here

.github/workflows/push_dockerhub.yml

@@ -0,0 +1,41 @@
+name: Docker push
+# This builds the docker image and pushes it to DockerHub
+# Runs on cov-lineages/pangolin repo releases
+# and push event to 'dev' branch (PR merges)
+on:
+  push:
+    branches:
+      - dev
+  release:
+    types: [published]
+
+jobs:
+  push_dockerhub:
+    name: Push new Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    # Only run for the cov-lineages/pangolin repo, for releases and merged PRs
+    if: ${{ github.repository == 'cov-lineages/pangolin' }}
+    env:
+      DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+      DOCKERHUB_PASS: ${{ secrets.DOCKERHUB_PASS }}
+    steps:
+      - name: Check out pipeline code
+        uses: actions/checkout@v2
+
+      - name: Build new docker image
+        run: docker build --no-cache . -t covlineages/pangolin:latest
+
+      - name: Push Docker image to DockerHub (dev)
+        if: ${{ github.event_name == 'push' }}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker tag covlineages/pangolin:latest covlineages/pangolin:dev
+          docker push covlineages/pangolin:dev
+
+      - name: Push Docker image to DockerHub (release)
+        if: ${{ github.event_name == 'release' }}
+        run: |
+          echo "$DOCKERHUB_PASS" | docker login -u "$DOCKERHUB_USERNAME" --password-stdin
+          docker push covlineages/pangolin:latest
+          docker tag covlineages/pangolin:latest covlineages/pangolin:${{ github.event.release.tag_name }}
+          docker push covlineages/pangolin:${{ github.event.release.tag_name }}

Dockerfile

@@ -0,0 +1,27 @@
+FROM continuumio/miniconda3:4.9.2-alpine
+LABEL version="2.1.2" \
+      description="Docker image for Pangolin"
+
+# Install git for pangolin
+RUN apk update && \
+    apk add git bash
+
+COPY environment.yml /environment.yml
+# Python 3.8.5 already installed along with recent version of pip
+# so remove Python and pip deps from environment.yml before installation
+RUN sed -i "$(grep -n python=3.7 /environment.yml | cut -f1 -d:)d" /environment.yml && \
+    sed -i "$(grep -n pip= /environment.yml | cut -f1 -d:)d" /environment.yml
+# Install the conda environment
+RUN conda env create --quiet -f /environment.yml && conda clean -a
+# Add conda installation dir to PATH (instead of doing 'conda activate')
+ENV PATH /opt/conda/envs/pangolin/bin:$PATH
+
+# Install Pangolin
+COPY . /pangolin/
+WORKDIR /pangolin/
+RUN pip install . && rm -rf /root/.cache/pip
+RUN pangolin --version &> /pangolin-version.txt
+
+# Dump the details of the installed packages to a file for posterity
+RUN conda env export --name pangolin > /pangolin.yml
+WORKDIR /tmp/

README.md

@@ -67,7 +67,12 @@ and you should see the versions of <strong>pangolin</strong>, and <strong>pangoL
 
 > Note: Even if you have previously installed <strong>pangolin</strong>, as it is being worked on intensively, we recommend you check for updates before running.
 
-To update:
+To update pangolin, pangoLEARN, and lineages automatically to the latest stable release:
+
+1. ``conda activate pangolin``
+2. ``pangolin --update``
+
+Alternatively, this can be done manually:
 
 1. ``conda activate pangolin``
 2. ``git pull`` \

environment.yml

@@ -16,5 +16,5 @@ dependencies:
     - pytools==2020.1
     - dendropy>=4.4.0
     - git+https://github.com/cov-ert/datafunk.git
-    - git+https://github.com/cov-lineages/pangoLEARN.git@2020-11-30_2
+    - git+https://github.com/cov-lineages/pangoLEARN.git
     - git+https://github.com/cov-lineages/lineages.git@2020-05-19-2 

pangolin/__init__.py

@@ -1,2 +1,2 @@
 _program = "pangolin"
-__version__ = "2.0.11"
+__version__ = "2.1.10"

pangolin/command.py

@@ -4,10 +4,16 @@
 import os.path
 import snakemake
 import sys
+from urllib import request
+from distutils.version import LooseVersion
+import subprocess
+import json
 from tempfile import gettempdir
 import tempfile
 import pprint
 import json
+import os
+import joblib
 import lineages
 import pangoLEARN
 
@@ -23,17 +29,18 @@
 
 def main(sysargs = sys.argv[1:]):
 
-    parser = argparse.ArgumentParser(prog = _program, 
-    description='pangolin: Phylogenetic Assignment of Named Global Outbreak LINeages', 
+    parser = argparse.ArgumentParser(prog = _program,
+    description='pangolin: Phylogenetic Assignment of Named Global Outbreak LINeages',
     usage='''pangolin <query> [options]''')
 
-    parser.add_argument('query', help='Query fasta file of sequences to analyse.')
+    parser.add_argument('query', nargs="*", help='Query fasta file of sequences to analyse.')
     parser.add_argument('-o','--outdir', action="store",help="Output directory. Default: current working directory")
     parser.add_argument('--outfile', action="store",help="Optional output file name. Default: lineage_report.csv")
     parser.add_argument('-d', '--data', action='store',help="Data directory minimally containing a fasta alignment and guide tree")
     parser.add_argument('-n', '--dry-run', action='store_true',help="Go through the motions but don't actually run")
     parser.add_argument('--tempdir',action="store",help="Specify where you want the temp stuff to go. Default: $TMPDIR")
     parser.add_argument("--no-temp",action="store_true",help="Output all intermediate files, for dev purposes.")
+    parser.add_argument('--decompress-model',action="store_true",dest="decompress",help="Permanently decompress the model file to save time running pangolin.")
     parser.add_argument('--max-ambig', action="store", default=0.5, type=float,help="Maximum proportion of Ns allowed for pangolin to attempt assignment. Default: 0.5",dest="maxambig")
     parser.add_argument('--min-length', action="store", default=10000, type=int,help="Minimum query length allowed for pangolin to attempt assignment. Default: 10000",dest="minlen")
     parser.add_argument('--panGUIlin', action='store_true',help="Run web-app version of pangolin",dest="panGUIlin")
@@ -45,12 +52,17 @@ def main(sysargs = sys.argv[1:]):
     parser.add_argument("-v","--version", action='version', version=f"pangolin {__version__}")
     parser.add_argument("-lv","--lineages-version", action='version', version=f"lineages {lineages.__version__}",help="show lineages's version number and exit")
     parser.add_argument("-pv","--pangoLEARN-version", action='version', version=f"pangoLEARN {pangoLEARN.__version__}",help="show pangoLEARN's version number and exit")
+    parser.add_argument("--update", action='store_true', default=False, help="Automatically updates to latest release of pangolin, pangoLEARN, and lineages then exits")
 
     if len(sysargs)<1:
         parser.print_help()
         sys.exit(-1)
     else:
         args = parser.parse_args(sysargs)
+    args = parser.parse_args()
+
+    if args.update:
+        update(__version__, lineages.__version__, pangoLEARN.__version__)
 
     if args.legacy:
         snakefile = os.path.join(thisdir, 'scripts','Snakefile')
@@ -63,13 +75,20 @@ def main(sysargs = sys.argv[1:]):
     else:
         print("Found the snakefile")
 
-    # find the query fasta
-    query = os.path.join(cwd, args.query)
-    if not os.path.exists(query):
-        sys.stderr.write('Error: cannot find query (input) fasta file at {}\nPlease enter your fasta sequence file and refer to pangolin usage at:\nhttps://github.com/hCoV-2019/pangolin#usage\n for detailed instructions\n'.format(query))
+    # to enable not having to pass a query if running update
+    # by allowing query to accept 0 to many arguments
+    if len(args.query) > 1:
+        print(f"Error: Too many query (input) fasta files supplied: {args.query}\nPlease supply one only")
+        parser.print_help()
         sys.exit(-1)
     else:
-        print(f"The query file is {query}")
+        # find the query fasta
+        query = os.path.join(cwd, args.query[0])
+        if not os.path.exists(query):
+            sys.stderr.write('Error: cannot find query (input) fasta file at {}\nPlease enter your fasta sequence file and refer to pangolin usage at:\nhttps://github.com/hCoV-2019/pangolin#usage\n for detailed instructions\n'.format(query))
+            sys.exit(-1)
+        else:
+            print(f"The query file is {query}")
 
         # default output dir
     outdir = ''
@@ -100,12 +119,14 @@ def main(sysargs = sys.argv[1:]):
     else:
         temporary_directory = tempfile.TemporaryDirectory(suffix=None, prefix=None, dir=None)
         tempdir = temporary_directory.name
-    
+
     if args.no_temp:
         print(f"--no-temp: All intermediate files will be written to {outdir}")
         tempdir = outdir
 
-    """ 
+
+
+    """
     QC steps:
     1) check no empty seqs
     2) check N content
@@ -127,7 +148,7 @@ def main(sysargs = sys.argv[1:]):
         else:
             num_N = str(record.seq).upper().count("N")
             prop_N = round((num_N)/len(record.seq), 2)
-            if prop_N > args.maxambig: 
+            if prop_N > args.maxambig:
                 record.description = record.description + f" fail=N_content:{prop_N}"
                 do_not_run.append(record)
                 print(f"{record.id}\thas an N content of {prop_N}")
@@ -147,7 +168,7 @@ def main(sysargs = sys.argv[1:]):
                     fw.write(f"{record.id},None,0,{pangoLEARN.__version__},fail,{reason}\n")
             print(f'Note: no query sequences have passed the qc\n')
             sys.exit(0)
-            
+
     post_qc_query = os.path.join(tempdir, 'query.post_qc.fasta')
     with open(post_qc_query,"w") as fw:
         SeqIO.write(run, fw, "fasta")
@@ -206,7 +227,7 @@ def main(sysargs = sys.argv[1:]):
                     elif fn.endswith(".csv") and fn.startswith("lineages"):
                         lineages_csv = os.path.join(r, fn)
 
-        
+
         if representative_aln=="" or guide_tree=="" or lineages_csv=="":
             print("""Check your environment, didn't find appropriate files from the lineages repo, please see https://cov-lineages.org/pangolin.html for installation instructions. \nTreefile must end with `.treefile`.\
 \nAlignment must be in `.fasta` format.\n Trained model must exist. \
@@ -242,6 +263,22 @@ def main(sysargs = sys.argv[1:]):
             print("""Check your environment, didn't find appropriate files from the pangoLEARN repo.\n Trained model must be installed, please see https://cov-lineages.org/pangolin.html for installation instructions.""")
             exit(1)
         else:
+            if args.decompress:
+                prev_size = os.path.getsize(trained_model)
+
+                print("Decompressing model and header files")
+                model = joblib.load(trained_model)
+                joblib.dump(model, trained_model, compress=0)
+                headers = joblib.load(header_file)
+                joblib.dump(headers, header_file, compress=0)
+
+                if os.path.getsize(trained_model) >= prev_size:
+                    print(f'Success! Decompressed the model file. Exiting\n')
+                    sys.exit(0)
+                else:
+                    print(f'Error: failed to decompress model. Exiting\n')
+                    sys.exit(0)
+
             print("\nData files found")
             print(f"Trained model:\t{trained_model}")
             print(f"Header file:\t{header_file}")
@@ -252,6 +289,18 @@ def main(sysargs = sys.argv[1:]):
     reference_fasta = pkg_resources.resource_filename('pangolin', 'data/reference.fasta')
     config["reference_fasta"] = reference_fasta
 
+    variants_file = pkg_resources.resource_filename('pangolin', 'data/config_b.1.1.7.csv')
+    config["b117_variants"] = variants_file
+
+    variants_file = pkg_resources.resource_filename('pangolin', 'data/config_b.1.351.csv')
+    config["b1351_variants"] = variants_file
+
+    variants_file = pkg_resources.resource_filename('pangolin', 'data/config_p.1.csv')
+    config["p1_variants"] = variants_file
+
+    variants_file = pkg_resources.resource_filename('pangolin', 'data/config_p.2.csv')
+    config["p2_variants"] = variants_file
+
     if args.write_tree:
         config["write_tree"]="True"
 
@@ -275,5 +324,61 @@ def main(sysargs = sys.argv[1:]):
 
     return 1
 
+
+def update(pangolin_version, lineages_version, pangoLEARN_version):
+    """
+    Using the github releases API check for the latest current release
+    of each pangolin, lineages, and pangoLEARN
+
+    Compare these to the currently running versions and if newer releases
+    exist update to them accordingly (or do nothing if current).
+    Afterwards, exit program safely with a 0 exit code.
+
+    pangolin_version: string containing the __version__ data for the currently
+                      running pangolin module
+    lineages_version: string containing the __version__ data for the imported
+                      lineages data module
+    pangoLEARN_version: string containing the __version__ data for the imported
+                       pangoLEARN data module
+    """
+    # flag if any element is update if everything is the latest release
+    # we want to just continue running
+    for dependency, version in [('pangolin', pangolin_version),
+                                ('pangoLEARN', pangoLEARN_version),
+                                ('lineages', lineages_version)]:
+        latest_release = request.urlopen(\
+            f"https://api.github.com/repos/cov-lineages/{dependency}/releases")
+        latest_release = json.load(latest_release)
+        latest_release = LooseVersion(latest_release[0]['tag_name'])
+
+        # to match the tag names add a v to the pangolin internal version
+        if dependency == 'pangolin':
+            version = "v" + version
+        # lineages doesn't need any changes for matching
+        # to match the tag names for pangoLEARN add data release
+        elif dependency == 'pangoLEARN':
+            version = version.replace(' ', ' data release ')
+
+        # convert to LooseVersion to have proper ordering of versions
+        # this prevents someone using the latest commit/HEAD from being
+        # downgraded to the last stable release
+        version = LooseVersion(version)
+
+        if version < latest_release:
+            subprocess.run([sys.executable, '-m', 'pip', 'install', '--upgrade',
+                            f"git+https://github.com/cov-lineages/{dependency}.git@{latest_release}"],
+                            check=True,
+                            stdout=subprocess.DEVNULL,
+                            stderr=subprocess.DEVNULL)
+            print(f"{dependency} updated to {latest_release}", file=sys.stderr)
+        elif version > latest_release:
+            print(f"{dependency} ({version}) is newer than latest stable "
+                  f"release ({latest_release}), not updating.", file=sys.stderr)
+        else:
+            print(f"{dependency} already latest release ({latest_release})",
+                    file=sys.stderr)
+
+    sys.exit(0)
+
 if __name__ == '__main__':
     main()

pangolin/data/config_b.1.1.7.csv

@@ -0,0 +1,17 @@
+aa:orf1ab:T1001I
+aa:orf1ab:A1708D
+aa:orf1ab:I2230T
+del:11288:9
+del:21765:6
+del:21991:3
+aa:S:N501Y
+aa:S:A570D
+aa:S:P681H
+aa:S:T716I
+aa:S:S982A
+aa:S:D1118H
+aa:Orf8:Q27*
+aa:Orf8:R52I
+aa:Orf8:Y73C
+aa:N:D3L
+aa:N:S235F

pangolin/data/config_b.1.351.csv

@@ -0,0 +1,9 @@
+aa:E:P71L
+aa:N:T205I
+aa:orf1a:K1655N
+aa:S:D80A
+aa:S:D215G
+aa:S:K417N
+aa:S:A701V
+aa:S:N501Y
+aa:S:E484K