diff --git a/Makefile b/Makefile index 90e48e4..7514f7b 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .DEFAULT_GOAL := pypitest test3: - python3 -m pytest -n 3 --disable-warnings --show-capture=no --cov=ngs_toolkit --cov-report xml tests/test_*.py --lf + python3 -m pytest -n 3 --disable-warnings --show-capture=no --cov=ngs_toolkit --cov-report xml ngs_toolkit/tests/test_*.py --lf test: test3 diff --git a/docs/source/changelog.rst b/docs/source/changelog.rst index 0157b91..09e3df5 100644 --- a/docs/source/changelog.rst +++ b/docs/source/changelog.rst @@ -7,6 +7,15 @@ Changelog ------------ +- **v0.16** (development, pre-1.0): + + - Fixed bug in general.get_genomic_context due to a bug in the timestamping workflow + - Distributing tests with library for more portable testing + +- **v0.15** (development, pre-1.0): + + - Shortlived, shipped with bug - please ignore + - **v0.14** (development, pre-1.0): - Add recording of analysis outputs under Analysis.output_files diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 6fea377..e6de8fb 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -7,7 +7,7 @@ Installation With pip ----------------------------- -``ngs_toolkit`` is available for Python 3 only. +``ngs_toolkit`` is available for Python 3 only. Is is tested in Python 3.6 and 3.7. To install, simply do: @@ -20,13 +20,6 @@ you might need to add a ``--user`` flag if not root or running in a virtual envi This will install all the Python dependencies needed too. See `here `_ a list of all Python dependencies used. -If you wish to install optional libraries that interface with R libraries, you can pass ``[rstats]`` to the following pip call: - -.. code-block:: bash - - pip install ngs-toolkit[rstats] - - To install the latest development version: .. code-block:: bash @@ -34,25 +27,6 @@ To install the latest development version: pip install git+https://github.com/afrendeiro/toolkit.git#egg=ngs-toolkit - -**Non-Python optional requirements** - - -``ngs_toolkit`` makes use of some non-Python dependencies. - -The following are required only for some data or analysis types: - - - `bedtools `_: required for some ATAC/ChIP-seq functions. It is underlying the Python interface library to bedtools (pybedtools) which can be installed without bedtools. - - `R `_ and some bioconductor libraries (optional): - - - `cqn `_: used for GC-content aware normalization of NGS data. - - `DESeq2 `_: used for differential testing of genes/regulatory elements. - - `Kent tools `_ (optional): the 'twoBitToFa' binary from UCSC's Kent bioinformatics toolkit is used to convert between the 2bit and FASTA formats. - -.. note:: - ``bedtools`` version should be below 2.24.0 (2.20.1 is used for testing) - - Using a conda environment ----------------------------- @@ -66,7 +40,7 @@ Setup the bioconda channel: conda config --add channels bioconda conda config --add channels conda-forge -Install the dependencies: +Install non-Python dependencies: .. code-block:: bash @@ -82,6 +56,41 @@ And then install the ``ngs-toolkit`` library with pip (available only through Py pip install ngs-toolkit +**Non-Python requirements** +----------------------------- + + +``ngs_toolkit`` makes use of some non-Python dependencies. + +The following are highly recommended only for some data or analysis types: + + - `bedtools `_: required for some ATAC/ChIP-seq functions. It is underlying the Python interface library to bedtools (pybedtools) which can be installed without bedtools. + - `R `_ and some bioconductor libraries (optional): + + - `cqn `_ (optional): used for GC-content aware normalization of NGS data. + - `DESeq2 `_ (optional): used for differential testing of genes/regulatory elements. + - `Kent tools `_ (optional): the 'twoBitToFa' binary from UCSC's Kent bioinformatics toolkit is used to convert between the 2bit and FASTA formats. + +.. note:: + ``bedtools`` version should be below 2.24.0 (2.20.1 is used for testing) + + +Testing +============================= + +To make sure everything is correctly configured, the user is encouraged to test the library prior to use. + +In order to do this, install testing requirements and simply run ``pytest``: + +.. code-block:: bash + + pip install ngs-toolkit[testing] + pytest --pyargs ngs_toolkit + + +Pytest will output summary results (`see for example `_) and further outputs can be seen in ``${TMPDIR}/pytest-of-${USER}/`` or ``/tmp/pytest-of-${USER}/`` if $TMPDIR is not defined. + + API usage ============================= diff --git a/docs/source/report.rst b/docs/source/report.rst index 9c54424..c8a6cab 100644 --- a/docs/source/report.rst +++ b/docs/source/report.rst @@ -3,10 +3,6 @@ Analysis reports .. _Report: - -Report -============================= - Each analysis object in the `ngs_toolkit` will by default record the outputs it produces (e.g. tables, figures). This allows the collection of all outputs in a standardized way and the generation of an HTML report. diff --git a/ngs_toolkit/__init__.py b/ngs_toolkit/__init__.py index e5e4c9c..cd6bbcb 100644 --- a/ngs_toolkit/__init__.py +++ b/ngs_toolkit/__init__.py @@ -1,5 +1,4 @@ from ._version import __version__ -import divvy as _divvy def setup_logger(level="INFO", logfile=None): @@ -194,7 +193,8 @@ def setup_timestamping(): pd.io.parsers.TextFileReader = read_csv_timestamped( pd.io.parsers.TextFileReader) pd.DataFrame.to_csv = to_csv_timestamped( - pd.DataFrame.to_csv) + pd.DataFrame.to_csv, + exclude_functions=["from_dataframe"]) # setup @@ -203,11 +203,6 @@ def setup_timestamping(): setup_graphic_preferences() setup_timestamping() -# reduce level of logging from divvy -# only for divvy <=0. -if "logging" in _divvy.__dict__.keys(): - _divvy.logging.getLogger("divvy").setLevel("ERROR") - # easier API from ngs_toolkit.analysis import Analysis diff --git a/ngs_toolkit/_version.py b/ngs_toolkit/_version.py index 9306ea7..e9ed7e5 100644 --- a/ngs_toolkit/_version.py +++ b/ngs_toolkit/_version.py @@ -1 +1 @@ -__version__ = "0.14" +__version__ = "0.16" diff --git a/tests/__init__.py b/ngs_toolkit/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to ngs_toolkit/tests/__init__.py diff --git a/tests/conftest.py b/ngs_toolkit/tests/conftest.py similarity index 100% rename from tests/conftest.py rename to ngs_toolkit/tests/conftest.py diff --git a/tests/data_generator.py b/ngs_toolkit/tests/data_generator.py similarity index 100% rename from tests/data_generator.py rename to ngs_toolkit/tests/data_generator.py diff --git a/tests/test_analysis.py b/ngs_toolkit/tests/test_analysis.py similarity index 100% rename from tests/test_analysis.py rename to ngs_toolkit/tests/test_analysis.py diff --git a/tests/test_atacseq_analysis.py b/ngs_toolkit/tests/test_atacseq_analysis.py similarity index 100% rename from tests/test_atacseq_analysis.py rename to ngs_toolkit/tests/test_atacseq_analysis.py diff --git a/tests/test_config.py b/ngs_toolkit/tests/test_config.py similarity index 100% rename from tests/test_config.py rename to ngs_toolkit/tests/test_config.py diff --git a/tests/test_decorators.py b/ngs_toolkit/tests/test_decorators.py similarity index 100% rename from tests/test_decorators.py rename to ngs_toolkit/tests/test_decorators.py diff --git a/tests/test_differential_analysis.py b/ngs_toolkit/tests/test_differential_analysis.py similarity index 100% rename from tests/test_differential_analysis.py rename to ngs_toolkit/tests/test_differential_analysis.py diff --git a/tests/test_differential_enrichment.py b/ngs_toolkit/tests/test_differential_enrichment.py similarity index 100% rename from tests/test_differential_enrichment.py rename to ngs_toolkit/tests/test_differential_enrichment.py diff --git a/tests/test_general.py b/ngs_toolkit/tests/test_general.py similarity index 100% rename from tests/test_general.py rename to ngs_toolkit/tests/test_general.py diff --git a/tests/test_install.py b/ngs_toolkit/tests/test_install.py similarity index 100% rename from tests/test_install.py rename to ngs_toolkit/tests/test_install.py diff --git a/tests/test_logger.py b/ngs_toolkit/tests/test_logger.py similarity index 100% rename from tests/test_logger.py rename to ngs_toolkit/tests/test_logger.py diff --git a/tests/test_plot_differential.py b/ngs_toolkit/tests/test_plot_differential.py similarity index 100% rename from tests/test_plot_differential.py rename to ngs_toolkit/tests/test_plot_differential.py diff --git a/tests/test_plot_differential_enrichment.py b/ngs_toolkit/tests/test_plot_differential_enrichment.py similarity index 100% rename from tests/test_plot_differential_enrichment.py rename to ngs_toolkit/tests/test_plot_differential_enrichment.py diff --git a/tests/test_project_manager.py b/ngs_toolkit/tests/test_project_manager.py similarity index 100% rename from tests/test_project_manager.py rename to ngs_toolkit/tests/test_project_manager.py diff --git a/tests/test_regression_tests.py b/ngs_toolkit/tests/test_regression_tests.py similarity index 100% rename from tests/test_regression_tests.py rename to ngs_toolkit/tests/test_regression_tests.py diff --git a/tests/test_report.py b/ngs_toolkit/tests/test_report.py similarity index 100% rename from tests/test_report.py rename to ngs_toolkit/tests/test_report.py diff --git a/tests/test_rnaseq_analysis.py b/ngs_toolkit/tests/test_rnaseq_analysis.py similarity index 100% rename from tests/test_rnaseq_analysis.py rename to ngs_toolkit/tests/test_rnaseq_analysis.py diff --git a/tests/test_sample_input_files.py b/ngs_toolkit/tests/test_sample_input_files.py similarity index 98% rename from tests/test_sample_input_files.py rename to ngs_toolkit/tests/test_sample_input_files.py index 4ee4cbe..36a91fa 100644 --- a/tests/test_sample_input_files.py +++ b/ngs_toolkit/tests/test_sample_input_files.py @@ -6,7 +6,7 @@ import pandas as pd import pytest -from tests.conftest import file_exists_and_not_empty +from .conftest import file_exists_and_not_empty from ngs_toolkit.utils import get_this_file_or_timestamped diff --git a/tests/test_unsupervised_analysis.py b/ngs_toolkit/tests/test_unsupervised_analysis.py similarity index 100% rename from tests/test_unsupervised_analysis.py rename to ngs_toolkit/tests/test_unsupervised_analysis.py diff --git a/ngs_toolkit/utils.py b/ngs_toolkit/utils.py index 7466d02..9a16f7b 100644 --- a/ngs_toolkit/utils.py +++ b/ngs_toolkit/utils.py @@ -221,6 +221,11 @@ def submit_job( import divvy from ngs_toolkit import _CONFIG, _LOGGER + # reduce level of logging from divvy + # only for divvy <=0. + if "logging" in divvy.__dict__.keys(): + divvy.logging.getLogger("divvy").setLevel("ERROR") + def count_jobs_running(check_cmd="squeue", sep="\n"): """ Count running jobs on a cluster by invoquing a command that lists the jobs. diff --git a/requirements/requirements.rstats.txt b/requirements/requirements.rstats.txt deleted file mode 100644 index 449e164..0000000 --- a/requirements/requirements.rstats.txt +++ /dev/null @@ -1,2 +0,0 @@ -rpy2 -tzlocal # undocumented requirement for rpy2 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 37c9478..ff87745 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -16,3 +16,5 @@ divvy>=0.4.0 attmap>=0.12.9 requests jinja2>=2.10.1 +rpy2 +tzlocal # undocumented requirement for rpy2 < 3.0 diff --git a/setup.py b/setup.py index cf25e91..f393ce1 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,6 @@ def parse_requirements(req_file): # Requirements requirements = parse_requirements("requirements/requirements.txt") test_requirements = parse_requirements("requirements/requirements.test.txt") -requirements_rstats = parse_requirements("requirements/requirements.rstats.txt") requirements_sc = parse_requirements("requirements/requirements.single_cell.txt") # Recipes @@ -84,16 +83,17 @@ def parse_requirements(req_file): "Source Code": "https://github.com/afrendeiro/toolkit", }, author=u"Andre Rendeiro", - author_email="afrendeiro@gmail.com", + author_email="andre.rendeiro@pm.me", license="GPL3", install_requires=requirements, tests_require=test_requirements, - extras_require={"rstats": requirements_rstats, "single_cell": requirements_sc}, + extras_require={ + "testing": test_requirements, + "single_cell": requirements_sc}, package_data={"ngs_toolkit": ["config/*.yaml", "templates/*.html"]}, data_files=[ "requirements/requirements.txt", "requirements/requirements.test.txt", - "requirements/requirements.rstats.txt", "requirements/requirements.single_cell.txt", ], **extra