diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index a9f1ccae9..c30294f91 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -56,7 +56,7 @@ jobs: if: ${{ steps.changes.outputs.utils == 'true' }} run: git tag delphi-utils/v${{ steps.utils.outputs.version }} - name: Publish tags - run: git push --tags + run: git push --tags outputs: indicators-version: ${{ steps.indicators.outputs.version }} utils-version: ${{ steps.utils.outputs.version }} @@ -76,10 +76,6 @@ jobs: uses: actions/setup-python@v2 with: python-version: 3.8 - - name: Install testing dependencies - run: | - python -m pip install --upgrade pip - pip install pylint pytest pydocstyle wheel twine - name: Install run: | make install diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 3e1ee9689..8f5a733ae 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -51,11 +51,7 @@ jobs: with: python-version: 3.8 cache: "pip" - cache-dependency-path: "setup.py" - - name: Install testing dependencies - run: | - python -m pip install --upgrade pip - pip install pylint pytest pydocstyle wheel + cache-dependency-path: "pyproject.toml" - name: Install run: | make install-ci diff --git a/_delphi_utils_python/.bumpversion.cfg b/_delphi_utils_python/.bumpversion.cfg index 722a91e30..af22c56eb 100644 --- a/_delphi_utils_python/.bumpversion.cfg +++ b/_delphi_utils_python/.bumpversion.cfg @@ -4,6 +4,6 @@ commit = True message = chore: bump delphi_utils to {new_version} tag = False -[bumpversion:file:setup.py] +[bumpversion:file:pyproject.toml] [bumpversion:file:delphi_utils/__init__.py] diff --git a/_delphi_utils_python/DEVELOP.md b/_delphi_utils_python/DEVELOP.md index 2407e29a8..53ffde93f 100644 --- a/_delphi_utils_python/DEVELOP.md +++ b/_delphi_utils_python/DEVELOP.md @@ -9,7 +9,7 @@ To install the module in your default version of Python, run the following from this directory: ``` -pip install . +pip install -e '.[dev]' ``` As described in each of the indicator code directories, you will want to install diff --git a/_delphi_utils_python/Makefile b/_delphi_utils_python/Makefile index 79d7f7943..6aa241e96 100644 --- a/_delphi_utils_python/Makefile +++ b/_delphi_utils_python/Makefile @@ -1,17 +1,17 @@ .PHONY = venv, lint, test, clean, release venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ - pip install -e . + pip install -e '.[dev]' install-ci: venv . env/bin/activate; \ - pip install wheel ; \ - pip install . + pip install '.[dev]' lint: . env/bin/activate; pylint delphi_utils --rcfile=../pyproject.toml @@ -30,4 +30,4 @@ clean: release: lint test . env/bin/activate ; \ - python setup.py sdist bdist_wheel + python -m build --sdist --wheel diff --git a/_delphi_utils_python/pyproject.toml b/_delphi_utils_python/pyproject.toml new file mode 100644 index 000000000..c72f4fb83 --- /dev/null +++ b/_delphi_utils_python/pyproject.toml @@ -0,0 +1,59 @@ +[build-system] +requires = ["setuptools>=70.1", "setuptools-scm>=8.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "delphi-utils" +version = "0.3.24" +description = "Shared Utility Functions for Indicators" +readme = "README.md" +requires-python = "== 3.8.*" +license = { text = "MIT License" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.8", + "License :: MIT", +] +dependencies = [ + "boto3", + "covidcast", + "cvxpy", + "epiweeks", + "gitpython", + "importlib_resources>=1.3", + "numpy", + "pandas>=1.1.0", + "requests", + "slackclient", + "scs<3.2.6", # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283 + "structlog", + "xlrd", # needed by Pandas to read Excel files +] + +[project.urls] +Homepage = "https://github.com/cmu-delphi/covidcast-indicators" + +[project.optional-dependencies] +dev = [ + "build[virtualenv]", + "darker[isort]~=2.1.1", + "pylint==2.8.3", + "pytest", + "pydocstyle", + "pytest-cov", + "mock", + "moto~=4.2.14", + "requests-mock", + "freezegun", + "twine" +] +flash = ["scipy"] + +[tool.setuptools.packages.find] +where = ["."] +include = ["delphi_utils"] +namespaces = true + +[tool.setuptools.package-data] +"delphi_utils.data" = ["20*/*.csv"] diff --git a/_delphi_utils_python/setup.py b/_delphi_utils_python/setup.py deleted file mode 100644 index 3dee89b53..000000000 --- a/_delphi_utils_python/setup.py +++ /dev/null @@ -1,48 +0,0 @@ -from setuptools import setup -from setuptools import find_packages - -with open("README.md", "r") as f: - long_description = f.read() - -required = [ - "boto3", - "covidcast", - "cvxpy", - "scs<3.2.6", # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283 - "darker[isort]~=2.1.1", - "epiweeks", - "freezegun", - "gitpython", - "importlib_resources>=1.3", - "mock", - "moto~=4.2.14", - "numpy", - "pandas>=1.1.0", - "pydocstyle", - "pylint==2.8.3", - "pytest-cov", - "pytest", - "requests-mock", - "slackclient", - "structlog", - "xlrd" -] - -setup( - name="delphi_utils", - version="0.3.24", - description="Shared Utility Functions for Indicators", - long_description=long_description, - long_description_content_type="text/markdown", - author="", - author_email="", - url="https://github.com/cmu-delphi/", - install_requires=required, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3.8", - ], - packages=find_packages(), - package_data={'': ['data/20*/*.csv']} -) diff --git a/_template_python/Makefile b/_template_python/Makefile index 390113eef..b34394efb 100644 --- a/_template_python/Makefile +++ b/_template_python/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/_template_python/pyproject.toml b/_template_python/pyproject.toml new file mode 100644 index 000000000..c29d9650e --- /dev/null +++ b/_template_python/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools", "setuptools-scm>=8.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "delphi_NAME" +version = "0.1.0" +description = "SHORT DESCRIPTION" +readme = "README.md" +requires-python = "== 3.8.*" +license = { text = "MIT License" } +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "Programming Language :: Python :: 3.8", + "License :: MIT", +] +dependencies = [ + "delphi-utils", + "numpy", + "pandas>=1.1.0", +] + +[project.urls] +Homepage = "https://github.com/cmu-delphi/covidcast-indicators" + +[project.optional-dependencies] +dev = [ + "darker[isort]~=2.1.1", + "pylint==2.8.3", + "pytest", + "pydocstyle", + "pytest-cov", + "mock", + "moto~=4.2.14", + "requests-mock", + "freezegun", +] + +[tool.setuptools.packages.find] +where = ["."] +namespaces = true diff --git a/_template_python/setup.py b/_template_python/setup.py deleted file mode 100644 index d7bc44078..000000000 --- a/_template_python/setup.py +++ /dev/null @@ -1,30 +0,0 @@ -from setuptools import setup -from setuptools import find_packages - -required = [ - "covidcast", - "darker[isort]~=2.1.1", - "delphi-utils", - "numpy", - "pandas", - "pydocstyle", - "pylint==2.8.3", - "pytest-cov", - "pytest", -] - -setup( - name="delphi_NAME", - version="0.1.0", - description="SHORT DESCRIPTION", - author="", - author_email="", - url="https://github.com/cmu-delphi/covidcast-indicators", - install_requires=required, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Intended Audience :: Developers", - "Programming Language :: Python :: 3.8", - ], - packages=find_packages(), -) diff --git a/backfill_corrections/Makefile b/backfill_corrections/Makefile index d479237be..ba0321734 100644 --- a/backfill_corrections/Makefile +++ b/backfill_corrections/Makefile @@ -56,7 +56,7 @@ install-python: fi python3 -m venv env source env/bin/activate && \ - python -m pip install --upgrade pip && \ + python -m pip install pip==23.0.1 && \ pip install wheel && \ pip install --timeout 1000 delphi_utils diff --git a/changehc/Makefile b/changehc/Makefile index 390113eef..b34394efb 100644 --- a/changehc/Makefile +++ b/changehc/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/changehc/setup.py b/changehc/setup.py index d95beb771..f386e9613 100644 --- a/changehc/setup.py +++ b/changehc/setup.py @@ -6,6 +6,7 @@ "covidcast", "darker[isort]~=2.1.1", "delphi-utils", + "mock", "moto~=4.2.14", "numpy", "pandas", diff --git a/claims_hosp/Makefile b/claims_hosp/Makefile index 390113eef..b34394efb 100644 --- a/claims_hosp/Makefile +++ b/claims_hosp/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/claims_hosp/setup.py b/claims_hosp/setup.py index 3b859c294..5eb23cabb 100644 --- a/claims_hosp/setup.py +++ b/claims_hosp/setup.py @@ -6,6 +6,7 @@ "darker[isort]~=2.1.1", "delphi-utils", "numpy", + "pydocstyle", "pandas", "paramiko", "pyarrow", diff --git a/doctor_visits/Makefile b/doctor_visits/Makefile index 390113eef..b34394efb 100644 --- a/doctor_visits/Makefile +++ b/doctor_visits/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/doctor_visits/setup.py b/doctor_visits/setup.py index 17d6fc9af..e224b5d8e 100644 --- a/doctor_visits/setup.py +++ b/doctor_visits/setup.py @@ -5,6 +5,7 @@ "darker[isort]~=2.1.1", "delphi-utils", "numpy", + "pydocstyle", "pandas", "paramiko", "pylint==2.8.3", diff --git a/google_symptoms/Makefile b/google_symptoms/Makefile index 6884278cf..62897971e 100644 --- a/google_symptoms/Makefile +++ b/google_symptoms/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/google_symptoms/setup.py b/google_symptoms/setup.py index ccba3c47a..56621942f 100644 --- a/google_symptoms/setup.py +++ b/google_symptoms/setup.py @@ -8,6 +8,7 @@ "freezegun", "mock", "numpy", + "pydocstyle", "pandas-gbq", "pandas", "pydocstyle", diff --git a/hhs_hosp/Makefile b/hhs_hosp/Makefile index 69529feb7..ca84eb7e6 100644 --- a/hhs_hosp/Makefile +++ b/hhs_hosp/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/jenkins/build-indicator.sh b/jenkins/build-indicator.sh index 3d2315741..3b8e3338a 100755 --- a/jenkins/build-indicator.sh +++ b/jenkins/build-indicator.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Jenkins build +# Jenkins build # set -eo pipefail @@ -14,7 +14,6 @@ cd "${WORKSPACE}/${local_indicator}" || exit # Set up venv python -m venv env source env/bin/activate -pip install pip==23.0.1 --retries 10 --timeout 20 -pip install numpy --retries 10 --timeout 20 +python -m pip install pip==23.0.1 --retries 10 --timeout 20 pip install ../_delphi_utils_python/. --retries 10 --timeout 20 [ ! -f setup.py ] || pip install . --retries 10 --timeout 20 diff --git a/nchs_mortality/Makefile b/nchs_mortality/Makefile index 390113eef..b34394efb 100644 --- a/nchs_mortality/Makefile +++ b/nchs_mortality/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/nchs_mortality/setup.py b/nchs_mortality/setup.py index 3fe354ba4..142db6f35 100644 --- a/nchs_mortality/setup.py +++ b/nchs_mortality/setup.py @@ -2,10 +2,12 @@ from setuptools import find_packages required = [ + "boto3", "darker[isort]~=2.1.1", "delphi-utils", "epiweeks", "freezegun", + "moto~=4.2.14", "numpy", "pandas", "pydocstyle", diff --git a/nssp/Makefile b/nssp/Makefile index 390113eef..b34394efb 100644 --- a/nssp/Makefile +++ b/nssp/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/nwss_wastewater/Makefile b/nwss_wastewater/Makefile index 390113eef..b34394efb 100644 --- a/nwss_wastewater/Makefile +++ b/nwss_wastewater/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/quidel_covidtest/Makefile b/quidel_covidtest/Makefile index 390113eef..b34394efb 100644 --- a/quidel_covidtest/Makefile +++ b/quidel_covidtest/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/quidel_covidtest/setup.py b/quidel_covidtest/setup.py index c2791930f..82c80832a 100644 --- a/quidel_covidtest/setup.py +++ b/quidel_covidtest/setup.py @@ -2,6 +2,7 @@ from setuptools import find_packages required = [ + "boto3", "covidcast", "darker[isort]~=2.1.1", "delphi-utils", @@ -14,7 +15,7 @@ "pylint==2.8.3", "pytest-cov", "pytest", - "xlrd==1.2.0", + "xlrd==1.2.0", # needed by Pandas to read Excel files ] setup( diff --git a/sir_complainsalot/Makefile b/sir_complainsalot/Makefile index 390113eef..b34394efb 100644 --- a/sir_complainsalot/Makefile +++ b/sir_complainsalot/Makefile @@ -2,17 +2,17 @@ dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1) venv: - python3.8 -m venv env + python3.8 -m venv env; \ + . env/bin/activate; \ + python -m pip install pip==23.0.1 install: venv . env/bin/activate; \ - pip install wheel ; \ pip install -e ../_delphi_utils_python ;\ pip install -e . install-ci: venv . env/bin/activate; \ - pip install wheel ; \ pip install ../_delphi_utils_python ;\ pip install . diff --git a/sir_complainsalot/setup.py b/sir_complainsalot/setup.py index 157c001b2..4346a1b1b 100644 --- a/sir_complainsalot/setup.py +++ b/sir_complainsalot/setup.py @@ -6,6 +6,7 @@ "darker[isort]~=2.1.1", "delphi-utils", "pandas", + "pydocstyle", "pylint==2.8.3", "pytest-cov", "pytest",