build+ci: make local, gh, and Jenkins envs identical

.github/workflows/publish-release.yml

@@ -56,7 +56,7 @@ jobs:
         if: ${{ steps.changes.outputs.utils == 'true' }}
         run: git tag delphi-utils/v${{ steps.utils.outputs.version }}
       - name: Publish tags
-        run: git push --tags 
+        run: git push --tags
     outputs:
       indicators-version: ${{ steps.indicators.outputs.version }}
       utils-version: ${{ steps.utils.outputs.version }}
@@ -76,10 +76,6 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.8
-      - name: Install testing dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pylint pytest pydocstyle wheel twine
       - name: Install
         run: |
           make install

.github/workflows/python-ci.yml

@@ -51,11 +51,7 @@ jobs:
         with:
           python-version: 3.8
           cache: "pip"
-          cache-dependency-path: "setup.py"
-      - name: Install testing dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install pylint pytest pydocstyle wheel
+          cache-dependency-path: "pyproject.toml"
       - name: Install
         run: |
           make install-ci

_delphi_utils_python/.bumpversion.cfg

@@ -4,6 +4,6 @@ commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False
 
-[bumpversion:file:setup.py]
+[bumpversion:file:pyproject.toml]
 
 [bumpversion:file:delphi_utils/__init__.py]

_delphi_utils_python/DEVELOP.md

@@ -9,7 +9,7 @@ To install the module in your default version of Python, run the
 following from this directory:
 
 ```
-pip install .
+pip install -e '.[dev]'
 ```
 
 As described in each of the indicator code directories, you will want to install

_delphi_utils_python/Makefile

@@ -1,17 +1,17 @@
 .PHONY = venv, lint, test, clean, release
 
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
-	pip install -e .
+	pip install -e '.[dev]'
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
-	pip install .
+	pip install '.[dev]'
 
 lint:
 	. env/bin/activate; pylint delphi_utils --rcfile=../pyproject.toml
@@ -30,4 +30,4 @@ clean:
 
 release: lint test
 	. env/bin/activate ; \
-	python setup.py sdist bdist_wheel
+	python -m build --sdist --wheel

_delphi_utils_python/pyproject.toml

@@ -0,0 +1,59 @@
+[build-system]
+requires = ["setuptools>=70.1", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "delphi-utils"
+version = "0.3.24"
+description = "Shared Utility Functions for Indicators"
+readme = "README.md"
+requires-python = "== 3.8.*"
+license = { text = "MIT License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.8",
+    "License :: MIT",
+]
+dependencies = [
+    "boto3",
+    "covidcast",
+    "cvxpy",
+    "epiweeks",
+    "gitpython",
+    "importlib_resources>=1.3",
+    "numpy",
+    "pandas>=1.1.0",
+    "requests",
+    "slackclient",
+    "scs<3.2.6",                # TODO: remove this ; it is a cvxpy dependency, and the excluded version appears to break our jenkins build. see: https://github.com/cvxgrp/scs/issues/283
+    "structlog",
+    "xlrd",                     # needed by Pandas to read Excel files
+]
+
+[project.urls]
+Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
+
+[project.optional-dependencies]
+dev = [
+    "build[virtualenv]",
+    "darker[isort]~=2.1.1",
+    "pylint==2.8.3",
+    "pytest",
+    "pydocstyle",
+    "pytest-cov",
+    "mock",
+    "moto~=4.2.14",
+    "requests-mock",
+    "freezegun",
+    "twine"
+]
+flash = ["scipy"]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["delphi_utils"]
+namespaces = true
+
+[tool.setuptools.package-data]
+"delphi_utils.data" = ["20*/*.csv"]

_template_python/Makefile

@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 

_template_python/pyproject.toml

@@ -0,0 +1,42 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm>=8.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "delphi_NAME"
+version = "0.1.0"
+description = "SHORT DESCRIPTION"
+readme = "README.md"
+requires-python = "== 3.8.*"
+license = { text = "MIT License" }
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python :: 3.8",
+    "License :: MIT",
+]
+dependencies = [
+    "delphi-utils",
+    "numpy",
+    "pandas>=1.1.0",
+]
+
+[project.urls]
+Homepage = "https://github.com/cmu-delphi/covidcast-indicators"
+
+[project.optional-dependencies]
+dev = [
+    "darker[isort]~=2.1.1",
+    "pylint==2.8.3",
+    "pytest",
+    "pydocstyle",
+    "pytest-cov",
+    "mock",
+    "moto~=4.2.14",
+    "requests-mock",
+    "freezegun",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+namespaces = true

backfill_corrections/Makefile

@@ -56,7 +56,7 @@ install-python:
 	fi
 	python3 -m venv env
 	source env/bin/activate && \
-	python -m pip install --upgrade pip && \
+	python -m pip install pip==23.0.1 && \
 	pip install wheel && \
 	pip install --timeout 1000 delphi_utils
 

changehc/Makefile

@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 

changehc/setup.py

@@ -6,6 +6,7 @@
     "covidcast",
     "darker[isort]~=2.1.1",
     "delphi-utils",
+    "mock",
     "moto~=4.2.14",
     "numpy",
     "pandas",

claims_hosp/Makefile

@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 

claims_hosp/setup.py

@@ -6,6 +6,7 @@
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "numpy",
+    "pydocstyle",
     "pandas",
     "paramiko",
     "pyarrow",

doctor_visits/Makefile

@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .
 

doctor_visits/setup.py

@@ -5,6 +5,7 @@
     "darker[isort]~=2.1.1",
     "delphi-utils",
     "numpy",
+    "pydocstyle",
     "pandas",
     "paramiko",
     "pylint==2.8.3",

google_symptoms/Makefile

@@ -2,17 +2,17 @@
 
 dir = $(shell find ./delphi_* -name __init__.py | grep -o 'delphi_[_[:alnum:]]*' | head -1)
 venv:
-	python3.8 -m venv env
+	python3.8 -m venv env; \
+	. env/bin/activate; \
+	python -m pip install pip==23.0.1
 
 install: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install -e ../_delphi_utils_python ;\
 	pip install -e .
 
 install-ci: venv
 	. env/bin/activate; \
-	pip install wheel ; \
 	pip install ../_delphi_utils_python ;\
 	pip install .