From 5771db589fcdb1bd86cb0cad8920f9ef5f5361ac Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Tue, 10 Dec 2024 19:23:38 -0600 Subject: [PATCH 1/4] Use flit for package installs instead of pip in github ci. --- .github/workflows/ci.yml | 46 +++++++++++++++++++++++++++------------- pyproject.toml | 1 + 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 039e2b9fb..e5d72a455 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: ci on: push: - # A branch github-ci-updates can be created and used for ci + # A branch github-ci can be created and used for ci # experiments and tweaks. - branches: [ "develop", "master", "github-ci", "windows", "pr_413_continued_plugins" ] + branches: [ "develop", "master", "github-ci", "windows" ] pull_request: branches: [ "develop", "master" ] @@ -20,7 +20,7 @@ jobs: strategy: matrix: - python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ] + python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] steps: @@ -45,7 +45,12 @@ jobs: python-version: ${{ matrix.python_version }} cache: 'pip' # caching pip dependencies - - run: pip install -r requirements-dev.txt + - name: Install packages using flit + run: | + pip install flit + # --only-deps: Installs only the dependencies, not Lute itself + # --deps develop: both regular and optional dependencies. + flit install --only-deps --deps develop - name: Setup config run: | @@ -63,7 +68,9 @@ jobs: export MECAB_PATH=/lib/x86_64-linux-gnu/libmecab.so.2 pytest + # Skipping acceptance tests for 3.8, just to save some time. - name: Acceptance testing + if: matrix.python_version != '3.8' uses: nick-fields/retry@v3 with: max_attempts: 3 @@ -130,7 +137,10 @@ jobs: python-version: ${{ matrix.python_version }} cache: 'pip' # caching pip dependencies - - run: pip install -r requirements-dev.txt + - name: Install packages using flit + run: | + pip install flit + flit install --only-deps --deps develop - name: Setup config run: | @@ -192,14 +202,14 @@ jobs: - name: test all plugins run: | - for plugin in $(ls plugins); do - # Lute reqs, such as pytest. - pip install -r requirements-dev.txt - - # Lute itself, via toml, so that it can be found - # by each plugin's own "pip install ." - pip install . + # Lute reqs, such as pytest. + # This also installs Lute itself, + # so that it can be found + # by each plugin's own "pip install ." + pip install flit + flit install --deps develop + for plugin in $(ls plugins); do pushd plugins/$plugin pip install . # Note for future: some plugins may have extra reqs not covered by pip @@ -237,7 +247,9 @@ jobs: python-version: ${{ matrix.python_version }} cache: 'pip' # caching pip dependencies - - run: pip install -r requirements-dev.txt + - run: | + pip install flit + flit install --only-deps --deps develop - name: Lint run: inv lint @@ -261,7 +273,9 @@ jobs: with: python-version: 3.11 cache: 'pip' # caching pip dependencies - - run: pip install -r requirements-dev.txt + - run: | + pip install flit + flit install --only-deps --deps develop - name: Test docker build run: | docker build -f docker/Dockerfile --build-arg INSTALL_EVERYTHING=false -t lute3 . @@ -310,7 +324,9 @@ jobs: [System.IO.File]::WriteAllText("${{ github.workspace }}\lute\config\config.yml", $content) Get-Content -Path "${{ github.workspace }}\lute\config\config.yml" - - run: pip install -r requirements-dev.txt + - run: | + pip install flit + flit install --only-deps --deps develop # Can't get playwright to run reliably on Windows. # - name: Playwright install diff --git a/pyproject.toml b/pyproject.toml index 89f252261..e0968525a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,7 @@ dev = [ "pylint>=2.17.5,<3", "pytest-bdd>=7.0.0,<8", "pytest-splinter>=3.3.2,<4", + "playwright>=1.39.0,<2", "pre-commit>=3.5.0,<4", "black>=23.10.1,<24", ] From 1771f6acb22184412c2d196f1b33946504af74cf Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Tue, 10 Dec 2024 19:09:45 -0600 Subject: [PATCH 2/4] Skip lint for newer versions of python due to pylint/astroid bug --- tasks.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tasks.py b/tasks.py index 52cbcbf57..05c0ab41f 100644 --- a/tasks.py +++ b/tasks.py @@ -26,6 +26,15 @@ @task def lint(c): "Run pylint on lute/ and tests/." + # pylint is currently busted for some versions ... :-( + # ref https://github.com/pylint-dev/pylint/issues/10133 + print("Starting lint") + pyversion = int(sys.version_info.minor) + if pyversion > 9: + print("WARNING: SKIPPING LINT for python version > 9", flush=True) + print("Ref https://github.com/pylint-dev/pylint/issues/10133", flush=True) + return + # Formats: https://pylint.pycqa.org/en/latest/user_guide/usage/output.html msgfmt = [ "--ignore-patterns='zz_.*.py'", From 4a7f81d59d451ff82f6806240e7e2dacf7807981 Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Tue, 10 Dec 2024 19:12:37 -0600 Subject: [PATCH 3/4] Ignore all .venv* folders. --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index aa9e7c2e9..86c057218 100644 --- a/.gitignore +++ b/.gitignore @@ -109,8 +109,8 @@ celerybeat.pid *.sage.py # Environments -.env -.venv +.env* +.venv* env/ venv/ ENV/ From 94762c3ded6846adadece44f87a8a12bb8040784 Mon Sep 17 00:00:00 2001 From: Jeff Zohrab Date: Tue, 10 Dec 2024 19:19:41 -0600 Subject: [PATCH 4/4] Remove requirements*.txt files, handle deps with flit and pyproject.toml --- requirements-all.txt | 92 -------------------------------------------- requirements-dev.txt | 68 -------------------------------- requirements.txt | 48 ----------------------- 3 files changed, 208 deletions(-) delete mode 100644 requirements-all.txt delete mode 100644 requirements-dev.txt delete mode 100644 requirements.txt diff --git a/requirements-all.txt b/requirements-all.txt deleted file mode 100644 index dddfdd657..000000000 --- a/requirements-all.txt +++ /dev/null @@ -1,92 +0,0 @@ -ahocorapy==1.6.2 -astroid==2.15.6 -attrs==23.1.0 -beautifulsoup4==4.12.2 -black==23.10.1 -blinker==1.6.2 -certifi==2023.7.22 -cffi==1.16.0 -cfgv==3.4.0 -chardet==5.2.0 -charset-normalizer==3.3.1 -click==8.1.7 -colorama==0.4.6 -coverage==7.3.1 -dill==0.3.7 -distlib==0.3.7 -docutils==0.20.1 -exceptiongroup==1.1.3 -filelock==3.13.1 -Flask==2.3.3 -Flask-SQLAlchemy==3.1.1 -Flask-WTF==1.2.1 -flit==3.9.0 -flit_core==3.9.0 -future==1.0.0 -greenlet==3.0.0 -h11==0.14.0 -identify==2.5.31 -idna==3.4 -importlib-metadata==6.8.0 -iniconfig==2.0.0 -invoke==2.2.0 -isort==5.12.0 -itsdangerous==2.1.2 -jaconv==0.3.4 -Jinja2==3.1.2 -lazy-object-proxy==1.9.0 -Mako==1.2.4 -MarkupSafe==2.1.3 -mccabe==0.7.0 -mypy-extensions==1.0.0 -natto-py==1.0.1 -nodeenv==1.8.0 -openepub==0.0.8 -outcome==1.3.0.post0 -packaging==23.1 -parse==1.19.1 -parse-type==0.6.2 -pathspec==0.11.2 -pipdeptree==2.13.0 -platformdirs==3.10.0 -playwright==1.39.0 -pluggy==1.3.0 -pre-commit==3.5.0 -pycparser==2.21 -pyee==11.0.1 -pylint==2.17.5 -pypdf==3.17.4 -PySocks==1.7.1 -pytest==7.4.2 -pytest-base-url==2.0.0 -pytest-bdd==7.0.0 -pytest-playwright==0.4.3 -pytest-splinter==3.3.2 -python-slugify==8.0.1 -PyYAML==6.0.1 -requests==2.31.0 -selenium==4.14.0 -six==1.16.0 -sniffio==1.3.0 -sortedcontainers==2.4.0 -soupsieve==2.5 -splinter==0.19.0 -SQLAlchemy==2.0.21 -subtitle-parser==1.3.0 -text-unidecode==1.3 -toml==0.10.2 -tomli==2.0.1 -tomli_w==1.0.0 -tomlkit==0.12.1 -trio==0.22.2 -trio-websocket==0.11.1 -typing_extensions==4.8.0 -urllib3==1.26.18 -virtualenv==20.24.6 -waitress==2.1.2 -Werkzeug==2.3.7 -wrapt==1.15.0 -wsproto==1.2.0 -WTForms==3.0.1 -xmltodict==0.13.0 -zipp==3.17.0 diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index b3a8dd867..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,68 +0,0 @@ -# Developer and CI only requirements. - -# Include all prod requirements. --r requirements.txt - -# invoke -invoke==2.2.0 - -# pytest -astroid==2.15.6 -coverage==7.3.1 -dill==0.3.7 -iniconfig==2.0.0 -isort==5.12.0 -lazy-object-proxy==1.9.0 -mccabe==0.7.0 -packaging==23.1 -pluggy==1.3.0 -pylint==2.17.5 -pytest==7.4.2 -tomlkit==0.12.1 -wrapt==1.15.0 - -# pytest-splinter/selenium -attrs==23.1.0 -h11==0.14.0 -outcome==1.3.0.post0 -PySocks==1.7.1 -pytest-splinter==3.3.2 -selenium==4.14.0 -sniffio==1.3.0 -sortedcontainers==2.4.0 -splinter==0.19.0 -trio==0.22.2 -trio-websocket==0.11.1 -wsproto==1.2.0 - -# playwright -playwright==1.39.0 -pyee==11.0.1 -pytest-base-url==2.0.0 -pytest-playwright==0.4.3 -python-slugify==8.0.1 - -# pytest-bdd -Mako==1.2.4 -parse==1.19.1 -parse-type==0.6.2 -pytest-bdd==7.0.0 -six==1.16.0 - -# flit -flit==3.9.0 -flit_core==3.9.0 - -# black -black==23.10.1 -mypy-extensions==1.0.0 -pathspec==0.11.2 - -# pre-commit -cfgv==3.4.0 -distlib==0.3.7 -filelock==3.13.1 -identify==2.5.31 -nodeenv==1.8.0 -pre-commit==3.5.0 -virtualenv==20.24.6 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 10807e77f..000000000 --- a/requirements.txt +++ /dev/null @@ -1,48 +0,0 @@ -# IMPORTANT: if any new requirements are added, -# you MUST also change pyproject.toml. -# If you don't, the pypi release will not be correct. - -ahocorapy==1.6.2 -beautifulsoup4==4.12.2 -blinker==1.6.2 -certifi==2023.7.22 -cffi==1.16.0 -chardet==5.2.0 -charset-normalizer==3.3.1 -click==8.1.7 -colorama==0.4.6 -docutils==0.20.1 -exceptiongroup==1.1.3 -Flask==2.3.3 -Flask-SQLAlchemy==3.1.1 -Flask-WTF==1.2.1 -future==1.0.0 -greenlet==3.0.0 -idna==3.4 -importlib-metadata==6.8.0 -itsdangerous==2.1.2 -jaconv==0.3.4 -Jinja2==3.1.2 -MarkupSafe==2.1.3 -natto-py==1.0.1 -openepub==0.0.8 -pipdeptree==2.13.0 -platformdirs==3.10.0 -pycparser==2.21 -pypdf==3.17.4 -PyYAML==6.0.1 -requests==2.31.0 -soupsieve==2.5 -SQLAlchemy==2.0.21 -subtitle-parser==1.3.0 -text-unidecode==1.3 -toml==0.10.2 -tomli==2.0.1 -tomli_w==1.0.0 -typing_extensions==4.8.0 -urllib3==1.26.18 -waitress==2.1.2 -Werkzeug==2.3.7 -WTForms==3.0.1 -xmltodict==0.13.0 -zipp==3.17.0