Drop python <3.10, general maintenance updates (#257)

- Drop python 3.8, 3.9, as per https://numpy.org/neps/nep-0029-deprecation_policy.html
- Update and run pre-commits.
- Replace pkg_resource with importlib.metadata, fixes #256.
- Ignore autogenerated models file in mypy.
- Use ruff-format instead of black.

Author: ndevenish

.github/workflows/build_and_test.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.10", "3.11", "3.12"]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/linting.yml

@@ -13,4 +13,4 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Ruff Check
-      uses: jpetrucciani/ruff-check@33f3725e751a187bb666b7d7a7094d38a2df12db # 0.0.239
+      uses: jpetrucciani/ruff-check@96f59856544cbfba111afdf2f6985e21756b8e7e # 0.6.2

.pre-commit-config.yaml

@@ -1,7 +1,11 @@
+ci:
+  autoupdate_schedule: quarterly
+  skip: [poetry-lock]
+
 repos:
 # Syntax validation and some basic sanity checks
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.6.0
   hooks:
   - id: check-merge-conflict
   - id: check-ast
@@ -11,34 +15,27 @@ repos:
     args: ['--maxkb=200']
   - id: check-yaml
 
-- repo: https://github.com/charliermarsh/ruff-pre-commit
-  rev: v0.0.286
+- repo: https://github.com/astral-sh/ruff-pre-commit
+  rev: v0.6.2
   hooks:
     - id: ruff
       args: [--fix, --show-fixes, --exit-non-zero-on-fix]
-
-# Automatic source code formatting
-- repo: https://github.com/psf/black
-  rev: 23.7.0
-  hooks:
-  - id: black
-    args: [--safe, --quiet]
+    - id: ruff-format
 
 # Type checking
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v1.5.1
+  rev: v1.11.2
   hooks:
   - id: mypy
     files: 'src/.*\.py$'
     additional_dependencies:
     - types-requests
-    - types-pkg_resources
     - types-pyyaml
 
 - repo: https://github.com/python-poetry/poetry
   rev: 1.8.0
-  additional_dependencies: ["poetry-plugin-export"]
   hooks:
     - id: poetry-lock
     - id: poetry-export
-      args: [-o, requirements_dev.txt, --with=dev, --without-hashes]
+      additional_dependencies: [poetry-plugin-export]
+      args: [-o, requirements_dev.txt, --with=dev, --without-hashes]

poetry.lock

@@ -14,7 +14,7 @@ classifiers = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.10"
 pyyaml = "^6.0"
 graypy = "^2.0"
 marshmallow = "^3.19"
@@ -89,29 +89,32 @@ iris = "zocalo.configuration.plugin_slurm:Slurm"
 smtp = "zocalo.configuration.plugin_smtp:SMTP"
 storage = "zocalo.configuration.plugin_storage:Storage"
 
-[tool.isort]
-profile = "black"
-
 [tool.pytest.ini_options]
 required_plugins = "pytest-mock requests-mock"
 testpaths = ["tests"]
 
-[tool.ruff]
-line-length = 88
+[tool.ruff.lint]
 ignore = ["E501", "E741"]
 select = ["C4", "E", "F", "W", "I"]
 unfixable = ["F841"]
 
-[tool.ruff.isort]
+[tool.ruff.lint.isort]
 required-imports = ["from __future__ import annotations"]
 
+[tool.ruff.lint.per-file-ignores]
+"src/zocalo/util/slurm/models.py" = ["W"]
+
 [tool.mypy]
 mypy_path = "src"
 
 [[tool.mypy.overrides]]
 module = ["graypy", "graypy.handler"]
 ignore_missing_imports = true
 
+[[tool.mypy.overrides]]
+module = ["zocalo.util.slurm.models"]
+ignore_errors = true
+
 [tool.bumpversion]
 current_version = "1.0.0"
 commit = true

pyproject.toml

@@ -1,30 +1,30 @@
-annotated-types==0.7.0 ; python_version >= "3.8" and python_version < "4.0"
-bidict==0.23.1 ; python_version >= "3.8" and python_version < "4.0"
-certifi==2024.7.4 ; python_version >= "3.8" and python_version < "4.0"
-charset-normalizer==3.3.2 ; python_version >= "3.8" and python_version < "4.0"
-colorama==0.4.6 ; python_version >= "3.8" and python_version < "4.0" and sys_platform == "win32"
-coverage[toml]==7.6.0 ; python_version >= "3.8" and python_version < "4.0"
-docopt==0.6.2 ; python_version >= "3.8" and python_version < "4.0"
-exceptiongroup==1.2.2 ; python_version >= "3.8" and python_version < "3.11"
-graypy==2.1.0 ; python_version >= "3.8" and python_version < "4.0"
-idna==3.7 ; python_version >= "3.8" and python_version < "4.0"
-iniconfig==2.0.0 ; python_version >= "3.8" and python_version < "4.0"
-marshmallow==3.21.3 ; python_version >= "3.8" and python_version < "4.0"
-packaging==24.1 ; python_version >= "3.8" and python_version < "4.0"
-pika==1.3.2 ; python_version >= "3.8" and python_version < "4.0"
-pluggy==1.5.0 ; python_version >= "3.8" and python_version < "4.0"
-pydantic-core==2.20.1 ; python_version >= "3.8" and python_version < "4.0"
-pydantic==2.8.2 ; python_version >= "3.8" and python_version < "4.0"
-pytest-cov==5.0.0 ; python_version >= "3.8" and python_version < "4.0"
-pytest-mock==3.14.0 ; python_version >= "3.8" and python_version < "4.0"
-pytest==8.2.2 ; python_version >= "3.8" and python_version < "4.0"
-pyyaml==6.0.1 ; python_version >= "3.8" and python_version < "4.0"
-requests-mock==1.12.1 ; python_version >= "3.8" and python_version < "4.0"
-requests==2.32.3 ; python_version >= "3.8" and python_version < "4.0"
-setuptools==71.0.3 ; python_version >= "3.8" and python_version < "4.0"
-stomp-py==8.1.2 ; python_version >= "3.8" and python_version < "4.0"
-tomli==2.0.1 ; python_version >= "3.8" and python_full_version <= "3.11.0a6"
-typing-extensions==4.12.2 ; python_version >= "3.8" and python_version < "4.0"
-urllib3==2.2.2 ; python_version >= "3.8" and python_version < "4.0"
-websocket-client==1.8.0 ; python_version >= "3.8" and python_version < "4.0"
-workflows==2.27 ; python_version >= "3.8" and python_version < "4.0"
+annotated-types==0.7.0 ; python_version >= "3.10" and python_version < "4.0"
+bidict==0.23.1 ; python_version >= "3.10" and python_version < "4.0"
+certifi==2024.7.4 ; python_version >= "3.10" and python_version < "4.0"
+charset-normalizer==3.3.2 ; python_version >= "3.10" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.10" and python_version < "4.0" and sys_platform == "win32"
+coverage[toml]==7.6.1 ; python_version >= "3.10" and python_version < "4.0"
+docopt==0.6.2 ; python_version >= "3.10" and python_version < "4.0"
+exceptiongroup==1.2.2 ; python_version >= "3.10" and python_version < "3.11"
+graypy==2.1.0 ; python_version >= "3.10" and python_version < "4.0"
+idna==3.8 ; python_version >= "3.10" and python_version < "4.0"
+iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "4.0"
+marshmallow==3.22.0 ; python_version >= "3.10" and python_version < "4.0"
+packaging==24.1 ; python_version >= "3.10" and python_version < "4.0"
+pika==1.3.2 ; python_version >= "3.10" and python_version < "4.0"
+pluggy==1.5.0 ; python_version >= "3.10" and python_version < "4.0"
+pydantic-core==2.20.1 ; python_version >= "3.10" and python_version < "4.0"
+pydantic==2.8.2 ; python_version >= "3.10" and python_version < "4.0"
+pytest-cov==5.0.0 ; python_version >= "3.10" and python_version < "4.0"
+pytest-mock==3.14.0 ; python_version >= "3.10" and python_version < "4.0"
+pytest==8.3.2 ; python_version >= "3.10" and python_version < "4.0"
+pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "4.0"
+requests-mock==1.12.1 ; python_version >= "3.10" and python_version < "4.0"
+requests==2.32.3 ; python_version >= "3.10" and python_version < "4.0"
+setuptools==74.0.0 ; python_version >= "3.10" and python_version < "4.0"
+stomp-py==8.1.2 ; python_version >= "3.10" and python_version < "4.0"
+tomli==2.0.1 ; python_version >= "3.10" and python_full_version <= "3.11.0a6"
+typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "4.0"
+urllib3==2.2.2 ; python_version >= "3.10" and python_version < "4.0"
+websocket-client==1.8.0 ; python_version >= "3.10" and python_version < "4.0"
+workflows==2.27 ; python_version >= "3.10" and python_version < "4.0"

requirements_dev.txt

@@ -11,8 +11,8 @@
 import logging
 import signal
 import sys
+from importlib.metadata import entry_points
 
-import pkg_resources
 import workflows.recipe.wrapper
 import workflows.services.common_service
 import workflows.transport
@@ -40,9 +40,7 @@ def run():
     zc = zocalo.configuration.from_file()
     zc.activate()
 
-    known_wrappers = {
-        e.name: e.load for e in pkg_resources.iter_entry_points("zocalo.wrappers")
-    }
+    known_wrappers = {e.name: e.load for e in entry_points(group="zocalo.wrappers")}
 
     # Set up parser
     parser = argparse.ArgumentParser(usage="zocalo.wrap [options]")

src/zocalo/cli/wrap.py

@@ -9,9 +9,9 @@
 import os
 import pathlib
 import typing
+from importlib.metadata import entry_points
 
 import marshmallow as mm
-import pkg_resources
 import yaml
 
 import zocalo.configuration.argparse
@@ -56,7 +56,7 @@ def _check_valid_plugin_name(name: str) -> bool:
 
 _configuration_plugins = {
     e.name: e
-    for e in pkg_resources.iter_entry_points("zocalo.configuration.plugins")
+    for e in entry_points(group="zocalo.configuration.plugins")
     if _check_valid_plugin_name(e.name)
 }
 
@@ -208,8 +208,7 @@ def add_command_line_options(self, parser):
         # attributes is allowed. We do this by setting a return type on __getattr__,
         # but only when in a type checking run, as to not affect the runtime class
         # behaviour.
-        def __getattr__(self, name: str) -> typing.Any:
-            ...
+        def __getattr__(self, name: str) -> typing.Any: ...
 
     def __str__(self):
         environments = len(self._environments)

src/zocalo/configuration/__init__.py

@@ -8,8 +8,8 @@
 import time
 import timeit
 import uuid
+from importlib.metadata import entry_points
 
-import pkg_resources
 import workflows.recipe
 from workflows.services.common_service import CommonService
 
@@ -110,9 +110,7 @@ def initializing(self):
         self.message_filters = {
             **{
                 f.name: f.load()
-                for f in pkg_resources.iter_entry_points(
-                    "zocalo.services.dispatcher.filters"
-                )
+                for f in entry_points(group="zocalo.services.dispatcher.filters")
             },
             "load_custom_recipe": self.filter_load_custom_recipe,
             "load_recipes_from_files": self.filter_load_recipes_from_files,
@@ -121,8 +119,8 @@ def initializing(self):
 
         self.ready_for_processing = {
             f.name: f.load()
-            for f in pkg_resources.iter_entry_points(
-                "zocalo.services.dispatcher.ready_for_processing"
+            for f in entry_points(
+                group="zocalo.services.dispatcher.ready_for_processing"
             )
         }
 

src/zocalo/service/dispatcher.py

@@ -186,14 +186,14 @@ def update_subscriptions(self):
             )
             if real_subscriber_count:
                 self.log.debug("subscribing to %s", destination)
-                self.known_queues[destination][
-                    "subscription"
-                ] = self._transport.subscribe(
-                    destination,
-                    self.quarantine,
-                    acknowledgement=True,
-                    selector="JMSXDeliveryCount>5",
-                    disable_mangling=True,
+                self.known_queues[destination]["subscription"] = (
+                    self._transport.subscribe(
+                        destination,
+                        self.quarantine,
+                        acknowledgement=True,
+                        selector="JMSXDeliveryCount>5",
+                        disable_mangling=True,
+                    )
                 )
 
     def garbage_collect(self):

src/zocalo/service/schlockmeister.py

@@ -5,9 +5,10 @@
 from unittest import mock
 
 import workflows.transport
-import zocalo.cli.dlq_purge as dlq_purge
 from workflows.transport.common_transport import CommonTransport
 
+import zocalo.cli.dlq_purge as dlq_purge
+
 
 def gen_header_activemq(i):
     return {

tests/cli/test_dlq_purge.py

@@ -8,6 +8,7 @@
 
 import workflows.transport
 from workflows.transport.common_transport import CommonTransport
+
 from zocalo.cli.dlq_reinject import run
 
 

tests/cli/test_dlq_reinject.py

@@ -8,8 +8,9 @@
 
 import pytest
 import workflows.transport
-import zocalo.configuration
 from workflows.transport.common_transport import CommonTransport
+
+import zocalo.configuration
 from zocalo.cli import pickup
 
 

tests/cli/test_pickup.py

@@ -4,6 +4,7 @@
 
 import workflows.transport
 from workflows.transport.common_transport import CommonTransport
+
 from zocalo.cli.queue_drain import run
 
 

tests/cli/test_queue_drain.py

@@ -6,6 +6,7 @@
 import workflows.transport
 from workflows.transport.common_transport import CommonTransport
 from workflows.util import generate_unique_host_id
+
 from zocalo.cli.shutdown import run
 
 

tests/cli/test_shutdown.py

@@ -7,6 +7,7 @@
 from unittest import mock
 
 import pytest
+
 import zocalo.configuration
 
 sample_configuration = """

tests/configuration/test_configuration.py

@@ -3,6 +3,7 @@
 from unittest import mock
 
 import pytest
+
 import zocalo.configuration
 
 sample_configuration = """

tests/configuration/test_plugin_graylog.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pytest
+
 import zocalo.configuration
 
 sample_configuration = """

tests/configuration/test_plugin_jmx.py

@@ -4,6 +4,7 @@
 from unittest import mock
 
 import pytest
+
 import zocalo.configuration
 import zocalo.configuration.plugin_logging
 

tests/configuration/test_plugin_logging.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pytest
+
 import zocalo.configuration
 
 sample_configuration = """

tests/configuration/test_plugin_slurm.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import pytest
+
 import zocalo.configuration
 
 sample_configuration = """

tests/configuration/test_plugin_smtp.py

@@ -4,9 +4,10 @@
 from unittest import mock
 
 import pytest
-import zocalo.configuration
 from workflows.recipe import Recipe
 from workflows.transport.offline_transport import OfflineTransport
+
+import zocalo.configuration
 from zocalo.service.dispatcher import Dispatcher
 
 

tests/service/test_dispatcher.py

@@ -4,6 +4,7 @@
 
 from workflows.recipe.wrapper import RecipeWrapper
 from workflows.transport.offline_transport import OfflineTransport
+
 from zocalo.service.jsonlines import JSONLines