diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0dc167e..ae83374 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,18 +6,18 @@ jobs: publish-pypi: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master - - name: Set up Python 3.8 - uses: actions/setup-python@v4.7.1 + - uses: actions/checkout@v4 + - name: Set up Python 3.13 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.13 - name: Install dependencies run: pip install -qU setuptools wheel twine - name: Generating distribution archives run: python setup.py sdist bdist_wheel - name: Publish distribution 馃摝 to PyPI if: startsWith(github.event.ref, 'refs/tags') - uses: pypa/gh-action-pypi-publish@master + uses: pypa/gh-action-pypi-publish@release/v1 with: user: __token__ password: ${{ secrets.pypi_password }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f85c56e..c56ca2c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,9 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.13 - name: Install dependencies run: make install-test - name: Lint @@ -20,11 +20,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: ['3.9', '3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies @@ -35,19 +35,20 @@ jobs: coverage: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5 with: - python-version: 3.8 + python-version: 3.13 - name: Install dependencies run: make install-test - name: Generate coverage report run: pytest --cov-report=xml - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v5 with: file: ./coverage.xml flags: unittests name: codecov-umbrella + token: ${{ secrets.CODECOV_TOKEN }} fail_ci_if_error: true diff --git a/Makefile b/Makefile index c062c7f..61ed6b7 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,9 @@ SHELL := bash PATH := ./venv/bin:${PATH} -PYTHON = python3.8 +PYTHON = python3.13 PROJECT = fast_agave isort = isort $(PROJECT) tests setup.py -black = black -S -l 79 --target-version py38 $(PROJECT) tests setup.py examples +black = black -S -l 79 --target-version py313 $(PROJECT) tests setup.py examples .PHONY: all diff --git a/examples/app.py b/examples/app.py index 71ea401..6b1697c 100644 --- a/examples/app.py +++ b/examples/app.py @@ -1,5 +1,4 @@ import asyncio -from typing import Dict import mongomock as mongomock from mongoengine import connect @@ -22,7 +21,7 @@ @app.get('/') -async def iam_healty() -> Dict: +async def iam_healty() -> dict: return dict(greeting="I'm healthy!!!") diff --git a/examples/middlewares/authed.py b/examples/middlewares/authed.py index 6691e7a..db9734d 100644 --- a/examples/middlewares/authed.py +++ b/examples/middlewares/authed.py @@ -14,6 +14,17 @@ class AuthedMiddleware(ContextMiddleware): + def __init__( + self, app, plugins=None, default_error_response=None, *args, **kwargs + ): + super().__init__( + app=app, + plugins=plugins, + default_error_response=default_error_response, + *args, + **kwargs, + ) + def required_user_id(self) -> bool: """ Example method so we can easily mock it in tests environment diff --git a/examples/resources/base.py b/examples/resources/base.py index ad8f541..4b7700c 100644 --- a/examples/resources/base.py +++ b/examples/resources/base.py @@ -1,5 +1,3 @@ -from typing import Dict, NoReturn - from cuenca_validations.errors import WrongCredsError from fast_agave.blueprints import RestApiBlueprint @@ -9,17 +7,17 @@ @app.get('/healthy_auth') -def health_auth_check() -> Dict: +def health_auth_check() -> dict: return dict(greeting="I'm authenticated and healthy !!!") @app.get('/raise_cuenca_errors') -def raise_cuenca_errors() -> NoReturn: +def raise_cuenca_errors() -> None: raise WrongCredsError('you are not lucky enough!') @app.get('/raise_fast_agave_errors') -def raise_fast_agave_errors() -> NoReturn: +def raise_fast_agave_errors() -> None: raise UnauthorizedError('nice try!') diff --git a/examples/resources/cards.py b/examples/resources/cards.py index abcace1..a65e1f4 100644 --- a/examples/resources/cards.py +++ b/examples/resources/cards.py @@ -1,5 +1,3 @@ -from typing import Dict - from fastapi.responses import JSONResponse as Response from fast_agave.filters import generic_query @@ -21,7 +19,7 @@ async def retrieve(card: CardModel) -> Response: return Response(content=data) @staticmethod - async def query(response: Dict): + async def query(response: dict): for item in response['items']: item['number'] = '*' * 16 return response diff --git a/examples/resources/users.py b/examples/resources/users.py index 53e4f91..b33d3a8 100644 --- a/examples/resources/users.py +++ b/examples/resources/users.py @@ -18,6 +18,6 @@ async def update( user: UserModel, request: UserUpdateRequest, api_request: Request ) -> Response: user.name = request.name - user.ip = api_request.client.host + user.ip = api_request.client.host if api_request.client else None await user.async_save() return Response(content=user.to_dict(), status_code=200) diff --git a/examples/tasks/retry_task_example.py b/examples/tasks/retry_task_example.py index 687830d..cd78a10 100644 --- a/examples/tasks/retry_task_example.py +++ b/examples/tasks/retry_task_example.py @@ -8,8 +8,7 @@ QUEUE_URL = 'http://127.0.0.1:4000/123456789012/core.fifo' -class YouCanTryAgain(Exception): - ... +class YouCanTryAgain(Exception): ... def test_your_luck(message): diff --git a/examples/tasks/task_example.py b/examples/tasks/task_example.py index 1a84c95..003fbf8 100644 --- a/examples/tasks/task_example.py +++ b/examples/tasks/task_example.py @@ -28,4 +28,4 @@ async def dummy_task(message) -> None: @task(queue_url=QUEUE2_URL, region_name='us-east-1') async def task_validator(message: Union[User, Company]) -> None: - print(message.dict()) + print(message.model_dump()) diff --git a/examples/validators.py b/examples/validators.py index f1dafb4..3288ef7 100644 --- a/examples/validators.py +++ b/examples/validators.py @@ -1,11 +1,9 @@ from typing import Optional from cuenca_validations.types import QueryParams -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict import datetime as dt -from pydantic.main import BaseConfig - class AccountQuery(QueryParams): name: Optional[str] = None @@ -23,21 +21,18 @@ class BillerQuery(QueryParams): class UserQuery(QueryParams): - platform_id: str + platform_id: Optional[str] = None class AccountRequest(BaseModel): name: str - - class Config(BaseConfig): - fields = { - 'name': {'description': 'Sample description'}, - } - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ 'example': { 'name': 'Doroteo Arango', } } + ) class AccountResponse(BaseModel): @@ -47,10 +42,8 @@ class AccountResponse(BaseModel): platform_id: str created_at: dt.datetime deactivated_at: Optional[dt.datetime] = None - - class Config(BaseConfig): - fields = {'name': {'description': 'Sample description'}} - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ 'example': { 'id': 'AC-123456', 'name': 'Doroteo Arango', @@ -60,17 +53,18 @@ class Config(BaseConfig): 'deactivated_at': None, } } + ) class AccountUpdateRequest(BaseModel): name: str - - class Config(BaseConfig): - schema_extra = { + model_config = ConfigDict( + json_schema_extra={ 'example': { 'name': 'Pancho Villa', } } + ) class FileQuery(QueryParams): diff --git a/fast_agave/blueprints/rest_api.py b/fast_agave/blueprints/rest_api.py index 5896749..65f0db5 100644 --- a/fast_agave/blueprints/rest_api.py +++ b/fast_agave/blueprints/rest_api.py @@ -1,5 +1,5 @@ import mimetypes -from typing import Any, Dict, List, Optional +from typing import Any, Optional from urllib.parse import urlencode from cuenca_validations.types import QueryParams @@ -7,8 +7,7 @@ from fastapi.responses import JSONResponse as Response from fastapi.responses import StreamingResponse from mongoengine import DoesNotExist, Q -from pydantic import ValidationError -from pydantic.main import BaseConfig, BaseModel +from pydantic import BaseModel, Field, ValidationError from starlette_context import context from ..exc import NotFoundError, UnprocessableEntity @@ -231,7 +230,7 @@ async def retrieve(id: str, request: Request): file, media_type=mimetype, headers={ - 'Content-Disposition': f'attachment; filename={filename}' + 'Content-Disposition': f'attachment; filename={filename}' # noqa: E702 }, ) elif hasattr(cls, 'retrieve'): @@ -255,30 +254,25 @@ async def retrieve(id: str, request: Request): return cls query_description = ( - f'Make queries in resource {cls.__name__} and filter the result using query parameters. \n' - f'The items are paginated, to iterate over them use the `next_page_uri` included in response. \n' - f'If you need only a counter not the data send value `true` in `count` param.' + f"Make queries in resource {cls.__name__} and filter the result using query parameters. \n" + f"The items are paginated, to iterate over them use the 'next_page_uri' included in response. \n" + f"If you need only a counter not the data send value 'true' in 'count' param." ) # Build dynamically types for query response class QueryResponse(BaseModel): - items: Optional[List[response_model]] = [] - next_page_uri: Optional[str] = None - count: Optional[int] = None - - class Config(BaseConfig): - fields = { - 'items': { - 'description': f'List of {cls.__name__} that match with query filters' - }, - 'next_page_uri': { - 'description': 'URL to fetch the next page of results' - }, - 'count': { - 'description': f'Counter of {cls.__name__} objects that match with query filters. \n' - f'Included in response only if `count` param was `true`' - }, - } + items: Optional[list[response_model]] = Field( + [], + description=f'List of {cls.__name__} that match with query filters', + ) + next_page_uri: Optional[str] = Field( + None, description='URL to fetch the next page of results' + ) + count: Optional[int] = Field( + None, + description=f'Counter of {cls.__name__} objects that match with query filters. \n' + 'If you need only a counter not the data send value `true` in `count` param.', + ) QueryResponse.__name__ = f'QueryResponse{cls.__name__}' @@ -369,7 +363,7 @@ async def _all(query: QueryParams, filters: Q, resource_path: str): next_page_uri: Optional[str] = None if wants_more and has_more: query.created_before = item_dicts[-1]['created_at'] - params = query.dict() + params = query.model_dump() if self.user_id_filter_required(): params.pop('user_id') if self.platform_id_filter_required(): @@ -382,7 +376,7 @@ async def _all(query: QueryParams, filters: Q, resource_path: str): return wrapper_resource_class -def json_openapi(code: int, description, samples: List[Dict]) -> dict: +def json_openapi(code: int, description, samples: list[dict]) -> dict: examples = {f'example_{i}': ex for i, ex in enumerate(samples)} return { code: { diff --git a/fast_agave/filters.py b/fast_agave/filters.py index 1c03315..f2d1066 100644 --- a/fast_agave/filters.py +++ b/fast_agave/filters.py @@ -1,10 +1,8 @@ -from typing import List - from cuenca_validations.types import QueryParams from mongoengine import Q -def generic_query(query: QueryParams, excluded: List[str] = []) -> Q: +def generic_query(query: QueryParams, excluded: list[str] = []) -> Q: filters = Q() if query.created_before: filters &= Q(created_at__lt=query.created_before) @@ -19,7 +17,7 @@ def generic_query(query: QueryParams, excluded: List[str] = []) -> Q: 'key', *excluded, } - fields = query.dict(exclude=exclude_fields) + fields = query.model_dump(exclude=exclude_fields) if 'count' in fields: del fields['count'] return filters & Q(**fields) diff --git a/fast_agave/tasks/sqs_celery_client.py b/fast_agave/tasks/sqs_celery_client.py index 13ad85d..825fad8 100644 --- a/fast_agave/tasks/sqs_celery_client.py +++ b/fast_agave/tasks/sqs_celery_client.py @@ -2,14 +2,14 @@ import json from base64 import b64encode from dataclasses import dataclass -from typing import Dict, Iterable, Optional +from typing import Iterable, Optional from uuid import uuid4 from fast_agave.tasks.sqs_client import SqsClient def _build_celery_message( - task_name: str, args_: Iterable, kwargs_: Dict + task_name: str, args_: Iterable, kwargs_: dict ) -> str: task_id = str(uuid4()) # la definici贸n de esta plantila se encuentra en: @@ -60,7 +60,7 @@ async def send_task( self, name: str, args: Optional[Iterable] = None, - kwargs: Optional[Dict] = None, + kwargs: Optional[dict] = None, ) -> None: celery_message = _build_celery_message(name, args or (), kwargs or {}) await super().send_message(celery_message) @@ -69,7 +69,7 @@ def send_background_task( self, name: str, args: Optional[Iterable] = None, - kwargs: Optional[Dict] = None, + kwargs: Optional[dict] = None, ) -> asyncio.Task: celery_message = _build_celery_message(name, args or (), kwargs or {}) return super().send_message_async(celery_message) diff --git a/fast_agave/tasks/sqs_client.py b/fast_agave/tasks/sqs_client.py index 46e53f8..ef8732e 100644 --- a/fast_agave/tasks/sqs_client.py +++ b/fast_agave/tasks/sqs_client.py @@ -1,7 +1,7 @@ import asyncio import json from dataclasses import dataclass, field -from typing import Dict, Optional, Union +from typing import Optional, Union from uuid import uuid4 from aiobotocore.session import get_session @@ -37,7 +37,7 @@ async def close(self): async def send_message( self, - data: Union[str, Dict], + data: Union[str, dict], message_group_id: Optional[str] = None, ) -> None: await self._sqs.send_message( @@ -48,7 +48,7 @@ async def send_message( def send_message_async( self, - data: Union[str, Dict], + data: Union[str, dict], message_group_id: Optional[str] = None, ) -> asyncio.Task: task = asyncio.create_task(self.send_message(data, message_group_id)) diff --git a/fast_agave/tasks/sqs_tasks.py b/fast_agave/tasks/sqs_tasks.py index 912bcfa..f3e9133 100644 --- a/fast_agave/tasks/sqs_tasks.py +++ b/fast_agave/tasks/sqs_tasks.py @@ -8,7 +8,7 @@ from aiobotocore.httpsession import HTTPClientError from aiobotocore.session import get_session -from pydantic import validate_arguments +from pydantic import validate_call from ..exc import RetryTask @@ -106,7 +106,7 @@ async def concurrency_controller(coro: Coroutine) -> None: session = get_session() - task_with_validators = validate_arguments(task_func) + task_with_validators = validate_call(task_func) async with session.create_client('sqs', region_name) as sqs: async for message in message_consumer( diff --git a/fast_agave/version.py b/fast_agave/version.py index 092052c..afced14 100644 --- a/fast_agave/version.py +++ b/fast_agave/version.py @@ -1 +1 @@ -__version__ = '0.14.1' +__version__ = '2.0.0' diff --git a/requirements-test.txt b/requirements-test.txt index 7918af1..61531d6 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,14 +1,13 @@ -black==22.3.0 -flake8==4.0.* -isort==5.10.* -mock==4.0.3 -mongomock==4.1.* -moto[server]==2.2.* -mypy==1.0.1 -pytest==7.4.* -pytest-cov==4.1.* -pytest-vcr==1.0.* -pytest-asyncio==0.15.* -requests==2.28.* -boto3==1.20.24 -botocore==1.23.24 +black==24.10.0 +flake8==7.1.1 +isort==5.13.2 +mock==5.1.0 +mongomock==4.3.0 +moto[server]==5.0.26 +mypy==1.14.1 +pytest==8.3.4 +pytest-cov==6.0.0 +pytest-vcr==1.0.2 +pytest-asyncio==0.18.* +requests==2.32.3 +httpx==0.28.1 diff --git a/requirements.txt b/requirements.txt index bbf0412..5e111ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -aiobotocore==2.1.0 -cuenca-validations==0.11.19 -fastapi==0.68.2 -mongoengine-plus==0.0.3 -python-multipart==0.0.5 -starlette-context==0.3.3 -types-aiobotocore-sqs==2.1.0.post1 \ No newline at end of file +aiobotocore==2.17.0 +cuenca-validations==2.0.0 +fastapi==0.115.6 +mongoengine-plus==1.0.0 +python-multipart==0.0.20 +starlette-context==0.3.6 +types-aiobotocore-sqs==2.17.0 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 6b110c0..0e775d8 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,6 +3,7 @@ test=pytest [tool:pytest] addopts = -p no:warnings -v --cov-report term-missing --cov=fast_agave +asyncio_mode = auto [flake8] inline-quotes = ' diff --git a/setup.py b/setup.py index 45d3f16..cdf0e83 100644 --- a/setup.py +++ b/setup.py @@ -21,17 +21,21 @@ packages=find_packages(), include_package_data=True, package_data=dict(agave=['py.typed']), - python_requires='>=3.8', + python_requires='>=3.9', install_requires=[ - 'aiobotocore>=1.0.0,<3.0.0', - 'types-aiobotocore-sqs>=2.1.0.post1,<3.0.0', - 'cuenca-validations>=0.9.4,<1.0.0', - 'fastapi>=0.63.0,<0.69.0', - 'mongoengine-plus>=0.0.2,<1.0.0', + 'aiobotocore>=2.0.0,<3.0.0', + 'types-aiobotocore-sqs>=2.1.0,<3.0.0', + 'cuenca-validations>=2.0.0,<3.0.0', + 'fastapi>=0.115.0,<0.120.0', + 'mongoengine-plus>=1.0.0,<2.0.0', 'starlette-context>=0.3.2,<0.4.0', ], classifiers=[ - 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', + 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', + 'Programming Language :: Python :: 3.13', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', ], diff --git a/tests/blueprint/test_blueprint.py b/tests/blueprint/test_blueprint.py index 7bf236c..8abb485 100644 --- a/tests/blueprint/test_blueprint.py +++ b/tests/blueprint/test_blueprint.py @@ -1,6 +1,5 @@ import datetime as dt from tempfile import TemporaryFile -from typing import List from unittest.mock import MagicMock, patch from urllib.parse import urlencode @@ -139,7 +138,7 @@ def test_query_all_with_limit(client: TestClient) -> None: def test_query_all_resource( - client: TestClient, accounts: List[Account] + client: TestClient, accounts: list[Account] ) -> None: accounts = list(reversed(accounts)) @@ -158,7 +157,7 @@ def test_query_all_resource( def test_query_all_created_after( - client: TestClient, accounts: List[Account] + client: TestClient, accounts: list[Account] ) -> None: created_at = dt.datetime(2020, 2, 1) expected_length = len([a for a in accounts if a.created_at > created_at]) @@ -173,7 +172,7 @@ def test_query_all_created_after( @patch(PLATFORM_ID_FILTER_REQUIRED, MagicMock(return_value=True)) def test_query_platform_id_filter_required( - client: TestClient, accounts: List[Account] + client: TestClient, accounts: list[Account] ) -> None: accounts = list( reversed( @@ -197,7 +196,7 @@ def test_query_platform_id_filter_required( @patch(USER_ID_FILTER_REQUIRED, MagicMock(return_value=True)) def test_query_user_id_filter_required( - client: TestClient, accounts: List[Account] + client: TestClient, accounts: list[Account] ) -> None: accounts = list( reversed([a for a in accounts if a.user_id == TEST_DEFAULT_USER_ID]) diff --git a/tests/blueprint/test_decorators.py b/tests/blueprint/test_decorators.py index 8bc698f..9a445a8 100644 --- a/tests/blueprint/test_decorators.py +++ b/tests/blueprint/test_decorators.py @@ -19,8 +19,7 @@ def retrieve(self) -> str: def test_copy_properties_from() -> None: - def retrieve(): - ... + def retrieve(): ... # noqa: E704 assert not hasattr(retrieve, 'i_am_test') retrieve = copy_attributes(TestResource)(retrieve) diff --git a/tests/conftest.py b/tests/conftest.py index ebb612c..1fe9e49 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,8 @@ import datetime as dt import functools import os -import subprocess from functools import partial -from typing import Callable, Dict, Generator, List +from typing import Callable, Generator import aiobotocore import boto3 @@ -12,6 +11,7 @@ from aiobotocore.session import AioSession from fastapi.testclient import TestClient from mongoengine import Document +from moto.server import ThreadedMotoServer from examples.app import app from examples.config import ( @@ -29,7 +29,7 @@ def collection_fixture(model: Document) -> Callable[..., FuncDecorator]: def collection_decorator(func: Callable) -> FuncDecorator: @functools.wraps(func) - def wrapper(*args, **kwargs) -> Generator[List, None, None]: + def wrapper(*args, **kwargs) -> Generator[list, None, None]: items = func(*args, **kwargs) for item in items: item.save() @@ -49,7 +49,7 @@ def client() -> Generator[TestClient, None, None]: @pytest.fixture @collection_fixture(Account) -def accounts() -> List[Account]: +def accounts() -> list[Account]: return [ Account( name='Frida Kahlo', @@ -91,23 +91,23 @@ def accounts() -> List[Account]: @pytest.fixture -def account(accounts: List[Account]) -> Generator[Account, None, None]: +def account(accounts: list[Account]) -> Generator[Account, None, None]: yield accounts[0] @pytest.fixture -def user(users: List[User]) -> Generator[User, None, None]: +def user(users: list[User]) -> Generator[User, None, None]: yield users[0] @pytest.fixture -def other_account(accounts: List[Account]) -> Generator[Account, None, None]: +def other_account(accounts: list[Account]) -> Generator[Account, None, None]: yield accounts[-1] @pytest.fixture @collection_fixture(File) -def files() -> List[File]: +def files() -> list[File]: return [ File( name='Frida Kahlo', @@ -117,13 +117,13 @@ def files() -> List[File]: @pytest.fixture -def file(files: List[File]) -> Generator[File, None, None]: +def file(files: list[File]) -> Generator[File, None, None]: yield files[0] @pytest.fixture @collection_fixture(Card) -def cards() -> List[Card]: +def cards() -> list[Card]: return [ Card( number='5434000000000001', @@ -149,13 +149,13 @@ def cards() -> List[Card]: @pytest.fixture -def card(cards: List[Card]) -> Generator[Card, None, None]: +def card(cards: list[Card]) -> Generator[Card, None, None]: yield cards[0] @pytest.fixture @collection_fixture(User) -def users() -> List[User]: +def users() -> list[User]: return [ User(name='User1', platform_id=TEST_DEFAULT_PLATFORM_ID), User(name='User2', platform_id=TEST_SECOND_PLATFORM_ID), @@ -164,7 +164,7 @@ def users() -> List[User]: @pytest.fixture @collection_fixture(Biller) -def billers() -> List[Biller]: +def billers() -> list[Biller]: return [ Biller(name='Telcel'), Biller(name='ATT'), @@ -184,14 +184,17 @@ def aws_credentials() -> None: @pytest.fixture(scope='session') def aws_endpoint_urls( aws_credentials, -) -> Generator[Dict[str, str], None, None]: - sqs = subprocess.Popen(['moto_server', 'sqs', '-p', '4000']) +) -> Generator[dict[str, str], None, None]: + + server = ThreadedMotoServer(port=4000) + server.start() endpoints = dict( sqs='http://127.0.0.1:4000/', ) yield endpoints - sqs.kill() + + server.stop() @pytest.fixture(autouse=True) @@ -222,7 +225,11 @@ async def sqs_client(): session = aiobotocore.session.get_session() async with session.create_client('sqs', 'us-east-1') as sqs: await sqs.create_queue( - QueueName='core.fifo', Attributes={'FifoQueue': 'true'} + QueueName='core.fifo', + Attributes={ + 'FifoQueue': 'true', + 'ContentBasedDeduplication': 'true', + }, ) resp = await sqs.get_queue_url(QueueName='core.fifo') sqs.send_message = partial(sqs.send_message, QueueUrl=resp['QueueUrl']) diff --git a/tests/tasks/test_sqs_celery_client.py b/tests/tasks/test_sqs_celery_client.py index 7683138..19e7dc9 100644 --- a/tests/tasks/test_sqs_celery_client.py +++ b/tests/tasks/test_sqs_celery_client.py @@ -1,14 +1,11 @@ import base64 import json -import pytest - from fast_agave.tasks.sqs_celery_client import SqsCeleryClient CORE_QUEUE_REGION = 'us-east-1' -@pytest.mark.asyncio async def test_send_task(sqs_client) -> None: args = [10, 'foo'] kwargs = dict(hola='mundo') @@ -32,7 +29,6 @@ async def test_send_task(sqs_client) -> None: await queue.close() -@pytest.mark.asyncio async def test_send_background_task(sqs_client) -> None: args = [10, 'foo'] kwargs = dict(hola='mundo') diff --git a/tests/tasks/test_sqs_client.py b/tests/tasks/test_sqs_client.py index a1d4747..35c99e0 100644 --- a/tests/tasks/test_sqs_client.py +++ b/tests/tasks/test_sqs_client.py @@ -1,13 +1,10 @@ import json -import pytest - from fast_agave.tasks.sqs_client import SqsClient CORE_QUEUE_REGION = 'us-east-1' -@pytest.mark.asyncio async def test_send_message(sqs_client) -> None: data1 = dict(hola='mundo') data2 = dict(foo='bar') @@ -25,7 +22,6 @@ async def test_send_message(sqs_client) -> None: assert message == data2 -@pytest.mark.asyncio async def test_send_message_async(sqs_client) -> None: data1 = dict(hola='mundo') diff --git a/tests/tasks/test_sqs_tasks.py b/tests/tasks/test_sqs_tasks.py index a6a780c..f76b8a6 100644 --- a/tests/tasks/test_sqs_tasks.py +++ b/tests/tasks/test_sqs_tasks.py @@ -2,11 +2,10 @@ import datetime as dt import json import uuid -from typing import Dict, Union +from typing import Union from unittest.mock import AsyncMock, call, patch import aiobotocore.client -import pytest from aiobotocore.httpsession import HTTPClientError from pydantic import BaseModel @@ -20,7 +19,6 @@ CORE_QUEUE_REGION = 'us-east-1' -@pytest.mark.asyncio async def test_execute_tasks(sqs_client) -> None: """ Happy path: Se obtiene el mensaje y se ejecuta el task exitosamente. @@ -35,7 +33,7 @@ async def test_execute_tasks(sqs_client) -> None: async_mock_function = AsyncMock() - async def my_task(data: Dict) -> None: + async def my_task(data: dict) -> None: await async_mock_function(data) await task( @@ -52,7 +50,6 @@ async def my_task(data: Dict) -> None: assert len(BACKGROUND_TASKS) == 0 -@pytest.mark.asyncio async def test_execute_tasks_with_validator(sqs_client) -> None: class Validator(BaseModel): id: str @@ -94,7 +91,6 @@ async def my_task(data: Validator) -> None: assert len(BACKGROUND_TASKS) == 0 -@pytest.mark.asyncio async def test_execute_tasks_with_union_validator(sqs_client) -> None: class User(BaseModel): id: str @@ -108,7 +104,7 @@ class Company(BaseModel): async_mock_function = AsyncMock(return_value=None) async def my_task(data: Union[User, Company]) -> None: - await async_mock_function(data) + await async_mock_function(data.model_dump()) task_params = dict( queue_url=sqs_client.queue_url, @@ -146,14 +142,13 @@ async def my_task(data: Union[User, Company]) -> None: assert len(BACKGROUND_TASKS) == 0 -@pytest.mark.asyncio async def test_not_execute_tasks(sqs_client) -> None: """ Este caso es cuando el queue est谩 vac铆o. No hay nada que ejecutar """ async_mock_function = AsyncMock() - async def my_task(data: Dict) -> None: + async def my_task(data: dict) -> None: await async_mock_function(data) # No escribimos un mensaje en el queue @@ -169,7 +164,6 @@ async def my_task(data: Dict) -> None: assert len(BACKGROUND_TASKS) == 0 -@pytest.mark.asyncio async def test_http_client_error_tasks(sqs_client) -> None: """ Este test prueba el caso cuando hay un error de conexi贸n al intentar @@ -205,7 +199,7 @@ async def mock_create_client(*args, **kwargs): async_mock_function = AsyncMock(return_value=None) - async def my_task(data: Dict) -> None: + async def my_task(data: dict) -> None: await async_mock_function(data) with patch( @@ -221,7 +215,6 @@ async def my_task(data: Dict) -> None: async_mock_function.assert_called_once() -@pytest.mark.asyncio async def test_retry_tasks_default_max_retries(sqs_client) -> None: """ Este test prueba la l贸gica de reintentos con la configuraci贸n default, @@ -242,7 +235,7 @@ async def test_retry_tasks_default_max_retries(sqs_client) -> None: async_mock_function = AsyncMock(side_effect=RetryTask) - async def my_task(data: Dict) -> None: + async def my_task(data: dict) -> None: await async_mock_function(data) await task( @@ -260,7 +253,6 @@ async def my_task(data: Dict) -> None: assert 'Messages' not in resp -@pytest.mark.asyncio async def test_retry_tasks_custom_max_retries(sqs_client) -> None: """ Este test prueba la l贸gica de reintentos con la configuraci贸n default, @@ -277,13 +269,13 @@ async def test_retry_tasks_custom_max_retries(sqs_client) -> None: async_mock_function = AsyncMock(side_effect=RetryTask) - async def my_task(data: Dict) -> None: + async def my_task(data: dict) -> None: await async_mock_function(data) await task( queue_url=sqs_client.queue_url, region_name=CORE_QUEUE_REGION, - wait_time_seconds=1, + wait_time_seconds=2, visibility_timeout=1, max_retries=3, )(my_task)() @@ -297,7 +289,6 @@ async def my_task(data: Dict) -> None: assert len(BACKGROUND_TASKS) == 0 -@pytest.mark.asyncio async def test_does_not_retry_on_unhandled_exceptions(sqs_client) -> None: """ Este caso prueba que las excepciones no controladas no se reintentan por @@ -316,7 +307,7 @@ async def test_does_not_retry_on_unhandled_exceptions(sqs_client) -> None: side_effect=Exception('something went wrong :(') ) - async def my_task(data: Dict) -> None: + async def my_task(data: dict) -> None: await async_mock_function(data) await task( @@ -335,7 +326,6 @@ async def my_task(data: Dict) -> None: assert len(BACKGROUND_TASKS) == 0 -@pytest.mark.asyncio async def test_retry_tasks_with_countdown(sqs_client) -> None: """ Este test prueba la l贸gica de reintentos con un countdown, @@ -359,7 +349,7 @@ async def test_retry_tasks_with_countdown(sqs_client) -> None: async_mock_function = AsyncMock(side_effect=RetryTask(countdown=2)) - async def countdown_tester(data: Dict): + async def countdown_tester(data: dict): await async_mock_function(data, dt.datetime.now()) await task( @@ -376,7 +366,6 @@ async def countdown_tester(data: Dict): assert 'Messages' not in resp -@pytest.mark.asyncio async def test_concurrency_controller( sqs_client, ) -> None: @@ -390,8 +379,8 @@ async def test_concurrency_controller( async_mock_function = AsyncMock() - async def task_counter(data: Dict) -> None: - await asyncio.sleep(1) + async def task_counter(data: dict) -> None: + await asyncio.sleep(2) running_tasks = len(await get_running_fast_agave_tasks()) await async_mock_function(running_tasks) @@ -406,3 +395,35 @@ async def task_counter(data: Dict) -> None: running_tasks = [call[0] for call, _ in async_mock_function.call_args_list] assert max(running_tasks) == 2 + + +async def test_invalid_json_message(sqs_client) -> None: + """ + Este test verifica que los mensajes con JSON inv谩lido son ignorados + y el mensaje es eliminado del queue sin ejecutar el task + """ + # Enviamos un mensaje con JSON inv谩lido + await sqs_client.send_message( + MessageBody='{invalid_json', + MessageGroupId='1234', + ) + + async_mock_function = AsyncMock() + + async def my_task(data: dict) -> None: + await async_mock_function(data) + + await task( + queue_url=sqs_client.queue_url, + region_name=CORE_QUEUE_REGION, + wait_time_seconds=1, + visibility_timeout=1, + )(my_task)() + + # Verificamos que el task nunca fue ejecutado + async_mock_function.assert_not_called() + + # Verificamos que el mensaje fue eliminado del queue + resp = await sqs_client.receive_message() + assert 'Messages' not in resp + assert len(BACKGROUND_TASKS) == 0