From be98386da302e4480222ec9cc0161ec5481f1518 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 15:59:18 +0530 Subject: [PATCH 01/39] Use gunicorn instead of uvicorn --- Dockerfile | 4 ++-- main.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b1f121..0a3cf70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,5 +27,5 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD python3 main.py migrate; \ - python3 main.py run +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "main:app"] + diff --git a/main.py b/main.py index a2f4cfc..3dd08e0 100755 --- a/main.py +++ b/main.py @@ -5,8 +5,10 @@ from openg2p_spar_mapper_api.app import Initializer from openg2p_fastapi_common.ping import PingInitializer +from openg2p_fastapi_common.context import app_registry main_init = Initializer() PingInitializer() -main_init.main() +# main_init.main() +app = app_registry.get() From 5df8ce0b5ce58ca1e9056435989f0d03da2ab4dc Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 16:01:41 +0530 Subject: [PATCH 02/39] Use gunicorn instead of uvicorn --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0a3cf70..109fd34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,6 +22,7 @@ ADD --chown=${container_user}:${container_user_group} main.py /app RUN python3 -m venv venv \ && . ./venv/bin/activate RUN python3 -m pip install \ + guniconrn \ openg2p-fastapi-common==1.0.0 \ openg2p-g2pconnect-common-lib==1.0.0 \ openg2p-g2pconnect-mapper-lib==1.0.0 \ From a897ee33ab21755581206d262ee3a39d03f5863d Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 16:33:29 +0530 Subject: [PATCH 03/39] Revert code --- Dockerfile | 5 ++--- main.py | 4 +--- src/openg2p_spar_mapper_api/app.py | 2 ++ src/openg2p_spar_mapper_api/services/__init__.py | 1 + src/openg2p_spar_mapper_api/services/mapper.py | 3 ++- .../services/session_service.py | 16 ++++++++++++++++ 6 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 src/openg2p_spar_mapper_api/services/session_service.py diff --git a/Dockerfile b/Dockerfile index 109fd34..2b1f121 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,11 +22,10 @@ ADD --chown=${container_user}:${container_user_group} main.py /app RUN python3 -m venv venv \ && . ./venv/bin/activate RUN python3 -m pip install \ - guniconrn \ openg2p-fastapi-common==1.0.0 \ openg2p-g2pconnect-common-lib==1.0.0 \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "main:app"] - +CMD python3 main.py migrate; \ + python3 main.py run diff --git a/main.py b/main.py index 3dd08e0..a2f4cfc 100755 --- a/main.py +++ b/main.py @@ -5,10 +5,8 @@ from openg2p_spar_mapper_api.app import Initializer from openg2p_fastapi_common.ping import PingInitializer -from openg2p_fastapi_common.context import app_registry main_init = Initializer() PingInitializer() -# main_init.main() -app = app_registry.get() +main_init.main() diff --git a/src/openg2p_spar_mapper_api/app.py b/src/openg2p_spar_mapper_api/app.py index 1eb6cfe..670d171 100644 --- a/src/openg2p_spar_mapper_api/app.py +++ b/src/openg2p_spar_mapper_api/app.py @@ -20,6 +20,7 @@ RequestValidation, SyncRequestHelper, SyncResponseHelper, + SessionInitializer, ) @@ -27,6 +28,7 @@ class Initializer(BaseInitializer): def initialize(self, **kwargs): super().initialize() + SessionInitializer() MapperService() IdFaMappingValidations() SyncRequestHelper() diff --git a/src/openg2p_spar_mapper_api/services/__init__.py b/src/openg2p_spar_mapper_api/services/__init__.py index 9b15056..3f400fd 100644 --- a/src/openg2p_spar_mapper_api/services/__init__.py +++ b/src/openg2p_spar_mapper_api/services/__init__.py @@ -4,3 +4,4 @@ from .request_helper import AsyncRequestHelper, SyncRequestHelper from .request_validations import RequestValidation from .response_helper import AsyncResponseHelper, SyncResponseHelper +from .session_service import SessionInitializer diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 2b586a8..97e5133 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -36,6 +36,7 @@ UpdateValidationException, ) from ..services.id_fa_mapping_validations import IdFaMappingValidations +from ..services.session_service import SessionInitializer _config = Settings.get_config() _logger = logging.getLogger(_config.logging_default_logger_name) @@ -204,7 +205,7 @@ def construct_single_update_response_for_failure( ) async def resolve(self, resolve_request: ResolveRequest): - session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) + session_maker = await SessionInitializer.get_component().retrieve_session() async with session_maker() as session: resolve_request_message: ResolveRequestMessage = resolve_request.message diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py new file mode 100644 index 0000000..c695fc8 --- /dev/null +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -0,0 +1,16 @@ +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from openg2p_fastapi_common.service import BaseService +from openg2p_fastapi_common.context import dbengine + + +class SessionInitializer(BaseService): + + def __init__(self): + super().__init__("SessionInitializer") + self.session_maker = None + + async def retrieve_session(self) -> AsyncSession: + if not self.session_maker: + session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) + return session_maker From 6647a0b7711d993577ecc4b68937da52bce64919 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 16:46:08 +0530 Subject: [PATCH 04/39] Revert code --- .../services/mapper.py | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 97e5133..471969e 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -207,38 +207,39 @@ def construct_single_update_response_for_failure( async def resolve(self, resolve_request: ResolveRequest): session_maker = await SessionInitializer.get_component().retrieve_session() async with session_maker() as session: - resolve_request_message: ResolveRequestMessage = resolve_request.message + async with session.begin(): + resolve_request_message: ResolveRequestMessage = resolve_request.message - single_resolve_responses: list[SingleResolveResponse] = [] + single_resolve_responses: list[SingleResolveResponse] = [] - for single_resolve_request in resolve_request_message.resolve_request: - try: - await IdFaMappingValidations.get_component().validate_resolve_request( - connection=session, - single_resolve_request=single_resolve_request, - ) - single_resolve_request: SingleResolveRequest = ( - SingleResolveRequest.model_validate(single_resolve_request) - ) - stmt, result = await self.construct_query( - session, single_resolve_request - ) - single_resolve_response = self.construct_single_resolve( - single_resolve_request, result - ) + for single_resolve_request in resolve_request_message.resolve_request: + try: + await IdFaMappingValidations.get_component().validate_resolve_request( + connection=session, + single_resolve_request=single_resolve_request, + ) + single_resolve_request: SingleResolveRequest = ( + SingleResolveRequest.model_validate(single_resolve_request) + ) + stmt, result = await self.construct_query( + session, single_resolve_request + ) + single_resolve_response = self.construct_single_resolve( + single_resolve_request, result + ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_success( - single_resolve_response + single_resolve_responses.append( + self.construct_single_resolve_response_for_success( + single_resolve_response + ) ) - ) - except ResolveValidationException as e: - single_resolve_responses.append( - self.construct_single_resolve_response_for_failure( - single_resolve_request, e + except ResolveValidationException as e: + single_resolve_responses.append( + self.construct_single_resolve_response_for_failure( + single_resolve_request, e + ) ) - ) - await session.commit() + await session.commit() return single_resolve_responses def construct_single_resolve( From d2fc19ff0ef20cca131eff5e9c14625ed6449f60 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 17:06:10 +0530 Subject: [PATCH 05/39] Revert code --- .../services/mapper.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 471969e..38b1a24 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -205,41 +205,41 @@ def construct_single_update_response_for_failure( ) async def resolve(self, resolve_request: ResolveRequest): - session_maker = await SessionInitializer.get_component().retrieve_session() - async with session_maker() as session: - async with session.begin(): - resolve_request_message: ResolveRequestMessage = resolve_request.message + session_initializer = SessionInitializer.get_component() + session: AsyncSession = await session_initializer.retrieve_session() + async with session.begin(): + resolve_request_message: ResolveRequestMessage = resolve_request.message - single_resolve_responses: list[SingleResolveResponse] = [] + single_resolve_responses: list[SingleResolveResponse] = [] - for single_resolve_request in resolve_request_message.resolve_request: - try: - await IdFaMappingValidations.get_component().validate_resolve_request( - connection=session, - single_resolve_request=single_resolve_request, - ) - single_resolve_request: SingleResolveRequest = ( - SingleResolveRequest.model_validate(single_resolve_request) - ) - stmt, result = await self.construct_query( - session, single_resolve_request - ) - single_resolve_response = self.construct_single_resolve( - single_resolve_request, result - ) + for single_resolve_request in resolve_request_message.resolve_request: + try: + await IdFaMappingValidations.get_component().validate_resolve_request( + connection=session, + single_resolve_request=single_resolve_request, + ) + single_resolve_request: SingleResolveRequest = ( + SingleResolveRequest.model_validate(single_resolve_request) + ) + stmt, result = await self.construct_query( + session, single_resolve_request + ) + single_resolve_response = self.construct_single_resolve( + single_resolve_request, result + ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_success( - single_resolve_response - ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_success( + single_resolve_response ) - except ResolveValidationException as e: - single_resolve_responses.append( - self.construct_single_resolve_response_for_failure( - single_resolve_request, e - ) + ) + except ResolveValidationException as e: + single_resolve_responses.append( + self.construct_single_resolve_response_for_failure( + single_resolve_request, e ) - await session.commit() + ) + await session.commit() return single_resolve_responses def construct_single_resolve( From 945f06473cce079fae3c79ad9040bbd60348d90e Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 17:28:42 +0530 Subject: [PATCH 06/39] Revert code --- src/openg2p_spar_mapper_api/services/mapper.py | 6 +++--- .../services/session_service.py | 15 ++++----------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 38b1a24..e7ace0f 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -205,9 +205,9 @@ def construct_single_update_response_for_failure( ) async def resolve(self, resolve_request: ResolveRequest): - session_initializer = SessionInitializer.get_component() - session: AsyncSession = await session_initializer.retrieve_session() - async with session.begin(): + session_init = SessionInitializer.get_component() + session_maker = await session_init.retrieve_session() + async with session_maker() as session: resolve_request_message: ResolveRequestMessage = resolve_request.message single_resolve_responses: list[SingleResolveResponse] = [] diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index c695fc8..27d5b52 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -1,16 +1,9 @@ -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker - -from openg2p_fastapi_common.service import BaseService from openg2p_fastapi_common.context import dbengine +from openg2p_fastapi_common.service import BaseService +from sqlalchemy.ext.asyncio import async_sessionmaker class SessionInitializer(BaseService): - - def __init__(self): - super().__init__("SessionInitializer") - self.session_maker = None - - async def retrieve_session(self) -> AsyncSession: - if not self.session_maker: - session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) + async def retrieve_session(self): + session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) return session_maker From 4d0cf9503060ccce6f46f4f0f51e83e222704290 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Mon, 13 May 2024 17:48:13 +0530 Subject: [PATCH 07/39] update self.session_maker --- .../services/session_service.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index 27d5b52..71590db 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -4,6 +4,13 @@ class SessionInitializer(BaseService): + def __init__(self): + super().__init__("SessionInitializer") + self.session_maker = None + async def retrieve_session(self): - session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) - return session_maker + if not self.session_maker: + self.session_maker = async_sessionmaker( + dbengine.get(), expire_on_commit=False + ) + return self.session_maker From eaa7fc2de4cb5904b3bdc2e7a1a3c6dcfa707e3e Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 08:12:49 +0530 Subject: [PATCH 08/39] Retrive same session --- src/openg2p_spar_mapper_api/services/mapper.py | 4 ++-- .../services/session_service.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index e7ace0f..bfb03e7 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -206,8 +206,8 @@ def construct_single_update_response_for_failure( async def resolve(self, resolve_request: ResolveRequest): session_init = SessionInitializer.get_component() - session_maker = await session_init.retrieve_session() - async with session_maker() as session: + session = await session_init.retrieve_session() + async with session.begin(): resolve_request_message: ResolveRequestMessage = resolve_request.message single_resolve_responses: list[SingleResolveResponse] = [] diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index 71590db..ee066d0 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -6,11 +6,11 @@ class SessionInitializer(BaseService): def __init__(self): super().__init__("SessionInitializer") - self.session_maker = None + self.session = None async def retrieve_session(self): - if not self.session_maker: - self.session_maker = async_sessionmaker( - dbengine.get(), expire_on_commit=False - ) - return self.session_maker + if not self.session: + session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) + async with session_maker() as session: + self.session = session + return self.session From 5f5b7f44b3f15ed61636e1af6f13645438181f2f Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 08:38:24 +0530 Subject: [PATCH 09/39] Retrive same session --- src/openg2p_spar_mapper_api/services/session_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index ee066d0..ee5618c 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -11,6 +11,6 @@ def __init__(self): async def retrieve_session(self): if not self.session: session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) - async with session_maker() as session: - self.session = session + async with session_maker() as session: + self.session = session return self.session From 0a667f7a07d3ebddb504cbbf838447b8192124bc Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 08:39:40 +0530 Subject: [PATCH 10/39] get session from pool --- src/openg2p_spar_mapper_api/services/mapper.py | 2 +- src/openg2p_spar_mapper_api/services/session_service.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index bfb03e7..745fbd6 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -206,7 +206,7 @@ def construct_single_update_response_for_failure( async def resolve(self, resolve_request: ResolveRequest): session_init = SessionInitializer.get_component() - session = await session_init.retrieve_session() + session = await session_init.get_session_from_pool() async with session.begin(): resolve_request_message: ResolveRequestMessage = resolve_request.message diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index ee5618c..33c294a 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -8,7 +8,7 @@ def __init__(self): super().__init__("SessionInitializer") self.session = None - async def retrieve_session(self): + async def get_session_from_pool(self): if not self.session: session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) async with session_maker() as session: From d5bd26299a8428e4950d4ad17bc417554c62672d Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 08:52:41 +0530 Subject: [PATCH 11/39] get session from pool --- src/openg2p_spar_mapper_api/services/mapper.py | 4 ++-- .../services/session_service.py | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 745fbd6..5e6df18 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -25,7 +25,7 @@ UpdateStatusReasonCode, ) from sqlalchemy import delete, select -from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession from ..config import Settings from ..models import IdFaMapping @@ -206,7 +206,7 @@ def construct_single_update_response_for_failure( async def resolve(self, resolve_request: ResolveRequest): session_init = SessionInitializer.get_component() - session = await session_init.get_session_from_pool() + session: AsyncSession = await session_init.get_session_from_pool() async with session.begin(): resolve_request_message: ResolveRequestMessage = resolve_request.message diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index 33c294a..732dfa4 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -1,16 +1,18 @@ from openg2p_fastapi_common.context import dbengine from openg2p_fastapi_common.service import BaseService -from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession class SessionInitializer(BaseService): def __init__(self): super().__init__("SessionInitializer") - self.session = None + self.session_maker = None - async def get_session_from_pool(self): - if not self.session: - session_maker = async_sessionmaker(dbengine.get(), expire_on_commit=False) - async with session_maker() as session: - self.session = session - return self.session + async def get_session_from_pool(self) -> AsyncSession: + if not self.session_maker: + self.session_maker = async_sessionmaker( + dbengine.get(), expire_on_commit=False + ) + + async with self.session_maker() as session: + return session From b4e3de11478e3a69df3a3fda2a0b9898dc35719f Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 08:53:07 +0530 Subject: [PATCH 12/39] refactor session_initializer --- src/openg2p_spar_mapper_api/services/mapper.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 5e6df18..cc13705 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -205,8 +205,8 @@ def construct_single_update_response_for_failure( ) async def resolve(self, resolve_request: ResolveRequest): - session_init = SessionInitializer.get_component() - session: AsyncSession = await session_init.get_session_from_pool() + session_initializer = SessionInitializer.get_component() + session: AsyncSession = await session_initializer.get_session_from_pool() async with session.begin(): resolve_request_message: ResolveRequestMessage = resolve_request.message From 6232a5a0183ac3670f3ca0070250fc36d8dd69b6 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 09:29:26 +0530 Subject: [PATCH 13/39] Add gunicorn command --- Dockerfile | 4 ++-- main.py | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b1f121..e159970 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,10 +22,10 @@ ADD --chown=${container_user}:${container_user_group} main.py /app RUN python3 -m venv venv \ && . ./venv/bin/activate RUN python3 -m pip install \ + gunicorn \ openg2p-fastapi-common==1.0.0 \ openg2p-g2pconnect-common-lib==1.0.0 \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD python3 main.py migrate; \ - python3 main.py run +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "1", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app"] diff --git a/main.py b/main.py index a2f4cfc..716455c 100755 --- a/main.py +++ b/main.py @@ -5,8 +5,9 @@ from openg2p_spar_mapper_api.app import Initializer from openg2p_fastapi_common.ping import PingInitializer +from openg2p_fastapi_common.context import app_registry -main_init = Initializer() - +initializer = Initializer() PingInitializer() -main_init.main() +# initializer.main() +mapper_app = app_registry.get() From e4cc108845d9e41242637e4c948d4886f037e52b Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 09:38:02 +0530 Subject: [PATCH 14/39] bind gunicorn to 0.0.0.0:8000 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e159970..261204a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "1", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app"] +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "1", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] From 98190150b369df9ed3150ad80a1b694c4bb2f0b7 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 09:39:15 +0530 Subject: [PATCH 15/39] Add more workers to gunicorn --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 261204a..bbaab16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "1", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] From 61ab9a585a220e2edbed5b743cb9785f8150abf4 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 10:55:25 +0530 Subject: [PATCH 16/39] Add 8 workers to gunicorn --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bbaab16..985822a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "4", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "8", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] From 1fb11331246f1014924c937331cc45be89132cfd Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Tue, 14 May 2024 11:19:20 +0530 Subject: [PATCH 17/39] update pip install to use forked repo for fastapi common --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 985822a..f487d31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ RUN python3 -m venv venv \ && . ./venv/bin/activate RUN python3 -m pip install \ gunicorn \ - openg2p-fastapi-common==1.0.0 \ + git+https://github.com/psnappz/openg2p-fastapi-common@develop\#subdirectory=openg2p-fastapi-common \ openg2p-g2pconnect-common-lib==1.0.0 \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src From 89cb367981a9cb1c2792ff65f59f72e73e332350 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 12:15:04 +0530 Subject: [PATCH 18/39] Remove db call from req validation --- .../services/id_fa_mapping_validations.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py b/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py index d867440..5fd31aa 100644 --- a/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py +++ b/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py @@ -94,7 +94,7 @@ async def validate_update_request( return None async def validate_resolve_request( - self, connection, single_resolve_request: SingleResolveRequest + self, single_resolve_request: SingleResolveRequest ) -> None: if not single_resolve_request.id and not single_resolve_request.fa: raise ResolveValidationException( @@ -103,19 +103,19 @@ async def validate_resolve_request( validation_error_type=ResolveStatusReasonCode.rjct_reference_id_invalid, ) - result = await connection.execute( - select(IdFaMapping).where( - IdFaMapping.id_value == single_resolve_request.id, - ) - ) - resolve_request_from_db = result.first() - - if not resolve_request_from_db: - raise ResolveValidationException( - message="ID doesnt exist please link first", - status=StatusEnum.succ, - validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, - ) + # result = await connection.execute( + # select(IdFaMapping).where( + # IdFaMapping.id_value == single_resolve_request.id, + # ) + # ) + # resolve_request_from_db = result.first() + # + # if not resolve_request_from_db: + # raise ResolveValidationException( + # message="ID doesnt exist please link first", + # status=StatusEnum.succ, + # validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, + # ) return None async def validate_unlink_request( From fb7a3e93decac8405a7793cf9f3f949e54719ad7 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 12:15:19 +0530 Subject: [PATCH 19/39] Add resolve_bulk API --- .../controllers/sync_mapper_controller.py | 54 ++++++++++++++----- .../services/mapper.py | 52 +++++++++++++++++- 2 files changed, 93 insertions(+), 13 deletions(-) diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 5d00139..433872b 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -54,6 +54,12 @@ def __init__(self, **kwargs): responses={200: {"model": UnlinkResponse}}, methods=["POST"], ) + self.router.add_api_route( + "/resolve_bulk", + self.resolve_bulk_sync, + responses={200: {"model": ResolveResponse}}, + methods=["POST"], + ) async def link_sync(self, link_request: LinkRequest): try: @@ -67,9 +73,9 @@ async def link_sync(self, link_request: LinkRequest): ) return error_response - single_link_responses: list[ - SingleLinkResponse - ] = await self.mapper_service.link(link_request) + single_link_responses: list[SingleLinkResponse] = ( + await self.mapper_service.link(link_request) + ) return SyncResponseHelper.get_component().construct_success_sync_link_response( link_request, single_link_responses, @@ -89,9 +95,9 @@ async def update_sync(self, update_request: UpdateRequest): ) return error_response - single_update_responses: list[ - SingleUpdateResponse - ] = await self.mapper_service.update(update_request) + single_update_responses: list[SingleUpdateResponse] = ( + await self.mapper_service.update(update_request) + ) return ( SyncResponseHelper.get_component().construct_success_sync_update_response( update_request, @@ -113,9 +119,9 @@ async def resolve_sync(self, resolve_request: ResolveRequest): ) return error_response - single_resolve_responses: list[ - SingleResolveResponse - ] = await self.mapper_service.resolve(resolve_request) + single_resolve_responses: list[SingleResolveResponse] = ( + await self.mapper_service.resolve(resolve_request) + ) return ( SyncResponseHelper.get_component().construct_success_sync_resolve_response( resolve_request, @@ -137,12 +143,36 @@ async def unlink_sync(self, unlink_request: UnlinkRequest): ) return error_response - single_unlink_responses: list[ - SingleResolveResponse - ] = await self.mapper_service.unlink(unlink_request) + single_unlink_responses: list[SingleResolveResponse] = ( + await self.mapper_service.unlink(unlink_request) + ) return ( SyncResponseHelper.get_component().construct_success_sync_unlink_response( unlink_request, single_unlink_responses, ) ) + + async def resolve_bulk_sync(self, resolve_request: ResolveRequest): + try: + RequestValidation.get_component().validate_request(resolve_request) + RequestValidation.get_component().validate_resolve_request_header( + resolve_request + ) + except RequestValidationException as e: + error_response = ( + SyncResponseHelper.get_component().construct_error_sync_response( + resolve_request, e + ) + ) + return error_response + + single_resolve_responses: list[SingleResolveResponse] = ( + await self.mapper_service.resolve_bulk(resolve_request) + ) + return ( + SyncResponseHelper.get_component().construct_success_sync_resolve_response( + resolve_request, + single_resolve_responses, + ) + ) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index cc13705..bc20889 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -215,7 +215,6 @@ async def resolve(self, resolve_request: ResolveRequest): for single_resolve_request in resolve_request_message.resolve_request: try: await IdFaMappingValidations.get_component().validate_resolve_request( - connection=session, single_resolve_request=single_resolve_request, ) single_resolve_request: SingleResolveRequest = ( @@ -281,6 +280,14 @@ async def construct_query(self, session, single_resolve_request): result = result.scalar() return stmt, result + async def construct_bulk_query(self, session, id_values): + stmt = None + id_query = IdFaMapping.id_value.in_(id_values) + stmt = select(IdFaMapping).where(id_query) + result = await session.execute(stmt) + result = result.scalars().all() + return stmt, result + def construct_single_resolve_response_for_success(self, single_resolve_request): return SingleResolveResponse( id=single_resolve_request.id, @@ -362,3 +369,46 @@ def construct_single_unlink_response_for_failure( additional_info=None, locale=single_unlink_request.locale, ) + + async def resolve_bulk(self, resolve_request: ResolveRequest): + session_initializer = SessionInitializer.get_component() + session: AsyncSession = await session_initializer.get_session_from_pool() + async with session.begin(): + resolve_request_message: ResolveRequestMessage = resolve_request.message + + id_values = [ + single_resolve_request.id + for single_resolve_request in resolve_request_message.resolve_request + ] + + for single_resolve_request in resolve_request_message.resolve_request: + try: + await IdFaMappingValidations.get_component().validate_resolve_request( + single_resolve_request=single_resolve_request, + ) + single_resolve_request: SingleResolveRequest = ( + SingleResolveRequest.model_validate(single_resolve_request) + ) + except ResolveValidationException as e: + print(e) + stmt, results = await self.construct_bulk_query(session, id_values) + single_resolve_responses = [] + for result in results: + single_resolve_request = SingleResolveRequest( + id=result.id_value, + fa=result.fa_value, + reference_id="None", + timestamp=datetime.now(), + additional_info=result.additional_info, + locale=None, + ) + single_resolve_response = self.construct_single_resolve( + single_resolve_request, result + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_success( + single_resolve_response + ) + ) + await session.commit() + return single_resolve_responses From d2408aa725cffb20fd015c0bfb73c079e72865d6 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 13:05:30 +0530 Subject: [PATCH 20/39] Fix resolve method --- .../services/mapper.py | 100 ++++++++---------- 1 file changed, 42 insertions(+), 58 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index bc20889..c962d87 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -209,35 +209,62 @@ async def resolve(self, resolve_request: ResolveRequest): session: AsyncSession = await session_initializer.get_session_from_pool() async with session.begin(): resolve_request_message: ResolveRequestMessage = resolve_request.message - single_resolve_responses: list[SingleResolveResponse] = [] + # Collect all ID values for bulk query + id_values = [ + single_resolve_request.id + for single_resolve_request in resolve_request_message.resolve_request + ] + # Validate all requests and collect validated requests + validated_requests = [] for single_resolve_request in resolve_request_message.resolve_request: try: await IdFaMappingValidations.get_component().validate_resolve_request( single_resolve_request=single_resolve_request, ) - single_resolve_request: SingleResolveRequest = ( - SingleResolveRequest.model_validate(single_resolve_request) - ) - stmt, result = await self.construct_query( - session, single_resolve_request - ) - single_resolve_response = self.construct_single_resolve( - single_resolve_request, result - ) - - single_resolve_responses.append( - self.construct_single_resolve_response_for_success( - single_resolve_response - ) + validated_request = SingleResolveRequest.model_validate( + single_resolve_request ) + validated_requests.append(validated_request) except ResolveValidationException as e: single_resolve_responses.append( self.construct_single_resolve_response_for_failure( single_resolve_request, e ) ) + + # Construct and execute bulk query + stmt, results = await self.construct_bulk_query(session, id_values) + + # Create responses for all validated requests + single_resolve_responses = [] + for validated_request in validated_requests: + + result = next( + (res for res in results if res.id_value == validated_request.id), None + ) + if result: + single_resolve_response = self.construct_single_resolve( + validated_request, result + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_success( + single_resolve_response + ) + ) + else: + error = ResolveValidationException( + message="ID doesnt exist please link first", + status=StatusEnum.succ, + validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_failure( + validated_request, error + ) + ) + await session.commit() return single_resolve_responses @@ -369,46 +396,3 @@ def construct_single_unlink_response_for_failure( additional_info=None, locale=single_unlink_request.locale, ) - - async def resolve_bulk(self, resolve_request: ResolveRequest): - session_initializer = SessionInitializer.get_component() - session: AsyncSession = await session_initializer.get_session_from_pool() - async with session.begin(): - resolve_request_message: ResolveRequestMessage = resolve_request.message - - id_values = [ - single_resolve_request.id - for single_resolve_request in resolve_request_message.resolve_request - ] - - for single_resolve_request in resolve_request_message.resolve_request: - try: - await IdFaMappingValidations.get_component().validate_resolve_request( - single_resolve_request=single_resolve_request, - ) - single_resolve_request: SingleResolveRequest = ( - SingleResolveRequest.model_validate(single_resolve_request) - ) - except ResolveValidationException as e: - print(e) - stmt, results = await self.construct_bulk_query(session, id_values) - single_resolve_responses = [] - for result in results: - single_resolve_request = SingleResolveRequest( - id=result.id_value, - fa=result.fa_value, - reference_id="None", - timestamp=datetime.now(), - additional_info=result.additional_info, - locale=None, - ) - single_resolve_response = self.construct_single_resolve( - single_resolve_request, result - ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_success( - single_resolve_response - ) - ) - await session.commit() - return single_resolve_responses From 7cee8d6c86aefaa4a4037c06ef630c026992757e Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 13:05:38 +0530 Subject: [PATCH 21/39] Remove bulk API --- .../controllers/sync_mapper_controller.py | 30 ------------------- 1 file changed, 30 deletions(-) diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 433872b..231e5c0 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -54,12 +54,6 @@ def __init__(self, **kwargs): responses={200: {"model": UnlinkResponse}}, methods=["POST"], ) - self.router.add_api_route( - "/resolve_bulk", - self.resolve_bulk_sync, - responses={200: {"model": ResolveResponse}}, - methods=["POST"], - ) async def link_sync(self, link_request: LinkRequest): try: @@ -152,27 +146,3 @@ async def unlink_sync(self, unlink_request: UnlinkRequest): single_unlink_responses, ) ) - - async def resolve_bulk_sync(self, resolve_request: ResolveRequest): - try: - RequestValidation.get_component().validate_request(resolve_request) - RequestValidation.get_component().validate_resolve_request_header( - resolve_request - ) - except RequestValidationException as e: - error_response = ( - SyncResponseHelper.get_component().construct_error_sync_response( - resolve_request, e - ) - ) - return error_response - - single_resolve_responses: list[SingleResolveResponse] = ( - await self.mapper_service.resolve_bulk(resolve_request) - ) - return ( - SyncResponseHelper.get_component().construct_success_sync_resolve_response( - resolve_request, - single_resolve_responses, - ) - ) From 63f5e7b32fd7bef27dc33bbb65116cf9bb535003 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 13:18:39 +0530 Subject: [PATCH 22/39] Add bulk API --- .../controllers/sync_mapper_controller.py | 30 +++++++++ .../services/mapper.py | 65 +++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 231e5c0..433872b 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -54,6 +54,12 @@ def __init__(self, **kwargs): responses={200: {"model": UnlinkResponse}}, methods=["POST"], ) + self.router.add_api_route( + "/resolve_bulk", + self.resolve_bulk_sync, + responses={200: {"model": ResolveResponse}}, + methods=["POST"], + ) async def link_sync(self, link_request: LinkRequest): try: @@ -146,3 +152,27 @@ async def unlink_sync(self, unlink_request: UnlinkRequest): single_unlink_responses, ) ) + + async def resolve_bulk_sync(self, resolve_request: ResolveRequest): + try: + RequestValidation.get_component().validate_request(resolve_request) + RequestValidation.get_component().validate_resolve_request_header( + resolve_request + ) + except RequestValidationException as e: + error_response = ( + SyncResponseHelper.get_component().construct_error_sync_response( + resolve_request, e + ) + ) + return error_response + + single_resolve_responses: list[SingleResolveResponse] = ( + await self.mapper_service.resolve_bulk(resolve_request) + ) + return ( + SyncResponseHelper.get_component().construct_success_sync_resolve_response( + resolve_request, + single_resolve_responses, + ) + ) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index c962d87..526f729 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -396,3 +396,68 @@ def construct_single_unlink_response_for_failure( additional_info=None, locale=single_unlink_request.locale, ) + + async def resolve_bulk(self, resolve_request: ResolveRequest): + session_initializer = SessionInitializer.get_component() + session: AsyncSession = await session_initializer.get_session_from_pool() + async with session.begin(): + resolve_request_message: ResolveRequestMessage = resolve_request.message + + # Collect all ID values for bulk query + id_values = [ + single_resolve_request.id + for single_resolve_request in resolve_request_message.resolve_request + ] + + # Validate all requests and collect validated requests + validated_requests = [] + single_resolve_responses = [] + for single_resolve_request in resolve_request_message.resolve_request: + try: + await IdFaMappingValidations.get_component().validate_resolve_request( + single_resolve_request=single_resolve_request, + ) + validated_request = SingleResolveRequest.model_validate( + single_resolve_request + ) + validated_requests.append(validated_request) + except ResolveValidationException as e: + single_resolve_responses.append( + self.construct_single_resolve_response_for_failure( + single_resolve_request, e + ) + ) + + # Construct and execute bulk query + if validated_requests: + stmt, results = await self.construct_bulk_query(session, id_values) + + # Create a dictionary for fast lookup of results by ID + result_dict = {result.id_value: result for result in results} + + # Create responses for all validated requests + for validated_request in validated_requests: + result = result_dict.get(validated_request.id) + if result: + single_resolve_response = self.construct_single_resolve( + validated_request, result + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_success( + single_resolve_response + ) + ) + else: + error = ResolveValidationException( + message="ID doesn't exist, please link first", + status=StatusEnum.succ, + validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_failure( + validated_request, error + ) + ) + + await session.commit() + return single_resolve_responses From 6f1972e042fb71246616ee296d1ae71f8fa99d12 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 13:19:42 +0530 Subject: [PATCH 23/39] Pre-commit fixes --- src/openg2p_spar_mapper_api/app.py | 2 +- .../controllers/sync_mapper_controller.py | 30 +++++++++---------- .../services/mapper.py | 3 +- .../services/session_service.py | 2 +- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/openg2p_spar_mapper_api/app.py b/src/openg2p_spar_mapper_api/app.py index 670d171..498b2e5 100644 --- a/src/openg2p_spar_mapper_api/app.py +++ b/src/openg2p_spar_mapper_api/app.py @@ -18,9 +18,9 @@ IdFaMappingValidations, MapperService, RequestValidation, + SessionInitializer, SyncRequestHelper, SyncResponseHelper, - SessionInitializer, ) diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 433872b..743b5d7 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -73,9 +73,9 @@ async def link_sync(self, link_request: LinkRequest): ) return error_response - single_link_responses: list[SingleLinkResponse] = ( - await self.mapper_service.link(link_request) - ) + single_link_responses: list[ + SingleLinkResponse + ] = await self.mapper_service.link(link_request) return SyncResponseHelper.get_component().construct_success_sync_link_response( link_request, single_link_responses, @@ -95,9 +95,9 @@ async def update_sync(self, update_request: UpdateRequest): ) return error_response - single_update_responses: list[SingleUpdateResponse] = ( - await self.mapper_service.update(update_request) - ) + single_update_responses: list[ + SingleUpdateResponse + ] = await self.mapper_service.update(update_request) return ( SyncResponseHelper.get_component().construct_success_sync_update_response( update_request, @@ -119,9 +119,9 @@ async def resolve_sync(self, resolve_request: ResolveRequest): ) return error_response - single_resolve_responses: list[SingleResolveResponse] = ( - await self.mapper_service.resolve(resolve_request) - ) + single_resolve_responses: list[ + SingleResolveResponse + ] = await self.mapper_service.resolve(resolve_request) return ( SyncResponseHelper.get_component().construct_success_sync_resolve_response( resolve_request, @@ -143,9 +143,9 @@ async def unlink_sync(self, unlink_request: UnlinkRequest): ) return error_response - single_unlink_responses: list[SingleResolveResponse] = ( - await self.mapper_service.unlink(unlink_request) - ) + single_unlink_responses: list[ + SingleResolveResponse + ] = await self.mapper_service.unlink(unlink_request) return ( SyncResponseHelper.get_component().construct_success_sync_unlink_response( unlink_request, @@ -167,9 +167,9 @@ async def resolve_bulk_sync(self, resolve_request: ResolveRequest): ) return error_response - single_resolve_responses: list[SingleResolveResponse] = ( - await self.mapper_service.resolve_bulk(resolve_request) - ) + single_resolve_responses: list[ + SingleResolveResponse + ] = await self.mapper_service.resolve_bulk(resolve_request) return ( SyncResponseHelper.get_component().construct_success_sync_resolve_response( resolve_request, diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index 526f729..cdf6c8e 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -25,7 +25,7 @@ UpdateStatusReasonCode, ) from sqlalchemy import delete, select -from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from ..config import Settings from ..models import IdFaMapping @@ -240,7 +240,6 @@ async def resolve(self, resolve_request: ResolveRequest): # Create responses for all validated requests single_resolve_responses = [] for validated_request in validated_requests: - result = next( (res for res in results if res.id_value == validated_request.id), None ) diff --git a/src/openg2p_spar_mapper_api/services/session_service.py b/src/openg2p_spar_mapper_api/services/session_service.py index 732dfa4..29d3d0b 100644 --- a/src/openg2p_spar_mapper_api/services/session_service.py +++ b/src/openg2p_spar_mapper_api/services/session_service.py @@ -1,6 +1,6 @@ from openg2p_fastapi_common.context import dbengine from openg2p_fastapi_common.service import BaseService -from sqlalchemy.ext.asyncio import async_sessionmaker, AsyncSession +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker class SessionInitializer(BaseService): From 3c6677f669d35149cbccecf2d50282bffc5e80be Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 14:00:32 +0530 Subject: [PATCH 24/39] Remove bulk API --- .../controllers/sync_mapper_controller.py | 30 ----- .../services/mapper.py | 125 +++++------------- 2 files changed, 31 insertions(+), 124 deletions(-) diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 743b5d7..5d00139 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -54,12 +54,6 @@ def __init__(self, **kwargs): responses={200: {"model": UnlinkResponse}}, methods=["POST"], ) - self.router.add_api_route( - "/resolve_bulk", - self.resolve_bulk_sync, - responses={200: {"model": ResolveResponse}}, - methods=["POST"], - ) async def link_sync(self, link_request: LinkRequest): try: @@ -152,27 +146,3 @@ async def unlink_sync(self, unlink_request: UnlinkRequest): single_unlink_responses, ) ) - - async def resolve_bulk_sync(self, resolve_request: ResolveRequest): - try: - RequestValidation.get_component().validate_request(resolve_request) - RequestValidation.get_component().validate_resolve_request_header( - resolve_request - ) - except RequestValidationException as e: - error_response = ( - SyncResponseHelper.get_component().construct_error_sync_response( - resolve_request, e - ) - ) - return error_response - - single_resolve_responses: list[ - SingleResolveResponse - ] = await self.mapper_service.resolve_bulk(resolve_request) - return ( - SyncResponseHelper.get_component().construct_success_sync_resolve_response( - resolve_request, - single_resolve_responses, - ) - ) diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index cdf6c8e..eb4646b 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -209,7 +209,7 @@ async def resolve(self, resolve_request: ResolveRequest): session: AsyncSession = await session_initializer.get_session_from_pool() async with session.begin(): resolve_request_message: ResolveRequestMessage = resolve_request.message - single_resolve_responses: list[SingleResolveResponse] = [] + # Collect all ID values for bulk query id_values = [ single_resolve_request.id @@ -218,6 +218,7 @@ async def resolve(self, resolve_request: ResolveRequest): # Validate all requests and collect validated requests validated_requests = [] + single_resolve_responses = [] for single_resolve_request in resolve_request_message.resolve_request: try: await IdFaMappingValidations.get_component().validate_resolve_request( @@ -235,36 +236,37 @@ async def resolve(self, resolve_request: ResolveRequest): ) # Construct and execute bulk query - stmt, results = await self.construct_bulk_query(session, id_values) + if validated_requests: + stmt, results = await self.construct_bulk_query(session, id_values) - # Create responses for all validated requests - single_resolve_responses = [] - for validated_request in validated_requests: - result = next( - (res for res in results if res.id_value == validated_request.id), None - ) - if result: - single_resolve_response = self.construct_single_resolve( - validated_request, result - ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_success( - single_resolve_response - ) - ) - else: - error = ResolveValidationException( - message="ID doesnt exist please link first", - status=StatusEnum.succ, - validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, - ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_failure( - validated_request, error - ) - ) + # Create a dictionary for fast lookup of results by ID + result_dict = {result.id_value: result for result in results} - await session.commit() + # Create responses for all validated requests + for validated_request in validated_requests: + result = result_dict.get(validated_request.id) + if result: + single_resolve_response = self.construct_single_resolve( + validated_request, result + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_success( + single_resolve_response + ) + ) + else: + error = ResolveValidationException( + message="ID doesn't exist, please link first", + status=StatusEnum.succ, + validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, + ) + single_resolve_responses.append( + self.construct_single_resolve_response_for_failure( + validated_request, error + ) + ) + + await session.commit() return single_resolve_responses def construct_single_resolve( @@ -395,68 +397,3 @@ def construct_single_unlink_response_for_failure( additional_info=None, locale=single_unlink_request.locale, ) - - async def resolve_bulk(self, resolve_request: ResolveRequest): - session_initializer = SessionInitializer.get_component() - session: AsyncSession = await session_initializer.get_session_from_pool() - async with session.begin(): - resolve_request_message: ResolveRequestMessage = resolve_request.message - - # Collect all ID values for bulk query - id_values = [ - single_resolve_request.id - for single_resolve_request in resolve_request_message.resolve_request - ] - - # Validate all requests and collect validated requests - validated_requests = [] - single_resolve_responses = [] - for single_resolve_request in resolve_request_message.resolve_request: - try: - await IdFaMappingValidations.get_component().validate_resolve_request( - single_resolve_request=single_resolve_request, - ) - validated_request = SingleResolveRequest.model_validate( - single_resolve_request - ) - validated_requests.append(validated_request) - except ResolveValidationException as e: - single_resolve_responses.append( - self.construct_single_resolve_response_for_failure( - single_resolve_request, e - ) - ) - - # Construct and execute bulk query - if validated_requests: - stmt, results = await self.construct_bulk_query(session, id_values) - - # Create a dictionary for fast lookup of results by ID - result_dict = {result.id_value: result for result in results} - - # Create responses for all validated requests - for validated_request in validated_requests: - result = result_dict.get(validated_request.id) - if result: - single_resolve_response = self.construct_single_resolve( - validated_request, result - ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_success( - single_resolve_response - ) - ) - else: - error = ResolveValidationException( - message="ID doesn't exist, please link first", - status=StatusEnum.succ, - validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, - ) - single_resolve_responses.append( - self.construct_single_resolve_response_for_failure( - validated_request, error - ) - ) - - await session.commit() - return single_resolve_responses From f189ce86512bc8ab8398d9d7b7841f0cc6e45959 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 14:59:52 +0530 Subject: [PATCH 25/39] Increase workers to 12 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index f487d31..e6de629 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "8", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "12", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] From 3b44b14ae5151185a3516102bf16bc03d0ab0308 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 15:38:54 +0530 Subject: [PATCH 26/39] Rebuild docker --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5ed1f89..98e0008 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,4 @@ FastAPI based micro service providing Beneficiary ID and Financial Account Mappi ## Licenses This repository is licensed under [MPL-2.0](LICENSE). + From 5fa7b8d560cb0dd9b72fe0f9ff8b73436dfc4541 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 16:11:15 +0530 Subject: [PATCH 27/39] Refactor --- .../services/id_fa_mapping_validations.py | 14 -------------- src/openg2p_spar_mapper_api/services/mapper.py | 4 ++-- 2 files changed, 2 insertions(+), 16 deletions(-) diff --git a/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py b/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py index 5fd31aa..56f424d 100644 --- a/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py +++ b/src/openg2p_spar_mapper_api/services/id_fa_mapping_validations.py @@ -102,20 +102,6 @@ async def validate_resolve_request( status=StatusEnum.rjct, validation_error_type=ResolveStatusReasonCode.rjct_reference_id_invalid, ) - - # result = await connection.execute( - # select(IdFaMapping).where( - # IdFaMapping.id_value == single_resolve_request.id, - # ) - # ) - # resolve_request_from_db = result.first() - # - # if not resolve_request_from_db: - # raise ResolveValidationException( - # message="ID doesnt exist please link first", - # status=StatusEnum.succ, - # validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, - # ) return None async def validate_unlink_request( diff --git a/src/openg2p_spar_mapper_api/services/mapper.py b/src/openg2p_spar_mapper_api/services/mapper.py index eb4646b..be20b8f 100644 --- a/src/openg2p_spar_mapper_api/services/mapper.py +++ b/src/openg2p_spar_mapper_api/services/mapper.py @@ -255,14 +255,14 @@ async def resolve(self, resolve_request: ResolveRequest): ) ) else: - error = ResolveValidationException( + resolve_validation_exception = ResolveValidationException( message="ID doesn't exist, please link first", status=StatusEnum.succ, validation_error_type=ResolveStatusReasonCode.succ_fa_not_linked_to_id, ) single_resolve_responses.append( self.construct_single_resolve_response_for_failure( - validated_request, error + validated_request, resolve_validation_exception ) ) From 5a11a4041347172ef4ca3bf9bc2786992b4a9b49 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 15 May 2024 16:11:36 +0530 Subject: [PATCH 28/39] Increase workers to 16 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e6de629..9a73e6d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "12", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "16", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] From d7876aadcf42ce9b937a8219902136f137f1fbec Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 16 May 2024 12:16:28 +0530 Subject: [PATCH 29/39] Set workers to 8 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9a73e6d..f487d31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,4 +28,4 @@ RUN python3 -m pip install \ openg2p-g2pconnect-mapper-lib==1.0.0 \ ./src -CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "16", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] +CMD ["/home/openg2p/.local/bin/gunicorn", "-w", "8", "-k", "uvicorn.workers.UvicornWorker", "main:mapper_app", "--bind", "0.0.0.0:8000"] From 92d0df2f2a7568a8b45b5540b74323e4e546cf05 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 19 Jun 2024 15:01:38 +0530 Subject: [PATCH 30/39] Sync API Signature Validation --- main.py | 4 +-- src/openg2p_spar_mapper_api/app.py | 2 ++ src/openg2p_spar_mapper_api/config.py | 2 +- .../controllers/sync_mapper_controller.py | 26 ++++++++++++++++--- .../services/request_validations.py | 14 +++++++--- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/main.py b/main.py index 4d282a8..56554ae 100755 --- a/main.py +++ b/main.py @@ -5,12 +5,12 @@ from openg2p_spar_mapper_api.app import Initializer from openg2p_fastapi_common.ping import PingInitializer -from openg2p_fastapi_common.context import app_registry + initializer = Initializer() PingInitializer() -app = main_init.return_app() +app = initializer.return_app() if __name__ == "__main__": initializer.main() diff --git a/src/openg2p_spar_mapper_api/app.py b/src/openg2p_spar_mapper_api/app.py index 498b2e5..509cb45 100644 --- a/src/openg2p_spar_mapper_api/app.py +++ b/src/openg2p_spar_mapper_api/app.py @@ -6,6 +6,7 @@ _config = Settings.get_config() from openg2p_fastapi_common.app import Initializer as BaseInitializer +from openg2p_g2pconnect_common_lib.oauth_token import OAuthTokenService from .controllers import ( AsyncMapperController, @@ -28,6 +29,7 @@ class Initializer(BaseInitializer): def initialize(self, **kwargs): super().initialize() + OAuthTokenService() SessionInitializer() MapperService() IdFaMappingValidations() diff --git a/src/openg2p_spar_mapper_api/config.py b/src/openg2p_spar_mapper_api/config.py index 276a2b0..eaccece 100644 --- a/src/openg2p_spar_mapper_api/config.py +++ b/src/openg2p_spar_mapper_api/config.py @@ -1,6 +1,6 @@ from typing import Optional -from openg2p_fastapi_common.config import Settings as BaseSettings +from openg2p_g2pconnect_common_lib.config import Settings as BaseSettings from pydantic import AnyUrl from pydantic_settings import SettingsConfigDict diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 5d00139..3c996e9 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -1,4 +1,8 @@ +from typing import Annotated + +from fastapi import Depends from openg2p_fastapi_common.controller import BaseController +from openg2p_g2pconnect_common_lib.jwt_signature_validator import JWTSignatureValidator from openg2p_g2pconnect_mapper_lib.schemas import ( LinkRequest, LinkResponse, @@ -55,8 +59,13 @@ def __init__(self, **kwargs): methods=["POST"], ) - async def link_sync(self, link_request: LinkRequest): + async def link_sync( + self, + link_request: LinkRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], + ): try: + RequestValidation.get_component().validate_signature(is_signature_valid) RequestValidation.get_component().validate_request(link_request) RequestValidation.get_component().validate_link_request_header(link_request) except RequestValidationException as e: @@ -75,7 +84,10 @@ async def link_sync(self, link_request: LinkRequest): single_link_responses, ) - async def update_sync(self, update_request: UpdateRequest): + async def update_sync( + self, + update_request: UpdateRequest, + ): try: RequestValidation.get_component().validate_request(update_request) RequestValidation.get_component().validate_update_request_header( @@ -99,7 +111,10 @@ async def update_sync(self, update_request: UpdateRequest): ) ) - async def resolve_sync(self, resolve_request: ResolveRequest): + async def resolve_sync( + self, + resolve_request: ResolveRequest, + ): try: RequestValidation.get_component().validate_request(resolve_request) RequestValidation.get_component().validate_resolve_request_header( @@ -123,7 +138,10 @@ async def resolve_sync(self, resolve_request: ResolveRequest): ) ) - async def unlink_sync(self, unlink_request: UnlinkRequest): + async def unlink_sync( + self, + unlink_request: UnlinkRequest, + ): try: RequestValidation.get_component().validate_request(unlink_request) RequestValidation.get_component().validate_unlink_request_header( diff --git a/src/openg2p_spar_mapper_api/services/request_validations.py b/src/openg2p_spar_mapper_api/services/request_validations.py index 7b08fe8..854e03a 100644 --- a/src/openg2p_spar_mapper_api/services/request_validations.py +++ b/src/openg2p_spar_mapper_api/services/request_validations.py @@ -1,15 +1,23 @@ from openg2p_fastapi_common.service import BaseService from openg2p_g2pconnect_common_lib.schemas import ( - SyncResponseStatusReasonCodeEnum, -) -from openg2p_g2pconnect_common_lib.schemas.async_schemas import ( AsyncResponseStatusReasonCodeEnum, + SecurityErrorCodes, + SyncResponseStatusReasonCodeEnum, ) from .exceptions import RequestValidationException class RequestValidation(BaseService): + def validate_signature(self, is_signature_valid) -> None: + if not is_signature_valid: + raise RequestValidationException( + code=SecurityErrorCodes.INVALID_JWT_SIGNATURE, + message=SecurityErrorCodes.INVALID_JWT_SIGNATURE, + ) + + return None + def validate_link_request_header(self, request) -> None: if request.header.action != "link": raise RequestValidationException( From 044148fa1e78dc3ce9b84a41fea03687c7b58af0 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 19 Jun 2024 15:02:01 +0530 Subject: [PATCH 31/39] Unit Tests: Sync API Signature Validation --- tests/test_sync_mapper_controller.py | 32 ++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/tests/test_sync_mapper_controller.py b/tests/test_sync_mapper_controller.py index 0432e34..2e34a25 100644 --- a/tests/test_sync_mapper_controller.py +++ b/tests/test_sync_mapper_controller.py @@ -4,6 +4,7 @@ import pytest from openg2p_g2pconnect_common_lib.schemas import ( RequestHeader, + SecurityErrorCodes, StatusEnum, SyncResponseHeader, SyncResponseStatusReasonCodeEnum, @@ -49,6 +50,14 @@ ) +def mock_validate_signature(is_signature_valid): + if not is_signature_valid: + raise RequestValidationException( + code=SecurityErrorCodes.INVALID_JWT_SIGNATURE, + message=SecurityErrorCodes.INVALID_JWT_SIGNATURE, + ) + + @pytest.fixture def setup_link_controller(): controller = SyncMapperController() @@ -56,7 +65,9 @@ def setup_link_controller(): request_validation_mock = MagicMock() request_validation_mock.validate_request = MagicMock(return_value=True) request_validation_mock.validate_link_request_header = MagicMock(return_value=True) - + request_validation_mock.validate_signature = MagicMock( + side_effect=mock_validate_signature + ) mock_link_response = LinkResponse( header=SyncResponseHeader( message_id="test_message_id", @@ -102,10 +113,10 @@ def setup_link_controller(): link_response=[], ), ) - # Mock SyncResponseHelper for error scenario response_helper_link_mock.construct_error_sync_response.return_value = ( mock_error_link_response ) + with patch( "openg2p_spar_mapper_api.services.RequestValidation.get_component", return_value=request_validation_mock, @@ -423,12 +434,23 @@ def setup_unlink_controller(): async def test_link_sync_success(setup_link_controller): controller, mock_link_request = setup_link_controller assert controller is not None - response = await controller.link_sync(mock_link_request) + + response = await controller.link_sync(mock_link_request, is_signature_valid=True) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.link.assert_called_once_with(mock_link_request) +@pytest.mark.asyncio +async def test_link_sync_invalid_signature(setup_link_controller): + controller, mock_link_request = setup_link_controller + assert controller is not None + + response = await controller.link_sync(mock_link_request, is_signature_valid=False) + assert response.header.status == StatusEnum.rjct + assert response.header.status_reason_message == "Validation error" + + @pytest.mark.asyncio async def test_update_sync_success(setup_update_controller): controller, mock_update_request = setup_update_controller @@ -475,7 +497,9 @@ async def test_link_sync_validation_error(setup_link_controller): "validate_link_request_header", side_effect=validation_error, ): - response = await controller.link_sync(mock_link_request) + response = await controller.link_sync( + mock_link_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.link.assert_not_called() From 3ef6221fa0b62236cbae420cc571d67e16d4e5e9 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 19 Jun 2024 15:18:50 +0530 Subject: [PATCH 32/39] Async Mapper Link API Security Signature Validation --- .../controllers/async_mapper_controller.py | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py index 967919e..269ace9 100644 --- a/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py @@ -1,9 +1,12 @@ import asyncio import logging import uuid +from typing import Annotated import httpx +from fastapi import Depends from openg2p_fastapi_common.controller import BaseController +from openg2p_g2pconnect_common_lib.jwt_signature_validator import JWTSignatureValidator from openg2p_g2pconnect_common_lib.schemas import ( AsyncCallbackRequest, AsyncResponse, @@ -89,8 +92,21 @@ def __init__(self, **kwargs): methods=["POST"], ) - async def link_async(self, link_request: LinkRequest): + async def link_async( + self, + link_request: LinkRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], + ): correlation_id = str(uuid.uuid4()) + try: + RequestValidation.get_component().validate_signature(is_signature_valid) + except RequestValidationException as e: + error_response = ( + AsyncResponseHelper.get_component().construct_error_async_response( + link_request, e + ) + ) + return error_response await asyncio.create_task( self.handle_service_and_link_callback(link_request, correlation_id, "link") ) @@ -148,16 +164,19 @@ async def unlink_async(self, unlink_request: UnlinkRequest): ) async def handle_service_and_link_callback( - self, link_request: LinkRequest, correlation_id: str, action: str + self, + link_request: LinkRequest, + correlation_id: str, + action: str, ): try: RequestValidation.get_component().validate_async_request(link_request) RequestValidation.get_component().validate_link_async_request_header( link_request ) - single_link_responses: list[ - SingleLinkResponse - ] = await self.action_to_method[action](link_request) + single_link_responses: list[SingleLinkResponse] = ( + await self.action_to_method[action](link_request) + ) async_call_back_request: ( AsyncCallbackRequest @@ -188,9 +207,9 @@ async def handle_service_and_update_callback( RequestValidation.get_component().validate_update_async_request_header( request ) - single_update_responses: list[ - SingleUpdateResponse - ] = await self.action_to_method[action](request) + single_update_responses: list[SingleUpdateResponse] = ( + await self.action_to_method[action](request) + ) async_call_back_request: ( AsyncCallbackRequest ) = AsyncResponseHelper.get_component().construct_success_async_callback_update_request( @@ -220,9 +239,9 @@ async def handle_service_and_resolve_callback( RequestValidation.get_component().validate_resolve_async_request_header( request ) - single_resolve_responses: list[ - SingleResolveResponse - ] = await self.action_to_method[action](request) + single_resolve_responses: list[SingleResolveResponse] = ( + await self.action_to_method[action](request) + ) async_call_back_request: ( AsyncCallbackRequest ) = AsyncResponseHelper.get_component().construct_success_async_callback_resolve_request( @@ -252,9 +271,9 @@ async def handle_service_and_unlink_callback( RequestValidation.get_component().validate_unlink_async_request_header( request ) - single_unlink_responses: list[ - SingleUnlinkResponse - ] = await self.action_to_method[action](request) + single_unlink_responses: list[SingleUnlinkResponse] = ( + await self.action_to_method[action](request) + ) async_call_back_request: ( AsyncCallbackRequest ) = AsyncResponseHelper.get_component().construct_success_async_callback_unlink_request( From 1abdd640445b44cba2545a0e626a17fc71105b17 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 19 Jun 2024 15:18:59 +0530 Subject: [PATCH 33/39] Unit Test: Async Mapper Link API Security Signature Validation --- tests/test_async_mapper_controller.py | 103 +++++++++++++++++++++++++- 1 file changed, 102 insertions(+), 1 deletion(-) diff --git a/tests/test_async_mapper_controller.py b/tests/test_async_mapper_controller.py index 3c113a9..02ee217 100644 --- a/tests/test_async_mapper_controller.py +++ b/tests/test_async_mapper_controller.py @@ -11,6 +11,7 @@ AsyncResponseMessage, RequestHeader, StatusEnum, + SecurityErrorCodes, ) from openg2p_g2pconnect_mapper_lib.schemas import ( LinkRequest, @@ -39,6 +40,18 @@ from openg2p_spar_mapper_api.controllers.async_mapper_controller import ( AsyncMapperController, ) +from openg2p_spar_mapper_api.services import ( + RequestValidationException, +) + + +# Setup side effect for validate_signature to raise RequestValidationException +def mock_validate_signature(is_signature_valid): + if not is_signature_valid: + raise RequestValidationException( + code=SecurityErrorCodes.INVALID_JWT_SIGNATURE, + message=SecurityErrorCodes.INVALID_JWT_SIGNATURE, + ) @pytest.mark.asyncio @@ -68,6 +81,11 @@ async def test_link_async( mock_request_validation_get_component.return_value = ( mock_request_validation_instance ) + + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + mock_async_response_helper_instance = MagicMock() expected_response = AsyncResponse( message=AsyncResponseMessage( @@ -106,7 +124,9 @@ async def test_link_async( ), ) - actual_response = await controller.link_async(mock_link_request) + actual_response = await controller.link_async( + mock_link_request, is_signature_valid=True + ) assert ( actual_response == expected_response ), "The response did not match the expected response." @@ -115,6 +135,87 @@ async def test_link_async( assert actual_response.message.timestamp == expected_response.message.timestamp +@pytest.mark.asyncio +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.RequestValidation.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.MapperService.get_component" +) +async def test_link_async_invalid_signature( + mock_mapper_service_get_component, + mock_request_validation_get_component, + mock_async_response_helper_get_component, +): + # Setup MagicMock for MapperService and RequestValidation components + mock_mapper_service_instance = MagicMock() + mock_mapper_service_instance.link = AsyncMock() + mock_request_validation_instance = MagicMock() + + # Assign return values to the get_component mocks + mock_mapper_service_get_component.return_value = mock_mapper_service_instance + mock_request_validation_get_component.return_value = ( + mock_request_validation_instance + ) + + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + + # Setup MagicMock for AsyncResponseHelper component + mock_async_response_helper_instance = MagicMock() + error_response = AsyncResponse( + message=AsyncResponseMessage( + correlation_id="error_correlation_id", + timestamp=datetime.utcnow().isoformat(), + ack_status="NACK", + ) + ) + mock_async_response_helper_instance.construct_error_async_response.return_value = ( + error_response + ) + mock_async_response_helper_get_component.return_value = ( + mock_async_response_helper_instance + ) + + controller = AsyncMapperController() + + mock_link_request = LinkRequest( + header=RequestHeader( + message_id="test_message_id", + message_ts=datetime.utcnow().isoformat(), + action="link", + sender_id="test_sender", + total_count=1, + ), + message=LinkRequestMessage( + transaction_id="test_transaction_id", + link_request=[ + SingleLinkRequest( + reference_id="test_ref", + timestamp=datetime.utcnow(), + id="test_id", + fa="test_fa", + ) + ], + ), + ) + + actual_response = await controller.link_async( + mock_link_request, is_signature_valid=False + ) + + assert ( + actual_response == error_response + ), "The response did not match the expected error response." + assert actual_response.message.correlation_id == "error_correlation_id" + assert actual_response.message.ack_status == AsyncAck.NACK + assert actual_response.message.timestamp == error_response.message.timestamp + + @pytest.mark.asyncio @patch( "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" From b1d45b5bff1c44c9ee3d75001ce2dc02e893c2f4 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Wed, 19 Jun 2024 15:28:02 +0530 Subject: [PATCH 34/39] Unit Test: Add Validation Error Message assertion --- tests/test_async_mapper_controller.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_async_mapper_controller.py b/tests/test_async_mapper_controller.py index 02ee217..27f03fc 100644 --- a/tests/test_async_mapper_controller.py +++ b/tests/test_async_mapper_controller.py @@ -172,6 +172,10 @@ async def test_link_async_invalid_signature( correlation_id="error_correlation_id", timestamp=datetime.utcnow().isoformat(), ack_status="NACK", + error={ + "code": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + "message": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + }, ) ) mock_async_response_helper_instance.construct_error_async_response.return_value = ( @@ -214,6 +218,8 @@ async def test_link_async_invalid_signature( assert actual_response.message.correlation_id == "error_correlation_id" assert actual_response.message.ack_status == AsyncAck.NACK assert actual_response.message.timestamp == error_response.message.timestamp + assert actual_response.message.error.code == "INVALID JWT SIGNATURE" + assert actual_response.message.error.message == "INVALID JWT SIGNATURE" @pytest.mark.asyncio From 5ca5d6d931ef33b996aeea54b6932ebddee4a43d Mon Sep 17 00:00:00 2001 From: vineela-afk Date: Wed, 19 Jun 2024 16:22:09 +0530 Subject: [PATCH 35/39] generated the rest test functions --- tests/test_async_mapper_controller.py | 277 +++++++++++++++++++++++++- tests/test_sync_mapper_controller.py | 66 ++++-- 2 files changed, 324 insertions(+), 19 deletions(-) diff --git a/tests/test_async_mapper_controller.py b/tests/test_async_mapper_controller.py index 27f03fc..24287fe 100644 --- a/tests/test_async_mapper_controller.py +++ b/tests/test_async_mapper_controller.py @@ -320,6 +320,10 @@ async def test_update_async( mock_request_validation_get_component.return_value = ( mock_request_validation_instance ) + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + mock_async_response_helper_instance = MagicMock() expected_response = AsyncResponse( message=AsyncResponseMessage( @@ -358,7 +362,7 @@ async def test_update_async( ), ) - actual_response = await controller.update_async(mock_update_request) + actual_response = await controller.update_async(mock_update_request, is_signature_valid=True) assert ( actual_response == expected_response ), "The response did not match the expected response." @@ -366,6 +370,92 @@ async def test_update_async( assert actual_response.message.ack_status == AsyncAck.ACK assert actual_response.message.timestamp == expected_response.message.timestamp +@pytest.mark.asyncio +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.RequestValidation.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.MapperService.get_component" +) +async def test_update_async_invalid_signature( + mock_mapper_service_get_component, + mock_request_validation_get_component, + mock_async_response_helper_get_component, +): + # Setup MagicMock for MapperService and RequestValidation components + mock_mapper_service_instance = MagicMock() + mock_mapper_service_instance.update = AsyncMock() + mock_request_validation_instance = MagicMock() + + # Assign return values to the get_component mocks + mock_mapper_service_get_component.return_value = mock_mapper_service_instance + mock_request_validation_get_component.return_value = ( + mock_request_validation_instance + ) + + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + + # Setup MagicMock for AsyncResponseHelper component + mock_async_response_helper_instance = MagicMock() + error_response = AsyncResponse( + message=AsyncResponseMessage( + correlation_id="error_correlation_id", + timestamp=datetime.utcnow().isoformat(), + ack_status="NACK", + error={ + "code": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + "message": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + }, + ) + ) + mock_async_response_helper_instance.construct_error_async_response.return_value = ( + error_response + ) + mock_async_response_helper_get_component.return_value = ( + mock_async_response_helper_instance + ) + + controller = AsyncMapperController() + + mock_update_request = UpdateRequest( + header=RequestHeader( + message_id="test_message_id", + message_ts=datetime.utcnow().isoformat(), + action="update", + sender_id="test_sender", + total_count=1, + ), + message=UpdateRequestMessage( + transaction_id="test_transaction_id", + update_request=[ + SingleUpdateRequest( + reference_id="test_ref", + timestamp=datetime.utcnow(), + id="test_id", + fa="test_fa", + ) + ], + ), + ) + + actual_response = await controller.update_async( + mock_update_request, is_signature_valid=False + ) + + assert ( + actual_response == error_response + ), "The response did not match the expected error response." + assert actual_response.message.correlation_id == "error_correlation_id" + assert actual_response.message.ack_status == AsyncAck.NACK + assert actual_response.message.timestamp == error_response.message.timestamp + assert actual_response.message.error.code == "INVALID JWT SIGNATURE" + assert actual_response.message.error.message == "INVALID JWT SIGNATURE" + @pytest.mark.asyncio @patch( @@ -463,6 +553,10 @@ async def test_resolve_async( mock_request_validation_get_component.return_value = ( mock_request_validation_instance ) + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + mock_async_response_helper_instance = MagicMock() expected_response = AsyncResponse( message=AsyncResponseMessage( @@ -501,7 +595,7 @@ async def test_resolve_async( ), ) - actual_response = await controller.resolve_async(mock_resolve_request) + actual_response = await controller.resolve_async(mock_resolve_request, is_signature_valid=True) assert ( actual_response == expected_response ), "The response did not match the expected response." @@ -509,6 +603,92 @@ async def test_resolve_async( assert actual_response.message.ack_status == AsyncAck.ACK assert actual_response.message.timestamp == expected_response.message.timestamp +@pytest.mark.asyncio +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.RequestValidation.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.MapperService.get_component" +) +async def test_resolve_async_invalid_signature( + mock_mapper_service_get_component, + mock_request_validation_get_component, + mock_async_response_helper_get_component, +): + # Setup MagicMock for MapperService and RequestValidation components + mock_mapper_service_instance = MagicMock() + mock_mapper_service_instance.resolve = AsyncMock() + mock_request_validation_instance = MagicMock() + + # Assign return values to the get_component mocks + mock_mapper_service_get_component.return_value = mock_mapper_service_instance + mock_request_validation_get_component.return_value = ( + mock_request_validation_instance + ) + + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + + # Setup MagicMock for AsyncResponseHelper component + mock_async_response_helper_instance = MagicMock() + error_response = AsyncResponse( + message=AsyncResponseMessage( + correlation_id="error_correlation_id", + timestamp=datetime.utcnow().isoformat(), + ack_status="NACK", + error={ + "code": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + "message": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + }, + ) + ) + mock_async_response_helper_instance.construct_error_async_response.return_value = ( + error_response + ) + mock_async_response_helper_get_component.return_value = ( + mock_async_response_helper_instance + ) + + controller = AsyncMapperController() + + mock_resolve_request = ResolveRequest( + header=RequestHeader( + message_id="test_message_id", + message_ts=datetime.utcnow().isoformat(), + action="resolve", + sender_id="test_sender", + total_count=1, + ), + message=ResolveRequestMessage( + transaction_id="test_transaction_id", + resolve_request=[ + SingleResolveRequest( + reference_id="test_ref", + timestamp=datetime.utcnow(), + id="test_id", + fa="test_fa", + ) + ], + ), + ) + + actual_response = await controller.resolve_async( + mock_resolve_request, is_signature_valid=False + ) + + assert ( + actual_response == error_response + ), "The response did not match the expected error response." + assert actual_response.message.correlation_id == "error_correlation_id" + assert actual_response.message.ack_status == AsyncAck.NACK + assert actual_response.message.timestamp == error_response.message.timestamp + assert actual_response.message.error.code == "INVALID JWT SIGNATURE" + assert actual_response.message.error.message == "INVALID JWT SIGNATURE" + @pytest.mark.asyncio @patch( @@ -606,6 +786,10 @@ async def test_unlink_async( mock_request_validation_get_component.return_value = ( mock_request_validation_instance ) + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + mock_async_response_helper_instance = MagicMock() expected_response = AsyncResponse( message=AsyncResponseMessage( @@ -644,13 +828,100 @@ async def test_unlink_async( ), ) - actual_response = await controller.unlink_async(mock_unlink_request) + actual_response = await controller.unlink_async(mock_unlink_request, is_signature_valid=True) assert ( actual_response == expected_response ), "The response did not match the expected response." assert actual_response.message.correlation_id == "1234" assert actual_response.message.ack_status == AsyncAck.ACK assert actual_response.message.timestamp == expected_response.message.timestamp + +@pytest.mark.asyncio +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.RequestValidation.get_component" +) +@patch( + "openg2p_spar_mapper_api.controllers.async_mapper_controller.MapperService.get_component" +) +async def test_unlink_async_invalid_signature( + mock_mapper_service_get_component, + mock_request_validation_get_component, + mock_async_response_helper_get_component, +): + # Setup MagicMock for MapperService and RequestValidation components + mock_mapper_service_instance = MagicMock() + mock_mapper_service_instance.unlink = AsyncMock() + mock_request_validation_instance = MagicMock() + + # Assign return values to the get_component mocks + mock_mapper_service_get_component.return_value = mock_mapper_service_instance + mock_request_validation_get_component.return_value = ( + mock_request_validation_instance + ) + + mock_request_validation_instance.validate_signature.side_effect = ( + mock_validate_signature + ) + + # Setup MagicMock for AsyncResponseHelper component + mock_async_response_helper_instance = MagicMock() + error_response = AsyncResponse( + message=AsyncResponseMessage( + correlation_id="error_correlation_id", + timestamp=datetime.utcnow().isoformat(), + ack_status="NACK", + error={ + "code": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + "message": SecurityErrorCodes.INVALID_JWT_SIGNATURE, + }, + ) + ) + mock_async_response_helper_instance.construct_error_async_response.return_value = ( + error_response + ) + mock_async_response_helper_get_component.return_value = ( + mock_async_response_helper_instance + ) + + controller = AsyncMapperController() + + mock_unlink_request = UnlinkRequest( + header=RequestHeader( + message_id="test_message_id", + message_ts=datetime.utcnow().isoformat(), + action="unlink", + sender_id="test_sender", + total_count=1, + ), + message=UnlinkRequestMessage( + transaction_id="test_transaction_id", + unlink_request=[ + SingleUnlinkRequest( + reference_id="test_ref", + timestamp=datetime.utcnow(), + id="test_id", + fa="test_fa", + ) + ], + ), + ) + + actual_response = await controller.unlink_async( + mock_unlink_request, is_signature_valid=False + ) + + assert ( + actual_response == error_response + ), "The response did not match the expected error response." + assert actual_response.message.correlation_id == "error_correlation_id" + assert actual_response.message.ack_status == AsyncAck.NACK + assert actual_response.message.timestamp == error_response.message.timestamp + assert actual_response.message.error.code == "INVALID JWT SIGNATURE" + assert actual_response.message.error.message == "INVALID JWT SIGNATURE" + @pytest.mark.asyncio diff --git a/tests/test_sync_mapper_controller.py b/tests/test_sync_mapper_controller.py index 2e34a25..5d203fa 100644 --- a/tests/test_sync_mapper_controller.py +++ b/tests/test_sync_mapper_controller.py @@ -157,7 +157,9 @@ def setup_update_controller(): request_validation_mock.validate_update_request_header = MagicMock( return_value=True ) - + request_validation_mock.validate_signature = MagicMock( + side_effect=mock_validate_signature + ) mock_update_response = UpdateResponse( header=SyncResponseHeader( message_id="test_message_id", @@ -249,7 +251,9 @@ def setup_resolve_controller(): request_validation_mock.validate_resolve_request_header = MagicMock( return_value=True ) - + request_validation_mock.validate_signature = MagicMock( + side_effect=mock_validate_signature + ) mock_resolve_response = ResolveResponse( header=SyncResponseHeader( message_id="test_message_id", @@ -338,17 +342,19 @@ def setup_unlink_controller(): request_validation_mock = MagicMock() request_validation_mock.validate_request = MagicMock(return_value=True) - request_validation_mock.validate_link_request_header = MagicMock(return_value=True) - request_validation_mock.validate_update_request_header = MagicMock( - return_value=True - ) - request_validation_mock.validate_resolve_request_header = MagicMock( - return_value=True - ) + # request_validation_mock.validate_link_request_header = MagicMock(return_value=True) + # request_validation_mock.validate_update_request_header = MagicMock( + # return_value=True + # ) + # request_validation_mock.validate_resolve_request_header = MagicMock( + # return_value=True + # ) request_validation_mock.validate_unlink_request_header = MagicMock( return_value=True ) - + request_validation_mock.validate_signature = MagicMock( + side_effect=mock_validate_signature + ) mock_unlink_response = UnlinkResponse( header=SyncResponseHeader( message_id="test_message_id", @@ -455,31 +461,59 @@ async def test_link_sync_invalid_signature(setup_link_controller): async def test_update_sync_success(setup_update_controller): controller, mock_update_request = setup_update_controller assert controller is not None - response = await controller.update_sync(mock_update_request) + response = await controller.update_sync(mock_update_request, is_signature_valid=True) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.update.assert_called_once_with(mock_update_request) +@pytest.mark.asyncio +async def test_update_sync_invalid_signature(setup_update_controller): + controller, mock_update_request = setup_update_controller + assert controller is not None + + response = await controller.link_sync(mock_update_request, is_signature_valid=False) + assert response.header.status == StatusEnum.rjct + assert response.header.status_reason_message == "Validation error" + + @pytest.mark.asyncio async def test_resolve_sync_success(setup_resolve_controller): controller, mock_resolve_request = setup_resolve_controller assert controller is not None - response = await controller.resolve_sync(mock_resolve_request) + response = await controller.resolve_sync(mock_resolve_request, is_signature_valid=True) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.resolve.assert_called_once_with(mock_resolve_request) +@pytest.mark.asyncio +async def test_resolve_sync_invalid_signature(setup_resolve_controller): + controller, mock_resolve_request = setup_resolve_controller + assert controller is not None + + response = await controller.link_sync(mock_resolve_request, is_signature_valid=False) + assert response.header.status == StatusEnum.rjct + assert response.header.status_reason_message == "Validation error" + @pytest.mark.asyncio async def test_unlink_sync_success(setup_unlink_controller): controller, mock_unlink_request = setup_unlink_controller assert controller is not None - response = await controller.unlink_sync(mock_unlink_request) + response = await controller.unlink_sync(mock_unlink_request, is_signature_valid=True) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.unlink.assert_called_once_with(mock_unlink_request) +@pytest.mark.asyncio +async def test_unlink_sync_invalid_signature(setup_unlink_controller): + controller, mock_unlink_request = setup_unlink_controller + assert controller is not None + + response = await controller.link_sync(mock_unlink_request, is_signature_valid=False) + assert response.header.status == StatusEnum.rjct + assert response.header.status_reason_message == "Validation error" + @pytest.mark.asyncio async def test_link_sync_validation_error(setup_link_controller): @@ -521,7 +555,7 @@ async def test_update_sync_validation_error(setup_update_controller): "validate_update_request_header", side_effect=validation_error, ): - response = await controller.update_sync(mock_update_request) + response = await controller.update_sync(mock_update_request, is_signature_valid=True) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.update.assert_not_called() @@ -543,7 +577,7 @@ async def test_resolve_sync_validation_error(setup_resolve_controller): "validate_resolve_request_header", side_effect=validation_error, ): - response = await controller.resolve_sync(mock_resolve_request) + response = await controller.resolve_sync(mock_resolve_request, is_signature_valid=True) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.resolve.assert_not_called() @@ -565,7 +599,7 @@ async def test_unlink_sync_validation_error(setup_unlink_controller): "validate_unlink_request_header", side_effect=validation_error, ): - response = await controller.unlink_sync(mock_unlink_request) + response = await controller.unlink_sync(mock_unlink_request, is_signature_valid=True) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.unlink.assert_not_called() From 39e454353ead58a2f0e0afc67cc2b0e7b4be408a Mon Sep 17 00:00:00 2001 From: vineela-afk Date: Wed, 19 Jun 2024 16:52:34 +0530 Subject: [PATCH 36/39] done pre-commit --- .../controllers/async_mapper_controller.py | 69 +++++++++++++++---- .../controllers/sync_mapper_controller.py | 6 ++ tests/test_async_mapper_controller.py | 20 ++++-- tests/test_sync_mapper_controller.py | 32 ++++++--- 4 files changed, 98 insertions(+), 29 deletions(-) diff --git a/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py index 269ace9..4e269c6 100644 --- a/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/async_mapper_controller.py @@ -115,8 +115,21 @@ async def link_async( correlation_id, ) - async def update_async(self, update_request: UpdateRequest): + async def update_async( + self, + update_request: UpdateRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], + ): correlation_id = str(uuid.uuid4()) + try: + RequestValidation.get_component().validate_signature(is_signature_valid) + except RequestValidationException as e: + error_response = ( + AsyncResponseHelper.get_component().construct_error_async_response( + update_request, e + ) + ) + return error_response await asyncio.create_task( self.handle_service_and_update_callback( update_request, correlation_id, "update" @@ -127,8 +140,21 @@ async def update_async(self, update_request: UpdateRequest): correlation_id, ) - async def resolve_async(self, resolve_request: ResolveRequest): + async def resolve_async( + self, + resolve_request: ResolveRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], + ): correlation_id = str(uuid.uuid4()) + try: + RequestValidation.get_component().validate_signature(is_signature_valid) + except RequestValidationException as e: + error_response = ( + AsyncResponseHelper.get_component().construct_error_async_response( + resolve_request, e + ) + ) + return error_response await asyncio.create_task( self.handle_service_and_resolve_callback( resolve_request, correlation_id, "resolve" @@ -139,8 +165,21 @@ async def resolve_async(self, resolve_request: ResolveRequest): correlation_id, ) - async def unlink_async(self, unlink_request: UnlinkRequest): + async def unlink_async( + self, + unlink_request: UnlinkRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], + ): correlation_id = str(uuid.uuid4()) + try: + RequestValidation.get_component().validate_signature(is_signature_valid) + except RequestValidationException as e: + error_response = ( + AsyncResponseHelper.get_component().construct_error_async_response( + unlink_request, e + ) + ) + return error_response try: RequestValidation.get_component().validate_request(unlink_request) RequestValidation.get_component().validate_unlink_async_request_header( @@ -174,9 +213,9 @@ async def handle_service_and_link_callback( RequestValidation.get_component().validate_link_async_request_header( link_request ) - single_link_responses: list[SingleLinkResponse] = ( - await self.action_to_method[action](link_request) - ) + single_link_responses: list[ + SingleLinkResponse + ] = await self.action_to_method[action](link_request) async_call_back_request: ( AsyncCallbackRequest @@ -207,9 +246,9 @@ async def handle_service_and_update_callback( RequestValidation.get_component().validate_update_async_request_header( request ) - single_update_responses: list[SingleUpdateResponse] = ( - await self.action_to_method[action](request) - ) + single_update_responses: list[ + SingleUpdateResponse + ] = await self.action_to_method[action](request) async_call_back_request: ( AsyncCallbackRequest ) = AsyncResponseHelper.get_component().construct_success_async_callback_update_request( @@ -239,9 +278,9 @@ async def handle_service_and_resolve_callback( RequestValidation.get_component().validate_resolve_async_request_header( request ) - single_resolve_responses: list[SingleResolveResponse] = ( - await self.action_to_method[action](request) - ) + single_resolve_responses: list[ + SingleResolveResponse + ] = await self.action_to_method[action](request) async_call_back_request: ( AsyncCallbackRequest ) = AsyncResponseHelper.get_component().construct_success_async_callback_resolve_request( @@ -271,9 +310,9 @@ async def handle_service_and_unlink_callback( RequestValidation.get_component().validate_unlink_async_request_header( request ) - single_unlink_responses: list[SingleUnlinkResponse] = ( - await self.action_to_method[action](request) - ) + single_unlink_responses: list[ + SingleUnlinkResponse + ] = await self.action_to_method[action](request) async_call_back_request: ( AsyncCallbackRequest ) = AsyncResponseHelper.get_component().construct_success_async_callback_unlink_request( diff --git a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py index 3c996e9..440726e 100644 --- a/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py +++ b/src/openg2p_spar_mapper_api/controllers/sync_mapper_controller.py @@ -87,8 +87,10 @@ async def link_sync( async def update_sync( self, update_request: UpdateRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], ): try: + RequestValidation.get_component().validate_signature(is_signature_valid) RequestValidation.get_component().validate_request(update_request) RequestValidation.get_component().validate_update_request_header( update_request @@ -114,8 +116,10 @@ async def update_sync( async def resolve_sync( self, resolve_request: ResolveRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], ): try: + RequestValidation.get_component().validate_signature(is_signature_valid) RequestValidation.get_component().validate_request(resolve_request) RequestValidation.get_component().validate_resolve_request_header( resolve_request @@ -141,8 +145,10 @@ async def resolve_sync( async def unlink_sync( self, unlink_request: UnlinkRequest, + is_signature_valid: Annotated[bool, Depends(JWTSignatureValidator())], ): try: + RequestValidation.get_component().validate_signature(is_signature_valid) RequestValidation.get_component().validate_request(unlink_request) RequestValidation.get_component().validate_unlink_request_header( unlink_request diff --git a/tests/test_async_mapper_controller.py b/tests/test_async_mapper_controller.py index 24287fe..2596069 100644 --- a/tests/test_async_mapper_controller.py +++ b/tests/test_async_mapper_controller.py @@ -10,8 +10,8 @@ AsyncResponse, AsyncResponseMessage, RequestHeader, - StatusEnum, SecurityErrorCodes, + StatusEnum, ) from openg2p_g2pconnect_mapper_lib.schemas import ( LinkRequest, @@ -362,7 +362,9 @@ async def test_update_async( ), ) - actual_response = await controller.update_async(mock_update_request, is_signature_valid=True) + actual_response = await controller.update_async( + mock_update_request, is_signature_valid=True + ) assert ( actual_response == expected_response ), "The response did not match the expected response." @@ -370,6 +372,7 @@ async def test_update_async( assert actual_response.message.ack_status == AsyncAck.ACK assert actual_response.message.timestamp == expected_response.message.timestamp + @pytest.mark.asyncio @patch( "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" @@ -595,7 +598,9 @@ async def test_resolve_async( ), ) - actual_response = await controller.resolve_async(mock_resolve_request, is_signature_valid=True) + actual_response = await controller.resolve_async( + mock_resolve_request, is_signature_valid=True + ) assert ( actual_response == expected_response ), "The response did not match the expected response." @@ -603,6 +608,7 @@ async def test_resolve_async( assert actual_response.message.ack_status == AsyncAck.ACK assert actual_response.message.timestamp == expected_response.message.timestamp + @pytest.mark.asyncio @patch( "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" @@ -828,14 +834,17 @@ async def test_unlink_async( ), ) - actual_response = await controller.unlink_async(mock_unlink_request, is_signature_valid=True) + actual_response = await controller.unlink_async( + mock_unlink_request, is_signature_valid=True + ) assert ( actual_response == expected_response ), "The response did not match the expected response." assert actual_response.message.correlation_id == "1234" assert actual_response.message.ack_status == AsyncAck.ACK assert actual_response.message.timestamp == expected_response.message.timestamp - + + @pytest.mark.asyncio @patch( "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" @@ -923,7 +932,6 @@ async def test_unlink_async_invalid_signature( assert actual_response.message.error.message == "INVALID JWT SIGNATURE" - @pytest.mark.asyncio @patch( "openg2p_spar_mapper_api.controllers.async_mapper_controller.AsyncResponseHelper.get_component" diff --git a/tests/test_sync_mapper_controller.py b/tests/test_sync_mapper_controller.py index 5d203fa..cf5ff73 100644 --- a/tests/test_sync_mapper_controller.py +++ b/tests/test_sync_mapper_controller.py @@ -461,11 +461,14 @@ async def test_link_sync_invalid_signature(setup_link_controller): async def test_update_sync_success(setup_update_controller): controller, mock_update_request = setup_update_controller assert controller is not None - response = await controller.update_sync(mock_update_request, is_signature_valid=True) + response = await controller.update_sync( + mock_update_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.update.assert_called_once_with(mock_update_request) + @pytest.mark.asyncio async def test_update_sync_invalid_signature(setup_update_controller): controller, mock_update_request = setup_update_controller @@ -476,22 +479,26 @@ async def test_update_sync_invalid_signature(setup_update_controller): assert response.header.status_reason_message == "Validation error" - @pytest.mark.asyncio async def test_resolve_sync_success(setup_resolve_controller): controller, mock_resolve_request = setup_resolve_controller assert controller is not None - response = await controller.resolve_sync(mock_resolve_request, is_signature_valid=True) + response = await controller.resolve_sync( + mock_resolve_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.resolve.assert_called_once_with(mock_resolve_request) + @pytest.mark.asyncio async def test_resolve_sync_invalid_signature(setup_resolve_controller): controller, mock_resolve_request = setup_resolve_controller assert controller is not None - response = await controller.link_sync(mock_resolve_request, is_signature_valid=False) + response = await controller.link_sync( + mock_resolve_request, is_signature_valid=False + ) assert response.header.status == StatusEnum.rjct assert response.header.status_reason_message == "Validation error" @@ -500,11 +507,14 @@ async def test_resolve_sync_invalid_signature(setup_resolve_controller): async def test_unlink_sync_success(setup_unlink_controller): controller, mock_unlink_request = setup_unlink_controller assert controller is not None - response = await controller.unlink_sync(mock_unlink_request, is_signature_valid=True) + response = await controller.unlink_sync( + mock_unlink_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.succ assert response.message.transaction_id == "trans_id" controller.mapper_service.unlink.assert_called_once_with(mock_unlink_request) + @pytest.mark.asyncio async def test_unlink_sync_invalid_signature(setup_unlink_controller): controller, mock_unlink_request = setup_unlink_controller @@ -555,7 +565,9 @@ async def test_update_sync_validation_error(setup_update_controller): "validate_update_request_header", side_effect=validation_error, ): - response = await controller.update_sync(mock_update_request, is_signature_valid=True) + response = await controller.update_sync( + mock_update_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.update.assert_not_called() @@ -577,7 +589,9 @@ async def test_resolve_sync_validation_error(setup_resolve_controller): "validate_resolve_request_header", side_effect=validation_error, ): - response = await controller.resolve_sync(mock_resolve_request, is_signature_valid=True) + response = await controller.resolve_sync( + mock_resolve_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.resolve.assert_not_called() @@ -599,7 +613,9 @@ async def test_unlink_sync_validation_error(setup_unlink_controller): "validate_unlink_request_header", side_effect=validation_error, ): - response = await controller.unlink_sync(mock_unlink_request, is_signature_valid=True) + response = await controller.unlink_sync( + mock_unlink_request, is_signature_valid=True + ) assert response.header.status == StatusEnum.rjct assert validation_error.message in response.header.status_reason_message controller.mapper_service.unlink.assert_not_called() From 0946c8416e86ca952e717b27b439db664059f38a Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 20 Jun 2024 11:40:04 +0530 Subject: [PATCH 37/39] Update branch name on pip install --- .github/workflows/openapi-push.yml | 6 +++--- Dockerfile | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/openapi-push.yml b/.github/workflows/openapi-push.yml index 19d91c2..6cd24fb 100644 --- a/.github/workflows/openapi-push.yml +++ b/.github/workflows/openapi-push.yml @@ -23,9 +23,9 @@ jobs: python-version: "3.10" - name: Install app run: | - python -m pip install git+https://github.com/openg2p/openg2p-fastapi-common@develop\#subdirectory=openg2p-fastapi-common - python -m pip install git+https://github.com/openg2p/openg2p-g2pconnect-common-lib@develop\#subdirectory=openg2p-g2pconnect-common-lib - python -m pip install git+https://github.com/openg2p/openg2p-g2pconnect-common-lib@develop\#subdirectory=openg2p-g2pconnect-mapper-lib + python -m pip install git+https://github.com/openg2p/openg2p-fastapi-common@1.0.0\#subdirectory=openg2p-fastapi-common + python -m pip install git+https://github.com/openg2p/openg2p-g2pconnect-common-lib@1.1.0\#subdirectory=openg2p-g2pconnect-common-lib + python -m pip install git+https://github.com/openg2p/openg2p-g2pconnect-common-lib@1.1.0\#subdirectory=openg2p-g2pconnect-mapper-lib python -m pip install . - name: Generate openapi json run: | diff --git a/Dockerfile b/Dockerfile index cf16a50..f327351 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,9 +20,9 @@ ADD --chown=${container_user}:${container_user_group} . /app/src ADD --chown=${container_user}:${container_user_group} main.py /app RUN python3 -m pip install \ - git+https://github.com/openg2p/openg2p-fastapi-common@develop\#subdirectory=openg2p-fastapi-common \ - git+https://github.com/OpenG2P/openg2p-g2pconnect-common-lib@develop\#subdirectory=openg2p-g2pconnect-common-lib \ - git+https://github.com/OpenG2P/openg2p-g2pconnect-common-lib@develop\#subdirectory=openg2p-g2pconnect-mapper-lib \ + git+https://github.com/openg2p/openg2p-fastapi-common@1.0.0\#subdirectory=openg2p-fastapi-common \ + git+https://github.com/OpenG2P/openg2p-g2pconnect-common-lib@1.1.0\#subdirectory=openg2p-g2pconnect-common-lib \ + git+https://github.com/OpenG2P/openg2p-g2pconnect-common-lib@1.1.0\#subdirectory=openg2p-g2pconnect-mapper-lib \ ./src ENV SPAR_MAPPER_WORKER_TYPE=gunicorn From 2e1a8a56a31e9e08afb7aa4e677ae2ae4d4fa895 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 20 Jun 2024 11:57:17 +0530 Subject: [PATCH 38/39] Fix test cases --- tests/test_sync_mapper_controller.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/test_sync_mapper_controller.py b/tests/test_sync_mapper_controller.py index cf5ff73..e95e0e1 100644 --- a/tests/test_sync_mapper_controller.py +++ b/tests/test_sync_mapper_controller.py @@ -342,13 +342,6 @@ def setup_unlink_controller(): request_validation_mock = MagicMock() request_validation_mock.validate_request = MagicMock(return_value=True) - # request_validation_mock.validate_link_request_header = MagicMock(return_value=True) - # request_validation_mock.validate_update_request_header = MagicMock( - # return_value=True - # ) - # request_validation_mock.validate_resolve_request_header = MagicMock( - # return_value=True - # ) request_validation_mock.validate_unlink_request_header = MagicMock( return_value=True ) @@ -474,7 +467,9 @@ async def test_update_sync_invalid_signature(setup_update_controller): controller, mock_update_request = setup_update_controller assert controller is not None - response = await controller.link_sync(mock_update_request, is_signature_valid=False) + response = await controller.update_sync( + mock_update_request, is_signature_valid=False + ) assert response.header.status == StatusEnum.rjct assert response.header.status_reason_message == "Validation error" @@ -496,7 +491,7 @@ async def test_resolve_sync_invalid_signature(setup_resolve_controller): controller, mock_resolve_request = setup_resolve_controller assert controller is not None - response = await controller.link_sync( + response = await controller.resolve_sync( mock_resolve_request, is_signature_valid=False ) assert response.header.status == StatusEnum.rjct @@ -520,7 +515,9 @@ async def test_unlink_sync_invalid_signature(setup_unlink_controller): controller, mock_unlink_request = setup_unlink_controller assert controller is not None - response = await controller.link_sync(mock_unlink_request, is_signature_valid=False) + response = await controller.unlink_sync( + mock_unlink_request, is_signature_valid=False + ) assert response.header.status == StatusEnum.rjct assert response.header.status_reason_message == "Validation error" From 2fb4c632cad2ca5a105e16915cd388dd91479209 Mon Sep 17 00:00:00 2001 From: PSNAppZ Date: Thu, 20 Jun 2024 12:09:06 +0530 Subject: [PATCH 39/39] Fix test dependencies --- test-requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index 5f3a815..03186a8 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,5 +1,5 @@ pytest-asyncio pytest-cov -git+https://github.com/openg2p/openg2p-fastapi-common@develop#subdirectory=openg2p-fastapi-common -git+https://github.com/openg2p/openg2p-g2pconnect-common@develop#subdirectory=openg2p-g2pconnect-common-lib -git+https://github.com/OpenG2P/openg2p-g2pconnect-common@develop#subdirectory=openg2p-g2pconnect-mapper-lib +git+https://github.com/openg2p/openg2p-fastapi-common@1.0.0#subdirectory=openg2p-fastapi-common +git+https://github.com/openg2p/openg2p-g2pconnect-common@1.1.0#subdirectory=openg2p-g2pconnect-common-lib +git+https://github.com/OpenG2P/openg2p-g2pconnect-common@1.1.0#subdirectory=openg2p-g2pconnect-mapper-lib