Endpoint tests with Sumo data (#710)

Co-authored-by: Anders Fredrik Kiær <[email protected]>

Author: HansKallekleiv

.github/workflows/backend_sumo_prod.yml

@@ -0,0 +1,57 @@
+name: integration
+
+on:
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * *"
+
+jobs:
+  sumo_prod:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: 🤖 Get shared key from Sumo
+        working-directory: ./backend_py/primary
+        env:
+          SHARED_KEY_SUMO_PROD: ${{ secrets.SHARED_KEY_DROGON_READ_PROD }}
+        run: |
+          if [ ${#SHARED_KEY_SUMO_PROD} -eq 0 ]; then
+            echo "Error: SHARED_KEY_SUMO_PROD is empty. Stopping the action."
+            exit 1
+          fi
+          mkdir ~/.sumo
+          echo $SHARED_KEY_SUMO_PROD > ~/.sumo/9e5443dd-3431-4690-9617-31eed61cb55a.sharedkey
+
+      - name: 🐍 Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          cache: pip
+
+      - name: 📦 Install poetry and dependencies
+        working-directory: ./backend_py/primary
+        run: |
+          pip install --upgrade pip
+          pip install poetry==1.8.5  # Pin Poetry to version 1.8.5
+          poetry config virtualenvs.create false
+          poetry check --lock  # Check lock file is consistent with pyproject.toml
+          poetry install --with dev
+
+      - name: 🤖 Run tests
+        working-directory: ./backend_py/primary
+        env:
+          WEBVIZ_CLIENT_SECRET: 0
+          WEBVIZ_SMDA_SUBSCRIPTION_KEY: 0
+          WEBVIZ_SMDA_RESOURCE_SCOPE: 0
+          WEBVIZ_VDS_HOST_ADDRESS: 0
+          WEBVIZ_ENTERPRISE_SUBSCRIPTION_KEY: 0
+          WEBVIZ_SSDL_RESOURCE_SCOPE: 0
+          WEBVIZ_SUMU_ENV: prod
+        run: |
+          pytest -s --timeout=300 ./tests/integration

.github/workflows/webviz.yml

@@ -106,7 +106,7 @@ jobs:
           black --check primary/ tests/
           pylint primary/ tests/
           bandit --recursive primary/
-          mypy primary/ tests/
+          mypy primary/
 
       - name: 🤖 Run tests
         working-directory: ./backend_py/primary

backend_py/primary/poetry.lock

@@ -13,7 +13,7 @@
 from primary.auth.enforce_logged_in_middleware import EnforceLoggedInMiddleware
 from primary.middleware.add_process_time_to_server_timing_middleware import AddProcessTimeToServerTimingMiddleware
 from primary.routers.dev.router import router as dev_router
-from primary.routers.explore import router as explore_router
+from primary.routers.explore.router import router as explore_router
 from primary.routers.general import router as general_router
 from primary.routers.graph.router import router as graph_router
 from primary.routers.grid3d.router import router as grid3d_router

backend_py/primary/primary/main.py

@@ -1,51 +1,27 @@
-from typing import List, Sequence
+from typing import List
 
 from fastapi import APIRouter, Depends, Path, Query
-from pydantic import BaseModel
 
 from primary.auth.auth_helper import AuthHelper
 from primary.services.sumo_access.case_inspector import CaseInspector
 from primary.services.sumo_access.sumo_inspector import SumoInspector
 from primary.services.utils.authenticated_user import AuthenticatedUser
 
-router = APIRouter()
-
-
-class FieldInfo(BaseModel):
-    field_identifier: str
-
-
-class CaseInfo(BaseModel):
-    uuid: str
-    name: str
-    status: str
-    user: str
+from . import schemas
 
-
-class EnsembleInfo(BaseModel):
-    name: str
-    realization_count: int
-
-
-class EnsembleDetails(BaseModel):
-    name: str
-    field_identifier: str
-    stratigraphic_column_identifier: str
-    case_name: str
-    case_uuid: str
-    realizations: Sequence[int]
+router = APIRouter()
 
 
 @router.get("/fields")
 async def get_fields(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
-) -> List[FieldInfo]:
+) -> List[schemas.FieldInfo]:
     """
     Get list of fields
     """
     sumo_inspector = SumoInspector(authenticated_user.get_sumo_access_token())
     field_ident_arr = await sumo_inspector.get_fields_async()
-    ret_arr = [FieldInfo(field_identifier=field_ident.identifier) for field_ident in field_ident_arr]
+    ret_arr = [schemas.FieldInfo(field_identifier=field_ident.identifier) for field_ident in field_ident_arr]
 
     return ret_arr
 
@@ -54,14 +30,14 @@ async def get_fields(
 async def get_cases(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
     field_identifier: str = Query(description="Field identifier"),
-) -> List[CaseInfo]:
+) -> List[schemas.CaseInfo]:
     """Get list of cases for specified field"""
     sumo_inspector = SumoInspector(authenticated_user.get_sumo_access_token())
     case_info_arr = await sumo_inspector.get_cases_async(field_identifier=field_identifier)
 
-    ret_arr: List[CaseInfo] = []
+    ret_arr: List[schemas.CaseInfo] = []
 
-    ret_arr = [CaseInfo(uuid=ci.uuid, name=ci.name, status=ci.status, user=ci.user) for ci in case_info_arr]
+    ret_arr = [schemas.CaseInfo(uuid=ci.uuid, name=ci.name, status=ci.status, user=ci.user) for ci in case_info_arr]
 
     return ret_arr
 
@@ -70,22 +46,20 @@ async def get_cases(
 async def get_ensembles(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
     case_uuid: str = Path(description="Sumo case uuid"),
-) -> List[EnsembleInfo]:
+) -> List[schemas.EnsembleInfo]:
     """Get list of ensembles for a case"""
     case_inspector = CaseInspector.from_case_uuid(authenticated_user.get_sumo_access_token(), case_uuid)
     iteration_info_arr = await case_inspector.get_iterations_async()
 
-    print(iteration_info_arr)
-
-    return [EnsembleInfo(name=it.name, realization_count=it.realization_count) for it in iteration_info_arr]
+    return [schemas.EnsembleInfo(name=it.name, realization_count=it.realization_count) for it in iteration_info_arr]
 
 
 @router.get("/cases/{case_uuid}/ensembles/{ensemble_name}")
 async def get_ensemble_details(
     authenticated_user: AuthenticatedUser = Depends(AuthHelper.get_authenticated_user),
     case_uuid: str = Path(description="Sumo case uuid"),
     ensemble_name: str = Path(description="Ensemble name"),
-) -> EnsembleDetails:
+) -> schemas.EnsembleDetails:
     """Get more detailed information for an ensemble"""
 
     case_inspector = CaseInspector.from_case_uuid(authenticated_user.get_sumo_access_token(), case_uuid)
@@ -97,7 +71,7 @@ async def get_ensemble_details(
     if len(field_identifiers) != 1:
         raise NotImplementedError("Multiple field identifiers not supported")
 
-    return EnsembleDetails(
+    return schemas.EnsembleDetails(
         name=ensemble_name,
         case_name=case_name,
         case_uuid=case_uuid,

backend_py/primary/primary/routers/explore.py backend_py/primary/primary/routers/explore/router.py

@@ -0,0 +1,28 @@
+from typing import Sequence
+
+from pydantic import BaseModel
+
+
+class FieldInfo(BaseModel):
+    field_identifier: str
+
+
+class CaseInfo(BaseModel):
+    uuid: str
+    name: str
+    status: str
+    user: str
+
+
+class EnsembleInfo(BaseModel):
+    name: str
+    realization_count: int
+
+
+class EnsembleDetails(BaseModel):
+    name: str
+    field_identifier: str
+    case_name: str
+    case_uuid: str
+    realizations: Sequence[int]
+    stratigraphic_column_identifier: str

backend_py/primary/primary/routers/explore/schemas.py

@@ -11,7 +11,10 @@
 
 
 def create_sumo_client(access_token: str) -> SumoClient:
-    sumo_client = SumoClient(env=config.SUMO_ENV, token=access_token, interactive=False)
+    if access_token == "DUMMY_TOKEN_FOR_TESTING":  # nosec bandit B105
+        sumo_client = SumoClient(env=config.SUMO_ENV, interactive=False)
+    else:
+        sumo_client = SumoClient(env=config.SUMO_ENV, token=access_token, interactive=False)
     return sumo_client
 
 

backend_py/primary/primary/services/sumo_access/_helpers.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
 from opentelemetry.sdk._logs import LoggingHandler
-
+from opentelemetry._logs import get_logger
 
 # This is a custom logging handler that does formatting of log messages before passing them on to open telemetry.
 # Note that the class we're inheriting from here *is* an OpenTelemetry derived Python logger.
@@ -18,8 +18,10 @@ def emit(self, record: logging.LogRecord) -> None:
         record.msg = formatted_msg
         record.args = None
 
+        logger = get_logger(record.name, logger_provider=self._logger_provider)
+
         # Note that the logger that we're calling emit on here is an Open Telemetry Logger, not a Python logger.
-        self._logger.emit(self._translate(record))
+        logger.emit(self._translate(record))
 
         # For inspecting and debugging the actual telemetry payload, uncomment the following lines.
         # log_record_as_json = self._translate(record).to_json()

backend_py/primary/primary/utils/azure_monitor_setup.py

@@ -22,7 +22,7 @@ pandas = {version = "2.0.1", extras = ["performance"]}
 httpx = "^0.24.0"
 fmu-sumo = "1.2.5"
 sumo-wrapper-python = "1.0.9"
-azure-monitor-opentelemetry = "^1.1.0"
+azure-monitor-opentelemetry = "1.1.0"
 requests-toolbelt = "^1.0.0"
 pottery = "^3.0.0"
 xtgeo = "^3.8.0"
@@ -33,11 +33,13 @@ polars = "^1.6.0"
 [tool.poetry.group.dev.dependencies]
 black = "^22.12.0"
 pylint = "^2.15.10"
-pytest = "^7.2.1"
+pytest = "^8.3.2"
 mypy = "^1.9.0"
 bandit = "^1.7.5"
 types-requests = "^2.31.0.1"
 types-redis = "^4.6.0"
+pytest-timeout = "^2.3.1"
+pytest-asyncio = "^0.24.0"
 
 
 [build-system]
@@ -64,5 +66,6 @@ disallow_untyped_defs = true
 [tool.pytest.ini_options]
 pythonpath = ["."]
 filterwarnings = "ignore::DeprecationWarning:pkg_resources"
-
+asyncio_mode="auto"
+asyncio_default_fixture_loop_scope="session"