diff --git a/.github/workflows/test-dev-deployment.yml b/.github/workflows/test-dev-deployment.yml new file mode 100644 index 00000000..52b96973 --- /dev/null +++ b/.github/workflows/test-dev-deployment.yml @@ -0,0 +1,41 @@ +name: Integration Tests for Development API + +on: + schedule: + - cron: '*/30 * * * *' + workflow_dispatch: + +jobs: + integration_tests: + runs-on: self-hosted + + steps: + - name: Print hostname + run: echo "The hostname is $(hostname)" + + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Java + uses: actions/setup-java@v4 + with: + distribution: 'zulu' + java-version: '21' + + - name: Download ijhttp.zip + run: curl -f -L -o ijhttp.zip "https://jb.gg/ijhttp/latest" + + - name: Unzip ijhttp.zip + run: unzip -q ijhttp.zip + + - name: Beamline endpoint tests + run: | + ./ijhttp/ijhttp --env-file ./integration-tests/http-client.env.json \ + --env dev \ + ./integration-tests/check_all_beamlines.http + + - name: Check API Service + run: | + ./ijhttp/ijhttp --env-file ./integration-tests/http-client.env.json \ + --env dev \ + ./integration-tests/check_service_health.http diff --git a/README.md b/README.md index 0d585fde..e60cba84 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,57 @@ # NSLS-II Facility API -This is the repository for the NSLS-II Facility API codebase. +This is the repository for the NSLS-II Facility API codebase. ## Current Status [![Integration Tests for Deployed API](https://github.com/NSLS2/nsls2api/actions/workflows/test-production-deployment.yml/badge.svg)](https://github.com/NSLS2/nsls2api/actions/workflows/test-production-deployment.yml) -## Developer Notes +[![Integration Tests for DEV API](https://github.com/NSLS2/nsls2api/actions/workflows/test-dev-deployment.yml/badge.svg)](https://github.com/NSLS2/nsls2api/actions/workflows/test-dev-deployment.yml) + +## Developer Notes + +In order to develop locally you will need to have a local MongoDB running. +This can be installed using your preferred method, a native install +or [running a container](https://hub.docker.com/_/mongo) work perfectly fine. + +Once you have MongoDB up and running you then need to 'seed' the facility and beamline information that +does not get pulled from any other source. + +The files for the collections can be found within the `/nsls2/software/dssi/nsls2core/nsls2core-development.tgz` + +1. Copy and unpack the archive into a directory (e.g. `nsls2core-development`) to your development machine + +2. Import facility information into the local mongodb``` + ```bash + mongorestore --uri="mongodb://localhost:27017" --nsFrom=nsls2core-development.facilities --nsTo=nsls2core-development.facilities ./nsls2core-development/facilities.bson + ``` +3. Import beamline information into the local mongodb + ```bash + mongorestore --uri="mongodb://localhost:27017" --nsFrom=nsls2core-development.beamlines --nsTo=nsls2core-development.beamlines ./nsls2core-development/beamlines.bson + ``` + +You will then need to create a `.env` file that contains the configuration (an example can also be found in the same +directory as the json files). + +1. Copy `/nsls2/software/dssi/nsls2core/.env.development` to your local machine +2. Rename to `.env` and place in the `src/nsls2api` directory in your cloned repo (in the same folder as `main.py`) +3. Ensure that you have the `bnlroot.crt` file (which is deployed to all BNL managed machines) in the location specified + within the `.env` file. ### Updating Dependencies -The project uses `pip-compile` to manage the `requirements.txt` and `requirements-dev.txt`. -In order to upgrade the packages you will need to install `pip-tools`. Then to upgrade simply run +The project uses `uv pip compile` to manage the `requirements.txt` and `requirements-dev.txt` files. + +In order to upgrade the packages versions you will need to simply run ``` -pip-compile requirements-dev.in --upgrade -pip-compile requirements.in --upgrade +uv pip compile requirements-dev.in --upgrade -o requirements-dev.txt +uv pip compile requirements.in --upgrade -o requirements.txt ``` +Then in order to actually upgrade the packages +``` +uv pip install -r requirements-dev.txt +uv pip install -r requirements.txt +``` +Of course, you can drop the `uv` from these last commands if you want to wait longer. \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt index a2071950..c5e54355 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,29 +1,29 @@ # This file was autogenerated by uv via the following command: # uv pip compile requirements-dev.in -o requirements-dev.txt -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.10.11 +aiohttp==3.11.11 # via # aiohttp-jinja2 # textual-dev # textual-serve aiohttp-jinja2==1.6 # via textual-serve -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.8.0 # via # asyncer # httpx asgi-lifespan==2.1.0 # via -r requirements-dev.in -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data asyncer==0.0.8 # via -r requirements-dev.in -attrs==24.2.0 +attrs==24.3.0 # via aiohttp black==24.10.0 # via -r requirements-dev.in @@ -33,17 +33,15 @@ brotli==1.1.0 # via geventhttpclient bunnet==1.3.0 # via -r requirements-dev.in -certifi==2024.8.30 +certifi==2024.12.14 # via # geventhttpclient # httpcore # httpx # requests -cffi==1.17.1 - # via cryptography -charset-normalizer==3.4.0 +charset-normalizer==3.4.1 # via requests -click==8.1.7 +click==8.1.8 # via # black # bunnet @@ -53,10 +51,8 @@ click==8.1.7 # userpath configargparse==1.7 # via locust -coverage==7.6.4 +coverage==7.6.10 # via -r requirements-dev.in -cryptography==43.0.3 - # via secretstorage decorator==5.1.1 # via ipython distlib==0.3.9 @@ -67,7 +63,7 @@ executing==2.1.0 # via stack-data filelock==3.16.1 # via virtualenv -flask==3.0.3 +flask==3.1.0 # via # flask-cors # flask-login @@ -81,30 +77,28 @@ frozenlist==1.5.0 # aiohttp # aiosignal gevent==24.11.1 - # via - # geventhttpclient - # locust -geventhttpclient==2.3.1 + # via geventhttpclient +geventhttpclient==2.3.3 # via locust greenlet==3.1.1 # via gevent h11==0.14.0 # via httpcore -hatch==1.13.0 +hatch==1.14.0 # via -r requirements-dev.in hatch-requirements-txt==0.4.1 # via -r requirements-dev.in hatch-vcs==0.4.0 # via -r requirements-dev.in -hatchling==1.26.3 +hatchling==1.27.0 # via # -r requirements-dev.in # hatch # hatch-requirements-txt # hatch-vcs -httpcore==1.0.6 +httpcore==1.0.7 # via httpx -httpx==0.27.2 +httpx==0.28.1 # via hatch hyperlink==21.0.0 # via hatch @@ -117,7 +111,7 @@ idna==3.10 # yarl iniconfig==2.0.0 # via pytest -ipython==8.29.0 +ipython==8.31.0 # via -r requirements-dev.in itsdangerous==2.2.0 # via flask @@ -129,24 +123,20 @@ jaraco-functools==4.1.0 # via keyring jedi==0.19.2 # via ipython -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.5 # via # aiohttp-jinja2 # flask # textual-serve -keyring==25.5.0 +keyring==25.6.0 # via hatch lazy-model==0.2.0 # via bunnet linkify-it-py==2.0.3 # via markdown-it-py -locust==2.32.2 +locust==2.32.5 # via -r requirements-dev.in -markdown-it-py[linkify,plugins]==3.0.0 +markdown-it-py==3.0.0 # via # mdit-py-plugins # rich @@ -207,35 +197,35 @@ pluggy==1.5.0 # pytest prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 - # via yarl -psutil==6.1.0 +propcache==0.2.1 + # via + # aiohttp + # yarl +psutil==6.1.1 # via locust ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pycparser==2.22 - # via cffi -pydantic==2.9.2 +pydantic==2.10.5 # via # bunnet # lazy-model -pydantic-core==2.23.4 +pydantic-core==2.27.2 # via pydantic -pygments==2.18.0 +pygments==2.19.1 # via # ipython # rich pymongo==4.10.1 # via bunnet -pyright==1.1.388 +pyright==1.1.391 # via -r requirements-dev.in -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements-dev.in # pytest-asyncio -pytest-asyncio==0.24.0 +pytest-asyncio==0.25.2 # via -r requirements-dev.in pyzmq==26.2.0 # via locust @@ -246,28 +236,29 @@ rich==13.9.4 # hatch # textual # textual-serve -ruff==0.7.3 +ruff==0.9.0 # via -r requirements-dev.in -secretstorage==3.3.3 - # via keyring +setuptools==75.8.0 + # via + # locust + # setuptools-scm + # zope-event + # zope-interface setuptools-scm==8.1.0 # via hatch-vcs shellingham==1.5.4 # via hatch -six==1.16.0 - # via asttokens sniffio==1.3.1 # via # anyio # asgi-lifespan - # httpx stack-data==0.6.3 # via ipython -textual==0.85.2 +textual==1.0.0 # via # textual-dev # textual-serve -textual-dev==1.6.1 +textual-dev==1.7.0 # via -r requirements-dev.in textual-serve==1.1.1 # via textual-dev @@ -281,10 +272,11 @@ traitlets==5.14.3 # via # ipython # matplotlib-inline -trove-classifiers==2024.10.21.16 +trove-classifiers==2025.1.7.14 # via hatchling typing-extensions==4.12.2 # via + # anyio # pydantic # pydantic-core # pyright @@ -292,17 +284,17 @@ typing-extensions==4.12.2 # textual-dev uc-micro-py==1.0.3 # via linkify-it-py -urllib3==2.2.3 +urllib3==2.3.0 # via # geventhttpclient # requests userpath==1.9.2 # via hatch -uv==0.5.1 +uv==0.5.16 # via # -r requirements-dev.in # hatch -virtualenv==20.27.1 +virtualenv==20.28.1 # via hatch wcwidth==0.2.13 # via prompt-toolkit @@ -311,11 +303,11 @@ werkzeug==3.1.3 # flask # flask-login # locust -yarl==1.17.1 +yarl==1.18.3 # via aiohttp zope-event==5.0 # via gevent -zope-interface==7.1.1 +zope-interface==7.2 # via gevent zstandard==0.23.0 # via hatch diff --git a/requirements.txt b/requirements.txt index fe6cdbed..a0b3b423 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ aiofiles==24.1.0 # via -r requirements.in annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.8.0 # via # httpx # starlette @@ -16,15 +16,15 @@ asgi-correlation-id==4.3.4 # via -r requirements.in async-timeout==5.0.1 # via httpx-socks -beanie==1.27.0 +beanie==1.29.0 # via -r requirements.in -certifi==2024.8.30 +certifi==2024.12.14 # via # httpcore # httpx cffi==1.17.1 # via argon2-cffi-bindings -click==8.1.7 +click==8.1.8 # via # beanie # typer @@ -33,9 +33,9 @@ decorator==5.1.1 # via gssapi dnspython==2.7.0 # via pymongo -faker==32.1.0 +faker==33.3.0 # via -r requirements.in -fastapi==0.115.5 +fastapi==0.115.6 # via -r requirements.in gssapi==1.9.0 # via n2snusertools @@ -45,15 +45,15 @@ h11==0.14.0 # via # httpcore # uvicorn -httpcore==1.0.6 +httpcore==1.0.7 # via # httpx # httpx-socks -httpx==0.27.2 +httpx==0.28.1 # via # -r requirements.in # httpx-socks -httpx-socks[asyncio]==0.9.2 +httpx-socks==0.10.0 # via -r requirements.in idna==3.10 # via @@ -71,7 +71,7 @@ ldap3==2.9.1 # via n2snusertools linkify-it-py==2.0.3 # via markdown-it-py -markdown-it-py[linkify,plugins]==3.0.0 +markdown-it-py==3.0.0 # via # mdit-py-plugins # rich @@ -102,18 +102,18 @@ pyasn1==0.6.1 # via ldap3 pycparser==2.22 # via cffi -pydantic==2.9.2 +pydantic==2.10.5 # via # -r requirements.in # beanie # fastapi # lazy-model # pydantic-settings -pydantic-core==2.23.4 +pydantic-core==2.27.2 # via pydantic -pydantic-settings==2.6.1 +pydantic-settings==2.7.1 # via -r requirements.in -pygments==2.18.0 +pygments==2.19.1 # via rich pymongo==4.9.2 # via motor @@ -121,9 +121,9 @@ python-dateutil==2.9.0.post0 # via faker python-dotenv==1.0.1 # via pydantic-settings -python-multipart==0.0.18 +python-multipart==0.0.20 # via -r requirements.in -python-socks==2.5.3 +python-socks==2.6.1 # via httpx-socks pyyaml==6.0.2 # via n2snusertools @@ -134,30 +134,29 @@ rich==13.9.4 # typer shellingham==1.5.4 # via typer -six==1.16.0 +six==1.17.0 # via python-dateutil -slack-bolt==1.21.2 +slack-bolt==1.22.0 # via -r requirements.in -slack-sdk==3.33.3 +slack-sdk==3.34.0 # via # -r requirements.in # slack-bolt sniffio==1.3.1 - # via - # anyio - # httpx -starlette==0.41.2 + # via anyio +starlette==0.41.3 # via # asgi-correlation-id # fastapi -textual==0.85.2 +textual==1.0.0 # via -r requirements.in toml==0.10.2 # via beanie -typer==0.13.0 +typer==0.15.1 # via -r requirements.in typing-extensions==4.12.2 # via + # anyio # beanie # faker # fastapi @@ -169,7 +168,7 @@ uc-micro-py==1.0.3 # via linkify-it-py uuid==1.30 # via -r requirements.in -uvicorn==0.32.0 +uvicorn==0.34.0 # via -r requirements.in wcwidth==0.2.13 # via prettytable diff --git a/src/nsls2api/infrastructure/security.py b/src/nsls2api/infrastructure/security.py index b56ebaae..4a4b8301 100644 --- a/src/nsls2api/infrastructure/security.py +++ b/src/nsls2api/infrastructure/security.py @@ -170,6 +170,8 @@ async def validate_admin_role( if api_key is not None: try: valid_key = await verify_api_key(api_key) + if valid_key is None: + return None key = await lookup_api_key(api_key) # await key.fetch_all_links() if key.user.role == ApiUserRole.admin: diff --git a/src/nsls2api/models/principals.py b/src/nsls2api/models/principals.py deleted file mode 100644 index 49947013..00000000 --- a/src/nsls2api/models/principals.py +++ /dev/null @@ -1,18 +0,0 @@ -from uuid import UUID, uuid4 - -import beanie -from pydantic import Field - - -class PrincipalType(str, enum.Enum): - user = "user" - service = "service" - - -class Principal(beanie.Document): - id: UUID = Field(default_factory=uuid4) - type: PrincipalType - identities: list[Identity] = [] - roles: list[Role] = [] - api_keys: list[APIKey] = [] - sessions: list[Session] = [] diff --git a/src/nsls2api/services/proposal_service.py b/src/nsls2api/services/proposal_service.py index 88e7f778..c548082d 100644 --- a/src/nsls2api/services/proposal_service.py +++ b/src/nsls2api/services/proposal_service.py @@ -1,7 +1,7 @@ import datetime from pathlib import Path from faker import Faker -from faker.providers import person, python, date_time +from faker.providers import python, date_time import random from typing import Optional @@ -461,7 +461,6 @@ async def generate_fake_test_proposal( user_list = [] fake = Faker() - fake.add_provider(person) fake.add_provider(python) fake.add_provider(date_time) diff --git a/src/nsls2api/services/slack_service.py b/src/nsls2api/services/slack_service.py index 66039f44..76a34639 100644 --- a/src/nsls2api/services/slack_service.py +++ b/src/nsls2api/services/slack_service.py @@ -265,7 +265,7 @@ def lookup_username_by_email(email: str) -> str | None: def add_users_to_channel(channel_id: str, user_ids: list[str]): try: userlist = ",".join(user_ids) - app.client.conversations_invite(channel=channel_id, users=userlist) + get_boring_app().client.conversations_invite(channel=channel_id, users=userlist) except SlackApiError as error: if error.response["error"] == "failed_for_some_users": channel_members = get_channel_members(channel_id) diff --git a/src/nsls2api/tests/api/test_beamline_api.py b/src/nsls2api/tests/api/test_beamline_api.py index 3a556c8f..a0641dec 100644 --- a/src/nsls2api/tests/api/test_beamline_api.py +++ b/src/nsls2api/tests/api/test_beamline_api.py @@ -62,6 +62,7 @@ async def test_get_beamline_directory_skeleton(): response = await ac.get("/v1/beamline/zzz/directory-skeleton") response_json = response.json() assert response.status_code == 200 + assert response_json["directory_count"] == 2 @pytest.mark.anyio diff --git a/src/nsls2api/views/home.py b/src/nsls2api/views/home.py index 71e53898..f7a09e19 100644 --- a/src/nsls2api/views/home.py +++ b/src/nsls2api/views/home.py @@ -20,6 +20,12 @@ def index(request: Request): return templates.TemplateResponse("home/index.html", data) +@router.get("/default", include_in_schema=False) +def default(request: Request): + data = {"request": request} + return templates.TemplateResponse("home/default.html", data) + + # This is a test endpoint to make sure the server is running # It is used by haproxy to determine if the server is healthy @router.get("/healthy", include_in_schema=False) @@ -29,12 +35,6 @@ async def healthy(): ) -@router.get("/default", include_in_schema=False) -def index(request: Request): - data = {"request": request} - return templates.TemplateResponse("home/default.html", data) - - @router.get("/search/proposals", include_in_schema=False) async def search_proposals(request: Request): vm = SearchViewModel(request)