diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index bd36cb0..5e5c806 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Poetry run: | - pipx install poetry + pipx install 'poetry<2' - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Poetry run: | - pipx install poetry + pipx install 'poetry<2' - uses: actions/setup-python@v5 with: python-version: "3.12" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d99285f..213992a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,7 +21,7 @@ jobs: python-version: "3.12" - name: Install dependencies run: | - pipx install poetry + pipx install 'poetry<2' - name: Build package run: | poetry build diff --git a/.github/workflows/sphinx.yaml b/.github/workflows/sphinx.yaml index 8ca09f0..1a2bdb5 100644 --- a/.github/workflows/sphinx.yaml +++ b/.github/workflows/sphinx.yaml @@ -40,7 +40,7 @@ jobs: python-version: "3.12" - name: Install Poetry run: | - pipx install poetry + pipx install 'poetry<2' - name: Install dependencies run: poetry install --with docs - name: Setup Pages diff --git a/esgf_playground_utils/config/kafka.py b/esgf_playground_utils/config/kafka.py index 53ec51b..86e0b66 100644 --- a/esgf_playground_utils/config/kafka.py +++ b/esgf_playground_utils/config/kafka.py @@ -24,5 +24,7 @@ class Settings(BaseSettings): bootstrap_servers: List[str] = ["localhost"] consumer_group: Optional[str] = None kafka_topics: Pattern[str] = compile(r".*\..*\..*") - error_topic: str = "esgf_consumer_error" + sasl_mechanism: str = "PLAIN" + sasl_plain_username: Optional[str] = None + sasl_plain_password: Optional[str] = None stac_server: HttpUrl = HttpUrl("http://localhost:9010") diff --git a/esgf_playground_utils/models/kafka.py b/esgf_playground_utils/models/kafka.py index 087124e..9475773 100644 --- a/esgf_playground_utils/models/kafka.py +++ b/esgf_playground_utils/models/kafka.py @@ -87,28 +87,30 @@ class Data(BaseModel): payload: Union[CreatePayload, RevokePayload, UpdatePayload, PartialUpdatePayload] -class Auth(BaseModel): +class RequesterData(BaseModel): """ - Model describing the ``AUTH`` component of a Kafka message. - - .. note:: - - This is not an authorisation token or other verified identity. It is the simply an indication of the institute - providing the message. + Model describing ``Requests Data`` for the ``Auth`` component of a Kafka message in more detail. """ - client_id: str - server: str + iss: str + sub: str + identity_provider: str + identity_provider_display_name: str -class AuthData(BaseModel): +class Auth(BaseModel): """ Model describing ``Auth`` component of a Kafka message in more detail. + + .. note:: + + This is not an authorisation token or other verified identity. It is the simply an indication of the institute + providing the message. """ auth_policy_id: str client_id: str - requester_data: Dict[str, str] + requester_data: RequesterData class Publisher(BaseModel): @@ -126,28 +128,21 @@ class Metadata(BaseModel): Multiple metadata attributes required for ESGF but not part of the STAC payload. """ - auth: Union[Auth, AuthData] + event_id: str + request_id: str + auth: Auth publisher: Publisher time: datetime schema_version: str -class ExtendedMetadata(Metadata): - """ - An extension of the Metadata Model for Publish Events. - """ - - event_id: str - request_id: str - - class KafkaEvent(BaseModel): """ The full content of a Kafka message, containing both the STAC payload, the request description and the ESGF mandated metadata. """ - metadata: Union[Metadata, ExtendedMetadata] + metadata: Metadata data: Data diff --git a/pyproject.toml b/pyproject.toml index a8b5722..f865fef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,13 +1,13 @@ [tool.poetry] name = "esgf-playground-utils" -version = "0.3.8" +version = "0.4.0" description = "Common library and models for the ESGF-Playground" +readme = "README.rst" authors = [ "David Poulter ", "Rhys Evans ", "Ed Borthwick ", ] -readme = "README.rst" [tool.poetry.dependencies] python = ">=3.9" diff --git a/test/functional_test/__init__.py b/test/functional_test/__init__.py index a5448a0..b1c0e13 100644 --- a/test/functional_test/__init__.py +++ b/test/functional_test/__init__.py @@ -149,15 +149,23 @@ def test_create_payload_valid(self) -> None: } }, "metadata": { + "event_id": "3a8a2536-323b-4259-a218-6002a3491d25", + "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501", "auth": { - "client_id": "", - "server": "" + "auth_policy_id": "ESGF-Publish-00012", + "client_id": "CEDA-transaction-client", + "requester_data": { + "iss": "auth.globus.org", + "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2", + "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47", + "identity_provider_display_name": "University of Chicago" + } }, "publisher": { - "package": "some_python_package", - "version": "0.3.0" + "package": "some_python_package", + "version": "0.3.0" }, - "time": "2022-01-20T00:00:00Z", + "time": "2024-07-04T14:17:35Z", "schema_version": "1.0.0" } } @@ -299,15 +307,23 @@ def test_create_payload_invalid(self) -> None: } }, "metadata": { + "event_id": "3a8a2536-323b-4259-a218-6002a3491d25", + "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501", "auth": { - "client_id": "", - "server": "" + "auth_policy_id": "ESGF-Publish-00012", + "client_id": "CEDA-transaction-client", + "requester_data": { + "iss": "auth.globus.org", + "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2", + "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47", + "identity_provider_display_name": "University of Chicago" + } }, "publisher": { - "package": "some_python_package", - "version": "0.3.0" + "package": "some_python_package", + "version": "0.3.0" }, - "time": "2022-01-20T00:00:00Z", + "time": "2024-07-04T14:17:35Z", "schema_version": "1.0.0" } } @@ -325,19 +341,27 @@ def test_revoke_payload_soft(self) -> None: "payload": { "collection_id": "cmip6", "method": "PATCH", - "item_id": "an_item" + "item_id": "an_item" } }, "metadata": { + "event_id": "3a8a2536-323b-4259-a218-6002a3491d25", + "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501", "auth": { - "client_id": "", - "server": "" + "auth_policy_id": "ESGF-Publish-00012", + "client_id": "CEDA-transaction-client", + "requester_data": { + "iss": "auth.globus.org", + "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2", + "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47", + "identity_provider_display_name": "University of Chicago" + } }, "publisher": { - "package": "some_python_package", - "version": "0.3.0" + "package": "some_python_package", + "version": "0.3.0" }, - "time": "2022-01-20T00:00:00Z", + "time": "2024-07-04T14:17:35Z", "schema_version": "1.0.0" } } @@ -360,15 +384,23 @@ def test_revoke_payload_hard(self) -> None: } }, "metadata": { + "event_id": "3a8a2536-323b-4259-a218-6002a3491d25", + "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501", "auth": { - "client_id": "", - "server": "" + "auth_policy_id": "ESGF-Publish-00012", + "client_id": "CEDA-transaction-client", + "requester_data": { + "iss": "auth.globus.org", + "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2", + "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47", + "identity_provider_display_name": "University of Chicago" + } }, "publisher": { - "package": "some_python_package", - "version": "0.3.0" + "package": "some_python_package", + "version": "0.3.0" }, - "time": "2022-01-20T00:00:00Z", + "time": "2024-07-04T14:17:35Z", "schema_version": "1.0.0" } } @@ -511,15 +543,23 @@ def test_update_payload_full(self) -> None: } }, "metadata": { + "event_id": "3a8a2536-323b-4259-a218-6002a3491d25", + "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501", "auth": { - "client_id": "", - "server": "" + "auth_policy_id": "ESGF-Publish-00012", + "client_id": "CEDA-transaction-client", + "requester_data": { + "iss": "auth.globus.org", + "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2", + "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47", + "identity_provider_display_name": "University of Chicago" + } }, "publisher": { - "package": "some_python_package", - "version": "0.3.0" + "package": "some_python_package", + "version": "0.3.0" }, - "time": "2022-01-20T00:00:00Z", + "time": "2024-07-04T14:17:35Z", "schema_version": "1.0.0" } }