Merging Kafka metadata.

.github/workflows/check.yml

@@ -15,7 +15,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install Poetry
         run: |
-          pipx install poetry
+          pipx install 'poetry<2'
       - uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
@@ -38,7 +38,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Install Poetry
         run: |
-          pipx install poetry
+          pipx install 'poetry<2'
       - uses: actions/setup-python@v5
         with:
           python-version: "3.12"

.github/workflows/publish.yml

@@ -21,7 +21,7 @@ jobs:
           python-version: "3.12"
       - name: Install dependencies
         run: |
-          pipx install poetry
+          pipx install 'poetry<2'
       - name: Build package
         run: |
           poetry build

.github/workflows/sphinx.yaml

@@ -40,7 +40,7 @@ jobs:
           python-version: "3.12"
       - name: Install Poetry
         run: |
-          pipx install poetry
+          pipx install 'poetry<2'
       - name: Install dependencies
         run: poetry install --with docs
       - name: Setup Pages

esgf_playground_utils/config/kafka.py

@@ -24,5 +24,7 @@ class Settings(BaseSettings):
     bootstrap_servers: List[str] = ["localhost"]
     consumer_group: Optional[str] = None
     kafka_topics: Pattern[str] = compile(r".*\..*\..*")
-    error_topic: str = "esgf_consumer_error"
+    sasl_mechanism: str = "PLAIN"
+    sasl_plain_username: Optional[str] = None
+    sasl_plain_password: Optional[str] = None
     stac_server: HttpUrl = HttpUrl("http://localhost:9010")

esgf_playground_utils/models/kafka.py

@@ -87,28 +87,30 @@ class Data(BaseModel):
     payload: Union[CreatePayload, RevokePayload, UpdatePayload, PartialUpdatePayload]
 
 
-class Auth(BaseModel):
+class RequesterData(BaseModel):
     """
-    Model describing the ``AUTH`` component of a Kafka message.
-
-    .. note::
-
-      This is not an authorisation token or other verified identity. It is the simply an indication of the institute
-      providing the message.
+    Model describing ``Requests Data`` for the ``Auth`` component of a Kafka message in more detail.
     """
 
-    client_id: str
-    server: str
+    iss: str
+    sub: str
+    identity_provider: str
+    identity_provider_display_name: str
 
 
-class AuthData(BaseModel):
+class Auth(BaseModel):
     """
     Model describing ``Auth`` component of a Kafka message in more detail.
+
+     .. note::
+
+      This is not an authorisation token or other verified identity. It is the simply an indication of the institute
+      providing the message.
     """
 
     auth_policy_id: str
     client_id: str
-    requester_data: Dict[str, str]
+    requester_data: RequesterData
 
 
 class Publisher(BaseModel):
@@ -126,28 +128,21 @@ class Metadata(BaseModel):
     Multiple metadata attributes required for ESGF but not part of the STAC payload.
     """
 
-    auth: Union[Auth, AuthData]
+    event_id: str
+    request_id: str
+    auth: Auth
     publisher: Publisher
     time: datetime
     schema_version: str
 
 
-class ExtendedMetadata(Metadata):
-    """
-    An extension of the Metadata Model for Publish Events.
-    """
-
-    event_id: str
-    request_id: str
-
-
 class KafkaEvent(BaseModel):
     """
     The full content of a Kafka message, containing both the STAC payload, the request description and the ESGF
     mandated metadata.
     """
 
-    metadata: Union[Metadata, ExtendedMetadata]
+    metadata: Metadata
     data: Data
 
 

pyproject.toml

@@ -1,13 +1,13 @@
 [tool.poetry]
 name = "esgf-playground-utils"
-version = "0.3.8"
+version = "0.4.0"
 description = "Common library and models for the ESGF-Playground"
+readme = "README.rst"
 authors = [
     "David Poulter <[email protected]>",
     "Rhys Evans <[email protected]>",
     "Ed Borthwick <[email protected]>",
 ]
-readme = "README.rst"
 
 [tool.poetry.dependencies]
 python = ">=3.9"

test/functional_test/__init__.py

@@ -149,15 +149,23 @@ def test_create_payload_valid(self) -> None:
     }
   },
   "metadata": {
+    "event_id": "3a8a2536-323b-4259-a218-6002a3491d25",
+    "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501",
     "auth": {
-      "client_id": "<KEY>",
-      "server": "<SERVER>"
+        "auth_policy_id": "ESGF-Publish-00012",
+        "client_id": "CEDA-transaction-client",
+        "requester_data": {
+          "iss": "auth.globus.org",
+          "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2",
+          "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47",
+          "identity_provider_display_name": "University of Chicago"
+        }
     },
     "publisher": {
-      "package": "some_python_package",
-      "version": "0.3.0"
+        "package": "some_python_package",
+        "version": "0.3.0"
     },
-    "time": "2022-01-20T00:00:00Z",
+    "time": "2024-07-04T14:17:35Z",
     "schema_version": "1.0.0"
   }
 }
@@ -299,15 +307,23 @@ def test_create_payload_invalid(self) -> None:
     }
   },
   "metadata": {
+    "event_id": "3a8a2536-323b-4259-a218-6002a3491d25",
+    "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501",
     "auth": {
-      "client_id": "<KEY>",
-      "server": "<SERVER>"
+        "auth_policy_id": "ESGF-Publish-00012",
+        "client_id": "CEDA-transaction-client",
+        "requester_data": {
+          "iss": "auth.globus.org",
+          "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2",
+          "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47",
+          "identity_provider_display_name": "University of Chicago"
+        }
     },
     "publisher": {
-      "package": "some_python_package",
-      "version": "0.3.0"
+        "package": "some_python_package",
+        "version": "0.3.0"
     },
-    "time": "2022-01-20T00:00:00Z",
+    "time": "2024-07-04T14:17:35Z",
     "schema_version": "1.0.0"
   }
 }
@@ -325,19 +341,27 @@ def test_revoke_payload_soft(self) -> None:
             "payload": {
               "collection_id": "cmip6",
               "method": "PATCH",
-              "item_id": "an_item" 
+              "item_id": "an_item"
             }
           },
           "metadata": {
+            "event_id": "3a8a2536-323b-4259-a218-6002a3491d25",
+            "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501",
             "auth": {
-              "client_id": "<KEY>",
-              "server": "<SERVER>"
+                "auth_policy_id": "ESGF-Publish-00012",
+                "client_id": "CEDA-transaction-client",
+                "requester_data": {
+                  "iss": "auth.globus.org",
+                  "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2",
+                  "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47",
+                  "identity_provider_display_name": "University of Chicago"
+                }
             },
             "publisher": {
-              "package": "some_python_package",
-              "version": "0.3.0"
+                "package": "some_python_package",
+                "version": "0.3.0"
             },
-            "time": "2022-01-20T00:00:00Z",
+            "time": "2024-07-04T14:17:35Z",
             "schema_version": "1.0.0"
           }
         }
@@ -360,15 +384,23 @@ def test_revoke_payload_hard(self) -> None:
     }
   },
   "metadata": {
+    "event_id": "3a8a2536-323b-4259-a218-6002a3491d25",
+    "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501",
     "auth": {
-      "client_id": "<KEY>",
-      "server": "<SERVER>"
+        "auth_policy_id": "ESGF-Publish-00012",
+        "client_id": "CEDA-transaction-client",
+        "requester_data": {
+          "iss": "auth.globus.org",
+          "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2",
+          "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47",
+          "identity_provider_display_name": "University of Chicago"
+        }
     },
     "publisher": {
-      "package": "some_python_package",
-      "version": "0.3.0"
+        "package": "some_python_package",
+        "version": "0.3.0"
     },
-    "time": "2022-01-20T00:00:00Z",
+    "time": "2024-07-04T14:17:35Z",
     "schema_version": "1.0.0"
   }
 }
@@ -511,15 +543,23 @@ def test_update_payload_full(self) -> None:
     }
   },
   "metadata": {
+    "event_id": "3a8a2536-323b-4259-a218-6002a3491d25",
+    "request_id": "dee8c54c-e815-4863-87fb-7b606efd2501",
     "auth": {
-      "client_id": "<KEY>",
-      "server": "<SERVER>"
+        "auth_policy_id": "ESGF-Publish-00012",
+        "client_id": "CEDA-transaction-client",
+        "requester_data": {
+          "iss": "auth.globus.org",
+          "sub": "b16b12b6-d274-11e5-8e41-5fea585a1aa2",
+          "identity_provider": "0dcf5063-bffd-40f7-b403-24f97e32fa47",
+          "identity_provider_display_name": "University of Chicago"
+        }
     },
     "publisher": {
-      "package": "some_python_package",
-      "version": "0.3.0"
+        "package": "some_python_package",
+        "version": "0.3.0"
     },
-    "time": "2022-01-20T00:00:00Z",
+    "time": "2024-07-04T14:17:35Z",
     "schema_version": "1.0.0"
   }
 }