Handle starknet event keys/data correctly

Author: m-kus

pyproject.toml

@@ -48,38 +48,37 @@ classifiers = [
     "Typing :: Typed",
 ]
 
-dependencies = [
-    "aiohttp~=3.10",
-    "aiolimiter~=1.1",
-    "anyio~=4.4",
-    "appdirs~=1.4",
-    "APScheduler~=3.10",
-    "async-lru~=2.0",
-    "asyncpg~=0.29",
-    "click~=8.1",
-    "datamodel-code-generator~=0.26",
-    "eth-abi~=5.0",
-    "lru-dict~=1.3",
-    "orjson~=3.10",
-    "prometheus-client~=0.20",
-    "pycryptodome~=3.20",
-    "pydantic~=2.9",
-    "pyhumps~=3.8",
-    "pysignalr~=1.0",
-    "python-dotenv~=1.0",
-    "python-json-logger~=2.0",
-    "ruamel.yaml~=0.18.6",
-    "sentry-sdk~=2.16",
-    "sqlparse~=0.5",
-    "starknet-py==0.24.0",
-    "strict-rfc3339~=0.7",
-    "survey~=5.4",
-    "tabulate~=0.9",
-    # NOTE: Heavily patched; don't update without testing.
-    "tortoise-orm==0.21.7",
-    "uvloop~=0.20",
-    "web3~=7.2",
-]
+[tool.poetry.dependencies]
+aiohttp = "~3.10"
+aiolimiter = "~1.1"
+anyio = "~4.4"
+appdirs = "~1.4"
+APScheduler = "~3.10"
+async-lru = "~2.0"
+asyncpg = "~0.29"
+click = "~8.1"
+datamodel-code-generator = "~0.26"
+eth-abi = "~5.0"
+lru-dict = "~1.3"
+orjson = "~3.10"
+prometheus-client = "~0.20"
+pycryptodome = "~3.20"
+pydantic = "~2.9"
+pyhumps = "~3.8"
+pysignalr = "~1.0"
+python-dotenv = "~1.0"
+python-json-logger = "~2.0"
+ruamel.yaml = "~0.18.6"
+sentry-sdk = "~2.16"
+sqlparse = "~0.5"
+starknet-py = { git = "https://github.com/m-kus/starknet.py", rev = "8d76ee0af683062bf63c6b32a5fd49608e6b731e" }
+strict-rfc3339 = "~0.7"
+survey = "~5.4"
+tabulate = "~0.9"
+# NOTE: Heavily patched; don't update without testing.
+tortoise-orm = "0.21.7"
+uvloop = "~0.20"
+web3 = "~7.2"
 
 [project.optional-dependencies]
 migrations = [

requirements.txt

@@ -83,7 +83,7 @@ sentry-sdk==2.16.0
 six==1.16.0
 sniffio==1.3.1
 sqlparse==0.5.1
-starknet-py==0.24.0
+git+https://github.com/m-kus/starknet.py@8d76ee0af683062bf63c6b32a5fd49608e6b731e#egg=starknet-py
 strict-rfc3339==0.7
 survey==5.4.0
 sympy==1.11.1

src/dipdup/abi/cairo.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections import OrderedDict
 from functools import cache
 from typing import TYPE_CHECKING
 from typing import Any
@@ -16,7 +17,7 @@
     from starknet_py.abi.v2 import Abi  # type: ignore[import-untyped]
     from starknet_py.cairo.data_types import CairoType  # type: ignore[import-untyped]
     from starknet_py.cairo.data_types import EventType
-    from starknet_py.serialization import PayloadSerializer  # type: ignore[import-untyped]
+    from starknet_py.serialization import CairoDataSerializer  # type: ignore[import-untyped]
 
     from dipdup.package import DipDupPackage
 
@@ -25,7 +26,7 @@ class CairoEventAbi(TypedDict):
     name: str
     event_identifier: str
     members: dict[str, CairoType]
-    serializer: PayloadSerializer
+    sorted_serializers: OrderedDict[str, CairoDataSerializer]
 
 
 class CairoAbi(TypedDict):
@@ -93,12 +94,25 @@ def convert_abi(package: DipDupPackage) -> dict[str, CairoAbi]:
         for name, event_type in parsed_abi.events.items():
             if name in converted_abi['events']:
                 raise NotImplementedError('Multiple events with the same name are not supported')
+            
+            serializers = serializer_for_event(event_type).serializers
+            
+            # Event payload is returned from RPC in two arrays: keys (including event selector) and data.
+            # Since any event field can be marked as key, the original ordering might be broken.
+            #
+            # We need to reorder deserializers so that the keys remain in the beginning and
+            # the rest of the fields are moved towards the end (preserving their inner ordering).
+            #
+            # That way we can apply the deserializers to the concatenation of keys (without first element) + data.
+            sorted_members = event_type.keys + [name for name in serializers if name not in event_type.keys]
+            sorted_serializers = OrderedDict((name, serializers[name]) for name in sorted_members)
+
             converted_abi['events'].append(
                 CairoEventAbi(
                     name=name,
                     event_identifier=sn_keccak(name),
                     members=event_type.types,
-                    serializer=serializer_for_event(event_type),
+                    sorted_serializers=sorted_serializers,
                 )
             )
         abi_by_typename[contract_typename] = converted_abi

src/dipdup/indexes/starknet_events/matcher.py

@@ -1,5 +1,5 @@
 import logging
-from collections import deque
+from collections import deque, OrderedDict
 from collections.abc import Iterable
 from typing import Any
 
@@ -69,15 +69,18 @@ def prepare_event_handler_args(
         name=snake_to_pascal(handler_config.name) + 'Payload',
     )
 
-    serializer = package._cairo_abis.get_event_abi(
+    event_abi = package._cairo_abis.get_event_abi(
         typename=typename,
         name=handler_config.name,
-    )['serializer']
-    data = [int(s, 16) for s in matched_event.data]
+    )
+
+    # Skipping first key which is the event selector
+    # Note that some fields might be encoded with more than one felt (complex types)
+    raw_data = [int(x, 16) for x in matched_event.keys[1:] + matched_event.data]
 
     # holding context for error building
-    with DeserializationContext.create(data) as context:
-        data_dict = deserialize_to_dict(serializer.serializers, context)
+    with DeserializationContext.create(raw_data) as context:
+        data_dict = deserialize_to_dict(event_abi['sorted_serializers'], context)
 
     typed_payload = parse_object(type_=type_, data=data_dict)
     return StarknetEvent(