Handle Starknet events keys/data correctly

Dockerfile

@@ -2,7 +2,7 @@
 FROM python:3.12-slim-bookworm AS compile-image
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 RUN apt update && \
-    apt install -y build-essential && \
+    apt install -y build-essential git && \
     python -m venv --without-pip --system-site-packages /opt/dipdup && \
     mkdir -p /opt/dipdup/src/dipdup/ && \
     touch /opt/dipdup/src/dipdup/__init__.py && \

pyproject.toml

@@ -71,7 +71,7 @@ dependencies = [
     "ruamel.yaml~=0.18.6",
     "sentry-sdk~=2.16",
     "sqlparse~=0.5",
-    "starknet-py==0.24.0",
+    "starknet-py @ git+https://github.com/software-mansion/starknet.py@a8d73538d409d9ef7c756921e43d10925f2838bc",
     "strict-rfc3339~=0.7",
     "survey~=5.4",
     "tabulate~=0.9",

requirements.txt

@@ -10,26 +10,26 @@ annotated-types==0.7.0
 anyio==4.7.0
 appdirs==1.4.4
 apscheduler==3.11.0
-argcomplete==3.5.0
+argcomplete==3.5.2
 asgiref==3.8.1
 async-lru==2.0.4
 asyncpg==0.30.0
 attrs==24.2.0
-bitarray==2.9.2
+bitarray==3.0.0
 black==24.10.0
 certifi==2024.8.30
 charset-normalizer==3.3.2
-ckzg==2.0.0
+ckzg==2.0.1
 click==8.1.7
 colorama==0.4.6; platform_system == "Windows" or sys_platform == "win32"
-crypto-cpp-py==1.4.4
+crypto-cpp-py==1.4.5
 cytoolz==0.12.3; implementation_name == "cpython"
-datamodel-code-generator==0.26.3
-dnspython==2.6.1; python_version ~= "3.11"
+datamodel-code-generator==0.26.4
+dnspython==2.7.0; python_version ~= "3.11"
 ecdsa==0.18.0
 email-validator==2.2.0; python_version ~= "3.11"
 eth-abi==5.1.0
-eth-account==0.13.3
+eth-account==0.13.4
 eth-hash[pycryptodome]==0.7.0
 eth-keyfile==0.8.1
 eth-keys==0.5.1
@@ -51,7 +51,7 @@ marshmallow==3.22.0
 marshmallow-dataclass==8.7.0
 marshmallow-oneofschema==3.1.1
 mpmath==1.3.0
-msgpack==1.0.8
+msgpack==1.1.0
 multidict==6.0.5
 mypy-extensions==1.0.0
 orjson==3.10.12
@@ -71,33 +71,33 @@ pysignalr==1.1.0
 python-dotenv==1.0.1
 python-json-logger==2.0.7
 pytz==2024.1
-pyunormalize==15.1.0
+pyunormalize==16.0.0
 pywin32==306; platform_system == "Windows" or sys_platform == "win32" or os_name == "nt"
 pyyaml==6.0.2
-regex==2024.7.24
+regex==2024.11.6
 requests==2.32.3
 rlp==4.0.1
 ruamel-yaml==0.18.6
-ruamel-yaml-clib==0.2.8; platform_python_implementation == "CPython" and python_version < "3.13"
+ruamel-yaml-clib==0.2.12; platform_python_implementation == "CPython" and python_version < "3.13"
 sentry-sdk==2.19.2
 six==1.16.0
 sniffio==1.3.1
 sqlparse==0.5.3
-starknet-py==0.24.0
+starknet-py @ git+https://github.com/software-mansion/starknet.py@a8d73538d409d9ef7c756921e43d10925f2838bc
 strict-rfc3339==0.7
 survey==5.4.2
-sympy==1.11.1
+sympy==1.12.1
 tabulate==0.9.0
 toolz==0.12.1; implementation_name == "pypy" or implementation_name == "cpython"
 tortoise-orm==0.21.7
 typeguard==4.0.1
-types-requests==2.32.0.20240712
+types-requests==2.32.0.20241016
 typing-extensions==4.12.2
 typing-inspect==0.9.0
-tzdata==2024.1; platform_system == "Windows"
+tzdata==2024.2; platform_system == "Windows"
 tzlocal==5.2
 urllib3==2.2.2
 uvloop==0.21.0
 web3==7.6.0
-websockets==12.0
+websockets==13.1
 yarl==1.18.0

src/demo_starknet_events/handlers/on_transfer.py

@@ -1,11 +1,12 @@
 from decimal import Decimal
 
-from demo_starknet_events import models as models
-from demo_starknet_events.types.stark_usdt.starknet_events.transfer import TransferPayload
 from dipdup.context import HandlerContext
 from dipdup.models.starknet import StarknetEvent
 from tortoise.exceptions import DoesNotExist
 
+from demo_starknet_events import models as models
+from demo_starknet_events.types.stark_usdt.starknet_events.transfer import TransferPayload
+
 
 async def on_transfer(
     ctx: HandlerContext,

src/dipdup/abi/cairo.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections import OrderedDict
 from functools import cache
 from typing import TYPE_CHECKING
 from typing import Any
@@ -16,7 +17,7 @@
     from starknet_py.abi.v2 import Abi  # type: ignore[import-untyped]
     from starknet_py.cairo.data_types import CairoType  # type: ignore[import-untyped]
     from starknet_py.cairo.data_types import EventType
-    from starknet_py.serialization import PayloadSerializer  # type: ignore[import-untyped]
+    from starknet_py.serialization import CairoDataSerializer  # type: ignore[import-untyped]
 
     from dipdup.package import DipDupPackage
 
@@ -25,7 +26,7 @@ class CairoEventAbi(TypedDict):
     name: str
     event_identifier: str
     members: dict[str, CairoType]
-    serializer: PayloadSerializer
+    sorted_serializers: OrderedDict[str, CairoDataSerializer]
 
 
 class CairoAbi(TypedDict):
@@ -93,12 +94,25 @@ def convert_abi(package: DipDupPackage) -> dict[str, CairoAbi]:
         for name, event_type in parsed_abi.events.items():
             if name in converted_abi['events']:
                 raise NotImplementedError('Multiple events with the same name are not supported')
+
+            serializers = serializer_for_event(event_type).serializers
+
+            # Event payload is returned from RPC in two arrays: keys (including event selector) and data.
+            # Since any event field can be marked as key, the original ordering might be broken.
+            #
+            # We need to reorder deserializers so that the keys remain in the beginning and
+            # the rest of the fields are moved towards the end (preserving their inner ordering).
+            #
+            # That way we can apply the deserializers to the concatenation of keys (without first element) + data.
+            sorted_members = event_type.keys + [name for name in serializers if name not in event_type.keys]
+            sorted_serializers = OrderedDict((name, serializers[name]) for name in sorted_members)
+
             converted_abi['events'].append(
                 CairoEventAbi(
                     name=name,
                     event_identifier=sn_keccak(name),
                     members=event_type.types,
-                    serializer=serializer_for_event(event_type),
+                    sorted_serializers=sorted_serializers,
                 )
             )
         abi_by_typename[contract_typename] = converted_abi

src/dipdup/indexes/starknet_events/matcher.py

@@ -69,15 +69,18 @@ def prepare_event_handler_args(
         name=snake_to_pascal(handler_config.name) + 'Payload',
     )
 
-    serializer = package._cairo_abis.get_event_abi(
+    event_abi = package._cairo_abis.get_event_abi(
         typename=typename,
         name=handler_config.name,
-    )['serializer']
-    data = [int(s, 16) for s in matched_event.data]
+    )
+
+    # Skipping first key which is the event selector
+    # Note that some fields might be encoded with more than one felt (complex types)
+    raw_data = [int(x, 16) for x in matched_event.keys[1:] + matched_event.data]
 
     # holding context for error building
-    with DeserializationContext.create(data) as context:
-        data_dict = deserialize_to_dict(serializer.serializers, context)
+    with DeserializationContext.create(raw_data) as context:
+        data_dict = deserialize_to_dict(event_abi['sorted_serializers'], context)
 
     typed_payload = parse_object(type_=type_, data=data_dict)
     return StarknetEvent(