Some improvements related to cursor paging model for the events endpoint in Crystal API. Re #194.

Author: kukabi

Cargo.toml

@@ -32,7 +32,7 @@ async-trait = "0.1"
 anyhow = "1"
 axum = "0.8"
 axum-embed = "0.1"
-base64 = "0.22.1"
+base64 = "0.22"
 chrono = { version = "0.4", features = ["serde"] }
 clap = { version = "4.5", features = ["env", "derive"] }
 convert_case = "0.10"

submerge-crystal/api-spec/submerge-crystal-api.json

@@ -1440,7 +1440,7 @@
           {
             "name": "next_cursor",
             "in": "query",
-            "description": "Opaque cursor for pagination. If provided, all filter params are ignored.",
+            "description": "Opaque cursor for pagination - returned in the endpoint response.\nThis parameter is mutually exclusive with all other parameters,\nand will return bad request if any other parameter is set.",
             "required": false,
             "schema": {
               "type": "string"
@@ -3545,10 +3545,7 @@
         ],
         "properties": {
           "nextCursor": {
-            "type": [
-              "string",
-              "null"
-            ],
+            "type": "string",
             "description": "Cursor for the next page, `null` if there's no next page."
           },
           "pageSize": {
@@ -4752,7 +4749,7 @@
       },
       "SignatureHexString": {
         "type": "string",
-        "description": "and ECDSA (65 bytes → 130 hex chars). **Always** `0x`-prefixed and lowercase in responses.\nInputs elsewhere may accept mixed case or missing `0x`; the API normalizes outputs.",
+        "description": "Hex-encoded Substrate extrinsic signature. Supports Sr25519/Ed25519 (64 bytes → 128 hex chars)\nand ECDSA (65 bytes → 130 hex chars). **Always** `0x`-prefixed and lowercase in responses.\nInputs elsewhere may accept mixed case or missing `0x`; the API normalizes outputs.",
         "examples": [
           "0xabababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab",
           "0xababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab"

submerge-crystal/src/api/mod.rs

@@ -57,13 +57,18 @@ pub(crate) mod legacy;
 pub(crate) mod v1;
 
 const DEFAULT_PAGE: u32 = 1;
-const DEFAULT_PAGE_SIZE: u32 = 1000;
-const MAX_PAGE_SIZE: u32 = 1000;
+const DEFAULT_PAGE_SIZE: u32 = 100;
+const MAX_PAGE_SIZE: u32 = 250;
+const DEFAULT_PAGE_SIZE_WITH_ARGS: u32 = 25;
 const MAX_PAGE_SIZE_WITH_ARGS: u32 = 25;
 const MAX_RESPONSE_MESSAGE_BYTES: usize = 64 * 1024;
 
 fn get_page_size(page_size: Option<u32>, include_args: bool) -> Result<u32, APIError> {
-    let page_size = page_size.unwrap_or(DEFAULT_PAGE_SIZE);
+    let page_size = page_size.unwrap_or(if include_args {
+        DEFAULT_PAGE_SIZE_WITH_ARGS
+    } else {
+        DEFAULT_PAGE_SIZE
+    });
     if page_size < 1 {
         Err(APIError::BadRequest(
             "Page size cannot be less than 1.".to_string(),
@@ -87,25 +92,12 @@ fn get_page_number_and_size(
     include_args: bool,
 ) -> Result<(u32, u32), APIError> {
     let page = page.unwrap_or(DEFAULT_PAGE);
-    let page_size = page_size.unwrap_or(DEFAULT_PAGE_SIZE);
     if page < 1 {
         Err(APIError::BadRequest(
             "Page number cannot be less than 1.".to_string(),
         ))
-    } else if page_size < 1 {
-        Err(APIError::BadRequest(
-            "Page size cannot be less than 1.".to_string(),
-        ))
-    } else if !include_args && page_size > MAX_PAGE_SIZE {
-        Err(APIError::BadRequest(format!(
-            "Page size cannot be greater than {MAX_PAGE_SIZE}."
-        )))
-    } else if include_args && page_size > MAX_PAGE_SIZE_WITH_ARGS {
-        Err(APIError::BadRequest(format!(
-            "Page size for requests with arguments included cannot be greater than {MAX_PAGE_SIZE_WITH_ARGS}."
-        )))
     } else {
-        Ok((page, page_size))
+        Ok((page, get_page_size(page_size, include_args)?))
     }
 }
 

submerge-crystal/src/api/v1/event.rs

@@ -17,14 +17,14 @@ use crate::{
             pagination::{CursorPaginationData, PaginationData},
             request::{
                 block::BlockReference,
-                event::{
-                    BlockEventQuery, EventCursorPayload, EventCursorPosition, EventQuery,
-                    IncludeEventArgsParam,
-                },
+                event::{BlockEventQuery, EventQuery, IncludeEventArgsParam},
             },
             response::{
                 error::{BadRequest, InternalServerError, NotFound, TooManyRequests},
-                event::{CursorEventList, EventArgs, EventDTO, EventList, PaginatedEventList},
+                event::{
+                    CursorEventList, EventArgs, EventCursorPayload, EventCursorPosition, EventDTO,
+                    EventList, PaginatedEventList,
+                },
             },
         },
         error::APIError,
@@ -61,6 +61,7 @@ pub(crate) async fn get_events(
     State(state): State<ServiceState>,
     Query(query): Query<EventQuery>,
 ) -> Result<Json<CursorEventList>, APIError> {
+    query.validate_next_cursor_mutually_exclusive()?;
     let (cursor_position, query) = if let Some(cursor) = query.next_cursor {
         // TODO validate that no other query params are set
         let decoded = URL_SAFE_NO_PAD.decode(cursor)?;

submerge-crystal/src/persistence/api/event.rs

@@ -1,4 +1,6 @@
-use crate::types::{api::dto::request::event::EventCursorPosition, persistence::EventCompositeRow};
+use crate::types::{
+    api::dto::response::event::EventCursorPosition, persistence::EventCompositeRow,
+};
 use serde_json::Value as JSONValue;
 use sqlx::{Postgres, QueryBuilder};
 use submerge_persistence::postgres::{escape_like_pattern, PostgreSQLStorage};
@@ -31,7 +33,7 @@ fn get_select_query(include_args: bool) -> String {
 pub(crate) trait CrystalEventAPIPostgreSQLStorage {
     async fn get_events(
         &self,
-        cursor: Option<EventCursorPosition>,
+        cursor_position: Option<EventCursorPosition>,
         min_block_number: Option<i64>,
         max_block_number: Option<i64>,
         pallet_name: &Option<String>,
@@ -141,7 +143,7 @@ pub(crate) trait CrystalEventAPIPostgreSQLStorage {
 impl CrystalEventAPIPostgreSQLStorage for PostgreSQLStorage {
     async fn get_events(
         &self,
-        cursor: Option<EventCursorPosition>,
+        cursor_position: Option<EventCursorPosition>,
         min_block_number: Option<i64>,
         max_block_number: Option<i64>,
         pallet_name: &Option<String>,
@@ -159,28 +161,28 @@ impl CrystalEventAPIPostgreSQLStorage for PostgreSQLStorage {
             query_builder.push(" AND E.block_number <= ").push_bind(max);
         }
 
-        if let Some(cursor) = cursor {
-            let block_hash = hex::decode(cursor.block_hash_hex.trim_start_matches("0x"))?;
+        if let Some(cursor_position) = cursor_position {
+            let block_hash = cursor_position.get_block_hash()?;
             query_builder.push(" AND (");
             query_builder
                 .push("E.block_number < ")
-                .push_bind(cursor.block_number as i64);
+                .push_bind(cursor_position.block_number as i64);
             query_builder
                 .push(" OR (E.block_number = ")
-                .push_bind(cursor.block_number as i64);
+                .push_bind(cursor_position.block_number as i64);
             query_builder
                 .push(" AND E.block_hash > ")
                 .push_bind(block_hash.clone());
             query_builder.push(")");
             query_builder
                 .push(" OR (E.block_number = ")
-                .push_bind(cursor.block_number as i64);
+                .push_bind(cursor_position.block_number as i64);
             query_builder
                 .push(" AND E.block_hash = ")
                 .push_bind(block_hash.clone());
             query_builder
                 .push(" AND E.index > ")
-                .push_bind(cursor.index as i32);
+                .push_bind(cursor_position.index as i32);
             query_builder.push(")");
             query_builder.push(")");
         }

submerge-crystal/src/types/api/dto/pagination.rs

@@ -45,5 +45,6 @@ pub(crate) struct CursorPaginationData {
     #[schema(example = 1)]
     pub page_size: u32,
     /// Cursor for the next page, `null` if there's no next page.
+    #[schema(required = false, nullable = false)]
     pub next_cursor: Option<String>,
 }

submerge-crystal/src/types/api/dto/request/event.rs

@@ -1,11 +1,15 @@
 use serde::{Deserialize, Serialize};
 use utoipa::IntoParams;
 
+use crate::types::api::error::APIError;
+
 /// Query parameters for fetching and filtering events.
 #[derive(Debug, Serialize, Deserialize, IntoParams)]
 #[serde(deny_unknown_fields)]
 pub(crate) struct EventQuery {
-    /// Opaque cursor for pagination. If provided, all filter params are ignored.
+    /// Opaque cursor for pagination - returned in the endpoint response.
+    /// This parameter is mutually exclusive with all other parameters,
+    /// and will return bad request if any other parameter is set.
     #[param(required = false, nullable = false)]
     #[serde(skip_serializing_if = "Option::is_none")]
     pub next_cursor: Option<String>,
@@ -70,17 +74,48 @@ pub(crate) struct EventQuery {
     pub include_args: bool,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
-pub(crate) struct EventCursorPosition {
-    pub(crate) block_number: u64,
-    pub(crate) block_hash_hex: String,
-    pub(crate) index: u32,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub(crate) struct EventCursorPayload {
-    pub(crate) cursor_position: EventCursorPosition,
-    pub(crate) query: EventQuery,
+impl EventQuery {
+    pub(crate) fn validate_next_cursor_mutually_exclusive(&self) -> Result<(), APIError> {
+        if self.next_cursor.is_some() {
+            let mut other_fields = Vec::new();
+            if self.page_size.is_some() {
+                other_fields.push("page_size");
+            }
+            if self.min_block_number.is_some() {
+                other_fields.push("min_block_number");
+            }
+            if self.max_block_number.is_some() {
+                other_fields.push("max_block_number");
+            }
+            if self.min_block_timestamp.is_some() {
+                other_fields.push("min_block_timestamp");
+            }
+            if self.max_block_timestamp.is_some() {
+                other_fields.push("max_block_timestamp");
+            }
+            if self.min_spec_version.is_some() {
+                other_fields.push("min_spec_version");
+            }
+            if self.max_spec_version.is_some() {
+                other_fields.push("max_spec_version");
+            }
+            if self.pallet_name.is_some() {
+                other_fields.push("pallet_name");
+            }
+            if self.event_name.is_some() {
+                other_fields.push("event_name");
+            }
+            if !other_fields.is_empty() {
+                return Err(APIError::BadRequest(
+                    format!(
+                        "No other parameter should not be set when next_cursor is set. Please remove these parameters and try again: {}",
+                        other_fields.join(", ")
+                    )
+                ));
+            }
+        }
+        Ok(())
+    }
 }
 
 /// Query parameters for fetching and filtering events within a block.

submerge-crystal/src/types/api/dto/response/event.rs

@@ -1,10 +1,11 @@
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 use serde_json::Value as JSONValue;
 use utoipa::{ToResponse, ToSchema};
 
 use crate::types::{
     api::dto::{
         pagination::{CursorPaginationData, PaginationData},
+        request::event::EventQuery,
         response::{
             example::event::event_example, hex::Hash256Hex, schema::event::event_args_schema,
         },
@@ -108,6 +109,25 @@ pub(crate) struct PaginatedEventList {
     pub pagination: PaginationData,
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct EventCursorPosition {
+    pub(crate) block_number: u64,
+    pub(crate) block_hash_hex: String,
+    pub(crate) index: u32,
+}
+
+impl EventCursorPosition {
+    pub(crate) fn get_block_hash(&self) -> anyhow::Result<Vec<u8>> {
+        Ok(hex::decode(self.block_hash_hex.trim_start_matches("0x"))?)
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub(crate) struct EventCursorPayload {
+    pub(crate) cursor_position: EventCursorPosition,
+    pub(crate) query: EventQuery,
+}
+
 #[derive(Debug, Serialize, ToResponse, ToSchema)]
 #[response(
     description = "List of matching events, with a cursor for the next page.",

submerge-crystal/src/types/api/dto/response/hex.rs

@@ -80,7 +80,7 @@ impl From<&[u8; 32]> for Address32Hex {
     }
 }
 
-// Hex-encoded Substrate extrinsic signature. Supports Sr25519/Ed25519 (64 bytes → 128 hex chars)
+/// Hex-encoded Substrate extrinsic signature. Supports Sr25519/Ed25519 (64 bytes → 128 hex chars)
 /// and ECDSA (65 bytes → 130 hex chars). **Always** `0x`-prefixed and lowercase in responses.
 /// Inputs elsewhere may accept mixed case or missing `0x`; the API normalizes outputs.
 #[derive(Debug, Clone, Serialize, ToSchema)]