Skip to content

Commit f7078f5

Browse files
kukabikukabi
committed
Switched to cursor paging model for the events endpoint in Crystal API. Re #194.
1 parent 2b1ce09 commit f7078f5

File tree

15 files changed

+300
-101
lines changed

15 files changed

+300
-101
lines changed

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ async-trait = "0.1"
3232
anyhow = "1"
3333
axum = "0.8"
3434
axum-embed = "0.1"
35+
base64 = "0.22.1"
3536
chrono = { version = "0.4", features = ["serde"] }
3637
clap = { version = "4.5", features = ["env", "derive"] }
3738
convert_case = "0.10"

‎README.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
![](https://github.com/helikon-labs/submerge/actions/workflows/rust_checks.yml/badge.svg)
66

7+
⚠️ This project is under heavy development. Development schedule can be viewed [here](https://github.com/orgs/helikon-labs/projects/3/views/1).
8+
79
[Submerge](https://submerge.io) is a multi-function blockchain data indexing, processing, analysis, and compliance platform. Below is a list of the components of Submerge:
810

911
1. **Crystal:** The main indexer component. Indexes genesis records, blocks, extrinsics, calls within extrinsics, events, traces, logs, and metadata (versions, pallets, calls, events, constants, errors). Deployed per chain.

‎_migrations/crystal/migrations/20250624065518_event.up.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ CREATE TABLE IF NOT EXISTS event
2020
index INTEGER NOT NULL,
2121
args JSONB NOT NULL,
2222
created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(),
23-
CONSTRAINT event_pk PRIMARY KEY (block_number, hash),
23+
CONSTRAINT event_pk PRIMARY KEY (block_number, hash), -- block_number has to be in the primary key due to partitioning
2424
CONSTRAINT event_fk_block
2525
FOREIGN KEY (block_hash)
2626
REFERENCES block (hash)

‎submerge-crystal/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ async-recursion = { workspace = true }
2222
async-trait = { workspace = true }
2323
axum = { workspace = true }
2424
axum-embed = { workspace = true }
25+
base64 = { workspace = true }
2526
clap = { workspace = true }
2627
convert_case = { workspace = true }
2728
frame-metadata = { workspace = true }

‎submerge-crystal/api-spec/submerge-crystal-api.json‎

Lines changed: 110 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@
483483
"/blocks/{block_ref}/extrinsics": {
484484
"get": {
485485
"tags": [
486-
"call"
486+
"extrinsic"
487487
],
488488
"summary": "Get block extrinsics",
489489
"description": "If a hash is passed, returns the extrinsics for the matching block. If a number is passed, gives the extrinsis for the block by that number - could be multiple blocks if there's a pruned block in that slot.",
@@ -729,7 +729,7 @@
729729
"/blocks/{block_ref}/extrinsics/{extrinsic_index}/events": {
730730
"get": {
731731
"tags": [
732-
"call"
732+
"event"
733733
],
734734
"summary": "Get block extrinsic events",
735735
"description": "Returns the events for extrinsic in a block by block reference and 0-based extrinsic index.",
@@ -1438,17 +1438,13 @@
14381438
"operationId": "get_events",
14391439
"parameters": [
14401440
{
1441-
"name": "page",
1441+
"name": "next_cursor",
14421442
"in": "query",
1443-
"description": "Events list page number to retrieve. 1-indexed.",
1443+
"description": "Opaque cursor for pagination. If provided, all filter params are ignored.",
14441444
"required": false,
14451445
"schema": {
1446-
"type": "integer",
1447-
"format": "int32",
1448-
"default": 1,
1449-
"minimum": 1
1450-
},
1451-
"example": 1
1446+
"type": "string"
1447+
}
14521448
},
14531449
{
14541450
"name": "page_size",
@@ -1569,7 +1565,7 @@
15691565
],
15701566
"responses": {
15711567
"200": {
1572-
"$ref": "#/components/responses/PaginatedEventList"
1568+
"$ref": "#/components/responses/CursorEventList"
15731569
},
15741570
"400": {
15751571
"$ref": "#/components/responses/BadRequest"
@@ -3541,6 +3537,33 @@
35413537
"type": "object",
35423538
"description": "Call arguments wrapper."
35433539
},
3540+
"CursorPaginationData": {
3541+
"type": "object",
3542+
"description": "Pagination data for cursor responses.",
3543+
"required": [
3544+
"pageSize"
3545+
],
3546+
"properties": {
3547+
"nextCursor": {
3548+
"type": [
3549+
"string",
3550+
"null"
3551+
],
3552+
"description": "Cursor for the next page, `null` if there's no next page."
3553+
},
3554+
"pageSize": {
3555+
"type": "integer",
3556+
"format": "int32",
3557+
"description": "Number of items per cursor page.",
3558+
"example": 1,
3559+
"minimum": 1
3560+
}
3561+
},
3562+
"example": {
3563+
"nextCursor": "eyJjdXJzb3JfcG9zaXRpb24iOnsiYmxvY2tfbnVtYmVyIjozMzY0MzMzLCJibG9ja19oYXNoX2hleCI6IjB4ZjdlMjkyYWQ3ZDNkYzE4MzUzOWYwOGM4NDgwMmNiMDc2ZTc5NjNkYjA2NTA3MjAwNTY1M2NjNWU4YzdkMTE3MyIsImluZGV4IjowfSwicXVlcnkiOnsiaW5jbHVkZV9hcmdzIjpmYWxzZX19",
3564+
"pageSize": 1
3565+
}
3566+
},
35443567
"Error": {
35453568
"type": "object",
35463569
"description": "Generic error type.",
@@ -4918,6 +4941,82 @@
49184941
}
49194942
}
49204943
},
4944+
"CursorEventList": {
4945+
"description": "List of matching events, with a cursor for the next page.",
4946+
"headers": {
4947+
"X-RateLimit-Limit": {
4948+
"schema": {
4949+
"type": "integer",
4950+
"format": "int32",
4951+
"minimum": 0
4952+
}
4953+
},
4954+
"X-RateLimit-Remaining": {
4955+
"schema": {
4956+
"type": "integer",
4957+
"format": "int32",
4958+
"minimum": 0
4959+
}
4960+
}
4961+
},
4962+
"content": {
4963+
"application/json": {
4964+
"schema": {
4965+
"type": "object",
4966+
"required": [
4967+
"data",
4968+
"pagination"
4969+
],
4970+
"properties": {
4971+
"data": {
4972+
"type": "array",
4973+
"items": {
4974+
"$ref": "#/components/schemas/Event"
4975+
},
4976+
"example": [
4977+
{
4978+
"args": {
4979+
"dispatchInfo": {
4980+
"class": {
4981+
"type": "Mandatory",
4982+
"value": []
4983+
},
4984+
"paysFee": {
4985+
"type": "Yes",
4986+
"value": []
4987+
},
4988+
"weight": {
4989+
"proofSize": "0",
4990+
"refTime": "125000000"
4991+
}
4992+
}
4993+
},
4994+
"blockHash": "0x5c4de7f2cea658d5d3804d495e8246354f709735d371fd54caaf59e80181bcaa",
4995+
"blockNumber": 10758052,
4996+
"blockStatus": "proposed",
4997+
"blockTimestamp": 1765456362000,
4998+
"extrinsicHash": "0x6963ce866a54258d9d6ca9222060f7270a8f5f6b83eaac88e899bb73fbbb68cb",
4999+
"extrinsicIndex": 0,
5000+
"hash": "0x2c923bb54d06dfb649aaaf1c198eb1af9e19ec52b8e90267984496c128ee7adc",
5001+
"index": 1,
5002+
"palletEventIndex": 0,
5003+
"palletEventName": "ExtrinsicSuccess",
5004+
"palletIndex": 0,
5005+
"palletName": "System",
5006+
"phase": "ApplyExtrinsic",
5007+
"specVersion": 2000003,
5008+
"traceIndex": 78
5009+
}
5010+
]
5011+
},
5012+
"pagination": {
5013+
"$ref": "#/components/schemas/CursorPaginationData"
5014+
}
5015+
}
5016+
}
5017+
}
5018+
}
5019+
},
49215020
"EventList": {
49225021
"description": "List of matching events.",
49235022
"headers": {

‎submerge-crystal/src/api/docs.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
use crate::types::api::{
22
dto::{
3-
pagination::PaginationData,
3+
pagination::{CursorPaginationData, PaginationData},
44
response::{
55
block::{BlockDTO, BlockList, PaginatedBlockList},
66
call::{CallDTO, PaginatedCallList},
77
error::{BadRequest, InternalServerError, NotFound, TooManyRequests},
8-
event::{EventDTO, EventList, PaginatedEventList},
8+
event::{CursorEventList, EventDTO, EventList, PaginatedEventList},
99
extrinsic::{ExtrinsicList, PaginatedExtrinsicList},
1010
genesis::{GenesisRecordDTO, PaginatedGenesisRecordList},
1111
hex::HexStringParam,
@@ -127,12 +127,12 @@ use crate::types::api::{
127127
components(
128128
schemas(
129129
BlockDTO, CallDTO, EventDTO, TraceDTO, GenesisRecordDTO, MetadataSummaryDTO, MetadataPalletSummaryDTO, MetadataPalletDTO,
130-
PaginationData, APIErrorBody, MetadataJSON, HexStringParam,
130+
PaginationData, CursorPaginationData, APIErrorBody, MetadataJSON, HexStringParam,
131131
),
132132
responses(
133133
BlockList, PaginatedBlockList, PaginatedCallList,
134134
ExtrinsicList, PaginatedExtrinsicList,
135-
EventList, PaginatedEventList,
135+
EventList, PaginatedEventList, CursorEventList,
136136
PaginatedTraceList,
137137
PaginatedGenesisRecordList,
138138
PaginatedMetadataList, MetadataPalletSummaryList, MetadataPalletCallList,

‎submerge-crystal/src/api/mod.rs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,30 @@ pub(crate) mod legacy;
5757
pub(crate) mod v1;
5858

5959
const DEFAULT_PAGE: u32 = 1;
60-
const DEFAULT_PAGE_SIZE: u32 = 25;
61-
const MAX_PAGE_SIZE: u32 = 100;
60+
const DEFAULT_PAGE_SIZE: u32 = 1000;
61+
const MAX_PAGE_SIZE: u32 = 1000;
6262
const MAX_PAGE_SIZE_WITH_ARGS: u32 = 25;
6363
const MAX_RESPONSE_MESSAGE_BYTES: usize = 64 * 1024;
6464

65+
fn get_page_size(page_size: Option<u32>, include_args: bool) -> Result<u32, APIError> {
66+
let page_size = page_size.unwrap_or(DEFAULT_PAGE_SIZE);
67+
if page_size < 1 {
68+
Err(APIError::BadRequest(
69+
"Page size cannot be less than 1.".to_string(),
70+
))
71+
} else if !include_args && page_size > MAX_PAGE_SIZE {
72+
Err(APIError::BadRequest(format!(
73+
"Page size cannot be greater than {MAX_PAGE_SIZE}."
74+
)))
75+
} else if include_args && page_size > MAX_PAGE_SIZE_WITH_ARGS {
76+
Err(APIError::BadRequest(format!(
77+
"Page size for requests with arguments included cannot be greater than {MAX_PAGE_SIZE_WITH_ARGS}."
78+
)))
79+
} else {
80+
Ok(page_size)
81+
}
82+
}
83+
6584
fn get_page_number_and_size(
6685
page: Option<u32>,
6786
page_size: Option<u32>,

0 commit comments

Comments
 (0)