Skip to content

Commit 236e053

Browse files
kukabikukabi
committed
Switched to cursor paging model for the extrinsics endpoint in Crystal API. Re #194.
1 parent d6861f6 commit 236e053

File tree

8 files changed

+276
-102
lines changed

8 files changed

+276
-102
lines changed

‎_migrations/crystal/migrations/20250624065515_extrinsic.up.sql‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS extrinsic
1414
is_successful BOOLEAN NOT NULL,
1515
extra JSONB,
1616
created_at TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT now(),
17-
CONSTRAINT extrinsic_pk PRIMARY KEY (block_hash, block_number, hash),
17+
CONSTRAINT extrinsic_pk PRIMARY KEY (block_hash, block_number, hash), -- block_number has to be in the primary key due to partitioning
1818
CONSTRAINT extrinsic_u_block_hash_block_number_index UNIQUE (block_hash, block_number, index),
1919
CONSTRAINT extrinsic_fk_block
2020
FOREIGN KEY (block_hash)
@@ -28,12 +28,12 @@ CREATE INDEX IF NOT EXISTS extrinsic_idx_hash
2828
CREATE INDEX IF NOT EXISTS extrinsic_idx_block_hash
2929
ON extrinsic (block_hash, index ASC);
3030
CREATE INDEX IF NOT EXISTS extrinsic_idx_block_number
31-
ON extrinsic (block_number DESC, index ASC);
31+
ON extrinsic (block_number DESC, block_hash ASC, index ASC);
3232
CREATE INDEX IF NOT EXISTS extrinsic_idx_block_number_signed
33-
ON extrinsic (block_number DESC, index ASC)
33+
ON extrinsic (block_number DESC, block_hash ASC, index ASC)
3434
WHERE multi_signature IS NOT NULL;
3535
CREATE INDEX IF NOT EXISTS extrinsic_idx_signer_block
36-
ON extrinsic (signer_multi_address, block_number DESC, index ASC);
36+
ON extrinsic (signer_multi_address, block_number DESC, block_hash ASC, index ASC);
3737

3838
CREATE TABLE extrinsic_0_1000000 PARTITION OF extrinsic FOR VALUES FROM (0) TO (1000000);
3939
CREATE TABLE extrinsic_1000000_2000000 PARTITION OF extrinsic FOR VALUES FROM (1000000) TO (2000000);

‎submerge-crystal/api-spec/submerge-crystal-api.json‎

Lines changed: 90 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1727,17 +1727,13 @@
17271727
"operationId": "get_extrinsics",
17281728
"parameters": [
17291729
{
1730-
"name": "page",
1730+
"name": "next_cursor",
17311731
"in": "query",
1732-
"description": "Extrinsic list page number to retrieve. 1-indexed.",
1732+
"description": "Opaque cursor for extrinsic pagination - returned in the endpoint response.\nThis parameter is mutually exclusive with all other parameters,\nand will return bad request if any other parameter is set.",
17331733
"required": false,
17341734
"schema": {
1735-
"type": "integer",
1736-
"format": "int32",
1737-
"default": 1,
1738-
"minimum": 1
1739-
},
1740-
"example": 1
1735+
"type": "string"
1736+
}
17411737
},
17421738
{
17431739
"name": "page_size",
@@ -1849,7 +1845,7 @@
18491845
],
18501846
"responses": {
18511847
"200": {
1852-
"$ref": "#/components/responses/PaginatedExtrinsicList"
1848+
"$ref": "#/components/responses/CursorExtrinsicList"
18531849
},
18541850
"400": {
18551851
"$ref": "#/components/responses/BadRequest"
@@ -5085,6 +5081,91 @@
50855081
}
50865082
}
50875083
},
5084+
"CursorExtrinsicList": {
5085+
"description": "List of matching extrinsic, with a cursor for the next page.",
5086+
"headers": {
5087+
"X-RateLimit-Limit": {
5088+
"schema": {
5089+
"type": "integer",
5090+
"format": "int32",
5091+
"minimum": 0
5092+
}
5093+
},
5094+
"X-RateLimit-Remaining": {
5095+
"schema": {
5096+
"type": "integer",
5097+
"format": "int32",
5098+
"minimum": 0
5099+
}
5100+
}
5101+
},
5102+
"content": {
5103+
"application/json": {
5104+
"schema": {
5105+
"type": "object",
5106+
"required": [
5107+
"data",
5108+
"pagination"
5109+
],
5110+
"properties": {
5111+
"data": {
5112+
"type": "array",
5113+
"items": {
5114+
"$ref": "#/components/schemas/Extrinsic"
5115+
},
5116+
"example": [
5117+
{
5118+
"blockHash": "0x758fadeb5004882de8ba39ee2105302ad0ce93ecd68fe26b6fa09de6608e7a77",
5119+
"blockNumber": 3172595,
5120+
"blockStatus": "proposed",
5121+
"blockTimestamp": 1765432302000,
5122+
"extra": {
5123+
"chargeAssetTxPayment": {
5124+
"assetId": null,
5125+
"tip": "0"
5126+
},
5127+
"checkGenesis": {},
5128+
"checkMetadataHash": {
5129+
"mode": {
5130+
"type": "Disabled",
5131+
"value": []
5132+
}
5133+
},
5134+
"checkMortality": {
5135+
"type": "Mortal84",
5136+
"value": "0"
5137+
},
5138+
"checkNonZeroSender": {},
5139+
"checkNonce": "8362",
5140+
"checkSpecVersion": {},
5141+
"checkTxVersion": {},
5142+
"checkWeight": {}
5143+
},
5144+
"hash": "0x91b171bb158e2d3848fa23a9f1c25182fb8e20313b2c1eb49219da7a70ce90c3",
5145+
"index": 2,
5146+
"isSuccessful": true,
5147+
"signature": {
5148+
"type": "sr25519",
5149+
"value": "0xabababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab"
5150+
},
5151+
"signer": {
5152+
"type": "accountId",
5153+
"value": "0x269a84431cd8dfc5762beadfa54a8f21597c12d4f31e51f9f6f985f65ba0c626"
5154+
},
5155+
"specVersion": 2000000,
5156+
"traceIndex": 2,
5157+
"version": 4
5158+
}
5159+
]
5160+
},
5161+
"pagination": {
5162+
"$ref": "#/components/schemas/CursorPaginationData"
5163+
}
5164+
}
5165+
}
5166+
}
5167+
}
5168+
},
50885169
"EventList": {
50895170
"description": "List of matching events.",
50905171
"headers": {

‎submerge-crystal/legacy-decoder/package-lock.json‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎submerge-crystal/src/api/docs.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use crate::types::api::{
66
call::{CallDTO, CursorCallList, PaginatedCallList},
77
error::{BadRequest, InternalServerError, NotFound, TooManyRequests},
88
event::{CursorEventList, EventDTO, EventList, PaginatedEventList},
9-
extrinsic::{ExtrinsicList, PaginatedExtrinsicList},
9+
extrinsic::{CursorExtrinsicList, ExtrinsicList, PaginatedExtrinsicList},
1010
genesis::{GenesisRecordDTO, PaginatedGenesisRecordList},
1111
hex::HexStringParam,
1212
metadata::{
@@ -132,7 +132,7 @@ use crate::types::api::{
132132
responses(
133133
BlockList, PaginatedBlockList,
134134
CursorCallList, PaginatedCallList,
135-
ExtrinsicList, PaginatedExtrinsicList,
135+
ExtrinsicList, PaginatedExtrinsicList, CursorExtrinsicList,
136136
EventList, PaginatedEventList, CursorEventList,
137137
PaginatedTraceList,
138138
PaginatedGenesisRecordList,

‎submerge-crystal/src/api/v1/extrinsic.rs‎

Lines changed: 46 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,10 @@ use axum::{
22
extract::{Path, Query, State},
33
Json,
44
};
5+
use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine};
56

67
use crate::{
7-
api::{get_page_number_and_size, ServiceState},
8+
api::{get_page_number_and_size, get_page_size, ServiceState},
89
persistence::{
910
api::{
1011
block::CrystalBlockAPIPostgreSQLStorage,
@@ -14,14 +15,17 @@ use crate::{
1415
},
1516
types::api::{
1617
dto::{
17-
pagination::PaginationData,
18+
pagination::{CursorPaginationData, PaginationData},
1819
request::{
1920
block::BlockReference,
2021
extrinsic::{BlockExtrinsicQuery, ExtrinsicQuery},
2122
},
2223
response::{
2324
error::{BadRequest, InternalServerError, NotFound, TooManyRequests},
24-
extrinsic::{ExtrinsicDTO, ExtrinsicList, PaginatedExtrinsicList},
25+
extrinsic::{
26+
CursorExtrinsicList, ExtrinsicCursorPayload, ExtrinsicCursorPosition,
27+
ExtrinsicDTO, ExtrinsicList, PaginatedExtrinsicList,
28+
},
2529
},
2630
},
2731
error::APIError,
@@ -38,7 +42,7 @@ use crate::{
3842
responses(
3943
(
4044
status = 200,
41-
response = PaginatedExtrinsicList,
45+
response = CursorExtrinsicList,
4246
),
4347
(
4448
status = 400,
@@ -57,8 +61,16 @@ use crate::{
5761
pub(crate) async fn get_extrinsics(
5862
State(state): State<ServiceState>,
5963
Query(query): Query<ExtrinsicQuery>,
60-
) -> Result<Json<PaginatedExtrinsicList>, APIError> {
61-
let (page, page_size) = get_page_number_and_size(query.page, query.page_size, false)?;
64+
) -> Result<Json<CursorExtrinsicList>, APIError> {
65+
query.validate_next_cursor_mutually_exclusive()?;
66+
let (cursor_position, query) = if let Some(cursor) = query.next_cursor {
67+
let decoded = URL_SAFE_NO_PAD.decode(cursor)?;
68+
let cursor_payload: ExtrinsicCursorPayload = serde_json::from_slice(&decoded)?;
69+
(Some(cursor_payload.cursor_position), cursor_payload.query)
70+
} else {
71+
(None, query)
72+
};
73+
let page_size = get_page_size(query.page_size, false)?;
6274
let Ok(signer_multi_address) = query.get_signer_multi_address() else {
6375
return Err(APIError::InvalidExtrinsicSigner(
6476
query.signer.unwrap_or("".to_string()),
@@ -76,31 +88,43 @@ pub(crate) async fn get_extrinsics(
7688
)
7789
.await?;
7890

79-
let (total_count, rows) = tokio::try_join!(
80-
state.postgres.get_extrinsic_count(
81-
min_block_number,
82-
max_block_number,
83-
query.is_signed,
84-
&signer_multi_address,
85-
),
86-
state.postgres.get_extrinsics(
91+
let rows = state
92+
.postgres
93+
.get_extrinsics(
94+
cursor_position,
8795
min_block_number,
8896
max_block_number,
8997
query.is_signed,
9098
&signer_multi_address,
91-
page,
9299
page_size,
93-
),
94-
)?;
95-
let mut data = Vec::new();
100+
)
101+
.await?;
102+
let mut data: Vec<ExtrinsicDTO> = Vec::new();
96103
for row in rows.iter() {
97104
data.push(row.try_into()?);
98105
}
99-
let response = PaginatedExtrinsicList {
100-
pagination: PaginationData {
101-
page,
106+
let next_cursor = if data.len() < page_size as usize {
107+
None
108+
} else if let Some(last_extrinsic) = data.last() {
109+
let cursor_payload = ExtrinsicCursorPayload {
110+
cursor_position: ExtrinsicCursorPosition {
111+
block_number: last_extrinsic.block_number,
112+
block_hash_hex: last_extrinsic.block_hash.0.clone(),
113+
index: last_extrinsic.index,
114+
},
115+
query,
116+
};
117+
let cursor = serde_json::to_string(&cursor_payload)?;
118+
let cursor_encoded = URL_SAFE_NO_PAD.encode(cursor.as_bytes());
119+
Some(cursor_encoded)
120+
} else {
121+
None
122+
};
123+
124+
let response = CursorExtrinsicList {
125+
pagination: CursorPaginationData {
102126
page_size,
103-
total: total_count,
127+
next_cursor,
104128
},
105129
data,
106130
};

0 commit comments

Comments
 (0)