Skip to content
This repository was archived by the owner on Jan 24, 2025. It is now read-only.

Commit 0be95a4

Browse files
jkrvivianjkrvivian
authored
Merge pull request #825 from iotaledger/feat/api-cache
Add cache for core API `validators/`
2 parents 39c9f43 + 417f372 commit 0be95a4

File tree

16 files changed

+517
-55
lines changed

16 files changed

+517
-55
lines changed

‎components/restapi/component.go

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/labstack/echo/v4"
1010
"github.com/labstack/echo/v4/middleware"
11+
"github.com/labstack/gommon/bytes"
1112
"github.com/libp2p/go-libp2p/core/crypto"
1213
"github.com/libp2p/go-libp2p/core/host"
1314
"go.uber.org/dig"
@@ -88,7 +89,12 @@ func provide(c *dig.Container) error {
8889
}
8990

9091
if err := c.Provide(func(deps requestHandlerDeps) *requesthandler.RequestHandler {
91-
return requesthandler.New(deps.Protocol)
92+
maxCacheSizeBytes, err := bytes.Parse(ParamsRestAPI.MaxCacheSize)
93+
if err != nil {
94+
Component.LogPanicf("parameter %s invalid", Component.App().Config().GetParameterPath(&(ParamsRestAPI.MaxCacheSize)))
95+
}
96+
97+
return requesthandler.New(deps.Protocol, requesthandler.WithCacheMaxSizeOptions(int(maxCacheSizeBytes)))
9298
}); err != nil {
9399
Component.LogPanic(err.Error())
94100
}

‎components/restapi/core/accounts.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,20 +61,23 @@ func validators(c echo.Context) (*api.ValidatorsResponse, error) {
6161
var err error
6262
pageSize := httpserver.ParsePageSizeQueryParam(c, api.ParameterPageSize, restapi.ParamsRestAPI.MaxPageSize)
6363
latestCommittedSlot := deps.RequestHandler.GetLatestCommitment().Slot()
64+
currentEpoch := deps.RequestHandler.APIProvider().APIForSlot(latestCommittedSlot).TimeProvider().EpochFromSlot(latestCommittedSlot)
65+
requestedEpoch := currentEpoch
6466

6567
// no cursor provided will be the first request
66-
requestedSlot := latestCommittedSlot
6768
var cursorIndex uint32
6869
if len(c.QueryParam(api.ParameterCursor)) != 0 {
69-
requestedSlot, cursorIndex, err = httpserver.ParseSlotCursorQueryParam(c, api.ParameterCursor)
70+
requestedEpoch, cursorIndex, err = httpserver.ParseEpochCursorQueryParam(c, api.ParameterCursor)
7071
if err != nil {
7172
return nil, err
7273
}
7374
}
7475

75-
slotRange := uint32(requestedSlot) / restapi.ParamsRestAPI.RequestsMemoryCacheGranularity
76+
if requestedEpoch > currentEpoch || requestedEpoch <= deps.RequestHandler.GetNodeStatus().PruningEpoch {
77+
return nil, ierrors.Wrapf(echo.ErrBadRequest, "epoch %d is larger than current epoch or already pruned", requestedEpoch)
78+
}
7679

77-
return deps.RequestHandler.Validators(slotRange, cursorIndex, pageSize)
80+
return deps.RequestHandler.Validators(requestedEpoch, cursorIndex, pageSize)
7881
}
7982

8083
func validatorByAccountAddress(c echo.Context) (*api.ValidatorResponse, error) {

‎components/restapi/params.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,8 @@ type ParametersRestAPI struct {
1616
DebugRequestLoggerEnabled bool `default:"false" usage:"whether the debug logging for requests should be enabled"`
1717
// MaxPageSize defines the maximum number of results per page.
1818
MaxPageSize uint32 `default:"100" usage:"the maximum number of results per page"`
19-
// RequestsMemoryCacheGranularity defines per how many slots a cache is created for big API requests.
20-
RequestsMemoryCacheGranularity uint32 `default:"10" usage:"defines per how many slots a cache is created for big API requests"`
21-
// MaxRequestedSlotAge defines the maximum age of a request that will be processed.
22-
MaxRequestedSlotAge uint32 `default:"10" usage:"the maximum age of a request that will be processed"`
19+
// MaxCacheSize defines the maximum size of cache for results.
20+
MaxCacheSize string `default:"50MB" usage:"the maximum size of cache for results"`
2321

2422
JWTAuth struct {
2523
// salt used inside the JWT tokens for the REST API. Change this to a different value to invalidate JWT tokens not matching this new value

‎config_defaults.json

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,7 @@
6464
],
6565
"debugRequestLoggerEnabled": false,
6666
"maxPageSize": 100,
67-
"requestsMemoryCacheGranularity": 10,
68-
"maxRequestedSlotAge": 10,
67+
"maxCacheSize": "50MB",
6968
"jwtAuth": {
7069
"salt": "IOTA"
7170
},

‎documentation/configuration.md

Lines changed: 11 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -166,17 +166,16 @@ Example:
166166

167167
## <a id="restapi"></a> 5. RestAPI
168168

169-
| Name | Description | Type | Default value |
170-
| ------------------------------ | ---------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
171-
| bindAddress | The bind address on which the REST API listens on | string | "0.0.0.0:14265" |
172-
| publicRoutes | The HTTP REST routes which can be called without authorization. Wildcards using \* are allowed | array | /health<br/>/api/routes<br/>/api/core/v3/info<br/>/api/core/v3/network\*<br/>/api/core/v3/blocks\*<br/>/api/core/v3/transactions\*<br/>/api/core/v3/commitments\*<br/>/api/core/v3/outputs\*<br/>/api/core/v3/accounts\*<br/>/api/core/v3/validators\*<br/>/api/core/v3/rewards\*<br/>/api/core/v3/committee\*<br/>/api/debug/v2/\*<br/>/api/indexer/v2/\*<br/>/api/mqtt/v2<br/>/api/blockissuer/v1/\* |
173-
| protectedRoutes | The HTTP REST routes which need to be called with authorization. Wildcards using \* are allowed | array | /api/\* |
174-
| debugRequestLoggerEnabled | Whether the debug logging for requests should be enabled | boolean | false |
175-
| maxPageSize | The maximum number of results per page | uint | 100 |
176-
| requestsMemoryCacheGranularity | Defines per how many slots a cache is created for big API requests | uint | 10 |
177-
| maxRequestedSlotAge | The maximum age of a request that will be processed | uint | 10 |
178-
| [jwtAuth](#restapi_jwtauth) | Configuration for jwtAuth | object | |
179-
| [limits](#restapi_limits) | Configuration for limits | object | |
169+
| Name | Description | Type | Default value |
170+
| --------------------------- | ---------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
171+
| bindAddress | The bind address on which the REST API listens on | string | "0.0.0.0:14265" |
172+
| publicRoutes | The HTTP REST routes which can be called without authorization. Wildcards using \* are allowed | array | /health<br/>/api/routes<br/>/api/core/v3/info<br/>/api/core/v3/network\*<br/>/api/core/v3/blocks\*<br/>/api/core/v3/transactions\*<br/>/api/core/v3/commitments\*<br/>/api/core/v3/outputs\*<br/>/api/core/v3/accounts\*<br/>/api/core/v3/validators\*<br/>/api/core/v3/rewards\*<br/>/api/core/v3/committee\*<br/>/api/debug/v2/\*<br/>/api/indexer/v2/\*<br/>/api/mqtt/v2<br/>/api/blockissuer/v1/\* |
173+
| protectedRoutes | The HTTP REST routes which need to be called with authorization. Wildcards using \* are allowed | array | /api/\* |
174+
| debugRequestLoggerEnabled | Whether the debug logging for requests should be enabled | boolean | false |
175+
| maxPageSize | The maximum number of results per page | uint | 100 |
176+
| maxCacheSize | The maximum size of cache for results | string | "50MB" |
177+
| [jwtAuth](#restapi_jwtauth) | Configuration for jwtAuth | object | |
178+
| [limits](#restapi_limits) | Configuration for limits | object | |
180179

181180
### <a id="restapi_jwtauth"></a> JwtAuth
182181

@@ -220,8 +219,7 @@ Example:
220219
],
221220
"debugRequestLoggerEnabled": false,
222221
"maxPageSize": 100,
223-
"requestsMemoryCacheGranularity": 10,
224-
"maxRequestedSlotAge": 10,
222+
"maxCacheSize": "50MB",
225223
"jwtAuth": {
226224
"salt": "IOTA"
227225
},

‎go.mod

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ go 1.22.0
55
replace github.com/goccy/go-graphviz => github.com/alexsporn/go-graphviz v0.0.0-20231011102718-04f10f0a9b59
66

77
require (
8+
github.com/VictoriaMetrics/fastcache v1.12.2
89
github.com/fjl/memsize v0.0.2
910
github.com/goccy/go-graphviz v0.1.2
1011
github.com/golang-jwt/jwt v3.2.2+incompatible
@@ -79,6 +80,7 @@ require (
7980
github.com/gogo/protobuf v1.3.2 // indirect
8081
github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
8182
github.com/golang/protobuf v1.5.4 // indirect
83+
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect
8284
github.com/google/go-github v17.0.0+incompatible // indirect
8385
github.com/google/go-querystring v1.1.0 // indirect
8486
github.com/google/gopacket v1.1.19 // indirect

‎go.sum

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,17 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
1010
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
1111
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
1212
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
13+
github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI=
14+
github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI=
1315
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
1416
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
1517
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
1618
github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
1719
github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
1820
github.com/alexsporn/go-graphviz v0.0.0-20231011102718-04f10f0a9b59 h1:6900O8HwoKgovNmOy7q4E2R6DurhAOrqe1WQv0EvUv8=
1921
github.com/alexsporn/go-graphviz v0.0.0-20231011102718-04f10f0a9b59/go.mod h1:lpnwvVDjskayq84ZxG8tGCPeZX/WxP88W+OJajh+gFk=
22+
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8=
23+
github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM=
2024
github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
2125
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
2226
github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -183,6 +187,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
183187
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
184188
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
185189
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
190+
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
191+
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk=
192+
github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
186193
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
187194
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
188195
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -840,6 +847,7 @@ golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBc
840847
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
841848
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
842849
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
850+
golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
843851
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
844852
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
845853
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

0 commit comments

Comments
 (0)