From 6ac280281c468e9a0de60e93c2dfbf18798abc12 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Tue, 5 Mar 2024 22:21:46 +0800 Subject: [PATCH 01/21] Add cache to validators API --- go.mod | 2 + go.sum | 7 ++ pkg/retainer/cache/cache.go | 23 +++++++ pkg/retainer/retainer.go | 4 +- pkg/retainer/retainer/retainer.go | 104 +++++++++++++++++++++--------- 5 files changed, 108 insertions(+), 32 deletions(-) create mode 100644 pkg/retainer/cache/cache.go diff --git a/go.mod b/go.mod index 2bc475856..27e10755c 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect + github.com/VictoriaMetrics/fastcache v1.12.2 // indirect github.com/benbjohnson/clock v1.3.5 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect @@ -78,6 +79,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/go-github v17.0.0+incompatible // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/gopacket v1.1.19 // indirect diff --git a/go.sum b/go.sum index 83dd05a45..7c1ef8a35 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/VictoriaMetrics/fastcache v1.12.2 h1:N0y9ASrJ0F6h0QaC3o6uJb3NIZ9VKLjCM7NQbSmF7WI= +github.com/VictoriaMetrics/fastcache v1.12.2/go.mod h1:AmC+Nzz1+3G2eCPapF6UcsnkThDcMsQicp4xDukwJYI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -17,6 +19,7 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexsporn/go-graphviz v0.0.0-20231011102718-04f10f0a9b59 h1:6900O8HwoKgovNmOy7q4E2R6DurhAOrqe1WQv0EvUv8= github.com/alexsporn/go-graphviz v0.0.0-20231011102718-04f10f0a9b59/go.mod h1:lpnwvVDjskayq84ZxG8tGCPeZX/WxP88W+OJajh+gFk= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -177,6 +180,9 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -817,6 +823,7 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pkg/retainer/cache/cache.go b/pkg/retainer/cache/cache.go new file mode 100644 index 000000000..9608b8e9c --- /dev/null +++ b/pkg/retainer/cache/cache.go @@ -0,0 +1,23 @@ +package cache + +import ( + "github.com/VictoriaMetrics/fastcache" +) + +type Cache struct { + *fastcache.Cache +} + +func NewCache(maxSize int) *Cache { + return &Cache{ + Cache: fastcache.New(maxSize), + } +} + +func (c *Cache) Get(key []byte) []byte { + if c.Has(key) { + return c.Get(key) + } + + return nil +} diff --git a/pkg/retainer/retainer.go b/pkg/retainer/retainer.go index 411e3c323..b10811100 100644 --- a/pkg/retainer/retainer.go +++ b/pkg/retainer/retainer.go @@ -9,9 +9,7 @@ import ( // Retainer keeps and resolves all the information needed in the API and INX. type Retainer interface { BlockMetadata(blockID iotago.BlockID) (*BlockMetadata, error) - - RegisteredValidatorsCache(uint32) ([]*api.ValidatorResponse, bool) - RetainRegisteredValidatorsCache(uint32, []*api.ValidatorResponse) + RegisteredValidators(iotago.EpochIndex) ([]*api.ValidatorResponse, error) RetainBlockFailure(iotago.BlockID, api.BlockFailureReason) RetainTransactionFailure(iotago.BlockID, error) diff --git a/pkg/retainer/retainer/retainer.go b/pkg/retainer/retainer/retainer.go index 21dde25fa..c8c9ebc37 100644 --- a/pkg/retainer/retainer/retainer.go +++ b/pkg/retainer/retainer/retainer.go @@ -1,16 +1,20 @@ package retainer import ( + "github.com/labstack/echo/v4" + "github.com/iotaledger/hive.go/ds/shrinkingmap" "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/runtime/event" "github.com/iotaledger/hive.go/runtime/module" + "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/runtime/workerpool" "github.com/iotaledger/iota-core/pkg/protocol/engine" "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" "github.com/iotaledger/iota-core/pkg/protocol/engine/filter/postsolidfilter" "github.com/iotaledger/iota-core/pkg/protocol/engine/mempool" "github.com/iotaledger/iota-core/pkg/retainer" + "github.com/iotaledger/iota-core/pkg/retainer/cache" "github.com/iotaledger/iota-core/pkg/storage/prunable/slotstore" iotago "github.com/iotaledger/iota.go/v4" "github.com/iotaledger/iota.go/v4/api" @@ -18,36 +22,48 @@ import ( type ( //nolint:revive - RetainerFunc func(iotago.SlotIndex) (*slotstore.Retainer, error) - LatestCommittedSlotFunc func() iotago.SlotIndex - FinalizedSlotFunc func() iotago.SlotIndex + RetainerFunc func(iotago.SlotIndex) (*slotstore.Retainer, error) + LatestCommittedSlotFunc func() iotago.SlotIndex + FinalizedSlotFunc func() iotago.SlotIndex + RegisteredValidatorsFunc func(iotago.EpochIndex) ([]*api.ValidatorResponse, error) + APIFunc func(iotago.EpochIndex) iotago.API ) const MaxStakersResponsesCacheNum = 10 // Retainer keeps and resolves all the information needed in the API and INX. type Retainer struct { - store RetainerFunc - latestCommittedSlotFunc LatestCommittedSlotFunc - finalizedSlotFunc FinalizedSlotFunc - errorHandler func(error) + store RetainerFunc + latestCommittedSlotFunc LatestCommittedSlotFunc + finalizedSlotFunc FinalizedSlotFunc + registeredValidatorsFunc RegisteredValidatorsFunc + apiForEpochFunc APIFunc + errorHandler func(error) - stakersResponses *shrinkingmap.ShrinkingMap[uint32, []*api.ValidatorResponse] + stakersResponses *shrinkingmap.ShrinkingMap[uint32, []*api.ValidatorResponse] + registeredValidatorsCache *cache.Cache workerPool *workerpool.WorkerPool + optsCacheMaxSize int + module.Module } -func New(workersGroup *workerpool.Group, retainerFunc RetainerFunc, latestCommittedSlotFunc LatestCommittedSlotFunc, finalizedSlotFunc FinalizedSlotFunc, errorHandler func(error)) *Retainer { - return &Retainer{ - workerPool: workersGroup.CreatePool("Retainer", workerpool.WithWorkerCount(1)), - store: retainerFunc, - stakersResponses: shrinkingmap.New[uint32, []*api.ValidatorResponse](), - latestCommittedSlotFunc: latestCommittedSlotFunc, - finalizedSlotFunc: finalizedSlotFunc, - errorHandler: errorHandler, - } +func New(workersGroup *workerpool.Group, retainerFunc RetainerFunc, latestCommittedSlotFunc LatestCommittedSlotFunc, finalizedSlotFunc FinalizedSlotFunc, registeredValidatorsFunc RegisteredValidatorsFunc, apiForEpochFunc APIFunc, errorHandler func(error), opts ...options.Option[Retainer]) *Retainer { + return options.Apply(&Retainer{ + workerPool: workersGroup.CreatePool("Retainer", workerpool.WithWorkerCount(1)), + store: retainerFunc, + stakersResponses: shrinkingmap.New[uint32, []*api.ValidatorResponse](), + latestCommittedSlotFunc: latestCommittedSlotFunc, + finalizedSlotFunc: finalizedSlotFunc, + apiForEpochFunc: apiForEpochFunc, + registeredValidatorsFunc: registeredValidatorsFunc, + errorHandler: errorHandler, + optsCacheMaxSize: 2 << 20, // 2MB + }, opts, func(r *Retainer) { + r.registeredValidatorsCache = cache.NewCache(r.optsCacheMaxSize) + }) } // NewProvider creates a new Retainer provider. @@ -71,6 +87,8 @@ func NewProvider() module.Provider[*engine.Engine, retainer.Retainer] { return e.SyncManager.LatestFinalizedSlot() }, + e.SybilProtection.OrderedRegisteredCandidateValidatorsList, + e.APIForEpoch, e.ErrorHandler("retainer")) asyncOpt := event.WithWorkerPool(r.workerPool) @@ -211,22 +229,44 @@ func (r *Retainer) RetainTransactionFailure(blockID iotago.BlockID, err error) { } } -func (r *Retainer) RegisteredValidatorsCache(index uint32) ([]*api.ValidatorResponse, bool) { - return r.stakersResponses.Get(index) +func (r *Retainer) RegisteredValidators(index iotago.EpochIndex) ([]*api.ValidatorResponse, error) { + apiForEpoch := r.apiForEpochFunc(index) + currentEpoch := apiForEpoch.TimeProvider().EpochFromSlot(r.latestCommittedSlotFunc()) + + // return registered validators of current epoch from node, because they are not yet finalized. + if index == currentEpoch { + return r.registeredValidatorsFunc(index) + } + + return r.registeredValidatorsFromCache(index) } -func (r *Retainer) RetainRegisteredValidatorsCache(index uint32, resp []*api.ValidatorResponse) { - r.stakersResponses.Set(index, resp) - if r.stakersResponses.Size() > MaxStakersResponsesCacheNum { - keys := r.stakersResponses.Keys() - minKey := index + 1 - for _, key := range keys { - if key < minKey { - minKey = key - } +func (r *Retainer) registeredValidatorsFromCache(index iotago.EpochIndex) ([]*api.ValidatorResponse, error) { + apiForEpoch := r.apiForEpochFunc(index) + + registeredValidatorsBytes := r.registeredValidatorsCache.Get(index.MustBytes()) + if registeredValidatorsBytes == nil { + // get the ordered registered validators list from engine. + registeredValidators, err := r.registeredValidatorsFunc(index) + if err != nil { + return nil, ierrors.Wrapf(echo.ErrNotFound, " ordered registered validators list for epoch %d not found: %s", index, err) } - r.stakersResponses.Delete(minKey) + + // store validator responses in cache. + registeredValidatorsBytes, err := apiForEpoch.Encode(registeredValidators) + if err != nil { + return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to encode ordered registered validators list for epoch %d : %s", index, err) + } + r.registeredValidatorsCache.Set(index.MustBytes(), registeredValidatorsBytes) + } + + validatorResp := make([]*api.ValidatorResponse, 0) + _, err := apiForEpoch.Decode(registeredValidatorsBytes, validatorResp) + if err != nil { + return nil, ierrors.Wrapf(err, "failed to decode validator responses for epoch %d", index) } + + return validatorResp, nil } func (r *Retainer) blockStatus(blockID iotago.BlockID) (api.BlockState, api.BlockFailureReason) { @@ -347,3 +387,9 @@ func (r *Retainer) onAttachmentUpdated(prevID iotago.BlockID, newID iotago.Block return store.StoreTransactionNoFailureStatus(newID, api.TransactionStatePending) } + +func WithCacheMaxSizeOptions(size int) options.Option[Retainer] { + return func(p *Retainer) { + p.optsCacheMaxSize = size + } +} From b39d85fa689711e51cd3dd2d6652dec9198bef70 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 6 Mar 2024 14:27:08 +0800 Subject: [PATCH 02/21] Adapt to RequestHandler --- components/restapi/core/accounts.go | 13 ++++++------- pkg/requesthandler/accounts.go | 17 +++++------------ 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/components/restapi/core/accounts.go b/components/restapi/core/accounts.go index 8c09be967..0734884f3 100644 --- a/components/restapi/core/accounts.go +++ b/components/restapi/core/accounts.go @@ -50,24 +50,23 @@ func validators(c echo.Context) (*api.ValidatorsResponse, error) { var err error pageSize := httpserver.ParsePageSizeQueryParam(c, api.ParameterPageSize, restapi.ParamsRestAPI.MaxPageSize) latestCommittedSlot := deps.RequestHandler.GetLatestCommitment().Slot() + currentEpoch := deps.RequestHandler.APIProvider().APIForSlot(latestCommittedSlot).TimeProvider().EpochFromSlot(latestCommittedSlot) + requestedEpoch := currentEpoch // no cursor provided will be the first request - requestedSlot := latestCommittedSlot var cursorIndex uint32 if len(c.QueryParam(api.ParameterCursor)) != 0 { - requestedSlot, cursorIndex, err = httpserver.ParseCursorQueryParam(c, api.ParameterCursor) + requestedEpoch, cursorIndex, err = httpserver.ParseCursorQueryParam(c, api.ParameterCursor) if err != nil { return nil, err } } - // do not respond to really old requests - if requestedSlot+iotago.SlotIndex(restapi.ParamsRestAPI.MaxRequestedSlotAge) < latestCommittedSlot { - return nil, ierrors.Wrapf(echo.ErrBadRequest, "request is too old, request started at %d, latest committed slot index is %d", requestedSlot, latestCommittedSlot) + if requestedEpoch > currentEpoch { + return nil, ierrors.Wrapf(echo.ErrBadRequest, "epoch %d is larger than current epoch %d", requestedEpoch, currentEpoch) } - slotRange := uint32(requestedSlot) / restapi.ParamsRestAPI.RequestsMemoryCacheGranularity - return deps.RequestHandler.Validators(slotRange, pageSize, cursorIndex) + return deps.RequestHandler.Validators(requestedEpoch, pageSize, cursorIndex) } func validatorByAccountAddress(c echo.Context) (*api.ValidatorResponse, error) { diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index 97f9c97e7..c450cbbdc 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -31,17 +31,10 @@ func (r *RequestHandler) CongestionByAccountAddress(accountAddress *iotago.Accou }, nil } -func (r *RequestHandler) Validators(slotRange, cursorIndex, pageSize uint32) (*api.ValidatorsResponse, error) { - latestCommittedSlot := r.protocol.Engines.Main.Get().SyncManager.LatestCommitment().Slot() - latestEpoch := r.protocol.APIForSlot(latestCommittedSlot).TimeProvider().EpochFromSlot(latestCommittedSlot) - - registeredValidators, exists := r.protocol.Engines.Main.Get().Retainer.RegisteredValidatorsCache(slotRange) - if !exists { - registeredValidators, err := r.protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(latestEpoch) - if err != nil { - return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to get ordered registered validators list for epoch %d : %s", latestEpoch, err) - } - r.protocol.Engines.Main.Get().Retainer.RetainRegisteredValidatorsCache(slotRange, registeredValidators) +func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, pageSize uint32) (*api.ValidatorsResponse, error) { + registeredValidators, err := r.protocol.Engines.Main.Get().Retainer.RegisteredValidators(epochIndex) + if err != nil { + return nil, err } page := registeredValidators[cursorIndex:lo.Min(cursorIndex+pageSize, uint32(len(registeredValidators)))] @@ -53,7 +46,7 @@ func (r *RequestHandler) Validators(slotRange, cursorIndex, pageSize uint32) (*a if int(cursorIndex+pageSize) > len(registeredValidators) { resp.Cursor = "" } else { - resp.Cursor = fmt.Sprintf("%d,%d", slotRange, cursorIndex+pageSize) + resp.Cursor = fmt.Sprintf("%d,%d", epochIndex, cursorIndex+pageSize) } return resp, nil From 0144ee7d59a6d3863b7303191aa47af243124753 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Thu, 7 Mar 2024 13:38:42 +0800 Subject: [PATCH 03/21] Fix passing wrong arguments to RequestHandler.Validators --- components/restapi/core/accounts.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/restapi/core/accounts.go b/components/restapi/core/accounts.go index 0734884f3..25734726b 100644 --- a/components/restapi/core/accounts.go +++ b/components/restapi/core/accounts.go @@ -66,7 +66,7 @@ func validators(c echo.Context) (*api.ValidatorsResponse, error) { return nil, ierrors.Wrapf(echo.ErrBadRequest, "epoch %d is larger than current epoch %d", requestedEpoch, currentEpoch) } - return deps.RequestHandler.Validators(requestedEpoch, pageSize, cursorIndex) + return deps.RequestHandler.Validators(requestedEpoch, cursorIndex, pageSize) } func validatorByAccountAddress(c echo.Context) (*api.ValidatorResponse, error) { From 129c5231adcc7a5e4f048d3e2f0236eac8557f19 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Thu, 7 Mar 2024 17:33:30 +0800 Subject: [PATCH 04/21] Fix Encode/Decode error from ValidatorResponse --- pkg/retainer/cache/cache.go | 4 +++- pkg/retainer/retainer/retainer.go | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pkg/retainer/cache/cache.go b/pkg/retainer/cache/cache.go index 9608b8e9c..f85b0382f 100644 --- a/pkg/retainer/cache/cache.go +++ b/pkg/retainer/cache/cache.go @@ -16,7 +16,9 @@ func NewCache(maxSize int) *Cache { func (c *Cache) Get(key []byte) []byte { if c.Has(key) { - return c.Get(key) + value := make([]byte, 0) + + return c.Cache.Get(value, key) } return nil diff --git a/pkg/retainer/retainer/retainer.go b/pkg/retainer/retainer/retainer.go index c8c9ebc37..ee48d1b80 100644 --- a/pkg/retainer/retainer/retainer.go +++ b/pkg/retainer/retainer/retainer.go @@ -9,6 +9,7 @@ import ( "github.com/iotaledger/hive.go/runtime/module" "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/runtime/workerpool" + "github.com/iotaledger/hive.go/serializer/v2/serix" "github.com/iotaledger/iota-core/pkg/protocol/engine" "github.com/iotaledger/iota-core/pkg/protocol/engine/blocks" "github.com/iotaledger/iota-core/pkg/protocol/engine/filter/postsolidfilter" @@ -29,6 +30,8 @@ type ( APIFunc func(iotago.EpochIndex) iotago.API ) +var validatorResponsesTypeSettings = serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsByte) + const MaxStakersResponsesCacheNum = 10 // Retainer keeps and resolves all the information needed in the API and INX. @@ -253,15 +256,17 @@ func (r *Retainer) registeredValidatorsFromCache(index iotago.EpochIndex) ([]*ap } // store validator responses in cache. - registeredValidatorsBytes, err := apiForEpoch.Encode(registeredValidators) + registeredValidatorsBytes, err := apiForEpoch.Encode(registeredValidators, serix.WithTypeSettings(validatorResponsesTypeSettings)) if err != nil { return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to encode ordered registered validators list for epoch %d : %s", index, err) } r.registeredValidatorsCache.Set(index.MustBytes(), registeredValidatorsBytes) + + return registeredValidators, nil } validatorResp := make([]*api.ValidatorResponse, 0) - _, err := apiForEpoch.Decode(registeredValidatorsBytes, validatorResp) + _, err := apiForEpoch.Decode(registeredValidatorsBytes, &validatorResp, serix.WithTypeSettings(validatorResponsesTypeSettings)) if err != nil { return nil, ierrors.Wrapf(err, "failed to decode validator responses for epoch %d", index) } From 4b0c18ffae01cb9f9115d5ff841a9e41fb80b950 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Thu, 7 Mar 2024 17:33:54 +0800 Subject: [PATCH 05/21] Add validator cache unit test --- pkg/retainer/retainer/retainer_test.go | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 pkg/retainer/retainer/retainer_test.go diff --git a/pkg/retainer/retainer/retainer_test.go b/pkg/retainer/retainer/retainer_test.go new file mode 100644 index 000000000..66c06acf7 --- /dev/null +++ b/pkg/retainer/retainer/retainer_test.go @@ -0,0 +1,78 @@ +package retainer + +import ( + "testing" + + "github.com/iotaledger/hive.go/runtime/workerpool" + "github.com/iotaledger/iota-core/pkg/storage/prunable/slotstore" + iotago "github.com/iotaledger/iota.go/v4" + "github.com/iotaledger/iota.go/v4/api" + "github.com/iotaledger/iota.go/v4/tpkg" + "github.com/labstack/echo/v4" + "github.com/stretchr/testify/require" +) + +func Test_ValidatorCache(t *testing.T) { + validators := make([]*api.ValidatorResponse, 0) + for i := 0; i < 10; i++ { + validators = append(validators, &api.ValidatorResponse{ + AddressBech32: tpkg.RandAccountID().ToAddress().Bech32(iotago.PrefixTestnet), + FixedCost: 1, + StakingEndEpoch: iotago.EpochIndex(i), + }) + } + + r := New(workerpool.NewGroup("retainer"), func(si iotago.SlotIndex) (*slotstore.Retainer, error) { + return nil, nil + }, func() iotago.SlotIndex { + return iotago.SlotIndex(5) + }, func() iotago.SlotIndex { + return iotago.SlotIndex(2) + }, func(ei iotago.EpochIndex) ([]*api.ValidatorResponse, error) { + if ei == 0 { + return validators, nil + } + return nil, echo.ErrNotFound + }, func(ei iotago.EpochIndex) iotago.API { + return tpkg.ZeroCostTestAPI + }, func(err error) {}) + + // epoch 0, with 10 validators + resp, err := r.RegisteredValidators(iotago.EpochIndex(0)) + require.NoError(t, err) + require.ElementsMatch(t, validators, resp) + + // epoch 0, 1 validator added, should return 11 validators + validators = append(validators, &api.ValidatorResponse{ + AddressBech32: tpkg.RandAccountID().ToAddress().Bech32(iotago.PrefixTestnet), + FixedCost: 1, + StakingEndEpoch: iotago.EpochIndex(10), + }) + resp, err = r.RegisteredValidators(iotago.EpochIndex(0)) + require.NoError(t, err) + require.ElementsMatch(t, validators, resp) + require.False(t, r.registeredValidatorsCache.Has(iotago.EpochIndex(0).MustBytes())) + + // epoch 2, we should have validators in epoch 0 added to cache + r.latestCommittedSlotFunc = func() iotago.SlotIndex { + return tpkg.ZeroCostTestAPI.TimeProvider().EpochStart(2) + } + resp, err = r.RegisteredValidators(iotago.EpochIndex(0)) + require.NoError(t, err) + require.ElementsMatch(t, validators, resp) + require.True(t, r.registeredValidatorsCache.Has(iotago.EpochIndex(0).MustBytes())) + + // epoch 3, we should have cache hit + r.latestCommittedSlotFunc = func() iotago.SlotIndex { + return tpkg.ZeroCostTestAPI.TimeProvider().EpochStart(3) + } + resp, err = r.RegisteredValidators(iotago.EpochIndex(0)) + require.NoError(t, err) + require.ElementsMatch(t, validators, resp) + require.True(t, r.registeredValidatorsCache.Has(iotago.EpochIndex(0).MustBytes())) + + // can not retrieve validators for epoch 1, should return not found + resp, err = r.RegisteredValidators(iotago.EpochIndex(1)) + require.ErrorAs(t, echo.ErrNotFound, &err) + +} From 475b43b20d44a2419b0c617d5c24f73e563282b0 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Thu, 7 Mar 2024 17:56:40 +0800 Subject: [PATCH 06/21] Add docker test for core API `validators` --- tools/docker-network/tests/coreapi_test.go | 96 +++++++++++++++++++ tools/docker-network/tests/dockerframework.go | 10 ++ tools/docker-network/tests/wallet.go | 42 +++++++- 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 tools/docker-network/tests/coreapi_test.go diff --git a/tools/docker-network/tests/coreapi_test.go b/tools/docker-network/tests/coreapi_test.go new file mode 100644 index 000000000..892fc8c43 --- /dev/null +++ b/tools/docker-network/tests/coreapi_test.go @@ -0,0 +1,96 @@ +package tests + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + iotago "github.com/iotaledger/iota.go/v4" + "github.com/iotaledger/iota.go/v4/nodeclient" + "github.com/stretchr/testify/require" +) + +func Test_ValidatorsAPI(t *testing.T) { + d := NewDockerTestFramework(t, + WithProtocolParametersOptions( + iotago.WithTimeProviderOptions(5, time.Now().Unix(), 10, 4), + iotago.WithLivenessOptions(10, 10, 2, 4, 8), + iotago.WithRewardsOptions(8, 10, 2, 384), + iotago.WithTargetCommitteeSize(4), + )) + defer d.Stop() + + d.AddValidatorNode("V1", "docker-network-inx-validator-1-1", "http://localhost:8050", "rms1pzg8cqhfxqhq7pt37y8cs4v5u4kcc48lquy2k73ehsdhf5ukhya3y5rx2w6") + d.AddValidatorNode("V2", "docker-network-inx-validator-2-1", "http://localhost:8060", "rms1pqm4xk8e9ny5w5rxjkvtp249tfhlwvcshyr3pc0665jvp7g3hc875k538hl") + d.AddValidatorNode("V3", "docker-network-inx-validator-3-1", "http://localhost:8070", "rms1pp4wuuz0y42caz48vv876qfpmffswsvg40zz8v79sy8cp0jfxm4kunflcgt") + d.AddValidatorNode("V4", "docker-network-inx-validator-4-1", "http://localhost:8040", "rms1pr8cxs3dzu9xh4cduff4dd4cxdthpjkpwmz2244f75m0urslrsvtsshrrjw") + d.AddNode("node5", "docker-network-node-5-1", "http://localhost:8090") + + err := d.Run() + require.NoError(t, err) + + d.WaitUntilNetworkReady() + hrp := d.wallet.DefaultClient().CommittedAPI().ProtocolParameters().Bech32HRP() + + // Create registered validators + var wg sync.WaitGroup + clt := d.wallet.DefaultClient() + status := d.NodeStatus("V1") + currentEpoch := clt.CommittedAPI().TimeProvider().EpochFromSlot(status.LatestAcceptedBlockSlot) + + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + + account := d.CreateAccount(WithStakingFeature(100, 1, 0)) + + // issue candidacy payload in the next epoch (currentEpoch + 1), in order to issue it before epochNearingThreshold + d.AwaitCommitment(clt.CommittedAPI().TimeProvider().EpochEnd(currentEpoch)) + blkID := d.IssueCandidacyPayloadFromAccount(account.ID) + fmt.Println("Candidacy payload:", blkID.ToHex(), blkID.Slot()) + d.AwaitCommitment(blkID.Slot()) + }() + } + wg.Wait() + + expectedValidators := d.AccountsFromNodes(d.Nodes()...) + for _, v := range d.wallet.Accounts() { + expectedValidators = append(expectedValidators, v.Address.Bech32(hrp)) + } + // get all validators of currentEpoch+1 with pageSize 10 + actualValidators := getAllValidatorsOnEpoch(t, clt, 0, 10) + require.ElementsMatch(t, expectedValidators, actualValidators) + + // wait until currentEpoch+3 and check the results again + targetSlot := clt.CommittedAPI().TimeProvider().EpochEnd(currentEpoch + 2) + d.AwaitCommitment(targetSlot) + actualValidators = getAllValidatorsOnEpoch(t, clt, currentEpoch+1, 10) + require.ElementsMatch(t, expectedValidators, actualValidators) +} + +func getAllValidatorsOnEpoch(t *testing.T, clt *nodeclient.Client, epoch iotago.EpochIndex, pageSize uint64) []string { + actualValidators := make([]string, 0) + cursor := "" + if epoch != 0 { + cursor = fmt.Sprintf("%d,%d", epoch, 0) + } + + for { + resp, err := clt.Validators(context.Background(), pageSize, cursor) + require.NoError(t, err) + + for _, v := range resp.Validators { + actualValidators = append(actualValidators, v.AddressBech32) + } + + cursor = resp.Cursor + if cursor == "" { + break + } + } + + return actualValidators +} diff --git a/tools/docker-network/tests/dockerframework.go b/tools/docker-network/tests/dockerframework.go index 7462bd934..d4de882d1 100644 --- a/tools/docker-network/tests/dockerframework.go +++ b/tools/docker-network/tests/dockerframework.go @@ -352,6 +352,16 @@ func (d *DockerTestFramework) StopIssueCandidacyPayload(nodes ...*Node) { d.DockerComposeUp(true) } +func (d *DockerTestFramework) IssueCandidacyPayloadFromAccount(issuerId iotago.AccountID) iotago.BlockID { + issuer := d.wallet.Account(issuerId) + ctx := context.TODO() + clt := d.wallet.DefaultClient() + + issuerResp, congestionResp := d.PrepareBlockIssuance(ctx, clt, issuer.Address) + + return d.SubmitPayload(ctx, &iotago.CandidacyAnnouncement{}, issuerId, congestionResp, issuerResp) +} + // CreateTaggedDataBlock creates a block of a tagged data payload. func (d *DockerTestFramework) CreateTaggedDataBlock(issuerId iotago.AccountID, tag []byte) *iotago.Block { issuer := d.wallet.Account(issuerId) diff --git a/tools/docker-network/tests/wallet.go b/tools/docker-network/tests/wallet.go index ba0e5e765..e0a0d81a5 100644 --- a/tools/docker-network/tests/wallet.go +++ b/tools/docker-network/tests/wallet.go @@ -5,6 +5,7 @@ package tests import ( "crypto/ed25519" "math/big" + "sync" "sync/atomic" "testing" @@ -32,8 +33,11 @@ type DockerWallet struct { lastUsedIndex atomic.Uint32 - outputs map[iotago.OutputID]*OutputData - accounts map[iotago.AccountID]*AccountData + outputs map[iotago.OutputID]*OutputData + outputsLock sync.RWMutex + + accounts map[iotago.AccountID]*AccountData + accountsLock sync.RWMutex } // OutputData holds the details of an output that can be used to build a transaction. @@ -77,14 +81,23 @@ func (w *DockerWallet) DefaultClient() *nodeclient.Client { } func (w *DockerWallet) AddOutput(outputId iotago.OutputID, output *OutputData) { + w.outputsLock.Lock() + defer w.outputsLock.Unlock() + w.outputs[outputId] = output } func (w *DockerWallet) AddAccount(accountId iotago.AccountID, data *AccountData) { + w.accountsLock.Lock() + defer w.accountsLock.Unlock() + w.accounts[accountId] = data } func (w *DockerWallet) Output(outputName iotago.OutputID) *OutputData { + w.outputsLock.RLock() + defer w.outputsLock.RUnlock() + output, exists := w.outputs[outputName] if !exists { panic(ierrors.Errorf("output %s not registered in wallet", outputName)) @@ -94,6 +107,9 @@ func (w *DockerWallet) Output(outputName iotago.OutputID) *OutputData { } func (w *DockerWallet) Account(accountId iotago.AccountID) *AccountData { + w.accountsLock.RLock() + defer w.accountsLock.RUnlock() + acc, exists := w.accounts[accountId] if !exists { panic(ierrors.Errorf("account %s not registered in wallet", accountId.ToHex())) @@ -102,6 +118,28 @@ func (w *DockerWallet) Account(accountId iotago.AccountID) *AccountData { return acc } +func (w *DockerWallet) Accounts(accountIds ...iotago.AccountID) []*AccountData { + w.accountsLock.RLock() + defer w.accountsLock.RUnlock() + + accounts := make([]*AccountData, 0) + if len(accountIds) == 0 { + for _, acc := range w.accounts { + accounts = append(accounts, acc) + } + } + + for _, id := range accountIds { + acc, exists := w.accounts[id] + if !exists { + panic(ierrors.Errorf("account %s not registered in wallet", id.ToHex())) + } + accounts = append(accounts, acc) + } + + return accounts +} + func (w *DockerWallet) Address(index ...uint32) (uint32, *iotago.Ed25519Address) { if len(index) == 0 { index = append(index, w.lastUsedIndex.Add(1)) From d7825b09e1265e85802ecda2ab7a2b5a5010d358 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Fri, 8 Mar 2024 20:54:45 +0800 Subject: [PATCH 07/21] Move cache package to RequestHandler --- pkg/requesthandler/accounts.go | 7 -- .../cache/cache.go | 0 pkg/requesthandler/requesthandler.go | 26 +++++-- pkg/retainer/retainer/retainer_test.go | 78 ------------------- 4 files changed, 20 insertions(+), 91 deletions(-) rename pkg/{retainer => requesthandler}/cache/cache.go (100%) delete mode 100644 pkg/retainer/retainer/retainer_test.go diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index c99f973d5..b880094f8 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -8,7 +8,6 @@ import ( "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/lo" - "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/serializer/v2/serix" "github.com/iotaledger/iota-core/pkg/core/account" "github.com/iotaledger/iota-core/pkg/model" @@ -251,9 +250,3 @@ func (r *RequestHandler) SelectedCommittee(epoch iotago.EpochIndex) (*api.Commit TotalValidatorStake: accounts.TotalValidatorStake(), }, nil } - -func WithCacheMaxSizeOptions(size int) options.Option[RequestHandler] { - return func(r *RequestHandler) { - r.optsCacheMaxSize = size - } -} diff --git a/pkg/retainer/cache/cache.go b/pkg/requesthandler/cache/cache.go similarity index 100% rename from pkg/retainer/cache/cache.go rename to pkg/requesthandler/cache/cache.go diff --git a/pkg/requesthandler/requesthandler.go b/pkg/requesthandler/requesthandler.go index ca28431dd..243ddaf42 100644 --- a/pkg/requesthandler/requesthandler.go +++ b/pkg/requesthandler/requesthandler.go @@ -1,22 +1,30 @@ package requesthandler import ( + "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/hive.go/runtime/workerpool" "github.com/iotaledger/iota-core/pkg/protocol" + "github.com/iotaledger/iota-core/pkg/requesthandler/cache" ) // RequestHandler contains the logic to handle api requests. type RequestHandler struct { workerPool *workerpool.WorkerPool - protocol *protocol.Protocol + registeredValidatorsCache *cache.Cache + protocol *protocol.Protocol + + optsCacheMaxSize int } -func New(p *protocol.Protocol) *RequestHandler { - return &RequestHandler{ - workerPool: p.Workers.CreatePool("BlockHandler"), - protocol: p, - } +func New(p *protocol.Protocol, opts ...options.Option[RequestHandler]) *RequestHandler { + return options.Apply(&RequestHandler{ + workerPool: p.Workers.CreatePool("BlockHandler"), + protocol: p, + optsCacheMaxSize: 2 << 20, // 2MB + }, opts, func(r *RequestHandler) { + r.registeredValidatorsCache = cache.NewCache(r.optsCacheMaxSize) + }) } // Shutdown shuts down the block issuer. @@ -24,3 +32,9 @@ func (r *RequestHandler) Shutdown() { r.workerPool.Shutdown() r.workerPool.ShutdownComplete.Wait() } + +func WithCacheMaxSizeOptions(size int) options.Option[RequestHandler] { + return func(r *RequestHandler) { + r.optsCacheMaxSize = size + } +} diff --git a/pkg/retainer/retainer/retainer_test.go b/pkg/retainer/retainer/retainer_test.go deleted file mode 100644 index 66c06acf7..000000000 --- a/pkg/retainer/retainer/retainer_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package retainer - -import ( - "testing" - - "github.com/iotaledger/hive.go/runtime/workerpool" - "github.com/iotaledger/iota-core/pkg/storage/prunable/slotstore" - iotago "github.com/iotaledger/iota.go/v4" - "github.com/iotaledger/iota.go/v4/api" - "github.com/iotaledger/iota.go/v4/tpkg" - "github.com/labstack/echo/v4" - "github.com/stretchr/testify/require" -) - -func Test_ValidatorCache(t *testing.T) { - validators := make([]*api.ValidatorResponse, 0) - for i := 0; i < 10; i++ { - validators = append(validators, &api.ValidatorResponse{ - AddressBech32: tpkg.RandAccountID().ToAddress().Bech32(iotago.PrefixTestnet), - FixedCost: 1, - StakingEndEpoch: iotago.EpochIndex(i), - }) - } - - r := New(workerpool.NewGroup("retainer"), func(si iotago.SlotIndex) (*slotstore.Retainer, error) { - return nil, nil - }, func() iotago.SlotIndex { - return iotago.SlotIndex(5) - }, func() iotago.SlotIndex { - return iotago.SlotIndex(2) - }, func(ei iotago.EpochIndex) ([]*api.ValidatorResponse, error) { - if ei == 0 { - return validators, nil - } - return nil, echo.ErrNotFound - }, func(ei iotago.EpochIndex) iotago.API { - return tpkg.ZeroCostTestAPI - }, func(err error) {}) - - // epoch 0, with 10 validators - resp, err := r.RegisteredValidators(iotago.EpochIndex(0)) - require.NoError(t, err) - require.ElementsMatch(t, validators, resp) - - // epoch 0, 1 validator added, should return 11 validators - validators = append(validators, &api.ValidatorResponse{ - AddressBech32: tpkg.RandAccountID().ToAddress().Bech32(iotago.PrefixTestnet), - FixedCost: 1, - StakingEndEpoch: iotago.EpochIndex(10), - }) - resp, err = r.RegisteredValidators(iotago.EpochIndex(0)) - require.NoError(t, err) - require.ElementsMatch(t, validators, resp) - require.False(t, r.registeredValidatorsCache.Has(iotago.EpochIndex(0).MustBytes())) - - // epoch 2, we should have validators in epoch 0 added to cache - r.latestCommittedSlotFunc = func() iotago.SlotIndex { - return tpkg.ZeroCostTestAPI.TimeProvider().EpochStart(2) - } - resp, err = r.RegisteredValidators(iotago.EpochIndex(0)) - require.NoError(t, err) - require.ElementsMatch(t, validators, resp) - require.True(t, r.registeredValidatorsCache.Has(iotago.EpochIndex(0).MustBytes())) - - // epoch 3, we should have cache hit - r.latestCommittedSlotFunc = func() iotago.SlotIndex { - return tpkg.ZeroCostTestAPI.TimeProvider().EpochStart(3) - } - resp, err = r.RegisteredValidators(iotago.EpochIndex(0)) - require.NoError(t, err) - require.ElementsMatch(t, validators, resp) - require.True(t, r.registeredValidatorsCache.Has(iotago.EpochIndex(0).MustBytes())) - - // can not retrieve validators for epoch 1, should return not found - resp, err = r.RegisteredValidators(iotago.EpochIndex(1)) - require.ErrorAs(t, echo.ErrNotFound, &err) - -} From a4846c58b11078205826bf4f02cbe7f626722cf8 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Fri, 8 Mar 2024 22:08:13 +0800 Subject: [PATCH 08/21] Fix linter error --- pkg/requesthandler/accounts.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index b880094f8..1e265404b 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -68,7 +68,7 @@ func (r *RequestHandler) registeredValidatorsFromCache(index iotago.EpochIndex) func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, pageSize uint32) (*api.ValidatorsResponse, error) { apiForEpoch := r.APIProvider().APIForEpoch(epochIndex) currentEpoch := apiForEpoch.TimeProvider().EpochFromSlot(r.protocol.Engines.Main.Get().SyncManager.LatestCommitment().Slot()) - registeredValidators := make([]*api.ValidatorResponse, 0) + var registeredValidators []*api.ValidatorResponse var err error // return registered validators of current epoch from node, because they are not yet finalized. @@ -77,7 +77,6 @@ func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, p } else { registeredValidators, err = r.registeredValidatorsFromCache(epochIndex) } - if err != nil { return nil, ierrors.Wrapf(err, "failed to get registered validators for epoch %d", epochIndex) } From b0918fb0561761c29da93e55e2975c116e71ee86 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Fri, 8 Mar 2024 22:10:12 +0800 Subject: [PATCH 09/21] Add RequestHandler Validators unit test --- pkg/requesthandler/requesthandler_test.go | 191 ++++++++++++++++++++++ pkg/testsuite/mock/blockissuer.go | 2 +- pkg/testsuite/sybilprotection.go | 25 +++ 3 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 pkg/requesthandler/requesthandler_test.go diff --git a/pkg/requesthandler/requesthandler_test.go b/pkg/requesthandler/requesthandler_test.go new file mode 100644 index 000000000..00dfa627a --- /dev/null +++ b/pkg/requesthandler/requesthandler_test.go @@ -0,0 +1,191 @@ +package requesthandler + +import ( + "testing" + + "github.com/iotaledger/hive.go/lo" + "github.com/iotaledger/hive.go/log" + "github.com/iotaledger/hive.go/runtime/options" + "github.com/iotaledger/iota-core/pkg/protocol" + "github.com/iotaledger/iota-core/pkg/protocol/engine/notarization/slotnotarization" + "github.com/iotaledger/iota-core/pkg/protocol/sybilprotection/seatmanager/topstakers" + "github.com/iotaledger/iota-core/pkg/protocol/sybilprotection/sybilprotectionv1" + "github.com/iotaledger/iota-core/pkg/testsuite" + iotago "github.com/iotaledger/iota.go/v4" + "github.com/iotaledger/iota.go/v4/api" + "github.com/stretchr/testify/require" +) + +func Test_ValidatorsAPI(t *testing.T) { + ts := testsuite.NewTestSuite(t, + testsuite.WithProtocolParametersOptions( + iotago.WithTimeProviderOptions( + 0, + testsuite.GenesisTimeWithOffsetBySlots(200, testsuite.DefaultSlotDurationInSeconds), + testsuite.DefaultSlotDurationInSeconds, + 4, + ), + iotago.WithLivenessOptions( + 10, + 10, + 2, + 4, + 5, + ), + iotago.WithTargetCommitteeSize(3), + ), + ) + defer ts.Shutdown() + + node1 := ts.AddValidatorNode("node1", testsuite.WithWalletAmount(1_000_006)) + ts.AddValidatorNode("node2", testsuite.WithWalletAmount(1_000_005)) + ts.AddValidatorNode("node3", testsuite.WithWalletAmount(1_000_004)) + ts.AddValidatorNode("node4", testsuite.WithWalletAmount(1_000_003)) + ts.AddDefaultWallet(node1) + + nodeOpts := []options.Option[protocol.Protocol]{ + protocol.WithNotarizationProvider( + slotnotarization.NewProvider(), + ), + protocol.WithSybilProtectionProvider( + sybilprotectionv1.NewProvider( + sybilprotectionv1.WithSeatManagerProvider( + topstakers.NewProvider(), + ), + ), + ), + } + + ts.Run(true, map[string][]options.Option[protocol.Protocol]{ + "node1": nodeOpts, + "node2": nodeOpts, + "node3": nodeOpts, + "node4": nodeOpts, + }) + ts.DefaultWallet().Node.Protocol.SetLogLevel(log.LevelTrace) + + ts.AssertSybilProtectionCommittee(0, []iotago.AccountID{ + ts.Node("node1").Validator.AccountID, + ts.Node("node2").Validator.AccountID, + ts.Node("node3").Validator.AccountID, + }, ts.Nodes()...) + + requestHandler := New(ts.DefaultWallet().Node.Protocol) + hrp := ts.API.ProtocolParameters().Bech32HRP() + + // Select committee for epoch 1 and test candidacy announcements at different times. + { + ts.IssueBlocksAtSlots("wave-1:", []iotago.SlotIndex{1, 2, 3, 4}, 4, "Genesis", ts.Nodes(), true, false) + + ts.IssueCandidacyAnnouncementInSlot("node1-candidacy:1", 4, "wave-1:4.3", ts.Wallet("node1")) + ts.IssueCandidacyAnnouncementInSlot("node4-candidacy:1", 5, "node1-candidacy:1", ts.Wallet("node4")) + + ts.IssueBlocksAtSlots("wave-2:", []iotago.SlotIndex{5, 6, 7, 8}, 4, "node4-candidacy:1", ts.Nodes(), true, false) + + // Assert that only candidates that issued before slot 11 are considered. + ts.AssertSybilProtectionCandidates(0, []iotago.AccountID{ + ts.Node("node1").Validator.AccountID, + ts.Node("node4").Validator.AccountID, + }, ts.Nodes()...) + + ts.AssertSybilProtectionRegisteredValidators(0, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, ts.Nodes()...) + + assertValidatorsFromRequestHandler(t, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, requestHandler, 0) + } + + { + ts.IssueCandidacyAnnouncementInSlot("node2-candidacy:1", 9, "wave-2:8.3", ts.Wallet("node2")) + ts.IssueBlocksAtSlots("wave-3:", []iotago.SlotIndex{9, 10}, 4, "node2-candidacy:1", ts.Nodes(), true, false) + + ts.AssertSybilProtectionCandidates(0, []iotago.AccountID{ + ts.Node("node1").Validator.AccountID, + ts.Node("node2").Validator.AccountID, + ts.Node("node4").Validator.AccountID, + }, ts.Nodes()...) + + ts.AssertSybilProtectionRegisteredValidators(0, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, ts.Nodes()...) + + assertValidatorsFromRequestHandler(t, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, requestHandler, 0) + } + + { + // Those candidacies should not be considered as they're issued after EpochNearingThreshold (slot 10). + ts.IssueCandidacyAnnouncementInSlot("node3-candidacy:1", 11, "wave-3:10.3", ts.Wallet("node3")) + ts.IssueBlocksAtSlots("wave-5:", []iotago.SlotIndex{11}, 4, "node3-candidacy:1", ts.Nodes(), true, false) + + ts.AssertSybilProtectionCandidates(0, []iotago.AccountID{ + ts.Node("node1").Validator.AccountID, + ts.Node("node2").Validator.AccountID, + ts.Node("node4").Validator.AccountID, + }, ts.Nodes()...) + + ts.AssertSybilProtectionRegisteredValidators(0, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, ts.Nodes()...) + + assertValidatorsFromRequestHandler(t, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, requestHandler, 0) + } + + { + ts.IssueBlocksAtSlots("wave-6:", []iotago.SlotIndex{12, 13, 14, 15, 16}, 4, "wave-5:11.3", ts.Nodes(), true, false) + + ts.IssueCandidacyAnnouncementInSlot("node2-candidacy:2", 17, "wave-6:16.3", ts.Wallet("node2")) + ts.IssueCandidacyAnnouncementInSlot("node3-candidacy:2", 17, "node2-candidacy:2", ts.Wallet("node3")) + ts.IssueBlocksAtSlots("wave-7:", []iotago.SlotIndex{18}, 4, "node3-candidacy:2", ts.Nodes(), true, false) + + // advance to next epoch, the validator cache should be used. + assertValidatorsFromRequestHandler(t, []string{ + ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), + }, requestHandler, 0) + + // new epoch should have 2 validators (node2, node3) + ts.AssertSybilProtectionCandidates(1, []iotago.AccountID{ + ts.Node("node2").Validator.AccountID, + ts.Node("node3").Validator.AccountID, + }, ts.Nodes()...) + + ts.AssertSybilProtectionRegisteredValidators(1, []string{ + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node3").Validator.AccountID.ToAddress().Bech32(hrp), + }, ts.Nodes()...) + + assertValidatorsFromRequestHandler(t, []string{ + ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), + ts.Node("node3").Validator.AccountID.ToAddress().Bech32(hrp), + }, requestHandler, 1) + + } + +} + +func assertValidatorsFromRequestHandler(t *testing.T, expectedValidators []string, requestHandler *RequestHandler, requestedEpoch iotago.EpochIndex) { + resp, err := requestHandler.Validators(requestedEpoch, 0, 10) + require.NoError(t, err) + actualValidators := lo.Map(resp.Validators, func(validator *api.ValidatorResponse) string { + return validator.AddressBech32 + }) + + require.ElementsMatch(t, expectedValidators, actualValidators) +} diff --git a/pkg/testsuite/mock/blockissuer.go b/pkg/testsuite/mock/blockissuer.go index 30e75a943..6d0deb4fa 100644 --- a/pkg/testsuite/mock/blockissuer.go +++ b/pkg/testsuite/mock/blockissuer.go @@ -213,7 +213,7 @@ func (i *BlockIssuer) retrieveAPI(blockParams *BlockHeaderParams, node *Node) (i // CreateBlock creates a new block with the options. func (i *BlockIssuer) CreateBasicBlock(ctx context.Context, alias string, node *Node, opts ...options.Option[BasicBlockParams]) (*blocks.Block, error) { - blockParams := options.Apply(&BasicBlockParams{}, opts) + blockParams := options.Apply(&BasicBlockParams{BlockHeader: &BlockHeaderParams{}}, opts) if blockParams.BlockHeader.IssuingTime == nil { issuingTime := time.Now().UTC() diff --git a/pkg/testsuite/sybilprotection.go b/pkg/testsuite/sybilprotection.go index 57d8e282f..85da211ff 100644 --- a/pkg/testsuite/sybilprotection.go +++ b/pkg/testsuite/sybilprotection.go @@ -10,6 +10,7 @@ import ( "github.com/iotaledger/iota-core/pkg/protocol/engine/accounts" "github.com/iotaledger/iota-core/pkg/testsuite/mock" iotago "github.com/iotaledger/iota.go/v4" + "github.com/iotaledger/iota.go/v4/api" ) func (t *TestSuite) AssertSybilProtectionCommittee(epoch iotago.EpochIndex, expectedAccounts []iotago.AccountID, nodes ...*mock.Node) { @@ -79,6 +80,30 @@ func (t *TestSuite) AssertReelectedCommitteeSeatIndices(prevEpoch iotago.EpochIn } } +func (t *TestSuite) AssertSybilProtectionRegisteredValidators(epoch iotago.EpochIndex, expectedAccounts []string, nodes ...*mock.Node) { + mustNodes(nodes) + + for _, node := range nodes { + t.Eventually(func() error { + candidates, err := node.Protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(epoch) + candidateIDs := lo.Map(candidates, func(candidate *api.ValidatorResponse) string { + return candidate.AddressBech32 + }) + require.NoError(t.Testing, err) + + if !assert.ElementsMatch(t.fakeTesting, expectedAccounts, candidateIDs) { + return ierrors.Errorf("AssertSybilProtectionRegisteredValidators: %s: expected %s, got %s", node.Name, expectedAccounts, candidateIDs) + } + + if len(expectedAccounts) != len(candidates) { + return ierrors.Errorf("AssertSybilProtectionRegisteredValidators: %s: expected %v, got %v", node.Name, len(expectedAccounts), len(candidateIDs)) + } + + return nil + }) + } +} + func (t *TestSuite) AssertSybilProtectionCandidates(epoch iotago.EpochIndex, expectedAccounts []iotago.AccountID, nodes ...*mock.Node) { mustNodes(nodes) From d4d58016b530449d8b9753850dd91a37a741932c Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 11 Mar 2024 14:42:18 +0800 Subject: [PATCH 10/21] Expose Reset,Get,Set of Cache --- pkg/requesthandler/cache/cache.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/requesthandler/cache/cache.go b/pkg/requesthandler/cache/cache.go index f85b0382f..06f2cef55 100644 --- a/pkg/requesthandler/cache/cache.go +++ b/pkg/requesthandler/cache/cache.go @@ -5,21 +5,29 @@ import ( ) type Cache struct { - *fastcache.Cache + cache *fastcache.Cache } func NewCache(maxSize int) *Cache { return &Cache{ - Cache: fastcache.New(maxSize), + cache: fastcache.New(maxSize), } } +func (c *Cache) Set(key, value []byte) { + c.cache.Set(key, value) +} + func (c *Cache) Get(key []byte) []byte { - if c.Has(key) { + if c.cache.Has(key) { value := make([]byte, 0) - return c.Cache.Get(value, key) + return c.cache.Get(value, key) } return nil } + +func (c *Cache) Reset() { + c.cache.Reset() +} From 0d05c283e6c2acec87afac7a716e7499230eadba Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 11 Mar 2024 14:49:31 +0800 Subject: [PATCH 11/21] Add comments to requesthandler unit test --- pkg/requesthandler/requesthandler_test.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/pkg/requesthandler/requesthandler_test.go b/pkg/requesthandler/requesthandler_test.go index 00dfa627a..ce8bb092a 100644 --- a/pkg/requesthandler/requesthandler_test.go +++ b/pkg/requesthandler/requesthandler_test.go @@ -4,7 +4,6 @@ import ( "testing" "github.com/iotaledger/hive.go/lo" - "github.com/iotaledger/hive.go/log" "github.com/iotaledger/hive.go/runtime/options" "github.com/iotaledger/iota-core/pkg/protocol" "github.com/iotaledger/iota-core/pkg/protocol/engine/notarization/slotnotarization" @@ -62,7 +61,6 @@ func Test_ValidatorsAPI(t *testing.T) { "node3": nodeOpts, "node4": nodeOpts, }) - ts.DefaultWallet().Node.Protocol.SetLogLevel(log.LevelTrace) ts.AssertSybilProtectionCommittee(0, []iotago.AccountID{ ts.Node("node1").Validator.AccountID, @@ -73,7 +71,7 @@ func Test_ValidatorsAPI(t *testing.T) { requestHandler := New(ts.DefaultWallet().Node.Protocol) hrp := ts.API.ProtocolParameters().Bech32HRP() - // Select committee for epoch 1 and test candidacy announcements at different times. + // Epoch 0, assert that node1 and node4 are the only candidates. { ts.IssueBlocksAtSlots("wave-1:", []iotago.SlotIndex{1, 2, 3, 4}, 4, "Genesis", ts.Nodes(), true, false) @@ -82,7 +80,6 @@ func Test_ValidatorsAPI(t *testing.T) { ts.IssueBlocksAtSlots("wave-2:", []iotago.SlotIndex{5, 6, 7, 8}, 4, "node4-candidacy:1", ts.Nodes(), true, false) - // Assert that only candidates that issued before slot 11 are considered. ts.AssertSybilProtectionCandidates(0, []iotago.AccountID{ ts.Node("node1").Validator.AccountID, ts.Node("node4").Validator.AccountID, @@ -99,6 +96,7 @@ func Test_ValidatorsAPI(t *testing.T) { }, requestHandler, 0) } + // Epoch 0, assert that node1, node2 and node4 are the only candidates. { ts.IssueCandidacyAnnouncementInSlot("node2-candidacy:1", 9, "wave-2:8.3", ts.Wallet("node2")) ts.IssueBlocksAtSlots("wave-3:", []iotago.SlotIndex{9, 10}, 4, "node2-candidacy:1", ts.Nodes(), true, false) @@ -122,8 +120,8 @@ func Test_ValidatorsAPI(t *testing.T) { }, requestHandler, 0) } + // Epoch 0, assert that node1, node2 and node4 are the only registered validators. Since node3 issued a candidacy payload after epoch nearing threshold (slot 11), it should not be a registered validators. { - // Those candidacies should not be considered as they're issued after EpochNearingThreshold (slot 10). ts.IssueCandidacyAnnouncementInSlot("node3-candidacy:1", 11, "wave-3:10.3", ts.Wallet("node3")) ts.IssueBlocksAtSlots("wave-5:", []iotago.SlotIndex{11}, 4, "node3-candidacy:1", ts.Nodes(), true, false) @@ -146,6 +144,7 @@ func Test_ValidatorsAPI(t *testing.T) { }, requestHandler, 0) } + // Epoch 1, assert that node1, node2 and node4 are the only registered validators in Epoch 0. And node2 and node3 are the only registered validators in Epoch 1. { ts.IssueBlocksAtSlots("wave-6:", []iotago.SlotIndex{12, 13, 14, 15, 16}, 4, "wave-5:11.3", ts.Nodes(), true, false) @@ -153,14 +152,14 @@ func Test_ValidatorsAPI(t *testing.T) { ts.IssueCandidacyAnnouncementInSlot("node3-candidacy:2", 17, "node2-candidacy:2", ts.Wallet("node3")) ts.IssueBlocksAtSlots("wave-7:", []iotago.SlotIndex{18}, 4, "node3-candidacy:2", ts.Nodes(), true, false) - // advance to next epoch, the validator cache should be used. + // request registered validators of epoch 0, the validator cache should be used. assertValidatorsFromRequestHandler(t, []string{ ts.Node("node1").Validator.AccountID.ToAddress().Bech32(hrp), ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), ts.Node("node4").Validator.AccountID.ToAddress().Bech32(hrp), }, requestHandler, 0) - // new epoch should have 2 validators (node2, node3) + // epoch 1 should have 2 validators (node2, node3) ts.AssertSybilProtectionCandidates(1, []iotago.AccountID{ ts.Node("node2").Validator.AccountID, ts.Node("node3").Validator.AccountID, From 955565ee0e22302b3c9ca60754f17d9709297235 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 11 Mar 2024 15:10:21 +0800 Subject: [PATCH 12/21] Add dockertests build tag to coreapi_test.go --- tools/docker-network/tests/coreapi_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/docker-network/tests/coreapi_test.go b/tools/docker-network/tests/coreapi_test.go index 892fc8c43..3965bc5e9 100644 --- a/tools/docker-network/tests/coreapi_test.go +++ b/tools/docker-network/tests/coreapi_test.go @@ -1,3 +1,5 @@ +//go:build dockertests + package tests import ( From a6878e928e140fca6baf899d21506a1855b6bc07 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 11 Mar 2024 15:52:07 +0800 Subject: [PATCH 13/21] Cache validatorsResponse by slot in current epoch --- pkg/requesthandler/accounts.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index 1e265404b..3d441c255 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -35,23 +35,23 @@ func (r *RequestHandler) CongestionByAccountAddress(accountAddress *iotago.Accou }, nil } -func (r *RequestHandler) registeredValidatorsFromCache(index iotago.EpochIndex) ([]*api.ValidatorResponse, error) { - apiForEpoch := r.APIProvider().APIForEpoch(index) +func (r *RequestHandler) registeredValidatorsFromCache(epochIndex iotago.EpochIndex, key []byte) ([]*api.ValidatorResponse, error) { + apiForEpoch := r.APIProvider().APIForEpoch(epochIndex) - registeredValidatorsBytes := r.registeredValidatorsCache.Get(index.MustBytes()) + registeredValidatorsBytes := r.registeredValidatorsCache.Get(key) if registeredValidatorsBytes == nil { // get the ordered registered validators list from engine. - registeredValidators, err := r.protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(index) + registeredValidators, err := r.protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(epochIndex) if err != nil { - return nil, ierrors.Wrapf(echo.ErrNotFound, " ordered registered validators list for epoch %d not found: %s", index, err) + return nil, ierrors.Wrapf(echo.ErrNotFound, " ordered registered validators list for epoch %d not found: %s", epochIndex, err) } // store validator responses in cache. registeredValidatorsBytes, err := apiForEpoch.Encode(registeredValidators, serix.WithTypeSettings(validatorResponsesTypeSettings)) if err != nil { - return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to encode ordered registered validators list for epoch %d : %s", index, err) + return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to encode ordered registered validators list for epoch %d : %s", epochIndex, err) } - r.registeredValidatorsCache.Set(index.MustBytes(), registeredValidatorsBytes) + r.registeredValidatorsCache.Set(key, registeredValidatorsBytes) return registeredValidators, nil } @@ -59,7 +59,7 @@ func (r *RequestHandler) registeredValidatorsFromCache(index iotago.EpochIndex) validatorResp := make([]*api.ValidatorResponse, 0) _, err := apiForEpoch.Decode(registeredValidatorsBytes, &validatorResp, serix.WithTypeSettings(validatorResponsesTypeSettings)) if err != nil { - return nil, ierrors.Wrapf(err, "failed to decode validator responses for epoch %d", index) + return nil, ierrors.Wrapf(err, "failed to decode validator responses for epoch %d", epochIndex) } return validatorResp, nil @@ -67,15 +67,17 @@ func (r *RequestHandler) registeredValidatorsFromCache(index iotago.EpochIndex) func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, pageSize uint32) (*api.ValidatorsResponse, error) { apiForEpoch := r.APIProvider().APIForEpoch(epochIndex) - currentEpoch := apiForEpoch.TimeProvider().EpochFromSlot(r.protocol.Engines.Main.Get().SyncManager.LatestCommitment().Slot()) + currentSlot := r.protocol.Engines.Main.Get().SyncManager.LatestCommitment().Slot() + currentEpoch := apiForEpoch.TimeProvider().EpochFromSlot(currentSlot) var registeredValidators []*api.ValidatorResponse var err error - // return registered validators of current epoch from node, because they are not yet finalized. if epochIndex == currentEpoch { - registeredValidators, err = r.protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(epochIndex) + // The key of validators cache is the combination of current epoch and current slot. So the results is updated when new commitment is created. + key := append(currentEpoch.MustBytes(), currentSlot.MustBytes()...) + registeredValidators, err = r.registeredValidatorsFromCache(epochIndex, key) } else { - registeredValidators, err = r.registeredValidatorsFromCache(epochIndex) + registeredValidators, err = r.registeredValidatorsFromCache(epochIndex, epochIndex.MustBytes()) } if err != nil { return nil, ierrors.Wrapf(err, "failed to get registered validators for epoch %d", epochIndex) From b0a3cd8bc742d3cbbf5820d90617771063dca9aa Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 11 Mar 2024 16:09:20 +0800 Subject: [PATCH 14/21] Check cursorIndex in Validators --- pkg/requesthandler/accounts.go | 7 ++++++- pkg/requesthandler/requesthandler_test.go | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index 3d441c255..8914a8b16 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -83,7 +83,12 @@ func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, p return nil, ierrors.Wrapf(err, "failed to get registered validators for epoch %d", epochIndex) } - page := registeredValidators[cursorIndex:lo.Min(cursorIndex+pageSize, uint32(len(registeredValidators)))] + pageEndIndex := lo.Min(cursorIndex+pageSize, uint32(len(registeredValidators))) + if cursorIndex >= pageEndIndex { + return nil, ierrors.Wrapf(echo.ErrBadRequest, "invalid pagination cursorIndex, requesting index %d to %d", cursorIndex, pageEndIndex) + } + + page := registeredValidators[cursorIndex:pageEndIndex] resp := &api.ValidatorsResponse{ Validators: page, PageSize: pageSize, diff --git a/pkg/requesthandler/requesthandler_test.go b/pkg/requesthandler/requesthandler_test.go index ce8bb092a..b78d18f9e 100644 --- a/pkg/requesthandler/requesthandler_test.go +++ b/pkg/requesthandler/requesthandler_test.go @@ -12,6 +12,7 @@ import ( "github.com/iotaledger/iota-core/pkg/testsuite" iotago "github.com/iotaledger/iota.go/v4" "github.com/iotaledger/iota.go/v4/api" + "github.com/labstack/echo/v4" "github.com/stretchr/testify/require" ) @@ -174,9 +175,13 @@ func Test_ValidatorsAPI(t *testing.T) { ts.Node("node2").Validator.AccountID.ToAddress().Bech32(hrp), ts.Node("node3").Validator.AccountID.ToAddress().Bech32(hrp), }, requestHandler, 1) - } + // error returned, requesting with invalid cursor index. + { + _, err := requestHandler.Validators(1, 6, 10) + require.ErrorAs(t, echo.ErrBadRequest, &err) + } } func assertValidatorsFromRequestHandler(t *testing.T, expectedValidators []string, requestHandler *RequestHandler, requestedEpoch iotago.EpochIndex) { From f0f98a5344fa79098013728cf19fcf3fc351f2b6 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Mon, 11 Mar 2024 16:18:46 +0800 Subject: [PATCH 15/21] Add comments to Test_ValidatorsAPI --- tools/docker-network/tests/coreapi_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tools/docker-network/tests/coreapi_test.go b/tools/docker-network/tests/coreapi_test.go index 3965bc5e9..26012e797 100644 --- a/tools/docker-network/tests/coreapi_test.go +++ b/tools/docker-network/tests/coreapi_test.go @@ -14,6 +14,12 @@ import ( "github.com/stretchr/testify/require" ) +// Test_ValidatorsAPI tests if the validators API returns the expected validators. +// 1. Run docker network. +// 2. Create 50 new accounts with staking feature. +// 3. Wait until next epoch then issue candidacy payload for each account. +// 4. Check if all 54 validators are returned from the validators API with pageSize 10, the pagination of api is also tested. +// 5. Wait until next epoch then check again if the results remain. func Test_ValidatorsAPI(t *testing.T) { d := NewDockerTestFramework(t, WithProtocolParametersOptions( From 74057a7fa7ac98a80068483b07d3052572132e98 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 13 Mar 2024 14:04:58 +0800 Subject: [PATCH 16/21] Add MaxCacheSize to restAPI params --- components/restapi/component.go | 8 +++++++- components/restapi/params.go | 6 ++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/components/restapi/component.go b/components/restapi/component.go index 4e624c6e2..59447c3d1 100644 --- a/components/restapi/component.go +++ b/components/restapi/component.go @@ -8,6 +8,7 @@ import ( "github.com/labstack/echo/v4" "github.com/labstack/echo/v4/middleware" + "github.com/labstack/gommon/bytes" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/host" "go.uber.org/dig" @@ -88,7 +89,12 @@ func provide(c *dig.Container) error { } if err := c.Provide(func(deps requestHandlerDeps) *requesthandler.RequestHandler { - return requesthandler.New(deps.Protocol) + maxCacheSizeBytes, err := bytes.Parse(ParamsRestAPI.MaxCacheSize) + if err != nil { + Component.LogPanicf("parameter %s invalid", Component.App().Config().GetParameterPath(&(ParamsRestAPI.MaxCacheSize))) + } + + return requesthandler.New(deps.Protocol, requesthandler.WithCacheMaxSizeOptions(int(maxCacheSizeBytes))) }); err != nil { Component.LogPanic(err.Error()) } diff --git a/components/restapi/params.go b/components/restapi/params.go index da4eb45e4..19d69bf7d 100644 --- a/components/restapi/params.go +++ b/components/restapi/params.go @@ -16,10 +16,8 @@ type ParametersRestAPI struct { DebugRequestLoggerEnabled bool `default:"false" usage:"whether the debug logging for requests should be enabled"` // MaxPageSize defines the maximum number of results per page. MaxPageSize uint32 `default:"100" usage:"the maximum number of results per page"` - // RequestsMemoryCacheGranularity defines per how many slots a cache is created for big API requests. - RequestsMemoryCacheGranularity uint32 `default:"10" usage:"defines per how many slots a cache is created for big API requests"` - // MaxRequestedSlotAge defines the maximum age of a request that will be processed. - MaxRequestedSlotAge uint32 `default:"10" usage:"the maximum age of a request that will be processed"` + // MaxCacheSize defines the maximum size of cache for results. + MaxCacheSize string `default:"2MB" usage:"the maximum size of cache for results"` JWTAuth struct { // salt used inside the JWT tokens for the REST API. Change this to a different value to invalidate JWT tokens not matching this new value From 321d26784c84fa93e58d745543bc4ab8444b558a Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 13 Mar 2024 14:51:23 +0800 Subject: [PATCH 17/21] Move cache operations from requestHandler to Cache --- pkg/requesthandler/accounts.go | 50 +++++----------------------- pkg/requesthandler/cache/cache.go | 38 +++++++++++++++++++++ pkg/requesthandler/requesthandler.go | 6 ++-- 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index 5f0d89cce..48d3260e4 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -9,15 +9,12 @@ import ( "github.com/iotaledger/hive.go/ierrors" "github.com/iotaledger/hive.go/kvstore" "github.com/iotaledger/hive.go/lo" - "github.com/iotaledger/hive.go/serializer/v2/serix" "github.com/iotaledger/iota-core/pkg/core/account" "github.com/iotaledger/iota-core/pkg/model" iotago "github.com/iotaledger/iota.go/v4" "github.com/iotaledger/iota.go/v4/api" ) -var validatorResponsesTypeSettings = serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsByte) - func (r *RequestHandler) CongestionByAccountAddress(accountAddress *iotago.AccountAddress, commitment *model.Commitment, workScores ...iotago.WorkScore) (*api.CongestionResponse, error) { accountID := accountAddress.AccountID() acc, exists, err := r.protocol.Engines.Main.Get().Ledger.Account(accountID, commitment.Slot()) @@ -36,36 +33,6 @@ func (r *RequestHandler) CongestionByAccountAddress(accountAddress *iotago.Accou }, nil } -func (r *RequestHandler) registeredValidatorsFromCache(epochIndex iotago.EpochIndex, key []byte) ([]*api.ValidatorResponse, error) { - apiForEpoch := r.APIProvider().APIForEpoch(epochIndex) - - registeredValidatorsBytes := r.registeredValidatorsCache.Get(key) - if registeredValidatorsBytes == nil { - // get the ordered registered validators list from engine. - registeredValidators, err := r.protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(epochIndex) - if err != nil { - return nil, ierrors.Wrapf(echo.ErrNotFound, " ordered registered validators list for epoch %d not found: %s", epochIndex, err) - } - - // store validator responses in cache. - registeredValidatorsBytes, err := apiForEpoch.Encode(registeredValidators, serix.WithTypeSettings(validatorResponsesTypeSettings)) - if err != nil { - return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to encode ordered registered validators list for epoch %d : %s", epochIndex, err) - } - r.registeredValidatorsCache.Set(key, registeredValidatorsBytes) - - return registeredValidators, nil - } - - validatorResp := make([]*api.ValidatorResponse, 0) - _, err := apiForEpoch.Decode(registeredValidatorsBytes, &validatorResp, serix.WithTypeSettings(validatorResponsesTypeSettings)) - if err != nil { - return nil, ierrors.Wrapf(err, "failed to decode validator responses for epoch %d", epochIndex) - } - - return validatorResp, nil -} - func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, pageSize uint32) (*api.ValidatorsResponse, error) { apiForEpoch := r.APIProvider().APIForEpoch(epochIndex) currentSlot := r.protocol.Engines.Main.Get().SyncManager.LatestCommitment().Slot() @@ -73,13 +40,16 @@ func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, p var registeredValidators []*api.ValidatorResponse var err error + key := epochIndex.MustBytes() + // The key of validators cache for current epoch is the combination of current epoch and current slot. So the results is updated when new commitment is created. if epochIndex == currentEpoch { - // The key of validators cache is the combination of current epoch and current slot. So the results is updated when new commitment is created. - key := append(currentEpoch.MustBytes(), currentSlot.MustBytes()...) - registeredValidators, err = r.registeredValidatorsFromCache(epochIndex, key) - } else { - registeredValidators, err = r.registeredValidatorsFromCache(epochIndex, epochIndex.MustBytes()) + key = append(key, currentSlot.MustBytes()...) } + + // get registered validators from cache, if not found, get from engine and store in cache. + registeredValidators, err = r.cache.GetOrCreateRegisteredValidators(apiForEpoch, key, func() ([]*api.ValidatorResponse, error) { + return r.protocol.Engines.Main.Get().SybilProtection.OrderedRegisteredCandidateValidatorsList(epochIndex) + }) if err != nil { return nil, ierrors.Wrapf(err, "failed to get registered validators for epoch %d", epochIndex) } @@ -95,9 +65,7 @@ func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, p PageSize: pageSize, } // this is the last page - if int(cursorIndex+pageSize) > len(registeredValidators) { - resp.Cursor = "" - } else { + if int(cursorIndex+pageSize) <= len(registeredValidators) { resp.Cursor = fmt.Sprintf("%d,%d", epochIndex, cursorIndex+pageSize) } diff --git a/pkg/requesthandler/cache/cache.go b/pkg/requesthandler/cache/cache.go index 06f2cef55..647af6b0e 100644 --- a/pkg/requesthandler/cache/cache.go +++ b/pkg/requesthandler/cache/cache.go @@ -2,8 +2,16 @@ package cache import ( "github.com/VictoriaMetrics/fastcache" + "github.com/labstack/echo/v4" + + "github.com/iotaledger/hive.go/ierrors" + "github.com/iotaledger/hive.go/serializer/v2/serix" + iotago "github.com/iotaledger/iota.go/v4" + "github.com/iotaledger/iota.go/v4/api" ) +var validatorResponsesTypeSettings = serix.TypeSettings{}.WithLengthPrefixType(serix.LengthPrefixTypeAsByte) + type Cache struct { cache *fastcache.Cache } @@ -31,3 +39,33 @@ func (c *Cache) Get(key []byte) []byte { func (c *Cache) Reset() { c.cache.Reset() } + +func (c *Cache) GetOrCreateRegisteredValidators(apiForEpoch iotago.API, key []byte, defaultValueFunc func() ([]*api.ValidatorResponse, error)) ([]*api.ValidatorResponse, error) { + + registeredValidatorsBytes := c.Get(key) + if registeredValidatorsBytes == nil { + // get the ordered registered validators list from engine. + registeredValidators, err := defaultValueFunc() + if err != nil { + return nil, ierrors.Wrapf(echo.ErrNotFound, "ordered registered validators list not found: %w", err) + } + + // store validator responses in cache. + registeredValidatorsBytes, err := apiForEpoch.Encode(registeredValidators, serix.WithTypeSettings(validatorResponsesTypeSettings)) + if err != nil { + return nil, ierrors.Wrapf(echo.ErrInternalServerError, "failed to encode registered validators list: %w", err) + } + + c.Set(key, registeredValidatorsBytes) + + return registeredValidators, nil + } + + validatorResp := make([]*api.ValidatorResponse, 0) + _, err := apiForEpoch.Decode(registeredValidatorsBytes, &validatorResp, serix.WithTypeSettings(validatorResponsesTypeSettings)) + if err != nil { + return nil, err + } + + return validatorResp, nil +} diff --git a/pkg/requesthandler/requesthandler.go b/pkg/requesthandler/requesthandler.go index 243ddaf42..897247c32 100644 --- a/pkg/requesthandler/requesthandler.go +++ b/pkg/requesthandler/requesthandler.go @@ -11,8 +11,8 @@ import ( type RequestHandler struct { workerPool *workerpool.WorkerPool - registeredValidatorsCache *cache.Cache - protocol *protocol.Protocol + cache *cache.Cache + protocol *protocol.Protocol optsCacheMaxSize int } @@ -23,7 +23,7 @@ func New(p *protocol.Protocol, opts ...options.Option[RequestHandler]) *RequestH protocol: p, optsCacheMaxSize: 2 << 20, // 2MB }, opts, func(r *RequestHandler) { - r.registeredValidatorsCache = cache.NewCache(r.optsCacheMaxSize) + r.cache = cache.NewCache(r.optsCacheMaxSize) }) } From f6cef02273f2ffe0bfd69d68f1c6f109df2e7a1a Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 13 Mar 2024 15:31:54 +0800 Subject: [PATCH 18/21] Simplify cursorIndex check in Validators --- pkg/requesthandler/accounts.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/requesthandler/accounts.go b/pkg/requesthandler/accounts.go index 48d3260e4..34069291e 100644 --- a/pkg/requesthandler/accounts.go +++ b/pkg/requesthandler/accounts.go @@ -54,11 +54,11 @@ func (r *RequestHandler) Validators(epochIndex iotago.EpochIndex, cursorIndex, p return nil, ierrors.Wrapf(err, "failed to get registered validators for epoch %d", epochIndex) } - pageEndIndex := lo.Min(cursorIndex+pageSize, uint32(len(registeredValidators))) - if cursorIndex >= pageEndIndex { - return nil, ierrors.Wrapf(echo.ErrBadRequest, "invalid pagination cursorIndex, requesting index %d to %d", cursorIndex, pageEndIndex) + if cursorIndex >= uint32(len(registeredValidators)) { + return nil, ierrors.Wrapf(echo.ErrBadRequest, "invalid pagination cursorIndex, cursorIndex %d is larger than the number of registered validators %d", cursorIndex, len(registeredValidators)) } + pageEndIndex := lo.Min(cursorIndex+pageSize, uint32(len(registeredValidators))) page := registeredValidators[cursorIndex:pageEndIndex] resp := &api.ValidatorsResponse{ Validators: page, From 9cf7b387bee8ac0b18adc32b62401ff1f4c47089 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 13 Mar 2024 15:49:48 +0800 Subject: [PATCH 19/21] Check if requested epoch is pruned --- components/restapi/core/accounts.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/restapi/core/accounts.go b/components/restapi/core/accounts.go index 7f81aa90b..c6869c53e 100644 --- a/components/restapi/core/accounts.go +++ b/components/restapi/core/accounts.go @@ -73,8 +73,8 @@ func validators(c echo.Context) (*api.ValidatorsResponse, error) { } } - if requestedEpoch > currentEpoch { - return nil, ierrors.Wrapf(echo.ErrBadRequest, "epoch %d is larger than current epoch %d", requestedEpoch, currentEpoch) + if requestedEpoch > currentEpoch || requestedEpoch <= deps.RequestHandler.GetNodeStatus().PruningEpoch { + return nil, ierrors.Wrapf(echo.ErrBadRequest, "epoch %d is larger than current epoch or already pruned", requestedEpoch) } return deps.RequestHandler.Validators(requestedEpoch, cursorIndex, pageSize) From 0994a7d7c149e9c7229ab75bddc25302f9ef95a1 Mon Sep 17 00:00:00 2001 From: jkrvivian Date: Wed, 13 Mar 2024 21:20:30 +0800 Subject: [PATCH 20/21] Increase the maximum size of cache --- components/restapi/params.go | 2 +- pkg/requesthandler/requesthandler.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/components/restapi/params.go b/components/restapi/params.go index 19d69bf7d..560c72f00 100644 --- a/components/restapi/params.go +++ b/components/restapi/params.go @@ -17,7 +17,7 @@ type ParametersRestAPI struct { // MaxPageSize defines the maximum number of results per page. MaxPageSize uint32 `default:"100" usage:"the maximum number of results per page"` // MaxCacheSize defines the maximum size of cache for results. - MaxCacheSize string `default:"2MB" usage:"the maximum size of cache for results"` + MaxCacheSize string `default:"50MB" usage:"the maximum size of cache for results"` JWTAuth struct { // salt used inside the JWT tokens for the REST API. Change this to a different value to invalidate JWT tokens not matching this new value diff --git a/pkg/requesthandler/requesthandler.go b/pkg/requesthandler/requesthandler.go index 897247c32..6d06ae753 100644 --- a/pkg/requesthandler/requesthandler.go +++ b/pkg/requesthandler/requesthandler.go @@ -21,7 +21,7 @@ func New(p *protocol.Protocol, opts ...options.Option[RequestHandler]) *RequestH return options.Apply(&RequestHandler{ workerPool: p.Workers.CreatePool("BlockHandler"), protocol: p, - optsCacheMaxSize: 2 << 20, // 2MB + optsCacheMaxSize: 50 << 20, // 50MB }, opts, func(r *RequestHandler) { r.cache = cache.NewCache(r.optsCacheMaxSize) }) From 417f372fde1b14354a33457dd3e840bf4e520da8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daria=20Dziuba=C5=82towska?= Date: Wed, 13 Mar 2024 14:25:26 +0100 Subject: [PATCH 21/21] Update gendoc --- config_defaults.json | 3 +-- documentation/configuration.md | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/config_defaults.json b/config_defaults.json index f216849fd..5ef1234ff 100644 --- a/config_defaults.json +++ b/config_defaults.json @@ -64,8 +64,7 @@ ], "debugRequestLoggerEnabled": false, "maxPageSize": 100, - "requestsMemoryCacheGranularity": 10, - "maxRequestedSlotAge": 10, + "maxCacheSize": "50MB", "jwtAuth": { "salt": "IOTA" }, diff --git a/documentation/configuration.md b/documentation/configuration.md index d81021021..1c635f49d 100644 --- a/documentation/configuration.md +++ b/documentation/configuration.md @@ -166,17 +166,16 @@ Example: ## 5. RestAPI -| Name | Description | Type | Default value | -| ------------------------------ | ---------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| bindAddress | The bind address on which the REST API listens on | string | "0.0.0.0:14265" | -| publicRoutes | The HTTP REST routes which can be called without authorization. Wildcards using \* are allowed | array | /health
/api/routes
/api/core/v3/info
/api/core/v3/network\*
/api/core/v3/blocks\*
/api/core/v3/transactions\*
/api/core/v3/commitments\*
/api/core/v3/outputs\*
/api/core/v3/accounts\*
/api/core/v3/validators\*
/api/core/v3/rewards\*
/api/core/v3/committee\*
/api/debug/v2/\*
/api/indexer/v2/\*
/api/mqtt/v2
/api/blockissuer/v1/\* | -| protectedRoutes | The HTTP REST routes which need to be called with authorization. Wildcards using \* are allowed | array | /api/\* | -| debugRequestLoggerEnabled | Whether the debug logging for requests should be enabled | boolean | false | -| maxPageSize | The maximum number of results per page | uint | 100 | -| requestsMemoryCacheGranularity | Defines per how many slots a cache is created for big API requests | uint | 10 | -| maxRequestedSlotAge | The maximum age of a request that will be processed | uint | 10 | -| [jwtAuth](#restapi_jwtauth) | Configuration for jwtAuth | object | | -| [limits](#restapi_limits) | Configuration for limits | object | | +| Name | Description | Type | Default value | +| --------------------------- | ---------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| bindAddress | The bind address on which the REST API listens on | string | "0.0.0.0:14265" | +| publicRoutes | The HTTP REST routes which can be called without authorization. Wildcards using \* are allowed | array | /health
/api/routes
/api/core/v3/info
/api/core/v3/network\*
/api/core/v3/blocks\*
/api/core/v3/transactions\*
/api/core/v3/commitments\*
/api/core/v3/outputs\*
/api/core/v3/accounts\*
/api/core/v3/validators\*
/api/core/v3/rewards\*
/api/core/v3/committee\*
/api/debug/v2/\*
/api/indexer/v2/\*
/api/mqtt/v2
/api/blockissuer/v1/\* | +| protectedRoutes | The HTTP REST routes which need to be called with authorization. Wildcards using \* are allowed | array | /api/\* | +| debugRequestLoggerEnabled | Whether the debug logging for requests should be enabled | boolean | false | +| maxPageSize | The maximum number of results per page | uint | 100 | +| maxCacheSize | The maximum size of cache for results | string | "50MB" | +| [jwtAuth](#restapi_jwtauth) | Configuration for jwtAuth | object | | +| [limits](#restapi_limits) | Configuration for limits | object | | ### JwtAuth @@ -220,8 +219,7 @@ Example: ], "debugRequestLoggerEnabled": false, "maxPageSize": 100, - "requestsMemoryCacheGranularity": 10, - "maxRequestedSlotAge": 10, + "maxCacheSize": "50MB", "jwtAuth": { "salt": "IOTA" },