From ebc9bcdd071f8f8ff4527c94f2d29093a8af3f63 Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Wed, 25 Sep 2024 11:12:52 +0100 Subject: [PATCH] Add blocksapi blocks loader --- go.mod | 36 +- go.sum | 92 +++-- indexer/blockapi.go | 173 +++++++++ indexer/config.go | 62 --- indexer/indexer.go | 374 +++++-------------- indexer/refiner.go | 298 +++++++++++++++ indexer/{indexer_test.go => refiner_test.go} | 38 +- main.go | 18 +- 8 files changed, 660 insertions(+), 431 deletions(-) create mode 100644 indexer/blockapi.go delete mode 100644 indexer/config.go create mode 100644 indexer/refiner.go rename indexer/{indexer_test.go => refiner_test.go} (94%) diff --git a/go.mod b/go.mod index 593e58f..787d35d 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/aurora-is-near/relayer2-public -go 1.18 +go 1.21 + +toolchain go1.23.0 //replace github.com/aurora-is-near/relayer2-base => "../relayer2-base" //replace github.com/aurora-is-near/near-api-go => github.com/aurora-is-near/near-api-go @@ -10,18 +12,20 @@ go 1.18 replace github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.23.2 require ( + github.com/aurora-is-near/borealis-prototypes-go v0.0.0-20240612174445-a1ca8d036252 github.com/aurora-is-near/near-api-go v0.0.14 github.com/aurora-is-near/relayer2-base v1.1.7 github.com/btcsuite/btcutil v1.0.2 github.com/buger/jsonparser v1.1.1 github.com/ethereum/go-ethereum v1.10.25 - github.com/google/uuid v1.3.0 + github.com/google/uuid v1.6.0 github.com/json-iterator/go v1.1.12 + github.com/planetscale/vtprotobuf v0.6.0 github.com/spf13/cobra v1.6.0 github.com/spf13/viper v1.13.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.4 github.com/valyala/fasthttp v1.47.0 - golang.org/x/net v0.17.0 + google.golang.org/grpc v1.65.0 ) require ( @@ -37,10 +41,11 @@ require ( github.com/dustin/go-humanize v1.0.0 // indirect github.com/fasthttp/websocket v1.5.2 // indirect github.com/fxamacker/cbor/v2 v2.4.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/glog v1.0.0 // indirect + github.com/golang/glog v1.2.1 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/google/flatbuffers v22.9.29+incompatible // indirect github.com/holiman/uint256 v1.2.0 // indirect @@ -63,38 +68,35 @@ require ( github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/valyala/tcplisten v1.0.0 // indirect github.com/x448/float16 v0.8.4 // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opencensus.io v0.23.0 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/sync v0.1.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sync v0.7.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect + google.golang.org/protobuf v1.34.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect ) require ( github.com/btcsuite/btcd/btcec/v2 v2.2.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect github.com/fsnotify/fsnotify v1.6.0 - github.com/go-stack/stack v1.8.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/shirou/gopsutil v3.21.11+incompatible // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect - github.com/tklauser/numcpus v0.5.0 // indirect - golang.org/x/crypto v0.14.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.13.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 9af9596..42a51a0 100644 --- a/go.sum +++ b/go.sum @@ -40,16 +40,22 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.1.0 h1:ksErzDEI1khOiGPgpwuI7x2ebx/uXQNw7xJpn9Eq1+I= +github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/adhityaramadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a h1:/Crz3EkXO3i1JsZ8VGRnP4EMQ91NMn2meNzZyBziv6Q= github.com/adhityaramadhanus/fasthttpcors v0.0.0-20170121111917-d4c07198763a/go.mod h1:nRKQcoNwlDDIGlkaivCFTccvuJj6SnKV7w8T/WVqwgo= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/aurora-is-near/borealis-prototypes-go v0.0.0-20240612174445-a1ca8d036252 h1:yD7zd+QLUMctvjMwkIJIOiQF0bMBLRNRfsKRSIEIVdM= +github.com/aurora-is-near/borealis-prototypes-go v0.0.0-20240612174445-a1ca8d036252/go.mod h1:bWm5wYU/BbGd3L9JC40lajbXD3wsjcTmKHuikCzEpT0= github.com/aurora-is-near/go-jsonrpc/v3 v3.1.2 h1:GKX/ga2vElkJYHSIpWEGsW3Z5c0xFx6bnXkSsU30GXg= github.com/aurora-is-near/go-jsonrpc/v3 v3.1.2/go.mod h1:Li013EFlPu3crtlFQtWJAeE7VmdhSsxOpRoop1J0icw= github.com/aurora-is-near/near-api-go v0.0.14 h1:6BHZ0jtUQ3frNx0h6Kucd2tOR/TZ5I7lScgN5khNhd8= @@ -82,8 +88,8 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -127,6 +133,7 @@ github.com/ethereum/go-ethereum v1.10.25/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbL github.com/fasthttp/websocket v1.5.2 h1:KdCb0EpLpdJpfE3IPA5YLK/aYBO3dhZcvwxz6tXe2LQ= github.com/fasthttp/websocket v1.5.2/go.mod h1:S0KC1VBlx1SaXGXq7yi1wKz4jMub58qEnHQG9oHuqBw= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -138,14 +145,14 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= -github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.2.1 h1:OptwRhECazUx5ix5TTWC3EZhsZEHWcYWY4FQHTIubm4= +github.com/golang/glog v1.2.1/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -172,9 +179,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -193,8 +199,8 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -211,8 +217,8 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= @@ -251,9 +257,11 @@ github.com/klauspost/compress v1.16.3/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQs github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= @@ -264,6 +272,7 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -278,6 +287,7 @@ github.com/near/borsh-go v0.3.1 h1:ukNbhJlPKxfua0/nIuMZhggSU8zvtRP/VyC25LLqPUA= github.com/near/borsh-go v0.3.1/go.mod h1:NeMochZp7jN/pYFuxLkrZtmLqbADmnp/y1+/dL+AsyQ= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= @@ -293,14 +303,18 @@ github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaF github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/planetscale/vtprotobuf v0.6.0 h1:nBeETjudeJ5ZgBHUz1fVHvbqUKnYOXNhsIEabROxmNA= +github.com/planetscale/vtprotobuf v0.6.0/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/puzpuzpuz/xsync/v2 v2.4.0 h1:5sXAMHrtx1bg9nbRZTOn8T4MkWe5V+o8yKRH02Eznag= github.com/puzpuzpuz/xsync/v2 v2.4.0/go.mod h1:gD2H2krq/w52MfPLE+Uy64TzJDVY7lP2znR9qmR35kU= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= @@ -308,8 +322,8 @@ github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee h1:8Iv5m6xEo1NR1AvpV+7XmhI4r39LGNzwUL4YpMuL5vk= github.com/savsgio/gotils v0.0.0-20230208104028-c358bd845dee/go.mod h1:qwtSXrKuJh/zsFQ12yEE89xfCrGKK63Rr7ctU/uCo4g= -github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= -github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= +github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -340,17 +354,17 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= -github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= -github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= -github.com/tklauser/numcpus v0.5.0 h1:ooe7gN0fg6myJ0EKoTAf5hebTZrH52px3New/D9iJ+A= -github.com/tklauser/numcpus v0.5.0/go.mod h1:OGzpTxpcIMNGYQdit2BYL1pvk/dSOaJWjKoflh+RQjo= +github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= +github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= +github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA= +github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= @@ -365,8 +379,6 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -386,8 +398,8 @@ golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -456,8 +468,8 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -477,8 +489,8 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -524,13 +536,11 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -538,8 +548,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -597,6 +607,7 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -659,6 +670,8 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -675,6 +688,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -685,14 +700,13 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/indexer/blockapi.go b/indexer/blockapi.go new file mode 100644 index 0000000..874cf6b --- /dev/null +++ b/indexer/blockapi.go @@ -0,0 +1,173 @@ +package indexer + +import ( + "context" + "errors" + "io" + "sync" + + blocksapi "github.com/aurora-is-near/borealis-prototypes-go/generated/blocksapi" + "github.com/aurora-is-near/relayer2-base/broker" + "github.com/aurora-is-near/relayer2-base/db" + "github.com/aurora-is-near/relayer2-base/log" + "github.com/aurora-is-near/relayer2-base/types" + "github.com/aurora-is-near/relayer2-base/types/common" + "github.com/aurora-is-near/relayer2-base/types/indexer" + "github.com/aurora-is-near/relayer2-base/utils" + jsoniter "github.com/json-iterator/go" + + vtgrpc "github.com/planetscale/vtprotobuf/codec/grpc" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding" + _ "google.golang.org/grpc/encoding/proto" + "google.golang.org/grpc/metadata" +) + +type IndexerBlocksAPI struct { + token string + stream string + + dbh db.Handler + l *log.Logger + b broker.Broker + grpc *grpc.ClientConn + height uint64 + mu sync.Mutex +} + +func init() { + // Register protobuf plugin for generate optimized marshall & unmarshal code + encoding.RegisterCodec(vtgrpc.Codec{}) +} + +func NewIndexerBlocksApi(config *Config, dbh db.Handler, b broker.Broker) (*IndexerBlocksAPI, error) { + logger := log.Log() + + client, err := grpc.NewClient( + // TODO: use client-side load balancer + config.BlocksApiUrl[0], + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithInitialConnWindowSize(64*1024*1024), // 64 MB + grpc.WithInitialWindowSize(64*1024*1024), // 64 MB + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(1*1024*1024*1024)), // 1GB + ) + if err != nil { + return nil, err + } + + return &IndexerBlocksAPI{ + stream: config.BlocksApiStream, + token: config.BlocksApiToken, + dbh: dbh, + l: logger, + b: b, + grpc: client, + height: config.FromBlock, + }, nil +} + +func (i *IndexerBlocksAPI) Start(ctx context.Context) { + i.mu.Lock() + defer i.mu.Unlock() + + // Attaching authorization token + md := metadata.New(make(map[string]string)) + md.Set("authorization", "Bearer "+i.token) + + callCtx := metadata.NewOutgoingContext(ctx, md) + + go i.run(callCtx) +} + +func (i *IndexerBlocksAPI) run(callCtx context.Context) { + blocksProviderClient := blocksapi.NewBlocksProviderClient(i.grpc) + + request := &blocksapi.ReceiveBlocksRequest{ + StreamName: i.stream, + StartPolicy: blocksapi.ReceiveBlocksRequest_START_EXACTLY_ON_TARGET, + StopPolicy: blocksapi.ReceiveBlocksRequest_STOP_NEVER, + StartTarget: &blocksapi.BlockMessage_ID{ + Kind: blocksapi.BlockMessage_MSG_WHOLE, + Height: i.height, + }, + } + + callClient, err := blocksProviderClient.ReceiveBlocks(callCtx, request) + if err != nil { + i.l.Error().Err(err).Msgf("unable to call ReceiveBlocks") + return + } + + defer callClient.CloseSend() + + for { + select { + case <-callCtx.Done(): + return + default: + } + + response, err := callClient.Recv() + if errors.Is(err, io.EOF) { + break + } else if err != nil { + i.l.Error().Err(err).Msg("unable to receive next response") + return + } + + switch r := response.Response.(type) { + case *blocksapi.ReceiveBlocksResponse_Message: + var block indexer.Block + + var payload []byte + if rawPayload, ok := r.Message.Message.Payload.(*blocksapi.BlockMessage_RawPayload); ok { + payload = rawPayload.RawPayload + } else { + i.l.Fatal().Msg("invalid payload type") + return + } + + err = jsoniter.Unmarshal(payload, &block) + if err != nil { + i.l.Error().Err(err).Msg("couln't parse block") + return + } + + err := i.dbh.InsertBlock(&block) + if err != nil { + i.l.Error().Err(err).Msg("couln't insert block") + return + } + + if i.b != nil { + ctx := utils.PutChainId(callCtx, block.ChainId) + bn := common.UintToBN64(block.Height) + block, err := i.dbh.GetBlockByNumber(ctx, bn, true) + if err != nil { + // just log, this is a best-effort operation + i.l.Error().Err(err).Msg("couln't get block number") + } else { + i.b.PublishNewHeads(block) + } + + nfilter := &(types.Filter{FromBlock: bn.Uint64(), ToBlock: bn.Uint64()}) + logs, err := i.dbh.GetLogs(ctx, nfilter.ToLogFilter()) + if err != nil { + // just log, this is a best-effort operation + i.l.Error().Err(err).Msg("couln't get block logs") + } else { + i.b.PublishLogs(logs) + } + } + + i.height += 1 + } + } +} + +func (i *IndexerBlocksAPI) Close() { + if err := i.grpc.Close(); err != nil { + log.Log().Printf("Unable to close connection: %v", err) + } +} diff --git a/indexer/config.go b/indexer/config.go deleted file mode 100644 index 24292f2..0000000 --- a/indexer/config.go +++ /dev/null @@ -1,62 +0,0 @@ -package indexer - -import ( - "github.com/aurora-is-near/relayer2-base/cmdutils" - "github.com/aurora-is-near/relayer2-base/log" - - "github.com/spf13/viper" -) - -const ( - DefaultGenesisBlock = 9820210 - DefaultKeepFiles = false - DefaultForceReindex = false - DefaultFromBlock = DefaultGenesisBlock - DefaultToBlock = 0 - DefaultSubFolderBatchSize = 10000 - DefaultRetryCountOnFailure = 10 - DefaultWaitForBlockMs = 500 - DefaultSourceFolder = "/tmp/relayer/json/" - - configPath = "indexer" -) - -type Config struct { - KeepFiles bool `mapstructure:"keepFiles"` - ForceReindex bool `mapstructure:"forceReindex"` - RetryCountOnFailure uint8 `mapstructure:"retryCountOnFailure"` - SubFolderBatchSize uint16 `mapstructure:"subFolderBatchSize"` - GenesisBlock uint64 `mapstructure:"genesisBlock"` - FromBlock uint64 `mapstructure:"fromBlock"` - ToBlock uint64 `mapstructure:"toBlock"` - SourceFolder string `mapstructure:"sourceFolder"` - WaitForBlockMs uint16 `mapstructure:"waitForBlockMs"` -} - -func defaultConfig() *Config { - return &Config{ - KeepFiles: DefaultKeepFiles, - ForceReindex: DefaultForceReindex, - RetryCountOnFailure: DefaultRetryCountOnFailure, - GenesisBlock: DefaultGenesisBlock, - FromBlock: DefaultFromBlock, // inclusive - ToBlock: DefaultToBlock, // exclusive - SubFolderBatchSize: DefaultSubFolderBatchSize, - SourceFolder: DefaultSourceFolder, - WaitForBlockMs: DefaultWaitForBlockMs, - } -} - -func GetConfig() *Config { - config := defaultConfig() - sub := viper.Sub(configPath) - if sub != nil { - cmdutils.BindSubViper(sub, configPath) - if err := sub.Unmarshal(&config); err != nil { - log.Log().Warn().Err(err).Msgf("failed to parse configuration [%s] from [%s], "+ - "falling back to defaults", configPath, viper.ConfigFileUsed()) - } - } - - return config -} diff --git a/indexer/indexer.go b/indexer/indexer.go index 508a2c5..84d2770 100644 --- a/indexer/indexer.go +++ b/indexer/indexer.go @@ -1,333 +1,135 @@ package indexer import ( + "context" "errors" "fmt" - "io/ioutil" - "os" - "path/filepath" - "sync" - "time" - - jsoniter "github.com/json-iterator/go" "github.com/aurora-is-near/relayer2-base/broker" + "github.com/aurora-is-near/relayer2-base/cmdutils" "github.com/aurora-is-near/relayer2-base/db" "github.com/aurora-is-near/relayer2-base/log" - "github.com/aurora-is-near/relayer2-base/types" - "github.com/aurora-is-near/relayer2-base/types/common" - "github.com/aurora-is-near/relayer2-base/types/indexer" - "github.com/aurora-is-near/relayer2-base/utils" - "golang.org/x/net/context" + + "github.com/spf13/viper" ) -type processIndexerState func(*Indexer) processIndexerState +const ( + SourceRefiner = "refiner" + SourceBlocksApi = "blocksApi" + + DefaultSource = SourceRefiner + DefaultGenesisBlock = 9820210 + DefaultKeepFiles = false + DefaultForceReindex = false + DefaultFromBlock = DefaultGenesisBlock + DefaultToBlock = 0 + DefaultSubFolderBatchSize = 10000 + DefaultRetryCountOnFailure = 10 + DefaultWaitForBlockMs = 500 + DefaultSourceFolder = "/tmp/relayer/json/" + + configPath = "indexer" +) -type indexerStats struct { - totalBlocksIndexed uint64 - avgBlockIndexingRateSeconds float32 // TODO add (EW) Moving Avg - blockIndexingStartTime time.Time -} +type Config struct { + Source string `mapstructure:"source"` + GenesisBlock uint64 `mapstructure:"genesisBlock"` + FromBlock uint64 `mapstructure:"fromBlock"` + ToBlock uint64 `mapstructure:"toBlock"` + ForceReindex bool `mapstructure:"forceReindex"` + + // Refiner source specific + KeepFiles bool `mapstructure:"keepFiles"` + SourceFolder string `mapstructure:"sourceFolder"` + SubFolderBatchSize uint16 `mapstructure:"subFolderBatchSize"` + RetryCountOnFailure uint8 `mapstructure:"retryCountOnFailure"` + WaitForBlockMs uint16 `mapstructure:"waitForBlockMs"` + + // BlocksApi source specific + BlocksApiUrl []string `mapstructure:"blocksApiUrl"` + BlocksApiStream string `mapstructure:"blocApiToken"` + BlocksApiToken string `mapstructure:"blocApiToken"` +} + +func defaultConfig() *Config { + return &Config{ + Source: DefaultSource, + GenesisBlock: DefaultGenesisBlock, + FromBlock: DefaultFromBlock, // inclusive + ToBlock: DefaultToBlock, // exclusive + ForceReindex: DefaultForceReindex, + + RetryCountOnFailure: DefaultRetryCountOnFailure, + WaitForBlockMs: DefaultWaitForBlockMs, + + KeepFiles: DefaultKeepFiles, + SubFolderBatchSize: DefaultSubFolderBatchSize, + SourceFolder: DefaultSourceFolder, + } +} + +func GetConfig() *Config { + config := defaultConfig() + sub := viper.Sub(configPath) + if sub != nil { + cmdutils.BindSubViper(sub, configPath) + if err := sub.Unmarshal(&config); err != nil { + log.Log().Warn().Err(err).Msgf("failed to parse configuration [%s] from [%s], "+ + "falling back to defaults", configPath, viper.ConfigFileUsed()) + } + } -type indexerState struct { - currBlock uint64 - subBlock uint64 - batchSize uint64 - filePath string - subBlockPath string - retryCount uint8 - started bool - block *indexer.Block - stats *indexerStats + return config } -type Indexer struct { - dbh db.Handler - l *log.Logger - c *Config - s *indexerState - b broker.Broker - lock *sync.Mutex - stopCh chan struct{} +type Indexer interface { + Start(ctx context.Context) + Close() } -// New creates the indexer, the db.Handler should not be nil -func New(dbh db.Handler) (*Indexer, error) { +func New(dbh db.Handler, b broker.Broker) (Indexer, error) { if dbh == nil { return nil, errors.New("db handler is not initialized") } - logger := log.Log() config := GetConfig() + logger := log.Log() + // validate & normalise config fromBlockUpdated := false - cfgFromBlockOrigin := config.FromBlock - fromBlock := config.GenesisBlock + if !config.ForceReindex { lb, err := dbh.BlockNumber(context.Background()) if err == nil && lb != nil { bn := uint64(*lb) logger.Info().Msgf("latest indexed block: [%d]", bn) + // fromBlock should not be updated for pre history blocks if bn >= config.GenesisBlock { - fromBlock = bn + 1 + logger.Warn().Msgf("overwriting fromBlock: [%d] as [%d]", config.FromBlock, bn+1) + + config.FromBlock = bn + 1 fromBlockUpdated = true } } } - if config.FromBlock < fromBlock { - logger.Warn().Msgf("overwriting fromBlock: [%d] as [%d]", config.FromBlock, fromBlock) - config.FromBlock = fromBlock - } - if (config.ToBlock > DefaultToBlock) && (config.ToBlock <= config.FromBlock) { - if fromBlockUpdated && (cfgFromBlockOrigin < config.ToBlock) { + if fromBlockUpdated && (config.FromBlock < config.ToBlock) { return nil, fmt.Errorf("given block range is already indexed till [%d]. Please either enable `forceReindex` config to re-index or update the range and re-start the application", config.FromBlock) } else { return nil, fmt.Errorf("invalid config, toBlock: [%d] must be greater than fromBlock: [%d]", config.ToBlock, config.FromBlock) } } - bs := uint64(config.SubFolderBatchSize) - sb := config.FromBlock / bs * bs - sbp := filepath.Join(config.SourceFolder, fmt.Sprintf("%v", sb)) - fp := filepath.Join(sbp, fmt.Sprintf("%v.json", config.FromBlock)) - i := &Indexer{ - dbh: dbh, - l: logger, - c: config, - s: &indexerState{ - currBlock: config.FromBlock, - batchSize: bs, - subBlock: sb, - subBlockPath: sbp, - filePath: fp, - retryCount: uint8(0), - block: nil, - stats: &indexerStats{ - totalBlocksIndexed: uint64(0), - avgBlockIndexingRateSeconds: float32(0), - blockIndexingStartTime: time.Now(), - }, - }, - lock: &sync.Mutex{}, - stopCh: make(chan struct{}), - } - - return i, nil -} + switch config.Source { + case SourceRefiner: + return NewIndexerRefiner(config, dbh, b) -// NewWithBroker is same as New but also sets broker for indexer. Both db.Handler and broker.Broker should not be nil. -// Initialize indexer with broker only if the JSON-RPC server supports subscription, otherwise use New instead. -func NewWithBroker(dbh db.Handler, b broker.Broker) (*Indexer, error) { - if b == nil { - return nil, errors.New("broker is not initialized") - } - i, err := New(dbh) - if err != nil { - return nil, err - } - i.b = b - return i, nil -} - -// Start indexer state machine as a goroutine, if it's not already started. -func (i *Indexer) Start() { - i.lock.Lock() - defer i.lock.Unlock() - if !i.s.started { - i.s.started = true - i.s.stats.blockIndexingStartTime = time.Now() - i.l.Info().Msgf("starting indexing fromBlock: [%d], source: [%s]", i.c.FromBlock, i.c.SourceFolder) - go i.index() - } -} - -// Close gracefully stops indexer state machine -func (i *Indexer) Close() { - i.lock.Lock() - defer i.lock.Unlock() - if i.s.started { - stop(i) - } -} - -// index is the entry point of indexer state machine -func (i *Indexer) index() { - f := read - for { - f = f(i) - select { - case <-i.stopCh: - i.lock.Lock() - defer i.lock.Unlock() - i.s.started = false - return - default: - } - } -} - -// evalError should be called by processIndexerState typed functions for all errors, returns; -// nil, if err is nil. i.e.: no state change required, hence caller should continue -// onRetry processIndexerState, if err is not nil and max retry count not exceeded -// onFail processIndexerState, if err is not nil and max retry count is exceeded -func (i *Indexer) evalError(err error, onRetry processIndexerState, onFail processIndexerState) processIndexerState { - if err == nil { - return nil - } - if i.s.retryCount < i.c.RetryCountOnFailure { - i.l.Error().Err(err).Msgf("retrying block: [%d] on failure", i.s.currBlock) - i.s.retryCount += 1 - time.Sleep(time.Duration(i.c.WaitForBlockMs) * time.Millisecond) - return onRetry - } - i.l.Error().Err(err).Msgf("exceeded max onRetry count: [%d] for block: [%d]", i.c.RetryCountOnFailure, i.s.currBlock) - return onFail -} - -// reads block data from file pointed by indexerState.filePath and parses file content into utils.Block. If file does not -// exit on the specified path, it retries/waits indefinitely -// On success, continues with insert -// On failure, retries for Config.RetryCountOnFailure and if all retries fails then stops indexer -func read(i *Indexer) processIndexerState { - i.l.Debug().Msgf("reading block: [%d], path: [%s]", i.s.currBlock, i.s.filePath) - buff, err := os.ReadFile(i.s.filePath) - if os.IsNotExist(err) { - time.Sleep(time.Duration(i.c.WaitForBlockMs) * time.Millisecond) - return read - } - if next := i.evalError(err, read, stop); next != nil { - return next - } - - err = jsoniter.Unmarshal(buff, &i.s.block) - if next := i.evalError(err, read, stop); next != nil { - return next - } - return insert -} - -// insert adds the block to persistent storage. -// On success, continues with publish. -// On failure, retries for Config.RetryCountOnFailure and if all retries fails then stops indexer -func insert(i *Indexer) processIndexerState { - i.l.Debug().Msgf("inserting block: [%d]", i.s.block.Height) - err := i.dbh.InsertBlock(i.s.block) - if next := i.evalError(err, insert, stop); next != nil { - return next - } - return publish -} - -// removeFile tries to remove file pointed by indexerState.filePath. If the file is the last file of the sub block it -// continues with removeFolder state otherwise increment to next block. This is a best-effort operation if there is a -// failure during remove operation, it retries for Config.RetryCountOnFailure and continues with the next block in any case. -func removeFile(i *Indexer) processIndexerState { - i.l.Debug().Msgf("removing file: [%s]", i.s.filePath) - err := os.Remove(i.s.filePath) - if next := i.evalError(err, removeFile, increment); next != nil { - return next - } - // if next block is equal to next sub block and current sub block dir is empty - if ((i.s.currBlock + 1) == (i.s.subBlock + i.s.batchSize)) && isDirEmpty(i.s.subBlockPath) { - return removeFolder - } - return increment -} - -// removeFolder tries to remove the directory pointed by indexerState.subBlockPath. This is a best-effort operation if -// there is a failure during remove operation, it retries for Config.RetryCountOnFailure and continues with the next -// block in any case. -func removeFolder(i *Indexer) processIndexerState { - i.l.Debug().Msgf("removing folder: [%s]", i.s.subBlockPath) - err := os.Remove(i.s.subBlockPath) - if next := i.evalError(err, removeFolder, increment); next != nil { - return next - } - return increment -} - -// increment prepares the indexerState for next block processing and sets indexerStats, returns; -// stop processIndexerState, if Config.ToBlock is specified and reached and re-indexing is disabled -// read processIndexerState, otherwise -func increment(i *Indexer) processIndexerState { - i.s.stats.totalBlocksIndexed += 1 - if (i.c.ToBlock > 0) && (i.c.ToBlock <= i.s.currBlock) { - if !i.c.ForceReindex { - i.l.Info().Msgf("indexing finished fromBlock: [%d], toBlock: [%d]", i.c.FromBlock, i.c.ToBlock) - return stop - } - // after re-indexing a range of blocks, continue with the latest block in db - lb, err := i.dbh.BlockNumber(context.Background()) - if err != nil || lb == nil { - i.l.Error().Err(err).Msgf("failed to read the latest block after re-indexing") - return stop - } - fb := uint64(*lb) - i.l.Info().Msgf("re-indexing finished fromBlock: [%d], toBlock: [%d], indexing will continue fromBlock: [%d]", i.c.FromBlock, i.c.ToBlock, fb) - i.c.ToBlock = 0 - i.s.currBlock = fb - } - i.s.currBlock += 1 - i.s.subBlock = i.s.currBlock / i.s.batchSize * i.s.batchSize - i.s.retryCount = 0 - i.s.block = nil - i.s.subBlockPath = filepath.Join(i.c.SourceFolder, fmt.Sprintf("%v", i.s.subBlock)) - i.s.filePath = filepath.Join(i.s.subBlockPath, fmt.Sprintf("%v.json", i.s.currBlock)) - i.s.stats.blockIndexingStartTime = time.Now() - i.l.Info().Msgf("indexing block: [%d] from: [%s]", i.s.currBlock, i.s.filePath) - return read -} - -// stop indexer state machine. -func stop(i *Indexer) processIndexerState { - lastBlock, err := i.dbh.BlockNumber(context.Background()) - if err != nil { - i.l.Error().Err(err).Msg("failed to get last indexed block") - } - i.l.Info().Msgf("stopping indexer, last indexed block: [%v], current processed block: [%d]", *lastBlock, i.s.currBlock) - close(i.stopCh) - return nil -} - -// publish sends block and log data to event broker if broker is initialized, returns either increment or removeFile -func publish(i *Indexer) processIndexerState { - if i.b != nil { - // TODO: this block can be optimized - i.l.Debug().Msgf("publishing block: [%d]", i.s.currBlock) - ctx := context.Background() - ctx = utils.PutChainId(ctx, i.s.block.ChainId) - bn := common.UintToBN64(i.s.block.Height) - block, err := i.dbh.GetBlockByNumber(ctx, bn, true) - if err != nil { - i.l.Error().Err(err) // just log, this is a best-effort operation - } else { - i.b.PublishNewHeads(block) - } - nfilter := &(types.Filter{FromBlock: bn.Uint64(), ToBlock: bn.Uint64()}) - logs, err := i.dbh.GetLogs(ctx, nfilter.ToLogFilter()) - if err != nil { - i.l.Error().Err(err) // just log, this is a best-effort operation - } else { - i.b.PublishLogs(logs) - } - } - if i.c.KeepFiles { - return increment - } else { - return removeFile - } -} + case SourceBlocksApi: + return NewIndexerBlocksApi(config, dbh, b) -// isDirEmpty checks whether a directory is empty or not, returns; -// false if directory is not empty or fails to get dir info -// true otherwise -func isDirEmpty(dirName string) bool { - files, err := ioutil.ReadDir(dirName) - if err != nil || len(files) != 0 { - return false + default: + return nil, fmt.Errorf("invalid config, source: the source must be %s or %s", SourceRefiner, SourceBlocksApi) } - return true } diff --git a/indexer/refiner.go b/indexer/refiner.go new file mode 100644 index 0000000..7c0f7ca --- /dev/null +++ b/indexer/refiner.go @@ -0,0 +1,298 @@ +package indexer + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/aurora-is-near/relayer2-base/broker" + "github.com/aurora-is-near/relayer2-base/db" + "github.com/aurora-is-near/relayer2-base/log" + "github.com/aurora-is-near/relayer2-base/types" + "github.com/aurora-is-near/relayer2-base/types/common" + "github.com/aurora-is-near/relayer2-base/types/indexer" + "github.com/aurora-is-near/relayer2-base/utils" + + jsoniter "github.com/json-iterator/go" +) + +type processIndexerState func(*IndexerRefiner) processIndexerState + +type indexerStats struct { + totalBlocksIndexed uint64 + avgBlockIndexingRateSeconds float32 // TODO add (EW) Moving Avg + blockIndexingStartTime time.Time +} + +type indexerState struct { + currBlock uint64 + subBlock uint64 + batchSize uint64 + filePath string + subBlockPath string + retryCount uint8 + started bool + block *indexer.Block + stats *indexerStats +} + +type IndexerRefiner struct { + dbh db.Handler + l *log.Logger + c *Config + s *indexerState + b broker.Broker + lock sync.Mutex + stopCh chan struct{} +} + +// NewIndexerRefiner creates the indexer, the db.Handler should not be nil +// Initialize indexer with broker only if the JSON-RPC server supports subscription, +// otherwise broker must be nil +func NewIndexerRefiner( + config *Config, + dbh db.Handler, + b broker.Broker, +) (*IndexerRefiner, error) { + logger := log.Log() + + batchSize := uint64(config.SubFolderBatchSize) + subBlock := config.FromBlock / batchSize * batchSize + subBlockPath := filepath.Join(config.SourceFolder, fmt.Sprintf("%v", subBlock)) + filePath := filepath.Join(subBlockPath, fmt.Sprintf("%v.json", config.FromBlock)) + + return &IndexerRefiner{ + dbh: dbh, + b: b, + l: logger, + c: config, + s: &indexerState{ + currBlock: config.FromBlock, + batchSize: batchSize, + subBlock: subBlock, + subBlockPath: subBlockPath, + filePath: filePath, + retryCount: uint8(0), + block: nil, + stats: &indexerStats{ + totalBlocksIndexed: uint64(0), + avgBlockIndexingRateSeconds: float32(0), + blockIndexingStartTime: time.Now(), + }, + }, + stopCh: make(chan struct{}), + }, nil +} + +// Start indexer state machine as a goroutine, if it's not already started. +// TODO: use ctx for requests fot gracefully shutdown +func (i *IndexerRefiner) Start(ctx context.Context) { + i.lock.Lock() + defer i.lock.Unlock() + + if !i.s.started { + i.s.started = true + i.s.stats.blockIndexingStartTime = time.Now() + i.l.Info().Msgf("starting indexing fromBlock: [%d], source: [%s]", i.c.FromBlock, i.c.SourceFolder) + go i.index() + } +} + +// Close gracefully stops indexer state machine +func (i *IndexerRefiner) Close() { + i.lock.Lock() + defer i.lock.Unlock() + + if i.s.started { + stop(i) + } +} + +// index is the entry point of indexer state machine +func (i *IndexerRefiner) index() { + f := read + for { + f = f(i) + select { + case <-i.stopCh: + i.lock.Lock() + defer i.lock.Unlock() + i.s.started = false + return + default: + } + } +} + +// evalError should be called by processIndexerState typed functions for all errors, returns; +// +// nil, if err is nil. i.e.: no state change required, hence caller should continue +// onRetry processIndexerState, if err is not nil and max retry count not exceeded +// onFail processIndexerState, if err is not nil and max retry count is exceeded +func (i *IndexerRefiner) evalError(err error, onRetry processIndexerState, onFail processIndexerState) processIndexerState { + if err == nil { + return nil + } + if i.s.retryCount < i.c.RetryCountOnFailure { + i.l.Error().Err(err).Msgf("retrying block: [%d] on failure", i.s.currBlock) + i.s.retryCount += 1 + time.Sleep(time.Duration(i.c.WaitForBlockMs) * time.Millisecond) + return onRetry + } + i.l.Error().Err(err).Msgf("exceeded max onRetry count: [%d] for block: [%d]", i.c.RetryCountOnFailure, i.s.currBlock) + return onFail +} + +// reads block data from file pointed by indexerState.filePath and parses file content into utils.Block. If file does not +// exit on the specified path, it retries/waits indefinitely +// +// On success, continues with insert +// On failure, retries for Config.RetryCountOnFailure and if all retries fails then stops indexer +func read(i *IndexerRefiner) processIndexerState { + i.l.Debug().Msgf("reading block: [%d], path: [%s]", i.s.currBlock, i.s.filePath) + buff, err := os.ReadFile(i.s.filePath) + if os.IsNotExist(err) { + time.Sleep(time.Duration(i.c.WaitForBlockMs) * time.Millisecond) + return read + } + if next := i.evalError(err, read, stop); next != nil { + return next + } + + err = jsoniter.Unmarshal(buff, &i.s.block) + if next := i.evalError(err, read, stop); next != nil { + return next + } + return insert +} + +// insert adds the block to persistent storage. +// +// On success, continues with publish. +// On failure, retries for Config.RetryCountOnFailure and if all retries fails then stops indexer +func insert(i *IndexerRefiner) processIndexerState { + i.l.Debug().Msgf("inserting block: [%d]", i.s.block.Height) + err := i.dbh.InsertBlock(i.s.block) + if next := i.evalError(err, insert, stop); next != nil { + return next + } + return publish +} + +// removeFile tries to remove file pointed by indexerState.filePath. If the file is the last file of the sub block it +// continues with removeFolder state otherwise increment to next block. This is a best-effort operation if there is a +// failure during remove operation, it retries for Config.RetryCountOnFailure and continues with the next block in any case. +func removeFile(i *IndexerRefiner) processIndexerState { + i.l.Debug().Msgf("removing file: [%s]", i.s.filePath) + err := os.Remove(i.s.filePath) + if next := i.evalError(err, removeFile, increment); next != nil { + return next + } + // if next block is equal to next sub block and current sub block dir is empty + if ((i.s.currBlock + 1) == (i.s.subBlock + i.s.batchSize)) && isDirEmpty(i.s.subBlockPath) { + return removeFolder + } + return increment +} + +// removeFolder tries to remove the directory pointed by indexerState.subBlockPath. This is a best-effort operation if +// there is a failure during remove operation, it retries for Config.RetryCountOnFailure and continues with the next +// block in any case. +func removeFolder(i *IndexerRefiner) processIndexerState { + i.l.Debug().Msgf("removing folder: [%s]", i.s.subBlockPath) + err := os.Remove(i.s.subBlockPath) + if next := i.evalError(err, removeFolder, increment); next != nil { + return next + } + return increment +} + +// increment prepares the indexerState for next block processing and sets indexerStats, returns; +// +// stop processIndexerState, if Config.ToBlock is specified and reached and re-indexing is disabled +// read processIndexerState, otherwise +func increment(i *IndexerRefiner) processIndexerState { + i.s.stats.totalBlocksIndexed += 1 + if (i.c.ToBlock > 0) && (i.c.ToBlock <= i.s.currBlock) { + if !i.c.ForceReindex { + i.l.Info().Msgf("indexing finished fromBlock: [%d], toBlock: [%d]", i.c.FromBlock, i.c.ToBlock) + return stop + } + // after re-indexing a range of blocks, continue with the latest block in db + lb, err := i.dbh.BlockNumber(context.Background()) + if err != nil || lb == nil { + i.l.Error().Err(err).Msgf("failed to read the latest block after re-indexing") + return stop + } + fb := uint64(*lb) + i.l.Info().Msgf("re-indexing finished fromBlock: [%d], toBlock: [%d], indexing will continue fromBlock: [%d]", i.c.FromBlock, i.c.ToBlock, fb) + i.c.ToBlock = 0 + i.s.currBlock = fb + } + i.s.currBlock += 1 + i.s.subBlock = i.s.currBlock / i.s.batchSize * i.s.batchSize + i.s.retryCount = 0 + i.s.block = nil + i.s.subBlockPath = filepath.Join(i.c.SourceFolder, fmt.Sprintf("%v", i.s.subBlock)) + i.s.filePath = filepath.Join(i.s.subBlockPath, fmt.Sprintf("%v.json", i.s.currBlock)) + i.s.stats.blockIndexingStartTime = time.Now() + i.l.Info().Msgf("indexing block: [%d] from: [%s]", i.s.currBlock, i.s.filePath) + return read +} + +// stop indexer state machine. +func stop(i *IndexerRefiner) processIndexerState { + lastBlock, err := i.dbh.BlockNumber(context.Background()) + if err != nil { + i.l.Error().Err(err).Msg("failed to get last indexed block") + } + i.l.Info().Msgf("stopping indexer, last indexed block: [%v], current processed block: [%d]", *lastBlock, i.s.currBlock) + close(i.stopCh) + return nil +} + +// publish sends block and log data to event broker if broker is initialized, returns either increment or removeFile +func publish(i *IndexerRefiner) processIndexerState { + if i.b != nil { + // TODO: this block can be optimized + i.l.Debug().Msgf("publishing block: [%d]", i.s.currBlock) + ctx := context.Background() + ctx = utils.PutChainId(ctx, i.s.block.ChainId) + bn := common.UintToBN64(i.s.block.Height) + block, err := i.dbh.GetBlockByNumber(ctx, bn, true) + if err != nil { + // just log, this is a best-effort operation + i.l.Error().Err(err).Msg("couln't get block number") + } else { + i.b.PublishNewHeads(block) + } + nfilter := &(types.Filter{FromBlock: bn.Uint64(), ToBlock: bn.Uint64()}) + logs, err := i.dbh.GetLogs(ctx, nfilter.ToLogFilter()) + if err != nil { + // just log, this is a best-effort operation + i.l.Error().Err(err).Msg("couln't get block logs") + } else { + i.b.PublishLogs(logs) + } + } + if i.c.KeepFiles { + return increment + } else { + return removeFile + } +} + +// isDirEmpty checks whether a directory is empty or not, returns; +// +// false if directory is not empty or fails to get dir info +// true otherwise +func isDirEmpty(dirName string) bool { + files, err := os.ReadDir(dirName) + if err != nil || len(files) != 0 { + return false + } + return true +} diff --git a/indexer/indexer_test.go b/indexer/refiner_test.go similarity index 94% rename from indexer/indexer_test.go rename to indexer/refiner_test.go index fc7ce4a..a5f6219 100644 --- a/indexer/indexer_test.go +++ b/indexer/refiner_test.go @@ -263,7 +263,7 @@ func TestConfiguration(t *testing.T) { args[0].(*db.StoreHandler).Close() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) return i.c.FromBlock, err }, want: uint64(DefaultGenesisBlock), @@ -283,7 +283,7 @@ func TestConfiguration(t *testing.T) { args[0].(*db.StoreHandler).Close() }, call: func(args ...interface{}) (interface{}, error) { - return New(args[0].(*db.StoreHandler)) + return NewIndexerRefiner(args[0].(*db.StoreHandler), nil) }, want: nil, errContains: "invalid config", @@ -303,7 +303,7 @@ func TestConfiguration(t *testing.T) { args[0].(*db.StoreHandler).Close() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) return i.c.FromBlock, err }, want: uint64(DefaultGenesisBlock+numBlocksToInsert) + 1, @@ -324,7 +324,7 @@ func TestConfiguration(t *testing.T) { args[0].(*db.StoreHandler).Close() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) return i.c.FromBlock, err }, want: uint64(DefaultGenesisBlock), @@ -356,7 +356,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) r := read for n := uint8(0); n < i.c.RetryCountOnFailure+1; n++ { r = r(i) @@ -382,7 +382,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) r := read for n := uint8(0); n < i.c.RetryCountOnFailure+1; n++ { r = r(i) @@ -408,7 +408,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) return nameOfFunc(read(i)), err }, want: nameOfFunc(insert), @@ -430,7 +430,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) r := read(i) r = r(i) return nameOfFunc(r), err @@ -454,7 +454,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) r := read(i) r = r(i) r = r(i) @@ -479,7 +479,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) r := read(i) r = r(i) r = r(i) @@ -504,7 +504,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) i.s.filePath = "invalid" r := removeFile for n := uint8(0); n < i.c.RetryCountOnFailure+1; n++ { @@ -531,7 +531,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) return nameOfFunc(removeFile(i)), err }, want: nameOfFunc(increment), @@ -553,7 +553,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) removeFile(i) increment(i) i.s.currBlock = i.s.subBlock + i.s.batchSize - 1 @@ -578,7 +578,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) i.s.currBlock = i.s.subBlock + i.s.batchSize - 1 return nameOfFunc(removeFile(i)), err }, @@ -601,7 +601,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) removeFile(i) increment(i) removeFile(i) @@ -626,7 +626,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) i.s.subBlockPath = "invalid" r := removeFolder for n := uint8(0); n < i.c.RetryCountOnFailure+1; n++ { @@ -653,7 +653,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) return nameOfFunc(increment(i)), err }, want: nameOfFunc(read), @@ -675,7 +675,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) increment(i) return nameOfFunc(increment(i)), err }, @@ -699,7 +699,7 @@ func TestStateTransitions(t *testing.T) { deleteFiles() }, call: func(args ...interface{}) (interface{}, error) { - i, err := New(args[0].(*db.StoreHandler)) + i, err := NewIndexerRefiner(args[0].(*db.StoreHandler), nil) increment(i) return nameOfFunc(increment(i)), err }, diff --git a/main.go b/main.go index c83fb0e..b2b8fd3 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "os" "os/signal" @@ -35,6 +36,9 @@ func main() { c.AddCommand(cmd.GetBlockCmd()) c.AddCommand(cmd.FlattenDB()) c.AddCommand(cmd.StartCmd(func(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // graceful interrupt net requests and io + logger := log.Log() bh, err := badger.NewBlockHandler() if err != nil { @@ -133,16 +137,12 @@ func main() { defer tarIndexer.Close() } - var indxr *indexer.Indexer - if rpcNode.Broker != nil { - indxr, err = indexer.NewWithBroker(handler, rpcNode.Broker) - } else { - indxr, err = indexer.New(handler) - } + indxr, err := indexer.New(handler, rpcNode.Broker) if err != nil { logger.Fatal().Err(err).Msg("failed to start indexer") } - indxr.Start() + + indxr.Start(ctx) defer indxr.Close() preIndxr, err := prehistory.New(handler) @@ -160,7 +160,9 @@ func main() { baseEndpoint.HandleConfigChange() }) - sig := make(chan os.Signal) + // We must use a buffered channel or risk missing the signal + // if we're not ready to receive when the signal is sent. + sig := make(chan os.Signal, 1) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) <-sig }))