diff --git a/cmd/update-binance-spot-history/main.go b/cmd/update-binance-spot-history/main.go new file mode 100644 index 00000000..c04d8a0d --- /dev/null +++ b/cmd/update-binance-spot-history/main.go @@ -0,0 +1,33 @@ +package main + +import ( + "time" + + "github.com/defipod/mochi/pkg/config" + "github.com/defipod/mochi/pkg/entities" + "github.com/defipod/mochi/pkg/logger" + "github.com/defipod/mochi/pkg/scheduler" + "github.com/defipod/mochi/pkg/service" +) + +func main() { + cfg := config.LoadConfig(config.DefaultConfigLoaders()) + log := logger.NewLogrusLogger() + // *** entities *** + err := entities.Init(cfg, log) + if err != nil { + log.Fatal(err, "failed to init entities") + } + + service, err := service.NewService(cfg, log) + if err != nil { + log.Fatal(err, "failed to init service") + } + defer service.Sentry.Flush(2 * time.Second) + + if err := scheduler.NewUpdateBinanceSpotHistory(entities.Get(), log, service, cfg).Run(); err != nil { + log.Fatal(err, "failed to run job") + } + + log.Info("done") +} diff --git a/go.mod b/go.mod index 9090d4a3..3dc0720c 100644 --- a/go.mod +++ b/go.mod @@ -161,7 +161,6 @@ require ( github.com/getsentry/sentry-go v0.18.0 github.com/k0kubun/pp/v3 v3.2.0 github.com/pkg/errors v0.9.1 - github.com/shopspring/decimal v1.2.0 github.com/xssnick/tonutils-go v1.8.5 github.com/yuin/goldmark v1.6.0 golang.org/x/exp v0.0.0-20230510235704-dd950f8aeaea diff --git a/go.sum b/go.sum index b20d20b7..d3715700 100644 --- a/go.sum +++ b/go.sum @@ -74,8 +74,6 @@ github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= @@ -728,7 +726,6 @@ github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAm github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU= github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sigurn/crc16 v0.0.0-20211026045750-20ab5afb07e3 h1:aQKxg3+2p+IFXXg97McgDGT5zcMrQoi0EICZs8Pgchs= diff --git a/migrations/schemas/20240423200849-create_binance_tracking_table.sql b/migrations/schemas/20240423200849-create_binance_tracking_table.sql index 89afd651..3b9fc3f5 100644 --- a/migrations/schemas/20240423200849-create_binance_tracking_table.sql +++ b/migrations/schemas/20240423200849-create_binance_tracking_table.sql @@ -31,8 +31,6 @@ create table binance_spot_transactions ( self_trade_prevention_mode text ); -create unique index binance_spot_transactions_profile_id_order_id_index on binance_spot_transactions (profile_id, order_id); -- +migrate Down drop table if exists binance_trackings; -drop index if exists binance_spot_transactions_profile_id_order_id_index; -drop table if exists binance_spot_transactions; \ No newline at end of file +drop table if exists binance_spot_transactions; diff --git a/pkg/entities/binance_data.go b/pkg/entities/binance_data.go index a9b16a63..1e76a49b 100644 --- a/pkg/entities/binance_data.go +++ b/pkg/entities/binance_data.go @@ -109,7 +109,7 @@ func (e *Entity) CrawlBinanceSpotTransactions() { // get spot transactions startTime := strconv.Itoa(int(binance.SpotLastTime.UnixMilli())) endTime := strconv.Itoa(int(time.Date(2024, 2, 20, 0, 0, 0, 0, time.UTC).UnixMilli())) - _, err := e.svc.Binance.GetSpotTransactions(binance.ApiKey, binance.ApiSecret, startTime, endTime) + _, err := e.svc.Binance.GetSpotTransactions(binance.ApiKey, binance.ApiSecret, "BTCUSDT", startTime, endTime) if err != nil { e.log.Fields(logger.Fields{"profileId": binance.ProfileId}).Error(err, "[entities.CrawlBinanceSpotTransactions] - fail to get spot transactions") continue diff --git a/pkg/model/binance_spot_transaction.go b/pkg/model/binance_spot_transaction.go index f713108b..30a5d947 100644 --- a/pkg/model/binance_spot_transaction.go +++ b/pkg/model/binance_spot_transaction.go @@ -12,7 +12,7 @@ type BinanceSpotTransaction struct { Price string `json:"price"` OrigQty string `json:"orig_qty"` ExecutedQty string `json:"executed_qty"` - CummulativeQuoteQty string `json:"cummulative_quote_qty"` + CumulativeQuoteQty string `json:"cumulative_quote_qty"` Status string `json:"status"` TimeInForce string `json:"time_in_force"` Type string `json:"type"` diff --git a/pkg/repo/binance_tracking/pg.go b/pkg/repo/binance_tracking/pg.go index a4335444..8fb634b3 100644 --- a/pkg/repo/binance_tracking/pg.go +++ b/pkg/repo/binance_tracking/pg.go @@ -17,3 +17,7 @@ func NewPG(db *gorm.DB) Store { func (pg *pg) FirstOrCreate(binanceTracking *model.BinanceTracking) (*model.BinanceTracking, error) { return binanceTracking, pg.db.Where("profile_id = ?", binanceTracking.ProfileId).FirstOrCreate(&binanceTracking).Error } + +func (pg *pg) Update(binanceTracking *model.BinanceTracking) error { + return pg.db.Save(binanceTracking).Error +} diff --git a/pkg/repo/binance_tracking/store.go b/pkg/repo/binance_tracking/store.go index bced36fe..2f4c8298 100644 --- a/pkg/repo/binance_tracking/store.go +++ b/pkg/repo/binance_tracking/store.go @@ -4,4 +4,5 @@ import "github.com/defipod/mochi/pkg/model" type Store interface { FirstOrCreate(binanceTracking *model.BinanceTracking) (*model.BinanceTracking, error) + Update(binanceTracking *model.BinanceTracking) error } diff --git a/pkg/scheduler/scheduler.go b/pkg/scheduler/scheduler.go new file mode 100644 index 00000000..25914e46 --- /dev/null +++ b/pkg/scheduler/scheduler.go @@ -0,0 +1,5 @@ +package scheduler + +type Scheduler interface { + Run() error +} diff --git a/pkg/scheduler/update-binance-spot-history.go b/pkg/scheduler/update-binance-spot-history.go new file mode 100644 index 00000000..6fe33732 --- /dev/null +++ b/pkg/scheduler/update-binance-spot-history.go @@ -0,0 +1,132 @@ +package scheduler + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/defipod/mochi/pkg/config" + "github.com/defipod/mochi/pkg/entities" + "github.com/defipod/mochi/pkg/logger" + "github.com/defipod/mochi/pkg/model" + "github.com/defipod/mochi/pkg/request" + "github.com/defipod/mochi/pkg/service" +) + +type updateBinanceSpotHistory struct { + entity *entities.Entity + log logger.Logger + svc *service.Service + config config.Config +} + +// NewCheckInvalidateEmoji returns a new job that checks for invalid emojis +func NewUpdateBinanceSpotHistory(e *entities.Entity, l logger.Logger, s *service.Service, cfg config.Config) Scheduler { + return &updateBinanceSpotHistory{ + entity: e, + log: l, + svc: s, + config: cfg, + } +} +func binanceStartTime() time.Time { + return time.Now().Add(-90 * 24 * time.Hour).UTC() +} +func (s *updateBinanceSpotHistory) Run() error { + for { + s.schedulerUpdate() + // Sleep for an hour interval before checking the database again + time.Sleep(5 * time.Second) + } +} + +func (s *updateBinanceSpotHistory) schedulerUpdate() error { + + res, err := s.svc.MochiProfile.GetAllBinanceAccount() + if err != nil { + s.log.Error(err, "[updateBinanceSpotHistory] - MochiProfile.GetAllBinanceAccount() fail to get all binance associated account") + return err + } + // get binance exchangeInfo + data, _, _ := s.svc.Binance.GetExchangeInfo("") + pairs := []string{} + for _, d := range data.Symbols { + pairs = append(pairs, d.Symbol) + } + + for _, acc := range res.Data { + binanceTracking, err := s.entity.GetRepo().BinanceTracking.FirstOrCreate(&model.BinanceTracking{ProfileId: acc.ProfileId, SpotLastTime: binanceStartTime()}) + if err != nil { + s.log.Fields(logger.Fields{"profileId": acc.ProfileId}).Error(err, "[updateBinanceSpotHistory] - BinanceTracking.FirstOrCreate() fail to first or create binance tracking ") + continue + } + + symbols := []string{} + assetBal, _, _, _ := s.entity.GetBinanceAssets(request.GetBinanceAssetsRequest{ + Id: acc.ProfileId, + Platform: "binance", + }) + for _, asset := range assetBal.Asset { + symbols = append(symbols, asset.Token.Symbol) + } + symbolPairs := make(map[string][]string) + + // Populate the map + for _, pair := range pairs { + for _, symbol := range symbols { + if strings.HasPrefix(pair, symbol) { + symbolPairs[symbol] = append(symbolPairs[symbol], pair) + break + } + } + } + + // Print pairs sorted by the order of symbols + for _, symbol := range symbols { + pairs := symbolPairs[symbol] + for _, p := range pairs { + startTime := strconv.Itoa(int(binanceTracking.SpotLastTime.UnixMilli())) + endTime := strconv.Itoa(int(binanceTracking.SpotLastTime.Add(24 * time.Hour).UnixMilli())) + txs, err := s.svc.Binance.GetSpotTransactions(acc.ApiKey, acc.ApiSecret, p, startTime, endTime) + if err != nil { + s.log.Fields(logger.Fields{"profileId": acc.ProfileId}).Error(err, "[updateBinanceSpotHistory] - svc.Binance.GetSpotTransactions() fail to get spot txs") + break + } + for _, tx := range txs { + err = s.entity.GetRepo().BinanceSpotTransaction.Create(&model.BinanceSpotTransaction{ + ProfileId: acc.ProfileId, + Symbol: tx.Symbol, + OrderId: tx.OrderId, + OrderListId: tx.OrderListId, + ClientOrderId: tx.ClientOrderId, + Price: tx.Price, + OrigQty: tx.OrigQty, + ExecutedQty: tx.ExecutedQty, + CumulativeQuoteQty: tx.CummulativeQuoteQty, + Status: tx.Status, + TimeInForce: tx.TimeInForce, + Type: tx.Type, + Side: tx.Side, + StopPrice: tx.StopPrice, + IcebergQty: tx.IcebergQty, + IsWorking: tx.IsWorking, + OrigQuoteOrderQty: tx.OrigQuoteOrderQty, + SelfTradePreventionMode: tx.SelfTradePreventionMode, + }) + if err != nil { + fmt.Printf("err: %v", err) + break + } + } + } + } + binanceTracking.SpotLastTime = binanceTracking.SpotLastTime.Add(24 * time.Hour) + err = s.entity.GetRepo().BinanceTracking.Update(binanceTracking) + if err != nil { + s.log.Fields(logger.Fields{"profileId": acc.ProfileId}).Error(err, "[updateBinanceSpotHistory] -BinanceTracking.Update() fail to update binance tracking") + break + } + } + return nil +} diff --git a/pkg/service/binance/adapter/adapter.go b/pkg/service/binance/adapter/adapter.go index 9df25fb2..f839d4fa 100644 --- a/pkg/service/binance/adapter/adapter.go +++ b/pkg/service/binance/adapter/adapter.go @@ -7,8 +7,6 @@ import ( "strconv" "time" - "github.com/k0kubun/pp/v3" - "github.com/defipod/mochi/pkg/logger" "github.com/defipod/mochi/pkg/response" butils "github.com/defipod/mochi/pkg/service/binance/utils" @@ -337,12 +335,12 @@ func GetTickerPrice(symbol string) (price *response.BinanceApiTickerPriceRespons return price, nil } -func GetSpotTransaction(apiKey, apiSecret, startTime, endTime string) (txs []response.BinanceSpotTransaction, err error) { +func GetSpotTransaction(apiKey, apiSecret, symbol, startTime, endTime string) (txs []response.BinanceSpotTransaction, err error) { q := map[string]string{ "timestamp": strconv.Itoa(int(time.Now().UnixMilli())), "startTime": startTime, "endTime": endTime, - "symbol": "BTCUSDT", + "symbol": symbol, "limit": "1000", } queryString := butils.QueryString(q, apiSecret) @@ -364,8 +362,6 @@ func GetSpotTransaction(apiKey, apiSecret, startTime, endTime string) (txs []res return nil, err } - pp.Println(string(resBody)) - // decode response json err = json.Unmarshal(resBody, &txs) if err != nil { diff --git a/pkg/service/binance/binance.go b/pkg/service/binance/binance.go index b11dd918..f92311f5 100644 --- a/pkg/service/binance/binance.go +++ b/pkg/service/binance/binance.go @@ -514,10 +514,10 @@ func (b *Binance) GetPrice(symbol string) (*response.BinanceApiTickerPriceRespon return badapter.GetTickerPrice(symbol) } -func (b *Binance) GetSpotTransactions(apiKey, apiSecret, startTime, endTime string) ([]response.BinanceSpotTransaction, error) { +func (b *Binance) GetSpotTransactions(apiKey, apiSecret, symbol, startTime, endTime string) ([]response.BinanceSpotTransaction, error) { b.logger.Debug("start binance.GetSpotTransaction()") defer b.logger.Debug("end binance.GetSpotTransaction()") // get spot transaction - return badapter.GetSpotTransaction(apiKey, apiSecret, startTime, endTime) + return badapter.GetSpotTransaction(apiKey, apiSecret, symbol, startTime, endTime) } diff --git a/pkg/service/binance/service.go b/pkg/service/binance/service.go index cab589b8..aaae7246 100644 --- a/pkg/service/binance/service.go +++ b/pkg/service/binance/service.go @@ -19,5 +19,5 @@ type Service interface { GetFutureAccount(apiKey, apiSecret string) (*response.BinanceFutureAccount, error) GetFutureAccountInfo(apiKey, apiSecret string) ([]response.BinanceFuturePositionInfo, error) GetPrice(symbol string) (*response.BinanceApiTickerPriceResponse, error) - GetSpotTransactions(apiKey, apiSecret, startTime, endTime string) ([]response.BinanceSpotTransaction, error) + GetSpotTransactions(apiKey, apiSecret, symbol, startTime, endTime string) ([]response.BinanceSpotTransaction, error) }