Feat/ implement sentry integration (#80)

Benzbeeb · web-flow · commit 95e8103ab73a · 2025-02-19T10:36:52.000+07:00
* feat: implement sentry integration

* fix: lint error

* fix: lint

* fix: as coderabbit

* fix: as comment
diff --git a/.gitignore b/.gitignore
@@ -14,4 +14,6 @@ build/
 # Dependency directories
 vendor/
 
-test_key/
+test_key/
+
+.env*
diff --git a/challenger/challenger.go b/challenger/challenger.go
@@ -10,6 +10,7 @@ import (
 	"github.com/initia-labs/opinit-bots/challenger/child"
 	challengerdb "github.com/initia-labs/opinit-bots/challenger/db"
 	"github.com/initia-labs/opinit-bots/challenger/host"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/server"
 
 	bottypes "github.com/initia-labs/opinit-bots/bot/types"
@@ -45,7 +46,12 @@ type Challenger struct {
 }
 
 func NewChallenger(cfg *challengertypes.Config, db types.DB, sv *server.Server) *Challenger {
-	err := cfg.Validate()
+	err := sentry_integration.Init("opinit-bots", string(bottypes.BotTypeChallenger))
+	if err != nil {
+		panic(err)
+	}
+
+	err = cfg.Validate()
 	if err != nil {
 		panic(err)
 	}
diff --git a/challenger/handler.go b/challenger/handler.go
@@ -1,10 +1,13 @@
 package challenger
 
 import (
+	"fmt"
 	"sort"
 
+	"github.com/getsentry/sentry-go"
 	challengerdb "github.com/initia-labs/opinit-bots/challenger/db"
 	challengertypes "github.com/initia-labs/opinit-bots/challenger/types"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/types"
 	"go.uber.org/zap"
 )
@@ -70,7 +73,10 @@ func (c *Challenger) getLatestChallenges() []challengertypes.Challenge {
 }
 
 func (c *Challenger) handleChallenge(ctx types.Context, challenge challengertypes.Challenge) error {
-	// TODO: warning log or send to alerting system
+	sentry_integration.CaptureCurrentHubException(
+		fmt.Errorf("challenge: %v", challenge),
+		sentry.LevelWarning,
+	)
 	ctx.Logger().Error("challenge", zap.Any("challenge", challenge))
 
 	return nil
diff --git a/executor/batchsubmitter/batch.go b/executor/batchsubmitter/batch.go
@@ -15,6 +15,7 @@ import (
 	executortypes "github.com/initia-labs/opinit-bots/executor/types"
 	"github.com/initia-labs/opinit-bots/node"
 	btypes "github.com/initia-labs/opinit-bots/node/broadcaster/types"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/types"
 )
 
@@ -115,7 +116,11 @@ func (bs *BatchSubmitter) handleBatch(blockBytes []byte) (int, error) {
 // finalizeBatch finalizes the batch by writing the last block's commit to the batch file.
 // it creates batch messages for the batch data and adds them to the processed messages.
 // the batch data is divided into chunks and each chunk is added to the processed messages.
-func (bs *BatchSubmitter) finalizeBatch(ctx types.Context, blockHeight int64) error {
+func (bs *BatchSubmitter) finalizeBatch(parentCtx types.Context, blockHeight int64) error {
+	transaction, ctx := sentry_integration.StartSentryTransaction(parentCtx, "finalizeBatch", "Finalizes the batch")
+	defer transaction.Finish()
+	transaction.SetTag("height", fmt.Sprintf("%d", blockHeight))
+
 	// write last block's commit to batch file
 	rawCommit, err := bs.node.GetRPCClient().QueryRawCommit(ctx, blockHeight)
 	if err != nil {
diff --git a/executor/executor.go b/executor/executor.go
@@ -9,6 +9,7 @@ import (
 	"github.com/initia-labs/opinit-bots/executor/celestia"
 	"github.com/initia-labs/opinit-bots/executor/child"
 	"github.com/initia-labs/opinit-bots/executor/host"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/server"
 
 	bottypes "github.com/initia-labs/opinit-bots/bot/types"
@@ -36,7 +37,12 @@ type Executor struct {
 }
 
 func NewExecutor(cfg *executortypes.Config, db types.DB, sv *server.Server) *Executor {
-	err := cfg.Validate()
+	err := sentry_integration.Init("opinit-bots", string(bottypes.BotTypeExecutor))
+	if err != nil {
+		panic(err)
+	}
+
+	err = cfg.Validate()
 	if err != nil {
 		panic(err)
 	}
diff --git a/go.mod b/go.mod
@@ -9,11 +9,13 @@ require (
 	cosmossdk.io/x/feegrant v0.1.1
 	cosmossdk.io/x/tx v0.13.7
 	github.com/celestiaorg/go-square/v2 v2.0.0
+	github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d
 	github.com/cometbft/cometbft v0.38.12
 	github.com/cosmos/cosmos-sdk v0.50.11
 	github.com/cosmos/go-bip39 v1.0.0
 	github.com/cosmos/gogoproto v1.7.0
 	github.com/cosmos/ibc-go/v8 v8.5.0
+	github.com/getsentry/sentry-go v0.27.0
 	github.com/gofiber/fiber/v2 v2.52.5
 	github.com/initia-labs/OPinit v0.6.2
 	github.com/pkg/errors v0.9.1
@@ -76,7 +78,6 @@ require (
 	github.com/fatih/color v1.17.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/getsentry/sentry-go v0.27.0 // indirect
 	github.com/go-kit/kit v0.13.0 // indirect
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.6.0 // indirect
diff --git a/go.sum b/go.sum
@@ -884,6 +884,8 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw=
+github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d h1:S2NE3iHSwP0XV47EEXL8mWmRdEfGscSJ+7EgePNgt0s=
+github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
diff --git a/node/block_handler.go b/node/block_handler.go
@@ -9,12 +9,17 @@ import (
 	rpccoretypes "github.com/cometbft/cometbft/rpc/core/types"
 	comettypes "github.com/cometbft/cometbft/types"
 	nodetypes "github.com/initia-labs/opinit-bots/node/types"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/types"
 	"github.com/pkg/errors"
 )
 
 // handleBeginBlock handles the begin block.
 func (n *Node) handleBeginBlock(ctx types.Context, blockID []byte, protoBlock *prototypes.Block, latestHeight int64) error {
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "handleBeginBlock", "Handles the begin block")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", latestHeight))
+
 	if n.beginBlockHandler != nil {
 		return n.beginBlockHandler(ctx, nodetypes.BeginBlockArgs{
 			BlockID:      blockID,
@@ -27,6 +32,10 @@ func (n *Node) handleBeginBlock(ctx types.Context, blockID []byte, protoBlock *p
 
 // handleBlockTxs handles the block transactions.
 func (n *Node) handleBlockTxs(ctx types.Context, block *rpccoretypes.ResultBlock, blockResult *rpccoretypes.ResultBlockResults, latestHeight int64) error {
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "handleBlockTxs", "Handles the block transactions")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", latestHeight))
+
 	if len(block.Block.Txs) != len(blockResult.TxsResults) {
 		return fmt.Errorf("mismatch in transactions and results count: %d vs %d", len(block.Block.Txs), len(blockResult.TxsResults))
 	}
@@ -55,6 +64,10 @@ func (n *Node) handleBlockTxs(ctx types.Context, block *rpccoretypes.ResultBlock
 
 // handleFinalizeBlock handles the finalize block.
 func (n *Node) handleFinalizeBlock(ctx types.Context, blockHeight int64, blockTime time.Time, blockResult *rpccoretypes.ResultBlockResults, latestHeight int64) error {
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "handleFinalizeBlock", "Handles the finalize block")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", latestHeight))
+
 	return n.handleEvents(ctx, blockHeight, blockTime, blockResult.FinalizeBlockEvents, latestHeight, nil, 0)
 }
 
@@ -73,6 +86,10 @@ func (n *Node) handleEvents(ctx types.Context, blockHeight int64, blockTime time
 
 // handleEndBlock handles the end block.
 func (n *Node) handleEndBlock(ctx types.Context, blockID []byte, protoBlock *prototypes.Block, latestHeight int64) error {
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "handleEndBlock", "Handles the end block")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", latestHeight))
+
 	if n.endBlockHandler != nil {
 		return n.endBlockHandler(ctx, nodetypes.EndBlockArgs{
 			BlockID:      blockID,
@@ -84,8 +101,13 @@ func (n *Node) handleEndBlock(ctx types.Context, blockID []byte, protoBlock *pro
 }
 
 // handleRawBlock handles the raw block bytes.
-func (n *Node) handleRawBlock(ctx types.Context, blockHeight int64, latestHeight int64, blockBytes []byte) error {
+func (n *Node) handleRawBlock(parentCtx types.Context, blockHeight int64, latestHeight int64, blockBytes []byte) error {
 	if n.rawBlockHandler != nil {
+		transactions, ctx := sentry_integration.StartSentryTransaction(parentCtx, "handleRawBlock", "Handles the raw block bytes")
+		defer transactions.Finish()
+		transactions.SetTag("height", fmt.Sprintf("%d", blockHeight))
+		transactions.SetTag("latest_height", fmt.Sprintf("%d", latestHeight))
+
 		return n.rawBlockHandler(ctx, nodetypes.RawBlockArgs{
 			BlockHeight:  blockHeight,
 			LatestHeight: latestHeight,
diff --git a/node/broadcaster/account.go b/node/broadcaster/account.go
@@ -4,11 +4,14 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"strings"
 
 	abci "github.com/cometbft/cometbft/abci/types"
+	"github.com/getsentry/sentry-go"
 	"github.com/initia-labs/opinit-bots/keys"
 	btypes "github.com/initia-labs/opinit-bots/node/broadcaster/types"
 	"github.com/initia-labs/opinit-bots/node/rpcclient"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/txutils"
 	"github.com/initia-labs/opinit-bots/types"
 
@@ -19,6 +22,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+
 	txtypes "github.com/cosmos/cosmos-sdk/types/tx"
 	"github.com/cosmos/cosmos-sdk/types/tx/signing"
 	authsigning "github.com/cosmos/cosmos-sdk/x/auth/signing"
@@ -221,6 +225,13 @@ func (b BroadcasterAccount) CalculateGas(ctx context.Context, msgs ...sdk.Msg) (
 
 	res, err := b.rpcClient.QueryABCI(ctx, simQuery)
 	if err != nil {
+		for _, e := range sentryCapturedErrors {
+			if strings.Contains(err.Error(), e.Error()) {
+				sentry_integration.CaptureCurrentHubException(err, sentry.LevelError)
+				return txtypes.SimulateResponse{}, 0, err
+			}
+		}
+
 		return txtypes.SimulateResponse{}, 0, err
 	}
 
diff --git a/node/broadcaster/process.go b/node/broadcaster/process.go
@@ -7,12 +7,14 @@ import (
 	"strings"
 	"time"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
 
 	rpccoretypes "github.com/cometbft/cometbft/rpc/core/types"
 
 	btypes "github.com/initia-labs/opinit-bots/node/broadcaster/types"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/types"
 )
 
@@ -122,7 +124,9 @@ func (b *Broadcaster) Start(ctx types.Context) error {
 				}
 			}
 			if err != nil {
-				return errors.Wrap(err, "failed to handle processed msgs")
+				err = errors.Wrap(err, "failed to handle processed msgs")
+				sentry_integration.CaptureCurrentHubException(err, sentry.LevelWarning)
+				return err
 			}
 		}
 	}
diff --git a/node/broadcaster/tx.go b/node/broadcaster/tx.go
@@ -9,12 +9,15 @@ import (
 
 	"go.uber.org/zap"
 
+	"github.com/getsentry/sentry-go"
 	"github.com/pkg/errors"
 
 	btypes "github.com/initia-labs/opinit-bots/node/broadcaster/types"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
+	"github.com/initia-labs/opinit-bots/types"
 
+	sdkerrors "github.com/cosmos/cosmos-sdk/types/errors"
 	opchildtypes "github.com/initia-labs/OPinit/x/opchild/types"
-	"github.com/initia-labs/opinit-bots/types"
 )
 
 var ignoringErrors = []error{
@@ -28,10 +31,16 @@ var ignoringErrors = []error{
 var accountSeqRegex = regexp.MustCompile("account sequence mismatch, expected ([0-9]+), got ([0-9]+)")
 var outputIndexRegex = regexp.MustCompile("expected ([0-9]+), got ([0-9]+): invalid output index")
 
+var sentryCapturedErrors = []error{
+	sdkerrors.ErrOutOfGas,
+	sdkerrors.ErrInsufficientFunds,
+}
+
 // handleMsgError handles error when processing messages.
 // If there is an error known to be ignored, it will be ignored.
 func (b *Broadcaster) handleMsgError(ctx types.Context, err error, broadcasterAccount *BroadcasterAccount) error {
 	if strs := accountSeqRegex.FindStringSubmatch(err.Error()); strs != nil {
+		sentry_integration.CaptureCurrentHubException(err, sentry.LevelWarning)
 		expected, parseErr := strconv.ParseUint(strs[1], 10, 64)
 		if parseErr != nil {
 			return parseErr
@@ -65,6 +74,13 @@ func (b *Broadcaster) handleMsgError(ctx types.Context, err error, broadcasterAc
 		return err
 	}
 
+	for _, e := range sentryCapturedErrors {
+		if strings.Contains(err.Error(), e.Error()) {
+			sentry_integration.CaptureCurrentHubException(err, sentry.LevelError)
+			return err
+		}
+	}
+
 	for _, e := range ignoringErrors {
 		if strings.Contains(err.Error(), e.Error()) {
 			ctx.Logger().Warn("ignoring error", zap.String("error", e.Error()))
diff --git a/node/process.go b/node/process.go
@@ -8,6 +8,7 @@ import (
 	rpccoretypes "github.com/cometbft/cometbft/rpc/core/types"
 	comettypes "github.com/cometbft/cometbft/types"
 	nodetypes "github.com/initia-labs/opinit-bots/node/types"
+	"github.com/initia-labs/opinit-bots/sentry_integration"
 	"github.com/initia-labs/opinit-bots/types"
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
@@ -123,15 +124,18 @@ func (n *Node) processBlocksTypeRaw(ctx types.Context, latestHeight int64) error
 }
 
 // fetchNewBlock fetches a new block and block results given the height
-func (n *Node) fetchNewBlock(ctx types.Context, height int64) (*rpccoretypes.ResultBlock, *rpccoretypes.ResultBlockResults, error) {
+func (n *Node) fetchNewBlock(parentCtx types.Context, height int64) (*rpccoretypes.ResultBlock, *rpccoretypes.ResultBlockResults, error) {
+	transaction, ctx := sentry_integration.StartSentryTransaction(parentCtx, "fetchNewBlock", "Fetches a new block and block results given the height")
+	defer transaction.Finish()
+	transaction.SetTag("height", fmt.Sprintf("%d", height))
 	ctx.Logger().Debug("fetch new block", zap.Int64("height", height))
 
-	block, err := n.rpcClient.Block(ctx, &height)
+	block, err := n.fetchBlockWithSentry(ctx, height)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	blockResult, err := n.rpcClient.BlockResults(ctx, &height)
+	blockResult, err := n.fetchBlockResultsWithSentry(ctx, height)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -140,7 +144,11 @@ func (n *Node) fetchNewBlock(ctx types.Context, height int64) (*rpccoretypes.Res
 
 // handleNewBlock handles a new block and block results given the height
 // it sends txs and events to the respective registered handlers
-func (n *Node) handleNewBlock(ctx types.Context, block *rpccoretypes.ResultBlock, blockResult *rpccoretypes.ResultBlockResults, latestChainHeight int64) error {
+func (n *Node) handleNewBlock(parentCtx types.Context, block *rpccoretypes.ResultBlock, blockResult *rpccoretypes.ResultBlockResults, latestChainHeight int64) error {
+	transaction, ctx := sentry_integration.StartSentryTransaction(parentCtx, "handleNewBlock", "Handles a new block and block results given the height")
+	defer transaction.Finish()
+	transaction.SetTag("height", fmt.Sprintf("%d", block.Block.Height))
+
 	protoBlock, err := block.Block.ToProto()
 	if err != nil {
 		return errors.Wrap(err, "failed to convert block to proto block")
@@ -174,6 +182,10 @@ func (n *Node) handleEvent(ctx types.Context, blockHeight int64, blockTime time.
 	if n.eventHandlers[event.GetType()] == nil {
 		return nil
 	}
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "handleEvent", "Handles the event for the given transaction")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", blockHeight))
+	span.SetTag("type", event.GetType())
 
 	ctx.Logger().Debug("handle event", zap.Int64("height", blockHeight), zap.String("type", event.GetType()))
 	return n.eventHandlers[event.Type](ctx, nodetypes.EventHandlerArgs{
@@ -185,3 +197,27 @@ func (n *Node) handleEvent(ctx types.Context, blockHeight int64, blockTime time.
 		EventAttributes: event.GetAttributes(),
 	})
 }
+
+func (n *Node) fetchBlockWithSentry(ctx types.Context, height int64) (*rpccoretypes.ResultBlock, error) {
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "rpc.Block", "Fetches a block given the height")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", height))
+
+	block, err := n.rpcClient.Block(ctx, &height)
+	if err != nil {
+		return nil, err
+	}
+	return block, nil
+}
+
+func (n *Node) fetchBlockResultsWithSentry(ctx types.Context, height int64) (*rpccoretypes.ResultBlockResults, error) {
+	span, ctx := sentry_integration.StartSentrySpan(ctx, "rpc.BlockResults", "Fetches block results given the height")
+	defer span.Finish()
+	span.SetTag("height", fmt.Sprintf("%d", height))
+
+	blockResult, err := n.rpcClient.BlockResults(ctx, &height)
+	if err != nil {
+		return nil, err
+	}
+	return blockResult, nil
+}
diff --git a/sentry_integration/sentry.go b/sentry_integration/sentry.go

Original file line number	Diff line number	Diff line change
`@@ -7,12 +7,14 @@ import (`
`7`	`7`	`"strings"`
`8`	`8`	`"time"`
`9`	`9`
	`10`	`+ "github.com/getsentry/sentry-go"`
`10`	`11`	`"github.com/pkg/errors"`
`11`	`12`	`"go.uber.org/zap"`
`12`	`13`
`13`	`14`	`rpccoretypes "github.com/cometbft/cometbft/rpc/core/types"`
`14`	`15`
`15`	`16`	`btypes "github.com/initia-labs/opinit-bots/node/broadcaster/types"`
	`17`	`+ "github.com/initia-labs/opinit-bots/sentry_integration"`
`16`	`18`	`"github.com/initia-labs/opinit-bots/types"`
`17`	`19`	`)`
`18`	`20`
`@@ -122,7 +124,9 @@ func (b *Broadcaster) Start(ctx types.Context) error {`
`122`	`124`	`}`
`123`	`125`	`}`
`124`	`126`	`if err != nil {`
`125`		`- return errors.Wrap(err, "failed to handle processed msgs")`
	`127`	`+ err = errors.Wrap(err, "failed to handle processed msgs")`
	`128`	`+ sentry_integration.CaptureCurrentHubException(err, sentry.LevelWarning)`
	`129`	`+ return err`
`126`	`130`	`}`
`127`	`131`	`}`
`128`	`132`	`}`