enhance: stop trigger when tool reference not updated (#1867)

enhance: stop trigger when tool reference not updated

The existing logic would update the tool reference every hour, even when there
were no updates to the tool reference because we would track the last checked
time on the object itself. This would cause a large storm of object processing
because runs, threads, agents, workflows, and workflow executions would all be
triggered in this case.

This change ensures that tool references only trigger related objects when the
tool reference itself changes. In addition to the changes that are explicitly
added here, changes to nah were also required. This change bumps that dependency
as well.

Signed-off-by: Donnie Adams <[email protected]>

Author: thedadams

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de
 	github.com/mhale/smtpd v0.8.3
 	github.com/obot-platform/kinm v0.0.0-20250116162656-270198b40c6d
-	github.com/obot-platform/nah v0.0.0-20250226155418-3825a76fcc28
+	github.com/obot-platform/nah v0.0.0-20250227230402-c8ac47e1add7
 	github.com/obot-platform/namegenerator v0.0.0-20241217121223-fc58bdb7dca2
 	github.com/obot-platform/obot/apiclient v0.0.0-00010101000000-000000000000
 	github.com/obot-platform/obot/logger v0.0.0-20241217130503-4004a5c69f32

go.sum

@@ -460,8 +460,8 @@ github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349 h1:t05Ww3DxZutOqbMN+
 github.com/oasdiff/yaml3 v0.0.0-20241210130736-a94c01f36349/go.mod h1:y5+oSEHCPT/DGrS++Wc/479ERge0zTFxaF8PbGKcg2o=
 github.com/obot-platform/kinm v0.0.0-20250116162656-270198b40c6d h1:GzMvRkssr4jAa2YvQiv9eXhjuNpaZVab3GajE7+cQ3s=
 github.com/obot-platform/kinm v0.0.0-20250116162656-270198b40c6d/go.mod h1:RzrH0geIlbiTHDGZ8bpCk5k1hwdU9uu3l4zJn9n0pZU=
-github.com/obot-platform/nah v0.0.0-20250226155418-3825a76fcc28 h1:NrUAP4e9UHjEHCItaHt9ZDmuWqy0Wba7vFaAU6MGtZE=
-github.com/obot-platform/nah v0.0.0-20250226155418-3825a76fcc28/go.mod h1:KG1jLO9FeYvCPGI0iDqe5oqDqOFLd3/dt/iwuMianmI=
+github.com/obot-platform/nah v0.0.0-20250227230402-c8ac47e1add7 h1:IuG8TZqqxz5P1ZHj2aCxEEn6LGgHXeUVhkG/ELh2a8U=
+github.com/obot-platform/nah v0.0.0-20250227230402-c8ac47e1add7/go.mod h1:KG1jLO9FeYvCPGI0iDqe5oqDqOFLd3/dt/iwuMianmI=
 github.com/obot-platform/namegenerator v0.0.0-20241217121223-fc58bdb7dca2 h1:jiyBM/TYxU6UNVS9ff8Y8n55DOKDYohKkIZjfHpjfTY=
 github.com/obot-platform/namegenerator v0.0.0-20241217121223-fc58bdb7dca2/go.mod h1:isbKX6EfvvG/ojjFB2ZLyz27+2xoG3yRmpTSE+ytWEs=
 github.com/olekukonko/tablewriter v0.0.6-0.20230925090304-df64c4bbad77 h1:3bMMZ1f+GPXFQ1uNaYbO/uECWvSfqEA+ZEXn1rFAT88=

pkg/controller/handlers/toolreference/toolreference.go

@@ -9,6 +9,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/gptscript-ai/go-gptscript"
@@ -30,9 +31,12 @@ import (
 	"sigs.k8s.io/yaml"
 )
 
-var log = logger.Package()
+var (
+	log           = logger.Package()
+	jsonErrRegexp = regexp.MustCompile(`\{.*"error":.*}`)
+)
 
-var jsonErrRegexp = regexp.MustCompile(`\{.*"error":.*}`)
+const toolRecheckPeriod = time.Hour
 
 type indexEntry struct {
 	Reference string `json:"reference,omitempty"`
@@ -49,10 +53,12 @@ type index struct {
 }
 
 type Handler struct {
-	gptClient     *gptscript.GPTScript
-	dispatcher    *dispatcher.Dispatcher
-	supportDocker bool
-	registryURLs  []string
+	gptClient      *gptscript.GPTScript
+	dispatcher     *dispatcher.Dispatcher
+	supportDocker  bool
+	registryURLs   []string
+	lastChecksLock *sync.RWMutex
+	lastChecks     map[string]time.Time
 }
 
 func New(gptClient *gptscript.GPTScript,
@@ -61,10 +67,12 @@ func New(gptClient *gptscript.GPTScript,
 	supportDocker bool,
 ) *Handler {
 	return &Handler{
-		gptClient:     gptClient,
-		dispatcher:    dispatcher,
-		registryURLs:  registryURLs,
-		supportDocker: supportDocker,
+		gptClient:      gptClient,
+		dispatcher:     dispatcher,
+		registryURLs:   registryURLs,
+		supportDocker:  supportDocker,
+		lastChecks:     make(map[string]time.Time),
+		lastChecksLock: new(sync.RWMutex),
 	}
 }
 
@@ -167,22 +175,41 @@ func (h *Handler) PollRegistries(ctx context.Context, c client.Client) {
 
 func (h *Handler) Populate(req router.Request, resp router.Response) error {
 	toolRef := req.Object.(*v1.ToolReference)
-	if retry := time.Until(toolRef.Status.LastReferenceCheck.Time); toolRef.Generation == toolRef.Status.ObservedGeneration && retry > -time.Hour {
-		resp.RetryAfter(time.Hour + retry)
+	h.lastChecksLock.RLock()
+	lastCheck, ok := h.lastChecks[toolRef.Name]
+	h.lastChecksLock.RUnlock()
+	if !ok {
+		// Tracking these times this way is not HA. However, we don't run this in an HA way right now.
+		// When we are ready to start exploring HA as an option, this will have to be changed.
+		lastCheck = time.Now()
+		h.lastChecksLock.Lock()
+		h.lastChecks[toolRef.Name] = time.Now()
+		h.lastChecksLock.Unlock()
+	}
+
+	var retry time.Duration
+	defer func() {
+		resp.RetryAfter(toolRecheckPeriod + retry)
+	}()
+
+	if retry = time.Until(lastCheck); ok && toolRef.Generation == toolRef.Status.ObservedGeneration && retry > -toolRecheckPeriod {
 		return nil
 	}
+	retry = 0
 
 	// Reset status
-	lastCheck := toolRef.Status.LastReferenceCheck
-	toolRef.Status.LastReferenceCheck = metav1.Now()
 	toolRef.Status.ObservedGeneration = toolRef.Generation
 	toolRef.Status.Reference = toolRef.Spec.Reference
 	toolRef.Status.Commit = ""
 	toolRef.Status.Tool = nil
 	toolRef.Status.Error = ""
 
+	h.lastChecksLock.Lock()
+	h.lastChecks[toolRef.Name] = time.Now()
+	h.lastChecksLock.Unlock()
+
 	prg, err := h.gptClient.LoadFile(req.Ctx, toolRef.Spec.Reference, gptscript.LoadOptions{
-		DisableCache: toolRef.Spec.ForceRefresh.After(lastCheck.Time),
+		DisableCache: toolRef.Spec.ForceRefresh.After(lastCheck),
 	})
 	if err != nil {
 		toolRef.Status.Error = err.Error()

pkg/controller/handlers/workflowstep/workflowstep.go

@@ -229,7 +229,6 @@ func NewStep(namespace, workflowExecutionName, afterStepName string, generation
 	stepName = strings.ReplaceAll(stepName, "--", "-")
 
 	return &v1.WorkflowStep{
-		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      stepName,
 			Namespace: namespace,

pkg/storage/apis/obot.obot.ai/v1/toolreference.go

@@ -80,7 +80,6 @@ type ToolReferenceStatus struct {
 	Reference          string                `json:"reference,omitempty"`
 	Commit             string                `json:"commit,omitempty"`
 	ObservedGeneration int64                 `json:"observedGeneration,omitempty"`
-	LastReferenceCheck metav1.Time           `json:"lastReferenceCheck,omitempty"`
 	Tool               *ToolShortDescription `json:"tool,omitempty"`
 	Error              string                `json:"error,omitempty"`
 }