Metrics gc

cmd/server.go

@@ -20,6 +20,7 @@ import (
 	"path/filepath"
 	"slices"
 	"strings"
+	"time"
 
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/moby/patternmatcher"
@@ -126,6 +127,7 @@ const (
 	ParallelPoolSize                 = "parallel-pool-size"
 	PendingApplyStatusFlag           = "pending-apply-status"
 	StatsNamespace                   = "stats-namespace"
+	MetricsInactivePRRetention       = "metrics-inactive-pr-retention"
 	AllowDraftPRs                    = "allow-draft-prs"
 	PortFlag                         = "port"
 	RedisDB                          = "redis-db"
@@ -188,6 +190,7 @@ const (
 	DefaultMaxCommentsPerCommand        = 100
 	DefaultParallelPoolSize             = 15
 	DefaultStatsNamespace               = "atlantis"
+	DefaultMetricsInactivePRRetention   = "24h"
 	DefaultPort                         = 4141
 	DefaultRedisDB                      = 0
 	DefaultRedisPort                    = 6379
@@ -414,6 +417,10 @@ var stringFlags = map[string]stringFlag{
 		description:  "Namespace for aggregating stats.",
 		defaultValue: DefaultStatsNamespace,
 	},
+	MetricsInactivePRRetention: {
+		description:  "Duration to retain metrics for inactive PRs before cleanup (e.g., '24h', '168h', '7d'). Cleanup runs at this same frequency. Set to 0 to disable cleanup.",
+		defaultValue: DefaultMetricsInactivePRRetention,
+	},
 	RedisHost: {
 		description: "The Redis Hostname for when using a Locking DB type of 'redis'.",
 	},
@@ -956,6 +963,9 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig, v *viper.Viper) {
 	if c.StatsNamespace == "" {
 		c.StatsNamespace = DefaultStatsNamespace
 	}
+	if c.MetricsInactivePRRetention == "" {
+		c.MetricsInactivePRRetention = DefaultMetricsInactivePRRetention
+	}
 	if c.Port == 0 {
 		c.Port = DefaultPort
 	}
@@ -1005,6 +1015,19 @@ func (s *ServerCmd) validate(userConfig server.UserConfig) error {
 			TFDistributionTerraform, TFDistributionOpenTofu)
 	}
 
+	if userConfig.MetricsInactivePRRetention != "" && userConfig.MetricsInactivePRRetention != "0" {
+		retention, err := time.ParseDuration(userConfig.MetricsInactivePRRetention)
+		if err != nil {
+			return fmt.Errorf("--%s must be a valid duration (e.g., '24h', '168h', '7d'): %w", MetricsInactivePRRetention, err)
+		}
+		if retention < 0 {
+			return fmt.Errorf("--%s must be positive", MetricsInactivePRRetention)
+		}
+		if retention > 30*24*time.Hour {
+			return fmt.Errorf("--%s must be <= 30 days", MetricsInactivePRRetention)
+		}
+	}
+
 	checkoutStrategy := userConfig.CheckoutStrategy
 	if checkoutStrategy != CheckoutStrategyBranch && checkoutStrategy != CheckoutStrategyMerge {
 		return fmt.Errorf("invalid checkout strategy: not one of %s or %s",

cmd/server_test.go

@@ -118,6 +118,7 @@ var testFlags = map[string]any{
 	LogLevelFlag:                     "debug",
 	MarkdownTemplateOverridesDirFlag: "/path2",
 	MaxCommentsPerCommand:            10,
+	MetricsInactivePRRetention:       "72h",
 	StatsNamespace:                   "atlantis",
 	AllowDraftPRs:                    true,
 	PortFlag:                         8181,

runatlantis.io/docs/server-configuration.md

@@ -1047,6 +1047,16 @@ ATLANTIS_MAX_COMMENTS_PER_COMMAND=100
 
 Limit the number of comments published after a command is executed, to prevent spamming your VCS and Atlantis to get throttled as a result. Defaults to `100`. Set this option to `0` to disable log truncation. Note that the truncation will happen on the top of the command output, to preserve the most important parts of the output, often displayed at the end.
 
+### `--metrics-inactive-pr-retention`
+
+```bash
+atlantis server --metrics-inactive-pr-retention=72h
+# or
+ATLANTIS_METRICS_INACTIVE_PR_RETENTION=72h
+```
+
+After the duration specified, Atlantis will stop reporting metrics for inactive pull requests.
+
 ### `--parallel-apply` <Badge text="v0.22.0+" type="info"/>
 
 ```bash

server/controllers/events/events_controller.go

@@ -196,11 +196,17 @@ func (e *VCSEventsController) handleGithubPost(w http.ResponseWriter, r *http.Re
 	case *github.IssueCommentEvent:
 		resp = e.HandleGithubCommentEvent(event, githubReqID, logger)
 		scope = scope.SubScope(fmt.Sprintf("comment_%s", *event.Action))
-		scope = common.SetGitScopeTags(scope, event.GetRepo().GetFullName(), event.GetIssue().GetNumber())
+		scope = scope.Tagged(map[string]string{
+			"base_repo": event.GetRepo().GetFullName(),
+			"pr_number": strconv.Itoa(event.GetIssue().GetNumber()),
+		})
 	case *github.PullRequestEvent:
 		resp = e.HandleGithubPullRequestEvent(logger, event, githubReqID)
 		scope = scope.SubScope(fmt.Sprintf("pr_%s", *event.Action))
-		scope = common.SetGitScopeTags(scope, event.GetRepo().GetFullName(), event.GetNumber())
+		scope = scope.Tagged(map[string]string{
+			"base_repo": event.GetRepo().GetFullName(),
+			"pr_number": strconv.Itoa(event.GetNumber()),
+		})
 	default:
 		resp = HTTPResponse{
 			body: fmt.Sprintf("Ignoring unsupported event %s", githubReqID),

server/events/command/project_context.go

@@ -12,6 +12,7 @@ import (
 	"github.com/runatlantis/atlantis/server/core/config/valid"
 	"github.com/runatlantis/atlantis/server/events/models"
 	"github.com/runatlantis/atlantis/server/logging"
+	"github.com/runatlantis/atlantis/server/metrics"
 	tally "github.com/uber-go/tally/v4"
 )
 
@@ -139,8 +140,9 @@ type ProjectContext struct {
 	TeamAllowlistChecker TeamAllowlistChecker
 }
 
-// SetProjectScopeTags adds ProjectContext tags to a new returned scope.
-func (p ProjectContext) SetProjectScopeTags(scope tally.Scope) tally.Scope {
+// SetProjectScopeTags sets project-specific tags on a scope using the PR scope manager.
+// Creates a closeable PR-specific root scope with project-level tags.
+func (p ProjectContext) SetProjectScopeTags(prScopeManager *metrics.PRScopeManager) tally.Scope {
 	v := ""
 	if p.TerraformVersion != nil {
 		v = p.TerraformVersion.String()
@@ -155,7 +157,8 @@ func (p ProjectContext) SetProjectScopeTags(scope tally.Scope) tally.Scope {
 		Workspace:        p.Workspace,
 	}
 
-	return scope.Tagged(tags.Loadtags())
+	// Use PR scope manager to create closeable PR-specific root scope
+	return prScopeManager.GetOrCreatePRScope(p.BaseRepo.FullName, p.Pull.Num, tags.Loadtags())
 }
 
 // GetShowResultFileName returns the filename (not the path) to store the tf show result

server/events/instrumented_project_command_runner.go

@@ -20,51 +20,54 @@ type IntrumentedCommandRunner interface {
 
 type InstrumentedProjectCommandRunner struct {
 	projectCommandRunner ProjectCommandRunner
-	scope                tally.Scope
+	prScopeManager       *metrics.PRScopeManager
+	scope                tally.Scope // fallback scope if PRScopeManager is nil
 }
 
-func NewInstrumentedProjectCommandRunner(scope tally.Scope, projectCommandRunner ProjectCommandRunner) *InstrumentedProjectCommandRunner {
+func NewInstrumentedProjectCommandRunner(scope tally.Scope, projectCommandRunner ProjectCommandRunner, prScopeManager *metrics.PRScopeManager) *InstrumentedProjectCommandRunner {
 	projectTags := command.ProjectScopeTags{}
 	scope = scope.SubScope("project").Tagged(projectTags.Loadtags())
 
-	for _, m := range []string{metrics.ExecutionSuccessMetric, metrics.ExecutionErrorMetric, metrics.ExecutionFailureMetric} {
+	for _, m := range metrics.ExecutionCounterMetrics {
 		metrics.InitCounter(scope, m)
 	}
 
 	return &InstrumentedProjectCommandRunner{
 		projectCommandRunner: projectCommandRunner,
+		prScopeManager:       prScopeManager,
 		scope:                scope,
 	}
 }
 
 func (p *InstrumentedProjectCommandRunner) Plan(ctx command.ProjectContext) command.ProjectCommandOutput {
-	return RunAndEmitStats(ctx, p.projectCommandRunner.Plan, p.scope)
+	return RunAndEmitStats(ctx, p.projectCommandRunner.Plan, p.prScopeManager, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) PolicyCheck(ctx command.ProjectContext) command.ProjectCommandOutput {
-	return RunAndEmitStats(ctx, p.projectCommandRunner.PolicyCheck, p.scope)
+	return RunAndEmitStats(ctx, p.projectCommandRunner.PolicyCheck, p.prScopeManager, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) Apply(ctx command.ProjectContext) command.ProjectCommandOutput {
-	return RunAndEmitStats(ctx, p.projectCommandRunner.Apply, p.scope)
+	return RunAndEmitStats(ctx, p.projectCommandRunner.Apply, p.prScopeManager, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) ApprovePolicies(ctx command.ProjectContext) command.ProjectCommandOutput {
-	return RunAndEmitStats(ctx, p.projectCommandRunner.ApprovePolicies, p.scope)
+	return RunAndEmitStats(ctx, p.projectCommandRunner.ApprovePolicies, p.prScopeManager, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) Import(ctx command.ProjectContext) command.ProjectCommandOutput {
-	return RunAndEmitStats(ctx, p.projectCommandRunner.Import, p.scope)
+	return RunAndEmitStats(ctx, p.projectCommandRunner.Import, p.prScopeManager, p.scope)
 }
 
 func (p *InstrumentedProjectCommandRunner) StateRm(ctx command.ProjectContext) command.ProjectCommandOutput {
-	return RunAndEmitStats(ctx, p.projectCommandRunner.StateRm, p.scope)
+	return RunAndEmitStats(ctx, p.projectCommandRunner.StateRm, p.prScopeManager, p.scope)
 }
 
-func RunAndEmitStats(ctx command.ProjectContext, execute func(ctx command.ProjectContext) command.ProjectCommandOutput, scope tally.Scope) command.ProjectCommandOutput {
+func RunAndEmitStats(ctx command.ProjectContext, execute func(ctx command.ProjectContext) command.ProjectCommandOutput, prScopeManager *metrics.PRScopeManager, fallbackScope tally.Scope) command.ProjectCommandOutput {
 	commandName := ctx.CommandName.String()
 	// ensures we are differentiating between project level command and overall command
-	scope = ctx.SetProjectScopeTags(scope).SubScope(commandName)
+
+	scope := ctx.SetProjectScopeTags(prScopeManager).SubScope(commandName)
 	logger := ctx.Log
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()

server/events/project_command_context_builder.go

@@ -74,11 +74,6 @@ func (cb *CommandScopedStatsProjectCommandContextBuilder) BuildProjectContext(
 	projectCmds = []command.ProjectContext{}
 
 	for _, cmd := range cmds {
-
-		// specifically use the command name in the context instead of the arg
-		// since we can return multiple commands worth of contexts for a given command name arg
-		// to effectively pipeline them.
-		cmd.Scope = cmd.SetProjectScopeTags(cmd.Scope)
 		projectCmds = append(projectCmds, cmd)
 	}
 

server/events/pull_closed_executor.go

@@ -46,6 +46,16 @@ type PullCleaner interface {
 	CleanUpPull(logger logging.SimpleLogging, repo models.Repo, pull models.PullRequest) error
 }
 
+//go:generate pegomock generate github.com/runatlantis/atlantis/server/events --package mocks -o mocks/mock_scope_cleaner.go ScopeCleaner
+
+// ScopeCleaner tracks and cleans up metric scopes for closed PRs.
+type ScopeCleaner interface {
+	// MarkPRClosed marks a PR as closed for metric cleanup.
+	MarkPRClosed(repoFullName string, pullNum int)
+	// CleanupStaleMetrics closes scopes that have exceeded the retention period.
+	CleanupStaleMetrics() int
+}
+
 // PullClosedExecutor executes the tasks required to clean up a closed pull
 // request.
 type PullClosedExecutor struct {
@@ -56,6 +66,7 @@ type PullClosedExecutor struct {
 	PullClosedTemplate       PullCleanupTemplate
 	LogStreamResourceCleaner ResourceCleaner
 	CancellationTracker      CancellationTracker
+	ScopeCleaner             ScopeCleaner
 }
 
 type templatedProject struct {
@@ -122,6 +133,10 @@ func (p *PullClosedExecutor) CleanUpPull(logger logging.SimpleLogging, repo mode
 		p.CancellationTracker.Clear(pull)
 	}
 
+	if p.ScopeCleaner != nil {
+		p.ScopeCleaner.MarkPRClosed(repo.FullName, pull.Num)
+	}
+
 	// If there are no locks then there's no need to comment.
 	if len(locks) == 0 {
 		return nil

server/events/vcs/common/instrumented_client.go

@@ -15,13 +15,13 @@ import (
 
 type InstrumentedClient struct {
 	vcs.Client
-	StatsScope tally.Scope
-	Logger     logging.SimpleLogging
+	StatsScope     tally.Scope
+	PRScopeManager *metrics.PRScopeManager
+	Logger         logging.SimpleLogging
 }
 
 func (c *InstrumentedClient) GetModifiedFiles(logger logging.SimpleLogging, repo models.Repo, pull models.PullRequest) ([]string, error) {
-	scope := c.StatsScope.SubScope("get_modified_files")
-	scope = SetGitScopeTags(scope, repo.FullName, pull.Num)
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pull.Num).SubScope("get_modified_files")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -42,8 +42,7 @@ func (c *InstrumentedClient) GetModifiedFiles(logger logging.SimpleLogging, repo
 }
 
 func (c *InstrumentedClient) CreateComment(logger logging.SimpleLogging, repo models.Repo, pullNum int, comment string, command string) error {
-	scope := c.StatsScope.SubScope("create_comment")
-	scope = SetGitScopeTags(scope, repo.FullName, pullNum)
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pullNum).SubScope("create_comment")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -62,7 +61,7 @@ func (c *InstrumentedClient) CreateComment(logger logging.SimpleLogging, repo mo
 }
 
 func (c *InstrumentedClient) ReactToComment(logger logging.SimpleLogging, repo models.Repo, pullNum int, commentID int64, reaction string) error {
-	scope := c.StatsScope.SubScope("react_to_comment")
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pullNum).SubScope("react_to_comment")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -81,8 +80,7 @@ func (c *InstrumentedClient) ReactToComment(logger logging.SimpleLogging, repo m
 }
 
 func (c *InstrumentedClient) HidePrevCommandComments(logger logging.SimpleLogging, repo models.Repo, pullNum int, command string, dir string) error {
-	scope := c.StatsScope.SubScope("hide_prev_plan_comments")
-	scope = SetGitScopeTags(scope, repo.FullName, pullNum)
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pullNum).SubScope("hide_prev_plan_comments")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -102,8 +100,7 @@ func (c *InstrumentedClient) HidePrevCommandComments(logger logging.SimpleLoggin
 }
 
 func (c *InstrumentedClient) PullIsApproved(logger logging.SimpleLogging, repo models.Repo, pull models.PullRequest) (models.ApprovalStatus, error) {
-	scope := c.StatsScope.SubScope("pull_is_approved")
-	scope = SetGitScopeTags(scope, repo.FullName, pull.Num)
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pull.Num).SubScope("pull_is_approved")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -124,8 +121,7 @@ func (c *InstrumentedClient) PullIsApproved(logger logging.SimpleLogging, repo m
 }
 
 func (c *InstrumentedClient) PullIsMergeable(logger logging.SimpleLogging, repo models.Repo, pull models.PullRequest, vcsstatusname string, ignoreVCSStatusNames []string) (models.MergeableStatus, error) {
-	scope := c.StatsScope.SubScope("pull_is_mergeable")
-	scope = SetGitScopeTags(scope, repo.FullName, pull.Num)
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pull.Num).SubScope("pull_is_mergeable")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -152,8 +148,7 @@ func (c *InstrumentedClient) UpdateStatus(logger logging.SimpleLogging, repo mod
 		return nil
 	}
 
-	scope := c.StatsScope.SubScope("update_status")
-	scope = SetGitScopeTags(scope, repo.FullName, pull.Num)
+	scope := SetGitScopeTags(c.PRScopeManager, repo.FullName, pull.Num).SubScope("update_status")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -172,8 +167,7 @@ func (c *InstrumentedClient) UpdateStatus(logger logging.SimpleLogging, repo mod
 }
 
 func (c *InstrumentedClient) MergePull(logger logging.SimpleLogging, pull models.PullRequest, pullOptions models.PullRequestOptions) error {
-	scope := c.StatsScope.SubScope("merge_pull")
-	scope = SetGitScopeTags(scope, pull.BaseRepo.FullName, pull.Num)
+	scope := SetGitScopeTags(c.PRScopeManager, pull.BaseRepo.FullName, pull.Num).SubScope("merge_pull")
 
 	executionTime := scope.Timer(metrics.ExecutionTimeMetric).Start()
 	defer executionTime.Stop()
@@ -191,9 +185,13 @@ func (c *InstrumentedClient) MergePull(logger logging.SimpleLogging, pull models
 	return nil
 }
 
-func SetGitScopeTags(scope tally.Scope, repoFullName string, pullNum int) tally.Scope {
-	return scope.Tagged(map[string]string{
+// SetGitScopeTags sets git-level tags (repo and PR) on a scope using the PR scope manager.
+// Creates a closeable PR-specific root scope with git-level tags.
+func SetGitScopeTags(prScopeManager *metrics.PRScopeManager, repoFullName string, pullNum int) tally.Scope {
+	tags := map[string]string{
 		"base_repo": repoFullName,
 		"pr_number": strconv.Itoa(pullNum),
-	})
+	}
+
+	return prScopeManager.GetOrCreatePRScope(repoFullName, pullNum, tags)
 }