Add rule information to query paramters sent to the QFE

Signed-off-by: SungJin1212 <[email protected]>

Author: SungJin1212

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## master / unreleased
 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458
 * [ENHANCEMENT] Add `compactor.auto-forget-delay` for compactor to auto forget compactors after X minutes without heartbeat. #6533
+* [ENHANCEMENT] Ruler: Add rule information (group name, namespace, name, and kind) to query parameters sent to the Query Frontend to leave rule information logs on query stats. #6539
 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526
 * [ENHANCEMENT] StoreGateway: Emit more histogram buckets on the `cortex_querier_storegateway_refetches_per_query` metric. #6570
 * [ENHANCEMENT] Querier: Apply bytes limiter to LabelNames and LabelValuesForLabelNames. #6568

docs/configuration/config-file-reference.md

@@ -4812,7 +4812,9 @@ ring:
 [disabled_tenants: <string> | default = ""]
 
 # Report query statistics for ruler queries to complete as a per user metric and
-# as an info level log message.
+# as an info level log message. It works only when the -ruler.frontend-address
+# is not configured. When -ruler.frontend-address enabled, the Query Frontend
+# tracks query statistics logs and metrics.
 # CLI flag: -ruler.query-stats-enabled
 [query_stats_enabled: <boolean> | default = false]
 

pkg/ruler/compat.go

@@ -177,7 +177,20 @@ func EngineQueryFunc(engine promql.QueryEngine, frontendClient *frontendClient,
 		}
 
 		if frontendClient != nil {
-			v, err := frontendClient.InstantQuery(ctx, qs, t)
+			// query parameters sent to the Query Frontend to leave rule information logs on query stats
+			queryParams := map[string]string{}
+
+			if origin := ctx.Value(promql.QueryOrigin{}); origin != nil {
+				queryLabels := origin.(map[string]interface{})
+				rgMap := queryLabels["ruleGroup"].(map[string]string)
+				queryParams["rule_group"] = rgMap["name"]
+				queryParams["rule_namespace"] = rgMap["file"]
+			}
+			ruleDetail := rules.FromOriginContext(ctx)
+			queryParams["rule"] = ruleDetail.Name
+			queryParams["rule_kind"] = ruleDetail.Kind
+
+			v, err := frontendClient.InstantQuery(ctx, qs, t, queryParams)
 			if err != nil {
 				return nil, err
 			}
@@ -326,7 +339,8 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi
 		totalWrites := evalMetrics.TotalWritesVec.WithLabelValues(userID)
 		failedWrites := evalMetrics.FailedWritesVec.WithLabelValues(userID)
 
-		if cfg.FrontendAddress != "" {
+		shouldEvalFromQFE := cfg.FrontendAddress != ""
+		if shouldEvalFromQFE {
 			c, err := frontendPool.GetClientFor(cfg.FrontendAddress)
 			if err != nil {
 				return nil, err
@@ -336,7 +350,7 @@ func DefaultTenantManagerFactory(cfg Config, p Pusher, q storage.Queryable, engi
 		var queryFunc rules.QueryFunc
 		engineQueryFunc := EngineQueryFunc(engine, client, q, overrides, userID, cfg.LookbackDelta)
 		metricsQueryFunc := MetricsQueryFunc(engineQueryFunc, totalQueries, failedQueries)
-		if cfg.EnableQueryStats {
+		if cfg.EnableQueryStats && !shouldEvalFromQFE {
 			queryFunc = RecordAndReportRuleQueryMetrics(metricsQueryFunc, userID, evalMetrics, logger)
 		} else {
 			queryFunc = metricsQueryFunc

pkg/ruler/frontend_client.go

@@ -49,12 +49,17 @@ func NewFrontendClient(client httpgrpc.HTTPClient, timeout time.Duration, promet
 	}
 }
 
-func (p *FrontendClient) makeRequest(ctx context.Context, qs string, ts time.Time) (*httpgrpc.HTTPRequest, error) {
+func (p *FrontendClient) makeRequest(ctx context.Context, qs string, ts time.Time, queryParams map[string]string) (*httpgrpc.HTTPRequest, error) {
 	args := make(url.Values)
 	args.Set("query", qs)
 	if !ts.IsZero() {
 		args.Set("time", ts.Format(time.RFC3339Nano))
 	}
+	// set query parameters sent to the Query Frontend to leave rule information logs on query stats
+	for k, v := range queryParams {
+		args.Set(k, v)
+	}
+
 	body := []byte(args.Encode())
 
 	//lint:ignore faillint wrapper around upstream method
@@ -87,11 +92,11 @@ func (p *FrontendClient) makeRequest(ctx context.Context, qs string, ts time.Tim
 	return req, nil
 }
 
-func (p *FrontendClient) InstantQuery(ctx context.Context, qs string, t time.Time) (promql.Vector, error) {
+func (p *FrontendClient) InstantQuery(ctx context.Context, qs string, t time.Time, queryParams map[string]string) (promql.Vector, error) {
 	log, ctx := spanlogger.New(ctx, "FrontendClient.InstantQuery")
 	defer log.Span.Finish()
 
-	req, err := p.makeRequest(ctx, qs, t)
+	req, err := p.makeRequest(ctx, qs, t, queryParams)
 	if err != nil {
 		level.Error(log).Log("err", err, "query", qs)
 		return nil, err

pkg/ruler/frontend_client_test.go

@@ -32,7 +32,7 @@ func TestTimeout(t *testing.T) {
 	ctx := context.Background()
 	ctx = user.InjectOrgID(ctx, "userID")
 	frontendClient := NewFrontendClient(mockHTTPGRPCClient(mockClientFn), time.Second*5, "/prometheus", "json")
-	_, err := frontendClient.InstantQuery(ctx, "query", time.Now())
+	_, err := frontendClient.InstantQuery(ctx, "query", time.Now(), nil)
 	require.Equal(t, context.DeadlineExceeded, err)
 }
 
@@ -41,7 +41,7 @@ func TestNoOrgId(t *testing.T) {
 		return nil, nil
 	}
 	frontendClient := NewFrontendClient(mockHTTPGRPCClient(mockClientFn), time.Second*5, "/prometheus", "json")
-	_, err := frontendClient.InstantQuery(context.Background(), "query", time.Now())
+	_, err := frontendClient.InstantQuery(context.Background(), "query", time.Now(), nil)
 	require.Equal(t, user.ErrNoOrgID, err)
 }
 
@@ -152,7 +152,7 @@ func TestInstantQueryJsonCodec(t *testing.T) {
 			ctx := context.Background()
 			ctx = user.InjectOrgID(ctx, "userID")
 			frontendClient := NewFrontendClient(mockHTTPGRPCClient(mockClientFn), time.Second*5, "/prometheus", "json")
-			vector, err := frontendClient.InstantQuery(ctx, "query", time.Now())
+			vector, err := frontendClient.InstantQuery(ctx, "query", time.Now(), nil)
 			require.Equal(t, test.expected, vector)
 			require.Equal(t, test.expectedErr, err)
 		})
@@ -301,7 +301,7 @@ func TestInstantQueryProtoCodec(t *testing.T) {
 			ctx := context.Background()
 			ctx = user.InjectOrgID(ctx, "userID")
 			frontendClient := NewFrontendClient(mockHTTPGRPCClient(mockClientFn), time.Second*5, "/prometheus", "protobuf")
-			vector, err := frontendClient.InstantQuery(ctx, "query", time.Now())
+			vector, err := frontendClient.InstantQuery(ctx, "query", time.Now(), nil)
 			require.Equal(t, test.expected, vector)
 			require.Equal(t, test.expectedErr, err)
 		})

pkg/ruler/manager_metrics.go

@@ -255,7 +255,7 @@ func NewRuleEvalMetrics(cfg Config, reg prometheus.Registerer) *RuleEvalMetrics
 			Help: "Number of failed queries by ruler.",
 		}, []string{"user"}),
 	}
-	if cfg.EnableQueryStats {
+	if cfg.EnableQueryStats && cfg.FrontendAddress == "" {
 		m.RulerQuerySeconds = promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Name: "cortex_ruler_query_seconds_total",
 			Help: "Total amount of wall clock time spent processing queries by the ruler.",

pkg/ruler/ruler.go

@@ -249,7 +249,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	f.Var(&cfg.EnabledTenants, "ruler.enabled-tenants", "Comma separated list of tenants whose rules this ruler can evaluate. If specified, only these tenants will be handled by ruler, otherwise this ruler can process rules from all tenants. Subject to sharding.")
 	f.Var(&cfg.DisabledTenants, "ruler.disabled-tenants", "Comma separated list of tenants whose rules this ruler cannot evaluate. If specified, a ruler that would normally pick the specified tenant(s) for processing will ignore them instead. Subject to sharding.")
 
-	f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report query statistics for ruler queries to complete as a per user metric and as an info level log message.")
+	f.BoolVar(&cfg.EnableQueryStats, "ruler.query-stats-enabled", false, "Report query statistics for ruler queries to complete as a per user metric and as an info level log message. It works only when the -ruler.frontend-address is not configured. When -ruler.frontend-address enabled, the Query Frontend tracks query statistics logs and metrics.")
 	f.BoolVar(&cfg.DisableRuleGroupLabel, "ruler.disable-rule-group-label", false, "Disable the rule_group label on exported metrics")
 
 	f.BoolVar(&cfg.EnableHAEvaluation, "ruler.enable-ha-evaluation", false, "Enable high availability")