From 775b07371e2f2d3d602130cfe9fb6c461e08d3d0 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Wed, 23 Apr 2025 06:10:17 -0700 Subject: [PATCH 1/4] Removing EnableNativeHistograms from TSDB config and replacing with EnableNativeHistogramPerUser limit Signed-off-by: Paurush Garg --- pkg/ingester/ingester.go | 10 ++++----- pkg/ingester/ingester_test.go | 41 +++++++++++++++++++++-------------- pkg/storage/tsdb/config.go | 4 ---- pkg/util/validation/limits.go | 17 ++++++++++----- 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index e4eebef9911..e502a00afba 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1347,7 +1347,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte return nil, wrapWithUser(err, userID) } - if i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms { + if i.limits.EnableNativeHistogramPerUser(userID) { for _, hp := range ts.Histograms { var ( err error @@ -1494,7 +1494,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID).Add(float64(perLabelSetSeriesLimitCount)) } - if !i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms && discardedNativeHistogramCount > 0 { + if !i.limits.EnableNativeHistogramPerUser(userID) && discardedNativeHistogramCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID).Add(float64(discardedNativeHistogramCount)) } @@ -2451,9 +2451,9 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { EnableMemorySnapshotOnShutdown: i.cfg.BlocksStorageConfig.TSDB.MemorySnapshotOnShutdown, OutOfOrderTimeWindow: time.Duration(oooTimeWindow).Milliseconds(), OutOfOrderCapMax: i.cfg.BlocksStorageConfig.TSDB.OutOfOrderCapMax, - EnableOOONativeHistograms: i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, // Automatically enabled when EnableNativeHistograms is true. - EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. - EnableNativeHistograms: i.cfg.BlocksStorageConfig.TSDB.EnableNativeHistograms, + EnableOOONativeHistograms: true, + EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. + EnableNativeHistograms: true, // Always enable Native Histograms BlockChunkQuerierFunc: i.blockChunkQuerierFunc(userID), }, nil) if err != nil { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index dda43c4ba97..2e1f78b1dff 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -124,6 +124,7 @@ func seriesSetFromResponseStream(s *mockQueryStreamServer) (storage.SeriesSet, e func TestMatcherCache(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = true userID := "1" tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits}) registry := prometheus.NewRegistry() @@ -135,7 +136,7 @@ func TestMatcherCache(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) cfg := defaultIngesterTestConfig(t) cfg.MatchersCacheMaxItems = 50 - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) @@ -204,7 +205,7 @@ func TestIngesterDeletionRace(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry, false) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck @@ -254,6 +255,7 @@ func TestIngesterDeletionRace(t *testing.T) { func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = true userID := "1" registry := prometheus.NewRegistry() @@ -287,7 +289,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -630,7 +632,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { // Should persist between restarts services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck registry = prometheus.NewRegistry() - ing, err = prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry, true) + ing, err = prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, tenantLimits, blocksDir, registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) ing.updateActiveSeries(ctx) @@ -661,6 +663,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { func TestPushRace(t *testing.T) { cfg := defaultIngesterTestConfig(t) l := defaultLimitsTestConfig() + l.EnableNativeHistogramPerUser = true cfg.LabelsStringInterningEnabled = true cfg.LifecyclerConfig.JoinAfter = 0 @@ -686,7 +689,7 @@ func TestPushRace(t *testing.T) { blocksDir := filepath.Join(dir, "blocks") require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, l, nil, blocksDir, prometheus.NewRegistry(), true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, l, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) @@ -747,6 +750,7 @@ func TestPushRace(t *testing.T) { func TestIngesterUserLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = true limits.MaxLocalSeriesPerUser = 1 limits.MaxLocalMetricsWithMetadataPerUser = 1 @@ -778,7 +782,7 @@ func TestIngesterUserLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -878,6 +882,7 @@ func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexp func TestIngesterMetricLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = true limits.MaxLocalSeriesPerMetric = 1 limits.MaxLocalMetadataPerMetric = 1 @@ -909,7 +914,7 @@ func TestIngesterMetricLimitExceeded(t *testing.T) { require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) blocksIngesterGenerator := func(reg prometheus.Registerer) *Ingester { - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg, true) + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, defaultIngesterTestConfig(t), limits, nil, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) // Wait until it's ACTIVE @@ -1933,6 +1938,7 @@ func TestIngester_Push(t *testing.T) { cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = !testData.disableNativeHistogram limits.MaxExemplars = testData.maxExemplars limits.OutOfOrderTimeWindow = model.Duration(testData.oooTimeWindow) limits.LimitsPerLabelSet = []validation.LimitsPerLabelSet{ @@ -1945,7 +1951,7 @@ func TestIngester_Push(t *testing.T) { Hash: 1, }, } - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry, !testData.disableNativeHistogram) + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -2174,7 +2180,8 @@ func TestIngester_PushNativeHistogramErrors(t *testing.T) { cfg.LifecyclerConfig.JoinAfter = 0 limits := defaultLimitsTestConfig() - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry, true) + limits.EnableNativeHistogramPerUser = true + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -2662,6 +2669,7 @@ func Benchmark_Ingester_PushOnError(b *testing.B) { cfg.LifecyclerConfig.JoinAfter = 0 limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = true if !testData.prepareConfig(&limits, instanceLimits) { b.SkipNow() } @@ -2670,7 +2678,7 @@ func Benchmark_Ingester_PushOnError(b *testing.B) { return instanceLimits } - ingester, err := prepareIngesterWithBlocksStorageAndLimits(b, cfg, limits, nil, "", registry, true) + ingester, err := prepareIngesterWithBlocksStorageAndLimits(b, cfg, limits, nil, "", registry) require.NoError(b, err) require.NoError(b, services.StartAndAwaitRunning(context.Background(), ingester)) defer services.StopAndAwaitTerminated(context.Background(), ingester) //nolint:errcheck @@ -3947,10 +3955,10 @@ func mockHistogramWriteRequest(t *testing.T, lbls labels.Labels, value int64, ti } func prepareIngesterWithBlocksStorage(t testing.TB, ingesterCfg Config, registerer prometheus.Registerer) (*Ingester, error) { - return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, defaultLimitsTestConfig(), nil, "", registerer, true) + return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, defaultLimitsTestConfig(), nil, "", registerer) } -func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, tenantLimits validation.TenantLimits, dataDir string, registerer prometheus.Registerer, nativeHistograms bool) (*Ingester, error) { +func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, tenantLimits validation.TenantLimits, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { // Create a data dir if none has been provided. if dataDir == "" { dataDir = t.TempDir() @@ -3966,7 +3974,6 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, ingesterCfg.BlocksStorageConfig.TSDB.Dir = dataDir ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir - ingesterCfg.BlocksStorageConfig.TSDB.EnableNativeHistograms = nativeHistograms ingester, err := New(ingesterCfg, overrides, registerer, log.NewNopLogger(), nil) if err != nil { @@ -6432,7 +6439,8 @@ func TestIngester_MaxExemplarsFallBack(t *testing.T) { dir := t.TempDir() blocksDir := filepath.Join(dir, "blocks") limits := defaultLimitsTestConfig() - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry(), true) + limits.EnableNativeHistogramPerUser = true + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) maxExemplars := i.getMaxExemplars("someTenant") @@ -6440,7 +6448,7 @@ func TestIngester_MaxExemplarsFallBack(t *testing.T) { // set max exemplars value in limits, and re-initialize the ingester limits.MaxExemplars = 5 - i, err = prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry(), true) + i, err = prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) // validate this value is picked up now @@ -6815,6 +6823,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{2 * time.Hour} reg := prometheus.NewRegistry() limits := defaultLimitsTestConfig() + limits.EnableNativeHistogramPerUser = true userID := "1" ctx := user.InjectOrgID(context.Background(), userID) @@ -6839,7 +6848,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { require.NoError(t, os.Mkdir(chunksDir, os.ModePerm)) require.NoError(t, os.Mkdir(blocksDir, os.ModePerm)) - i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, reg, false) + i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, tenantLimits, blocksDir, reg) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index 2a2e16c58d8..13daaff3f4c 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -172,9 +172,6 @@ type TSDBConfig struct { // OutOfOrderCapMax is maximum capacity for OOO chunks (in samples). OutOfOrderCapMax int64 `yaml:"out_of_order_cap_max"` - // Enable native histogram ingestion. - EnableNativeHistograms bool `yaml:"enable_native_histograms"` - // Posting Cache Configuration for TSDB PostingsCache TSDBPostingsCacheConfig `yaml:"expanded_postings_cache" doc:"description=[EXPERIMENTAL] If enabled, ingesters will cache expanded postings when querying blocks. Caching can be configured separately for the head and compacted blocks."` } @@ -204,7 +201,6 @@ func (cfg *TSDBConfig) RegisterFlags(f *flag.FlagSet) { f.IntVar(&cfg.MaxExemplars, "blocks-storage.tsdb.max-exemplars", 0, "Deprecated, use maxExemplars in limits instead. If the MaxExemplars value in limits is set to zero, cortex will fallback on this value. This setting enables support for exemplars in TSDB and sets the maximum number that will be stored. 0 or less means disabled.") f.BoolVar(&cfg.MemorySnapshotOnShutdown, "blocks-storage.tsdb.memory-snapshot-on-shutdown", false, "True to enable snapshotting of in-memory TSDB data on disk when shutting down.") f.Int64Var(&cfg.OutOfOrderCapMax, "blocks-storage.tsdb.out-of-order-cap-max", tsdb.DefaultOutOfOrderCapMax, "[EXPERIMENTAL] Configures the maximum number of samples per chunk that can be out-of-order.") - f.BoolVar(&cfg.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") flagext.DeprecatedFlag(f, "blocks-storage.tsdb.wal-compression-enabled", "Deprecated (use blocks-storage.tsdb.wal-compression-type instead): True to enable TSDB WAL compression.", util_log.Logger) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 84596f17950..bd9f4e4e3c4 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -144,11 +144,12 @@ type Limits struct { // Ingester enforced limits. // Series - MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` - MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` - MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` - MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` - LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` + MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` + MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` + MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` + MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` + LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` + EnableNativeHistogramPerUser bool `yaml:"enable_native_histogram_per_user" json:"enable_native_histogram_per_user"` // Metadata MaxLocalMetricsWithMetadataPerUser int `yaml:"max_metadata_per_user" json:"max_metadata_per_user"` @@ -257,6 +258,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") + f.BoolVar(&l.EnableNativeHistogramPerUser, "ingester.enable_native_histogram_per_user", false, "Flag to enable NativeHistograms per user.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") @@ -659,6 +661,11 @@ func (o *Overrides) MaxGlobalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxGlobalSeriesPerUser } +// EnableNativeHistogramPerUser returns whether the Ingester should accept NativeHistograms samples from this user. +func (o *Overrides) EnableNativeHistogramPerUser(userID string) bool { + return o.GetOverridesForUser(userID).EnableNativeHistogramPerUser +} + // OutOfOrderTimeWindow returns the allowed time window for ingestion of out-of-order samples. func (o *Overrides) OutOfOrderTimeWindow(userID string) model.Duration { return o.GetOverridesForUser(userID).OutOfOrderTimeWindow From c6b2e6a5cf22e9b0e15408cd29d8d725bb289204 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Thu, 24 Apr 2025 05:03:25 -0700 Subject: [PATCH 2/4] Adding experimental flag Signed-off-by: Paurush Garg --- pkg/ingester/ingester.go | 2 +- pkg/util/validation/limits.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index e502a00afba..26bd9567693 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -2453,7 +2453,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { OutOfOrderCapMax: i.cfg.BlocksStorageConfig.TSDB.OutOfOrderCapMax, EnableOOONativeHistograms: true, EnableOverlappingCompaction: false, // Always let compactors handle overlapped blocks, e.g. OOO blocks. - EnableNativeHistograms: true, // Always enable Native Histograms + EnableNativeHistograms: true, // Always enable Native Histograms. Gate keeping is done though a per-tenant limit at ingestion. BlockChunkQuerierFunc: i.blockChunkQuerierFunc(userID), }, nil) if err != nil { diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index bd9f4e4e3c4..b517ebc7249 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -258,7 +258,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") - f.BoolVar(&l.EnableNativeHistogramPerUser, "ingester.enable_native_histogram_per_user", false, "Flag to enable NativeHistograms per user.") + f.BoolVar(&l.EnableNativeHistogramPerUser, "ingester.enable_native_histogram_per_user", false, "[Experimental] True to enable native histogram per user.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") From f5f203e48efa864bc05e11168b5727d6beaa5700 Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Thu, 24 Apr 2025 06:38:53 -0700 Subject: [PATCH 3/4] Adding changelog entry and make doc Signed-off-by: Paurush Garg --- CHANGELOG.md | 1 + docs/blocks-storage/querier.md | 4 ---- docs/blocks-storage/store-gateway.md | 4 ---- docs/configuration/config-file-reference.md | 8 ++++---- 4 files changed, 5 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5432fc045dd..968835cdda4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new EnableNativeHistogramPerUser limit. #6718 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 5d09d1169c2..6177adea64f 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -1561,10 +1561,6 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: | default = 32] - # [EXPERIMENTAL] True to enable native histogram. - # CLI flag: -blocks-storage.tsdb.enable-native-histograms - [enable_native_histograms: | default = false] - # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 58b4c08eb3b..5053ea86ee7 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -1679,10 +1679,6 @@ blocks_storage: # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: | default = 32] - # [EXPERIMENTAL] True to enable native histogram. - # CLI flag: -blocks-storage.tsdb.enable-native-histograms - [enable_native_histograms: | default = false] - # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 252887a15e5..44b8f946b47 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -2130,10 +2130,6 @@ tsdb: # CLI flag: -blocks-storage.tsdb.out-of-order-cap-max [out_of_order_cap_max: | default = 32] - # [EXPERIMENTAL] True to enable native histogram. - # CLI flag: -blocks-storage.tsdb.enable-native-histograms - [enable_native_histograms: | default = false] - # [EXPERIMENTAL] If enabled, ingesters will cache expanded postings when # querying blocks. Caching can be configured separately for the head and # compacted blocks. @@ -3516,6 +3512,10 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # [max_series] [limits_per_label_set: | default = []] +# [Experimental] True to enable native histogram per user. +# CLI flag: -ingester.enable_native_histogram_per_user +[enable_native_histogram_per_user: | default = false] + # The maximum number of active metrics with metadata per user, per ingester. 0 # to disable. # CLI flag: -ingester.max-metadata-per-user From a022a966a5992dbfc9d0f400e44d1e4844b5a14e Mon Sep 17 00:00:00 2001 From: Paurush Garg Date: Fri, 25 Apr 2025 06:58:42 -0700 Subject: [PATCH 4/4] Updating to keep the EnableNativeHistograms limit name same as existing config name Signed-off-by: Paurush Garg --- CHANGELOG.md | 2 +- docs/configuration/config-file-reference.md | 6 +++--- pkg/ingester/ingester.go | 4 ++-- pkg/ingester/ingester_test.go | 24 +++++++++++---------- pkg/util/validation/limits.go | 20 ++++++++--------- 5 files changed, 29 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 968835cdda4..bd8d71ef895 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog ## master / unreleased -* [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new EnableNativeHistogramPerUser limit. #6718 +* [CHANGE] Ingester: Remove EnableNativeHistograms config flag and instead gate keep through new per-tenant limit at ingestion. #6718 * [CHANGE] StoreGateway/Alertmanager: Add default 5s connection timeout on client. #6603 * [FEATURE] Query Frontend: Add dynamic interval size for query splitting. This is enabled by configuring experimental flags `querier.max-shards-per-query` and/or `querier.max-fetched-data-duration-per-query`. The split interval size is dynamically increased to maintain a number of shards and total duration fetched below the configured values. #6458 * [FEATURE] Querier/Ruler: Add `query_partial_data` and `rules_partial_data` limits to allow queries/rules to be evaluated with data from a single zone, if other zones are not available. #6526 diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 44b8f946b47..a64b106ef71 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -3512,9 +3512,9 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s # [max_series] [limits_per_label_set: | default = []] -# [Experimental] True to enable native histogram per user. -# CLI flag: -ingester.enable_native_histogram_per_user -[enable_native_histogram_per_user: | default = false] +# [EXPERIMENTAL] True to enable native histogram. +# CLI flag: -blocks-storage.tsdb.enable-native-histograms +[enable_native_histograms: | default = false] # The maximum number of active metrics with metadata per user, per ingester. 0 # to disable. diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 26bd9567693..0dfe0219265 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1347,7 +1347,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte return nil, wrapWithUser(err, userID) } - if i.limits.EnableNativeHistogramPerUser(userID) { + if i.limits.EnableNativeHistograms(userID) { for _, hp := range ts.Histograms { var ( err error @@ -1494,7 +1494,7 @@ func (i *Ingester) Push(ctx context.Context, req *cortexpb.WriteRequest) (*corte i.validateMetrics.DiscardedSamples.WithLabelValues(perLabelsetSeriesLimit, userID).Add(float64(perLabelSetSeriesLimitCount)) } - if !i.limits.EnableNativeHistogramPerUser(userID) && discardedNativeHistogramCount > 0 { + if !i.limits.EnableNativeHistograms(userID) && discardedNativeHistogramCount > 0 { i.validateMetrics.DiscardedSamples.WithLabelValues(nativeHistogramSample, userID).Add(float64(discardedNativeHistogramCount)) } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 2e1f78b1dff..0da6a49161c 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -124,7 +124,7 @@ func seriesSetFromResponseStream(s *mockQueryStreamServer) (storage.SeriesSet, e func TestMatcherCache(t *testing.T) { limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true userID := "1" tenantLimits := newMockTenantLimits(map[string]*validation.Limits{userID: &limits}) registry := prometheus.NewRegistry() @@ -255,7 +255,7 @@ func TestIngesterDeletionRace(t *testing.T) { func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true userID := "1" registry := prometheus.NewRegistry() @@ -663,7 +663,7 @@ func TestIngesterPerLabelsetLimitExceeded(t *testing.T) { func TestPushRace(t *testing.T) { cfg := defaultIngesterTestConfig(t) l := defaultLimitsTestConfig() - l.EnableNativeHistogramPerUser = true + l.EnableNativeHistograms = true cfg.LabelsStringInterningEnabled = true cfg.LifecyclerConfig.JoinAfter = 0 @@ -750,7 +750,7 @@ func TestPushRace(t *testing.T) { func TestIngesterUserLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true limits.MaxLocalSeriesPerUser = 1 limits.MaxLocalMetricsWithMetadataPerUser = 1 @@ -882,7 +882,7 @@ func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []cortexp func TestIngesterMetricLimitExceeded(t *testing.T) { limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true limits.MaxLocalSeriesPerMetric = 1 limits.MaxLocalMetadataPerMetric = 1 @@ -1938,7 +1938,7 @@ func TestIngester_Push(t *testing.T) { cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = !testData.disableNativeHistogram + limits.EnableNativeHistograms = !testData.disableNativeHistogram limits.MaxExemplars = testData.maxExemplars limits.OutOfOrderTimeWindow = model.Duration(testData.oooTimeWindow) limits.LimitsPerLabelSet = []validation.LimitsPerLabelSet{ @@ -2180,7 +2180,7 @@ func TestIngester_PushNativeHistogramErrors(t *testing.T) { cfg.LifecyclerConfig.JoinAfter = 0 limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, "", registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) @@ -2669,7 +2669,7 @@ func Benchmark_Ingester_PushOnError(b *testing.B) { cfg.LifecyclerConfig.JoinAfter = 0 limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true if !testData.prepareConfig(&limits, instanceLimits) { b.SkipNow() } @@ -3955,7 +3955,9 @@ func mockHistogramWriteRequest(t *testing.T, lbls labels.Labels, value int64, ti } func prepareIngesterWithBlocksStorage(t testing.TB, ingesterCfg Config, registerer prometheus.Registerer) (*Ingester, error) { - return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, defaultLimitsTestConfig(), nil, "", registerer) + limits := defaultLimitsTestConfig() + limits.EnableNativeHistograms = true + return prepareIngesterWithBlocksStorageAndLimits(t, ingesterCfg, limits, nil, "", registerer) } func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, tenantLimits validation.TenantLimits, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { @@ -6439,7 +6441,7 @@ func TestIngester_MaxExemplarsFallBack(t *testing.T) { dir := t.TempDir() blocksDir := filepath.Join(dir, "blocks") limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true i, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, limits, nil, blocksDir, prometheus.NewRegistry()) require.NoError(t, err) @@ -6823,7 +6825,7 @@ func TestIngester_UpdateLabelSetMetrics(t *testing.T) { cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{2 * time.Hour} reg := prometheus.NewRegistry() limits := defaultLimitsTestConfig() - limits.EnableNativeHistogramPerUser = true + limits.EnableNativeHistograms = true userID := "1" ctx := user.InjectOrgID(context.Background(), userID) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index b517ebc7249..03302b4c2b0 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -144,12 +144,12 @@ type Limits struct { // Ingester enforced limits. // Series - MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` - MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` - MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` - MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` - LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` - EnableNativeHistogramPerUser bool `yaml:"enable_native_histogram_per_user" json:"enable_native_histogram_per_user"` + MaxLocalSeriesPerUser int `yaml:"max_series_per_user" json:"max_series_per_user"` + MaxLocalSeriesPerMetric int `yaml:"max_series_per_metric" json:"max_series_per_metric"` + MaxGlobalSeriesPerUser int `yaml:"max_global_series_per_user" json:"max_global_series_per_user"` + MaxGlobalSeriesPerMetric int `yaml:"max_global_series_per_metric" json:"max_global_series_per_metric"` + LimitsPerLabelSet []LimitsPerLabelSet `yaml:"limits_per_label_set" json:"limits_per_label_set" doc:"nocli|description=[Experimental] Enable limits per LabelSet. Supported limits per labelSet: [max_series]"` + EnableNativeHistograms bool `yaml:"enable_native_histograms" json:"enable_native_histograms"` // Metadata MaxLocalMetricsWithMetadataPerUser int `yaml:"max_metadata_per_user" json:"max_metadata_per_user"` @@ -258,7 +258,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxLocalSeriesPerMetric, "ingester.max-series-per-metric", 50000, "The maximum number of active series per metric name, per ingester. 0 to disable.") f.IntVar(&l.MaxGlobalSeriesPerUser, "ingester.max-global-series-per-user", 0, "The maximum number of active series per user, across the cluster before replication. 0 to disable. Supported only if -distributor.shard-by-all-labels is true.") f.IntVar(&l.MaxGlobalSeriesPerMetric, "ingester.max-global-series-per-metric", 0, "The maximum number of active series per metric name, across the cluster before replication. 0 to disable.") - f.BoolVar(&l.EnableNativeHistogramPerUser, "ingester.enable_native_histogram_per_user", false, "[Experimental] True to enable native histogram per user.") + f.BoolVar(&l.EnableNativeHistograms, "blocks-storage.tsdb.enable-native-histograms", false, "[EXPERIMENTAL] True to enable native histogram.") f.IntVar(&l.MaxExemplars, "ingester.max-exemplars", 0, "Enables support for exemplars in TSDB and sets the maximum number that will be stored. less than zero means disabled. If the value is set to zero, cortex will fallback to blocks-storage.tsdb.max-exemplars value.") f.Var(&l.OutOfOrderTimeWindow, "ingester.out-of-order-time-window", "[Experimental] Configures the allowed time window for ingestion of out-of-order samples. Disabled (0s) by default.") @@ -661,9 +661,9 @@ func (o *Overrides) MaxGlobalSeriesPerUser(userID string) int { return o.GetOverridesForUser(userID).MaxGlobalSeriesPerUser } -// EnableNativeHistogramPerUser returns whether the Ingester should accept NativeHistograms samples from this user. -func (o *Overrides) EnableNativeHistogramPerUser(userID string) bool { - return o.GetOverridesForUser(userID).EnableNativeHistogramPerUser +// EnableNativeHistograms returns whether the Ingester should accept NativeHistograms samples from this user. +func (o *Overrides) EnableNativeHistograms(userID string) bool { + return o.GetOverridesForUser(userID).EnableNativeHistograms } // OutOfOrderTimeWindow returns the allowed time window for ingestion of out-of-order samples.