perf(ingester): invalidate head postings cache per-label

CHANGELOG.md

@@ -37,6 +37,7 @@
 * [ENHANCEMENT] Querier: Detach series label and chunk data from gRPC unmarshal buffers in store-gateway streaming path, allowing the Go GC to reclaim receive buffers. #7519
 * [ENHANCEMENT] Distributor: Added `cortex_distributor_received_histogram_buckets` metric to track number of buckets in received native histogram samples before validation, per user. #7569
 * [ENHANCEMENT] Ingester: Add lazy regex evaluation on head postings cache miss. Defers expensive regex matchers on high-cardinality labels to per-series filtering when a selective equality matcher already narrows the result set. Configured via `-blocks-storage.expanded_postings_cache.head.lazy-matcher-max-cardinality` (disabled by default). #7553
+* [ENHANCEMENT] Ingester: Improve head expanded postings cache hit rate by invalidating only the cache entries whose matched labels changed when a series is created or deleted, instead of invalidating all entries for the metric name. #7678
 * [ENHANCEMENT] Store Gateway: Resolve the parquet shard count from the bucket index instead of reading the converter mark for each block, reducing object storage calls when the bucket index is enabled. A `component` label is added to the bucket index loader metrics to distinguish store-queryable and store-gateway. #7648
 * [ENHANCEMENT] Query Frontend: Improve the slow query log with `source`, `user_agent`, `engine_type`, `block_store_type`, and query stats fields to aid slow query diagnosis. #7601
 * [ENHANCEMENT] Ring: Add ring metric to count number of duplicate tokens. #7626

docs/blocks-storage/querier.md

@@ -2274,6 +2274,12 @@ blocks_storage:
       # CLI flag: -blocks-storage.expanded_postings_cache.head.lazy-matcher-complex-cost-ratio
       [lazy_matcher_complex_cost_ratio: <int> | default = 2]
 
+      # The number of counts stored in the label counter used for head cache
+      # invalidation. Note one label counter is shared by all tenants. 0 sets to
+      # the default of 4_000_000.
+      # CLI flag: -blocks-storage.expanded_postings_cache.head.label-counter-size
+      [label_counter_size: <int> | default = 4000000]
+
   users_scanner:
     # Strategy to use to scan users. Supported values are: list, user_index.
     # CLI flag: -blocks-storage.users-scanner.strategy

docs/blocks-storage/store-gateway.md

@@ -2332,6 +2332,12 @@ blocks_storage:
       # CLI flag: -blocks-storage.expanded_postings_cache.head.lazy-matcher-complex-cost-ratio
       [lazy_matcher_complex_cost_ratio: <int> | default = 2]
 
+      # The number of counts stored in the label counter used for head cache
+      # invalidation. Note one label counter is shared by all tenants. 0 sets to
+      # the default of 4_000_000.
+      # CLI flag: -blocks-storage.expanded_postings_cache.head.label-counter-size
+      [label_counter_size: <int> | default = 4000000]
+
   users_scanner:
     # Strategy to use to scan users. Supported values are: list, user_index.
     # CLI flag: -blocks-storage.users-scanner.strategy

docs/configuration/config-file-reference.md

@@ -2923,6 +2923,12 @@ tsdb:
     # CLI flag: -blocks-storage.expanded_postings_cache.head.lazy-matcher-complex-cost-ratio
     [lazy_matcher_complex_cost_ratio: <int> | default = 2]
 
+    # The number of counts stored in the label counter used for head cache
+    # invalidation. Note one label counter is shared by all tenants. 0 sets to
+    # the default of 4_000_000.
+    # CLI flag: -blocks-storage.expanded_postings_cache.head.label-counter-size
+    [label_counter_size: <int> | default = 4000000]
+
 users_scanner:
   # Strategy to use to scan users. Supported values are: list, user_index.
   # CLI flag: -blocks-storage.users-scanner.strategy

pkg/ingester/ingester_test.go

@@ -6670,7 +6670,7 @@ func TestExpendedPostingsCacheIsolation(t *testing.T) {
 	cfg.BlocksStorageConfig.TSDB.BlockRanges = []time.Duration{2 * time.Hour}
 	cfg.LifecyclerConfig.JoinAfter = 0
 	cfg.BlocksStorageConfig.TSDB.PostingsCache = cortex_tsdb.TSDBPostingsCacheConfig{
-		SeedSize: 3, // lets make sure all metric names collide
+		LabelCounterSize: 3, // lets make sure all metric names collide
 		Head: cortex_tsdb.PostingsCacheConfig{
 			Enabled:  true,
 			Ttl:      time.Hour,
@@ -7082,9 +7082,15 @@ func TestExpendedPostingsCache(t *testing.T) {
 			extraMatcher := []struct {
 				matchers       []*client.LabelMatcher
 				expectedLenght int
+				// invalidatedByPush is true if pushing a series with a="new" below should
+				// invalidate this matcher's cached head entry. Postings are invalidated per
+				// label: an equality matcher is keyed by value, so a="new" does not touch the
+				// a="aaa1" entry; a regex matcher is keyed by name, so a="new" invalidates it.
+				invalidatedByPush bool
 			}{
 				{
-					expectedLenght: 10,
+					expectedLenght:    10,
+					invalidatedByPush: true,
 					matchers: []*client.LabelMatcher{
 						{
 							Type:  client.REGEX_MATCH,
@@ -7094,7 +7100,8 @@ func TestExpendedPostingsCache(t *testing.T) {
 					},
 				},
 				{
-					expectedLenght: 1,
+					expectedLenght:    1,
+					invalidatedByPush: false,
 					matchers: []*client.LabelMatcher{
 						{
 							Type:  client.EQUAL,
@@ -7171,9 +7178,9 @@ func TestExpendedPostingsCache(t *testing.T) {
 			// Query block and head
 			require.Equal(t, postingsForMatchersCalls.Load(), int64(c.expectedBlockPostingCall+c.expectedHeadPostingCall))
 
-			// Adding a metric for the first metric name so we expire all caches for that metric name
+			// Adding a series with the queried label "a" to expire the caches for that metric name and label
 			_, err = i.Push(ctx, cortexpb.ToWriteRequest(
-				[]labels.Labels{labels.FromStrings(labels.MetricName, metricNames[0], "extra", "1")}, []cortexpb.Sample{{Value: 2, TimestampMs: 4 * 60 * 60 * 1000}}, nil, nil, cortexpb.API))
+				[]labels.Labels{labels.FromStrings(labels.MetricName, metricNames[0], "a", "new", "extra", "1")}, []cortexpb.Sample{{Value: 2, TimestampMs: 4 * 60 * 60 * 1000}}, nil, nil, cortexpb.API))
 			require.NoError(t, err)
 
 			for in, name := range metricNames {
@@ -7182,8 +7189,11 @@ func TestExpendedPostingsCache(t *testing.T) {
 
 					require.Len(t, runQuery(t, ctx, i, append(m.matchers, &client.LabelMatcher{Type: client.EQUAL, Name: labels.MetricName, Value: name})), m.expectedLenght)
 
-					// first metric name should be expired
-					if in == 0 {
+					// Only the first metric name had a series pushed, and only matchers whose
+					// tracked slot was bumped by that push are invalidated (see invalidatedByPush).
+					// An invalidated head entry refetches; the block stays cached, so everything
+					// else is served from cache.
+					if in == 0 && m.invalidatedByPush {
 						// Query only head as the block is already cached and the head was expired
 						require.Equal(t, postingsForMatchersCalls.Load(), int64(c.expectedHeadPostingCall))
 					} else {