[HiCache] Label radix-cache metrics per rank and split the "shrunk" prefetch reason (#39280)

test/registered/unit/managers/test_prefill_adder.py

@@ -291,7 +291,7 @@ class TestPrefillAdder(CustomTestCase):
         self.mock_tree_cache.finish_storage_prefetch_admission.assert_called_once_with(
             req.cache_request_handle,
             fulfilled_tokens=4,
-            reason="shrunk",
+            reason="cache_admission_shortfall",
         )
         self.assertEqual(adder.log_device_hit_tokens, 8)
         self.assertEqual(adder.log_host_hit_tokens, 0)

test/registered/unit/mem_cache/test_unified_radix_cache_unittest.py

@@ -10177,7 +10177,7 @@ class TestAnchorLockOutcomePolicy(CustomTestCase):
         cache.dec_host_lock_ref.assert_not_called()
         self.assertEqual(controller.prefetch_tokens_occupied, 8)
 
-    def test_positive_hit_with_lost_anchor_is_reported_as_shrunk(self):
+    def test_positive_hit_with_lost_anchor_reports_anchor_lost(self):
         cache = UnifiedRadixCache.__new__(UnifiedRadixCache)
         cache.storage_prefetch_retries = StoragePrefetchRetries()
         cache.ongoing_prefetch = {
@@ -10191,7 +10191,7 @@ class TestAnchorLockOutcomePolicy(CustomTestCase):
         cache._handle_storage_prefetch_anchor_loss(self._REQ)
 
         cache._finish_storage_prefetch.assert_called_once_with(
-            self._REQ, fulfilled_tokens=0, reason="shrunk"
+            self._REQ, fulfilled_tokens=0, reason="anchor_lost"
         )
         # The eviction widened the span, so the request replans over it --
         # skipping the query, the prior hit having proved it stored, and the

test/registered/unit/observability/test_stat_loggers_di.py

@@ -23,6 +23,7 @@ from sglang.test.ci.ci_register import register_cpu_ci
 register_cpu_ci(est_time=10, suite="base-a-test-cpu")
 
 import unittest
+from types import SimpleNamespace
 
 import prometheus_client
 
@@ -37,6 +38,7 @@ from sglang.srt.observability.metrics_collector import (
     SchedulerMetricsCollector,
     StorageMetricsCollector,
     TokenizerMetricsCollector,
+    radix_cache_metric_labels,
     resolve_collector_class,
 )
 from sglang.srt.runtime_context import get_context, reset_context
@@ -179,6 +181,29 @@ class TestDefaultBackend(unittest.TestCase):
         )
 
 
+class TestRadixCacheMetricLabels(unittest.TestCase):
+    """Radix-cache series must stay distinct per scheduler rank: an unlabeled
+    family is summed across local ranks by the multiprocess registry, which
+    reported TP x the logical token count in production. The rank keys follow
+    the storage collector's DP-aware convention so L2 and L3 series line up."""
+
+    def test_labels_follow_the_storage_collector_rank_keys(self):
+        parallel = SimpleNamespace(tp_rank=3, pp_rank=1, attn_tp_rank=1, attn_dp_rank=2)
+        self.assertEqual(
+            radix_cache_metric_labels("UnifiedRadixCache", parallel, True),
+            {
+                "cache_type": "UnifiedRadixCache",
+                "tp_rank": 1,
+                "pp_rank": 1,
+                "dp_rank": 2,
+            },
+        )
+        self.assertEqual(
+            radix_cache_metric_labels("RadixCache", parallel, False),
+            {"cache_type": "RadixCache", "tp_rank": 3, "pp_rank": 1, "dp_rank": 0},
+        )
+
+
 class TestHiCacheMetrics(unittest.TestCase):
     def test_cached_tokens_uses_literal_storage_source(self):
         labels = {"model_name": "test"}