From c911d1b5723cebe8acf0866d84b784d9ec6ade70 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Fri, 6 Nov 2026 10:33:52 +0800 Subject: [PATCH] observability: expose SWA prefix-match gate rejections The unified-cache SWA component gates prefix matching on a live sliding window: when the window is missing, the match stops before an otherwise available Full-KV prefix. The existing hit counters do not show the tokens lost to that gate, which makes the gate-on/gate-off trade-off invisible. Record the gap between the longest Full-KV match and the accepted prefix (full_kv_hit_length - (device_prefix_len + host_hit_length)) on the Req and export it as: sglang:swa_gate_reject_tokens_total sglang:swa_gate_reject_reqs_total This is the token volume that will be re-prefilled because a component validator (SWA window; Mamba where present) rejected the match. With the gate enabled it measures the hit length lost to a missing window; with the gate disabled it only captures the host-only / load-back-gap fallback. Includes unit tests for the gap and the host-hit case. Not runtime verified yet (no GPU run on this build). --- python/sglang/srt/managers/schedule_batch.py | 3 + python/sglang/srt/mem_cache/common.py | 16 +++++ .../srt/observability/metrics_collector.py | 27 +++++++ .../unit/mem_cache/test_radix_force_miss.py | 71 +++++++++++++++++++ 4 files changed, 117 insertions(+) diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 6cab5b8066..999880cf86 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1148,6 +1148,8 @@ class Req(ReqDllmMixin): # Full-KV-derived boundary whose SWA window should be inserted after # the current prefill pass. self.swa_branching_seqlen: Optional[int] = None + # Observability: Full-KV tokens rejected by the SWA prefix-match gate. + self.swa_gate_reject_tokens: int = 0 # for cross-encoder model self.token_type_ids = token_type_ids @@ -2083,6 +2085,7 @@ class Req(ReqDllmMixin): self.num_matched_prefix_tokens = 0 self.lock = None self.swa_branching_seqlen = None + self.swa_gate_reject_tokens = 0 self.extend_end = None self.dllm_initialized = False self.is_retracted = True diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index c4aa18e6f5..7cab1bee92 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -268,6 +268,22 @@ def match_kv_cache( max_len = req._compute_max_prefix_len(len(token_ids)) req.num_matched_prefix_tokens = min(req.prefix_len + req.host_hit_length, max_len) req.swa_branching_seqlen = match_result.swa_branching_seqlen + # Observability: Full-KV tokens that a component validator (SWA/Mamba) + # refused to match. They remain in the request's re-prefill range; with the + # SWA prefix-match gate enabled this is the hit length lost to a missing + # sliding window. With the gate disabled it only captures the host-only / + # load-back-gap fallback, so it should be near 0. + accepted_prefix_len = ( + match_result.device_prefix_len + match_result.host_hit_length + ) + rejected_tokens = match_result.full_kv_hit_length - accepted_prefix_len + req.swa_gate_reject_tokens = max(0, rejected_tokens) + if rejected_tokens > 0: + time_stats = getattr(req, "time_stats", None) + collector = getattr(time_stats, "metrics_collector", None) + observe = getattr(collector, "observe_swa_gate_reject", None) + if observe is not None and getattr(time_stats, "enable_metrics", False): + observe(rejected_tokens) # A probe match keeps what it did not report; a new round resets both. if match_result.mamba_branching_seqlen is not None: req.mamba_branching_seqlen = match_result.mamba_branching_seqlen diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index f956f98f17..dd517d8276 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -562,6 +562,27 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): documentation="Total number of retracted output tokens.", labelnames=labels.keys(), ) + # Prefix-cache component gate (SWA window): Full-KV tokens that a + # component validator refused to match, so they are re-prefilled. + # With the SWA prefix-match gate enabled this is the hit length lost + # to a missing sliding-window; with the gate disabled it only captures + # the host-only / load-back-gap fallback, so it should be near 0. + self.swa_gate_reject_tokens_total = Counter( + name="sglang:swa_gate_reject_tokens_total", + documentation=( + "Full-KV matched tokens rejected by the SWA prefix-match gate " + "and re-prefilled (any component-validator rejection)." + ), + labelnames=labels.keys(), + ) + self.swa_gate_reject_reqs_total = Counter( + name="sglang:swa_gate_reject_reqs_total", + documentation=( + "Requests whose Full-KV match was truncated by the SWA " + "prefix-match gate." + ), + labelnames=labels.keys(), + ) self.num_paused_reqs = Gauge( name="sglang:num_paused_reqs", documentation="The number of paused requests by async weight sync.", @@ -1422,6 +1443,12 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): if count > 0: self.num_prefill_retries_total.labels(**self.labels).inc(count) + def observe_swa_gate_reject(self, tokens: int) -> None: + """Record Full-KV tokens a component validator refused to match.""" + if tokens > 0: + self.swa_gate_reject_tokens_total.labels(**self.labels).inc(tokens) + self.swa_gate_reject_reqs_total.labels(**self.labels).inc(1) + def observe_kv_transfer_metrics( self, latency_ms: float, diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index 0879a19c97..9063e1725d 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -121,5 +121,76 @@ class TestMatchKvCacheForceMiss(unittest.TestCase): self.assertEqual(forced_req.host_hit_length, 0) +class TestSwaGateRejectMetric(unittest.TestCase): + """Full-KV tokens rejected by the SWA prefix-match gate are observable.""" + + def test_rejected_full_kv_tokens_are_observed(self): + class _StubTreeCache: + def swa_reprefill_tail_tokens(self, req=None): + return 0 + + def match_prefix(self, params): + # Full KV covers 164 tokens, but the component validators only + # accept 100: the 64-token gap is the SWA window gate's cost. + return MatchResult( + device_prefix_len=100, + last_device_node=None, + last_host_node=None, + best_match_node=None, + host_hit_length=0, + full_kv_hit_length=164, + swa_branching_seqlen=128, + ) + + class _StubCollector: + def __init__(self): + self.rejected = [] + + def observe_swa_gate_reject(self, tokens): + self.rejected.append(tokens) + + req = _StubReq(list(range(200))) + req.time_stats = SimpleNamespace( + metrics_collector=_StubCollector(), enable_metrics=True + ) + + match_kv_cache(req, _StubTreeCache()) + + self.assertEqual(req.swa_gate_reject_tokens, 64) + self.assertEqual(req.time_stats.metrics_collector.rejected, [64]) + + def test_host_hit_closes_the_gap(self): + class _StubTreeCache: + def swa_reprefill_tail_tokens(self, req=None): + return 0 + + def match_prefix(self, params): + return MatchResult( + device_prefix_len=100, + last_device_node=None, + last_host_node=None, + best_match_node=None, + host_hit_length=64, + full_kv_hit_length=164, + ) + + class _StubCollector: + def __init__(self): + self.rejected = [] + + def observe_swa_gate_reject(self, tokens): + self.rejected.append(tokens) + + req = _StubReq(list(range(200))) + req.time_stats = SimpleNamespace( + metrics_collector=_StubCollector(), enable_metrics=True + ) + + match_kv_cache(req, _StubTreeCache()) + + self.assertEqual(req.swa_gate_reject_tokens, 0) + self.assertEqual(req.time_stats.metrics_collector.rejected, []) + + if __name__ == "__main__": unittest.main()