diff --git a/python/sglang/srt/managers/schedule_batch.py b/python/sglang/srt/managers/schedule_batch.py index 999880cf86..750bfa63a9 100644 --- a/python/sglang/srt/managers/schedule_batch.py +++ b/python/sglang/srt/managers/schedule_batch.py @@ -1150,6 +1150,8 @@ class Req(ReqDllmMixin): self.swa_branching_seqlen: Optional[int] = None # Observability: Full-KV tokens rejected by the SWA prefix-match gate. self.swa_gate_reject_tokens: int = 0 + # Longest Full-KV match before component gates (set by match_kv_cache). + self.full_kv_hit_length: int = 0 # for cross-encoder model self.token_type_ids = token_type_ids @@ -2086,6 +2088,7 @@ class Req(ReqDllmMixin): self.lock = None self.swa_branching_seqlen = None self.swa_gate_reject_tokens = 0 + self.full_kv_hit_length = 0 self.extend_end = None self.dllm_initialized = False self.is_retracted = True diff --git a/python/sglang/srt/managers/schedule_policy.py b/python/sglang/srt/managers/schedule_policy.py index 197d7bbaca..2dad9dc228 100644 --- a/python/sglang/srt/managers/schedule_policy.py +++ b/python/sglang/srt/managers/schedule_policy.py @@ -28,6 +28,25 @@ logger = logging.getLogger(__name__) # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== + + +def observe_swa_gate_reject_on_admission(req, admitted_prefix_len: int) -> int: + """Emit the SWA prefix-match gate's cost once per prefill admission. + + ``match_kv_cache`` also runs on waiting-queue probes, PD decode and HiCache + rematches; only the prefill admission knows the final accepted prefix (after + any host load-back), so the counter is incremented here. A retracted and + re-admitted request is counted again because it is re-prefilled. + """ + gap = max(0, getattr(req, "full_kv_hit_length", 0) - admitted_prefix_len) + req.swa_gate_reject_tokens = gap + if gap > 0: + time_stats = getattr(req, "time_stats", None) + collector = getattr(time_stats, "metrics_collector", None) + observe = getattr(collector, "observe_swa_gate_reject_admission", None) + if observe is not None and getattr(time_stats, "enable_metrics", False): + observe(gap) + return gap """Request scheduler policy""" import os @@ -1622,6 +1641,7 @@ class PrefillAdder: ) -> None: assert req.prefix_len == admission.prefix_len req.extend_end = admission.prefix_len + admission.extend_len + observe_swa_gate_reject_on_admission(req, admission.prefix_len) req.lock = self.tree_cache.lock(req.last_node) self.can_run_list.append(req) if admission.is_chunked: diff --git a/python/sglang/srt/mem_cache/common.py b/python/sglang/srt/mem_cache/common.py index 7cab1bee92..7122c3fab6 100644 --- a/python/sglang/srt/mem_cache/common.py +++ b/python/sglang/srt/mem_cache/common.py @@ -268,22 +268,17 @@ def match_kv_cache( max_len = req._compute_max_prefix_len(len(token_ids)) req.num_matched_prefix_tokens = min(req.prefix_len + req.host_hit_length, max_len) req.swa_branching_seqlen = match_result.swa_branching_seqlen - # Observability: Full-KV tokens that a component validator (SWA/Mamba) - # refused to match. They remain in the request's re-prefill range; with the - # SWA prefix-match gate enabled this is the hit length lost to a missing - # sliding window. With the gate disabled it only captures the host-only / - # load-back-gap fallback, so it should be near 0. - accepted_prefix_len = ( - match_result.device_prefix_len + match_result.host_hit_length + # Observability: record the Full-KV length the components could have used. + # This function also runs on waiting-queue probes, PD decode and HiCache + # rematches, so it only stores the gap; the metric is emitted exactly once + # per prefill admission in PrefillAdder._commit_prefill_admission, where the + # accepted prefix length is final (after any host load-back). + req.full_kv_hit_length = match_result.full_kv_hit_length + req.swa_gate_reject_tokens = max( + 0, + match_result.full_kv_hit_length + - (match_result.device_prefix_len + match_result.host_hit_length), ) - rejected_tokens = match_result.full_kv_hit_length - accepted_prefix_len - req.swa_gate_reject_tokens = max(0, rejected_tokens) - if rejected_tokens > 0: - time_stats = getattr(req, "time_stats", None) - collector = getattr(time_stats, "metrics_collector", None) - observe = getattr(collector, "observe_swa_gate_reject", None) - if observe is not None and getattr(time_stats, "enable_metrics", False): - observe(rejected_tokens) # A probe match keeps what it did not report; a new round resets both. if match_result.mamba_branching_seqlen is not None: req.mamba_branching_seqlen = match_result.mamba_branching_seqlen diff --git a/python/sglang/srt/observability/metrics_collector.py b/python/sglang/srt/observability/metrics_collector.py index dd517d8276..99d8bfccfe 100644 --- a/python/sglang/srt/observability/metrics_collector.py +++ b/python/sglang/srt/observability/metrics_collector.py @@ -564,25 +564,32 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): ) # Prefix-cache component gate (SWA window): Full-KV tokens that a # component validator refused to match, so they are re-prefilled. - # With the SWA prefix-match gate enabled this is the hit length lost - # to a missing sliding-window; with the gate disabled it only captures - # the host-only / load-back-gap fallback, so it should be near 0. + # Counted once per prefill admission (not on waiting-queue probes, PD + # decode matches or HiCache rematches). With the SWA prefix-match gate + # enabled this is the hit length lost to a missing sliding window; with + # the gate disabled it only captures the host-only / load-back-gap + # fallback, so it should be near 0. self.swa_gate_reject_tokens_total = Counter( name="sglang:swa_gate_reject_tokens_total", documentation=( "Full-KV matched tokens rejected by the SWA prefix-match gate " - "and re-prefilled (any component-validator rejection)." + "at prefill admission and therefore re-prefilled." ), labelnames=labels.keys(), ) - self.swa_gate_reject_reqs_total = Counter( - name="sglang:swa_gate_reject_reqs_total", + self.swa_gate_reject_admissions_total = Counter( + name="sglang:swa_gate_reject_admissions_total", documentation=( - "Requests whose Full-KV match was truncated by the SWA " - "prefix-match gate." + "Prefill admissions whose Full-KV match was truncated by the " + "SWA prefix-match gate. A retracted and re-admitted request is " + "counted again because it is re-prefilled." ), labelnames=labels.keys(), ) + # Create the zero-valued children so a gate-off deployment exposes the + # series before the first rejection. + self.swa_gate_reject_tokens_total.labels(**self.labels).inc(0) + self.swa_gate_reject_admissions_total.labels(**self.labels).inc(0) self.num_paused_reqs = Gauge( name="sglang:num_paused_reqs", documentation="The number of paused requests by async weight sync.", @@ -1443,11 +1450,11 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin): if count > 0: self.num_prefill_retries_total.labels(**self.labels).inc(count) - def observe_swa_gate_reject(self, tokens: int) -> None: - """Record Full-KV tokens a component validator refused to match.""" + def observe_swa_gate_reject_admission(self, tokens: int) -> None: + """Record one prefill admission whose match the SWA gate truncated.""" if tokens > 0: self.swa_gate_reject_tokens_total.labels(**self.labels).inc(tokens) - self.swa_gate_reject_reqs_total.labels(**self.labels).inc(1) + self.swa_gate_reject_admissions_total.labels(**self.labels).inc(1) def observe_kv_transfer_metrics( self, diff --git a/test/registered/unit/mem_cache/test_radix_force_miss.py b/test/registered/unit/mem_cache/test_radix_force_miss.py index 9063e1725d..40f816c267 100644 --- a/test/registered/unit/mem_cache/test_radix_force_miss.py +++ b/test/registered/unit/mem_cache/test_radix_force_miss.py @@ -121,74 +121,85 @@ class TestMatchKvCacheForceMiss(unittest.TestCase): self.assertEqual(forced_req.host_hit_length, 0) -class TestSwaGateRejectMetric(unittest.TestCase): - """Full-KV tokens rejected by the SWA prefix-match gate are observable.""" +class TestSwaGateRejectObservability(unittest.TestCase): + """The gate gap is recorded on match and emitted once per prefill admission.""" - def test_rejected_full_kv_tokens_are_observed(self): + @staticmethod + def _req_with_collector(): + class _StubCollector: + def __init__(self): + self.rejected = [] + + def observe_swa_gate_reject_admission(self, tokens): + self.rejected.append(tokens) + + req = _StubReq(list(range(200))) + req.time_stats = SimpleNamespace( + metrics_collector=_StubCollector(), enable_metrics=True + ) + return req + + @staticmethod + def _tree_cache(device_prefix_len, host_hit_length, full_kv_hit_length): class _StubTreeCache: def swa_reprefill_tail_tokens(self, req=None): return 0 def match_prefix(self, params): - # Full KV covers 164 tokens, but the component validators only - # accept 100: the 64-token gap is the SWA window gate's cost. return MatchResult( - device_prefix_len=100, + device_prefix_len=device_prefix_len, last_device_node=None, last_host_node=None, best_match_node=None, - host_hit_length=0, - full_kv_hit_length=164, + host_hit_length=host_hit_length, + full_kv_hit_length=full_kv_hit_length, swa_branching_seqlen=128, ) - class _StubCollector: - def __init__(self): - self.rejected = [] + return _StubTreeCache() - def observe_swa_gate_reject(self, tokens): - self.rejected.append(tokens) + def test_match_records_gap_but_does_not_emit(self): + req = self._req_with_collector() + tree = self._tree_cache(100, 0, 164) - req = _StubReq(list(range(200))) - req.time_stats = SimpleNamespace( - metrics_collector=_StubCollector(), enable_metrics=True + # Waiting-queue probes and the final init_next_round_input both match. + match_kv_cache(req, tree) + match_kv_cache(req, tree) + + self.assertEqual(req.full_kv_hit_length, 164) + self.assertEqual(req.swa_gate_reject_tokens, 64) + self.assertEqual(req.time_stats.metrics_collector.rejected, []) + + def test_match_time_gap_accounts_for_host_hit(self): + req = self._req_with_collector() + match_kv_cache(req, self._tree_cache(100, 64, 164)) + self.assertEqual(req.swa_gate_reject_tokens, 0) + + def test_admission_emits_once_and_counts_retries(self): + from sglang.srt.managers.schedule_policy import ( + observe_swa_gate_reject_on_admission, ) - match_kv_cache(req, _StubTreeCache()) + req = self._req_with_collector() + req.full_kv_hit_length = 164 - self.assertEqual(req.swa_gate_reject_tokens, 64) + self.assertEqual(observe_swa_gate_reject_on_admission(req, 100), 64) self.assertEqual(req.time_stats.metrics_collector.rejected, [64]) - def test_host_hit_closes_the_gap(self): - class _StubTreeCache: - def swa_reprefill_tail_tokens(self, req=None): - return 0 + # A retracted request that is admitted again is re-prefilled: count again. + self.assertEqual(observe_swa_gate_reject_on_admission(req, 100), 64) + self.assertEqual(req.time_stats.metrics_collector.rejected, [64, 64]) - def match_prefix(self, params): - return MatchResult( - device_prefix_len=100, - last_device_node=None, - last_host_node=None, - best_match_node=None, - host_hit_length=64, - full_kv_hit_length=164, - ) - - class _StubCollector: - def __init__(self): - self.rejected = [] - - def observe_swa_gate_reject(self, tokens): - self.rejected.append(tokens) - - req = _StubReq(list(range(200))) - req.time_stats = SimpleNamespace( - metrics_collector=_StubCollector(), enable_metrics=True + def test_host_load_back_closes_the_gap(self): + from sglang.srt.managers.schedule_policy import ( + observe_swa_gate_reject_on_admission, ) - match_kv_cache(req, _StubTreeCache()) + req = self._req_with_collector() + req.full_kv_hit_length = 164 - self.assertEqual(req.swa_gate_reject_tokens, 0) + # 64 host tokens were loaded back before admission: final prefix is 164. + self.assertEqual(observe_swa_gate_reject_on_admission(req, 164), 0) self.assertEqual(req.time_stats.metrics_collector.rejected, [])