observability: count SWA gate rejections per prefill admission

Review follow-up. match_kv_cache also runs on waiting-queue probes
(calc_priority), PD decode _match_prefix_and_lock and HiCache rematches.
Emitting the counter there counted the same gap once per scheduling round
and attributed "re-prefilled" tokens to decode engines.

- match_kv_cache now only records req.full_kv_hit_length (plus the match-time
  gap); it no longer emits.
- the metric is emitted once in PrefillAdder._commit_prefill_admission, where
  the accepted prefix is final after any host load-back.
- rename the request counter to swa_gate_reject_admissions_total and document
  that a retracted and re-admitted request is counted again (it is re-prefilled).
- pre-create zero-valued children so gate-off deployments expose the series
  before the first rejection.
- tests: repeated match does not emit, admission emits once, a retry counts
  again, and host load-back closes the gap.

Not runtime verified (no GPU run on this build).
This commit is contained in:
2026-11-06 13:18:07 +08:00
parent bf8cf7f1dc
commit 23053730fc
5 changed files with 106 additions and 70 deletions
@@ -1150,6 +1150,8 @@ class Req(ReqDllmMixin):
self.swa_branching_seqlen: Optional[int] = None
# Observability: Full-KV tokens rejected by the SWA prefix-match gate.
self.swa_gate_reject_tokens: int = 0
# Longest Full-KV match before component gates (set by match_kv_cache).
self.full_kv_hit_length: int = 0
# for cross-encoder model
self.token_type_ids = token_type_ids
@@ -2086,6 +2088,7 @@ class Req(ReqDllmMixin):
self.lock = None
self.swa_branching_seqlen = None
self.swa_gate_reject_tokens = 0
self.full_kv_hit_length = 0
self.extend_end = None
self.dllm_initialized = False
self.is_retracted = True
@@ -28,6 +28,25 @@ logger = logging.getLogger(__name__)
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
def observe_swa_gate_reject_on_admission(req, admitted_prefix_len: int) -> int:
"""Emit the SWA prefix-match gate's cost once per prefill admission.
``match_kv_cache`` also runs on waiting-queue probes, PD decode and HiCache
rematches; only the prefill admission knows the final accepted prefix (after
any host load-back), so the counter is incremented here. A retracted and
re-admitted request is counted again because it is re-prefilled.
"""
gap = max(0, getattr(req, "full_kv_hit_length", 0) - admitted_prefix_len)
req.swa_gate_reject_tokens = gap
if gap > 0:
time_stats = getattr(req, "time_stats", None)
collector = getattr(time_stats, "metrics_collector", None)
observe = getattr(collector, "observe_swa_gate_reject_admission", None)
if observe is not None and getattr(time_stats, "enable_metrics", False):
observe(gap)
return gap
"""Request scheduler policy"""
import os
@@ -1622,6 +1641,7 @@ class PrefillAdder:
) -> None:
assert req.prefix_len == admission.prefix_len
req.extend_end = admission.prefix_len + admission.extend_len
observe_swa_gate_reject_on_admission(req, admission.prefix_len)
req.lock = self.tree_cache.lock(req.last_node)
self.can_run_list.append(req)
if admission.is_chunked:
+10 -15
View File
@@ -268,22 +268,17 @@ def match_kv_cache(
max_len = req._compute_max_prefix_len(len(token_ids))
req.num_matched_prefix_tokens = min(req.prefix_len + req.host_hit_length, max_len)
req.swa_branching_seqlen = match_result.swa_branching_seqlen
# Observability: Full-KV tokens that a component validator (SWA/Mamba)
# refused to match. They remain in the request's re-prefill range; with the
# SWA prefix-match gate enabled this is the hit length lost to a missing
# sliding window. With the gate disabled it only captures the host-only /
# load-back-gap fallback, so it should be near 0.
accepted_prefix_len = (
match_result.device_prefix_len + match_result.host_hit_length
# Observability: record the Full-KV length the components could have used.
# This function also runs on waiting-queue probes, PD decode and HiCache
# rematches, so it only stores the gap; the metric is emitted exactly once
# per prefill admission in PrefillAdder._commit_prefill_admission, where the
# accepted prefix length is final (after any host load-back).
req.full_kv_hit_length = match_result.full_kv_hit_length
req.swa_gate_reject_tokens = max(
0,
match_result.full_kv_hit_length
- (match_result.device_prefix_len + match_result.host_hit_length),
)
rejected_tokens = match_result.full_kv_hit_length - accepted_prefix_len
req.swa_gate_reject_tokens = max(0, rejected_tokens)
if rejected_tokens > 0:
time_stats = getattr(req, "time_stats", None)
collector = getattr(time_stats, "metrics_collector", None)
observe = getattr(collector, "observe_swa_gate_reject", None)
if observe is not None and getattr(time_stats, "enable_metrics", False):
observe(rejected_tokens)
# A probe match keeps what it did not report; a new round resets both.
if match_result.mamba_branching_seqlen is not None:
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
@@ -564,25 +564,32 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
)
# Prefix-cache component gate (SWA window): Full-KV tokens that a
# component validator refused to match, so they are re-prefilled.
# With the SWA prefix-match gate enabled this is the hit length lost
# to a missing sliding-window; with the gate disabled it only captures
# the host-only / load-back-gap fallback, so it should be near 0.
# Counted once per prefill admission (not on waiting-queue probes, PD
# decode matches or HiCache rematches). With the SWA prefix-match gate
# enabled this is the hit length lost to a missing sliding window; with
# the gate disabled it only captures the host-only / load-back-gap
# fallback, so it should be near 0.
self.swa_gate_reject_tokens_total = Counter(
name="sglang:swa_gate_reject_tokens_total",
documentation=(
"Full-KV matched tokens rejected by the SWA prefix-match gate "
"and re-prefilled (any component-validator rejection)."
"at prefill admission and therefore re-prefilled."
),
labelnames=labels.keys(),
)
self.swa_gate_reject_reqs_total = Counter(
name="sglang:swa_gate_reject_reqs_total",
self.swa_gate_reject_admissions_total = Counter(
name="sglang:swa_gate_reject_admissions_total",
documentation=(
"Requests whose Full-KV match was truncated by the SWA "
"prefix-match gate."
"Prefill admissions whose Full-KV match was truncated by the "
"SWA prefix-match gate. A retracted and re-admitted request is "
"counted again because it is re-prefilled."
),
labelnames=labels.keys(),
)
# Create the zero-valued children so a gate-off deployment exposes the
# series before the first rejection.
self.swa_gate_reject_tokens_total.labels(**self.labels).inc(0)
self.swa_gate_reject_admissions_total.labels(**self.labels).inc(0)
self.num_paused_reqs = Gauge(
name="sglang:num_paused_reqs",
documentation="The number of paused requests by async weight sync.",
@@ -1443,11 +1450,11 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
if count > 0:
self.num_prefill_retries_total.labels(**self.labels).inc(count)
def observe_swa_gate_reject(self, tokens: int) -> None:
"""Record Full-KV tokens a component validator refused to match."""
def observe_swa_gate_reject_admission(self, tokens: int) -> None:
"""Record one prefill admission whose match the SWA gate truncated."""
if tokens > 0:
self.swa_gate_reject_tokens_total.labels(**self.labels).inc(tokens)
self.swa_gate_reject_reqs_total.labels(**self.labels).inc(1)
self.swa_gate_reject_admissions_total.labels(**self.labels).inc(1)
def observe_kv_transfer_metrics(
self,
@@ -121,74 +121,85 @@ class TestMatchKvCacheForceMiss(unittest.TestCase):
self.assertEqual(forced_req.host_hit_length, 0)
class TestSwaGateRejectMetric(unittest.TestCase):
"""Full-KV tokens rejected by the SWA prefix-match gate are observable."""
class TestSwaGateRejectObservability(unittest.TestCase):
"""The gate gap is recorded on match and emitted once per prefill admission."""
def test_rejected_full_kv_tokens_are_observed(self):
@staticmethod
def _req_with_collector():
class _StubCollector:
def __init__(self):
self.rejected = []
def observe_swa_gate_reject_admission(self, tokens):
self.rejected.append(tokens)
req = _StubReq(list(range(200)))
req.time_stats = SimpleNamespace(
metrics_collector=_StubCollector(), enable_metrics=True
)
return req
@staticmethod
def _tree_cache(device_prefix_len, host_hit_length, full_kv_hit_length):
class _StubTreeCache:
def swa_reprefill_tail_tokens(self, req=None):
return 0
def match_prefix(self, params):
# Full KV covers 164 tokens, but the component validators only
# accept 100: the 64-token gap is the SWA window gate's cost.
return MatchResult(
device_prefix_len=100,
device_prefix_len=device_prefix_len,
last_device_node=None,
last_host_node=None,
best_match_node=None,
host_hit_length=0,
full_kv_hit_length=164,
host_hit_length=host_hit_length,
full_kv_hit_length=full_kv_hit_length,
swa_branching_seqlen=128,
)
class _StubCollector:
def __init__(self):
self.rejected = []
return _StubTreeCache()
def observe_swa_gate_reject(self, tokens):
self.rejected.append(tokens)
def test_match_records_gap_but_does_not_emit(self):
req = self._req_with_collector()
tree = self._tree_cache(100, 0, 164)
req = _StubReq(list(range(200)))
req.time_stats = SimpleNamespace(
metrics_collector=_StubCollector(), enable_metrics=True
# Waiting-queue probes and the final init_next_round_input both match.
match_kv_cache(req, tree)
match_kv_cache(req, tree)
self.assertEqual(req.full_kv_hit_length, 164)
self.assertEqual(req.swa_gate_reject_tokens, 64)
self.assertEqual(req.time_stats.metrics_collector.rejected, [])
def test_match_time_gap_accounts_for_host_hit(self):
req = self._req_with_collector()
match_kv_cache(req, self._tree_cache(100, 64, 164))
self.assertEqual(req.swa_gate_reject_tokens, 0)
def test_admission_emits_once_and_counts_retries(self):
from sglang.srt.managers.schedule_policy import (
observe_swa_gate_reject_on_admission,
)
match_kv_cache(req, _StubTreeCache())
req = self._req_with_collector()
req.full_kv_hit_length = 164
self.assertEqual(req.swa_gate_reject_tokens, 64)
self.assertEqual(observe_swa_gate_reject_on_admission(req, 100), 64)
self.assertEqual(req.time_stats.metrics_collector.rejected, [64])
def test_host_hit_closes_the_gap(self):
class _StubTreeCache:
def swa_reprefill_tail_tokens(self, req=None):
return 0
# A retracted request that is admitted again is re-prefilled: count again.
self.assertEqual(observe_swa_gate_reject_on_admission(req, 100), 64)
self.assertEqual(req.time_stats.metrics_collector.rejected, [64, 64])
def match_prefix(self, params):
return MatchResult(
device_prefix_len=100,
last_device_node=None,
last_host_node=None,
best_match_node=None,
host_hit_length=64,
full_kv_hit_length=164,
)
class _StubCollector:
def __init__(self):
self.rejected = []
def observe_swa_gate_reject(self, tokens):
self.rejected.append(tokens)
req = _StubReq(list(range(200)))
req.time_stats = SimpleNamespace(
metrics_collector=_StubCollector(), enable_metrics=True
def test_host_load_back_closes_the_gap(self):
from sglang.srt.managers.schedule_policy import (
observe_swa_gate_reject_on_admission,
)
match_kv_cache(req, _StubTreeCache())
req = self._req_with_collector()
req.full_kv_hit_length = 164
self.assertEqual(req.swa_gate_reject_tokens, 0)
# 64 host tokens were loaded back before admission: final prefix is 164.
self.assertEqual(observe_swa_gate_reject_on_admission(req, 164), 0)
self.assertEqual(req.time_stats.metrics_collector.rejected, [])