observability: expose SWA prefix-match gate rejections
The unified-cache SWA component gates prefix matching on a live sliding window: when the window is missing, the match stops before an otherwise available Full-KV prefix. The existing hit counters do not show the tokens lost to that gate, which makes the gate-on/gate-off trade-off invisible. Record the gap between the longest Full-KV match and the accepted prefix (full_kv_hit_length - (device_prefix_len + host_hit_length)) on the Req and export it as: sglang:swa_gate_reject_tokens_total sglang:swa_gate_reject_reqs_total This is the token volume that will be re-prefilled because a component validator (SWA window; Mamba where present) rejected the match. With the gate enabled it measures the hit length lost to a missing window; with the gate disabled it only captures the host-only / load-back-gap fallback. Includes unit tests for the gap and the host-hit case. Not runtime verified yet (no GPU run on this build).
This commit is contained in:
@@ -1148,6 +1148,8 @@ class Req(ReqDllmMixin):
|
||||
# Full-KV-derived boundary whose SWA window should be inserted after
|
||||
# the current prefill pass.
|
||||
self.swa_branching_seqlen: Optional[int] = None
|
||||
# Observability: Full-KV tokens rejected by the SWA prefix-match gate.
|
||||
self.swa_gate_reject_tokens: int = 0
|
||||
|
||||
# for cross-encoder model
|
||||
self.token_type_ids = token_type_ids
|
||||
@@ -2083,6 +2085,7 @@ class Req(ReqDllmMixin):
|
||||
self.num_matched_prefix_tokens = 0
|
||||
self.lock = None
|
||||
self.swa_branching_seqlen = None
|
||||
self.swa_gate_reject_tokens = 0
|
||||
self.extend_end = None
|
||||
self.dllm_initialized = False
|
||||
self.is_retracted = True
|
||||
|
||||
@@ -268,6 +268,22 @@ def match_kv_cache(
|
||||
max_len = req._compute_max_prefix_len(len(token_ids))
|
||||
req.num_matched_prefix_tokens = min(req.prefix_len + req.host_hit_length, max_len)
|
||||
req.swa_branching_seqlen = match_result.swa_branching_seqlen
|
||||
# Observability: Full-KV tokens that a component validator (SWA/Mamba)
|
||||
# refused to match. They remain in the request's re-prefill range; with the
|
||||
# SWA prefix-match gate enabled this is the hit length lost to a missing
|
||||
# sliding window. With the gate disabled it only captures the host-only /
|
||||
# load-back-gap fallback, so it should be near 0.
|
||||
accepted_prefix_len = (
|
||||
match_result.device_prefix_len + match_result.host_hit_length
|
||||
)
|
||||
rejected_tokens = match_result.full_kv_hit_length - accepted_prefix_len
|
||||
req.swa_gate_reject_tokens = max(0, rejected_tokens)
|
||||
if rejected_tokens > 0:
|
||||
time_stats = getattr(req, "time_stats", None)
|
||||
collector = getattr(time_stats, "metrics_collector", None)
|
||||
observe = getattr(collector, "observe_swa_gate_reject", None)
|
||||
if observe is not None and getattr(time_stats, "enable_metrics", False):
|
||||
observe(rejected_tokens)
|
||||
# A probe match keeps what it did not report; a new round resets both.
|
||||
if match_result.mamba_branching_seqlen is not None:
|
||||
req.mamba_branching_seqlen = match_result.mamba_branching_seqlen
|
||||
|
||||
@@ -562,6 +562,27 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
documentation="Total number of retracted output tokens.",
|
||||
labelnames=labels.keys(),
|
||||
)
|
||||
# Prefix-cache component gate (SWA window): Full-KV tokens that a
|
||||
# component validator refused to match, so they are re-prefilled.
|
||||
# With the SWA prefix-match gate enabled this is the hit length lost
|
||||
# to a missing sliding-window; with the gate disabled it only captures
|
||||
# the host-only / load-back-gap fallback, so it should be near 0.
|
||||
self.swa_gate_reject_tokens_total = Counter(
|
||||
name="sglang:swa_gate_reject_tokens_total",
|
||||
documentation=(
|
||||
"Full-KV matched tokens rejected by the SWA prefix-match gate "
|
||||
"and re-prefilled (any component-validator rejection)."
|
||||
),
|
||||
labelnames=labels.keys(),
|
||||
)
|
||||
self.swa_gate_reject_reqs_total = Counter(
|
||||
name="sglang:swa_gate_reject_reqs_total",
|
||||
documentation=(
|
||||
"Requests whose Full-KV match was truncated by the SWA "
|
||||
"prefix-match gate."
|
||||
),
|
||||
labelnames=labels.keys(),
|
||||
)
|
||||
self.num_paused_reqs = Gauge(
|
||||
name="sglang:num_paused_reqs",
|
||||
documentation="The number of paused requests by async weight sync.",
|
||||
@@ -1422,6 +1443,12 @@ class SchedulerMetricsCollector(_StatLoggerDIMixin):
|
||||
if count > 0:
|
||||
self.num_prefill_retries_total.labels(**self.labels).inc(count)
|
||||
|
||||
def observe_swa_gate_reject(self, tokens: int) -> None:
|
||||
"""Record Full-KV tokens a component validator refused to match."""
|
||||
if tokens > 0:
|
||||
self.swa_gate_reject_tokens_total.labels(**self.labels).inc(tokens)
|
||||
self.swa_gate_reject_reqs_total.labels(**self.labels).inc(1)
|
||||
|
||||
def observe_kv_transfer_metrics(
|
||||
self,
|
||||
latency_ms: float,
|
||||
|
||||
@@ -121,5 +121,76 @@ class TestMatchKvCacheForceMiss(unittest.TestCase):
|
||||
self.assertEqual(forced_req.host_hit_length, 0)
|
||||
|
||||
|
||||
class TestSwaGateRejectMetric(unittest.TestCase):
|
||||
"""Full-KV tokens rejected by the SWA prefix-match gate are observable."""
|
||||
|
||||
def test_rejected_full_kv_tokens_are_observed(self):
|
||||
class _StubTreeCache:
|
||||
def swa_reprefill_tail_tokens(self, req=None):
|
||||
return 0
|
||||
|
||||
def match_prefix(self, params):
|
||||
# Full KV covers 164 tokens, but the component validators only
|
||||
# accept 100: the 64-token gap is the SWA window gate's cost.
|
||||
return MatchResult(
|
||||
device_prefix_len=100,
|
||||
last_device_node=None,
|
||||
last_host_node=None,
|
||||
best_match_node=None,
|
||||
host_hit_length=0,
|
||||
full_kv_hit_length=164,
|
||||
swa_branching_seqlen=128,
|
||||
)
|
||||
|
||||
class _StubCollector:
|
||||
def __init__(self):
|
||||
self.rejected = []
|
||||
|
||||
def observe_swa_gate_reject(self, tokens):
|
||||
self.rejected.append(tokens)
|
||||
|
||||
req = _StubReq(list(range(200)))
|
||||
req.time_stats = SimpleNamespace(
|
||||
metrics_collector=_StubCollector(), enable_metrics=True
|
||||
)
|
||||
|
||||
match_kv_cache(req, _StubTreeCache())
|
||||
|
||||
self.assertEqual(req.swa_gate_reject_tokens, 64)
|
||||
self.assertEqual(req.time_stats.metrics_collector.rejected, [64])
|
||||
|
||||
def test_host_hit_closes_the_gap(self):
|
||||
class _StubTreeCache:
|
||||
def swa_reprefill_tail_tokens(self, req=None):
|
||||
return 0
|
||||
|
||||
def match_prefix(self, params):
|
||||
return MatchResult(
|
||||
device_prefix_len=100,
|
||||
last_device_node=None,
|
||||
last_host_node=None,
|
||||
best_match_node=None,
|
||||
host_hit_length=64,
|
||||
full_kv_hit_length=164,
|
||||
)
|
||||
|
||||
class _StubCollector:
|
||||
def __init__(self):
|
||||
self.rejected = []
|
||||
|
||||
def observe_swa_gate_reject(self, tokens):
|
||||
self.rejected.append(tokens)
|
||||
|
||||
req = _StubReq(list(range(200)))
|
||||
req.time_stats = SimpleNamespace(
|
||||
metrics_collector=_StubCollector(), enable_metrics=True
|
||||
)
|
||||
|
||||
match_kv_cache(req, _StubTreeCache())
|
||||
|
||||
self.assertEqual(req.swa_gate_reject_tokens, 0)
|
||||
self.assertEqual(req.time_stats.metrics_collector.rejected, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user