From 7f6fe22445597dec3a437f9779c482716c25108d Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Mon, 26 Oct 2026 13:23:17 +0000 Subject: [PATCH] mem_cache: fail-soft on SWA load-back gaps instead of asserting With long prompts and L3 under pressure, a matched prefix can contain a node whose SWA component has neither a device nor a host value (host tombstone evicted between match and load-back). build_hicache_transfers asserted on this and killed the prefill scheduler. Add SWAComponent.has_load_back_gap() and have build_load_back_spec return an empty transfer spec when a gap is found, so the caller backs off and the request is recomputed. Measured on peak15-v6 (1M, cap 64): 0 scheduler exceptions, hit 59.4%->94.0%, TTFT p95 47.0s->3.4s with the SWA prefix-match gate disabled. --- .../mem_cache/unified_cache/components/swa.py | 19 +++++++++++++++++++ .../unified_cache/unified_tree_core.py | 14 ++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/python/sglang/srt/mem_cache/unified_cache/components/swa.py b/python/sglang/srt/mem_cache/unified_cache/components/swa.py index e30a21c487..ac8cdda1a8 100644 --- a/python/sglang/srt/mem_cache/unified_cache/components/swa.py +++ b/python/sglang/srt/mem_cache/unified_cache/components/swa.py @@ -1123,6 +1123,25 @@ class SWAComponent(TreeComponent): return {self.pool_name: []} return {self.pool_name: list(hash_values[-num_pages:])} + def has_load_back_gap(self, node: UnifiedTreeNode) -> bool: + """True if some node of the SWA window has neither a device nor a host + value. With the prefix-match gate disabled such a gap can sit inside a + matched prefix; the L3 load-back must then back off (recompute) instead + of asserting in build_hicache_transfers.""" + ct = self.component_type + n_swa = 0 + cur = node + while cur is not self.tree_core.root_node and n_swa < self.sliding_window_size: + cd = cur.component_data[ct] + if cd.value is not None: + n_swa += len(cd.value) + elif cd.host_value is not None: + n_swa += len(cd.host_value) + else: + return True + cur = cur.parent + return False + def build_hicache_transfers( self, node: UnifiedTreeNode, diff --git a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py index 7f73fe2d54..4ad096393e 100644 --- a/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py +++ b/python/sglang/srt/mem_cache/unified_cache/unified_tree_core.py @@ -2569,6 +2569,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface): # Component hooks take primitives, not Req: extract its fields here. mamba_pool_idx = req.kv.mamba_pool_idx if req is not None else None node = self.node_by_id(node_id) + # Gate-off safety: an aux component window with a hole (no device and + # no host value) must not be L3-loaded; emit an empty spec so the + # caller backs off and recomputes instead of asserting downstream. + for _comp in self.components: + _gap = getattr(_comp, "has_load_back_gap", None) + if _gap is not None and _gap(node): + return ( + PoolTransfer( + name=PoolName.KV, + host_indices=torch.empty((0,), dtype=torch.int64, device="cpu"), + nodes_to_load=[], + ), + {}, + ) kv_xfer = self.components_by_type[BASE_COMPONENT_TYPE].build_hicache_transfers( node, CacheTransferPhase.LOAD_BACK )[0]