mem_cache: fail-soft on SWA load-back gaps instead of asserting
With long prompts and L3 under pressure, a matched prefix can contain a node whose SWA component has neither a device nor a host value (host tombstone evicted between match and load-back). build_hicache_transfers asserted on this and killed the prefill scheduler. Add SWAComponent.has_load_back_gap() and have build_load_back_spec return an empty transfer spec when a gap is found, so the caller backs off and the request is recomputed. Measured on peak15-v6 (1M, cap 64): 0 scheduler exceptions, hit 59.4%->94.0%, TTFT p95 47.0s->3.4s with the SWA prefix-match gate disabled.
This commit is contained in:
@@ -1123,6 +1123,25 @@ class SWAComponent(TreeComponent):
|
||||
return {self.pool_name: []}
|
||||
return {self.pool_name: list(hash_values[-num_pages:])}
|
||||
|
||||
def has_load_back_gap(self, node: UnifiedTreeNode) -> bool:
|
||||
"""True if some node of the SWA window has neither a device nor a host
|
||||
value. With the prefix-match gate disabled such a gap can sit inside a
|
||||
matched prefix; the L3 load-back must then back off (recompute) instead
|
||||
of asserting in build_hicache_transfers."""
|
||||
ct = self.component_type
|
||||
n_swa = 0
|
||||
cur = node
|
||||
while cur is not self.tree_core.root_node and n_swa < self.sliding_window_size:
|
||||
cd = cur.component_data[ct]
|
||||
if cd.value is not None:
|
||||
n_swa += len(cd.value)
|
||||
elif cd.host_value is not None:
|
||||
n_swa += len(cd.host_value)
|
||||
else:
|
||||
return True
|
||||
cur = cur.parent
|
||||
return False
|
||||
|
||||
def build_hicache_transfers(
|
||||
self,
|
||||
node: UnifiedTreeNode,
|
||||
|
||||
@@ -2569,6 +2569,20 @@ class UnifiedTreeCore(UnifiedTreeCoreInterface):
|
||||
# Component hooks take primitives, not Req: extract its fields here.
|
||||
mamba_pool_idx = req.kv.mamba_pool_idx if req is not None else None
|
||||
node = self.node_by_id(node_id)
|
||||
# Gate-off safety: an aux component window with a hole (no device and
|
||||
# no host value) must not be L3-loaded; emit an empty spec so the
|
||||
# caller backs off and recomputes instead of asserting downstream.
|
||||
for _comp in self.components:
|
||||
_gap = getattr(_comp, "has_load_back_gap", None)
|
||||
if _gap is not None and _gap(node):
|
||||
return (
|
||||
PoolTransfer(
|
||||
name=PoolName.KV,
|
||||
host_indices=torch.empty((0,), dtype=torch.int64, device="cpu"),
|
||||
nodes_to_load=[],
|
||||
),
|
||||
{},
|
||||
)
|
||||
kv_xfer = self.components_by_type[BASE_COMPONENT_TYPE].build_hicache_transfers(
|
||||
node, CacheTransferPhase.LOAD_BACK
|
||||
)[0]
|
||||
|
||||
Reference in New Issue
Block a user