Fix SWA eviction behind the replay-trimmed prefix frontier

(cherry picked from commit f399d1e9a2bf7c33c7244ecf405e756292ddaf2b)
This commit is contained in:
Florian Wunderlich
2026-11-04 09:45:08 +08:00
committed by minke.yu
parent 01087e689c
commit a3401b3e65
4 changed files with 166 additions and 0 deletions
@@ -703,6 +703,7 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
supports_prefix_sharing=self.supports_prefix_sharing(),
retain_floor=self.swa_retain_floor(req),
eviction_interval=eviction_interval,
reprefill_tail_tokens=self.swa_reprefill_tail_tokens(req),
)
def swa_retain_floor(self, req) -> int | None:
+9
View File
@@ -84,6 +84,7 @@ def free_swa_out_of_window_slots(
component_type: ComponentType = ComponentType.SWA,
free_segment: Callable[..., None] | None = None,
eviction_interval: int = 1,
reprefill_tail_tokens: int = 0,
) -> None:
if not req.kv.holds_kv:
return
@@ -115,6 +116,14 @@ def free_swa_out_of_window_slots(
evict_threshold = pre_len - ceil_align(
max(sliding_window_size, page_size), page_size
)
if reprefill_tail_tokens:
# Matching excludes the replay tail before page alignment. Keep the
# SWA window behind that earlier boundary too, or a first replay
# falls back to an older checkpoint despite a Full-KV cache hit.
replay_boundary = (pre_len - reprefill_tail_tokens) // page_size * page_size
evict_threshold = min(
evict_threshold, replay_boundary - sliding_window_size
)
if retain_floor is not None and supports_prefix_sharing:
# The caller owns where the floor is (see BasePrefixCache.swa_retain_floor);
# this only promises not to free past it. Without prefix sharing a retained
@@ -1046,6 +1046,7 @@ class SWAComponent(TreeComponent):
req_to_token_pool=self.cache.req_to_token_pool,
token_to_kv_pool_allocator=self.cache.token_to_kv_pool_allocator,
retain_floor=self.cache.swa_retain_floor(req),
reprefill_tail_tokens=self.cache.swa_reprefill_tail_tokens(req),
)
def free_out_of_window_slots(