[Scheduler] Derive prefix KV indices from last_node at allocation; drop Req.prefix_indices (#42825)

This commit is contained in:
Liangsheng Yin
2026-10-07 16:03:09 -07:00
committed by GitHub
parent e9240b3243
commit 5349d3e9d2
40 changed files with 160 additions and 272 deletions
+2 -5
View File
@@ -438,11 +438,8 @@ def prepare_extend_inputs_for_correctness_test(
req.full_untruncated_fill_ids.extend(input_ids[i][bench_args.cut_len :])
if model_runner is not None:
# Use req.kv.req_pool_idx instead of i to handle slot 0 padding correctly
req.set_prefix_indices(
model_runner.req_to_token_pool.req_to_token[
req.kv.req_pool_idx, : bench_args.cut_len
].to(torch.int64)
)
# The cut prefix is already in the request's row from the first extend.
req.prefix_len = bench_args.cut_len
req.logprob_start_len = -1
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
return reqs
+2 -4
View File
@@ -145,7 +145,7 @@ def _bootstrap_addr(req: Req) -> str:
def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
"""Start a decode-radix request that owns its whole KV row at the root."""
req.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
req.prefix_len = 0
req.last_node = tree_cache.root_node_handle(req.extra_key)
req.last_host_node = req.last_node
req.best_match_node = req.last_node
@@ -2204,9 +2204,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
# Decode-disagg allocates before batch assembly, so it binds the prefix
# here instead of in init_next_round_input.
req.set_prefix_indices(
prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64)
)
req.prefix_len = prefix_len
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
self.tree_cache.maybe_hand_to_session(req)
@@ -211,7 +211,7 @@ class DecodeHiCacheTransferMixin:
return False
self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle)
# Re-match: req.last_node / prefix_indices updated to current device state.
# Re-match: req.last_node / prefix_len updated to current device state.
rematch = match_kv_cache(
dr.req,
self.tree_cache,
@@ -316,11 +316,7 @@ class DecodeHiCacheTransferMixin:
),
decode_req.hicache_restored_kv_indices,
)
req.set_prefix_indices(
torch.cat(
[prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices]
)
)
req.prefix_len = prefix_match.decode_prefix_len
req.last_node = restore_lock.node
req.lock = restore_lock
# Prevent abort cleanup from releasing the transferred lock.
+3 -12
View File
@@ -1176,14 +1176,9 @@ class Req(ReqDllmMixin):
self.mm_video_tokens: int = 0
# Prefix info
# Tokens [0, prefix_len) are already in the row or come from a match;
# the next extend starts at prefix_len.
# Tokens [0, prefix_len) are already in the row, or, before allocation,
# on the path to last_node; the next extend starts at prefix_len.
self.prefix_len: int = 0
# The matched prefix's KV indices, consumed when the next allocation
# writes them into the row; None once written.
self.prefix_indices: Optional[torch.Tensor] = torch.empty(
(0,), dtype=torch.int64
)
# TODO(ispobock): rename to last_device_node
self.last_node: Any = None
self.last_host_node: Any = None
@@ -1571,10 +1566,6 @@ class Req(ReqDllmMixin):
def set_extend_range(self, start: int, end: int) -> None:
self.extend_range = Range(start, end)
def set_prefix_indices(self, prefix_indices: torch.Tensor) -> None:
self.prefix_indices = prefix_indices
self.prefix_len = len(prefix_indices)
def get_fill_ids(self) -> array:
return self.full_untruncated_fill_ids[: self.extend_range.end]
@@ -1934,7 +1925,7 @@ class Req(ReqDllmMixin):
# since we are tracking the total number of retractions for each request.
self.retraction_count += 1
self.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
self.prefix_len = 0
self.routed_experts = None
self.indexer_topk = None
self.last_node = None
@@ -1412,7 +1412,7 @@ class PrefillAdder:
)
if isinstance(admission, AddReqResult):
return admission
req.set_prefix_indices(torch.cat([req.prefix_indices, new_indices]))
req.prefix_len += len(new_indices)
req.kv.cache_protected_len = req.prefix_len
# Sharded pools cannot load host KV; reserve scratch after all other gates.
+14 -12
View File
@@ -48,6 +48,8 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
_EMPTY_PREFIX = torch.empty((0,), dtype=torch.int64)
def write_cache_indices(
out_cache_loc: torch.Tensor,
@@ -348,6 +350,17 @@ def _alloc_page_size(batch: ScheduleBatch) -> int:
return batch.tree_cache.page_size
def _prefix_kv_indices(batch: ScheduleBatch, req: Req) -> torch.Tensor:
# A request holding its row (a later chunk, a borrowed session record)
# already has its prefix there; a new one reads its match off the tree.
if req.kv.holds_kv:
row = batch.req_to_token_pool.req_to_token[req.kv.req_pool_idx]
return row[: req.prefix_len].to(torch.int64)
if req.prefix_len == 0:
return _EMPTY_PREFIX
return batch.tree_cache.prefix_device_indices(req)
def alloc_for_extend(
batch: ScheduleBatch,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
@@ -361,17 +374,7 @@ def alloc_for_extend(
# free out-of-window swa tokens
batch.maybe_evict_swa()
# A request holding its row (a later chunk, a borrowed session record)
# already has its prefix there; a new one writes its matched prefix.
req_to_token = batch.req_to_token_pool.req_to_token
prefix_tensors = [
(
req_to_token[r.kv.req_pool_idx, : r.prefix_len].to(torch.int64)
if r.kv.holds_kv
else r.prefix_indices
)
for r in batch.reqs
]
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
reuse_kv = None
if batch.is_dllm():
@@ -465,7 +468,6 @@ def alloc_for_extend(
for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()):
req.kv.kv_allocated_len = seq_len
req.kv.kv_committed_len = seq_len
req.prefix_indices = None
batch.tree_cache.maybe_hand_to_session(req)
return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu
@@ -528,6 +528,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
if lock is not None:
self.dec_lock_ref(lock.node, lock.receipt)
def prefix_device_indices(self, req: Req) -> torch.Tensor:
"""KV indices of req's matched prefix, read off the path to its locked
match node; valid from match until allocation writes them into the row."""
raise NotImplementedError
def maybe_hand_to_session(self, req: Req) -> None:
"""A cache that keeps records across requests (a streaming session) takes
the just-allocated row and the request's tree lock; the request borrows it."""
@@ -1249,12 +1249,7 @@ class BufferModePipeline:
)
self._refetch_staged(f)
return False
root_id = self._cache.tree_core.empty_match_result.last_device_node
full_indices = self._cache.tree_core.collect_full_device_indices(
node_id, root_id
)[:matched_len]
assert len(full_indices) == matched_len
req.set_prefix_indices(full_indices)
req.prefix_len = matched_len
req.last_node = node_id
req.kv.cache_protected_len = matched_len
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
@@ -1441,6 +1436,7 @@ class BufferModePipeline:
splice_base = plan.device_prefix_len
assert req.prefix_len == splice_base
prefix_indices = self._cache.prefix_device_indices(req)
trim_tokens = splice_base - f.matched_len
assert trim_tokens % cache.page_size == 0, (
f"staged splice trim not page-aligned req={req.rid}: "
@@ -1497,7 +1493,7 @@ class BufferModePipeline:
anchor_parts = []
host_parts = []
for repair_start, repair_end_ in repair_ranges:
anchor_parts.append(req.prefix_indices[repair_start:repair_end_])
anchor_parts.append(prefix_indices[repair_start:repair_end_])
host_parts.append(
slice(repair_start - window_start, repair_end_ - window_start)
)
@@ -1560,16 +1556,14 @@ class BufferModePipeline:
key,
repair_start,
repair_end_,
req.prefix_indices[repair_start:repair_end_],
prefix_indices[repair_start:repair_end_],
):
cache._apply_cache_action(action)
elif swa_dev is not None:
# Register the window's FULL->SWA translation now (attention reads
# through it). Keep SWA slots another request may still hold; their
# redundant H2D destinations are reclaimed at the transfer ack.
full_window = torch.cat([req.prefix_indices, device_indices])[
-len(swa_dev) :
]
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
allocator = cache.token_to_kv_pool_allocator
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
missing = old_swa <= 0
@@ -1619,7 +1613,7 @@ class BufferModePipeline:
insert_result = cache.insert(
InsertParams(
key=key,
value=torch.cat([req.prefix_indices, device_indices]),
value=torch.cat([prefix_indices, device_indices]),
prev_prefix_len=splice_base,
component_evicted_seqlens={
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
+1 -1
View File
@@ -209,7 +209,7 @@ def match_kv_cache(
match_result = zero_match_result(
tree_cache, match_result, extra_key=req.extra_key
)
req.set_prefix_indices(match_result.device_indices)
req.prefix_len = len(match_result.device_indices)
(
req.last_node,
req.last_host_node,
@@ -624,6 +624,17 @@ class RadixCache(BasePrefixCache):
##### Internal Helper Functions #####
def prefix_device_indices(self, req: Req) -> torch.Tensor:
values = []
node = req.last_node
while node is not self.root_node:
values.append(node.value)
node = node.parent
values.reverse()
path = torch.cat(values)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
def _match_prefix_helper(self, node: TreeNode, key: RadixKey):
access_time = time.monotonic()
node.last_access_time = access_time
@@ -87,6 +87,20 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
# LMCache rebuilds lookup keys from tokens, so any L1 node is valid.
return True
def prefix_device_indices(self, req: Req) -> torch.Tensor:
# match_prefix may append loaded slots not yet published to the tree,
# so the path alone can be shorter than the prefix.
root = self.root_node_handle(req.extra_key)
path = self.tree_core.collect_full_device_indices(req.last_node, root)
indices = path[: req.prefix_len]
missing = req.prefix_len - len(indices)
if missing > 0:
load = self._external_flows[req.rid].load
skip = max(len(indices) - load.local_hit_tokens, 0)
indices = torch.cat([indices, load.device_indices[skip : skip + missing]])
assert len(indices) == req.prefix_len, (req.rid, len(indices), req.prefix_len)
return indices
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
"""Report LMCache hits through host fields until GPU slots are ready."""
requested_key_len = len(params.key)
@@ -338,7 +338,7 @@ class UnifiedCacheLinkerWrapper:
# Insert the newly loaded tail into the tree.
prefix_indices = torch.cat(
[req.prefix_indices.to(torch.int64), full_transfer.device_indices]
[cache.prefix_device_indices(req), full_transfer.device_indices]
)
mamba_transfer = next(
(
@@ -3801,6 +3801,12 @@ class UnifiedRadixCache(BasePrefixCache):
# Internal callers (and the session sentinel / None) pass a non-int through.
return node_handle
def prefix_device_indices(self, req: Req) -> torch.Tensor:
root = self.root_node_handle(req.extra_key)
path = self.tree_core.collect_full_device_indices(req.last_node, root)
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
return path[: req.prefix_len]
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
"""The root's NodeId -- URC match results carry NodeIds."""
return self.tree_core.root_node_handle(extra_key)