[Scheduler] Derive prefix KV indices from last_node at allocation; drop Req.prefix_indices (#42825)
This commit is contained in:
@@ -438,11 +438,8 @@ def prepare_extend_inputs_for_correctness_test(
|
||||
req.full_untruncated_fill_ids.extend(input_ids[i][bench_args.cut_len :])
|
||||
if model_runner is not None:
|
||||
# Use req.kv.req_pool_idx instead of i to handle slot 0 padding correctly
|
||||
req.set_prefix_indices(
|
||||
model_runner.req_to_token_pool.req_to_token[
|
||||
req.kv.req_pool_idx, : bench_args.cut_len
|
||||
].to(torch.int64)
|
||||
)
|
||||
# The cut prefix is already in the request's row from the first extend.
|
||||
req.prefix_len = bench_args.cut_len
|
||||
req.logprob_start_len = -1
|
||||
req.set_extend_range(req.prefix_len, len(req.full_untruncated_fill_ids))
|
||||
return reqs
|
||||
|
||||
@@ -145,7 +145,7 @@ def _bootstrap_addr(req: Req) -> str:
|
||||
|
||||
def _bind_root_prefix(req: Req, tree_cache: BasePrefixCache) -> None:
|
||||
"""Start a decode-radix request that owns its whole KV row at the root."""
|
||||
req.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
|
||||
req.prefix_len = 0
|
||||
req.last_node = tree_cache.root_node_handle(req.extra_key)
|
||||
req.last_host_node = req.last_node
|
||||
req.best_match_node = req.last_node
|
||||
@@ -2204,9 +2204,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin):
|
||||
req.full_untruncated_fill_ids = req.origin_input_ids + req.output_ids
|
||||
# Decode-disagg allocates before batch assembly, so it binds the prefix
|
||||
# here instead of in init_next_round_input.
|
||||
req.set_prefix_indices(
|
||||
prefix_indices if prefix_len > 0 else torch.empty((0,), dtype=torch.int64)
|
||||
)
|
||||
req.prefix_len = prefix_len
|
||||
req.set_extend_range(total_prefix_len, req.kv.kv_committed_len)
|
||||
self.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
|
||||
@@ -211,7 +211,7 @@ class DecodeHiCacheTransferMixin:
|
||||
return False
|
||||
self.tree_cache.pop_prefetch_loaded_tokens(dr.req.cache_request_handle)
|
||||
|
||||
# Re-match: req.last_node / prefix_indices updated to current device state.
|
||||
# Re-match: req.last_node / prefix_len updated to current device state.
|
||||
rematch = match_kv_cache(
|
||||
dr.req,
|
||||
self.tree_cache,
|
||||
@@ -316,11 +316,7 @@ class DecodeHiCacheTransferMixin:
|
||||
),
|
||||
decode_req.hicache_restored_kv_indices,
|
||||
)
|
||||
req.set_prefix_indices(
|
||||
torch.cat(
|
||||
[prefix_match.prefix_indices, decode_req.hicache_restored_kv_indices]
|
||||
)
|
||||
)
|
||||
req.prefix_len = prefix_match.decode_prefix_len
|
||||
req.last_node = restore_lock.node
|
||||
req.lock = restore_lock
|
||||
# Prevent abort cleanup from releasing the transferred lock.
|
||||
|
||||
@@ -1176,14 +1176,9 @@ class Req(ReqDllmMixin):
|
||||
self.mm_video_tokens: int = 0
|
||||
|
||||
# Prefix info
|
||||
# Tokens [0, prefix_len) are already in the row or come from a match;
|
||||
# the next extend starts at prefix_len.
|
||||
# Tokens [0, prefix_len) are already in the row, or, before allocation,
|
||||
# on the path to last_node; the next extend starts at prefix_len.
|
||||
self.prefix_len: int = 0
|
||||
# The matched prefix's KV indices, consumed when the next allocation
|
||||
# writes them into the row; None once written.
|
||||
self.prefix_indices: Optional[torch.Tensor] = torch.empty(
|
||||
(0,), dtype=torch.int64
|
||||
)
|
||||
# TODO(ispobock): rename to last_device_node
|
||||
self.last_node: Any = None
|
||||
self.last_host_node: Any = None
|
||||
@@ -1571,10 +1566,6 @@ class Req(ReqDllmMixin):
|
||||
def set_extend_range(self, start: int, end: int) -> None:
|
||||
self.extend_range = Range(start, end)
|
||||
|
||||
def set_prefix_indices(self, prefix_indices: torch.Tensor) -> None:
|
||||
self.prefix_indices = prefix_indices
|
||||
self.prefix_len = len(prefix_indices)
|
||||
|
||||
def get_fill_ids(self) -> array:
|
||||
return self.full_untruncated_fill_ids[: self.extend_range.end]
|
||||
|
||||
@@ -1934,7 +1925,7 @@ class Req(ReqDllmMixin):
|
||||
# since we are tracking the total number of retractions for each request.
|
||||
self.retraction_count += 1
|
||||
|
||||
self.set_prefix_indices(torch.empty((0,), dtype=torch.int64))
|
||||
self.prefix_len = 0
|
||||
self.routed_experts = None
|
||||
self.indexer_topk = None
|
||||
self.last_node = None
|
||||
|
||||
@@ -1412,7 +1412,7 @@ class PrefillAdder:
|
||||
)
|
||||
if isinstance(admission, AddReqResult):
|
||||
return admission
|
||||
req.set_prefix_indices(torch.cat([req.prefix_indices, new_indices]))
|
||||
req.prefix_len += len(new_indices)
|
||||
req.kv.cache_protected_len = req.prefix_len
|
||||
|
||||
# Sharded pools cannot load host KV; reserve scratch after all other gates.
|
||||
|
||||
@@ -48,6 +48,8 @@ if TYPE_CHECKING:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_EMPTY_PREFIX = torch.empty((0,), dtype=torch.int64)
|
||||
|
||||
|
||||
def write_cache_indices(
|
||||
out_cache_loc: torch.Tensor,
|
||||
@@ -348,6 +350,17 @@ def _alloc_page_size(batch: ScheduleBatch) -> int:
|
||||
return batch.tree_cache.page_size
|
||||
|
||||
|
||||
def _prefix_kv_indices(batch: ScheduleBatch, req: Req) -> torch.Tensor:
|
||||
# A request holding its row (a later chunk, a borrowed session record)
|
||||
# already has its prefix there; a new one reads its match off the tree.
|
||||
if req.kv.holds_kv:
|
||||
row = batch.req_to_token_pool.req_to_token[req.kv.req_pool_idx]
|
||||
return row[: req.prefix_len].to(torch.int64)
|
||||
if req.prefix_len == 0:
|
||||
return _EMPTY_PREFIX
|
||||
return batch.tree_cache.prefix_device_indices(req)
|
||||
|
||||
|
||||
def alloc_for_extend(
|
||||
batch: ScheduleBatch,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
@@ -361,17 +374,7 @@ def alloc_for_extend(
|
||||
# free out-of-window swa tokens
|
||||
batch.maybe_evict_swa()
|
||||
|
||||
# A request holding its row (a later chunk, a borrowed session record)
|
||||
# already has its prefix there; a new one writes its matched prefix.
|
||||
req_to_token = batch.req_to_token_pool.req_to_token
|
||||
prefix_tensors = [
|
||||
(
|
||||
req_to_token[r.kv.req_pool_idx, : r.prefix_len].to(torch.int64)
|
||||
if r.kv.holds_kv
|
||||
else r.prefix_indices
|
||||
)
|
||||
for r in batch.reqs
|
||||
]
|
||||
prefix_tensors = [_prefix_kv_indices(batch, r) for r in batch.reqs]
|
||||
|
||||
reuse_kv = None
|
||||
if batch.is_dllm():
|
||||
@@ -465,7 +468,6 @@ def alloc_for_extend(
|
||||
for req, seq_len in zip(batch.reqs, batch.seq_lens_cpu.tolist()):
|
||||
req.kv.kv_allocated_len = seq_len
|
||||
req.kv.kv_committed_len = seq_len
|
||||
req.prefix_indices = None
|
||||
batch.tree_cache.maybe_hand_to_session(req)
|
||||
|
||||
return out_cache_loc, req_pool_indices_device, req_pool_indices_cpu
|
||||
|
||||
@@ -528,6 +528,11 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
if lock is not None:
|
||||
self.dec_lock_ref(lock.node, lock.receipt)
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
"""KV indices of req's matched prefix, read off the path to its locked
|
||||
match node; valid from match until allocation writes them into the row."""
|
||||
raise NotImplementedError
|
||||
|
||||
def maybe_hand_to_session(self, req: Req) -> None:
|
||||
"""A cache that keeps records across requests (a streaming session) takes
|
||||
the just-allocated row and the request's tree lock; the request borrows it."""
|
||||
|
||||
@@ -1249,12 +1249,7 @@ class BufferModePipeline:
|
||||
)
|
||||
self._refetch_staged(f)
|
||||
return False
|
||||
root_id = self._cache.tree_core.empty_match_result.last_device_node
|
||||
full_indices = self._cache.tree_core.collect_full_device_indices(
|
||||
node_id, root_id
|
||||
)[:matched_len]
|
||||
assert len(full_indices) == matched_len
|
||||
req.set_prefix_indices(full_indices)
|
||||
req.prefix_len = matched_len
|
||||
req.last_node = node_id
|
||||
req.kv.cache_protected_len = matched_len
|
||||
full_tokens = max(0, f.matched_len + f.num_tokens - matched_len)
|
||||
@@ -1441,6 +1436,7 @@ class BufferModePipeline:
|
||||
|
||||
splice_base = plan.device_prefix_len
|
||||
assert req.prefix_len == splice_base
|
||||
prefix_indices = self._cache.prefix_device_indices(req)
|
||||
trim_tokens = splice_base - f.matched_len
|
||||
assert trim_tokens % cache.page_size == 0, (
|
||||
f"staged splice trim not page-aligned req={req.rid}: "
|
||||
@@ -1497,7 +1493,7 @@ class BufferModePipeline:
|
||||
anchor_parts = []
|
||||
host_parts = []
|
||||
for repair_start, repair_end_ in repair_ranges:
|
||||
anchor_parts.append(req.prefix_indices[repair_start:repair_end_])
|
||||
anchor_parts.append(prefix_indices[repair_start:repair_end_])
|
||||
host_parts.append(
|
||||
slice(repair_start - window_start, repair_end_ - window_start)
|
||||
)
|
||||
@@ -1560,16 +1556,14 @@ class BufferModePipeline:
|
||||
key,
|
||||
repair_start,
|
||||
repair_end_,
|
||||
req.prefix_indices[repair_start:repair_end_],
|
||||
prefix_indices[repair_start:repair_end_],
|
||||
):
|
||||
cache._apply_cache_action(action)
|
||||
elif swa_dev is not None:
|
||||
# Register the window's FULL->SWA translation now (attention reads
|
||||
# through it). Keep SWA slots another request may still hold; their
|
||||
# redundant H2D destinations are reclaimed at the transfer ack.
|
||||
full_window = torch.cat([req.prefix_indices, device_indices])[
|
||||
-len(swa_dev) :
|
||||
]
|
||||
full_window = torch.cat([prefix_indices, device_indices])[-len(swa_dev) :]
|
||||
allocator = cache.token_to_kv_pool_allocator
|
||||
old_swa = allocator.translate_swa_indices_for_transfer(full_window)
|
||||
missing = old_swa <= 0
|
||||
@@ -1619,7 +1613,7 @@ class BufferModePipeline:
|
||||
insert_result = cache.insert(
|
||||
InsertParams(
|
||||
key=key,
|
||||
value=torch.cat([req.prefix_indices, device_indices]),
|
||||
value=torch.cat([prefix_indices, device_indices]),
|
||||
prev_prefix_len=splice_base,
|
||||
component_evicted_seqlens={
|
||||
ComponentType.SWA: (span_end - staged_swa) if staged_swa else 0
|
||||
|
||||
@@ -209,7 +209,7 @@ def match_kv_cache(
|
||||
match_result = zero_match_result(
|
||||
tree_cache, match_result, extra_key=req.extra_key
|
||||
)
|
||||
req.set_prefix_indices(match_result.device_indices)
|
||||
req.prefix_len = len(match_result.device_indices)
|
||||
(
|
||||
req.last_node,
|
||||
req.last_host_node,
|
||||
|
||||
@@ -624,6 +624,17 @@ class RadixCache(BasePrefixCache):
|
||||
|
||||
##### Internal Helper Functions #####
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
values = []
|
||||
node = req.last_node
|
||||
while node is not self.root_node:
|
||||
values.append(node.value)
|
||||
node = node.parent
|
||||
values.reverse()
|
||||
path = torch.cat(values)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
|
||||
def _match_prefix_helper(self, node: TreeNode, key: RadixKey):
|
||||
access_time = time.monotonic()
|
||||
node.last_access_time = access_time
|
||||
|
||||
@@ -87,6 +87,20 @@ class LMCacheUnifiedRadixCache(UnifiedRadixCache):
|
||||
# LMCache rebuilds lookup keys from tokens, so any L1 node is valid.
|
||||
return True
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
# match_prefix may append loaded slots not yet published to the tree,
|
||||
# so the path alone can be shorter than the prefix.
|
||||
root = self.root_node_handle(req.extra_key)
|
||||
path = self.tree_core.collect_full_device_indices(req.last_node, root)
|
||||
indices = path[: req.prefix_len]
|
||||
missing = req.prefix_len - len(indices)
|
||||
if missing > 0:
|
||||
load = self._external_flows[req.rid].load
|
||||
skip = max(len(indices) - load.local_hit_tokens, 0)
|
||||
indices = torch.cat([indices, load.device_indices[skip : skip + missing]])
|
||||
assert len(indices) == req.prefix_len, (req.rid, len(indices), req.prefix_len)
|
||||
return indices
|
||||
|
||||
def match_prefix(self, params: MatchPrefixParams) -> MatchResult:
|
||||
"""Report LMCache hits through host fields until GPU slots are ready."""
|
||||
requested_key_len = len(params.key)
|
||||
|
||||
@@ -338,7 +338,7 @@ class UnifiedCacheLinkerWrapper:
|
||||
|
||||
# Insert the newly loaded tail into the tree.
|
||||
prefix_indices = torch.cat(
|
||||
[req.prefix_indices.to(torch.int64), full_transfer.device_indices]
|
||||
[cache.prefix_device_indices(req), full_transfer.device_indices]
|
||||
)
|
||||
mamba_transfer = next(
|
||||
(
|
||||
|
||||
@@ -3801,6 +3801,12 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
# Internal callers (and the session sentinel / None) pass a non-int through.
|
||||
return node_handle
|
||||
|
||||
def prefix_device_indices(self, req: Req) -> torch.Tensor:
|
||||
root = self.root_node_handle(req.extra_key)
|
||||
path = self.tree_core.collect_full_device_indices(req.last_node, root)
|
||||
assert len(path) >= req.prefix_len, (req.rid, len(path), req.prefix_len)
|
||||
return path[: req.prefix_len]
|
||||
|
||||
def root_node_handle(self, extra_key: Optional[str] = None) -> NodeId:
|
||||
"""The root's NodeId -- URC match results carry NodeIds."""
|
||||
return self.tree_core.root_node_handle(extra_key)
|
||||
|
||||
Reference in New Issue
Block a user