[Fix] spec: Take post-verify KV move targets from the verify window (#45491)

This commit is contained in:
Liangsheng Yin
2026-10-20 21:12:41 -07:00
committed by GitHub
parent 1c2f42470c
commit 573b5716f5
2 changed files with 105 additions and 10 deletions
+16 -10
View File
@@ -823,23 +823,29 @@ def move_accept_tokens_to_target_kvcache(
device=device,
)
accept_out_cache_loc = torch.zeros(size, dtype=torch.int64, device=device)
# The targets lead each request's verify window. Read them from that window,
# not req_to_token: after verify's read-done the scheduler may hand a finished
# request's row to a new request.
window = batch.out_cache_loc.view(bs, -1)
window_rows = torch.arange(bs, device=device)
window_starts = torch.zeros_like(num_correct_drafts)
if _is_cpu:
assign_extend_cache_locs_cpu(
batch.req_pool_indices,
batch.req_to_token_pool.req_to_token,
batch.seq_lens,
batch.seq_lens + num_correct_drafts + 1,
window_rows,
window.int(),
window_starts,
num_correct_drafts + 1,
tgt_cache_loc,
batch.req_to_token_pool.req_to_token.shape[1],
window.shape[1],
)
else:
assign_extend_cache_locs[(bs,)](
batch.req_pool_indices,
batch.req_to_token_pool.req_to_token,
batch.seq_lens,
batch.seq_lens + num_correct_drafts + 1,
window_rows,
window,
window_starts,
num_correct_drafts + 1,
tgt_cache_loc,
batch.req_to_token_pool.req_to_token.shape[1],
window.shape[1],
next_power_of_2(bs),
)
fill_accept_out_cache_loc_func(