[Fix] overlap scheduler: record_stream mix_running_indices for the forward-stream relay gather (#34076)

Co-authored-by: Qiaolin-Yu <liin1211@outlook.com>
This commit is contained in:
Shenxiu Liu
2026-10-08 12:19:35 -07:00
committed by GitHub
co-authored by Qiaolin-Yu
parent 1638c5d699
commit 393eaed509
@@ -93,6 +93,13 @@ def resolve_forward_inputs(batch: ScheduleBatch, future_map: FutureMap) -> None:
if batch.prefill_input_ids_cpu is not None:
prefill_gpu = batch.prefill_input_ids_cpu.to(batch.device, non_blocking=True)
if batch.mix_running_indices is not None:
# Keep mix_running_indices (= the mixed-in running batch's req_pool_indices, freed
# after merge_batch) alive on the forward stream until this gather completes.
# CPU has no cross-stream reuse hazard (and record_stream rejects CPU tensors).
if batch.mix_running_indices.device.type != "cpu":
batch.mix_running_indices.record_stream(
torch.get_device_module(batch.device).current_stream()
)
if batch.enable_overlap and not batch.spec_algorithm.is_none():
future_map.resolve_mixed_spec_tails(batch)
decode_gpu = future_map.output_tokens_buf[batch.mix_running_indices]