[Fix] overlap scheduler: record_stream mix_running_indices for the forward-stream relay gather (#34076)
Co-authored-by: Qiaolin-Yu <liin1211@outlook.com>
This commit is contained in:
co-authored by
Qiaolin-Yu
parent
1638c5d699
commit
393eaed509
@@ -93,6 +93,13 @@ def resolve_forward_inputs(batch: ScheduleBatch, future_map: FutureMap) -> None:
|
||||
if batch.prefill_input_ids_cpu is not None:
|
||||
prefill_gpu = batch.prefill_input_ids_cpu.to(batch.device, non_blocking=True)
|
||||
if batch.mix_running_indices is not None:
|
||||
# Keep mix_running_indices (= the mixed-in running batch's req_pool_indices, freed
|
||||
# after merge_batch) alive on the forward stream until this gather completes.
|
||||
# CPU has no cross-stream reuse hazard (and record_stream rejects CPU tensors).
|
||||
if batch.mix_running_indices.device.type != "cpu":
|
||||
batch.mix_running_indices.record_stream(
|
||||
torch.get_device_module(batch.device).current_stream()
|
||||
)
|
||||
if batch.enable_overlap and not batch.spec_algorithm.is_none():
|
||||
future_map.resolve_mixed_spec_tails(batch)
|
||||
decode_gpu = future_map.output_tokens_buf[batch.mix_running_indices]
|
||||
|
||||
Reference in New Issue
Block a user