diff --git a/python/sglang/srt/managers/overlap_utils.py b/python/sglang/srt/managers/overlap_utils.py index e43920cd41..4fb1981633 100644 --- a/python/sglang/srt/managers/overlap_utils.py +++ b/python/sglang/srt/managers/overlap_utils.py @@ -93,6 +93,13 @@ def resolve_forward_inputs(batch: ScheduleBatch, future_map: FutureMap) -> None: if batch.prefill_input_ids_cpu is not None: prefill_gpu = batch.prefill_input_ids_cpu.to(batch.device, non_blocking=True) if batch.mix_running_indices is not None: + # Keep mix_running_indices (= the mixed-in running batch's req_pool_indices, freed + # after merge_batch) alive on the forward stream until this gather completes. + # CPU has no cross-stream reuse hazard (and record_stream rejects CPU tensors). + if batch.mix_running_indices.device.type != "cpu": + batch.mix_running_indices.record_stream( + torch.get_device_module(batch.device).current_stream() + ) if batch.enable_overlap and not batch.spec_algorithm.is_none(): future_map.resolve_mixed_spec_tails(batch) decode_gpu = future_map.output_tokens_buf[batch.mix_running_indices]