From 393eaed5093ec106c43658257d08e59fa670ad28 Mon Sep 17 00:00:00 2001 From: Shenxiu Liu Date: Thu, 8 Oct 2026 12:19:35 -0700 Subject: [PATCH] [Fix] overlap scheduler: record_stream mix_running_indices for the forward-stream relay gather (#34076) Co-authored-by: Qiaolin-Yu --- python/sglang/srt/managers/overlap_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/sglang/srt/managers/overlap_utils.py b/python/sglang/srt/managers/overlap_utils.py index e43920cd41..4fb1981633 100644 --- a/python/sglang/srt/managers/overlap_utils.py +++ b/python/sglang/srt/managers/overlap_utils.py @@ -93,6 +93,13 @@ def resolve_forward_inputs(batch: ScheduleBatch, future_map: FutureMap) -> None: if batch.prefill_input_ids_cpu is not None: prefill_gpu = batch.prefill_input_ids_cpu.to(batch.device, non_blocking=True) if batch.mix_running_indices is not None: + # Keep mix_running_indices (= the mixed-in running batch's req_pool_indices, freed + # after merge_batch) alive on the forward stream until this gather completes. + # CPU has no cross-stream reuse hazard (and record_stream rejects CPU tensors). + if batch.mix_running_indices.device.type != "cpu": + batch.mix_running_indices.record_stream( + torch.get_device_module(batch.device).current_stream() + ) if batch.enable_overlap and not batch.spec_algorithm.is_none(): future_map.resolve_mixed_spec_tails(batch) decode_gpu = future_map.output_tokens_buf[batch.mix_running_indices]