From 7dd93b01a5421bd0a52cd78a9ef1dce3457db07b Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 2 Oct 2026 13:48:39 -0700 Subject: [PATCH] [PD] Make the prebuilt last-token H2D copy non-blocking under overlap (#42113) Co-authored-by: merrymercy Co-authored-by: bbus --- .../srt/disaggregation/decode_schedule_batch_mixin.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py index 11fafbab4b..23dad06d19 100644 --- a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload from sglang.srt.mem_cache.common import checkpoint_kv_cache from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo +from sglang.srt.utils.common import is_pin_memory_available logger = logging.getLogger(__name__) @@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin: error_message, HTTPStatus.INTERNAL_SERVER_ERROR ) req.grammar.finished = req.finished() + # Non-blocking H2D: with overlap this runs on the schedule stream after + # its wait on the in-flight forward, so a blocking copy would stall the + # host until that forward ends and leave the GPU idle. last_tokens_tensor = torch.tensor( - last_tokens, dtype=torch.int64, device=self.device - ) + last_tokens, + dtype=torch.int64, + pin_memory=is_pin_memory_available(self.device), + ).to(self.device, non_blocking=True) spec_info = self.spec_algorithm.build_disagg_draft_input( self,