diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py index 5b5aada9f8..a828c7fc5a 100644 --- a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload from sglang.srt.mem_cache.common import maybe_cache_unfinished_req from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo +from sglang.srt.utils.common import is_pin_memory_available logger = logging.getLogger(__name__) @@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin: error_message, HTTPStatus.INTERNAL_SERVER_ERROR ) req.grammar.finished = req.finished() + # Non-blocking H2D: with overlap this runs on the schedule stream after + # its wait on the in-flight forward, so a blocking copy would stall the + # host until that forward ends and leave the GPU idle. last_tokens_tensor = torch.tensor( - last_tokens, dtype=torch.int64, device=self.device - ) + last_tokens, + dtype=torch.int64, + pin_memory=is_pin_memory_available(self.device), + ).to(self.device, non_blocking=True) spec_info = self.spec_algorithm.build_disagg_draft_input( self,