[PD] Make the prebuilt last-token H2D copy non-blocking under overlap (#42113)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: bbus <bbus@users.noreply.github.com>
(cherry picked from commit 7dd93b01a5)
This commit is contained in:
metamergebot
2026-10-08 15:32:46 +08:00
committed by minke.yu
parent 842ad7c4d4
commit 6deac260bf
@@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.utils.common import is_pin_memory_available
logger = logging.getLogger(__name__)
@@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin:
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
)
req.grammar.finished = req.finished()
# Non-blocking H2D: with overlap this runs on the schedule stream after
# its wait on the in-flight forward, so a blocking copy would stall the
# host until that forward ends and leave the GPU idle.
last_tokens_tensor = torch.tensor(
last_tokens, dtype=torch.int64, device=self.device
)
last_tokens,
dtype=torch.int64,
pin_memory=is_pin_memory_available(self.device),
).to(self.device, non_blocking=True)
spec_info = self.spec_algorithm.build_disagg_draft_input(
self,