[PD] Make the prebuilt last-token H2D copy non-blocking under overlap (#42113)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: bbus <bbus@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-02 13:48:39 -07:00
committed by GitHub
co-authored by merrymercy bbus
parent 07e1a4f082
commit 7dd93b01a5
@@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.mem_cache.common import checkpoint_kv_cache
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.utils.common import is_pin_memory_available
logger = logging.getLogger(__name__)
@@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin:
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
)
req.grammar.finished = req.finished()
# Non-blocking H2D: with overlap this runs on the schedule stream after
# its wait on the in-flight forward, so a blocking copy would stall the
# host until that forward ends and leave the GPU idle.
last_tokens_tensor = torch.tensor(
last_tokens, dtype=torch.int64, device=self.device
)
last_tokens,
dtype=torch.int64,
pin_memory=is_pin_memory_available(self.device),
).to(self.device, non_blocking=True)
spec_info = self.spec_algorithm.build_disagg_draft_input(
self,