[PD] Make the prebuilt last-token H2D copy non-blocking under overlap (#42113)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: bbus <bbus@users.noreply.github.com>
(cherry picked from commit 7dd93b01a5)
This commit is contained in:
@@ -10,6 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.mem_cache.common import maybe_cache_unfinished_req
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -140,9 +141,14 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
req.grammar.finished = req.finished()
|
||||
# Non-blocking H2D: with overlap this runs on the schedule stream after
|
||||
# its wait on the in-flight forward, so a blocking copy would stall the
|
||||
# host until that forward ends and leave the GPU idle.
|
||||
last_tokens_tensor = torch.tensor(
|
||||
last_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
last_tokens,
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(self.device),
|
||||
).to(self.device, non_blocking=True)
|
||||
|
||||
spec_info = self.spec_algorithm.build_disagg_draft_input(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user