[Perf] pd: Copy prebuilt decode batch inputs to the GPU without host syncs (#44598)

This commit is contained in:
Liangsheng Yin
2026-10-15 17:20:02 -07:00
committed by GitHub
parent 4cca7fcd3e
commit d14fa5db9b
3 changed files with 32 additions and 33 deletions
@@ -10,7 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.mem_cache.common import checkpoint_kv_cache
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
from sglang.srt.utils.common import is_pin_memory_available
from sglang.srt.utils.common import async_h2d
logger = logging.getLogger(__name__)
@@ -81,15 +81,13 @@ class ScheduleBatchDisaggregationDecodeMixin:
# process_prebuilt through FutureMap, so merge/forward-entry rebuilds
# input_ids from the relay.
self.input_ids = None
self.req_pool_indices = torch.tensor(
self.req_pool_indices = async_h2d(
req_pool_indices, dtype=torch.int64, device=self.device
)
self.req_pool_indices_cpu = torch.tensor(req_pool_indices, dtype=torch.int64)
self.seq_lens = torch.tensor(seq_lens, dtype=torch.int64, device=self.device)
self.seq_lens = async_h2d(seq_lens, dtype=torch.int64, device=self.device)
self.seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.int64)
self.orig_seq_lens = torch.tensor(
seq_lens, dtype=torch.int32, device=self.device
)
self.orig_seq_lens = async_h2d(seq_lens, dtype=torch.int32, device=self.device)
self.out_cache_loc = out_cache_loc
self.seq_lens_sum = sum(seq_lens)
@@ -141,14 +139,9 @@ class ScheduleBatchDisaggregationDecodeMixin:
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
)
req.grammar.finished = req.finished()
# Non-blocking H2D: with overlap this runs on the schedule stream after
# its wait on the in-flight forward, so a blocking copy would stall the
# host until that forward ends and leave the GPU idle.
last_tokens_tensor = torch.tensor(
last_tokens,
dtype=torch.int64,
pin_memory=is_pin_memory_available(self.device),
).to(self.device, non_blocking=True)
last_tokens_tensor = async_h2d(
last_tokens, dtype=torch.int64, device=self.device
)
spec_info = self.spec_algorithm.build_disagg_draft_input(
self,
@@ -12,6 +12,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
from sglang.srt.runtime_context import get_spec
from sglang.srt.speculative.eagle_info import EagleDraftInput
from sglang.srt.utils.common import async_h2d_stack
if TYPE_CHECKING:
from sglang.srt.managers.overlap_utils import FutureMap
@@ -58,37 +59,28 @@ def build_eagle_disagg_draft_input(
if spec.enable_multi_layer_eagle:
num_states *= spec.speculative_num_steps
topk_p = torch.stack(
topk_p = async_h2d_stack(
[
torch.as_tensor(
req.output_topk_p[:num_states],
device=batch.device,
dtype=torch.float32,
)
torch.as_tensor(req.output_topk_p[:num_states], dtype=torch.float32)
for req in batch.reqs
],
dim=0,
device=batch.device,
)
topk_index = torch.stack(
topk_index = async_h2d_stack(
[
torch.as_tensor(
req.output_topk_index[:num_states],
device=batch.device,
dtype=torch.int64,
)
torch.as_tensor(req.output_topk_index[:num_states], dtype=torch.int64)
for req in batch.reqs
],
dim=0,
device=batch.device,
)
hidden_states = async_h2d_stack(
[req.hidden_states_tensor for req in batch.reqs], device=batch.device
)
hidden_states = torch.stack(
[req.hidden_states_tensor for req in batch.reqs], dim=0
).to(batch.device)
dsa_topk_indices = None
dsa_indices_list = [req.output_dsa_topk_indices for req in batch.reqs]
if dsa_indices_list and all(t is not None for t in dsa_indices_list):
dsa_topk_indices = torch.stack(dsa_indices_list, dim=0).to(batch.device)
dsa_topk_indices = async_h2d_stack(dsa_indices_list, device=batch.device)
if should_remap_pd_dsa_seed_to_local_slots():
# PD sends request-relative positions; fused TopK consumes
# decode-local physical slots. Remap once before the draft loop/graph.
+14
View File
@@ -593,6 +593,20 @@ def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch.
return torch.tensor(args, dtype=dtype, pin_memory=pin).to(device, non_blocking=pin)
def async_h2d_stack(
tensors: List[torch.Tensor], *, device: torch.device
) -> torch.Tensor:
"""``torch.stack(tensors)`` on ``device``; host tensors are stacked straight
into pinned memory and copied like ``async_h2d``."""
if not tensors[0].is_cpu:
return torch.stack(tensors).to(device)
pin = is_pin_memory_available(device)
out = torch.empty(
(len(tensors), *tensors[0].shape), dtype=tensors[0].dtype, pin_memory=pin
)
return torch.stack(tensors, out=out).to(device, non_blocking=pin)
def async_d2h(tensor: torch.Tensor) -> torch.Tensor:
"""Enqueue an accelerator-to-pinned-host copy on the current stream."""
if tensor.device.type == "cpu":