[Perf] pd: Copy prebuilt decode batch inputs to the GPU without host syncs (#44598)
This commit is contained in:
@@ -10,7 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.mem_cache.common import checkpoint_kv_cache
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardMode
|
||||
from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo
|
||||
from sglang.srt.utils.common import is_pin_memory_available
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -81,15 +81,13 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
# process_prebuilt through FutureMap, so merge/forward-entry rebuilds
|
||||
# input_ids from the relay.
|
||||
self.input_ids = None
|
||||
self.req_pool_indices = torch.tensor(
|
||||
self.req_pool_indices = async_h2d(
|
||||
req_pool_indices, dtype=torch.int64, device=self.device
|
||||
)
|
||||
self.req_pool_indices_cpu = torch.tensor(req_pool_indices, dtype=torch.int64)
|
||||
self.seq_lens = torch.tensor(seq_lens, dtype=torch.int64, device=self.device)
|
||||
self.seq_lens = async_h2d(seq_lens, dtype=torch.int64, device=self.device)
|
||||
self.seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.int64)
|
||||
self.orig_seq_lens = torch.tensor(
|
||||
seq_lens, dtype=torch.int32, device=self.device
|
||||
)
|
||||
self.orig_seq_lens = async_h2d(seq_lens, dtype=torch.int32, device=self.device)
|
||||
self.out_cache_loc = out_cache_loc
|
||||
self.seq_lens_sum = sum(seq_lens)
|
||||
|
||||
@@ -141,14 +139,9 @@ class ScheduleBatchDisaggregationDecodeMixin:
|
||||
error_message, HTTPStatus.INTERNAL_SERVER_ERROR
|
||||
)
|
||||
req.grammar.finished = req.finished()
|
||||
# Non-blocking H2D: with overlap this runs on the schedule stream after
|
||||
# its wait on the in-flight forward, so a blocking copy would stall the
|
||||
# host until that forward ends and leave the GPU idle.
|
||||
last_tokens_tensor = torch.tensor(
|
||||
last_tokens,
|
||||
dtype=torch.int64,
|
||||
pin_memory=is_pin_memory_available(self.device),
|
||||
).to(self.device, non_blocking=True)
|
||||
last_tokens_tensor = async_h2d(
|
||||
last_tokens, dtype=torch.int64, device=self.device
|
||||
)
|
||||
|
||||
spec_info = self.spec_algorithm.build_disagg_draft_input(
|
||||
self,
|
||||
|
||||
@@ -12,6 +12,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload
|
||||
from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode
|
||||
from sglang.srt.runtime_context import get_spec
|
||||
from sglang.srt.speculative.eagle_info import EagleDraftInput
|
||||
from sglang.srt.utils.common import async_h2d_stack
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.overlap_utils import FutureMap
|
||||
@@ -58,37 +59,28 @@ def build_eagle_disagg_draft_input(
|
||||
if spec.enable_multi_layer_eagle:
|
||||
num_states *= spec.speculative_num_steps
|
||||
|
||||
topk_p = torch.stack(
|
||||
topk_p = async_h2d_stack(
|
||||
[
|
||||
torch.as_tensor(
|
||||
req.output_topk_p[:num_states],
|
||||
device=batch.device,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
torch.as_tensor(req.output_topk_p[:num_states], dtype=torch.float32)
|
||||
for req in batch.reqs
|
||||
],
|
||||
dim=0,
|
||||
device=batch.device,
|
||||
)
|
||||
topk_index = torch.stack(
|
||||
topk_index = async_h2d_stack(
|
||||
[
|
||||
torch.as_tensor(
|
||||
req.output_topk_index[:num_states],
|
||||
device=batch.device,
|
||||
dtype=torch.int64,
|
||||
)
|
||||
torch.as_tensor(req.output_topk_index[:num_states], dtype=torch.int64)
|
||||
for req in batch.reqs
|
||||
],
|
||||
dim=0,
|
||||
device=batch.device,
|
||||
)
|
||||
hidden_states = async_h2d_stack(
|
||||
[req.hidden_states_tensor for req in batch.reqs], device=batch.device
|
||||
)
|
||||
|
||||
hidden_states = torch.stack(
|
||||
[req.hidden_states_tensor for req in batch.reqs], dim=0
|
||||
).to(batch.device)
|
||||
|
||||
dsa_topk_indices = None
|
||||
dsa_indices_list = [req.output_dsa_topk_indices for req in batch.reqs]
|
||||
if dsa_indices_list and all(t is not None for t in dsa_indices_list):
|
||||
dsa_topk_indices = torch.stack(dsa_indices_list, dim=0).to(batch.device)
|
||||
dsa_topk_indices = async_h2d_stack(dsa_indices_list, device=batch.device)
|
||||
if should_remap_pd_dsa_seed_to_local_slots():
|
||||
# PD sends request-relative positions; fused TopK consumes
|
||||
# decode-local physical slots. Remap once before the draft loop/graph.
|
||||
|
||||
@@ -593,6 +593,20 @@ def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch.
|
||||
return torch.tensor(args, dtype=dtype, pin_memory=pin).to(device, non_blocking=pin)
|
||||
|
||||
|
||||
def async_h2d_stack(
|
||||
tensors: List[torch.Tensor], *, device: torch.device
|
||||
) -> torch.Tensor:
|
||||
"""``torch.stack(tensors)`` on ``device``; host tensors are stacked straight
|
||||
into pinned memory and copied like ``async_h2d``."""
|
||||
if not tensors[0].is_cpu:
|
||||
return torch.stack(tensors).to(device)
|
||||
pin = is_pin_memory_available(device)
|
||||
out = torch.empty(
|
||||
(len(tensors), *tensors[0].shape), dtype=tensors[0].dtype, pin_memory=pin
|
||||
)
|
||||
return torch.stack(tensors, out=out).to(device, non_blocking=pin)
|
||||
|
||||
|
||||
def async_d2h(tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""Enqueue an accelerator-to-pinned-host copy on the current stream."""
|
||||
if tensor.device.type == "cpu":
|
||||
|
||||
Reference in New Issue
Block a user