diff --git a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py index 11602ed6cb..4c48656614 100644 --- a/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py +++ b/python/sglang/srt/disaggregation/decode_schedule_batch_mixin.py @@ -10,7 +10,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload from sglang.srt.mem_cache.common import checkpoint_kv_cache from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.srt.sampling.sampling_batch_info import SamplingBatchInfo -from sglang.srt.utils.common import is_pin_memory_available +from sglang.srt.utils.common import async_h2d logger = logging.getLogger(__name__) @@ -81,15 +81,13 @@ class ScheduleBatchDisaggregationDecodeMixin: # process_prebuilt through FutureMap, so merge/forward-entry rebuilds # input_ids from the relay. self.input_ids = None - self.req_pool_indices = torch.tensor( + self.req_pool_indices = async_h2d( req_pool_indices, dtype=torch.int64, device=self.device ) self.req_pool_indices_cpu = torch.tensor(req_pool_indices, dtype=torch.int64) - self.seq_lens = torch.tensor(seq_lens, dtype=torch.int64, device=self.device) + self.seq_lens = async_h2d(seq_lens, dtype=torch.int64, device=self.device) self.seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.int64) - self.orig_seq_lens = torch.tensor( - seq_lens, dtype=torch.int32, device=self.device - ) + self.orig_seq_lens = async_h2d(seq_lens, dtype=torch.int32, device=self.device) self.out_cache_loc = out_cache_loc self.seq_lens_sum = sum(seq_lens) @@ -141,14 +139,9 @@ class ScheduleBatchDisaggregationDecodeMixin: error_message, HTTPStatus.INTERNAL_SERVER_ERROR ) req.grammar.finished = req.finished() - # Non-blocking H2D: with overlap this runs on the schedule stream after - # its wait on the in-flight forward, so a blocking copy would stall the - # host until that forward ends and leave the GPU idle. - last_tokens_tensor = torch.tensor( - last_tokens, - dtype=torch.int64, - pin_memory=is_pin_memory_available(self.device), - ).to(self.device, non_blocking=True) + last_tokens_tensor = async_h2d( + last_tokens, dtype=torch.int64, device=self.device + ) spec_info = self.spec_algorithm.build_disagg_draft_input( self, diff --git a/python/sglang/srt/speculative/eagle_disaggregation.py b/python/sglang/srt/speculative/eagle_disaggregation.py index b8541c5fc2..f8b5a23bb6 100644 --- a/python/sglang/srt/speculative/eagle_disaggregation.py +++ b/python/sglang/srt/speculative/eagle_disaggregation.py @@ -12,6 +12,7 @@ from sglang.srt.managers.overlap_utils import RelayPayload from sglang.srt.model_executor.forward_batch_info import CaptureHiddenMode from sglang.srt.runtime_context import get_spec from sglang.srt.speculative.eagle_info import EagleDraftInput +from sglang.srt.utils.common import async_h2d_stack if TYPE_CHECKING: from sglang.srt.managers.overlap_utils import FutureMap @@ -58,37 +59,28 @@ def build_eagle_disagg_draft_input( if spec.enable_multi_layer_eagle: num_states *= spec.speculative_num_steps - topk_p = torch.stack( + topk_p = async_h2d_stack( [ - torch.as_tensor( - req.output_topk_p[:num_states], - device=batch.device, - dtype=torch.float32, - ) + torch.as_tensor(req.output_topk_p[:num_states], dtype=torch.float32) for req in batch.reqs ], - dim=0, + device=batch.device, ) - topk_index = torch.stack( + topk_index = async_h2d_stack( [ - torch.as_tensor( - req.output_topk_index[:num_states], - device=batch.device, - dtype=torch.int64, - ) + torch.as_tensor(req.output_topk_index[:num_states], dtype=torch.int64) for req in batch.reqs ], - dim=0, + device=batch.device, + ) + hidden_states = async_h2d_stack( + [req.hidden_states_tensor for req in batch.reqs], device=batch.device ) - - hidden_states = torch.stack( - [req.hidden_states_tensor for req in batch.reqs], dim=0 - ).to(batch.device) dsa_topk_indices = None dsa_indices_list = [req.output_dsa_topk_indices for req in batch.reqs] if dsa_indices_list and all(t is not None for t in dsa_indices_list): - dsa_topk_indices = torch.stack(dsa_indices_list, dim=0).to(batch.device) + dsa_topk_indices = async_h2d_stack(dsa_indices_list, device=batch.device) if should_remap_pd_dsa_seed_to_local_slots(): # PD sends request-relative positions; fused TopK consumes # decode-local physical slots. Remap once before the draft loop/graph. diff --git a/python/sglang/srt/utils/common.py b/python/sglang/srt/utils/common.py index e201e8b5d7..11fa646841 100644 --- a/python/sglang/srt/utils/common.py +++ b/python/sglang/srt/utils/common.py @@ -593,6 +593,20 @@ def async_h2d(args: List, *, dtype: torch.dtype, device: torch.device) -> torch. return torch.tensor(args, dtype=dtype, pin_memory=pin).to(device, non_blocking=pin) +def async_h2d_stack( + tensors: List[torch.Tensor], *, device: torch.device +) -> torch.Tensor: + """``torch.stack(tensors)`` on ``device``; host tensors are stacked straight + into pinned memory and copied like ``async_h2d``.""" + if not tensors[0].is_cpu: + return torch.stack(tensors).to(device) + pin = is_pin_memory_available(device) + out = torch.empty( + (len(tensors), *tensors[0].shape), dtype=tensors[0].dtype, pin_memory=pin + ) + return torch.stack(tensors, out=out).to(device, non_blocking=pin) + + def async_d2h(tensor: torch.Tensor) -> torch.Tensor: """Enqueue an accelerator-to-pinned-host copy on the current stream.""" if tensor.device.type == "cpu":