From 06342acad2ed320e729aa40628c7745620565439 Mon Sep 17 00:00:00 2001 From: weireweire Date: Mon, 19 Oct 2026 11:19:26 +0800 Subject: [PATCH] [Perf] deepseek-v4.1: Stage late-layer DP row counts through pinned memory (#45062) Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: kpham-sgl Co-authored-by: Claude Fable 5.1 --- .../deepseek_common/decoder_swa_bounded_replay.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py b/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py index 2feb9fa117..260dc617bb 100644 --- a/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py +++ b/python/sglang/srt/models/deepseek_common/decoder_swa_bounded_replay.py @@ -7,6 +7,7 @@ import torch from sglang.srt.distributed.utils import all_gather_single from sglang.srt.layers.dp_attention import DpPaddingMode, set_dp_buffer_len_from_batch from sglang.srt.runtime_context import get_parallel +from sglang.srt.utils.common import async_h2d if TYPE_CHECKING: from sglang.srt.model_executor.forward_batch_info import ForwardBatch @@ -70,11 +71,10 @@ def enter_late_layer_dp(forward_batch: ForwardBatch, counts: List[int]) -> tuple # Device ops only: this runs inside prefill graph capture, where a host copy # would be recorded against pageable memory. gpu = fb.global_num_tokens_gpu - fb.global_num_tokens_gpu = ( - torch.full_like(gpu, counts[0]) - if len(set(counts)) == 1 - else gpu.new_tensor(counts) - ) + if len(set(counts)) == 1: + fb.global_num_tokens_gpu = torch.full_like(gpu, counts[0]) + else: + fb.global_num_tokens_gpu = async_h2d(counts, dtype=gpu.dtype, device=gpu.device) fb.global_num_tokens_padded_cpu = counts # The MoE gatherv reads its sizes from the logprob counts when present. fb.global_num_tokens_for_logprob_cpu = counts