[Perf] deepseek-v4.1: Stage late-layer DP row counts through pinned memory (#45062)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: kpham-sgl <khoa.pham@radixark.ai>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
weireweire
2026-10-18 20:19:26 -07:00
committed by GitHub
co-authored by weireweire kpham-sgl Claude Fable 5.1
parent f02458fff1
commit 06342acad2
@@ -7,6 +7,7 @@ import torch
from sglang.srt.distributed.utils import all_gather_single
from sglang.srt.layers.dp_attention import DpPaddingMode, set_dp_buffer_len_from_batch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils.common import async_h2d
if TYPE_CHECKING:
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
@@ -70,11 +71,10 @@ def enter_late_layer_dp(forward_batch: ForwardBatch, counts: List[int]) -> tuple
# Device ops only: this runs inside prefill graph capture, where a host copy
# would be recorded against pageable memory.
gpu = fb.global_num_tokens_gpu
fb.global_num_tokens_gpu = (
torch.full_like(gpu, counts[0])
if len(set(counts)) == 1
else gpu.new_tensor(counts)
)
if len(set(counts)) == 1:
fb.global_num_tokens_gpu = torch.full_like(gpu, counts[0])
else:
fb.global_num_tokens_gpu = async_h2d(counts, dtype=gpu.dtype, device=gpu.device)
fb.global_num_tokens_padded_cpu = counts
# The MoE gatherv reads its sizes from the logprob counts when present.
fb.global_num_tokens_for_logprob_cpu = counts