[Perf] deepseek-v4.1: Stage late-layer DP row counts through pinned memory (#45062)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: kpham-sgl <khoa.pham@radixark.ai> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
co-authored by
weireweire
kpham-sgl
Claude Fable 5.1
parent
f02458fff1
commit
06342acad2
@@ -7,6 +7,7 @@ import torch
|
||||
from sglang.srt.distributed.utils import all_gather_single
|
||||
from sglang.srt.layers.dp_attention import DpPaddingMode, set_dp_buffer_len_from_batch
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils.common import async_h2d
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
@@ -70,11 +71,10 @@ def enter_late_layer_dp(forward_batch: ForwardBatch, counts: List[int]) -> tuple
|
||||
# Device ops only: this runs inside prefill graph capture, where a host copy
|
||||
# would be recorded against pageable memory.
|
||||
gpu = fb.global_num_tokens_gpu
|
||||
fb.global_num_tokens_gpu = (
|
||||
torch.full_like(gpu, counts[0])
|
||||
if len(set(counts)) == 1
|
||||
else gpu.new_tensor(counts)
|
||||
)
|
||||
if len(set(counts)) == 1:
|
||||
fb.global_num_tokens_gpu = torch.full_like(gpu, counts[0])
|
||||
else:
|
||||
fb.global_num_tokens_gpu = async_h2d(counts, dtype=gpu.dtype, device=gpu.device)
|
||||
fb.global_num_tokens_padded_cpu = counts
|
||||
# The MoE gatherv reads its sizes from the logprob counts when present.
|
||||
fb.global_num_tokens_for_logprob_cpu = counts
|
||||
|
||||
Reference in New Issue
Block a user