[Fix] Keep the target's DP sync slot in draft scopes (#41062)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
Mohammad Miadh Angkad
2026-09-24 21:48:39 -07:00
committed by GitHub
co-authored by Mohammad Angkad Cheng Wan
parent 3fc7a669bf
commit cec70a4708
10 changed files with 330 additions and 24 deletions
@@ -54,6 +54,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
from sglang.srt.platforms.device_mixin import _DEVICE_TO_DISTRIBUTED_BACKEND
from sglang.srt.runtime_context import (
derive_parallel_widths,
get_flags,
get_global_dwdp_manager,
get_parallel,
set_global_dwdp_manager,
@@ -3067,6 +3068,16 @@ def patch_tensor_parallel_group(tp_group: GroupCoordinator, *, owns_attention: b
global _TP_STATE_PATCHED
assert not _TP_STATE_PATCHED, "Should not call when it's already patched"
# A draft forwards the target's DP sync, and the narrowed ranks cannot recover
# this process's slot in it; read it before narrowing (config only, not _TP).
dp_flags = get_flags().dp
saved_gather_slot = dp_flags.scoped_gather_slot
scoped_gather_slot = saved_gather_slot
if owns_attention and dp_flags.enabled:
from sglang.srt.layers.dp_attention import dp_gather_slot
scoped_gather_slot = dp_gather_slot()
_TP_STATE_PATCHED = True
global _TP
old_tp_group = _TP
@@ -3093,10 +3104,12 @@ def patch_tensor_parallel_group(tp_group: GroupCoordinator, *, owns_attention: b
moe_tp_size=tp_group.world_size,
moe_tp_rank=tp_group.rank_in_group,
)
dp_flags.scoped_gather_slot = scoped_gather_slot
try:
with get_parallel().override(**narrowed):
yield
finally:
dp_flags.scoped_gather_slot = saved_gather_slot
_TP_STATE_PATCHED = False
_TP = old_tp_group
@@ -5,7 +5,7 @@ import torch
import triton
from sglang.srt.environ import envs
from sglang.srt.layers.dp_attention import DpPaddingMode
from sglang.srt.layers.dp_attention import DpPaddingMode, dp_slot_in
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
)
@@ -184,10 +184,8 @@ def cal_padded_tokens(forward_batch: "ForwardBatch"):
)
if dp_padding_mode.is_max_len():
tokens = max(global_num_tokens)
elif len(global_num_tokens) > 1:
tokens = global_num_tokens[get_parallel().attn_dp_rank]
else:
tokens = global_num_tokens[0]
tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
if can_dsa_prefill_cp_interleave(forward_batch):
tokens = ceil_div(tokens, attn_cp_size)
return tokens
+33 -7
View File
@@ -64,18 +64,45 @@ def dp_gather_width() -> int:
"""Return the DP gather width.
After elastic scale-up, the gather spans the expanded WORLD; otherwise
it spans the attention-DP replicas.
it spans the attention-DP replicas. Inside a draft scope that owns its
attention, it is still the target's gather (see patch_tensor_parallel_group).
"""
parallel = get_parallel()
return parallel.dp_size if world_dp_gather_enabled() else parallel.attn_dp_size
if world_dp_gather_enabled():
return parallel.dp_size
if get_flags().dp.scoped_gather_slot is not None:
return deployment_attn_dp_size()
return parallel.attn_dp_size
def dp_slot_in(per_rank) -> int:
"""Return this process's slot in a per-DP-replica sequence.
The sequence carries one entry per replica of the gather this process
takes part in, so its length is the gather width. Length one is the
all-gather-skipped batch, which carries this process's entry alone.
"""
if len(per_rank) == 1:
return 0
width = dp_gather_width()
if len(per_rank) != width:
raise ValueError(
f"a per-replica sequence of {len(per_rank)} entries does not "
f"belong to a DP gather of width {width}"
)
return dp_gather_slot()
def dp_gather_slot() -> int:
"""Return this process's index in the DP gather.
After elastic scale-up, use the TP rank plus the join offset; otherwise
use the attention-DP rank.
use the attention-DP rank. Inside a draft scope that owns its attention,
it is still the slot in the target's gather (see patch_tensor_parallel_group).
"""
scoped = get_flags().dp.scoped_gather_slot
if scoped is not None:
return scoped
parallel = get_parallel()
if world_dp_gather_enabled():
return parallel.tp_rank + parallel.ep_join_rank_offset
@@ -458,9 +485,8 @@ def is_allocation_symmetric() -> bool:
def get_dp_local_info(forward_batch: ForwardBatch) -> Tuple[torch.Tensor, torch.Tensor]:
# `get_dp_local_info` is only called in global DP gather and scatter. We use global DP rank here.
dp_rank = dp_gather_slot()
if forward_batch.dp_local_start_pos is None:
dp_rank = dp_slot_in(forward_batch.global_num_tokens_gpu)
cumtokens = torch.cumsum(forward_batch.global_num_tokens_gpu, dim=0)
if dp_rank == 0:
local_start_pos = torch.zeros_like(cumtokens[0])
@@ -482,7 +508,7 @@ def get_dp_local_slice_cpu(
# CPU (start, length) slice for DP-local data in a rank-padded buffer.
# Returns Python ints (no D2H sync) and handles the cuda-graph-padded layout.
global_num_tokens = forward_batch.global_num_tokens_cpu
dp_rank = dp_gather_slot()
dp_rank = dp_slot_in(global_num_tokens)
local_num_tokens = global_num_tokens[dp_rank]
if can_run_graph:
local_start_pos = dp_rank * cuda_graph_batch
@@ -800,7 +826,7 @@ def _dp_gather_via_all_gatherv(
# each rank's local tensor up to sizes[rank] with zeros (matching the
# buffer's reserved per-rank slot) so sum(sizes) == buffer rows and there
# is no uninitialized tail for the MoE to read.
rank = get_parallel().attn_dp_rank
rank = dp_slot_in(sizes)
local_rows = sizes[rank]
if local_tokens.shape[0] == local_rows:
local_real = local_tokens
@@ -45,7 +45,7 @@ from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
)
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_gather_slot,
dp_slot_in,
set_dp_buffer_len_from_batch,
set_is_extend_in_batch,
world_dp_gather_enabled,
@@ -1193,7 +1193,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
if self.global_num_tokens_cpu is not None:
# DP / MLP-sync path: per-DP padded width.
if require_mlp_tp_gather():
num_tokens_per_dp = self.global_num_tokens_cpu[dp_gather_slot()]
num_tokens_per_dp = self.global_num_tokens_cpu[
dp_slot_in(self.global_num_tokens_cpu)
]
else:
num_tokens_per_dp = self.global_num_tokens_cpu[0]
else:
@@ -1592,10 +1594,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
else:
buffer_len = sum(global_num_tokens)
if len(global_num_tokens) > 1:
num_tokens = global_num_tokens[dp_gather_slot()]
else:
num_tokens = global_num_tokens[0]
num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
self.attn_tp_sequence_sharded = model_runner.attn_tp_sequence_sharded(
num_tokens
@@ -1736,6 +1735,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
if self.input_embeds is not None:
# Keep token-aligned inputs consistent after padding.
self.input_embeds = self._pad_tensor_to_size(self.input_embeds, num_tokens)
if (
self.mm_input_embeds is not None
and self.mm_input_embeds.shape[0] < num_tokens
):
# A draft reads the target's mm embeds in place of its input_ids.
self.mm_input_embeds = self._pad_tensor_to_size(
self.mm_input_embeds, num_tokens
)
self.req_pool_indices = self._pad_tensor_to_size(self.req_pool_indices, bs)
if self.req_pool_indices_cpu is not None:
self.req_pool_indices_cpu = self._pad_tensor_to_size(
@@ -70,6 +70,7 @@ from sglang.srt.layers.cp.utils import (
)
from sglang.srt.layers.dp_attention import (
DpPaddingMode,
dp_slot_in,
set_dp_buffer_len,
set_is_extend_in_batch,
)
@@ -672,8 +673,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
global_num_tokens = forward_batch.global_num_tokens_for_logprob_cpu
if global_num_tokens is not None:
dp_rank = get_parallel().attn_dp_rank
return int(global_num_tokens[dp_rank if len(global_num_tokens) > 1 else 0])
return int(global_num_tokens[dp_slot_in(global_num_tokens)])
return sum(
max(int(seq_len) - int(start_len), 1)
+3
View File
@@ -577,6 +577,9 @@ class DpFlags(_FlagGroupBase):
# gather/scatter helpers, whose captured geometry needs one shared bucket.
capturing_prefill_graph: bool = False
prefill_graph_has_dp_gather: bool = False
# This process's slot in the target's DP sync while a draft scope narrows
# attention; dp_gather_width() also reads it as the in-scope marker.
scoped_gather_slot: Optional[int] = None
# DP gathered-buffer allocation metadata (model hidden size / dtype /
# device), set by initialize_dp_attention alongside the flags above.
buffer_hidden_size: Any = None