[Fix] Keep the target's DP sync slot in draft scopes (#41062)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai> Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
co-authored by
Mohammad Angkad
Cheng Wan
parent
3fc7a669bf
commit
cec70a4708
@@ -54,6 +54,7 @@ from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph impo
|
||||
from sglang.srt.platforms.device_mixin import _DEVICE_TO_DISTRIBUTED_BACKEND
|
||||
from sglang.srt.runtime_context import (
|
||||
derive_parallel_widths,
|
||||
get_flags,
|
||||
get_global_dwdp_manager,
|
||||
get_parallel,
|
||||
set_global_dwdp_manager,
|
||||
@@ -3067,6 +3068,16 @@ def patch_tensor_parallel_group(tp_group: GroupCoordinator, *, owns_attention: b
|
||||
global _TP_STATE_PATCHED
|
||||
assert not _TP_STATE_PATCHED, "Should not call when it's already patched"
|
||||
|
||||
# A draft forwards the target's DP sync, and the narrowed ranks cannot recover
|
||||
# this process's slot in it; read it before narrowing (config only, not _TP).
|
||||
dp_flags = get_flags().dp
|
||||
saved_gather_slot = dp_flags.scoped_gather_slot
|
||||
scoped_gather_slot = saved_gather_slot
|
||||
if owns_attention and dp_flags.enabled:
|
||||
from sglang.srt.layers.dp_attention import dp_gather_slot
|
||||
|
||||
scoped_gather_slot = dp_gather_slot()
|
||||
|
||||
_TP_STATE_PATCHED = True
|
||||
global _TP
|
||||
old_tp_group = _TP
|
||||
@@ -3093,10 +3104,12 @@ def patch_tensor_parallel_group(tp_group: GroupCoordinator, *, owns_attention: b
|
||||
moe_tp_size=tp_group.world_size,
|
||||
moe_tp_rank=tp_group.rank_in_group,
|
||||
)
|
||||
dp_flags.scoped_gather_slot = scoped_gather_slot
|
||||
try:
|
||||
with get_parallel().override(**narrowed):
|
||||
yield
|
||||
finally:
|
||||
dp_flags.scoped_gather_slot = saved_gather_slot
|
||||
_TP_STATE_PATCHED = False
|
||||
_TP = old_tp_group
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import torch
|
||||
import triton
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.dp_attention import DpPaddingMode
|
||||
from sglang.srt.layers.dp_attention import DpPaddingMode, dp_slot_in
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
)
|
||||
@@ -184,10 +184,8 @@ def cal_padded_tokens(forward_batch: "ForwardBatch"):
|
||||
)
|
||||
if dp_padding_mode.is_max_len():
|
||||
tokens = max(global_num_tokens)
|
||||
elif len(global_num_tokens) > 1:
|
||||
tokens = global_num_tokens[get_parallel().attn_dp_rank]
|
||||
else:
|
||||
tokens = global_num_tokens[0]
|
||||
tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
|
||||
if can_dsa_prefill_cp_interleave(forward_batch):
|
||||
tokens = ceil_div(tokens, attn_cp_size)
|
||||
return tokens
|
||||
|
||||
@@ -64,18 +64,45 @@ def dp_gather_width() -> int:
|
||||
"""Return the DP gather width.
|
||||
|
||||
After elastic scale-up, the gather spans the expanded WORLD; otherwise
|
||||
it spans the attention-DP replicas.
|
||||
it spans the attention-DP replicas. Inside a draft scope that owns its
|
||||
attention, it is still the target's gather (see patch_tensor_parallel_group).
|
||||
"""
|
||||
parallel = get_parallel()
|
||||
return parallel.dp_size if world_dp_gather_enabled() else parallel.attn_dp_size
|
||||
if world_dp_gather_enabled():
|
||||
return parallel.dp_size
|
||||
if get_flags().dp.scoped_gather_slot is not None:
|
||||
return deployment_attn_dp_size()
|
||||
return parallel.attn_dp_size
|
||||
|
||||
|
||||
def dp_slot_in(per_rank) -> int:
|
||||
"""Return this process's slot in a per-DP-replica sequence.
|
||||
|
||||
The sequence carries one entry per replica of the gather this process
|
||||
takes part in, so its length is the gather width. Length one is the
|
||||
all-gather-skipped batch, which carries this process's entry alone.
|
||||
"""
|
||||
if len(per_rank) == 1:
|
||||
return 0
|
||||
width = dp_gather_width()
|
||||
if len(per_rank) != width:
|
||||
raise ValueError(
|
||||
f"a per-replica sequence of {len(per_rank)} entries does not "
|
||||
f"belong to a DP gather of width {width}"
|
||||
)
|
||||
return dp_gather_slot()
|
||||
|
||||
|
||||
def dp_gather_slot() -> int:
|
||||
"""Return this process's index in the DP gather.
|
||||
|
||||
After elastic scale-up, use the TP rank plus the join offset; otherwise
|
||||
use the attention-DP rank.
|
||||
use the attention-DP rank. Inside a draft scope that owns its attention,
|
||||
it is still the slot in the target's gather (see patch_tensor_parallel_group).
|
||||
"""
|
||||
scoped = get_flags().dp.scoped_gather_slot
|
||||
if scoped is not None:
|
||||
return scoped
|
||||
parallel = get_parallel()
|
||||
if world_dp_gather_enabled():
|
||||
return parallel.tp_rank + parallel.ep_join_rank_offset
|
||||
@@ -458,9 +485,8 @@ def is_allocation_symmetric() -> bool:
|
||||
|
||||
def get_dp_local_info(forward_batch: ForwardBatch) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
# `get_dp_local_info` is only called in global DP gather and scatter. We use global DP rank here.
|
||||
dp_rank = dp_gather_slot()
|
||||
|
||||
if forward_batch.dp_local_start_pos is None:
|
||||
dp_rank = dp_slot_in(forward_batch.global_num_tokens_gpu)
|
||||
cumtokens = torch.cumsum(forward_batch.global_num_tokens_gpu, dim=0)
|
||||
if dp_rank == 0:
|
||||
local_start_pos = torch.zeros_like(cumtokens[0])
|
||||
@@ -482,7 +508,7 @@ def get_dp_local_slice_cpu(
|
||||
# CPU (start, length) slice for DP-local data in a rank-padded buffer.
|
||||
# Returns Python ints (no D2H sync) and handles the cuda-graph-padded layout.
|
||||
global_num_tokens = forward_batch.global_num_tokens_cpu
|
||||
dp_rank = dp_gather_slot()
|
||||
dp_rank = dp_slot_in(global_num_tokens)
|
||||
local_num_tokens = global_num_tokens[dp_rank]
|
||||
if can_run_graph:
|
||||
local_start_pos = dp_rank * cuda_graph_batch
|
||||
@@ -800,7 +826,7 @@ def _dp_gather_via_all_gatherv(
|
||||
# each rank's local tensor up to sizes[rank] with zeros (matching the
|
||||
# buffer's reserved per-rank slot) so sum(sizes) == buffer rows and there
|
||||
# is no uninitialized tail for the MoE to read.
|
||||
rank = get_parallel().attn_dp_rank
|
||||
rank = dp_slot_in(sizes)
|
||||
local_rows = sizes[rank]
|
||||
if local_tokens.shape[0] == local_rows:
|
||||
local_real = local_tokens
|
||||
|
||||
@@ -45,7 +45,7 @@ from sglang.srt.kv_canary.req_to_expected_token_ids_manager import (
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_gather_slot,
|
||||
dp_slot_in,
|
||||
set_dp_buffer_len_from_batch,
|
||||
set_is_extend_in_batch,
|
||||
world_dp_gather_enabled,
|
||||
@@ -1193,7 +1193,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
if self.global_num_tokens_cpu is not None:
|
||||
# DP / MLP-sync path: per-DP padded width.
|
||||
if require_mlp_tp_gather():
|
||||
num_tokens_per_dp = self.global_num_tokens_cpu[dp_gather_slot()]
|
||||
num_tokens_per_dp = self.global_num_tokens_cpu[
|
||||
dp_slot_in(self.global_num_tokens_cpu)
|
||||
]
|
||||
else:
|
||||
num_tokens_per_dp = self.global_num_tokens_cpu[0]
|
||||
else:
|
||||
@@ -1592,10 +1594,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
else:
|
||||
buffer_len = sum(global_num_tokens)
|
||||
|
||||
if len(global_num_tokens) > 1:
|
||||
num_tokens = global_num_tokens[dp_gather_slot()]
|
||||
else:
|
||||
num_tokens = global_num_tokens[0]
|
||||
num_tokens = global_num_tokens[dp_slot_in(global_num_tokens)]
|
||||
|
||||
self.attn_tp_sequence_sharded = model_runner.attn_tp_sequence_sharded(
|
||||
num_tokens
|
||||
@@ -1736,6 +1735,14 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
if self.input_embeds is not None:
|
||||
# Keep token-aligned inputs consistent after padding.
|
||||
self.input_embeds = self._pad_tensor_to_size(self.input_embeds, num_tokens)
|
||||
if (
|
||||
self.mm_input_embeds is not None
|
||||
and self.mm_input_embeds.shape[0] < num_tokens
|
||||
):
|
||||
# A draft reads the target's mm embeds in place of its input_ids.
|
||||
self.mm_input_embeds = self._pad_tensor_to_size(
|
||||
self.mm_input_embeds, num_tokens
|
||||
)
|
||||
self.req_pool_indices = self._pad_tensor_to_size(self.req_pool_indices, bs)
|
||||
if self.req_pool_indices_cpu is not None:
|
||||
self.req_pool_indices_cpu = self._pad_tensor_to_size(
|
||||
|
||||
@@ -70,6 +70,7 @@ from sglang.srt.layers.cp.utils import (
|
||||
)
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
DpPaddingMode,
|
||||
dp_slot_in,
|
||||
set_dp_buffer_len,
|
||||
set_is_extend_in_batch,
|
||||
)
|
||||
@@ -672,8 +673,7 @@ class PrefillCudaGraphRunner(BaseCudaGraphRunner):
|
||||
|
||||
global_num_tokens = forward_batch.global_num_tokens_for_logprob_cpu
|
||||
if global_num_tokens is not None:
|
||||
dp_rank = get_parallel().attn_dp_rank
|
||||
return int(global_num_tokens[dp_rank if len(global_num_tokens) > 1 else 0])
|
||||
return int(global_num_tokens[dp_slot_in(global_num_tokens)])
|
||||
|
||||
return sum(
|
||||
max(int(seq_len) - int(start_len), 1)
|
||||
|
||||
@@ -577,6 +577,9 @@ class DpFlags(_FlagGroupBase):
|
||||
# gather/scatter helpers, whose captured geometry needs one shared bucket.
|
||||
capturing_prefill_graph: bool = False
|
||||
prefill_graph_has_dp_gather: bool = False
|
||||
# This process's slot in the target's DP sync while a draft scope narrows
|
||||
# attention; dp_gather_width() also reads it as the in-scope marker.
|
||||
scoped_gather_slot: Optional[int] = None
|
||||
# DP gathered-buffer allocation metadata (model hidden size / dtype /
|
||||
# device), set by initialize_dp_attention alongside the flags above.
|
||||
buffer_hidden_size: Any = None
|
||||
|
||||
Reference in New Issue
Block a user