[PD] Keep EAGLE DP graph and token metadata consistent (#32196)
Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com> Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com> Co-authored-by: Khoa Pham <khoa.pham@radixark.ai>
This commit is contained in:
co-authored by
weireweire
Po-Han Huang
Khoa Pham
parent
89e1316eae
commit
7889fbab9a
@@ -756,6 +756,7 @@ class TboForwardBatchPreparer:
|
||||
"dp_spec_prefill_coordination_applied",
|
||||
"return_logprob",
|
||||
"can_run_decode_cuda_graph",
|
||||
"can_run_dp_draft_cuda_graph",
|
||||
"can_run_dp_prefill_cuda_graph",
|
||||
"dp_prefill_cuda_graph_max_prefix_len",
|
||||
"dp_padding_mode",
|
||||
|
||||
@@ -201,6 +201,19 @@ def _broadcast_indexer_topk_from_rank0(
|
||||
return topk_indices
|
||||
|
||||
|
||||
def _make_eager_idle_topk_result(
|
||||
x: torch.Tensor, index_topk: int, return_indices: bool
|
||||
) -> Optional[torch.Tensor]:
|
||||
if not return_indices:
|
||||
return None
|
||||
return torch.full(
|
||||
(x.shape[0], index_topk),
|
||||
-1,
|
||||
dtype=torch.int32,
|
||||
device=x.device,
|
||||
)
|
||||
|
||||
|
||||
def rotate_activation(x: torch.Tensor) -> torch.Tensor:
|
||||
if _is_hip:
|
||||
from fast_hadamard_transform import hadamard_transform
|
||||
@@ -1591,6 +1604,21 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
layer_id: int,
|
||||
return_indices: bool = True,
|
||||
) -> Optional[torch.Tensor]:
|
||||
# A padded eager IDLE rank has no real requests and therefore no valid
|
||||
# DSA page-table rows to index. Return invalid rows with the physical DP
|
||||
# shape so later MLP/EP collectives still agree across ranks.
|
||||
x_meta = x[0] if isinstance(x, tuple) else x
|
||||
if (
|
||||
_is_cuda
|
||||
and forward_batch.forward_mode.is_idle()
|
||||
and not get_is_capture_mode()
|
||||
):
|
||||
topk_result = _make_eager_idle_topk_result(
|
||||
x_meta, self.index_topk, return_indices
|
||||
)
|
||||
topk_result = _broadcast_indexer_topk_from_rank0(topk_result)
|
||||
return maybe_capture_indexer_topk(layer_id, topk_result)
|
||||
|
||||
if _is_hip:
|
||||
from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant
|
||||
elif not _is_npu:
|
||||
@@ -1599,10 +1627,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
|
||||
if TYPE_CHECKING:
|
||||
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
|
||||
|
||||
# When upstream uses fused FP8 RMSNorm+quant, activations may be passed as
|
||||
# a tuple like (x_fp8, x_scale[, y]). Use `x_meta` for shape/device queries.
|
||||
x_meta = x[0] if isinstance(x, tuple) else x
|
||||
|
||||
in_piecewise_or_breakable_cuda_graph = (
|
||||
_is_in_piecewise_or_breakable_cuda_graph()
|
||||
)
|
||||
|
||||
@@ -173,6 +173,46 @@ def _to_2d_context_lens(seqlens_32: torch.Tensor, batch_size: int) -> torch.Tens
|
||||
return seqlens_32.contiguous().view(-1, 1)
|
||||
|
||||
|
||||
def _trim_trtllm_decode_dp_padding(
|
||||
q_all: torch.Tensor,
|
||||
topk_indices: Optional[torch.Tensor],
|
||||
real_batch_size: int,
|
||||
) -> Tuple[torch.Tensor, Optional[torch.Tensor], int]:
|
||||
"""Align eager decode inputs with metadata planned before DP padding."""
|
||||
physical_batch_size = q_all.shape[0]
|
||||
assert real_batch_size <= physical_batch_size, (
|
||||
f"DSA metadata batch size ({real_batch_size}) exceeds q batch size "
|
||||
f"({physical_batch_size})"
|
||||
)
|
||||
if topk_indices is not None:
|
||||
assert real_batch_size <= topk_indices.shape[0], (
|
||||
f"DSA metadata batch size ({real_batch_size}) exceeds topk batch size "
|
||||
f"({topk_indices.shape[0]})"
|
||||
)
|
||||
|
||||
num_padding_rows = physical_batch_size - real_batch_size
|
||||
if num_padding_rows == 0:
|
||||
return q_all, topk_indices, 0
|
||||
|
||||
return (
|
||||
q_all[:real_batch_size],
|
||||
topk_indices[:real_batch_size] if topk_indices is not None else None,
|
||||
num_padding_rows,
|
||||
)
|
||||
|
||||
|
||||
def _restore_trtllm_decode_dp_padding(
|
||||
output: torch.Tensor, num_padding_rows: int
|
||||
) -> torch.Tensor:
|
||||
"""Restore the physical DP shape required by downstream MLP collectives."""
|
||||
if num_padding_rows == 0:
|
||||
return output
|
||||
return torch.cat(
|
||||
[output, output.new_zeros((num_padding_rows, *output.shape[1:]))],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DSAFlashMLAMetadata:
|
||||
"""Metadata only needed by FlashMLA"""
|
||||
@@ -3441,9 +3481,24 @@ class DeepseekSparseAttnBackend(
|
||||
else:
|
||||
q_all = q.view(-1, layer.tp_q_head_num, layer.head_dim)
|
||||
|
||||
# Eager DP attention can pad q beyond metadata that was deliberately
|
||||
# planned on the real draft batch. Pad top-k to the physical q shape,
|
||||
# then run decode attention only on metadata-backed rows. The output is
|
||||
# restored below before downstream MLP/EP collectives.
|
||||
if (self.use_fused_topk or not is_prefill) and topk_indices is not None:
|
||||
topk_indices = self._pad_topk_indices(topk_indices, q.shape[0])
|
||||
|
||||
num_decode_padding_rows = 0
|
||||
if not is_prefill:
|
||||
q_all, topk_indices, num_decode_padding_rows = (
|
||||
_trim_trtllm_decode_dp_padding(
|
||||
q_all,
|
||||
topk_indices,
|
||||
metadata.cache_seqlens_int32.shape[0],
|
||||
)
|
||||
)
|
||||
|
||||
if self.use_fused_topk:
|
||||
if topk_indices is not None:
|
||||
topk_indices = self._pad_topk_indices(topk_indices, q.shape[0])
|
||||
page_table_1 = self._get_fused_topk_page_table(topk_indices)
|
||||
elif is_prefill:
|
||||
page_table_1 = transform_index_page_table_prefill(
|
||||
@@ -3459,8 +3514,6 @@ class DeepseekSparseAttnBackend(
|
||||
cu_seqlens_q=metadata.cu_seqlens_q,
|
||||
)
|
||||
else:
|
||||
if topk_indices is not None:
|
||||
topk_indices = self._pad_topk_indices(topk_indices, q.shape[0])
|
||||
page_table_1 = transform_index_page_table_decode(
|
||||
page_table=metadata.page_table_1,
|
||||
topk_indices=topk_indices,
|
||||
@@ -3509,7 +3562,7 @@ class DeepseekSparseAttnBackend(
|
||||
multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer,
|
||||
)
|
||||
|
||||
return out
|
||||
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows)
|
||||
|
||||
def _pad_topk_indices(
|
||||
self, topk_indices: torch.Tensor, num_tokens: int
|
||||
|
||||
@@ -2471,6 +2471,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# For DP attention
|
||||
is_extend_in_batch: bool = False
|
||||
can_run_decode_cuda_graph: bool = False
|
||||
# Rank-consistent EAGLE draft replay gate. Keep it separate so missing
|
||||
# draft-only state does not disable target verification or draft extend.
|
||||
can_run_dp_draft_cuda_graph: bool = False
|
||||
can_run_dp_prefill_cuda_graph: bool = False
|
||||
dp_prefill_cuda_graph_max_prefix_len: int = 0
|
||||
tbo_split_seq_index: Optional[int] = None
|
||||
@@ -2524,6 +2527,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
# For DP attention
|
||||
global_num_tokens: Optional[List[int]] = None
|
||||
global_num_tokens_for_logprob: Optional[List[int]] = None
|
||||
# The draft model can use a different MoE A2A backend than the target.
|
||||
draft_global_num_tokens: Optional[List[int]] = None
|
||||
draft_global_num_tokens_for_logprob: Optional[List[int]] = None
|
||||
# Full DP token vector retained for Aiter MegaMoE even when the normal MLP
|
||||
# TP gather path stores only this rank's token count.
|
||||
global_spec_verify_tier_num_tokens: Optional[List[int]] = None
|
||||
@@ -3831,6 +3837,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
|
||||
global_num_tokens=self.global_num_tokens,
|
||||
global_num_tokens_for_logprob=self.global_num_tokens_for_logprob,
|
||||
can_run_decode_cuda_graph=self.can_run_decode_cuda_graph,
|
||||
can_run_dp_draft_cuda_graph=self.can_run_dp_draft_cuda_graph,
|
||||
can_run_dp_prefill_cuda_graph=self.can_run_dp_prefill_cuda_graph,
|
||||
dp_prefill_cuda_graph_max_prefix_len=self.dp_prefill_cuda_graph_max_prefix_len,
|
||||
is_extend_in_batch=self.is_extend_in_batch,
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.runtime_context import (
|
||||
get_memory,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
get_spec,
|
||||
)
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
from sglang.srt.utils.common import require_mlp_tp_gather
|
||||
@@ -45,6 +46,25 @@ if TYPE_CHECKING:
|
||||
_ENABLE_METRICS_DP_ATTENTION = envs.SGLANG_ENABLE_METRICS_DP_ATTENTION.get()
|
||||
|
||||
|
||||
def _spec_input_cuda_graph_compatible(
|
||||
local_batch: Optional[ScheduleBatch],
|
||||
) -> bool:
|
||||
"""Return the local speculative-draft graph admission bit.
|
||||
|
||||
None/idle/prebuilt inputs stay permissive so an active rank with complete
|
||||
runtime state can still use graphs. Any active incompatible input is
|
||||
min-reduced by ``MLPSyncBatchInfo`` and forces every DP rank eager.
|
||||
"""
|
||||
if (
|
||||
local_batch is None
|
||||
or local_batch.forward_mode.is_idle()
|
||||
or local_batch.forward_mode.is_prebuilt()
|
||||
):
|
||||
return True
|
||||
spec_info = local_batch.spec_info
|
||||
return spec_info is None or spec_info.cuda_graph_compatible
|
||||
|
||||
|
||||
def _resolve_elastic_world_dp_size(
|
||||
dp_size: int,
|
||||
*,
|
||||
@@ -92,6 +112,7 @@ class MLPSyncBatchInfo:
|
||||
num_tokens: int
|
||||
num_tokens_for_logprob: int
|
||||
can_run_decode_cuda_graph: bool
|
||||
can_run_draft_cuda_graph: bool
|
||||
can_run_prefill_cuda_graph: bool
|
||||
is_extend_in_batch: bool
|
||||
local_can_run_tbo: bool
|
||||
@@ -117,6 +138,7 @@ class MLPSyncBatchInfo:
|
||||
self.local_forward_mode,
|
||||
int(self.can_run_prefill_cuda_graph),
|
||||
self.prefill_cuda_graph_max_prefix_len,
|
||||
int(self.can_run_draft_cuda_graph),
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -133,6 +155,7 @@ class MLPSyncBatchInfo:
|
||||
ForwardMode.IDLE.value, # local_forward_mode
|
||||
0, # can_run_prefill_cuda_graph
|
||||
0, # prefill_cuda_graph_max_prefix_len
|
||||
1, # can_run_draft_cuda_graph
|
||||
],
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
@@ -215,6 +238,7 @@ class MLPSyncBatchInfo:
|
||||
self.is_extend_in_batch = bool(tp0_info_cpu[:, 3].max())
|
||||
self.can_run_prefill_cuda_graph = bool(tp0_info_cpu[:, 6].min())
|
||||
self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max())
|
||||
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
|
||||
if _ENABLE_METRICS_DP_ATTENTION:
|
||||
self.dp_cooperation_info = DPCooperationInfo.create(
|
||||
tp0_info_cpu[:, 5].tolist()
|
||||
@@ -225,6 +249,7 @@ def _update_gather_batch(
|
||||
batch: ScheduleBatch,
|
||||
mlp_sync_info: MLPSyncBatchInfo,
|
||||
require_mlp_tp_gather: bool,
|
||||
draft_require_mlp_tp_gather: Optional[bool] = None,
|
||||
skip_global_metadata=False,
|
||||
):
|
||||
if not require_mlp_tp_gather:
|
||||
@@ -235,6 +260,19 @@ def _update_gather_batch(
|
||||
batch.global_num_tokens_for_logprob = (
|
||||
mlp_sync_info.global_num_tokens_for_logprob
|
||||
)
|
||||
# Reuse the same all-gather result for a draft model whose A2A backend
|
||||
# requires a different local/full token-count representation.
|
||||
if draft_require_mlp_tp_gather is not None:
|
||||
if draft_require_mlp_tp_gather:
|
||||
batch.draft_global_num_tokens = mlp_sync_info.global_num_tokens
|
||||
batch.draft_global_num_tokens_for_logprob = (
|
||||
mlp_sync_info.global_num_tokens_for_logprob
|
||||
)
|
||||
else:
|
||||
batch.draft_global_num_tokens = [mlp_sync_info.num_tokens]
|
||||
batch.draft_global_num_tokens_for_logprob = [
|
||||
mlp_sync_info.num_tokens_for_logprob
|
||||
]
|
||||
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
|
||||
# Fresh counts have not yet been adjusted by the coordination plan.
|
||||
batch.dp_spec_prefill_coordination_applied = False
|
||||
@@ -251,6 +289,7 @@ def _update_gather_batch(
|
||||
|
||||
# Check forward mode for cuda graph
|
||||
batch.can_run_decode_cuda_graph = mlp_sync_info.can_run_decode_cuda_graph
|
||||
batch.can_run_dp_draft_cuda_graph = mlp_sync_info.can_run_draft_cuda_graph
|
||||
batch.can_run_dp_prefill_cuda_graph = mlp_sync_info.can_run_prefill_cuda_graph
|
||||
batch.dp_prefill_cuda_graph_max_prefix_len = (
|
||||
mlp_sync_info.prefill_cuda_graph_max_prefix_len
|
||||
@@ -378,6 +417,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
require_mlp_tp_gather: bool,
|
||||
disable_overlap_schedule: bool,
|
||||
offload_tags: set[str],
|
||||
draft_require_mlp_tp_gather: Optional[bool] = None,
|
||||
dwdp: bool = False,
|
||||
):
|
||||
parallel = get_parallel()
|
||||
@@ -413,6 +453,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
can_run_decode_cuda_graph = _local_decode_cuda_graph_vote(
|
||||
local_batch=local_batch, disable_cuda_graph=disable_cuda_graph
|
||||
)
|
||||
can_run_draft_cuda_graph = _spec_input_cuda_graph_compatible(local_batch)
|
||||
breakable_prefill = check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
|
||||
full_prefill = check_cuda_graph_backend(Phase.PREFILL, Backend.FULL)
|
||||
coordinated_prefill = breakable_prefill or full_prefill
|
||||
@@ -472,6 +513,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
num_tokens=num_tokens,
|
||||
num_tokens_for_logprob=num_tokens_for_logprob,
|
||||
can_run_decode_cuda_graph=can_run_decode_cuda_graph,
|
||||
can_run_draft_cuda_graph=can_run_draft_cuda_graph,
|
||||
can_run_prefill_cuda_graph=can_run_prefill_cuda_graph,
|
||||
is_extend_in_batch=is_extend_in_batch,
|
||||
local_can_run_tbo=local_can_run_tbo,
|
||||
@@ -515,6 +557,7 @@ def prepare_mlp_sync_batch_raw(
|
||||
batch_to_gather,
|
||||
mlp_sync_info,
|
||||
require_mlp_tp_gather,
|
||||
draft_require_mlp_tp_gather,
|
||||
skip_global_metadata=not metadata_ready,
|
||||
)
|
||||
|
||||
@@ -546,12 +589,22 @@ class SchedulerDPAttnAdapter:
|
||||
get_require_mlp_sync: Callable[[], bool]
|
||||
|
||||
def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
|
||||
draft_require_mlp_tp_gather = None
|
||||
if self.spec_algorithm.is_eagle() or self.spec_algorithm.is_standalone():
|
||||
draft_moe_a2a_backend = get_spec().speculative_moe_a2a_backend
|
||||
if draft_moe_a2a_backend is None:
|
||||
draft_moe_a2a_backend = get_exec().moe.moe_a2a_backend
|
||||
draft_require_mlp_tp_gather = require_mlp_tp_gather(
|
||||
moe_a2a_backend=draft_moe_a2a_backend,
|
||||
)
|
||||
|
||||
return prepare_mlp_sync_batch_raw(
|
||||
local_batch,
|
||||
model_runner=self.model_runner,
|
||||
get_idle_batch=self.get_idle_batch,
|
||||
disable_cuda_graph=cuda_graph_fully_disabled(),
|
||||
require_mlp_tp_gather=require_mlp_tp_gather(),
|
||||
draft_require_mlp_tp_gather=draft_require_mlp_tp_gather,
|
||||
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
|
||||
offload_tags=self.offload_tags,
|
||||
dwdp=get_parallel().dwdp_size > 1,
|
||||
|
||||
@@ -558,6 +558,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
is_extend_in_batch: bool = False
|
||||
dp_spec_prefill_coordination_applied: bool = False
|
||||
can_run_decode_cuda_graph: bool = False
|
||||
# Draft-only companion to the generic DP decode graph vote.
|
||||
can_run_dp_draft_cuda_graph: bool = False
|
||||
can_run_dp_prefill_cuda_graph: bool = False
|
||||
dp_prefill_cuda_graph_max_prefix_len: int = 0
|
||||
global_forward_mode: Optional[ForwardMode] = None
|
||||
@@ -835,16 +837,33 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
self.mark_forward_metadata_ready()
|
||||
|
||||
def init_mlp_sync_metadata(
|
||||
self, batch: ScheduleBatch, device: Union[str, torch.device]
|
||||
self,
|
||||
batch: ScheduleBatch,
|
||||
device: Union[str, torch.device],
|
||||
*,
|
||||
is_draft_worker: bool = False,
|
||||
) -> None:
|
||||
"""Populate per-rank token counts for DP-attention MLP synchronization."""
|
||||
self.dp_spec_prefill_coordination_applied = (
|
||||
batch.dp_spec_prefill_coordination_applied
|
||||
)
|
||||
if batch.global_num_tokens is None:
|
||||
global_num_tokens_source = batch.global_num_tokens
|
||||
global_num_tokens_for_logprob_source = batch.global_num_tokens_for_logprob
|
||||
# A DP spec/prefill coordination plan already wrote this phase's counts.
|
||||
if (
|
||||
is_draft_worker
|
||||
and batch.draft_global_num_tokens is not None
|
||||
and not batch.dp_spec_prefill_coordination_applied
|
||||
):
|
||||
global_num_tokens_source = batch.draft_global_num_tokens
|
||||
global_num_tokens_for_logprob_source = (
|
||||
batch.draft_global_num_tokens_for_logprob
|
||||
)
|
||||
|
||||
if global_num_tokens_source is None:
|
||||
return
|
||||
|
||||
assert batch.global_num_tokens_for_logprob is not None
|
||||
assert global_num_tokens_for_logprob_source is not None
|
||||
if (
|
||||
self.spec_info is not None
|
||||
and not batch.dp_spec_prefill_coordination_applied
|
||||
@@ -854,15 +873,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
global_num_tokens, global_num_tokens_for_logprob = (
|
||||
spec_scale_global_num_tokens(
|
||||
self.spec_info,
|
||||
batch.global_num_tokens,
|
||||
batch.global_num_tokens_for_logprob,
|
||||
global_num_tokens_source,
|
||||
global_num_tokens_for_logprob_source,
|
||||
)
|
||||
)
|
||||
else:
|
||||
global_num_tokens = batch.global_num_tokens
|
||||
global_num_tokens_for_logprob = batch.global_num_tokens_for_logprob
|
||||
global_num_tokens = global_num_tokens_source
|
||||
global_num_tokens_for_logprob = global_num_tokens_for_logprob_source
|
||||
|
||||
self.original_global_num_tokens_cpu = batch.global_num_tokens
|
||||
self.original_global_num_tokens_cpu = global_num_tokens_source
|
||||
self.global_num_tokens_cpu = global_num_tokens
|
||||
pin_memory = is_pin_memory_available(device)
|
||||
self.global_num_tokens_gpu = torch.tensor(
|
||||
@@ -875,6 +894,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
pin_memory=pin_memory,
|
||||
).to(device, non_blocking=True)
|
||||
self.can_run_decode_cuda_graph = batch.can_run_decode_cuda_graph
|
||||
self.can_run_dp_draft_cuda_graph = batch.can_run_dp_draft_cuda_graph
|
||||
|
||||
@classmethod
|
||||
def init_new(
|
||||
@@ -976,6 +996,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
return_logprob=batch.return_logprob,
|
||||
is_extend_in_batch=batch.is_extend_in_batch,
|
||||
can_run_decode_cuda_graph=batch.can_run_decode_cuda_graph,
|
||||
can_run_dp_draft_cuda_graph=batch.can_run_dp_draft_cuda_graph,
|
||||
can_run_dp_prefill_cuda_graph=batch.can_run_dp_prefill_cuda_graph,
|
||||
dp_prefill_cuda_graph_max_prefix_len=batch.dp_prefill_cuda_graph_max_prefix_len,
|
||||
global_forward_mode=batch.global_forward_mode,
|
||||
@@ -1045,7 +1066,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
).to(device, non_blocking=True)
|
||||
ret.global_num_token_non_padded_cpu = num_tokens
|
||||
|
||||
ret.init_mlp_sync_metadata(batch, device)
|
||||
ret.init_mlp_sync_metadata(
|
||||
batch, device, is_draft_worker=model_runner.is_draft_worker
|
||||
)
|
||||
|
||||
if ret.forward_mode.is_idle():
|
||||
ret.positions = torch.empty((0,), dtype=torch.int64, device=device)
|
||||
@@ -1979,6 +2002,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
|
||||
]
|
||||
logits_output.hidden_states = logits_output.hidden_states[:bs]
|
||||
elif self.forward_mode.is_extend() or self.forward_mode.is_idle():
|
||||
if self.forward_mode.is_idle():
|
||||
self.positions = self.positions[:bs]
|
||||
self.seq_lens = self.seq_lens[:bs]
|
||||
self.req_pool_indices = self.req_pool_indices[:bs]
|
||||
if self.seq_lens_cpu is not None:
|
||||
self.seq_lens_cpu = self.seq_lens_cpu[:bs]
|
||||
if logits_output.next_token_logits is not None:
|
||||
logits_output.next_token_logits = logits_output.next_token_logits[
|
||||
:bs
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.disaggregation.utils import get_dsa_seed_metadata_dim
|
||||
from sglang.srt.layers.attention.dsa.utils import (
|
||||
should_remap_pd_dsa_seed_to_local_slots,
|
||||
)
|
||||
@@ -17,6 +18,10 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ScheduleBatch
|
||||
|
||||
|
||||
def _requires_dsa_seed_for_cuda_graph(hf_config, topk: int) -> bool:
|
||||
return topk == 1 and get_dsa_seed_metadata_dim(hf_config) > 0
|
||||
|
||||
|
||||
def build_eagle_disagg_draft_input(
|
||||
batch: ScheduleBatch,
|
||||
last_tokens_tensor: torch.Tensor,
|
||||
@@ -87,12 +92,20 @@ def build_eagle_disagg_draft_input(
|
||||
if torch.any(torch.all(dsa_topk_indices < 0, dim=1)).item():
|
||||
dsa_topk_indices = None
|
||||
|
||||
requires_dsa_seed_for_cuda_graph = _requires_dsa_seed_for_cuda_graph(
|
||||
batch.model_config.hf_config,
|
||||
spec.speculative_eagle_topk,
|
||||
)
|
||||
|
||||
spec_info = EagleDraftInput(
|
||||
topk_p=topk_p,
|
||||
topk_index=topk_index,
|
||||
hidden_states=hidden_states,
|
||||
bonus_tokens=last_tokens_tensor,
|
||||
dsa_topk_indices=dsa_topk_indices,
|
||||
cuda_graph_compatible=not (
|
||||
requires_dsa_seed_for_cuda_graph and dsa_topk_indices is None
|
||||
),
|
||||
)
|
||||
spec_info.capture_hidden_mode = CaptureHiddenMode.LAST
|
||||
|
||||
|
||||
@@ -340,7 +340,9 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
|
||||
|
||||
if self.require_mlp_sync:
|
||||
is_bs_supported = (
|
||||
is_bs_supported and forward_batch.can_run_decode_cuda_graph
|
||||
is_bs_supported
|
||||
and forward_batch.can_run_decode_cuda_graph
|
||||
and forward_batch.can_run_dp_draft_cuda_graph
|
||||
)
|
||||
|
||||
return is_bs_supported
|
||||
|
||||
@@ -177,6 +177,10 @@ class EagleDraftInput(SpecInput):
|
||||
future_indices: Optional[torch.Tensor] = None
|
||||
future_dsa_topk_indices_available: bool = False
|
||||
|
||||
# False when this input is missing model-required state for graph replay.
|
||||
# DP attention synchronizes the bit across ranks before the worker runs.
|
||||
cuda_graph_compatible: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
super().__init__(SpecInputType.EAGLE_DRAFT)
|
||||
|
||||
@@ -237,19 +241,31 @@ class EagleDraftInput(SpecInput):
|
||||
def merge_batch(self, spec_info: "EagleDraftInput"):
|
||||
if self.future_indices is not None:
|
||||
assert spec_info.future_indices is not None
|
||||
self_was_empty = len(self.future_indices) == 0
|
||||
spec_info_is_empty = len(spec_info.future_indices) == 0
|
||||
if self_was_empty:
|
||||
self.cuda_graph_compatible = spec_info.cuda_graph_compatible
|
||||
self.future_dsa_topk_indices_available = (
|
||||
spec_info.future_dsa_topk_indices_available
|
||||
)
|
||||
elif not spec_info_is_empty:
|
||||
self.cuda_graph_compatible = (
|
||||
self.cuda_graph_compatible and spec_info.cuda_graph_compatible
|
||||
)
|
||||
self.future_dsa_topk_indices_available = (
|
||||
self.future_dsa_topk_indices_available
|
||||
and spec_info.future_dsa_topk_indices_available
|
||||
)
|
||||
self.future_indices = torch.cat(
|
||||
[self.future_indices, spec_info.future_indices]
|
||||
)
|
||||
self.future_dsa_topk_indices_available = (
|
||||
self.future_dsa_topk_indices_available
|
||||
and spec_info.future_dsa_topk_indices_available
|
||||
)
|
||||
return
|
||||
|
||||
# Detect idle stub by `topk_index` length (idle inputs have
|
||||
# shape[0] == 0 across all fields). Don't use `hidden_states is None`:
|
||||
# for STANDALONE all non-idle inputs also have None hidden_states.
|
||||
if len(self.topk_index) == 0:
|
||||
self.cuda_graph_compatible = spec_info.cuda_graph_compatible
|
||||
self.hidden_states = spec_info.hidden_states
|
||||
self.bonus_tokens = spec_info.bonus_tokens
|
||||
self.topk_p = spec_info.topk_p
|
||||
@@ -259,6 +275,9 @@ class EagleDraftInput(SpecInput):
|
||||
return
|
||||
if len(spec_info.topk_index) == 0:
|
||||
return
|
||||
self.cuda_graph_compatible = (
|
||||
self.cuda_graph_compatible and spec_info.cuda_graph_compatible
|
||||
)
|
||||
if self.hidden_states is not None and spec_info.hidden_states is not None:
|
||||
self.hidden_states = torch.cat(
|
||||
[self.hidden_states, spec_info.hidden_states], axis=0
|
||||
|
||||
@@ -1021,6 +1021,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
num_tokens_per_req=1,
|
||||
num_tokens_for_logprob_per_req=1,
|
||||
dsa_topk_indices=prefill_dsa_topk,
|
||||
cuda_graph_compatible=not (seed_from_extend and prefill_dsa_topk is None),
|
||||
)
|
||||
|
||||
def _get_dsa_extend_topk_buf(self, num_tokens: int) -> torch.Tensor:
|
||||
@@ -1190,6 +1191,9 @@ class EagleDraftWorker(EagleDraftWorkerBase):
|
||||
next_draft_input.draft_probs = ret_draft_probs
|
||||
if self.seed_dsa_topk_from_draft_extend:
|
||||
next_draft_input.dsa_topk_indices = dsa_seed_topk_indices
|
||||
next_draft_input.cuda_graph_compatible = not (
|
||||
self.seed_dsa_topk_from_draft_extend and dsa_seed_topk_indices is None
|
||||
)
|
||||
|
||||
|
||||
class EAGLEWorkerV2(BaseSpecWorker):
|
||||
@@ -1507,7 +1511,12 @@ class EAGLEWorkerV2(BaseSpecWorker):
|
||||
"""Run draft, target, and draft-extend with each rank's local mode."""
|
||||
rank = get_parallel().attn_dp_rank
|
||||
target_local_only = len(batch.global_num_tokens) == 1
|
||||
draft_local_only = target_local_only or self.draft_worker.draft_owns_attention
|
||||
# The draft MoE A2A backend may gather differently from the target.
|
||||
draft_local_only = (
|
||||
target_local_only
|
||||
if batch.draft_global_num_tokens is None
|
||||
else len(batch.draft_global_num_tokens) == 1
|
||||
) or self.draft_worker.draft_owns_attention
|
||||
is_prefill = batch.forward_mode.is_extend()
|
||||
draft_batch = batch
|
||||
if is_prefill:
|
||||
|
||||
@@ -382,6 +382,10 @@ class SpecInputType(IntEnum):
|
||||
|
||||
|
||||
class SpecInput(ABC):
|
||||
# Whether this input carries all state required for CUDA graph replay.
|
||||
# Algorithm-specific dataclass subclasses can override this per input.
|
||||
cuda_graph_compatible: bool = True
|
||||
|
||||
# Per-request verify lengths for the ragged-verify graphs (see
|
||||
# sglang.srt.speculative.ragged_verify); verify inputs of algorithms with
|
||||
# supports_ragged_verify() override it per step. Must stay a class-level
|
||||
|
||||
@@ -3958,11 +3958,16 @@ class Withable(Generic[T]):
|
||||
self._value = None
|
||||
|
||||
|
||||
def require_mlp_tp_gather():
|
||||
def require_mlp_tp_gather(*, moe_a2a_backend=None):
|
||||
"""
|
||||
Check if the input of MLP is obtained by all-gather rather than all-reduce. This only happens when each MLP TP group contains multiple attention DP groups.
|
||||
"""
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
from sglang.srt.layers.moe.utils import MoeA2ABackend, get_moe_a2a_backend
|
||||
|
||||
if moe_a2a_backend is None:
|
||||
moe_a2a_backend = get_moe_a2a_backend()
|
||||
elif not isinstance(moe_a2a_backend, MoeA2ABackend):
|
||||
moe_a2a_backend = MoeA2ABackend(moe_a2a_backend)
|
||||
|
||||
# elastic-EP scale-up rewrites dp_size on the published config
|
||||
if get_parallel().enable_dp_attention:
|
||||
@@ -3980,9 +3985,9 @@ def require_mlp_tp_gather():
|
||||
return True
|
||||
elif not get_parallel().enable_dp_lm_head:
|
||||
return True
|
||||
elif get_moe_a2a_backend().is_none():
|
||||
elif moe_a2a_backend.is_none():
|
||||
return True
|
||||
elif get_moe_a2a_backend().is_flashinfer():
|
||||
elif moe_a2a_backend.is_flashinfer():
|
||||
# FlashInfer MoE A2A needs a rank-invariant, DP-synchronized per-rank
|
||||
# token count: MoeAlltoAll uses fixed-geometry buffers and the decode
|
||||
# cuda-graph bucket must be identical across EP ranks, otherwise ranks
|
||||
@@ -3992,7 +3997,7 @@ def require_mlp_tp_gather():
|
||||
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
|
||||
# max-based graph bucket). See #30432 re: the misleading flag name.
|
||||
return True
|
||||
elif get_moe_a2a_backend().is_mori() and get_bool_env_var(
|
||||
elif moe_a2a_backend.is_mori() and get_bool_env_var(
|
||||
"SGLANG_MORI_RECV_BOUND", "false"
|
||||
):
|
||||
# Same bookkeeping, for the same reason. Bounding mori's receive
|
||||
|
||||
Reference in New Issue
Block a user