[PD] Keep EAGLE DP graph and token metadata consistent (#32196)

Co-authored-by: weireweire <20922698+weireweire@users.noreply.github.com>
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Khoa Pham <khoa.pham@radixark.ai>
This commit is contained in:
weireweire
2026-09-28 17:15:51 -07:00
committed by GitHub
co-authored by weireweire Po-Han Huang Khoa Pham
parent 89e1316eae
commit 7889fbab9a
21 changed files with 649 additions and 29 deletions
@@ -756,6 +756,7 @@ class TboForwardBatchPreparer:
"dp_spec_prefill_coordination_applied",
"return_logprob",
"can_run_decode_cuda_graph",
"can_run_dp_draft_cuda_graph",
"can_run_dp_prefill_cuda_graph",
"dp_prefill_cuda_graph_max_prefix_len",
"dp_padding_mode",
@@ -201,6 +201,19 @@ def _broadcast_indexer_topk_from_rank0(
return topk_indices
def _make_eager_idle_topk_result(
x: torch.Tensor, index_topk: int, return_indices: bool
) -> Optional[torch.Tensor]:
if not return_indices:
return None
return torch.full(
(x.shape[0], index_topk),
-1,
dtype=torch.int32,
device=x.device,
)
def rotate_activation(x: torch.Tensor) -> torch.Tensor:
if _is_hip:
from fast_hadamard_transform import hadamard_transform
@@ -1591,6 +1604,21 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
layer_id: int,
return_indices: bool = True,
) -> Optional[torch.Tensor]:
# A padded eager IDLE rank has no real requests and therefore no valid
# DSA page-table rows to index. Return invalid rows with the physical DP
# shape so later MLP/EP collectives still agree across ranks.
x_meta = x[0] if isinstance(x, tuple) else x
if (
_is_cuda
and forward_batch.forward_mode.is_idle()
and not get_is_capture_mode()
):
topk_result = _make_eager_idle_topk_result(
x_meta, self.index_topk, return_indices
)
topk_result = _broadcast_indexer_topk_from_rank0(topk_result)
return maybe_capture_indexer_topk(layer_id, topk_result)
if _is_hip:
from sglang.kernels.ops.attention.dsa.tilelang_kernel import act_quant
elif not _is_npu:
@@ -1599,10 +1627,6 @@ class Indexer(DSANPUIndexerMixin, BaseFusedOp):
if TYPE_CHECKING:
assert isinstance(get_token_to_kv_pool(), DSATokenToKVPool)
# When upstream uses fused FP8 RMSNorm+quant, activations may be passed as
# a tuple like (x_fp8, x_scale[, y]). Use `x_meta` for shape/device queries.
x_meta = x[0] if isinstance(x, tuple) else x
in_piecewise_or_breakable_cuda_graph = (
_is_in_piecewise_or_breakable_cuda_graph()
)
@@ -173,6 +173,46 @@ def _to_2d_context_lens(seqlens_32: torch.Tensor, batch_size: int) -> torch.Tens
return seqlens_32.contiguous().view(-1, 1)
def _trim_trtllm_decode_dp_padding(
q_all: torch.Tensor,
topk_indices: Optional[torch.Tensor],
real_batch_size: int,
) -> Tuple[torch.Tensor, Optional[torch.Tensor], int]:
"""Align eager decode inputs with metadata planned before DP padding."""
physical_batch_size = q_all.shape[0]
assert real_batch_size <= physical_batch_size, (
f"DSA metadata batch size ({real_batch_size}) exceeds q batch size "
f"({physical_batch_size})"
)
if topk_indices is not None:
assert real_batch_size <= topk_indices.shape[0], (
f"DSA metadata batch size ({real_batch_size}) exceeds topk batch size "
f"({topk_indices.shape[0]})"
)
num_padding_rows = physical_batch_size - real_batch_size
if num_padding_rows == 0:
return q_all, topk_indices, 0
return (
q_all[:real_batch_size],
topk_indices[:real_batch_size] if topk_indices is not None else None,
num_padding_rows,
)
def _restore_trtllm_decode_dp_padding(
output: torch.Tensor, num_padding_rows: int
) -> torch.Tensor:
"""Restore the physical DP shape required by downstream MLP collectives."""
if num_padding_rows == 0:
return output
return torch.cat(
[output, output.new_zeros((num_padding_rows, *output.shape[1:]))],
dim=0,
)
@dataclass(frozen=True)
class DSAFlashMLAMetadata:
"""Metadata only needed by FlashMLA"""
@@ -3441,9 +3481,24 @@ class DeepseekSparseAttnBackend(
else:
q_all = q.view(-1, layer.tp_q_head_num, layer.head_dim)
# Eager DP attention can pad q beyond metadata that was deliberately
# planned on the real draft batch. Pad top-k to the physical q shape,
# then run decode attention only on metadata-backed rows. The output is
# restored below before downstream MLP/EP collectives.
if (self.use_fused_topk or not is_prefill) and topk_indices is not None:
topk_indices = self._pad_topk_indices(topk_indices, q.shape[0])
num_decode_padding_rows = 0
if not is_prefill:
q_all, topk_indices, num_decode_padding_rows = (
_trim_trtllm_decode_dp_padding(
q_all,
topk_indices,
metadata.cache_seqlens_int32.shape[0],
)
)
if self.use_fused_topk:
if topk_indices is not None:
topk_indices = self._pad_topk_indices(topk_indices, q.shape[0])
page_table_1 = self._get_fused_topk_page_table(topk_indices)
elif is_prefill:
page_table_1 = transform_index_page_table_prefill(
@@ -3459,8 +3514,6 @@ class DeepseekSparseAttnBackend(
cu_seqlens_q=metadata.cu_seqlens_q,
)
else:
if topk_indices is not None:
topk_indices = self._pad_topk_indices(topk_indices, q.shape[0])
page_table_1 = transform_index_page_table_decode(
page_table=metadata.page_table_1,
topk_indices=topk_indices,
@@ -3509,7 +3562,7 @@ class DeepseekSparseAttnBackend(
multi_ctas_kv_counter_buffer=multi_ctas_kv_counter_buffer,
)
return out
return _restore_trtllm_decode_dp_padding(out, num_decode_padding_rows)
def _pad_topk_indices(
self, topk_indices: torch.Tensor, num_tokens: int
@@ -2471,6 +2471,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# For DP attention
is_extend_in_batch: bool = False
can_run_decode_cuda_graph: bool = False
# Rank-consistent EAGLE draft replay gate. Keep it separate so missing
# draft-only state does not disable target verification or draft extend.
can_run_dp_draft_cuda_graph: bool = False
can_run_dp_prefill_cuda_graph: bool = False
dp_prefill_cuda_graph_max_prefix_len: int = 0
tbo_split_seq_index: Optional[int] = None
@@ -2524,6 +2527,9 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
# For DP attention
global_num_tokens: Optional[List[int]] = None
global_num_tokens_for_logprob: Optional[List[int]] = None
# The draft model can use a different MoE A2A backend than the target.
draft_global_num_tokens: Optional[List[int]] = None
draft_global_num_tokens_for_logprob: Optional[List[int]] = None
# Full DP token vector retained for Aiter MegaMoE even when the normal MLP
# TP gather path stores only this rank's token count.
global_spec_verify_tier_num_tokens: Optional[List[int]] = None
@@ -3831,6 +3837,7 @@ class ScheduleBatch(ScheduleBatchDisaggregationDecodeMixin):
global_num_tokens=self.global_num_tokens,
global_num_tokens_for_logprob=self.global_num_tokens_for_logprob,
can_run_decode_cuda_graph=self.can_run_decode_cuda_graph,
can_run_dp_draft_cuda_graph=self.can_run_dp_draft_cuda_graph,
can_run_dp_prefill_cuda_graph=self.can_run_dp_prefill_cuda_graph,
dp_prefill_cuda_graph_max_prefix_len=self.dp_prefill_cuda_graph_max_prefix_len,
is_extend_in_batch=self.is_extend_in_batch,
@@ -34,6 +34,7 @@ from sglang.srt.runtime_context import (
get_memory,
get_parallel,
get_schedule,
get_spec,
)
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
from sglang.srt.utils.common import require_mlp_tp_gather
@@ -45,6 +46,25 @@ if TYPE_CHECKING:
_ENABLE_METRICS_DP_ATTENTION = envs.SGLANG_ENABLE_METRICS_DP_ATTENTION.get()
def _spec_input_cuda_graph_compatible(
local_batch: Optional[ScheduleBatch],
) -> bool:
"""Return the local speculative-draft graph admission bit.
None/idle/prebuilt inputs stay permissive so an active rank with complete
runtime state can still use graphs. Any active incompatible input is
min-reduced by ``MLPSyncBatchInfo`` and forces every DP rank eager.
"""
if (
local_batch is None
or local_batch.forward_mode.is_idle()
or local_batch.forward_mode.is_prebuilt()
):
return True
spec_info = local_batch.spec_info
return spec_info is None or spec_info.cuda_graph_compatible
def _resolve_elastic_world_dp_size(
dp_size: int,
*,
@@ -92,6 +112,7 @@ class MLPSyncBatchInfo:
num_tokens: int
num_tokens_for_logprob: int
can_run_decode_cuda_graph: bool
can_run_draft_cuda_graph: bool
can_run_prefill_cuda_graph: bool
is_extend_in_batch: bool
local_can_run_tbo: bool
@@ -117,6 +138,7 @@ class MLPSyncBatchInfo:
self.local_forward_mode,
int(self.can_run_prefill_cuda_graph),
self.prefill_cuda_graph_max_prefix_len,
int(self.can_run_draft_cuda_graph),
],
device=device,
dtype=dtype,
@@ -133,6 +155,7 @@ class MLPSyncBatchInfo:
ForwardMode.IDLE.value, # local_forward_mode
0, # can_run_prefill_cuda_graph
0, # prefill_cuda_graph_max_prefix_len
1, # can_run_draft_cuda_graph
],
device=device,
dtype=dtype,
@@ -215,6 +238,7 @@ class MLPSyncBatchInfo:
self.is_extend_in_batch = bool(tp0_info_cpu[:, 3].max())
self.can_run_prefill_cuda_graph = bool(tp0_info_cpu[:, 6].min())
self.prefill_cuda_graph_max_prefix_len = int(tp0_info_cpu[:, 7].max())
self.can_run_draft_cuda_graph = bool(tp0_info_cpu[:, 8].min())
if _ENABLE_METRICS_DP_ATTENTION:
self.dp_cooperation_info = DPCooperationInfo.create(
tp0_info_cpu[:, 5].tolist()
@@ -225,6 +249,7 @@ def _update_gather_batch(
batch: ScheduleBatch,
mlp_sync_info: MLPSyncBatchInfo,
require_mlp_tp_gather: bool,
draft_require_mlp_tp_gather: Optional[bool] = None,
skip_global_metadata=False,
):
if not require_mlp_tp_gather:
@@ -235,6 +260,19 @@ def _update_gather_batch(
batch.global_num_tokens_for_logprob = (
mlp_sync_info.global_num_tokens_for_logprob
)
# Reuse the same all-gather result for a draft model whose A2A backend
# requires a different local/full token-count representation.
if draft_require_mlp_tp_gather is not None:
if draft_require_mlp_tp_gather:
batch.draft_global_num_tokens = mlp_sync_info.global_num_tokens
batch.draft_global_num_tokens_for_logprob = (
mlp_sync_info.global_num_tokens_for_logprob
)
else:
batch.draft_global_num_tokens = [mlp_sync_info.num_tokens]
batch.draft_global_num_tokens_for_logprob = [
mlp_sync_info.num_tokens_for_logprob
]
if envs.SGLANG_ENABLE_DP_SPEC_PREFILL_COORDINATION.get():
# Fresh counts have not yet been adjusted by the coordination plan.
batch.dp_spec_prefill_coordination_applied = False
@@ -251,6 +289,7 @@ def _update_gather_batch(
# Check forward mode for cuda graph
batch.can_run_decode_cuda_graph = mlp_sync_info.can_run_decode_cuda_graph
batch.can_run_dp_draft_cuda_graph = mlp_sync_info.can_run_draft_cuda_graph
batch.can_run_dp_prefill_cuda_graph = mlp_sync_info.can_run_prefill_cuda_graph
batch.dp_prefill_cuda_graph_max_prefix_len = (
mlp_sync_info.prefill_cuda_graph_max_prefix_len
@@ -378,6 +417,7 @@ def prepare_mlp_sync_batch_raw(
require_mlp_tp_gather: bool,
disable_overlap_schedule: bool,
offload_tags: set[str],
draft_require_mlp_tp_gather: Optional[bool] = None,
dwdp: bool = False,
):
parallel = get_parallel()
@@ -413,6 +453,7 @@ def prepare_mlp_sync_batch_raw(
can_run_decode_cuda_graph = _local_decode_cuda_graph_vote(
local_batch=local_batch, disable_cuda_graph=disable_cuda_graph
)
can_run_draft_cuda_graph = _spec_input_cuda_graph_compatible(local_batch)
breakable_prefill = check_cuda_graph_backend(Phase.PREFILL, Backend.BREAKABLE)
full_prefill = check_cuda_graph_backend(Phase.PREFILL, Backend.FULL)
coordinated_prefill = breakable_prefill or full_prefill
@@ -472,6 +513,7 @@ def prepare_mlp_sync_batch_raw(
num_tokens=num_tokens,
num_tokens_for_logprob=num_tokens_for_logprob,
can_run_decode_cuda_graph=can_run_decode_cuda_graph,
can_run_draft_cuda_graph=can_run_draft_cuda_graph,
can_run_prefill_cuda_graph=can_run_prefill_cuda_graph,
is_extend_in_batch=is_extend_in_batch,
local_can_run_tbo=local_can_run_tbo,
@@ -515,6 +557,7 @@ def prepare_mlp_sync_batch_raw(
batch_to_gather,
mlp_sync_info,
require_mlp_tp_gather,
draft_require_mlp_tp_gather,
skip_global_metadata=not metadata_ready,
)
@@ -546,12 +589,22 @@ class SchedulerDPAttnAdapter:
get_require_mlp_sync: Callable[[], bool]
def prepare_mlp_sync_batch(self, local_batch: ScheduleBatch):
draft_require_mlp_tp_gather = None
if self.spec_algorithm.is_eagle() or self.spec_algorithm.is_standalone():
draft_moe_a2a_backend = get_spec().speculative_moe_a2a_backend
if draft_moe_a2a_backend is None:
draft_moe_a2a_backend = get_exec().moe.moe_a2a_backend
draft_require_mlp_tp_gather = require_mlp_tp_gather(
moe_a2a_backend=draft_moe_a2a_backend,
)
return prepare_mlp_sync_batch_raw(
local_batch,
model_runner=self.model_runner,
get_idle_batch=self.get_idle_batch,
disable_cuda_graph=cuda_graph_fully_disabled(),
require_mlp_tp_gather=require_mlp_tp_gather(),
draft_require_mlp_tp_gather=draft_require_mlp_tp_gather,
disable_overlap_schedule=get_schedule().disable_overlap_schedule,
offload_tags=self.offload_tags,
dwdp=get_parallel().dwdp_size > 1,
@@ -558,6 +558,8 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
is_extend_in_batch: bool = False
dp_spec_prefill_coordination_applied: bool = False
can_run_decode_cuda_graph: bool = False
# Draft-only companion to the generic DP decode graph vote.
can_run_dp_draft_cuda_graph: bool = False
can_run_dp_prefill_cuda_graph: bool = False
dp_prefill_cuda_graph_max_prefix_len: int = 0
global_forward_mode: Optional[ForwardMode] = None
@@ -835,16 +837,33 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
self.mark_forward_metadata_ready()
def init_mlp_sync_metadata(
self, batch: ScheduleBatch, device: Union[str, torch.device]
self,
batch: ScheduleBatch,
device: Union[str, torch.device],
*,
is_draft_worker: bool = False,
) -> None:
"""Populate per-rank token counts for DP-attention MLP synchronization."""
self.dp_spec_prefill_coordination_applied = (
batch.dp_spec_prefill_coordination_applied
)
if batch.global_num_tokens is None:
global_num_tokens_source = batch.global_num_tokens
global_num_tokens_for_logprob_source = batch.global_num_tokens_for_logprob
# A DP spec/prefill coordination plan already wrote this phase's counts.
if (
is_draft_worker
and batch.draft_global_num_tokens is not None
and not batch.dp_spec_prefill_coordination_applied
):
global_num_tokens_source = batch.draft_global_num_tokens
global_num_tokens_for_logprob_source = (
batch.draft_global_num_tokens_for_logprob
)
if global_num_tokens_source is None:
return
assert batch.global_num_tokens_for_logprob is not None
assert global_num_tokens_for_logprob_source is not None
if (
self.spec_info is not None
and not batch.dp_spec_prefill_coordination_applied
@@ -854,15 +873,15 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
global_num_tokens, global_num_tokens_for_logprob = (
spec_scale_global_num_tokens(
self.spec_info,
batch.global_num_tokens,
batch.global_num_tokens_for_logprob,
global_num_tokens_source,
global_num_tokens_for_logprob_source,
)
)
else:
global_num_tokens = batch.global_num_tokens
global_num_tokens_for_logprob = batch.global_num_tokens_for_logprob
global_num_tokens = global_num_tokens_source
global_num_tokens_for_logprob = global_num_tokens_for_logprob_source
self.original_global_num_tokens_cpu = batch.global_num_tokens
self.original_global_num_tokens_cpu = global_num_tokens_source
self.global_num_tokens_cpu = global_num_tokens
pin_memory = is_pin_memory_available(device)
self.global_num_tokens_gpu = torch.tensor(
@@ -875,6 +894,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
pin_memory=pin_memory,
).to(device, non_blocking=True)
self.can_run_decode_cuda_graph = batch.can_run_decode_cuda_graph
self.can_run_dp_draft_cuda_graph = batch.can_run_dp_draft_cuda_graph
@classmethod
def init_new(
@@ -976,6 +996,7 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
return_logprob=batch.return_logprob,
is_extend_in_batch=batch.is_extend_in_batch,
can_run_decode_cuda_graph=batch.can_run_decode_cuda_graph,
can_run_dp_draft_cuda_graph=batch.can_run_dp_draft_cuda_graph,
can_run_dp_prefill_cuda_graph=batch.can_run_dp_prefill_cuda_graph,
dp_prefill_cuda_graph_max_prefix_len=batch.dp_prefill_cuda_graph_max_prefix_len,
global_forward_mode=batch.global_forward_mode,
@@ -1045,7 +1066,9 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
).to(device, non_blocking=True)
ret.global_num_token_non_padded_cpu = num_tokens
ret.init_mlp_sync_metadata(batch, device)
ret.init_mlp_sync_metadata(
batch, device, is_draft_worker=model_runner.is_draft_worker
)
if ret.forward_mode.is_idle():
ret.positions = torch.empty((0,), dtype=torch.int64, device=device)
@@ -1979,6 +2002,12 @@ class ForwardBatch(ForwardBatchDeepSeekMHAMixin):
]
logits_output.hidden_states = logits_output.hidden_states[:bs]
elif self.forward_mode.is_extend() or self.forward_mode.is_idle():
if self.forward_mode.is_idle():
self.positions = self.positions[:bs]
self.seq_lens = self.seq_lens[:bs]
self.req_pool_indices = self.req_pool_indices[:bs]
if self.seq_lens_cpu is not None:
self.seq_lens_cpu = self.seq_lens_cpu[:bs]
if logits_output.next_token_logits is not None:
logits_output.next_token_logits = logits_output.next_token_logits[
:bs
@@ -4,6 +4,7 @@ from typing import TYPE_CHECKING
import torch
from sglang.srt.disaggregation.utils import get_dsa_seed_metadata_dim
from sglang.srt.layers.attention.dsa.utils import (
should_remap_pd_dsa_seed_to_local_slots,
)
@@ -17,6 +18,10 @@ if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import ScheduleBatch
def _requires_dsa_seed_for_cuda_graph(hf_config, topk: int) -> bool:
return topk == 1 and get_dsa_seed_metadata_dim(hf_config) > 0
def build_eagle_disagg_draft_input(
batch: ScheduleBatch,
last_tokens_tensor: torch.Tensor,
@@ -87,12 +92,20 @@ def build_eagle_disagg_draft_input(
if torch.any(torch.all(dsa_topk_indices < 0, dim=1)).item():
dsa_topk_indices = None
requires_dsa_seed_for_cuda_graph = _requires_dsa_seed_for_cuda_graph(
batch.model_config.hf_config,
spec.speculative_eagle_topk,
)
spec_info = EagleDraftInput(
topk_p=topk_p,
topk_index=topk_index,
hidden_states=hidden_states,
bonus_tokens=last_tokens_tensor,
dsa_topk_indices=dsa_topk_indices,
cuda_graph_compatible=not (
requires_dsa_seed_for_cuda_graph and dsa_topk_indices is None
),
)
spec_info.capture_hidden_mode = CaptureHiddenMode.LAST
@@ -340,7 +340,9 @@ class EAGLEDraftCudaGraphRunner(DecodeCudaGraphRunner):
if self.require_mlp_sync:
is_bs_supported = (
is_bs_supported and forward_batch.can_run_decode_cuda_graph
is_bs_supported
and forward_batch.can_run_decode_cuda_graph
and forward_batch.can_run_dp_draft_cuda_graph
)
return is_bs_supported
+23 -4
View File
@@ -177,6 +177,10 @@ class EagleDraftInput(SpecInput):
future_indices: Optional[torch.Tensor] = None
future_dsa_topk_indices_available: bool = False
# False when this input is missing model-required state for graph replay.
# DP attention synchronizes the bit across ranks before the worker runs.
cuda_graph_compatible: bool = True
def __post_init__(self):
super().__init__(SpecInputType.EAGLE_DRAFT)
@@ -237,19 +241,31 @@ class EagleDraftInput(SpecInput):
def merge_batch(self, spec_info: "EagleDraftInput"):
if self.future_indices is not None:
assert spec_info.future_indices is not None
self_was_empty = len(self.future_indices) == 0
spec_info_is_empty = len(spec_info.future_indices) == 0
if self_was_empty:
self.cuda_graph_compatible = spec_info.cuda_graph_compatible
self.future_dsa_topk_indices_available = (
spec_info.future_dsa_topk_indices_available
)
elif not spec_info_is_empty:
self.cuda_graph_compatible = (
self.cuda_graph_compatible and spec_info.cuda_graph_compatible
)
self.future_dsa_topk_indices_available = (
self.future_dsa_topk_indices_available
and spec_info.future_dsa_topk_indices_available
)
self.future_indices = torch.cat(
[self.future_indices, spec_info.future_indices]
)
self.future_dsa_topk_indices_available = (
self.future_dsa_topk_indices_available
and spec_info.future_dsa_topk_indices_available
)
return
# Detect idle stub by `topk_index` length (idle inputs have
# shape[0] == 0 across all fields). Don't use `hidden_states is None`:
# for STANDALONE all non-idle inputs also have None hidden_states.
if len(self.topk_index) == 0:
self.cuda_graph_compatible = spec_info.cuda_graph_compatible
self.hidden_states = spec_info.hidden_states
self.bonus_tokens = spec_info.bonus_tokens
self.topk_p = spec_info.topk_p
@@ -259,6 +275,9 @@ class EagleDraftInput(SpecInput):
return
if len(spec_info.topk_index) == 0:
return
self.cuda_graph_compatible = (
self.cuda_graph_compatible and spec_info.cuda_graph_compatible
)
if self.hidden_states is not None and spec_info.hidden_states is not None:
self.hidden_states = torch.cat(
[self.hidden_states, spec_info.hidden_states], axis=0
@@ -1021,6 +1021,7 @@ class EagleDraftWorker(EagleDraftWorkerBase):
num_tokens_per_req=1,
num_tokens_for_logprob_per_req=1,
dsa_topk_indices=prefill_dsa_topk,
cuda_graph_compatible=not (seed_from_extend and prefill_dsa_topk is None),
)
def _get_dsa_extend_topk_buf(self, num_tokens: int) -> torch.Tensor:
@@ -1190,6 +1191,9 @@ class EagleDraftWorker(EagleDraftWorkerBase):
next_draft_input.draft_probs = ret_draft_probs
if self.seed_dsa_topk_from_draft_extend:
next_draft_input.dsa_topk_indices = dsa_seed_topk_indices
next_draft_input.cuda_graph_compatible = not (
self.seed_dsa_topk_from_draft_extend and dsa_seed_topk_indices is None
)
class EAGLEWorkerV2(BaseSpecWorker):
@@ -1507,7 +1511,12 @@ class EAGLEWorkerV2(BaseSpecWorker):
"""Run draft, target, and draft-extend with each rank's local mode."""
rank = get_parallel().attn_dp_rank
target_local_only = len(batch.global_num_tokens) == 1
draft_local_only = target_local_only or self.draft_worker.draft_owns_attention
# The draft MoE A2A backend may gather differently from the target.
draft_local_only = (
target_local_only
if batch.draft_global_num_tokens is None
else len(batch.draft_global_num_tokens) == 1
) or self.draft_worker.draft_owns_attention
is_prefill = batch.forward_mode.is_extend()
draft_batch = batch
if is_prefill:
@@ -382,6 +382,10 @@ class SpecInputType(IntEnum):
class SpecInput(ABC):
# Whether this input carries all state required for CUDA graph replay.
# Algorithm-specific dataclass subclasses can override this per input.
cuda_graph_compatible: bool = True
# Per-request verify lengths for the ragged-verify graphs (see
# sglang.srt.speculative.ragged_verify); verify inputs of algorithms with
# supports_ragged_verify() override it per step. Must stay a class-level
+10 -5
View File
@@ -3958,11 +3958,16 @@ class Withable(Generic[T]):
self._value = None
def require_mlp_tp_gather():
def require_mlp_tp_gather(*, moe_a2a_backend=None):
"""
Check if the input of MLP is obtained by all-gather rather than all-reduce. This only happens when each MLP TP group contains multiple attention DP groups.
"""
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
from sglang.srt.layers.moe.utils import MoeA2ABackend, get_moe_a2a_backend
if moe_a2a_backend is None:
moe_a2a_backend = get_moe_a2a_backend()
elif not isinstance(moe_a2a_backend, MoeA2ABackend):
moe_a2a_backend = MoeA2ABackend(moe_a2a_backend)
# elastic-EP scale-up rewrites dp_size on the published config
if get_parallel().enable_dp_attention:
@@ -3980,9 +3985,9 @@ def require_mlp_tp_gather():
return True
elif not get_parallel().enable_dp_lm_head:
return True
elif get_moe_a2a_backend().is_none():
elif moe_a2a_backend.is_none():
return True
elif get_moe_a2a_backend().is_flashinfer():
elif moe_a2a_backend.is_flashinfer():
# FlashInfer MoE A2A needs a rank-invariant, DP-synchronized per-rank
# token count: MoeAlltoAll uses fixed-geometry buffers and the decode
# cuda-graph bucket must be identical across EP ranks, otherwise ranks
@@ -3992,7 +3997,7 @@ def require_mlp_tp_gather():
# reuse this flag's DP-sync bookkeeping (uniform global_num_tokens +
# max-based graph bucket). See #30432 re: the misleading flag name.
return True
elif get_moe_a2a_backend().is_mori() and get_bool_env_var(
elif moe_a2a_backend.is_mori() and get_bool_env_var(
"SGLANG_MORI_RECV_BOUND", "false"
):
# Same bookkeeping, for the same reason. Bounding mori's receive