[Feature] DeepEP v2: expanded (do_expand=True) prefill dispatch (#37261)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai> Co-authored-by: Cheng Wan <54331508+ch-wan@users.noreply.github.com>
This commit is contained in:
co-authored by
Cheng Wan
Cheng Wan
parent
cafb37a10d
commit
438d9d2074
@@ -1337,46 +1337,6 @@ def ep_scatter_from_psum(
|
||||
return
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kernel_ep_expand_m_indices_init(
|
||||
psum_num_recv_tokens_per_expert,
|
||||
m_indices,
|
||||
BLOCK_E: tl.constexpr,
|
||||
):
|
||||
cur_expert = tl.program_id(0)
|
||||
cur_end = tl.load(psum_num_recv_tokens_per_expert + cur_expert)
|
||||
prev_end = tl.load(
|
||||
psum_num_recv_tokens_per_expert + cur_expert - 1,
|
||||
mask=cur_expert > 0,
|
||||
other=0,
|
||||
)
|
||||
cur_start = ((prev_end + BLOCK_E - 1) // BLOCK_E) * BLOCK_E
|
||||
aligned_end = ((cur_end + BLOCK_E - 1) // BLOCK_E) * BLOCK_E
|
||||
|
||||
off_expert = tl.arange(0, BLOCK_E)
|
||||
for start_m in tl.range(0, aligned_end - cur_start, BLOCK_E, num_stages=4):
|
||||
idx = cur_start + start_m + off_expert
|
||||
tl.store(m_indices + idx, cur_expert, mask=idx < aligned_end)
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def ep_expand_init_m_indices_from_psum(
|
||||
psum_num_recv_tokens_per_expert: torch.Tensor,
|
||||
m_indices: torch.Tensor,
|
||||
):
|
||||
BLOCK_E = 128
|
||||
num_warps = 8
|
||||
num_experts = psum_num_recv_tokens_per_expert.shape[0]
|
||||
assert m_indices.shape[0] % BLOCK_E == 0
|
||||
_fwd_kernel_ep_expand_m_indices_init[(num_experts,)](
|
||||
psum_num_recv_tokens_per_expert,
|
||||
m_indices,
|
||||
num_warps=num_warps,
|
||||
BLOCK_E=BLOCK_E,
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kernel_ep_gather(
|
||||
total_token_num,
|
||||
@@ -2578,6 +2538,95 @@ def masked_slab_to_expand(
|
||||
return output_tensor
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _fwd_kernel_fill_m_indices_from_psum(
|
||||
psum_ptr,
|
||||
m_indices_ptr,
|
||||
total_rows,
|
||||
ALIGN: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
):
|
||||
# psum is DeepEP's inclusive per-expert count: psum[i] = align(psum[i-1]) +
|
||||
# count_i (only earlier experts aligned), so both start and seg_end round up.
|
||||
# Each expert's whole aligned segment gets its id, padding rows included --
|
||||
# combine ignores them, so the junk the GEMM computes there is discarded.
|
||||
# Only rows past the last segment keep the wrapper's -1 sentinel; with
|
||||
# do_cpu_sync=True there are none, so that fill is pure insurance.
|
||||
e = tl.program_id(0)
|
||||
prev_end = tl.load(psum_ptr + e - 1, mask=e > 0, other=0)
|
||||
start = ((prev_end + ALIGN - 1) // ALIGN) * ALIGN
|
||||
end = tl.load(psum_ptr + e)
|
||||
seg_end = ((end + ALIGN - 1) // ALIGN) * ALIGN
|
||||
count = seg_end - start
|
||||
off = tl.arange(0, BLOCK_M)
|
||||
for base in tl.range(0, count, BLOCK_M):
|
||||
idx = start + base + off
|
||||
tl.store(m_indices_ptr + idx, e, mask=(base + off < count) & (idx < total_rows))
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def fill_m_indices_from_psum(
|
||||
psum_num_recv_tokens_per_expert: torch.Tensor,
|
||||
num_local_experts: int,
|
||||
total_rows: int,
|
||||
expert_alignment: int,
|
||||
) -> torch.Tensor:
|
||||
"""Build contiguous-GEMM `m_indices` from the device psum (deepep_v2
|
||||
`do_expand=True` prefill).
|
||||
|
||||
The psum from DeepEP is already the alignment-padded per-expert prefix sum,
|
||||
so this only labels rows.
|
||||
"""
|
||||
# do_cpu_sync=True sizes recv_x to align(psum[-1]); the last expert's segment
|
||||
# therefore ends exactly at total_rows (no capacity tail to skip).
|
||||
assert total_rows % expert_alignment == 0, (
|
||||
f"total_rows {total_rows} not a multiple of expert_alignment {expert_alignment}"
|
||||
)
|
||||
# -1 is DeepGEMM's "empty token" sentinel: uncovered rows are zero-filled and
|
||||
# skipped instead of indexing the weight tensor out of bounds.
|
||||
m_indices = torch.full(
|
||||
(total_rows,),
|
||||
-1,
|
||||
device=psum_num_recv_tokens_per_expert.device,
|
||||
dtype=torch.int32,
|
||||
)
|
||||
_fwd_kernel_fill_m_indices_from_psum[(num_local_experts,)](
|
||||
psum_num_recv_tokens_per_expert,
|
||||
m_indices,
|
||||
total_rows,
|
||||
ALIGN=expert_alignment,
|
||||
BLOCK_M=128,
|
||||
num_warps=4,
|
||||
)
|
||||
return m_indices
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def scale_expanded_rows_(
|
||||
x: torch.Tensor,
|
||||
row_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""In-place `x[r, :] *= row_weights[r]` for a 2D `x`, any strides.
|
||||
|
||||
deepep_v2 `do_expand=True` prefill folds the router weights into the
|
||||
down-proj input scale here, because ElasticBuffer.combine ignores
|
||||
topk_weights in expand mode. `row_weights` must be a 1-D `[rows]` tensor
|
||||
(what DeepEP hands back). `x` must be fp32: a packed UE8M0 scale cannot
|
||||
absorb the weight, and a lower-precision `x` would round it.
|
||||
"""
|
||||
assert x.dim() == 2, f"expected 2D x, got {tuple(x.shape)}"
|
||||
assert x.dtype == torch.float32, f"expected fp32 x, got {x.dtype}"
|
||||
rows, _ = x.shape
|
||||
assert row_weights.dim() == 1, (
|
||||
f"expected 1-D row_weights, got {row_weights.dim()}-D {tuple(row_weights.shape)}"
|
||||
)
|
||||
assert row_weights.numel() == rows, (
|
||||
f"row_weights has {row_weights.numel()} entries but x has {rows} rows"
|
||||
)
|
||||
x.mul_(row_weights.unsqueeze(1))
|
||||
return x
|
||||
|
||||
|
||||
def _moe_permute_rows(
|
||||
inputs: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
|
||||
@@ -35,14 +35,13 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict:
|
||||
and cfg.moe_runner_backend == "auto"
|
||||
and get_quantization_config(hf_config) == "fp8"
|
||||
):
|
||||
if model_config_of(server_args).is_fp4_experts:
|
||||
# Let the all-to-all backend choose its compatible runner.
|
||||
if cfg.moe_a2a_backend == "none":
|
||||
# Let the all-to-all backend choose its compatible runner.
|
||||
if cfg.moe_a2a_backend == "none":
|
||||
if model_config_of(server_args).is_fp4_experts:
|
||||
overrides["moe_runner_backend"] = "flashinfer_mxfp4"
|
||||
else:
|
||||
# Avoid the slower Triton default for ordinary FP8 checkpoints.
|
||||
overrides["moe_runner_backend"] = "flashinfer_trtllm"
|
||||
if "moe_runner_backend" in overrides:
|
||||
else:
|
||||
# Avoid the slower Triton default for ordinary FP8 checkpoints.
|
||||
overrides["moe_runner_backend"] = "flashinfer_trtllm"
|
||||
logger.info(
|
||||
"MiMoV2 on SM100: moe_runner_backend=%s.",
|
||||
overrides["moe_runner_backend"],
|
||||
|
||||
@@ -360,6 +360,11 @@ def handle_a2a_moe(server_args: Any):
|
||||
"Remove --enforce-shared-experts-fusion when using "
|
||||
"--moe-a2a-backend deepep_v2."
|
||||
)
|
||||
if cfg.deepep_v2_mode == "direct" and cfg.nnodes > 1:
|
||||
raise ValueError(
|
||||
"--deepep-v2-mode direct is NVLink-only and cannot run across "
|
||||
f"nodes (nnodes={cfg.nnodes}); pass --deepep-v2-mode hybrid."
|
||||
)
|
||||
# Prefill reads host counts and is not graph-capturable.
|
||||
declare_resolution(
|
||||
server_args,
|
||||
@@ -542,13 +547,17 @@ def validate_deepep_v2_speculative_draft(server_args: Any) -> None:
|
||||
|
||||
|
||||
def required_deepep_v2_prefill_tokens_per_rank(server_args: Any) -> int:
|
||||
"""Largest prefill dispatch on one rank, after model-specific sharding."""
|
||||
"""Largest prefill dispatch on one rank, after topology and model sharding."""
|
||||
view = resolved_view(server_args)
|
||||
tokens = max_prefill_buffer_tokens(server_args) or (view.max_prefill_tokens or 0)
|
||||
ceiling = max_prefill_buffer_tokens(server_args) or (view.max_prefill_tokens or 0)
|
||||
# A per-DP chunk is scattered across tp_size // attn_dp_size ranks before
|
||||
# dispatch, so that is the per-EP-rank divisor (pure TP scatters across all).
|
||||
scatter_ranks = max(1, view.tp_size // view.attn_dp_size)
|
||||
return model_deepep_v2_prefill_dispatch_tokens(
|
||||
hf_config=model_config_of(server_args).hf_config,
|
||||
cfg=view,
|
||||
default_tokens=tokens,
|
||||
ceiling=ceiling,
|
||||
default_tokens=-(-ceiling // scatter_ranks),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -22,6 +22,8 @@ _DEEPEP_V2_MODELS: dict[str, _DeepEPv2ModelPolicy] = {
|
||||
"DeepseekV4ForCausalLM": _DeepEPv2ModelPolicy(),
|
||||
"Qwen3MoeForCausalLM": _DeepEPv2ModelPolicy(),
|
||||
"Glm5NextForConditionalGeneration": _DeepEPv2ModelPolicy(),
|
||||
"MiMoV2ForCausalLM": _DeepEPv2ModelPolicy(),
|
||||
"MiMoV2FlashForCausalLM": _DeepEPv2ModelPolicy(),
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +38,8 @@ def register_deepep_v2_model(
|
||||
``prefill_dispatch_tokens(cfg, tokens)`` receives a read-only resolved
|
||||
configuration and the prefill-buffer ceiling. It returns the maximum
|
||||
tokens one rank dispatches, without changing scheduler/buffer settings.
|
||||
Omitting it preserves the default, unsharded ceiling.
|
||||
Omitting it keeps the default: the ceiling split across the ranks each
|
||||
per-DP chunk is scattered over before dispatch.
|
||||
"""
|
||||
|
||||
policy = _DeepEPv2ModelPolicy(
|
||||
@@ -69,11 +72,12 @@ def model_requires_fp32_silu_mul(hf_config: Any) -> bool:
|
||||
|
||||
|
||||
def model_deepep_v2_prefill_dispatch_tokens(
|
||||
hf_config: Any, cfg: Any, default_tokens: int
|
||||
hf_config: Any, cfg: Any, *, ceiling: int, default_tokens: int
|
||||
) -> int:
|
||||
"""Apply the primary model's dispatch sizing, or keep the buffer ceiling."""
|
||||
"""Apply the primary model's dispatch sizing to the prefill-buffer ceiling,
|
||||
or return default_tokens when the model registers none."""
|
||||
|
||||
policy = _policy_of(hf_config)
|
||||
if policy is None or policy.prefill_dispatch_tokens is None:
|
||||
return default_tokens
|
||||
return policy.prefill_dispatch_tokens(cfg, default_tokens)
|
||||
return policy.prefill_dispatch_tokens(cfg, ceiling)
|
||||
|
||||
@@ -1275,6 +1275,7 @@ class Envs:
|
||||
SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128)
|
||||
# 0 lets ElasticBuffer select its theoretical communication SM/QP counts.
|
||||
SGLANG_DEEPEP_V2_NUM_SMS = EnvInt(0)
|
||||
SGLANG_DEEPEP_V2_ENABLE_PREFILL_EXPAND = EnvBool(None)
|
||||
SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32)
|
||||
# A5 DSV4 FP4 + DeepEP low-latency dispatch wire format. This is read only
|
||||
# by the model-specific dispatcher configuration; all other paths retain
|
||||
|
||||
@@ -267,7 +267,6 @@ class DeepGemmRunnerInput(RunnerInput):
|
||||
masked_m: Optional[torch.Tensor] = None
|
||||
expected_m: Optional[int] = None
|
||||
m_indices: Optional[torch.Tensor] = None
|
||||
hidden_states_scale_tma_aligned: bool = False
|
||||
# Number of activation elements sharing one scale along K.
|
||||
# Records the actual input quantization group, independently of weight scales.
|
||||
activation_scale_block_size: Optional[int] = None
|
||||
@@ -446,10 +445,7 @@ class DeepGemmRunnerCore(MoeRunnerCore):
|
||||
device=hidden_states_device,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
if (
|
||||
deep_gemm_wrapper.DEEPGEMM_NEED_TMA_ALIGNED_SCALES
|
||||
and not runner_input.hidden_states_scale_tma_aligned
|
||||
):
|
||||
if deep_gemm_wrapper.DEEPGEMM_NEED_TMA_ALIGNED_SCALES:
|
||||
hidden_states_scale = tma_align_input_scale(hidden_states_scale)
|
||||
|
||||
deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_contig(
|
||||
@@ -590,6 +586,22 @@ class DeepGemmRunnerCore(MoeRunnerCore):
|
||||
torch.cuda.synchronize()
|
||||
logger.warning("DeepEP v2 expanded contig activation returned")
|
||||
|
||||
deepep_v2_expanded = running_state.get("deepep_v2_expanded", False)
|
||||
# Folding the row weight into down_input_scale needs a non-power-of-two
|
||||
# scale; ue8m0 is a power of two, so it weights down_output before combine.
|
||||
# no_combine asks for raw rows, so the fold must not pre-apply the weight.
|
||||
fuse_weight_into_scale = (
|
||||
deepep_v2_expanded
|
||||
and not self.config.no_combine
|
||||
and not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0
|
||||
and running_state.get("topk_weights") is not None
|
||||
)
|
||||
if fuse_weight_into_scale:
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import scale_expanded_rows_
|
||||
|
||||
scale_expanded_rows_(down_input_scale, running_state["topk_weights"])
|
||||
running_state["deepep_v2_weight_prefused"] = True
|
||||
|
||||
# Allocate the MoE output in the NCCL symmetric memory pool when symmetric
|
||||
# allocation is required, so the downstream all-reduce takes the low-latency
|
||||
# symmetric path. Only this final output enters the pool; intermediate
|
||||
@@ -1703,8 +1715,8 @@ def pre_permute_deepep_v2_to_deep_gemm(
|
||||
running_state: dict,
|
||||
) -> DeepGemmRunnerInput:
|
||||
from sglang.kernels.ops.moe.ep_moe_kernels import (
|
||||
ep_expand_init_m_indices_from_psum,
|
||||
ep_scatter_from_psum,
|
||||
fill_m_indices_from_psum,
|
||||
)
|
||||
|
||||
hidden_states = dispatch_output.hidden_states
|
||||
@@ -1713,7 +1725,6 @@ def pre_permute_deepep_v2_to_deep_gemm(
|
||||
topk_weights = dispatch_output.topk_weights
|
||||
psum_num_recv_tokens_per_expert = dispatch_output.psum_num_recv_tokens_per_expert
|
||||
is_expanded = dispatch_output.is_expanded
|
||||
hidden_states_scale_tma_aligned = dispatch_output.hidden_states_scale_tma_aligned
|
||||
deepep_v2_use_masked = dispatch_output.use_masked_gemm
|
||||
deepep_v2_expected_m = dispatch_output.expected_m
|
||||
deepep_v2_masked_max_m = dispatch_output.masked_max_m
|
||||
@@ -1771,17 +1782,18 @@ def pre_permute_deepep_v2_to_deep_gemm(
|
||||
),
|
||||
)
|
||||
|
||||
# Mark aligned expert rows and leave the unused receive tail at -1.
|
||||
m_indices = torch.full(
|
||||
(all_tokens,), -1, device=hidden_states.device, dtype=torch.int32
|
||||
num_local_experts = psum_num_recv_tokens_per_expert.shape[0]
|
||||
m_indices = fill_m_indices_from_psum(
|
||||
psum_num_recv_tokens_per_expert,
|
||||
num_local_experts,
|
||||
all_tokens,
|
||||
deepep_v2_expert_alignment,
|
||||
)
|
||||
ep_expand_init_m_indices_from_psum(psum_num_recv_tokens_per_expert, m_indices)
|
||||
return DeepGemmRunnerInput(
|
||||
hidden_states=hidden_states,
|
||||
hidden_states_scale=hidden_states_scale,
|
||||
use_masked_gemm=False,
|
||||
m_indices=m_indices,
|
||||
hidden_states_scale_tma_aligned=hidden_states_scale_tma_aligned,
|
||||
activation_scale_block_size=dispatch_output.activation_scale_block_size,
|
||||
)
|
||||
|
||||
@@ -1888,11 +1900,14 @@ def post_permute_deep_gemm_to_deepep_v2(
|
||||
return DeepEPv2CombineInput(
|
||||
hidden_states, topk_weights, RoutewiseLayout.EXPANDED
|
||||
)
|
||||
if topk_weights is not None:
|
||||
# Expanded combine does not consume top-k weights.
|
||||
hidden_states = hidden_states * topk_weights.to(
|
||||
hidden_states.dtype
|
||||
).unsqueeze(-1)
|
||||
if topk_weights is not None and not running_state.get(
|
||||
"deepep_v2_weight_prefused", False
|
||||
):
|
||||
# Expanded combine does not consume top-k weights;
|
||||
# skip when fold-into-scale already applied them before down_proj.
|
||||
# In-place with fp32 weights rounds once; casting the weights to
|
||||
# bf16 first costs measurable accuracy.
|
||||
hidden_states.mul_(topk_weights.unsqueeze(-1))
|
||||
return DeepEPv2CombineInput(hidden_states, None)
|
||||
|
||||
hidden_states = runner_output.hidden_states
|
||||
|
||||
@@ -61,7 +61,6 @@ class DeepEPv2DispatchOutput(NamedTuple):
|
||||
topk_weights: torch.Tensor
|
||||
psum_num_recv_tokens_per_expert: Optional[torch.Tensor] = None
|
||||
is_expanded: bool = False
|
||||
hidden_states_scale_tma_aligned: bool = False
|
||||
use_masked_gemm: bool = False
|
||||
expected_m: int = 0
|
||||
masked_max_m: int = 0
|
||||
@@ -120,7 +119,8 @@ def _ensure_fp8_quant_available() -> None:
|
||||
|
||||
|
||||
def _get_allow_hybrid_mode() -> bool:
|
||||
|
||||
# Multi-node is rejected with direct in server-args validation, so a hybrid
|
||||
# mode here already covers every multi-node run.
|
||||
return get_exec().moe.deepep_v2_mode == "hybrid"
|
||||
|
||||
|
||||
@@ -246,6 +246,12 @@ class _DeepEPv2Impl:
|
||||
self.use_fp8_dispatch = use_fp8_dispatch
|
||||
self._handle = None
|
||||
self._pad_empty_combine = False
|
||||
prefill_expand = envs.SGLANG_DEEPEP_V2_ENABLE_PREFILL_EXPAND.get()
|
||||
if prefill_expand is None:
|
||||
# Unset: expanded prefill is validated end to end only without
|
||||
# UE8M0 scales (Hopper), so Blackwell keeps the non-expanded path.
|
||||
prefill_expand = not scale_format.ue8m0
|
||||
self._prefill_expand_enabled = prefill_expand
|
||||
|
||||
def _destroy_handle(self) -> None:
|
||||
self._handle = None
|
||||
@@ -259,6 +265,15 @@ class _DeepEPv2Impl:
|
||||
self.use_fp8_dispatch,
|
||||
)
|
||||
|
||||
def prebuild_buffer(self) -> None:
|
||||
"""Build the ElasticBuffer now instead of lazily on the first dispatch.
|
||||
|
||||
Avoids the ~2GB alloc + cross-rank NCCL barrier stalling the first request
|
||||
when decode CUDA-graph capture did not already build it. Needs only
|
||||
host-known config already on this impl; key-cached so dispatch reuses it.
|
||||
"""
|
||||
self._get_buffer()
|
||||
|
||||
def _validate_common(
|
||||
self, hidden_states: torch.Tensor, topk_ids: torch.Tensor
|
||||
) -> None:
|
||||
@@ -297,9 +312,9 @@ class _DeepEPv2Impl:
|
||||
topk_weights = topk_output.topk_weights
|
||||
topk_ids = topk_output.topk_ids.to(torch.int64)
|
||||
self._validate_common(hidden_states, topk_ids)
|
||||
# Decode uses expanded/masked layout; extend uses contiguous in both modes.
|
||||
use_expand_layout = not get_is_extend_in_batch()
|
||||
use_masked = use_expand_layout
|
||||
is_decode = not get_is_extend_in_batch()
|
||||
use_masked = is_decode
|
||||
use_expand_layout = is_decode or self._prefill_expand_enabled
|
||||
|
||||
# CPU-synced dispatch needs a dummy token to notify from an idle rank.
|
||||
self._pad_empty_combine = (not use_masked) and hidden_states.shape[0] == 0
|
||||
@@ -314,7 +329,7 @@ class _DeepEPv2Impl:
|
||||
if not self.use_fp8_dispatch:
|
||||
dispatch_x = hidden_states
|
||||
use_tma_aligned_col_major_sf = False
|
||||
elif use_masked:
|
||||
elif use_expand_layout:
|
||||
_ensure_fp8_quant_available()
|
||||
_ue8m0 = self.scale_format.ue8m0
|
||||
dispatch_x = sglang_per_token_group_quant_fp8(
|
||||
@@ -364,6 +379,11 @@ class _DeepEPv2Impl:
|
||||
if use_expand_layout:
|
||||
# Expanded combine uses handle metadata instead of recv_topk_idx.
|
||||
local_topk_ids = None
|
||||
if recv_topk_weights.shape != (recv_hidden_states.shape[0],):
|
||||
raise ValueError(
|
||||
"DeepEP v2 expanded activations and router weights must "
|
||||
"have the same receive capacity"
|
||||
)
|
||||
else:
|
||||
num_recv_tokens = int(
|
||||
handle.psum_num_recv_tokens_per_scaleup_rank[-1].item()
|
||||
@@ -383,11 +403,6 @@ class _DeepEPv2Impl:
|
||||
total_expanded = 0
|
||||
if use_masked:
|
||||
recv_capacity = recv_hidden_states.shape[0]
|
||||
if recv_topk_weights.shape != (recv_capacity,):
|
||||
raise ValueError(
|
||||
"DeepEP v2 expanded activations and router weights must "
|
||||
"have the same receive capacity"
|
||||
)
|
||||
# expected_m is only a schedule hint; masked_m is the actual bound.
|
||||
ep_group_size = max(1, self.num_experts // self.num_local_experts)
|
||||
expected_m = max(
|
||||
@@ -408,7 +423,6 @@ class _DeepEPv2Impl:
|
||||
recv_topk_weights,
|
||||
handle.psum_num_recv_tokens_per_expert,
|
||||
use_expand_layout,
|
||||
use_tma_aligned_col_major_sf,
|
||||
use_masked,
|
||||
expected_m,
|
||||
masked_max_m,
|
||||
@@ -490,3 +504,7 @@ class DeepEPv2Dispatcher(BaseDispatcher):
|
||||
f"Expected DeepEP v2 combine input, got {combine_input.format}"
|
||||
)
|
||||
return self._impl.combine(combine_input)
|
||||
|
||||
def prebuild(self) -> None:
|
||||
"""Build the ElasticBuffer eagerly at deployment time (no forward needed)."""
|
||||
self._impl.prebuild_buffer()
|
||||
|
||||
@@ -153,6 +153,7 @@ from sglang.srt.model_executor.model_runner_components.load_model_utils import (
|
||||
from sglang.srt.model_executor.model_runner_components.moe_ep_setup import (
|
||||
check_quantized_moe_compatibility,
|
||||
init_lplb_solvers,
|
||||
prebuild_deepep_v2_buffer,
|
||||
prepare_moe_topk,
|
||||
)
|
||||
from sglang.srt.model_executor.model_runner_components.ngram_embedding_manager import (
|
||||
@@ -1146,6 +1147,8 @@ class ModelRunner:
|
||||
target_size = get_parallel().ep_join_rank_offset + get_parallel().tp_size
|
||||
self._finalize_elastic_ep_joiner(target_size)
|
||||
|
||||
prebuild_deepep_v2_buffer(model=self.model)
|
||||
|
||||
def init_routed_experts_capturer(self):
|
||||
if self.is_draft_worker:
|
||||
# Capture is target-only. The draft worker runs in the same process
|
||||
|
||||
@@ -68,6 +68,32 @@ def prepare_moe_topk(*, model, model_config: ModelConfig) -> None:
|
||||
log_info_on_rank0(logger, f"Prepared {num_prepared} Waterfill TopK modules.")
|
||||
|
||||
|
||||
def prebuild_deepep_v2_buffer(*, model) -> None:
|
||||
"""Build the deepep_v2 ElasticBuffer at deployment time.
|
||||
|
||||
No-op unless the a2a backend is deepep_v2. The buffer is process-wide and
|
||||
shared by every MoE layer, so the first deepep_v2 dispatcher builds it. The
|
||||
per-rank cap is validated in validate_deepep_v2_dispatch_token_budget at
|
||||
server-args time.
|
||||
"""
|
||||
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
|
||||
from sglang.srt.layers.moe.token_dispatcher.deepep_v2 import DeepEPv2Dispatcher
|
||||
from sglang.srt.layers.moe.utils import get_moe_a2a_backend
|
||||
|
||||
if not get_moe_a2a_backend().is_deepep_v2():
|
||||
return
|
||||
|
||||
for module in model.modules():
|
||||
if isinstance(module, FusedMoE) and isinstance(
|
||||
module.dispatcher, DeepEPv2Dispatcher
|
||||
):
|
||||
module.dispatcher.prebuild()
|
||||
log_info_on_rank0(
|
||||
logger, "Prebuilt the DeepEP-V2 ElasticBuffer at startup."
|
||||
)
|
||||
return
|
||||
|
||||
|
||||
def init_lplb_solvers(*, model_config: ModelConfig) -> None:
|
||||
"""Initialize per-layer LPLB solvers from current expert location metadata."""
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
@@ -55,6 +55,7 @@ from sglang.srt.layers.moe import (
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.utils import is_deepep_class_backend
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
@@ -443,11 +444,10 @@ class MiMoV2MoE(nn.Module):
|
||||
)
|
||||
|
||||
# todo : implement tbo forward needed
|
||||
if (
|
||||
get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
):
|
||||
self._enable_a2a_moe = (
|
||||
is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
)
|
||||
if self._enable_a2a_moe:
|
||||
# TODO: we will support tp < ep in the future
|
||||
self.ep_size = get_parallel().moe_ep_size
|
||||
self.num_experts = (
|
||||
@@ -462,12 +462,6 @@ class MiMoV2MoE(nn.Module):
|
||||
else None
|
||||
)
|
||||
|
||||
self._enable_a2a_moe = (
|
||||
get_moe_a2a_backend().is_deepep()
|
||||
or get_moe_a2a_backend().is_mooncake()
|
||||
or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
)
|
||||
|
||||
def get_moe_weights(self):
|
||||
return [
|
||||
x.data
|
||||
|
||||
Reference in New Issue
Block a user