diff --git a/docs/src/snippets/configs/Qwen/qwen3.8.jsx b/docs/src/snippets/configs/Qwen/qwen3.8.jsx index 337e890d00..2f0ab447d2 100644 --- a/docs/src/snippets/configs/Qwen/qwen3.8.jsx +++ b/docs/src/snippets/configs/Qwen/qwen3.8.jsx @@ -358,7 +358,6 @@ export const config = { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "balanced", nodes: "multi-4" }, verified: true, env: [ - "SGLANG_DEEPEP_V2_EXPAND_PREFILL=1", "SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK=2048", "SGLANG_DEEPEP_V2_MASKED_NUM_MAX_DISPATCH_TOKENS_PER_RANK=384", "EP_DISABLE_GIN=1", @@ -442,7 +441,6 @@ export const config = { match: { hw: "gb300", variant: "default", quant: "fp8", strategy: "high-throughput", nodes: "multi-4" }, verified: true, env: [ - "SGLANG_DEEPEP_V2_EXPAND_PREFILL=1", "SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK=2048", "SGLANG_DEEPEP_V2_MASKED_NUM_MAX_DISPATCH_TOKENS_PER_RANK=384", "EP_DISABLE_GIN=1", diff --git a/python/sglang/kernels/ops/moe/ep_moe_kernels.py b/python/sglang/kernels/ops/moe/ep_moe_kernels.py index 8f5caca654..ba4d737303 100644 --- a/python/sglang/kernels/ops/moe/ep_moe_kernels.py +++ b/python/sglang/kernels/ops/moe/ep_moe_kernels.py @@ -1337,46 +1337,6 @@ def ep_scatter_from_psum( return -@triton.jit -def _fwd_kernel_ep_expand_m_indices_init( - psum_num_recv_tokens_per_expert, - m_indices, - BLOCK_E: tl.constexpr, -): - cur_expert = tl.program_id(0) - cur_end = tl.load(psum_num_recv_tokens_per_expert + cur_expert) - prev_end = tl.load( - psum_num_recv_tokens_per_expert + cur_expert - 1, - mask=cur_expert > 0, - other=0, - ) - cur_start = ((prev_end + BLOCK_E - 1) // BLOCK_E) * BLOCK_E - aligned_end = ((cur_end + BLOCK_E - 1) // BLOCK_E) * BLOCK_E - - off_expert = tl.arange(0, BLOCK_E) - for start_m in tl.range(0, aligned_end - cur_start, BLOCK_E, num_stages=4): - idx = cur_start + start_m + off_expert - tl.store(m_indices + idx, cur_expert, mask=idx < aligned_end) - - -@torch.no_grad() -def ep_expand_init_m_indices_from_psum( - psum_num_recv_tokens_per_expert: torch.Tensor, - m_indices: torch.Tensor, -): - BLOCK_E = 128 - num_warps = 8 - num_experts = psum_num_recv_tokens_per_expert.shape[0] - assert m_indices.shape[0] % BLOCK_E == 0 - _fwd_kernel_ep_expand_m_indices_init[(num_experts,)]( - psum_num_recv_tokens_per_expert, - m_indices, - num_warps=num_warps, - BLOCK_E=BLOCK_E, - ) - return - - @triton.jit def _fwd_kernel_ep_gather( total_token_num, @@ -2578,6 +2538,95 @@ def masked_slab_to_expand( return output_tensor +@triton.jit +def _fwd_kernel_fill_m_indices_from_psum( + psum_ptr, + m_indices_ptr, + total_rows, + ALIGN: tl.constexpr, + BLOCK_M: tl.constexpr, +): + # psum is DeepEP's inclusive per-expert count: psum[i] = align(psum[i-1]) + + # count_i (only earlier experts aligned), so both start and seg_end round up. + # Each expert's whole aligned segment gets its id, padding rows included -- + # combine ignores them, so the junk the GEMM computes there is discarded. + # Only rows past the last segment keep the wrapper's -1 sentinel; with + # do_cpu_sync=True there are none, so that fill is pure insurance. + e = tl.program_id(0) + prev_end = tl.load(psum_ptr + e - 1, mask=e > 0, other=0) + start = ((prev_end + ALIGN - 1) // ALIGN) * ALIGN + end = tl.load(psum_ptr + e) + seg_end = ((end + ALIGN - 1) // ALIGN) * ALIGN + count = seg_end - start + off = tl.arange(0, BLOCK_M) + for base in tl.range(0, count, BLOCK_M): + idx = start + base + off + tl.store(m_indices_ptr + idx, e, mask=(base + off < count) & (idx < total_rows)) + + +@torch.no_grad() +def fill_m_indices_from_psum( + psum_num_recv_tokens_per_expert: torch.Tensor, + num_local_experts: int, + total_rows: int, + expert_alignment: int, +) -> torch.Tensor: + """Build contiguous-GEMM `m_indices` from the device psum (deepep_v2 + `do_expand=True` prefill). + + The psum from DeepEP is already the alignment-padded per-expert prefix sum, + so this only labels rows. + """ + # do_cpu_sync=True sizes recv_x to align(psum[-1]); the last expert's segment + # therefore ends exactly at total_rows (no capacity tail to skip). + assert total_rows % expert_alignment == 0, ( + f"total_rows {total_rows} not a multiple of expert_alignment {expert_alignment}" + ) + # -1 is DeepGEMM's "empty token" sentinel: uncovered rows are zero-filled and + # skipped instead of indexing the weight tensor out of bounds. + m_indices = torch.full( + (total_rows,), + -1, + device=psum_num_recv_tokens_per_expert.device, + dtype=torch.int32, + ) + _fwd_kernel_fill_m_indices_from_psum[(num_local_experts,)]( + psum_num_recv_tokens_per_expert, + m_indices, + total_rows, + ALIGN=expert_alignment, + BLOCK_M=128, + num_warps=4, + ) + return m_indices + + +@torch.no_grad() +def scale_expanded_rows_( + x: torch.Tensor, + row_weights: torch.Tensor, +) -> torch.Tensor: + """In-place `x[r, :] *= row_weights[r]` for a 2D `x`, any strides. + + deepep_v2 `do_expand=True` prefill folds the router weights into the + down-proj input scale here, because ElasticBuffer.combine ignores + topk_weights in expand mode. `row_weights` must be a 1-D `[rows]` tensor + (what DeepEP hands back). `x` must be fp32: a packed UE8M0 scale cannot + absorb the weight, and a lower-precision `x` would round it. + """ + assert x.dim() == 2, f"expected 2D x, got {tuple(x.shape)}" + assert x.dtype == torch.float32, f"expected fp32 x, got {x.dtype}" + rows, _ = x.shape + assert row_weights.dim() == 1, ( + f"expected 1-D row_weights, got {row_weights.dim()}-D {tuple(row_weights.shape)}" + ) + assert row_weights.numel() == rows, ( + f"row_weights has {row_weights.numel()} entries but x has {rows} rows" + ) + x.mul_(row_weights.unsqueeze(1)) + return x + + def _moe_permute_rows( inputs: torch.Tensor, topk_ids: torch.Tensor, diff --git a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py index 8ab773b07a..da7f66aa8c 100644 --- a/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py +++ b/python/sglang/srt/arg_groups/model_overrides/mimo_v2.py @@ -35,14 +35,13 @@ def _mimo_v2_overrides(server_args: Any, hf_config: Any) -> dict: and cfg.moe_runner_backend == "auto" and get_quantization_config(hf_config) == "fp8" ): - if model_config_of(server_args).is_fp4_experts: - # Let the all-to-all backend choose its compatible runner. - if cfg.moe_a2a_backend == "none": + # Let the all-to-all backend choose its compatible runner. + if cfg.moe_a2a_backend == "none": + if model_config_of(server_args).is_fp4_experts: overrides["moe_runner_backend"] = "flashinfer_mxfp4" - else: - # Avoid the slower Triton default for ordinary FP8 checkpoints. - overrides["moe_runner_backend"] = "flashinfer_trtllm" - if "moe_runner_backend" in overrides: + else: + # Avoid the slower Triton default for ordinary FP8 checkpoints. + overrides["moe_runner_backend"] = "flashinfer_trtllm" logger.info( "MiMoV2 on SM100: moe_runner_backend=%s.", overrides["moe_runner_backend"], diff --git a/python/sglang/srt/arg_groups/moe_hook.py b/python/sglang/srt/arg_groups/moe_hook.py index e34cd3373a..7760431cf9 100644 --- a/python/sglang/srt/arg_groups/moe_hook.py +++ b/python/sglang/srt/arg_groups/moe_hook.py @@ -360,6 +360,11 @@ def handle_a2a_moe(server_args: Any): "Remove --enforce-shared-experts-fusion when using " "--moe-a2a-backend deepep_v2." ) + if cfg.deepep_v2_mode == "direct" and cfg.nnodes > 1: + raise ValueError( + "--deepep-v2-mode direct is NVLink-only and cannot run across " + f"nodes (nnodes={cfg.nnodes}); pass --deepep-v2-mode hybrid." + ) # Prefill reads host counts and is not graph-capturable. declare_resolution( server_args, @@ -542,13 +547,17 @@ def validate_deepep_v2_speculative_draft(server_args: Any) -> None: def required_deepep_v2_prefill_tokens_per_rank(server_args: Any) -> int: - """Largest prefill dispatch on one rank, after model-specific sharding.""" + """Largest prefill dispatch on one rank, after topology and model sharding.""" view = resolved_view(server_args) - tokens = max_prefill_buffer_tokens(server_args) or (view.max_prefill_tokens or 0) + ceiling = max_prefill_buffer_tokens(server_args) or (view.max_prefill_tokens or 0) + # A per-DP chunk is scattered across tp_size // attn_dp_size ranks before + # dispatch, so that is the per-EP-rank divisor (pure TP scatters across all). + scatter_ranks = max(1, view.tp_size // view.attn_dp_size) return model_deepep_v2_prefill_dispatch_tokens( hf_config=model_config_of(server_args).hf_config, cfg=view, - default_tokens=tokens, + ceiling=ceiling, + default_tokens=-(-ceiling // scatter_ranks), ) diff --git a/python/sglang/srt/configs/moe_model_registry.py b/python/sglang/srt/configs/moe_model_registry.py index 2bfb1f7815..2932436bdf 100644 --- a/python/sglang/srt/configs/moe_model_registry.py +++ b/python/sglang/srt/configs/moe_model_registry.py @@ -22,6 +22,8 @@ _DEEPEP_V2_MODELS: dict[str, _DeepEPv2ModelPolicy] = { "DeepseekV4ForCausalLM": _DeepEPv2ModelPolicy(), "Qwen3MoeForCausalLM": _DeepEPv2ModelPolicy(), "Glm5NextForConditionalGeneration": _DeepEPv2ModelPolicy(), + "MiMoV2ForCausalLM": _DeepEPv2ModelPolicy(), + "MiMoV2FlashForCausalLM": _DeepEPv2ModelPolicy(), } @@ -36,7 +38,8 @@ def register_deepep_v2_model( ``prefill_dispatch_tokens(cfg, tokens)`` receives a read-only resolved configuration and the prefill-buffer ceiling. It returns the maximum tokens one rank dispatches, without changing scheduler/buffer settings. - Omitting it preserves the default, unsharded ceiling. + Omitting it keeps the default: the ceiling split across the ranks each + per-DP chunk is scattered over before dispatch. """ policy = _DeepEPv2ModelPolicy( @@ -69,11 +72,12 @@ def model_requires_fp32_silu_mul(hf_config: Any) -> bool: def model_deepep_v2_prefill_dispatch_tokens( - hf_config: Any, cfg: Any, default_tokens: int + hf_config: Any, cfg: Any, *, ceiling: int, default_tokens: int ) -> int: - """Apply the primary model's dispatch sizing, or keep the buffer ceiling.""" + """Apply the primary model's dispatch sizing to the prefill-buffer ceiling, + or return default_tokens when the model registers none.""" policy = _policy_of(hf_config) if policy is None or policy.prefill_dispatch_tokens is None: return default_tokens - return policy.prefill_dispatch_tokens(cfg, default_tokens) + return policy.prefill_dispatch_tokens(cfg, ceiling) diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index a0c27a51f5..83e7f1671c 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1275,6 +1275,7 @@ class Envs: SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK = EnvInt(128) # 0 lets ElasticBuffer select its theoretical communication SM/QP counts. SGLANG_DEEPEP_V2_NUM_SMS = EnvInt(0) + SGLANG_DEEPEP_V2_ENABLE_PREFILL_EXPAND = EnvBool(None) SGLANG_DEEPEP_LL_COMBINE_SEND_NUM_SMS = EnvInt(32) # A5 DSV4 FP4 + DeepEP low-latency dispatch wire format. This is read only # by the model-specific dispatcher configuration; all other paths retain diff --git a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py index bbff20fac6..9656cc1bb4 100644 --- a/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py +++ b/python/sglang/srt/layers/moe/moe_runner/deep_gemm.py @@ -267,7 +267,6 @@ class DeepGemmRunnerInput(RunnerInput): masked_m: Optional[torch.Tensor] = None expected_m: Optional[int] = None m_indices: Optional[torch.Tensor] = None - hidden_states_scale_tma_aligned: bool = False # Number of activation elements sharing one scale along K. # Records the actual input quantization group, independently of weight scales. activation_scale_block_size: Optional[int] = None @@ -446,10 +445,7 @@ class DeepGemmRunnerCore(MoeRunnerCore): device=hidden_states_device, dtype=torch.bfloat16, ) - if ( - deep_gemm_wrapper.DEEPGEMM_NEED_TMA_ALIGNED_SCALES - and not runner_input.hidden_states_scale_tma_aligned - ): + if deep_gemm_wrapper.DEEPGEMM_NEED_TMA_ALIGNED_SCALES: hidden_states_scale = tma_align_input_scale(hidden_states_scale) deep_gemm_wrapper.grouped_gemm_nt_f8f8bf16_contig( @@ -590,6 +586,22 @@ class DeepGemmRunnerCore(MoeRunnerCore): torch.cuda.synchronize() logger.warning("DeepEP v2 expanded contig activation returned") + deepep_v2_expanded = running_state.get("deepep_v2_expanded", False) + # Folding the row weight into down_input_scale needs a non-power-of-two + # scale; ue8m0 is a power of two, so it weights down_output before combine. + # no_combine asks for raw rows, so the fold must not pre-apply the weight. + fuse_weight_into_scale = ( + deepep_v2_expanded + and not self.config.no_combine + and not deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 + and running_state.get("topk_weights") is not None + ) + if fuse_weight_into_scale: + from sglang.kernels.ops.moe.ep_moe_kernels import scale_expanded_rows_ + + scale_expanded_rows_(down_input_scale, running_state["topk_weights"]) + running_state["deepep_v2_weight_prefused"] = True + # Allocate the MoE output in the NCCL symmetric memory pool when symmetric # allocation is required, so the downstream all-reduce takes the low-latency # symmetric path. Only this final output enters the pool; intermediate @@ -1703,8 +1715,8 @@ def pre_permute_deepep_v2_to_deep_gemm( running_state: dict, ) -> DeepGemmRunnerInput: from sglang.kernels.ops.moe.ep_moe_kernels import ( - ep_expand_init_m_indices_from_psum, ep_scatter_from_psum, + fill_m_indices_from_psum, ) hidden_states = dispatch_output.hidden_states @@ -1713,7 +1725,6 @@ def pre_permute_deepep_v2_to_deep_gemm( topk_weights = dispatch_output.topk_weights psum_num_recv_tokens_per_expert = dispatch_output.psum_num_recv_tokens_per_expert is_expanded = dispatch_output.is_expanded - hidden_states_scale_tma_aligned = dispatch_output.hidden_states_scale_tma_aligned deepep_v2_use_masked = dispatch_output.use_masked_gemm deepep_v2_expected_m = dispatch_output.expected_m deepep_v2_masked_max_m = dispatch_output.masked_max_m @@ -1771,17 +1782,18 @@ def pre_permute_deepep_v2_to_deep_gemm( ), ) - # Mark aligned expert rows and leave the unused receive tail at -1. - m_indices = torch.full( - (all_tokens,), -1, device=hidden_states.device, dtype=torch.int32 + num_local_experts = psum_num_recv_tokens_per_expert.shape[0] + m_indices = fill_m_indices_from_psum( + psum_num_recv_tokens_per_expert, + num_local_experts, + all_tokens, + deepep_v2_expert_alignment, ) - ep_expand_init_m_indices_from_psum(psum_num_recv_tokens_per_expert, m_indices) return DeepGemmRunnerInput( hidden_states=hidden_states, hidden_states_scale=hidden_states_scale, use_masked_gemm=False, m_indices=m_indices, - hidden_states_scale_tma_aligned=hidden_states_scale_tma_aligned, activation_scale_block_size=dispatch_output.activation_scale_block_size, ) @@ -1888,11 +1900,14 @@ def post_permute_deep_gemm_to_deepep_v2( return DeepEPv2CombineInput( hidden_states, topk_weights, RoutewiseLayout.EXPANDED ) - if topk_weights is not None: - # Expanded combine does not consume top-k weights. - hidden_states = hidden_states * topk_weights.to( - hidden_states.dtype - ).unsqueeze(-1) + if topk_weights is not None and not running_state.get( + "deepep_v2_weight_prefused", False + ): + # Expanded combine does not consume top-k weights; + # skip when fold-into-scale already applied them before down_proj. + # In-place with fp32 weights rounds once; casting the weights to + # bf16 first costs measurable accuracy. + hidden_states.mul_(topk_weights.unsqueeze(-1)) return DeepEPv2CombineInput(hidden_states, None) hidden_states = runner_output.hidden_states diff --git a/python/sglang/srt/layers/moe/token_dispatcher/deepep_v2.py b/python/sglang/srt/layers/moe/token_dispatcher/deepep_v2.py index 46bc552dc1..b8050c5910 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/deepep_v2.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/deepep_v2.py @@ -61,7 +61,6 @@ class DeepEPv2DispatchOutput(NamedTuple): topk_weights: torch.Tensor psum_num_recv_tokens_per_expert: Optional[torch.Tensor] = None is_expanded: bool = False - hidden_states_scale_tma_aligned: bool = False use_masked_gemm: bool = False expected_m: int = 0 masked_max_m: int = 0 @@ -120,7 +119,8 @@ def _ensure_fp8_quant_available() -> None: def _get_allow_hybrid_mode() -> bool: - + # Multi-node is rejected with direct in server-args validation, so a hybrid + # mode here already covers every multi-node run. return get_exec().moe.deepep_v2_mode == "hybrid" @@ -246,6 +246,12 @@ class _DeepEPv2Impl: self.use_fp8_dispatch = use_fp8_dispatch self._handle = None self._pad_empty_combine = False + prefill_expand = envs.SGLANG_DEEPEP_V2_ENABLE_PREFILL_EXPAND.get() + if prefill_expand is None: + # Unset: expanded prefill is validated end to end only without + # UE8M0 scales (Hopper), so Blackwell keeps the non-expanded path. + prefill_expand = not scale_format.ue8m0 + self._prefill_expand_enabled = prefill_expand def _destroy_handle(self) -> None: self._handle = None @@ -259,6 +265,15 @@ class _DeepEPv2Impl: self.use_fp8_dispatch, ) + def prebuild_buffer(self) -> None: + """Build the ElasticBuffer now instead of lazily on the first dispatch. + + Avoids the ~2GB alloc + cross-rank NCCL barrier stalling the first request + when decode CUDA-graph capture did not already build it. Needs only + host-known config already on this impl; key-cached so dispatch reuses it. + """ + self._get_buffer() + def _validate_common( self, hidden_states: torch.Tensor, topk_ids: torch.Tensor ) -> None: @@ -297,9 +312,9 @@ class _DeepEPv2Impl: topk_weights = topk_output.topk_weights topk_ids = topk_output.topk_ids.to(torch.int64) self._validate_common(hidden_states, topk_ids) - # Decode uses expanded/masked layout; extend uses contiguous in both modes. - use_expand_layout = not get_is_extend_in_batch() - use_masked = use_expand_layout + is_decode = not get_is_extend_in_batch() + use_masked = is_decode + use_expand_layout = is_decode or self._prefill_expand_enabled # CPU-synced dispatch needs a dummy token to notify from an idle rank. self._pad_empty_combine = (not use_masked) and hidden_states.shape[0] == 0 @@ -314,7 +329,7 @@ class _DeepEPv2Impl: if not self.use_fp8_dispatch: dispatch_x = hidden_states use_tma_aligned_col_major_sf = False - elif use_masked: + elif use_expand_layout: _ensure_fp8_quant_available() _ue8m0 = self.scale_format.ue8m0 dispatch_x = sglang_per_token_group_quant_fp8( @@ -364,6 +379,11 @@ class _DeepEPv2Impl: if use_expand_layout: # Expanded combine uses handle metadata instead of recv_topk_idx. local_topk_ids = None + if recv_topk_weights.shape != (recv_hidden_states.shape[0],): + raise ValueError( + "DeepEP v2 expanded activations and router weights must " + "have the same receive capacity" + ) else: num_recv_tokens = int( handle.psum_num_recv_tokens_per_scaleup_rank[-1].item() @@ -383,11 +403,6 @@ class _DeepEPv2Impl: total_expanded = 0 if use_masked: recv_capacity = recv_hidden_states.shape[0] - if recv_topk_weights.shape != (recv_capacity,): - raise ValueError( - "DeepEP v2 expanded activations and router weights must " - "have the same receive capacity" - ) # expected_m is only a schedule hint; masked_m is the actual bound. ep_group_size = max(1, self.num_experts // self.num_local_experts) expected_m = max( @@ -408,7 +423,6 @@ class _DeepEPv2Impl: recv_topk_weights, handle.psum_num_recv_tokens_per_expert, use_expand_layout, - use_tma_aligned_col_major_sf, use_masked, expected_m, masked_max_m, @@ -490,3 +504,7 @@ class DeepEPv2Dispatcher(BaseDispatcher): f"Expected DeepEP v2 combine input, got {combine_input.format}" ) return self._impl.combine(combine_input) + + def prebuild(self) -> None: + """Build the ElasticBuffer eagerly at deployment time (no forward needed).""" + self._impl.prebuild_buffer() diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index d20c68f8e0..1c513641cb 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -153,6 +153,7 @@ from sglang.srt.model_executor.model_runner_components.load_model_utils import ( from sglang.srt.model_executor.model_runner_components.moe_ep_setup import ( check_quantized_moe_compatibility, init_lplb_solvers, + prebuild_deepep_v2_buffer, prepare_moe_topk, ) from sglang.srt.model_executor.model_runner_components.ngram_embedding_manager import ( @@ -1146,6 +1147,8 @@ class ModelRunner: target_size = get_parallel().ep_join_rank_offset + get_parallel().tp_size self._finalize_elastic_ep_joiner(target_size) + prebuild_deepep_v2_buffer(model=self.model) + def init_routed_experts_capturer(self): if self.is_draft_worker: # Capture is target-only. The draft worker runs in the same process diff --git a/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py b/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py index 4f734ac096..9f6a047f46 100644 --- a/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py +++ b/python/sglang/srt/model_executor/model_runner_components/moe_ep_setup.py @@ -68,6 +68,32 @@ def prepare_moe_topk(*, model, model_config: ModelConfig) -> None: log_info_on_rank0(logger, f"Prepared {num_prepared} Waterfill TopK modules.") +def prebuild_deepep_v2_buffer(*, model) -> None: + """Build the deepep_v2 ElasticBuffer at deployment time. + + No-op unless the a2a backend is deepep_v2. The buffer is process-wide and + shared by every MoE layer, so the first deepep_v2 dispatcher builds it. The + per-rank cap is validated in validate_deepep_v2_dispatch_token_budget at + server-args time. + """ + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.moe.token_dispatcher.deepep_v2 import DeepEPv2Dispatcher + from sglang.srt.layers.moe.utils import get_moe_a2a_backend + + if not get_moe_a2a_backend().is_deepep_v2(): + return + + for module in model.modules(): + if isinstance(module, FusedMoE) and isinstance( + module.dispatcher, DeepEPv2Dispatcher + ): + module.dispatcher.prebuild() + log_info_on_rank0( + logger, "Prebuilt the DeepEP-V2 ElasticBuffer at startup." + ) + return + + def init_lplb_solvers(*, model_config: ModelConfig) -> None: """Initialize per-layer LPLB solvers from current expert location metadata.""" from sglang.srt.runtime_context import get_parallel diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 5aa378cace..c79e645483 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -55,6 +55,7 @@ from sglang.srt.layers.moe import ( ) from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat +from sglang.srt.layers.moe.utils import is_deepep_class_backend from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -443,11 +444,10 @@ class MiMoV2MoE(nn.Module): ) # todo : implement tbo forward needed - if ( - get_moe_a2a_backend().is_deepep() - or get_moe_a2a_backend().is_mooncake() - or get_moe_a2a_backend().is_ascend_fuseep() - ): + self._enable_a2a_moe = ( + is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep() + ) + if self._enable_a2a_moe: # TODO: we will support tp < ep in the future self.ep_size = get_parallel().moe_ep_size self.num_experts = ( @@ -462,12 +462,6 @@ class MiMoV2MoE(nn.Module): else None ) - self._enable_a2a_moe = ( - get_moe_a2a_backend().is_deepep() - or get_moe_a2a_backend().is_mooncake() - or get_moe_a2a_backend().is_ascend_fuseep() - ) - def get_moe_weights(self): return [ x.data diff --git a/test/registered/kernels/ops/moe/test_deepep_v2_expand_kernels.py b/test/registered/kernels/ops/moe/test_deepep_v2_expand_kernels.py new file mode 100644 index 0000000000..5192db0ef5 --- /dev/null +++ b/test/registered/kernels/ops/moe/test_deepep_v2_expand_kernels.py @@ -0,0 +1,125 @@ +"""Tests for the DeepEP v2 expand-path helper kernels. + +fill_m_indices_from_psum and scale_expanded_rows_ only run on the do_expand=True +prefill path, which no other CI test exercises. Both are pure and deterministic +with obvious reference implementations, so pin them directly. +""" + +import unittest + +import torch + +from sglang.kernels.ops.moe.ep_moe_kernels import ( + fill_m_indices_from_psum, + scale_expanded_rows_, +) +from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.test_utils import CustomTestCase + +register_cuda_ci(est_time=10, stage="base-b-kernel-unit", runner_config="1-gpu-large") + +DEVICE = "cuda" + + +def _reference_m_indices(counts, align, total_rows): + """Label each expert's whole *aligned* segment with its id, matching the + kernel; only rows past the last segment keep the -1 sentinel.""" + m_indices = [-1] * total_rows + start = 0 + for e, count in enumerate(counts): + # psum is inclusive: seg_end = align(start + count), and the padding + # inside the segment gets the expert id too (combine ignores it). + seg_end = ((start + count + align - 1) // align) * align + for r in range(start, min(seg_end, total_rows)): + m_indices[r] = e + start = seg_end + return m_indices + + +class TestFillMIndicesFromPsum(CustomTestCase): + """m_indices must label exactly the aligned per-expert segments.""" + + def _run(self, counts, align, extra_rows=0): + # psum[i] = align(psum[i-1]) + count_i (DeepEP's inclusive prefix sum). + psum_vals = [] + prev_end = 0 + for count in counts: + start = ((prev_end + align - 1) // align) * align + prev_end = start + count + psum_vals.append(prev_end) + num_local_experts = len(counts) + # do_cpu_sync=True sizes recv_x to align(psum[-1]); extra_rows models a + # capacity-sized buffer where the tail past the segments stays sentinel. + total_rows = ((psum_vals[-1] + align - 1) // align) * align + extra_rows + + psum = torch.tensor(psum_vals, dtype=torch.int32, device=DEVICE) + m_indices = fill_m_indices_from_psum(psum, num_local_experts, total_rows, align) + + expected = _reference_m_indices(counts, align, total_rows) + self.assertEqual(m_indices.tolist(), expected) + return m_indices + + def test_align_128(self): + self._run([200, 50, 128], align=128) + + def test_align_not_128(self): + # The kernel this replaces silently assumed expert_alignment == 128; a + # separate ALIGN parameter is the fix, so cover a non-128 alignment. + self._run([100, 30, 64], align=64) + + def test_empty_expert_segment(self): + # An expert with count == 0 (start == seg_end) must label no rows. + self._run([128, 0, 64], align=128) + + def test_segments_tile_exactly(self): + # do_cpu_sync=True => total_rows == align(psum[-1]), so the aligned + # segments tile the array and no -1 sentinel survives. + m_indices = self._run([200, 50, 128], align=128) + self.assertNotIn(-1, m_indices.tolist()) + + def test_sentinel_survives_capacity_tail(self): + # A capacity-sized buffer leaves rows past the last segment at -1 -- the + # defensive torch.full(-1) fill that guards the out-of-bounds weight read. + m_indices = self._run([200, 50, 128], align=128, extra_rows=128) + self.assertEqual(m_indices[-128:].tolist(), [-1] * 128) + + +class TestScaleExpandedRows(CustomTestCase): + """In-place row scaling must match x * w[:, None] on any strides.""" + + def _check(self, x, weights): + ref = x * weights.unsqueeze(1) + scale_expanded_rows_(x, weights) + torch.testing.assert_close(x, ref) + + def test_contiguous(self): + x = torch.randn(37, 128, device=DEVICE) + weights = torch.rand(37, device=DEVICE) + self._check(x, weights) + + def test_transposed_view(self): + # mul_ must update a non-contiguous (transposed) view in place. + base = torch.randn(16, 40, device=DEVICE) + x = base.t() # [40, 16], stride(1) != 1 + self.assertNotEqual(x.stride(1), 1) + weights = torch.rand(40, device=DEVICE) + self._check(x, weights) + + def test_rows_zero(self): + x = torch.randn(0, 128, device=DEVICE) + weights = torch.empty(0, device=DEVICE) + # Must not launch or raise on an empty dispatch. + scale_expanded_rows_(x, weights) + self.assertEqual(x.shape, (0, 128)) + + def test_rejects_non_fp32(self): + # A bf16 x would round the weights; a packed UE8M0 scale cannot take them. + for dtype in (torch.bfloat16, torch.int32): + with self.subTest(dtype=dtype): + x = torch.ones(4, 8, device=DEVICE, dtype=dtype) + with self.assertRaises(AssertionError): + scale_expanded_rows_(x, torch.rand(4, device=DEVICE)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/moe/test_deepep_v2_masked_slab.py b/test/registered/unit/layers/moe/test_deepep_v2_masked_slab.py index 61ed8ac97a..4e60294ac3 100644 --- a/test/registered/unit/layers/moe/test_deepep_v2_masked_slab.py +++ b/test/registered/unit/layers/moe/test_deepep_v2_masked_slab.py @@ -156,6 +156,78 @@ class TestDeepEPv2MaskedSlab(CustomTestCase): if no_combine: self.assertTrue(torch.equal(output.topk_weights, weights)) + def test_runner_expanded_nonmasked_route_weighting(self): + # Non-masked expanded (do_expand=True prefill) post_permute. Guards the + # no_combine / deepep_v2_weight_prefused interaction: no_combine hands + # back raw rows + EXPANDED weights; a prefused fold must not re-weight. + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.moe_runner.deep_gemm import ( + DeepGemmRunnerOutput, + post_permute_deep_gemm_to_deepep_v2, + ) + from sglang.srt.layers.moe.token_dispatcher.base import RoutewiseLayout + + rows = torch.tensor([[2.0], [4.0], [8.0]], device=DEVICE) + weights = torch.tensor([0.25, 0.5, 0.75], device=DEVICE) + # (no_combine, prefused) -> (expected rows, expected weights, layout) + cases = [ + (True, False, rows, weights, RoutewiseLayout.EXPANDED), + (False, True, rows, None, None), + (False, False, rows * weights.unsqueeze(-1), None, None), + ] + for no_combine, prefused, want_rows, want_w, want_layout in cases: + with self.subTest(no_combine=no_combine, prefused=prefused): + state = { + "deepep_v2_expanded": True, + "topk_weights": weights, + "deepep_v2_weight_prefused": prefused, + } + output = post_permute_deep_gemm_to_deepep_v2( + DeepGemmRunnerOutput(rows.clone()), + None, + MoeRunnerConfig(no_combine=no_combine), + state, + ) + self.assertTrue(torch.equal(output.hidden_states, want_rows)) + self.assertEqual(output.routewise_layout, want_layout) + if want_w is None: + self.assertIsNone(output.topk_weights) + else: + self.assertTrue(torch.equal(output.topk_weights, want_w)) + + def test_runner_expanded_weighting_rounds_bf16_rows_once(self): + # Production rows are bf16 and DeepEP's expanded weights are fp32. The + # weighting must round once; casting the weights to bf16 first rounds + # twice and costs accuracy. + from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig + from sglang.srt.layers.moe.moe_runner.deep_gemm import ( + DeepGemmRunnerOutput, + post_permute_deep_gemm_to_deepep_v2, + ) + + generator = torch.Generator(device=DEVICE).manual_seed(0) + rows = torch.randn( + (64, 128), device=DEVICE, dtype=torch.bfloat16, generator=generator + ) + weights = torch.rand(64, device=DEVICE, generator=generator) + want = (rows.float() * weights.unsqueeze(-1)).to(torch.bfloat16) + weights_cast_first = rows * weights.to(torch.bfloat16).unsqueeze(-1) + # Otherwise the check below could not tell the two roundings apart. + self.assertFalse(torch.equal(want, weights_cast_first)) + + output = post_permute_deep_gemm_to_deepep_v2( + DeepGemmRunnerOutput(rows.clone()), + None, + MoeRunnerConfig(no_combine=False), + { + "deepep_v2_expanded": True, + "topk_weights": weights, + "deepep_v2_weight_prefused": False, + }, + ) + self.assertEqual(output.hidden_states.dtype, torch.bfloat16) + self.assertTrue(torch.equal(output.hidden_states, want)) + def test_runner_restores_token_topk_routes_and_masks_nonlocal_slots(self): from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.deep_gemm import ( diff --git a/test/registered/unit/layers/moe/test_deepep_v2_wire_dtype.py b/test/registered/unit/layers/moe/test_deepep_v2_wire_dtype.py index 47a0636b28..d4437ddd63 100644 --- a/test/registered/unit/layers/moe/test_deepep_v2_wire_dtype.py +++ b/test/registered/unit/layers/moe/test_deepep_v2_wire_dtype.py @@ -6,6 +6,7 @@ from unittest.mock import patch import torch +from sglang.srt.environ import envs from sglang.srt.layers.moe.token_dispatcher import deepep_v2 from sglang.srt.layers.moe.utils import ( DeepEPv2Fp8ScaleFormat, @@ -136,14 +137,36 @@ class TestDeepEPv2WireDtype(_DeepEPv2WireDtypeBase): ) self.assertEqual(out.activation_scale_block_size, 32) self.assertEqual(out.hidden_states_scale.shape[-1], HIDDEN // 32) - self.assertEqual(out.is_expanded, not is_extend) + # Prefill expands but stays non-masked; only decode is masked. + self.assertTrue(out.is_expanded) + self.assertEqual(out.use_masked_gemm, not is_extend) + + def test_prefill_expand_default_follows_scale_format(self): + # Unset, prefill expands only without UE8M0 scales; an explicit value wins. + cases = [ + (False, None, True), + (True, None, False), + (True, True, True), + (False, False, False), + ] + for ue8m0, setting, want_expanded in cases: + scale_format = DeepEPv2Fp8ScaleFormat(tma_aligned=ue8m0, ue8m0=ue8m0) + with ( + self.subTest(ue8m0=ue8m0, setting=setting), + envs.SGLANG_DEEPEP_V2_ENABLE_PREFILL_EXPAND.override(setting), + patch.object( + deepep_v2, "get_deepep_v2_fp8_scale_format", lambda: scale_format + ), + ): + _, out = self._dispatch(use_fp8_dispatch=False) + self.assertEqual(out.is_expanded, want_expanded) + self.assertFalse(out.use_masked_gemm) def test_bf16_dispatch_sends_unquantized_activations(self): hidden_states, out = self._dispatch(use_fp8_dispatch=False) self.assertIs(_FakeBuffer.last.dispatch_x, hidden_states) self.assertIsNone(out.hidden_states_scale) self.assertEqual(out.hidden_states.dtype, torch.bfloat16) - self.assertFalse(out.hidden_states_scale_tma_aligned) def test_fp8_dispatch_still_sends_activations_and_scales(self): _, out = self._dispatch(use_fp8_dispatch=True) diff --git a/test/registered/unit/layers/moe/test_hpc_ops_runner_guard.py b/test/registered/unit/layers/moe/test_hpc_ops_runner_guard.py index dfb0c1d334..21adea49b2 100644 --- a/test/registered/unit/layers/moe/test_hpc_ops_runner_guard.py +++ b/test/registered/unit/layers/moe/test_hpc_ops_runner_guard.py @@ -174,11 +174,13 @@ def test_deepep_v2_registration_uses_primary_architecture_and_rejects_conflicts( cfg = SimpleNamespace(tp_size=8) assert model_supports_deepep_v2(config) assert model_requires_fp32_silu_mul(config) - assert model_deepep_v2_prefill_dispatch_tokens(config, cfg, 32768) == 4096 + # The callback sizes the ceiling; the default only applies without one. + sizing = dict(ceiling=32768, default_tokens=2048) + assert model_deepep_v2_prefill_dispatch_tokens(config, cfg, **sizing) == 4096 config.architectures = ["UnsupportedMoe", "TestRoutewiseMoe"] assert not model_supports_deepep_v2(config) assert not model_requires_fp32_silu_mul(config) - assert model_deepep_v2_prefill_dispatch_tokens(config, cfg, 32768) == 32768 + assert model_deepep_v2_prefill_dispatch_tokens(config, cfg, **sizing) == 2048 with pytest.raises(ValueError, match="Conflicting"): register_deepep_v2_model("TestRoutewiseMoe", silu_mul_keep_fp32=False) with pytest.raises(ValueError, match="Conflicting"): diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py index be58ab769c..0d26063d56 100644 --- a/test/registered/unit/server_args/test_server_args.py +++ b/test/registered/unit/server_args/test_server_args.py @@ -2971,6 +2971,9 @@ class TestDeepEPv2Args(CustomTestCase): "DeepseekV3ForCausalLM", "DeepseekV4ForCausalLM", "Qwen3MoeForCausalLM", + "Glm5NextForConditionalGeneration", + "MiMoV2ForCausalLM", + "MiMoV2FlashForCausalLM", ): args = self._args(moe_runner_backend="deep_gemm") args._model_config.hf_config.architectures = [architecture] @@ -2996,6 +2999,17 @@ class TestDeepEPv2Args(CustomTestCase): with self.assertRaisesRegex(ValueError, "instance connector"): handle_a2a_moe(args) + def test_direct_mode_rejected_across_nodes(self): + # direct is NVLink-only; multi-node must ask for hybrid up front. + args = self._args(moe_runner_backend="deep_gemm", nnodes=2) + with self.assertRaisesRegex(ValueError, "--deepep-v2-mode hybrid"): + handle_a2a_moe(args) + handle_a2a_moe( + self._args( + moe_runner_backend="deep_gemm", nnodes=2, deepep_v2_mode="hybrid" + ) + ) + def test_deterministic_inference_accepted(self): args = self._args( moe_runner_backend="deep_gemm", @@ -3142,6 +3156,65 @@ class TestDeepEPv2Args(CustomTestCase): with self.assertRaisesRegex(ValueError, "required=1280"): validate_deepep_v2_dispatch_token_budget(args) + def test_prefill_budget_divides_by_scatter_ranks(self): + # Budget divides by tp_size // attn_dp_size; dp=2 -> 2048/(16/2)=256. + # Pin the exact required value: a looser cap would pass for a wrong divisor. + args = self._args( + chunked_prefill_size=2048, + tp_size=16, + attn_dp_size=2, + max_running_requests=16, + ) + with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(255): + with self.assertRaisesRegex(ValueError, "required=256"): + validate_deepep_v2_dispatch_token_budget(args) + with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(256): + validate_deepep_v2_dispatch_token_budget(args) + + def test_prefill_budget_divides_under_pure_tp(self): + # Pure TP still scatters the dispatch input across all tp_size ranks, so + # the budget divides by tp_size: 2048/16=128, cap 127 must raise. + args = self._args( + chunked_prefill_size=2048, + tp_size=16, + attn_dp_size=1, + max_running_requests=16, + ) + with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(127): + with self.assertRaisesRegex(ValueError, "required=128"): + validate_deepep_v2_dispatch_token_budget(args) + with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128): + validate_deepep_v2_dispatch_token_budget(args) + + def test_prefill_budget_model_policy_receives_undivided_ceiling(self): + # A registered policy owns the whole per-rank computation, so it gets + # the prefill-buffer ceiling, not the generic scatter quotient. + from sglang.srt.configs import moe_model_registry + + seen = [] + + def prefill_dispatch_tokens(cfg, tokens): + seen.append(tokens) + return -(-tokens // (cfg.tp_size // cfg.attn_dp_size)) + + moe_model_registry.register_deepep_v2_model( + "TestPrefillPolicyMoe", prefill_dispatch_tokens=prefill_dispatch_tokens + ) + self.addCleanup( + moe_model_registry._DEEPEP_V2_MODELS.pop, "TestPrefillPolicyMoe" + ) + args = self._args( + chunked_prefill_size=2048, + tp_size=16, + attn_dp_size=2, + max_running_requests=16, + ) + args._model_config.hf_config.architectures = ["TestPrefillPolicyMoe"] + with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(255): + with self.assertRaisesRegex(ValueError, "required=256"): + validate_deepep_v2_dispatch_token_budget(args) + self.assertEqual(seen, [2048]) + def test_disabled_chunking_uses_max_prefill_tokens(self): for disabled in (None, 0, -1): args = self._args( diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index b7642b81c1..0524cd87de 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1168,6 +1168,7 @@ class TestGoldenModelOverrides(_IsolatedPublish): defaults = dict( speculative_algorithm=None, moe_runner_backend="auto", + moe_a2a_backend="none", attention_backend=None, prefill_attention_backend=None, decode_attention_backend=None, @@ -1181,6 +1182,12 @@ class TestGoldenModelOverrides(_IsolatedPublish): _mimo_v2_overrides(_args(), _hf("fp8")), {"attention_backend": "fa4", "moe_runner_backend": "flashinfer_trtllm"}, ) + # An all-to-all backend chooses its own runner (deepep_v2 accepts + # only deep_gemm), so the FP8 pin must not fire. + self.assertEqual( + _mimo_v2_overrides(_args(moe_a2a_backend="deepep_v2"), _hf("fp8")), + {"attention_backend": "fa4"}, + ) # An explicit user choice is never overwritten. self.assertEqual( _mimo_v2_overrides(_args(moe_runner_backend="triton"), _hf("fp8")),