[Perf] Enable native MXFP4 routing and CuTeDSL fusion for MiMo (#43639)

Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com>
Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
leonardHONG
2026-10-14 15:39:44 +08:00
committed by GitHub
co-authored by lianghougong Xiaoyu Zhang
parent 6daae9c98e
commit c54ba66cd4
8 changed files with 203 additions and 48 deletions
@@ -89,8 +89,8 @@ def qwen_moe_topk_output(block, hidden_states):
from sglang.srt.layers.moe import get_moe_runner_backend
from sglang.srt.layers.moe.topk import (
StandardTopKOutput,
_eplb_remap_enabled,
capture_routed_experts_if_allowed,
is_expert_remapping_enabled,
)
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.context_manager import (
is_in_tc_piecewise_cuda_graph,
@@ -115,7 +115,7 @@ def qwen_moe_topk_output(block, hidden_states):
and sw.is_contiguous()
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
and not _eplb_remap_enabled()
and not is_expert_remapping_enabled()
)
if (
not ok
+2 -1
View File
@@ -711,7 +711,8 @@ class ExecComm(msgspec.Struct):
help=(
"Enable FlashInfer allreduce fusion and choose backend. "
"Requires SM90 or SM10X NVIDIA GPUs. "
"Defaults to auto. "
"When unspecified, supported MiMo configurations use cutedsl; "
"other supported models use auto. "
"'auto': choose mnnvl on Blackwell (SM100/SM103) systems "
"(single- and multi-node) and trtllm on SM90 single-node systems. "
"'trtllm': available on single-node systems only. "
+22 -5
View File
@@ -931,10 +931,26 @@ _FLASHINFER_ALLREDUCE_FUSION_ARCHS = frozenset(
def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
"""Enable supported FlashInfer AllReduce fusion after CP and A2A resolution.
Auto selects MNNVL on Blackwell and TRTLLM on single-node SM90.
MiMo defaults to CuTeDSL on Blackwell; other supported models use auto.
"""
hf_config = model_config_of(view).hf_config
model_config = model_config_of(view)
hf_config = model_config.hf_config
model_arch = hf_config.architectures[0]
is_mimo = model_arch in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM")
if is_mimo:
import torch
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import SUPPORTED_TP_SIZES
if not (
get_platform().is_sm100
and model_config.dtype == torch.bfloat16
and view.tp_size in SUPPORTED_TP_SIZES
and view.attn_cp_size == 1
and not view.encoder_only
and not view.enable_pdmux
):
return {}
# V4.1 TP4 uses the custom push plane for decode and fused MoE finalize.
prefer_custom_dsv41 = (
getattr(hf_config, "model_type", None) == "deepseek_v41"
@@ -946,7 +962,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
)
if (
view.flashinfer_allreduce_fusion_backend is None
and model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS
and (is_mimo or model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS)
and not prefer_custom_dsv41
and (get_platform().is_sm90 or get_platform().is_sm100)
and view.tp_size > 1
@@ -954,10 +970,11 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
and (view.nnodes == 1 or get_platform().is_sm100)
and view.moe_a2a_backend == "none"
):
backend = "cutedsl" if is_mimo else "auto"
logger.info(
f"Auto-enabling FlashInfer AllReduce Fusion on SM90/SM10X for {model_arch}"
"Auto-enabling FlashInfer AllReduce Fusion (%s) for %s", backend, model_arch
)
return {"flashinfer_allreduce_fusion_backend": "auto"}
return {"flashinfer_allreduce_fusion_backend": backend}
return {}
@@ -41,7 +41,7 @@ _CUDA_BLOCK_THREADS = 1024
_HT_NON_REDUCTION_WARPS = 2
_HT_REDUCTION_WARP_CHOICES = (1, 2, 4, 8)
_SUPPORTED_TP_SIZES = (2, 4, 8, 16)
SUPPORTED_TP_SIZES = (2, 4, 8, 16)
def _ht_shard_split(
@@ -320,6 +320,7 @@ class FlashInferMNNVLCuteDSLARFusion:
weight_bias: float,
process_group: ProcessGroup,
device: torch.device,
include_shared_expert: bool = True,
) -> None:
if hidden_size <= 0 or top_k <= 0 or max_m <= 0:
raise ValueError("hidden_size, top_k, and max_m must be positive")
@@ -334,9 +335,9 @@ class FlashInferMNNVLCuteDSLARFusion:
self.process_group = process_group
# Cached: supports() runs per layer and must not re-enter c10d.
self.tp_size = dist.get_world_size(process_group)
if self.tp_size not in _SUPPORTED_TP_SIZES:
if self.tp_size not in SUPPORTED_TP_SIZES:
raise ValueError(
f"MNNVL CuTe DSL fusion supports tp_size in {_SUPPORTED_TP_SIZES}, "
f"MNNVL CuTe DSL fusion supports tp_size in {SUPPORTED_TP_SIZES}, "
f"got {self.tp_size}"
)
self.device = torch.device(device)
@@ -373,9 +374,10 @@ class FlashInferMNNVLCuteDSLARFusion:
hidden_size=self.hidden_size,
top_k=self.top_k,
)
if get_spec().speculative_algorithm is None:
self.workspace_config = shaped_config
if include_shared_expert and get_spec().speculative_algorithm is None:
self.workspace_config = _with_early_finalize_shared_load(shaped_config)
else:
elif include_shared_expert:
# Early shared load is safe only for a single looping decode graph;
# alternating draft/verify replays can read an unfinished buffer.
logger.info(
@@ -383,7 +385,6 @@ class FlashInferMNNVLCuteDSLARFusion:
"CuTe DSL finalize presets on the safe (non-early-load) "
"ordering."
)
self.workspace_config = shaped_config
self.workspace = workspace_type(
tp_size=self.tp_size,
tp_rank=dist.get_rank(process_group),
@@ -395,7 +396,7 @@ class FlashInferMNNVLCuteDSLARFusion:
rms_eps=self.rms_epsilon,
routed_scaling_factor=1.0,
weight_bias=self.weight_bias,
include_shared_expert=True,
include_shared_expert=include_shared_expert,
add_residual=True,
write_residual_output=True,
config=self.workspace_config,
@@ -422,7 +423,7 @@ class FlashInferMNNVLCuteDSLARFusion:
routed_output: torch.Tensor,
expert_weights: torch.Tensor,
permuted_indices: torch.Tensor,
gated_shared_output: torch.Tensor,
gated_shared_output: torch.Tensor | None,
residual: torch.Tensor,
gamma: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
@@ -491,6 +492,7 @@ def get_flashinfer_mnnvl_cutedsl_ar_fusion(
max_m: int,
rms_epsilon: float,
weight_bias: float,
include_shared_expert: bool = True,
) -> FlashInferMNNVLCuteDSLARFusion:
"""Build the process-local workspace. Must run before graph capture."""
global _WORKSPACE
@@ -518,5 +520,6 @@ def get_flashinfer_mnnvl_cutedsl_ar_fusion(
weight_bias=weight_bias,
process_group=get_parallel().tp_group.device_group,
device=torch.device("cuda", torch.cuda.current_device()),
include_shared_expert=include_shared_expert,
)
return _WORKSPACE
@@ -83,7 +83,7 @@ class MoeDeferredFinalize(DeferredFinalize, frozen=True):
routed_output: torch.Tensor
expert_weights: torch.Tensor
permuted_indices: torch.Tensor
gated_shared_output: torch.Tensor
gated_shared_output: torch.Tensor | None
m: int
finish: Callable[[], torch.Tensor]
@@ -95,7 +95,7 @@ class MoeDeferredFinalize(DeferredFinalize, frozen=True):
cls,
deferred_output,
*,
gated_shared_output: torch.Tensor,
gated_shared_output: torch.Tensor | None,
m: int,
reduce: Callable[[torch.Tensor], torch.Tensor],
) -> MoeDeferredFinalize:
@@ -130,7 +130,9 @@ class CuteDSLWorkspace:
hidden_size: int,
top_k: int,
rms_epsilon: float,
include_shared_expert: bool = True,
) -> None:
self.include_shared_expert = include_shared_expert
self.hidden_size = int(hidden_size)
self.top_k = int(top_k)
self.rms_epsilon = float(rms_epsilon)
@@ -157,6 +159,7 @@ class CuteDSLWorkspace:
rms_epsilon=self.rms_epsilon,
# _fused_norm_gamma() already returns the multiplier as applied.
weight_bias=0.0,
include_shared_expert=self.include_shared_expert,
)
self._workspace = workspace
self.max_m = workspace.max_m
@@ -368,6 +371,7 @@ def install_cutedsl_fusion(
can_defer_finalize: _LayerPredicate,
label: str,
terminal_finalize: bool = False,
include_shared_expert: bool = True,
) -> CuteDSLWorkspace | None:
"""One shared workspace handle per fusion-enabled layer, or None.
@@ -403,6 +407,7 @@ def install_cutedsl_fusion(
hidden_size=hidden_size,
top_k=top_k,
rms_epsilon=rms_epsilon,
include_shared_expert=include_shared_expert,
)
hands_off = 0
for layer in layers:
+3 -3
View File
@@ -1688,7 +1688,7 @@ def is_power_of_two(n):
return n > 0 and math.log2(n).is_integer()
def _eplb_remap_enabled() -> bool:
def is_expert_remapping_enabled() -> bool:
# A real logical->physical mapping only exists when EPLB is enabled, the
# initial expert placement is non-trivial, or there are redundant physical
# experts. Otherwise the map is identity and the remap must be skipped (it is
@@ -2393,7 +2393,7 @@ def _post_process_topk_ids(
# MoE routing for models like DeepSeek-R1-MXFP4 (accuracy ~0.09 vs 0.94+).
#
# Let append kernels materialize padded ids when remapping is disabled.
eplb_remap_enabled = _eplb_remap_enabled()
eplb_remap_enabled = is_expert_remapping_enabled()
_fold_pad_into_append = (
_aiter_append
and not eplb_remap_enabled
@@ -2597,7 +2597,7 @@ def select_experts(
and (
num_fused_shared_experts == 0
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
or not _eplb_remap_enabled()
or not is_expert_remapping_enabled()
)
):
# only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits
@@ -29,7 +29,10 @@ _MXFP8_QUANTIZE_BACKEND = "cute-dsl" if get_platform().is_sm100 else "cuda"
if is_flashinfer_available():
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
from flashinfer.fp4_quantization import block_scale_interleave
from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe
from flashinfer.fused_moe import (
trtllm_fp4_block_scale_moe,
trtllm_fp4_block_scale_routed_moe,
)
from flashinfer.fused_moe.core import (
_maybe_get_cached_w3_w1_permute_indices,
get_w2_permute_indices_with_cache,
@@ -39,6 +42,7 @@ logger = logging.getLogger(__name__)
if TYPE_CHECKING:
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
from sglang.srt.layers.moe.topk import TopKConfig
from sglang.srt.utils.common import get_bool_env_var
@@ -115,9 +119,25 @@ class Mxfp8RoutedInputPreQuant(NamedTuple):
class Mxfp4FlashinferTrtllmMoEMethod:
fuse_routed_scaling_factor_in_topk = True
def configure_bypassed_topk(self, config: TopKConfig) -> bool:
"""Select the FlashInfer routing contract before Top-K can be bypassed."""
supported = (
config.use_grouped_topk
and config.scoring_func == "sigmoid"
and config.correction_bias is not None
and config.custom_routing_function is None
and config.num_fused_shared_experts == 0
and not config.torch_native
)
self._bypassed_routing_method = (
RoutingMethodType.DeepSeekV3 if supported else None
)
return supported
def __init__(self, fp8_method, prefix: str):
self._fp8 = fp8_method
self.prefix = prefix
self._bypassed_routing_method = None
# precision=fp8 is an SM90 knob (Humming W4A8); this SM100 trtllm path
# already runs MXFP8 activations, so the flag is inert here rather than
# an error -- one config can move across hardware.
@@ -370,16 +390,41 @@ class Mxfp4FlashinferTrtllmMoEMethod:
if w2_scale.dim() == 2:
w2_scale = w2_scale.reshape(num_local_experts, hidden_size, -1)
if TopKOutputChecker.format_is_bypassed(topk_output):
raise NotImplementedError(
"the old code in this branch is WRONG. e.g. it does not consider HashTopK, and may miss args"
bypassed = TopKOutputChecker.format_is_bypassed(topk_output)
if TopKOutputChecker.format_is_standard(topk_output):
top_k = topk_output.topk_ids.shape[1]
moe_call = trtllm_fp4_block_scale_routed_moe
routing = dict(
topk_ids=(topk_output.topk_ids, topk_output.topk_weights),
routing_bias=None,
routing_method_type=int(RoutingMethodType.TopK),
n_group=1,
topk_group=1,
routed_scaling_factor=1.0,
)
if not TopKOutputChecker.format_is_standard(topk_output):
elif bypassed:
if self._bypassed_routing_method is None:
raise NotImplementedError("MxFP4 bypassed routing is not configured")
config = topk_output.topk_config
top_k = config.top_k
moe_call = trtllm_fp4_block_scale_moe
routing = dict(
routing_logits=topk_output.router_logits,
routing_bias=config.correction_bias,
routing_method_type=int(self._bypassed_routing_method),
norm_topk_prob=config.renormalize,
n_group=config.num_expert_group,
topk_group=config.topk_group,
routed_scaling_factor=(
config.routed_scaling_factor
if config.apply_routed_scaling_factor_on_output
and config.routed_scaling_factor is not None
else 1.0
),
)
else:
raise ValueError(f"Unsupported topk output format: {topk_output.format}")
topk_ids = topk_output.topk_ids
topk_weights = topk_output.topk_weights
precision = self.flashinfer_mxfp4_moe_precision
input_ready: Optional[torch.cuda.Event] = None
if precision == "bf16":
@@ -439,9 +484,8 @@ class Mxfp4FlashinferTrtllmMoEMethod:
# The op launches the routing kernel, so the join must precede it.
torch.cuda.current_stream().wait_event(input_ready)
result = trtllm_fp4_block_scale_routed_moe(
topk_ids=(topk_ids, topk_weights),
routing_bias=None,
result = moe_call(
**routing,
hidden_states=x_quant,
hidden_states_scale=x_scale,
gemm1_weights=w13,
@@ -457,21 +501,17 @@ class Mxfp4FlashinferTrtllmMoEMethod:
output1_scale_gate_scalar=layer.output1_scale_gate_scalar,
output2_scale_scalar=layer.output2_scale_scalar,
num_experts=layer.num_experts,
top_k=topk_ids.shape[1],
n_group=1,
topk_group=1,
top_k=top_k,
intermediate_size=intermediate_size,
local_expert_offset=layer.moe_ep_rank * layer.num_local_experts,
local_num_experts=num_local_experts,
routed_scaling_factor=1.0,
routing_method_type=int(RoutingMethodType.TopK),
do_finalize=not defer_finalize,
tune_max_num_tokens=next_power_of_2(num_tokens),
output=symm_output,
enable_pdl=trtllm_moe_enable_pdl(num_tokens),
)
if defer_finalize:
output = _make_deferred_finalize_output(result, top_k=topk_ids.shape[1])
output = _make_deferred_finalize_output(result, top_k=top_k)
else:
output = result[0]
+99 -10
View File
@@ -26,6 +26,7 @@ from torch import nn
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
from sglang.srt.distributed.utils import get_group_rank_size
from sglang.srt.environ import envs
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
@@ -37,12 +38,18 @@ from sglang.srt.layers.aux_hidden_states import (
from sglang.srt.layers.dp_attention import (
is_dp_attention_enabled,
)
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
from sglang.srt.layers.layer_boundary import (
append_stages,
declare_attn,
declare_ffn,
is_dense_ffn_fully_dp,
)
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
CuteDSLFusion,
MoeDeferredFinalize,
install_cutedsl_fusion,
)
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
from sglang.srt.layers.layernorm import RMSNorm
from sglang.srt.layers.linear import (
@@ -58,9 +65,17 @@ from sglang.srt.layers.moe import (
get_moe_runner_backend,
)
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.moe.utils import is_deepep_class_backend
from sglang.srt.layers.moe.topk import (
TopK,
TopKOutputChecker,
TopKOutputFormat,
is_expert_remapping_enabled,
)
from sglang.srt.layers.moe.utils import is_deepep_class_backend, sum_post_experts_output
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
Mxfp4FlashinferTrtllmMoEMethod,
)
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
@@ -85,7 +100,12 @@ from sglang.srt.model_loader.weight_utils import (
)
from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig
from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig
from sglang.srt.runtime_context import get_exec, get_parallel
from sglang.srt.runtime_context import (
get_exec,
get_flags,
get_forward,
get_parallel,
)
from sglang.srt.utils import (
LazyValue,
add_prefix,
@@ -450,6 +470,34 @@ class MiMoV2MoE(nn.Module):
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
)
uses_mxfp4_runner = isinstance(
self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod
)
self._uses_expert_remap = (
not get_flags().moe.in_speculative_scope and is_expert_remapping_enabled()
)
use_bypassed_topk = (
uses_cutedsl_ar_fusion()
and not get_flags().moe.in_speculative_scope
and not self._uses_expert_remap
and not get_exec().features.enable_return_routed_experts
and get_exec().moe.expert_distribution_recorder_mode is None
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
and uses_mxfp4_runner
and get_moe_a2a_backend().is_none()
and self.experts.quant_method.configure_bypassed_topk(self.topk.topk_config)
)
if uses_mxfp4_runner:
self.topk.topk_config.output_format = (
TopKOutputFormat.BYPASSED
if use_bypassed_topk
else TopKOutputFormat.STANDARD
)
self.supports_deferred_finalize = (
use_bypassed_topk and envs.SGLANG_ENABLE_MOE_DEFERRED_FINALIZE.get()
)
# todo : implement tbo forward needed
self._enable_a2a_moe = (
is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep()
@@ -480,7 +528,7 @@ class MiMoV2MoE(nn.Module):
self,
hidden_states: torch.Tensor,
forward_batch: Optional[ForwardBatch] = None,
) -> torch.Tensor:
) -> torch.Tensor | MoeDeferredFinalize:
if not self._enable_a2a_moe:
return self.forward_normal(hidden_states)
else:
@@ -489,14 +537,36 @@ class MiMoV2MoE(nn.Module):
def forward_normal(
self,
hidden_states: torch.Tensor,
) -> torch.Tensor:
) -> torch.Tensor | MoeDeferredFinalize:
if hidden_states.shape[0] > 0:
router_logits = self.gate(hidden_states)
topk_output = self.topk(hidden_states, router_logits)
topk_output = self.topk(
hidden_states,
router_logits,
expert_location_dispatch_info=(
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
if self._uses_expert_remap
else None
),
)
else:
topk_output = self.topk.empty_topk_output(hidden_states.device)
# Precomputed routing keeps FP32 weights and finalizes before the reduction.
if get_forward().defer_moe_finalize and TopKOutputChecker.format_is_bypassed(
topk_output
):
deferred = self.experts.forward_deferred_finalize(
hidden_states, topk_output
)
return MoeDeferredFinalize.from_flashinfer(
deferred,
gated_shared_output=None,
m=hidden_states.shape[0],
reduce=sum_post_experts_output,
)
final_hidden_states = self.experts(hidden_states, topk_output)
return final_hidden_states
@@ -850,9 +920,14 @@ class MiMoV2DecoderLayer(nn.Module):
)
attn, ffn = self.stage_facts(config, layer_id)
fusions = (
CuteDSLFusion()
if uses_cutedsl_ar_fusion() and not get_flags().moe.in_speculative_scope
else None
)
self.attn_boundary, self.ffn_boundary = append_stages(
(attn, self.input_layernorm),
(ffn, self.post_attention_layernorm),
(attn, self.input_layernorm, {"fusions": fusions}),
(ffn, self.post_attention_layernorm, {"fusions": fusions}),
)
@classmethod
@@ -892,8 +967,9 @@ class MiMoV2DecoderLayer(nn.Module):
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
hidden_states = self.mlp(hidden_states, forward_batch)
hidden_states = ffn_exit.finish(hidden_states)
return hidden_states
@@ -1002,6 +1078,19 @@ class MiMoV2Model(nn.Module):
self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
else:
self.norm = PPMissingLayer(return_tuple=True)
if uses_cutedsl_ar_fusion():
install_cutedsl_fusion(
self.layers[self.start_layer : self.end_layer],
hidden_size=config.hidden_size,
top_k=config.num_experts_per_tok,
rms_epsilon=config.layernorm_epsilon,
can_defer_finalize=lambda layer: (
isinstance(layer.mlp, MiMoV2MoE)
and layer.mlp.supports_deferred_finalize
),
label="MiMo",
include_shared_expert=False,
)
def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor:
if hasattr(self.config, "scale_emb"):