[Perf] Enable native MXFP4 routing and CuTeDSL fusion for MiMo (#43639)
Co-authored-by: lianghougong <lianghouhong@xiaohongshu.com> Co-authored-by: Xiaoyu Zhang <1182563586@qq.com>
This commit is contained in:
co-authored by
lianghougong
Xiaoyu Zhang
parent
6daae9c98e
commit
c54ba66cd4
@@ -89,8 +89,8 @@ def qwen_moe_topk_output(block, hidden_states):
|
||||
from sglang.srt.layers.moe import get_moe_runner_backend
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
StandardTopKOutput,
|
||||
_eplb_remap_enabled,
|
||||
capture_routed_experts_if_allowed,
|
||||
is_expert_remapping_enabled,
|
||||
)
|
||||
from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.context_manager import (
|
||||
is_in_tc_piecewise_cuda_graph,
|
||||
@@ -115,7 +115,7 @@ def qwen_moe_topk_output(block, hidden_states):
|
||||
and sw.is_contiguous()
|
||||
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
|
||||
and not _eplb_remap_enabled()
|
||||
and not is_expert_remapping_enabled()
|
||||
)
|
||||
if (
|
||||
not ok
|
||||
|
||||
@@ -711,7 +711,8 @@ class ExecComm(msgspec.Struct):
|
||||
help=(
|
||||
"Enable FlashInfer allreduce fusion and choose backend. "
|
||||
"Requires SM90 or SM10X NVIDIA GPUs. "
|
||||
"Defaults to auto. "
|
||||
"When unspecified, supported MiMo configurations use cutedsl; "
|
||||
"other supported models use auto. "
|
||||
"'auto': choose mnnvl on Blackwell (SM100/SM103) systems "
|
||||
"(single- and multi-node) and trtllm on SM90 single-node systems. "
|
||||
"'trtllm': available on single-node systems only. "
|
||||
|
||||
@@ -931,10 +931,26 @@ _FLASHINFER_ALLREDUCE_FUSION_ARCHS = frozenset(
|
||||
def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
|
||||
"""Enable supported FlashInfer AllReduce fusion after CP and A2A resolution.
|
||||
|
||||
Auto selects MNNVL on Blackwell and TRTLLM on single-node SM90.
|
||||
MiMo defaults to CuTeDSL on Blackwell; other supported models use auto.
|
||||
"""
|
||||
hf_config = model_config_of(view).hf_config
|
||||
model_config = model_config_of(view)
|
||||
hf_config = model_config.hf_config
|
||||
model_arch = hf_config.architectures[0]
|
||||
is_mimo = model_arch in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM")
|
||||
if is_mimo:
|
||||
import torch
|
||||
|
||||
from sglang.srt.layers.flashinfer_mnnvl_cutedsl import SUPPORTED_TP_SIZES
|
||||
|
||||
if not (
|
||||
get_platform().is_sm100
|
||||
and model_config.dtype == torch.bfloat16
|
||||
and view.tp_size in SUPPORTED_TP_SIZES
|
||||
and view.attn_cp_size == 1
|
||||
and not view.encoder_only
|
||||
and not view.enable_pdmux
|
||||
):
|
||||
return {}
|
||||
# V4.1 TP4 uses the custom push plane for decode and fused MoE finalize.
|
||||
prefer_custom_dsv41 = (
|
||||
getattr(hf_config, "model_type", None) == "deepseek_v41"
|
||||
@@ -946,7 +962,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
|
||||
)
|
||||
if (
|
||||
view.flashinfer_allreduce_fusion_backend is None
|
||||
and model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS
|
||||
and (is_mimo or model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS)
|
||||
and not prefer_custom_dsv41
|
||||
and (get_platform().is_sm90 or get_platform().is_sm100)
|
||||
and view.tp_size > 1
|
||||
@@ -954,10 +970,11 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict:
|
||||
and (view.nnodes == 1 or get_platform().is_sm100)
|
||||
and view.moe_a2a_backend == "none"
|
||||
):
|
||||
backend = "cutedsl" if is_mimo else "auto"
|
||||
logger.info(
|
||||
f"Auto-enabling FlashInfer AllReduce Fusion on SM90/SM10X for {model_arch}"
|
||||
"Auto-enabling FlashInfer AllReduce Fusion (%s) for %s", backend, model_arch
|
||||
)
|
||||
return {"flashinfer_allreduce_fusion_backend": "auto"}
|
||||
return {"flashinfer_allreduce_fusion_backend": backend}
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
@@ -41,7 +41,7 @@ _CUDA_BLOCK_THREADS = 1024
|
||||
_HT_NON_REDUCTION_WARPS = 2
|
||||
_HT_REDUCTION_WARP_CHOICES = (1, 2, 4, 8)
|
||||
|
||||
_SUPPORTED_TP_SIZES = (2, 4, 8, 16)
|
||||
SUPPORTED_TP_SIZES = (2, 4, 8, 16)
|
||||
|
||||
|
||||
def _ht_shard_split(
|
||||
@@ -320,6 +320,7 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
weight_bias: float,
|
||||
process_group: ProcessGroup,
|
||||
device: torch.device,
|
||||
include_shared_expert: bool = True,
|
||||
) -> None:
|
||||
if hidden_size <= 0 or top_k <= 0 or max_m <= 0:
|
||||
raise ValueError("hidden_size, top_k, and max_m must be positive")
|
||||
@@ -334,9 +335,9 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
self.process_group = process_group
|
||||
# Cached: supports() runs per layer and must not re-enter c10d.
|
||||
self.tp_size = dist.get_world_size(process_group)
|
||||
if self.tp_size not in _SUPPORTED_TP_SIZES:
|
||||
if self.tp_size not in SUPPORTED_TP_SIZES:
|
||||
raise ValueError(
|
||||
f"MNNVL CuTe DSL fusion supports tp_size in {_SUPPORTED_TP_SIZES}, "
|
||||
f"MNNVL CuTe DSL fusion supports tp_size in {SUPPORTED_TP_SIZES}, "
|
||||
f"got {self.tp_size}"
|
||||
)
|
||||
self.device = torch.device(device)
|
||||
@@ -373,9 +374,10 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
hidden_size=self.hidden_size,
|
||||
top_k=self.top_k,
|
||||
)
|
||||
if get_spec().speculative_algorithm is None:
|
||||
self.workspace_config = shaped_config
|
||||
if include_shared_expert and get_spec().speculative_algorithm is None:
|
||||
self.workspace_config = _with_early_finalize_shared_load(shaped_config)
|
||||
else:
|
||||
elif include_shared_expert:
|
||||
# Early shared load is safe only for a single looping decode graph;
|
||||
# alternating draft/verify replays can read an unfinished buffer.
|
||||
logger.info(
|
||||
@@ -383,7 +385,6 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
"CuTe DSL finalize presets on the safe (non-early-load) "
|
||||
"ordering."
|
||||
)
|
||||
self.workspace_config = shaped_config
|
||||
self.workspace = workspace_type(
|
||||
tp_size=self.tp_size,
|
||||
tp_rank=dist.get_rank(process_group),
|
||||
@@ -395,7 +396,7 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
rms_eps=self.rms_epsilon,
|
||||
routed_scaling_factor=1.0,
|
||||
weight_bias=self.weight_bias,
|
||||
include_shared_expert=True,
|
||||
include_shared_expert=include_shared_expert,
|
||||
add_residual=True,
|
||||
write_residual_output=True,
|
||||
config=self.workspace_config,
|
||||
@@ -422,7 +423,7 @@ class FlashInferMNNVLCuteDSLARFusion:
|
||||
routed_output: torch.Tensor,
|
||||
expert_weights: torch.Tensor,
|
||||
permuted_indices: torch.Tensor,
|
||||
gated_shared_output: torch.Tensor,
|
||||
gated_shared_output: torch.Tensor | None,
|
||||
residual: torch.Tensor,
|
||||
gamma: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
@@ -491,6 +492,7 @@ def get_flashinfer_mnnvl_cutedsl_ar_fusion(
|
||||
max_m: int,
|
||||
rms_epsilon: float,
|
||||
weight_bias: float,
|
||||
include_shared_expert: bool = True,
|
||||
) -> FlashInferMNNVLCuteDSLARFusion:
|
||||
"""Build the process-local workspace. Must run before graph capture."""
|
||||
global _WORKSPACE
|
||||
@@ -518,5 +520,6 @@ def get_flashinfer_mnnvl_cutedsl_ar_fusion(
|
||||
weight_bias=weight_bias,
|
||||
process_group=get_parallel().tp_group.device_group,
|
||||
device=torch.device("cuda", torch.cuda.current_device()),
|
||||
include_shared_expert=include_shared_expert,
|
||||
)
|
||||
return _WORKSPACE
|
||||
|
||||
@@ -83,7 +83,7 @@ class MoeDeferredFinalize(DeferredFinalize, frozen=True):
|
||||
routed_output: torch.Tensor
|
||||
expert_weights: torch.Tensor
|
||||
permuted_indices: torch.Tensor
|
||||
gated_shared_output: torch.Tensor
|
||||
gated_shared_output: torch.Tensor | None
|
||||
m: int
|
||||
finish: Callable[[], torch.Tensor]
|
||||
|
||||
@@ -95,7 +95,7 @@ class MoeDeferredFinalize(DeferredFinalize, frozen=True):
|
||||
cls,
|
||||
deferred_output,
|
||||
*,
|
||||
gated_shared_output: torch.Tensor,
|
||||
gated_shared_output: torch.Tensor | None,
|
||||
m: int,
|
||||
reduce: Callable[[torch.Tensor], torch.Tensor],
|
||||
) -> MoeDeferredFinalize:
|
||||
@@ -130,7 +130,9 @@ class CuteDSLWorkspace:
|
||||
hidden_size: int,
|
||||
top_k: int,
|
||||
rms_epsilon: float,
|
||||
include_shared_expert: bool = True,
|
||||
) -> None:
|
||||
self.include_shared_expert = include_shared_expert
|
||||
self.hidden_size = int(hidden_size)
|
||||
self.top_k = int(top_k)
|
||||
self.rms_epsilon = float(rms_epsilon)
|
||||
@@ -157,6 +159,7 @@ class CuteDSLWorkspace:
|
||||
rms_epsilon=self.rms_epsilon,
|
||||
# _fused_norm_gamma() already returns the multiplier as applied.
|
||||
weight_bias=0.0,
|
||||
include_shared_expert=self.include_shared_expert,
|
||||
)
|
||||
self._workspace = workspace
|
||||
self.max_m = workspace.max_m
|
||||
@@ -368,6 +371,7 @@ def install_cutedsl_fusion(
|
||||
can_defer_finalize: _LayerPredicate,
|
||||
label: str,
|
||||
terminal_finalize: bool = False,
|
||||
include_shared_expert: bool = True,
|
||||
) -> CuteDSLWorkspace | None:
|
||||
"""One shared workspace handle per fusion-enabled layer, or None.
|
||||
|
||||
@@ -403,6 +407,7 @@ def install_cutedsl_fusion(
|
||||
hidden_size=hidden_size,
|
||||
top_k=top_k,
|
||||
rms_epsilon=rms_epsilon,
|
||||
include_shared_expert=include_shared_expert,
|
||||
)
|
||||
hands_off = 0
|
||||
for layer in layers:
|
||||
|
||||
@@ -1688,7 +1688,7 @@ def is_power_of_two(n):
|
||||
return n > 0 and math.log2(n).is_integer()
|
||||
|
||||
|
||||
def _eplb_remap_enabled() -> bool:
|
||||
def is_expert_remapping_enabled() -> bool:
|
||||
# A real logical->physical mapping only exists when EPLB is enabled, the
|
||||
# initial expert placement is non-trivial, or there are redundant physical
|
||||
# experts. Otherwise the map is identity and the remap must be skipped (it is
|
||||
@@ -2393,7 +2393,7 @@ def _post_process_topk_ids(
|
||||
# MoE routing for models like DeepSeek-R1-MXFP4 (accuracy ~0.09 vs 0.94+).
|
||||
#
|
||||
# Let append kernels materialize padded ids when remapping is disabled.
|
||||
eplb_remap_enabled = _eplb_remap_enabled()
|
||||
eplb_remap_enabled = is_expert_remapping_enabled()
|
||||
_fold_pad_into_append = (
|
||||
_aiter_append
|
||||
and not eplb_remap_enabled
|
||||
@@ -2597,7 +2597,7 @@ def select_experts(
|
||||
and (
|
||||
num_fused_shared_experts == 0
|
||||
or has_per_rank_fused_shared_slots(num_fused_shared_experts)
|
||||
or not _eplb_remap_enabled()
|
||||
or not is_expert_remapping_enabled()
|
||||
)
|
||||
):
|
||||
# only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits
|
||||
|
||||
@@ -29,7 +29,10 @@ _MXFP8_QUANTIZE_BACKEND = "cute-dsl" if get_platform().is_sm100 else "cuda"
|
||||
if is_flashinfer_available():
|
||||
from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a
|
||||
from flashinfer.fp4_quantization import block_scale_interleave
|
||||
from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe
|
||||
from flashinfer.fused_moe import (
|
||||
trtllm_fp4_block_scale_moe,
|
||||
trtllm_fp4_block_scale_routed_moe,
|
||||
)
|
||||
from flashinfer.fused_moe.core import (
|
||||
_maybe_get_cached_w3_w1_permute_indices,
|
||||
get_w2_permute_indices_with_cache,
|
||||
@@ -39,6 +42,7 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput
|
||||
from sglang.srt.layers.moe.topk import TopKConfig
|
||||
|
||||
from sglang.srt.utils.common import get_bool_env_var
|
||||
|
||||
@@ -115,9 +119,25 @@ class Mxfp8RoutedInputPreQuant(NamedTuple):
|
||||
class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
fuse_routed_scaling_factor_in_topk = True
|
||||
|
||||
def configure_bypassed_topk(self, config: TopKConfig) -> bool:
|
||||
"""Select the FlashInfer routing contract before Top-K can be bypassed."""
|
||||
supported = (
|
||||
config.use_grouped_topk
|
||||
and config.scoring_func == "sigmoid"
|
||||
and config.correction_bias is not None
|
||||
and config.custom_routing_function is None
|
||||
and config.num_fused_shared_experts == 0
|
||||
and not config.torch_native
|
||||
)
|
||||
self._bypassed_routing_method = (
|
||||
RoutingMethodType.DeepSeekV3 if supported else None
|
||||
)
|
||||
return supported
|
||||
|
||||
def __init__(self, fp8_method, prefix: str):
|
||||
self._fp8 = fp8_method
|
||||
self.prefix = prefix
|
||||
self._bypassed_routing_method = None
|
||||
# precision=fp8 is an SM90 knob (Humming W4A8); this SM100 trtllm path
|
||||
# already runs MXFP8 activations, so the flag is inert here rather than
|
||||
# an error -- one config can move across hardware.
|
||||
@@ -370,16 +390,41 @@ class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
if w2_scale.dim() == 2:
|
||||
w2_scale = w2_scale.reshape(num_local_experts, hidden_size, -1)
|
||||
|
||||
if TopKOutputChecker.format_is_bypassed(topk_output):
|
||||
raise NotImplementedError(
|
||||
"the old code in this branch is WRONG. e.g. it does not consider HashTopK, and may miss args"
|
||||
bypassed = TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
if TopKOutputChecker.format_is_standard(topk_output):
|
||||
top_k = topk_output.topk_ids.shape[1]
|
||||
moe_call = trtllm_fp4_block_scale_routed_moe
|
||||
routing = dict(
|
||||
topk_ids=(topk_output.topk_ids, topk_output.topk_weights),
|
||||
routing_bias=None,
|
||||
routing_method_type=int(RoutingMethodType.TopK),
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
routed_scaling_factor=1.0,
|
||||
)
|
||||
if not TopKOutputChecker.format_is_standard(topk_output):
|
||||
elif bypassed:
|
||||
if self._bypassed_routing_method is None:
|
||||
raise NotImplementedError("MxFP4 bypassed routing is not configured")
|
||||
config = topk_output.topk_config
|
||||
top_k = config.top_k
|
||||
moe_call = trtllm_fp4_block_scale_moe
|
||||
routing = dict(
|
||||
routing_logits=topk_output.router_logits,
|
||||
routing_bias=config.correction_bias,
|
||||
routing_method_type=int(self._bypassed_routing_method),
|
||||
norm_topk_prob=config.renormalize,
|
||||
n_group=config.num_expert_group,
|
||||
topk_group=config.topk_group,
|
||||
routed_scaling_factor=(
|
||||
config.routed_scaling_factor
|
||||
if config.apply_routed_scaling_factor_on_output
|
||||
and config.routed_scaling_factor is not None
|
||||
else 1.0
|
||||
),
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported topk output format: {topk_output.format}")
|
||||
|
||||
topk_ids = topk_output.topk_ids
|
||||
topk_weights = topk_output.topk_weights
|
||||
|
||||
precision = self.flashinfer_mxfp4_moe_precision
|
||||
input_ready: Optional[torch.cuda.Event] = None
|
||||
if precision == "bf16":
|
||||
@@ -439,9 +484,8 @@ class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
# The op launches the routing kernel, so the join must precede it.
|
||||
torch.cuda.current_stream().wait_event(input_ready)
|
||||
|
||||
result = trtllm_fp4_block_scale_routed_moe(
|
||||
topk_ids=(topk_ids, topk_weights),
|
||||
routing_bias=None,
|
||||
result = moe_call(
|
||||
**routing,
|
||||
hidden_states=x_quant,
|
||||
hidden_states_scale=x_scale,
|
||||
gemm1_weights=w13,
|
||||
@@ -457,21 +501,17 @@ class Mxfp4FlashinferTrtllmMoEMethod:
|
||||
output1_scale_gate_scalar=layer.output1_scale_gate_scalar,
|
||||
output2_scale_scalar=layer.output2_scale_scalar,
|
||||
num_experts=layer.num_experts,
|
||||
top_k=topk_ids.shape[1],
|
||||
n_group=1,
|
||||
topk_group=1,
|
||||
top_k=top_k,
|
||||
intermediate_size=intermediate_size,
|
||||
local_expert_offset=layer.moe_ep_rank * layer.num_local_experts,
|
||||
local_num_experts=num_local_experts,
|
||||
routed_scaling_factor=1.0,
|
||||
routing_method_type=int(RoutingMethodType.TopK),
|
||||
do_finalize=not defer_finalize,
|
||||
tune_max_num_tokens=next_power_of_2(num_tokens),
|
||||
output=symm_output,
|
||||
enable_pdl=trtllm_moe_enable_pdl(num_tokens),
|
||||
)
|
||||
if defer_finalize:
|
||||
output = _make_deferred_finalize_output(result, top_k=topk_ids.shape[1])
|
||||
output = _make_deferred_finalize_output(result, top_k=top_k)
|
||||
else:
|
||||
output = result[0]
|
||||
|
||||
|
||||
@@ -26,6 +26,7 @@ from torch import nn
|
||||
from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages
|
||||
from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size
|
||||
from sglang.srt.distributed.utils import get_group_rank_size
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder
|
||||
from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation
|
||||
from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo
|
||||
@@ -37,12 +38,18 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
from sglang.srt.layers.dp_attention import (
|
||||
is_dp_attention_enabled,
|
||||
)
|
||||
from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion
|
||||
from sglang.srt.layers.layer_boundary import (
|
||||
append_stages,
|
||||
declare_attn,
|
||||
declare_ffn,
|
||||
is_dense_ffn_fully_dp,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.fusions.cutedsl import (
|
||||
CuteDSLFusion,
|
||||
MoeDeferredFinalize,
|
||||
install_cutedsl_fusion,
|
||||
)
|
||||
from sglang.srt.layers.layer_boundary.residual import batch as residual_batch
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.linear import (
|
||||
@@ -58,9 +65,17 @@ from sglang.srt.layers.moe import (
|
||||
get_moe_runner_backend,
|
||||
)
|
||||
from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class
|
||||
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
|
||||
from sglang.srt.layers.moe.utils import is_deepep_class_backend
|
||||
from sglang.srt.layers.moe.topk import (
|
||||
TopK,
|
||||
TopKOutputChecker,
|
||||
TopKOutputFormat,
|
||||
is_expert_remapping_enabled,
|
||||
)
|
||||
from sglang.srt.layers.moe.utils import is_deepep_class_backend, sum_post_experts_output
|
||||
from sglang.srt.layers.quantization.base_config import QuantizationConfig
|
||||
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
|
||||
Mxfp4FlashinferTrtllmMoEMethod,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.layers.rotary_embedding import get_rope
|
||||
from sglang.srt.layers.utils import PPMissingLayer, get_layer_id
|
||||
@@ -85,7 +100,12 @@ from sglang.srt.model_loader.weight_utils import (
|
||||
)
|
||||
from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig
|
||||
from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_flags,
|
||||
get_forward,
|
||||
get_parallel,
|
||||
)
|
||||
from sglang.srt.utils import (
|
||||
LazyValue,
|
||||
add_prefix,
|
||||
@@ -450,6 +470,34 @@ class MiMoV2MoE(nn.Module):
|
||||
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
|
||||
)
|
||||
|
||||
uses_mxfp4_runner = isinstance(
|
||||
self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod
|
||||
)
|
||||
self._uses_expert_remap = (
|
||||
not get_flags().moe.in_speculative_scope and is_expert_remapping_enabled()
|
||||
)
|
||||
use_bypassed_topk = (
|
||||
uses_cutedsl_ar_fusion()
|
||||
and not get_flags().moe.in_speculative_scope
|
||||
and not self._uses_expert_remap
|
||||
and not get_exec().features.enable_return_routed_experts
|
||||
and get_exec().moe.expert_distribution_recorder_mode is None
|
||||
and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get()
|
||||
and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get()
|
||||
and uses_mxfp4_runner
|
||||
and get_moe_a2a_backend().is_none()
|
||||
and self.experts.quant_method.configure_bypassed_topk(self.topk.topk_config)
|
||||
)
|
||||
if uses_mxfp4_runner:
|
||||
self.topk.topk_config.output_format = (
|
||||
TopKOutputFormat.BYPASSED
|
||||
if use_bypassed_topk
|
||||
else TopKOutputFormat.STANDARD
|
||||
)
|
||||
self.supports_deferred_finalize = (
|
||||
use_bypassed_topk and envs.SGLANG_ENABLE_MOE_DEFERRED_FINALIZE.get()
|
||||
)
|
||||
|
||||
# todo : implement tbo forward needed
|
||||
self._enable_a2a_moe = (
|
||||
is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep()
|
||||
@@ -480,7 +528,7 @@ class MiMoV2MoE(nn.Module):
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
forward_batch: Optional[ForwardBatch] = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | MoeDeferredFinalize:
|
||||
if not self._enable_a2a_moe:
|
||||
return self.forward_normal(hidden_states)
|
||||
else:
|
||||
@@ -489,14 +537,36 @@ class MiMoV2MoE(nn.Module):
|
||||
def forward_normal(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | MoeDeferredFinalize:
|
||||
|
||||
if hidden_states.shape[0] > 0:
|
||||
router_logits = self.gate(hidden_states)
|
||||
topk_output = self.topk(hidden_states, router_logits)
|
||||
topk_output = self.topk(
|
||||
hidden_states,
|
||||
router_logits,
|
||||
expert_location_dispatch_info=(
|
||||
ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id)
|
||||
if self._uses_expert_remap
|
||||
else None
|
||||
),
|
||||
)
|
||||
else:
|
||||
topk_output = self.topk.empty_topk_output(hidden_states.device)
|
||||
|
||||
# Precomputed routing keeps FP32 weights and finalizes before the reduction.
|
||||
if get_forward().defer_moe_finalize and TopKOutputChecker.format_is_bypassed(
|
||||
topk_output
|
||||
):
|
||||
deferred = self.experts.forward_deferred_finalize(
|
||||
hidden_states, topk_output
|
||||
)
|
||||
return MoeDeferredFinalize.from_flashinfer(
|
||||
deferred,
|
||||
gated_shared_output=None,
|
||||
m=hidden_states.shape[0],
|
||||
reduce=sum_post_experts_output,
|
||||
)
|
||||
|
||||
final_hidden_states = self.experts(hidden_states, topk_output)
|
||||
|
||||
return final_hidden_states
|
||||
@@ -850,9 +920,14 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
)
|
||||
|
||||
attn, ffn = self.stage_facts(config, layer_id)
|
||||
fusions = (
|
||||
CuteDSLFusion()
|
||||
if uses_cutedsl_ar_fusion() and not get_flags().moe.in_speculative_scope
|
||||
else None
|
||||
)
|
||||
self.attn_boundary, self.ffn_boundary = append_stages(
|
||||
(attn, self.input_layernorm),
|
||||
(ffn, self.post_attention_layernorm),
|
||||
(attn, self.input_layernorm, {"fusions": fusions}),
|
||||
(ffn, self.post_attention_layernorm, {"fusions": fusions}),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@@ -892,8 +967,9 @@ class MiMoV2DecoderLayer(nn.Module):
|
||||
hidden_states = self.attn_boundary.finish(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch)
|
||||
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch)
|
||||
with self.ffn_boundary.exit(forward_batch) as ffn_exit:
|
||||
hidden_states = self.mlp(hidden_states, forward_batch)
|
||||
hidden_states = ffn_exit.finish(hidden_states)
|
||||
|
||||
return hidden_states
|
||||
|
||||
@@ -1002,6 +1078,19 @@ class MiMoV2Model(nn.Module):
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon)
|
||||
else:
|
||||
self.norm = PPMissingLayer(return_tuple=True)
|
||||
if uses_cutedsl_ar_fusion():
|
||||
install_cutedsl_fusion(
|
||||
self.layers[self.start_layer : self.end_layer],
|
||||
hidden_size=config.hidden_size,
|
||||
top_k=config.num_experts_per_tok,
|
||||
rms_epsilon=config.layernorm_epsilon,
|
||||
can_defer_finalize=lambda layer: (
|
||||
isinstance(layer.mlp, MiMoV2MoE)
|
||||
and layer.mlp.supports_deferred_finalize
|
||||
),
|
||||
label="MiMo",
|
||||
include_shared_expert=False,
|
||||
)
|
||||
|
||||
def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
if hasattr(self.config, "scale_emb"):
|
||||
|
||||
Reference in New Issue
Block a user