diff --git a/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py b/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py index f0a1f9b7a9..13e6a9a8d6 100644 --- a/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py +++ b/python/sglang/kernels/ops/moe/smallm_router_gfx950/__init__.py @@ -89,8 +89,8 @@ def qwen_moe_topk_output(block, hidden_states): from sglang.srt.layers.moe import get_moe_runner_backend from sglang.srt.layers.moe.topk import ( StandardTopKOutput, - _eplb_remap_enabled, capture_routed_experts_if_allowed, + is_expert_remapping_enabled, ) from sglang.srt.model_executor.runner_backend_utils.tc_piecewise_cuda_graph.context_manager import ( is_in_tc_piecewise_cuda_graph, @@ -115,7 +115,7 @@ def qwen_moe_topk_output(block, hidden_states): and sw.is_contiguous() and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get() and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get() - and not _eplb_remap_enabled() + and not is_expert_remapping_enabled() ) if ( not ok diff --git a/python/sglang/srt/arg_groups/fields/exec_.py b/python/sglang/srt/arg_groups/fields/exec_.py index 66158cfacb..94e6bc08be 100644 --- a/python/sglang/srt/arg_groups/fields/exec_.py +++ b/python/sglang/srt/arg_groups/fields/exec_.py @@ -711,7 +711,8 @@ class ExecComm(msgspec.Struct): help=( "Enable FlashInfer allreduce fusion and choose backend. " "Requires SM90 or SM10X NVIDIA GPUs. " - "Defaults to auto. " + "When unspecified, supported MiMo configurations use cutedsl; " + "other supported models use auto. " "'auto': choose mnnvl on Blackwell (SM100/SM103) systems " "(single- and multi-node) and trtllm on SM90 single-node systems. " "'trtllm': available on single-node systems only. " diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 20d3fb15c8..ac79251c93 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -931,10 +931,26 @@ _FLASHINFER_ALLREDUCE_FUSION_ARCHS = frozenset( def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: """Enable supported FlashInfer AllReduce fusion after CP and A2A resolution. - Auto selects MNNVL on Blackwell and TRTLLM on single-node SM90. + MiMo defaults to CuTeDSL on Blackwell; other supported models use auto. """ - hf_config = model_config_of(view).hf_config + model_config = model_config_of(view) + hf_config = model_config.hf_config model_arch = hf_config.architectures[0] + is_mimo = model_arch in ("MiMoV2ForCausalLM", "MiMoV2FlashForCausalLM") + if is_mimo: + import torch + + from sglang.srt.layers.flashinfer_mnnvl_cutedsl import SUPPORTED_TP_SIZES + + if not ( + get_platform().is_sm100 + and model_config.dtype == torch.bfloat16 + and view.tp_size in SUPPORTED_TP_SIZES + and view.attn_cp_size == 1 + and not view.encoder_only + and not view.enable_pdmux + ): + return {} # V4.1 TP4 uses the custom push plane for decode and fused MoE finalize. prefer_custom_dsv41 = ( getattr(hf_config, "model_type", None) == "deepseek_v41" @@ -946,7 +962,7 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: ) if ( view.flashinfer_allreduce_fusion_backend is None - and model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS + and (is_mimo or model_arch in _FLASHINFER_ALLREDUCE_FUSION_ARCHS) and not prefer_custom_dsv41 and (get_platform().is_sm90 or get_platform().is_sm100) and view.tp_size > 1 @@ -954,10 +970,11 @@ def _flashinfer_allreduce_fusion_auto_enable(view: Any) -> dict: and (view.nnodes == 1 or get_platform().is_sm100) and view.moe_a2a_backend == "none" ): + backend = "cutedsl" if is_mimo else "auto" logger.info( - f"Auto-enabling FlashInfer AllReduce Fusion on SM90/SM10X for {model_arch}" + "Auto-enabling FlashInfer AllReduce Fusion (%s) for %s", backend, model_arch ) - return {"flashinfer_allreduce_fusion_backend": "auto"} + return {"flashinfer_allreduce_fusion_backend": backend} return {} diff --git a/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py b/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py index 2d630c1d61..6492d07f54 100644 --- a/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py +++ b/python/sglang/srt/layers/flashinfer_mnnvl_cutedsl.py @@ -41,7 +41,7 @@ _CUDA_BLOCK_THREADS = 1024 _HT_NON_REDUCTION_WARPS = 2 _HT_REDUCTION_WARP_CHOICES = (1, 2, 4, 8) -_SUPPORTED_TP_SIZES = (2, 4, 8, 16) +SUPPORTED_TP_SIZES = (2, 4, 8, 16) def _ht_shard_split( @@ -320,6 +320,7 @@ class FlashInferMNNVLCuteDSLARFusion: weight_bias: float, process_group: ProcessGroup, device: torch.device, + include_shared_expert: bool = True, ) -> None: if hidden_size <= 0 or top_k <= 0 or max_m <= 0: raise ValueError("hidden_size, top_k, and max_m must be positive") @@ -334,9 +335,9 @@ class FlashInferMNNVLCuteDSLARFusion: self.process_group = process_group # Cached: supports() runs per layer and must not re-enter c10d. self.tp_size = dist.get_world_size(process_group) - if self.tp_size not in _SUPPORTED_TP_SIZES: + if self.tp_size not in SUPPORTED_TP_SIZES: raise ValueError( - f"MNNVL CuTe DSL fusion supports tp_size in {_SUPPORTED_TP_SIZES}, " + f"MNNVL CuTe DSL fusion supports tp_size in {SUPPORTED_TP_SIZES}, " f"got {self.tp_size}" ) self.device = torch.device(device) @@ -373,9 +374,10 @@ class FlashInferMNNVLCuteDSLARFusion: hidden_size=self.hidden_size, top_k=self.top_k, ) - if get_spec().speculative_algorithm is None: + self.workspace_config = shaped_config + if include_shared_expert and get_spec().speculative_algorithm is None: self.workspace_config = _with_early_finalize_shared_load(shaped_config) - else: + elif include_shared_expert: # Early shared load is safe only for a single looping decode graph; # alternating draft/verify replays can read an unfinished buffer. logger.info( @@ -383,7 +385,6 @@ class FlashInferMNNVLCuteDSLARFusion: "CuTe DSL finalize presets on the safe (non-early-load) " "ordering." ) - self.workspace_config = shaped_config self.workspace = workspace_type( tp_size=self.tp_size, tp_rank=dist.get_rank(process_group), @@ -395,7 +396,7 @@ class FlashInferMNNVLCuteDSLARFusion: rms_eps=self.rms_epsilon, routed_scaling_factor=1.0, weight_bias=self.weight_bias, - include_shared_expert=True, + include_shared_expert=include_shared_expert, add_residual=True, write_residual_output=True, config=self.workspace_config, @@ -422,7 +423,7 @@ class FlashInferMNNVLCuteDSLARFusion: routed_output: torch.Tensor, expert_weights: torch.Tensor, permuted_indices: torch.Tensor, - gated_shared_output: torch.Tensor, + gated_shared_output: torch.Tensor | None, residual: torch.Tensor, gamma: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: @@ -491,6 +492,7 @@ def get_flashinfer_mnnvl_cutedsl_ar_fusion( max_m: int, rms_epsilon: float, weight_bias: float, + include_shared_expert: bool = True, ) -> FlashInferMNNVLCuteDSLARFusion: """Build the process-local workspace. Must run before graph capture.""" global _WORKSPACE @@ -518,5 +520,6 @@ def get_flashinfer_mnnvl_cutedsl_ar_fusion( weight_bias=weight_bias, process_group=get_parallel().tp_group.device_group, device=torch.device("cuda", torch.cuda.current_device()), + include_shared_expert=include_shared_expert, ) return _WORKSPACE diff --git a/python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py b/python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py index ed32bff45e..739f1923eb 100644 --- a/python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py +++ b/python/sglang/srt/layers/layer_boundary/fusions/cutedsl.py @@ -83,7 +83,7 @@ class MoeDeferredFinalize(DeferredFinalize, frozen=True): routed_output: torch.Tensor expert_weights: torch.Tensor permuted_indices: torch.Tensor - gated_shared_output: torch.Tensor + gated_shared_output: torch.Tensor | None m: int finish: Callable[[], torch.Tensor] @@ -95,7 +95,7 @@ class MoeDeferredFinalize(DeferredFinalize, frozen=True): cls, deferred_output, *, - gated_shared_output: torch.Tensor, + gated_shared_output: torch.Tensor | None, m: int, reduce: Callable[[torch.Tensor], torch.Tensor], ) -> MoeDeferredFinalize: @@ -130,7 +130,9 @@ class CuteDSLWorkspace: hidden_size: int, top_k: int, rms_epsilon: float, + include_shared_expert: bool = True, ) -> None: + self.include_shared_expert = include_shared_expert self.hidden_size = int(hidden_size) self.top_k = int(top_k) self.rms_epsilon = float(rms_epsilon) @@ -157,6 +159,7 @@ class CuteDSLWorkspace: rms_epsilon=self.rms_epsilon, # _fused_norm_gamma() already returns the multiplier as applied. weight_bias=0.0, + include_shared_expert=self.include_shared_expert, ) self._workspace = workspace self.max_m = workspace.max_m @@ -368,6 +371,7 @@ def install_cutedsl_fusion( can_defer_finalize: _LayerPredicate, label: str, terminal_finalize: bool = False, + include_shared_expert: bool = True, ) -> CuteDSLWorkspace | None: """One shared workspace handle per fusion-enabled layer, or None. @@ -403,6 +407,7 @@ def install_cutedsl_fusion( hidden_size=hidden_size, top_k=top_k, rms_epsilon=rms_epsilon, + include_shared_expert=include_shared_expert, ) hands_off = 0 for layer in layers: diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 6d8db4e2c3..8d90c269ff 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -1688,7 +1688,7 @@ def is_power_of_two(n): return n > 0 and math.log2(n).is_integer() -def _eplb_remap_enabled() -> bool: +def is_expert_remapping_enabled() -> bool: # A real logical->physical mapping only exists when EPLB is enabled, the # initial expert placement is non-trivial, or there are redundant physical # experts. Otherwise the map is identity and the remap must be skipped (it is @@ -2393,7 +2393,7 @@ def _post_process_topk_ids( # MoE routing for models like DeepSeek-R1-MXFP4 (accuracy ~0.09 vs 0.94+). # # Let append kernels materialize padded ids when remapping is disabled. - eplb_remap_enabled = _eplb_remap_enabled() + eplb_remap_enabled = is_expert_remapping_enabled() _fold_pad_into_append = ( _aiter_append and not eplb_remap_enabled @@ -2597,7 +2597,7 @@ def select_experts( and ( num_fused_shared_experts == 0 or has_per_rank_fused_shared_slots(num_fused_shared_experts) - or not _eplb_remap_enabled() + or not is_expert_remapping_enabled() ) ): # only the aiter sqrtsoftplus gate takes the partials; every other route reads router_logits diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py index 440e16cbcb..491cd585cd 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py @@ -29,7 +29,10 @@ _MXFP8_QUANTIZE_BACKEND = "cute-dsl" if get_platform().is_sm100 else "cuda" if is_flashinfer_available(): from flashinfer import shuffle_matrix_a, shuffle_matrix_sf_a from flashinfer.fp4_quantization import block_scale_interleave - from flashinfer.fused_moe import trtllm_fp4_block_scale_routed_moe + from flashinfer.fused_moe import ( + trtllm_fp4_block_scale_moe, + trtllm_fp4_block_scale_routed_moe, + ) from flashinfer.fused_moe.core import ( _maybe_get_cached_w3_w1_permute_indices, get_w2_permute_indices_with_cache, @@ -39,6 +42,7 @@ logger = logging.getLogger(__name__) if TYPE_CHECKING: from sglang.srt.layers.moe.token_dispatcher import CombineInput, DispatchOutput + from sglang.srt.layers.moe.topk import TopKConfig from sglang.srt.utils.common import get_bool_env_var @@ -115,9 +119,25 @@ class Mxfp8RoutedInputPreQuant(NamedTuple): class Mxfp4FlashinferTrtllmMoEMethod: fuse_routed_scaling_factor_in_topk = True + def configure_bypassed_topk(self, config: TopKConfig) -> bool: + """Select the FlashInfer routing contract before Top-K can be bypassed.""" + supported = ( + config.use_grouped_topk + and config.scoring_func == "sigmoid" + and config.correction_bias is not None + and config.custom_routing_function is None + and config.num_fused_shared_experts == 0 + and not config.torch_native + ) + self._bypassed_routing_method = ( + RoutingMethodType.DeepSeekV3 if supported else None + ) + return supported + def __init__(self, fp8_method, prefix: str): self._fp8 = fp8_method self.prefix = prefix + self._bypassed_routing_method = None # precision=fp8 is an SM90 knob (Humming W4A8); this SM100 trtllm path # already runs MXFP8 activations, so the flag is inert here rather than # an error -- one config can move across hardware. @@ -370,16 +390,41 @@ class Mxfp4FlashinferTrtllmMoEMethod: if w2_scale.dim() == 2: w2_scale = w2_scale.reshape(num_local_experts, hidden_size, -1) - if TopKOutputChecker.format_is_bypassed(topk_output): - raise NotImplementedError( - "the old code in this branch is WRONG. e.g. it does not consider HashTopK, and may miss args" + bypassed = TopKOutputChecker.format_is_bypassed(topk_output) + if TopKOutputChecker.format_is_standard(topk_output): + top_k = topk_output.topk_ids.shape[1] + moe_call = trtllm_fp4_block_scale_routed_moe + routing = dict( + topk_ids=(topk_output.topk_ids, topk_output.topk_weights), + routing_bias=None, + routing_method_type=int(RoutingMethodType.TopK), + n_group=1, + topk_group=1, + routed_scaling_factor=1.0, ) - if not TopKOutputChecker.format_is_standard(topk_output): + elif bypassed: + if self._bypassed_routing_method is None: + raise NotImplementedError("MxFP4 bypassed routing is not configured") + config = topk_output.topk_config + top_k = config.top_k + moe_call = trtllm_fp4_block_scale_moe + routing = dict( + routing_logits=topk_output.router_logits, + routing_bias=config.correction_bias, + routing_method_type=int(self._bypassed_routing_method), + norm_topk_prob=config.renormalize, + n_group=config.num_expert_group, + topk_group=config.topk_group, + routed_scaling_factor=( + config.routed_scaling_factor + if config.apply_routed_scaling_factor_on_output + and config.routed_scaling_factor is not None + else 1.0 + ), + ) + else: raise ValueError(f"Unsupported topk output format: {topk_output.format}") - topk_ids = topk_output.topk_ids - topk_weights = topk_output.topk_weights - precision = self.flashinfer_mxfp4_moe_precision input_ready: Optional[torch.cuda.Event] = None if precision == "bf16": @@ -439,9 +484,8 @@ class Mxfp4FlashinferTrtllmMoEMethod: # The op launches the routing kernel, so the join must precede it. torch.cuda.current_stream().wait_event(input_ready) - result = trtllm_fp4_block_scale_routed_moe( - topk_ids=(topk_ids, topk_weights), - routing_bias=None, + result = moe_call( + **routing, hidden_states=x_quant, hidden_states_scale=x_scale, gemm1_weights=w13, @@ -457,21 +501,17 @@ class Mxfp4FlashinferTrtllmMoEMethod: output1_scale_gate_scalar=layer.output1_scale_gate_scalar, output2_scale_scalar=layer.output2_scale_scalar, num_experts=layer.num_experts, - top_k=topk_ids.shape[1], - n_group=1, - topk_group=1, + top_k=top_k, intermediate_size=intermediate_size, local_expert_offset=layer.moe_ep_rank * layer.num_local_experts, local_num_experts=num_local_experts, - routed_scaling_factor=1.0, - routing_method_type=int(RoutingMethodType.TopK), do_finalize=not defer_finalize, tune_max_num_tokens=next_power_of_2(num_tokens), output=symm_output, enable_pdl=trtllm_moe_enable_pdl(num_tokens), ) if defer_finalize: - output = _make_deferred_finalize_output(result, top_k=topk_ids.shape[1]) + output = _make_deferred_finalize_output(result, top_k=top_k) else: output = result[0] diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index f6f40c22f3..33393d623a 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -26,6 +26,7 @@ from torch import nn from sglang.srt.batch_overlap.two_batch_overlap import model_forward_stages from sglang.srt.configs.model_config import get_mimo_v2_fused_qkv_expected_tp_size from sglang.srt.distributed.utils import get_group_rank_size +from sglang.srt.environ import envs from sglang.srt.eplb.expert_distribution import get_global_expert_distribution_recorder from sglang.srt.eplb.expert_location import ModelConfigForExpertLocation from sglang.srt.eplb.expert_location_dispatch import ExpertLocationDispatchInfo @@ -37,12 +38,18 @@ from sglang.srt.layers.aux_hidden_states import ( from sglang.srt.layers.dp_attention import ( is_dp_attention_enabled, ) +from sglang.srt.layers.flashinfer_comm_fusion import uses_cutedsl_ar_fusion from sglang.srt.layers.layer_boundary import ( append_stages, declare_attn, declare_ffn, is_dense_ffn_fully_dp, ) +from sglang.srt.layers.layer_boundary.fusions.cutedsl import ( + CuteDSLFusion, + MoeDeferredFinalize, + install_cutedsl_fusion, +) from sglang.srt.layers.layer_boundary.residual import batch as residual_batch from sglang.srt.layers.layernorm import RMSNorm from sglang.srt.layers.linear import ( @@ -58,9 +65,17 @@ from sglang.srt.layers.moe import ( get_moe_runner_backend, ) from sglang.srt.layers.moe.ep_moe.layer import DeepEPMoE, get_moe_impl_class -from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat -from sglang.srt.layers.moe.utils import is_deepep_class_backend +from sglang.srt.layers.moe.topk import ( + TopK, + TopKOutputChecker, + TopKOutputFormat, + is_expert_remapping_enabled, +) +from sglang.srt.layers.moe.utils import is_deepep_class_backend, sum_post_experts_output from sglang.srt.layers.quantization.base_config import QuantizationConfig +from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + Mxfp4FlashinferTrtllmMoEMethod, +) from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope from sglang.srt.layers.utils import PPMissingLayer, get_layer_id @@ -85,7 +100,12 @@ from sglang.srt.model_loader.weight_utils import ( ) from sglang.srt.models.mimo_audio import AudioEncoderMixin, MiMoAudioEncoderConfig from sglang.srt.models.mimo_vl import MiMoVisionTransformer, MiMoVLVisionConfig -from sglang.srt.runtime_context import get_exec, get_parallel +from sglang.srt.runtime_context import ( + get_exec, + get_flags, + get_forward, + get_parallel, +) from sglang.srt.utils import ( LazyValue, add_prefix, @@ -450,6 +470,34 @@ class MiMoV2MoE(nn.Module): output_format=TopKOutputFormat.STANDARD if quant_config is None else None, ) + uses_mxfp4_runner = isinstance( + self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod + ) + self._uses_expert_remap = ( + not get_flags().moe.in_speculative_scope and is_expert_remapping_enabled() + ) + use_bypassed_topk = ( + uses_cutedsl_ar_fusion() + and not get_flags().moe.in_speculative_scope + and not self._uses_expert_remap + and not get_exec().features.enable_return_routed_experts + and get_exec().moe.expert_distribution_recorder_mode is None + and not envs.SGLANG_SIMULATE_UNIFORM_EXPERTS.get() + and not envs.SGLANG_SIMULATE_ROUND_ROBIN_EXPERTS.get() + and uses_mxfp4_runner + and get_moe_a2a_backend().is_none() + and self.experts.quant_method.configure_bypassed_topk(self.topk.topk_config) + ) + if uses_mxfp4_runner: + self.topk.topk_config.output_format = ( + TopKOutputFormat.BYPASSED + if use_bypassed_topk + else TopKOutputFormat.STANDARD + ) + self.supports_deferred_finalize = ( + use_bypassed_topk and envs.SGLANG_ENABLE_MOE_DEFERRED_FINALIZE.get() + ) + # todo : implement tbo forward needed self._enable_a2a_moe = ( is_deepep_class_backend() or get_moe_a2a_backend().is_ascend_fuseep() @@ -480,7 +528,7 @@ class MiMoV2MoE(nn.Module): self, hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None, - ) -> torch.Tensor: + ) -> torch.Tensor | MoeDeferredFinalize: if not self._enable_a2a_moe: return self.forward_normal(hidden_states) else: @@ -489,14 +537,36 @@ class MiMoV2MoE(nn.Module): def forward_normal( self, hidden_states: torch.Tensor, - ) -> torch.Tensor: + ) -> torch.Tensor | MoeDeferredFinalize: if hidden_states.shape[0] > 0: router_logits = self.gate(hidden_states) - topk_output = self.topk(hidden_states, router_logits) + topk_output = self.topk( + hidden_states, + router_logits, + expert_location_dispatch_info=( + ExpertLocationDispatchInfo.init_new(layer_id=self.layer_id) + if self._uses_expert_remap + else None + ), + ) else: topk_output = self.topk.empty_topk_output(hidden_states.device) + # Precomputed routing keeps FP32 weights and finalizes before the reduction. + if get_forward().defer_moe_finalize and TopKOutputChecker.format_is_bypassed( + topk_output + ): + deferred = self.experts.forward_deferred_finalize( + hidden_states, topk_output + ) + return MoeDeferredFinalize.from_flashinfer( + deferred, + gated_shared_output=None, + m=hidden_states.shape[0], + reduce=sum_post_experts_output, + ) + final_hidden_states = self.experts(hidden_states, topk_output) return final_hidden_states @@ -850,9 +920,14 @@ class MiMoV2DecoderLayer(nn.Module): ) attn, ffn = self.stage_facts(config, layer_id) + fusions = ( + CuteDSLFusion() + if uses_cutedsl_ar_fusion() and not get_flags().moe.in_speculative_scope + else None + ) self.attn_boundary, self.ffn_boundary = append_stages( - (attn, self.input_layernorm), - (ffn, self.post_attention_layernorm), + (attn, self.input_layernorm, {"fusions": fusions}), + (ffn, self.post_attention_layernorm, {"fusions": fusions}), ) @classmethod @@ -892,8 +967,9 @@ class MiMoV2DecoderLayer(nn.Module): hidden_states = self.attn_boundary.finish(hidden_states, forward_batch) hidden_states = self.ffn_boundary.prepare(hidden_states, forward_batch) - hidden_states = self.mlp(hidden_states, forward_batch) - hidden_states = self.ffn_boundary.finish(hidden_states, forward_batch) + with self.ffn_boundary.exit(forward_batch) as ffn_exit: + hidden_states = self.mlp(hidden_states, forward_batch) + hidden_states = ffn_exit.finish(hidden_states) return hidden_states @@ -1002,6 +1078,19 @@ class MiMoV2Model(nn.Module): self.norm = RMSNorm(config.hidden_size, eps=config.layernorm_epsilon) else: self.norm = PPMissingLayer(return_tuple=True) + if uses_cutedsl_ar_fusion(): + install_cutedsl_fusion( + self.layers[self.start_layer : self.end_layer], + hidden_size=config.hidden_size, + top_k=config.num_experts_per_tok, + rms_epsilon=config.layernorm_epsilon, + can_defer_finalize=lambda layer: ( + isinstance(layer.mlp, MiMoV2MoE) + and layer.mlp.supports_deferred_finalize + ), + label="MiMo", + include_shared_expert=False, + ) def get_input_embedding(self, input_ids: torch.Tensor) -> torch.Tensor: if hasattr(self.config, "scale_emb"):