diff --git a/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py b/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py index 11e792e320..1b402fe7d6 100644 --- a/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py +++ b/python/sglang/multimodal_gen/runtime/models/dits/hunyuan_image3.py @@ -513,6 +513,7 @@ class HunYuanSparseMoeBlock(nn.Module): layer_id=layer_id, prefix=f"{prefix}.experts", ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states): orig_shape = hidden_states.shape diff --git a/python/sglang/srt/batch_overlap/two_batch_overlap.py b/python/sglang/srt/batch_overlap/two_batch_overlap.py index 83569c2ea6..424c8c578d 100644 --- a/python/sglang/srt/batch_overlap/two_batch_overlap.py +++ b/python/sglang/srt/batch_overlap/two_batch_overlap.py @@ -1155,12 +1155,13 @@ def _model_forward_tbo_merge_key(output_a, output_b, name, original_len): class MaybeTboDeepEPDispatcher(BaseDispatcher): - def __init__(self, **kwargs): + def __init__(self, runner_backend=None, **kwargs): super().__init__() num_inner_dispatchers = 2 if is_tbo_enabled() else 1 if get_moe_a2a_backend().is_deepep(): self._inners = [ - DeepEPDispatcher(**kwargs) for _ in range(num_inner_dispatchers) + DeepEPDispatcher(runner_backend=runner_backend, **kwargs) + for _ in range(num_inner_dispatchers) ] elif get_moe_a2a_backend().is_mooncake(): self._inners = [ @@ -1177,7 +1178,8 @@ class MaybeTboDeepEPDispatcher(BaseDispatcher): ] elif get_moe_a2a_backend().is_pplx(): self._inners = [ - PplxDispatcher(**kwargs) for _ in range(num_inner_dispatchers) + PplxDispatcher(runner_backend=runner_backend, **kwargs) + for _ in range(num_inner_dispatchers) ] @property diff --git a/python/sglang/srt/layers/moe/ep_moe/layer.py b/python/sglang/srt/layers/moe/ep_moe/layer.py index cd5164c3be..948ffcdb9b 100644 --- a/python/sglang/srt/layers/moe/ep_moe/layer.py +++ b/python/sglang/srt/layers/moe/ep_moe/layer.py @@ -15,7 +15,6 @@ from sglang.srt.layers.dp_attention import ( from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, - get_moe_runner_backend, ) from sglang.srt.layers.moe.fused_moe_triton.layer import ( FusedMoE, @@ -97,12 +96,8 @@ class DeepEPMoE(FusedMoE): routed_scaling_factor=routed_scaling_factor, **kwargs, ) - is_humming = ( - get_moe_runner_backend().is_humming() - or get_moe_runner_backend().is_auto() - and quant_config is not None - and quant_config.get_name() == "humming" - ) + backend = self.runner_backend + is_humming = backend is not None and backend.is_humming() if get_moe_a2a_backend().is_deepep_v2(): self.deprecate_flag = True elif is_humming: @@ -121,14 +116,16 @@ class DeepEPMoE(FusedMoE): ): self.deprecate_flag = True elif ( - get_moe_runner_backend().is_flashinfer_cutedsl() + backend is not None + and backend.is_flashinfer_cutedsl() and quant_config is not None and quant_config.get_name() in ("modelopt_fp4", "modelopt_mixed") ): self.deprecate_flag = True elif ( deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM - and get_moe_runner_backend().is_deep_gemm() + and backend is not None + and backend.is_deep_gemm() and quant_config is not None and quant_config.get_name() == "mxfp4" ): @@ -138,7 +135,8 @@ class DeepEPMoE(FusedMoE): elif ( quant_config is None and self.w13_weight.dtype == torch.bfloat16 - and get_moe_runner_backend().is_deep_gemm() + and backend is not None + and backend.is_deep_gemm() and (get_moe_a2a_backend().is_deepep() or get_moe_a2a_backend().is_pplx()) and not _is_npu and not _is_hip diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 40fdec171c..3a3e527690 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -101,26 +101,24 @@ _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip _deferred_finalize_info_logged = False -def _fuses_routed_scaling_factor_in_topk(quant_method) -> bool: +def _fuses_routed_scaling_factor_in_topk(quant_method, runner_backend) -> bool: return ( getattr(quant_method, "fuse_routed_scaling_factor_in_topk", False) or ( isinstance(quant_method, ModelOptNvFp4FusedMoEMethod) - and not getattr( - quant_method, "_moe_runner_backend", get_moe_runner_backend() - ).is_marlin() + and (runner_backend is None or not runner_backend.is_marlin()) ) or ( - isinstance(quant_method, Fp8MoEMethod) + runner_backend is not None and ( - get_moe_runner_backend().is_cutlass() - or get_moe_runner_backend().is_flashinfer_trtllm_routed() + # Materialized TRT-LLM routes carry the final weights, including + # when a quantization wrapper delegates to a layer-owned scheme. + runner_backend.is_flashinfer_trtllm() + or runner_backend.is_flashinfer_trtllm_routed() + or isinstance(quant_method, Fp8MoEMethod) + and runner_backend.is_cutlass() ) ) - or ( - isinstance(quant_method, UnquantizedFusedMoEMethod) - and get_moe_runner_backend().is_flashinfer_trtllm_routed() - ) ) @@ -165,15 +163,6 @@ def _get_deepep_comm_group(a2a_backend): return group -def _built_runner_backend(quant_method: FusedMoEMethodBase): - # The layer calls create_moe_runner before building its dispatcher, so the - # runner the quant method actually picked is known here. Methods that keep - # no runner of their own (including wrappers that build one on an inner - # scheme) return None and the dispatcher follows --moe-runner-backend. - runner = quant_method.runner - return None if runner is None else runner.runner_backend - - def create_moe_dispatcher( moe_runner_config: MoeRunnerConfig, quant_method: FusedMoEMethodBase, @@ -192,9 +181,7 @@ def create_moe_dispatcher( # StandardDispatcher is created but never invoked. # flashinfer_megamoe does its EP all-to-all inside the kernel, so the # dispatcher stays a pure noop passthrough. - return StandardDispatcher( - moe_runner_config, runner_backend=_built_runner_backend(quant_method) - ) + return StandardDispatcher(moe_runner_config) elif ( a2a_backend.is_deepep() or a2a_backend.is_mooncake() @@ -203,6 +190,7 @@ def create_moe_dispatcher( or a2a_backend.is_pplx() ): return MaybeTboDeepEPDispatcher( + runner_backend=moe_runner_config.runner_backend, group=_get_deepep_comm_group(a2a_backend), router_topk=moe_runner_config.top_k, permute_fusion=True, @@ -546,19 +534,7 @@ class FusedMoE(torch.nn.Module): and num_experts == 512 and top_k == 10 ) - self.supports_deferred_finalize = ( - get_moe_runner_backend().is_flashinfer_trtllm() - and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred) - ) - global _deferred_finalize_info_logged - if not _deferred_finalize_info_logged: - _deferred_finalize_info_logged = True - logging.getLogger(__name__).debug( - "FlashInfer TRTLLM MoE deferred finalize is " - f"{'enabled' if self.supports_deferred_finalize else 'disabled'} " - f"(moe_runner_backend={get_exec().moe.moe_runner_backend}, " - f"quant_method={type(self.quant_method).__name__})." - ) + self.supports_deferred_finalize = False self.quant_method.create_weights( layer=self, @@ -576,6 +552,24 @@ class FusedMoE(torch.nn.Module): ) self.quant_method.create_moe_runner(self, self.moe_runner_config) + self.runner_backend = self.quant_method.get_moe_runner_backend(self) + self.moe_runner_config.runner_backend = self.runner_backend + self._set_runner_flags() + self.supports_deferred_finalize = ( + self.runner_backend is not None + and self.runner_backend.is_flashinfer_trtllm() + and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred) + ) + global _deferred_finalize_info_logged + if not _deferred_finalize_info_logged: + _deferred_finalize_info_logged = True + logging.getLogger(__name__).debug( + "FlashInfer TRTLLM MoE deferred finalize is " + f"{'enabled' if self.supports_deferred_finalize else 'disabled'} " + f"(moe_runner_backend={self.runner_backend}, " + f"quant_method={type(self.quant_method).__name__})." + ) + self.dispatcher = create_moe_dispatcher( self.moe_runner_config, quant_method=self.quant_method ) @@ -593,14 +587,11 @@ class FusedMoE(torch.nn.Module): self.swiglu_alpha = gemm1_alpha self.swiglu_clamp_limit = gemm1_clamp_limit - if ( - get_moe_runner_backend().is_flashinfer_trtllm_routed() - or get_moe_runner_backend().is_flashinfer_trtllm() - ): + if self.use_flashinfer_trtllm_moe: self.moe_runner_config.inplace = False self.should_fuse_routed_scaling_factor_in_topk = ( - _fuses_routed_scaling_factor_in_topk(self.quant_method) + _fuses_routed_scaling_factor_in_topk(self.quant_method, self.runner_backend) ) self.routing_method_type = routing_method_type @@ -613,6 +604,20 @@ class FusedMoE(torch.nn.Module): self.runner = self.quant_method.runner + def _set_runner_flags(self) -> None: + backend = self.runner_backend + self.enable_flashinfer_cutlass_moe = ( + backend is not None and backend.is_flashinfer_cutlass() + ) + self.use_flashinfer_mxfp4_moe = ( + backend is not None and backend.is_flashinfer_mxfp4() + ) + self.use_triton_kernels = backend is not None and backend.is_triton_kernels() + self.use_deep_gemm = backend is not None and backend.is_deep_gemm() + self.use_flashinfer_trtllm_moe = backend is not None and ( + backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed() + ) + @property def num_global_routed_experts(self) -> int: return self._num_global_routed @@ -828,17 +833,19 @@ class FusedMoE(torch.nn.Module): loaded_weight = _maybe_copy_weight_view_before_h2d(loaded_weight) # loaded_weight may be smaller than expert_data along shard_dim when - # the buffer is padded. Copy into the leading slice and leave the - # trailing padding as zeros. Rank-mismatched tensors (bias / scalar + # the buffer is padded. Initialize padding even if a runner fallback + # disabled use_padded_loading. Rank-mismatched tensors (bias / scalar # scales) don't carry shard_dim; they keep the plain broadcast copy. if ( loaded_weight.dim() == expert_data.dim() and shard_dim < expert_data.dim() and loaded_weight.shape[shard_dim] < expert_data.shape[shard_dim] ): - expert_data.narrow(shard_dim, 0, loaded_weight.shape[shard_dim]).copy_( - loaded_weight - ) + loaded_size = loaded_weight.shape[shard_dim] + expert_data.narrow( + shard_dim, loaded_size, expert_data.shape[shard_dim] - loaded_size + ).zero_() + expert_data.narrow(shard_dim, 0, loaded_size).copy_(loaded_weight) else: expert_data.copy_(loaded_weight) @@ -913,16 +920,18 @@ class FusedMoE(torch.nn.Module): # w2, down_proj: Load into only logical weight of w2. loaded_weight = _maybe_copy_weight_view_before_h2d(loaded_weight) # loaded_weight may be smaller than expert_data along shard_dim when - # the buffer is padded. Copy into the leading slice only. See the - # rank-mismatch note in _load_w13. + # the buffer is padded. Zero the untouched tail before copying the + # leading slice. See the rank-mismatch note in _load_w13. if ( loaded_weight.dim() == expert_data.dim() and shard_dim < expert_data.dim() and loaded_weight.shape[shard_dim] < expert_data.shape[shard_dim] ): - expert_data.narrow(shard_dim, 0, loaded_weight.shape[shard_dim]).copy_( - loaded_weight - ) + loaded_size = loaded_weight.shape[shard_dim] + expert_data.narrow( + shard_dim, loaded_size, expert_data.shape[shard_dim] - loaded_size + ).zero_() + expert_data.narrow(shard_dim, 0, loaded_size).copy_(loaded_weight) else: expert_data.copy_(loaded_weight) @@ -1230,10 +1239,7 @@ class FusedMoE(torch.nn.Module): param=param, weight_name=weight_name, ) - elif isinstance(method, Fp8MoEMethod) and ( - get_moe_runner_backend().is_flashinfer_trtllm_routed() - or get_moe_runner_backend().is_flashinfer_trtllm() - ): + elif isinstance(method, Fp8MoEMethod) and (self.use_flashinfer_trtllm_moe): # Drop the GPU mxfp8 shuffle-index cache on every reload for mxfp8 trtllm, trtllm_routed from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( clear_mxfp8_shuffle_index_cache, @@ -1482,10 +1488,7 @@ class FusedMoE(torch.nn.Module): method = _weight_layout_method(self) if self.scheme is not None: method = self.scheme - if isinstance(method, Fp8MoEMethod) and ( - get_moe_runner_backend().is_flashinfer_trtllm_routed() - or get_moe_runner_backend().is_flashinfer_trtllm() - ): + if isinstance(method, Fp8MoEMethod) and (self.use_flashinfer_trtllm_moe): # Drop the GPU mxfp8 shuffle-index cache on every reload for mxfp8 trtllm, trtllm_routed from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( clear_mxfp8_shuffle_index_cache, diff --git a/python/sglang/srt/layers/moe/kt_ep_wrapper.py b/python/sglang/srt/layers/moe/kt_ep_wrapper.py index 36c15cf139..1849fb0fa0 100644 --- a/python/sglang/srt/layers/moe/kt_ep_wrapper.py +++ b/python/sglang/srt/layers/moe/kt_ep_wrapper.py @@ -272,6 +272,9 @@ class KTEPWrapperMethod(FusedMoEMethodBase): # Delegate to GPU method to create its runner self.gpu_method.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.gpu_method.get_moe_runner_backend(layer) + def submit( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/moe/moe_runner/base.py b/python/sglang/srt/layers/moe/moe_runner/base.py index 6b8ba0890d..6b2d00c7c2 100644 --- a/python/sglang/srt/layers/moe/moe_runner/base.py +++ b/python/sglang/srt/layers/moe/moe_runner/base.py @@ -47,6 +47,9 @@ class MoeRunnerConfig: params_dtype: Optional[torch.dtype] = None routing_method_type: Optional[RoutingMethodType] = None + # None denotes a direct kernel with standard TopK and local expert IDs. + runner_backend: Optional[MoeRunnerBackendLike] = None + # Runner configuration activation: str = "silu" is_gated: bool = True diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index fcbaab28a3..631dd2a06b 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -800,6 +800,10 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( else hidden_states.dtype ) topk_output = dispatch_output.topk_output + materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard( + topk_output + ) + use_routed_topk |= materialized_topk if TopKOutputChecker.format_is_bypassed(topk_output): router_logits = topk_output.router_logits topk_config = topk_output.topk_config @@ -809,7 +813,9 @@ def fused_experts_none_to_flashinfer_trtllm_fp8( topk_config = None correction_bias = None - routing_method_type = quant_info.routing_method_type + routing_method_type = ( + RoutingMethodType.TopK if materialized_topk else quant_info.routing_method_type + ) fp8_quantization_type = ( Fp8QuantizationType.MxFp8 if quant_info.use_mxfp8 @@ -1622,6 +1628,10 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( from sglang.srt.layers.moe.topk import TopKOutputChecker from sglang.srt.layers.moe.utils import RoutingMethodType + materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard( + dispatch_output.topk_output + ) + use_routed_topk |= materialized_topk trtllm_bf16_routed_moe = None trtllm_bf16_moe = None if use_routed_topk: @@ -1676,7 +1686,11 @@ def fused_experts_none_to_flashinfer_trtllm_bf16( assert runner_config.top_k is not None, ( "runner_config.top_k is required for flashinfer_trtllm_routed." ) - routing_method_type = runner_config.routing_method_type + routing_method_type = ( + RoutingMethodType.TopK + if materialized_topk + else runner_config.routing_method_type + ) if routing_method_type is None: routing_method_type = RoutingMethodType.Default elif routing_method_type == RoutingMethodType.DeepSeekV3: diff --git a/python/sglang/srt/layers/moe/token_dispatcher/base.py b/python/sglang/srt/layers/moe/token_dispatcher/base.py index 94a7bd0624..0eae148db1 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/base.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/base.py @@ -329,6 +329,7 @@ class BaseDispatcher(ABC): def __init__(self): self.quant_config: dict = {} + self.runner_backend = None # Overlap args self.overlap_args: Optional[CombineOverlapArgs] = None diff --git a/python/sglang/srt/layers/moe/token_dispatcher/deepep.py b/python/sglang/srt/layers/moe/token_dispatcher/deepep.py index 2c3670a839..d7ef1b820e 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/deepep.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/deepep.py @@ -376,6 +376,7 @@ class _DeepEPDispatcherImplBase: hidden_size: int, params_dtype: torch.dtype, deepep_mode: DeepEPMode, + runner_backend=None, ): if not use_deepep: raise ImportError( @@ -388,6 +389,7 @@ class _DeepEPDispatcherImplBase: ) ) from _deepep_import_error + self.runner_backend = runner_backend self.group = group self.router_topk = router_topk self.permute_fusion = permute_fusion @@ -991,12 +993,14 @@ class DeepEPDispatcher(BaseDispatcher): deepep_mode: DeepEPMode = DeepEPMode.AUTO, async_finish: bool = False, return_recv_hook: bool = False, + runner_backend=None, ): super().__init__() self.deepep_mode = deepep_mode common_kwargs = dict( + runner_backend=runner_backend, group=group, router_topk=router_topk, permute_fusion=permute_fusion, diff --git a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py index ad685fe533..261e9fc2bd 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/flashinfer.py @@ -36,7 +36,6 @@ from sglang.srt.layers.moe.topk import ( from sglang.srt.layers.moe.utils import ( FlashinferA2ADispatchType, get_flashinfer_a2a_dispatch_type, - get_moe_runner_backend, ) from sglang.srt.runtime_context import ( get_flags, @@ -162,24 +161,33 @@ class FlashinferDispatcher(BaseDispatcher): "Please install the appropriate version of Flashinfer." ) + self.runner_backend = ( + moe_runner_config.runner_backend if moe_runner_config is not None else None + ) self.ep_size = group.size() self.ep_rank = group.rank() self.router_topk = router_topk self.hidden_size = hidden_size self.num_experts = num_experts self.num_local_experts = num_local_experts - runner_backend = get_moe_runner_backend() + runner_backend = self.runner_backend self.invalid_token_expert_id = ( -1 if ( - runner_backend.is_deep_gemm() - or runner_backend.is_flashinfer_trtllm() - or runner_backend.is_flashinfer_trtllm_routed() + runner_backend is not None + and ( + runner_backend.is_deep_gemm() + or runner_backend.is_flashinfer_trtllm() + or runner_backend.is_flashinfer_trtllm_routed() + ) ) else self.num_experts ) # TODO: Can other moe runners use payload_in_workspace too? - self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass() + self.payload_in_workspace = ( + self.runner_backend is not None + and self.runner_backend.is_flashinfer_cutlass() + ) self.dispatch_type = get_flashinfer_a2a_dispatch_type() if moe_runner_config is None: from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig @@ -389,15 +397,18 @@ class FlashinferDispatcher(BaseDispatcher): ) # Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16; # FP4 retains its packed wire path keyed by input_global_scale. - runner_backend = get_moe_runner_backend() + runner_backend = self.runner_backend weight_dtype = self.quant_config.get("weight_dtype") uses_bf16_fp8_payload = weight_dtype in ( torch.float8_e4m3fn, torch.float8_e5m2, ) and ( - runner_backend.is_deep_gemm() - or runner_backend.is_flashinfer_trtllm() - or runner_backend.is_flashinfer_trtllm_routed() + runner_backend is not None + and ( + runner_backend.is_deep_gemm() + or runner_backend.is_flashinfer_trtllm() + or runner_backend.is_flashinfer_trtllm_routed() + ) ) if uses_bf16_fp8_payload and hidden_states.dtype != torch.bfloat16: raise TypeError( @@ -537,9 +548,9 @@ class FlashinferDispatcher(BaseDispatcher): x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1]) # TODO: fuse interleave into cutlass moe - if ( - dispatch_type == FlashinferA2ADispatchType.NVFP4 - and get_moe_runner_backend().is_flashinfer_cutlass() + if dispatch_type == FlashinferA2ADispatchType.NVFP4 and ( + self.runner_backend is not None + and self.runner_backend.is_flashinfer_cutlass() ): x_sf = nvfp4_block_scale_interleave(x_sf) else: @@ -593,13 +604,16 @@ class FlashinferDispatcher(BaseDispatcher): ) weight_dtype = self.quant_config.get("weight_dtype") - runner_backend = get_moe_runner_backend() + runner_backend = self.runner_backend if ( weight_dtype in (torch.float8_e4m3fn, torch.float8_e5m2) and ( - runner_backend.is_deep_gemm() - or runner_backend.is_flashinfer_trtllm() - or runner_backend.is_flashinfer_trtllm_routed() + runner_backend is not None + and ( + runner_backend.is_deep_gemm() + or runner_backend.is_flashinfer_trtllm() + or runner_backend.is_flashinfer_trtllm_routed() + ) ) and hidden_states.dtype != torch.bfloat16 ): diff --git a/python/sglang/srt/layers/moe/token_dispatcher/pplx.py b/python/sglang/srt/layers/moe/token_dispatcher/pplx.py index 6337bb9448..ecd423c51e 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/pplx.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/pplx.py @@ -198,6 +198,7 @@ class _PplxDispatcherImpl: hidden_size: int, params_dtype: torch.dtype, deepep_mode: DeepEPMode, + runner_backend=None, ): if not use_pplx: raise ImportError( @@ -208,6 +209,7 @@ class _PplxDispatcherImpl: "backend." ) + self.runner_backend = runner_backend self.group = group self.router_topk = router_topk self.permute_fusion = permute_fusion @@ -431,6 +433,7 @@ class PplxDispatcher(BaseDispatcher): deepep_mode: DeepEPMode = DeepEPMode.AUTO, async_finish: bool = False, return_recv_hook: bool = False, + runner_backend=None, ): super().__init__() @@ -442,6 +445,7 @@ class PplxDispatcher(BaseDispatcher): ) self._low_latency_dispatcher = _PplxDispatcherImpl( + runner_backend=runner_backend, group=group, router_topk=router_topk, permute_fusion=permute_fusion, diff --git a/python/sglang/srt/layers/moe/token_dispatcher/standard.py b/python/sglang/srt/layers/moe/token_dispatcher/standard.py index 7ae47fa9d2..9da5e46088 100644 --- a/python/sglang/srt/layers/moe/token_dispatcher/standard.py +++ b/python/sglang/srt/layers/moe/token_dispatcher/standard.py @@ -24,7 +24,6 @@ from sglang.srt.layers.moe.token_dispatcher.base import ( ) from sglang.srt.layers.moe.topk import StandardTopKOutput, TopKOutput, TopKOutputChecker from sglang.srt.layers.moe.utils import ( - get_moe_a2a_backend, get_moe_runner_backend, should_use_flashinfer_cutlass_moe_fp4_allgather, ) @@ -103,17 +102,11 @@ def _map_topk_ids_to_local_experts( ) -def _keeps_global_expert_ids(backend: MoeRunnerBackendLike) -> bool: - # These runners handle EP with global expert IDs: - # - cutlass / cutedsl / trtllm_routed handle EP internally - # - mxfp4 dispatcher mapping is already global - # - hpc_ops consumes global ids together with rank_ep / num_expert_total - # - flashinfer_megamoe routes by global expert ID inside the mega kernel - return ( +def _keeps_global_expert_ids(backend: Optional[MoeRunnerBackendLike]) -> bool: + return backend is not None and ( backend.is_flashinfer_cutlass() or backend.is_flashinfer_cutedsl() or backend.is_flashinfer_trtllm() - or backend.is_experimental_sgl_trtllm() or backend.is_flashinfer_trtllm_routed() or backend.is_hpc_ops() or backend.is_flashinfer_megamoe() @@ -122,42 +115,34 @@ def _keeps_global_expert_ids(backend: MoeRunnerBackendLike) -> bool: class StandardDispatcher(BaseDispatcher): - def __init__( - self, - moe_runner_config: MoeRunnerConfig, - runner_backend: Optional[MoeRunnerBackendLike] = None, - ): + def __init__(self, moe_runner_config: MoeRunnerConfig): super().__init__() self.moe_ep_size = get_parallel().moe_ep_size - backend = get_moe_runner_backend() - self.enable_flashinfer_cutlass_moe = backend.is_flashinfer_cutlass() - self.enable_flashinfer_mxfp4_moe = backend.is_flashinfer_mxfp4() - self.enable_flashinfer_trtllm_routed_moe = backend.is_flashinfer_trtllm_routed() - # AITER fast paths can be on while the MoE runner stays Triton; only the - # AITER runner keeps global expert IDs, so Triton must remap to local range. - self.use_aiter_moe_runner = backend.is_aiter() or ( - backend.is_auto() and _use_aiter and get_moe_a2a_backend().supports_aiter() + backend = moe_runner_config.runner_backend + self.runner_backend = backend + self.enable_flashinfer_cutlass_moe = ( + backend is not None and backend.is_flashinfer_cutlass() ) + self.enable_flashinfer_mxfp4_moe = ( + backend is not None and backend.is_flashinfer_mxfp4() + ) + self.enable_flashinfer_trtllm_routed_moe = ( + backend is not None and backend.is_flashinfer_trtllm_routed() + ) + self.use_aiter_moe_runner = backend is not None and backend.is_aiter() self.skip_local_expert_mapping = _keeps_global_expert_ids(backend) - # A built runner may enable local-ID mapping; all other flags above - # follow the requested backend. if ( - self.skip_local_expert_mapping - and runner_backend is not None - and not _keeps_global_expert_ids(runner_backend) + self.moe_ep_size > 1 + and ( + _keeps_global_expert_ids(get_moe_runner_backend()) + != self.skip_local_expert_mapping + ) + and (get_lora().enable_lora or get_lora().lora_paths) ): - # The LoRA pool keys experts by the global backend's convention, so - # it would read the wrong adapters for this layer under EP. - if self.moe_ep_size > 1 and ( - get_lora().enable_lora or get_lora().lora_paths - ): - raise ValueError( - "LoRA with expert parallelism is not supported on this MoE " - f"layer: --moe-runner-backend {backend.value} keeps global " - f"expert ids, but the layer's quantization method runs the " - f"'{runner_backend.value}' runner, which needs local ids." - ) - self.skip_local_expert_mapping = False + raise ValueError( + "LoRA with expert parallelism requires the layer's MoE runner " + "to use the requested runner's expert-ID convention." + ) self.num_experts = moe_runner_config.num_experts self.num_local_experts = moe_runner_config.num_local_experts self.num_local_shared_experts = moe_runner_config.num_fused_shared_experts @@ -172,7 +157,24 @@ class StandardDispatcher(BaseDispatcher): self, hidden_states: torch.Tensor, topk_output: TopKOutput ) -> StandardDispatchOutput: - if should_use_flashinfer_cutlass_moe_fp4_allgather(): + needs_dp_allgather = should_use_flashinfer_cutlass_moe_fp4_allgather() + use_fp4_payload = ( + needs_dp_allgather + and self.enable_flashinfer_cutlass_moe + and self.quant_config.get("input_global_scale") is not None + ) + if needs_dp_allgather and not use_fp4_payload: + if TopKOutputChecker.format_is_bypassed(topk_output): + topk_output = topk_output.to_standard() + topk_weights, topk_ids, hidden_states = get_parallel().tp_group.all_gatherv( + [topk_output.topk_weights, topk_output.topk_ids, hidden_states], + sizes=get_dp_global_num_tokens(), + ) + topk_output = StandardTopKOutput( + topk_weights, topk_ids, topk_output.router_logits + ) + hidden_states_scale = None + elif needs_dp_allgather: # all-gather fp4 hidden states if ( fp4_quantize_flashinfer is None diff --git a/python/sglang/srt/layers/moe/topk.py b/python/sglang/srt/layers/moe/topk.py index 52b5754bc5..5a3aa48ef1 100644 --- a/python/sglang/srt/layers/moe/topk.py +++ b/python/sglang/srt/layers/moe/topk.py @@ -103,10 +103,11 @@ from sglang.srt.eplb.expert_location_dispatch import ( from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe import ( get_moe_a2a_backend, - get_moe_runner_backend, is_moe_input_scattered_across_dp_ranks, ) from sglang.srt.layers.moe.utils import ( + MoeRunnerBackendLike, + RoutingMethodType, has_per_rank_fused_shared_slots, ) from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer @@ -268,6 +269,8 @@ class TopKConfig: # router's shared weights on the other standard path. fused_shared_experts_scaling_factor: Optional[float] = None output_format: Optional[TopKOutputFormat] = None + runner_backend: Optional[MoeRunnerBackendLike] = None + routing_method_type: Optional[RoutingMethodType] = None scoring_func: str = "softmax" # sqrtsoftplus through log1p with NaNs ranked first (DeepSeek-V4.1 routing). sqrtsoftplus_log1p: bool = False @@ -584,6 +587,46 @@ def _simulate_balanced_routing( ) +def _can_bypass_trtllm_routing(config: TopKConfig) -> bool: + """Match the native selector to the materialized router's policy.""" + method = config.routing_method_type + if ( + method is None + or method == RoutingMethodType.Unspecified + or config.sqrtsoftplus_log1p + or config.num_shared_topk_slots != 0 + ): + return False + plain = not config.use_grouped_topk and config.correction_bias is None + if method == RoutingMethodType.Llama4: + return ( + plain + and config.top_k == 1 + and not config.renormalize + and config.custom_routing_function is not None + ) + if config.custom_routing_function is not None: + return False + if method == RoutingMethodType.DeepSeekV3: + # The biased grouped router uses sigmoid regardless of scoring_func. + return ( + config.use_grouped_topk + and config.correction_bias is not None + and config.renormalize + ) + if not plain: + return False + if method in (RoutingMethodType.Renormalize, RoutingMethodType.RenormalizeNaive): + return config.scoring_func == "softmax" and config.renormalize + if method == RoutingMethodType.Default: + return config.scoring_func == "softmax" and not config.renormalize + if method == RoutingMethodType.SigmoidRenorm: + return config.scoring_func == "sigmoid" and config.renormalize + if method == RoutingMethodType.Sigmoid: + return config.scoring_func == "sigmoid" and not config.renormalize + return False + + # -------------------------------- TopK --------------------------------------- @@ -631,6 +674,7 @@ class TopK(BaseFusedOp): assert num_expert_group is not None and topk_group is not None self.layer_id = layer_id + self._runner_backend_bound = False shared_experts = get_model().shared_experts self.enable_waterfill = shared_experts.use_waterfill @@ -662,6 +706,46 @@ class TopK(BaseFusedOp): allow_routed_experts_capture=allow_routed_experts_capture, ) + def bind_runner_backend( + self, + runner_backend: Optional[MoeRunnerBackendLike], + *, + routing_method_type: Optional[RoutingMethodType] = None, + ) -> None: + """Associate the owning experts once; repeating the same association is safe.""" + if self._runner_backend_bound: + if ( + runner_backend != self.topk_config.runner_backend + or routing_method_type != self.topk_config.routing_method_type + ): + raise RuntimeError("TopK cannot bind a different MoE runner backend.") + return + self._runner_backend_bound = True + self.topk_config.runner_backend = runner_backend + self.topk_config.routing_method_type = routing_method_type + self.topk_config.fused_gate_packed_ids &= ( + runner_backend is not None and runner_backend.is_flashinfer_mxfp4() + ) + if self.topk_config.output_format is not None: + return + if runner_backend is not None and runner_backend.is_triton_kernels(): + output_format = TopKOutputFormat.TRITON_KERNEL + elif runner_backend is not None and ( + runner_backend.is_flashinfer_trtllm() + and _can_bypass_trtllm_routing(self.topk_config) + or runner_backend.is_flashinfer_mxfp4() + and not self.is_fp4_experts + ): + output_format = ( + TopKOutputFormat.STANDARD + if runner_backend.is_experimental_sgl_trtllm() + and get_lora().enable_lora + else TopKOutputFormat.BYPASSED + ) + else: + output_format = TopKOutputFormat.STANDARD + self.topk_config.output_format = output_format + def _apply_waterfill(self, topk_output: TopKOutput, num_tokens: int) -> TopKOutput: if self.enable_waterfill and self.waterfill_balancer is None: raise RuntimeError( @@ -725,26 +809,10 @@ class TopK(BaseFusedOp): output_format = TopKOutputFormat.STANDARD elif self.topk_config.output_format is not None: output_format = self.topk_config.output_format - elif get_moe_runner_backend().is_triton_kernels(): - output_format = TopKOutputFormat.TRITON_KERNEL - # ===== TO BE REFACTORED ==== - elif get_moe_runner_backend().is_experimental_sgl_trtllm(): - try: - use_standard_for_lora = bool(get_lora().enable_lora) - except ValueError: - use_standard_for_lora = False - output_format = ( - TopKOutputFormat.STANDARD - if use_standard_for_lora - else TopKOutputFormat.BYPASSED - ) - # ===== END TO BE REFACTORED ==== - elif get_moe_runner_backend().is_flashinfer_trtllm() or ( - get_moe_runner_backend().is_flashinfer_mxfp4() and not self.is_fp4_experts - ): - output_format = TopKOutputFormat.BYPASSED else: - output_format = TopKOutputFormat.STANDARD + raise RuntimeError( + "TopK requires the owning experts' runner backend or an explicit output format." + ) if ( router_logits_partials is not None @@ -1265,8 +1333,8 @@ def grouped_topk_gpu( ) if routed_scaling_factor is not None: topk_weights[:, -1] = ( - topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor - ) + topk_weights[:, :-1].sum(dim=-1) if renormalize else 1.0 + ) / routed_scaling_factor if renormalize: # fp32 like the reference gate (the epsilon is not representable in @@ -1278,8 +1346,8 @@ def grouped_topk_gpu( else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32) ) topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON) - if apply_routed_scaling_factor_on_output: - topk_weights *= routed_scaling_factor + if apply_routed_scaling_factor_on_output: + topk_weights *= routed_scaling_factor topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) @@ -1403,8 +1471,8 @@ def kimi_k2_biased_topk_impl( # the existing kernels, so no extra cast launch is needed topk_weights_sum = topk_weights.sum(dim=-1, keepdim=True, dtype=torch.float32) topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON) - if apply_routed_scaling_factor_on_output: - topk_weights *= routed_scaling_factor + if apply_routed_scaling_factor_on_output: + topk_weights *= routed_scaling_factor topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) return topk_weights, topk_ids @@ -1455,8 +1523,8 @@ def biased_topk_impl( ) if routed_scaling_factor is not None: topk_weights[:, -1] = ( - topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor - ) + topk_weights[:, :-1].sum(dim=-1) if renormalize else 1.0 + ) / routed_scaling_factor if renormalize: # fp32 like the reference gate (the epsilon is not representable in @@ -1468,8 +1536,8 @@ def biased_topk_impl( else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32) ) topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON) - if apply_routed_scaling_factor_on_output: - topk_weights *= routed_scaling_factor + if apply_routed_scaling_factor_on_output: + topk_weights *= routed_scaling_factor topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32) return topk_weights, topk_ids @@ -1735,7 +1803,6 @@ def _fused_gate_emits_packed_ids( return ( enabled and _fused_gate_masks_padded_rows(scoring_func) - and get_moe_runner_backend().is_flashinfer_mxfp4() and expert_location_dispatch_info is None and num_shared_topk_slots == 0 and not routing_overridden @@ -2623,6 +2690,7 @@ def select_experts( ): correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype) + fold_scale_after_routing = False num_routed_topk = top_k - num_shared_topk_slots # On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends # the shared columns, so the router selects and renormalizes routed experts only. @@ -2686,7 +2754,7 @@ def select_experts( "num_token_non_padded is not yet supported in fused_topk_native" ) assert expert_location_dispatch_info is None - assert not apply_routed_scaling_factor_on_output, "Not implemented" + fold_scale_after_routing = True topk_weights, topk_ids = fused_topk_native( hidden_states=hidden_states, gating_output=router_logits, @@ -2696,8 +2764,7 @@ def select_experts( scoring_func=scoring_func, ) elif custom_routing_function is None: - if scoring_func not in ("sqrtsoftplus", "sigmoid"): - assert not apply_routed_scaling_factor_on_output, "Not implemented" + fold_scale_after_routing = scoring_func not in ("sqrtsoftplus", "sigmoid") # The JIT route depends on GPU-only topk_sigmoid/topk_softmax imports _can_use_jit_kernel = not _is_cpu @@ -2747,7 +2814,8 @@ def select_experts( ) padded_rows_masked = _fused_gate_masks_padded_rows(scoring_func) elif ( - get_moe_runner_backend().is_flashinfer_trtllm_routed() + topk_config.runner_backend is not None + and topk_config.runner_backend.is_flashinfer_trtllm_routed() and scoring_func == "softmax" and correction_bias is None ): @@ -2775,6 +2843,10 @@ def select_experts( and expert_location_dispatch_info is None and num_shared_topk_slots == 0 and not routing_overridden + and not ( + apply_routed_scaling_factor_on_output + and routed_scaling_factor not in (None, 1.0) + ) ): num_experts = router_logits.shape[-1] if num_experts & (num_experts - 1) == 0 and num_experts <= 512: @@ -2804,7 +2876,10 @@ def select_experts( scoring_func=scoring_func, num_fused_shared_experts=num_shared_topk_slots_for_gate, routed_scaling_factor=routed_scaling_factor, - apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output, + apply_routed_scaling_factor_on_output=( + apply_routed_scaling_factor_on_output + and not fold_scale_after_routing + ), **_fused_topk_kwargs, ) else: @@ -2812,7 +2887,7 @@ def select_experts( # padded rows is garbage. That is fine because _post_process_topk_ids # below masks rows >= num_token_non_padded (-1 on CUDA, 0 + zeroed # weights on HIP) after the logical->physical remap. - assert not apply_routed_scaling_factor_on_output, "Not implemented" + fold_scale_after_routing = True topk_weights, topk_ids = custom_routing_function( hidden_states=hidden_states, gating_output=router_logits, @@ -2820,6 +2895,15 @@ def select_experts( renormalize=renormalize, ) + if ( + fold_scale_after_routing + and apply_routed_scaling_factor_on_output + and routed_scaling_factor is not None + ): + # Shared columns keep their own weight; only routed experts take this scale. + num_routed_for_gate = top_k_for_gate - num_shared_topk_slots_for_gate + topk_weights[:, :num_routed_for_gate] *= routed_scaling_factor + if routing_overridden: # Benchmark-only: override gating with a balanced expert assignment (so # dummy/random benchmark tokens don't skew MoE load) via a single fused diff --git a/python/sglang/srt/layers/moe/utils.py b/python/sglang/srt/layers/moe/utils.py index f7e346b189..abea470255 100644 --- a/python/sglang/srt/layers/moe/utils.py +++ b/python/sglang/srt/layers/moe/utils.py @@ -379,11 +379,9 @@ def get_deepep_output_dtype(self) -> DispatcherOutputDtype: if dispatcher_output_dtype is not None: return DispatcherOutputDtype(dispatcher_output_dtype) - # 5. flashinfer_cutedsl / cutlass / humming expects BF16 dispatch - if ( - get_moe_runner_backend().is_flashinfer_cutedsl() - or get_moe_runner_backend().is_cutlass() - or get_moe_runner_backend().is_humming() + backend = self.runner_backend + if backend is not None and ( + backend.is_flashinfer_cutedsl() or backend.is_cutlass() or backend.is_humming() ): return DispatcherOutputDtype.BF16 diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py index 24f1e1ccd1..787a10dfac 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_bf16_moe.py @@ -22,6 +22,7 @@ from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import ( fused_post_expert_norm_weighted_sum, ) +from sglang.srt.layers.moe import MoeRunnerBackend from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import ( finalize_with_post_expert_norm, @@ -110,6 +111,7 @@ class MuseSpark1xBf16Experts(nn.Module): self.moe_runner_config = MoeRunnerConfig( num_experts=num_experts, num_local_experts=num_local_experts, + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, hidden_size=expert_hidden_size, intermediate_size_per_partition=intermediate_size, top_k=top_k, diff --git a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py index f4a564ac23..cd7beaf8bb 100644 --- a/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py +++ b/python/sglang/srt/layers/muse_spark/muse_spark_fp4_moe.py @@ -108,11 +108,13 @@ class MuseSpark1xFp4Experts(nn.Module): self.top_k = top_k self.params_dtype = params_dtype or torch.get_default_dtype() self.quant_config = quant_config + self.moe_runner_backend = _resolve_moe_runner_backend() # SwiGLU, and no routed_scaling here -- Muse folds its route_scale into # the topk weights before they reach us. self.moe_runner_config = MoeRunnerConfig( num_experts=num_experts, num_local_experts=num_local_experts, + runner_backend=self.moe_runner_backend, hidden_size=hidden_size, intermediate_size_per_partition=intermediate_size, top_k=top_k, @@ -127,7 +129,6 @@ class MuseSpark1xFp4Experts(nn.Module): self.dispatcher = _DirectDispatcher() capability = get_device_capability() if is_cuda() else None - self.moe_runner_backend = _resolve_moe_runner_backend() self.use_marlin = self.moe_runner_backend.is_marlin() self._sm_major = capability[0] if capability is not None else 0 if not self.use_marlin and capability is not None and capability < (10, 0): diff --git a/python/sglang/srt/layers/quantization/awq/awq.py b/python/sglang/srt/layers/quantization/awq/awq.py index 6199ae8f1c..c5d4828520 100644 --- a/python/sglang/srt/layers/quantization/awq/awq.py +++ b/python/sglang/srt/layers/quantization/awq/awq.py @@ -500,6 +500,9 @@ class AWQMoEMethod(FusedMoEMethodBase): ): layer.scheme.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return layer.scheme.get_moe_runner_backend(layer) + def apply( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/quantization/awq/schemes/awq_cpu.py b/python/sglang/srt/layers/quantization/awq/schemes/awq_cpu.py index 0965a76b47..e8b24f8541 100644 --- a/python/sglang/srt/layers/quantization/awq/schemes/awq_cpu.py +++ b/python/sglang/srt/layers/quantization/awq/schemes/awq_cpu.py @@ -38,3 +38,6 @@ class AWQIntelAMXMoEScheme(AWQMoEScheme): ): self.moe_runner_config = moe_runner_config self.kernel.create_moe_runner(layer, moe_runner_config) + + def get_moe_runner_backend(self, layer): + return None diff --git a/python/sglang/srt/layers/quantization/awq/schemes/awq_moe.py b/python/sglang/srt/layers/quantization/awq/schemes/awq_moe.py index 92bb0cdf5a..5d523cd6d2 100644 --- a/python/sglang/srt/layers/quantization/awq/schemes/awq_moe.py +++ b/python/sglang/srt/layers/quantization/awq/schemes/awq_moe.py @@ -133,6 +133,9 @@ class AWQMoEScheme(AWQMoESchemeBase): self.moe_runner_config = moe_runner_config self.kernel.runner = MoeRunner(MoeRunnerBackend.MARLIN, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.kernel.runner.runner_backend + def apply_weights( self, layer: torch.nn.Module, @@ -168,6 +171,9 @@ class AWQAscendMoEScheme(AWQMoEScheme): backend = MoeRunnerBackend.ASCEND self.runner = MoeRunner(backend, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.runner.runner_backend + def apply_weights( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/quantization/base_config.py b/python/sglang/srt/layers/quantization/base_config.py index a080d0c19d..b81b37ecb0 100644 --- a/python/sglang/srt/layers/quantization/base_config.py +++ b/python/sglang/srt/layers/quantization/base_config.py @@ -113,6 +113,16 @@ class FusedMoEMethodBase(QuantizeMethodBase): ): raise NotImplementedError + def get_moe_runner_backend( + self, layer: torch.nn.Module + ) -> MoeRunnerBackendLike | None: + """Return the execution contract after create_moe_runner. + + None declares a direct kernel consuming standard TopK with local IDs. + Override when apply uses a different path from the owned runner. + """ + return self.runner.runner_backend if self.runner is not None else None + @abstractmethod def apply( self, diff --git a/python/sglang/srt/layers/quantization/base_scheme.py b/python/sglang/srt/layers/quantization/base_scheme.py index 2e62d6c261..2847d09f6e 100644 --- a/python/sglang/srt/layers/quantization/base_scheme.py +++ b/python/sglang/srt/layers/quantization/base_scheme.py @@ -59,6 +59,12 @@ class BaseMoEScheme(ABC): of different quantization schemes. """ + runner = None + + def get_moe_runner_backend(self, layer): + """Return the runner contract, or None for a direct standard/local-ID kernel.""" + return self.runner.runner_backend if self.runner is not None else None + @abstractmethod def create_weights(self, *args, **kwargs): """ diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py index fbb155ec91..4aea9944f0 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/compressed_tensors.py @@ -1378,6 +1378,9 @@ class CompressedTensorsFusedMoEMethod(FusedMoEMethodBase): ): return layer.scheme.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return layer.scheme.get_moe_runner_backend(layer) + def get_triton_quant_info(self, layer: torch.nn.Module): return layer.scheme.get_triton_quant_info(layer) diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py index d24b3b2223..219fe4699a 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_mxint4_moe.py @@ -11,7 +11,11 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( ) from sglang.srt.layers.dp_attention import is_allocation_symmetric from sglang.srt.layers.moe import MoeRunnerConfig -from sglang.srt.layers.moe.utils import RoutingMethodType, get_moe_runner_backend +from sglang.srt.layers.moe.utils import ( + MoeRunnerBackend, + RoutingMethodType, + get_moe_runner_backend, +) from sglang.srt.layers.quantization.compressed_tensors.schemes import ( CompressedTensorsMoEScheme, ) @@ -292,6 +296,9 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme): ): self.moe_runner_config = moe_runner_config + def get_moe_runner_backend(self, layer): + return MoeRunnerBackend.FLASHINFER_TRTLLM + def apply_weights( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index 27096041a3..c4560909cd 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -1436,6 +1436,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): # The MxFP4 wrapper methods borrow this instance for weight loading; # they never call create_moe_runner, so moe_runner_config is unset. self._owns_moe_runner = False + self._moe_runner_backend = None if get_moe_runner_backend().is_cutlass(): assert cutlass_fp8_supported(), ( "cutlass_fp8 MoE requires CUDA 12.0+ with SM90 or CUDA 12.4+ with SM89" @@ -2063,6 +2064,9 @@ class Fp8MoEMethod(FusedMoEMethodBase): ) return + runner_backend = self.get_moe_runner_backend(layer) + runner_is_aiter = runner_backend is not None and runner_backend.is_aiter() + # If ROCm, normalize the weights and scales to e4m3fnuz if _is_fp8_fnuz: # activation_scheme: dynamic @@ -2087,7 +2091,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): w2_weight_scale, requires_grad=False ) layer.w2_input_scale = None - if _use_aiter: + if _use_aiter and runner_is_aiter: layer.w13_weight.data = shuffle_weight( layer.w13_weight.contiguous(), (16, 16) ) @@ -2098,16 +2102,17 @@ class Fp8MoEMethod(FusedMoEMethodBase): layer.w2_weight.is_shuffled = True layer._aiter_gate_up_interleaved = False elif _use_aiter: - # Pre-shuffle weights - t = shuffle_weight(layer.w13_weight, (16, 16)) - layer.w13_weight.copy_(t) - del t - t = shuffle_weight(layer.w2_weight, (16, 16)) - layer.w2_weight.copy_(t) - del t - layer.w13_weight.is_shuffled = True - layer.w2_weight.is_shuffled = True - layer._aiter_gate_up_interleaved = False + if runner_is_aiter: + # Pre-shuffle weights + t = shuffle_weight(layer.w13_weight, (16, 16)) + layer.w13_weight.copy_(t) + del t + t = shuffle_weight(layer.w2_weight, (16, 16)) + layer.w2_weight.copy_(t) + del t + layer.w13_weight.is_shuffled = True + layer.w2_weight.is_shuffled = True + layer._aiter_gate_up_interleaved = False elif _is_cpu: assert _is_cpu_amx_available, ( "Fp8MoEMethod on CPU requires that CPU has AMX support" @@ -2913,8 +2918,13 @@ class Fp8MoEMethod(FusedMoEMethodBase): self.runner = MoeRunner(moe_runner_backend, moe_runner_config) self._owns_moe_runner = True else: - # TODO(cwan): refactor other backends - pass + # Direct kernels declare the layout they consume. + self._moe_runner_backend = moe_runner_backend + + def get_moe_runner_backend(self, layer): + if self.runner is not None: + return self.runner.runner_backend + return self._moe_runner_backend def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo: use_rocm_mxfp8 = self.use_mxfp8 and _is_hip and _is_gfx95_supported @@ -3208,6 +3218,10 @@ class Fp8MoEMethod(FusedMoEMethodBase): is_gated=self.moe_runner_config.is_gated, ) + routing_method_type = layer.routing_method_type + if routing_method_type is None: + routing_method_type = RoutingMethodType.DeepSeekV3 + quant_info = FlashInferTrtllmFp8MoeQuantInfo( w13_weight=layer.w13_weight, w2_weight=layer.w2_weight, @@ -3215,10 +3229,7 @@ class Fp8MoEMethod(FusedMoEMethodBase): local_expert_offset=moe_ep_rank * num_local_experts, local_num_experts=num_local_experts, intermediate_size=layer.w2_weight.shape[2], - routing_method_type=int( - getattr(layer, "routing_method_type", None) - or RoutingMethodType.DeepSeekV3 - ), + routing_method_type=int(routing_method_type), block_quant=self.block_quant, use_mxfp8=getattr(self.quant_config, "use_mxfp8", False), weight_block_k=( diff --git a/python/sglang/srt/layers/quantization/gptq/gptq.py b/python/sglang/srt/layers/quantization/gptq/gptq.py index d53f628448..0797d872fa 100644 --- a/python/sglang/srt/layers/quantization/gptq/gptq.py +++ b/python/sglang/srt/layers/quantization/gptq/gptq.py @@ -559,6 +559,9 @@ class GPTQMoEMethod(FusedMoEMethodBase): ): layer.scheme.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return layer.scheme.get_moe_runner_backend(layer) + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: layer.scheme.process_weights_after_loading(layer) @@ -651,6 +654,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase): ): layer.scheme.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return layer.scheme.get_moe_runner_backend(layer) + def apply( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/quantization/gptq/schemes/gptq_cpu.py b/python/sglang/srt/layers/quantization/gptq/schemes/gptq_cpu.py index f3ce556004..8423b54df3 100644 --- a/python/sglang/srt/layers/quantization/gptq/schemes/gptq_cpu.py +++ b/python/sglang/srt/layers/quantization/gptq/schemes/gptq_cpu.py @@ -274,6 +274,9 @@ class GPTQIntelAMXMoEScheme(GPTQMoESchemeBase): ): self.kernel.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return None + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/python/sglang/srt/layers/quantization/gptq/schemes/gptq_moe.py b/python/sglang/srt/layers/quantization/gptq/schemes/gptq_moe.py index d3481cb5e4..ba8b8267fe 100644 --- a/python/sglang/srt/layers/quantization/gptq/schemes/gptq_moe.py +++ b/python/sglang/srt/layers/quantization/gptq/schemes/gptq_moe.py @@ -325,6 +325,9 @@ class GPTQMarlinMoEScheme(GPTQMoESchemeBase): ): self.kernel.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.kernel.runner.runner_backend + def process_weights_after_loading(self, layer: torch.nn.Module) -> None: self.kernel.process_weights_after_loading(layer) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index b12086b86b..8774bd15a8 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -23,6 +23,7 @@ from sglang.srt.layers.moe import ( from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo from sglang.srt.layers.moe.utils import ( FlashinferA2ADispatchType, + RoutingMethodType, get_flashinfer_a2a_dispatch_type, is_flashinfer_cutedsl_v1_path, should_use_flashinfer_cutlass_moe_fp4_allgather, @@ -1255,14 +1256,14 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): ) # Align FP8 weights to FlashInfer per-tensor kernel layout if enabled - if get_moe_runner_backend().is_flashinfer_trtllm(): + if self._moe_runner_backend.is_flashinfer_trtllm(): from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( align_fp8_moe_weights_for_flashinfer_trtllm, ) # ModelOpt FP8 stores weights in [Up, Gate] order, so we need to swap align_fp8_moe_weights_for_flashinfer_trtllm(layer, swap_w13_halves=True) - elif get_moe_runner_backend().is_flashinfer_cutlass(): + elif self._moe_runner_backend.is_flashinfer_cutlass(): assert ( hasattr(layer, "w13_input_scale") and layer.w13_input_scale is not None ) @@ -1352,6 +1353,18 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): ) else: self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config) + # This per-tensor TRT-LLM adapter only supports logits routing. An + # undeclared router keeps Triton's weight layout and local expert IDs. + self._moe_runner_backend = ( + moe_runner_backend + if moe_runner_backend.is_flashinfer_trtllm() + and moe_runner_config.routing_method_type + not in (None, RoutingMethodType.Unspecified) + else self.runner.runner_backend + ) + + def get_moe_runner_backend(self, layer): + return self._moe_runner_backend def apply( self, @@ -1362,10 +1375,18 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): topk_output = dispatch_output.topk_output from sglang.srt.layers.moe.topk import TopKOutputChecker + if self._moe_runner_backend.is_flashinfer_trtllm() and not ( + TopKOutputChecker.format_is_bypassed(topk_output) + ): + raise NotImplementedError( + "TRT-LLM per-tensor FP8 cannot consume materialized routing. " + "Use --moe-runner-backend triton for this router." + ) + # Fast path: TRT-LLM FP8 per-tensor MoE using BYPASSED TopK routing if ( - get_moe_runner_backend().is_flashinfer_trtllm() + self._moe_runner_backend.is_flashinfer_trtllm() and TopKOutputChecker.format_is_bypassed(topk_output) ): from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( @@ -1409,7 +1430,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): dispatch_output, quant_info, self.moe_runner_config ) - if get_moe_runner_backend().is_flashinfer_cutlass(): + if self._moe_runner_backend.is_flashinfer_cutlass(): activation_str = self.moe_runner_config.activation assert activation_str in _SUPPORTED_ACT_STRS, ( f"Activation {activation_str!r} is not supported for " @@ -3114,6 +3135,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): self.runner = MoeRunner(moe_runner_backend, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self._moe_runner_backend + def get_marlin_quant_info(self, layer: torch.nn.Module): """Marlin payload for the fp4-marlin (W4A16) fallback; the weights were repacked by prepare_moe_nvfp4_layer_for_marlin. Also consumed by diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py index b8e6ff0810..96300858ac 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_cutlass_moe.py @@ -124,6 +124,9 @@ class Mxfp4FlashinferCutlassMoEMethod: self.runner = MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.runner.runner_backend + def process_weights_after_loading(self, layer: Module) -> None: # Preserve the base FP4 post-load handling. self._fp8.process_weights_after_loading(layer) diff --git a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py index d100e78a0b..4d873c36ae 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_flashinfer_trtllm_moe.py @@ -11,7 +11,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import ( use_symmetric_memory, ) from sglang.srt.layers.dp_attention import is_allocation_symmetric -from sglang.srt.layers.moe.utils import RoutingMethodType +from sglang.srt.layers.moe.utils import MoeRunnerBackend, RoutingMethodType from sglang.srt.runtime_context import ( get_exec, get_parallel, @@ -164,6 +164,9 @@ class Mxfp4FlashinferTrtllmMoEMethod: else None ) + def get_moe_runner_backend(self, layer): + return MoeRunnerBackend.FLASHINFER_MXFP4 + def create_weights( self, layer, diff --git a/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py b/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py index 736d1109b1..e34a1acd7a 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_humming_moe.py @@ -35,6 +35,9 @@ class Mxfp4HummingMoEMethod: self.runner = MoeRunner(MoeRunnerBackend.HUMMING, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.runner.runner_backend + def create_weights( self, layer: Module, diff --git a/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py b/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py index b66a2f465d..43450d96dc 100644 --- a/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py +++ b/python/sglang/srt/layers/quantization/mxfp4_marlin_moe.py @@ -56,6 +56,9 @@ class Mxfp4MarlinMoEMethod: self.runner = MoeRunner(MoeRunnerBackend.MARLIN, moe_runner_config) + def get_moe_runner_backend(self, layer): + return self.runner.runner_backend + def create_weights( self, layer: Module, diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index 19effc3dbd..519781b8d2 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -1180,6 +1180,9 @@ class QuarkFusedMoEMethod(FusedMoEMethodBase): ): layer.scheme.create_moe_runner(layer, moe_runner_config) + def get_moe_runner_backend(self, layer): + return layer.scheme.get_moe_runner_backend(layer) + def apply( self, layer: torch.nn.Module, diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index f46c88ea48..fc5fcdbafa 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -749,6 +749,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): self.use_flashinfer_trtllm_moe = use_flashinfer_trtllm_moe self.use_deep_gemm = use_deep_gemm self._cache_permute_indices = dict({}) + self.runner: Optional[MoeRunner] = None + self._aiter_runner: Optional[MoeRunner] = None # Set by process_weights_after_loading when w13 rows are permuted to # interleave gate/up for the fused swiglu up-GEMM epilogue. self.w13_swiglu_interleaved = False @@ -982,10 +984,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): return moe_runner_config = layer.moe_runner_config + runner_backend = self.get_moe_runner_backend(layer) if not ( _is_cuda - and self._aiter_runner is None - and self.runner.runner_backend.is_triton() + and runner_backend is not None + and runner_backend.is_triton() and get_moe_a2a_backend().is_none() and not self.with_bias and layer.w13_weight.dtype == torch.bfloat16 @@ -1029,7 +1032,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): block layout. During weight update, checkpoint tensors are in canonical layout and need a temporary shape restore for copy. """ - if not get_moe_runner_backend().is_flashinfer_trtllm_routed(): + runner_backend = self.get_moe_runner_backend(layer) + if runner_backend is None or not runner_backend.is_flashinfer_trtllm_routed(): return expected_shape = None @@ -1139,6 +1143,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): "using triton MoE runner." ) + def get_moe_runner_backend(self, layer): + runner = self._aiter_runner if self._aiter_runner is not None else self.runner + return runner.runner_backend if runner is not None else None + @property def load_up_proj_weight_first(self) -> bool: # FlashInfer CUTLASS kernel assumes [Up, Gate] Proj as W13 diff --git a/python/sglang/srt/models/afmoe.py b/python/sglang/srt/models/afmoe.py index 0efd7cd11b..98ca356d1c 100644 --- a/python/sglang/srt/models/afmoe.py +++ b/python/sglang/srt/models/afmoe.py @@ -46,7 +46,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -250,6 +250,7 @@ class AfmoeMoE(nn.Module): if _is_npu else {} ), + output_format=TopKOutputFormat.STANDARD, ) def pack_params(self) -> None: diff --git a/python/sglang/srt/models/bailing_moe.py b/python/sglang/srt/models/bailing_moe.py index fef8818a9d..6c3857a0d7 100644 --- a/python/sglang/srt/models/bailing_moe.py +++ b/python/sglang/srt/models/bailing_moe.py @@ -61,7 +61,10 @@ from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.token_dispatcher import DeepEPDispatcher from sglang.srt.layers.moe.topk import TopK -from sglang.srt.layers.moe.utils import filter_moe_weight_param_global_expert +from sglang.srt.layers.moe.utils import ( + RoutingMethodType, + filter_moe_weight_param_global_expert, +) from sglang.srt.layers.multi_gate import ( create_multi_gate_mm_indices, multi_gate_triton_kernel, @@ -300,6 +303,13 @@ class BailingMoESparseMoeBlock(nn.Module): routed_scaling_factor=self.routed_scaling_factor, ) + routing_method_type = None + if self.norm_topk_prob: + if self.use_grouped_topk and self.correction_bias is not None: + routing_method_type = RoutingMethodType.DeepSeekV3 + elif not self.use_grouped_topk and self.correction_bias is None: + routing_method_type = RoutingMethodType.RenormalizeNaive + self.experts = get_moe_impl_class(quant_config)( num_experts=self.num_experts, top_k=self.top_k, @@ -308,8 +318,16 @@ class BailingMoESparseMoeBlock(nn.Module): intermediate_size=config.moe_intermediate_size, quant_config=quant_config, routed_scaling_factor=self.routed_scaling_factor, + routing_method_type=routing_method_type, prefix=add_prefix("experts", prefix), ) + self.topk.topk_config.apply_routed_scaling_factor_on_output = ( + self.experts.should_fuse_routed_scaling_factor_in_topk + ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) # shared expert if config.num_shared_experts is not None: if hasattr(config, "moe_shared_expert_intermediate_size"): diff --git a/python/sglang/srt/models/bailing_moe_linear.py b/python/sglang/srt/models/bailing_moe_linear.py index 9b6e0973e4..0c141da70e 100644 --- a/python/sglang/srt/models/bailing_moe_linear.py +++ b/python/sglang/srt/models/bailing_moe_linear.py @@ -97,17 +97,6 @@ _use_aiter_gfx95 = _use_aiter and _is_gfx95_supported if _use_aiter_gfx95: pass -if _is_cuda: - from sglang.kernels.ops.quantization.awq_dequantize import awq_dequantize -elif _is_cpu and _is_cpu_amx_available: - pass -elif _is_hip: - from sglang.kernels.ops.quantization.awq_triton import ( - awq_dequantize_triton as awq_dequantize, - ) -else: - from vllm._custom_ops import awq_dequantize - if _is_hip: pass @@ -318,6 +307,10 @@ class BailingMoE(nn.Module): routed_scaling_factor=self.routed_scaling_factor, prefix=f"{prefix}.experts", ) + self.topk.topk_config.apply_routed_scaling_factor_on_output = ( + self.experts.should_fuse_routed_scaling_factor_in_topk + ) + self.topk.bind_runner_backend(self.experts.runner_backend) if self.num_shared_experts > 0: intermediate_size = self.intermediate_size * self.num_shared_experts @@ -1109,6 +1102,19 @@ class BailingMoELinearForCausalLM(nn.Module): if not hasattr(self_attn, "kv_b_proj"): continue if hasattr(self_attn.kv_b_proj, "qweight"): + if _is_cuda: + from sglang.kernels.ops.quantization.awq_dequantize import ( + awq_dequantize, + ) + elif _is_cpu and _is_cpu_amx_available: + pass + elif _is_hip: + from sglang.kernels.ops.quantization.awq_triton import ( + awq_dequantize_triton as awq_dequantize, + ) + else: + from vllm._custom_ops import awq_dequantize + # AWQ compatible if _is_cuda or _is_hip: w = awq_dequantize( diff --git a/python/sglang/srt/models/bailing_moe_v3.py b/python/sglang/srt/models/bailing_moe_v3.py index dfc4231688..6002c8ac74 100644 --- a/python/sglang/srt/models/bailing_moe_v3.py +++ b/python/sglang/srt/models/bailing_moe_v3.py @@ -598,6 +598,7 @@ class BailingMoE(nn.Module): and getattr(quant_config, "is_fp4_experts", False) ), ) + self.topk.bind_runner_backend(self.experts.runner_backend) # Whether to apply routed_scaling_factor at model layer. # For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it, diff --git a/python/sglang/srt/models/cohere2_moe.py b/python/sglang/srt/models/cohere2_moe.py index 10c4282432..4af0597175 100644 --- a/python/sglang/srt/models/cohere2_moe.py +++ b/python/sglang/srt/models/cohere2_moe.py @@ -248,7 +248,9 @@ class Cohere2MoeSparseMoeBlock(nn.Module): self.norm_topk_prob = getattr(config, "norm_topk_prob", True) if self.expert_selection_fn == "sigmoid": - custom_routing_function = cohere2_sigmoid_topk + custom_routing_function = ( + None if self.norm_topk_prob else cohere2_sigmoid_topk + ) scoring_func = "sigmoid" routing_method_type = ( RoutingMethodType.SigmoidRenorm @@ -293,6 +295,10 @@ class Cohere2MoeSparseMoeBlock(nn.Module): # The decoder reads this buffer on another stream, so in-place races it. inplace=False, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) num_shared_experts = getattr(config, "num_shared_experts", 0) self.num_shared_experts = num_shared_experts diff --git a/python/sglang/srt/models/dbrx.py b/python/sglang/srt/models/dbrx.py index bc8c7ef8bd..9998030aee 100644 --- a/python/sglang/srt/models/dbrx.py +++ b/python/sglang/srt/models/dbrx.py @@ -38,7 +38,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -118,6 +118,7 @@ class DbrxExperts(nn.Module): self.top_k, layer_id=layer_id, renormalize=True, + output_format=TopKOutputFormat.STANDARD, ) self.moe_runner_config = MoeRunnerConfig(inplace=True) self.ws = nn.Parameter( diff --git a/python/sglang/srt/models/deepseek.py b/python/sglang/srt/models/deepseek.py index 67ac59ba2e..9dbc720e4b 100644 --- a/python/sglang/srt/models/deepseek.py +++ b/python/sglang/srt/models/deepseek.py @@ -37,7 +37,7 @@ from sglang.srt.layers.linear import ( ) from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -127,6 +127,7 @@ class DeepseekMoE(nn.Module): top_k=self.top_k, layer_id=layer_id, renormalize=config.norm_topk_prob, + output_format=TopKOutputFormat.STANDARD, ) self.experts = nn.ModuleList( [ diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index adbb76c4a2..bf2d2ace7e 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -827,6 +827,10 @@ class DeepseekV2MoE(nn.Module): ), ) self.topk = TopK(**topk_kwargs) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.routed_scaling_factor_fused_in_topk = ( is_routed_scaling_factor_fused_in_topk(self.topk.topk_config) ) @@ -1030,6 +1034,7 @@ class DeepseekV2MoE(nn.Module): and hidden_states.shape[0] > 0 and hasattr(self, "shared_experts") and getattr(self.experts, "use_flashinfer_trtllm_moe", False) + and self.topk.topk_config.output_format == TopKOutputFormat.BYPASSED and not self._enable_a2a_moe and not self._fuse_shared_experts_inside_sbo and not getattr(self, "is_hash", False) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index 04821464c4..4895e6b878 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -322,6 +322,7 @@ class Dots3MoE(nn.Module): routed_scaling_factor=self.routed_scaling_factor, apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk, ) + self.topk.bind_runner_backend(self.experts.runner_backend) self.shared_experts = None if config.n_shared_experts is not None and self.num_fused_shared_experts == 0: diff --git a/python/sglang/srt/models/ernie4.py b/python/sglang/srt/models/ernie4.py index 5d30b1e77e..5486d83824 100644 --- a/python/sglang/srt/models/ernie4.py +++ b/python/sglang/srt/models/ernie4.py @@ -110,6 +110,7 @@ class Ernie4Moe(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) if self.moe_num_shared_experts > 0: intermediate_size = ( diff --git a/python/sglang/srt/models/ernie45_moe_vl.py b/python/sglang/srt/models/ernie45_moe_vl.py index 29761990b4..6511561b17 100644 --- a/python/sglang/srt/models/ernie45_moe_vl.py +++ b/python/sglang/srt/models/ernie45_moe_vl.py @@ -224,6 +224,7 @@ class Ernie4_5_VLMoeMoE(nn.Module): quant_config=quant_config, prefix=add_prefix("text_experts", prefix), ) + self.text_experts_topk.bind_runner_backend(self.text_experts.runner_backend) assert vision_moe_layer_start_index <= vision_moe_layer_end_index if ( @@ -256,6 +257,9 @@ class Ernie4_5_VLMoeMoE(nn.Module): quant_config=quant_config, prefix=add_prefix("vision_experts", prefix), ) + self.vision_experts_topk.bind_runner_backend( + self.vision_experts.runner_backend + ) if self.moe_num_shared_experts > 0: intermediate_size = ( diff --git a/python/sglang/srt/models/exaone_moe.py b/python/sglang/srt/models/exaone_moe.py index 4d759482ae..a3bde21cb0 100755 --- a/python/sglang/srt/models/exaone_moe.py +++ b/python/sglang/srt/models/exaone_moe.py @@ -182,6 +182,10 @@ class ExaoneMoESparseMoEBlock(nn.Module): apply_routed_scaling_factor_on_output=True, scoring_func="sigmoid", ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) if config.num_shared_experts is not None: intermediate_size = config.moe_intermediate_size * config.num_shared_experts diff --git a/python/sglang/srt/models/gemma4_causal.py b/python/sglang/srt/models/gemma4_causal.py index f0983560b9..9874bcb8b2 100644 --- a/python/sglang/srt/models/gemma4_causal.py +++ b/python/sglang/srt/models/gemma4_causal.py @@ -295,6 +295,7 @@ class Gemma4MoE(nn.Module): activation=activation, reduce_results=reduce_results, ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward( self, hidden_states: torch.Tensor, router_logits: torch.Tensor diff --git a/python/sglang/srt/models/glm4_moe.py b/python/sglang/srt/models/glm4_moe.py index e54a6894ab..234985fa14 100644 --- a/python/sglang/srt/models/glm4_moe.py +++ b/python/sglang/srt/models/glm4_moe.py @@ -461,6 +461,10 @@ class Glm4MoeSparseMoeBlock(nn.Module): ), fused_shared_experts_scaling_factor=1, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.shared_experts_is_int8 = False self.shared_experts_is_fp8 = False diff --git a/python/sglang/srt/models/glm4_moe_lite.py b/python/sglang/srt/models/glm4_moe_lite.py index c8f026f399..a23bbf6b38 100644 --- a/python/sglang/srt/models/glm4_moe_lite.py +++ b/python/sglang/srt/models/glm4_moe_lite.py @@ -237,6 +237,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module): output_format=TopKOutputFormat.STANDARD if quant_config is None else None, **({"scoring_func": "sigmoid"} if _is_npu else {}), ) + self.topk.bind_runner_backend(self.experts.runner_backend) self.shared_experts_is_int8 = False self.shared_experts_is_fp8 = False diff --git a/python/sglang/srt/models/gpt_oss.py b/python/sglang/srt/models/gpt_oss.py index 84ae3fa328..daed28916a 100644 --- a/python/sglang/srt/models/gpt_oss.py +++ b/python/sglang/srt/models/gpt_oss.py @@ -252,6 +252,7 @@ class GptOssSparseMoeBlock(nn.Module): prefix=add_prefix("experts", prefix), **extra_kwargs, ) + self.topk.bind_runner_backend(self.experts.runner_backend) self.router = TinyGemmLinear( config.hidden_size, diff --git a/python/sglang/srt/models/granitemoe.py b/python/sglang/srt/models/granitemoe.py index 8010fde045..dbf84d9aa5 100644 --- a/python/sglang/srt/models/granitemoe.py +++ b/python/sglang/srt/models/granitemoe.py @@ -150,6 +150,7 @@ class GraniteMoeMoE(nn.Module): quant_config=quant_config, prefix=f"{prefix}.experts", ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: # NOTE: hidden_states can have either 1D or 2D shape. diff --git a/python/sglang/srt/models/grok.py b/python/sglang/srt/models/grok.py index 089eea2f8e..ceed324f28 100644 --- a/python/sglang/srt/models/grok.py +++ b/python/sglang/srt/models/grok.py @@ -161,6 +161,7 @@ class Grok1MoE(nn.Module): inplace=inplace, no_combine=no_combine, ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: if not _is_npu: diff --git a/python/sglang/srt/models/hunyuan.py b/python/sglang/srt/models/hunyuan.py index 65ae511e8e..b8a5bffbd3 100644 --- a/python/sglang/srt/models/hunyuan.py +++ b/python/sglang/srt/models/hunyuan.py @@ -161,6 +161,7 @@ class HunYuanSparseMoeBlock(nn.Module): layer_id=layer_id, quant_config=quant_config, ) + self.topk.bind_runner_backend(self.experts.runner_backend) self.gate = ReplicatedLinear( config.hidden_size, config.num_experts, bias=False, quant_config=None diff --git a/python/sglang/srt/models/hunyuan_v3.py b/python/sglang/srt/models/hunyuan_v3.py index fe61d554c2..094a8db651 100644 --- a/python/sglang/srt/models/hunyuan_v3.py +++ b/python/sglang/srt/models/hunyuan_v3.py @@ -152,6 +152,7 @@ class HYV3MoEFused(nn.Module): quant_config=quant_config, prefix=f"{prefix}.experts", ) + self.topk.bind_runner_backend(self.experts.runner_backend) @staticmethod def ebias_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor) -> None: diff --git a/python/sglang/srt/models/interns1pro.py b/python/sglang/srt/models/interns1pro.py index 17851f46a0..78a93aca4a 100644 --- a/python/sglang/srt/models/interns1pro.py +++ b/python/sglang/srt/models/interns1pro.py @@ -126,6 +126,7 @@ class InternS1ProTextDecoderLayer(Qwen3MoeDecoderLayer): layer_id=layer_id, custom_routing_function=self._custom_routing_function, ) + self.mlp.topk.bind_runner_backend(self.mlp.experts.runner_backend) @staticmethod @functools.lru_cache diff --git a/python/sglang/srt/models/interns2_mobius.py b/python/sglang/srt/models/interns2_mobius.py index bd0781cefe..0b6cfc86bd 100644 --- a/python/sglang/srt/models/interns2_mobius.py +++ b/python/sglang/srt/models/interns2_mobius.py @@ -339,6 +339,10 @@ class InternS2MobiusRoutedExpertBank(nn.Module): # The layer-local shared path consumes the same post-attention input. inplace=False, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.gate = ReplicatedLinear( config.hidden_size, config.num_experts, diff --git a/python/sglang/srt/models/iquest_q1.py b/python/sglang/srt/models/iquest_q1.py index 5affe7ee57..c84a4daead 100644 --- a/python/sglang/srt/models/iquest_q1.py +++ b/python/sglang/srt/models/iquest_q1.py @@ -222,6 +222,10 @@ class IQuestQ1MoEBlock(nn.Module): prefix=add_prefix("experts", prefix), routing_method_type=RoutingMethodType.Renormalize, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: original_shape = hidden_states.shape diff --git a/python/sglang/srt/models/kimi_k3.py b/python/sglang/srt/models/kimi_k3.py index 33e27f6c81..5375382611 100644 --- a/python/sglang/srt/models/kimi_k3.py +++ b/python/sglang/srt/models/kimi_k3.py @@ -551,6 +551,10 @@ class KimiK3MoE(nn.Module): else None ), ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) # MegaMoE (deep_gemm fused a2a+GEMM over the EP symm buffer) replaces # the routed experts call below. K3 routes ALL batches through it when diff --git a/python/sglang/srt/models/kimi_linear.py b/python/sglang/srt/models/kimi_linear.py index 015ced9575..be6f1fbd6d 100644 --- a/python/sglang/srt/models/kimi_linear.py +++ b/python/sglang/srt/models/kimi_linear.py @@ -141,6 +141,7 @@ class KimiMoE(nn.Module): # and requires the output format to be standard. We use quant_config to determine the output format. output_format=TopKOutputFormat.STANDARD if quant_config is None else None, ) + self.topk.bind_runner_backend(self.experts.runner_backend) if self.num_shared_experts is not None: intermediate_size = moe_intermediate_size * self.num_shared_experts diff --git a/python/sglang/srt/models/laguna.py b/python/sglang/srt/models/laguna.py index 71bfdac86f..bb585c975d 100644 --- a/python/sglang/srt/models/laguna.py +++ b/python/sglang/srt/models/laguna.py @@ -172,6 +172,7 @@ class LagunaMoE(nn.Module): scoring_func=config.moe_router_score_func, correction_bias=self.gate.e_score_correction_bias, ) + self.topk.bind_runner_backend(self.experts.runner_backend) # HF safetensors key is singular `shared_expert.…`; mirror so the # default loader picks it up without remapping. diff --git a/python/sglang/srt/models/lfm2_moe.py b/python/sglang/srt/models/lfm2_moe.py index c8864cc92a..d80e1beddd 100644 --- a/python/sglang/srt/models/lfm2_moe.py +++ b/python/sglang/srt/models/lfm2_moe.py @@ -162,6 +162,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: """Optimized expert forward pass using FusedMoE.""" diff --git a/python/sglang/srt/models/llada2.py b/python/sglang/srt/models/llada2.py index 1394d1125d..ecc75ed52a 100644 --- a/python/sglang/srt/models/llada2.py +++ b/python/sglang/srt/models/llada2.py @@ -57,7 +57,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe import ( get_deepep_mode, get_moe_a2a_backend, - get_moe_runner_backend, ) from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE @@ -127,42 +126,22 @@ def _require_block_routing_simulated_experts_compatibility() -> None: ) -def _get_effective_moe_runner_backend(experts): - """Return the runner selected by the quant method, including auto selection.""" - quant_method = getattr(experts, "quant_method", None) - - # UnquantizedFusedMoEMethod keeps a Triton fallback in ``runner`` but - # forward_cuda executes this AITER runner first whenever auto selection - # created it. Resolve the runner that actually executes, not the fallback. - aiter_runner = getattr(quant_method, "_aiter_runner", None) - backend = getattr(aiter_runner, "runner_backend", None) - if backend is not None: - return backend - - backend = getattr(quant_method, "_moe_runner_backend", None) - if backend is not None: - return backend - - runner = getattr(quant_method, "runner", None) - backend = getattr(runner, "runner_backend", None) - if backend is not None: - return backend - - runner = getattr(experts, "runner", None) - backend = getattr(runner, "runner_backend", None) - if backend is not None: - return backend - - return get_moe_runner_backend() - - -def _require_block_routing_runner_compatibility(experts) -> None: - backend = _get_effective_moe_runner_backend(experts) +def _require_block_routing_runner_compatibility(experts, top_k: int) -> None: + backend = experts.runner_backend + if backend is None: + # Direct standard/local-ID kernels consume the block router's materialized output. + return if backend.is_flashinfer_trtllm(): raise ValueError( "LLaDA2 block routing does not support --moe-runner-backend " "flashinfer_trtllm; use a compatible runner instead" ) + if backend.is_triton_kernels() and top_k & (top_k - 1): + raise ValueError( + "LLaDA2 block routing with --moe-runner-backend " + "triton_kernel requires num_experts_per_tok to be a " + f"power of two, got {top_k}" + ) def _make_block_routing_triton_output( @@ -667,14 +646,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): f"expert_capacity={self.expert_capacity}, " f"num_experts={self.num_experts}" ) - if get_moe_runner_backend().is_triton_kernels() and self.top_k & ( - self.top_k - 1 - ): - raise ValueError( - "LLaDA2 block routing with --moe-runner-backend " - "triton_kernel requires num_experts_per_tok to be a " - f"power of two, got {self.top_k}" - ) if self.score_function is not None: assert ( @@ -709,8 +680,12 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): routed_scaling_factor=self.routed_scaling_factor, prefix=add_prefix("experts", prefix), ) + self.topk.topk_config.apply_routed_scaling_factor_on_output = ( + _is_npu or self.experts.should_fuse_routed_scaling_factor_in_topk + ) + self.topk.bind_runner_backend(self.experts.runner_backend) if self.use_block_routing: - _require_block_routing_runner_compatibility(self.experts) + _require_block_routing_runner_compatibility(self.experts, self.top_k) # shared expert if config.num_shared_experts is not None: if hasattr(config, "moe_shared_expert_intermediate_size"): @@ -791,15 +766,12 @@ class LLaDA2MoeSparseMoeBlock(nn.Module): ) get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids) - runner_backend = _get_effective_moe_runner_backend(self.experts) - if not runner_backend.is_triton_kernels(): + runner_backend = self.experts.runner_backend + if runner_backend is None or not runner_backend.is_triton_kernels(): if ( - runner_backend.is_aiter() - or getattr( - self.experts, - "should_fuse_routed_scaling_factor_in_topk", - False, - ) + runner_backend is not None + and runner_backend.is_aiter() + or self.experts.should_fuse_routed_scaling_factor_in_topk ) and self.routed_scaling_factor is not None: topk_weights = topk_weights * self.routed_scaling_factor return StandardTopKOutput( diff --git a/python/sglang/srt/models/llama4.py b/python/sglang/srt/models/llama4.py index 0d14cebc14..bb11de0646 100644 --- a/python/sglang/srt/models/llama4.py +++ b/python/sglang/srt/models/llama4.py @@ -43,6 +43,7 @@ from sglang.srt.layers.linear import ( ) from sglang.srt.layers.moe.fused_moe_triton import FusedMoE from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.utils import RoutingMethodType from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -124,8 +125,13 @@ class Llama4MoE(nn.Module): reduce_results=False, quant_config=quant_config, apply_router_weight_on_input=True, + routing_method_type=RoutingMethodType.Llama4, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.shared_expert = LlamaMLP( hidden_size=config.hidden_size, diff --git a/python/sglang/srt/models/longcat_flash.py b/python/sglang/srt/models/longcat_flash.py index 14d213dfb6..63c5e490c3 100644 --- a/python/sglang/srt/models/longcat_flash.py +++ b/python/sglang/srt/models/longcat_flash.py @@ -296,6 +296,7 @@ class LongcatFlashMoE(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: num_tokens, hidden_dim = hidden_states.shape diff --git a/python/sglang/srt/models/mimo_v2.py b/python/sglang/srt/models/mimo_v2.py index 1a63f34fce..8a7e3a65be 100644 --- a/python/sglang/srt/models/mimo_v2.py +++ b/python/sglang/srt/models/mimo_v2.py @@ -71,7 +71,11 @@ from sglang.srt.layers.moe.topk import ( TopKOutputFormat, is_expert_remapping_enabled, ) -from sglang.srt.layers.moe.utils import is_deepep_class_backend, sum_post_experts_output +from sglang.srt.layers.moe.utils import ( + RoutingMethodType, + is_deepep_class_backend, + sum_post_experts_output, +) from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( Mxfp4FlashinferTrtllmMoEMethod, @@ -440,6 +444,11 @@ class MiMoV2MoE(nn.Module): is_nextn=is_nextn, ) + routing_method_type = ( + RoutingMethodType.DeepSeekV3 + if self.gate.e_score_correction_bias is not None and config.norm_topk_prob + else None + ) experts_type = get_moe_impl_class(quant_config) self.experts = experts_type( num_experts=config.n_routed_experts @@ -450,6 +459,7 @@ class MiMoV2MoE(nn.Module): layer_id=self.layer_id, quant_config=quant_config, routed_scaling_factor=1.0, + routing_method_type=routing_method_type, prefix=add_prefix("experts", prefix), ) @@ -470,6 +480,10 @@ class MiMoV2MoE(nn.Module): # and requires the output format to be standard. We use quant_config to determine the output format. output_format=TopKOutputFormat.STANDARD if quant_config is None else None, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) uses_mxfp4_runner = isinstance( self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod diff --git a/python/sglang/srt/models/minimax_m2.py b/python/sglang/srt/models/minimax_m2.py index 6f06bea80f..6e94ec1319 100644 --- a/python/sglang/srt/models/minimax_m2.py +++ b/python/sglang/srt/models/minimax_m2.py @@ -552,6 +552,7 @@ class MiniMaxM2MoE(nn.Module): correction_bias=self.e_score_correction_bias, routed_scaling_factor=1.0, ) + self.topk.bind_runner_backend(self.experts.runner_backend) self.gate = ReplicatedLinear( config.hidden_size, diff --git a/python/sglang/srt/models/minimax_m3.py b/python/sglang/srt/models/minimax_m3.py index 4fd904e9ac..1e82cf9ceb 100644 --- a/python/sglang/srt/models/minimax_m3.py +++ b/python/sglang/srt/models/minimax_m3.py @@ -430,6 +430,7 @@ class MiniMaxM3MoE(nn.Module): routed_scaling_factor=self.routed_scaling_factor, apply_routed_scaling_factor_on_output=True, ) + self.topk.bind_runner_backend(self.experts.runner_backend) if self.n_shared_experts is not None and self.num_fused_shared_experts == 0: intermediate_size = config.intermediate_size * self.n_shared_experts diff --git a/python/sglang/srt/models/mixtral.py b/python/sglang/srt/models/mixtral.py index ac0e8b4e4d..5bbe5a9688 100644 --- a/python/sglang/srt/models/mixtral.py +++ b/python/sglang/srt/models/mixtral.py @@ -107,6 +107,7 @@ class MixtralMoE(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: # NOTE: hidden_states can have either 1D or 2D shape. diff --git a/python/sglang/srt/models/nemotron_h.py b/python/sglang/srt/models/nemotron_h.py index d03994f4e7..8847bd2955 100644 --- a/python/sglang/srt/models/nemotron_h.py +++ b/python/sglang/srt/models/nemotron_h.py @@ -290,6 +290,10 @@ class NemotronHMoE(nn.Module): routed_scaling_factor=self.routed_scaling_factor, apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) if config.n_shared_experts: self.shared_experts = NemotronHMLP( config, diff --git a/python/sglang/srt/models/olmoe.py b/python/sglang/srt/models/olmoe.py index ac97c72996..32199a83ea 100644 --- a/python/sglang/srt/models/olmoe.py +++ b/python/sglang/srt/models/olmoe.py @@ -94,6 +94,7 @@ class OlmoeMoE(nn.Module): layer_id=layer_id, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: # NOTE: hidden_states can have either 1D or 2D shape. diff --git a/python/sglang/srt/models/phimoe.py b/python/sglang/srt/models/phimoe.py index 0a4ec8c469..035fe791a1 100644 --- a/python/sglang/srt/models/phimoe.py +++ b/python/sglang/srt/models/phimoe.py @@ -215,6 +215,7 @@ class PhiMoE(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) def forward( self, hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None diff --git a/python/sglang/srt/models/qwen2_moe.py b/python/sglang/srt/models/qwen2_moe.py index e5305134fe..d30064f28f 100644 --- a/python/sglang/srt/models/qwen2_moe.py +++ b/python/sglang/srt/models/qwen2_moe.py @@ -341,6 +341,10 @@ class Qwen2MoeSparseMoeBlock(nn.Module): config.model_type == "qwen3_5_moe_text" and uses_cutedsl_ar_fusion() ), ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.gate = ReplicatedLinear( config.hidden_size, diff --git a/python/sglang/srt/models/qwen3_moe.py b/python/sglang/srt/models/qwen3_moe.py index 7d35ab6fc5..eab3a112bc 100644 --- a/python/sglang/srt/models/qwen3_moe.py +++ b/python/sglang/srt/models/qwen3_moe.py @@ -273,6 +273,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module): prefix=add_prefix("experts", prefix), routing_method_type=RoutingMethodType.Renormalize, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) # Router gate: description-driven quant. Only the offline ModelSlim path # (which carries a per-layer quant_model_description) may quantise the diff --git a/python/sglang/srt/models/sarvam_moe.py b/python/sglang/srt/models/sarvam_moe.py index a921a70c2a..3592624dba 100644 --- a/python/sglang/srt/models/sarvam_moe.py +++ b/python/sglang/srt/models/sarvam_moe.py @@ -284,6 +284,10 @@ class SarvamMoESparseMoeBlock(nn.Module): prefix=add_prefix("experts", prefix), routing_method_type=RoutingMethodType.Renormalize, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.gate = ReplicatedLinear( config.hidden_size, diff --git a/python/sglang/srt/models/sdar_moe.py b/python/sglang/srt/models/sdar_moe.py index df9738f4cd..77126a789f 100644 --- a/python/sglang/srt/models/sdar_moe.py +++ b/python/sglang/srt/models/sdar_moe.py @@ -106,6 +106,10 @@ class SDARMoeSparseMoeBlock(nn.Module): prefix=add_prefix("experts", prefix), routing_method_type=RoutingMethodType.Renormalize, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.gate = ReplicatedLinear( config.hidden_size, diff --git a/python/sglang/srt/models/step3_vl.py b/python/sglang/srt/models/step3_vl.py index 34ae96c43f..7effe67e9b 100644 --- a/python/sglang/srt/models/step3_vl.py +++ b/python/sglang/srt/models/step3_vl.py @@ -145,6 +145,7 @@ class Step3TextMoEMLP(nn.Module): quant_config=quant_config, prefix=add_prefix("experts", prefix), ) + self.topk.bind_runner_backend(self.experts.runner_backend) self.gate = ReplicatedLinear( config.hidden_size, diff --git a/python/sglang/srt/models/step3p5.py b/python/sglang/srt/models/step3p5.py index 7c7c45b8d4..d9918b826f 100644 --- a/python/sglang/srt/models/step3p5.py +++ b/python/sglang/srt/models/step3p5.py @@ -162,6 +162,10 @@ class Step3p5MoEMLP(nn.Module): routing_method_type=RoutingMethodType.Renormalize, gemm1_clamp_limit=self.limit, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) self.gate = ReplicatedLinear( config.hidden_size, diff --git a/python/sglang/srt/models/xllm.py b/python/sglang/srt/models/xllm.py index 7c75d842ea..7f0d0ad61e 100644 --- a/python/sglang/srt/models/xllm.py +++ b/python/sglang/srt/models/xllm.py @@ -942,6 +942,10 @@ class XllmSparseMoeBlock(nn.Module): prefix=add_prefix("experts", prefix), routing_method_type=RoutingMethodType.RenormalizeNaive, ) + self.topk.bind_runner_backend( + self.experts.runner_backend, + routing_method_type=self.experts.routing_method_type, + ) # Shared expert (no gating — output added directly) num_shared_experts = getattr(config, "num_shared_experts", 0) diff --git a/python/sglang/srt/models/xverse_moe.py b/python/sglang/srt/models/xverse_moe.py index 81029f308c..7457cdee5f 100644 --- a/python/sglang/srt/models/xverse_moe.py +++ b/python/sglang/srt/models/xverse_moe.py @@ -36,7 +36,7 @@ from sglang.srt.layers.linear import ( from sglang.srt.layers.logits_processor import LogitsProcessor from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe -from sglang.srt.layers.moe.topk import TopK +from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat from sglang.srt.layers.quantization.base_config import QuantizationConfig from sglang.srt.layers.radix_attention import RadixAttention from sglang.srt.layers.rotary_embedding import get_rope @@ -136,6 +136,7 @@ class XverseMoE(nn.Module): top_k=self.top_k, layer_id=layer_id, renormalize=getattr(self.config, "norm_topk_prob", False), + output_format=TopKOutputFormat.STANDARD, ) if config.num_shared_experts is not None: diff --git a/test/registered/amd/test_fp8_moe_triton_runner_with_aiter.py b/test/registered/amd/test_fp8_moe_triton_runner_with_aiter.py new file mode 100644 index 0000000000..64366d6e19 --- /dev/null +++ b/test/registered/amd/test_fp8_moe_triton_runner_with_aiter.py @@ -0,0 +1,146 @@ +"""With AITER on, a block-FP8 MoE that the Triton runner serves must compute what +it computes with AITER off: AITER's expert layout is for AITER's kernels only. +""" + +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.runtime_context import get_context, get_flags +from sglang.srt.utils import is_hip +from sglang.test.ci.ci_register import register_amd_ci +from sglang.test.test_utils import CustomTestCase + +register_amd_ci(est_time=60, suite="stage-b-test-1-gpu-small-amd") + +MODULE = "sglang.srt.layers.quantization.fp8" +BLOCK = 128 +EXPERTS, HIDDEN, INTERMEDIATE, TOKENS, TOP_K = 4, 256, 256, 64, 2 + + +def _on_fnuz_gpu() -> bool: + if not (is_hip() and torch.cuda.is_available()): + return False + from sglang.kernels.ops.quantization.fp8_kernel import is_fp8_fnuz + + return is_fp8_fnuz() + + +def _aiter_shuffle(): + try: + from aiter.ops.shuffle import shuffle_weight + except ImportError: + return None + return shuffle_weight + + +def _finalized(runner_backend, *, use_aiter): + """A block-FP8 MoE layer after the finalization the given runner would see.""" + from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod + + method = Fp8MoEMethod( + Fp8Config(is_checkpoint_fp8_serialized=True, weight_block_size=[BLOCK, BLOCK]) + ) + method.runner = SimpleNamespace(runner_backend=runner_backend) + layer = torch.nn.Module() + generator = torch.Generator(device="cuda").manual_seed(0) + for name, shape in ( + ("w13_weight", (EXPERTS, 2 * INTERMEDIATE, HIDDEN)), + ("w2_weight", (EXPERTS, HIDDEN, INTERMEDIATE)), + ): + weight = ( + ((torch.rand(shape, generator=generator, device="cuda") - 0.5) * 2 * 448) + .clamp(-448, 448) + .to(torch.float8_e4m3fn) + ) + scale = ( + torch.rand( + (EXPERTS, shape[1] // BLOCK, shape[2] // BLOCK), + generator=generator, + device="cuda", + ) + * 1e-2 + ) + layer.register_parameter(name, torch.nn.Parameter(weight, requires_grad=False)) + layer.register_parameter( + name + "_scale_inv", torch.nn.Parameter(scale, requires_grad=False) + ) + with ( + patch(f"{MODULE}._use_aiter", use_aiter), + patch(f"{MODULE}._use_hip_int4", False), + patch(f"{MODULE}.shuffle_weight", _aiter_shuffle(), create=True), + ): + method.process_weights_after_loading_block_quant(layer) + return layer + + +def _triton_moe(layer, hidden_states, topk_output): + from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe + + return fused_moe( + hidden_states.clone(), # fused_moe writes its output into its input + layer.w13_weight, + layer.w2_weight, + topk_output, + use_fp8_w8a8=True, + w1_scale=layer.w13_weight_scale_inv, + w2_scale=layer.w2_weight_scale_inv, + block_shape=[BLOCK, BLOCK], + ) + + +@unittest.skipUnless( + _on_fnuz_gpu() and _aiter_shuffle() is not None, + "requires a gfx94x (E4M3FNUZ) GPU with AITER", +) +class TestFp8MoeTritonRunnerWithAiter(CustomTestCase): + def setUp(self): + self.enterContext( + get_context().override_server_args(model_path="dummy", device="cuda") + ) + self.enterContext( + get_flags().moe.override(runner_backend=MoeRunnerBackend.TRITON) + ) + + def test_the_triton_runner_computes_what_it_computes_without_aiter(self): + from sglang.srt.layers.moe.topk import TopKConfig, select_experts + + torch.manual_seed(0) + hidden_states = ( + torch.randn(TOKENS, HIDDEN, dtype=torch.bfloat16, device="cuda") / 10 + ) + router_logits = torch.randn( + TOKENS, EXPERTS, dtype=torch.bfloat16, device="cuda" + ) + topk_output = select_experts( + hidden_states=hidden_states, + router_logits=router_logits, + topk_config=TopKConfig(top_k=TOP_K, renormalize=False), + ) + + with torch.inference_mode(): + reference = _triton_moe( + _finalized(MoeRunnerBackend.TRITON, use_aiter=False), + hidden_states, + topk_output, + ) + served_layer = _finalized(MoeRunnerBackend.TRITON, use_aiter=True) + served = _triton_moe(served_layer, hidden_states, topk_output) + # What the Triton runner computed when it was handed AITER's layout. + aiter_layout = _triton_moe( + _finalized(MoeRunnerBackend.AITER, use_aiter=True), + hidden_states, + topk_output, + ) + + torch.testing.assert_close(served, reference, rtol=0, atol=0) + # The layout matters to the Triton kernel: fed AITER's, it is not even close. + error = (aiter_layout.float() - reference.float()).abs().mean() + self.assertGreater(error / reference.float().abs().mean(), 0.5) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py b/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py index f437351f42..96801c79f0 100644 --- a/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py +++ b/test/registered/cuda_graph/breakable/test_deepseek_dual_stream_outputs.py @@ -6,6 +6,8 @@ from unittest.mock import patch import torch +from sglang.srt.layers.moe.topk import TopKOutputFormat +from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.models import deepseek_v2 from sglang.srt.runtime_context import get_forward from sglang.test.ci.ci_register import register_cuda_ci @@ -28,7 +30,11 @@ class DualStreamOperand: self.shared_experts = object() self.is_hash = False self.gate = SimpleNamespace(e_score_correction_bias_vl=None) + self.topk = SimpleNamespace( + topk_config=SimpleNamespace(output_format=TopKOutputFormat.BYPASSED) + ) self.experts = SimpleNamespace( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, use_flashinfer_trtllm_moe=True, quant_method=None, should_fuse_routed_scaling_factor_in_topk=False, @@ -71,13 +77,15 @@ class TestDualStreamOutputs(CustomTestCase): patch.object( deepseek_v2, "get_moe_runner_backend", - return_value=SimpleNamespace(is_flashinfer_trtllm=lambda: True), + return_value=MoeRunnerBackend.FLASHINFER_TRTLLM, ), patch.object( deepseek_v2, "get_exec", return_value=SimpleNamespace( - moe=SimpleNamespace(enable_eplb=False) + moe=SimpleNamespace( + enable_eplb=False, ep_dispatch_algorithm="static" + ) ), ), patch.object( diff --git a/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py b/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py index 3582335aac..adaa18d8a2 100644 --- a/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py +++ b/test/registered/unit/layers/moe/test_flashinfer_dispatcher.py @@ -30,6 +30,7 @@ class _RecordingMoeAlltoAll: def _make_dispatcher(*, dispatch_type, max_num_tokens, quant_config): dispatcher = object.__new__(FlashinferDispatcher) + dispatcher.runner_backend = None dispatcher.dispatch_type = dispatch_type dispatcher.hidden_size = 128 dispatcher.max_num_tokens = max_num_tokens diff --git a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py index 771160978e..a90c424a9d 100644 --- a/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py +++ b/test/registered/unit/layers/moe/test_fused_shared_expert_scaling.py @@ -198,7 +198,9 @@ class TestFusedSharedExpertScaling(CustomTestCase): e_score_correction_bias=torch.zeros(16) if correction_bias else None ) experts = SimpleNamespace( - should_fuse_routed_scaling_factor_in_topk=folded_in_topk + should_fuse_routed_scaling_factor_in_topk=folded_in_topk, + runner_backend=None, + routing_method_type=None, ) exec_config = SimpleNamespace( moe=SimpleNamespace(enable_waterfill=False, ep_num_redundant_experts=0) diff --git a/test/registered/unit/layers/moe/test_moe_runner_extensions.py b/test/registered/unit/layers/moe/test_moe_runner_extensions.py index 05969b2d7d..3dede2654b 100644 --- a/test/registered/unit/layers/moe/test_moe_runner_extensions.py +++ b/test/registered/unit/layers/moe/test_moe_runner_extensions.py @@ -141,7 +141,7 @@ def test_fused_moe_uses_explicit_quant_method_for_full_lifecycle(monkeypatch) -> def create_runner(layer, config) -> None: calls.append("runner") - method.runner = SimpleNamespace() + method.runner = SimpleNamespace(runner_backend=MoeRunnerBackend.TRITON) monkeypatch.setattr(method, "create_moe_runner", create_runner) monkeypatch.setattr( diff --git a/test/registered/unit/layers/moe/test_standard_dispatcher.py b/test/registered/unit/layers/moe/test_standard_dispatcher.py index 2e0e42772e..64ddc0bd12 100644 --- a/test/registered/unit/layers/moe/test_standard_dispatcher.py +++ b/test/registered/unit/layers/moe/test_standard_dispatcher.py @@ -1,15 +1,31 @@ import sys +from functools import partial +from types import SimpleNamespace from unittest.mock import patch import torch +import sglang.srt.layers.moe.topk as topk_module +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.moe.token_dispatcher.standard import ( StandardDispatcher, + StandardDispatchOutput, ) -from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.moe.topk import StandardTopKOutput, TopK, TopKOutputFormat +from sglang.srt.layers.moe.utils import ( + MoeA2ABackend, + MoeRunnerBackend, + RoutingMethodType, +) +from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod +from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod +from sglang.srt.runtime_context import get_context, get_flags, get_model, get_parallel from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase -register_cpu_ci(est_time=1, suite="base-a-test-cpu") +register_cpu_ci(est_time=12, suite="base-a-test-cpu") def test_dispatch_preserves_negative_routing_sentinel(): @@ -38,6 +54,793 @@ def test_dispatch_preserves_negative_routing_sentinel(): assert torch.equal(output.topk_output.topk_ids, expected) +class TestLayerRunnerDispatch(CustomTestCase): + def setUp(self): + self.enterContext( + patch( + "sglang.srt.layers.moe.token_dispatcher.standard.get_device", + return_value="cpu", + ) + ) + self.enterContext( + get_context().override_server_args(model_path="dummy", device="cpu") + ) + self.enterContext( + get_flags().moe.override( + runner_backend=MoeRunnerBackend.TRITON, + a2a_backend=MoeA2ABackend.NONE, + ) + ) + self.enterContext( + get_parallel().override( + tp_size=2, + tp_rank=1, + tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + attn_tp_size=2, + attn_tp_rank=1, + moe_ep_size=2, + moe_ep_rank=1, + moe_tp_size=1, + moe_tp_rank=0, + ) + ) + + @staticmethod + def _config(): + return MoeRunnerConfig( + num_experts=4, + num_local_experts=2, + num_fused_shared_experts=0, + hidden_size=4, + intermediate_size_per_partition=96, + top_k=2, + ) + + def _build_layer(self, method, *, routing_method_type=None): + with patch.object(method, "create_weights"): + return FusedMoE( + num_experts=4, + top_k=2, + hidden_size=4, + intermediate_size=96, + layer_id=0, + quant_method=method, + routing_method_type=routing_method_type, + ) + + def _assert_local_routes(self, config): + dispatcher = StandardDispatcher(config) + routes = StandardTopKOutput( + topk_weights=torch.ones((2, 2)), + topk_ids=torch.tensor([[-1, 3], [2, 0]], dtype=torch.int32), + router_logits=None, + ) + with patch( + "sglang.srt.layers.moe.token_dispatcher.standard." + "should_use_flashinfer_cutlass_moe_fp4_allgather", + return_value=False, + ): + output = dispatcher.dispatch(torch.ones((2, 4)), routes) + self.assertTrue( + torch.equal( + output.topk_output.topk_ids, + torch.tensor([[-1, 1], [0, -1]], dtype=torch.int32), + ) + ) + topk = TopK(top_k=2, layer_id=0, allow_routed_experts_capture=False) + topk.bind_runner_backend(config.runner_backend) + self.assertEqual(topk.topk_config.output_format, TopKOutputFormat.STANDARD) + self.assertFalse(dispatcher.use_aiter_moe_runner) + + def test_compressed_routes_apply_scale_once_for_the_effective_runner(self): + """Wrapped FP8/NVFP4 experts consume final materialized route weights.""" + from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsFusedMoEMethod, + ) + from sglang.srt.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_nvfp4_moe import ( + CompressedTensorsW4A4Nvfp4MoE, + ) + from sglang.srt.layers.quantization.compressed_tensors.schemes.compressed_tensors_w8a8_fp8_moe import ( + CompressedTensorsW8A8Fp8MoE, + ) + + with ( + get_flags().moe.override(runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM), + get_parallel().override( + tp_size=1, + tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + ), + get_model().override(shared_experts=SharedExpertsConfig()), + patch.object( + topk_module, + "grouped_topk_gpu", + topk_module.grouped_topk_gpu._torchdynamo_orig_callable, + ), + patch( + "sglang.srt.layers.quantization.compressed_tensors.schemes." + "compressed_tensors_w4a4_nvfp4_moe.get_platform", + return_value=SimpleNamespace(is_blackwell=True), + ), + ): + schemes = ( + CompressedTensorsW8A8Fp8MoE( + SimpleNamespace( + strategy="block", block_structure=[128, 128], dynamic=False + ), + SimpleNamespace(strategy="token", dynamic=True), + ), + CompressedTensorsW4A4Nvfp4MoE(), + ) + hidden = torch.ones((1, 128)) + logits = torch.tensor([[3.0, 2.0, 1.0, 0.0, -1.0, -2.0, -3.0, -4.0]]) + expert_outputs = torch.tensor( + [4.0, 10.0, 20.0, 30.0, 40.0, 50.0, 60.0, 70.0] + ) + for scheme in schemes: + for renormalize in (True, False): + for factor in (1.0, 2.5): + with self.subTest( + scheme=type(scheme).__name__, + renormalize=renormalize, + factor=factor, + ): + method = CompressedTensorsFusedMoEMethod(None) + + def select_method(layer, prefix): + layer.scheme = scheme + return method + + layer = FusedMoE( + num_experts=8, + top_k=2, + hidden_size=128, + intermediate_size=128, + layer_id=0, + quant_config=SimpleNamespace( + get_quant_method=select_method, + get_name=lambda: "compressed-tensors", + ), + routed_scaling_factor=factor, + routing_method_type=RoutingMethodType.DeepSeekV3, + ) + topk = TopK( + top_k=2, + layer_id=0, + use_grouped_topk=True, + num_expert_group=2, + topk_group=1, + renormalize=renormalize, + routed_scaling_factor=factor, + apply_routed_scaling_factor_on_output=layer.should_fuse_routed_scaling_factor_in_topk, + allow_routed_experts_capture=False, + ) + topk.bind_runner_backend( + layer.runner_backend, + routing_method_type=layer.routing_method_type, + ) + routes = topk.forward_cuda(hidden, logits) + self.assertEqual(routes.format, TopKOutputFormat.STANDARD) + dispatched = layer.dispatcher.dispatch(hidden, routes) + actual = ( + expert_outputs[dispatched.topk_output.topk_ids.long()] + * dispatched.topk_output.topk_weights + ).sum() + reference = logits.softmax(-1)[:, :2] + if renormalize: + reference = reference / reference.sum(-1, keepdim=True) + expected = factor * (reference * expert_outputs[:2]).sum() + torch.testing.assert_close(actual, expected) + + def test_materialized_router_scale_is_independent_of_renormalization(self): + """Folding a routed factor does not require selected-weight normalization.""" + hidden = torch.ones((1, 4)) + logits = torch.tensor([[3.0, 2.0, 1.0, 0.0, -1.0, -2.0, -3.0, -4.0]]) + for name in ( + "grouped_topk_gpu", + "kimi_k2_biased_topk_impl", + "biased_topk_impl", + ): + router = getattr(topk_module, name)._torchdynamo_orig_callable + for renormalize in (True, False): + for fold in (True, False): + with self.subTest(router=name, renormalize=renormalize, fold=fold): + kwargs = dict( + hidden_states=hidden, + gating_output=logits, + topk=2, + renormalize=renormalize, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=fold, + ) + if name == "grouped_topk_gpu": + kwargs.update(num_expert_group=2, topk_group=1) + scores = logits.softmax(-1) + else: + kwargs["correction_bias"] = torch.zeros(8) + scores = logits.sigmoid() + weights, ids = router(**kwargs) + expected = scores.gather(1, ids.long()) + if renormalize: + expected = expected / expected.sum(-1, keepdim=True) + if fold: + expected *= 2.5 + torch.testing.assert_close(weights, expected) + if name != "kimi_k2_biased_topk_impl": + kwargs.update(topk=3, num_fused_shared_experts=1) + shared_weights, shared_ids = router(**kwargs) + torch.testing.assert_close(shared_weights[:, :2], expected) + self.assertEqual(int(shared_ids[0, -1]), logits.shape[-1]) + torch.testing.assert_close( + shared_weights[:, -1], + torch.tensor([1.0 if fold else 1.0 / 2.5]), + ) + + def test_fp8_fallback_uses_local_ids_and_materialized_routes(self): + """FP8 experts under an FP4 runner request must not receive global IDs.""" + for requested in ( + MoeRunnerBackend.FLASHINFER_CUTLASS, + MoeRunnerBackend.FLASHINFER_CUTEDSL, + ): + with ( + self.subTest(requested=requested), + get_flags().moe.override(runner_backend=requested), + ): + method = object.__new__(Fp8MoEMethod) + method.is_fp4_expert = False + method.use_mxfp8 = False + layer = self._build_layer(method) + self._assert_local_routes(layer.moe_runner_config) + + def test_unquantized_fallback_is_independent_of_requested_runner(self): + """A draft BF16 layer may fall back while its target keeps global IDs.""" + for requested in ( + MoeRunnerBackend.FLASHINFER_MEGAMOE, + MoeRunnerBackend.FLASHINFER_CUTEDSL, + ): + with ( + self.subTest(requested=requested), + get_flags().moe.override(runner_backend=requested), + ): + layer = self._build_layer(UnquantizedFusedMoEMethod()) + self._assert_local_routes(layer.moe_runner_config) + + def test_unsupported_aiter_shape_keeps_triton_local_ids(self): + """Enabling AITER must not suppress local remapping after a shape fallback.""" + with ( + get_flags().moe.override(runner_backend=MoeRunnerBackend.AUTO), + patch("sglang.srt.layers.quantization.unquant._use_aiter", True), + ): + layer = self._build_layer(UnquantizedFusedMoEMethod()) + self._assert_local_routes(layer.moe_runner_config) + + def test_fp8_fallback_keeps_dp_allgather_in_bf16(self): + """FP8 fallback experts still gather DP-local rows without FP4 quantization.""" + captured = {} + + class Group: + world_size = 2 + + def all_gatherv(self, payloads, sizes): + captured["payloads"] = payloads + captured["sizes"] = sizes + weights, ids, hidden = payloads + return [ + torch.cat([weights, weights]), + torch.tensor([[0, 3], [2, 1]], dtype=ids.dtype), + torch.cat([hidden, hidden + 1]), + ] + + config = self._config() + config.runner_backend = MoeRunnerBackend.TRITON + dispatcher = StandardDispatcher(config) + hidden = torch.zeros((1, 4), dtype=torch.bfloat16) + topk = StandardTopKOutput( + torch.ones((1, 2)), torch.tensor([[0, 3]], dtype=torch.int32), None + ) + with ( + get_parallel().override(tp_group=Group()), + patch( + "sglang.srt.layers.moe.token_dispatcher.standard." + "should_use_flashinfer_cutlass_moe_fp4_allgather", + return_value=True, + ), + patch( + "sglang.srt.layers.moe.token_dispatcher.standard.get_dp_global_num_tokens", + return_value=[1, 1], + ), + ): + output = dispatcher.dispatch(hidden, topk) + self.assertIs(captured["payloads"][2], hidden) + self.assertEqual(captured["sizes"], [1, 1]) + self.assertEqual(output.hidden_states.dtype, torch.bfloat16) + self.assertIsNone(output.hidden_states_scale) + self.assertTrue( + torch.equal( + output.topk_output.topk_ids, + torch.tensor([[-1, 1], [0, -1]], dtype=torch.int32), + ) + ) + + def test_direct_kernels_declare_their_execution_contract(self): + """A missing or dormant runner must not change a direct kernel's ID ABI.""" + from sglang.srt.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxint4_moe import ( + CompressedTensorsMxInt4MoE, + ) + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8MoEMethod + from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import ( + Mxfp4FlashinferTrtllmMoEMethod, + ) + + cases = [ + (CompressedTensorsMxInt4MoE, MoeRunnerBackend.FLASHINFER_TRTLLM, False), + (Mxfp4FlashinferTrtllmMoEMethod, MoeRunnerBackend.FLASHINFER_MXFP4, True), + (ModelOptFp8MoEMethod, MoeRunnerBackend.FLASHINFER_TRTLLM, False), + ] + for method_type, backend, fp4 in cases: + with ( + self.subTest(method=method_type.__name__), + get_flags().moe.override(runner_backend=backend), + ): + method = object.__new__(method_type) + layer = self._build_layer( + method, routing_method_type=RoutingMethodType.DeepSeekV3 + ) + self.assertEqual(layer.runner_backend, backend) + self.assertTrue(layer.dispatcher.skip_local_expert_mapping) + topk = TopK( + top_k=2, + layer_id=0, + is_fp4_experts=fp4, + use_grouped_topk=True, + num_expert_group=1, + topk_group=1, + scoring_func="sigmoid", + correction_bias=torch.zeros(4), + ) + topk.bind_runner_backend( + layer.runner_backend, + routing_method_type=layer.routing_method_type, + ) + self.assertEqual( + topk.topk_config.output_format, + TopKOutputFormat.STANDARD if fp4 else TopKOutputFormat.BYPASSED, + ) + if method_type is ModelOptFp8MoEMethod: + self.assertTrue(method.runner.runner_backend.is_triton()) + self.assertTrue(layer.use_flashinfer_trtllm_moe) + self.assertFalse(layer.moe_runner_config.inplace) + + def test_compressed_wrapper_propagates_the_inner_direct_contract(self): + """Wrapper methods expose their scheme's format and global-ID contract.""" + from sglang.srt.layers.quantization.compressed_tensors.compressed_tensors import ( + CompressedTensorsFusedMoEMethod, + ) + from sglang.srt.layers.quantization.compressed_tensors.schemes.compressed_tensors_w4a4_mxint4_moe import ( + CompressedTensorsMxInt4MoE, + ) + + scheme = object.__new__(CompressedTensorsMxInt4MoE) + method = object.__new__(CompressedTensorsFusedMoEMethod) + + def create_weights(*, layer, **kwargs): + layer.scheme = scheme + + with patch.object(method, "create_weights", side_effect=create_weights): + layer = FusedMoE( + num_experts=4, + top_k=2, + hidden_size=4, + intermediate_size=96, + layer_id=0, + quant_method=method, + ) + self.assertEqual(layer.runner_backend, MoeRunnerBackend.FLASHINFER_TRTLLM) + self.assertTrue(layer.use_flashinfer_trtllm_moe) + self.assertTrue(layer.dispatcher.skip_local_expert_mapping) + + def test_gfx1250_internal_router_requires_full_expert_weights(self): + """Internal raw-logit routing cannot address a rank-local expert shard.""" + from sglang.srt.layers.quantization.quark.schemes.quark_w4a8_mxfp4_moe import ( + QuarkW4A8MXFp4MoE, + ) + + with patch( + "sglang.srt.layers.quantization.quark.schemes.quark_w4a8_mxfp4_moe." + "_use_gfx1250_w4a8", + True, + ): + with self.assertRaisesRegex(ValueError, "expert parallelism"): + self._build_layer(object.__new__(QuarkW4A8MXFp4MoE)) + with get_parallel().override( + tp_size=1, + tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + ): + layer = self._build_layer(object.__new__(QuarkW4A8MXFp4MoE)) + self.assertIsNone(layer.runner_backend) + self.assertFalse(layer.dispatcher.skip_local_expert_mapping) + + def test_borrowed_inkling_dense_loader_keeps_canonical_weights(self): + from sglang.srt.environ import envs + from sglang.srt.layers.quantization import unquant + from sglang.srt.models.inkling_common.dense_mlp import InklingBatchDenseMLP + + layer = InklingBatchDenseMLP( + n_shared_experts=2, + d_model=4, + shared_d_mlp=8, + layer_id=0, + prefix="model.layers.0.mlp.shared_experts", + linearized_bf16=True, + ) + self.assertIsNone(layer.quant_method.get_moe_runner_backend(layer)) + for offset in (0, 100): + w13 = torch.arange(128).reshape(2, 16, 4).float() + offset + w2 = torch.arange(64).reshape(2, 4, 8).float() + offset + layer.weight_loader_fused( + layer.w13_weight, w13, "model.layers.0.mlp.experts.w13_weight", "w13" + ) + layer.weight_loader_fused( + layer.w2_weight, w2, "model.layers.0.mlp.experts.w2_weight", "w2" + ) + with ( + envs.SGLANG_OPT_FUSE_SWIGLU_INTERLEAVED.override(True), + patch.object(unquant, "_is_cuda", True), + patch.object(unquant, "_is_cpu", False), + ): + layer.quant_method.process_weights_after_loading(layer) + torch.testing.assert_close(layer.w13_weight, w13, rtol=0, atol=0) + torch.testing.assert_close(layer.w2_weight, w2, rtol=0, atol=0) + w13_linear, w2_linear = layer.get_bf16_linearized_weights() + torch.testing.assert_close(w13_linear, w13.reshape(32, 4), rtol=0, atol=0) + torch.testing.assert_close( + w2_linear, w2.transpose(1, 2).reshape(16, 4), rtol=0, atol=0 + ) + + def test_topk_does_not_re_read_the_target_backend(self): + """A constructed router keeps its layer's format across draft scopes.""" + topk = TopK(top_k=2, layer_id=0, allow_routed_experts_capture=False) + topk.bind_runner_backend( + MoeRunnerBackend.FLASHINFER_TRTLLM, + routing_method_type=RoutingMethodType.Renormalize, + ) + with get_flags().moe.override(runner_backend=MoeRunnerBackend.TRITON): + output = topk.forward_cuda(torch.ones((1, 4)), torch.ones((1, 4))) + self.assertEqual(output.format, TopKOutputFormat.BYPASSED) + + def test_auto_nvfp4_preserves_undeclared_routing_policy(self): + """An AUTO-selected FP4 kernel must preserve renormalization and custom routes.""" + from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptNvFp4FusedMoEMethod, + ) + + logits = torch.tensor([[2.0, 1.0, 0.0, -1.0]]) + hidden = torch.ones((1, 4)) + custom_ids = torch.tensor([[3, 1]], dtype=torch.int32) + custom_weights = torch.tensor([[0.25, 0.75]]) + + def custom_router(**kwargs): + return custom_weights.clone(), custom_ids.clone() + + def cpu_router(hidden_states, gating_output, topk, renormalize, **kwargs): + return topk_module.fused_topk_torch_native( + hidden_states, + gating_output, + topk, + renormalize, + correction_bias=kwargs["correction_bias"], + scoring_func=kwargs["scoring_func"], + ) + + for custom, routing_method_type in ( + (None, None), + (custom_router, None), + (custom_router, RoutingMethodType.Renormalize), + ): + with ( + self.subTest(custom=custom is not None, native=routing_method_type), + get_flags().moe.override(runner_backend=MoeRunnerBackend.AUTO), + patch( + "sglang.srt.layers.quantization.modelopt_quant.is_cuda", + return_value=True, + ), + patch( + "sglang.srt.layers.quantization.modelopt_quant.get_device_capability", + return_value=(10, 0), + ), + patch.object(topk_module, "fused_topk", side_effect=cpu_router), + ): + method = object.__new__(ModelOptNvFp4FusedMoEMethod) + layer = self._build_layer( + method, routing_method_type=routing_method_type + ) + self.assertEqual( + layer.runner_backend, MoeRunnerBackend.FLASHINFER_TRTLLM + ) + self.assertTrue(layer.dispatcher.skip_local_expert_mapping) + topk = TopK( + top_k=2, + layer_id=0, + renormalize=True, + custom_routing_function=custom, + allow_routed_experts_capture=False, + ) + topk.bind_runner_backend( + layer.runner_backend, + routing_method_type=layer.routing_method_type, + ) + output = topk.forward_cuda(hidden, logits) + self.assertEqual(output.format, TopKOutputFormat.STANDARD) + expected_ids = ( + logits.topk(2, dim=-1).indices.to(torch.int32) + if custom is None + else custom_ids + ) + expected_weights = ( + torch.softmax(logits[:, :2], dim=-1) + if custom is None + else custom_weights + ) + self.assertTrue(torch.equal(output.topk_ids, expected_ids)) + torch.testing.assert_close(output.topk_weights, expected_weights) + + def test_per_tensor_fp8_undeclared_router_selects_local_layout(self): + """A router the per-tensor kernel cannot represent must not use its packed weights.""" + from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp8MoEMethod + + with get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM + ): + for routing_method_type, expected in ( + (None, MoeRunnerBackend.TRITON), + (RoutingMethodType.Unspecified, MoeRunnerBackend.TRITON), + (RoutingMethodType.Llama4, MoeRunnerBackend.FLASHINFER_TRTLLM), + ): + with self.subTest(routing_method_type=routing_method_type): + layer = self._build_layer( + object.__new__(ModelOptFp8MoEMethod), + routing_method_type=routing_method_type, + ) + self.assertEqual(layer.runner_backend, expected) + self.assertEqual( + layer.dispatcher.skip_local_expert_mapping, + expected == MoeRunnerBackend.FLASHINFER_TRTLLM, + ) + self.assertEqual( + layer.use_flashinfer_trtllm_moe, + expected == MoeRunnerBackend.FLASHINFER_TRTLLM, + ) + topk = TopK( + top_k=1, + layer_id=0, + renormalize=False, + custom_routing_function=lambda **kwargs: None, + ) + topk.bind_runner_backend( + layer.runner_backend, + routing_method_type=layer.routing_method_type, + ) + self.assertEqual( + topk.topk_config.output_format, + TopKOutputFormat.BYPASSED + if routing_method_type == RoutingMethodType.Llama4 + else TopKOutputFormat.STANDARD, + ) + if expected == MoeRunnerBackend.FLASHINFER_TRTLLM: + with self.assertRaisesRegex( + NotImplementedError, "materialized" + ): + layer.quant_method.apply( + layer, + StandardDispatchOutput( + torch.ones((1, 4)), + None, + StandardTopKOutput( + torch.ones((1, 2)), + torch.tensor([[0, 1]], dtype=torch.int32), + None, + ), + ), + ) + + def test_declared_softmax_kernel_cannot_replace_other_router_policies(self): + """A native softmax enum must not override sigmoid, grouping, bias, or renormalization.""" + for kwargs in ( + dict( + scoring_func="sigmoid", + use_grouped_topk=True, + num_expert_group=2, + topk_group=1, + correction_bias=torch.zeros(4), + ), + dict(scoring_func="sigmoid", correction_bias=torch.zeros(4)), + dict(renormalize=False), + ): + with self.subTest(kwargs=kwargs): + topk = TopK(top_k=2, layer_id=0, **kwargs) + topk.bind_runner_backend( + MoeRunnerBackend.FLASHINFER_TRTLLM, + routing_method_type=RoutingMethodType.RenormalizeNaive, + ) + self.assertEqual( + topk.topk_config.output_format, TopKOutputFormat.STANDARD + ) + + def test_materialized_softmax_and_custom_routes_fold_scale_once(self): + """Materializers scale routed weights once while keeping shared weights unchanged.""" + logits = torch.tensor([[2.0, 1.0, 0.0, -1.0]]) + hidden = torch.ones((1, 4)) + + def cpu_router(hidden_states, gating_output, topk, renormalize, **kwargs): + return topk_module.fused_topk_torch_native( + hidden_states, gating_output, topk, renormalize + ) + + def custom_router(**kwargs): + ids = [3, 1] + ([4] if kwargs["topk"] == 3 else []) + weights = [0.25, 0.75] + ([1.0] if kwargs["topk"] == 3 else []) + return torch.tensor([weights]), torch.tensor([ids], dtype=torch.int32) + + for custom, shared in ((None, 0), (custom_router, 0), (custom_router, 1)): + with ( + self.subTest(custom=custom is not None, shared=shared), + get_model().override( + shared_experts=SharedExpertsConfig(num_fused_shared_experts=shared) + ), + patch.object(topk_module, "fused_topk", side_effect=cpu_router), + ): + topk = TopK( + top_k=2, + layer_id=0, + custom_routing_function=custom, + routed_scaling_factor=2.5, + apply_routed_scaling_factor_on_output=True, + allow_routed_experts_capture=False, + ) + topk.bind_runner_backend(MoeRunnerBackend.FLASHINFER_TRTLLM) + output = topk.forward_cuda(hidden, logits) + expected = ( + torch.softmax(logits[:, :2], dim=-1) + if custom is None + else torch.tensor([[0.25, 0.75]]) + ) * 2.5 + torch.testing.assert_close(output.topk_weights[:, :2], expected) + if shared: + self.assertEqual(output.topk_weights[0, 2].item(), 1.0) + + def test_model_constructors_preserve_materialized_routed_scale(self): + """Bailing and normal LLaDA2 routing must retain scale when AUTO selects an FP4 kernel.""" + from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptNvFp4FusedMoEMethod, + ) + from sglang.srt.models import bailing_moe, bailing_moe_linear, llada2 + + logits = torch.tensor([[2.0, 1.0, 0.0, -1.0]]) + + def cpu_router(hidden_states, gating_output, topk, renormalize, **kwargs): + return topk_module.fused_topk_torch_native( + hidden_states, gating_output, topk, renormalize + ) + + cases = ( + (bailing_moe, bailing_moe.BailingMoESparseMoeBlock, None), + (bailing_moe_linear, bailing_moe_linear.BailingMoE, 0), + (llada2, llada2.LLaDA2MoeSparseMoeBlock, None), + ) + for module, block_type, shared in cases: + config = SimpleNamespace( + num_experts_per_tok=2, + norm_topk_prob=True, + hidden_size=4, + moe_intermediate_size=96, + num_experts=4, + num_shared_experts=shared, + hidden_act="silu", + routed_scaling_factor=2.5, + score_function="softmax", + router_dtype="fp32", + ) + if module is bailing_moe: + # Grouped softmax still requires materialized routes. + config.n_group = 2 + config.topk_group = 1 + method = object.__new__(ModelOptNvFp4FusedMoEMethod) + with ( + self.subTest(model=block_type.__name__), + get_flags().moe.override(runner_backend=MoeRunnerBackend.AUTO), + patch.object(method, "create_weights"), + patch.object( + module, + "get_moe_impl_class", + return_value=partial(FusedMoE, quant_method=method), + ), + patch( + "sglang.srt.layers.quantization.modelopt_quant.is_cuda", + return_value=True, + ), + patch( + "sglang.srt.layers.quantization.modelopt_quant.get_device_capability", + return_value=(10, 0), + ), + patch.object(llada2, "_is_npu", False), + patch.object(topk_module, "fused_topk", side_effect=cpu_router), + patch.object( + topk_module, + "grouped_topk_gpu", + topk_module.grouped_topk_gpu._torchdynamo_orig_callable, + ), + ): + block = block_type(config=config, layer_id=0) + self.assertEqual( + block.experts.runner_backend, MoeRunnerBackend.FLASHINFER_TRTLLM + ) + output = block.topk.forward_cuda(torch.ones((1, 4)), logits) + self.assertEqual(output.format, TopKOutputFormat.STANDARD) + torch.testing.assert_close( + output.topk_weights, + torch.softmax(logits[:, :2], dim=-1) * 2.5, + ) + + def test_router_rebinding_rejects_conflicting_contract_before_mutation(self): + for first, second in ( + (MoeRunnerBackend.FLASHINFER_TRTLLM, MoeRunnerBackend.TRITON), + (MoeRunnerBackend.FLASHINFER_MXFP4, None), + (None, MoeRunnerBackend.FLASHINFER_MXFP4), + ): + with self.subTest(first=first, second=second): + topk = TopK( + top_k=2, + layer_id=0, + fused_gate_packed_ids=True, + allow_routed_experts_capture=False, + ) + topk.bind_runner_backend(first) + original = vars(topk.topk_config).copy() + with self.assertRaisesRegex(RuntimeError, "different MoE runner"): + topk.bind_runner_backend(second) + self.assertEqual(vars(topk.topk_config), original) + topk.bind_runner_backend(first) + self.assertEqual(vars(topk.topk_config), original) + + def test_router_rebinding_rejects_a_different_native_policy(self): + topk = TopK(top_k=2, layer_id=0) + topk.bind_runner_backend( + MoeRunnerBackend.FLASHINFER_TRTLLM, + routing_method_type=RoutingMethodType.Renormalize, + ) + with self.assertRaisesRegex(RuntimeError, "different MoE runner"): + topk.bind_runner_backend( + MoeRunnerBackend.FLASHINFER_TRTLLM, + routing_method_type=RoutingMethodType.Default, + ) + self.assertEqual( + topk.topk_config.routing_method_type, RoutingMethodType.Renormalize + ) + + def test_implicit_router_without_a_layer_contract_fails_closed(self): + """A forgotten model association must fail before invoking a wrong kernel.""" + topk = TopK(top_k=2, layer_id=0) + with self.assertRaisesRegex(RuntimeError, "owning experts"): + topk.forward_cuda(torch.ones((1, 4)), torch.ones((1, 4))) + + def test_explicit_standard_router_preserves_its_format(self): + """Routers consumed by model-side operations must stay materialized.""" + topk = TopK(top_k=2, layer_id=0, output_format=TopKOutputFormat.STANDARD) + topk.bind_runner_backend(MoeRunnerBackend.FLASHINFER_TRTLLM) + self.assertEqual(topk.topk_config.output_format, TopKOutputFormat.STANDARD) + + if __name__ == "__main__": import pytest diff --git a/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py b/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py index e0ea1e401d..ee5639bb9c 100644 --- a/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py +++ b/test/registered/unit/layers/moe/test_standard_dispatcher_runner_backend.py @@ -9,6 +9,7 @@ nvidia/Qwen3.8-Flash-Next-NVFP4 at --tp 2 --ep 2). """ from types import SimpleNamespace +from unittest.mock import patch import pytest import torch @@ -17,7 +18,7 @@ from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig from sglang.srt.layers.moe.token_dispatcher import standard from sglang.srt.layers.moe.topk import StandardTopKOutput from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend -from sglang.srt.runtime_context import get_flags +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=8, suite="stage-b-test-cpu-intel") @@ -28,23 +29,29 @@ NUM_LOCAL_EXPERTS = 4 @pytest.fixture def ep2_rank0(monkeypatch): - moe = get_flags().moe - saved = (moe.runner_backend, moe.a2a_backend) - moe.a2a_backend = MoeA2ABackend.NONE monkeypatch.setattr(standard, "get_device", lambda: "cpu") - monkeypatch.setattr( - standard, - "get_parallel", - lambda: SimpleNamespace(moe_ep_size=2, moe_ep_rank=0), - ) monkeypatch.setattr( standard, "get_lora", lambda: SimpleNamespace(enable_lora=False, lora_paths=None), - raising=False, ) - yield moe - moe.runner_backend, moe.a2a_backend = saved + with ( + get_context().override_server_args(model_path="dummy", device="cpu"), + get_flags().moe.override( + a2a_backend=MoeA2ABackend.NONE, runner_backend=MoeRunnerBackend.AUTO + ), + get_parallel().override( + tp_size=2, + tp_rank=0, + attn_tp_size=2, + attn_tp_rank=0, + moe_ep_size=2, + moe_ep_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + ), + ): + yield get_flags().moe @pytest.fixture @@ -53,12 +60,13 @@ def cutlass_pin(ep2_rank0): return ep2_rank0 -def _config(): +def _config(runner_backend=None): return MoeRunnerConfig( num_experts=NUM_EXPERTS, num_local_experts=NUM_LOCAL_EXPERTS, num_fused_shared_experts=0, top_k=2, + runner_backend=runner_backend, ) @@ -74,15 +82,13 @@ def _dispatch_ids(dispatcher, topk_ids): return dispatcher.dispatch(hidden_states, topk_output).topk_output.topk_ids -def test_global_backend_still_decides_without_a_runner(cutlass_pin): +def test_direct_local_backend_is_independent_of_request(cutlass_pin): dispatcher = standard.StandardDispatcher(_config()) - assert dispatcher.skip_local_expert_mapping + assert not dispatcher.skip_local_expert_mapping def test_triton_runner_gets_local_ids_under_cutlass_pin(cutlass_pin): - dispatcher = standard.StandardDispatcher( - _config(), runner_backend=MoeRunnerBackend.TRITON - ) + dispatcher = standard.StandardDispatcher(_config(MoeRunnerBackend.TRITON)) assert not dispatcher.skip_local_expert_mapping topk_ids = torch.tensor([[0, 5], [3, 7]], dtype=torch.int32, device="cpu") @@ -94,30 +100,25 @@ def test_triton_runner_gets_local_ids_under_cutlass_pin(cutlass_pin): def test_runner_that_keeps_global_ids_changes_nothing(cutlass_pin): dispatcher = standard.StandardDispatcher( - _config(), runner_backend=MoeRunnerBackend.FLASHINFER_CUTLASS + _config(MoeRunnerBackend.FLASHINFER_CUTLASS) ) assert dispatcher.skip_local_expert_mapping -def test_runner_never_turns_mapping_off(ep2_rank0): - # Under auto a built runner is never AUTO, e.g. ModelOpt NVFP4 resolving - # to flashinfer_trtllm on sm100; the global layout must stay in charge. +def test_effective_global_backend_keeps_global_ids(ep2_rank0): + # A resolved global-ID backend owns the layout even under an AUTO request. ep2_rank0.runner_backend = MoeRunnerBackend.AUTO dispatcher = standard.StandardDispatcher( - _config(), runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM + _config(MoeRunnerBackend.FLASHINFER_TRTLLM) ) - assert not dispatcher.skip_local_expert_mapping + assert dispatcher.skip_local_expert_mapping -def test_aiter_flag_follows_global_backend(ep2_rank0, monkeypatch): - # Unquantized MoE on ROCm builds a triton runner next to its AITER one; - # the AITER expert mask must still be built from the global backend. +def test_aiter_flag_follows_effective_backend(ep2_rank0, monkeypatch): + # A Triton fallback uses local IDs even if AITER is available to other layers. ep2_rank0.runner_backend = MoeRunnerBackend.AUTO - monkeypatch.setattr(standard, "_use_aiter", True) - dispatcher = standard.StandardDispatcher( - _config(), runner_backend=MoeRunnerBackend.TRITON - ) - assert dispatcher.use_aiter_moe_runner + dispatcher = standard.StandardDispatcher(_config(MoeRunnerBackend.TRITON)) + assert not dispatcher.use_aiter_moe_runner def test_lora_with_ep_is_rejected_on_a_remapped_layer(cutlass_pin, monkeypatch): @@ -127,20 +128,28 @@ def test_lora_with_ep_is_rejected_on_a_remapped_layer(cutlass_pin, monkeypatch): lambda: SimpleNamespace(enable_lora=True, lora_paths=None), ) with pytest.raises(ValueError, match="LoRA"): - standard.StandardDispatcher(_config(), runner_backend=MoeRunnerBackend.TRITON) + standard.StandardDispatcher(_config(MoeRunnerBackend.TRITON)) def test_fp8_triton_fallback_reaches_the_dispatcher(cutlass_pin): - from sglang.srt.layers.moe.fused_moe_triton.layer import create_moe_dispatcher + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod method = object.__new__(Fp8MoEMethod) method.is_fp4_expert = False - method.create_moe_runner(SimpleNamespace(), _config()) + method.use_mxfp8 = False + with patch.object(method, "create_weights"): + layer = FusedMoE( + num_experts=NUM_EXPERTS, + top_k=2, + hidden_size=16, + intermediate_size=128, + layer_id=0, + quant_method=method, + ) assert method.runner.runner_backend.is_triton() - - dispatcher = create_moe_dispatcher(_config(), method) - assert not dispatcher.skip_local_expert_mapping + assert layer.moe_runner_config.runner_backend.is_triton() + assert not layer.dispatcher.skip_local_expert_mapping if __name__ == "__main__": diff --git a/test/registered/unit/layers/muse_spark/test_muse_spark_runner_dispatch.py b/test/registered/unit/layers/muse_spark/test_muse_spark_runner_dispatch.py new file mode 100644 index 0000000000..c76859f053 --- /dev/null +++ b/test/registered/unit/layers/muse_spark/test_muse_spark_runner_dispatch.py @@ -0,0 +1,129 @@ +"""Muse Spark's custom experts must declare the TRT-LLM dispatch contract.""" + +import unittest +from contextlib import ExitStack +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.layers.moe.token_dispatcher import flashinfer +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.moe.utils import FlashinferA2ADispatchType +from sglang.srt.layers.muse_spark.muse_spark_bf16_moe import MuseSpark1xBf16Experts +from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import MuseSpark1xFp4Experts +from sglang.srt.layers.quantization import modelopt_quant +from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config +from sglang.srt.runtime_context import get_context, get_flags, get_forward, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + + +class _CpuMoeAlltoAll: + def __init__(self, **_kwargs): + self.invalid_token_expert_id = None + + def dispatch(self, _topk_ids, payloads, _max_tokens, **kwargs): + self.invalid_token_expert_id = kwargs["invalid_token_expert_id"] + return payloads + + +class TestMuseSparkRunnerDispatch(CustomTestCase): + def test_custom_expert_config_selects_trtllm_receive_sentinel(self): + """Unoccupied SP receive slots must use the direct TRT-LLM consumer's + sentinel even though these experts do not construct a generic runner. + """ + with ( + get_context().override_server_args(model_path="dummy"), + get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + a2a_backend=MoeA2ABackend.NONE, + ), + get_parallel().override( + tp_size=2, + tp_rank=1, + attn_tp_size=2, + attn_tp_rank=1, + moe_ep_size=2, + moe_ep_rank=1, + ), + get_forward().scoped(sp_active=True), + patch.object( + modelopt_quant, + "get_platform", + return_value=SimpleNamespace(is_blackwell=True), + ), + ): + experts = ( + MuseSpark1xBf16Experts( + num_experts=4, + num_local_experts=2, + local_expert_offset=2, + expert_hidden_size=128, + intermediate_size=128, + top_k=2, + ), + MuseSpark1xFp4Experts( + num_experts=4, + num_local_experts=2, + local_expert_offset=2, + hidden_size=128, + intermediate_size=128, + top_k=2, + quant_config=ModelOptFp4Config( + is_checkpoint_nvfp4_serialized=True, group_size=16 + ), + ), + ) + # Replace FlashInfer's distributed workspace allocation, not the + # SGLang constructor or its runner/sentinel selection. + with ExitStack() as stack: + for name, value in { + "use_flashinfer": True, + "moe_a2a_get_workspace_size_per_rank": lambda **_: 128, + "Mapping": lambda **_: None, + "MnnvlConfig": lambda **_: None, + "TorchDistributedCommBackend": lambda _: None, + "MoeAlltoAll": _CpuMoeAlltoAll, + "get_flashinfer_a2a_dispatch_type": lambda: ( + FlashinferA2ADispatchType.BF16 + ), + "get_is_extend_in_batch": lambda: True, + "get_dp_global_num_tokens": lambda: None, + "is_dp_attention_enabled": lambda: False, + }.items(): + stack.enter_context( + patch.object(flashinfer, name, value, create=True) + ) + for expert in experts: + with self.subTest(expert=type(expert).__name__): + dispatcher = flashinfer.FlashinferDispatcher( + group=SimpleNamespace(size=lambda: 2, rank=lambda: 1), + router_topk=2, + num_experts=4, + num_local_experts=2, + hidden_size=128, + moe_runner_config=expert.moe_runner_config, + enable_sp_prefill_a2a=True, + ) + topk = StandardTopKOutput( + torch.full((2, 2), 0.5), + torch.tensor([[0, 3], [2, 1]], dtype=torch.int32), + None, + ) + output = dispatcher.dispatch( + torch.zeros((2, 128), dtype=torch.bfloat16), topk + ) + self.assertEqual( + dispatcher.sp_prefill_moe_a2a.invalid_token_expert_id, -1 + ) + torch.testing.assert_close( + output.topk_output.topk_ids, topk.topk_ids + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/layers/quantization/test_deepgemm_ue8m0_requant.py b/test/registered/unit/layers/quantization/test_deepgemm_ue8m0_requant.py index c6db7e15c6..e71ef4c205 100644 --- a/test/registered/unit/layers/quantization/test_deepgemm_ue8m0_requant.py +++ b/test/registered/unit/layers/quantization/test_deepgemm_ue8m0_requant.py @@ -202,12 +202,11 @@ class TestDeepGemmUE8M0Requant(CustomTestCase): requant.assert_called_once() def test_fp8_moe_requants_standard_layer_for_deepgemm(self): - method = fp8_quant.Fp8MoEMethod.__new__(fp8_quant.Fp8MoEMethod) - method.convert_mxfp8_to_block = False - method.use_mxfp8 = False - method.is_fp4_expert = False - method.dequant_fp4_to_fp8 = False - method.quant_config = unittest.mock.Mock(weight_block_size=BLOCK_SIZE) + method = fp8_quant.Fp8MoEMethod( + fp8_quant.Fp8Config( + is_checkpoint_fp8_serialized=True, weight_block_size=BLOCK_SIZE + ) + ) layer = torch.nn.Module() layer.w13_weight, layer.w13_weight_scale_inv = _make_params() diff --git a/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py b/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py index 55f6799410..80fa2c81fc 100644 --- a/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py +++ b/test/registered/unit/layers/quantization/test_fp8_moe_runner_ownership.py @@ -7,6 +7,7 @@ the MxFP4 wrapper methods borrow an `Fp8MoEMethod` for weight loading only and never give it a `moe_runner_config` (issue #36264). """ +import math import sys import types import unittest @@ -162,5 +163,80 @@ class TestFp8MoEAiterQuantInfo(CustomTestCase): self.assertIs(quant_info.expert_mask, layer.dispatcher.expert_mask_gpu) +class TestFp8MoEAiterWeightLayout(CustomTestCase): + @staticmethod + def _make_layer(): + layer = torch.nn.Module() + for name, shape in (("w13_weight", (2, 32, 16)), ("w2_weight", (2, 16, 16))): + weight = (torch.arange(math.prod(shape)).reshape(shape) % 13).to( + torch.float8_e4m3fn + ) + weight.view(torch.int8).reshape(-1)[0] = -128 + layer.register_parameter( + name, torch.nn.Parameter(weight, requires_grad=False) + ) + layer.register_parameter( + f"{name}_scale_inv", + torch.nn.Parameter(torch.ones((2, 1, 1)), requires_grad=False), + ) + return layer + + @staticmethod + def _shuffle(weight, block): + return weight.view(torch.uint8).flip(-1).contiguous().view(weight.dtype) + + def _check_layout(self, fnuz): + with get_flags().moe.override(runner_backend=MoeRunnerBackend.TRITON): + for backend in (MoeRunnerBackend.TRITON, None, MoeRunnerBackend.AITER): + with self.subTest(backend=backend, fnuz=fnuz): + method = Fp8MoEMethod( + Fp8Config( + is_checkpoint_fp8_serialized=True, + weight_block_size=[128, 128], + ) + ) + if backend is not None: + method.runner = SimpleNamespace(runner_backend=backend) + layer = self._make_layer() + expected = {} + for name in ("w13_weight", "w2_weight"): + weight = getattr(layer, name).detach().clone() + if fnuz: + weight, _, _ = fp8_module.normalize_e4m3fn_to_e4m3fnuz( + weight, getattr(layer, f"{name}_scale_inv"), None + ) + if backend == MoeRunnerBackend.AITER: + weight = self._shuffle(weight, (16, 16)) + expected[name] = weight + with ( + patch.object(fp8_module, "_is_fp8_fnuz", fnuz), + patch.object(fp8_module, "_use_aiter", True), + patch.object( + fp8_module, "shuffle_weight", self._shuffle, create=True + ), + ): + method.process_weights_after_loading_block_quant(layer) + for name, weight in expected.items(): + actual = getattr(layer, name) + self.assertEqual(actual.dtype, weight.dtype) + self.assertTrue( + torch.equal( + actual.view(torch.uint8), weight.view(torch.uint8) + ) + ) + self.assertTrue( + torch.equal( + getattr(layer, f"{name}_scale_inv"), + torch.full((2, 1, 1), 2.0 if fnuz else 1.0), + ) + ) + + def test_fnuz_shuffles_only_for_aiter_runner(self): + self._check_layout(fnuz=True) + + def test_non_fnuz_shuffles_only_for_aiter_runner(self): + self._check_layout(fnuz=False) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py b/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py index 09428cc325..c955c10079 100644 --- a/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py +++ b/test/registered/unit/layers/quantization/test_fp8_moe_weight_gating.py @@ -12,7 +12,7 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=10, suite="base-a-test-cpu") import unittest -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import torch @@ -89,5 +89,165 @@ class TestFp8MoEWeightGating(CustomTestCase): self.assertEqual(params["w13_weight_scale"].shape, (NUM_EXPERTS, 1)) +class TestFallbackWeightPadding(CustomTestCase): + def setUp(self): + from sglang.srt.layers.moe import MoeA2ABackend, MoeRunnerBackend + from sglang.srt.runtime_context import get_context, get_flags + + self.enterContext( + get_context().override_server_args(model_path="dummy", device="cpu") + ) + self.enterContext( + get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED, + a2a_backend=MoeA2ABackend.NONE, + ) + ) + # CPU's always-padded loader would hide a GPU fallback's uninitialized tail. + self.enterContext( + patch("sglang.srt.layers.moe.fused_moe_triton.layer._is_cpu", False) + ) + self.enterContext( + patch("sglang.srt.layers.moe.fused_moe_triton.layer._use_aiter", False) + ) + self.enterContext( + patch( + "sglang.srt.layers.quantization.modelopt_quant.cutlass_fp8_supported", + return_value=False, + ) + ) + + @staticmethod + def _expert_output(hidden, w13, w2, is_gated): + projected = hidden @ w13.float().t() + if is_gated: + gate, up = projected.chunk(2, dim=-1) + activated = torch.nn.functional.silu(gate) * up + else: + activated = projected.relu().square() + return activated @ w2.float().t() + + @staticmethod + def _build_layer(logical_size, tp_size, rank, presharded, is_gated): + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptFp8Config, + ModelOptFp8MoEMethod, + ) + from sglang.srt.runtime_context import get_parallel + + with get_parallel().override( + tp_size=tp_size, + tp_rank=rank, + attn_tp_size=tp_size, + attn_tp_rank=rank, + moe_tp_size=tp_size, + moe_tp_rank=rank, + moe_ep_size=1, + moe_ep_rank=0, + ): + config = ModelOptFp8Config(is_checkpoint_fp8_serialized=True) + return FusedMoE( + num_experts=1, + top_k=1, + hidden_size=128, + intermediate_size=logical_size * tp_size, + layer_id=0, + quant_config=config, + quant_method=ModelOptFp8MoEMethod(config), + use_presharded_weights=presharded, + is_gated=is_gated, + activation="silu" if is_gated else "relu2", + ) + + def _load_and_run(self, layer, checkpoint, hidden, logical_size): + layer.w13_weight.data.fill_(float("nan")) + layer.w2_weight.data.fill_(float("nan")) + is_gated = layer.moe_runner_config.is_gated + shard_ids = ("w3", "w1", "w2") if is_gated else ("w1", "w2") + for shard_id in shard_ids: + weight = checkpoint[shard_id] + if layer.use_presharded_weights: + dim = 1 if shard_id == "w2" else 0 + weight = weight.narrow( + dim, layer.moe_tp_rank * logical_size, logical_size + ) + param = layer.w2_weight if shard_id == "w2" else layer.w13_weight + layer.weight_loader( + param=param, + loaded_weight=weight, + weight_name="weight", + shard_id=shard_id, + expert_id=0, + ) + physical_size = layer.intermediate_size_per_partition + for part in layer.w13_weight[0].float().split(physical_size): + self.assertEqual(torch.count_nonzero(part[logical_size:]).item(), 0) + self.assertEqual( + torch.count_nonzero(layer.w2_weight[0].float()[:, logical_size:]).item(), 0 + ) + return self._expert_output( + hidden, layer.w13_weight[0], layer.w2_weight[0], is_gated + ) + + def test_fallback_loading_initializes_padding_on_every_load(self): + """Padded FP8 fallback weights must produce the unpadded expert's output.""" + from sglang.srt.layers.moe import MoeRunnerBackend + + hidden_size = 128 + hidden = torch.arange(3 * hidden_size).reshape(3, hidden_size).remainder(5) + hidden = (hidden.float() - 2) / 8 + cases = ( + (192, 1, False, True), + (192, 2, False, True), + (192, 2, True, True), + (256, 1, False, True), + (192, 1, False, False), + ) + for logical_size, tp_size, presharded, is_gated in cases: + with self.subTest( + intermediate=logical_size, + tp_size=tp_size, + presharded=presharded, + is_gated=is_gated, + ): + global_size = logical_size * tp_size + values = torch.arange(global_size * hidden_size).reshape( + global_size, hidden_size + ) + gate = (values.remainder(7).float() - 3) / 32 + up = (values.remainder(5).float() - 2) / 16 + down = (values.t().remainder(3).float() - 1) / 16 + layers = [ + self._build_layer(logical_size, tp_size, rank, presharded, is_gated) + for rank in range(tp_size) + ] + for layer in layers: + self.assertEqual(layer.runner_backend, MoeRunnerBackend.TRITON) + self.assertFalse(layer.use_padded_loading) + + for multiplier in (1, 2): + checkpoint = { + "w1": (gate * multiplier).to(torch.float8_e4m3fn), + "w2": (down * multiplier).to(torch.float8_e4m3fn), + "w3": (up * multiplier).to(torch.float8_e4m3fn), + } + outputs = [ + self._load_and_run(layer, checkpoint, hidden, logical_size) + for layer in layers + ] + reference_w13 = checkpoint["w1"].float() + if is_gated: + reference_w13 = torch.cat( + (reference_w13, checkpoint["w3"].float()) + ) + expected = self._expert_output( + hidden, reference_w13, checkpoint["w2"], is_gated + ) + actual = torch.stack(outputs).sum(dim=0) + self.assertTrue(torch.isfinite(actual).all().item()) + torch.testing.assert_close(actual, expected, rtol=2e-5, atol=2e-5) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py index e434dd7cdf..84a6213351 100644 --- a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_dispatch.py @@ -14,9 +14,10 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=6, suite="base-a-test-cpu") +import sys import unittest from contextlib import contextmanager -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace from unittest import mock import torch @@ -32,7 +33,9 @@ from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( # isort: on from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig -from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import StandardTopKOutput +from sglang.srt.layers.moe.utils import MoeRunnerBackend, RoutingMethodType from sglang.test.test_utils import CustomTestCase NUM_EXPERTS = 8 @@ -174,6 +177,204 @@ class TestNvFp4MoeDispatch(CustomTestCase): ((_, quant_info),) = method.runner.calls self.assertIsInstance(quant_info, FlashInferTrtllmFp4MoeQuantInfo) + def test_materialized_router_weights_reach_fp8_and_bf16_routed_kernels(self): + """Materialized custom weights must not be sent to a logits router or normalized again.""" + from sglang.srt.layers.moe import topk as topk_module + from sglang.srt.layers.moe.fused_moe_triton.layer import ( + _fuses_routed_scaling_factor_in_topk, + ) + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as kernels + from sglang.srt.layers.quantization.fp8 import Fp8MoEMethod + from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod + from sglang.srt.runtime_context import get_parallel + + logits = torch.tensor([[0.0, 1.0, -2.0, 2.0]]) + ids = torch.tensor([[3, 1]], dtype=torch.int32) + routed_scale = 2.5 + weights = torch.sigmoid(logits).gather(1, ids.long()) + weights = weights / weights.sum(dim=-1, keepdim=True) * routed_scale + hidden = torch.zeros((1, 4), dtype=torch.bfloat16) + config = MoeRunnerConfig( + top_k=2, + activation="silu", + num_local_experts=4, + num_fused_shared_experts=0, + intermediate_size_per_partition=128, + routed_scaling_factor=routed_scale, + ) + expected = (ids.float() * weights).sum(dim=-1, keepdim=True).expand_as(hidden) + fake_flashinfer = ModuleType("flashinfer.fused_moe") + fake_flashinfer.Fp8QuantizationType = SimpleNamespace(MxFp8=1, DeepSeekFp8=0) + fake_core = ModuleType("flashinfer.fused_moe.core") + fake_core.ActivationType = SimpleNamespace( + Swiglu=SimpleNamespace(value=0), + Geglu=SimpleNamespace(value=1), + Situ=SimpleNamespace(value=2), + ) + + def routed_result(kwargs): + self.assertEqual(kwargs["routing_method_type"], RoutingMethodType.TopK) + routing = kwargs["topk_ids"] + if isinstance(routing, tuple): + seen_ids, seen_weights = routing + else: + seen_ids, seen_weights = routing, kwargs["topk_weights"] + torch.testing.assert_close(seen_ids, ids) + torch.testing.assert_close(seen_weights, weights) + return ( + (seen_ids.float() * seen_weights) + .sum(dim=-1, keepdim=True) + .expand_as(hidden) + .to(hidden.dtype) + ) + + def fp8_kernel(**kwargs): + kwargs["output"].copy_(routed_result(kwargs)) + + def bf16_kernel(**kwargs): + return routed_result(kwargs) + + fake_flashinfer.trtllm_bf16_routed_moe = bf16_kernel + common = dict( + w13_weight=torch.zeros((4, 256, 4)), + w2_weight=torch.zeros((4, 4, 128)), + global_num_experts=4, + local_num_experts=4, + local_expert_offset=0, + routing_method_type=None, + intermediate_size=128, + ) + quant_info = SimpleNamespace( + **common, + block_quant=True, + use_mxfp8=False, + weight_block_k=128, + w13_weight_scale_inv=torch.ones((4, 2, 1)), + w2_weight_scale_inv=torch.ones((4, 1, 1)), + gemm1_alpha=None, + gemm1_beta=None, + gemm1_clamp_limit=None, + activation_type=None, + ) + bf16_info = SimpleNamespace( + gemm1_weights=common["w13_weight"], + gemm2_weights=common["w2_weight"], + global_num_experts=4, + local_expert_offset=0, + ) + with ( + get_parallel().override(tp_group=None), + mock.patch.dict( + sys.modules, + { + "flashinfer.fused_moe": fake_flashinfer, + "flashinfer.fused_moe.core": fake_core, + }, + ), + mock.patch.object(kernels, "is_allocation_symmetric", return_value=False), + mock.patch.object(kernels, "trtllm_moe_enable_pdl", return_value=False), + mock.patch.object( + kernels, + "per_token_group_quant_fp8", + return_value=(hidden, torch.ones((1, 1))), + ), + mock.patch.object( + kernels, "trtllm_fp8_block_scale_routed_moe_out_wrapper", fp8_kernel + ), + ): + for method_type, kernel, info in ( + ( + Fp8MoEMethod, + kernels.fused_experts_none_to_flashinfer_trtllm_fp8, + quant_info, + ), + ( + UnquantizedFusedMoEMethod, + kernels.fused_experts_none_to_flashinfer_trtllm_bf16, + bf16_info, + ), + ): + with self.subTest(method=method_type.__name__): + fold_scale = _fuses_routed_scaling_factor_in_topk( + object.__new__(method_type), MoeRunnerBackend.FLASHINFER_TRTLLM + ) + produced_weights, produced_ids = ( + topk_module.biased_grouped_topk_impl.__wrapped__( + hidden, + logits, + torch.zeros(4), + topk=2, + renormalize=True, + num_expert_group=1, + topk_group=1, + routed_scaling_factor=routed_scale, + apply_routed_scaling_factor_on_output=fold_scale, + ) + ) + dispatch = StandardDispatchOutput( + hidden, + None, + StandardTopKOutput( + produced_weights, produced_ids.to(torch.int32), None + ), + ) + output = kernel(dispatch, info, config) + torch.testing.assert_close( + output.hidden_states, expected.to(hidden.dtype) + ) + + def test_fp8_default_routing_policy_keeps_zero_enum(self): + """The native softmax selector's zero value must not select grouped sigmoid routing.""" + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm as kernels + from sglang.srt.layers.quantization import fp8 + + method = object.__new__(fp8.Fp8MoEMethod) + method.is_fp4_expert = False + method.block_quant = True + method.quant_config = SimpleNamespace( + use_mxfp8=False, weight_block_size=[128, 128] + ) + method.moe_runner_config = MoeRunnerConfig(activation="silu") + method.runner = SimpleNamespace( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + run=lambda dispatch, quant_info: quant_info.routing_method_type, + ) + layer = SimpleNamespace( + num_experts=4, + num_local_experts=4, + moe_ep_rank=0, + w13_weight=torch.zeros((4, 256, 4)), + w2_weight=torch.zeros((4, 4, 128)), + w13_weight_scale_inv=torch.ones((4, 2, 1)), + w2_weight_scale_inv=torch.ones((4, 1, 1)), + _flashinfer_trtllm_gemm1_alpha=None, + _flashinfer_trtllm_gemm1_beta=None, + _flashinfer_trtllm_gemm1_clamp_limit=None, + ) + with ( + mock.patch.object(fp8, "use_intel_amx_backend", return_value=False), + mock.patch.object(fp8, "is_xpu", return_value=False), + mock.patch.object( + fp8, + "get_moe_runner_backend", + return_value=MoeRunnerBackend.FLASHINFER_TRTLLM, + ), + mock.patch.object(kernels, "get_activation_type", return_value=0), + ): + for policy, expected in ( + (RoutingMethodType.Default, 0), + (RoutingMethodType.Renormalize, 1), + (None, 2), + ): + with self.subTest(policy=policy): + layer.routing_method_type = policy + self.assertEqual( + method.apply( + layer, SimpleNamespace(hidden_states=torch.zeros((1, 4))) + ), + expected, + ) + def test_missing_trtllm_prep_names_the_missing_field(self): method = _method_set_up_for(MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED) layer = _trtllm_prepared_layer() diff --git a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py index 57915f8eac..aea072cb06 100644 --- a/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py +++ b/test/registered/unit/layers/test_kimi_mlp_parallel_groups.py @@ -10,6 +10,7 @@ import torch.nn.functional as F from sglang.srt.layers import linear from sglang.srt.layers.dp_attention import initialize_dp_attention_flags +from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.models import kimi_k3 from sglang.srt.runtime_context import SpawnRanks, reset_context from sglang.srt.server_args import ServerArgs @@ -32,6 +33,8 @@ def build_mlp(group=None, *, width=16, quant_config=None): class _RoutedExperts(torch.nn.Module): + routing_method_type = None + runner_backend = MoeRunnerBackend.TRITON should_fuse_routed_scaling_factor_in_topk = False def __init__(self, **kwargs): diff --git a/test/registered/unit/lora/test_mem_pool_ep_unit.py b/test/registered/unit/lora/test_mem_pool_ep_unit.py index 4d1136f079..1f2dd1dac2 100644 --- a/test/registered/unit/lora/test_mem_pool_ep_unit.py +++ b/test/registered/unit/lora/test_mem_pool_ep_unit.py @@ -176,6 +176,7 @@ def _load_standard_dispatcher(get_parallel, get_backend): "Optional": typing.Optional, "get_parallel": get_parallel, "get_moe_runner_backend": get_backend, + "get_lora": lambda: types.SimpleNamespace(enable_lora=False, lora_paths=None), "_use_aiter": False, "get_moe_a2a_backend": lambda: types.SimpleNamespace( supports_aiter=lambda: False @@ -945,6 +946,7 @@ class TestModuleLevelHelpers(unittest.TestCase): moe_utils.get_moe_runner_backend = lambda: state.backend for backend in backends: state.backend = backend + config.runner_backend = backend with mock.patch.dict( "sys.modules", {"sglang.srt.layers.moe.utils": moe_utils} ): diff --git a/test/registered/unit/models/test_llada2.py b/test/registered/unit/models/test_llada2.py index 84ff53d4c2..e109f7e6ad 100644 --- a/test/registered/unit/models/test_llada2.py +++ b/test/registered/unit/models/test_llada2.py @@ -11,7 +11,8 @@ from torch import nn from sglang.srt.dllm.config import DllmConfig from sglang.srt.environ import envs -from sglang.srt.layers.moe.utils import MoeRunnerBackend +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend from sglang.srt.layers.quantization.modelopt_quant import ( ModelOptNvFp4FusedMoEMethod, ) @@ -19,13 +20,12 @@ from sglang.srt.layers.quantization.unquant import UnquantizedFusedMoEMethod from sglang.srt.models.llada2 import ( LLaDA2MoeGate, LLaDA2MoeSparseMoeBlock, - _get_effective_moe_runner_backend, _make_block_routing_triton_output, _prepare_llada2_language_weights, _require_block_routing_ep1, _require_block_routing_runner_compatibility, ) -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_context, get_flags, get_parallel from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -33,6 +33,39 @@ register_cpu_ci(est_time=5, suite="base-a-test-cpu") class TestLLaDA2BlockRoutingValidation(CustomTestCase): + def setUp(self): + self.enterContext( + get_context().override_server_args(model_path="dummy", device="cpu") + ) + self.enterContext( + get_flags().moe.override( + runner_backend=MoeRunnerBackend.TRITON, a2a_backend=MoeA2ABackend.NONE + ) + ) + self.enterContext( + get_parallel().override( + tp_size=1, + tp_rank=0, + attn_tp_size=1, + attn_tp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + ) + ) + + def _build_experts(self, method): + with patch.object(method, "create_weights"): + return FusedMoE( + num_experts=4, + top_k=2, + hidden_size=8, + intermediate_size=256, + layer_id=0, + quant_method=method, + ) + def test_block_routing_accepts_ep1(self): _require_block_routing_ep1(1) @@ -45,7 +78,7 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): expert_capacity=48, ) - with get_parallel().override(tp_size=4, moe_ep_size=4): + with get_parallel().override(tp_size=4, attn_tp_size=4, moe_ep_size=4): with self.assertRaisesRegex( ValueError, r"does not support expert parallelism.*moe_ep_size=4", @@ -79,38 +112,24 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): return_value=(10, 0), ) @patch("sglang.srt.layers.quantization.modelopt_quant.is_cuda", return_value=True) - @patch("sglang.srt.layers.quantization.modelopt_quant.get_moe_runner_backend") def test_block_routing_rejects_auto_selected_plain_flashinfer( self, - backend, _is_cuda, _capability, runner, ): - backend.return_value = MoeRunnerBackend.AUTO quant_method = ModelOptNvFp4FusedMoEMethod.__new__(ModelOptNvFp4FusedMoEMethod) - quant_method.create_moe_runner( - SimpleNamespace(), - SimpleNamespace(), - ) - experts = SimpleNamespace(quant_method=quant_method) - - self.assertIs( - _get_effective_moe_runner_backend(experts), - MoeRunnerBackend.FLASHINFER_TRTLLM, - ) + with get_flags().moe.override(runner_backend=MoeRunnerBackend.AUTO): + experts = self._build_experts(quant_method) + self.assertIs(experts.runner_backend, MoeRunnerBackend.FLASHINFER_TRTLLM) runner.assert_called_once_with( MoeRunnerBackend.FLASHINFER_TRTLLM, quant_method.moe_runner_config, ) with self.assertRaisesRegex(ValueError, r"does not support.*flashinfer_trtllm"): - _require_block_routing_runner_compatibility(experts) - - @patch("sglang.srt.models.llada2.get_moe_runner_backend") - def test_fused_scaling_contract_applies_scale_once(self, backend): - backend.return_value.is_triton_kernels.return_value = False - backend.return_value.is_aiter.return_value = False + _require_block_routing_runner_compatibility(experts, top_k=2) + def test_fused_scaling_contract_applies_scale_once(self): block = LLaDA2MoeSparseMoeBlock.__new__(LLaDA2MoeSparseMoeBlock) nn.Module.__init__(block) block.layer_id = 0 @@ -118,7 +137,9 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): block.topk = SimpleNamespace( topk_config=SimpleNamespace(allow_routed_experts_capture=False) ) - block.experts = SimpleNamespace(should_fuse_routed_scaling_factor_in_topk=True) + block.experts = SimpleNamespace( + runner_backend=None, should_fuse_routed_scaling_factor_in_topk=True + ) router_logits = torch.zeros((1, 4), dtype=torch.float32) topk_weights = torch.tensor([[0.25, 0.75]], dtype=torch.float32) @@ -144,13 +165,7 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): "sglang.srt.layers.quantization.unquant._use_aiter", True, ), - patch( - "sglang.srt.layers.quantization.unquant.get_moe_runner_backend", - return_value=MoeRunnerBackend.AUTO, - ), - patch( - "sglang.srt.layers.quantization.unquant.get_moe_a2a_backend" - ) as get_a2a_backend, + get_flags().moe.override(runner_backend=MoeRunnerBackend.AUTO), patch( "sglang.srt.layers.quantization.unquant.MoeRunner", side_effect=lambda backend, config: SimpleNamespace( @@ -158,12 +173,8 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): ), ), ): - get_a2a_backend.return_value.supports_aiter.return_value = True quant_method = UnquantizedFusedMoEMethod() - quant_method.create_moe_runner( - SimpleNamespace(intermediate_size_per_partition=256), - SimpleNamespace(), - ) + experts = self._build_experts(quant_method) self.assertIs( quant_method.runner.runner_backend, @@ -181,16 +192,8 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): block.topk = SimpleNamespace( topk_config=SimpleNamespace(allow_routed_experts_capture=False) ) - block.experts = SimpleNamespace( - quant_method=quant_method, - runner=quant_method.runner, - should_fuse_routed_scaling_factor_in_topk=False, - ) - - self.assertIs( - _get_effective_moe_runner_backend(block.experts), - MoeRunnerBackend.AITER, - ) + block.experts = experts + self.assertIs(block.experts.runner_backend, MoeRunnerBackend.AITER) router_logits = torch.zeros((1, 4), dtype=torch.float32) topk_weights = torch.tensor([[0.25, 0.75]], dtype=torch.float32) @@ -210,24 +213,21 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): self.assertIs(result.topk_ids, topk_ids) self.assertIs(result.router_logits, router_logits) - @patch("sglang.srt.models.llada2.get_moe_runner_backend") @patch("sglang.srt.models.llada2.get_global_expert_distribution_recorder") @patch("sglang.srt.models.llada2.capture_routed_experts_if_allowed") def test_block_routing_runs_precomputed_route_hooks( self, capture_routed_experts, get_recorder, - backend, ): - backend.return_value.is_triton_kernels.return_value = False - backend.return_value.is_aiter.return_value = False - block = LLaDA2MoeSparseMoeBlock.__new__(LLaDA2MoeSparseMoeBlock) nn.Module.__init__(block) block.layer_id = 7 block.routed_scaling_factor = 1.0 block.topk = SimpleNamespace(topk_config=object()) - block.experts = SimpleNamespace(should_fuse_routed_scaling_factor_in_topk=False) + block.experts = SimpleNamespace( + runner_backend=None, should_fuse_routed_scaling_factor_in_topk=False + ) topk_ids = torch.tensor([[1, 3]], dtype=torch.int32) block._make_block_topk_output( @@ -243,6 +243,46 @@ class TestLLaDA2BlockRoutingValidation(CustomTestCase): topk_ids=topk_ids ) + def test_direct_standard_contract_does_not_fall_back_to_global_backend(self): + block = LLaDA2MoeSparseMoeBlock.__new__(LLaDA2MoeSparseMoeBlock) + nn.Module.__init__(block) + block.layer_id = 0 + block.routed_scaling_factor = 2.5 + block.topk = SimpleNamespace( + topk_config=SimpleNamespace(allow_routed_experts_capture=False) + ) + block.experts = SimpleNamespace( + runner_backend=None, should_fuse_routed_scaling_factor_in_topk=False + ) + weights = torch.tensor([[0.25, 0.75]]) + ids = torch.tensor([[0, 1]], dtype=torch.int32) + for requested in (MoeRunnerBackend.FLASHINFER_TRTLLM, MoeRunnerBackend.AITER): + with ( + self.subTest(requested=requested), + get_flags().moe.override(runner_backend=requested), + ): + _require_block_routing_runner_compatibility(block.experts, top_k=3) + result = block._make_block_topk_output( + torch.zeros((1, 4)), weights, ids + ) + torch.testing.assert_close(result.topk_weights, weights, rtol=0, atol=0) + self.assertIs(result.topk_ids, ids) + + def test_block_shape_guard_follows_the_layer_backend(self): + with get_flags().moe.override(runner_backend=MoeRunnerBackend.TRITON): + with self.assertRaisesRegex(ValueError, "power of two"): + _require_block_routing_runner_compatibility( + SimpleNamespace(runner_backend=MoeRunnerBackend.TRITON_KERNELS), + top_k=3, + ) + _require_block_routing_runner_compatibility( + SimpleNamespace(runner_backend=MoeRunnerBackend.TRITON_KERNELS), top_k=2 + ) + with get_flags().moe.override(runner_backend=MoeRunnerBackend.TRITON_KERNELS): + _require_block_routing_runner_compatibility( + SimpleNamespace(runner_backend=MoeRunnerBackend.TRITON), top_k=3 + ) + def test_block_routing_rejects_uniform_expert_simulation(self): with ( get_parallel().override(tp_size=1, moe_ep_size=1), diff --git a/test/registered/unit/models/test_native_moe_routing_policy.py b/test/registered/unit/models/test_native_moe_routing_policy.py new file mode 100644 index 0000000000..c8276ee6fa --- /dev/null +++ b/test/registered/unit/models/test_native_moe_routing_policy.py @@ -0,0 +1,302 @@ +"""Model routing policies retain equivalent native selectors and materialized fallbacks.""" + +import sys +import unittest +from types import ModuleType, SimpleNamespace +from unittest.mock import patch + +import torch + +import sglang.srt.layers.moe.topk as topk_module +from sglang.srt.configs.shared_experts import SharedExpertsConfig +from sglang.srt.layers.moe.token_dispatcher.standard import StandardDispatchOutput +from sglang.srt.layers.moe.topk import TopKOutputFormat +from sglang.srt.layers.moe.utils import ( + MoeA2ABackend, + MoeRunnerBackend, + RoutingMethodType, +) +from sglang.srt.layers.quantization.fp8 import Fp8Config, Fp8MoEMethod +from sglang.srt.runtime_context import get_context, get_flags, get_model, get_parallel +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=8, suite="base-a-test-cpu") + + +class TestNativeMoeRoutingPolicy(CustomTestCase): + def setUp(self): + self.enterContext( + get_context().override_server_args(model_path="dummy", device="cpu") + ) + self.enterContext( + get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + a2a_backend=MoeA2ABackend.NONE, + ) + ) + self.enterContext( + get_parallel().override( + tp_size=1, + tp_rank=0, + tp_group=None, + attn_tp_size=1, + attn_tp_rank=0, + attn_tp_group=None, + attn_dp_size=1, + attn_dp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + moe_tp_size=1, + moe_tp_rank=0, + moe_ep_size=1, + moe_ep_rank=0, + ) + ) + self.enterContext(get_model().override(shared_experts=SharedExpertsConfig())) + self.enterContext(patch.object(Fp8MoEMethod, "create_weights")) + self.enterContext( + patch( + "sglang.srt.layers.moe.token_dispatcher.standard.get_device", + return_value="cpu", + ) + ) + self.quant_config = Fp8Config( + is_checkpoint_fp8_serialized=True, weight_block_size=[128, 128] + ) + self.logits = torch.tensor([[3.0, 2.0, 1.0, 0.0, -1.0, -2.0, -3.0, -4.0]]) + self.bias = torch.tensor([0.0, 0.0, 0.0, 0.0, 2.0, 2.0, 0.0, 0.0]) + + def _mimo(self, *, bias=True, renormalize=True): + from sglang.srt.models.mimo_v2 import MiMoV2MoE + + config = SimpleNamespace( + n_routed_experts=8, + num_experts_per_tok=2, + hidden_size=4, + moe_intermediate_size=128, + hidden_act="silu", + topk_method="noaux_tc" if bias else "group_limited_greedy", + norm_topk_prob=renormalize, + n_group=2, + topk_group=1, + scoring_func="sigmoid" if bias else "softmax", + ) + block = MiMoV2MoE(config, layer_id=0, quant_config=self.quant_config) + if bias: + block.gate.e_score_correction_bias.data.copy_(self.bias) + return block + + def _bailing(self, *, grouped, bias, renormalize=True): + from sglang.srt.models.bailing_moe import BailingMoESparseMoeBlock + + config = SimpleNamespace( + num_experts=8, + num_experts_per_tok=2, + num_shared_experts=None, + hidden_size=4, + moe_intermediate_size=128, + hidden_act="silu", + n_group=2 if grouped else 0, + topk_group=1 if grouped else 0, + norm_topk_prob=renormalize, + routed_scaling_factor=2.5, + moe_router_enable_expert_bias=bias, + ) + block = BailingMoESparseMoeBlock( + config=config, layer_id=0, quant_config=self.quant_config + ) + if bias: + block.gate.expert_bias.data.copy_(self.bias) + return block + + def _cohere(self, *, renormalize): + from sglang.srt.models.cohere2_moe import Cohere2MoeSparseMoeBlock + + return Cohere2MoeSparseMoeBlock( + config=SimpleNamespace( + num_experts=8, + num_experts_per_tok=2, + hidden_size=4, + intermediate_size=128, + expert_selection_fn="sigmoid", + norm_topk_prob=renormalize, + ), + layer_id=0, + quant_config=self.quant_config, + ) + + @staticmethod + def _weighted_experts(ids, weights, hidden_states): + values = ids.float() + 1 + return (values * weights).sum(dim=-1, keepdim=True).expand_as(hidden_states) + + def _native_output(self, block): + from sglang.srt.layers.moe.moe_runner import flashinfer_trtllm + + hidden_states = torch.ones((1, 4)) + self.assertEqual( + block.topk.topk_config.output_format, TopKOutputFormat.BYPASSED + ) + routing = block.topk.forward_cuda(hidden_states, self.logits) + policy = block.experts.routing_method_type + self.assertEqual(block.experts.moe_runner_config.routing_method_type, policy) + + def native_kernel(*, routing_logits, routing_method_type, output, **kwargs): + self.assertEqual(routing_method_type, policy) + if routing_method_type == RoutingMethodType.DeepSeekV3: + scores = routing_logits.sigmoid() + choices = scores + kwargs["routing_bias"] + group_scores = choices.reshape(1, 2, 4).topk(2, dim=-1).values.sum(-1) + selected = group_scores.argmax(-1).item() + ids = choices[:, selected * 4 : (selected + 1) * 4].topk(2).indices + ids = ids + selected * 4 + weights = scores.gather(1, ids) + elif routing_method_type == RoutingMethodType.SigmoidRenorm: + weights, ids = routing_logits.sigmoid().topk(2) + else: + self.assertEqual( + routing_method_type, RoutingMethodType.RenormalizeNaive + ) + weights, ids = routing_logits.softmax(-1).topk(2) + weights = weights / weights.sum(-1, keepdim=True) + weights = weights * kwargs["routed_scaling_factor"] + output.copy_(self._weighted_experts(ids, weights, hidden_states)) + + quant_info = flashinfer_trtllm.FlashInferTrtllmFp8MoeQuantInfo( + w13_weight=torch.empty(8, 256, 4), + w2_weight=torch.empty(8, 4, 128), + global_num_experts=8, + local_expert_offset=0, + local_num_experts=8, + intermediate_size=128, + routing_method_type=policy, + block_quant=True, + weight_block_k=128, + w13_weight_scale_inv=torch.ones(8, 2, 1), + w2_weight_scale_inv=torch.ones(8, 1, 1), + ) + provider = ModuleType("flashinfer.fused_moe") + provider.Fp8QuantizationType = SimpleNamespace(MxFp8=1, DeepSeekFp8=0) + with ( + patch.dict(sys.modules, {"flashinfer.fused_moe": provider}), + patch.object( + flashinfer_trtllm, + "per_token_group_quant_fp8", + return_value=(hidden_states, torch.ones((1, 1))), + ), + patch.object( + flashinfer_trtllm, + "trtllm_fp8_block_scale_moe_out_wrapper", + side_effect=native_kernel, + ), + ): + return flashinfer_trtllm.fused_experts_none_to_flashinfer_trtllm_fp8( + StandardDispatchOutput(hidden_states, None, routing), + quant_info, + block.experts.moe_runner_config, + ).hidden_states + + def test_mimo_grouped_bias_retains_native_selection_and_weights(self): + """Equivalent grouped routing must not incur materialized dispatch.""" + block = self._mimo() + output = self._native_output(block) + weights = self.logits[:, 4:6].sigmoid() + weights = weights / weights.sum(-1, keepdim=True) + expected = self._weighted_experts(torch.tensor([[4, 5]]), weights, output) + torch.testing.assert_close(output, expected) + for bias, renormalize in ((False, True), (True, False)): + with self.subTest(bias=bias, renormalize=renormalize): + fallback = self._mimo(bias=bias, renormalize=renormalize) + self.assertEqual( + fallback.topk.topk_config.output_format, TopKOutputFormat.STANDARD + ) + + def test_bailing_native_subsets_preserve_nonunit_scale(self): + """Native declarations must match the configured router, including its factor.""" + for grouped, bias, ids in ((True, True, [4, 5]), (False, False, [0, 1])): + with self.subTest(grouped=grouped): + block = self._bailing(grouped=grouped, bias=bias) + output = self._native_output(block) + scores = self.logits.sigmoid() if bias else self.logits.softmax(-1) + indices = torch.tensor([ids]) + weights = scores.gather(1, indices) + weights = weights / weights.sum(-1, keepdim=True) * 2.5 + torch.testing.assert_close( + output, self._weighted_experts(indices, weights, output) + ) + for grouped, bias, renormalize in ( + (True, False, True), + (False, True, True), + (False, False, False), + ): + with self.subTest(grouped=grouped, bias=bias, renormalize=renormalize): + fallback = self._bailing( + grouped=grouped, bias=bias, renormalize=renormalize + ) + self.assertEqual( + fallback.topk.topk_config.output_format, TopKOutputFormat.STANDARD + ) + + def test_cohere_normalized_sigmoid_keeps_native_and_finite_fallback(self): + """Sigmoid normalization keeps the native policy and finite zero-score fallback.""" + block = self._cohere(renormalize=True) + output = self._native_output(block) + weights = self.logits[:, :2].sigmoid() + weights = weights / weights.sum(-1, keepdim=True) + torch.testing.assert_close( + output, self._weighted_experts(torch.tensor([[0, 1]]), weights, output) + ) + with ( + get_flags().moe.override(runner_backend=MoeRunnerBackend.TRITON), + patch.object( + topk_module, + "fused_topk_native", + side_effect=topk_module.fused_topk_torch_native, + ), + ): + fallback = self._cohere(renormalize=True) + routing = fallback.topk.forward_native( + torch.ones((1, 4)), torch.full((1, 8), -1000.0) + ) + self.assertTrue(torch.isfinite(routing.topk_weights).all().item()) + torch.testing.assert_close(routing.topk_weights, torch.zeros((1, 2))) + fallback = self._cohere(renormalize=False) + self.assertEqual( + fallback.topk.topk_config.output_format, TopKOutputFormat.STANDARD + ) + routing = fallback.topk.forward_native(torch.ones((1, 4)), self.logits) + expected = self.logits.sigmoid().topk(2).values.sort(-1).values + torch.testing.assert_close(routing.topk_weights.sort(-1).values, expected) + + def test_deepseek_dual_stream_graph_rejects_materialized_routing(self): + """The logits-only graph cannot bypass a router requiring materialization.""" + from sglang.srt.models import deepseek_v2 + + block = object.__new__(deepseek_v2.DeepseekV2MoE) + torch.nn.Module.__init__(block) + block.alt_stream = object() + block.num_fused_shared_experts = 0 + block.shared_experts = torch.nn.Identity() + block.experts = SimpleNamespace( + use_flashinfer_trtllm_moe=True, + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + ) + block._enable_a2a_moe = False + block._fuse_shared_experts_inside_sbo = False + block.topk = SimpleNamespace( + topk_config=SimpleNamespace(output_format=TopKOutputFormat.BYPASSED) + ) + with ( + patch.object(deepseek_v2, "_enable_pcg_dsv2_dual_stream", True), + patch.object( + deepseek_v2, "is_in_tc_piecewise_cuda_graph", return_value=True + ), + ): + self.assertTrue(block._can_dual_stream_graph(torch.ones((1, 4)))) + block.topk.topk_config.output_format = TopKOutputFormat.STANDARD + self.assertFalse(block._can_dual_stream_graph(torch.ones((1, 4)))) + + +if __name__ == "__main__": + unittest.main()