[Refactor] moe: Derive layer contracts from the selected quant method (#44993)

Co-authored-by: Aditya Suresh <152752311+adi-suresh01@users.noreply.github.com>
This commit is contained in:
Cheng Wan
2026-10-20 16:18:45 -07:00
committed by GitHub
co-authored by Aditya Suresh
parent 7b1d5cd02d
commit 4f8f06faf3
95 changed files with 2599 additions and 372 deletions
@@ -513,6 +513,7 @@ class HunYuanSparseMoeBlock(nn.Module):
layer_id=layer_id,
prefix=f"{prefix}.experts",
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states):
orig_shape = hidden_states.shape
@@ -1155,12 +1155,13 @@ def _model_forward_tbo_merge_key(output_a, output_b, name, original_len):
class MaybeTboDeepEPDispatcher(BaseDispatcher):
def __init__(self, **kwargs):
def __init__(self, runner_backend=None, **kwargs):
super().__init__()
num_inner_dispatchers = 2 if is_tbo_enabled() else 1
if get_moe_a2a_backend().is_deepep():
self._inners = [
DeepEPDispatcher(**kwargs) for _ in range(num_inner_dispatchers)
DeepEPDispatcher(runner_backend=runner_backend, **kwargs)
for _ in range(num_inner_dispatchers)
]
elif get_moe_a2a_backend().is_mooncake():
self._inners = [
@@ -1177,7 +1178,8 @@ class MaybeTboDeepEPDispatcher(BaseDispatcher):
]
elif get_moe_a2a_backend().is_pplx():
self._inners = [
PplxDispatcher(**kwargs) for _ in range(num_inner_dispatchers)
PplxDispatcher(runner_backend=runner_backend, **kwargs)
for _ in range(num_inner_dispatchers)
]
@property
+8 -10
View File
@@ -15,7 +15,6 @@ from sglang.srt.layers.dp_attention import (
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
get_moe_runner_backend,
)
from sglang.srt.layers.moe.fused_moe_triton.layer import (
FusedMoE,
@@ -97,12 +96,8 @@ class DeepEPMoE(FusedMoE):
routed_scaling_factor=routed_scaling_factor,
**kwargs,
)
is_humming = (
get_moe_runner_backend().is_humming()
or get_moe_runner_backend().is_auto()
and quant_config is not None
and quant_config.get_name() == "humming"
)
backend = self.runner_backend
is_humming = backend is not None and backend.is_humming()
if get_moe_a2a_backend().is_deepep_v2():
self.deprecate_flag = True
elif is_humming:
@@ -121,14 +116,16 @@ class DeepEPMoE(FusedMoE):
):
self.deprecate_flag = True
elif (
get_moe_runner_backend().is_flashinfer_cutedsl()
backend is not None
and backend.is_flashinfer_cutedsl()
and quant_config is not None
and quant_config.get_name() in ("modelopt_fp4", "modelopt_mixed")
):
self.deprecate_flag = True
elif (
deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
and get_moe_runner_backend().is_deep_gemm()
and backend is not None
and backend.is_deep_gemm()
and quant_config is not None
and quant_config.get_name() == "mxfp4"
):
@@ -138,7 +135,8 @@ class DeepEPMoE(FusedMoE):
elif (
quant_config is None
and self.w13_weight.dtype == torch.bfloat16
and get_moe_runner_backend().is_deep_gemm()
and backend is not None
and backend.is_deep_gemm()
and (get_moe_a2a_backend().is_deepep() or get_moe_a2a_backend().is_pplx())
and not _is_npu
and not _is_hip
@@ -101,26 +101,24 @@ _use_aiter = get_bool_env_var("SGLANG_USE_AITER") and _is_hip
_deferred_finalize_info_logged = False
def _fuses_routed_scaling_factor_in_topk(quant_method) -> bool:
def _fuses_routed_scaling_factor_in_topk(quant_method, runner_backend) -> bool:
return (
getattr(quant_method, "fuse_routed_scaling_factor_in_topk", False)
or (
isinstance(quant_method, ModelOptNvFp4FusedMoEMethod)
and not getattr(
quant_method, "_moe_runner_backend", get_moe_runner_backend()
).is_marlin()
and (runner_backend is None or not runner_backend.is_marlin())
)
or (
isinstance(quant_method, Fp8MoEMethod)
runner_backend is not None
and (
get_moe_runner_backend().is_cutlass()
or get_moe_runner_backend().is_flashinfer_trtllm_routed()
# Materialized TRT-LLM routes carry the final weights, including
# when a quantization wrapper delegates to a layer-owned scheme.
runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
or isinstance(quant_method, Fp8MoEMethod)
and runner_backend.is_cutlass()
)
)
or (
isinstance(quant_method, UnquantizedFusedMoEMethod)
and get_moe_runner_backend().is_flashinfer_trtllm_routed()
)
)
@@ -165,15 +163,6 @@ def _get_deepep_comm_group(a2a_backend):
return group
def _built_runner_backend(quant_method: FusedMoEMethodBase):
# The layer calls create_moe_runner before building its dispatcher, so the
# runner the quant method actually picked is known here. Methods that keep
# no runner of their own (including wrappers that build one on an inner
# scheme) return None and the dispatcher follows --moe-runner-backend.
runner = quant_method.runner
return None if runner is None else runner.runner_backend
def create_moe_dispatcher(
moe_runner_config: MoeRunnerConfig,
quant_method: FusedMoEMethodBase,
@@ -192,9 +181,7 @@ def create_moe_dispatcher(
# StandardDispatcher is created but never invoked.
# flashinfer_megamoe does its EP all-to-all inside the kernel, so the
# dispatcher stays a pure noop passthrough.
return StandardDispatcher(
moe_runner_config, runner_backend=_built_runner_backend(quant_method)
)
return StandardDispatcher(moe_runner_config)
elif (
a2a_backend.is_deepep()
or a2a_backend.is_mooncake()
@@ -203,6 +190,7 @@ def create_moe_dispatcher(
or a2a_backend.is_pplx()
):
return MaybeTboDeepEPDispatcher(
runner_backend=moe_runner_config.runner_backend,
group=_get_deepep_comm_group(a2a_backend),
router_topk=moe_runner_config.top_k,
permute_fusion=True,
@@ -546,19 +534,7 @@ class FusedMoE(torch.nn.Module):
and num_experts == 512
and top_k == 10
)
self.supports_deferred_finalize = (
get_moe_runner_backend().is_flashinfer_trtllm()
and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred)
)
global _deferred_finalize_info_logged
if not _deferred_finalize_info_logged:
_deferred_finalize_info_logged = True
logging.getLogger(__name__).debug(
"FlashInfer TRTLLM MoE deferred finalize is "
f"{'enabled' if self.supports_deferred_finalize else 'disabled'} "
f"(moe_runner_backend={get_exec().moe.moe_runner_backend}, "
f"quant_method={type(self.quant_method).__name__})."
)
self.supports_deferred_finalize = False
self.quant_method.create_weights(
layer=self,
@@ -576,6 +552,24 @@ class FusedMoE(torch.nn.Module):
)
self.quant_method.create_moe_runner(self, self.moe_runner_config)
self.runner_backend = self.quant_method.get_moe_runner_backend(self)
self.moe_runner_config.runner_backend = self.runner_backend
self._set_runner_flags()
self.supports_deferred_finalize = (
self.runner_backend is not None
and self.runner_backend.is_flashinfer_trtllm()
and (nvfp4_deferred or qwen35_fp8_deferred or qwen4_bf16_deferred)
)
global _deferred_finalize_info_logged
if not _deferred_finalize_info_logged:
_deferred_finalize_info_logged = True
logging.getLogger(__name__).debug(
"FlashInfer TRTLLM MoE deferred finalize is "
f"{'enabled' if self.supports_deferred_finalize else 'disabled'} "
f"(moe_runner_backend={self.runner_backend}, "
f"quant_method={type(self.quant_method).__name__})."
)
self.dispatcher = create_moe_dispatcher(
self.moe_runner_config, quant_method=self.quant_method
)
@@ -593,14 +587,11 @@ class FusedMoE(torch.nn.Module):
self.swiglu_alpha = gemm1_alpha
self.swiglu_clamp_limit = gemm1_clamp_limit
if (
get_moe_runner_backend().is_flashinfer_trtllm_routed()
or get_moe_runner_backend().is_flashinfer_trtllm()
):
if self.use_flashinfer_trtllm_moe:
self.moe_runner_config.inplace = False
self.should_fuse_routed_scaling_factor_in_topk = (
_fuses_routed_scaling_factor_in_topk(self.quant_method)
_fuses_routed_scaling_factor_in_topk(self.quant_method, self.runner_backend)
)
self.routing_method_type = routing_method_type
@@ -613,6 +604,20 @@ class FusedMoE(torch.nn.Module):
self.runner = self.quant_method.runner
def _set_runner_flags(self) -> None:
backend = self.runner_backend
self.enable_flashinfer_cutlass_moe = (
backend is not None and backend.is_flashinfer_cutlass()
)
self.use_flashinfer_mxfp4_moe = (
backend is not None and backend.is_flashinfer_mxfp4()
)
self.use_triton_kernels = backend is not None and backend.is_triton_kernels()
self.use_deep_gemm = backend is not None and backend.is_deep_gemm()
self.use_flashinfer_trtllm_moe = backend is not None and (
backend.is_flashinfer_trtllm() or backend.is_flashinfer_trtllm_routed()
)
@property
def num_global_routed_experts(self) -> int:
return self._num_global_routed
@@ -828,17 +833,19 @@ class FusedMoE(torch.nn.Module):
loaded_weight = _maybe_copy_weight_view_before_h2d(loaded_weight)
# loaded_weight may be smaller than expert_data along shard_dim when
# the buffer is padded. Copy into the leading slice and leave the
# trailing padding as zeros. Rank-mismatched tensors (bias / scalar
# the buffer is padded. Initialize padding even if a runner fallback
# disabled use_padded_loading. Rank-mismatched tensors (bias / scalar
# scales) don't carry shard_dim; they keep the plain broadcast copy.
if (
loaded_weight.dim() == expert_data.dim()
and shard_dim < expert_data.dim()
and loaded_weight.shape[shard_dim] < expert_data.shape[shard_dim]
):
expert_data.narrow(shard_dim, 0, loaded_weight.shape[shard_dim]).copy_(
loaded_weight
)
loaded_size = loaded_weight.shape[shard_dim]
expert_data.narrow(
shard_dim, loaded_size, expert_data.shape[shard_dim] - loaded_size
).zero_()
expert_data.narrow(shard_dim, 0, loaded_size).copy_(loaded_weight)
else:
expert_data.copy_(loaded_weight)
@@ -913,16 +920,18 @@ class FusedMoE(torch.nn.Module):
# w2, down_proj: Load into only logical weight of w2.
loaded_weight = _maybe_copy_weight_view_before_h2d(loaded_weight)
# loaded_weight may be smaller than expert_data along shard_dim when
# the buffer is padded. Copy into the leading slice only. See the
# rank-mismatch note in _load_w13.
# the buffer is padded. Zero the untouched tail before copying the
# leading slice. See the rank-mismatch note in _load_w13.
if (
loaded_weight.dim() == expert_data.dim()
and shard_dim < expert_data.dim()
and loaded_weight.shape[shard_dim] < expert_data.shape[shard_dim]
):
expert_data.narrow(shard_dim, 0, loaded_weight.shape[shard_dim]).copy_(
loaded_weight
)
loaded_size = loaded_weight.shape[shard_dim]
expert_data.narrow(
shard_dim, loaded_size, expert_data.shape[shard_dim] - loaded_size
).zero_()
expert_data.narrow(shard_dim, 0, loaded_size).copy_(loaded_weight)
else:
expert_data.copy_(loaded_weight)
@@ -1230,10 +1239,7 @@ class FusedMoE(torch.nn.Module):
param=param,
weight_name=weight_name,
)
elif isinstance(method, Fp8MoEMethod) and (
get_moe_runner_backend().is_flashinfer_trtllm_routed()
or get_moe_runner_backend().is_flashinfer_trtllm()
):
elif isinstance(method, Fp8MoEMethod) and (self.use_flashinfer_trtllm_moe):
# Drop the GPU mxfp8 shuffle-index cache on every reload for mxfp8 trtllm, trtllm_routed
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
clear_mxfp8_shuffle_index_cache,
@@ -1482,10 +1488,7 @@ class FusedMoE(torch.nn.Module):
method = _weight_layout_method(self)
if self.scheme is not None:
method = self.scheme
if isinstance(method, Fp8MoEMethod) and (
get_moe_runner_backend().is_flashinfer_trtllm_routed()
or get_moe_runner_backend().is_flashinfer_trtllm()
):
if isinstance(method, Fp8MoEMethod) and (self.use_flashinfer_trtllm_moe):
# Drop the GPU mxfp8 shuffle-index cache on every reload for mxfp8 trtllm, trtllm_routed
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
clear_mxfp8_shuffle_index_cache,
@@ -272,6 +272,9 @@ class KTEPWrapperMethod(FusedMoEMethodBase):
# Delegate to GPU method to create its runner
self.gpu_method.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.gpu_method.get_moe_runner_backend(layer)
def submit(
self,
layer: torch.nn.Module,
@@ -47,6 +47,9 @@ class MoeRunnerConfig:
params_dtype: Optional[torch.dtype] = None
routing_method_type: Optional[RoutingMethodType] = None
# None denotes a direct kernel with standard TopK and local expert IDs.
runner_backend: Optional[MoeRunnerBackendLike] = None
# Runner configuration
activation: str = "silu"
is_gated: bool = True
@@ -800,6 +800,10 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
else hidden_states.dtype
)
topk_output = dispatch_output.topk_output
materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard(
topk_output
)
use_routed_topk |= materialized_topk
if TopKOutputChecker.format_is_bypassed(topk_output):
router_logits = topk_output.router_logits
topk_config = topk_output.topk_config
@@ -809,7 +813,9 @@ def fused_experts_none_to_flashinfer_trtllm_fp8(
topk_config = None
correction_bias = None
routing_method_type = quant_info.routing_method_type
routing_method_type = (
RoutingMethodType.TopK if materialized_topk else quant_info.routing_method_type
)
fp8_quantization_type = (
Fp8QuantizationType.MxFp8
if quant_info.use_mxfp8
@@ -1622,6 +1628,10 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
from sglang.srt.layers.moe.topk import TopKOutputChecker
from sglang.srt.layers.moe.utils import RoutingMethodType
materialized_topk = not use_routed_topk and TopKOutputChecker.format_is_standard(
dispatch_output.topk_output
)
use_routed_topk |= materialized_topk
trtllm_bf16_routed_moe = None
trtllm_bf16_moe = None
if use_routed_topk:
@@ -1676,7 +1686,11 @@ def fused_experts_none_to_flashinfer_trtllm_bf16(
assert runner_config.top_k is not None, (
"runner_config.top_k is required for flashinfer_trtllm_routed."
)
routing_method_type = runner_config.routing_method_type
routing_method_type = (
RoutingMethodType.TopK
if materialized_topk
else runner_config.routing_method_type
)
if routing_method_type is None:
routing_method_type = RoutingMethodType.Default
elif routing_method_type == RoutingMethodType.DeepSeekV3:
@@ -329,6 +329,7 @@ class BaseDispatcher(ABC):
def __init__(self):
self.quant_config: dict = {}
self.runner_backend = None
# Overlap args
self.overlap_args: Optional[CombineOverlapArgs] = None
@@ -376,6 +376,7 @@ class _DeepEPDispatcherImplBase:
hidden_size: int,
params_dtype: torch.dtype,
deepep_mode: DeepEPMode,
runner_backend=None,
):
if not use_deepep:
raise ImportError(
@@ -388,6 +389,7 @@ class _DeepEPDispatcherImplBase:
)
) from _deepep_import_error
self.runner_backend = runner_backend
self.group = group
self.router_topk = router_topk
self.permute_fusion = permute_fusion
@@ -991,12 +993,14 @@ class DeepEPDispatcher(BaseDispatcher):
deepep_mode: DeepEPMode = DeepEPMode.AUTO,
async_finish: bool = False,
return_recv_hook: bool = False,
runner_backend=None,
):
super().__init__()
self.deepep_mode = deepep_mode
common_kwargs = dict(
runner_backend=runner_backend,
group=group,
router_topk=router_topk,
permute_fusion=permute_fusion,
@@ -36,7 +36,6 @@ from sglang.srt.layers.moe.topk import (
from sglang.srt.layers.moe.utils import (
FlashinferA2ADispatchType,
get_flashinfer_a2a_dispatch_type,
get_moe_runner_backend,
)
from sglang.srt.runtime_context import (
get_flags,
@@ -162,24 +161,33 @@ class FlashinferDispatcher(BaseDispatcher):
"Please install the appropriate version of Flashinfer."
)
self.runner_backend = (
moe_runner_config.runner_backend if moe_runner_config is not None else None
)
self.ep_size = group.size()
self.ep_rank = group.rank()
self.router_topk = router_topk
self.hidden_size = hidden_size
self.num_experts = num_experts
self.num_local_experts = num_local_experts
runner_backend = get_moe_runner_backend()
runner_backend = self.runner_backend
self.invalid_token_expert_id = (
-1
if (
runner_backend.is_deep_gemm()
or runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
runner_backend is not None
and (
runner_backend.is_deep_gemm()
or runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
)
)
else self.num_experts
)
# TODO: Can other moe runners use payload_in_workspace too?
self.payload_in_workspace = get_moe_runner_backend().is_flashinfer_cutlass()
self.payload_in_workspace = (
self.runner_backend is not None
and self.runner_backend.is_flashinfer_cutlass()
)
self.dispatch_type = get_flashinfer_a2a_dispatch_type()
if moe_runner_config is None:
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
@@ -389,15 +397,18 @@ class FlashinferDispatcher(BaseDispatcher):
)
# Block-wise FP8 runners quantize before GEMM, so keep dispatch/combine BF16;
# FP4 retains its packed wire path keyed by input_global_scale.
runner_backend = get_moe_runner_backend()
runner_backend = self.runner_backend
weight_dtype = self.quant_config.get("weight_dtype")
uses_bf16_fp8_payload = weight_dtype in (
torch.float8_e4m3fn,
torch.float8_e5m2,
) and (
runner_backend.is_deep_gemm()
or runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
runner_backend is not None
and (
runner_backend.is_deep_gemm()
or runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
)
)
if uses_bf16_fp8_payload and hidden_states.dtype != torch.bfloat16:
raise TypeError(
@@ -537,9 +548,9 @@ class FlashinferDispatcher(BaseDispatcher):
x_recv, x_sf_recv, topk_ids_recv, topk_weights_recv = recv_tensors
x_sf = x_sf_recv.view(-1, x_sf_recv.shape[-1])
# TODO: fuse interleave into cutlass moe
if (
dispatch_type == FlashinferA2ADispatchType.NVFP4
and get_moe_runner_backend().is_flashinfer_cutlass()
if dispatch_type == FlashinferA2ADispatchType.NVFP4 and (
self.runner_backend is not None
and self.runner_backend.is_flashinfer_cutlass()
):
x_sf = nvfp4_block_scale_interleave(x_sf)
else:
@@ -593,13 +604,16 @@ class FlashinferDispatcher(BaseDispatcher):
)
weight_dtype = self.quant_config.get("weight_dtype")
runner_backend = get_moe_runner_backend()
runner_backend = self.runner_backend
if (
weight_dtype in (torch.float8_e4m3fn, torch.float8_e5m2)
and (
runner_backend.is_deep_gemm()
or runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
runner_backend is not None
and (
runner_backend.is_deep_gemm()
or runner_backend.is_flashinfer_trtllm()
or runner_backend.is_flashinfer_trtllm_routed()
)
)
and hidden_states.dtype != torch.bfloat16
):
@@ -198,6 +198,7 @@ class _PplxDispatcherImpl:
hidden_size: int,
params_dtype: torch.dtype,
deepep_mode: DeepEPMode,
runner_backend=None,
):
if not use_pplx:
raise ImportError(
@@ -208,6 +209,7 @@ class _PplxDispatcherImpl:
"backend."
)
self.runner_backend = runner_backend
self.group = group
self.router_topk = router_topk
self.permute_fusion = permute_fusion
@@ -431,6 +433,7 @@ class PplxDispatcher(BaseDispatcher):
deepep_mode: DeepEPMode = DeepEPMode.AUTO,
async_finish: bool = False,
return_recv_hook: bool = False,
runner_backend=None,
):
super().__init__()
@@ -442,6 +445,7 @@ class PplxDispatcher(BaseDispatcher):
)
self._low_latency_dispatcher = _PplxDispatcherImpl(
runner_backend=runner_backend,
group=group,
router_topk=router_topk,
permute_fusion=permute_fusion,
@@ -24,7 +24,6 @@ from sglang.srt.layers.moe.token_dispatcher.base import (
)
from sglang.srt.layers.moe.topk import StandardTopKOutput, TopKOutput, TopKOutputChecker
from sglang.srt.layers.moe.utils import (
get_moe_a2a_backend,
get_moe_runner_backend,
should_use_flashinfer_cutlass_moe_fp4_allgather,
)
@@ -103,17 +102,11 @@ def _map_topk_ids_to_local_experts(
)
def _keeps_global_expert_ids(backend: MoeRunnerBackendLike) -> bool:
# These runners handle EP with global expert IDs:
# - cutlass / cutedsl / trtllm_routed handle EP internally
# - mxfp4 dispatcher mapping is already global
# - hpc_ops consumes global ids together with rank_ep / num_expert_total
# - flashinfer_megamoe routes by global expert ID inside the mega kernel
return (
def _keeps_global_expert_ids(backend: Optional[MoeRunnerBackendLike]) -> bool:
return backend is not None and (
backend.is_flashinfer_cutlass()
or backend.is_flashinfer_cutedsl()
or backend.is_flashinfer_trtllm()
or backend.is_experimental_sgl_trtllm()
or backend.is_flashinfer_trtllm_routed()
or backend.is_hpc_ops()
or backend.is_flashinfer_megamoe()
@@ -122,42 +115,34 @@ def _keeps_global_expert_ids(backend: MoeRunnerBackendLike) -> bool:
class StandardDispatcher(BaseDispatcher):
def __init__(
self,
moe_runner_config: MoeRunnerConfig,
runner_backend: Optional[MoeRunnerBackendLike] = None,
):
def __init__(self, moe_runner_config: MoeRunnerConfig):
super().__init__()
self.moe_ep_size = get_parallel().moe_ep_size
backend = get_moe_runner_backend()
self.enable_flashinfer_cutlass_moe = backend.is_flashinfer_cutlass()
self.enable_flashinfer_mxfp4_moe = backend.is_flashinfer_mxfp4()
self.enable_flashinfer_trtllm_routed_moe = backend.is_flashinfer_trtllm_routed()
# AITER fast paths can be on while the MoE runner stays Triton; only the
# AITER runner keeps global expert IDs, so Triton must remap to local range.
self.use_aiter_moe_runner = backend.is_aiter() or (
backend.is_auto() and _use_aiter and get_moe_a2a_backend().supports_aiter()
backend = moe_runner_config.runner_backend
self.runner_backend = backend
self.enable_flashinfer_cutlass_moe = (
backend is not None and backend.is_flashinfer_cutlass()
)
self.enable_flashinfer_mxfp4_moe = (
backend is not None and backend.is_flashinfer_mxfp4()
)
self.enable_flashinfer_trtllm_routed_moe = (
backend is not None and backend.is_flashinfer_trtllm_routed()
)
self.use_aiter_moe_runner = backend is not None and backend.is_aiter()
self.skip_local_expert_mapping = _keeps_global_expert_ids(backend)
# A built runner may enable local-ID mapping; all other flags above
# follow the requested backend.
if (
self.skip_local_expert_mapping
and runner_backend is not None
and not _keeps_global_expert_ids(runner_backend)
self.moe_ep_size > 1
and (
_keeps_global_expert_ids(get_moe_runner_backend())
!= self.skip_local_expert_mapping
)
and (get_lora().enable_lora or get_lora().lora_paths)
):
# The LoRA pool keys experts by the global backend's convention, so
# it would read the wrong adapters for this layer under EP.
if self.moe_ep_size > 1 and (
get_lora().enable_lora or get_lora().lora_paths
):
raise ValueError(
"LoRA with expert parallelism is not supported on this MoE "
f"layer: --moe-runner-backend {backend.value} keeps global "
f"expert ids, but the layer's quantization method runs the "
f"'{runner_backend.value}' runner, which needs local ids."
)
self.skip_local_expert_mapping = False
raise ValueError(
"LoRA with expert parallelism requires the layer's MoE runner "
"to use the requested runner's expert-ID convention."
)
self.num_experts = moe_runner_config.num_experts
self.num_local_experts = moe_runner_config.num_local_experts
self.num_local_shared_experts = moe_runner_config.num_fused_shared_experts
@@ -172,7 +157,24 @@ class StandardDispatcher(BaseDispatcher):
self, hidden_states: torch.Tensor, topk_output: TopKOutput
) -> StandardDispatchOutput:
if should_use_flashinfer_cutlass_moe_fp4_allgather():
needs_dp_allgather = should_use_flashinfer_cutlass_moe_fp4_allgather()
use_fp4_payload = (
needs_dp_allgather
and self.enable_flashinfer_cutlass_moe
and self.quant_config.get("input_global_scale") is not None
)
if needs_dp_allgather and not use_fp4_payload:
if TopKOutputChecker.format_is_bypassed(topk_output):
topk_output = topk_output.to_standard()
topk_weights, topk_ids, hidden_states = get_parallel().tp_group.all_gatherv(
[topk_output.topk_weights, topk_output.topk_ids, hidden_states],
sizes=get_dp_global_num_tokens(),
)
topk_output = StandardTopKOutput(
topk_weights, topk_ids, topk_output.router_logits
)
hidden_states_scale = None
elif needs_dp_allgather:
# all-gather fp4 hidden states
if (
fp4_quantize_flashinfer is None
+121 -37
View File
@@ -103,10 +103,11 @@ from sglang.srt.eplb.expert_location_dispatch import (
from sglang.srt.layers.dp_attention import is_allocation_symmetric
from sglang.srt.layers.moe import (
get_moe_a2a_backend,
get_moe_runner_backend,
is_moe_input_scattered_across_dp_ranks,
)
from sglang.srt.layers.moe.utils import (
MoeRunnerBackendLike,
RoutingMethodType,
has_per_rank_fused_shared_slots,
)
from sglang.srt.state_capturer.routed_experts import get_global_experts_capturer
@@ -268,6 +269,8 @@ class TopKConfig:
# router's shared weights on the other standard path.
fused_shared_experts_scaling_factor: Optional[float] = None
output_format: Optional[TopKOutputFormat] = None
runner_backend: Optional[MoeRunnerBackendLike] = None
routing_method_type: Optional[RoutingMethodType] = None
scoring_func: str = "softmax"
# sqrtsoftplus through log1p with NaNs ranked first (DeepSeek-V4.1 routing).
sqrtsoftplus_log1p: bool = False
@@ -584,6 +587,46 @@ def _simulate_balanced_routing(
)
def _can_bypass_trtllm_routing(config: TopKConfig) -> bool:
"""Match the native selector to the materialized router's policy."""
method = config.routing_method_type
if (
method is None
or method == RoutingMethodType.Unspecified
or config.sqrtsoftplus_log1p
or config.num_shared_topk_slots != 0
):
return False
plain = not config.use_grouped_topk and config.correction_bias is None
if method == RoutingMethodType.Llama4:
return (
plain
and config.top_k == 1
and not config.renormalize
and config.custom_routing_function is not None
)
if config.custom_routing_function is not None:
return False
if method == RoutingMethodType.DeepSeekV3:
# The biased grouped router uses sigmoid regardless of scoring_func.
return (
config.use_grouped_topk
and config.correction_bias is not None
and config.renormalize
)
if not plain:
return False
if method in (RoutingMethodType.Renormalize, RoutingMethodType.RenormalizeNaive):
return config.scoring_func == "softmax" and config.renormalize
if method == RoutingMethodType.Default:
return config.scoring_func == "softmax" and not config.renormalize
if method == RoutingMethodType.SigmoidRenorm:
return config.scoring_func == "sigmoid" and config.renormalize
if method == RoutingMethodType.Sigmoid:
return config.scoring_func == "sigmoid" and not config.renormalize
return False
# -------------------------------- TopK ---------------------------------------
@@ -631,6 +674,7 @@ class TopK(BaseFusedOp):
assert num_expert_group is not None and topk_group is not None
self.layer_id = layer_id
self._runner_backend_bound = False
shared_experts = get_model().shared_experts
self.enable_waterfill = shared_experts.use_waterfill
@@ -662,6 +706,46 @@ class TopK(BaseFusedOp):
allow_routed_experts_capture=allow_routed_experts_capture,
)
def bind_runner_backend(
self,
runner_backend: Optional[MoeRunnerBackendLike],
*,
routing_method_type: Optional[RoutingMethodType] = None,
) -> None:
"""Associate the owning experts once; repeating the same association is safe."""
if self._runner_backend_bound:
if (
runner_backend != self.topk_config.runner_backend
or routing_method_type != self.topk_config.routing_method_type
):
raise RuntimeError("TopK cannot bind a different MoE runner backend.")
return
self._runner_backend_bound = True
self.topk_config.runner_backend = runner_backend
self.topk_config.routing_method_type = routing_method_type
self.topk_config.fused_gate_packed_ids &= (
runner_backend is not None and runner_backend.is_flashinfer_mxfp4()
)
if self.topk_config.output_format is not None:
return
if runner_backend is not None and runner_backend.is_triton_kernels():
output_format = TopKOutputFormat.TRITON_KERNEL
elif runner_backend is not None and (
runner_backend.is_flashinfer_trtllm()
and _can_bypass_trtllm_routing(self.topk_config)
or runner_backend.is_flashinfer_mxfp4()
and not self.is_fp4_experts
):
output_format = (
TopKOutputFormat.STANDARD
if runner_backend.is_experimental_sgl_trtllm()
and get_lora().enable_lora
else TopKOutputFormat.BYPASSED
)
else:
output_format = TopKOutputFormat.STANDARD
self.topk_config.output_format = output_format
def _apply_waterfill(self, topk_output: TopKOutput, num_tokens: int) -> TopKOutput:
if self.enable_waterfill and self.waterfill_balancer is None:
raise RuntimeError(
@@ -725,26 +809,10 @@ class TopK(BaseFusedOp):
output_format = TopKOutputFormat.STANDARD
elif self.topk_config.output_format is not None:
output_format = self.topk_config.output_format
elif get_moe_runner_backend().is_triton_kernels():
output_format = TopKOutputFormat.TRITON_KERNEL
# ===== TO BE REFACTORED ====
elif get_moe_runner_backend().is_experimental_sgl_trtllm():
try:
use_standard_for_lora = bool(get_lora().enable_lora)
except ValueError:
use_standard_for_lora = False
output_format = (
TopKOutputFormat.STANDARD
if use_standard_for_lora
else TopKOutputFormat.BYPASSED
)
# ===== END TO BE REFACTORED ====
elif get_moe_runner_backend().is_flashinfer_trtllm() or (
get_moe_runner_backend().is_flashinfer_mxfp4() and not self.is_fp4_experts
):
output_format = TopKOutputFormat.BYPASSED
else:
output_format = TopKOutputFormat.STANDARD
raise RuntimeError(
"TopK requires the owning experts' runner backend or an explicit output format."
)
if (
router_logits_partials is not None
@@ -1265,8 +1333,8 @@ def grouped_topk_gpu(
)
if routed_scaling_factor is not None:
topk_weights[:, -1] = (
topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor
)
topk_weights[:, :-1].sum(dim=-1) if renormalize else 1.0
) / routed_scaling_factor
if renormalize:
# fp32 like the reference gate (the epsilon is not representable in
@@ -1278,8 +1346,8 @@ def grouped_topk_gpu(
else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32)
)
topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON)
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32)
@@ -1403,8 +1471,8 @@ def kimi_k2_biased_topk_impl(
# the existing kernels, so no extra cast launch is needed
topk_weights_sum = topk_weights.sum(dim=-1, keepdim=True, dtype=torch.float32)
topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON)
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32)
return topk_weights, topk_ids
@@ -1455,8 +1523,8 @@ def biased_topk_impl(
)
if routed_scaling_factor is not None:
topk_weights[:, -1] = (
topk_weights[:, :-1].sum(dim=-1) / routed_scaling_factor
)
topk_weights[:, :-1].sum(dim=-1) if renormalize else 1.0
) / routed_scaling_factor
if renormalize:
# fp32 like the reference gate (the epsilon is not representable in
@@ -1468,8 +1536,8 @@ def biased_topk_impl(
else topk_weights[:, :-1].sum(dim=-1, keepdim=True, dtype=torch.float32)
)
topk_weights = topk_weights / (topk_weights_sum + _RENORMALIZE_SUM_EPSILON)
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
if apply_routed_scaling_factor_on_output:
topk_weights *= routed_scaling_factor
topk_weights, topk_ids = topk_weights.to(torch.float32), topk_ids.to(torch.int32)
return topk_weights, topk_ids
@@ -1735,7 +1803,6 @@ def _fused_gate_emits_packed_ids(
return (
enabled
and _fused_gate_masks_padded_rows(scoring_func)
and get_moe_runner_backend().is_flashinfer_mxfp4()
and expert_location_dispatch_info is None
and num_shared_topk_slots == 0
and not routing_overridden
@@ -2623,6 +2690,7 @@ def select_experts(
):
correction_bias = topk_config.correction_bias_for_dtype(router_logits.dtype)
fold_scale_after_routing = False
num_routed_topk = top_k - num_shared_topk_slots
# On aiter and on the per-rank shared-slot path, _post_process_topk_ids appends
# the shared columns, so the router selects and renormalizes routed experts only.
@@ -2686,7 +2754,7 @@ def select_experts(
"num_token_non_padded is not yet supported in fused_topk_native"
)
assert expert_location_dispatch_info is None
assert not apply_routed_scaling_factor_on_output, "Not implemented"
fold_scale_after_routing = True
topk_weights, topk_ids = fused_topk_native(
hidden_states=hidden_states,
gating_output=router_logits,
@@ -2696,8 +2764,7 @@ def select_experts(
scoring_func=scoring_func,
)
elif custom_routing_function is None:
if scoring_func not in ("sqrtsoftplus", "sigmoid"):
assert not apply_routed_scaling_factor_on_output, "Not implemented"
fold_scale_after_routing = scoring_func not in ("sqrtsoftplus", "sigmoid")
# The JIT route depends on GPU-only topk_sigmoid/topk_softmax imports
_can_use_jit_kernel = not _is_cpu
@@ -2747,7 +2814,8 @@ def select_experts(
)
padded_rows_masked = _fused_gate_masks_padded_rows(scoring_func)
elif (
get_moe_runner_backend().is_flashinfer_trtllm_routed()
topk_config.runner_backend is not None
and topk_config.runner_backend.is_flashinfer_trtllm_routed()
and scoring_func == "softmax"
and correction_bias is None
):
@@ -2775,6 +2843,10 @@ def select_experts(
and expert_location_dispatch_info is None
and num_shared_topk_slots == 0
and not routing_overridden
and not (
apply_routed_scaling_factor_on_output
and routed_scaling_factor not in (None, 1.0)
)
):
num_experts = router_logits.shape[-1]
if num_experts & (num_experts - 1) == 0 and num_experts <= 512:
@@ -2804,7 +2876,10 @@ def select_experts(
scoring_func=scoring_func,
num_fused_shared_experts=num_shared_topk_slots_for_gate,
routed_scaling_factor=routed_scaling_factor,
apply_routed_scaling_factor_on_output=apply_routed_scaling_factor_on_output,
apply_routed_scaling_factor_on_output=(
apply_routed_scaling_factor_on_output
and not fold_scale_after_routing
),
**_fused_topk_kwargs,
)
else:
@@ -2812,7 +2887,7 @@ def select_experts(
# padded rows is garbage. That is fine because _post_process_topk_ids
# below masks rows >= num_token_non_padded (-1 on CUDA, 0 + zeroed
# weights on HIP) after the logical->physical remap.
assert not apply_routed_scaling_factor_on_output, "Not implemented"
fold_scale_after_routing = True
topk_weights, topk_ids = custom_routing_function(
hidden_states=hidden_states,
gating_output=router_logits,
@@ -2820,6 +2895,15 @@ def select_experts(
renormalize=renormalize,
)
if (
fold_scale_after_routing
and apply_routed_scaling_factor_on_output
and routed_scaling_factor is not None
):
# Shared columns keep their own weight; only routed experts take this scale.
num_routed_for_gate = top_k_for_gate - num_shared_topk_slots_for_gate
topk_weights[:, :num_routed_for_gate] *= routed_scaling_factor
if routing_overridden:
# Benchmark-only: override gating with a balanced expert assignment (so
# dummy/random benchmark tokens don't skew MoE load) via a single fused
+3 -5
View File
@@ -379,11 +379,9 @@ def get_deepep_output_dtype(self) -> DispatcherOutputDtype:
if dispatcher_output_dtype is not None:
return DispatcherOutputDtype(dispatcher_output_dtype)
# 5. flashinfer_cutedsl / cutlass / humming expects BF16 dispatch
if (
get_moe_runner_backend().is_flashinfer_cutedsl()
or get_moe_runner_backend().is_cutlass()
or get_moe_runner_backend().is_humming()
backend = self.runner_backend
if backend is not None and (
backend.is_flashinfer_cutedsl() or backend.is_cutlass() or backend.is_humming()
):
return DispatcherOutputDtype.BF16
@@ -22,6 +22,7 @@ from sglang.kernels.ops.communication.muse_spark_moe_finalize_all_reduce import
from sglang.kernels.ops.moe.muse_spark_post_expert_finalize_triton import (
fused_post_expert_norm_weighted_sum,
)
from sglang.srt.layers.moe import MoeRunnerBackend
from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig
from sglang.srt.layers.muse_spark.muse_spark_fp4_moe import (
finalize_with_post_expert_norm,
@@ -110,6 +111,7 @@ class MuseSpark1xBf16Experts(nn.Module):
self.moe_runner_config = MoeRunnerConfig(
num_experts=num_experts,
num_local_experts=num_local_experts,
runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM,
hidden_size=expert_hidden_size,
intermediate_size_per_partition=intermediate_size,
top_k=top_k,
@@ -108,11 +108,13 @@ class MuseSpark1xFp4Experts(nn.Module):
self.top_k = top_k
self.params_dtype = params_dtype or torch.get_default_dtype()
self.quant_config = quant_config
self.moe_runner_backend = _resolve_moe_runner_backend()
# SwiGLU, and no routed_scaling here -- Muse folds its route_scale into
# the topk weights before they reach us.
self.moe_runner_config = MoeRunnerConfig(
num_experts=num_experts,
num_local_experts=num_local_experts,
runner_backend=self.moe_runner_backend,
hidden_size=hidden_size,
intermediate_size_per_partition=intermediate_size,
top_k=top_k,
@@ -127,7 +129,6 @@ class MuseSpark1xFp4Experts(nn.Module):
self.dispatcher = _DirectDispatcher()
capability = get_device_capability() if is_cuda() else None
self.moe_runner_backend = _resolve_moe_runner_backend()
self.use_marlin = self.moe_runner_backend.is_marlin()
self._sm_major = capability[0] if capability is not None else 0
if not self.use_marlin and capability is not None and capability < (10, 0):
@@ -500,6 +500,9 @@ class AWQMoEMethod(FusedMoEMethodBase):
):
layer.scheme.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return layer.scheme.get_moe_runner_backend(layer)
def apply(
self,
layer: torch.nn.Module,
@@ -38,3 +38,6 @@ class AWQIntelAMXMoEScheme(AWQMoEScheme):
):
self.moe_runner_config = moe_runner_config
self.kernel.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return None
@@ -133,6 +133,9 @@ class AWQMoEScheme(AWQMoESchemeBase):
self.moe_runner_config = moe_runner_config
self.kernel.runner = MoeRunner(MoeRunnerBackend.MARLIN, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.kernel.runner.runner_backend
def apply_weights(
self,
layer: torch.nn.Module,
@@ -168,6 +171,9 @@ class AWQAscendMoEScheme(AWQMoEScheme):
backend = MoeRunnerBackend.ASCEND
self.runner = MoeRunner(backend, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def apply_weights(
self,
layer: torch.nn.Module,
@@ -113,6 +113,16 @@ class FusedMoEMethodBase(QuantizeMethodBase):
):
raise NotImplementedError
def get_moe_runner_backend(
self, layer: torch.nn.Module
) -> MoeRunnerBackendLike | None:
"""Return the execution contract after create_moe_runner.
None declares a direct kernel consuming standard TopK with local IDs.
Override when apply uses a different path from the owned runner.
"""
return self.runner.runner_backend if self.runner is not None else None
@abstractmethod
def apply(
self,
@@ -59,6 +59,12 @@ class BaseMoEScheme(ABC):
of different quantization schemes.
"""
runner = None
def get_moe_runner_backend(self, layer):
"""Return the runner contract, or None for a direct standard/local-ID kernel."""
return self.runner.runner_backend if self.runner is not None else None
@abstractmethod
def create_weights(self, *args, **kwargs):
"""
@@ -1378,6 +1378,9 @@ class CompressedTensorsFusedMoEMethod(FusedMoEMethodBase):
):
return layer.scheme.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return layer.scheme.get_moe_runner_backend(layer)
def get_triton_quant_info(self, layer: torch.nn.Module):
return layer.scheme.get_triton_quant_info(layer)
@@ -11,7 +11,11 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
)
from sglang.srt.layers.dp_attention import is_allocation_symmetric
from sglang.srt.layers.moe import MoeRunnerConfig
from sglang.srt.layers.moe.utils import RoutingMethodType, get_moe_runner_backend
from sglang.srt.layers.moe.utils import (
MoeRunnerBackend,
RoutingMethodType,
get_moe_runner_backend,
)
from sglang.srt.layers.quantization.compressed_tensors.schemes import (
CompressedTensorsMoEScheme,
)
@@ -292,6 +296,9 @@ class CompressedTensorsMxInt4MoE(CompressedTensorsMoEScheme):
):
self.moe_runner_config = moe_runner_config
def get_moe_runner_backend(self, layer):
return MoeRunnerBackend.FLASHINFER_TRTLLM
def apply_weights(
self,
layer: torch.nn.Module,
+28 -17
View File
@@ -1436,6 +1436,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
# The MxFP4 wrapper methods borrow this instance for weight loading;
# they never call create_moe_runner, so moe_runner_config is unset.
self._owns_moe_runner = False
self._moe_runner_backend = None
if get_moe_runner_backend().is_cutlass():
assert cutlass_fp8_supported(), (
"cutlass_fp8 MoE requires CUDA 12.0+ with SM90 or CUDA 12.4+ with SM89"
@@ -2063,6 +2064,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
)
return
runner_backend = self.get_moe_runner_backend(layer)
runner_is_aiter = runner_backend is not None and runner_backend.is_aiter()
# If ROCm, normalize the weights and scales to e4m3fnuz
if _is_fp8_fnuz:
# activation_scheme: dynamic
@@ -2087,7 +2091,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
w2_weight_scale, requires_grad=False
)
layer.w2_input_scale = None
if _use_aiter:
if _use_aiter and runner_is_aiter:
layer.w13_weight.data = shuffle_weight(
layer.w13_weight.contiguous(), (16, 16)
)
@@ -2098,16 +2102,17 @@ class Fp8MoEMethod(FusedMoEMethodBase):
layer.w2_weight.is_shuffled = True
layer._aiter_gate_up_interleaved = False
elif _use_aiter:
# Pre-shuffle weights
t = shuffle_weight(layer.w13_weight, (16, 16))
layer.w13_weight.copy_(t)
del t
t = shuffle_weight(layer.w2_weight, (16, 16))
layer.w2_weight.copy_(t)
del t
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
layer._aiter_gate_up_interleaved = False
if runner_is_aiter:
# Pre-shuffle weights
t = shuffle_weight(layer.w13_weight, (16, 16))
layer.w13_weight.copy_(t)
del t
t = shuffle_weight(layer.w2_weight, (16, 16))
layer.w2_weight.copy_(t)
del t
layer.w13_weight.is_shuffled = True
layer.w2_weight.is_shuffled = True
layer._aiter_gate_up_interleaved = False
elif _is_cpu:
assert _is_cpu_amx_available, (
"Fp8MoEMethod on CPU requires that CPU has AMX support"
@@ -2913,8 +2918,13 @@ class Fp8MoEMethod(FusedMoEMethodBase):
self.runner = MoeRunner(moe_runner_backend, moe_runner_config)
self._owns_moe_runner = True
else:
# TODO(cwan): refactor other backends
pass
# Direct kernels declare the layout they consume.
self._moe_runner_backend = moe_runner_backend
def get_moe_runner_backend(self, layer):
if self.runner is not None:
return self.runner.runner_backend
return self._moe_runner_backend
def get_triton_quant_info(self, layer: torch.nn.Module) -> TritonMoeQuantInfo:
use_rocm_mxfp8 = self.use_mxfp8 and _is_hip and _is_gfx95_supported
@@ -3208,6 +3218,10 @@ class Fp8MoEMethod(FusedMoEMethodBase):
is_gated=self.moe_runner_config.is_gated,
)
routing_method_type = layer.routing_method_type
if routing_method_type is None:
routing_method_type = RoutingMethodType.DeepSeekV3
quant_info = FlashInferTrtllmFp8MoeQuantInfo(
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
@@ -3215,10 +3229,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
local_expert_offset=moe_ep_rank * num_local_experts,
local_num_experts=num_local_experts,
intermediate_size=layer.w2_weight.shape[2],
routing_method_type=int(
getattr(layer, "routing_method_type", None)
or RoutingMethodType.DeepSeekV3
),
routing_method_type=int(routing_method_type),
block_quant=self.block_quant,
use_mxfp8=getattr(self.quant_config, "use_mxfp8", False),
weight_block_k=(
@@ -559,6 +559,9 @@ class GPTQMoEMethod(FusedMoEMethodBase):
):
layer.scheme.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return layer.scheme.get_moe_runner_backend(layer)
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
layer.scheme.process_weights_after_loading(layer)
@@ -651,6 +654,9 @@ class GPTQMarlinMoEMethod(FusedMoEMethodBase):
):
layer.scheme.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return layer.scheme.get_moe_runner_backend(layer)
def apply(
self,
layer: torch.nn.Module,
@@ -274,6 +274,9 @@ class GPTQIntelAMXMoEScheme(GPTQMoESchemeBase):
):
self.kernel.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return None
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
@@ -325,6 +325,9 @@ class GPTQMarlinMoEScheme(GPTQMoESchemeBase):
):
self.kernel.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.kernel.runner.runner_backend
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
self.kernel.process_weights_after_loading(layer)
@@ -23,6 +23,7 @@ from sglang.srt.layers.moe import (
from sglang.srt.layers.moe.moe_runner.triton import TritonMoeQuantInfo
from sglang.srt.layers.moe.utils import (
FlashinferA2ADispatchType,
RoutingMethodType,
get_flashinfer_a2a_dispatch_type,
is_flashinfer_cutedsl_v1_path,
should_use_flashinfer_cutlass_moe_fp4_allgather,
@@ -1255,14 +1256,14 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
)
# Align FP8 weights to FlashInfer per-tensor kernel layout if enabled
if get_moe_runner_backend().is_flashinfer_trtllm():
if self._moe_runner_backend.is_flashinfer_trtllm():
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
align_fp8_moe_weights_for_flashinfer_trtllm,
)
# ModelOpt FP8 stores weights in [Up, Gate] order, so we need to swap
align_fp8_moe_weights_for_flashinfer_trtllm(layer, swap_w13_halves=True)
elif get_moe_runner_backend().is_flashinfer_cutlass():
elif self._moe_runner_backend.is_flashinfer_cutlass():
assert (
hasattr(layer, "w13_input_scale") and layer.w13_input_scale is not None
)
@@ -1352,6 +1353,18 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
)
else:
self.runner = MoeRunner(MoeRunnerBackend.TRITON, moe_runner_config)
# This per-tensor TRT-LLM adapter only supports logits routing. An
# undeclared router keeps Triton's weight layout and local expert IDs.
self._moe_runner_backend = (
moe_runner_backend
if moe_runner_backend.is_flashinfer_trtllm()
and moe_runner_config.routing_method_type
not in (None, RoutingMethodType.Unspecified)
else self.runner.runner_backend
)
def get_moe_runner_backend(self, layer):
return self._moe_runner_backend
def apply(
self,
@@ -1362,10 +1375,18 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
topk_output = dispatch_output.topk_output
from sglang.srt.layers.moe.topk import TopKOutputChecker
if self._moe_runner_backend.is_flashinfer_trtllm() and not (
TopKOutputChecker.format_is_bypassed(topk_output)
):
raise NotImplementedError(
"TRT-LLM per-tensor FP8 cannot consume materialized routing. "
"Use --moe-runner-backend triton for this router."
)
# Fast path: TRT-LLM FP8 per-tensor MoE using BYPASSED TopK routing
if (
get_moe_runner_backend().is_flashinfer_trtllm()
self._moe_runner_backend.is_flashinfer_trtllm()
and TopKOutputChecker.format_is_bypassed(topk_output)
):
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
@@ -1409,7 +1430,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
dispatch_output, quant_info, self.moe_runner_config
)
if get_moe_runner_backend().is_flashinfer_cutlass():
if self._moe_runner_backend.is_flashinfer_cutlass():
activation_str = self.moe_runner_config.activation
assert activation_str in _SUPPORTED_ACT_STRS, (
f"Activation {activation_str!r} is not supported for "
@@ -3114,6 +3135,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
self.runner = MoeRunner(moe_runner_backend, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self._moe_runner_backend
def get_marlin_quant_info(self, layer: torch.nn.Module):
"""Marlin payload for the fp4-marlin (W4A16) fallback; the weights were
repacked by prepare_moe_nvfp4_layer_for_marlin. Also consumed by
@@ -124,6 +124,9 @@ class Mxfp4FlashinferCutlassMoEMethod:
self.runner = MoeRunner(MoeRunnerBackend.FLASHINFER_MXFP4, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def process_weights_after_loading(self, layer: Module) -> None:
# Preserve the base FP4 post-load handling.
self._fp8.process_weights_after_loading(layer)
@@ -11,7 +11,7 @@ from sglang.srt.distributed.device_communicators.pynccl_allocator import (
use_symmetric_memory,
)
from sglang.srt.layers.dp_attention import is_allocation_symmetric
from sglang.srt.layers.moe.utils import RoutingMethodType
from sglang.srt.layers.moe.utils import MoeRunnerBackend, RoutingMethodType
from sglang.srt.runtime_context import (
get_exec,
get_parallel,
@@ -164,6 +164,9 @@ class Mxfp4FlashinferTrtllmMoEMethod:
else None
)
def get_moe_runner_backend(self, layer):
return MoeRunnerBackend.FLASHINFER_MXFP4
def create_weights(
self,
layer,
@@ -35,6 +35,9 @@ class Mxfp4HummingMoEMethod:
self.runner = MoeRunner(MoeRunnerBackend.HUMMING, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def create_weights(
self,
layer: Module,
@@ -56,6 +56,9 @@ class Mxfp4MarlinMoEMethod:
self.runner = MoeRunner(MoeRunnerBackend.MARLIN, moe_runner_config)
def get_moe_runner_backend(self, layer):
return self.runner.runner_backend
def create_weights(
self,
layer: Module,
@@ -1180,6 +1180,9 @@ class QuarkFusedMoEMethod(FusedMoEMethodBase):
):
layer.scheme.create_moe_runner(layer, moe_runner_config)
def get_moe_runner_backend(self, layer):
return layer.scheme.get_moe_runner_backend(layer)
def apply(
self,
layer: torch.nn.Module,
@@ -749,6 +749,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
self.use_flashinfer_trtllm_moe = use_flashinfer_trtllm_moe
self.use_deep_gemm = use_deep_gemm
self._cache_permute_indices = dict({})
self.runner: Optional[MoeRunner] = None
self._aiter_runner: Optional[MoeRunner] = None
# Set by process_weights_after_loading when w13 rows are permuted to
# interleave gate/up for the fused swiglu up-GEMM epilogue.
self.w13_swiglu_interleaved = False
@@ -982,10 +984,11 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
return
moe_runner_config = layer.moe_runner_config
runner_backend = self.get_moe_runner_backend(layer)
if not (
_is_cuda
and self._aiter_runner is None
and self.runner.runner_backend.is_triton()
and runner_backend is not None
and runner_backend.is_triton()
and get_moe_a2a_backend().is_none()
and not self.with_bias
and layer.w13_weight.dtype == torch.bfloat16
@@ -1029,7 +1032,8 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
block layout. During weight update, checkpoint tensors are in
canonical layout and need a temporary shape restore for copy.
"""
if not get_moe_runner_backend().is_flashinfer_trtllm_routed():
runner_backend = self.get_moe_runner_backend(layer)
if runner_backend is None or not runner_backend.is_flashinfer_trtllm_routed():
return
expected_shape = None
@@ -1139,6 +1143,10 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
"using triton MoE runner."
)
def get_moe_runner_backend(self, layer):
runner = self._aiter_runner if self._aiter_runner is not None else self.runner
return runner.runner_backend if runner is not None else None
@property
def load_up_proj_weight_first(self) -> bool:
# FlashInfer CUTLASS kernel assumes [Up, Gate] Proj as W13
+2 -1
View File
@@ -46,7 +46,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -250,6 +250,7 @@ class AfmoeMoE(nn.Module):
if _is_npu
else {}
),
output_format=TopKOutputFormat.STANDARD,
)
def pack_params(self) -> None:
+19 -1
View File
@@ -61,7 +61,10 @@ from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
from sglang.srt.layers.moe.token_dispatcher import DeepEPDispatcher
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import filter_moe_weight_param_global_expert
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
filter_moe_weight_param_global_expert,
)
from sglang.srt.layers.multi_gate import (
create_multi_gate_mm_indices,
multi_gate_triton_kernel,
@@ -300,6 +303,13 @@ class BailingMoESparseMoeBlock(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
)
routing_method_type = None
if self.norm_topk_prob:
if self.use_grouped_topk and self.correction_bias is not None:
routing_method_type = RoutingMethodType.DeepSeekV3
elif not self.use_grouped_topk and self.correction_bias is None:
routing_method_type = RoutingMethodType.RenormalizeNaive
self.experts = get_moe_impl_class(quant_config)(
num_experts=self.num_experts,
top_k=self.top_k,
@@ -308,8 +318,16 @@ class BailingMoESparseMoeBlock(nn.Module):
intermediate_size=config.moe_intermediate_size,
quant_config=quant_config,
routed_scaling_factor=self.routed_scaling_factor,
routing_method_type=routing_method_type,
prefix=add_prefix("experts", prefix),
)
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
self.experts.should_fuse_routed_scaling_factor_in_topk
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
# shared expert
if config.num_shared_experts is not None:
if hasattr(config, "moe_shared_expert_intermediate_size"):
+17 -11
View File
@@ -97,17 +97,6 @@ _use_aiter_gfx95 = _use_aiter and _is_gfx95_supported
if _use_aiter_gfx95:
pass
if _is_cuda:
from sglang.kernels.ops.quantization.awq_dequantize import awq_dequantize
elif _is_cpu and _is_cpu_amx_available:
pass
elif _is_hip:
from sglang.kernels.ops.quantization.awq_triton import (
awq_dequantize_triton as awq_dequantize,
)
else:
from vllm._custom_ops import awq_dequantize
if _is_hip:
pass
@@ -318,6 +307,10 @@ class BailingMoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
prefix=f"{prefix}.experts",
)
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
self.experts.should_fuse_routed_scaling_factor_in_topk
)
self.topk.bind_runner_backend(self.experts.runner_backend)
if self.num_shared_experts > 0:
intermediate_size = self.intermediate_size * self.num_shared_experts
@@ -1109,6 +1102,19 @@ class BailingMoELinearForCausalLM(nn.Module):
if not hasattr(self_attn, "kv_b_proj"):
continue
if hasattr(self_attn.kv_b_proj, "qweight"):
if _is_cuda:
from sglang.kernels.ops.quantization.awq_dequantize import (
awq_dequantize,
)
elif _is_cpu and _is_cpu_amx_available:
pass
elif _is_hip:
from sglang.kernels.ops.quantization.awq_triton import (
awq_dequantize_triton as awq_dequantize,
)
else:
from vllm._custom_ops import awq_dequantize
# AWQ compatible
if _is_cuda or _is_hip:
w = awq_dequantize(
@@ -598,6 +598,7 @@ class BailingMoE(nn.Module):
and getattr(quant_config, "is_fp4_experts", False)
),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
# Whether to apply routed_scaling_factor at model layer.
# For A2A MoE paths (e.g., DeepEP), the runner/post_permute does not apply it,
+7 -1
View File
@@ -248,7 +248,9 @@ class Cohere2MoeSparseMoeBlock(nn.Module):
self.norm_topk_prob = getattr(config, "norm_topk_prob", True)
if self.expert_selection_fn == "sigmoid":
custom_routing_function = cohere2_sigmoid_topk
custom_routing_function = (
None if self.norm_topk_prob else cohere2_sigmoid_topk
)
scoring_func = "sigmoid"
routing_method_type = (
RoutingMethodType.SigmoidRenorm
@@ -293,6 +295,10 @@ class Cohere2MoeSparseMoeBlock(nn.Module):
# The decoder reads this buffer on another stream, so in-place races it.
inplace=False,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
num_shared_experts = getattr(config, "num_shared_experts", 0)
self.num_shared_experts = num_shared_experts
+2 -1
View File
@@ -38,7 +38,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -118,6 +118,7 @@ class DbrxExperts(nn.Module):
self.top_k,
layer_id=layer_id,
renormalize=True,
output_format=TopKOutputFormat.STANDARD,
)
self.moe_runner_config = MoeRunnerConfig(inplace=True)
self.ws = nn.Parameter(
+2 -1
View File
@@ -37,7 +37,7 @@ from sglang.srt.layers.linear import (
)
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -127,6 +127,7 @@ class DeepseekMoE(nn.Module):
top_k=self.top_k,
layer_id=layer_id,
renormalize=config.norm_topk_prob,
output_format=TopKOutputFormat.STANDARD,
)
self.experts = nn.ModuleList(
[
+5
View File
@@ -827,6 +827,10 @@ class DeepseekV2MoE(nn.Module):
),
)
self.topk = TopK(**topk_kwargs)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.routed_scaling_factor_fused_in_topk = (
is_routed_scaling_factor_fused_in_topk(self.topk.topk_config)
)
@@ -1030,6 +1034,7 @@ class DeepseekV2MoE(nn.Module):
and hidden_states.shape[0] > 0
and hasattr(self, "shared_experts")
and getattr(self.experts, "use_flashinfer_trtllm_moe", False)
and self.topk.topk_config.output_format == TopKOutputFormat.BYPASSED
and not self._enable_a2a_moe
and not self._fuse_shared_experts_inside_sbo
and not getattr(self, "is_hash", False)
@@ -322,6 +322,7 @@ class Dots3MoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.shared_experts = None
if config.n_shared_experts is not None and self.num_fused_shared_experts == 0:
+1
View File
@@ -110,6 +110,7 @@ class Ernie4Moe(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
if self.moe_num_shared_experts > 0:
intermediate_size = (
@@ -224,6 +224,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
quant_config=quant_config,
prefix=add_prefix("text_experts", prefix),
)
self.text_experts_topk.bind_runner_backend(self.text_experts.runner_backend)
assert vision_moe_layer_start_index <= vision_moe_layer_end_index
if (
@@ -256,6 +257,9 @@ class Ernie4_5_VLMoeMoE(nn.Module):
quant_config=quant_config,
prefix=add_prefix("vision_experts", prefix),
)
self.vision_experts_topk.bind_runner_backend(
self.vision_experts.runner_backend
)
if self.moe_num_shared_experts > 0:
intermediate_size = (
+4
View File
@@ -182,6 +182,10 @@ class ExaoneMoESparseMoEBlock(nn.Module):
apply_routed_scaling_factor_on_output=True,
scoring_func="sigmoid",
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
if config.num_shared_experts is not None:
intermediate_size = config.moe_intermediate_size * config.num_shared_experts
@@ -295,6 +295,7 @@ class Gemma4MoE(nn.Module):
activation=activation,
reduce_results=reduce_results,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(
self, hidden_states: torch.Tensor, router_logits: torch.Tensor
+4
View File
@@ -461,6 +461,10 @@ class Glm4MoeSparseMoeBlock(nn.Module):
),
fused_shared_experts_scaling_factor=1,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.shared_experts_is_int8 = False
self.shared_experts_is_fp8 = False
@@ -237,6 +237,7 @@ class Glm4MoeLiteSparseMoeBlock(nn.Module):
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
**({"scoring_func": "sigmoid"} if _is_npu else {}),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.shared_experts_is_int8 = False
self.shared_experts_is_fp8 = False
+1
View File
@@ -252,6 +252,7 @@ class GptOssSparseMoeBlock(nn.Module):
prefix=add_prefix("experts", prefix),
**extra_kwargs,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.router = TinyGemmLinear(
config.hidden_size,
+1
View File
@@ -150,6 +150,7 @@ class GraniteMoeMoE(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
# NOTE: hidden_states can have either 1D or 2D shape.
+1
View File
@@ -161,6 +161,7 @@ class Grok1MoE(nn.Module):
inplace=inplace,
no_combine=no_combine,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
if not _is_npu:
+1
View File
@@ -161,6 +161,7 @@ class HunYuanSparseMoeBlock(nn.Module):
layer_id=layer_id,
quant_config=quant_config,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.gate = ReplicatedLinear(
config.hidden_size, config.num_experts, bias=False, quant_config=None
+1
View File
@@ -152,6 +152,7 @@ class HYV3MoEFused(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.experts",
)
self.topk.bind_runner_backend(self.experts.runner_backend)
@staticmethod
def ebias_weight_loader(param: nn.Parameter, loaded_weight: torch.Tensor) -> None:
+1
View File
@@ -126,6 +126,7 @@ class InternS1ProTextDecoderLayer(Qwen3MoeDecoderLayer):
layer_id=layer_id,
custom_routing_function=self._custom_routing_function,
)
self.mlp.topk.bind_runner_backend(self.mlp.experts.runner_backend)
@staticmethod
@functools.lru_cache
@@ -339,6 +339,10 @@ class InternS2MobiusRoutedExpertBank(nn.Module):
# The layer-local shared path consumes the same post-attention input.
inplace=False,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.gate = ReplicatedLinear(
config.hidden_size,
config.num_experts,
+4
View File
@@ -222,6 +222,10 @@ class IQuestQ1MoEBlock(nn.Module):
prefix=add_prefix("experts", prefix),
routing_method_type=RoutingMethodType.Renormalize,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
original_shape = hidden_states.shape
+4
View File
@@ -551,6 +551,10 @@ class KimiK3MoE(nn.Module):
else None
),
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
# MegaMoE (deep_gemm fused a2a+GEMM over the EP symm buffer) replaces
# the routed experts call below. K3 routes ALL batches through it when
+1
View File
@@ -141,6 +141,7 @@ class KimiMoE(nn.Module):
# and requires the output format to be standard. We use quant_config to determine the output format.
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
if self.num_shared_experts is not None:
intermediate_size = moe_intermediate_size * self.num_shared_experts
+1
View File
@@ -172,6 +172,7 @@ class LagunaMoE(nn.Module):
scoring_func=config.moe_router_score_func,
correction_bias=self.gate.e_score_correction_bias,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
# HF safetensors key is singular `shared_expert.…`; mirror so the
# default loader picks it up without remapping.
+1
View File
@@ -162,6 +162,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
"""Optimized expert forward pass using FusedMoE."""
+21 -49
View File
@@ -57,7 +57,6 @@ from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe import (
get_deepep_mode,
get_moe_a2a_backend,
get_moe_runner_backend,
)
from sglang.srt.layers.moe.ep_moe.layer import get_moe_impl_class
from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE
@@ -127,42 +126,22 @@ def _require_block_routing_simulated_experts_compatibility() -> None:
)
def _get_effective_moe_runner_backend(experts):
"""Return the runner selected by the quant method, including auto selection."""
quant_method = getattr(experts, "quant_method", None)
# UnquantizedFusedMoEMethod keeps a Triton fallback in ``runner`` but
# forward_cuda executes this AITER runner first whenever auto selection
# created it. Resolve the runner that actually executes, not the fallback.
aiter_runner = getattr(quant_method, "_aiter_runner", None)
backend = getattr(aiter_runner, "runner_backend", None)
if backend is not None:
return backend
backend = getattr(quant_method, "_moe_runner_backend", None)
if backend is not None:
return backend
runner = getattr(quant_method, "runner", None)
backend = getattr(runner, "runner_backend", None)
if backend is not None:
return backend
runner = getattr(experts, "runner", None)
backend = getattr(runner, "runner_backend", None)
if backend is not None:
return backend
return get_moe_runner_backend()
def _require_block_routing_runner_compatibility(experts) -> None:
backend = _get_effective_moe_runner_backend(experts)
def _require_block_routing_runner_compatibility(experts, top_k: int) -> None:
backend = experts.runner_backend
if backend is None:
# Direct standard/local-ID kernels consume the block router's materialized output.
return
if backend.is_flashinfer_trtllm():
raise ValueError(
"LLaDA2 block routing does not support --moe-runner-backend "
"flashinfer_trtllm; use a compatible runner instead"
)
if backend.is_triton_kernels() and top_k & (top_k - 1):
raise ValueError(
"LLaDA2 block routing with --moe-runner-backend "
"triton_kernel requires num_experts_per_tok to be a "
f"power of two, got {top_k}"
)
def _make_block_routing_triton_output(
@@ -667,14 +646,6 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
f"expert_capacity={self.expert_capacity}, "
f"num_experts={self.num_experts}"
)
if get_moe_runner_backend().is_triton_kernels() and self.top_k & (
self.top_k - 1
):
raise ValueError(
"LLaDA2 block routing with --moe-runner-backend "
"triton_kernel requires num_experts_per_tok to be a "
f"power of two, got {self.top_k}"
)
if self.score_function is not None:
assert (
@@ -709,8 +680,12 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
prefix=add_prefix("experts", prefix),
)
self.topk.topk_config.apply_routed_scaling_factor_on_output = (
_is_npu or self.experts.should_fuse_routed_scaling_factor_in_topk
)
self.topk.bind_runner_backend(self.experts.runner_backend)
if self.use_block_routing:
_require_block_routing_runner_compatibility(self.experts)
_require_block_routing_runner_compatibility(self.experts, self.top_k)
# shared expert
if config.num_shared_experts is not None:
if hasattr(config, "moe_shared_expert_intermediate_size"):
@@ -791,15 +766,12 @@ class LLaDA2MoeSparseMoeBlock(nn.Module):
)
get_global_expert_distribution_recorder().on_select_experts(topk_ids=topk_ids)
runner_backend = _get_effective_moe_runner_backend(self.experts)
if not runner_backend.is_triton_kernels():
runner_backend = self.experts.runner_backend
if runner_backend is None or not runner_backend.is_triton_kernels():
if (
runner_backend.is_aiter()
or getattr(
self.experts,
"should_fuse_routed_scaling_factor_in_topk",
False,
)
runner_backend is not None
and runner_backend.is_aiter()
or self.experts.should_fuse_routed_scaling_factor_in_topk
) and self.routed_scaling_factor is not None:
topk_weights = topk_weights * self.routed_scaling_factor
return StandardTopKOutput(
+6
View File
@@ -43,6 +43,7 @@ from sglang.srt.layers.linear import (
)
from sglang.srt.layers.moe.fused_moe_triton import FusedMoE
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.utils import RoutingMethodType
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -124,8 +125,13 @@ class Llama4MoE(nn.Module):
reduce_results=False,
quant_config=quant_config,
apply_router_weight_on_input=True,
routing_method_type=RoutingMethodType.Llama4,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.shared_expert = LlamaMLP(
hidden_size=config.hidden_size,
@@ -296,6 +296,7 @@ class LongcatFlashMoE(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
num_tokens, hidden_dim = hidden_states.shape
+15 -1
View File
@@ -71,7 +71,11 @@ from sglang.srt.layers.moe.topk import (
TopKOutputFormat,
is_expert_remapping_enabled,
)
from sglang.srt.layers.moe.utils import is_deepep_class_backend, sum_post_experts_output
from sglang.srt.layers.moe.utils import (
RoutingMethodType,
is_deepep_class_backend,
sum_post_experts_output,
)
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe import (
Mxfp4FlashinferTrtllmMoEMethod,
@@ -440,6 +444,11 @@ class MiMoV2MoE(nn.Module):
is_nextn=is_nextn,
)
routing_method_type = (
RoutingMethodType.DeepSeekV3
if self.gate.e_score_correction_bias is not None and config.norm_topk_prob
else None
)
experts_type = get_moe_impl_class(quant_config)
self.experts = experts_type(
num_experts=config.n_routed_experts
@@ -450,6 +459,7 @@ class MiMoV2MoE(nn.Module):
layer_id=self.layer_id,
quant_config=quant_config,
routed_scaling_factor=1.0,
routing_method_type=routing_method_type,
prefix=add_prefix("experts", prefix),
)
@@ -470,6 +480,10 @@ class MiMoV2MoE(nn.Module):
# and requires the output format to be standard. We use quant_config to determine the output format.
output_format=TopKOutputFormat.STANDARD if quant_config is None else None,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
uses_mxfp4_runner = isinstance(
self.experts.quant_method, Mxfp4FlashinferTrtllmMoEMethod
+1
View File
@@ -552,6 +552,7 @@ class MiniMaxM2MoE(nn.Module):
correction_bias=self.e_score_correction_bias,
routed_scaling_factor=1.0,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.gate = ReplicatedLinear(
config.hidden_size,
+1
View File
@@ -430,6 +430,7 @@ class MiniMaxM3MoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=True,
)
self.topk.bind_runner_backend(self.experts.runner_backend)
if self.n_shared_experts is not None and self.num_fused_shared_experts == 0:
intermediate_size = config.intermediate_size * self.n_shared_experts
+1
View File
@@ -107,6 +107,7 @@ class MixtralMoE(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
# NOTE: hidden_states can have either 1D or 2D shape.
+4
View File
@@ -290,6 +290,10 @@ class NemotronHMoE(nn.Module):
routed_scaling_factor=self.routed_scaling_factor,
apply_routed_scaling_factor_on_output=self.experts.should_fuse_routed_scaling_factor_in_topk,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
if config.n_shared_experts:
self.shared_experts = NemotronHMLP(
config,
+1
View File
@@ -94,6 +94,7 @@ class OlmoeMoE(nn.Module):
layer_id=layer_id,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
# NOTE: hidden_states can have either 1D or 2D shape.
+1
View File
@@ -215,6 +215,7 @@ class PhiMoE(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
def forward(
self, hidden_states: torch.Tensor, forward_batch: Optional[ForwardBatch] = None
+4
View File
@@ -341,6 +341,10 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
config.model_type == "qwen3_5_moe_text" and uses_cutedsl_ar_fusion()
),
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.gate = ReplicatedLinear(
config.hidden_size,
+4
View File
@@ -273,6 +273,10 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
prefix=add_prefix("experts", prefix),
routing_method_type=RoutingMethodType.Renormalize,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
# Router gate: description-driven quant. Only the offline ModelSlim path
# (which carries a per-layer quant_model_description) may quantise the
+4
View File
@@ -284,6 +284,10 @@ class SarvamMoESparseMoeBlock(nn.Module):
prefix=add_prefix("experts", prefix),
routing_method_type=RoutingMethodType.Renormalize,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.gate = ReplicatedLinear(
config.hidden_size,
+4
View File
@@ -106,6 +106,10 @@ class SDARMoeSparseMoeBlock(nn.Module):
prefix=add_prefix("experts", prefix),
routing_method_type=RoutingMethodType.Renormalize,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.gate = ReplicatedLinear(
config.hidden_size,
+1
View File
@@ -145,6 +145,7 @@ class Step3TextMoEMLP(nn.Module):
quant_config=quant_config,
prefix=add_prefix("experts", prefix),
)
self.topk.bind_runner_backend(self.experts.runner_backend)
self.gate = ReplicatedLinear(
config.hidden_size,
+4
View File
@@ -162,6 +162,10 @@ class Step3p5MoEMLP(nn.Module):
routing_method_type=RoutingMethodType.Renormalize,
gemm1_clamp_limit=self.limit,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
self.gate = ReplicatedLinear(
config.hidden_size,
+4
View File
@@ -942,6 +942,10 @@ class XllmSparseMoeBlock(nn.Module):
prefix=add_prefix("experts", prefix),
routing_method_type=RoutingMethodType.RenormalizeNaive,
)
self.topk.bind_runner_backend(
self.experts.runner_backend,
routing_method_type=self.experts.routing_method_type,
)
# Shared expert (no gating — output added directly)
num_shared_experts = getattr(config, "num_shared_experts", 0)
+2 -1
View File
@@ -36,7 +36,7 @@ from sglang.srt.layers.linear import (
from sglang.srt.layers.logits_processor import LogitsProcessor
from sglang.srt.layers.moe.moe_runner import MoeRunnerConfig
from sglang.srt.layers.moe.moe_runner.triton_utils.fused_moe import fused_moe
from sglang.srt.layers.moe.topk import TopK
from sglang.srt.layers.moe.topk import TopK, TopKOutputFormat
from sglang.srt.layers.quantization.base_config import QuantizationConfig
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.layers.rotary_embedding import get_rope
@@ -136,6 +136,7 @@ class XverseMoE(nn.Module):
top_k=self.top_k,
layer_id=layer_id,
renormalize=getattr(self.config, "norm_topk_prob", False),
output_format=TopKOutputFormat.STANDARD,
)
if config.num_shared_experts is not None: