[Moe] Honor swiglu_limit clamped activation in flashinfer_cutlass runner (#39939)

Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com>
Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca>
This commit is contained in:
Shijin Zhang
2026-09-23 20:10:03 -07:00
committed by GitHub
co-authored by Brayden Zhong
parent 5c44214d6a
commit 77983865d8
5 changed files with 207 additions and 0 deletions
@@ -54,6 +54,10 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
w2_weight: torch.Tensor
quant_scales: Optional[list[torch.Tensor]] = None
output_dtype: Optional[torch.dtype] = None
# Optional per-expert SwiGLU overrides, fp32 [num_local_experts].
swiglu_alpha: Optional[torch.Tensor] = None
swiglu_beta: Optional[torch.Tensor] = None
swiglu_limit: Optional[torch.Tensor] = None
moe_tp_size: int = 1
moe_tp_rank: int = 0
moe_ep_size: int = 1
@@ -61,6 +65,45 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
apply_routed_scaling_factor: bool = True
def materialize_swiglu_params_for_cutlass(
runner_config: MoeRunnerConfig,
num_local_experts: int,
device: torch.device,
) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]:
"""Per-expert SwiGLU (alpha, beta, limit) tensors for the CUTLASS kernel.
Returns all-None unless a clamp limit is configured. ``gemm1_alpha``
implies the GPT-OSS-style ``+1`` up term, so beta then defaults to 1.0;
without alpha, alpha/beta stay silu-neutral at 1.0 / 0.0. The kernel
clamps dequantized values, so the limit is in physical units (no
g1_alphas conversion, unlike trtllm-gen). SiTU carries its clamp in the
activation itself.
"""
if runner_config.activation == "situ":
return None, None, None
clamp_limit = runner_config.gemm1_clamp_limit or runner_config.swiglu_limit
if clamp_limit is None:
return None, None, None
alpha = runner_config.gemm1_alpha if runner_config.gemm1_alpha is not None else 1.0
beta = runner_config.gemm1_beta
if beta is None:
beta = 1.0 if runner_config.gemm1_alpha is not None else 0.0
return (
torch.full(
(num_local_experts,), float(alpha), dtype=torch.float32, device=device
),
torch.full(
(num_local_experts,), float(beta), dtype=torch.float32, device=device
),
torch.full(
(num_local_experts,),
float(clamp_limit),
dtype=torch.float32,
device=device,
),
)
@dataclass
class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo):
"""Quantization payload for CUTLASS MXFP4 MoE.
@@ -272,6 +315,9 @@ def _run_flashinfer_cutlass(
output_dtype=output_dtype,
input_sf=x_sf,
quant_scales=quant_scales,
swiglu_alpha=quant_info.swiglu_alpha,
swiglu_beta=quant_info.swiglu_beta,
swiglu_limit=quant_info.swiglu_limit,
ep_size=quant_info.moe_ep_size,
ep_rank=quant_info.moe_ep_rank,
tp_size=quant_info.moe_tp_size,
@@ -276,6 +276,17 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
swizzle_blockscale(layer.w2_weight_scale), requires_grad=False
)
# All-None unless the runner config carries a clamp limit.
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
materialize_swiglu_params_for_cutlass,
)
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
self.moe_runner_config,
int(layer.num_local_experts),
layer.w13_weight.device,
)
def create_moe_runner(
self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig
):
@@ -340,6 +351,7 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
"apply_router_weight_on_input is not supported for Flashinfer"
)
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
quant_info = FlashInferCutlassMoeQuantInfo(
quant_type="fp4",
w13_weight=layer.w13_weight,
@@ -353,6 +365,9 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
layer.w2_weight_scale,
layer.g2_alphas,
],
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
@@ -1308,6 +1308,17 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
requires_grad=False,
)
# All-None unless the runner config carries a clamp limit.
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
materialize_swiglu_params_for_cutlass,
)
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
layer.moe_runner_config,
int(layer.num_local_experts),
layer.w13_weight.device,
)
def create_moe_runner(
self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig
):
@@ -1388,6 +1399,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
FlashInferCutlassMoeQuantInfo,
)
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
quant_info = FlashInferCutlassMoeQuantInfo(
quant_type="fp8",
w13_weight=layer.w13_weight,
@@ -1399,6 +1411,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
layer.fc1_input_dequant,
],
output_dtype=x.dtype,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
@@ -2772,6 +2787,17 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
)
copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta)
if self.enable_flashinfer_cutlass_moe:
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
materialize_swiglu_params_for_cutlass,
)
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
layer.moe_runner_config,
int(layer.num_local_experts),
layer.w13_weight.device,
)
# TODO: for flashinfer always do MOE_NVFP4_DISPATCH
use_dispatch_fp4 = (
not self.quant_config.use_per_token_activation
@@ -3187,6 +3213,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
FlashInferCutlassMoeQuantInfo,
)
assert not self.moe_runner_config.apply_router_weight_on_input, (
"apply_router_weight_on_input is not supported for Flashinfer"
)
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
quant_info = FlashInferCutlassMoeQuantInfo(
quant_type="fp4",
w13_weight=layer.w13_weight,
@@ -3200,6 +3230,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
layer.w2_blockscale_swizzled,
layer.g2_alphas,
],
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,
@@ -760,6 +760,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
):
layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"})
if self.use_flashinfer_cutlass:
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
materialize_swiglu_params_for_cutlass,
)
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
layer.moe_runner_config,
int(layer.num_local_experts),
layer.w13_weight.device,
)
# Reorder rows of W1 for fused gated activation
if self.use_flashinfer_trtllm_moe:
# The cached indices are GPU tensors. Colocated weight offloading
@@ -1063,11 +1074,15 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
FlashInferCutlassMoeQuantInfo,
)
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
quant_info = FlashInferCutlassMoeQuantInfo(
quant_type="bf16",
w13_weight=layer.w13_weight,
w2_weight=layer.w2_weight,
output_dtype=x.dtype,
swiglu_alpha=swiglu_alpha,
swiglu_beta=swiglu_beta,
swiglu_limit=swiglu_limit,
moe_ep_size=layer.moe_ep_size,
moe_ep_rank=layer.moe_ep_rank,
moe_tp_size=layer.moe_tp_size,