[Moe] Honor swiglu_limit clamped activation in flashinfer_cutlass runner (#39939)
Signed-off-by: Shijin Zhang <75300765+Dovis01@users.noreply.github.com> Co-authored-by: Brayden Zhong <b8zhong@uwaterloo.ca>
This commit is contained in:
co-authored by
Brayden Zhong
parent
5c44214d6a
commit
77983865d8
@@ -54,6 +54,10 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
|
||||
w2_weight: torch.Tensor
|
||||
quant_scales: Optional[list[torch.Tensor]] = None
|
||||
output_dtype: Optional[torch.dtype] = None
|
||||
# Optional per-expert SwiGLU overrides, fp32 [num_local_experts].
|
||||
swiglu_alpha: Optional[torch.Tensor] = None
|
||||
swiglu_beta: Optional[torch.Tensor] = None
|
||||
swiglu_limit: Optional[torch.Tensor] = None
|
||||
moe_tp_size: int = 1
|
||||
moe_tp_rank: int = 0
|
||||
moe_ep_size: int = 1
|
||||
@@ -61,6 +65,45 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo):
|
||||
apply_routed_scaling_factor: bool = True
|
||||
|
||||
|
||||
def materialize_swiglu_params_for_cutlass(
|
||||
runner_config: MoeRunnerConfig,
|
||||
num_local_experts: int,
|
||||
device: torch.device,
|
||||
) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]:
|
||||
"""Per-expert SwiGLU (alpha, beta, limit) tensors for the CUTLASS kernel.
|
||||
|
||||
Returns all-None unless a clamp limit is configured. ``gemm1_alpha``
|
||||
implies the GPT-OSS-style ``+1`` up term, so beta then defaults to 1.0;
|
||||
without alpha, alpha/beta stay silu-neutral at 1.0 / 0.0. The kernel
|
||||
clamps dequantized values, so the limit is in physical units (no
|
||||
g1_alphas conversion, unlike trtllm-gen). SiTU carries its clamp in the
|
||||
activation itself.
|
||||
"""
|
||||
if runner_config.activation == "situ":
|
||||
return None, None, None
|
||||
clamp_limit = runner_config.gemm1_clamp_limit or runner_config.swiglu_limit
|
||||
if clamp_limit is None:
|
||||
return None, None, None
|
||||
alpha = runner_config.gemm1_alpha if runner_config.gemm1_alpha is not None else 1.0
|
||||
beta = runner_config.gemm1_beta
|
||||
if beta is None:
|
||||
beta = 1.0 if runner_config.gemm1_alpha is not None else 0.0
|
||||
return (
|
||||
torch.full(
|
||||
(num_local_experts,), float(alpha), dtype=torch.float32, device=device
|
||||
),
|
||||
torch.full(
|
||||
(num_local_experts,), float(beta), dtype=torch.float32, device=device
|
||||
),
|
||||
torch.full(
|
||||
(num_local_experts,),
|
||||
float(clamp_limit),
|
||||
dtype=torch.float32,
|
||||
device=device,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo):
|
||||
"""Quantization payload for CUTLASS MXFP4 MoE.
|
||||
@@ -272,6 +315,9 @@ def _run_flashinfer_cutlass(
|
||||
output_dtype=output_dtype,
|
||||
input_sf=x_sf,
|
||||
quant_scales=quant_scales,
|
||||
swiglu_alpha=quant_info.swiglu_alpha,
|
||||
swiglu_beta=quant_info.swiglu_beta,
|
||||
swiglu_limit=quant_info.swiglu_limit,
|
||||
ep_size=quant_info.moe_ep_size,
|
||||
ep_rank=quant_info.moe_ep_rank,
|
||||
tp_size=quant_info.moe_tp_size,
|
||||
|
||||
+15
@@ -276,6 +276,17 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
|
||||
swizzle_blockscale(layer.w2_weight_scale), requires_grad=False
|
||||
)
|
||||
|
||||
# All-None unless the runner config carries a clamp limit.
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
materialize_swiglu_params_for_cutlass,
|
||||
)
|
||||
|
||||
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
|
||||
self.moe_runner_config,
|
||||
int(layer.num_local_experts),
|
||||
layer.w13_weight.device,
|
||||
)
|
||||
|
||||
def create_moe_runner(
|
||||
self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig
|
||||
):
|
||||
@@ -340,6 +351,7 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
|
||||
"apply_router_weight_on_input is not supported for Flashinfer"
|
||||
)
|
||||
|
||||
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
|
||||
quant_info = FlashInferCutlassMoeQuantInfo(
|
||||
quant_type="fp4",
|
||||
w13_weight=layer.w13_weight,
|
||||
@@ -353,6 +365,9 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme):
|
||||
layer.w2_weight_scale,
|
||||
layer.g2_alphas,
|
||||
],
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
|
||||
@@ -1308,6 +1308,17 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
requires_grad=False,
|
||||
)
|
||||
|
||||
# All-None unless the runner config carries a clamp limit.
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
materialize_swiglu_params_for_cutlass,
|
||||
)
|
||||
|
||||
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
|
||||
layer.moe_runner_config,
|
||||
int(layer.num_local_experts),
|
||||
layer.w13_weight.device,
|
||||
)
|
||||
|
||||
def create_moe_runner(
|
||||
self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig
|
||||
):
|
||||
@@ -1388,6 +1399,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
FlashInferCutlassMoeQuantInfo,
|
||||
)
|
||||
|
||||
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
|
||||
quant_info = FlashInferCutlassMoeQuantInfo(
|
||||
quant_type="fp8",
|
||||
w13_weight=layer.w13_weight,
|
||||
@@ -1399,6 +1411,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase):
|
||||
layer.fc1_input_dequant,
|
||||
],
|
||||
output_dtype=x.dtype,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
@@ -2772,6 +2787,17 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
)
|
||||
copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta)
|
||||
|
||||
if self.enable_flashinfer_cutlass_moe:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
materialize_swiglu_params_for_cutlass,
|
||||
)
|
||||
|
||||
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
|
||||
layer.moe_runner_config,
|
||||
int(layer.num_local_experts),
|
||||
layer.w13_weight.device,
|
||||
)
|
||||
|
||||
# TODO: for flashinfer always do MOE_NVFP4_DISPATCH
|
||||
use_dispatch_fp4 = (
|
||||
not self.quant_config.use_per_token_activation
|
||||
@@ -3187,6 +3213,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
FlashInferCutlassMoeQuantInfo,
|
||||
)
|
||||
|
||||
assert not self.moe_runner_config.apply_router_weight_on_input, (
|
||||
"apply_router_weight_on_input is not supported for Flashinfer"
|
||||
)
|
||||
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
|
||||
quant_info = FlashInferCutlassMoeQuantInfo(
|
||||
quant_type="fp4",
|
||||
w13_weight=layer.w13_weight,
|
||||
@@ -3200,6 +3230,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
layer.w2_blockscale_swizzled,
|
||||
layer.g2_alphas,
|
||||
],
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
|
||||
@@ -760,6 +760,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
|
||||
):
|
||||
layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"})
|
||||
|
||||
if self.use_flashinfer_cutlass:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
materialize_swiglu_params_for_cutlass,
|
||||
)
|
||||
|
||||
layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass(
|
||||
layer.moe_runner_config,
|
||||
int(layer.num_local_experts),
|
||||
layer.w13_weight.device,
|
||||
)
|
||||
|
||||
# Reorder rows of W1 for fused gated activation
|
||||
if self.use_flashinfer_trtllm_moe:
|
||||
# The cached indices are GPU tensors. Colocated weight offloading
|
||||
@@ -1063,11 +1074,15 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp):
|
||||
FlashInferCutlassMoeQuantInfo,
|
||||
)
|
||||
|
||||
swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params
|
||||
quant_info = FlashInferCutlassMoeQuantInfo(
|
||||
quant_type="bf16",
|
||||
w13_weight=layer.w13_weight,
|
||||
w2_weight=layer.w2_weight,
|
||||
output_dtype=x.dtype,
|
||||
swiglu_alpha=swiglu_alpha,
|
||||
swiglu_beta=swiglu_beta,
|
||||
swiglu_limit=swiglu_limit,
|
||||
moe_ep_size=layer.moe_ep_size,
|
||||
moe_ep_rank=layer.moe_ep_rank,
|
||||
moe_tp_size=layer.moe_tp_size,
|
||||
|
||||
Reference in New Issue
Block a user