diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py index 1aa43e3e3d..8e21077460 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_cutlass.py @@ -54,6 +54,10 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo): w2_weight: torch.Tensor quant_scales: Optional[list[torch.Tensor]] = None output_dtype: Optional[torch.dtype] = None + # Optional per-expert SwiGLU overrides, fp32 [num_local_experts]. + swiglu_alpha: Optional[torch.Tensor] = None + swiglu_beta: Optional[torch.Tensor] = None + swiglu_limit: Optional[torch.Tensor] = None moe_tp_size: int = 1 moe_tp_rank: int = 0 moe_ep_size: int = 1 @@ -61,6 +65,45 @@ class FlashInferCutlassMoeQuantInfo(MoeQuantInfo): apply_routed_scaling_factor: bool = True +def materialize_swiglu_params_for_cutlass( + runner_config: MoeRunnerConfig, + num_local_experts: int, + device: torch.device, +) -> tuple[Optional[torch.Tensor], Optional[torch.Tensor], Optional[torch.Tensor]]: + """Per-expert SwiGLU (alpha, beta, limit) tensors for the CUTLASS kernel. + + Returns all-None unless a clamp limit is configured. ``gemm1_alpha`` + implies the GPT-OSS-style ``+1`` up term, so beta then defaults to 1.0; + without alpha, alpha/beta stay silu-neutral at 1.0 / 0.0. The kernel + clamps dequantized values, so the limit is in physical units (no + g1_alphas conversion, unlike trtllm-gen). SiTU carries its clamp in the + activation itself. + """ + if runner_config.activation == "situ": + return None, None, None + clamp_limit = runner_config.gemm1_clamp_limit or runner_config.swiglu_limit + if clamp_limit is None: + return None, None, None + alpha = runner_config.gemm1_alpha if runner_config.gemm1_alpha is not None else 1.0 + beta = runner_config.gemm1_beta + if beta is None: + beta = 1.0 if runner_config.gemm1_alpha is not None else 0.0 + return ( + torch.full( + (num_local_experts,), float(alpha), dtype=torch.float32, device=device + ), + torch.full( + (num_local_experts,), float(beta), dtype=torch.float32, device=device + ), + torch.full( + (num_local_experts,), + float(clamp_limit), + dtype=torch.float32, + device=device, + ), + ) + + @dataclass class FlashInferCutlassMxfp4MoeQuantInfo(MoeQuantInfo): """Quantization payload for CUTLASS MXFP4 MoE. @@ -272,6 +315,9 @@ def _run_flashinfer_cutlass( output_dtype=output_dtype, input_sf=x_sf, quant_scales=quant_scales, + swiglu_alpha=quant_info.swiglu_alpha, + swiglu_beta=quant_info.swiglu_beta, + swiglu_limit=quant_info.swiglu_limit, ep_size=quant_info.moe_ep_size, ep_rank=quant_info.moe_ep_rank, tp_size=quant_info.moe_tp_size, diff --git a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py index c724431834..8a3e12b3b3 100644 --- a/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py +++ b/python/sglang/srt/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a4_nvfp4_moe.py @@ -276,6 +276,17 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme): swizzle_blockscale(layer.w2_weight_scale), requires_grad=False ) + # All-None unless the runner config carries a clamp limit. + from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + materialize_swiglu_params_for_cutlass, + ) + + layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass( + self.moe_runner_config, + int(layer.num_local_experts), + layer.w13_weight.device, + ) + def create_moe_runner( self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig ): @@ -340,6 +351,7 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme): "apply_router_weight_on_input is not supported for Flashinfer" ) + swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params quant_info = FlashInferCutlassMoeQuantInfo( quant_type="fp4", w13_weight=layer.w13_weight, @@ -353,6 +365,9 @@ class CompressedTensorsW4A4Nvfp4MoE(CompressedTensorsMoEScheme): layer.w2_weight_scale, layer.g2_alphas, ], + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta, + swiglu_limit=swiglu_limit, moe_ep_size=layer.moe_ep_size, moe_ep_rank=layer.moe_ep_rank, moe_tp_size=layer.moe_tp_size, diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 7a25369330..d5215e8661 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -1308,6 +1308,17 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): requires_grad=False, ) + # All-None unless the runner config carries a clamp limit. + from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + materialize_swiglu_params_for_cutlass, + ) + + layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass( + layer.moe_runner_config, + int(layer.num_local_experts), + layer.w13_weight.device, + ) + def create_moe_runner( self, layer: torch.nn.Module, moe_runner_config: MoeRunnerConfig ): @@ -1388,6 +1399,7 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): FlashInferCutlassMoeQuantInfo, ) + swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params quant_info = FlashInferCutlassMoeQuantInfo( quant_type="fp8", w13_weight=layer.w13_weight, @@ -1399,6 +1411,9 @@ class ModelOptFp8MoEMethod(FusedMoEMethodBase): layer.fc1_input_dequant, ], output_dtype=x.dtype, + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta, + swiglu_limit=swiglu_limit, moe_ep_size=layer.moe_ep_size, moe_ep_rank=layer.moe_ep_rank, moe_tp_size=layer.moe_tp_size, @@ -2772,6 +2787,17 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): ) copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta) + if self.enable_flashinfer_cutlass_moe: + from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + materialize_swiglu_params_for_cutlass, + ) + + layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass( + layer.moe_runner_config, + int(layer.num_local_experts), + layer.w13_weight.device, + ) + # TODO: for flashinfer always do MOE_NVFP4_DISPATCH use_dispatch_fp4 = ( not self.quant_config.use_per_token_activation @@ -3187,6 +3213,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): FlashInferCutlassMoeQuantInfo, ) + assert not self.moe_runner_config.apply_router_weight_on_input, ( + "apply_router_weight_on_input is not supported for Flashinfer" + ) + swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params quant_info = FlashInferCutlassMoeQuantInfo( quant_type="fp4", w13_weight=layer.w13_weight, @@ -3200,6 +3230,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): layer.w2_blockscale_swizzled, layer.g2_alphas, ], + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta, + swiglu_limit=swiglu_limit, moe_ep_size=layer.moe_ep_size, moe_ep_rank=layer.moe_ep_rank, moe_tp_size=layer.moe_tp_size, diff --git a/python/sglang/srt/layers/quantization/unquant.py b/python/sglang/srt/layers/quantization/unquant.py index c0dc373a4b..0ca43541d6 100644 --- a/python/sglang/srt/layers/quantization/unquant.py +++ b/python/sglang/srt/layers/quantization/unquant.py @@ -760,6 +760,17 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): ): layer.dispatcher.set_quant_config({"dispatcher_output_dtype": "bf16"}) + if self.use_flashinfer_cutlass: + from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + materialize_swiglu_params_for_cutlass, + ) + + layer._cutlass_swiglu_params = materialize_swiglu_params_for_cutlass( + layer.moe_runner_config, + int(layer.num_local_experts), + layer.w13_weight.device, + ) + # Reorder rows of W1 for fused gated activation if self.use_flashinfer_trtllm_moe: # The cached indices are GPU tensors. Colocated weight offloading @@ -1063,11 +1074,15 @@ class UnquantizedFusedMoEMethod(FusedMoEMethodBase, BaseFusedOp): FlashInferCutlassMoeQuantInfo, ) + swiglu_alpha, swiglu_beta, swiglu_limit = layer._cutlass_swiglu_params quant_info = FlashInferCutlassMoeQuantInfo( quant_type="bf16", w13_weight=layer.w13_weight, w2_weight=layer.w2_weight, output_dtype=x.dtype, + swiglu_alpha=swiglu_alpha, + swiglu_beta=swiglu_beta, + swiglu_limit=swiglu_limit, moe_ep_size=layer.moe_ep_size, moe_ep_rank=layer.moe_ep_rank, moe_tp_size=layer.moe_tp_size, diff --git a/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py b/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py new file mode 100644 index 0000000000..6b3b46d2c3 --- /dev/null +++ b/test/registered/unit/layers/moe/test_flashinfer_cutlass_swiglu_params.py @@ -0,0 +1,98 @@ +"""Unit tests for the FlashInfer CUTLASS MoE SwiGLU parameter materialization.""" + +import unittest + +import torch + +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( + materialize_swiglu_params_for_cutlass, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=2, suite="base-a-test-cpu") + +GPT_OSS_ALPHA = 1.702 +GPT_OSS_LIMIT = 7.0 +NUM_LOCAL_EXPERTS = 4 + + +class TestMaterializeSwigluParamsForCutlass(CustomTestCase): + def _materialize(self, activation="silu", **config_kwargs): + config = MoeRunnerConfig(activation=activation, is_gated=True, **config_kwargs) + return materialize_swiglu_params_for_cutlass( + config, num_local_experts=NUM_LOCAL_EXPERTS, device=torch.device("cpu") + ) + + def test_gpt_oss_triple(self): + # GPT-OSS ships gemm1_alpha=1.702 with a 7.0 clamp and never sets + # gemm1_beta; alpha implies the +1 up term. + alpha, beta, limit = self._materialize( + gemm1_alpha=GPT_OSS_ALPHA, gemm1_clamp_limit=GPT_OSS_LIMIT + ) + # 1.702 is not exactly representable in fp32; compare with tolerance. + self.assertTrue(torch.allclose(alpha, torch.full_like(alpha, GPT_OSS_ALPHA))) + self.assertTrue(torch.allclose(beta, torch.ones_like(beta))) + self.assertTrue(torch.allclose(limit, torch.full_like(limit, GPT_OSS_LIMIT))) + + def test_gpt_oss_triple_matches_reference_epilogue(self): + # The reference epilogue is silu(gate) * clamp(alpha*up + 1, ±limit). + # With the old beta=0.0 default the up term lost its +1. + torch.manual_seed(0) + gate = torch.randn(64, dtype=torch.float32) + up = torch.randn(64, dtype=torch.float32) + + def reference(alpha_v, beta_v, limit_v): + return torch.nn.functional.silu(gate) * torch.clamp( + alpha_v * up + beta_v, -limit_v, limit_v + ) + + alpha, beta, limit = self._materialize( + gemm1_alpha=GPT_OSS_ALPHA, gemm1_clamp_limit=GPT_OSS_LIMIT + ) + got = reference(alpha[0], beta[0], limit[0]) + want = reference(GPT_OSS_ALPHA, 1.0, GPT_OSS_LIMIT) + self.assertTrue(torch.allclose(got, want)) + wrong = reference(GPT_OSS_ALPHA, 0.0, GPT_OSS_LIMIT) + self.assertFalse(torch.allclose(got, wrong)) + + def test_explicit_beta_wins(self): + _, beta, _ = self._materialize( + gemm1_alpha=GPT_OSS_ALPHA, + gemm1_beta=0.5, + gemm1_clamp_limit=GPT_OSS_LIMIT, + ) + self.assertEqual(beta.tolist(), [0.5] * NUM_LOCAL_EXPERTS) + + def test_no_alpha_stays_neutral(self): + alpha, beta, _ = self._materialize(gemm1_clamp_limit=GPT_OSS_LIMIT) + self.assertEqual(alpha.tolist(), [1.0] * NUM_LOCAL_EXPERTS) + self.assertEqual(beta.tolist(), [0.0] * NUM_LOCAL_EXPERTS) + + def test_swiglu_limit_alias(self): + _, _, limit = self._materialize(swiglu_limit=GPT_OSS_LIMIT) + self.assertEqual(limit.tolist(), [GPT_OSS_LIMIT] * NUM_LOCAL_EXPERTS) + + def test_no_clamp_disables(self): + self.assertEqual( + self._materialize(gemm1_alpha=GPT_OSS_ALPHA), (None, None, None) + ) + + def test_situ_disables(self): + # SiTU carries its clamp inside the activation itself. + self.assertEqual( + self._materialize(activation="situ", gemm1_clamp_limit=GPT_OSS_LIMIT), + (None, None, None), + ) + + def test_dtype_and_shape(self): + for tensor in self._materialize( + gemm1_alpha=GPT_OSS_ALPHA, gemm1_clamp_limit=GPT_OSS_LIMIT + ): + self.assertEqual(tensor.dtype, torch.float32) + self.assertEqual(tuple(tensor.shape), (NUM_LOCAL_EXPERTS,)) + + +if __name__ == "__main__": + unittest.main()