diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 0365d52aff..b12086b86b 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2316,20 +2316,24 @@ def _compute_gemm1_alphas( w13_weight_scale_2: torch.Tensor, w13_input_scale: torch.Tensor, is_gated: bool, + *, + up_first: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: """GEMM1 weight x input alphas for the gate (w1) and up (w3) halves of w13. w13 fuses the gate and up projections, which may carry separate NVFP4 weight - scales stored as [num_experts, 2] (col 0 = gate, col 1 = up). A 1-D (or - [num_experts, 1]) scale, and any non-gated layer, shares one scale across - both halves; the col-1 read is guarded so those cases stay in bounds. + scales stored as [num_experts, 2] in w13 row order: [gate, up] by default, + [up, gate] when up_first (the FlashInfer TRT-LLM loader layout). + A 1-D (or [num_experts, 1]) scale, and any non-gated layer, shares one scale + across both halves; the col-1 read is guarded so those cases stay in bounds. Returns (g1_alphas, g1_alphas_up), equal for a shared scale. Single-alpha backends use g1_alphas; the TRT-LLM path also uses g1_alphas_up. """ if is_gated and w13_weight_scale_2.dim() == 2 and w13_weight_scale_2.shape[1] >= 2: - gate_scale = w13_weight_scale_2[:, 0] - up_scale = w13_weight_scale_2[:, 1] + gate_col, up_col = (1, 0) if up_first else (0, 1) + gate_scale = w13_weight_scale_2[:, gate_col] + up_scale = w13_weight_scale_2[:, up_col] else: gate_scale = w13_weight_scale_2.reshape(w13_weight_scale_2.shape[0]) up_scale = gate_scale @@ -2338,6 +2342,41 @@ def _compute_gemm1_alphas( return g1_alphas, g1_alphas_up +def _compute_trtllm_gated_act_params( + *, + activation: Optional[str], + gemm1_alpha: Optional[float], + gemm1_clamp_limit: Optional[float], + swiglu_limit: Optional[float], + g1_alphas: torch.Tensor, + g1_alphas_up: torch.Tensor, +) -> dict[str, torch.Tensor]: + params = {} + is_situ = activation == "situ" + clamp_limit = None if is_situ else (gemm1_clamp_limit or swiglu_limit) + if clamp_limit is not None: + # The kernel clamps both raw GEMM1 halves at this one value, + # so it is exact for the up half only when the gate/up scales match. + if not torch.equal(g1_alphas, g1_alphas_up): + logger.warning_once( + "NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU clamp " + "limit is exact for the gate half only. Accuracy may be affected." + ) + params["gemm1_clamp_limit"] = (clamp_limit / g1_alphas).to(torch.float32) + + if gemm1_alpha is not None: + params["gemm1_alpha"] = torch.full_like( + g1_alphas, gemm1_alpha, dtype=torch.float32 + ) + # Beta is added to the raw up half, so it is in up-alpha units. + params["gemm1_beta"] = ( + torch.full_like(g1_alphas, gemm1_clamp_limit, dtype=torch.float32) + if is_situ + else (1.0 / g1_alphas_up).to(torch.float32) + ) + return params + + class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): """ MoE Method for FP4 Quantization with Blockscales and PerTensorScales @@ -2795,9 +2834,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): # Create shared parameters. g1_alphas / g1_alphas_up are the gate (w1) # and up (w3) GEMM1 scales (equal for shared-scale checkpoints). g1_alphas, g1_alphas_up = _compute_gemm1_alphas( - layer.w13_weight_scale_2, - w13_input_scale, - layer.moe_runner_config.is_gated, + w13_weight_scale_2=layer.w13_weight_scale_2, + w13_input_scale=w13_input_scale, + is_gated=layer.moe_runner_config.is_gated, + up_first=self.use_flashinfer_trtllm_weight_layout, ) copy_or_rebind_param(layer, "g1_alphas", g1_alphas) copy_or_rebind_param(layer, "g1_alphas_up", g1_alphas_up) @@ -2819,39 +2859,16 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): if layer.moe_runner_config.is_gated and self.enable_flashinfer_trtllm_moe: runner_config = layer.moe_runner_config - is_situ = runner_config.activation == "situ" - gemm1_clamp_limit = ( - None - if is_situ - else (runner_config.gemm1_clamp_limit or runner_config.swiglu_limit) + gated_act_params = _compute_trtllm_gated_act_params( + activation=runner_config.activation, + gemm1_alpha=runner_config.gemm1_alpha, + gemm1_clamp_limit=runner_config.gemm1_clamp_limit, + swiglu_limit=runner_config.swiglu_limit, + g1_alphas=layer.g1_alphas, + g1_alphas_up=layer.g1_alphas_up, ) - if gemm1_clamp_limit is not None: - copy_or_rebind_param( - layer, - "gemm1_clamp_limit", - (gemm1_clamp_limit / layer.g1_alphas).to(torch.float32), - ) - - if runner_config.gemm1_alpha is not None: - copy_or_rebind_param( - layer, - "gemm1_alpha", - torch.full_like( - layer.g1_alphas, - runner_config.gemm1_alpha, - dtype=torch.float32, - ), - ) - gemm1_beta = ( - torch.full_like( - layer.g1_alphas, - runner_config.gemm1_clamp_limit, - dtype=torch.float32, - ) - if is_situ - else (1.0 / layer.g1_alphas).to(torch.float32) - ) - copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta) + for name, value in gated_act_params.items(): + copy_or_rebind_param(layer, name, value) if self.enable_flashinfer_cutlass_moe: from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index 0a27b3f07e..39d1107c8c 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -4,6 +4,7 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci +from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -12,7 +13,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=1770, stage="weekly", runner_config="4-gpu-b200") +register_cuda_ci(est_time=2070, stage="weekly", runner_config="4-gpu-b200") class FlashinferTrtllmGenMoeBackendFP8Base: @@ -305,6 +306,29 @@ class TestFlashinferTrtllmGenMoeBackendNVFP4( backend = "flashinfer_trtllm" +class TestFlashinferTrtllmGenMoeBackendNvFp4UntiedGateUp( + FlashinferTrtllmGenMoeBackendNVFP4Base, CustomTestCase +): + """nvidia/Qwen3-30B-A3B-NVFP4 with each expert's gate and up global scales + re-split to differ but the same dequantized weights; accuracy must match.""" + + model = "mmangkad/Qwen3-30B-A3B-NVFP4-untied-gate-up" + backend = "flashinfer_trtllm" + + def test_gsm8k(self): + _run_sgl_eval( + self, + eval_name="gsm8k", + score_threshold=0.9, + num_examples=200, + num_threads=128, + chat_template_kwargs={"enable_thinking": False}, + max_tokens=2048, + temperature=0.7, + top_p=0.8, + ) + + class TestFlashinferTrtllmGenMoeBackendMXFP8Routed( FlashinferTrtllmGenMoeBackendMXFP8Base, CustomTestCase ): diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py index 7a698431db..b59f949317 100644 --- a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py @@ -1,16 +1,17 @@ """CPU unit tests for NVFP4 fused-MoE GEMM1 global-scale handling. w13 fuses the gate (w1) and up (w3) projections, which can carry separate NVFP4 -weight scales. These tests pin the two helpers that handle that: the gate/up -split in _compute_gemm1_alphas and the TRT-LLM up-half output scalar in -_compute_g1_scale_c. Both are plain tensor ops, so they run on CPU. +weight scales. These tests pin the helpers that handle that: the gate/up split +in _compute_gemm1_alphas, the TRT-LLM up-half output scalar in +_compute_g1_scale_c, and the TRT-LLM SwiGLU beta/clamp in +_compute_trtllm_gated_act_params. All are plain tensor ops, so they run on CPU. The full TRT-LLM path only runs on Blackwell, and most checkpoints ship near-equal gate/up scales, so this regression wouldn't surface in an accuracy eval -- hence the direct contract tests. The wiring around the helpers (Marlin's single-scale collapse, the w1/w3 mismatch warning, and registering g1_alphas / g1_alphas_up in process_weights_after_loading) needs the GPU kernels and is -covered on-device. +not covered here. Shapes and scale magnitudes follow real NVFP4 MoE checkpoints rather than toy sizes, so a failure looks like one a real checkpoint would hit. @@ -33,7 +34,10 @@ import torch # schemes package; the quantization-package-first order masks it. The isort # guards keep that order. # isort: off -from sglang.srt.layers.quantization.modelopt_quant import _compute_gemm1_alphas +from sglang.srt.layers.quantization.modelopt_quant import ( + _compute_gemm1_alphas, + _compute_trtllm_gated_act_params, +) from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( _align_fp4_moe_weights, _compute_g1_scale_c, @@ -352,6 +356,56 @@ def _byte_sum(t: torch.Tensor) -> int: return t.view(torch.uint8).to(torch.int64).sum().item() +class TestTrtllmGatedActParams(CustomTestCase): + """TRT-LLM SwiGLU params in raw GEMM1 accumulator units.""" + + def test_beta_uses_up_alpha_and_clamp_uses_gate_alpha(self): + """With distinct gate/up global scales, beta must be in up-half units + and the shared clamp limit in gate-half units.""" + swiglu_limit = 7.0 + for i, (name, num_experts, _, _, _) in enumerate(GATED_CONFIGS): + with self.subTest(config=name): + g1_alphas = _global_scales(num_experts, 1, seed=i) + g1_alphas_up = _global_scales(num_experts, 1, seed=300 + i) + + params = _compute_trtllm_gated_act_params( + activation="silu", + gemm1_alpha=1.702, + gemm1_clamp_limit=None, + swiglu_limit=swiglu_limit, + g1_alphas=g1_alphas, + g1_alphas_up=g1_alphas_up, + ) + + torch.testing.assert_close(params["gemm1_beta"], 1.0 / g1_alphas_up) + torch.testing.assert_close( + params["gemm1_clamp_limit"], swiglu_limit / g1_alphas + ) + + def test_situ_passes_linear_scale_unscaled_without_clamp(self): + """SiTU gets no clamp limit, and its beta is the configured linear scale + as-is, independent of the GEMM1 alphas.""" + num_experts = GATED_CONFIGS[0][1] + g1_alphas = _global_scales(num_experts, 1, seed=0) + + params = _compute_trtllm_gated_act_params( + activation="situ", + gemm1_alpha=4.0, + gemm1_clamp_limit=25.0, + swiglu_limit=7.0, + g1_alphas=g1_alphas, + g1_alphas_up=_global_scales(num_experts, 1, seed=1), + ) + + self.assertNotIn("gemm1_clamp_limit", params) + torch.testing.assert_close( + params["gemm1_beta"], torch.full((num_experts,), 25.0) + ) + torch.testing.assert_close( + params["gemm1_alpha"], torch.full((num_experts,), 4.0) + ) + + class TestTrtllmNvfp4HiddenAlignment(CustomTestCase): def test_only_per_token_non_gated_is_padded(self): for per_token, is_gated, expected in [