diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index 6371452dff..e8629f0895 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -1266,16 +1266,11 @@ class FusedMoE(torch.nn.Module): if shard_id not in ("w1", "w2", "w3"): raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.") - use_flashinfer_trtllm_weight_layout = ( - method.use_flashinfer_trtllm_weight_layout - if isinstance(method, ModelOptNvFp4FusedMoEMethod) - else self.use_flashinfer_trtllm_moe - ) - - # Flashinfer assumes w31 format for w13_weight. Same for the scales. - if use_flashinfer_trtllm_weight_layout and ( - isinstance(method, ModelOptNvFp4FusedMoEMethod) - or isinstance(method, Fp8MoEMethod) + # ModelOpt NVFP4 reorders weights and block scales in _load_w13; + # its per-tensor global scales retain the logical [gate, up] order. + # These other methods also reorder their per-tensor scales. + if self.use_flashinfer_trtllm_moe and ( + isinstance(method, Fp8MoEMethod) or isinstance(method, UnquantizedFusedMoEMethod) or isinstance(method, CompressedTensorsMxInt4MoE) ): diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 8774bd15a8..9c2786bd22 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2337,24 +2337,20 @@ def _compute_gemm1_alphas( w13_weight_scale_2: torch.Tensor, w13_input_scale: torch.Tensor, is_gated: bool, - *, - up_first: bool = False, ) -> tuple[torch.Tensor, torch.Tensor]: """GEMM1 weight x input alphas for the gate (w1) and up (w3) halves of w13. w13 fuses the gate and up projections, which may carry separate NVFP4 weight - scales stored as [num_experts, 2] in w13 row order: [gate, up] by default, - [up, gate] when up_first (the FlashInfer TRT-LLM loader layout). - A 1-D (or [num_experts, 1]) scale, and any non-gated layer, shares one scale - across both halves; the col-1 read is guarded so those cases stay in bounds. + scales stored as [num_experts, 2] (col 0 = gate, col 1 = up). A 1-D (or + [num_experts, 1]) scale, and any non-gated layer, shares one scale across + both halves; the col-1 read is guarded so those cases stay in bounds. Returns (g1_alphas, g1_alphas_up), equal for a shared scale. Single-alpha backends use g1_alphas; the TRT-LLM path also uses g1_alphas_up. """ if is_gated and w13_weight_scale_2.dim() == 2 and w13_weight_scale_2.shape[1] >= 2: - gate_col, up_col = (1, 0) if up_first else (0, 1) - gate_scale = w13_weight_scale_2[:, gate_col] - up_scale = w13_weight_scale_2[:, up_col] + gate_scale = w13_weight_scale_2[:, 0] + up_scale = w13_weight_scale_2[:, 1] else: gate_scale = w13_weight_scale_2.reshape(w13_weight_scale_2.shape[0]) up_scale = gate_scale @@ -2363,41 +2359,6 @@ def _compute_gemm1_alphas( return g1_alphas, g1_alphas_up -def _compute_trtllm_gated_act_params( - *, - activation: Optional[str], - gemm1_alpha: Optional[float], - gemm1_clamp_limit: Optional[float], - swiglu_limit: Optional[float], - g1_alphas: torch.Tensor, - g1_alphas_up: torch.Tensor, -) -> dict[str, torch.Tensor]: - params = {} - is_situ = activation == "situ" - clamp_limit = None if is_situ else (gemm1_clamp_limit or swiglu_limit) - if clamp_limit is not None: - # The kernel clamps both raw GEMM1 halves at this one value, - # so it is exact for the up half only when the gate/up scales match. - if not torch.equal(g1_alphas, g1_alphas_up): - logger.warning_once( - "NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU clamp " - "limit is exact for the gate half only. Accuracy may be affected." - ) - params["gemm1_clamp_limit"] = (clamp_limit / g1_alphas).to(torch.float32) - - if gemm1_alpha is not None: - params["gemm1_alpha"] = torch.full_like( - g1_alphas, gemm1_alpha, dtype=torch.float32 - ) - # Beta is added to the raw up half, so it is in up-alpha units. - params["gemm1_beta"] = ( - torch.full_like(g1_alphas, gemm1_clamp_limit, dtype=torch.float32) - if is_situ - else (1.0 / g1_alphas_up).to(torch.float32) - ) - return params - - class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): """ MoE Method for FP4 Quantization with Blockscales and PerTensorScales @@ -2855,10 +2816,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): # Create shared parameters. g1_alphas / g1_alphas_up are the gate (w1) # and up (w3) GEMM1 scales (equal for shared-scale checkpoints). g1_alphas, g1_alphas_up = _compute_gemm1_alphas( - w13_weight_scale_2=layer.w13_weight_scale_2, - w13_input_scale=w13_input_scale, - is_gated=layer.moe_runner_config.is_gated, - up_first=self.use_flashinfer_trtllm_weight_layout, + layer.w13_weight_scale_2, + w13_input_scale, + layer.moe_runner_config.is_gated, ) copy_or_rebind_param(layer, "g1_alphas", g1_alphas) copy_or_rebind_param(layer, "g1_alphas_up", g1_alphas_up) @@ -2880,16 +2840,48 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): if layer.moe_runner_config.is_gated and self.enable_flashinfer_trtllm_moe: runner_config = layer.moe_runner_config - gated_act_params = _compute_trtllm_gated_act_params( - activation=runner_config.activation, - gemm1_alpha=runner_config.gemm1_alpha, - gemm1_clamp_limit=runner_config.gemm1_clamp_limit, - swiglu_limit=runner_config.swiglu_limit, - g1_alphas=layer.g1_alphas, - g1_alphas_up=layer.g1_alphas_up, + is_situ = runner_config.activation == "situ" + gemm1_clamp_limit = ( + None + if is_situ + else (runner_config.gemm1_clamp_limit or runner_config.swiglu_limit) ) - for name, value in gated_act_params.items(): - copy_or_rebind_param(layer, name, value) + if gemm1_clamp_limit is not None: + # The kernel clamps both raw halves with one limit, which is + # exact for both only when their dequantization scales match. + if not torch.equal(layer.g1_alphas, layer.g1_alphas_up): + logger.warning_once( + "NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU " + "clamp limit is exact for the gate half only. " + "Accuracy may be affected." + ) + copy_or_rebind_param( + layer, + "gemm1_clamp_limit", + (gemm1_clamp_limit / layer.g1_alphas).to(torch.float32), + ) + + if runner_config.gemm1_alpha is not None: + copy_or_rebind_param( + layer, + "gemm1_alpha", + torch.full_like( + layer.g1_alphas, + runner_config.gemm1_alpha, + dtype=torch.float32, + ), + ) + # SwiGLU adds beta to the raw up half before dequantization. + gemm1_beta = ( + torch.full_like( + layer.g1_alphas, + runner_config.gemm1_clamp_limit, + dtype=torch.float32, + ) + if is_situ + else (1.0 / layer.g1_alphas_up).to(torch.float32) + ) + copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta) if self.enable_flashinfer_cutlass_moe: from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import ( @@ -3092,10 +3084,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): @property def load_up_proj_weight_first(self) -> bool: - # Load W13 as [Up, Gate] for FlashInfer CUTLASS and CuteDSL v2 kernels. - # The CuteDSL v1 (deepep) path uses [Gate, Up] -- do NOT flip. + # Only weights and block scales follow the kernel's physical row order; + # per-tensor global scales always remain [Gate, Up]. MegaMoE and + # CuteDSL v1 (deepep) consume canonical [Gate, Up] weights as well. return self.moe_runner_config.is_gated and ( - self.enable_flashinfer_cutlass_moe or self._is_cutedsl_v2_standard + self.use_flashinfer_trtllm_weight_layout + or self.enable_flashinfer_cutlass_moe + or self._is_cutedsl_v2_standard ) def create_moe_runner( diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index 39d1107c8c..0a27b3f07e 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -4,7 +4,6 @@ from types import SimpleNamespace from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.eval_accuracy_kit import _run_sgl_eval from sglang.test.run_eval import run_eval from sglang.test.test_utils import ( DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, @@ -13,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=2070, stage="weekly", runner_config="4-gpu-b200") +register_cuda_ci(est_time=1770, stage="weekly", runner_config="4-gpu-b200") class FlashinferTrtllmGenMoeBackendFP8Base: @@ -306,29 +305,6 @@ class TestFlashinferTrtllmGenMoeBackendNVFP4( backend = "flashinfer_trtllm" -class TestFlashinferTrtllmGenMoeBackendNvFp4UntiedGateUp( - FlashinferTrtllmGenMoeBackendNVFP4Base, CustomTestCase -): - """nvidia/Qwen3-30B-A3B-NVFP4 with each expert's gate and up global scales - re-split to differ but the same dequantized weights; accuracy must match.""" - - model = "mmangkad/Qwen3-30B-A3B-NVFP4-untied-gate-up" - backend = "flashinfer_trtllm" - - def test_gsm8k(self): - _run_sgl_eval( - self, - eval_name="gsm8k", - score_threshold=0.9, - num_examples=200, - num_threads=128, - chat_template_kwargs={"enable_thinking": False}, - max_tokens=2048, - temperature=0.7, - top_p=0.8, - ) - - class TestFlashinferTrtllmGenMoeBackendMXFP8Routed( FlashinferTrtllmGenMoeBackendMXFP8Base, CustomTestCase ): diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py index b59f949317..242f733d3d 100644 --- a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py @@ -1,17 +1,14 @@ """CPU unit tests for NVFP4 fused-MoE GEMM1 global-scale handling. w13 fuses the gate (w1) and up (w3) projections, which can carry separate NVFP4 -weight scales. These tests pin the helpers that handle that: the gate/up split -in _compute_gemm1_alphas, the TRT-LLM up-half output scalar in -_compute_g1_scale_c, and the TRT-LLM SwiGLU beta/clamp in -_compute_trtllm_gated_act_params. All are plain tensor ops, so they run on CPU. +weight scales. These tests cover their checkpoint loading, the gate/up split +in _compute_gemm1_alphas, the TRT-LLM up-half output scalar in _compute_g1_scale_c, +and the post-load SwiGLU/SiTU parameters. GPU weight packing is stubbed. The full TRT-LLM path only runs on Blackwell, and most checkpoints ship near-equal gate/up scales, so this regression wouldn't surface in an accuracy -eval -- hence the direct contract tests. The wiring around the helpers (Marlin's -single-scale collapse, the w1/w3 mismatch warning, and registering g1_alphas / -g1_alphas_up in process_weights_after_loading) needs the GPU kernels and is -not covered here. +eval -- hence the direct contract tests with distinct scales. Kernel execution +and end-to-end model accuracy are not covered here. Shapes and scale magnitudes follow real NVFP4 MoE checkpoints rather than toy sizes, so a failure looks like one a real checkpoint would hit. @@ -26,6 +23,8 @@ from sglang.test.ci.ci_register import register_cpu_ci register_cpu_ci(est_time=13, suite="base-a-test-cpu") import unittest +from types import SimpleNamespace +from unittest import mock import torch @@ -34,9 +33,10 @@ import torch # schemes package; the quantization-package-first order masks it. The isort # guards keep that order. # isort: off +from sglang.srt.layers.quantization import modelopt_quant from sglang.srt.layers.quantization.modelopt_quant import ( + ModelOptNvFp4FusedMoEMethod, _compute_gemm1_alphas, - _compute_trtllm_gated_act_params, ) from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( _align_fp4_moe_weights, @@ -45,6 +45,10 @@ from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( ) # isort: on +from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE +from sglang.srt.layers.moe.moe_runner.base import MoeRunnerConfig +from sglang.srt.layers.moe.utils import MoeA2ABackend, MoeRunnerBackend +from sglang.srt.runtime_context import get_flags from sglang.test.test_utils import CustomTestCase # Representative NVFP4 MoE shapes (name, num_experts, hidden, intermediate, @@ -73,6 +77,172 @@ def _global_scales(num_experts: int, num_cols: int, seed: int) -> torch.Tensor: return scales if num_cols > 1 else scales.squeeze(1) +def _nvfp4_loader_layer(*, is_gated=True, tp_rank=0): + """Small CPU buffers with the real quant method and checkpoint loader.""" + config = MoeRunnerConfig(is_gated=is_gated) + with mock.patch.object( + modelopt_quant, "get_platform", return_value=SimpleNamespace(is_blackwell=True) + ): + method = ModelOptNvFp4FusedMoEMethod( + SimpleNamespace(use_per_token_activation=False) + ) + method.moe_runner_config = config + layer = FusedMoE.__new__(FusedMoE) + torch.nn.Module.__init__(layer) + layer.quant_method = method + layer.quant_config = method.quant_config + layer.moe_runner_config = config + layer.scheme = None + layer._has_fused_shared = False + layer.use_flashinfer_trtllm_moe = method.enable_flashinfer_trtllm_moe + layer.use_triton_kernels = False + layer.use_presharded_weights = False + layer.moe_tp_size = 2 + layer.moe_tp_rank = tp_rank + layer.dispatcher = SimpleNamespace(set_quant_config=lambda config: None) + num_shards = 2 if is_gated else 1 + for name, data in { + "w13_weight": torch.zeros(1, num_shards * 64, 32, dtype=torch.uint8), + "w13_weight_scale": torch.zeros( + 1, num_shards * 64, 4, dtype=torch.float8_e4m3fn + ), + "w13_weight_scale_2": torch.zeros(1, num_shards), + "w13_input_scale": torch.full((1, num_shards), 0.5), + "w2_weight": torch.zeros(1, 64, 32, dtype=torch.uint8), + "w2_weight_scale": torch.zeros(1, 64, 4, dtype=torch.float8_e4m3fn), + "w2_weight_scale_2": torch.ones(1), + "w2_input_scale": torch.ones(1), + }.items(): + layer.register_parameter(name, torch.nn.Parameter(data, requires_grad=False)) + return layer + + +class TestNvfp4MoeScaleLoading(CustomTestCase): + def test_global_scales_keep_gate_up_order_across_weight_layouts(self): + """Changing backend layout must not change which projection an alpha scales. + + Exercise the full shard dispatch, including TP slicing and a second load + in reverse order, so a loader-wide w1/w3 swap cannot hide behind tied scales. + """ + layouts = [ + (MoeRunnerBackend.FLASHINFER_TRTLLM, MoeA2ABackend.NONE, True), + (MoeRunnerBackend.FLASHINFER_TRTLLM_ROUTED, MoeA2ABackend.NONE, True), + (MoeRunnerBackend.FLASHINFER_TRTLLM, MoeA2ABackend.MEGAMOE, False), + (MoeRunnerBackend.FLASHINFER_CUTLASS, MoeA2ABackend.NONE, True), + (MoeRunnerBackend.FLASHINFER_CUTEDSL, MoeA2ABackend.NONE, True), + (MoeRunnerBackend.FLASHINFER_CUTEDSL, MoeA2ABackend.DEEPEP, False), + ] + for backend, a2a, up_first in layouts: + for tp_rank in (0, 1): + with ( + self.subTest(backend=backend, a2a=a2a, tp_rank=tp_rank), + get_flags().moe.override(runner_backend=backend, a2a_backend=a2a), + ): + layer = _nvfp4_loader_layer(tp_rank=tp_rank) + gate = torch.arange(128, dtype=torch.uint8)[:, None].expand(128, 32) + up = gate + 128 + for reload, shards in enumerate((("w1", "w3"), ("w3", "w1"))): + for shard in shards: + is_gate = shard == "w1" + values = { + "w13_weight": gate if is_gate else up, + "w13_weight_scale": torch.full( + (128, 4), + 1.0 if is_gate else 2.0, + dtype=torch.float8_e4m3fn, + ), + "w13_weight_scale_2": torch.tensor( + (0.25 if is_gate else 1.0) * (reload + 1) + ), + } + for name, value in values.items(): + layer._weight_loader_impl( + getattr(layer, name), value, name, shard, 0 + ) + halves = (up, gate) if up_first else (gate, up) + expected = torch.cat( + [half[tp_rank * 64 : (tp_rank + 1) * 64] for half in halves] + ) + torch.testing.assert_close(layer.w13_weight[0], expected) + expected_blocks = ( + torch.tensor([2.0, 1.0] if up_first else [1.0, 2.0]) + .repeat_interleave(64)[:, None] + .expand(128, 4) + ) + torch.testing.assert_close( + layer.w13_weight_scale[0].float(), expected_blocks + ) + gate_alpha, up_alpha = _compute_gemm1_alphas( + layer.w13_weight_scale_2, torch.tensor(0.5), True + ) + torch.testing.assert_close( + gate_alpha, torch.tensor([0.125 * (reload + 1)]) + ) + torch.testing.assert_close( + up_alpha, torch.tensor([0.5 * (reload + 1)]) + ) + + def test_non_gated_trtllm_loads_one_projection(self): + with get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + a2a_backend=MoeA2ABackend.NONE, + ): + layer = _nvfp4_loader_layer(is_gated=False, tp_rank=1) + weight = torch.arange(128, dtype=torch.uint8)[:, None].expand(128, 32) + layer._weight_loader_impl(layer.w13_weight, weight, "w13_weight", "w1", 0) + layer._weight_loader_impl( + layer.w13_weight_scale_2, + torch.tensor(0.25), + "w13_weight_scale_2", + "w1", + 0, + ) + torch.testing.assert_close(layer.w13_weight[0], weight[64:]) + gate_alpha, up_alpha = _compute_gemm1_alphas( + layer.w13_weight_scale_2, torch.tensor(0.5), False + ) + torch.testing.assert_close(gate_alpha, torch.tensor([0.125])) + torch.testing.assert_close(up_alpha, gate_alpha) + + def test_trtllm_post_load_activation_parameters(self): + """Beta adds one in dequantized up units; SiTU keeps its linear scale.""" + for activation in ("silu", "situ"): + with ( + self.subTest(activation=activation), + get_flags().moe.override( + runner_backend=MoeRunnerBackend.FLASHINFER_TRTLLM, + a2a_backend=MoeA2ABackend.NONE, + ), + mock.patch.object(modelopt_quant, "reorder_rows_for_gated_act_gemm"), + mock.patch.object(modelopt_quant, "shuffle_matrix_sf_a"), + ): + layer = _nvfp4_loader_layer() + layer.w13_weight_scale_2.data.copy_(torch.tensor([[0.25, 1.0]])) + config = layer.moe_runner_config + config.activation = activation + config.gemm1_alpha = 1.702 if activation == "silu" else 4.0 + config.gemm1_clamp_limit = None if activation == "silu" else 25.0 + config.swiglu_limit = 7.0 + # Only GPU packing is stubbed; scale conversion and registration run. + with mock.patch.object( + layer.quant_method, "_align_fp4_moe_weights_for_flashinfer_trtllm" + ): + layer.quant_method.process_weights_after_loading(layer) + torch.testing.assert_close( + layer.gemm1_alpha, torch.tensor([config.gemm1_alpha]) + ) + if activation == "silu": + torch.testing.assert_close( + layer.gemm1_beta * layer.g1_alphas_up, torch.ones(1) + ) + torch.testing.assert_close( + layer.gemm1_clamp_limit * layer.g1_alphas, torch.tensor([7.0]) + ) + else: + self.assertFalse(hasattr(layer, "gemm1_clamp_limit")) + torch.testing.assert_close(layer.gemm1_beta, torch.tensor([25.0])) + + class TestGemm1Alphas(CustomTestCase): """w13 gate/up global-scale split feeding g1_alphas / g1_alphas_up.""" @@ -356,56 +526,6 @@ def _byte_sum(t: torch.Tensor) -> int: return t.view(torch.uint8).to(torch.int64).sum().item() -class TestTrtllmGatedActParams(CustomTestCase): - """TRT-LLM SwiGLU params in raw GEMM1 accumulator units.""" - - def test_beta_uses_up_alpha_and_clamp_uses_gate_alpha(self): - """With distinct gate/up global scales, beta must be in up-half units - and the shared clamp limit in gate-half units.""" - swiglu_limit = 7.0 - for i, (name, num_experts, _, _, _) in enumerate(GATED_CONFIGS): - with self.subTest(config=name): - g1_alphas = _global_scales(num_experts, 1, seed=i) - g1_alphas_up = _global_scales(num_experts, 1, seed=300 + i) - - params = _compute_trtllm_gated_act_params( - activation="silu", - gemm1_alpha=1.702, - gemm1_clamp_limit=None, - swiglu_limit=swiglu_limit, - g1_alphas=g1_alphas, - g1_alphas_up=g1_alphas_up, - ) - - torch.testing.assert_close(params["gemm1_beta"], 1.0 / g1_alphas_up) - torch.testing.assert_close( - params["gemm1_clamp_limit"], swiglu_limit / g1_alphas - ) - - def test_situ_passes_linear_scale_unscaled_without_clamp(self): - """SiTU gets no clamp limit, and its beta is the configured linear scale - as-is, independent of the GEMM1 alphas.""" - num_experts = GATED_CONFIGS[0][1] - g1_alphas = _global_scales(num_experts, 1, seed=0) - - params = _compute_trtllm_gated_act_params( - activation="situ", - gemm1_alpha=4.0, - gemm1_clamp_limit=25.0, - swiglu_limit=7.0, - g1_alphas=g1_alphas, - g1_alphas_up=_global_scales(num_experts, 1, seed=1), - ) - - self.assertNotIn("gemm1_clamp_limit", params) - torch.testing.assert_close( - params["gemm1_beta"], torch.full((num_experts,), 25.0) - ) - torch.testing.assert_close( - params["gemm1_alpha"], torch.full((num_experts,), 4.0) - ) - - class TestTrtllmNvfp4HiddenAlignment(CustomTestCase): def test_only_per_token_non_gated_is_padded(self): for per_token, is_gated, expected in [