From d20bdc1443c535996ca7a47d8e34df12882af5e6 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang <44627253+xuantengh@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:28:22 +0800 Subject: [PATCH] Add per-token NVFP4 MoE support for ReLU2 activation (#39199) Signed-off-by: Xuanteng Huang --- .../moe/moe_runner/flashinfer_trtllm.py | 190 ++++++++++++------ .../srt/layers/quantization/modelopt_quant.py | 10 +- .../test_flashinfer_trtllm_gen_moe_backend.py | 16 +- .../test_nvidia_nemotron_3_super_nvfp4.py | 33 ++- .../test_modelopt_nvfp4_moe_scales.py | 117 ++++++++++- 5 files changed, 302 insertions(+), 64 deletions(-) diff --git a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py index 12f2c0ecd9..4132e33451 100644 --- a/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py +++ b/python/sglang/srt/layers/moe/moe_runner/flashinfer_trtllm.py @@ -516,46 +516,61 @@ def _align_fp4_moe_weights( w2: torch.Tensor, w2_scale: torch.Tensor, is_gated: bool, - min_alignment: int = 16, -) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]: - """Pad intermediate size so FlashInfer TRTLLM FP4 kernels' alignment holds. + intermediate_alignment: int = 16, + hidden_alignment: int = 1, +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]: + """Zero-pad intermediate and hidden sizes to the TRT-LLM FP4 kernel alignments. - Returns (w13, w13_scale, w2, w2_scale, padded_intermediate). + Intermediate padding is transparent to callers. Hidden padding (K of GEMM1, + N of GEMM2) also needs the activations padded and the output sliced at runtime. + + Returns (w13, w13_scale, w2, w2_scale, padded_intermediate, padded_hidden). """ num_experts, hidden_size, intermediate_packed = w2.shape intermediate = intermediate_packed * 2 # FP4 packs 2 values per byte - padded_intermediate = round_up_to_multiple(intermediate, min_alignment) - if padded_intermediate == intermediate: - return w13, w13_scale, w2, w2_scale, intermediate + padded_intermediate = round_up_to_multiple(intermediate, intermediate_alignment) + padded_hidden = round_up_to_multiple(hidden_size, hidden_alignment) + if padded_intermediate == intermediate and padded_hidden == hidden_size: + return w13, w13_scale, w2, w2_scale, intermediate, hidden_size logger.info( - "FP4 MoE: padding intermediate size from %d to %d (alignment=%d)", + "FP4 MoE: padding (intermediate, hidden) from (%d, %d) to (%d, %d)", intermediate, + hidden_size, padded_intermediate, - min_alignment, + padded_hidden, ) up_mult = 2 if is_gated else 1 padded_gate_up = up_mult * padded_intermediate - padded_w13 = w13.new_zeros((num_experts, padded_gate_up, w13.shape[2])) - padded_w13[:, : w13.shape[1], :] = w13 - - padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2)) - padded_w2[:, :, : w2.shape[2]] = w2 + # w13: [E, gate_up, hidden // 2], scale [E, gate_up, hidden // 16] + padded_w13 = w13.new_zeros((num_experts, padded_gate_up, padded_hidden // 2)) + padded_w13[:, : w13.shape[1], : w13.shape[2]] = w13 padded_w13_scale = w13_scale.new_zeros( - (num_experts, padded_gate_up, w13_scale.shape[2]) + (num_experts, padded_gate_up, padded_hidden // 16) ) - padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale + padded_w13_scale[:, : w13_scale.shape[1], : w13_scale.shape[2]] = w13_scale + + # w2: [E, hidden, intermediate // 2], scale [E, hidden, intermediate // 16] + padded_w2 = w2.new_zeros((num_experts, padded_hidden, padded_intermediate // 2)) + padded_w2[:, :hidden_size, : w2.shape[2]] = w2 padded_w2_scale = w2_scale.new_zeros( - (num_experts, hidden_size, padded_intermediate // 16) + (num_experts, padded_hidden, padded_intermediate // 16) ) - padded_w2_scale[:, :, : w2_scale.shape[2]] = w2_scale + padded_w2_scale[:, :hidden_size, : w2_scale.shape[2]] = w2_scale - return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate + return ( + padded_w13, + padded_w13_scale, + padded_w2, + padded_w2_scale, + padded_intermediate, + padded_hidden, + ) def _compute_g1_scale_c( @@ -584,11 +599,30 @@ def _compute_g1_scale_c( return w2_input_scale_quant.to(torch.float32).expand(num_experts).contiguous() -def align_fp4_moe_weights_for_flashinfer_trtllm(layer: Module) -> None: +def trtllm_nvfp4_hidden_alignment( + use_per_token_activation: bool, is_gated: bool +) -> int: + """Hidden-dim alignment required by the TRT-LLM NVFP4 MoE cubins. + + The per-token (dynamic activation scale) non-gated variant ships far fewer + tile configs than the static one. At Nemotron's hidden size 2688 it runs + but produces garbage (GSM8K ~0.02 on FlashInfer 0.7.0.post1), while a + 512-aligned hidden size is correct. Every other variant runs unpadded. + """ + if use_per_token_activation and not is_gated: + return 512 + return 1 + + +def align_fp4_moe_weights_for_flashinfer_trtllm( + layer: Module, hidden_alignment: int = 1 +) -> None: """Prepare FP4 MoE weights/scales for FlashInfer TRT-LLM kernels. This function handles the weight transformation needed for FP4 TRTLLM MoE: - Pads intermediate dimension for kernel alignment constraints + - Pads hidden dimension to `hidden_alignment` and records the padded size + on `layer.trtllm_padded_hidden_size` (None when unpadded) - Reorders weights for gated activation GEMM - Shuffles weights and scales for transposed MMA output - Computes the output scale factors @@ -603,18 +637,27 @@ def align_fp4_moe_weights_for_flashinfer_trtllm(layer: Module) -> None: w2_weight_scale = cast(torch.Tensor, layer.w2_weight_scale) is_gated = layer.moe_runner_config.is_gated - min_alignment = 16 if is_gated else 128 + hidden_size = w2_weight.shape[1] # Pad for kernel alignment before shuffle/reorder - w13_weight, w13_weight_scale, w2_weight, w2_weight_scale, intermediate_size = ( - _align_fp4_moe_weights( - w13_weight, - w13_weight_scale, - w2_weight, - w2_weight_scale, - is_gated, - min_alignment, - ) + ( + w13_weight, + w13_weight_scale, + w2_weight, + w2_weight_scale, + intermediate_size, + padded_hidden_size, + ) = _align_fp4_moe_weights( + w13_weight, + w13_weight_scale, + w2_weight, + w2_weight_scale, + is_gated=is_gated, + intermediate_alignment=16 if is_gated else 128, + hidden_alignment=hidden_alignment, + ) + layer.trtllm_padded_hidden_size = ( + padded_hidden_size if padded_hidden_size != hidden_size else None ) ( @@ -1249,6 +1292,8 @@ class FlashInferTrtllmFp4MoeQuantInfo(MoeQuantInfo): routing_method_type: int use_per_token_activation: bool = False + # Hidden size the weights were zero-padded to; None means unpadded. + padded_hidden_size: Optional[int] = None gemm1_alpha: Optional[torch.Tensor] = None gemm1_beta: Optional[torch.Tensor] = None @@ -1286,6 +1331,34 @@ def quantize_hidden_states_fp4( return hs_fp4, hs_sf +def _get_fp4_moe_output_buffer( + num_tokens: int, + hidden_size: int, + dtype: torch.dtype, + device: torch.device, + symmetric: bool, +) -> torch.Tensor: + """Reuse the forward's MoE output buffer when it fits, else allocate one.""" + from sglang.srt.runtime_context import get_forward + + provided = get_forward().moe_output_buffer + symm_required = symmetric and is_allocation_symmetric() + if ( + provided is not None + and provided.shape == (num_tokens, hidden_size) + and provided.dtype == dtype + and provided.device == device + and ( + not symm_required + or not is_symmetric_memory_enabled() + or is_tensor_in_symmetric_mempool(provided) + ) + ): + return provided + with use_symmetric_memory(get_parallel().tp_group, disabled=not symm_required): + return torch.empty(num_tokens, hidden_size, dtype=dtype, device=device) + + def fused_experts_none_to_flashinfer_trtllm_fp4( dispatch_output: StandardDispatchOutput, quant_info: FlashInferTrtllmFp4MoeQuantInfo, @@ -1318,6 +1391,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( # Quantize hidden states to FP4 hidden_states_scale = dispatch_output.hidden_states_scale per_token_scale = None + unpadded_hidden_size = hidden_states.shape[-1] + hidden_pad = 0 + if quant_info.padded_hidden_size is not None: + assert hidden_states_scale is None, ( + "Hidden-dim padded TRTLLM NVFP4 MoE weights need unquantized " + "activations; FP4 dispatch is not supported." + ) + hidden_pad = quant_info.padded_hidden_size - unpadded_hidden_size + hidden_states = torch.nn.functional.pad(hidden_states, (0, hidden_pad)) if hidden_states_scale is not None: # NVFP4 dispatch (flashinfer a2a): inputs are already FP4-quantized by # the dispatcher, so pass them through unchanged. @@ -1367,10 +1449,12 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( if not use_routed_topk and TopKOutputChecker.format_is_standard(topk_output): use_routed_topk = True + # Deferred outputs keep the padded width, so padded weights always finalize. defer_finalize = ( _deferred_finalize_enabled.get() and not use_routed_topk and TopKOutputChecker.format_is_bypassed(topk_output) + and hidden_pad == 0 ) symm_output = None @@ -1384,32 +1468,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( output_dtype = ( hidden_states.dtype if hidden_states_scale is None else torch.bfloat16 ) - from sglang.srt.runtime_context import get_forward - - _provided = get_forward().moe_output_buffer - _symm_required = is_allocation_symmetric() - if ( - _provided is not None - and _provided.shape == (num_tokens, hidden_size) - and _provided.dtype == output_dtype - and _provided.device == hs_fp4.device - and ( - not _symm_required - or not is_symmetric_memory_enabled() - or is_tensor_in_symmetric_mempool(_provided) - ) - ): - symm_output = _provided - else: - with use_symmetric_memory( - get_parallel().tp_group, disabled=not _symm_required - ): - symm_output = torch.empty( - num_tokens, - hidden_size, - dtype=output_dtype, - device=hs_fp4.device, - ) + # A padded kernel output is a scratch buffer; the sliced copy is the + # tensor that reaches the all-reduce. + symm_output = _get_fp4_moe_output_buffer( + num_tokens=num_tokens, + hidden_size=hidden_size, + dtype=output_dtype, + device=hs_fp4.device, + symmetric=hidden_pad == 0, + ) if use_routed_topk: routing = _get_routing_for_flashinfer_routed(topk_output) @@ -1499,6 +1566,17 @@ def fused_experts_none_to_flashinfer_trtllm_fp4( else: result = result[0] + if hidden_pad > 0: + unpadded_result = _get_fp4_moe_output_buffer( + num_tokens=result.shape[0], + hidden_size=unpadded_hidden_size, + dtype=result.dtype, + device=result.device, + symmetric=True, + ) + unpadded_result.copy_(result[:, :unpadded_hidden_size]) + result = unpadded_result + return StandardCombineInput(hidden_states=result) diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 272460cd41..a91d928ef8 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2882,10 +2882,17 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): ): from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( align_fp4_moe_weights_for_flashinfer_trtllm, + trtllm_nvfp4_hidden_alignment, ) # FlashInfer TRTLLM processing - handles both w13 and w2 - align_fp4_moe_weights_for_flashinfer_trtllm(layer) + align_fp4_moe_weights_for_flashinfer_trtllm( + layer, + hidden_alignment=trtllm_nvfp4_hidden_alignment( + use_per_token_activation=self.quant_config.use_per_token_activation, + is_gated=layer.moe_runner_config.is_gated, + ), + ) # TRTLLM doesn't read *_blockscale_swizzled; alias to free the # placeholders from create_weights. layer.w13_blockscale_swizzled = layer.w13_weight_scale @@ -3169,6 +3176,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): intermediate_size_per_partition=layer.intermediate_size_per_partition, routing_method_type=routing_method_type, use_per_token_activation=self.quant_config.use_per_token_activation, + padded_hidden_size=layer.trtllm_padded_hidden_size, gemm1_alpha=gemm1_alpha.data if gemm1_alpha is not None else None, gemm1_beta=gemm1_beta.data if gemm1_beta is not None else None, gemm1_clamp_limit=gemm1_clamp.data if gemm1_clamp is not None else None, diff --git a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py index b89d74e968..0a27b3f07e 100644 --- a/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py +++ b/test/registered/backends/test_flashinfer_trtllm_gen_moe_backend.py @@ -250,11 +250,12 @@ class FlashinferTrtllmGenMoeBackendMXFP8MixedBF16Base: class FlashinferTrtllmGenMoeBackendNVFP4Base: backend = None + model = "nvidia/Qwen3-30B-A3B-NVFP4" extra_env = {} + gsm8k_threshold = 0.89 @classmethod def setUpClass(cls): - cls.model = "nvidia/Qwen3-30B-A3B-NVFP4" cls.base_url = DEFAULT_URL_FOR_TEST cls.process = popen_launch_server( cls.model, @@ -289,7 +290,7 @@ class FlashinferTrtllmGenMoeBackendNVFP4Base: ) metrics = run_eval(args) print(f"{metrics=}") - self.assertGreater(metrics["score"], 0.89) + self.assertGreater(metrics["score"], self.gsm8k_threshold) class TestFlashinferTrtllmGenMoeBackendFP8( @@ -335,5 +336,16 @@ class TestFlashinferTrtllmGenMoeBackendNvFp4PerTokenActivationRouted( backend = "flashinfer_trtllm_routed" +class TestFlashinferTrtllmGenMoeBackendNvFp4PerTokenActivationRelu2( + FlashinferTrtllmGenMoeBackendNVFP4Base, CustomTestCase +): + """Non-gated ReLU2 experts with per-token NVFP4 activations.""" + + model = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" + extra_env = {"SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION": "1"} + backend = "flashinfer_trtllm" + gsm8k_threshold = 0.93 + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py index 43f5a878f9..ae7e96f2d1 100644 --- a/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py +++ b/test/registered/e2e/models_large/test_nvidia_nemotron_3_super_nvfp4.py @@ -12,7 +12,7 @@ from sglang.test.test_utils import ( popen_launch_server, ) -register_cuda_ci(est_time=1200, stage="nightly", runner_config="4-gpu-b200") +register_cuda_ci(est_time=1600, stage="nightly", runner_config="4-gpu-b200") NEMOTRON_3_SUPER_NVFP4_MODEL = "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" @@ -69,7 +69,7 @@ MTP_ARGS = [ ] -def _run_gsm8k(test_case): +def _run_gsm8k(test_case, threshold=0.96): args = SimpleNamespace( model=test_case.model, eval_name="gsm8k", @@ -86,7 +86,7 @@ def _run_gsm8k(test_case): ) metrics = run_eval(args) print(f"{metrics=}") - test_case.assertGreaterEqual(metrics["score"], 0.96) + test_case.assertGreaterEqual(metrics["score"], threshold) class TestNvidiaNemotron3SuperNVFP4(CustomTestCase): @@ -110,6 +110,33 @@ class TestNvidiaNemotron3SuperNVFP4(CustomTestCase): _run_gsm8k(self) +class TestNvidiaNemotron3SuperNVFP4PerToken(CustomTestCase): + """Per-token NVFP4 activations on the FlashInfer TRT-LLM ReLU2 MoE.""" + + @classmethod + def setUpClass(cls): + cls.model = NEMOTRON_3_SUPER_NVFP4_MODEL + cls.base_url = DEFAULT_URL_FOR_TEST + with ( + envs.SGLANG_ENABLE_ASYNC_ASSERT.override(0), + envs.SGLANG_FLASHINFER_NVFP4_PER_TOKEN_ACTIVATION.override(True), + ): + cls.process = popen_launch_server( + cls.model, + cls.base_url, + timeout=DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, + other_args=NEMOTRON_3_SUPER_NVFP4_ARGS + + ["--moe-runner-backend", "flashinfer_trtllm"], + ) + + @classmethod + def tearDownClass(cls): + kill_process_tree(cls.process.pid) + + def test_gsm8k(self): + _run_gsm8k(self, threshold=0.93) + + class TestNvidiaNemotron3SuperNVFP4MTP(CustomTestCase): @classmethod def setUpClass(cls): diff --git a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py index 4ebb9f5feb..7a698431db 100644 --- a/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py +++ b/test/registered/unit/layers/quantization/test_modelopt_nvfp4_moe_scales.py @@ -14,11 +14,15 @@ covered on-device. Shapes and scale magnitudes follow real NVFP4 MoE checkpoints rather than toy sizes, so a failure looks like one a real checkpoint would hit. + +The TRT-LLM weight padding is pinned here too: Nemotron 3 (hidden 2688, +intermediate 1856, ReLU2 experts) needs a 128-aligned intermediate and, for +per-token activations, a 512-aligned hidden; padding must be zero-filled. """ from sglang.test.ci.ci_register import register_cpu_ci -register_cpu_ci(est_time=11, suite="base-a-test-cpu") +register_cpu_ci(est_time=13, suite="base-a-test-cpu") import unittest @@ -30,7 +34,11 @@ import torch # guards keep that order. # isort: off from sglang.srt.layers.quantization.modelopt_quant import _compute_gemm1_alphas -from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import _compute_g1_scale_c +from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import ( + _align_fp4_moe_weights, + _compute_g1_scale_c, + trtllm_nvfp4_hidden_alignment, +) # isort: on from sglang.test.test_utils import CustomTestCase @@ -324,5 +332,110 @@ class TestG1ScaleC(CustomTestCase): self.assertEqual(g1_scale_c.dtype, torch.float32) +def _generate_fp4_moe_weights( + num_experts: int, hidden: int, inter: int, is_gated: bool +): + """Packed FP4 weights + E4M3 block scales in checkpoint layout, no zeros.""" + gate_up = (2 if is_gated else 1) * inter + + def u8(*shape): + return torch.randint(1, 255, shape, dtype=torch.uint8) + + w13 = u8(num_experts, gate_up, hidden // 2) + w13_scale = u8(num_experts, gate_up, hidden // 16).view(torch.float8_e4m3fn) + w2 = u8(num_experts, hidden, inter // 2) + w2_scale = u8(num_experts, hidden, inter // 16).view(torch.float8_e4m3fn) + return w13, w13_scale, w2, w2_scale + + +def _byte_sum(t: torch.Tensor) -> int: + return t.view(torch.uint8).to(torch.int64).sum().item() + + +class TestTrtllmNvfp4HiddenAlignment(CustomTestCase): + def test_only_per_token_non_gated_is_padded(self): + for per_token, is_gated, expected in [ + (True, False, 512), + (True, True, 1), + (False, False, 1), + (False, True, 1), + ]: + self.assertEqual( + trtllm_nvfp4_hidden_alignment( + use_per_token_activation=per_token, is_gated=is_gated + ), + expected, + ) + + +class TestAlignFp4MoeWeights(CustomTestCase): + def test_aligned_shapes_are_returned_untouched(self): + weights = _generate_fp4_moe_weights(4, 3072, 1920, is_gated=False) + + out = _align_fp4_moe_weights( + *weights, is_gated=False, intermediate_alignment=128, hidden_alignment=512 + ) + + self.assertEqual(out[4:], (1920, 3072)) + for original, returned in zip(weights, out[:4]): + self.assertIs(returned, original) + + def test_nemotron_pads_intermediate_and_hidden_with_zero_fill(self): + num_experts, hidden, inter = 4, 2688, 1856 + padded_hidden, padded_inter = 3072, 1920 + w13, w13_scale, w2, w2_scale = _generate_fp4_moe_weights( + num_experts, hidden, inter, is_gated=False + ) + + p_w13, p_w13_scale, p_w2, p_w2_scale, out_inter, out_hidden = ( + _align_fp4_moe_weights( + w13, + w13_scale, + w2, + w2_scale, + is_gated=False, + intermediate_alignment=128, + hidden_alignment=512, + ) + ) + + self.assertEqual((out_inter, out_hidden), (padded_inter, padded_hidden)) + # GEMM1: rows are intermediate, K is hidden (2 values/byte, 16/scale). + self.assertEqual(p_w13.shape, (num_experts, padded_inter, padded_hidden // 2)) + self.assertEqual( + p_w13_scale.shape, (num_experts, padded_inter, padded_hidden // 16) + ) + # GEMM2: rows are hidden, K is intermediate. + self.assertEqual(p_w2.shape, (num_experts, padded_hidden, padded_inter // 2)) + self.assertEqual( + p_w2_scale.shape, (num_experts, padded_hidden, padded_inter // 16) + ) + torch.testing.assert_close(p_w13[:, :inter, : hidden // 2], w13) + torch.testing.assert_close(p_w2[:, :hidden, : inter // 2], w2) + # Equal byte sums with the originals embedded means the padding is zero. + for padded, original in [ + (p_w13, w13), + (p_w13_scale, w13_scale), + (p_w2, w2), + (p_w2_scale, w2_scale), + ]: + self.assertEqual(padded.dtype, original.dtype) + self.assertEqual(_byte_sum(padded), _byte_sum(original)) + + def test_gated_pads_only_intermediate_by_default(self): + num_experts, hidden, inter = 2, 2688, 1000 + w13, w13_scale, w2, w2_scale = _generate_fp4_moe_weights( + num_experts, hidden, inter, is_gated=True + ) + + p_w13, _, p_w2, _, out_inter, out_hidden = _align_fp4_moe_weights( + w13, w13_scale, w2, w2_scale, is_gated=True, intermediate_alignment=16 + ) + + self.assertEqual((out_inter, out_hidden), (1008, hidden)) + self.assertEqual(p_w13.shape, (num_experts, 2 * 1008, hidden // 2)) + self.assertEqual(p_w2.shape, (num_experts, hidden, 1008 // 2)) + + if __name__ == "__main__": unittest.main()