diff --git a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py index d4ca349e73..38ed6cb5c3 100644 --- a/python/sglang/srt/layers/moe/fused_moe_triton/layer.py +++ b/python/sglang/srt/layers/moe/fused_moe_triton/layer.py @@ -425,16 +425,6 @@ class FusedMoE(torch.nn.Module): ) self.use_deep_gemm = get_moe_runner_backend().is_deep_gemm() - # flashinfer_trtllm kernel requires intermediate_size to be a multiple of 128 - # Pad the intermediate_size_per_partition if necessary - if ( - self.use_flashinfer_trtllm_moe - and self.intermediate_size_per_partition % 128 != 0 - ): - self.intermediate_size_per_partition = round_up( - self.intermediate_size_per_partition, 128 - ) - self.quant_config = quant_config self.use_flashinfer_mxfp4_moe = get_moe_runner_backend().is_flashinfer_mxfp4() # TODO maybe we should remove this `if`, since `Mxfp4MoEMethod` does another round-up logic @@ -504,6 +494,20 @@ class FusedMoE(torch.nn.Module): self.quant_method = make_for_layer( layer=self, base_method=self.quant_method ) + # flashinfer_trtllm pads intermediate_size to a multiple of 128 here, except + # NVFP4 which aligns it in create_weights + if ( + self.use_flashinfer_trtllm_moe + and not isinstance(self.quant_method, ModelOptNvFp4FusedMoEMethod) + and self.intermediate_size_per_partition % 128 != 0 + ): + self.intermediate_size_per_partition = round_up( + self.intermediate_size_per_partition, 128 + ) + self.moe_runner_config.intermediate_size_per_partition = ( + self.intermediate_size_per_partition + ) + _validate_hpc_ops_quant_method(self.quant_method) _validate_deepep_v2_quant_method(self.quant_method) if ( @@ -661,7 +665,7 @@ class FusedMoE(torch.nn.Module): # This handles the case where the loaded weights are smaller than the padded expert_data # Use narrow_padded_param_and_loaded_weight for: # 1. CPU (always) - # 2. GPU with flashinfer_trtllm padding (when intermediate_size is padded to 128) + # 2. GPU with flashinfer_trtllm padding (intermediate_size padded to 128, or 64 for gated NVFP4) # 3. GPU with Aiter padding aiter_padded = ( _use_aiter diff --git a/python/sglang/srt/layers/quantization/modelopt_quant.py b/python/sglang/srt/layers/quantization/modelopt_quant.py index 0a84ce8fe0..5d90372611 100755 --- a/python/sglang/srt/layers/quantization/modelopt_quant.py +++ b/python/sglang/srt/layers/quantization/modelopt_quant.py @@ -2447,6 +2447,19 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase): params_dtype: torch.dtype, **extra_weight_attrs, ): + if self.enable_flashinfer_trtllm_moe: + alignment = ( + 64 + if layer.moe_runner_config.is_gated + and self.use_flashinfer_trtllm_weight_layout + else 128 + ) + intermediate_size_per_partition = round_up( + intermediate_size_per_partition, alignment + ) + layer.moe_runner_config.intermediate_size_per_partition = ( + intermediate_size_per_partition + ) # TODO(ch-wan): check if this is needed layer.intermediate_size_per_partition = intermediate_size_per_partition layer.params_dtype = params_dtype diff --git a/test/registered/unit/layers/quantization/test_nvfp4_moe_backends.py b/test/registered/unit/layers/quantization/test_nvfp4_moe_backends.py index 8ba5f246bc..81444834a0 100644 --- a/test/registered/unit/layers/quantization/test_nvfp4_moe_backends.py +++ b/test/registered/unit/layers/quantization/test_nvfp4_moe_backends.py @@ -12,7 +12,7 @@ import torch from sglang.srt.layers.moe.utils import MoeRunnerBackend from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config from sglang.srt.runtime_context import get_context, get_flags, get_parallel -from sglang.srt.utils import get_device_sm +from sglang.srt.utils import get_device_sm, round_up from sglang.test.ci.ci_register import register_cuda_ci from sglang.test.layer_ut_utils import assert_output_close, init_single_process_dist from sglang.test.quant_ref_utils import ( @@ -35,7 +35,7 @@ class TestNvFp4MoeBackends(CustomTestCase): init_single_process_dist(master_port=29631) torch.set_default_device("cuda") - def _run_backend(self, backend: str): + def _run_backend(self, backend: str, intermediate_size: int = I): from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE from sglang.srt.layers.moe.topk import TopKConfig, select_experts @@ -58,7 +58,7 @@ class TestNvFp4MoeBackends(CustomTestCase): layer = FusedMoE( num_experts=E, hidden_size=H, - intermediate_size=I, + intermediate_size=intermediate_size, layer_id=0, top_k=TOPK, params_dtype=torch.bfloat16, @@ -69,14 +69,24 @@ class TestNvFp4MoeBackends(CustomTestCase): # Checkpoint-format shards through the real weight_loader; # gate/up placement stays the loader's job. refs = { - "w1": torch.zeros(E, I, H, dtype=torch.float32, device="cuda"), - "w3": torch.zeros(E, I, H, dtype=torch.float32, device="cuda"), - "w2": torch.zeros(E, H, I, dtype=torch.float32, device="cuda"), + "w1": torch.zeros( + E, intermediate_size, H, dtype=torch.float32, device="cuda" + ), + "w3": torch.zeros( + E, intermediate_size, H, dtype=torch.float32, device="cuda" + ), + "w2": torch.zeros( + E, H, intermediate_size, dtype=torch.float32, device="cuda" + ), } act_scale = 1.0 / (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) for e in range(E): for shard_id in ("w1", "w3", "w2"): - rows, cols = (I, H) if shard_id in ("w1", "w3") else (H, I) + rows, cols = ( + (intermediate_size, H) + if shard_id in ("w1", "w3") + else (H, intermediate_size) + ) w = ( torch.randn(rows, cols, dtype=torch.bfloat16, device="cuda") / 10 @@ -96,6 +106,21 @@ class TestNvFp4MoeBackends(CustomTestCase): ) refs[shard_id][e] = w_dequant layer.quant_method.process_weights_after_loading(layer) + if backend == "flashinfer_trtllm": + expected_intermediate = round_up(intermediate_size, 64) + self.assertEqual( + layer.intermediate_size_per_partition, expected_intermediate + ) + self.assertEqual( + layer.moe_runner_config.intermediate_size_per_partition, + expected_intermediate, + ) + self.assertEqual( + layer.w13_weight.shape, (E, 2 * expected_intermediate, H // 2) + ) + self.assertEqual( + layer.w2_weight.shape, (E, H, expected_intermediate // 2) + ) x = torch.randn(M, H, dtype=torch.bfloat16, device="cuda") / 10 router_logits = torch.randn(M, E, dtype=torch.float32, device="cuda") @@ -149,6 +174,12 @@ class TestNvFp4MoeBackends(CustomTestCase): def test_flashinfer_trtllm(self): self._run_backend("flashinfer_trtllm") + def test_flashinfer_trtllm_intermediate_alignment(self): + """Gated NVFP4 keeps 64-aligned expert widths without padding to 128.""" + for intermediate_size in (160, 192, 256): + with self.subTest(intermediate_size=intermediate_size): + self._run_backend("flashinfer_trtllm", intermediate_size) + def test_flashinfer_cutedsl(self): self._run_backend("flashinfer_cutedsl")