Add per-token NVFP4 MoE support for ReLU2 activation (#39199)

Signed-off-by: Xuanteng Huang <xuanteng.huang@outlook.com>
This commit is contained in:
Xuanteng Huang
2026-10-08 14:28:22 -07:00
committed by GitHub
parent 3d8d44a4a9
commit d20bdc1443
5 changed files with 302 additions and 64 deletions
@@ -516,46 +516,61 @@ def _align_fp4_moe_weights(
w2: torch.Tensor,
w2_scale: torch.Tensor,
is_gated: bool,
min_alignment: int = 16,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""Pad intermediate size so FlashInfer TRTLLM FP4 kernels' alignment holds.
intermediate_alignment: int = 16,
hidden_alignment: int = 1,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
"""Zero-pad intermediate and hidden sizes to the TRT-LLM FP4 kernel alignments.
Returns (w13, w13_scale, w2, w2_scale, padded_intermediate).
Intermediate padding is transparent to callers. Hidden padding (K of GEMM1,
N of GEMM2) also needs the activations padded and the output sliced at runtime.
Returns (w13, w13_scale, w2, w2_scale, padded_intermediate, padded_hidden).
"""
num_experts, hidden_size, intermediate_packed = w2.shape
intermediate = intermediate_packed * 2 # FP4 packs 2 values per byte
padded_intermediate = round_up_to_multiple(intermediate, min_alignment)
if padded_intermediate == intermediate:
return w13, w13_scale, w2, w2_scale, intermediate
padded_intermediate = round_up_to_multiple(intermediate, intermediate_alignment)
padded_hidden = round_up_to_multiple(hidden_size, hidden_alignment)
if padded_intermediate == intermediate and padded_hidden == hidden_size:
return w13, w13_scale, w2, w2_scale, intermediate, hidden_size
logger.info(
"FP4 MoE: padding intermediate size from %d to %d (alignment=%d)",
"FP4 MoE: padding (intermediate, hidden) from (%d, %d) to (%d, %d)",
intermediate,
hidden_size,
padded_intermediate,
min_alignment,
padded_hidden,
)
up_mult = 2 if is_gated else 1
padded_gate_up = up_mult * padded_intermediate
padded_w13 = w13.new_zeros((num_experts, padded_gate_up, w13.shape[2]))
padded_w13[:, : w13.shape[1], :] = w13
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
padded_w2[:, :, : w2.shape[2]] = w2
# w13: [E, gate_up, hidden // 2], scale [E, gate_up, hidden // 16]
padded_w13 = w13.new_zeros((num_experts, padded_gate_up, padded_hidden // 2))
padded_w13[:, : w13.shape[1], : w13.shape[2]] = w13
padded_w13_scale = w13_scale.new_zeros(
(num_experts, padded_gate_up, w13_scale.shape[2])
(num_experts, padded_gate_up, padded_hidden // 16)
)
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
padded_w13_scale[:, : w13_scale.shape[1], : w13_scale.shape[2]] = w13_scale
# w2: [E, hidden, intermediate // 2], scale [E, hidden, intermediate // 16]
padded_w2 = w2.new_zeros((num_experts, padded_hidden, padded_intermediate // 2))
padded_w2[:, :hidden_size, : w2.shape[2]] = w2
padded_w2_scale = w2_scale.new_zeros(
(num_experts, hidden_size, padded_intermediate // 16)
(num_experts, padded_hidden, padded_intermediate // 16)
)
padded_w2_scale[:, :, : w2_scale.shape[2]] = w2_scale
padded_w2_scale[:, :hidden_size, : w2_scale.shape[2]] = w2_scale
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
return (
padded_w13,
padded_w13_scale,
padded_w2,
padded_w2_scale,
padded_intermediate,
padded_hidden,
)
def _compute_g1_scale_c(
@@ -584,11 +599,30 @@ def _compute_g1_scale_c(
return w2_input_scale_quant.to(torch.float32).expand(num_experts).contiguous()
def align_fp4_moe_weights_for_flashinfer_trtllm(layer: Module) -> None:
def trtllm_nvfp4_hidden_alignment(
use_per_token_activation: bool, is_gated: bool
) -> int:
"""Hidden-dim alignment required by the TRT-LLM NVFP4 MoE cubins.
The per-token (dynamic activation scale) non-gated variant ships far fewer
tile configs than the static one. At Nemotron's hidden size 2688 it runs
but produces garbage (GSM8K ~0.02 on FlashInfer 0.7.0.post1), while a
512-aligned hidden size is correct. Every other variant runs unpadded.
"""
if use_per_token_activation and not is_gated:
return 512
return 1
def align_fp4_moe_weights_for_flashinfer_trtllm(
layer: Module, hidden_alignment: int = 1
) -> None:
"""Prepare FP4 MoE weights/scales for FlashInfer TRT-LLM kernels.
This function handles the weight transformation needed for FP4 TRTLLM MoE:
- Pads intermediate dimension for kernel alignment constraints
- Pads hidden dimension to `hidden_alignment` and records the padded size
on `layer.trtllm_padded_hidden_size` (None when unpadded)
- Reorders weights for gated activation GEMM
- Shuffles weights and scales for transposed MMA output
- Computes the output scale factors
@@ -603,18 +637,27 @@ def align_fp4_moe_weights_for_flashinfer_trtllm(layer: Module) -> None:
w2_weight_scale = cast(torch.Tensor, layer.w2_weight_scale)
is_gated = layer.moe_runner_config.is_gated
min_alignment = 16 if is_gated else 128
hidden_size = w2_weight.shape[1]
# Pad for kernel alignment before shuffle/reorder
w13_weight, w13_weight_scale, w2_weight, w2_weight_scale, intermediate_size = (
_align_fp4_moe_weights(
w13_weight,
w13_weight_scale,
w2_weight,
w2_weight_scale,
is_gated,
min_alignment,
)
(
w13_weight,
w13_weight_scale,
w2_weight,
w2_weight_scale,
intermediate_size,
padded_hidden_size,
) = _align_fp4_moe_weights(
w13_weight,
w13_weight_scale,
w2_weight,
w2_weight_scale,
is_gated=is_gated,
intermediate_alignment=16 if is_gated else 128,
hidden_alignment=hidden_alignment,
)
layer.trtllm_padded_hidden_size = (
padded_hidden_size if padded_hidden_size != hidden_size else None
)
(
@@ -1249,6 +1292,8 @@ class FlashInferTrtllmFp4MoeQuantInfo(MoeQuantInfo):
routing_method_type: int
use_per_token_activation: bool = False
# Hidden size the weights were zero-padded to; None means unpadded.
padded_hidden_size: Optional[int] = None
gemm1_alpha: Optional[torch.Tensor] = None
gemm1_beta: Optional[torch.Tensor] = None
@@ -1286,6 +1331,34 @@ def quantize_hidden_states_fp4(
return hs_fp4, hs_sf
def _get_fp4_moe_output_buffer(
num_tokens: int,
hidden_size: int,
dtype: torch.dtype,
device: torch.device,
symmetric: bool,
) -> torch.Tensor:
"""Reuse the forward's MoE output buffer when it fits, else allocate one."""
from sglang.srt.runtime_context import get_forward
provided = get_forward().moe_output_buffer
symm_required = symmetric and is_allocation_symmetric()
if (
provided is not None
and provided.shape == (num_tokens, hidden_size)
and provided.dtype == dtype
and provided.device == device
and (
not symm_required
or not is_symmetric_memory_enabled()
or is_tensor_in_symmetric_mempool(provided)
)
):
return provided
with use_symmetric_memory(get_parallel().tp_group, disabled=not symm_required):
return torch.empty(num_tokens, hidden_size, dtype=dtype, device=device)
def fused_experts_none_to_flashinfer_trtllm_fp4(
dispatch_output: StandardDispatchOutput,
quant_info: FlashInferTrtllmFp4MoeQuantInfo,
@@ -1318,6 +1391,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
# Quantize hidden states to FP4
hidden_states_scale = dispatch_output.hidden_states_scale
per_token_scale = None
unpadded_hidden_size = hidden_states.shape[-1]
hidden_pad = 0
if quant_info.padded_hidden_size is not None:
assert hidden_states_scale is None, (
"Hidden-dim padded TRTLLM NVFP4 MoE weights need unquantized "
"activations; FP4 dispatch is not supported."
)
hidden_pad = quant_info.padded_hidden_size - unpadded_hidden_size
hidden_states = torch.nn.functional.pad(hidden_states, (0, hidden_pad))
if hidden_states_scale is not None:
# NVFP4 dispatch (flashinfer a2a): inputs are already FP4-quantized by
# the dispatcher, so pass them through unchanged.
@@ -1367,10 +1449,12 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
if not use_routed_topk and TopKOutputChecker.format_is_standard(topk_output):
use_routed_topk = True
# Deferred outputs keep the padded width, so padded weights always finalize.
defer_finalize = (
_deferred_finalize_enabled.get()
and not use_routed_topk
and TopKOutputChecker.format_is_bypassed(topk_output)
and hidden_pad == 0
)
symm_output = None
@@ -1384,32 +1468,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
output_dtype = (
hidden_states.dtype if hidden_states_scale is None else torch.bfloat16
)
from sglang.srt.runtime_context import get_forward
_provided = get_forward().moe_output_buffer
_symm_required = is_allocation_symmetric()
if (
_provided is not None
and _provided.shape == (num_tokens, hidden_size)
and _provided.dtype == output_dtype
and _provided.device == hs_fp4.device
and (
not _symm_required
or not is_symmetric_memory_enabled()
or is_tensor_in_symmetric_mempool(_provided)
)
):
symm_output = _provided
else:
with use_symmetric_memory(
get_parallel().tp_group, disabled=not _symm_required
):
symm_output = torch.empty(
num_tokens,
hidden_size,
dtype=output_dtype,
device=hs_fp4.device,
)
# A padded kernel output is a scratch buffer; the sliced copy is the
# tensor that reaches the all-reduce.
symm_output = _get_fp4_moe_output_buffer(
num_tokens=num_tokens,
hidden_size=hidden_size,
dtype=output_dtype,
device=hs_fp4.device,
symmetric=hidden_pad == 0,
)
if use_routed_topk:
routing = _get_routing_for_flashinfer_routed(topk_output)
@@ -1499,6 +1566,17 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
else:
result = result[0]
if hidden_pad > 0:
unpadded_result = _get_fp4_moe_output_buffer(
num_tokens=result.shape[0],
hidden_size=unpadded_hidden_size,
dtype=result.dtype,
device=result.device,
symmetric=True,
)
unpadded_result.copy_(result[:, :unpadded_hidden_size])
result = unpadded_result
return StandardCombineInput(hidden_states=result)
@@ -2882,10 +2882,17 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
):
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
align_fp4_moe_weights_for_flashinfer_trtllm,
trtllm_nvfp4_hidden_alignment,
)
# FlashInfer TRTLLM processing - handles both w13 and w2
align_fp4_moe_weights_for_flashinfer_trtllm(layer)
align_fp4_moe_weights_for_flashinfer_trtllm(
layer,
hidden_alignment=trtllm_nvfp4_hidden_alignment(
use_per_token_activation=self.quant_config.use_per_token_activation,
is_gated=layer.moe_runner_config.is_gated,
),
)
# TRTLLM doesn't read *_blockscale_swizzled; alias to free the
# placeholders from create_weights.
layer.w13_blockscale_swizzled = layer.w13_weight_scale
@@ -3169,6 +3176,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
intermediate_size_per_partition=layer.intermediate_size_per_partition,
routing_method_type=routing_method_type,
use_per_token_activation=self.quant_config.use_per_token_activation,
padded_hidden_size=layer.trtllm_padded_hidden_size,
gemm1_alpha=gemm1_alpha.data if gemm1_alpha is not None else None,
gemm1_beta=gemm1_beta.data if gemm1_beta is not None else None,
gemm1_clamp_limit=gemm1_clamp.data if gemm1_clamp is not None else None,