Add per-token NVFP4 MoE support for ReLU2 activation (#39199)
Signed-off-by: Xuanteng Huang <xuanteng.huang@outlook.com>
This commit is contained in:
@@ -516,46 +516,61 @@ def _align_fp4_moe_weights(
|
||||
w2: torch.Tensor,
|
||||
w2_scale: torch.Tensor,
|
||||
is_gated: bool,
|
||||
min_alignment: int = 16,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
"""Pad intermediate size so FlashInfer TRTLLM FP4 kernels' alignment holds.
|
||||
intermediate_alignment: int = 16,
|
||||
hidden_alignment: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, int, int]:
|
||||
"""Zero-pad intermediate and hidden sizes to the TRT-LLM FP4 kernel alignments.
|
||||
|
||||
Returns (w13, w13_scale, w2, w2_scale, padded_intermediate).
|
||||
Intermediate padding is transparent to callers. Hidden padding (K of GEMM1,
|
||||
N of GEMM2) also needs the activations padded and the output sliced at runtime.
|
||||
|
||||
Returns (w13, w13_scale, w2, w2_scale, padded_intermediate, padded_hidden).
|
||||
"""
|
||||
num_experts, hidden_size, intermediate_packed = w2.shape
|
||||
intermediate = intermediate_packed * 2 # FP4 packs 2 values per byte
|
||||
|
||||
padded_intermediate = round_up_to_multiple(intermediate, min_alignment)
|
||||
if padded_intermediate == intermediate:
|
||||
return w13, w13_scale, w2, w2_scale, intermediate
|
||||
padded_intermediate = round_up_to_multiple(intermediate, intermediate_alignment)
|
||||
padded_hidden = round_up_to_multiple(hidden_size, hidden_alignment)
|
||||
if padded_intermediate == intermediate and padded_hidden == hidden_size:
|
||||
return w13, w13_scale, w2, w2_scale, intermediate, hidden_size
|
||||
|
||||
logger.info(
|
||||
"FP4 MoE: padding intermediate size from %d to %d (alignment=%d)",
|
||||
"FP4 MoE: padding (intermediate, hidden) from (%d, %d) to (%d, %d)",
|
||||
intermediate,
|
||||
hidden_size,
|
||||
padded_intermediate,
|
||||
min_alignment,
|
||||
padded_hidden,
|
||||
)
|
||||
|
||||
up_mult = 2 if is_gated else 1
|
||||
padded_gate_up = up_mult * padded_intermediate
|
||||
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up, w13.shape[2]))
|
||||
padded_w13[:, : w13.shape[1], :] = w13
|
||||
|
||||
padded_w2 = w2.new_zeros((num_experts, hidden_size, padded_intermediate // 2))
|
||||
padded_w2[:, :, : w2.shape[2]] = w2
|
||||
# w13: [E, gate_up, hidden // 2], scale [E, gate_up, hidden // 16]
|
||||
padded_w13 = w13.new_zeros((num_experts, padded_gate_up, padded_hidden // 2))
|
||||
padded_w13[:, : w13.shape[1], : w13.shape[2]] = w13
|
||||
|
||||
padded_w13_scale = w13_scale.new_zeros(
|
||||
(num_experts, padded_gate_up, w13_scale.shape[2])
|
||||
(num_experts, padded_gate_up, padded_hidden // 16)
|
||||
)
|
||||
padded_w13_scale[:, : w13_scale.shape[1], :] = w13_scale
|
||||
padded_w13_scale[:, : w13_scale.shape[1], : w13_scale.shape[2]] = w13_scale
|
||||
|
||||
# w2: [E, hidden, intermediate // 2], scale [E, hidden, intermediate // 16]
|
||||
padded_w2 = w2.new_zeros((num_experts, padded_hidden, padded_intermediate // 2))
|
||||
padded_w2[:, :hidden_size, : w2.shape[2]] = w2
|
||||
|
||||
padded_w2_scale = w2_scale.new_zeros(
|
||||
(num_experts, hidden_size, padded_intermediate // 16)
|
||||
(num_experts, padded_hidden, padded_intermediate // 16)
|
||||
)
|
||||
padded_w2_scale[:, :, : w2_scale.shape[2]] = w2_scale
|
||||
padded_w2_scale[:, :hidden_size, : w2_scale.shape[2]] = w2_scale
|
||||
|
||||
return padded_w13, padded_w13_scale, padded_w2, padded_w2_scale, padded_intermediate
|
||||
return (
|
||||
padded_w13,
|
||||
padded_w13_scale,
|
||||
padded_w2,
|
||||
padded_w2_scale,
|
||||
padded_intermediate,
|
||||
padded_hidden,
|
||||
)
|
||||
|
||||
|
||||
def _compute_g1_scale_c(
|
||||
@@ -584,11 +599,30 @@ def _compute_g1_scale_c(
|
||||
return w2_input_scale_quant.to(torch.float32).expand(num_experts).contiguous()
|
||||
|
||||
|
||||
def align_fp4_moe_weights_for_flashinfer_trtllm(layer: Module) -> None:
|
||||
def trtllm_nvfp4_hidden_alignment(
|
||||
use_per_token_activation: bool, is_gated: bool
|
||||
) -> int:
|
||||
"""Hidden-dim alignment required by the TRT-LLM NVFP4 MoE cubins.
|
||||
|
||||
The per-token (dynamic activation scale) non-gated variant ships far fewer
|
||||
tile configs than the static one. At Nemotron's hidden size 2688 it runs
|
||||
but produces garbage (GSM8K ~0.02 on FlashInfer 0.7.0.post1), while a
|
||||
512-aligned hidden size is correct. Every other variant runs unpadded.
|
||||
"""
|
||||
if use_per_token_activation and not is_gated:
|
||||
return 512
|
||||
return 1
|
||||
|
||||
|
||||
def align_fp4_moe_weights_for_flashinfer_trtllm(
|
||||
layer: Module, hidden_alignment: int = 1
|
||||
) -> None:
|
||||
"""Prepare FP4 MoE weights/scales for FlashInfer TRT-LLM kernels.
|
||||
|
||||
This function handles the weight transformation needed for FP4 TRTLLM MoE:
|
||||
- Pads intermediate dimension for kernel alignment constraints
|
||||
- Pads hidden dimension to `hidden_alignment` and records the padded size
|
||||
on `layer.trtllm_padded_hidden_size` (None when unpadded)
|
||||
- Reorders weights for gated activation GEMM
|
||||
- Shuffles weights and scales for transposed MMA output
|
||||
- Computes the output scale factors
|
||||
@@ -603,18 +637,27 @@ def align_fp4_moe_weights_for_flashinfer_trtllm(layer: Module) -> None:
|
||||
w2_weight_scale = cast(torch.Tensor, layer.w2_weight_scale)
|
||||
|
||||
is_gated = layer.moe_runner_config.is_gated
|
||||
min_alignment = 16 if is_gated else 128
|
||||
hidden_size = w2_weight.shape[1]
|
||||
|
||||
# Pad for kernel alignment before shuffle/reorder
|
||||
w13_weight, w13_weight_scale, w2_weight, w2_weight_scale, intermediate_size = (
|
||||
_align_fp4_moe_weights(
|
||||
w13_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight,
|
||||
w2_weight_scale,
|
||||
is_gated,
|
||||
min_alignment,
|
||||
)
|
||||
(
|
||||
w13_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight,
|
||||
w2_weight_scale,
|
||||
intermediate_size,
|
||||
padded_hidden_size,
|
||||
) = _align_fp4_moe_weights(
|
||||
w13_weight,
|
||||
w13_weight_scale,
|
||||
w2_weight,
|
||||
w2_weight_scale,
|
||||
is_gated=is_gated,
|
||||
intermediate_alignment=16 if is_gated else 128,
|
||||
hidden_alignment=hidden_alignment,
|
||||
)
|
||||
layer.trtllm_padded_hidden_size = (
|
||||
padded_hidden_size if padded_hidden_size != hidden_size else None
|
||||
)
|
||||
|
||||
(
|
||||
@@ -1249,6 +1292,8 @@ class FlashInferTrtllmFp4MoeQuantInfo(MoeQuantInfo):
|
||||
|
||||
routing_method_type: int
|
||||
use_per_token_activation: bool = False
|
||||
# Hidden size the weights were zero-padded to; None means unpadded.
|
||||
padded_hidden_size: Optional[int] = None
|
||||
|
||||
gemm1_alpha: Optional[torch.Tensor] = None
|
||||
gemm1_beta: Optional[torch.Tensor] = None
|
||||
@@ -1286,6 +1331,34 @@ def quantize_hidden_states_fp4(
|
||||
return hs_fp4, hs_sf
|
||||
|
||||
|
||||
def _get_fp4_moe_output_buffer(
|
||||
num_tokens: int,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
symmetric: bool,
|
||||
) -> torch.Tensor:
|
||||
"""Reuse the forward's MoE output buffer when it fits, else allocate one."""
|
||||
from sglang.srt.runtime_context import get_forward
|
||||
|
||||
provided = get_forward().moe_output_buffer
|
||||
symm_required = symmetric and is_allocation_symmetric()
|
||||
if (
|
||||
provided is not None
|
||||
and provided.shape == (num_tokens, hidden_size)
|
||||
and provided.dtype == dtype
|
||||
and provided.device == device
|
||||
and (
|
||||
not symm_required
|
||||
or not is_symmetric_memory_enabled()
|
||||
or is_tensor_in_symmetric_mempool(provided)
|
||||
)
|
||||
):
|
||||
return provided
|
||||
with use_symmetric_memory(get_parallel().tp_group, disabled=not symm_required):
|
||||
return torch.empty(num_tokens, hidden_size, dtype=dtype, device=device)
|
||||
|
||||
|
||||
def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
dispatch_output: StandardDispatchOutput,
|
||||
quant_info: FlashInferTrtllmFp4MoeQuantInfo,
|
||||
@@ -1318,6 +1391,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
# Quantize hidden states to FP4
|
||||
hidden_states_scale = dispatch_output.hidden_states_scale
|
||||
per_token_scale = None
|
||||
unpadded_hidden_size = hidden_states.shape[-1]
|
||||
hidden_pad = 0
|
||||
if quant_info.padded_hidden_size is not None:
|
||||
assert hidden_states_scale is None, (
|
||||
"Hidden-dim padded TRTLLM NVFP4 MoE weights need unquantized "
|
||||
"activations; FP4 dispatch is not supported."
|
||||
)
|
||||
hidden_pad = quant_info.padded_hidden_size - unpadded_hidden_size
|
||||
hidden_states = torch.nn.functional.pad(hidden_states, (0, hidden_pad))
|
||||
if hidden_states_scale is not None:
|
||||
# NVFP4 dispatch (flashinfer a2a): inputs are already FP4-quantized by
|
||||
# the dispatcher, so pass them through unchanged.
|
||||
@@ -1367,10 +1449,12 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
if not use_routed_topk and TopKOutputChecker.format_is_standard(topk_output):
|
||||
use_routed_topk = True
|
||||
|
||||
# Deferred outputs keep the padded width, so padded weights always finalize.
|
||||
defer_finalize = (
|
||||
_deferred_finalize_enabled.get()
|
||||
and not use_routed_topk
|
||||
and TopKOutputChecker.format_is_bypassed(topk_output)
|
||||
and hidden_pad == 0
|
||||
)
|
||||
|
||||
symm_output = None
|
||||
@@ -1384,32 +1468,15 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
output_dtype = (
|
||||
hidden_states.dtype if hidden_states_scale is None else torch.bfloat16
|
||||
)
|
||||
from sglang.srt.runtime_context import get_forward
|
||||
|
||||
_provided = get_forward().moe_output_buffer
|
||||
_symm_required = is_allocation_symmetric()
|
||||
if (
|
||||
_provided is not None
|
||||
and _provided.shape == (num_tokens, hidden_size)
|
||||
and _provided.dtype == output_dtype
|
||||
and _provided.device == hs_fp4.device
|
||||
and (
|
||||
not _symm_required
|
||||
or not is_symmetric_memory_enabled()
|
||||
or is_tensor_in_symmetric_mempool(_provided)
|
||||
)
|
||||
):
|
||||
symm_output = _provided
|
||||
else:
|
||||
with use_symmetric_memory(
|
||||
get_parallel().tp_group, disabled=not _symm_required
|
||||
):
|
||||
symm_output = torch.empty(
|
||||
num_tokens,
|
||||
hidden_size,
|
||||
dtype=output_dtype,
|
||||
device=hs_fp4.device,
|
||||
)
|
||||
# A padded kernel output is a scratch buffer; the sliced copy is the
|
||||
# tensor that reaches the all-reduce.
|
||||
symm_output = _get_fp4_moe_output_buffer(
|
||||
num_tokens=num_tokens,
|
||||
hidden_size=hidden_size,
|
||||
dtype=output_dtype,
|
||||
device=hs_fp4.device,
|
||||
symmetric=hidden_pad == 0,
|
||||
)
|
||||
|
||||
if use_routed_topk:
|
||||
routing = _get_routing_for_flashinfer_routed(topk_output)
|
||||
@@ -1499,6 +1566,17 @@ def fused_experts_none_to_flashinfer_trtllm_fp4(
|
||||
else:
|
||||
result = result[0]
|
||||
|
||||
if hidden_pad > 0:
|
||||
unpadded_result = _get_fp4_moe_output_buffer(
|
||||
num_tokens=result.shape[0],
|
||||
hidden_size=unpadded_hidden_size,
|
||||
dtype=result.dtype,
|
||||
device=result.device,
|
||||
symmetric=True,
|
||||
)
|
||||
unpadded_result.copy_(result[:, :unpadded_hidden_size])
|
||||
result = unpadded_result
|
||||
|
||||
return StandardCombineInput(hidden_states=result)
|
||||
|
||||
|
||||
|
||||
@@ -2882,10 +2882,17 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
):
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_trtllm import (
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm,
|
||||
trtllm_nvfp4_hidden_alignment,
|
||||
)
|
||||
|
||||
# FlashInfer TRTLLM processing - handles both w13 and w2
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm(layer)
|
||||
align_fp4_moe_weights_for_flashinfer_trtllm(
|
||||
layer,
|
||||
hidden_alignment=trtllm_nvfp4_hidden_alignment(
|
||||
use_per_token_activation=self.quant_config.use_per_token_activation,
|
||||
is_gated=layer.moe_runner_config.is_gated,
|
||||
),
|
||||
)
|
||||
# TRTLLM doesn't read *_blockscale_swizzled; alias to free the
|
||||
# placeholders from create_weights.
|
||||
layer.w13_blockscale_swizzled = layer.w13_weight_scale
|
||||
@@ -3169,6 +3176,7 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
intermediate_size_per_partition=layer.intermediate_size_per_partition,
|
||||
routing_method_type=routing_method_type,
|
||||
use_per_token_activation=self.quant_config.use_per_token_activation,
|
||||
padded_hidden_size=layer.trtllm_padded_hidden_size,
|
||||
gemm1_alpha=gemm1_alpha.data if gemm1_alpha is not None else None,
|
||||
gemm1_beta=gemm1_beta.data if gemm1_beta is not None else None,
|
||||
gemm1_clamp_limit=gemm1_clamp.data if gemm1_clamp is not None else None,
|
||||
|
||||
Reference in New Issue
Block a user