[Fix] [NVIDIA] modelopt-nvfp4: Keep MoE global scales in gate/up order (#45289)

This commit is contained in:
Baizhou Zhang
2026-10-20 16:49:13 -07:00
committed by GitHub
parent 9686edcf80
commit 5b1dbc2c80
4 changed files with 240 additions and 154 deletions
@@ -1266,16 +1266,11 @@ class FusedMoE(torch.nn.Module):
if shard_id not in ("w1", "w2", "w3"):
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
use_flashinfer_trtllm_weight_layout = (
method.use_flashinfer_trtllm_weight_layout
if isinstance(method, ModelOptNvFp4FusedMoEMethod)
else self.use_flashinfer_trtllm_moe
)
# Flashinfer assumes w31 format for w13_weight. Same for the scales.
if use_flashinfer_trtllm_weight_layout and (
isinstance(method, ModelOptNvFp4FusedMoEMethod)
or isinstance(method, Fp8MoEMethod)
# ModelOpt NVFP4 reorders weights and block scales in _load_w13;
# its per-tensor global scales retain the logical [gate, up] order.
# These other methods also reorder their per-tensor scales.
if self.use_flashinfer_trtllm_moe and (
isinstance(method, Fp8MoEMethod)
or isinstance(method, UnquantizedFusedMoEMethod)
or isinstance(method, CompressedTensorsMxInt4MoE)
):
@@ -2337,24 +2337,20 @@ def _compute_gemm1_alphas(
w13_weight_scale_2: torch.Tensor,
w13_input_scale: torch.Tensor,
is_gated: bool,
*,
up_first: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""GEMM1 weight x input alphas for the gate (w1) and up (w3) halves of w13.
w13 fuses the gate and up projections, which may carry separate NVFP4 weight
scales stored as [num_experts, 2] in w13 row order: [gate, up] by default,
[up, gate] when up_first (the FlashInfer TRT-LLM loader layout).
A 1-D (or [num_experts, 1]) scale, and any non-gated layer, shares one scale
across both halves; the col-1 read is guarded so those cases stay in bounds.
scales stored as [num_experts, 2] (col 0 = gate, col 1 = up). A 1-D (or
[num_experts, 1]) scale, and any non-gated layer, shares one scale across
both halves; the col-1 read is guarded so those cases stay in bounds.
Returns (g1_alphas, g1_alphas_up), equal for a shared scale. Single-alpha
backends use g1_alphas; the TRT-LLM path also uses g1_alphas_up.
"""
if is_gated and w13_weight_scale_2.dim() == 2 and w13_weight_scale_2.shape[1] >= 2:
gate_col, up_col = (1, 0) if up_first else (0, 1)
gate_scale = w13_weight_scale_2[:, gate_col]
up_scale = w13_weight_scale_2[:, up_col]
gate_scale = w13_weight_scale_2[:, 0]
up_scale = w13_weight_scale_2[:, 1]
else:
gate_scale = w13_weight_scale_2.reshape(w13_weight_scale_2.shape[0])
up_scale = gate_scale
@@ -2363,41 +2359,6 @@ def _compute_gemm1_alphas(
return g1_alphas, g1_alphas_up
def _compute_trtllm_gated_act_params(
*,
activation: Optional[str],
gemm1_alpha: Optional[float],
gemm1_clamp_limit: Optional[float],
swiglu_limit: Optional[float],
g1_alphas: torch.Tensor,
g1_alphas_up: torch.Tensor,
) -> dict[str, torch.Tensor]:
params = {}
is_situ = activation == "situ"
clamp_limit = None if is_situ else (gemm1_clamp_limit or swiglu_limit)
if clamp_limit is not None:
# The kernel clamps both raw GEMM1 halves at this one value,
# so it is exact for the up half only when the gate/up scales match.
if not torch.equal(g1_alphas, g1_alphas_up):
logger.warning_once(
"NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU clamp "
"limit is exact for the gate half only. Accuracy may be affected."
)
params["gemm1_clamp_limit"] = (clamp_limit / g1_alphas).to(torch.float32)
if gemm1_alpha is not None:
params["gemm1_alpha"] = torch.full_like(
g1_alphas, gemm1_alpha, dtype=torch.float32
)
# Beta is added to the raw up half, so it is in up-alpha units.
params["gemm1_beta"] = (
torch.full_like(g1_alphas, gemm1_clamp_limit, dtype=torch.float32)
if is_situ
else (1.0 / g1_alphas_up).to(torch.float32)
)
return params
class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
"""
MoE Method for FP4 Quantization with Blockscales and PerTensorScales
@@ -2855,10 +2816,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
# Create shared parameters. g1_alphas / g1_alphas_up are the gate (w1)
# and up (w3) GEMM1 scales (equal for shared-scale checkpoints).
g1_alphas, g1_alphas_up = _compute_gemm1_alphas(
w13_weight_scale_2=layer.w13_weight_scale_2,
w13_input_scale=w13_input_scale,
is_gated=layer.moe_runner_config.is_gated,
up_first=self.use_flashinfer_trtllm_weight_layout,
layer.w13_weight_scale_2,
w13_input_scale,
layer.moe_runner_config.is_gated,
)
copy_or_rebind_param(layer, "g1_alphas", g1_alphas)
copy_or_rebind_param(layer, "g1_alphas_up", g1_alphas_up)
@@ -2880,16 +2840,48 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
if layer.moe_runner_config.is_gated and self.enable_flashinfer_trtllm_moe:
runner_config = layer.moe_runner_config
gated_act_params = _compute_trtllm_gated_act_params(
activation=runner_config.activation,
gemm1_alpha=runner_config.gemm1_alpha,
gemm1_clamp_limit=runner_config.gemm1_clamp_limit,
swiglu_limit=runner_config.swiglu_limit,
g1_alphas=layer.g1_alphas,
g1_alphas_up=layer.g1_alphas_up,
is_situ = runner_config.activation == "situ"
gemm1_clamp_limit = (
None
if is_situ
else (runner_config.gemm1_clamp_limit or runner_config.swiglu_limit)
)
for name, value in gated_act_params.items():
copy_or_rebind_param(layer, name, value)
if gemm1_clamp_limit is not None:
# The kernel clamps both raw halves with one limit, which is
# exact for both only when their dequantization scales match.
if not torch.equal(layer.g1_alphas, layer.g1_alphas_up):
logger.warning_once(
"NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU "
"clamp limit is exact for the gate half only. "
"Accuracy may be affected."
)
copy_or_rebind_param(
layer,
"gemm1_clamp_limit",
(gemm1_clamp_limit / layer.g1_alphas).to(torch.float32),
)
if runner_config.gemm1_alpha is not None:
copy_or_rebind_param(
layer,
"gemm1_alpha",
torch.full_like(
layer.g1_alphas,
runner_config.gemm1_alpha,
dtype=torch.float32,
),
)
# SwiGLU adds beta to the raw up half before dequantization.
gemm1_beta = (
torch.full_like(
layer.g1_alphas,
runner_config.gemm1_clamp_limit,
dtype=torch.float32,
)
if is_situ
else (1.0 / layer.g1_alphas_up).to(torch.float32)
)
copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta)
if self.enable_flashinfer_cutlass_moe:
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
@@ -3092,10 +3084,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
@property
def load_up_proj_weight_first(self) -> bool:
# Load W13 as [Up, Gate] for FlashInfer CUTLASS and CuteDSL v2 kernels.
# The CuteDSL v1 (deepep) path uses [Gate, Up] -- do NOT flip.
# Only weights and block scales follow the kernel's physical row order;
# per-tensor global scales always remain [Gate, Up]. MegaMoE and
# CuteDSL v1 (deepep) consume canonical [Gate, Up] weights as well.
return self.moe_runner_config.is_gated and (
self.enable_flashinfer_cutlass_moe or self._is_cutedsl_v2_standard
self.use_flashinfer_trtllm_weight_layout
or self.enable_flashinfer_cutlass_moe
or self._is_cutedsl_v2_standard
)
def create_moe_runner(