[Fix] [NVIDIA] modelopt-nvfp4: Read w13 global scales in the TRT-LLM [up, gate] order (#44858)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-20 07:38:45 +00:00
committed by GitHub
co-authored by Mohammad Angkad
parent 1739d962a4
commit 78185f4b3f
3 changed files with 141 additions and 46 deletions
@@ -2316,20 +2316,24 @@ def _compute_gemm1_alphas(
w13_weight_scale_2: torch.Tensor,
w13_input_scale: torch.Tensor,
is_gated: bool,
*,
up_first: bool = False,
) -> tuple[torch.Tensor, torch.Tensor]:
"""GEMM1 weight x input alphas for the gate (w1) and up (w3) halves of w13.
w13 fuses the gate and up projections, which may carry separate NVFP4 weight
scales stored as [num_experts, 2] (col 0 = gate, col 1 = up). A 1-D (or
[num_experts, 1]) scale, and any non-gated layer, shares one scale across
both halves; the col-1 read is guarded so those cases stay in bounds.
scales stored as [num_experts, 2] in w13 row order: [gate, up] by default,
[up, gate] when up_first (the FlashInfer TRT-LLM loader layout).
A 1-D (or [num_experts, 1]) scale, and any non-gated layer, shares one scale
across both halves; the col-1 read is guarded so those cases stay in bounds.
Returns (g1_alphas, g1_alphas_up), equal for a shared scale. Single-alpha
backends use g1_alphas; the TRT-LLM path also uses g1_alphas_up.
"""
if is_gated and w13_weight_scale_2.dim() == 2 and w13_weight_scale_2.shape[1] >= 2:
gate_scale = w13_weight_scale_2[:, 0]
up_scale = w13_weight_scale_2[:, 1]
gate_col, up_col = (1, 0) if up_first else (0, 1)
gate_scale = w13_weight_scale_2[:, gate_col]
up_scale = w13_weight_scale_2[:, up_col]
else:
gate_scale = w13_weight_scale_2.reshape(w13_weight_scale_2.shape[0])
up_scale = gate_scale
@@ -2338,6 +2342,41 @@ def _compute_gemm1_alphas(
return g1_alphas, g1_alphas_up
def _compute_trtllm_gated_act_params(
*,
activation: Optional[str],
gemm1_alpha: Optional[float],
gemm1_clamp_limit: Optional[float],
swiglu_limit: Optional[float],
g1_alphas: torch.Tensor,
g1_alphas_up: torch.Tensor,
) -> dict[str, torch.Tensor]:
params = {}
is_situ = activation == "situ"
clamp_limit = None if is_situ else (gemm1_clamp_limit or swiglu_limit)
if clamp_limit is not None:
# The kernel clamps both raw GEMM1 halves at this one value,
# so it is exact for the up half only when the gate/up scales match.
if not torch.equal(g1_alphas, g1_alphas_up):
logger.warning_once(
"NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU clamp "
"limit is exact for the gate half only. Accuracy may be affected."
)
params["gemm1_clamp_limit"] = (clamp_limit / g1_alphas).to(torch.float32)
if gemm1_alpha is not None:
params["gemm1_alpha"] = torch.full_like(
g1_alphas, gemm1_alpha, dtype=torch.float32
)
# Beta is added to the raw up half, so it is in up-alpha units.
params["gemm1_beta"] = (
torch.full_like(g1_alphas, gemm1_clamp_limit, dtype=torch.float32)
if is_situ
else (1.0 / g1_alphas_up).to(torch.float32)
)
return params
class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
"""
MoE Method for FP4 Quantization with Blockscales and PerTensorScales
@@ -2795,9 +2834,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
# Create shared parameters. g1_alphas / g1_alphas_up are the gate (w1)
# and up (w3) GEMM1 scales (equal for shared-scale checkpoints).
g1_alphas, g1_alphas_up = _compute_gemm1_alphas(
layer.w13_weight_scale_2,
w13_input_scale,
layer.moe_runner_config.is_gated,
w13_weight_scale_2=layer.w13_weight_scale_2,
w13_input_scale=w13_input_scale,
is_gated=layer.moe_runner_config.is_gated,
up_first=self.use_flashinfer_trtllm_weight_layout,
)
copy_or_rebind_param(layer, "g1_alphas", g1_alphas)
copy_or_rebind_param(layer, "g1_alphas_up", g1_alphas_up)
@@ -2819,39 +2859,16 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
if layer.moe_runner_config.is_gated and self.enable_flashinfer_trtllm_moe:
runner_config = layer.moe_runner_config
is_situ = runner_config.activation == "situ"
gemm1_clamp_limit = (
None
if is_situ
else (runner_config.gemm1_clamp_limit or runner_config.swiglu_limit)
gated_act_params = _compute_trtllm_gated_act_params(
activation=runner_config.activation,
gemm1_alpha=runner_config.gemm1_alpha,
gemm1_clamp_limit=runner_config.gemm1_clamp_limit,
swiglu_limit=runner_config.swiglu_limit,
g1_alphas=layer.g1_alphas,
g1_alphas_up=layer.g1_alphas_up,
)
if gemm1_clamp_limit is not None:
copy_or_rebind_param(
layer,
"gemm1_clamp_limit",
(gemm1_clamp_limit / layer.g1_alphas).to(torch.float32),
)
if runner_config.gemm1_alpha is not None:
copy_or_rebind_param(
layer,
"gemm1_alpha",
torch.full_like(
layer.g1_alphas,
runner_config.gemm1_alpha,
dtype=torch.float32,
),
)
gemm1_beta = (
torch.full_like(
layer.g1_alphas,
runner_config.gemm1_clamp_limit,
dtype=torch.float32,
)
if is_situ
else (1.0 / layer.g1_alphas).to(torch.float32)
)
copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta)
for name, value in gated_act_params.items():
copy_or_rebind_param(layer, name, value)
if self.enable_flashinfer_cutlass_moe:
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (