[Fix] [NVIDIA] modelopt-nvfp4: Keep MoE global scales in gate/up order (#45289)
This commit is contained in:
@@ -1266,16 +1266,11 @@ class FusedMoE(torch.nn.Module):
|
||||
if shard_id not in ("w1", "w2", "w3"):
|
||||
raise ValueError(f"shard_id must be ['w1','w2','w3'] but got {shard_id}.")
|
||||
|
||||
use_flashinfer_trtllm_weight_layout = (
|
||||
method.use_flashinfer_trtllm_weight_layout
|
||||
if isinstance(method, ModelOptNvFp4FusedMoEMethod)
|
||||
else self.use_flashinfer_trtllm_moe
|
||||
)
|
||||
|
||||
# Flashinfer assumes w31 format for w13_weight. Same for the scales.
|
||||
if use_flashinfer_trtllm_weight_layout and (
|
||||
isinstance(method, ModelOptNvFp4FusedMoEMethod)
|
||||
or isinstance(method, Fp8MoEMethod)
|
||||
# ModelOpt NVFP4 reorders weights and block scales in _load_w13;
|
||||
# its per-tensor global scales retain the logical [gate, up] order.
|
||||
# These other methods also reorder their per-tensor scales.
|
||||
if self.use_flashinfer_trtllm_moe and (
|
||||
isinstance(method, Fp8MoEMethod)
|
||||
or isinstance(method, UnquantizedFusedMoEMethod)
|
||||
or isinstance(method, CompressedTensorsMxInt4MoE)
|
||||
):
|
||||
|
||||
@@ -2337,24 +2337,20 @@ def _compute_gemm1_alphas(
|
||||
w13_weight_scale_2: torch.Tensor,
|
||||
w13_input_scale: torch.Tensor,
|
||||
is_gated: bool,
|
||||
*,
|
||||
up_first: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GEMM1 weight x input alphas for the gate (w1) and up (w3) halves of w13.
|
||||
|
||||
w13 fuses the gate and up projections, which may carry separate NVFP4 weight
|
||||
scales stored as [num_experts, 2] in w13 row order: [gate, up] by default,
|
||||
[up, gate] when up_first (the FlashInfer TRT-LLM loader layout).
|
||||
A 1-D (or [num_experts, 1]) scale, and any non-gated layer, shares one scale
|
||||
across both halves; the col-1 read is guarded so those cases stay in bounds.
|
||||
scales stored as [num_experts, 2] (col 0 = gate, col 1 = up). A 1-D (or
|
||||
[num_experts, 1]) scale, and any non-gated layer, shares one scale across
|
||||
both halves; the col-1 read is guarded so those cases stay in bounds.
|
||||
|
||||
Returns (g1_alphas, g1_alphas_up), equal for a shared scale. Single-alpha
|
||||
backends use g1_alphas; the TRT-LLM path also uses g1_alphas_up.
|
||||
"""
|
||||
if is_gated and w13_weight_scale_2.dim() == 2 and w13_weight_scale_2.shape[1] >= 2:
|
||||
gate_col, up_col = (1, 0) if up_first else (0, 1)
|
||||
gate_scale = w13_weight_scale_2[:, gate_col]
|
||||
up_scale = w13_weight_scale_2[:, up_col]
|
||||
gate_scale = w13_weight_scale_2[:, 0]
|
||||
up_scale = w13_weight_scale_2[:, 1]
|
||||
else:
|
||||
gate_scale = w13_weight_scale_2.reshape(w13_weight_scale_2.shape[0])
|
||||
up_scale = gate_scale
|
||||
@@ -2363,41 +2359,6 @@ def _compute_gemm1_alphas(
|
||||
return g1_alphas, g1_alphas_up
|
||||
|
||||
|
||||
def _compute_trtllm_gated_act_params(
|
||||
*,
|
||||
activation: Optional[str],
|
||||
gemm1_alpha: Optional[float],
|
||||
gemm1_clamp_limit: Optional[float],
|
||||
swiglu_limit: Optional[float],
|
||||
g1_alphas: torch.Tensor,
|
||||
g1_alphas_up: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
params = {}
|
||||
is_situ = activation == "situ"
|
||||
clamp_limit = None if is_situ else (gemm1_clamp_limit or swiglu_limit)
|
||||
if clamp_limit is not None:
|
||||
# The kernel clamps both raw GEMM1 halves at this one value,
|
||||
# so it is exact for the up half only when the gate/up scales match.
|
||||
if not torch.equal(g1_alphas, g1_alphas_up):
|
||||
logger.warning_once(
|
||||
"NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU clamp "
|
||||
"limit is exact for the gate half only. Accuracy may be affected."
|
||||
)
|
||||
params["gemm1_clamp_limit"] = (clamp_limit / g1_alphas).to(torch.float32)
|
||||
|
||||
if gemm1_alpha is not None:
|
||||
params["gemm1_alpha"] = torch.full_like(
|
||||
g1_alphas, gemm1_alpha, dtype=torch.float32
|
||||
)
|
||||
# Beta is added to the raw up half, so it is in up-alpha units.
|
||||
params["gemm1_beta"] = (
|
||||
torch.full_like(g1_alphas, gemm1_clamp_limit, dtype=torch.float32)
|
||||
if is_situ
|
||||
else (1.0 / g1_alphas_up).to(torch.float32)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
"""
|
||||
MoE Method for FP4 Quantization with Blockscales and PerTensorScales
|
||||
@@ -2855,10 +2816,9 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
# Create shared parameters. g1_alphas / g1_alphas_up are the gate (w1)
|
||||
# and up (w3) GEMM1 scales (equal for shared-scale checkpoints).
|
||||
g1_alphas, g1_alphas_up = _compute_gemm1_alphas(
|
||||
w13_weight_scale_2=layer.w13_weight_scale_2,
|
||||
w13_input_scale=w13_input_scale,
|
||||
is_gated=layer.moe_runner_config.is_gated,
|
||||
up_first=self.use_flashinfer_trtllm_weight_layout,
|
||||
layer.w13_weight_scale_2,
|
||||
w13_input_scale,
|
||||
layer.moe_runner_config.is_gated,
|
||||
)
|
||||
copy_or_rebind_param(layer, "g1_alphas", g1_alphas)
|
||||
copy_or_rebind_param(layer, "g1_alphas_up", g1_alphas_up)
|
||||
@@ -2880,16 +2840,48 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
if layer.moe_runner_config.is_gated and self.enable_flashinfer_trtllm_moe:
|
||||
runner_config = layer.moe_runner_config
|
||||
gated_act_params = _compute_trtllm_gated_act_params(
|
||||
activation=runner_config.activation,
|
||||
gemm1_alpha=runner_config.gemm1_alpha,
|
||||
gemm1_clamp_limit=runner_config.gemm1_clamp_limit,
|
||||
swiglu_limit=runner_config.swiglu_limit,
|
||||
g1_alphas=layer.g1_alphas,
|
||||
g1_alphas_up=layer.g1_alphas_up,
|
||||
is_situ = runner_config.activation == "situ"
|
||||
gemm1_clamp_limit = (
|
||||
None
|
||||
if is_situ
|
||||
else (runner_config.gemm1_clamp_limit or runner_config.swiglu_limit)
|
||||
)
|
||||
for name, value in gated_act_params.items():
|
||||
copy_or_rebind_param(layer, name, value)
|
||||
if gemm1_clamp_limit is not None:
|
||||
# The kernel clamps both raw halves with one limit, which is
|
||||
# exact for both only when their dequantization scales match.
|
||||
if not torch.equal(layer.g1_alphas, layer.g1_alphas_up):
|
||||
logger.warning_once(
|
||||
"NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU "
|
||||
"clamp limit is exact for the gate half only. "
|
||||
"Accuracy may be affected."
|
||||
)
|
||||
copy_or_rebind_param(
|
||||
layer,
|
||||
"gemm1_clamp_limit",
|
||||
(gemm1_clamp_limit / layer.g1_alphas).to(torch.float32),
|
||||
)
|
||||
|
||||
if runner_config.gemm1_alpha is not None:
|
||||
copy_or_rebind_param(
|
||||
layer,
|
||||
"gemm1_alpha",
|
||||
torch.full_like(
|
||||
layer.g1_alphas,
|
||||
runner_config.gemm1_alpha,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
# SwiGLU adds beta to the raw up half before dequantization.
|
||||
gemm1_beta = (
|
||||
torch.full_like(
|
||||
layer.g1_alphas,
|
||||
runner_config.gemm1_clamp_limit,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if is_situ
|
||||
else (1.0 / layer.g1_alphas_up).to(torch.float32)
|
||||
)
|
||||
copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta)
|
||||
|
||||
if self.enable_flashinfer_cutlass_moe:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
@@ -3092,10 +3084,13 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
@property
|
||||
def load_up_proj_weight_first(self) -> bool:
|
||||
# Load W13 as [Up, Gate] for FlashInfer CUTLASS and CuteDSL v2 kernels.
|
||||
# The CuteDSL v1 (deepep) path uses [Gate, Up] -- do NOT flip.
|
||||
# Only weights and block scales follow the kernel's physical row order;
|
||||
# per-tensor global scales always remain [Gate, Up]. MegaMoE and
|
||||
# CuteDSL v1 (deepep) consume canonical [Gate, Up] weights as well.
|
||||
return self.moe_runner_config.is_gated and (
|
||||
self.enable_flashinfer_cutlass_moe or self._is_cutedsl_v2_standard
|
||||
self.use_flashinfer_trtllm_weight_layout
|
||||
or self.enable_flashinfer_cutlass_moe
|
||||
or self._is_cutedsl_v2_standard
|
||||
)
|
||||
|
||||
def create_moe_runner(
|
||||
|
||||
Reference in New Issue
Block a user