[Fix] [NVIDIA] modelopt-nvfp4: Read w13 global scales in the TRT-LLM [up, gate] order (#44858)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
1739d962a4
commit
78185f4b3f
@@ -2316,20 +2316,24 @@ def _compute_gemm1_alphas(
|
||||
w13_weight_scale_2: torch.Tensor,
|
||||
w13_input_scale: torch.Tensor,
|
||||
is_gated: bool,
|
||||
*,
|
||||
up_first: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""GEMM1 weight x input alphas for the gate (w1) and up (w3) halves of w13.
|
||||
|
||||
w13 fuses the gate and up projections, which may carry separate NVFP4 weight
|
||||
scales stored as [num_experts, 2] (col 0 = gate, col 1 = up). A 1-D (or
|
||||
[num_experts, 1]) scale, and any non-gated layer, shares one scale across
|
||||
both halves; the col-1 read is guarded so those cases stay in bounds.
|
||||
scales stored as [num_experts, 2] in w13 row order: [gate, up] by default,
|
||||
[up, gate] when up_first (the FlashInfer TRT-LLM loader layout).
|
||||
A 1-D (or [num_experts, 1]) scale, and any non-gated layer, shares one scale
|
||||
across both halves; the col-1 read is guarded so those cases stay in bounds.
|
||||
|
||||
Returns (g1_alphas, g1_alphas_up), equal for a shared scale. Single-alpha
|
||||
backends use g1_alphas; the TRT-LLM path also uses g1_alphas_up.
|
||||
"""
|
||||
if is_gated and w13_weight_scale_2.dim() == 2 and w13_weight_scale_2.shape[1] >= 2:
|
||||
gate_scale = w13_weight_scale_2[:, 0]
|
||||
up_scale = w13_weight_scale_2[:, 1]
|
||||
gate_col, up_col = (1, 0) if up_first else (0, 1)
|
||||
gate_scale = w13_weight_scale_2[:, gate_col]
|
||||
up_scale = w13_weight_scale_2[:, up_col]
|
||||
else:
|
||||
gate_scale = w13_weight_scale_2.reshape(w13_weight_scale_2.shape[0])
|
||||
up_scale = gate_scale
|
||||
@@ -2338,6 +2342,41 @@ def _compute_gemm1_alphas(
|
||||
return g1_alphas, g1_alphas_up
|
||||
|
||||
|
||||
def _compute_trtllm_gated_act_params(
|
||||
*,
|
||||
activation: Optional[str],
|
||||
gemm1_alpha: Optional[float],
|
||||
gemm1_clamp_limit: Optional[float],
|
||||
swiglu_limit: Optional[float],
|
||||
g1_alphas: torch.Tensor,
|
||||
g1_alphas_up: torch.Tensor,
|
||||
) -> dict[str, torch.Tensor]:
|
||||
params = {}
|
||||
is_situ = activation == "situ"
|
||||
clamp_limit = None if is_situ else (gemm1_clamp_limit or swiglu_limit)
|
||||
if clamp_limit is not None:
|
||||
# The kernel clamps both raw GEMM1 halves at this one value,
|
||||
# so it is exact for the up half only when the gate/up scales match.
|
||||
if not torch.equal(g1_alphas, g1_alphas_up):
|
||||
logger.warning_once(
|
||||
"NVFP4 MoE gate/up global scales differ; the TRT-LLM SwiGLU clamp "
|
||||
"limit is exact for the gate half only. Accuracy may be affected."
|
||||
)
|
||||
params["gemm1_clamp_limit"] = (clamp_limit / g1_alphas).to(torch.float32)
|
||||
|
||||
if gemm1_alpha is not None:
|
||||
params["gemm1_alpha"] = torch.full_like(
|
||||
g1_alphas, gemm1_alpha, dtype=torch.float32
|
||||
)
|
||||
# Beta is added to the raw up half, so it is in up-alpha units.
|
||||
params["gemm1_beta"] = (
|
||||
torch.full_like(g1_alphas, gemm1_clamp_limit, dtype=torch.float32)
|
||||
if is_situ
|
||||
else (1.0 / g1_alphas_up).to(torch.float32)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
"""
|
||||
MoE Method for FP4 Quantization with Blockscales and PerTensorScales
|
||||
@@ -2795,9 +2834,10 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
# Create shared parameters. g1_alphas / g1_alphas_up are the gate (w1)
|
||||
# and up (w3) GEMM1 scales (equal for shared-scale checkpoints).
|
||||
g1_alphas, g1_alphas_up = _compute_gemm1_alphas(
|
||||
layer.w13_weight_scale_2,
|
||||
w13_input_scale,
|
||||
layer.moe_runner_config.is_gated,
|
||||
w13_weight_scale_2=layer.w13_weight_scale_2,
|
||||
w13_input_scale=w13_input_scale,
|
||||
is_gated=layer.moe_runner_config.is_gated,
|
||||
up_first=self.use_flashinfer_trtllm_weight_layout,
|
||||
)
|
||||
copy_or_rebind_param(layer, "g1_alphas", g1_alphas)
|
||||
copy_or_rebind_param(layer, "g1_alphas_up", g1_alphas_up)
|
||||
@@ -2819,39 +2859,16 @@ class ModelOptNvFp4FusedMoEMethod(FusedMoEMethodBase):
|
||||
|
||||
if layer.moe_runner_config.is_gated and self.enable_flashinfer_trtllm_moe:
|
||||
runner_config = layer.moe_runner_config
|
||||
is_situ = runner_config.activation == "situ"
|
||||
gemm1_clamp_limit = (
|
||||
None
|
||||
if is_situ
|
||||
else (runner_config.gemm1_clamp_limit or runner_config.swiglu_limit)
|
||||
gated_act_params = _compute_trtllm_gated_act_params(
|
||||
activation=runner_config.activation,
|
||||
gemm1_alpha=runner_config.gemm1_alpha,
|
||||
gemm1_clamp_limit=runner_config.gemm1_clamp_limit,
|
||||
swiglu_limit=runner_config.swiglu_limit,
|
||||
g1_alphas=layer.g1_alphas,
|
||||
g1_alphas_up=layer.g1_alphas_up,
|
||||
)
|
||||
if gemm1_clamp_limit is not None:
|
||||
copy_or_rebind_param(
|
||||
layer,
|
||||
"gemm1_clamp_limit",
|
||||
(gemm1_clamp_limit / layer.g1_alphas).to(torch.float32),
|
||||
)
|
||||
|
||||
if runner_config.gemm1_alpha is not None:
|
||||
copy_or_rebind_param(
|
||||
layer,
|
||||
"gemm1_alpha",
|
||||
torch.full_like(
|
||||
layer.g1_alphas,
|
||||
runner_config.gemm1_alpha,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
)
|
||||
gemm1_beta = (
|
||||
torch.full_like(
|
||||
layer.g1_alphas,
|
||||
runner_config.gemm1_clamp_limit,
|
||||
dtype=torch.float32,
|
||||
)
|
||||
if is_situ
|
||||
else (1.0 / layer.g1_alphas).to(torch.float32)
|
||||
)
|
||||
copy_or_rebind_param(layer, "gemm1_beta", gemm1_beta)
|
||||
for name, value in gated_act_params.items():
|
||||
copy_or_rebind_param(layer, name, value)
|
||||
|
||||
if self.enable_flashinfer_cutlass_moe:
|
||||
from sglang.srt.layers.moe.moe_runner.flashinfer_cutlass import (
|
||||
|
||||
Reference in New Issue
Block a user