[Fix] [SM120] quantization: Requantize block-FP8 MoE weights to UE8M0 for DeepGEMM (#43556)

Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com>
Co-authored-by: number-eleven11 <1635219023@qq.com>
This commit is contained in:
Gabriel Wu
2026-10-19 19:06:21 -07:00
committed by GitHub
co-authored by number-eleven11
parent ce329c85fa
commit 67d9ee156c
4 changed files with 40 additions and 2 deletions
@@ -2205,6 +2205,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
use_deepgemm_runner=will_use_deepgemm,
output_dtype=torch.bfloat16,
weight_shape=weight.shape[-2:],
grouped_moe=True,
)
def _convert_mxfp8_moe_to_block_fp8(self, layer: Module) -> None:
@@ -2023,6 +2023,8 @@ def requant_block_scale_ue8m0_for_deepgemm(
use_deepgemm_runner: bool,
output_dtype: Optional[torch.dtype] = None,
weight_shape=None,
*,
grouped_moe: bool = False,
) -> bool:
"""Requantize block-FP8 weight scales to UE8M0 in place for DeepGEMM.
@@ -2041,6 +2043,7 @@ def requant_block_scale_ue8m0_for_deepgemm(
weight_block_size=weight_block_size,
output_dtype=output_dtype,
weight_shape=weight_shape,
grouped_moe=grouped_moe,
)
):
return False
+2 -2
View File
@@ -264,7 +264,7 @@ def get_architecture_class_name(model_config: ModelConfig) -> str:
def should_deepgemm_weight_requant_ue8m0(
weight_block_size, output_dtype=None, weight_shape=None
weight_block_size, output_dtype=None, weight_shape=None, *, grouped_moe=False
):
"""Should we requant fp8 weights into UE8M0 format when loading the model.
@@ -281,7 +281,7 @@ def should_deepgemm_weight_requant_ue8m0(
return False
# SM120 routes dense block-FP8 GEMMs to CUTLASS/Triton (fp32 scales);
# only the grouped MoE GEMM consumes DeepGEMM layouts there.
if get_device_sm() == 120:
if get_device_sm() == 120 and not grouped_moe:
return False
if output_dtype is not None and output_dtype != torch.bfloat16:
return False