[Fix] [SM120] quantization: Requantize block-FP8 MoE weights to UE8M0 for DeepGEMM (#43556)
Signed-off-by: Zihua Wu <13583761+lucifer1004@users.noreply.github.com> Co-authored-by: number-eleven11 <1635219023@qq.com>
This commit is contained in:
co-authored by
number-eleven11
parent
ce329c85fa
commit
67d9ee156c
@@ -2205,6 +2205,7 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
use_deepgemm_runner=will_use_deepgemm,
|
||||
output_dtype=torch.bfloat16,
|
||||
weight_shape=weight.shape[-2:],
|
||||
grouped_moe=True,
|
||||
)
|
||||
|
||||
def _convert_mxfp8_moe_to_block_fp8(self, layer: Module) -> None:
|
||||
|
||||
@@ -2023,6 +2023,8 @@ def requant_block_scale_ue8m0_for_deepgemm(
|
||||
use_deepgemm_runner: bool,
|
||||
output_dtype: Optional[torch.dtype] = None,
|
||||
weight_shape=None,
|
||||
*,
|
||||
grouped_moe: bool = False,
|
||||
) -> bool:
|
||||
"""Requantize block-FP8 weight scales to UE8M0 in place for DeepGEMM.
|
||||
|
||||
@@ -2041,6 +2043,7 @@ def requant_block_scale_ue8m0_for_deepgemm(
|
||||
weight_block_size=weight_block_size,
|
||||
output_dtype=output_dtype,
|
||||
weight_shape=weight_shape,
|
||||
grouped_moe=grouped_moe,
|
||||
)
|
||||
):
|
||||
return False
|
||||
|
||||
@@ -264,7 +264,7 @@ def get_architecture_class_name(model_config: ModelConfig) -> str:
|
||||
|
||||
|
||||
def should_deepgemm_weight_requant_ue8m0(
|
||||
weight_block_size, output_dtype=None, weight_shape=None
|
||||
weight_block_size, output_dtype=None, weight_shape=None, *, grouped_moe=False
|
||||
):
|
||||
"""Should we requant fp8 weights into UE8M0 format when loading the model.
|
||||
|
||||
@@ -281,7 +281,7 @@ def should_deepgemm_weight_requant_ue8m0(
|
||||
return False
|
||||
# SM120 routes dense block-FP8 GEMMs to CUTLASS/Triton (fp32 scales);
|
||||
# only the grouped MoE GEMM consumes DeepGEMM layouts there.
|
||||
if get_device_sm() == 120:
|
||||
if get_device_sm() == 120 and not grouped_moe:
|
||||
return False
|
||||
if output_dtype is not None and output_dtype != torch.bfloat16:
|
||||
return False
|
||||
|
||||
Reference in New Issue
Block a user