MiniMax-M3: MXFP8 dense-only block convert + aiter MXFP8 MoE on gfx950 (#36574)
Co-authored-by: Claude Opus 5 <noreply@anthropic.com> Co-authored-by: mikevin920 <mikevin920@yahoo.com> Co-authored-by: Kevin Mi <mikevin920@gmail.com> Co-authored-by: Kevin Mi <45493463+kevin-mii@users.noreply.github.com> Co-authored-by: YC Yen-Ching Tseng <yctseng@amd.com>
This commit is contained in:
co-authored by
Claude Opus 5
mikevin920
Kevin Mi
Kevin Mi
YC Yen-Ching Tseng
parent
b129504f0e
commit
0b53305f48
@@ -1414,6 +1414,9 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
allowed = list(MXFP8_MOE_RUNNER_BACKEND_CHOICES)
|
||||
if is_gfx95_mxfp8:
|
||||
allowed.append("triton")
|
||||
# the aiter MXFP8 MoE quant info is built only when aiter is enabled
|
||||
if envs.SGLANG_USE_AITER.get():
|
||||
allowed.append("aiter")
|
||||
|
||||
if view.moe_a2a_backend == "flashinfer_megamoe":
|
||||
mxfp8_default = "flashinfer_megamoe"
|
||||
|
||||
@@ -1001,6 +1001,9 @@ class Envs:
|
||||
# on load. Unrelated to the NVFP4 block-FP8 NextN path above.
|
||||
SGLANG_GLM_NEXTN_MOE_PTPC = EnvBool(False)
|
||||
SGLANG_QUANT_ALLOW_DOWNCASTING = EnvBool(False)
|
||||
# HIP: convert only MXFP8 dense linears to block-fp8; fused MoE stays MX 1x32,
|
||||
# since the block-scale MoE kernel lacks SwiGLU-OAI
|
||||
SGLANG_FORCE_MXFP8_BLOCK_CONVERT_DENSE = EnvBool(False)
|
||||
SGLANG_FP8_IGNORED_LAYERS = EnvStr("")
|
||||
SGLANG_FP4_IGNORED_LAYERS = EnvStr("")
|
||||
# On by default; set SGLANG_ENABLE_FP8_GEMM_CONFIG_TUNE=0 as a kill switch.
|
||||
|
||||
@@ -510,7 +510,10 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
self.block_quant = (
|
||||
self.use_mxfp8 or self.quant_config.weight_block_size is not None
|
||||
)
|
||||
self.convert_mxfp8_to_block = self.use_mxfp8 and _mxfp8_to_block_fp8_required
|
||||
self.convert_mxfp8_to_block = self.use_mxfp8 and (
|
||||
_mxfp8_to_block_fp8_required
|
||||
or (_is_hip and envs.SGLANG_FORCE_MXFP8_BLOCK_CONVERT_DENSE.get())
|
||||
)
|
||||
self.weight_block_size = self.quant_config.weight_block_size
|
||||
self.w8a8_block_fp8_linear = None
|
||||
self.w8a8_mxfp8_linear = None
|
||||
@@ -722,12 +725,34 @@ class Fp8LinearMethod(LinearMethodBase):
|
||||
if self.convert_mxfp8_to_block:
|
||||
from sglang.srt.layers.quantization.mxfp8_block_convert import (
|
||||
convert_mxfp8_weight_to_block_fp8,
|
||||
dequant_mxfp8_2d_to_bf16,
|
||||
)
|
||||
|
||||
mx_weight, mx_scale = layer.weight.data, layer.weight_scale_inv.data
|
||||
qweight, scale = convert_mxfp8_weight_to_block_fp8(
|
||||
layer.weight.data, layer.weight_scale_inv.data, block=128
|
||||
mx_weight, mx_scale, block=128
|
||||
)
|
||||
layer.weight = Parameter(qweight, requires_grad=False)
|
||||
if (
|
||||
_use_aiter
|
||||
and _is_gfx95_supported
|
||||
and self.w8a8_block_fp8_linear is aiter_w8a8_block_fp8_linear
|
||||
):
|
||||
# rowwise-fp8 copy for the small-M path of aiter_w8a8_block_fp8_linear;
|
||||
# the later bpreshuffle is an in-place copy_, so these attrs survive
|
||||
weight_fp32 = dequant_mxfp8_2d_to_bf16(mx_weight, mx_scale).float()
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
row_scale = (
|
||||
weight_fp32.abs().amax(dim=1, keepdim=True).clamp(min=1e-12)
|
||||
/ fp8_max
|
||||
)
|
||||
layer.weight._ptpc_weight = shuffle_weight(
|
||||
(weight_fp32 / row_scale)
|
||||
.clamp(-fp8_max, fp8_max)
|
||||
.to(torch.float8_e4m3fn),
|
||||
(16, 16),
|
||||
)
|
||||
layer.weight._ptpc_scale = row_scale
|
||||
layer.weight_scale_inv = Parameter(scale, requires_grad=False)
|
||||
self.use_mxfp8 = False
|
||||
self.convert_mxfp8_to_block = False
|
||||
@@ -2347,6 +2372,30 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
|
||||
align_mxfp8_moe_weights_for_flashinfer_trtllm(layer)
|
||||
|
||||
if _is_hip and _is_gfx95_supported and get_moe_runner_backend().is_aiter():
|
||||
from aiter.ops.shuffle import shuffle_scale_a16w4, shuffle_weight_a16w4
|
||||
from aiter.utility import fp4_utils
|
||||
|
||||
num_experts = layer.w13_weight.shape[0]
|
||||
layer.w13_weight.data = shuffle_weight_a16w4(
|
||||
layer.w13_weight.data.contiguous(), 16, True
|
||||
)
|
||||
w13_s3d = layer.w13_weight_scale_inv.data
|
||||
layer.w13_weight_scale_inv.data = shuffle_scale_a16w4(
|
||||
w13_s3d.reshape(-1, w13_s3d.shape[-1]).contiguous(),
|
||||
num_experts,
|
||||
True,
|
||||
)
|
||||
layer.w2_weight.data = shuffle_weight_a16w4(
|
||||
layer.w2_weight.data.contiguous(), 16, False
|
||||
)
|
||||
w2_s3d = layer.w2_weight_scale_inv.data
|
||||
layer.w2_weight_scale_inv.data = fp4_utils.e8m0_shuffle(
|
||||
w2_s3d.reshape(-1, w2_s3d.shape[-1]).contiguous()
|
||||
)
|
||||
layer.w13_weight.is_shuffled = True
|
||||
layer.w2_weight.is_shuffled = True
|
||||
|
||||
def process_weights_after_loading(self, layer: Module) -> None:
|
||||
if _is_hip and _use_hip_int4:
|
||||
self.process_weights_hip_int4(layer)
|
||||
@@ -3142,6 +3191,34 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
w13_weight = layer.w13_weight
|
||||
w2_weight = layer.w2_weight
|
||||
|
||||
if self.use_mxfp8:
|
||||
gemm1_alpha = self.moe_runner_config.gemm1_alpha
|
||||
if gemm1_alpha != 1.702:
|
||||
raise NotImplementedError(
|
||||
f"AITER MXFP8 MoE only supports swiglu-oai "
|
||||
f"alpha=1.702, got {gemm1_alpha=}."
|
||||
)
|
||||
from aiter import ActivationType
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
return AiterMoeQuantInfo(
|
||||
w13_weight=w13_weight,
|
||||
w2_weight=w2_weight,
|
||||
quant_type=AiterQuantType.PER_1X32,
|
||||
w13_scale=layer.w13_weight_scale_inv,
|
||||
w2_scale=layer.w2_weight_scale_inv,
|
||||
expert_mask=layer.dispatcher.expert_mask_gpu if _use_aiter else None,
|
||||
swiglu_limit=self.moe_runner_config.swiglu_limit
|
||||
or self.moe_runner_config.gemm1_clamp_limit
|
||||
or 0.0,
|
||||
hidden_pad=getattr(layer, "hidden_pad", 0),
|
||||
intermediate_pad=getattr(layer, "intermediate_pad", 0),
|
||||
fused_moe_kwargs={
|
||||
"activation": ActivationType.Swiglu,
|
||||
"gate_mode": GateMode.INTERLEAVE.value,
|
||||
},
|
||||
)
|
||||
|
||||
if self.block_quant:
|
||||
quant_type = (
|
||||
AiterQuantType.PER_1X32
|
||||
|
||||
@@ -69,6 +69,9 @@ _use_aiter = (
|
||||
get_bool_env_var("SGLANG_USE_AITER") and _is_hip and not _is_gfx1250_supported
|
||||
)
|
||||
_use_aiter_gfx95 = _use_aiter and _is_gfx95_supported
|
||||
# Conservative, not a tuned crossover: ptpc beat block-fp8 up to M=512 on
|
||||
# MiniMax-M3 TP4 gfx950 shapes, but lost past M=64 on narrow ones (K=768).
|
||||
MXFP8_DENSE_PTPC_DECODE_MAX_M = 128
|
||||
# ROCm 7.0 hipcc miscompiles gemm_a8w8_blockscale_bpreshuffle on gfx95 (#23319).
|
||||
_use_aiter_bpreshuffle_gfx95 = _use_aiter_gfx95 and get_hip_version() >= (7, 2, 0)
|
||||
# gfx95 + ROCm < 7.2: bpreshuffle CK is disabled (above), and the non-bpreshuffle
|
||||
@@ -1334,6 +1337,22 @@ def aiter_w8a8_block_fp8_linear(
|
||||
input_2d = input.view(-1, input.shape[-1])
|
||||
output_shape = [*input.shape[:-1], weight.shape[0]]
|
||||
|
||||
# dense linears converted from MXFP8 carry a rowwise-fp8 copy; its ptpc GEMM
|
||||
# beats the block-fp8 GEMMs at decode-sized M
|
||||
if input_scale is None:
|
||||
ptpc_weight = getattr(weight, "_ptpc_weight", None)
|
||||
if (
|
||||
ptpc_weight is not None
|
||||
and input_2d.shape[0] <= MXFP8_DENSE_PTPC_DECODE_MAX_M
|
||||
):
|
||||
out = apply_fp8_ptpc_linear(
|
||||
input=input_2d,
|
||||
weight=ptpc_weight,
|
||||
weight_scale=weight._ptpc_scale,
|
||||
bias=bias,
|
||||
)
|
||||
return out.to(input.dtype).view(*output_shape)
|
||||
|
||||
n, k = weight.shape
|
||||
|
||||
if _use_aiter_bpreshuffle_gfx95:
|
||||
|
||||
Reference in New Issue
Block a user