[Fix] [AMD] dsv41: Set moe act to fp8 by default (#45480)

This commit is contained in:
Thomas Wang
2026-10-20 23:09:27 -07:00
committed by GitHub
parent 574f9cdb2e
commit bd5a445bf7
3 changed files with 26 additions and 1 deletions
@@ -68,6 +68,8 @@ class AiterMoeQuantInfo(MoeQuantInfo):
hidden_pad: int = 0
intermediate_pad: int = 0
swiglu_limit: float = 0.0
# Quantize the activations of MXFP4 experts to MXFP8 at any token count
fp8_activations: bool = False
fused_moe_kwargs: Optional[dict[str, Any]] = None
@@ -256,7 +258,11 @@ class AiterRunnerCore(MoeRunnerCore):
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
if _SMALLM_MOE_ON and quant_info.w13_weight.element_size() == 1:
if (
_SMALLM_MOE_ON
and quant_info.w13_weight.element_size() == 1
and not quant_info.fp8_activations
):
from sglang.kernels.ops.moe import smallm_moe_gfx950 as _smallm
try:
@@ -291,6 +297,18 @@ class AiterRunnerCore(MoeRunnerCore):
except _smallm.SmallMMoeUnavailable:
pass
if (
quant_info.fp8_activations
and is_gfx95
and runner_input.hidden_states.dtype == torch.bfloat16
):
from aiter import dtypes
from aiter.ops.flydsl.moe_common import GateMode
# aiter's A8W4 kernels take interleaved gate/up rows only
if extra.get("gate_mode") == GateMode.INTERLEAVE.value:
extra.setdefault("quant_dtype_a", dtypes.fp8)
activation = extra.pop("activation", None) if is_gfx95 else None
if activation is None:
activation = _aiter_activation(self.config)
@@ -3413,6 +3413,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
swiglu_limit=self.moe_runner_config.swiglu_limit or 0.0,
hidden_pad=getattr(layer, "hidden_pad", 0),
intermediate_pad=getattr(layer, "intermediate_pad", 0),
fp8_activations=(
self.is_fp4_expert and getattr(layer, "fp8_activations", False)
),
fused_moe_kwargs=fused_moe_kwargs,
)
+4
View File
@@ -3008,6 +3008,10 @@ class DeepseekV4DecoderLayerBase(nn.Module):
and not getattr(config, "language_model_only", False),
)
if _use_aiter:
# V4.1 runs its FP4 routed experts on MXFP8 activations
self.mlp.experts.fp8_activations = config.model_type == "deepseek_v41"
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.post_attention_layernorm = RMSNorm(
config.hidden_size, eps=config.rms_norm_eps