[Fix] [AMD] dsv41: Set moe act to fp8 by default (#45480)
This commit is contained in:
@@ -68,6 +68,8 @@ class AiterMoeQuantInfo(MoeQuantInfo):
|
||||
hidden_pad: int = 0
|
||||
intermediate_pad: int = 0
|
||||
swiglu_limit: float = 0.0
|
||||
# Quantize the activations of MXFP4 experts to MXFP8 at any token count
|
||||
fp8_activations: bool = False
|
||||
fused_moe_kwargs: Optional[dict[str, Any]] = None
|
||||
|
||||
|
||||
@@ -256,7 +258,11 @@ class AiterRunnerCore(MoeRunnerCore):
|
||||
extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY
|
||||
|
||||
# gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations.
|
||||
if _SMALLM_MOE_ON and quant_info.w13_weight.element_size() == 1:
|
||||
if (
|
||||
_SMALLM_MOE_ON
|
||||
and quant_info.w13_weight.element_size() == 1
|
||||
and not quant_info.fp8_activations
|
||||
):
|
||||
from sglang.kernels.ops.moe import smallm_moe_gfx950 as _smallm
|
||||
|
||||
try:
|
||||
@@ -291,6 +297,18 @@ class AiterRunnerCore(MoeRunnerCore):
|
||||
except _smallm.SmallMMoeUnavailable:
|
||||
pass
|
||||
|
||||
if (
|
||||
quant_info.fp8_activations
|
||||
and is_gfx95
|
||||
and runner_input.hidden_states.dtype == torch.bfloat16
|
||||
):
|
||||
from aiter import dtypes
|
||||
from aiter.ops.flydsl.moe_common import GateMode
|
||||
|
||||
# aiter's A8W4 kernels take interleaved gate/up rows only
|
||||
if extra.get("gate_mode") == GateMode.INTERLEAVE.value:
|
||||
extra.setdefault("quant_dtype_a", dtypes.fp8)
|
||||
|
||||
activation = extra.pop("activation", None) if is_gfx95 else None
|
||||
if activation is None:
|
||||
activation = _aiter_activation(self.config)
|
||||
|
||||
@@ -3413,6 +3413,9 @@ class Fp8MoEMethod(FusedMoEMethodBase):
|
||||
swiglu_limit=self.moe_runner_config.swiglu_limit or 0.0,
|
||||
hidden_pad=getattr(layer, "hidden_pad", 0),
|
||||
intermediate_pad=getattr(layer, "intermediate_pad", 0),
|
||||
fp8_activations=(
|
||||
self.is_fp4_expert and getattr(layer, "fp8_activations", False)
|
||||
),
|
||||
fused_moe_kwargs=fused_moe_kwargs,
|
||||
)
|
||||
|
||||
|
||||
@@ -3008,6 +3008,10 @@ class DeepseekV4DecoderLayerBase(nn.Module):
|
||||
and not getattr(config, "language_model_only", False),
|
||||
)
|
||||
|
||||
if _use_aiter:
|
||||
# V4.1 runs its FP4 routed experts on MXFP8 activations
|
||||
self.mlp.experts.fp8_activations = config.model_type == "deepseek_v41"
|
||||
|
||||
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
self.post_attention_layernorm = RMSNorm(
|
||||
config.hidden_size, eps=config.rms_norm_eps
|
||||
|
||||
Reference in New Issue
Block a user