diff --git a/python/sglang/srt/layers/moe/moe_runner/aiter.py b/python/sglang/srt/layers/moe/moe_runner/aiter.py index 9dabbfa41c..0a12f923ec 100644 --- a/python/sglang/srt/layers/moe/moe_runner/aiter.py +++ b/python/sglang/srt/layers/moe/moe_runner/aiter.py @@ -68,6 +68,8 @@ class AiterMoeQuantInfo(MoeQuantInfo): hidden_pad: int = 0 intermediate_pad: int = 0 swiglu_limit: float = 0.0 + # Quantize the activations of MXFP4 experts to MXFP8 at any token count + fp8_activations: bool = False fused_moe_kwargs: Optional[dict[str, Any]] = None @@ -256,7 +258,11 @@ class AiterRunnerCore(MoeRunnerCore): extra["moe_sorting_dispatch_policy"] = _MOE_SORTING_DISPATCH_POLICY # gfx950 small-M MXFP4 kernel (on by default, SGLANG_ROCM_SMALLM_MOE=0 disables): same layouts as aiter, bf16 activations. - if _SMALLM_MOE_ON and quant_info.w13_weight.element_size() == 1: + if ( + _SMALLM_MOE_ON + and quant_info.w13_weight.element_size() == 1 + and not quant_info.fp8_activations + ): from sglang.kernels.ops.moe import smallm_moe_gfx950 as _smallm try: @@ -291,6 +297,18 @@ class AiterRunnerCore(MoeRunnerCore): except _smallm.SmallMMoeUnavailable: pass + if ( + quant_info.fp8_activations + and is_gfx95 + and runner_input.hidden_states.dtype == torch.bfloat16 + ): + from aiter import dtypes + from aiter.ops.flydsl.moe_common import GateMode + + # aiter's A8W4 kernels take interleaved gate/up rows only + if extra.get("gate_mode") == GateMode.INTERLEAVE.value: + extra.setdefault("quant_dtype_a", dtypes.fp8) + activation = extra.pop("activation", None) if is_gfx95 else None if activation is None: activation = _aiter_activation(self.config) diff --git a/python/sglang/srt/layers/quantization/fp8.py b/python/sglang/srt/layers/quantization/fp8.py index c4560909cd..916363a0e1 100644 --- a/python/sglang/srt/layers/quantization/fp8.py +++ b/python/sglang/srt/layers/quantization/fp8.py @@ -3413,6 +3413,9 @@ class Fp8MoEMethod(FusedMoEMethodBase): swiglu_limit=self.moe_runner_config.swiglu_limit or 0.0, hidden_pad=getattr(layer, "hidden_pad", 0), intermediate_pad=getattr(layer, "intermediate_pad", 0), + fp8_activations=( + self.is_fp4_expert and getattr(layer, "fp8_activations", False) + ), fused_moe_kwargs=fused_moe_kwargs, ) diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 1899d0dfbe..c558c512e7 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -3008,6 +3008,10 @@ class DeepseekV4DecoderLayerBase(nn.Module): and not getattr(config, "language_model_only", False), ) + if _use_aiter: + # V4.1 runs its FP4 routed experts on MXFP8 activations + self.mlp.experts.fp8_activations = config.model_type == "deepseek_v41" + self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.post_attention_layernorm = RMSNorm( config.hidden_size, eps=config.rms_norm_eps