Accept MXFP8 dispatch in FlashInfer A2A TRT-LLM MoE (#39613)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-09-15 19:23:15 -07:00
committed by GitHub
co-authored by Mohammad Angkad
parent d4ad368ed9
commit 935cbf24ec
@@ -1707,6 +1707,17 @@ def fused_experts_flashinfer_to_flashinfer_trtllm(
use_routed_topk=True, use_routed_topk=True,
) )
elif isinstance(quant_info, FlashInferTrtllmFp8MoeQuantInfo): elif isinstance(quant_info, FlashInferTrtllmFp8MoeQuantInfo):
mxfp8_dispatch = (
quant_info.use_mxfp8
and dispatch_output.hidden_states.dtype == torch.float8_e4m3fn
)
if mxfp8_dispatch:
if dispatch_output.hidden_states_scale is None:
raise ValueError(
"FlashInfer A2A + TRT-LLM Gen MXFP8 MoE requires activation "
"scales alongside the FP8 dispatch payload."
)
else:
if dispatch_output.hidden_states.dtype != torch.bfloat16: if dispatch_output.hidden_states.dtype != torch.bfloat16:
raise TypeError( raise TypeError(
"FlashInfer A2A + TRT-LLM Gen FP8 MoE requires a BF16 " "FlashInfer A2A + TRT-LLM Gen FP8 MoE requires a BF16 "