Make the mxfp8 MoE runner backend list extensible (#30828)
This commit is contained in:
@@ -1928,13 +1928,10 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
"flashinfer_trtllm_routed."
|
||||
)
|
||||
if view.quantization == "mxfp8":
|
||||
from sglang.srt.server_args import MXFP8_MOE_RUNNER_BACKEND_CHOICES
|
||||
|
||||
is_gfx95_mxfp8 = is_hip() and is_gfx95_supported()
|
||||
allowed = [
|
||||
"cutlass",
|
||||
"deep_gemm",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_trtllm_routed",
|
||||
]
|
||||
allowed = list(MXFP8_MOE_RUNNER_BACKEND_CHOICES)
|
||||
if is_gfx95_mxfp8:
|
||||
allowed.append("triton")
|
||||
mxfp8_default = "triton" if is_gfx95_mxfp8 else "flashinfer_trtllm"
|
||||
|
||||
@@ -268,6 +268,13 @@ MOE_A2A_BACKEND_CHOICES = [
|
||||
"megamoe",
|
||||
]
|
||||
|
||||
MXFP8_MOE_RUNNER_BACKEND_CHOICES = [
|
||||
"cutlass",
|
||||
"deep_gemm",
|
||||
"flashinfer_trtllm",
|
||||
"flashinfer_trtllm_routed",
|
||||
]
|
||||
|
||||
FP8_GEMM_RUNNER_BACKEND_CHOICES = [
|
||||
"auto",
|
||||
"deep_gemm",
|
||||
@@ -367,6 +374,10 @@ def add_moe_runner_backend_choices(choices):
|
||||
MOE_RUNNER_BACKEND_CHOICES.extend(choices)
|
||||
|
||||
|
||||
def add_mxfp8_moe_runner_backend_choices(choices):
|
||||
MXFP8_MOE_RUNNER_BACKEND_CHOICES.extend(choices)
|
||||
|
||||
|
||||
def add_fp8_gemm_runner_backend_choices(choices):
|
||||
FP8_GEMM_RUNNER_BACKEND_CHOICES.extend(choices)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user