Make the mxfp8 MoE runner backend list extensible (#30828)
This commit is contained in:
@@ -1928,13 +1928,10 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
|||||||
"flashinfer_trtllm_routed."
|
"flashinfer_trtllm_routed."
|
||||||
)
|
)
|
||||||
if view.quantization == "mxfp8":
|
if view.quantization == "mxfp8":
|
||||||
|
from sglang.srt.server_args import MXFP8_MOE_RUNNER_BACKEND_CHOICES
|
||||||
|
|
||||||
is_gfx95_mxfp8 = is_hip() and is_gfx95_supported()
|
is_gfx95_mxfp8 = is_hip() and is_gfx95_supported()
|
||||||
allowed = [
|
allowed = list(MXFP8_MOE_RUNNER_BACKEND_CHOICES)
|
||||||
"cutlass",
|
|
||||||
"deep_gemm",
|
|
||||||
"flashinfer_trtllm",
|
|
||||||
"flashinfer_trtllm_routed",
|
|
||||||
]
|
|
||||||
if is_gfx95_mxfp8:
|
if is_gfx95_mxfp8:
|
||||||
allowed.append("triton")
|
allowed.append("triton")
|
||||||
mxfp8_default = "triton" if is_gfx95_mxfp8 else "flashinfer_trtllm"
|
mxfp8_default = "triton" if is_gfx95_mxfp8 else "flashinfer_trtllm"
|
||||||
|
|||||||
@@ -268,6 +268,13 @@ MOE_A2A_BACKEND_CHOICES = [
|
|||||||
"megamoe",
|
"megamoe",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
MXFP8_MOE_RUNNER_BACKEND_CHOICES = [
|
||||||
|
"cutlass",
|
||||||
|
"deep_gemm",
|
||||||
|
"flashinfer_trtllm",
|
||||||
|
"flashinfer_trtllm_routed",
|
||||||
|
]
|
||||||
|
|
||||||
FP8_GEMM_RUNNER_BACKEND_CHOICES = [
|
FP8_GEMM_RUNNER_BACKEND_CHOICES = [
|
||||||
"auto",
|
"auto",
|
||||||
"deep_gemm",
|
"deep_gemm",
|
||||||
@@ -367,6 +374,10 @@ def add_moe_runner_backend_choices(choices):
|
|||||||
MOE_RUNNER_BACKEND_CHOICES.extend(choices)
|
MOE_RUNNER_BACKEND_CHOICES.extend(choices)
|
||||||
|
|
||||||
|
|
||||||
|
def add_mxfp8_moe_runner_backend_choices(choices):
|
||||||
|
MXFP8_MOE_RUNNER_BACKEND_CHOICES.extend(choices)
|
||||||
|
|
||||||
|
|
||||||
def add_fp8_gemm_runner_backend_choices(choices):
|
def add_fp8_gemm_runner_backend_choices(choices):
|
||||||
FP8_GEMM_RUNNER_BACKEND_CHOICES.extend(choices)
|
FP8_GEMM_RUNNER_BACKEND_CHOICES.extend(choices)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user