fix: route Thor SM110 FP8 and ModelOpt NVFP4 auto backends (#41765)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
This commit is contained in:
co-authored by
Po-Han Huang
parent
6b737fd4c6
commit
0a0fe001f1
@@ -1446,11 +1446,11 @@ def _moe_runner_backend_quant_constraints(view: Any) -> dict:
|
||||
if (
|
||||
moe_runner_backend == "auto"
|
||||
and view.quantization == "modelopt_fp4"
|
||||
and get_platform().is_sm120
|
||||
and (get_platform().is_sm120 or get_platform().is_sm110)
|
||||
):
|
||||
moe_runner_backend = "flashinfer_cutlass"
|
||||
logger.info(
|
||||
"Use flashinfer_cutlass as MoE runner backend on SM120 for "
|
||||
"Use flashinfer_cutlass as MoE runner backend on SM110/SM120 for "
|
||||
"modelopt_fp4 (trtllm-gen MoE kernels are SM100-only)"
|
||||
)
|
||||
if moe_runner_backend != view.moe_runner_backend:
|
||||
|
||||
@@ -1009,6 +1009,9 @@ def initialize_fp8_gemm_config() -> None:
|
||||
backend = get_exec().kernel.fp8_gemm_runner_backend
|
||||
if backend == "auto" and get_platform().is_sm120:
|
||||
backend = "cutlass"
|
||||
elif backend == "auto" and get_platform().is_sm110:
|
||||
# DeepGEMM's block-FP8 kernel does not support SM110.
|
||||
backend = "triton"
|
||||
|
||||
backend = Fp8GemmRunnerBackend(backend)
|
||||
|
||||
|
||||
@@ -1776,6 +1776,7 @@ _PLATFORM_PROBES: Dict[str, str] = {
|
||||
"is_sm90": "is_sm90_supported",
|
||||
"is_sm100": "is_sm100_supported",
|
||||
"is_sm100_or_sm110": "is_sm100_or_sm110_supported",
|
||||
"is_sm110": "is_sm110_supported",
|
||||
"is_sm120": "is_sm120_supported",
|
||||
"is_blackwell": "is_blackwell_supported",
|
||||
"is_hopper_with_cuda_12_3": "is_hopper_with_cuda_12_3",
|
||||
|
||||
@@ -311,6 +311,11 @@ is_sm100_or_sm110_supported = lru_cache(maxsize=1)(
|
||||
cuda_version=(12, 8),
|
||||
)
|
||||
)
|
||||
is_sm110_supported = lru_cache(maxsize=1)(
|
||||
partial(
|
||||
_check_cuda_device_version, device_capability_majors=[11], cuda_version=(12, 8)
|
||||
)
|
||||
)
|
||||
is_sm80_supported = lru_cache(maxsize=1)(
|
||||
partial(
|
||||
_check_cuda_device_version, device_capability_majors=[8], cuda_version=(11, 0)
|
||||
|
||||
Reference in New Issue
Block a user