[NVIDIA] Allow modelopt_mixed quantization with flashinfer_cutedsl MoE runner (#30443)
This commit is contained in:
@@ -99,7 +99,7 @@ class DeepEPMoE(FusedMoE):
|
|||||||
elif (
|
elif (
|
||||||
get_moe_runner_backend().is_flashinfer_cutedsl()
|
get_moe_runner_backend().is_flashinfer_cutedsl()
|
||||||
and quant_config is not None
|
and quant_config is not None
|
||||||
and quant_config.get_name() == "modelopt_fp4"
|
and quant_config.get_name() in ("modelopt_fp4", "modelopt_mixed")
|
||||||
):
|
):
|
||||||
self.deprecate_flag = True
|
self.deprecate_flag = True
|
||||||
elif (
|
elif (
|
||||||
|
|||||||
@@ -5169,10 +5169,11 @@ class ServerArgs:
|
|||||||
], "The expert parallel size must be 1 or the same as the tensor parallel size"
|
], "The expert parallel size must be 1 or the same as the tensor parallel size"
|
||||||
|
|
||||||
if view.moe_runner_backend == "flashinfer_cutedsl":
|
if view.moe_runner_backend == "flashinfer_cutedsl":
|
||||||
|
# modelopt_mixed with non-NVFP4 MoE layers is rejected at load time.
|
||||||
assert (
|
assert (
|
||||||
view.quantization in ["modelopt_fp4"]
|
view.quantization in ["modelopt_fp4", "modelopt_mixed"]
|
||||||
or self.get_model_config().nvfp4_moe_meta is not None
|
or self.get_model_config().nvfp4_moe_meta is not None
|
||||||
), f"Invalid quantization '{view.quantization}'. \nFlashInfer CuteDSL MOE currently supports only: 'modelopt_fp4' or hybrid NVFP4 models."
|
), f"Invalid quantization '{view.quantization}'. \nFlashInfer CuteDSL MOE currently supports only: 'modelopt_fp4', 'modelopt_mixed' (with NVFP4 MoE layers), or hybrid NVFP4 models."
|
||||||
assert view.ep_size in [
|
assert view.ep_size in [
|
||||||
1,
|
1,
|
||||||
self.tp_size,
|
self.tp_size,
|
||||||
|
|||||||
Reference in New Issue
Block a user