[NVIDIA] Allow modelopt_mixed quantization with flashinfer_cutedsl MoE runner (#30443)
This commit is contained in:
@@ -99,7 +99,7 @@ class DeepEPMoE(FusedMoE):
|
||||
elif (
|
||||
get_moe_runner_backend().is_flashinfer_cutedsl()
|
||||
and quant_config is not None
|
||||
and quant_config.get_name() == "modelopt_fp4"
|
||||
and quant_config.get_name() in ("modelopt_fp4", "modelopt_mixed")
|
||||
):
|
||||
self.deprecate_flag = True
|
||||
elif (
|
||||
|
||||
@@ -5169,10 +5169,11 @@ class ServerArgs:
|
||||
], "The expert parallel size must be 1 or the same as the tensor parallel size"
|
||||
|
||||
if view.moe_runner_backend == "flashinfer_cutedsl":
|
||||
# modelopt_mixed with non-NVFP4 MoE layers is rejected at load time.
|
||||
assert (
|
||||
view.quantization in ["modelopt_fp4"]
|
||||
view.quantization in ["modelopt_fp4", "modelopt_mixed"]
|
||||
or self.get_model_config().nvfp4_moe_meta is not None
|
||||
), f"Invalid quantization '{view.quantization}'. \nFlashInfer CuteDSL MOE currently supports only: 'modelopt_fp4' or hybrid NVFP4 models."
|
||||
), f"Invalid quantization '{view.quantization}'. \nFlashInfer CuteDSL MOE currently supports only: 'modelopt_fp4', 'modelopt_mixed' (with NVFP4 MoE layers), or hybrid NVFP4 models."
|
||||
assert view.ep_size in [
|
||||
1,
|
||||
self.tp_size,
|
||||
|
||||
Reference in New Issue
Block a user