minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Expend compatibility check for all quantized MoE models (#13465)

Browse Source 
Signed-off-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
Xinyuan Tong
2025-11-20 01:24:27 +08:00
committed by GitHub
parent e72cf13693
commit a355794905
1 changed files with 25 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/model_runner.py
+25 -24
View File
@@ -298,6 +298,7 @@ class ModelRunner:
# Model-specific adjustment # Model-specific adjustment
self.model_specific_adjustment() self.model_specific_adjustment()
self.check_quantized_moe_compatibility()
# Set the global server_args in the scheduler process # Set the global server_args in the scheduler process
set_global_server_args_for_scheduler(server_args) set_global_server_args_for_scheduler(server_args)
@@ -570,34 +571,34 @@ class ModelRunner:
if not server_args.disable_chunked_prefix_cache: if not server_args.disable_chunked_prefix_cache:
log_info_on_rank0(logger, "Chunked prefix cache is turned on.") log_info_on_rank0(logger, "Chunked prefix cache is turned on.")
if self.model_config.hf_config.model_type == "qwen3_vl_moe": def check_quantized_moe_compatibility(self):
if ( if (
quantization_config := getattr( quantization_config := getattr(
self.model_config.hf_config, "quantization_config", None self.model_config.hf_config, "quantization_config", None
)
) is not None and "weight_block_size" in quantization_config:
weight_block_size_n = quantization_config["weight_block_size"][0]
if self.tp_size % self.moe_ep_size != 0:
raise ValueError(
f"tp_size {self.tp_size} must be divisible by ep_size {self.moe_ep_size}"
) )
) is not None and "weight_block_size" in quantization_config: moe_tp_size = self.tp_size // self.moe_ep_size
weight_block_size_n = quantization_config["weight_block_size"][0]
if self.tp_size % self.moe_ep_size != 0: moe_intermediate_size = (
raise ValueError( self.model_config.hf_text_config.moe_intermediate_size
f"tp_size {self.tp_size} must be divisible by moe_ep_size {self.moe_ep_size}" )
) if moe_intermediate_size % moe_tp_size != 0:
moe_tp_size = self.tp_size // self.moe_ep_size raise ValueError(
f"moe_intermediate_size {moe_intermediate_size} must be divisible by moe_tp_size ({moe_tp_size}) which is tp_size ({self.tp_size}) divided by moe_ep_size ({self.moe_ep_size})."
moe_intermediate_size = (
self.model_config.hf_text_config.moe_intermediate_size
) )
if moe_intermediate_size % moe_tp_size != 0:
raise ValueError(
f"moe_intermediate_size {moe_intermediate_size} must be divisible by moe_tp_size ({moe_tp_size}) which is tp_size ({self.tp_size}) divided by moe_ep_size ({self.moe_ep_size})."
)
if (moe_intermediate_size // moe_tp_size) % weight_block_size_n != 0: if (moe_intermediate_size // moe_tp_size) % weight_block_size_n != 0:
raise ValueError( raise ValueError(
f"For qwen3-vl-fp8 models, please make sure ({moe_intermediate_size=} / {moe_tp_size=}) % {weight_block_size_n=} == 0 " f"For quantized MoE models, please make sure ({moe_intermediate_size=} / {moe_tp_size=}) % {weight_block_size_n=} == 0 "
f"where moe_tp_size is equal to tp_size ({self.tp_size}) divided by moe_ep_size ({self.moe_ep_size}). " f"where moe_tp_size is equal to tp_size ({self.tp_size}) divided by ep_size ({self.moe_ep_size}). "
f"You can fix this by setting arguments `--tp-size` and `--ep-size` correctly." f"You can fix this by setting arguments `--tp` and `--ep` correctly."
) )
def init_torch_distributed(self): def init_torch_distributed(self):
logger.info("Init torch distributed begin.") logger.info("Init torch distributed begin.")