[ROCm][Bugfix] Keep quantization for mixed Quark Qwen3.5 MTP checkpoints (#39064)
Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: jacky.cheng <yichiche@amd.com>
This commit is contained in:
co-authored by
Cursor
jacky.cheng
parent
67bb6a58d0
commit
0154f72b48
@@ -69,21 +69,35 @@ def _mtp_quant_config(quant_config):
|
||||
return None
|
||||
if is_npu() and get_spec().speculative_draft_model_quantization is None:
|
||||
return None
|
||||
# Quark-quantized Qwen3.5 MXFP4 checkpoints ship the MTP module in bf16;
|
||||
# every `mtp.*` layer appears under the quantization exclude list. Detect
|
||||
# that and skip quantization here so linear/MoE weight loaders allocate
|
||||
# bf16 shapes (see sgl-project/sglang#23113).
|
||||
# Some Quark-quantized Qwen3.5 MXFP4 checkpoints ship the MTP module
|
||||
# entirely in bf16, listing every `mtp.*` layer under the quantization
|
||||
# exclude list. Skip quantization for those so linear/MoE weight loaders
|
||||
# allocate bf16 shapes (see sgl-project/sglang#23146).
|
||||
#
|
||||
# Others are mixed: the routed experts stay MXFP4 while attention, the
|
||||
# shared expert and fc are excluded. Skipping there would make the MoE
|
||||
# loader allocate bf16 experts that the MXFP4 checkpoint shards no longer
|
||||
# fit. The routed experts are the bulk of the draft, so use them as the
|
||||
# signal and skip only when they are excluded too; the per-layer
|
||||
# exclusions keep the remaining bf16 modules bf16 on their own.
|
||||
if quant_config and quant_config.get_name() == "quark":
|
||||
exclude_layers = getattr(quant_config, "exclude_layers", [])
|
||||
if any(
|
||||
isinstance(layer, str) and layer.startswith("mtp.")
|
||||
for layer in exclude_layers
|
||||
):
|
||||
mtp_excludes = [
|
||||
layer
|
||||
for layer in getattr(quant_config, "exclude_layers", [])
|
||||
if isinstance(layer, str) and layer.startswith("mtp.")
|
||||
]
|
||||
if mtp_excludes and any("mlp.experts" in layer for layer in mtp_excludes):
|
||||
return None
|
||||
return quant_config
|
||||
|
||||
|
||||
class Qwen3_5ForCausalLMMTP(nn.Module):
|
||||
# The loader reads this off the model class and hands it to the quant
|
||||
# config, which needs it to expand fused module names (qkv_proj ->
|
||||
# q/k/v_proj) before matching them against an exclude list. Without it an
|
||||
# excluded attention projection is not recognised as excluded.
|
||||
packed_modules_mapping = Qwen3_5ForCausalLM.packed_modules_mapping
|
||||
|
||||
@staticmethod
|
||||
def shared_experts_fusion_disable_reason(hf_config, quant_config):
|
||||
return Qwen3_5ForCausalLM.shared_experts_fusion_disable_reason(
|
||||
|
||||
Reference in New Issue
Block a user