[AMD] Support Quark DeepSeek-V4-MXFP4 checkpoints on the DSV4 stack, enabling shared-expert fusion and MTP (#31054)

This commit is contained in:
Colin Z
2026-10-08 22:54:49 -07:00
committed by GitHub
parent 9a1aeacc3b
commit 3c1703d2ff
5 changed files with 266 additions and 7 deletions
@@ -282,6 +282,10 @@ _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: tuple[str, ...] = (
"model.language_model.layers.0",
)
# A NextN draft builds its block under ``model.decoder``; only the draft's own
# weight-name mapper renames per-layer specs onto it, so the target never matches.
_MOE_SHARED_EXPERT_QUANT_DRAFT_BASES: tuple[str, ...] = ("model.decoder",)
_SHARED_EXPERT_BODY_PROJ_SUFFIXES: tuple[str, ...] = (
"gate_proj",
"up_proj",
@@ -381,9 +385,15 @@ class QuarkConfig(QuantizationConfig):
layer_quant_config = self.quant_config.get("layer_quant_config")
if layer_quant_config:
self.quant_config["layer_quant_config"] = hf_to_sglang_mapper.apply_dict(
layer_quant_config
)
layer_quant_config = hf_to_sglang_mapper.apply_dict(layer_quant_config)
# DeepSeek-V4 fuses wq_a + wkv into one wqkv_a module that has no spec
# of its own; both halves share one, so mirror it.
for name in list(layer_quant_config):
if name.endswith(".wq_a"):
layer_quant_config.setdefault(
name.removesuffix(".wq_a") + ".wqkv_a", layer_quant_config[name]
)
self.quant_config["layer_quant_config"] = layer_quant_config
if self.kv_cache_group:
self.kv_cache_group = hf_to_sglang_mapper.apply_list(self.kv_cache_group)
@@ -1054,15 +1064,26 @@ class QuarkConfig(QuantizationConfig):
if not layer_quant_config:
return True
# Compare routed vs shared specs at layer 0 (stub module needed by
# _find_matched_config; an unmatched name -> ValueError -> cannot fuse).
# DeepSeek-family modules are "shared_experts"; a misspelled lookup would
# fall through to the global spec and hide a mismatch.
shared_module = (
"shared_experts"
if any(".shared_experts." in name for name in layer_quant_config)
else "shared_expert"
)
# Compare routed vs shared specs at layer 0 and the draft block (stub module
# needed by _find_matched_config; an unmatched name -> ValueError -> cannot fuse).
lookup_stub = torch.nn.Module()
try:
for base in _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES:
for base in (
_MOE_SHARED_EXPERT_QUANT_LAYER0_BASES
+ _MOE_SHARED_EXPERT_QUANT_DRAFT_BASES
):
moe_name = f"{base}.mlp.experts"
moe_cfg = self._find_matched_config(moe_name, lookup_stub)
for suffix in _SHARED_EXPERT_BODY_PROJ_SUFFIXES:
shared_name = f"{base}.mlp.shared_expert.{suffix}"
shared_name = f"{base}.mlp.{shared_module}.{suffix}"
shared_cfg = self._find_matched_config(shared_name, lookup_stub)
if not deep_compare(moe_cfg, shared_cfg):
return False
@@ -59,6 +59,42 @@ if _use_aiter:
from aiter.utility.fp4_utils import e8m0_shuffle
def _pad_tp_chunks(
loaded_weight: torch.Tensor, *, dim: int, tp_size: int, padded: int
) -> torch.Tensor:
"""Zero-pad each of the tp_size equal chunks of loaded_weight along dim to padded."""
real = loaded_weight.shape[dim] // tp_size
if real == padded:
return loaded_weight
assert real * tp_size == loaded_weight.shape[dim] and real < padded
chunks = loaded_weight.unflatten(dim, (tp_size, real))
pad_shape = list(chunks.shape)
pad_shape[dim + 1] = padded - real
return torch.cat([chunks, chunks.new_zeros(pad_shape)], dim=dim + 1).flatten(
dim, dim + 1
)
def _rank_chunk_padding_loader(weight_loader, *, tp_size: int):
"""FusedMoE's padded loader slices rank r at r * padded_width, but aiter treats
the last intermediate_pad channels of each rank as padding; pad every rank's
checkpoint chunk to match aiter's assumption."""
def loader(param, loaded_weight, weight_name, shard_id, expert_id, *args, **kw):
if shard_id in ("w1", "w2", "w3") and loaded_weight.dim() == 2:
# Intermediate axis: rows of w1/w3 (into half of w13), cols of w2.
dim = 1 if shard_id == "w2" else 0
padded = param.shape[dim + 1] // (1 if shard_id == "w2" else 2)
loaded_weight = _pad_tp_chunks(
loaded_weight, dim=dim, tp_size=tp_size, padded=padded
)
return weight_loader(
param, loaded_weight, weight_name, shard_id, expert_id, *args, **kw
)
return loader
# gfx1250's grouped MoE GEMM reads weight scales in the n32k4 layout
# (moe_shuffle_scale -> shuffle_scale_n32k4), not the e8m0_shuffle layout used by
# gfx950. Using the wrong layout silently corrupts the dequant scales.
@@ -220,6 +256,10 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
)
layer.hidden_pad = 0
layer.intermediate_pad = w13_up_dim // 2 - intermediate_size_per_partition
if layer.intermediate_pad and not layer.use_presharded_weights:
original_weight_loader = _rank_chunk_padding_loader(
original_weight_loader, tp_size=layer.moe_tp_size
)
# Add the quantization method used (per tensor/grouped/channel)
# to ensure the weight scales are loaded in properly
+34
View File
@@ -176,6 +176,7 @@ from sglang.srt.models.deepseek_v2 import (
_is_xpu,
)
from sglang.srt.models.deepseek_v41_vit import Aligner, ViT
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.multimodal.deepseek_v41_image_processing import (
GPU_PLAN_KEY,
image_token_types,
@@ -4407,9 +4408,29 @@ class DeepseekV4Model(nn.Module):
return hidden_states, pre_hc_head
# Quark keys its per-layer specs by checkpoint module names (layers.N.attn.wq_a,
# mtp.0.ffn.experts.N.w1, ...); this maps them onto the module prefixes built here.
_QUARK_CKPT_TO_SGLANG_MAPPER = WeightsMapper(
orig_to_new_prefix={"layers.": "model.layers.", "mtp.": "model.mtp."},
orig_to_new_substr={".attn.": ".self_attn.", ".ffn.": ".mlp."},
orig_to_new_suffix={".w1": ".gate_proj", ".w2": ".down_proj", ".w3": ".up_proj"},
)
def _is_quark_checkpoint(config: DeepSeekV4Config) -> bool:
quantization_config = getattr(config, "quantization_config", None) or {}
return quantization_config.get("quant_method") == "quark"
class DeepseekV4ForCausalLM(nn.Module):
supports_cuda_vmm_feature_transport = True
@classmethod
def get_hf_to_sglang_mapper(cls, config) -> Optional[WeightsMapper]:
# The loader hands this mapper to every quant config; other formats were
# written against different names, so only Quark checkpoints get it.
return _QUARK_CKPT_TO_SGLANG_MAPPER if _is_quark_checkpoint(config) else None
def __init__(
self,
config: DeepSeekV4Config,
@@ -5478,9 +5499,22 @@ def _prepare_deepseek_v4_weights(
if quant_config is not None and quant_config.get_name() == "expert_pack":
logger.info("Keep Expert Pack GGUF weights on the streaming load path")
return weights
if quant_config is not None and quant_config.get_name() == "quark":
weights = _rename_quark_fp8_block_scales(weights)
return _dequant_fp8_wo_a_streaming(weights)
def _rename_quark_fp8_block_scales(
weights: Iterable[Tuple[str, torch.Tensor]],
) -> Iterable[Tuple[str, torch.Tensor]]:
# Quark spells the FP8-block scale ".weight_scale" where native V4 spells it
# ".scale" (both e8m0); MXFP4 scales are uint8 and keep their name.
for name, tensor in weights:
if name.endswith(".weight_scale") and tensor.dtype == torch.float8_e8m0fnu:
name = name.removesuffix(".weight_scale") + ".scale"
yield name, tensor
def _fuse_deepseek_v4_wqkv_a_pair(
param_name: str, bucket: dict[str, torch.Tensor]
) -> torch.Tensor:
@@ -36,6 +36,7 @@ from sglang.srt.models.deepseek_v4 import (
_is_npu,
wo_a_fp8_gemm_enabled,
)
from sglang.srt.models.utils import WeightsMapper
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import add_prefix
@@ -209,6 +210,20 @@ class DeepseekV4ModelNextN(nn.Module):
class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
@classmethod
def get_hf_to_sglang_mapper(cls, config) -> Optional[WeightsMapper]:
mapper = super().get_hf_to_sglang_mapper(config)
if mapper is None:
return None
# The draft builds mtp.0 as model.decoder, with e_proj/h_proj beside it.
return mapper | WeightsMapper(
orig_to_new_prefix={
"mtp.0.e_proj": "model.e_proj",
"mtp.0.h_proj": "model.h_proj",
"mtp.0.": "model.decoder.",
}
)
def __init__(
self,
config: PretrainedConfig,