[AMD] Support Quark DeepSeek-V4-MXFP4 checkpoints on the DSV4 stack, enabling shared-expert fusion and MTP (#31054)
This commit is contained in:
@@ -282,6 +282,10 @@ _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: tuple[str, ...] = (
|
||||
"model.language_model.layers.0",
|
||||
)
|
||||
|
||||
# A NextN draft builds its block under ``model.decoder``; only the draft's own
|
||||
# weight-name mapper renames per-layer specs onto it, so the target never matches.
|
||||
_MOE_SHARED_EXPERT_QUANT_DRAFT_BASES: tuple[str, ...] = ("model.decoder",)
|
||||
|
||||
_SHARED_EXPERT_BODY_PROJ_SUFFIXES: tuple[str, ...] = (
|
||||
"gate_proj",
|
||||
"up_proj",
|
||||
@@ -381,9 +385,15 @@ class QuarkConfig(QuantizationConfig):
|
||||
|
||||
layer_quant_config = self.quant_config.get("layer_quant_config")
|
||||
if layer_quant_config:
|
||||
self.quant_config["layer_quant_config"] = hf_to_sglang_mapper.apply_dict(
|
||||
layer_quant_config
|
||||
)
|
||||
layer_quant_config = hf_to_sglang_mapper.apply_dict(layer_quant_config)
|
||||
# DeepSeek-V4 fuses wq_a + wkv into one wqkv_a module that has no spec
|
||||
# of its own; both halves share one, so mirror it.
|
||||
for name in list(layer_quant_config):
|
||||
if name.endswith(".wq_a"):
|
||||
layer_quant_config.setdefault(
|
||||
name.removesuffix(".wq_a") + ".wqkv_a", layer_quant_config[name]
|
||||
)
|
||||
self.quant_config["layer_quant_config"] = layer_quant_config
|
||||
|
||||
if self.kv_cache_group:
|
||||
self.kv_cache_group = hf_to_sglang_mapper.apply_list(self.kv_cache_group)
|
||||
@@ -1054,15 +1064,26 @@ class QuarkConfig(QuantizationConfig):
|
||||
if not layer_quant_config:
|
||||
return True
|
||||
|
||||
# Compare routed vs shared specs at layer 0 (stub module needed by
|
||||
# _find_matched_config; an unmatched name -> ValueError -> cannot fuse).
|
||||
# DeepSeek-family modules are "shared_experts"; a misspelled lookup would
|
||||
# fall through to the global spec and hide a mismatch.
|
||||
shared_module = (
|
||||
"shared_experts"
|
||||
if any(".shared_experts." in name for name in layer_quant_config)
|
||||
else "shared_expert"
|
||||
)
|
||||
|
||||
# Compare routed vs shared specs at layer 0 and the draft block (stub module
|
||||
# needed by _find_matched_config; an unmatched name -> ValueError -> cannot fuse).
|
||||
lookup_stub = torch.nn.Module()
|
||||
try:
|
||||
for base in _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES:
|
||||
for base in (
|
||||
_MOE_SHARED_EXPERT_QUANT_LAYER0_BASES
|
||||
+ _MOE_SHARED_EXPERT_QUANT_DRAFT_BASES
|
||||
):
|
||||
moe_name = f"{base}.mlp.experts"
|
||||
moe_cfg = self._find_matched_config(moe_name, lookup_stub)
|
||||
for suffix in _SHARED_EXPERT_BODY_PROJ_SUFFIXES:
|
||||
shared_name = f"{base}.mlp.shared_expert.{suffix}"
|
||||
shared_name = f"{base}.mlp.{shared_module}.{suffix}"
|
||||
shared_cfg = self._find_matched_config(shared_name, lookup_stub)
|
||||
if not deep_compare(moe_cfg, shared_cfg):
|
||||
return False
|
||||
|
||||
@@ -59,6 +59,42 @@ if _use_aiter:
|
||||
from aiter.utility.fp4_utils import e8m0_shuffle
|
||||
|
||||
|
||||
def _pad_tp_chunks(
|
||||
loaded_weight: torch.Tensor, *, dim: int, tp_size: int, padded: int
|
||||
) -> torch.Tensor:
|
||||
"""Zero-pad each of the tp_size equal chunks of loaded_weight along dim to padded."""
|
||||
real = loaded_weight.shape[dim] // tp_size
|
||||
if real == padded:
|
||||
return loaded_weight
|
||||
assert real * tp_size == loaded_weight.shape[dim] and real < padded
|
||||
chunks = loaded_weight.unflatten(dim, (tp_size, real))
|
||||
pad_shape = list(chunks.shape)
|
||||
pad_shape[dim + 1] = padded - real
|
||||
return torch.cat([chunks, chunks.new_zeros(pad_shape)], dim=dim + 1).flatten(
|
||||
dim, dim + 1
|
||||
)
|
||||
|
||||
|
||||
def _rank_chunk_padding_loader(weight_loader, *, tp_size: int):
|
||||
"""FusedMoE's padded loader slices rank r at r * padded_width, but aiter treats
|
||||
the last intermediate_pad channels of each rank as padding; pad every rank's
|
||||
checkpoint chunk to match aiter's assumption."""
|
||||
|
||||
def loader(param, loaded_weight, weight_name, shard_id, expert_id, *args, **kw):
|
||||
if shard_id in ("w1", "w2", "w3") and loaded_weight.dim() == 2:
|
||||
# Intermediate axis: rows of w1/w3 (into half of w13), cols of w2.
|
||||
dim = 1 if shard_id == "w2" else 0
|
||||
padded = param.shape[dim + 1] // (1 if shard_id == "w2" else 2)
|
||||
loaded_weight = _pad_tp_chunks(
|
||||
loaded_weight, dim=dim, tp_size=tp_size, padded=padded
|
||||
)
|
||||
return weight_loader(
|
||||
param, loaded_weight, weight_name, shard_id, expert_id, *args, **kw
|
||||
)
|
||||
|
||||
return loader
|
||||
|
||||
|
||||
# gfx1250's grouped MoE GEMM reads weight scales in the n32k4 layout
|
||||
# (moe_shuffle_scale -> shuffle_scale_n32k4), not the e8m0_shuffle layout used by
|
||||
# gfx950. Using the wrong layout silently corrupts the dequant scales.
|
||||
@@ -220,6 +256,10 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme):
|
||||
)
|
||||
layer.hidden_pad = 0
|
||||
layer.intermediate_pad = w13_up_dim // 2 - intermediate_size_per_partition
|
||||
if layer.intermediate_pad and not layer.use_presharded_weights:
|
||||
original_weight_loader = _rank_chunk_padding_loader(
|
||||
original_weight_loader, tp_size=layer.moe_tp_size
|
||||
)
|
||||
|
||||
# Add the quantization method used (per tensor/grouped/channel)
|
||||
# to ensure the weight scales are loaded in properly
|
||||
|
||||
@@ -176,6 +176,7 @@ from sglang.srt.models.deepseek_v2 import (
|
||||
_is_xpu,
|
||||
)
|
||||
from sglang.srt.models.deepseek_v41_vit import Aligner, ViT
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.multimodal.deepseek_v41_image_processing import (
|
||||
GPU_PLAN_KEY,
|
||||
image_token_types,
|
||||
@@ -4407,9 +4408,29 @@ class DeepseekV4Model(nn.Module):
|
||||
return hidden_states, pre_hc_head
|
||||
|
||||
|
||||
# Quark keys its per-layer specs by checkpoint module names (layers.N.attn.wq_a,
|
||||
# mtp.0.ffn.experts.N.w1, ...); this maps them onto the module prefixes built here.
|
||||
_QUARK_CKPT_TO_SGLANG_MAPPER = WeightsMapper(
|
||||
orig_to_new_prefix={"layers.": "model.layers.", "mtp.": "model.mtp."},
|
||||
orig_to_new_substr={".attn.": ".self_attn.", ".ffn.": ".mlp."},
|
||||
orig_to_new_suffix={".w1": ".gate_proj", ".w2": ".down_proj", ".w3": ".up_proj"},
|
||||
)
|
||||
|
||||
|
||||
def _is_quark_checkpoint(config: DeepSeekV4Config) -> bool:
|
||||
quantization_config = getattr(config, "quantization_config", None) or {}
|
||||
return quantization_config.get("quant_method") == "quark"
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLM(nn.Module):
|
||||
supports_cuda_vmm_feature_transport = True
|
||||
|
||||
@classmethod
|
||||
def get_hf_to_sglang_mapper(cls, config) -> Optional[WeightsMapper]:
|
||||
# The loader hands this mapper to every quant config; other formats were
|
||||
# written against different names, so only Quark checkpoints get it.
|
||||
return _QUARK_CKPT_TO_SGLANG_MAPPER if _is_quark_checkpoint(config) else None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DeepSeekV4Config,
|
||||
@@ -5478,9 +5499,22 @@ def _prepare_deepseek_v4_weights(
|
||||
if quant_config is not None and quant_config.get_name() == "expert_pack":
|
||||
logger.info("Keep Expert Pack GGUF weights on the streaming load path")
|
||||
return weights
|
||||
if quant_config is not None and quant_config.get_name() == "quark":
|
||||
weights = _rename_quark_fp8_block_scales(weights)
|
||||
return _dequant_fp8_wo_a_streaming(weights)
|
||||
|
||||
|
||||
def _rename_quark_fp8_block_scales(
|
||||
weights: Iterable[Tuple[str, torch.Tensor]],
|
||||
) -> Iterable[Tuple[str, torch.Tensor]]:
|
||||
# Quark spells the FP8-block scale ".weight_scale" where native V4 spells it
|
||||
# ".scale" (both e8m0); MXFP4 scales are uint8 and keep their name.
|
||||
for name, tensor in weights:
|
||||
if name.endswith(".weight_scale") and tensor.dtype == torch.float8_e8m0fnu:
|
||||
name = name.removesuffix(".weight_scale") + ".scale"
|
||||
yield name, tensor
|
||||
|
||||
|
||||
def _fuse_deepseek_v4_wqkv_a_pair(
|
||||
param_name: str, bucket: dict[str, torch.Tensor]
|
||||
) -> torch.Tensor:
|
||||
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.models.deepseek_v4 import (
|
||||
_is_npu,
|
||||
wo_a_fp8_gemm_enabled,
|
||||
)
|
||||
from sglang.srt.models.utils import WeightsMapper
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
@@ -209,6 +210,20 @@ class DeepseekV4ModelNextN(nn.Module):
|
||||
|
||||
|
||||
class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM):
|
||||
@classmethod
|
||||
def get_hf_to_sglang_mapper(cls, config) -> Optional[WeightsMapper]:
|
||||
mapper = super().get_hf_to_sglang_mapper(config)
|
||||
if mapper is None:
|
||||
return None
|
||||
# The draft builds mtp.0 as model.decoder, with e_proj/h_proj beside it.
|
||||
return mapper | WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"mtp.0.e_proj": "model.e_proj",
|
||||
"mtp.0.h_proj": "model.h_proj",
|
||||
"mtp.0.": "model.decoder.",
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: PretrainedConfig,
|
||||
|
||||
Reference in New Issue
Block a user