From 3c1703d2ffddb7592893cdf632c6d8228df2e0e6 Mon Sep 17 00:00:00 2001 From: Colin Z <59755453+ColinZ22@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:54:49 -0700 Subject: [PATCH] [AMD] Support Quark DeepSeek-V4-MXFP4 checkpoints on the DSV4 stack, enabling shared-expert fusion and MTP (#31054) --- .../srt/layers/quantization/quark/quark.py | 35 +++- .../quark/schemes/quark_w4a4_mxfp4_moe.py | 40 +++++ python/sglang/srt/models/deepseek_v4.py | 34 ++++ python/sglang/srt/models/deepseek_v4_nextn.py | 15 ++ .../layers/quantization/test_quark_config.py | 149 ++++++++++++++++++ 5 files changed, 266 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/layers/quantization/quark/quark.py b/python/sglang/srt/layers/quantization/quark/quark.py index 943005c79a..19effc3dbd 100644 --- a/python/sglang/srt/layers/quantization/quark/quark.py +++ b/python/sglang/srt/layers/quantization/quark/quark.py @@ -282,6 +282,10 @@ _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: tuple[str, ...] = ( "model.language_model.layers.0", ) +# A NextN draft builds its block under ``model.decoder``; only the draft's own +# weight-name mapper renames per-layer specs onto it, so the target never matches. +_MOE_SHARED_EXPERT_QUANT_DRAFT_BASES: tuple[str, ...] = ("model.decoder",) + _SHARED_EXPERT_BODY_PROJ_SUFFIXES: tuple[str, ...] = ( "gate_proj", "up_proj", @@ -381,9 +385,15 @@ class QuarkConfig(QuantizationConfig): layer_quant_config = self.quant_config.get("layer_quant_config") if layer_quant_config: - self.quant_config["layer_quant_config"] = hf_to_sglang_mapper.apply_dict( - layer_quant_config - ) + layer_quant_config = hf_to_sglang_mapper.apply_dict(layer_quant_config) + # DeepSeek-V4 fuses wq_a + wkv into one wqkv_a module that has no spec + # of its own; both halves share one, so mirror it. + for name in list(layer_quant_config): + if name.endswith(".wq_a"): + layer_quant_config.setdefault( + name.removesuffix(".wq_a") + ".wqkv_a", layer_quant_config[name] + ) + self.quant_config["layer_quant_config"] = layer_quant_config if self.kv_cache_group: self.kv_cache_group = hf_to_sglang_mapper.apply_list(self.kv_cache_group) @@ -1054,15 +1064,26 @@ class QuarkConfig(QuantizationConfig): if not layer_quant_config: return True - # Compare routed vs shared specs at layer 0 (stub module needed by - # _find_matched_config; an unmatched name -> ValueError -> cannot fuse). + # DeepSeek-family modules are "shared_experts"; a misspelled lookup would + # fall through to the global spec and hide a mismatch. + shared_module = ( + "shared_experts" + if any(".shared_experts." in name for name in layer_quant_config) + else "shared_expert" + ) + + # Compare routed vs shared specs at layer 0 and the draft block (stub module + # needed by _find_matched_config; an unmatched name -> ValueError -> cannot fuse). lookup_stub = torch.nn.Module() try: - for base in _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES: + for base in ( + _MOE_SHARED_EXPERT_QUANT_LAYER0_BASES + + _MOE_SHARED_EXPERT_QUANT_DRAFT_BASES + ): moe_name = f"{base}.mlp.experts" moe_cfg = self._find_matched_config(moe_name, lookup_stub) for suffix in _SHARED_EXPERT_BODY_PROJ_SUFFIXES: - shared_name = f"{base}.mlp.shared_expert.{suffix}" + shared_name = f"{base}.mlp.{shared_module}.{suffix}" shared_cfg = self._find_matched_config(shared_name, lookup_stub) if not deep_compare(moe_cfg, shared_cfg): return False diff --git a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py index e3efb0833f..5342591171 100644 --- a/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py +++ b/python/sglang/srt/layers/quantization/quark/schemes/quark_w4a4_mxfp4_moe.py @@ -59,6 +59,42 @@ if _use_aiter: from aiter.utility.fp4_utils import e8m0_shuffle +def _pad_tp_chunks( + loaded_weight: torch.Tensor, *, dim: int, tp_size: int, padded: int +) -> torch.Tensor: + """Zero-pad each of the tp_size equal chunks of loaded_weight along dim to padded.""" + real = loaded_weight.shape[dim] // tp_size + if real == padded: + return loaded_weight + assert real * tp_size == loaded_weight.shape[dim] and real < padded + chunks = loaded_weight.unflatten(dim, (tp_size, real)) + pad_shape = list(chunks.shape) + pad_shape[dim + 1] = padded - real + return torch.cat([chunks, chunks.new_zeros(pad_shape)], dim=dim + 1).flatten( + dim, dim + 1 + ) + + +def _rank_chunk_padding_loader(weight_loader, *, tp_size: int): + """FusedMoE's padded loader slices rank r at r * padded_width, but aiter treats + the last intermediate_pad channels of each rank as padding; pad every rank's + checkpoint chunk to match aiter's assumption.""" + + def loader(param, loaded_weight, weight_name, shard_id, expert_id, *args, **kw): + if shard_id in ("w1", "w2", "w3") and loaded_weight.dim() == 2: + # Intermediate axis: rows of w1/w3 (into half of w13), cols of w2. + dim = 1 if shard_id == "w2" else 0 + padded = param.shape[dim + 1] // (1 if shard_id == "w2" else 2) + loaded_weight = _pad_tp_chunks( + loaded_weight, dim=dim, tp_size=tp_size, padded=padded + ) + return weight_loader( + param, loaded_weight, weight_name, shard_id, expert_id, *args, **kw + ) + + return loader + + # gfx1250's grouped MoE GEMM reads weight scales in the n32k4 layout # (moe_shuffle_scale -> shuffle_scale_n32k4), not the e8m0_shuffle layout used by # gfx950. Using the wrong layout silently corrupts the dequant scales. @@ -220,6 +256,10 @@ class QuarkW4A4MXFp4MoE(QuarkMoEScheme): ) layer.hidden_pad = 0 layer.intermediate_pad = w13_up_dim // 2 - intermediate_size_per_partition + if layer.intermediate_pad and not layer.use_presharded_weights: + original_weight_loader = _rank_chunk_padding_loader( + original_weight_loader, tp_size=layer.moe_tp_size + ) # Add the quantization method used (per tensor/grouped/channel) # to ensure the weight scales are loaded in properly diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index b5e2ce5c92..5234e214a8 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -176,6 +176,7 @@ from sglang.srt.models.deepseek_v2 import ( _is_xpu, ) from sglang.srt.models.deepseek_v41_vit import Aligner, ViT +from sglang.srt.models.utils import WeightsMapper from sglang.srt.multimodal.deepseek_v41_image_processing import ( GPU_PLAN_KEY, image_token_types, @@ -4407,9 +4408,29 @@ class DeepseekV4Model(nn.Module): return hidden_states, pre_hc_head +# Quark keys its per-layer specs by checkpoint module names (layers.N.attn.wq_a, +# mtp.0.ffn.experts.N.w1, ...); this maps them onto the module prefixes built here. +_QUARK_CKPT_TO_SGLANG_MAPPER = WeightsMapper( + orig_to_new_prefix={"layers.": "model.layers.", "mtp.": "model.mtp."}, + orig_to_new_substr={".attn.": ".self_attn.", ".ffn.": ".mlp."}, + orig_to_new_suffix={".w1": ".gate_proj", ".w2": ".down_proj", ".w3": ".up_proj"}, +) + + +def _is_quark_checkpoint(config: DeepSeekV4Config) -> bool: + quantization_config = getattr(config, "quantization_config", None) or {} + return quantization_config.get("quant_method") == "quark" + + class DeepseekV4ForCausalLM(nn.Module): supports_cuda_vmm_feature_transport = True + @classmethod + def get_hf_to_sglang_mapper(cls, config) -> Optional[WeightsMapper]: + # The loader hands this mapper to every quant config; other formats were + # written against different names, so only Quark checkpoints get it. + return _QUARK_CKPT_TO_SGLANG_MAPPER if _is_quark_checkpoint(config) else None + def __init__( self, config: DeepSeekV4Config, @@ -5478,9 +5499,22 @@ def _prepare_deepseek_v4_weights( if quant_config is not None and quant_config.get_name() == "expert_pack": logger.info("Keep Expert Pack GGUF weights on the streaming load path") return weights + if quant_config is not None and quant_config.get_name() == "quark": + weights = _rename_quark_fp8_block_scales(weights) return _dequant_fp8_wo_a_streaming(weights) +def _rename_quark_fp8_block_scales( + weights: Iterable[Tuple[str, torch.Tensor]], +) -> Iterable[Tuple[str, torch.Tensor]]: + # Quark spells the FP8-block scale ".weight_scale" where native V4 spells it + # ".scale" (both e8m0); MXFP4 scales are uint8 and keep their name. + for name, tensor in weights: + if name.endswith(".weight_scale") and tensor.dtype == torch.float8_e8m0fnu: + name = name.removesuffix(".weight_scale") + ".scale" + yield name, tensor + + def _fuse_deepseek_v4_wqkv_a_pair( param_name: str, bucket: dict[str, torch.Tensor] ) -> torch.Tensor: diff --git a/python/sglang/srt/models/deepseek_v4_nextn.py b/python/sglang/srt/models/deepseek_v4_nextn.py index 6694abb603..d278403808 100644 --- a/python/sglang/srt/models/deepseek_v4_nextn.py +++ b/python/sglang/srt/models/deepseek_v4_nextn.py @@ -36,6 +36,7 @@ from sglang.srt.models.deepseek_v4 import ( _is_npu, wo_a_fp8_gemm_enabled, ) +from sglang.srt.models.utils import WeightsMapper from sglang.srt.runtime_context import get_parallel from sglang.srt.utils import add_prefix @@ -209,6 +210,20 @@ class DeepseekV4ModelNextN(nn.Module): class DeepseekV4ForCausalLMNextN(DeepseekV4ForCausalLM): + @classmethod + def get_hf_to_sglang_mapper(cls, config) -> Optional[WeightsMapper]: + mapper = super().get_hf_to_sglang_mapper(config) + if mapper is None: + return None + # The draft builds mtp.0 as model.decoder, with e_proj/h_proj beside it. + return mapper | WeightsMapper( + orig_to_new_prefix={ + "mtp.0.e_proj": "model.e_proj", + "mtp.0.h_proj": "model.h_proj", + "mtp.0.": "model.decoder.", + } + ) + def __init__( self, config: PretrainedConfig, diff --git a/test/registered/unit/layers/quantization/test_quark_config.py b/test/registered/unit/layers/quantization/test_quark_config.py index 0f70be224d..edf6e9a2cf 100644 --- a/test/registered/unit/layers/quantization/test_quark_config.py +++ b/test/registered/unit/layers/quantization/test_quark_config.py @@ -399,6 +399,85 @@ class TestQuarkPerLayerBlockFp8(CustomTestCase): self.assertEqual(matched["weight"]["dtype"], "fp4") +class TestDeepseekV4QuarkMapping(CustomTestCase): + """amd/DeepSeek-V4-*-MXFP4 layout: MXFP4 experts everywhere, but the MTP + block keeps FP8-block shared experts; specs are keyed by checkpoint names.""" + + _HF_CONFIG = SimpleNamespace(quantization_config={"quant_method": "quark"}) + + _FP8 = TestQuarkPerLayerBlockFp8._BLOCK_FP8_CONFIG + _MXFP4 = { + "weight": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "is_dynamic": False, + "scale_format": "e8m0", + }, + "input_tensors": { + "dtype": "fp4", + "qscheme": "per_group", + "group_size": 32, + "is_dynamic": True, + "scale_format": "e8m0", + }, + "output_tensors": None, + "bias": None, + } + + def _config_for(self, model_cls) -> QuarkConfig: + config = _bare_config() + config.quant_config = { + "layer_quant_config": { + "layers.0.attn.wq_a": deepcopy(self._FP8), + "layers.0.attn.wkv": deepcopy(self._FP8), + **{ + f"mtp.0.ffn.shared_experts.w{i}": deepcopy(self._FP8) + for i in (1, 2, 3) + }, + **{ + f"mtp.0.ffn.experts.{e}.w{i}": deepcopy(self._MXFP4) + for e in range(2) + for i in (1, 2, 3) + }, + }, + "layer_type_quant_config": {}, + "global_quant_config": deepcopy(self._MXFP4), + } + config.exclude_layers = [] + config.kv_cache_group = [] + config.packed_modules_mapping = {"gate_up_proj": ["gate_proj", "up_proj"]} + config.excluded_fp8_config = None + config.is_prequantized = True + config.dequantization_config = None + config._online_quantized_layers = set() + config.apply_weight_name_mapper( + model_cls.get_hf_to_sglang_mapper(self._HF_CONFIG) + ) + return config + + def test_only_draft_vetoes_fusion_on_fp8_mtp_shared_experts(self): + from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM + from sglang.srt.models.deepseek_v4_nextn import DeepseekV4ForCausalLMNextN + + self.assertTrue( + self._config_for(DeepseekV4ForCausalLM).can_fuse_shared_expert() + ) + self.assertFalse( + self._config_for(DeepseekV4ForCausalLMNextN).can_fuse_shared_expert() + ) + + def test_fused_wqkv_a_takes_the_wq_a_block_fp8_spec(self): + from sglang.srt.models.deepseek_v4 import DeepseekV4ForCausalLM + + config = self._config_for(DeepseekV4ForCausalLM) + layer = LinearBase.__new__(LinearBase) + + method = config.get_quant_method(layer, "model.layers.0.self_attn.wqkv_a") + + self.assertIsInstance(method, Fp8LinearMethod) + + class _Runner: """Records the quant_info apply_weights() hands to the runner.""" @@ -459,5 +538,75 @@ class TestQuarkMxfp4MoEAiterQuantInfo(CustomTestCase): self.assertTrue(quant_info.w2_weight.is_shuffled) +class TestQuarkMxfp4MoEPaddedTpLoading(CustomTestCase): + """aiter skips the last intermediate_pad channels of every TP rank, so each + rank's real checkpoint channels must load at the head of its padded buffer. + Regression: DeepSeek-V4-Pro TP8 (384 -> 512) silently lost 25% of channels.""" + + TP, REAL, PADDED, COLS = 8, 384, 512, 3 + + def _load_rank(self, rank, loader): + from sglang.srt.layers.moe.fused_moe_triton.layer import FusedMoE + + moe = SimpleNamespace( + moe_runner_config=SimpleNamespace(is_gated=True), + quant_method=SimpleNamespace(), + quant_config=None, + use_padded_loading=True, + use_presharded_weights=False, + moe_tp_rank=rank, + moe_tp_size=self.TP, + ) + + def fused_moe_weight_loader(param, w, name, shard_id, expert_id): + load = FusedMoE._load_w2 if shard_id == "w2" else FusedMoE._load_w13 + load( + moe, + expert_data=param.data[expert_id], + shard_dim=1 if shard_id == "w2" else 0, + shard_id=shard_id, + loaded_weight=w, + ) + + load = loader(fused_moe_weight_loader) + full = self.TP * self.REAL + channel = torch.arange(1, full + 1, dtype=torch.float32) + gate, up = channel[:, None].repeat(1, self.COLS), -channel[:, None] + down = channel[None, :].repeat(self.COLS, 1) + w13 = torch.full((1, 2 * self.PADDED, self.COLS), float("nan")) + w2 = torch.full((1, self.COLS, self.PADDED), float("nan")) + load(w13, gate, "w13_weight", "w1", 0) + load(w13, up.repeat(1, self.COLS), "w13_weight", "w3", 0) + load(w2, down, "w2_weight", "w2", 0) + return w13[0], w2[0] + + def test_each_rank_holds_its_own_channels_then_zero_padding(self): + from sglang.srt.layers.quantization.quark.schemes.quark_w4a4_mxfp4_moe import ( + _rank_chunk_padding_loader, + ) + + loader = lambda wl: _rank_chunk_padding_loader(wl, tp_size=self.TP) + real, padded = self.REAL, self.PADDED + for rank in range(self.TP): + w13, w2 = self._load_rank(rank, loader) + expected = torch.arange(rank * real + 1, (rank + 1) * real + 1).float() + with self.subTest(rank=rank): + self.assertTrue(torch.equal(w13[:real, 0], expected)) + self.assertTrue(torch.equal(w13[padded : padded + real, 0], -expected)) + self.assertTrue(torch.equal(w2[0, :real], expected)) + self.assertEqual(w13[real:padded].abs().sum().item(), 0) + self.assertEqual(w13[padded + real :].abs().sum().item(), 0) + self.assertEqual(w2[:, real:].abs().sum().item(), 0) + + def test_unpadded_weights_pass_through_unchanged(self): + from sglang.srt.layers.quantization.quark.schemes.quark_w4a4_mxfp4_moe import ( + _pad_tp_chunks, + ) + + weight = torch.randn(self.TP * self.REAL, self.COLS) + out = _pad_tp_chunks(weight, dim=0, tp_size=self.TP, padded=self.REAL) + self.assertIs(out, weight) + + if __name__ == "__main__": unittest.main()