[Fix] [AMD] Quark: fix Quark block-FP8 loading in DeepSeek loader (#40776)

Co-authored-by: clintg6 <7388379+clintg6@users.noreply.github.com>
Co-authored-by: Thomas Wang <thomawan@amd.com>
Co-authored-by: Bowen Bao <bowenbao@amd.com>
Co-authored-by: Hubert Lu <Hubert.Lu@amd.com>
This commit is contained in:
Clint
2026-10-23 12:11:21 -07:00
committed by GitHub
co-authored by clintg6 Thomas Wang Bowen Bao Hubert Lu
parent baf80c61b7
commit 9d7ad308bd
2 changed files with 232 additions and 17 deletions
@@ -86,6 +86,32 @@ def _get_indexer_weight_block_size(
return [128, 128]
def _resolve_quark_fp8_block_scale_name(
name: str, params_dict: Dict[str, torch.Tensor]
) -> str:
"""Map Quark block-FP8 scale names to the shared FP8 parameter name."""
if name.endswith("weight_scale") and name not in params_dict:
candidate = name.removesuffix("weight_scale") + "weight_scale_inv"
if candidate in params_dict:
return candidate
return name
def _resolve_layer_weight_block_size(
layer: nn.Module,
quant_config: Optional[QuantizationConfig],
) -> Optional[List[int]]:
"""Return the resolved layer's block size, with legacy config fallback."""
quant_method = getattr(layer, "quant_method", None)
if quant_method is not None and hasattr(quant_method, "weight_block_size"):
return quant_method.weight_block_size
selected_quant_config = getattr(quant_config, "linear_fp8_config", None)
if selected_quant_config is None:
selected_quant_config = quant_config
return getattr(selected_quant_config, "weight_block_size", None)
def _load_fused_indexer_wk(
name: str,
loaded_weight: torch.Tensor,
@@ -107,7 +133,7 @@ def _load_fused_indexer_wk(
return False
if ".indexer.weights_proj." in name:
is_scale = name.endswith(".weight_scale_inv")
is_scale = name.endswith((".weight_scale_inv", ".weight_scale"))
if not is_scale and loaded_weight.dtype != torch.float8_e4m3fn:
w = _clone_if_runai_streamed_tensor(loaded_weight)
fused_param.data[-w.shape[0] :].copy_(w)
@@ -127,7 +153,7 @@ def _load_fused_indexer_wk(
return True
# wk: a bf16 checkpoint copies straight in; block-fp8 needs weight + scale.
is_scale = name.endswith(".weight_scale_inv")
is_scale = name.endswith((".weight_scale_inv", ".weight_scale"))
if not is_scale and loaded_weight.dtype != torch.float8_e4m3fn:
w = _clone_if_runai_streamed_tensor(loaded_weight)
fused_param.data[: w.shape[0]].copy_(w)
@@ -461,6 +487,16 @@ class DeepseekV2WeightLoaderMixin:
"fused_qkv_a_proj_with_mqa",
)
)
param_name = _resolve_quark_fp8_block_scale_name(
param_name, params_dict
)
if param_name not in params_dict:
logger.warning(
f"{param_name} not found in params_dict."
)
cached_a_proj.pop(q_a_proj_name, None)
cached_a_proj.pop(kv_a_proj_name, None)
continue
param = params_dict[param_name]
weight_loader = getattr(
@@ -486,6 +522,9 @@ class DeepseekV2WeightLoaderMixin:
f"{scale[0]}_proj", "attn_mqa"
)
break
name = _resolve_quark_fp8_block_scale_name(
name, params_dict
)
if name not in params_dict:
# modelopt ckpt contains not needed weights for MTP module:
# model.decoder.self_attn.attn_mqa.v_scale and
@@ -608,16 +647,9 @@ class DeepseekV2WeightLoaderMixin:
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
):
# For mixed quantization (experts int4, linear fp8), use linear_fp8_config
selected_quant_config = getattr(
self.quant_config, "linear_fp8_config", None
)
if selected_quant_config is None:
selected_quant_config = self.quant_config
weight_block_size = (
selected_quant_config.weight_block_size
if selected_quant_config is not None
else None
weight_block_size = _resolve_layer_weight_block_size(
self_attn.kv_b_proj,
self.quant_config,
)
if weight_block_size is not None:
assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") or hasattr(
@@ -647,11 +679,7 @@ class DeepseekV2WeightLoaderMixin:
)
elif (
should_deepgemm_weight_requant_ue8m0(
weight_block_size=(
self.quant_config.weight_block_size
if self.quant_config is not None
else None
)
weight_block_size=weight_block_size
)
and weight_scale.format_ue8m0
):
@@ -0,0 +1,187 @@
"""Regression tests for Quark block-FP8 weights in the DeepSeek loader."""
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
import unittest
from types import SimpleNamespace
from unittest.mock import patch
import torch
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
DeepseekV2WeightLoaderMixin,
_load_fused_indexer_wk,
_resolve_layer_weight_block_size,
_resolve_quark_fp8_block_scale_name,
)
from sglang.test.test_utils import CustomTestCase
class _FakeParam:
def __init__(self):
self.loaded = None
def weight_loader(self, param, loaded_weight):
self.loaded = loaded_weight
class _FakeDeepseekLoader(DeepseekV2WeightLoaderMixin):
def __init__(self, params):
self.config = SimpleNamespace(
n_routed_experts=0,
q_lora_rank=2048,
num_hidden_layers=1,
num_nextn_predict_layers=0,
)
self.model = SimpleNamespace()
self.quant_config = SimpleNamespace(get_name=lambda: "quark")
self.num_fused_shared_experts = 0
self.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
self._params = params
self.post_load_called = False
def named_parameters(self):
return iter(self._params.items())
def _maybe_quant_weights_to_fp8_ue8m0(
self, weights, attn_quant_modules, nextn_conf
):
return weights
def post_load_weights(self, is_nextn=False, weight_names=None):
self.post_load_called = True
class TestQuarkBlockFp8WeightLoading(CustomTestCase):
def test_scale_name_maps_only_when_scale_inv_is_registered(self):
scale_inv_name = (
"model.layers.0.self_attn.fused_qkv_a_proj_with_mqa.weight_scale_inv"
)
params_dict = {scale_inv_name: torch.empty(1)}
checkpoint_name = scale_inv_name.removesuffix("_inv")
self.assertEqual(
_resolve_quark_fp8_block_scale_name(checkpoint_name, params_dict),
scale_inv_name,
)
mxfp4_name = "model.layers.0.mlp.experts.w13_weight_scale"
params_dict = {mxfp4_name: torch.empty(1)}
self.assertEqual(
_resolve_quark_fp8_block_scale_name(mxfp4_name, params_dict),
mxfp4_name,
)
def test_layer_quant_method_precedes_config_level_summary(self):
layer = SimpleNamespace(
quant_method=SimpleNamespace(weight_block_size=[64, 128])
)
quant_config = SimpleNamespace(
linear_fp8_config=SimpleNamespace(weight_block_size=[128, 128]),
weight_block_size=None,
)
self.assertEqual(
_resolve_layer_weight_block_size(layer, quant_config),
[64, 128],
)
layer.quant_method.weight_block_size = None
self.assertIsNone(_resolve_layer_weight_block_size(layer, quant_config))
def test_fused_qkv_a_quark_scales_load_into_scale_inv(self):
target_name = (
"model.layers.0.self_attn.fused_qkv_a_proj_with_mqa.weight_scale_inv"
)
target = _FakeParam()
loader = _FakeDeepseekLoader({target_name: target})
q_scale = torch.ones((2, 3), dtype=torch.float32)
kv_scale = torch.full((1, 3), 2.0, dtype=torch.float32)
loader.do_load_weights(
[
(
"model.layers.0.self_attn.q_a_proj.weight_scale",
q_scale,
),
(
"model.layers.0.self_attn.kv_a_proj_with_mqa.weight_scale",
kv_scale,
),
]
)
torch.testing.assert_close(
target.loaded,
torch.cat([q_scale, kv_scale], dim=0),
)
self.assertTrue(loader.post_load_called)
def test_unfused_quark_scale_loads_into_scale_inv(self):
target_name = "model.layers.0.self_attn.o_proj.weight_scale_inv"
target = _FakeParam()
loader = _FakeDeepseekLoader({target_name: target})
loaded_scale = torch.ones((2, 3), dtype=torch.float32)
loader.do_load_weights(
[
(
"model.layers.0.self_attn.o_proj.weight_scale",
loaded_scale,
),
]
)
self.assertIs(target.loaded, loaded_scale)
self.assertTrue(loader.post_load_called)
def test_quark_weight_scale_is_not_misclassified_as_indexer_weight(self):
fused_name = "model.layers.0.self_attn.indexer.wk_weights_proj.weight"
fused_param = torch.nn.Parameter(
torch.zeros((4, 4), dtype=torch.bfloat16),
requires_grad=False,
)
params_dict = {fused_name: fused_param}
pending = {}
loaded_weight = torch.ones((2, 4), dtype=torch.float8_e4m3fn)
loaded_scale = torch.ones((1, 1), dtype=torch.float32)
dequantized = torch.ones((2, 4), dtype=torch.bfloat16)
with patch(
"sglang.srt.models.deepseek_common.deepseek_weight_loader."
"block_quant_dequant",
return_value=dequantized,
) as block_quant_dequant:
self.assertTrue(
_load_fused_indexer_wk(
"model.layers.0.self_attn.indexer.wk.weight",
loaded_weight,
params_dict,
pending,
None,
)
)
self.assertTrue(
_load_fused_indexer_wk(
"model.layers.0.self_attn.indexer.wk.weight_scale",
loaded_scale,
params_dict,
pending,
None,
)
)
block_quant_dequant.assert_called_once_with(
loaded_weight,
loaded_scale,
[128, 128],
torch.bfloat16,
)
torch.testing.assert_close(fused_param[:2], dequantized)
self.assertEqual(pending, {})
if __name__ == "__main__":
unittest.main()