[Fix] [AMD] Quark: fix Quark block-FP8 loading in DeepSeek loader (#40776)
Co-authored-by: clintg6 <7388379+clintg6@users.noreply.github.com> Co-authored-by: Thomas Wang <thomawan@amd.com> Co-authored-by: Bowen Bao <bowenbao@amd.com> Co-authored-by: Hubert Lu <Hubert.Lu@amd.com>
This commit is contained in:
co-authored by
clintg6
Thomas Wang
Bowen Bao
Hubert Lu
parent
baf80c61b7
commit
9d7ad308bd
@@ -86,6 +86,32 @@ def _get_indexer_weight_block_size(
|
||||
return [128, 128]
|
||||
|
||||
|
||||
def _resolve_quark_fp8_block_scale_name(
|
||||
name: str, params_dict: Dict[str, torch.Tensor]
|
||||
) -> str:
|
||||
"""Map Quark block-FP8 scale names to the shared FP8 parameter name."""
|
||||
if name.endswith("weight_scale") and name not in params_dict:
|
||||
candidate = name.removesuffix("weight_scale") + "weight_scale_inv"
|
||||
if candidate in params_dict:
|
||||
return candidate
|
||||
return name
|
||||
|
||||
|
||||
def _resolve_layer_weight_block_size(
|
||||
layer: nn.Module,
|
||||
quant_config: Optional[QuantizationConfig],
|
||||
) -> Optional[List[int]]:
|
||||
"""Return the resolved layer's block size, with legacy config fallback."""
|
||||
quant_method = getattr(layer, "quant_method", None)
|
||||
if quant_method is not None and hasattr(quant_method, "weight_block_size"):
|
||||
return quant_method.weight_block_size
|
||||
|
||||
selected_quant_config = getattr(quant_config, "linear_fp8_config", None)
|
||||
if selected_quant_config is None:
|
||||
selected_quant_config = quant_config
|
||||
return getattr(selected_quant_config, "weight_block_size", None)
|
||||
|
||||
|
||||
def _load_fused_indexer_wk(
|
||||
name: str,
|
||||
loaded_weight: torch.Tensor,
|
||||
@@ -107,7 +133,7 @@ def _load_fused_indexer_wk(
|
||||
return False
|
||||
|
||||
if ".indexer.weights_proj." in name:
|
||||
is_scale = name.endswith(".weight_scale_inv")
|
||||
is_scale = name.endswith((".weight_scale_inv", ".weight_scale"))
|
||||
if not is_scale and loaded_weight.dtype != torch.float8_e4m3fn:
|
||||
w = _clone_if_runai_streamed_tensor(loaded_weight)
|
||||
fused_param.data[-w.shape[0] :].copy_(w)
|
||||
@@ -127,7 +153,7 @@ def _load_fused_indexer_wk(
|
||||
return True
|
||||
|
||||
# wk: a bf16 checkpoint copies straight in; block-fp8 needs weight + scale.
|
||||
is_scale = name.endswith(".weight_scale_inv")
|
||||
is_scale = name.endswith((".weight_scale_inv", ".weight_scale"))
|
||||
if not is_scale and loaded_weight.dtype != torch.float8_e4m3fn:
|
||||
w = _clone_if_runai_streamed_tensor(loaded_weight)
|
||||
fused_param.data[: w.shape[0]].copy_(w)
|
||||
@@ -461,6 +487,16 @@ class DeepseekV2WeightLoaderMixin:
|
||||
"fused_qkv_a_proj_with_mqa",
|
||||
)
|
||||
)
|
||||
param_name = _resolve_quark_fp8_block_scale_name(
|
||||
param_name, params_dict
|
||||
)
|
||||
if param_name not in params_dict:
|
||||
logger.warning(
|
||||
f"{param_name} not found in params_dict."
|
||||
)
|
||||
cached_a_proj.pop(q_a_proj_name, None)
|
||||
cached_a_proj.pop(kv_a_proj_name, None)
|
||||
continue
|
||||
param = params_dict[param_name]
|
||||
|
||||
weight_loader = getattr(
|
||||
@@ -486,6 +522,9 @@ class DeepseekV2WeightLoaderMixin:
|
||||
f"{scale[0]}_proj", "attn_mqa"
|
||||
)
|
||||
break
|
||||
name = _resolve_quark_fp8_block_scale_name(
|
||||
name, params_dict
|
||||
)
|
||||
if name not in params_dict:
|
||||
# modelopt ckpt contains not needed weights for MTP module:
|
||||
# model.decoder.self_attn.attn_mqa.v_scale and
|
||||
@@ -608,16 +647,9 @@ class DeepseekV2WeightLoaderMixin:
|
||||
torch.float8_e4m3fn,
|
||||
torch.float8_e4m3fnuz,
|
||||
):
|
||||
# For mixed quantization (experts int4, linear fp8), use linear_fp8_config
|
||||
selected_quant_config = getattr(
|
||||
self.quant_config, "linear_fp8_config", None
|
||||
)
|
||||
if selected_quant_config is None:
|
||||
selected_quant_config = self.quant_config
|
||||
weight_block_size = (
|
||||
selected_quant_config.weight_block_size
|
||||
if selected_quant_config is not None
|
||||
else None
|
||||
weight_block_size = _resolve_layer_weight_block_size(
|
||||
self_attn.kv_b_proj,
|
||||
self.quant_config,
|
||||
)
|
||||
if weight_block_size is not None:
|
||||
assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") or hasattr(
|
||||
@@ -647,11 +679,7 @@ class DeepseekV2WeightLoaderMixin:
|
||||
)
|
||||
elif (
|
||||
should_deepgemm_weight_requant_ue8m0(
|
||||
weight_block_size=(
|
||||
self.quant_config.weight_block_size
|
||||
if self.quant_config is not None
|
||||
else None
|
||||
)
|
||||
weight_block_size=weight_block_size
|
||||
)
|
||||
and weight_scale.format_ue8m0
|
||||
):
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Regression tests for Quark block-FP8 weights in the DeepSeek loader."""
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
|
||||
register_cpu_ci(est_time=5, suite="base-a-test-cpu")
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.models.deepseek_common.deepseek_weight_loader import (
|
||||
DeepseekV2WeightLoaderMixin,
|
||||
_load_fused_indexer_wk,
|
||||
_resolve_layer_weight_block_size,
|
||||
_resolve_quark_fp8_block_scale_name,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
|
||||
class _FakeParam:
|
||||
def __init__(self):
|
||||
self.loaded = None
|
||||
|
||||
def weight_loader(self, param, loaded_weight):
|
||||
self.loaded = loaded_weight
|
||||
|
||||
|
||||
class _FakeDeepseekLoader(DeepseekV2WeightLoaderMixin):
|
||||
def __init__(self, params):
|
||||
self.config = SimpleNamespace(
|
||||
n_routed_experts=0,
|
||||
q_lora_rank=2048,
|
||||
num_hidden_layers=1,
|
||||
num_nextn_predict_layers=0,
|
||||
)
|
||||
self.model = SimpleNamespace()
|
||||
self.quant_config = SimpleNamespace(get_name=lambda: "quark")
|
||||
self.num_fused_shared_experts = 0
|
||||
self.pp_group = SimpleNamespace(is_first_rank=True, is_last_rank=True)
|
||||
self._params = params
|
||||
self.post_load_called = False
|
||||
|
||||
def named_parameters(self):
|
||||
return iter(self._params.items())
|
||||
|
||||
def _maybe_quant_weights_to_fp8_ue8m0(
|
||||
self, weights, attn_quant_modules, nextn_conf
|
||||
):
|
||||
return weights
|
||||
|
||||
def post_load_weights(self, is_nextn=False, weight_names=None):
|
||||
self.post_load_called = True
|
||||
|
||||
|
||||
class TestQuarkBlockFp8WeightLoading(CustomTestCase):
|
||||
def test_scale_name_maps_only_when_scale_inv_is_registered(self):
|
||||
scale_inv_name = (
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj_with_mqa.weight_scale_inv"
|
||||
)
|
||||
params_dict = {scale_inv_name: torch.empty(1)}
|
||||
checkpoint_name = scale_inv_name.removesuffix("_inv")
|
||||
|
||||
self.assertEqual(
|
||||
_resolve_quark_fp8_block_scale_name(checkpoint_name, params_dict),
|
||||
scale_inv_name,
|
||||
)
|
||||
|
||||
mxfp4_name = "model.layers.0.mlp.experts.w13_weight_scale"
|
||||
params_dict = {mxfp4_name: torch.empty(1)}
|
||||
self.assertEqual(
|
||||
_resolve_quark_fp8_block_scale_name(mxfp4_name, params_dict),
|
||||
mxfp4_name,
|
||||
)
|
||||
|
||||
def test_layer_quant_method_precedes_config_level_summary(self):
|
||||
layer = SimpleNamespace(
|
||||
quant_method=SimpleNamespace(weight_block_size=[64, 128])
|
||||
)
|
||||
quant_config = SimpleNamespace(
|
||||
linear_fp8_config=SimpleNamespace(weight_block_size=[128, 128]),
|
||||
weight_block_size=None,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
_resolve_layer_weight_block_size(layer, quant_config),
|
||||
[64, 128],
|
||||
)
|
||||
|
||||
layer.quant_method.weight_block_size = None
|
||||
self.assertIsNone(_resolve_layer_weight_block_size(layer, quant_config))
|
||||
|
||||
def test_fused_qkv_a_quark_scales_load_into_scale_inv(self):
|
||||
target_name = (
|
||||
"model.layers.0.self_attn.fused_qkv_a_proj_with_mqa.weight_scale_inv"
|
||||
)
|
||||
target = _FakeParam()
|
||||
loader = _FakeDeepseekLoader({target_name: target})
|
||||
q_scale = torch.ones((2, 3), dtype=torch.float32)
|
||||
kv_scale = torch.full((1, 3), 2.0, dtype=torch.float32)
|
||||
|
||||
loader.do_load_weights(
|
||||
[
|
||||
(
|
||||
"model.layers.0.self_attn.q_a_proj.weight_scale",
|
||||
q_scale,
|
||||
),
|
||||
(
|
||||
"model.layers.0.self_attn.kv_a_proj_with_mqa.weight_scale",
|
||||
kv_scale,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
torch.testing.assert_close(
|
||||
target.loaded,
|
||||
torch.cat([q_scale, kv_scale], dim=0),
|
||||
)
|
||||
self.assertTrue(loader.post_load_called)
|
||||
|
||||
def test_unfused_quark_scale_loads_into_scale_inv(self):
|
||||
target_name = "model.layers.0.self_attn.o_proj.weight_scale_inv"
|
||||
target = _FakeParam()
|
||||
loader = _FakeDeepseekLoader({target_name: target})
|
||||
loaded_scale = torch.ones((2, 3), dtype=torch.float32)
|
||||
|
||||
loader.do_load_weights(
|
||||
[
|
||||
(
|
||||
"model.layers.0.self_attn.o_proj.weight_scale",
|
||||
loaded_scale,
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
self.assertIs(target.loaded, loaded_scale)
|
||||
self.assertTrue(loader.post_load_called)
|
||||
|
||||
def test_quark_weight_scale_is_not_misclassified_as_indexer_weight(self):
|
||||
fused_name = "model.layers.0.self_attn.indexer.wk_weights_proj.weight"
|
||||
fused_param = torch.nn.Parameter(
|
||||
torch.zeros((4, 4), dtype=torch.bfloat16),
|
||||
requires_grad=False,
|
||||
)
|
||||
params_dict = {fused_name: fused_param}
|
||||
pending = {}
|
||||
loaded_weight = torch.ones((2, 4), dtype=torch.float8_e4m3fn)
|
||||
loaded_scale = torch.ones((1, 1), dtype=torch.float32)
|
||||
dequantized = torch.ones((2, 4), dtype=torch.bfloat16)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.models.deepseek_common.deepseek_weight_loader."
|
||||
"block_quant_dequant",
|
||||
return_value=dequantized,
|
||||
) as block_quant_dequant:
|
||||
self.assertTrue(
|
||||
_load_fused_indexer_wk(
|
||||
"model.layers.0.self_attn.indexer.wk.weight",
|
||||
loaded_weight,
|
||||
params_dict,
|
||||
pending,
|
||||
None,
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
_load_fused_indexer_wk(
|
||||
"model.layers.0.self_attn.indexer.wk.weight_scale",
|
||||
loaded_scale,
|
||||
params_dict,
|
||||
pending,
|
||||
None,
|
||||
)
|
||||
)
|
||||
|
||||
block_quant_dequant.assert_called_once_with(
|
||||
loaded_weight,
|
||||
loaded_scale,
|
||||
[128, 128],
|
||||
torch.bfloat16,
|
||||
)
|
||||
torch.testing.assert_close(fused_param[:2], dequantized)
|
||||
self.assertEqual(pending, {})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user