[Fix] [AMD] Quark: fix Quark block-FP8 loading in DeepSeek loader (#40776)

Co-authored-by: clintg6 <7388379+clintg6@users.noreply.github.com>
Co-authored-by: Thomas Wang <thomawan@amd.com>
Co-authored-by: Bowen Bao <bowenbao@amd.com>
Co-authored-by: Hubert Lu <Hubert.Lu@amd.com>
This commit is contained in:
Clint
2026-10-23 12:11:21 -07:00
committed by GitHub
co-authored by clintg6 Thomas Wang Bowen Bao Hubert Lu
parent baf80c61b7
commit 9d7ad308bd
2 changed files with 232 additions and 17 deletions
@@ -86,6 +86,32 @@ def _get_indexer_weight_block_size(
return [128, 128]
def _resolve_quark_fp8_block_scale_name(
name: str, params_dict: Dict[str, torch.Tensor]
) -> str:
"""Map Quark block-FP8 scale names to the shared FP8 parameter name."""
if name.endswith("weight_scale") and name not in params_dict:
candidate = name.removesuffix("weight_scale") + "weight_scale_inv"
if candidate in params_dict:
return candidate
return name
def _resolve_layer_weight_block_size(
layer: nn.Module,
quant_config: Optional[QuantizationConfig],
) -> Optional[List[int]]:
"""Return the resolved layer's block size, with legacy config fallback."""
quant_method = getattr(layer, "quant_method", None)
if quant_method is not None and hasattr(quant_method, "weight_block_size"):
return quant_method.weight_block_size
selected_quant_config = getattr(quant_config, "linear_fp8_config", None)
if selected_quant_config is None:
selected_quant_config = quant_config
return getattr(selected_quant_config, "weight_block_size", None)
def _load_fused_indexer_wk(
name: str,
loaded_weight: torch.Tensor,
@@ -107,7 +133,7 @@ def _load_fused_indexer_wk(
return False
if ".indexer.weights_proj." in name:
is_scale = name.endswith(".weight_scale_inv")
is_scale = name.endswith((".weight_scale_inv", ".weight_scale"))
if not is_scale and loaded_weight.dtype != torch.float8_e4m3fn:
w = _clone_if_runai_streamed_tensor(loaded_weight)
fused_param.data[-w.shape[0] :].copy_(w)
@@ -127,7 +153,7 @@ def _load_fused_indexer_wk(
return True
# wk: a bf16 checkpoint copies straight in; block-fp8 needs weight + scale.
is_scale = name.endswith(".weight_scale_inv")
is_scale = name.endswith((".weight_scale_inv", ".weight_scale"))
if not is_scale and loaded_weight.dtype != torch.float8_e4m3fn:
w = _clone_if_runai_streamed_tensor(loaded_weight)
fused_param.data[: w.shape[0]].copy_(w)
@@ -461,6 +487,16 @@ class DeepseekV2WeightLoaderMixin:
"fused_qkv_a_proj_with_mqa",
)
)
param_name = _resolve_quark_fp8_block_scale_name(
param_name, params_dict
)
if param_name not in params_dict:
logger.warning(
f"{param_name} not found in params_dict."
)
cached_a_proj.pop(q_a_proj_name, None)
cached_a_proj.pop(kv_a_proj_name, None)
continue
param = params_dict[param_name]
weight_loader = getattr(
@@ -486,6 +522,9 @@ class DeepseekV2WeightLoaderMixin:
f"{scale[0]}_proj", "attn_mqa"
)
break
name = _resolve_quark_fp8_block_scale_name(
name, params_dict
)
if name not in params_dict:
# modelopt ckpt contains not needed weights for MTP module:
# model.decoder.self_attn.attn_mqa.v_scale and
@@ -608,16 +647,9 @@ class DeepseekV2WeightLoaderMixin:
torch.float8_e4m3fn,
torch.float8_e4m3fnuz,
):
# For mixed quantization (experts int4, linear fp8), use linear_fp8_config
selected_quant_config = getattr(
self.quant_config, "linear_fp8_config", None
)
if selected_quant_config is None:
selected_quant_config = self.quant_config
weight_block_size = (
selected_quant_config.weight_block_size
if selected_quant_config is not None
else None
weight_block_size = _resolve_layer_weight_block_size(
self_attn.kv_b_proj,
self.quant_config,
)
if weight_block_size is not None:
assert hasattr(self_attn.kv_b_proj, "weight_scale_inv") or hasattr(
@@ -647,11 +679,7 @@ class DeepseekV2WeightLoaderMixin:
)
elif (
should_deepgemm_weight_requant_ue8m0(
weight_block_size=(
self.quant_config.weight_block_size
if self.quant_config is not None
else None
)
weight_block_size=weight_block_size
)
and weight_scale.format_ue8m0
):