[Fix] deepseek-v4: Refresh attn_sink after loading weights (#41630)
Co-authored-by: parkeychen <parkeychen@tencent.com>
This commit is contained in:
@@ -1149,10 +1149,19 @@ class MqaAttentionBase(nn.Module):
|
||||
sink_num_heads = max(kernel_num_heads, padded_num_heads)
|
||||
if self._attn_sink_local is None:
|
||||
sink = self.attn_sink.new_zeros(sink_num_heads)
|
||||
sink[:num_heads] = self.attn_sink[rank * num_heads : (rank + 1) * num_heads]
|
||||
self._attn_sink_local_slice = slice(
|
||||
rank * num_heads, (rank + 1) * num_heads
|
||||
)
|
||||
sink[:num_heads] = self.attn_sink[self._attn_sink_local_slice]
|
||||
self._attn_sink_local = sink
|
||||
return self._attn_sink_local[:kernel_num_heads]
|
||||
|
||||
def refresh_attn_sink_cache(self) -> None:
|
||||
if self._attn_sink_local is None:
|
||||
return
|
||||
src = self.attn_sink.data[self._attn_sink_local_slice]
|
||||
self._attn_sink_local[: src.numel()].copy_(src)
|
||||
|
||||
def _q_lora_quant_e8m0(self, with_indexer: bool) -> bool:
|
||||
"""Whether the fused q_norm quant feeds wq_b (and the indexer's wq_b, which
|
||||
reuses it) with ue8m0 scales: only when each consumer runs the e8m0 mxscale
|
||||
@@ -4860,6 +4869,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
and not self_attn.indexer.compressor.ape_converted
|
||||
):
|
||||
self_attn.indexer.compressor.apply_ape_hotfix()
|
||||
self_attn.refresh_attn_sink_cache()
|
||||
layer.refresh_mhc_norm_weight_cache()
|
||||
layers = self.model.layers
|
||||
for i, layer in enumerate(layers):
|
||||
|
||||
Reference in New Issue
Block a user