diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index dc07f75a91..fae7c4ce94 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -1149,10 +1149,19 @@ class MqaAttentionBase(nn.Module): sink_num_heads = max(kernel_num_heads, padded_num_heads) if self._attn_sink_local is None: sink = self.attn_sink.new_zeros(sink_num_heads) - sink[:num_heads] = self.attn_sink[rank * num_heads : (rank + 1) * num_heads] + self._attn_sink_local_slice = slice( + rank * num_heads, (rank + 1) * num_heads + ) + sink[:num_heads] = self.attn_sink[self._attn_sink_local_slice] self._attn_sink_local = sink return self._attn_sink_local[:kernel_num_heads] + def refresh_attn_sink_cache(self) -> None: + if self._attn_sink_local is None: + return + src = self.attn_sink.data[self._attn_sink_local_slice] + self._attn_sink_local[: src.numel()].copy_(src) + def _q_lora_quant_e8m0(self, with_indexer: bool) -> bool: """Whether the fused q_norm quant feeds wq_b (and the indexer's wq_b, which reuses it) with ue8m0 scales: only when each consumer runs the e8m0 mxscale @@ -4860,6 +4869,7 @@ class DeepseekV4ForCausalLM(nn.Module): and not self_attn.indexer.compressor.ape_converted ): self_attn.indexer.compressor.apply_ape_hotfix() + self_attn.refresh_attn_sink_cache() layer.refresh_mhc_norm_weight_cache() layers = self.model.layers for i, layer in enumerate(layers):