[DFlash] Build the attention QKV projection through an overridable hook (#43179)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
842974287
parent
00b3a48753
commit
e3c5719713
@@ -228,15 +228,7 @@ class DFlashAttention(nn.Module):
|
||||
attention_bias = bool(getattr(config, "attention_bias", False))
|
||||
rms_norm_eps = float(getattr(config, "rms_norm_eps", 1e-6))
|
||||
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
hidden_size=hidden_size,
|
||||
head_size=head_dim,
|
||||
total_num_heads=self.total_num_heads,
|
||||
total_num_kv_heads=self.total_num_kv_heads,
|
||||
bias=attention_bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj" if prefix else "qkv_proj",
|
||||
)
|
||||
self.qkv_proj = self._build_qkv_proj(attention_bias, quant_config, prefix)
|
||||
self.o_proj = RowParallelLinear(
|
||||
self.total_num_heads * head_dim,
|
||||
hidden_size,
|
||||
@@ -319,6 +311,17 @@ class DFlashAttention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
)
|
||||
|
||||
def _build_qkv_proj(self, bias: bool, quant_config, prefix: str) -> nn.Module:
|
||||
return QKVParallelLinear(
|
||||
hidden_size=self.hidden_size,
|
||||
head_size=self.head_dim,
|
||||
total_num_heads=self.total_num_heads,
|
||||
total_num_kv_heads=self.total_num_kv_heads,
|
||||
bias=bias,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.qkv_proj" if prefix else "qkv_proj",
|
||||
)
|
||||
|
||||
def forward_prepare_npu(self, positions, hidden_states):
|
||||
qkv, _ = self.qkv_proj(hidden_states)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user