[NPU] bugfix for chunkedprefill (#15166)
This commit is contained in:
@@ -737,6 +737,7 @@ class AscendAttnBackend(AttentionBackend):
|
|||||||
causal=causal,
|
causal=causal,
|
||||||
)
|
)
|
||||||
elif sum(forward_batch.extend_prefix_lens_cpu) > 0:
|
elif sum(forward_batch.extend_prefix_lens_cpu) > 0:
|
||||||
|
num_token_padding = q.shape[0]
|
||||||
q, k, v = [
|
q, k, v = [
|
||||||
data[: forward_batch.num_token_non_padded_cpu] for data in [q, k, v]
|
data[: forward_batch.num_token_non_padded_cpu] for data in [q, k, v]
|
||||||
]
|
]
|
||||||
@@ -824,6 +825,17 @@ class AscendAttnBackend(AttentionBackend):
|
|||||||
attn_output = attn_output.reshape(
|
attn_output = attn_output.reshape(
|
||||||
[-1, layer.tp_q_head_num, layer.v_head_dim]
|
[-1, layer.tp_q_head_num, layer.v_head_dim]
|
||||||
)
|
)
|
||||||
|
if num_token_padding != forward_batch.num_token_non_padded_cpu:
|
||||||
|
attn_output = torch.cat(
|
||||||
|
[
|
||||||
|
attn_output,
|
||||||
|
attn_output.new_zeros(
|
||||||
|
num_token_padding - attn_output.shape[0],
|
||||||
|
*attn_output.shape[1:],
|
||||||
|
),
|
||||||
|
],
|
||||||
|
dim=0,
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
assert (
|
assert (
|
||||||
layer.qk_head_dim != layer.v_head_dim
|
layer.qk_head_dim != layer.v_head_dim
|
||||||
|
|||||||
Reference in New Issue
Block a user