【NPU】【bugfix】accuracy fix when enable both nsa cp and prefixcache (#23268)
This commit is contained in:
@@ -359,7 +359,9 @@ def forward_dsa_prepare_npu(
|
|||||||
if q_event is not None:
|
if q_event is not None:
|
||||||
torch.npu.current_stream().wait_event(q_event)
|
torch.npu.current_stream().wait_event(q_event)
|
||||||
else:
|
else:
|
||||||
if fused_qkv_a_proj_out.shape[0] < 65535:
|
if fused_qkv_a_proj_out.shape[0] < 65535 and not nsa_use_prefill_cp(
|
||||||
|
forward_batch
|
||||||
|
):
|
||||||
q_lora, k_nope, k_pe = fused_split_qk_norm(
|
q_lora, k_nope, k_pe = fused_split_qk_norm(
|
||||||
fused_qkv_a_proj_out,
|
fused_qkv_a_proj_out,
|
||||||
m.q_a_layernorm,
|
m.q_a_layernorm,
|
||||||
|
|||||||
@@ -1463,6 +1463,20 @@ class Indexer(MultiPlatformOp):
|
|||||||
forward_batch.attn_cp_metadata.actual_seq_q_prev_tensor,
|
forward_batch.attn_cp_metadata.actual_seq_q_prev_tensor,
|
||||||
forward_batch.attn_cp_metadata.actual_seq_q_next_tensor,
|
forward_batch.attn_cp_metadata.actual_seq_q_next_tensor,
|
||||||
)
|
)
|
||||||
|
if sum(forward_batch.extend_prefix_lens_cpu) > 0:
|
||||||
|
total_kv_len_prev_tensor = (
|
||||||
|
forward_batch.attn_cp_metadata.kv_len_prev_tensor
|
||||||
|
+ forward_batch.extend_prefix_lens.squeeze()
|
||||||
|
)
|
||||||
|
total_kv_len_next_tensor = (
|
||||||
|
forward_batch.attn_cp_metadata.kv_len_next_tensor
|
||||||
|
+ forward_batch.extend_prefix_lens.squeeze()
|
||||||
|
)
|
||||||
|
forward_batch.attn_backend.forward_metadata.actual_seq_lengths_kv = (
|
||||||
|
total_kv_len_prev_tensor,
|
||||||
|
total_kv_len_next_tensor,
|
||||||
|
)
|
||||||
|
else:
|
||||||
forward_batch.attn_backend.forward_metadata.actual_seq_lengths_kv = (
|
forward_batch.attn_backend.forward_metadata.actual_seq_lengths_kv = (
|
||||||
forward_batch.attn_cp_metadata.kv_len_prev_tensor,
|
forward_batch.attn_cp_metadata.kv_len_prev_tensor,
|
||||||
forward_batch.attn_cp_metadata.kv_len_next_tensor,
|
forward_batch.attn_cp_metadata.kv_len_next_tensor,
|
||||||
|
|||||||
Reference in New Issue
Block a user