[Fix] [NPU] qwen: adapt pcp-v2 (#41564)

This commit is contained in:
gjsheu
2026-10-22 20:54:11 +08:00
committed by GitHub
parent 851419717a
commit a9902f47e3
2 changed files with 270 additions and 84 deletions
@@ -30,13 +30,12 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
)
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
from sglang.srt.layers.dcp.layout import (
get_dcp_chain_spec_lens,
get_dcp_lens,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
@@ -48,7 +47,6 @@ from sglang.srt.runtime_context import (
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
from sglang.srt.utils import (
get_bool_env_var,
get_current_device_stream_fast,
next_power_of_2,
)
@@ -290,29 +288,25 @@ def _cp_allgather_and_save_kv_npu(
if not layer.is_cross_attention
else forward_batch.encoder_out_cache_loc
)
# Save original trailing shapes for reshape after gather.
k_tail = k.shape[1:] # (tp_k_head_num, qk_head_dim)
v_tail = v.shape[1:] # (tp_v_head_num, v_head_dim)
# CP v2: all-gather the rank-local K/V through the active CP strategy and
# write the full sequence to the KV pool. Same contract as the
# `cp_active` branch of flashattention_backend.
cp_strategy = get_cp_strategy()
if cp_strategy is None or cp_strategy.cp_size <= 1:
token_to_kv_pool.set_kv_buffer(
layer,
KVWriteLoc(cache_loc, swa_loc),
k,
v,
)
return
# Flatten trailing dims then concat → one all-gather instead of two.
# Works for GQA where tp_k_head_num != tp_v_head_num.
k_flat = k.contiguous().reshape(k.shape[0], -1) # [S_local, k_feat]
v_flat = v.contiguous().reshape(v.shape[0], -1) # [S_local, v_feat]
k_feat_size = k_flat.shape[-1]
kv_flat = torch.cat([k_flat, v_flat], dim=-1) # [S_local, k_feat + v_feat]
kv_full = cp_all_gather_rerange_kv_cache(
kv_flat, cp_size, forward_batch, get_current_device_stream_fast()
) # [S_full, k_feat + v_feat]
key_cache_full = kv_full[..., :k_feat_size].reshape(-1, *k_tail)
value_cache_full = kv_full[..., k_feat_size:].reshape(-1, *v_tail)
token_to_kv_pool.set_kv_buffer(
cp_strategy.materialize_full_kv(
forward_batch,
layer,
KVWriteLoc(cache_loc, swa_loc),
key_cache_full,
value_cache_full,
k,
v,
swa_loc=swa_loc,
)
@@ -1380,70 +1374,51 @@ class AscendAttnBackend(AttentionBackend):
The KV cache must already contain the full gathered sequence
(written by _cp_allgather_and_save_kv_npu before this call).
Args:
q: Query tensor, shape [total_q_tokens, tp_q_head_num * qk_head_dim]
k_cache: Full key cache from token_to_kv_pool
v_cache: Full value cache from token_to_kv_pool
layer: RadixAttention layer
forward_batch: ForwardBatch with attn_cp_metadata populated
Returns:
attn_output [total_q_tokens, tp_q_head_num * v_head_dim]
Token slicing into the prev/next zigzag chunks, pad-row trimming and
pad-row re-appending are owned by the CP-v2 strategy
(``ZigzagCPStrategy.run_attention``); this backend only supplies the FIA
computation closure. ``attention_backend`` is borrowed from
``CPAttentionBackendKind.FLASH_ATTENTION`` because FIA uses the same
two-call (prev + next) dispatch convention.
"""
cp_meta = forward_batch.attn_cp_metadata
# Local tokens are laid out [all_seqs_prev, all_seqs_next]; split at
# total_q_prev_tokens rather than the midpoint to support bs > 1.
split = cp_meta.total_q_prev_tokens
q_prev = (
q[:split].contiguous().reshape(-1, layer.tp_q_head_num, layer.qk_head_dim)
)
q_next = (
q[split:].contiguous().reshape(-1, layer.tp_q_head_num, layer.qk_head_dim)
)
def _fia_cp_attn(q_chunk, cu_seqlens_q_cp, cache_seqlens_cp, max_seqlen_q_cp):
# The strategy passes cu_seqlens_q (with a leading 0) and plain KV
# lengths as int32 tensors; FIA wants cumulative query lengths
# (no leading 0) and KV lengths as Python lists.
k_cache_paged = k_cache.view(
-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim
)
v_cache_paged = v_cache.view(
-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim
)
return torch_npu.npu_fused_infer_attention_score(
query=q_chunk.contiguous(),
key=k_cache_paged,
value=v_cache_paged,
block_table=self.forward_metadata.block_tables,
block_size=self.page_size,
num_heads=layer.tp_q_head_num,
num_key_value_heads=layer.tp_k_head_num,
input_layout="TND",
atten_mask=self.fia_mask,
sparse_mode=3,
next_tokens=0,
scale=layer.scaling,
actual_seq_lengths=cu_seqlens_q_cp[1:].tolist(),
actual_seq_lengths_kv=cache_seqlens_cp.tolist(),
)[0]
k_cache_paged = k_cache.view(
-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim
cp_strategy = get_cp_strategy()
assert cp_strategy is not None
q_cp = q.contiguous().view(-1, layer.tp_q_head_num, layer.qk_head_dim)
attn_out = cp_strategy.run_attention(
q_cp,
forward_batch,
self.device,
_fia_cp_attn,
attention_backend=CPAttentionBackendKind.FLASH_ATTENTION,
)
v_cache_paged = v_cache.view(
-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim
)
attn_out_prev, _ = torch.ops.npu.npu_fused_infer_attention_score(
q_prev,
k_cache_paged,
v_cache_paged,
block_table=self.forward_metadata.block_tables,
block_size=self.page_size,
num_heads=layer.tp_q_head_num,
num_key_value_heads=layer.tp_k_head_num,
input_layout="TND",
atten_mask=self.fia_mask,
sparse_mode=3,
next_tokens=0,
scale=layer.scaling,
actual_seq_lengths=np.cumsum(cp_meta.actual_seq_q_prev_list).tolist(),
actual_seq_lengths_kv=cp_meta.kv_len_prev_list,
)
attn_out_next, _ = torch.ops.npu.npu_fused_infer_attention_score(
q_next,
k_cache_paged,
v_cache_paged,
block_table=self.forward_metadata.block_tables,
block_size=self.page_size,
num_heads=layer.tp_q_head_num,
num_key_value_heads=layer.tp_k_head_num,
input_layout="TND",
atten_mask=self.fia_mask,
sparse_mode=3,
next_tokens=0,
scale=layer.scaling,
actual_seq_lengths=np.cumsum(cp_meta.actual_seq_q_next_list).tolist(),
actual_seq_lengths_kv=cp_meta.kv_len_next_list,
)
attn_out = torch.cat([attn_out_prev, attn_out_next], dim=0)
return attn_out.view(-1, layer.tp_q_head_num * layer.v_head_dim)
def forward_sparse(