[Fix] [NPU] qwen: adapt pcp-v2 (#41564)
This commit is contained in:
@@ -30,13 +30,12 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
|
||||
)
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.cp.base import get_cp_strategy
|
||||
from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
|
||||
from sglang.srt.layers.dcp.layout import (
|
||||
get_dcp_chain_spec_lens,
|
||||
get_dcp_lens,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
@@ -48,7 +47,6 @@ from sglang.srt.runtime_context import (
|
||||
from sglang.srt.speculative.spec_info import SpecInput, SpecInputType
|
||||
from sglang.srt.utils import (
|
||||
get_bool_env_var,
|
||||
get_current_device_stream_fast,
|
||||
next_power_of_2,
|
||||
)
|
||||
|
||||
@@ -290,29 +288,25 @@ def _cp_allgather_and_save_kv_npu(
|
||||
if not layer.is_cross_attention
|
||||
else forward_batch.encoder_out_cache_loc
|
||||
)
|
||||
# Save original trailing shapes for reshape after gather.
|
||||
k_tail = k.shape[1:] # (tp_k_head_num, qk_head_dim)
|
||||
v_tail = v.shape[1:] # (tp_v_head_num, v_head_dim)
|
||||
# CP v2: all-gather the rank-local K/V through the active CP strategy and
|
||||
# write the full sequence to the KV pool. Same contract as the
|
||||
# `cp_active` branch of flashattention_backend.
|
||||
cp_strategy = get_cp_strategy()
|
||||
if cp_strategy is None or cp_strategy.cp_size <= 1:
|
||||
token_to_kv_pool.set_kv_buffer(
|
||||
layer,
|
||||
KVWriteLoc(cache_loc, swa_loc),
|
||||
k,
|
||||
v,
|
||||
)
|
||||
return
|
||||
|
||||
# Flatten trailing dims then concat → one all-gather instead of two.
|
||||
# Works for GQA where tp_k_head_num != tp_v_head_num.
|
||||
k_flat = k.contiguous().reshape(k.shape[0], -1) # [S_local, k_feat]
|
||||
v_flat = v.contiguous().reshape(v.shape[0], -1) # [S_local, v_feat]
|
||||
k_feat_size = k_flat.shape[-1]
|
||||
kv_flat = torch.cat([k_flat, v_flat], dim=-1) # [S_local, k_feat + v_feat]
|
||||
|
||||
kv_full = cp_all_gather_rerange_kv_cache(
|
||||
kv_flat, cp_size, forward_batch, get_current_device_stream_fast()
|
||||
) # [S_full, k_feat + v_feat]
|
||||
|
||||
key_cache_full = kv_full[..., :k_feat_size].reshape(-1, *k_tail)
|
||||
value_cache_full = kv_full[..., k_feat_size:].reshape(-1, *v_tail)
|
||||
|
||||
token_to_kv_pool.set_kv_buffer(
|
||||
cp_strategy.materialize_full_kv(
|
||||
forward_batch,
|
||||
layer,
|
||||
KVWriteLoc(cache_loc, swa_loc),
|
||||
key_cache_full,
|
||||
value_cache_full,
|
||||
k,
|
||||
v,
|
||||
swa_loc=swa_loc,
|
||||
)
|
||||
|
||||
|
||||
@@ -1380,70 +1374,51 @@ class AscendAttnBackend(AttentionBackend):
|
||||
The KV cache must already contain the full gathered sequence
|
||||
(written by _cp_allgather_and_save_kv_npu before this call).
|
||||
|
||||
Args:
|
||||
q: Query tensor, shape [total_q_tokens, tp_q_head_num * qk_head_dim]
|
||||
k_cache: Full key cache from token_to_kv_pool
|
||||
v_cache: Full value cache from token_to_kv_pool
|
||||
layer: RadixAttention layer
|
||||
forward_batch: ForwardBatch with attn_cp_metadata populated
|
||||
|
||||
Returns:
|
||||
attn_output [total_q_tokens, tp_q_head_num * v_head_dim]
|
||||
Token slicing into the prev/next zigzag chunks, pad-row trimming and
|
||||
pad-row re-appending are owned by the CP-v2 strategy
|
||||
(``ZigzagCPStrategy.run_attention``); this backend only supplies the FIA
|
||||
computation closure. ``attention_backend`` is borrowed from
|
||||
``CPAttentionBackendKind.FLASH_ATTENTION`` because FIA uses the same
|
||||
two-call (prev + next) dispatch convention.
|
||||
"""
|
||||
cp_meta = forward_batch.attn_cp_metadata
|
||||
|
||||
# Local tokens are laid out [all_seqs_prev, all_seqs_next]; split at
|
||||
# total_q_prev_tokens rather than the midpoint to support bs > 1.
|
||||
split = cp_meta.total_q_prev_tokens
|
||||
q_prev = (
|
||||
q[:split].contiguous().reshape(-1, layer.tp_q_head_num, layer.qk_head_dim)
|
||||
)
|
||||
q_next = (
|
||||
q[split:].contiguous().reshape(-1, layer.tp_q_head_num, layer.qk_head_dim)
|
||||
)
|
||||
def _fia_cp_attn(q_chunk, cu_seqlens_q_cp, cache_seqlens_cp, max_seqlen_q_cp):
|
||||
# The strategy passes cu_seqlens_q (with a leading 0) and plain KV
|
||||
# lengths as int32 tensors; FIA wants cumulative query lengths
|
||||
# (no leading 0) and KV lengths as Python lists.
|
||||
k_cache_paged = k_cache.view(
|
||||
-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim
|
||||
)
|
||||
v_cache_paged = v_cache.view(
|
||||
-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim
|
||||
)
|
||||
return torch_npu.npu_fused_infer_attention_score(
|
||||
query=q_chunk.contiguous(),
|
||||
key=k_cache_paged,
|
||||
value=v_cache_paged,
|
||||
block_table=self.forward_metadata.block_tables,
|
||||
block_size=self.page_size,
|
||||
num_heads=layer.tp_q_head_num,
|
||||
num_key_value_heads=layer.tp_k_head_num,
|
||||
input_layout="TND",
|
||||
atten_mask=self.fia_mask,
|
||||
sparse_mode=3,
|
||||
next_tokens=0,
|
||||
scale=layer.scaling,
|
||||
actual_seq_lengths=cu_seqlens_q_cp[1:].tolist(),
|
||||
actual_seq_lengths_kv=cache_seqlens_cp.tolist(),
|
||||
)[0]
|
||||
|
||||
k_cache_paged = k_cache.view(
|
||||
-1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim
|
||||
cp_strategy = get_cp_strategy()
|
||||
assert cp_strategy is not None
|
||||
q_cp = q.contiguous().view(-1, layer.tp_q_head_num, layer.qk_head_dim)
|
||||
attn_out = cp_strategy.run_attention(
|
||||
q_cp,
|
||||
forward_batch,
|
||||
self.device,
|
||||
_fia_cp_attn,
|
||||
attention_backend=CPAttentionBackendKind.FLASH_ATTENTION,
|
||||
)
|
||||
v_cache_paged = v_cache.view(
|
||||
-1, self.page_size, layer.tp_v_head_num * layer.v_head_dim
|
||||
)
|
||||
|
||||
attn_out_prev, _ = torch.ops.npu.npu_fused_infer_attention_score(
|
||||
q_prev,
|
||||
k_cache_paged,
|
||||
v_cache_paged,
|
||||
block_table=self.forward_metadata.block_tables,
|
||||
block_size=self.page_size,
|
||||
num_heads=layer.tp_q_head_num,
|
||||
num_key_value_heads=layer.tp_k_head_num,
|
||||
input_layout="TND",
|
||||
atten_mask=self.fia_mask,
|
||||
sparse_mode=3,
|
||||
next_tokens=0,
|
||||
scale=layer.scaling,
|
||||
actual_seq_lengths=np.cumsum(cp_meta.actual_seq_q_prev_list).tolist(),
|
||||
actual_seq_lengths_kv=cp_meta.kv_len_prev_list,
|
||||
)
|
||||
|
||||
attn_out_next, _ = torch.ops.npu.npu_fused_infer_attention_score(
|
||||
q_next,
|
||||
k_cache_paged,
|
||||
v_cache_paged,
|
||||
block_table=self.forward_metadata.block_tables,
|
||||
block_size=self.page_size,
|
||||
num_heads=layer.tp_q_head_num,
|
||||
num_key_value_heads=layer.tp_k_head_num,
|
||||
input_layout="TND",
|
||||
atten_mask=self.fia_mask,
|
||||
sparse_mode=3,
|
||||
next_tokens=0,
|
||||
scale=layer.scaling,
|
||||
actual_seq_lengths=np.cumsum(cp_meta.actual_seq_q_next_list).tolist(),
|
||||
actual_seq_lengths_kv=cp_meta.kv_len_next_list,
|
||||
)
|
||||
|
||||
attn_out = torch.cat([attn_out_prev, attn_out_next], dim=0)
|
||||
return attn_out.view(-1, layer.tp_q_head_num * layer.v_head_dim)
|
||||
|
||||
def forward_sparse(
|
||||
|
||||
Reference in New Issue
Block a user