From a9902f47e392ad544a8a521bc4075190c57bf74a Mon Sep 17 00:00:00 2001 From: gjsheu Date: Thu, 22 Oct 2026 20:54:11 +0800 Subject: [PATCH] [Fix] [NPU] qwen: adapt pcp-v2 (#41564) --- .../npu/attention/ascend_backend.py | 143 +++++------- .../ascend/test_npu_prefill_cp_backend.py | 211 ++++++++++++++++++ 2 files changed, 270 insertions(+), 84 deletions(-) create mode 100644 test/manual/ascend/test_npu_prefill_cp_backend.py diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index c4432c357b..7694724117 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -30,13 +30,12 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import ( ) from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp -from sglang.srt.layers.cp.base import get_cp_strategy +from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy from sglang.srt.layers.dcp.layout import ( get_dcp_chain_spec_lens, get_dcp_lens, ) from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache from sglang.srt.mem_cache.memory_pool import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode @@ -48,7 +47,6 @@ from sglang.srt.runtime_context import ( from sglang.srt.speculative.spec_info import SpecInput, SpecInputType from sglang.srt.utils import ( get_bool_env_var, - get_current_device_stream_fast, next_power_of_2, ) @@ -290,29 +288,25 @@ def _cp_allgather_and_save_kv_npu( if not layer.is_cross_attention else forward_batch.encoder_out_cache_loc ) - # Save original trailing shapes for reshape after gather. - k_tail = k.shape[1:] # (tp_k_head_num, qk_head_dim) - v_tail = v.shape[1:] # (tp_v_head_num, v_head_dim) + # CP v2: all-gather the rank-local K/V through the active CP strategy and + # write the full sequence to the KV pool. Same contract as the + # `cp_active` branch of flashattention_backend. + cp_strategy = get_cp_strategy() + if cp_strategy is None or cp_strategy.cp_size <= 1: + token_to_kv_pool.set_kv_buffer( + layer, + KVWriteLoc(cache_loc, swa_loc), + k, + v, + ) + return - # Flatten trailing dims then concat → one all-gather instead of two. - # Works for GQA where tp_k_head_num != tp_v_head_num. - k_flat = k.contiguous().reshape(k.shape[0], -1) # [S_local, k_feat] - v_flat = v.contiguous().reshape(v.shape[0], -1) # [S_local, v_feat] - k_feat_size = k_flat.shape[-1] - kv_flat = torch.cat([k_flat, v_flat], dim=-1) # [S_local, k_feat + v_feat] - - kv_full = cp_all_gather_rerange_kv_cache( - kv_flat, cp_size, forward_batch, get_current_device_stream_fast() - ) # [S_full, k_feat + v_feat] - - key_cache_full = kv_full[..., :k_feat_size].reshape(-1, *k_tail) - value_cache_full = kv_full[..., k_feat_size:].reshape(-1, *v_tail) - - token_to_kv_pool.set_kv_buffer( + cp_strategy.materialize_full_kv( + forward_batch, layer, - KVWriteLoc(cache_loc, swa_loc), - key_cache_full, - value_cache_full, + k, + v, + swa_loc=swa_loc, ) @@ -1380,70 +1374,51 @@ class AscendAttnBackend(AttentionBackend): The KV cache must already contain the full gathered sequence (written by _cp_allgather_and_save_kv_npu before this call). - Args: - q: Query tensor, shape [total_q_tokens, tp_q_head_num * qk_head_dim] - k_cache: Full key cache from token_to_kv_pool - v_cache: Full value cache from token_to_kv_pool - layer: RadixAttention layer - forward_batch: ForwardBatch with attn_cp_metadata populated - - Returns: - attn_output [total_q_tokens, tp_q_head_num * v_head_dim] + Token slicing into the prev/next zigzag chunks, pad-row trimming and + pad-row re-appending are owned by the CP-v2 strategy + (``ZigzagCPStrategy.run_attention``); this backend only supplies the FIA + computation closure. ``attention_backend`` is borrowed from + ``CPAttentionBackendKind.FLASH_ATTENTION`` because FIA uses the same + two-call (prev + next) dispatch convention. """ - cp_meta = forward_batch.attn_cp_metadata - # Local tokens are laid out [all_seqs_prev, all_seqs_next]; split at - # total_q_prev_tokens rather than the midpoint to support bs > 1. - split = cp_meta.total_q_prev_tokens - q_prev = ( - q[:split].contiguous().reshape(-1, layer.tp_q_head_num, layer.qk_head_dim) - ) - q_next = ( - q[split:].contiguous().reshape(-1, layer.tp_q_head_num, layer.qk_head_dim) - ) + def _fia_cp_attn(q_chunk, cu_seqlens_q_cp, cache_seqlens_cp, max_seqlen_q_cp): + # The strategy passes cu_seqlens_q (with a leading 0) and plain KV + # lengths as int32 tensors; FIA wants cumulative query lengths + # (no leading 0) and KV lengths as Python lists. + k_cache_paged = k_cache.view( + -1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim + ) + v_cache_paged = v_cache.view( + -1, self.page_size, layer.tp_v_head_num * layer.v_head_dim + ) + return torch_npu.npu_fused_infer_attention_score( + query=q_chunk.contiguous(), + key=k_cache_paged, + value=v_cache_paged, + block_table=self.forward_metadata.block_tables, + block_size=self.page_size, + num_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="TND", + atten_mask=self.fia_mask, + sparse_mode=3, + next_tokens=0, + scale=layer.scaling, + actual_seq_lengths=cu_seqlens_q_cp[1:].tolist(), + actual_seq_lengths_kv=cache_seqlens_cp.tolist(), + )[0] - k_cache_paged = k_cache.view( - -1, self.page_size, layer.tp_k_head_num * layer.qk_head_dim + cp_strategy = get_cp_strategy() + assert cp_strategy is not None + q_cp = q.contiguous().view(-1, layer.tp_q_head_num, layer.qk_head_dim) + attn_out = cp_strategy.run_attention( + q_cp, + forward_batch, + self.device, + _fia_cp_attn, + attention_backend=CPAttentionBackendKind.FLASH_ATTENTION, ) - v_cache_paged = v_cache.view( - -1, self.page_size, layer.tp_v_head_num * layer.v_head_dim - ) - - attn_out_prev, _ = torch.ops.npu.npu_fused_infer_attention_score( - q_prev, - k_cache_paged, - v_cache_paged, - block_table=self.forward_metadata.block_tables, - block_size=self.page_size, - num_heads=layer.tp_q_head_num, - num_key_value_heads=layer.tp_k_head_num, - input_layout="TND", - atten_mask=self.fia_mask, - sparse_mode=3, - next_tokens=0, - scale=layer.scaling, - actual_seq_lengths=np.cumsum(cp_meta.actual_seq_q_prev_list).tolist(), - actual_seq_lengths_kv=cp_meta.kv_len_prev_list, - ) - - attn_out_next, _ = torch.ops.npu.npu_fused_infer_attention_score( - q_next, - k_cache_paged, - v_cache_paged, - block_table=self.forward_metadata.block_tables, - block_size=self.page_size, - num_heads=layer.tp_q_head_num, - num_key_value_heads=layer.tp_k_head_num, - input_layout="TND", - atten_mask=self.fia_mask, - sparse_mode=3, - next_tokens=0, - scale=layer.scaling, - actual_seq_lengths=np.cumsum(cp_meta.actual_seq_q_next_list).tolist(), - actual_seq_lengths_kv=cp_meta.kv_len_next_list, - ) - - attn_out = torch.cat([attn_out_prev, attn_out_next], dim=0) return attn_out.view(-1, layer.tp_q_head_num * layer.v_head_dim) def forward_sparse( diff --git a/test/manual/ascend/test_npu_prefill_cp_backend.py b/test/manual/ascend/test_npu_prefill_cp_backend.py new file mode 100644 index 0000000000..0d06b21f4b --- /dev/null +++ b/test/manual/ascend/test_npu_prefill_cp_backend.py @@ -0,0 +1,211 @@ +""" +Unit tests for the NPU prefill context parallel paths in +sglang.srt.hardware_backend.npu.attention.ascend_backend. + +These guard the two defects left behind when the CP v1 runtime was removed: + +* the KV all-gather for prefill CP must go through the CP v2 strategy, not + through the deleted ``cp_all_gather_rerange_kv_cache`` shim; +* ``do_cp_attn_fia`` must hand FIA a query whose row count matches the + cumulative ``actual_seq_lengths`` it advertises -- the model runner pads the + CP shard to ``per_rank_actual_token[0]`` rows, which previously made FIA + tiling fail with error 561002. + +Everything runs on CPU tensors, so the checks are hardware independent. +""" + +import sys +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +# Mock NPU-only modules before importing the source module. +for _ in ( + "torch_npu", + "torch_npu.contrib", + "sgl_kernel_npu", + "sgl_kernel_npu.attention", + "sgl_kernel_npu.attention.sinks_attention", + "sglang.srt.speculative", + "sglang.srt.speculative.decoupled_spec_io", + "sglang.srt.speculative.spec_info", + "sglang.srt.speculative.eagle_info", +): + sys.modules.setdefault(_, unittest.mock.MagicMock()) + +from sglang.srt.hardware_backend.npu.attention import ascend_backend +from sglang.srt.hardware_backend.npu.attention.ascend_backend import ( + AscendAttnBackend, + _cp_allgather_and_save_kv_npu, +) +from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy + + +class _RecordingPool: + def __init__(self): + self.calls = [] + + def set_kv_buffer(self, layer, loc, k, v, *args, **kwargs): + self.calls.append((tuple(k.shape), tuple(v.shape))) + + +class _RecordingStrategy: + def __init__(self, cp_size=2): + self.cp_size = cp_size + self.kv_calls = [] + + def materialize_full_kv(self, forward_batch, layer, k, v, swa_loc=None): + self.kv_calls.append((tuple(k.shape), tuple(v.shape))) + + +def _cp_batch(out_cache_loc): + return SimpleNamespace(out_cache_loc=out_cache_loc, encoder_out_cache_loc=None) + + +def _attn_layer(): + return SimpleNamespace(is_cross_attention=False, k_scale=None, v_scale=None) + + +def _fia_layer(): + return SimpleNamespace( + tp_q_head_num=16, + tp_k_head_num=8, + tp_v_head_num=8, + qk_head_dim=128, + v_head_dim=128, + scaling=0.08838834764831845, + ) + + +class TestCPKVWrite(unittest.TestCase): + """`_cp_allgather_and_save_kv_npu` must use the CP v2 strategy API.""" + + def setUp(self): + self.k = torch.zeros(3, 8, 128, dtype=torch.bfloat16) + self.v = torch.zeros(3, 8, 128, dtype=torch.bfloat16) + self.batch = _cp_batch(torch.arange(6)) + self.layer = _attn_layer() + + def test_uses_cp_v2_strategy(self): + strategy = _RecordingStrategy(cp_size=2) + pool = _RecordingPool() + with patch.object(ascend_backend, "get_cp_strategy", return_value=strategy): + _cp_allgather_and_save_kv_npu( + self.batch, self.layer, self.k, self.v, 2, pool, swa_loc=None + ) + self.assertEqual(len(strategy.kv_calls), 1) + self.assertEqual(strategy.kv_calls[0], ((3, 8, 128), (3, 8, 128))) + # the strategy owns the pool write; the helper must not bypass it + self.assertEqual(pool.calls, []) + + def test_falls_back_to_plain_write_without_cp(self): + strategy = _RecordingStrategy(cp_size=1) + pool = _RecordingPool() + with patch.object(ascend_backend, "get_cp_strategy", return_value=strategy): + _cp_allgather_and_save_kv_npu( + self.batch, self.layer, self.k, self.v, 1, pool, swa_loc=None + ) + self.assertEqual(strategy.kv_calls, []) + self.assertEqual(pool.calls, [((3, 8, 128), (3, 8, 128))]) + + def test_deleted_cp_v1_helper_is_not_referenced(self): + with open(ascend_backend.__file__) as handle: + source = handle.read() + self.assertNotIn("cp_all_gather_rerange_kv_cache", source) + + +class TestDoCpAttnFia(unittest.TestCase): + """`do_cp_attn_fia` delegates CP slicing to the strategy and only supplies FIA. + + The ZigzagCPStrategy.run_attention owns prev/next slicing, concatenation and + pad-row re-appending; the backend's job is to hand it an FIA computation + closure and translate the strategy's cu_seqlens / KV-length tensors into the + lists FIA expects. + """ + + PHYSICAL_ROWS = 4 + + def setUp(self): + self.backend = AscendAttnBackend.__new__(AscendAttnBackend) + self.backend.page_size = 128 + self.backend.device = torch.device("cpu") + self.backend.fia_mask = torch.zeros(2048, 2048, dtype=torch.bool) + self.backend.forward_metadata = SimpleNamespace( + block_tables=torch.zeros(1, 1, dtype=torch.int32) + ) + self.layer = _fia_layer() + self.q = torch.randn(self.PHYSICAL_ROWS, 16 * 128, dtype=torch.bfloat16) + self.k_cache = torch.zeros(2163 * 128, 8 * 128, dtype=torch.bfloat16) + self.v_cache = torch.zeros(2163 * 128, 8 * 128, dtype=torch.bfloat16) + self.calls = [] + + def _fia(self, query, key, value, **kwargs): + record = dict(kwargs) + record["query"] = query + self.calls.append(record) + return ( + torch.zeros(query.shape[0], query.shape[1], 128, dtype=query.dtype), + None, + ) + + def _meta(self, prev, next_len): + # single-batch CP metadata with the tensor fields run_attention reads + return SimpleNamespace( + total_q_prev_tokens=prev, + total_q_next_tokens=next_len, + cu_seqlens_q_prev_tensor=torch.tensor([0, prev], dtype=torch.int32), + kv_len_prev_tensor=torch.tensor([prev], dtype=torch.int32), + max_seqlen_q_prev=prev, + cu_seqlens_q_next_tensor=torch.tensor([0, next_len], dtype=torch.int32), + kv_len_next_tensor=torch.tensor([prev + next_len], dtype=torch.int32), + max_seqlen_q_next=next_len, + ) + + def _run(self, prev, next_len): + forward_batch = SimpleNamespace(attn_cp_metadata=self._meta(prev, next_len)) + strategy = ZigzagCPStrategy(cp_size=2) + with patch.object(ascend_backend, "get_cp_strategy", return_value=strategy): + with patch.object( + ascend_backend.torch_npu, + "npu_fused_infer_attention_score", + self._fia, + ): + return self.backend.do_cp_attn_fia( + self.q, self.k_cache, self.v_cache, self.layer, forward_batch + ) + + def test_slices_into_prev_and_next_chunks(self): + # logical tokens 2 + 1 = 3 of the 4 physical rows; strategy slices them + self._run(prev=2, next_len=1) + self.assertEqual(len(self.calls), 2) + self.assertEqual(self.calls[0]["query"].shape[0], 2) # prev chunk + self.assertEqual(self.calls[1]["query"].shape[0], 1) # next chunk + + def test_cu_seqlens_translated_to_fia_actual_seq_lengths(self): + self._run(prev=2, next_len=1) + # strategy passes cu_seqlens_q [0, prev]; FIA wants cumsum without the + # leading zero + self.assertEqual(self.calls[0]["actual_seq_lengths"], [2]) + self.assertEqual(self.calls[1]["actual_seq_lengths"], [1]) + + def test_kv_lengths_forwarded(self): + self._run(prev=2, next_len=1) + self.assertEqual(self.calls[0]["actual_seq_lengths_kv"], [2]) + self.assertEqual(self.calls[1]["actual_seq_lengths_kv"], [3]) + + def test_output_keeps_physical_row_count(self): + out = self._run(prev=2, next_len=1) + self.assertEqual(out.shape[0], self.PHYSICAL_ROWS) + self.assertEqual(out.shape[1], 16 * 128) + + def test_no_padding_keeps_all_rows(self): + # 2 + 2 == physical rows: strategy does not pad + self._run(prev=2, next_len=2) + self.assertEqual(self.calls[0]["query"].shape[0], 2) + self.assertEqual(self.calls[1]["query"].shape[0], 2) + + +if __name__ == "__main__": + unittest.main()