[Fix] [NPU] glm5.2: support pcp v2 with interleave & zigzag (#40165)

This commit is contained in:
gjsheu
2026-10-20 21:26:30 +08:00
committed by GitHub
parent ee2f95a522
commit aa91108f64
3 changed files with 158 additions and 101 deletions
@@ -10,7 +10,7 @@ from sgl_kernel_npu.attention.sinks_attention import (
attention_sinks_triton,
)
from sglang.srt.configs.model_config import AttentionArch
from sglang.srt.configs.model_config import AttentionArch, is_deepseek_dsa
from sglang.srt.dllm.config import DllmConfig
from sglang.srt.environ import envs
from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import (
@@ -30,6 +30,7 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
)
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
from sglang.srt.layers.cp.base import get_cp_strategy
from sglang.srt.layers.dcp.layout import (
get_dcp_chain_spec_lens,
get_dcp_lens,
@@ -329,6 +330,9 @@ class AscendAttnBackend(AttentionBackend):
self.model_dtype = model_runner.model_config.dtype
self.kv_cache_dtype = model_runner.kv_cache_dtype
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
# Only DSA models (GLM-5.2, DeepSeek-V3-DSA, ...) consume the per-token DSA
# CP metadata built by prepare_dsa_cp_metadata; V4 keeps its own variant.
self.use_dsa = is_deepseek_dsa(model_runner.model_config.hf_config)
if self.use_mla:
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim
@@ -1212,6 +1216,118 @@ class AscendAttnBackend(AttentionBackend):
attn_output = attn_output.view(-1, num_heads * head_size)
return attn_output
def prepare_dsa_cp_metadata(self, forward_batch: ForwardBatch) -> None:
if getattr(forward_batch, "dsa_cp_metadata_prepared", False):
return
if not self.use_dsa:
return
if getattr(forward_batch, "attn_cp_metadata", None) is None:
return
if not forward_batch.forward_mode.is_context_parallel_extend():
return
strategy = get_cp_strategy()
if strategy is None or strategy.cp_size <= 1:
return
fm = self.forward_metadata
global_positions = forward_batch.positions
if global_positions is None:
return
device = global_positions.device
num_tokens = int(global_positions.shape[0])
local_idx = strategy.local_q_indices(num_tokens, forward_batch).to(
device=device, dtype=torch.long
)
if local_idx.numel() > 0:
shard_bound = int(
getattr(forward_batch.attn_cp_metadata, "total_seq_lens", num_tokens)
)
local_idx = local_idx[local_idx < shard_bound]
local_positions = global_positions.index_select(0, local_idx)
from sglang.srt.layers.cp.padding import pad_local_rows
local_positions = pad_local_rows(
local_positions, forward_batch.attn_cp_metadata, dim=0
)
# Per-token request ids for the local rows: the Ascend paged-attention
# operators consume per-token block-table rows under CP (same
# convention as prepare_dsv4_cp_metadata).
extend_lens = getattr(forward_batch, "extend_seq_lens_cpu", None)
if extend_lens is None:
seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None)
if seq_lens_cpu is not None:
extend_lens = seq_lens_cpu.tolist()
else:
extend_lens = [num_tokens]
extend_lens = [int(x) for x in extend_lens]
real_num_tokens = min(sum(extend_lens), num_tokens)
batch_ids_parts = []
for batch_id, length in enumerate(extend_lens):
if length <= 0:
continue
batch_ids_parts.append(
torch.full((length,), batch_id, dtype=torch.long, device=device)
)
if batch_ids_parts:
batch_ids = torch.cat(batch_ids_parts, dim=0)
else:
batch_ids = torch.empty(0, dtype=torch.long, device=device)
if batch_ids.shape[0] < num_tokens:
batch_ids = torch.cat(
[
batch_ids,
torch.zeros(
num_tokens - batch_ids.shape[0],
dtype=torch.long,
device=device,
),
]
)
elif batch_ids.shape[0] > num_tokens:
batch_ids = batch_ids[:num_tokens]
local_batch_ids = batch_ids.index_select(0, local_idx)
valid_rows = local_idx < real_num_tokens
pad_rows = local_positions.shape[0] - local_batch_ids.shape[0]
if pad_rows > 0:
local_batch_ids = torch.cat(
[local_batch_ids, local_batch_ids.new_zeros(pad_rows)]
)
valid_rows = torch.cat([valid_rows, valid_rows.new_zeros(pad_rows)])
def _select_rows(table: Optional[torch.Tensor]):
if table is None:
return None
if local_batch_ids.numel() == 0:
return table.new_empty((0, *table.shape[1:]))
return table.index_select(0, local_batch_ids)
fm.block_tables = _select_rows(fm.block_tables)
local_t = int(local_positions.shape[0])
fm.actual_seq_lengths_q = torch.arange(
1, local_t + 1, dtype=torch.int32, device=device
)
# KV length seen by each local token = its global position + 1 (causal
# DSA); pad rows (position 0) stay at 1 so the sparse kernel sees a
# valid length for every row.
fm.actual_seq_lengths_kv = torch.where(
valid_rows,
local_positions.to(torch.int32) + 1,
torch.ones_like(local_positions, dtype=torch.int32),
).clamp(min=1)
forward_batch.dsa_cp_local_positions = local_positions
# Keep forward_batch.positions aligned with the rank-local shard so any
# downstream reader sees the local rows (parity with the model hook).
forward_batch.positions = local_positions
forward_batch.dsa_cp_metadata_prepared = True
def do_cp_balance_attn(
self,
q_nope,
@@ -1223,34 +1339,21 @@ class AscendAttnBackend(AttentionBackend):
actual_seq_qlen,
actual_seq_lengths_kv,
):
seq_len = q_nope.shape[0]
split_len = (seq_len + 1) // 2
q_nope_prev, q_nope_next = torch.split(q_nope, split_len, dim=0)
q_rope_prev, q_rope_next = torch.split(q_pe, split_len, dim=0)
q_nope_prev = q_nope_prev.contiguous()
q_nope_next = q_nope_next.contiguous()
q_rope_prev = q_rope_prev.contiguous()
q_rope_next = q_rope_next.contiguous()
# V2: Q is already sharded to local by the strategy; single sparse-attn
# call against the full (gathered) KV, no prev/next balance split.
topk_indices = _expand_dsa_sparse_indices(topk_indices)
topk_indices_prev, topk_indices_next = torch.split(
topk_indices, split_len, dim=0
)
actual_seq_qlen_prev, actual_seq_qlen_next = actual_seq_qlen
actual_seq_lengths_kv_prev, actual_seq_lengths_kv_next = actual_seq_lengths_kv
attn_out_prev, _, _ = torch_npu.npu_sparse_flash_attention(
query=q_nope_prev,
attn_out, _, _ = torch_npu.npu_sparse_flash_attention(
query=q_nope,
key=k_nope,
value=k_nope,
query_rope=q_rope_prev,
query_rope=q_pe,
key_rope=k_pe,
sparse_indices=topk_indices_prev,
sparse_indices=topk_indices,
scale_value=layer.scaling,
actual_seq_lengths_query=actual_seq_qlen_prev.to(
actual_seq_lengths_query=actual_seq_qlen.to(
device=q_nope.device, dtype=torch.int32
),
actual_seq_lengths_kv=actual_seq_lengths_kv_prev.to(
actual_seq_lengths_kv=actual_seq_lengths_kv.to(
device=q_nope.device, dtype=torch.int32
),
block_table=self.forward_metadata.block_tables,
@@ -1261,29 +1364,7 @@ class AscendAttnBackend(AttentionBackend):
attention_mode=2,
return_softmax_lse=False,
)
attn_out_next, _, _ = torch_npu.npu_sparse_flash_attention(
query=q_nope_next,
key=k_nope,
value=k_nope,
query_rope=q_rope_next,
key_rope=k_pe,
sparse_indices=topk_indices_next,
scale_value=layer.scaling,
actual_seq_lengths_query=actual_seq_qlen_next.to(
device=q_nope.device, dtype=torch.int32
),
actual_seq_lengths_kv=actual_seq_lengths_kv_next.to(
device=q_nope.device, dtype=torch.int32
),
block_table=self.forward_metadata.block_tables,
sparse_block_size=1,
layout_query="TND",
layout_kv="PA_BSND",
sparse_mode=3,
attention_mode=2,
return_softmax_lse=False,
)
return torch.cat([attn_out_prev, attn_out_next], dim=0)
return attn_out
def do_cp_attn_fia(
self,
@@ -3350,7 +3431,7 @@ class AscendAttnBackend(AttentionBackend):
attn_output = attn_output[:, :, : layer.tp_q_head_num, :]
else:
assert (
self.graph_mode == False
not self.graph_mode
) # _npu_paged_attention_mla not support graph mode
if q_rope is not None:
q = torch.cat([q, q_rope], dim=-1)
@@ -64,6 +64,12 @@ class DSANPUIndexerMixin:
input_on_attn_tp_slices: bool = False,
dynamic_scale: torch.Tensor = None,
) -> torch.Tensor:
# DSA prefill CP metadata is Ascend-local and is first consumed by this
# indexer (before the attention backend's forward_extend), so prepare it
# here rather than in the shared eager runner.
attn_backend = get_attn_backend()
if hasattr(attn_backend, "prepare_dsa_cp_metadata"):
attn_backend.prepare_dsa_cp_metadata(forward_batch)
if get_attn_backend().forward_metadata.seq_lens_cpu_int is None:
actual_seq_lengths_kv = get_attn_backend().forward_metadata.seq_lens
else:
@@ -238,28 +244,7 @@ class DSANPUIndexerMixin:
self.dsa_enable_prefill_cp
and forward_batch.attn_cp_metadata is not None
):
get_attn_backend().forward_metadata.actual_seq_lengths_q = (
forward_batch.attn_cp_metadata.actual_seq_q_prev_tensor,
forward_batch.attn_cp_metadata.actual_seq_q_next_tensor,
)
if sum(forward_batch.extend_prefix_lens_cpu) > 0:
total_kv_len_prev_tensor = (
forward_batch.attn_cp_metadata.kv_len_prev_tensor
+ forward_batch.extend_prefix_lens.squeeze()
)
total_kv_len_next_tensor = (
forward_batch.attn_cp_metadata.kv_len_next_tensor
+ forward_batch.extend_prefix_lens.squeeze()
)
get_attn_backend().forward_metadata.actual_seq_lengths_kv = (
total_kv_len_prev_tensor,
total_kv_len_next_tensor,
)
else:
get_attn_backend().forward_metadata.actual_seq_lengths_kv = (
forward_batch.attn_cp_metadata.kv_len_prev_tensor,
forward_batch.attn_cp_metadata.kv_len_next_tensor,
)
# V2: local metadata already prepared by prepare_dsa_cp_metadata.
actual_seq_lengths_q = (
get_attn_backend().forward_metadata.actual_seq_lengths_q
)
@@ -308,7 +293,6 @@ class DSANPUIndexerMixin:
and self.dsa_enable_prefill_cp
and forward_batch.attn_cp_metadata is not None
):
block_table = block_table[: actual_seq_lengths_q[0].numel()]
topk_indices = self.do_npu_cp_balance_indexer(
q.view(-1, self.n_heads, self.head_dim),
past_key_states,
@@ -377,26 +361,19 @@ class DSANPUIndexerMixin:
actual_seq_lengths_kv,
block_table,
):
q_prev, q_next = torch.split(q, (q.size(0) + 1) // 2, dim=0)
weights_prev, weights_next = None, None
# V2: Q is local (sharded); single indexer call against full KV.
if indexer_weights is not None:
weights_prev, weights_next = torch.split(
indexer_weights, (indexer_weights.size(0) + 1) // 2, dim=0
indexer_weights = indexer_weights.contiguous().view(
-1, indexer_weights.shape[-1]
)
weights_prev = weights_prev.contiguous().view(-1, weights_prev.shape[-1])
weights_next = weights_next.contiguous().view(-1, weights_next.shape[-1])
actual_seq_lengths_q_prev, actual_seq_lengths_q_next = actual_seq_lengths_q
actual_seq_lengths_kv_prev, actual_seq_lengths_kv_next = actual_seq_lengths_kv
topk_indices_prev = torch_npu.npu_lightning_indexer(
query=q_prev,
topk_indices = torch_npu.npu_lightning_indexer(
query=q,
key=past_key_states,
weights=weights_prev,
actual_seq_lengths_query=actual_seq_lengths_q_prev.to(
weights=indexer_weights,
actual_seq_lengths_query=actual_seq_lengths_q.to(
device=q.device, dtype=torch.int32
),
actual_seq_lengths_key=actual_seq_lengths_kv_prev.to(
actual_seq_lengths_key=actual_seq_lengths_kv.to(
device=q.device, dtype=torch.int32
),
block_table=block_table,
@@ -405,23 +382,7 @@ class DSANPUIndexerMixin:
sparse_count=self.index_topk,
sparse_mode=3,
)
topk_indices_next = torch_npu.npu_lightning_indexer(
query=q_next,
key=past_key_states,
weights=weights_next,
actual_seq_lengths_query=actual_seq_lengths_q_next.to(
device=q.device, dtype=torch.int32
),
actual_seq_lengths_key=actual_seq_lengths_kv_next.to(
device=q.device, dtype=torch.int32
),
block_table=block_table,
layout_query="TND",
layout_key="PA_BSND",
sparse_count=self.index_topk,
sparse_mode=3,
)
return torch.cat([topk_indices_prev[0], topk_indices_next[0]], dim=0).squeeze(1)
return topk_indices[0].squeeze(1)
def scattered_to_tp_attn_full(
+15
View File
@@ -81,6 +81,7 @@ from sglang.srt.layers.aux_hidden_states import (
AuxHiddenStatePacker,
)
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
from sglang.srt.layers.cp.utils import cp_gather_full_sequence_states
from sglang.srt.layers.dcp.planner import (
prepare_decode_context_parallel_metadata,
)
@@ -2366,6 +2367,20 @@ class DeepseekV2AttentionMLA(
else:
state.hidden_states_after_attn = result
def rebuild_cp_kv_cache(self, latent_cache, forward_batch, k_nope, k_pe):
# CP V2: gather the rank-local latent KV into full sequence order
# (the Ascend sparse attention reads the full KV pool).
latent_cache[..., : self.kv_lora_rank] = k_nope.squeeze(1)
latent_cache[..., self.kv_lora_rank :] = k_pe.squeeze(1)
latent_cache_output = cp_gather_full_sequence_states(
latent_cache.contiguous(),
forward_batch,
torch.npu.current_stream(),
)
k_nope = latent_cache_output[..., : self.kv_lora_rank].unsqueeze(1)
k_pe = latent_cache_output[..., self.kv_lora_rank :].unsqueeze(1)
return k_nope, k_pe
def forward(
self,
positions: torch.Tensor,