[Fix] [NPU] glm5.2: support pcp v2 with interleave & zigzag (#40165)
This commit is contained in:
@@ -10,7 +10,7 @@ from sgl_kernel_npu.attention.sinks_attention import (
|
||||
attention_sinks_triton,
|
||||
)
|
||||
|
||||
from sglang.srt.configs.model_config import AttentionArch
|
||||
from sglang.srt.configs.model_config import AttentionArch, is_deepseek_dsa
|
||||
from sglang.srt.dllm.config import DllmConfig
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import (
|
||||
@@ -30,6 +30,7 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import (
|
||||
)
|
||||
from sglang.srt.layers.attention.base_attn_backend import AttentionBackend
|
||||
from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp
|
||||
from sglang.srt.layers.cp.base import get_cp_strategy
|
||||
from sglang.srt.layers.dcp.layout import (
|
||||
get_dcp_chain_spec_lens,
|
||||
get_dcp_lens,
|
||||
@@ -329,6 +330,9 @@ class AscendAttnBackend(AttentionBackend):
|
||||
self.model_dtype = model_runner.model_config.dtype
|
||||
self.kv_cache_dtype = model_runner.kv_cache_dtype
|
||||
self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA
|
||||
# Only DSA models (GLM-5.2, DeepSeek-V3-DSA, ...) consume the per-token DSA
|
||||
# CP metadata built by prepare_dsa_cp_metadata; V4 keeps its own variant.
|
||||
self.use_dsa = is_deepseek_dsa(model_runner.model_config.hf_config)
|
||||
if self.use_mla:
|
||||
self.kv_lora_rank = model_runner.model_config.kv_lora_rank
|
||||
self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim
|
||||
@@ -1212,6 +1216,118 @@ class AscendAttnBackend(AttentionBackend):
|
||||
attn_output = attn_output.view(-1, num_heads * head_size)
|
||||
return attn_output
|
||||
|
||||
def prepare_dsa_cp_metadata(self, forward_batch: ForwardBatch) -> None:
|
||||
if getattr(forward_batch, "dsa_cp_metadata_prepared", False):
|
||||
return
|
||||
if not self.use_dsa:
|
||||
return
|
||||
if getattr(forward_batch, "attn_cp_metadata", None) is None:
|
||||
return
|
||||
if not forward_batch.forward_mode.is_context_parallel_extend():
|
||||
return
|
||||
|
||||
strategy = get_cp_strategy()
|
||||
if strategy is None or strategy.cp_size <= 1:
|
||||
return
|
||||
|
||||
fm = self.forward_metadata
|
||||
global_positions = forward_batch.positions
|
||||
if global_positions is None:
|
||||
return
|
||||
|
||||
device = global_positions.device
|
||||
num_tokens = int(global_positions.shape[0])
|
||||
local_idx = strategy.local_q_indices(num_tokens, forward_batch).to(
|
||||
device=device, dtype=torch.long
|
||||
)
|
||||
if local_idx.numel() > 0:
|
||||
shard_bound = int(
|
||||
getattr(forward_batch.attn_cp_metadata, "total_seq_lens", num_tokens)
|
||||
)
|
||||
local_idx = local_idx[local_idx < shard_bound]
|
||||
local_positions = global_positions.index_select(0, local_idx)
|
||||
|
||||
from sglang.srt.layers.cp.padding import pad_local_rows
|
||||
|
||||
local_positions = pad_local_rows(
|
||||
local_positions, forward_batch.attn_cp_metadata, dim=0
|
||||
)
|
||||
|
||||
# Per-token request ids for the local rows: the Ascend paged-attention
|
||||
# operators consume per-token block-table rows under CP (same
|
||||
# convention as prepare_dsv4_cp_metadata).
|
||||
extend_lens = getattr(forward_batch, "extend_seq_lens_cpu", None)
|
||||
if extend_lens is None:
|
||||
seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None)
|
||||
if seq_lens_cpu is not None:
|
||||
extend_lens = seq_lens_cpu.tolist()
|
||||
else:
|
||||
extend_lens = [num_tokens]
|
||||
extend_lens = [int(x) for x in extend_lens]
|
||||
real_num_tokens = min(sum(extend_lens), num_tokens)
|
||||
|
||||
batch_ids_parts = []
|
||||
for batch_id, length in enumerate(extend_lens):
|
||||
if length <= 0:
|
||||
continue
|
||||
batch_ids_parts.append(
|
||||
torch.full((length,), batch_id, dtype=torch.long, device=device)
|
||||
)
|
||||
if batch_ids_parts:
|
||||
batch_ids = torch.cat(batch_ids_parts, dim=0)
|
||||
else:
|
||||
batch_ids = torch.empty(0, dtype=torch.long, device=device)
|
||||
if batch_ids.shape[0] < num_tokens:
|
||||
batch_ids = torch.cat(
|
||||
[
|
||||
batch_ids,
|
||||
torch.zeros(
|
||||
num_tokens - batch_ids.shape[0],
|
||||
dtype=torch.long,
|
||||
device=device,
|
||||
),
|
||||
]
|
||||
)
|
||||
elif batch_ids.shape[0] > num_tokens:
|
||||
batch_ids = batch_ids[:num_tokens]
|
||||
|
||||
local_batch_ids = batch_ids.index_select(0, local_idx)
|
||||
valid_rows = local_idx < real_num_tokens
|
||||
pad_rows = local_positions.shape[0] - local_batch_ids.shape[0]
|
||||
if pad_rows > 0:
|
||||
local_batch_ids = torch.cat(
|
||||
[local_batch_ids, local_batch_ids.new_zeros(pad_rows)]
|
||||
)
|
||||
valid_rows = torch.cat([valid_rows, valid_rows.new_zeros(pad_rows)])
|
||||
|
||||
def _select_rows(table: Optional[torch.Tensor]):
|
||||
if table is None:
|
||||
return None
|
||||
if local_batch_ids.numel() == 0:
|
||||
return table.new_empty((0, *table.shape[1:]))
|
||||
return table.index_select(0, local_batch_ids)
|
||||
|
||||
fm.block_tables = _select_rows(fm.block_tables)
|
||||
|
||||
local_t = int(local_positions.shape[0])
|
||||
fm.actual_seq_lengths_q = torch.arange(
|
||||
1, local_t + 1, dtype=torch.int32, device=device
|
||||
)
|
||||
# KV length seen by each local token = its global position + 1 (causal
|
||||
# DSA); pad rows (position 0) stay at 1 so the sparse kernel sees a
|
||||
# valid length for every row.
|
||||
fm.actual_seq_lengths_kv = torch.where(
|
||||
valid_rows,
|
||||
local_positions.to(torch.int32) + 1,
|
||||
torch.ones_like(local_positions, dtype=torch.int32),
|
||||
).clamp(min=1)
|
||||
|
||||
forward_batch.dsa_cp_local_positions = local_positions
|
||||
# Keep forward_batch.positions aligned with the rank-local shard so any
|
||||
# downstream reader sees the local rows (parity with the model hook).
|
||||
forward_batch.positions = local_positions
|
||||
forward_batch.dsa_cp_metadata_prepared = True
|
||||
|
||||
def do_cp_balance_attn(
|
||||
self,
|
||||
q_nope,
|
||||
@@ -1223,34 +1339,21 @@ class AscendAttnBackend(AttentionBackend):
|
||||
actual_seq_qlen,
|
||||
actual_seq_lengths_kv,
|
||||
):
|
||||
seq_len = q_nope.shape[0]
|
||||
split_len = (seq_len + 1) // 2
|
||||
q_nope_prev, q_nope_next = torch.split(q_nope, split_len, dim=0)
|
||||
q_rope_prev, q_rope_next = torch.split(q_pe, split_len, dim=0)
|
||||
q_nope_prev = q_nope_prev.contiguous()
|
||||
q_nope_next = q_nope_next.contiguous()
|
||||
q_rope_prev = q_rope_prev.contiguous()
|
||||
q_rope_next = q_rope_next.contiguous()
|
||||
# V2: Q is already sharded to local by the strategy; single sparse-attn
|
||||
# call against the full (gathered) KV, no prev/next balance split.
|
||||
topk_indices = _expand_dsa_sparse_indices(topk_indices)
|
||||
topk_indices_prev, topk_indices_next = torch.split(
|
||||
topk_indices, split_len, dim=0
|
||||
)
|
||||
|
||||
actual_seq_qlen_prev, actual_seq_qlen_next = actual_seq_qlen
|
||||
actual_seq_lengths_kv_prev, actual_seq_lengths_kv_next = actual_seq_lengths_kv
|
||||
|
||||
attn_out_prev, _, _ = torch_npu.npu_sparse_flash_attention(
|
||||
query=q_nope_prev,
|
||||
attn_out, _, _ = torch_npu.npu_sparse_flash_attention(
|
||||
query=q_nope,
|
||||
key=k_nope,
|
||||
value=k_nope,
|
||||
query_rope=q_rope_prev,
|
||||
query_rope=q_pe,
|
||||
key_rope=k_pe,
|
||||
sparse_indices=topk_indices_prev,
|
||||
sparse_indices=topk_indices,
|
||||
scale_value=layer.scaling,
|
||||
actual_seq_lengths_query=actual_seq_qlen_prev.to(
|
||||
actual_seq_lengths_query=actual_seq_qlen.to(
|
||||
device=q_nope.device, dtype=torch.int32
|
||||
),
|
||||
actual_seq_lengths_kv=actual_seq_lengths_kv_prev.to(
|
||||
actual_seq_lengths_kv=actual_seq_lengths_kv.to(
|
||||
device=q_nope.device, dtype=torch.int32
|
||||
),
|
||||
block_table=self.forward_metadata.block_tables,
|
||||
@@ -1261,29 +1364,7 @@ class AscendAttnBackend(AttentionBackend):
|
||||
attention_mode=2,
|
||||
return_softmax_lse=False,
|
||||
)
|
||||
attn_out_next, _, _ = torch_npu.npu_sparse_flash_attention(
|
||||
query=q_nope_next,
|
||||
key=k_nope,
|
||||
value=k_nope,
|
||||
query_rope=q_rope_next,
|
||||
key_rope=k_pe,
|
||||
sparse_indices=topk_indices_next,
|
||||
scale_value=layer.scaling,
|
||||
actual_seq_lengths_query=actual_seq_qlen_next.to(
|
||||
device=q_nope.device, dtype=torch.int32
|
||||
),
|
||||
actual_seq_lengths_kv=actual_seq_lengths_kv_next.to(
|
||||
device=q_nope.device, dtype=torch.int32
|
||||
),
|
||||
block_table=self.forward_metadata.block_tables,
|
||||
sparse_block_size=1,
|
||||
layout_query="TND",
|
||||
layout_kv="PA_BSND",
|
||||
sparse_mode=3,
|
||||
attention_mode=2,
|
||||
return_softmax_lse=False,
|
||||
)
|
||||
return torch.cat([attn_out_prev, attn_out_next], dim=0)
|
||||
return attn_out
|
||||
|
||||
def do_cp_attn_fia(
|
||||
self,
|
||||
@@ -3350,7 +3431,7 @@ class AscendAttnBackend(AttentionBackend):
|
||||
attn_output = attn_output[:, :, : layer.tp_q_head_num, :]
|
||||
else:
|
||||
assert (
|
||||
self.graph_mode == False
|
||||
not self.graph_mode
|
||||
) # _npu_paged_attention_mla not support graph mode
|
||||
if q_rope is not None:
|
||||
q = torch.cat([q, q_rope], dim=-1)
|
||||
|
||||
@@ -64,6 +64,12 @@ class DSANPUIndexerMixin:
|
||||
input_on_attn_tp_slices: bool = False,
|
||||
dynamic_scale: torch.Tensor = None,
|
||||
) -> torch.Tensor:
|
||||
# DSA prefill CP metadata is Ascend-local and is first consumed by this
|
||||
# indexer (before the attention backend's forward_extend), so prepare it
|
||||
# here rather than in the shared eager runner.
|
||||
attn_backend = get_attn_backend()
|
||||
if hasattr(attn_backend, "prepare_dsa_cp_metadata"):
|
||||
attn_backend.prepare_dsa_cp_metadata(forward_batch)
|
||||
if get_attn_backend().forward_metadata.seq_lens_cpu_int is None:
|
||||
actual_seq_lengths_kv = get_attn_backend().forward_metadata.seq_lens
|
||||
else:
|
||||
@@ -238,28 +244,7 @@ class DSANPUIndexerMixin:
|
||||
self.dsa_enable_prefill_cp
|
||||
and forward_batch.attn_cp_metadata is not None
|
||||
):
|
||||
get_attn_backend().forward_metadata.actual_seq_lengths_q = (
|
||||
forward_batch.attn_cp_metadata.actual_seq_q_prev_tensor,
|
||||
forward_batch.attn_cp_metadata.actual_seq_q_next_tensor,
|
||||
)
|
||||
if sum(forward_batch.extend_prefix_lens_cpu) > 0:
|
||||
total_kv_len_prev_tensor = (
|
||||
forward_batch.attn_cp_metadata.kv_len_prev_tensor
|
||||
+ forward_batch.extend_prefix_lens.squeeze()
|
||||
)
|
||||
total_kv_len_next_tensor = (
|
||||
forward_batch.attn_cp_metadata.kv_len_next_tensor
|
||||
+ forward_batch.extend_prefix_lens.squeeze()
|
||||
)
|
||||
get_attn_backend().forward_metadata.actual_seq_lengths_kv = (
|
||||
total_kv_len_prev_tensor,
|
||||
total_kv_len_next_tensor,
|
||||
)
|
||||
else:
|
||||
get_attn_backend().forward_metadata.actual_seq_lengths_kv = (
|
||||
forward_batch.attn_cp_metadata.kv_len_prev_tensor,
|
||||
forward_batch.attn_cp_metadata.kv_len_next_tensor,
|
||||
)
|
||||
# V2: local metadata already prepared by prepare_dsa_cp_metadata.
|
||||
actual_seq_lengths_q = (
|
||||
get_attn_backend().forward_metadata.actual_seq_lengths_q
|
||||
)
|
||||
@@ -308,7 +293,6 @@ class DSANPUIndexerMixin:
|
||||
and self.dsa_enable_prefill_cp
|
||||
and forward_batch.attn_cp_metadata is not None
|
||||
):
|
||||
block_table = block_table[: actual_seq_lengths_q[0].numel()]
|
||||
topk_indices = self.do_npu_cp_balance_indexer(
|
||||
q.view(-1, self.n_heads, self.head_dim),
|
||||
past_key_states,
|
||||
@@ -377,26 +361,19 @@ class DSANPUIndexerMixin:
|
||||
actual_seq_lengths_kv,
|
||||
block_table,
|
||||
):
|
||||
q_prev, q_next = torch.split(q, (q.size(0) + 1) // 2, dim=0)
|
||||
weights_prev, weights_next = None, None
|
||||
# V2: Q is local (sharded); single indexer call against full KV.
|
||||
if indexer_weights is not None:
|
||||
weights_prev, weights_next = torch.split(
|
||||
indexer_weights, (indexer_weights.size(0) + 1) // 2, dim=0
|
||||
indexer_weights = indexer_weights.contiguous().view(
|
||||
-1, indexer_weights.shape[-1]
|
||||
)
|
||||
weights_prev = weights_prev.contiguous().view(-1, weights_prev.shape[-1])
|
||||
weights_next = weights_next.contiguous().view(-1, weights_next.shape[-1])
|
||||
|
||||
actual_seq_lengths_q_prev, actual_seq_lengths_q_next = actual_seq_lengths_q
|
||||
actual_seq_lengths_kv_prev, actual_seq_lengths_kv_next = actual_seq_lengths_kv
|
||||
|
||||
topk_indices_prev = torch_npu.npu_lightning_indexer(
|
||||
query=q_prev,
|
||||
topk_indices = torch_npu.npu_lightning_indexer(
|
||||
query=q,
|
||||
key=past_key_states,
|
||||
weights=weights_prev,
|
||||
actual_seq_lengths_query=actual_seq_lengths_q_prev.to(
|
||||
weights=indexer_weights,
|
||||
actual_seq_lengths_query=actual_seq_lengths_q.to(
|
||||
device=q.device, dtype=torch.int32
|
||||
),
|
||||
actual_seq_lengths_key=actual_seq_lengths_kv_prev.to(
|
||||
actual_seq_lengths_key=actual_seq_lengths_kv.to(
|
||||
device=q.device, dtype=torch.int32
|
||||
),
|
||||
block_table=block_table,
|
||||
@@ -405,23 +382,7 @@ class DSANPUIndexerMixin:
|
||||
sparse_count=self.index_topk,
|
||||
sparse_mode=3,
|
||||
)
|
||||
topk_indices_next = torch_npu.npu_lightning_indexer(
|
||||
query=q_next,
|
||||
key=past_key_states,
|
||||
weights=weights_next,
|
||||
actual_seq_lengths_query=actual_seq_lengths_q_next.to(
|
||||
device=q.device, dtype=torch.int32
|
||||
),
|
||||
actual_seq_lengths_key=actual_seq_lengths_kv_next.to(
|
||||
device=q.device, dtype=torch.int32
|
||||
),
|
||||
block_table=block_table,
|
||||
layout_query="TND",
|
||||
layout_key="PA_BSND",
|
||||
sparse_count=self.index_topk,
|
||||
sparse_mode=3,
|
||||
)
|
||||
return torch.cat([topk_indices_prev[0], topk_indices_next[0]], dim=0).squeeze(1)
|
||||
return topk_indices[0].squeeze(1)
|
||||
|
||||
|
||||
def scattered_to_tp_attn_full(
|
||||
|
||||
@@ -81,6 +81,7 @@ from sglang.srt.layers.aux_hidden_states import (
|
||||
AuxHiddenStatePacker,
|
||||
)
|
||||
from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx
|
||||
from sglang.srt.layers.cp.utils import cp_gather_full_sequence_states
|
||||
from sglang.srt.layers.dcp.planner import (
|
||||
prepare_decode_context_parallel_metadata,
|
||||
)
|
||||
@@ -2366,6 +2367,20 @@ class DeepseekV2AttentionMLA(
|
||||
else:
|
||||
state.hidden_states_after_attn = result
|
||||
|
||||
def rebuild_cp_kv_cache(self, latent_cache, forward_batch, k_nope, k_pe):
|
||||
# CP V2: gather the rank-local latent KV into full sequence order
|
||||
# (the Ascend sparse attention reads the full KV pool).
|
||||
latent_cache[..., : self.kv_lora_rank] = k_nope.squeeze(1)
|
||||
latent_cache[..., self.kv_lora_rank :] = k_pe.squeeze(1)
|
||||
latent_cache_output = cp_gather_full_sequence_states(
|
||||
latent_cache.contiguous(),
|
||||
forward_batch,
|
||||
torch.npu.current_stream(),
|
||||
)
|
||||
k_nope = latent_cache_output[..., : self.kv_lora_rank].unsqueeze(1)
|
||||
k_pe = latent_cache_output[..., self.kv_lora_rank :].unsqueeze(1)
|
||||
return k_nope, k_pe
|
||||
|
||||
def forward(
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user