diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index e608474493..c4432c357b 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -10,7 +10,7 @@ from sgl_kernel_npu.attention.sinks_attention import ( attention_sinks_triton, ) -from sglang.srt.configs.model_config import AttentionArch +from sglang.srt.configs.model_config import AttentionArch, is_deepseek_dsa from sglang.srt.dllm.config import DllmConfig from sglang.srt.environ import envs from sglang.srt.hardware_backend.npu.attention.ascend_torch_native_backend import ( @@ -30,6 +30,7 @@ from sglang.srt.hardware_backend.npu.sparsity_driven_kv_offload.config import ( ) from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp +from sglang.srt.layers.cp.base import get_cp_strategy from sglang.srt.layers.dcp.layout import ( get_dcp_chain_spec_lens, get_dcp_lens, @@ -329,6 +330,9 @@ class AscendAttnBackend(AttentionBackend): self.model_dtype = model_runner.model_config.dtype self.kv_cache_dtype = model_runner.kv_cache_dtype self.use_mla = model_runner.model_config.attention_arch == AttentionArch.MLA + # Only DSA models (GLM-5.2, DeepSeek-V3-DSA, ...) consume the per-token DSA + # CP metadata built by prepare_dsa_cp_metadata; V4 keeps its own variant. + self.use_dsa = is_deepseek_dsa(model_runner.model_config.hf_config) if self.use_mla: self.kv_lora_rank = model_runner.model_config.kv_lora_rank self.qk_rope_head_dim = model_runner.model_config.qk_rope_head_dim @@ -1212,6 +1216,118 @@ class AscendAttnBackend(AttentionBackend): attn_output = attn_output.view(-1, num_heads * head_size) return attn_output + def prepare_dsa_cp_metadata(self, forward_batch: ForwardBatch) -> None: + if getattr(forward_batch, "dsa_cp_metadata_prepared", False): + return + if not self.use_dsa: + return + if getattr(forward_batch, "attn_cp_metadata", None) is None: + return + if not forward_batch.forward_mode.is_context_parallel_extend(): + return + + strategy = get_cp_strategy() + if strategy is None or strategy.cp_size <= 1: + return + + fm = self.forward_metadata + global_positions = forward_batch.positions + if global_positions is None: + return + + device = global_positions.device + num_tokens = int(global_positions.shape[0]) + local_idx = strategy.local_q_indices(num_tokens, forward_batch).to( + device=device, dtype=torch.long + ) + if local_idx.numel() > 0: + shard_bound = int( + getattr(forward_batch.attn_cp_metadata, "total_seq_lens", num_tokens) + ) + local_idx = local_idx[local_idx < shard_bound] + local_positions = global_positions.index_select(0, local_idx) + + from sglang.srt.layers.cp.padding import pad_local_rows + + local_positions = pad_local_rows( + local_positions, forward_batch.attn_cp_metadata, dim=0 + ) + + # Per-token request ids for the local rows: the Ascend paged-attention + # operators consume per-token block-table rows under CP (same + # convention as prepare_dsv4_cp_metadata). + extend_lens = getattr(forward_batch, "extend_seq_lens_cpu", None) + if extend_lens is None: + seq_lens_cpu = getattr(forward_batch, "seq_lens_cpu", None) + if seq_lens_cpu is not None: + extend_lens = seq_lens_cpu.tolist() + else: + extend_lens = [num_tokens] + extend_lens = [int(x) for x in extend_lens] + real_num_tokens = min(sum(extend_lens), num_tokens) + + batch_ids_parts = [] + for batch_id, length in enumerate(extend_lens): + if length <= 0: + continue + batch_ids_parts.append( + torch.full((length,), batch_id, dtype=torch.long, device=device) + ) + if batch_ids_parts: + batch_ids = torch.cat(batch_ids_parts, dim=0) + else: + batch_ids = torch.empty(0, dtype=torch.long, device=device) + if batch_ids.shape[0] < num_tokens: + batch_ids = torch.cat( + [ + batch_ids, + torch.zeros( + num_tokens - batch_ids.shape[0], + dtype=torch.long, + device=device, + ), + ] + ) + elif batch_ids.shape[0] > num_tokens: + batch_ids = batch_ids[:num_tokens] + + local_batch_ids = batch_ids.index_select(0, local_idx) + valid_rows = local_idx < real_num_tokens + pad_rows = local_positions.shape[0] - local_batch_ids.shape[0] + if pad_rows > 0: + local_batch_ids = torch.cat( + [local_batch_ids, local_batch_ids.new_zeros(pad_rows)] + ) + valid_rows = torch.cat([valid_rows, valid_rows.new_zeros(pad_rows)]) + + def _select_rows(table: Optional[torch.Tensor]): + if table is None: + return None + if local_batch_ids.numel() == 0: + return table.new_empty((0, *table.shape[1:])) + return table.index_select(0, local_batch_ids) + + fm.block_tables = _select_rows(fm.block_tables) + + local_t = int(local_positions.shape[0]) + fm.actual_seq_lengths_q = torch.arange( + 1, local_t + 1, dtype=torch.int32, device=device + ) + # KV length seen by each local token = its global position + 1 (causal + # DSA); pad rows (position 0) stay at 1 so the sparse kernel sees a + # valid length for every row. + fm.actual_seq_lengths_kv = torch.where( + valid_rows, + local_positions.to(torch.int32) + 1, + torch.ones_like(local_positions, dtype=torch.int32), + ).clamp(min=1) + + forward_batch.dsa_cp_local_positions = local_positions + # Keep forward_batch.positions aligned with the rank-local shard so any + # downstream reader sees the local rows (parity with the model hook). + forward_batch.positions = local_positions + forward_batch.dsa_cp_metadata_prepared = True + def do_cp_balance_attn( self, q_nope, @@ -1223,34 +1339,21 @@ class AscendAttnBackend(AttentionBackend): actual_seq_qlen, actual_seq_lengths_kv, ): - seq_len = q_nope.shape[0] - split_len = (seq_len + 1) // 2 - q_nope_prev, q_nope_next = torch.split(q_nope, split_len, dim=0) - q_rope_prev, q_rope_next = torch.split(q_pe, split_len, dim=0) - q_nope_prev = q_nope_prev.contiguous() - q_nope_next = q_nope_next.contiguous() - q_rope_prev = q_rope_prev.contiguous() - q_rope_next = q_rope_next.contiguous() + # V2: Q is already sharded to local by the strategy; single sparse-attn + # call against the full (gathered) KV, no prev/next balance split. topk_indices = _expand_dsa_sparse_indices(topk_indices) - topk_indices_prev, topk_indices_next = torch.split( - topk_indices, split_len, dim=0 - ) - - actual_seq_qlen_prev, actual_seq_qlen_next = actual_seq_qlen - actual_seq_lengths_kv_prev, actual_seq_lengths_kv_next = actual_seq_lengths_kv - - attn_out_prev, _, _ = torch_npu.npu_sparse_flash_attention( - query=q_nope_prev, + attn_out, _, _ = torch_npu.npu_sparse_flash_attention( + query=q_nope, key=k_nope, value=k_nope, - query_rope=q_rope_prev, + query_rope=q_pe, key_rope=k_pe, - sparse_indices=topk_indices_prev, + sparse_indices=topk_indices, scale_value=layer.scaling, - actual_seq_lengths_query=actual_seq_qlen_prev.to( + actual_seq_lengths_query=actual_seq_qlen.to( device=q_nope.device, dtype=torch.int32 ), - actual_seq_lengths_kv=actual_seq_lengths_kv_prev.to( + actual_seq_lengths_kv=actual_seq_lengths_kv.to( device=q_nope.device, dtype=torch.int32 ), block_table=self.forward_metadata.block_tables, @@ -1261,29 +1364,7 @@ class AscendAttnBackend(AttentionBackend): attention_mode=2, return_softmax_lse=False, ) - attn_out_next, _, _ = torch_npu.npu_sparse_flash_attention( - query=q_nope_next, - key=k_nope, - value=k_nope, - query_rope=q_rope_next, - key_rope=k_pe, - sparse_indices=topk_indices_next, - scale_value=layer.scaling, - actual_seq_lengths_query=actual_seq_qlen_next.to( - device=q_nope.device, dtype=torch.int32 - ), - actual_seq_lengths_kv=actual_seq_lengths_kv_next.to( - device=q_nope.device, dtype=torch.int32 - ), - block_table=self.forward_metadata.block_tables, - sparse_block_size=1, - layout_query="TND", - layout_kv="PA_BSND", - sparse_mode=3, - attention_mode=2, - return_softmax_lse=False, - ) - return torch.cat([attn_out_prev, attn_out_next], dim=0) + return attn_out def do_cp_attn_fia( self, @@ -3350,7 +3431,7 @@ class AscendAttnBackend(AttentionBackend): attn_output = attn_output[:, :, : layer.tp_q_head_num, :] else: assert ( - self.graph_mode == False + not self.graph_mode ) # _npu_paged_attention_mla not support graph mode if q_rope is not None: q = torch.cat([q, q_rope], dim=-1) diff --git a/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py b/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py index 64330f8073..77df4b7877 100644 --- a/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py +++ b/python/sglang/srt/layers/attention/dsa/dsa_npu_indexer.py @@ -64,6 +64,12 @@ class DSANPUIndexerMixin: input_on_attn_tp_slices: bool = False, dynamic_scale: torch.Tensor = None, ) -> torch.Tensor: + # DSA prefill CP metadata is Ascend-local and is first consumed by this + # indexer (before the attention backend's forward_extend), so prepare it + # here rather than in the shared eager runner. + attn_backend = get_attn_backend() + if hasattr(attn_backend, "prepare_dsa_cp_metadata"): + attn_backend.prepare_dsa_cp_metadata(forward_batch) if get_attn_backend().forward_metadata.seq_lens_cpu_int is None: actual_seq_lengths_kv = get_attn_backend().forward_metadata.seq_lens else: @@ -238,28 +244,7 @@ class DSANPUIndexerMixin: self.dsa_enable_prefill_cp and forward_batch.attn_cp_metadata is not None ): - get_attn_backend().forward_metadata.actual_seq_lengths_q = ( - forward_batch.attn_cp_metadata.actual_seq_q_prev_tensor, - forward_batch.attn_cp_metadata.actual_seq_q_next_tensor, - ) - if sum(forward_batch.extend_prefix_lens_cpu) > 0: - total_kv_len_prev_tensor = ( - forward_batch.attn_cp_metadata.kv_len_prev_tensor - + forward_batch.extend_prefix_lens.squeeze() - ) - total_kv_len_next_tensor = ( - forward_batch.attn_cp_metadata.kv_len_next_tensor - + forward_batch.extend_prefix_lens.squeeze() - ) - get_attn_backend().forward_metadata.actual_seq_lengths_kv = ( - total_kv_len_prev_tensor, - total_kv_len_next_tensor, - ) - else: - get_attn_backend().forward_metadata.actual_seq_lengths_kv = ( - forward_batch.attn_cp_metadata.kv_len_prev_tensor, - forward_batch.attn_cp_metadata.kv_len_next_tensor, - ) + # V2: local metadata already prepared by prepare_dsa_cp_metadata. actual_seq_lengths_q = ( get_attn_backend().forward_metadata.actual_seq_lengths_q ) @@ -308,7 +293,6 @@ class DSANPUIndexerMixin: and self.dsa_enable_prefill_cp and forward_batch.attn_cp_metadata is not None ): - block_table = block_table[: actual_seq_lengths_q[0].numel()] topk_indices = self.do_npu_cp_balance_indexer( q.view(-1, self.n_heads, self.head_dim), past_key_states, @@ -377,26 +361,19 @@ class DSANPUIndexerMixin: actual_seq_lengths_kv, block_table, ): - q_prev, q_next = torch.split(q, (q.size(0) + 1) // 2, dim=0) - weights_prev, weights_next = None, None + # V2: Q is local (sharded); single indexer call against full KV. if indexer_weights is not None: - weights_prev, weights_next = torch.split( - indexer_weights, (indexer_weights.size(0) + 1) // 2, dim=0 + indexer_weights = indexer_weights.contiguous().view( + -1, indexer_weights.shape[-1] ) - weights_prev = weights_prev.contiguous().view(-1, weights_prev.shape[-1]) - weights_next = weights_next.contiguous().view(-1, weights_next.shape[-1]) - - actual_seq_lengths_q_prev, actual_seq_lengths_q_next = actual_seq_lengths_q - actual_seq_lengths_kv_prev, actual_seq_lengths_kv_next = actual_seq_lengths_kv - - topk_indices_prev = torch_npu.npu_lightning_indexer( - query=q_prev, + topk_indices = torch_npu.npu_lightning_indexer( + query=q, key=past_key_states, - weights=weights_prev, - actual_seq_lengths_query=actual_seq_lengths_q_prev.to( + weights=indexer_weights, + actual_seq_lengths_query=actual_seq_lengths_q.to( device=q.device, dtype=torch.int32 ), - actual_seq_lengths_key=actual_seq_lengths_kv_prev.to( + actual_seq_lengths_key=actual_seq_lengths_kv.to( device=q.device, dtype=torch.int32 ), block_table=block_table, @@ -405,23 +382,7 @@ class DSANPUIndexerMixin: sparse_count=self.index_topk, sparse_mode=3, ) - topk_indices_next = torch_npu.npu_lightning_indexer( - query=q_next, - key=past_key_states, - weights=weights_next, - actual_seq_lengths_query=actual_seq_lengths_q_next.to( - device=q.device, dtype=torch.int32 - ), - actual_seq_lengths_key=actual_seq_lengths_kv_next.to( - device=q.device, dtype=torch.int32 - ), - block_table=block_table, - layout_query="TND", - layout_key="PA_BSND", - sparse_count=self.index_topk, - sparse_mode=3, - ) - return torch.cat([topk_indices_prev[0], topk_indices_next[0]], dim=0).squeeze(1) + return topk_indices[0].squeeze(1) def scattered_to_tp_attn_full( diff --git a/python/sglang/srt/models/deepseek_v2.py b/python/sglang/srt/models/deepseek_v2.py index c02f70f157..adbb76c4a2 100644 --- a/python/sglang/srt/models/deepseek_v2.py +++ b/python/sglang/srt/models/deepseek_v2.py @@ -81,6 +81,7 @@ from sglang.srt.layers.aux_hidden_states import ( AuxHiddenStatePacker, ) from sglang.srt.layers.cp.cp_decode_attn_tp import get_cp_decode_attn_tp_ctx +from sglang.srt.layers.cp.utils import cp_gather_full_sequence_states from sglang.srt.layers.dcp.planner import ( prepare_decode_context_parallel_metadata, ) @@ -2366,6 +2367,20 @@ class DeepseekV2AttentionMLA( else: state.hidden_states_after_attn = result + def rebuild_cp_kv_cache(self, latent_cache, forward_batch, k_nope, k_pe): + # CP V2: gather the rank-local latent KV into full sequence order + # (the Ascend sparse attention reads the full KV pool). + latent_cache[..., : self.kv_lora_rank] = k_nope.squeeze(1) + latent_cache[..., self.kv_lora_rank :] = k_pe.squeeze(1) + latent_cache_output = cp_gather_full_sequence_states( + latent_cache.contiguous(), + forward_batch, + torch.npu.current_stream(), + ) + k_nope = latent_cache_output[..., : self.kv_lora_rank].unsqueeze(1) + k_pe = latent_cache_output[..., self.kv_lora_rank :].unsqueeze(1) + return k_nope, k_pe + def forward( self, positions: torch.Tensor,