From 2d3dd3168d4a8e5b69ebd4c9b41a28fa93e6c16e Mon Sep 17 00:00:00 2001 From: McZyWu Date: Sat, 17 Oct 2026 10:12:41 +0800 Subject: [PATCH] [Fix] [NPU] attention: Use FIA TND for MLA prefix attention and add Gemma norm fallback (#40131) --- .../npu/attention/ascend_backend.py | 136 +++++++------ python/sglang/srt/layers/layernorm.py | 2 + .../npu/attention/test_npu_ascend_backend.py | 180 +++++++++++++++++- 3 files changed, 245 insertions(+), 73 deletions(-) diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index dd13e21641..bf63aa1079 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -156,13 +156,6 @@ class AscendAttnMaskBuilder: mixed_mask_len = 2048 self.mixed_chunk_attn_mask = self.get_splitfuse_attn_mask(mixed_mask_len) - if use_mla: - # Initialize RingMla mask - ringmla_mask_len = 512 - self.ringmla_mask = self.generate_attn_mask( - ringmla_mask_len, "norm", torch.bfloat16 - ).to(self.device) - @staticmethod def generate_mask_flag(max_seq_len): """ @@ -413,8 +406,6 @@ class AscendAttnBackend(AttentionBackend): self.ascend_attn_mask_builder.mtp_mask, self.ascend_attn_mask_builder.mixed_chunk_attn_mask, ) - if self.use_mla: - self.ringmla_mask = self.ascend_attn_mask_builder.ringmla_mask self.is_hybrid_swa = model_runner.is_hybrid_swa if self.is_hybrid_swa: self.full_to_swa_index_mapping = ( @@ -2069,84 +2060,85 @@ class AscendAttnBackend(AttentionBackend): [layer.v_head_dim, self.qk_rope_head_dim], dim=-1 ) - # 1st, compute extend tokens to get attn_output and attn_lse - num_tokens = q_nope.size(0) - attn_output = torch.zeros( - num_tokens, - layer.tp_q_head_num, - layer.v_head_dim, - dtype=q_nope.dtype, - device=q_nope.device, - ) - attn_lse = torch.zeros( - layer.tp_q_head_num, - num_tokens, - dtype=torch.float32, - device=q_nope.device, - ) - torch_npu.atb.npu_ring_mla( - q_nope=q_nope, - q_rope=q_rope, - k_nope=k_nope, - k_rope=k_rope, - value=v, - mask=self.ringmla_mask, - seqlen=self.forward_metadata.extend_seq_lens_cpu_int, - head_num=layer.tp_q_head_num, - kv_head_num=layer.tp_k_head_num, - pre_out=None, - prev_lse=None, - qk_scale=layer.scaling, - kernel_type="kernel_type_high_precision", - mask_type="mask_type_triu", - calc_type="calc_type_first_ring", - output=attn_output, - softmax_lse=attn_lse, + query_lens = self.forward_metadata.extend_seq_lens_cpu_int + cu_query_lens = query_lens.cumsum(0).tolist() + q_nope, q_rope = q_nope.contiguous(), q_rope.contiguous() + + attn_output, attn_lse = torch_npu.npu_fused_infer_attention_score_v2( + query=q_nope, + key=k_nope.contiguous(), + value=v.contiguous(), + query_rope=q_rope, + key_rope=k_rope.contiguous(), + num_query_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="TND", + actual_seq_qlen=cu_query_lens, + actual_seq_kvlen=cu_query_lens, + atten_mask=self.fia_mask, + sparse_mode=3, + next_tokens=0, + softmax_scale=layer.scaling, + return_softmax_lse=True, ) - # 2nd, load history kvcache(kv_a and k_pe) and calculate k_nope k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id) v_buffer = self.token_to_kv_pool.get_value_buffer(layer.layer_id) - kv_cached = torch.index_select( - k_buffer, 0, self.forward_metadata.flatten_prefix_block_tables + kv_cached = gather_mla_cache_pages( + k_buffer, + self.forward_metadata.flatten_prefix_block_tables, + is_nz=is_fia_nz(), ) - k_rope_cached = torch.index_select( - v_buffer, 0, self.forward_metadata.flatten_prefix_block_tables + k_rope_cached = gather_mla_cache_pages( + v_buffer, + self.forward_metadata.flatten_prefix_block_tables, + is_nz=is_fia_nz(), ).flatten(0, 1) assert layer.kv_b_proj is not None kv = layer.kv_b_proj(kv_cached)[0].view( - -1, layer.tp_k_head_num, self.qk_nope_head_dim + layer.v_head_dim + -1, + layer.tp_k_head_num, + self.qk_nope_head_dim + layer.v_head_dim, ) k_nope, v = kv.split([self.qk_nope_head_dim, layer.v_head_dim], dim=-1) - # 3rd, compute history kv to attn_out k_rope = k_rope_cached.expand(-1, layer.tp_k_head_num, -1) - seq_len = torch.stack( - [ - self.forward_metadata.extend_seq_lens_cpu_int, - self.forward_metadata.prefix_lens, - ] + prefix_lens = self.forward_metadata.prefix_lens + prefix_output, prefix_lse = ( + torch_npu.npu_fused_infer_attention_score_v2( + query=q_nope, + key=k_nope.contiguous(), + value=v.contiguous(), + query_rope=q_rope, + key_rope=k_rope.contiguous(), + num_query_heads=layer.tp_q_head_num, + num_key_value_heads=layer.tp_k_head_num, + input_layout="TND", + actual_seq_qlen=cu_query_lens, + actual_seq_kvlen=prefix_lens.cumsum(0).tolist(), + sparse_mode=0, + softmax_scale=layer.scaling, + return_softmax_lse=True, + ) ) - torch_npu.atb.npu_ring_mla( - q_nope=q_nope, - q_rope=q_rope, - k_nope=k_nope, - k_rope=k_rope, - value=v, - mask=self.ringmla_mask, - seqlen=seq_len, - head_num=layer.tp_q_head_num, - kv_head_num=layer.tp_k_head_num, - pre_out=attn_output, - prev_lse=attn_lse, - qk_scale=layer.scaling, - kernel_type="kernel_type_high_precision", - mask_type="no_mask", - calc_type="calc_type_default", - output=attn_output, - softmax_lse=attn_lse, + + query_start = 0 + for query_end, prefix_len in zip(cu_query_lens, prefix_lens.tolist()): + if prefix_len == 0: + prefix_output[query_start:query_end].zero_() + prefix_lse[query_start:query_end].fill_(-float("inf")) + query_start = query_end + + attn_output, _ = torch_npu.npu_attention_update( + (attn_lse.reshape(-1), prefix_lse.reshape(-1)), + ( + attn_output.float().reshape(-1, layer.v_head_dim), + prefix_output.float().reshape(-1, layer.v_head_dim), + ), + 0, ) + attn_output = attn_output.to(q.dtype) attn_output = attn_output.reshape( [-1, layer.tp_q_head_num, layer.v_head_dim] ) diff --git a/python/sglang/srt/layers/layernorm.py b/python/sglang/srt/layers/layernorm.py index 6068e8f209..e1560c4e76 100644 --- a/python/sglang/srt/layers/layernorm.py +++ b/python/sglang/srt/layers/layernorm.py @@ -1368,6 +1368,8 @@ class Gemma3RMSNorm(BaseFusedOp): return self.forward_native(x, residual) def forward_npu(self, x, residual: Optional[torch.Tensor] = None): + if envs.SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM.get(): + return self.forward_native(x, residual) if residual is not None: return self.forward_native(x, residual) output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps) diff --git a/test/registered/unit/npu/attention/test_npu_ascend_backend.py b/test/registered/unit/npu/attention/test_npu_ascend_backend.py index 1917c04239..929d34a015 100644 --- a/test/registered/unit/npu/attention/test_npu_ascend_backend.py +++ b/test/registered/unit/npu/attention/test_npu_ascend_backend.py @@ -4,7 +4,8 @@ Unit tests for sglang.srt.hardware_backend.npu.attention.ascend_backend. import sys import unittest -from unittest.mock import MagicMock +from types import SimpleNamespace +from unittest.mock import MagicMock, patch import torch @@ -29,6 +30,7 @@ for _ in ( from sglang.srt.hardware_backend.npu.attention.ascend_backend import ( AscendAttnBackend, AscendAttnMaskBuilder, + ForwardMetadata, _expand_dsa_sparse_indices, _reshape_kv_for_fia_nz, ) @@ -554,5 +556,181 @@ class TestGenerateAlibiBias(unittest.TestCase): self.assertEqual(result.dtype, torch.bfloat16) +class TestMlaPrefixFia(unittest.TestCase): + @staticmethod + def _fia_reference(**kwargs): + q, k, v = (kwargs[name] for name in ("query", "key", "value")) + for name in ("query", "key", "value", "query_rope", "key_rope"): + assert kwargs[name].is_contiguous(), name + assert kwargs["input_layout"] == "TND" + assert kwargs["return_softmax_lse"] + assert kwargs["num_query_heads"] == q.shape[1] + assert kwargs["num_key_value_heads"] == k.shape[1] + q = torch.cat((q, kwargs["query_rope"]), dim=-1).float() + k = torch.cat((k, kwargs["key_rope"]), dim=-1).float() + output = v.new_empty(q.shape[0], q.shape[1], v.shape[-1]) + lse = q.new_empty(q.shape[0], q.shape[1], 1) + q_start = kv_start = 0 + for q_end, kv_end in zip(kwargs["actual_seq_qlen"], kwargs["actual_seq_kvlen"]): + if kv_start == kv_end: + output[q_start:q_end] = float("nan") + lse[q_start:q_end] = float("inf") + else: + scores = ( + torch.einsum("thd,shd->hts", q[q_start:q_end], k[kv_start:kv_end]) + * kwargs["softmax_scale"] + ) + if kwargs["sparse_mode"] == 3: + assert kwargs["next_tokens"] == 0 + assert kwargs["atten_mask"].dtype == torch.bool + mask = torch.ones_like(scores, dtype=torch.bool).triu(1) + scores.masked_fill_(mask, -float("inf")) + else: + assert kwargs["sparse_mode"] == 0 + assert kwargs.get("atten_mask") is None + output[q_start:q_end] = torch.einsum( + "hts,shd->thd", scores.softmax(-1), v[kv_start:kv_end].float() + ).to(v.dtype) + lse[q_start:q_end] = scores.logsumexp(-1).T.unsqueeze(-1) + q_start, kv_start = q_end, kv_end + assert q_start == q.shape[0] + assert kv_start == k.shape[0] + return output, lse + + @staticmethod + def _update_reference(lse, outputs, update_type): + assert update_type == 0 + assert all(x.ndim == 1 and x.dtype == torch.float32 for x in lse) + assert all(x.ndim == 2 and x.dtype == torch.float32 for x in outputs) + assert all(x.is_contiguous() for x in (*lse, *outputs)) + weights = torch.stack(lse).softmax(0).unsqueeze(-1) + return (torch.stack(outputs) * weights).sum(0), None + + def _check_prefix( + self, query_lens, prefix_lens, dtype, padding, heads, magnitude=1, use_fia=False + ): + torch.manual_seed(47) + tokens, prefix_tokens = sum(query_lens), sum(prefix_lens) + q = torch.randn(tokens + padding, heads, 208, dtype=dtype)[..., :192] + k = torch.randn(tokens + padding, heads, 208, dtype=dtype)[..., :192] + v = torch.randn(tokens + padding, heads, 144, dtype=dtype)[..., :128] + q.mul_(magnitude) + k.mul_(magnitude) + cached = torch.randn(prefix_tokens + 3, 1, 32, dtype=dtype) + cached_rope = torch.randn(prefix_tokens + 3, 1, 1, 64, dtype=dtype) + indices = torch.randperm(prefix_tokens + 3)[:prefix_tokens] + weight = torch.randn(32, heads * 256, dtype=dtype) / 32**0.5 + + def project(x): + return x @ weight, None + + backend = AscendAttnBackend.__new__(AscendAttnBackend) + backend.use_mla = True + backend.use_fia = use_fia + backend.is_dllm_model = False + backend.qk_nope_head_dim = 128 + backend.qk_rope_head_dim = 64 + backend.fia_mask = torch.ones(2048, 2048, dtype=torch.bool).triu(1) + backend.forward_metadata = ForwardMetadata( + extend_seq_lens_cpu_int=torch.tensor(query_lens, dtype=torch.int32), + prefix_lens=torch.tensor(prefix_lens, dtype=torch.int32), + flatten_prefix_block_tables=indices, + ) + backend.token_to_kv_pool = SimpleNamespace( + get_key_buffer=lambda _: cached, + get_value_buffer=lambda _: cached_rope, + ) + layer = SimpleNamespace( + layer_id=0, + tp_q_head_num=heads, + tp_k_head_num=heads, + qk_head_dim=192, + v_head_dim=128, + kv_b_proj=project, + scaling=192**-0.5, + ) + batch = SimpleNamespace( + global_num_token_non_padded_cpu=tokens, + extend_prefix_lens_cpu=prefix_lens, + forward_mode=SimpleNamespace( + is_target_verify=lambda: False, + is_draft_extend_v2=lambda: False, + ), + ) + module_name = "sglang.srt.hardware_backend.npu.attention.ascend_backend" + with ( + patch(f"{module_name}.is_mla_preprocess_enabled", return_value=False), + patch( + f"{module_name}.torch_npu.npu_fused_infer_attention_score_v2", + side_effect=self._fia_reference, + ) as fia, + patch( + f"{module_name}.torch_npu.npu_attention_update", + side_effect=self._update_reference, + ) as update, + patch(f"{module_name}.torch_npu.atb", None), + ): + actual = backend.forward_extend(q, k, v, layer, batch, save_kv_cache=False) + self.assertEqual(fia.call_count, 2) + self.assertEqual(update.call_count, 1) + + prefix_k, prefix_v = ( + project(cached[indices])[0] + .view(prefix_tokens, heads, 256) + .split(128, dim=-1) + ) + prefix_k = torch.cat( + (prefix_k, cached_rope[indices].flatten(0, 1).expand(-1, heads, -1)), + dim=-1, + ) + expected = torch.zeros(tokens + padding, heads, 128, dtype=dtype) + q_start = prefix_start = 0 + for q_len, prefix_len in zip(query_lens, prefix_lens): + q_end, prefix_end = q_start + q_len, prefix_start + prefix_len + full_k = torch.cat((prefix_k[prefix_start:prefix_end], k[q_start:q_end])) + full_v = torch.cat((prefix_v[prefix_start:prefix_end], v[q_start:q_end])) + allowed = torch.arange(prefix_len + q_len)[None, :] <= ( + prefix_len + torch.arange(q_len)[:, None] + ) + expected[q_start:q_end] = ( + torch.nn.functional.scaled_dot_product_attention( + q[q_start:q_end].float().transpose(0, 1), + full_k.float().transpose(0, 1), + full_v.float().transpose(0, 1), + attn_mask=allowed, + scale=layer.scaling, + ) + .transpose(0, 1) + .to(dtype) + ) + q_start, prefix_start = q_end, prefix_end + self.assertEqual(actual.dtype, dtype) + self.assertTrue(torch.isfinite(actual).all()) + torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02) + self.assertEqual(torch.count_nonzero(actual[tokens:]).item(), 0) + + def test_prefix_matches_dense_attention(self): + for dtype in (torch.float16, torch.bfloat16): + for queries, prefixes, padding, heads in ( + ([3, 1, 2], [0, 5, 2], 3, 4), + ([2, 4, 1], [5, 0, 0], 0, 2), + ([1, 1], [2, 3], 2, 4), + ([4], [7], 0, 2), + ): + with self.subTest(dtype=dtype, queries=queries, prefixes=prefixes): + self._check_prefix(queries, prefixes, dtype, padding, heads) + + def test_large_logits_merge_stably(self): + self._check_prefix([3, 2], [4, 7], torch.bfloat16, 1, 4, magnitude=8) + + def test_prefix_ignores_fia_flag(self): + for dtype in (torch.float16, torch.bfloat16): + for use_fia in (False, True): + with self.subTest(dtype=dtype, use_fia=use_fia): + self._check_prefix( + [3, 1, 2], [0, 5, 2], dtype, 3, 4, use_fia=use_fia + ) + + if __name__ == "__main__": unittest.main()