[Fix] [NPU] attention: Use FIA TND for MLA prefix attention and add Gemma norm fallback (#40131)

This commit is contained in:
McZyWu
2026-10-17 10:12:41 +08:00
committed by GitHub
parent ff979e0816
commit 2d3dd3168d
3 changed files with 245 additions and 73 deletions
@@ -156,13 +156,6 @@ class AscendAttnMaskBuilder:
mixed_mask_len = 2048
self.mixed_chunk_attn_mask = self.get_splitfuse_attn_mask(mixed_mask_len)
if use_mla:
# Initialize RingMla mask
ringmla_mask_len = 512
self.ringmla_mask = self.generate_attn_mask(
ringmla_mask_len, "norm", torch.bfloat16
).to(self.device)
@staticmethod
def generate_mask_flag(max_seq_len):
"""
@@ -413,8 +406,6 @@ class AscendAttnBackend(AttentionBackend):
self.ascend_attn_mask_builder.mtp_mask,
self.ascend_attn_mask_builder.mixed_chunk_attn_mask,
)
if self.use_mla:
self.ringmla_mask = self.ascend_attn_mask_builder.ringmla_mask
self.is_hybrid_swa = model_runner.is_hybrid_swa
if self.is_hybrid_swa:
self.full_to_swa_index_mapping = (
@@ -2069,84 +2060,85 @@ class AscendAttnBackend(AttentionBackend):
[layer.v_head_dim, self.qk_rope_head_dim], dim=-1
)
# 1st, compute extend tokens to get attn_output and attn_lse
num_tokens = q_nope.size(0)
attn_output = torch.zeros(
num_tokens,
layer.tp_q_head_num,
layer.v_head_dim,
dtype=q_nope.dtype,
device=q_nope.device,
)
attn_lse = torch.zeros(
layer.tp_q_head_num,
num_tokens,
dtype=torch.float32,
device=q_nope.device,
)
torch_npu.atb.npu_ring_mla(
q_nope=q_nope,
q_rope=q_rope,
k_nope=k_nope,
k_rope=k_rope,
value=v,
mask=self.ringmla_mask,
seqlen=self.forward_metadata.extend_seq_lens_cpu_int,
head_num=layer.tp_q_head_num,
kv_head_num=layer.tp_k_head_num,
pre_out=None,
prev_lse=None,
qk_scale=layer.scaling,
kernel_type="kernel_type_high_precision",
mask_type="mask_type_triu",
calc_type="calc_type_first_ring",
output=attn_output,
softmax_lse=attn_lse,
query_lens = self.forward_metadata.extend_seq_lens_cpu_int
cu_query_lens = query_lens.cumsum(0).tolist()
q_nope, q_rope = q_nope.contiguous(), q_rope.contiguous()
attn_output, attn_lse = torch_npu.npu_fused_infer_attention_score_v2(
query=q_nope,
key=k_nope.contiguous(),
value=v.contiguous(),
query_rope=q_rope,
key_rope=k_rope.contiguous(),
num_query_heads=layer.tp_q_head_num,
num_key_value_heads=layer.tp_k_head_num,
input_layout="TND",
actual_seq_qlen=cu_query_lens,
actual_seq_kvlen=cu_query_lens,
atten_mask=self.fia_mask,
sparse_mode=3,
next_tokens=0,
softmax_scale=layer.scaling,
return_softmax_lse=True,
)
# 2nd, load history kvcache(kv_a and k_pe) and calculate k_nope
k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
v_buffer = self.token_to_kv_pool.get_value_buffer(layer.layer_id)
kv_cached = torch.index_select(
k_buffer, 0, self.forward_metadata.flatten_prefix_block_tables
kv_cached = gather_mla_cache_pages(
k_buffer,
self.forward_metadata.flatten_prefix_block_tables,
is_nz=is_fia_nz(),
)
k_rope_cached = torch.index_select(
v_buffer, 0, self.forward_metadata.flatten_prefix_block_tables
k_rope_cached = gather_mla_cache_pages(
v_buffer,
self.forward_metadata.flatten_prefix_block_tables,
is_nz=is_fia_nz(),
).flatten(0, 1)
assert layer.kv_b_proj is not None
kv = layer.kv_b_proj(kv_cached)[0].view(
-1, layer.tp_k_head_num, self.qk_nope_head_dim + layer.v_head_dim
-1,
layer.tp_k_head_num,
self.qk_nope_head_dim + layer.v_head_dim,
)
k_nope, v = kv.split([self.qk_nope_head_dim, layer.v_head_dim], dim=-1)
# 3rd, compute history kv to attn_out
k_rope = k_rope_cached.expand(-1, layer.tp_k_head_num, -1)
seq_len = torch.stack(
[
self.forward_metadata.extend_seq_lens_cpu_int,
self.forward_metadata.prefix_lens,
]
prefix_lens = self.forward_metadata.prefix_lens
prefix_output, prefix_lse = (
torch_npu.npu_fused_infer_attention_score_v2(
query=q_nope,
key=k_nope.contiguous(),
value=v.contiguous(),
query_rope=q_rope,
key_rope=k_rope.contiguous(),
num_query_heads=layer.tp_q_head_num,
num_key_value_heads=layer.tp_k_head_num,
input_layout="TND",
actual_seq_qlen=cu_query_lens,
actual_seq_kvlen=prefix_lens.cumsum(0).tolist(),
sparse_mode=0,
softmax_scale=layer.scaling,
return_softmax_lse=True,
)
)
torch_npu.atb.npu_ring_mla(
q_nope=q_nope,
q_rope=q_rope,
k_nope=k_nope,
k_rope=k_rope,
value=v,
mask=self.ringmla_mask,
seqlen=seq_len,
head_num=layer.tp_q_head_num,
kv_head_num=layer.tp_k_head_num,
pre_out=attn_output,
prev_lse=attn_lse,
qk_scale=layer.scaling,
kernel_type="kernel_type_high_precision",
mask_type="no_mask",
calc_type="calc_type_default",
output=attn_output,
softmax_lse=attn_lse,
query_start = 0
for query_end, prefix_len in zip(cu_query_lens, prefix_lens.tolist()):
if prefix_len == 0:
prefix_output[query_start:query_end].zero_()
prefix_lse[query_start:query_end].fill_(-float("inf"))
query_start = query_end
attn_output, _ = torch_npu.npu_attention_update(
(attn_lse.reshape(-1), prefix_lse.reshape(-1)),
(
attn_output.float().reshape(-1, layer.v_head_dim),
prefix_output.float().reshape(-1, layer.v_head_dim),
),
0,
)
attn_output = attn_output.to(q.dtype)
attn_output = attn_output.reshape(
[-1, layer.tp_q_head_num, layer.v_head_dim]
)
+2
View File
@@ -1368,6 +1368,8 @@ class Gemma3RMSNorm(BaseFusedOp):
return self.forward_native(x, residual)
def forward_npu(self, x, residual: Optional[torch.Tensor] = None):
if envs.SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM.get():
return self.forward_native(x, residual)
if residual is not None:
return self.forward_native(x, residual)
output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps)
@@ -4,7 +4,8 @@ Unit tests for sglang.srt.hardware_backend.npu.attention.ascend_backend.
import sys
import unittest
from unittest.mock import MagicMock
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
@@ -29,6 +30,7 @@ for _ in (
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
AscendAttnBackend,
AscendAttnMaskBuilder,
ForwardMetadata,
_expand_dsa_sparse_indices,
_reshape_kv_for_fia_nz,
)
@@ -554,5 +556,181 @@ class TestGenerateAlibiBias(unittest.TestCase):
self.assertEqual(result.dtype, torch.bfloat16)
class TestMlaPrefixFia(unittest.TestCase):
@staticmethod
def _fia_reference(**kwargs):
q, k, v = (kwargs[name] for name in ("query", "key", "value"))
for name in ("query", "key", "value", "query_rope", "key_rope"):
assert kwargs[name].is_contiguous(), name
assert kwargs["input_layout"] == "TND"
assert kwargs["return_softmax_lse"]
assert kwargs["num_query_heads"] == q.shape[1]
assert kwargs["num_key_value_heads"] == k.shape[1]
q = torch.cat((q, kwargs["query_rope"]), dim=-1).float()
k = torch.cat((k, kwargs["key_rope"]), dim=-1).float()
output = v.new_empty(q.shape[0], q.shape[1], v.shape[-1])
lse = q.new_empty(q.shape[0], q.shape[1], 1)
q_start = kv_start = 0
for q_end, kv_end in zip(kwargs["actual_seq_qlen"], kwargs["actual_seq_kvlen"]):
if kv_start == kv_end:
output[q_start:q_end] = float("nan")
lse[q_start:q_end] = float("inf")
else:
scores = (
torch.einsum("thd,shd->hts", q[q_start:q_end], k[kv_start:kv_end])
* kwargs["softmax_scale"]
)
if kwargs["sparse_mode"] == 3:
assert kwargs["next_tokens"] == 0
assert kwargs["atten_mask"].dtype == torch.bool
mask = torch.ones_like(scores, dtype=torch.bool).triu(1)
scores.masked_fill_(mask, -float("inf"))
else:
assert kwargs["sparse_mode"] == 0
assert kwargs.get("atten_mask") is None
output[q_start:q_end] = torch.einsum(
"hts,shd->thd", scores.softmax(-1), v[kv_start:kv_end].float()
).to(v.dtype)
lse[q_start:q_end] = scores.logsumexp(-1).T.unsqueeze(-1)
q_start, kv_start = q_end, kv_end
assert q_start == q.shape[0]
assert kv_start == k.shape[0]
return output, lse
@staticmethod
def _update_reference(lse, outputs, update_type):
assert update_type == 0
assert all(x.ndim == 1 and x.dtype == torch.float32 for x in lse)
assert all(x.ndim == 2 and x.dtype == torch.float32 for x in outputs)
assert all(x.is_contiguous() for x in (*lse, *outputs))
weights = torch.stack(lse).softmax(0).unsqueeze(-1)
return (torch.stack(outputs) * weights).sum(0), None
def _check_prefix(
self, query_lens, prefix_lens, dtype, padding, heads, magnitude=1, use_fia=False
):
torch.manual_seed(47)
tokens, prefix_tokens = sum(query_lens), sum(prefix_lens)
q = torch.randn(tokens + padding, heads, 208, dtype=dtype)[..., :192]
k = torch.randn(tokens + padding, heads, 208, dtype=dtype)[..., :192]
v = torch.randn(tokens + padding, heads, 144, dtype=dtype)[..., :128]
q.mul_(magnitude)
k.mul_(magnitude)
cached = torch.randn(prefix_tokens + 3, 1, 32, dtype=dtype)
cached_rope = torch.randn(prefix_tokens + 3, 1, 1, 64, dtype=dtype)
indices = torch.randperm(prefix_tokens + 3)[:prefix_tokens]
weight = torch.randn(32, heads * 256, dtype=dtype) / 32**0.5
def project(x):
return x @ weight, None
backend = AscendAttnBackend.__new__(AscendAttnBackend)
backend.use_mla = True
backend.use_fia = use_fia
backend.is_dllm_model = False
backend.qk_nope_head_dim = 128
backend.qk_rope_head_dim = 64
backend.fia_mask = torch.ones(2048, 2048, dtype=torch.bool).triu(1)
backend.forward_metadata = ForwardMetadata(
extend_seq_lens_cpu_int=torch.tensor(query_lens, dtype=torch.int32),
prefix_lens=torch.tensor(prefix_lens, dtype=torch.int32),
flatten_prefix_block_tables=indices,
)
backend.token_to_kv_pool = SimpleNamespace(
get_key_buffer=lambda _: cached,
get_value_buffer=lambda _: cached_rope,
)
layer = SimpleNamespace(
layer_id=0,
tp_q_head_num=heads,
tp_k_head_num=heads,
qk_head_dim=192,
v_head_dim=128,
kv_b_proj=project,
scaling=192**-0.5,
)
batch = SimpleNamespace(
global_num_token_non_padded_cpu=tokens,
extend_prefix_lens_cpu=prefix_lens,
forward_mode=SimpleNamespace(
is_target_verify=lambda: False,
is_draft_extend_v2=lambda: False,
),
)
module_name = "sglang.srt.hardware_backend.npu.attention.ascend_backend"
with (
patch(f"{module_name}.is_mla_preprocess_enabled", return_value=False),
patch(
f"{module_name}.torch_npu.npu_fused_infer_attention_score_v2",
side_effect=self._fia_reference,
) as fia,
patch(
f"{module_name}.torch_npu.npu_attention_update",
side_effect=self._update_reference,
) as update,
patch(f"{module_name}.torch_npu.atb", None),
):
actual = backend.forward_extend(q, k, v, layer, batch, save_kv_cache=False)
self.assertEqual(fia.call_count, 2)
self.assertEqual(update.call_count, 1)
prefix_k, prefix_v = (
project(cached[indices])[0]
.view(prefix_tokens, heads, 256)
.split(128, dim=-1)
)
prefix_k = torch.cat(
(prefix_k, cached_rope[indices].flatten(0, 1).expand(-1, heads, -1)),
dim=-1,
)
expected = torch.zeros(tokens + padding, heads, 128, dtype=dtype)
q_start = prefix_start = 0
for q_len, prefix_len in zip(query_lens, prefix_lens):
q_end, prefix_end = q_start + q_len, prefix_start + prefix_len
full_k = torch.cat((prefix_k[prefix_start:prefix_end], k[q_start:q_end]))
full_v = torch.cat((prefix_v[prefix_start:prefix_end], v[q_start:q_end]))
allowed = torch.arange(prefix_len + q_len)[None, :] <= (
prefix_len + torch.arange(q_len)[:, None]
)
expected[q_start:q_end] = (
torch.nn.functional.scaled_dot_product_attention(
q[q_start:q_end].float().transpose(0, 1),
full_k.float().transpose(0, 1),
full_v.float().transpose(0, 1),
attn_mask=allowed,
scale=layer.scaling,
)
.transpose(0, 1)
.to(dtype)
)
q_start, prefix_start = q_end, prefix_end
self.assertEqual(actual.dtype, dtype)
self.assertTrue(torch.isfinite(actual).all())
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
self.assertEqual(torch.count_nonzero(actual[tokens:]).item(), 0)
def test_prefix_matches_dense_attention(self):
for dtype in (torch.float16, torch.bfloat16):
for queries, prefixes, padding, heads in (
([3, 1, 2], [0, 5, 2], 3, 4),
([2, 4, 1], [5, 0, 0], 0, 2),
([1, 1], [2, 3], 2, 4),
([4], [7], 0, 2),
):
with self.subTest(dtype=dtype, queries=queries, prefixes=prefixes):
self._check_prefix(queries, prefixes, dtype, padding, heads)
def test_large_logits_merge_stably(self):
self._check_prefix([3, 2], [4, 7], torch.bfloat16, 1, 4, magnitude=8)
def test_prefix_ignores_fia_flag(self):
for dtype in (torch.float16, torch.bfloat16):
for use_fia in (False, True):
with self.subTest(dtype=dtype, use_fia=use_fia):
self._check_prefix(
[3, 1, 2], [0, 5, 2], dtype, 3, 4, use_fia=use_fia
)
if __name__ == "__main__":
unittest.main()