[Fix] [NPU] attention: Use FIA TND for MLA prefix attention and add Gemma norm fallback (#40131)
This commit is contained in:
@@ -156,13 +156,6 @@ class AscendAttnMaskBuilder:
|
||||
mixed_mask_len = 2048
|
||||
self.mixed_chunk_attn_mask = self.get_splitfuse_attn_mask(mixed_mask_len)
|
||||
|
||||
if use_mla:
|
||||
# Initialize RingMla mask
|
||||
ringmla_mask_len = 512
|
||||
self.ringmla_mask = self.generate_attn_mask(
|
||||
ringmla_mask_len, "norm", torch.bfloat16
|
||||
).to(self.device)
|
||||
|
||||
@staticmethod
|
||||
def generate_mask_flag(max_seq_len):
|
||||
"""
|
||||
@@ -413,8 +406,6 @@ class AscendAttnBackend(AttentionBackend):
|
||||
self.ascend_attn_mask_builder.mtp_mask,
|
||||
self.ascend_attn_mask_builder.mixed_chunk_attn_mask,
|
||||
)
|
||||
if self.use_mla:
|
||||
self.ringmla_mask = self.ascend_attn_mask_builder.ringmla_mask
|
||||
self.is_hybrid_swa = model_runner.is_hybrid_swa
|
||||
if self.is_hybrid_swa:
|
||||
self.full_to_swa_index_mapping = (
|
||||
@@ -2069,84 +2060,85 @@ class AscendAttnBackend(AttentionBackend):
|
||||
[layer.v_head_dim, self.qk_rope_head_dim], dim=-1
|
||||
)
|
||||
|
||||
# 1st, compute extend tokens to get attn_output and attn_lse
|
||||
num_tokens = q_nope.size(0)
|
||||
attn_output = torch.zeros(
|
||||
num_tokens,
|
||||
layer.tp_q_head_num,
|
||||
layer.v_head_dim,
|
||||
dtype=q_nope.dtype,
|
||||
device=q_nope.device,
|
||||
)
|
||||
attn_lse = torch.zeros(
|
||||
layer.tp_q_head_num,
|
||||
num_tokens,
|
||||
dtype=torch.float32,
|
||||
device=q_nope.device,
|
||||
)
|
||||
torch_npu.atb.npu_ring_mla(
|
||||
q_nope=q_nope,
|
||||
q_rope=q_rope,
|
||||
k_nope=k_nope,
|
||||
k_rope=k_rope,
|
||||
value=v,
|
||||
mask=self.ringmla_mask,
|
||||
seqlen=self.forward_metadata.extend_seq_lens_cpu_int,
|
||||
head_num=layer.tp_q_head_num,
|
||||
kv_head_num=layer.tp_k_head_num,
|
||||
pre_out=None,
|
||||
prev_lse=None,
|
||||
qk_scale=layer.scaling,
|
||||
kernel_type="kernel_type_high_precision",
|
||||
mask_type="mask_type_triu",
|
||||
calc_type="calc_type_first_ring",
|
||||
output=attn_output,
|
||||
softmax_lse=attn_lse,
|
||||
query_lens = self.forward_metadata.extend_seq_lens_cpu_int
|
||||
cu_query_lens = query_lens.cumsum(0).tolist()
|
||||
q_nope, q_rope = q_nope.contiguous(), q_rope.contiguous()
|
||||
|
||||
attn_output, attn_lse = torch_npu.npu_fused_infer_attention_score_v2(
|
||||
query=q_nope,
|
||||
key=k_nope.contiguous(),
|
||||
value=v.contiguous(),
|
||||
query_rope=q_rope,
|
||||
key_rope=k_rope.contiguous(),
|
||||
num_query_heads=layer.tp_q_head_num,
|
||||
num_key_value_heads=layer.tp_k_head_num,
|
||||
input_layout="TND",
|
||||
actual_seq_qlen=cu_query_lens,
|
||||
actual_seq_kvlen=cu_query_lens,
|
||||
atten_mask=self.fia_mask,
|
||||
sparse_mode=3,
|
||||
next_tokens=0,
|
||||
softmax_scale=layer.scaling,
|
||||
return_softmax_lse=True,
|
||||
)
|
||||
|
||||
# 2nd, load history kvcache(kv_a and k_pe) and calculate k_nope
|
||||
k_buffer = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
v_buffer = self.token_to_kv_pool.get_value_buffer(layer.layer_id)
|
||||
kv_cached = torch.index_select(
|
||||
k_buffer, 0, self.forward_metadata.flatten_prefix_block_tables
|
||||
kv_cached = gather_mla_cache_pages(
|
||||
k_buffer,
|
||||
self.forward_metadata.flatten_prefix_block_tables,
|
||||
is_nz=is_fia_nz(),
|
||||
)
|
||||
k_rope_cached = torch.index_select(
|
||||
v_buffer, 0, self.forward_metadata.flatten_prefix_block_tables
|
||||
k_rope_cached = gather_mla_cache_pages(
|
||||
v_buffer,
|
||||
self.forward_metadata.flatten_prefix_block_tables,
|
||||
is_nz=is_fia_nz(),
|
||||
).flatten(0, 1)
|
||||
|
||||
assert layer.kv_b_proj is not None
|
||||
kv = layer.kv_b_proj(kv_cached)[0].view(
|
||||
-1, layer.tp_k_head_num, self.qk_nope_head_dim + layer.v_head_dim
|
||||
-1,
|
||||
layer.tp_k_head_num,
|
||||
self.qk_nope_head_dim + layer.v_head_dim,
|
||||
)
|
||||
k_nope, v = kv.split([self.qk_nope_head_dim, layer.v_head_dim], dim=-1)
|
||||
|
||||
# 3rd, compute history kv to attn_out
|
||||
k_rope = k_rope_cached.expand(-1, layer.tp_k_head_num, -1)
|
||||
seq_len = torch.stack(
|
||||
[
|
||||
self.forward_metadata.extend_seq_lens_cpu_int,
|
||||
self.forward_metadata.prefix_lens,
|
||||
]
|
||||
prefix_lens = self.forward_metadata.prefix_lens
|
||||
prefix_output, prefix_lse = (
|
||||
torch_npu.npu_fused_infer_attention_score_v2(
|
||||
query=q_nope,
|
||||
key=k_nope.contiguous(),
|
||||
value=v.contiguous(),
|
||||
query_rope=q_rope,
|
||||
key_rope=k_rope.contiguous(),
|
||||
num_query_heads=layer.tp_q_head_num,
|
||||
num_key_value_heads=layer.tp_k_head_num,
|
||||
input_layout="TND",
|
||||
actual_seq_qlen=cu_query_lens,
|
||||
actual_seq_kvlen=prefix_lens.cumsum(0).tolist(),
|
||||
sparse_mode=0,
|
||||
softmax_scale=layer.scaling,
|
||||
return_softmax_lse=True,
|
||||
)
|
||||
)
|
||||
torch_npu.atb.npu_ring_mla(
|
||||
q_nope=q_nope,
|
||||
q_rope=q_rope,
|
||||
k_nope=k_nope,
|
||||
k_rope=k_rope,
|
||||
value=v,
|
||||
mask=self.ringmla_mask,
|
||||
seqlen=seq_len,
|
||||
head_num=layer.tp_q_head_num,
|
||||
kv_head_num=layer.tp_k_head_num,
|
||||
pre_out=attn_output,
|
||||
prev_lse=attn_lse,
|
||||
qk_scale=layer.scaling,
|
||||
kernel_type="kernel_type_high_precision",
|
||||
mask_type="no_mask",
|
||||
calc_type="calc_type_default",
|
||||
output=attn_output,
|
||||
softmax_lse=attn_lse,
|
||||
|
||||
query_start = 0
|
||||
for query_end, prefix_len in zip(cu_query_lens, prefix_lens.tolist()):
|
||||
if prefix_len == 0:
|
||||
prefix_output[query_start:query_end].zero_()
|
||||
prefix_lse[query_start:query_end].fill_(-float("inf"))
|
||||
query_start = query_end
|
||||
|
||||
attn_output, _ = torch_npu.npu_attention_update(
|
||||
(attn_lse.reshape(-1), prefix_lse.reshape(-1)),
|
||||
(
|
||||
attn_output.float().reshape(-1, layer.v_head_dim),
|
||||
prefix_output.float().reshape(-1, layer.v_head_dim),
|
||||
),
|
||||
0,
|
||||
)
|
||||
attn_output = attn_output.to(q.dtype)
|
||||
attn_output = attn_output.reshape(
|
||||
[-1, layer.tp_q_head_num, layer.v_head_dim]
|
||||
)
|
||||
|
||||
@@ -1368,6 +1368,8 @@ class Gemma3RMSNorm(BaseFusedOp):
|
||||
return self.forward_native(x, residual)
|
||||
|
||||
def forward_npu(self, x, residual: Optional[torch.Tensor] = None):
|
||||
if envs.SGLANG_NPU_FORWARD_NATIVE_GEMMA_RMS_NORM.get():
|
||||
return self.forward_native(x, residual)
|
||||
if residual is not None:
|
||||
return self.forward_native(x, residual)
|
||||
output, _ = torch_npu.npu_gemma_rms_norm(x, self.weight, self.eps)
|
||||
|
||||
@@ -4,7 +4,8 @@ Unit tests for sglang.srt.hardware_backend.npu.attention.ascend_backend.
|
||||
|
||||
import sys
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -29,6 +30,7 @@ for _ in (
|
||||
from sglang.srt.hardware_backend.npu.attention.ascend_backend import (
|
||||
AscendAttnBackend,
|
||||
AscendAttnMaskBuilder,
|
||||
ForwardMetadata,
|
||||
_expand_dsa_sparse_indices,
|
||||
_reshape_kv_for_fia_nz,
|
||||
)
|
||||
@@ -554,5 +556,181 @@ class TestGenerateAlibiBias(unittest.TestCase):
|
||||
self.assertEqual(result.dtype, torch.bfloat16)
|
||||
|
||||
|
||||
class TestMlaPrefixFia(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _fia_reference(**kwargs):
|
||||
q, k, v = (kwargs[name] for name in ("query", "key", "value"))
|
||||
for name in ("query", "key", "value", "query_rope", "key_rope"):
|
||||
assert kwargs[name].is_contiguous(), name
|
||||
assert kwargs["input_layout"] == "TND"
|
||||
assert kwargs["return_softmax_lse"]
|
||||
assert kwargs["num_query_heads"] == q.shape[1]
|
||||
assert kwargs["num_key_value_heads"] == k.shape[1]
|
||||
q = torch.cat((q, kwargs["query_rope"]), dim=-1).float()
|
||||
k = torch.cat((k, kwargs["key_rope"]), dim=-1).float()
|
||||
output = v.new_empty(q.shape[0], q.shape[1], v.shape[-1])
|
||||
lse = q.new_empty(q.shape[0], q.shape[1], 1)
|
||||
q_start = kv_start = 0
|
||||
for q_end, kv_end in zip(kwargs["actual_seq_qlen"], kwargs["actual_seq_kvlen"]):
|
||||
if kv_start == kv_end:
|
||||
output[q_start:q_end] = float("nan")
|
||||
lse[q_start:q_end] = float("inf")
|
||||
else:
|
||||
scores = (
|
||||
torch.einsum("thd,shd->hts", q[q_start:q_end], k[kv_start:kv_end])
|
||||
* kwargs["softmax_scale"]
|
||||
)
|
||||
if kwargs["sparse_mode"] == 3:
|
||||
assert kwargs["next_tokens"] == 0
|
||||
assert kwargs["atten_mask"].dtype == torch.bool
|
||||
mask = torch.ones_like(scores, dtype=torch.bool).triu(1)
|
||||
scores.masked_fill_(mask, -float("inf"))
|
||||
else:
|
||||
assert kwargs["sparse_mode"] == 0
|
||||
assert kwargs.get("atten_mask") is None
|
||||
output[q_start:q_end] = torch.einsum(
|
||||
"hts,shd->thd", scores.softmax(-1), v[kv_start:kv_end].float()
|
||||
).to(v.dtype)
|
||||
lse[q_start:q_end] = scores.logsumexp(-1).T.unsqueeze(-1)
|
||||
q_start, kv_start = q_end, kv_end
|
||||
assert q_start == q.shape[0]
|
||||
assert kv_start == k.shape[0]
|
||||
return output, lse
|
||||
|
||||
@staticmethod
|
||||
def _update_reference(lse, outputs, update_type):
|
||||
assert update_type == 0
|
||||
assert all(x.ndim == 1 and x.dtype == torch.float32 for x in lse)
|
||||
assert all(x.ndim == 2 and x.dtype == torch.float32 for x in outputs)
|
||||
assert all(x.is_contiguous() for x in (*lse, *outputs))
|
||||
weights = torch.stack(lse).softmax(0).unsqueeze(-1)
|
||||
return (torch.stack(outputs) * weights).sum(0), None
|
||||
|
||||
def _check_prefix(
|
||||
self, query_lens, prefix_lens, dtype, padding, heads, magnitude=1, use_fia=False
|
||||
):
|
||||
torch.manual_seed(47)
|
||||
tokens, prefix_tokens = sum(query_lens), sum(prefix_lens)
|
||||
q = torch.randn(tokens + padding, heads, 208, dtype=dtype)[..., :192]
|
||||
k = torch.randn(tokens + padding, heads, 208, dtype=dtype)[..., :192]
|
||||
v = torch.randn(tokens + padding, heads, 144, dtype=dtype)[..., :128]
|
||||
q.mul_(magnitude)
|
||||
k.mul_(magnitude)
|
||||
cached = torch.randn(prefix_tokens + 3, 1, 32, dtype=dtype)
|
||||
cached_rope = torch.randn(prefix_tokens + 3, 1, 1, 64, dtype=dtype)
|
||||
indices = torch.randperm(prefix_tokens + 3)[:prefix_tokens]
|
||||
weight = torch.randn(32, heads * 256, dtype=dtype) / 32**0.5
|
||||
|
||||
def project(x):
|
||||
return x @ weight, None
|
||||
|
||||
backend = AscendAttnBackend.__new__(AscendAttnBackend)
|
||||
backend.use_mla = True
|
||||
backend.use_fia = use_fia
|
||||
backend.is_dllm_model = False
|
||||
backend.qk_nope_head_dim = 128
|
||||
backend.qk_rope_head_dim = 64
|
||||
backend.fia_mask = torch.ones(2048, 2048, dtype=torch.bool).triu(1)
|
||||
backend.forward_metadata = ForwardMetadata(
|
||||
extend_seq_lens_cpu_int=torch.tensor(query_lens, dtype=torch.int32),
|
||||
prefix_lens=torch.tensor(prefix_lens, dtype=torch.int32),
|
||||
flatten_prefix_block_tables=indices,
|
||||
)
|
||||
backend.token_to_kv_pool = SimpleNamespace(
|
||||
get_key_buffer=lambda _: cached,
|
||||
get_value_buffer=lambda _: cached_rope,
|
||||
)
|
||||
layer = SimpleNamespace(
|
||||
layer_id=0,
|
||||
tp_q_head_num=heads,
|
||||
tp_k_head_num=heads,
|
||||
qk_head_dim=192,
|
||||
v_head_dim=128,
|
||||
kv_b_proj=project,
|
||||
scaling=192**-0.5,
|
||||
)
|
||||
batch = SimpleNamespace(
|
||||
global_num_token_non_padded_cpu=tokens,
|
||||
extend_prefix_lens_cpu=prefix_lens,
|
||||
forward_mode=SimpleNamespace(
|
||||
is_target_verify=lambda: False,
|
||||
is_draft_extend_v2=lambda: False,
|
||||
),
|
||||
)
|
||||
module_name = "sglang.srt.hardware_backend.npu.attention.ascend_backend"
|
||||
with (
|
||||
patch(f"{module_name}.is_mla_preprocess_enabled", return_value=False),
|
||||
patch(
|
||||
f"{module_name}.torch_npu.npu_fused_infer_attention_score_v2",
|
||||
side_effect=self._fia_reference,
|
||||
) as fia,
|
||||
patch(
|
||||
f"{module_name}.torch_npu.npu_attention_update",
|
||||
side_effect=self._update_reference,
|
||||
) as update,
|
||||
patch(f"{module_name}.torch_npu.atb", None),
|
||||
):
|
||||
actual = backend.forward_extend(q, k, v, layer, batch, save_kv_cache=False)
|
||||
self.assertEqual(fia.call_count, 2)
|
||||
self.assertEqual(update.call_count, 1)
|
||||
|
||||
prefix_k, prefix_v = (
|
||||
project(cached[indices])[0]
|
||||
.view(prefix_tokens, heads, 256)
|
||||
.split(128, dim=-1)
|
||||
)
|
||||
prefix_k = torch.cat(
|
||||
(prefix_k, cached_rope[indices].flatten(0, 1).expand(-1, heads, -1)),
|
||||
dim=-1,
|
||||
)
|
||||
expected = torch.zeros(tokens + padding, heads, 128, dtype=dtype)
|
||||
q_start = prefix_start = 0
|
||||
for q_len, prefix_len in zip(query_lens, prefix_lens):
|
||||
q_end, prefix_end = q_start + q_len, prefix_start + prefix_len
|
||||
full_k = torch.cat((prefix_k[prefix_start:prefix_end], k[q_start:q_end]))
|
||||
full_v = torch.cat((prefix_v[prefix_start:prefix_end], v[q_start:q_end]))
|
||||
allowed = torch.arange(prefix_len + q_len)[None, :] <= (
|
||||
prefix_len + torch.arange(q_len)[:, None]
|
||||
)
|
||||
expected[q_start:q_end] = (
|
||||
torch.nn.functional.scaled_dot_product_attention(
|
||||
q[q_start:q_end].float().transpose(0, 1),
|
||||
full_k.float().transpose(0, 1),
|
||||
full_v.float().transpose(0, 1),
|
||||
attn_mask=allowed,
|
||||
scale=layer.scaling,
|
||||
)
|
||||
.transpose(0, 1)
|
||||
.to(dtype)
|
||||
)
|
||||
q_start, prefix_start = q_end, prefix_end
|
||||
self.assertEqual(actual.dtype, dtype)
|
||||
self.assertTrue(torch.isfinite(actual).all())
|
||||
torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02)
|
||||
self.assertEqual(torch.count_nonzero(actual[tokens:]).item(), 0)
|
||||
|
||||
def test_prefix_matches_dense_attention(self):
|
||||
for dtype in (torch.float16, torch.bfloat16):
|
||||
for queries, prefixes, padding, heads in (
|
||||
([3, 1, 2], [0, 5, 2], 3, 4),
|
||||
([2, 4, 1], [5, 0, 0], 0, 2),
|
||||
([1, 1], [2, 3], 2, 4),
|
||||
([4], [7], 0, 2),
|
||||
):
|
||||
with self.subTest(dtype=dtype, queries=queries, prefixes=prefixes):
|
||||
self._check_prefix(queries, prefixes, dtype, padding, heads)
|
||||
|
||||
def test_large_logits_merge_stably(self):
|
||||
self._check_prefix([3, 2], [4, 7], torch.bfloat16, 1, 4, magnitude=8)
|
||||
|
||||
def test_prefix_ignores_fia_flag(self):
|
||||
for dtype in (torch.float16, torch.bfloat16):
|
||||
for use_fia in (False, True):
|
||||
with self.subTest(dtype=dtype, use_fia=use_fia):
|
||||
self._check_prefix(
|
||||
[3, 1, 2], [0, 5, 2], dtype, 3, 4, use_fia=use_fia
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user