[Perf] [NPU] mem_cache: Add optional ScatterPa KV cache writes (#40535)

Co-authored-by: sunguozhong <sun_gz2772@163.com>
This commit is contained in:
Anihc
2026-10-20 09:33:34 +08:00
committed by GitHub
co-authored by sunguozhong
parent 71ee2dc950
commit ebc71e650e
3 changed files with 264 additions and 18 deletions
+3
View File
@@ -1068,6 +1068,9 @@ class Envs:
SGLANG_USE_AG_AFTER_QLORA = EnvBool(False)
# Enable int4x2 weights loading
SGLANG_NPU_W4A4_NEW_PACKING = EnvBool(False)
# Opt in to fused MHA KV writes. Disabled by default because ScatterPa
# can increase memory usage during NPU compiled graph capture.
SGLANG_NPU_USE_SCATTER_PA_KV_CACHE = EnvBool(False)
# Use the graph-safe Triton-Ascend kernel for masked speculative KV commits.
SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE = EnvBool(False)
# Quantize x to int8 in the dispatch operator (vendor alias consumed by the
@@ -106,6 +106,7 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
**kwargs,
):
self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False")
self.use_scatter_pa_kv_cache = envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get()
self.use_triton_prefix_kv_cache_store = (
envs.SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE.get()
)
@@ -286,7 +287,15 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
cache_k = cache_k.view(self.store_dtype)
cache_v = cache_v.view(self.store_dtype)
if self.use_fia:
# Both FIA and paged-attention buffers share the same physical layout.
# Opt in at pool initialization, keeping environment reads outside graph
# capture. Otherwise retain the original writers and their memory usage.
use_scatter_pa = (
self.use_scatter_pa_kv_cache
and hasattr(torch_npu, "npu_scatter_pa_kv_cache")
and self.store_dtype in (torch.float16, torch.bfloat16, torch.int8)
)
if self.use_fia or use_scatter_pa:
k_buffer_layer = self.k_buffer[layer_id - self.start_layer]
v_buffer_layer = self.v_buffer[layer_id - self.start_layer]
num_rows = loc.numel()
@@ -297,29 +306,43 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
or cache_v.numel() != expected_v_numel
):
raise ValueError(
"NPU FIA KV scatter row mismatch: "
"NPU KV scatter row mismatch: "
f"loc_rows={num_rows}, cache_k_shape={tuple(cache_k.shape)}, "
f"cache_v_shape={tuple(cache_v.shape)}, "
f"head_num={self.head_num}, head_dim={self.head_dim}, "
f"v_head_dim={self.v_head_dim}."
)
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
# the singleton FIA layout axis and scatter through an equivalent
# 3-D view; the underlying KV storage and attention layout stay
# unchanged.
loc_indices = loc.contiguous().view(-1, 1)
torch_npu.npu_scatter_nd_update_(
k_buffer_layer.view(-1, self.head_num, self.head_dim),
loc_indices,
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
)
torch_npu.npu_scatter_nd_update_(
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
loc_indices,
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
)
if use_scatter_pa:
torch_npu.npu_scatter_pa_kv_cache(
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
k_buffer_layer.view(
-1, self.page_size, self.head_num, self.head_dim
),
v_buffer_layer.view(
-1, self.page_size, self.head_num, self.v_head_dim
),
loc.reshape(-1).to(torch.int32).contiguous(),
cache_mode="Norm",
)
else:
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
# the singleton FIA layout axis and scatter through an equivalent
# 3-D view; the underlying KV storage and attention layout stay
# unchanged.
loc_indices = loc.contiguous().view(-1, 1)
torch_npu.npu_scatter_nd_update_(
k_buffer_layer.view(-1, self.head_num, self.head_dim),
loc_indices,
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
)
torch_npu.npu_scatter_nd_update_(
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
loc_indices,
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
)
else:
loc = loc.to(torch.int32)
torch_npu._npu_reshape_and_cache(
@@ -0,0 +1,220 @@
import os
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
from sglang.test.ci.ci_register import register_npu_ci
register_npu_ci(est_time=1, suite="stage-a-unit-test-npu")
from sglang.srt.hardware_backend.npu import memory_pool_npu
from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMHATokenToKVPool
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
class TestNPUScatterPaKVCache(unittest.TestCase):
def setUp(self):
self.enterContext(
patch.dict(os.environ, {"SGLANG_NPU_USE_SCATTER_PA_KV_CACHE": "1"})
)
@staticmethod
def _make_pool(*, use_fia=True, dtype=torch.bfloat16):
pool = object.__new__(NPUMHATokenToKVPool)
pool.use_fia = use_fia
pool.use_scatter_pa_kv_cache = (
memory_pool_npu.envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get()
)
pool.dtype = dtype
pool.store_dtype = dtype
pool.start_layer = 0
pool.page_size = 4
pool.head_num = 2
pool.head_dim = 3
pool.v_head_dim = 5
if use_fia:
pool.k_buffer = [torch.zeros((8, 1, 2, 3), dtype=dtype)]
pool.v_buffer = [torch.zeros((8, 1, 2, 5), dtype=dtype)]
else:
pool.k_buffer = [torch.zeros((2, 4, 2, 3), dtype=dtype)]
pool.v_buffer = [torch.zeros((2, 4, 2, 5), dtype=dtype)]
return pool
@staticmethod
def _inputs(dtype=torch.bfloat16):
layer = SimpleNamespace(layer_id=0)
loc = torch.tensor([6, 1], dtype=torch.int64)
cache_k = torch.arange(12, dtype=torch.float32).to(dtype).view(2, 2, 3)
cache_v = torch.arange(20, dtype=torch.float32).to(dtype).view(2, 2, 5)
return layer, loc, cache_k, cache_v
def test_enabled_fia_write(self):
for dtype in (torch.float16, torch.bfloat16, torch.int8):
with self.subTest(dtype=dtype):
self._check_default_write(use_fia=True, dtype=dtype)
def test_enabled_paged_write(self):
for dtype in (torch.float16, torch.bfloat16, torch.int8):
with self.subTest(dtype=dtype):
self._check_default_write(use_fia=False, dtype=dtype)
def _check_default_write(self, *, use_fia, dtype):
pool = self._make_pool(use_fia=use_fia, dtype=dtype)
layer, loc, cache_k, cache_v = self._inputs(dtype=dtype)
# Projections can expose non-contiguous views of fused QKV outputs.
cache_k = cache_k.transpose(1, 2).contiguous().transpose(1, 2)
cache_v = cache_v.transpose(1, 2).contiguous().transpose(1, 2)
def scatter_pa(key, value, key_cache, value_cache, slot_mapping, **kwargs):
self.assertEqual(kwargs, {"cache_mode": "Norm"})
key_cache.view(-1, pool.head_num, pool.head_dim).index_copy_(
0, slot_mapping.long(), key
)
value_cache.view(-1, pool.head_num, pool.v_head_dim).index_copy_(
0, slot_mapping.long(), value
)
fake_torch_npu = SimpleNamespace(
npu_scatter_pa_kv_cache=MagicMock(side_effect=scatter_pa),
npu_scatter_nd_update_=MagicMock(),
)
with patch.object(memory_pool_npu, "torch_npu", fake_torch_npu, create=True):
pool.set_kv_buffer(
layer,
loc,
cache_k,
cache_v,
)
fake_torch_npu.npu_scatter_nd_update_.assert_not_called()
call = fake_torch_npu.npu_scatter_pa_kv_cache.call_args
self.assertIsNotNone(call)
key, value, key_cache, value_cache, slot_mapping = call.args
self.assertEqual(key.shape, (2, 2, 3))
self.assertEqual(value.shape, (2, 2, 5))
self.assertEqual(key_cache.shape, (2, 4, 2, 3))
self.assertEqual(value_cache.shape, (2, 4, 2, 5))
self.assertEqual(slot_mapping.dtype, torch.int32)
self.assertTrue(all(tensor.is_contiguous() for tensor in (key, value)))
self.assertTrue(slot_mapping.is_contiguous())
self.assertEqual(call.kwargs, {"cache_mode": "Norm"})
torch.testing.assert_close(
pool.k_buffer[0].view(-1, pool.head_num, pool.head_dim)[loc], cache_k
)
torch.testing.assert_close(
pool.v_buffer[0].view(-1, pool.head_num, pool.v_head_dim)[loc], cache_v
)
def test_missing_operator_uses_existing_writers(self):
for use_fia in (True, False):
with self.subTest(use_fia=use_fia):
pool = self._make_pool(use_fia=use_fia)
layer, loc, cache_k, cache_v = self._inputs()
fake = SimpleNamespace(
npu_scatter_nd_update_=MagicMock(),
_npu_reshape_and_cache=MagicMock(),
)
with patch.object(memory_pool_npu, "torch_npu", fake, create=True):
pool.set_kv_buffer(layer, loc, cache_k, cache_v)
self.assertEqual(
fake.npu_scatter_nd_update_.call_count, 2 if use_fia else 0
)
self.assertEqual(
fake._npu_reshape_and_cache.call_count, 0 if use_fia else 1
)
def test_unset_or_disabled_flag_uses_existing_writers(self):
for value in (None, "0"):
for use_fia in (True, False):
with self.subTest(value=value, use_fia=use_fia):
with patch.dict(os.environ):
os.environ.pop("SGLANG_NPU_USE_SCATTER_PA_KV_CACHE", None)
if value is not None:
os.environ["SGLANG_NPU_USE_SCATTER_PA_KV_CACHE"] = value
pool = self._make_pool(use_fia=use_fia)
self.assertFalse(pool.use_scatter_pa_kv_cache)
layer, loc, cache_k, cache_v = self._inputs()
fake = SimpleNamespace(
npu_scatter_pa_kv_cache=MagicMock(),
npu_scatter_nd_update_=MagicMock(),
_npu_reshape_and_cache=MagicMock(),
)
with patch.object(memory_pool_npu, "torch_npu", fake, create=True):
pool.set_kv_buffer(layer, loc, cache_k, cache_v)
fake.npu_scatter_pa_kv_cache.assert_not_called()
self.assertEqual(
fake.npu_scatter_nd_update_.call_count, 2 if use_fia else 0
)
self.assertEqual(
fake._npu_reshape_and_cache.call_count, 0 if use_fia else 1
)
def test_unsupported_storage_dtype_uses_existing_writers(self):
for use_fia in (True, False):
with self.subTest(use_fia=use_fia):
pool = self._make_pool(use_fia=use_fia, dtype=torch.uint8)
layer, loc, cache_k, cache_v = self._inputs(dtype=torch.uint8)
fake = SimpleNamespace(
npu_scatter_pa_kv_cache=MagicMock(),
npu_scatter_nd_update_=MagicMock(),
_npu_reshape_and_cache=MagicMock(),
)
with patch.object(memory_pool_npu, "torch_npu", fake, create=True):
pool.set_kv_buffer(layer, loc, cache_k, cache_v)
fake.npu_scatter_pa_kv_cache.assert_not_called()
self.assertEqual(
fake.npu_scatter_nd_update_.call_count, 2 if use_fia else 0
)
self.assertEqual(
fake._npu_reshape_and_cache.call_count, 0 if use_fia else 1
)
def test_row_mismatch_is_rejected(self):
pool = self._make_pool()
layer, loc, cache_k, cache_v = self._inputs()
fake = SimpleNamespace(npu_scatter_pa_kv_cache=MagicMock())
with (
patch.object(memory_pool_npu, "torch_npu", fake, create=True),
self.assertRaisesRegex(ValueError, "row mismatch"),
):
pool.set_kv_buffer(layer, loc[:1], cache_k, cache_v)
fake.npu_scatter_pa_kv_cache.assert_not_called()
def test_swa_pool_forwards_locations_to_selected_npu_pool(self):
inner_pool = SimpleNamespace(
set_kv_buffer=MagicMock(),
)
pool = object.__new__(SWAKVPool)
pool.full_kv_pool = SimpleNamespace(set_kv_buffer=MagicMock())
pool.swa_kv_pool = inner_pool
pool.layers_mapping = {3: (0, True)}
layer = SimpleNamespace(layer_id=3)
loc = torch.tensor([4, 5])
swa_loc = torch.tensor([0, 1])
cache_k = torch.zeros((2, 2, 3), dtype=torch.bfloat16)
cache_v = torch.zeros((2, 2, 5), dtype=torch.bfloat16)
pool.set_kv_buffer(
layer,
KVWriteLoc(loc, swa_loc),
cache_k,
cache_v,
)
inner_pool.set_kv_buffer.assert_called_once_with(
None,
swa_loc,
cache_k,
cache_v,
1.0,
1.0,
layer_id_override=0,
)
if __name__ == "__main__":
unittest.main()