From ebc71e650e28faf1dbdd0ecabde68d638ebdcefc Mon Sep 17 00:00:00 2001 From: Anihc <114674170+Guozhong23@users.noreply.github.com> Date: Tue, 20 Oct 2026 09:33:34 +0800 Subject: [PATCH] [Perf] [NPU] mem_cache: Add optional ScatterPa KV cache writes (#40535) Co-authored-by: sunguozhong --- python/sglang/srt/environ.py | 3 + .../hardware_backend/npu/memory_pool_npu.py | 59 +++-- .../unit/npu/test_npu_scatter_pa_kv_cache.py | 220 ++++++++++++++++++ 3 files changed, 264 insertions(+), 18 deletions(-) create mode 100644 test/registered/unit/npu/test_npu_scatter_pa_kv_cache.py diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index 90b5ab122e..da44f5811e 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1068,6 +1068,9 @@ class Envs: SGLANG_USE_AG_AFTER_QLORA = EnvBool(False) # Enable int4x2 weights loading SGLANG_NPU_W4A4_NEW_PACKING = EnvBool(False) + # Opt in to fused MHA KV writes. Disabled by default because ScatterPa + # can increase memory usage during NPU compiled graph capture. + SGLANG_NPU_USE_SCATTER_PA_KV_CACHE = EnvBool(False) # Use the graph-safe Triton-Ascend kernel for masked speculative KV commits. SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE = EnvBool(False) # Quantize x to int8 in the dispatch operator (vendor alias consumed by the diff --git a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py index 904e241c3c..e00f398571 100644 --- a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py +++ b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py @@ -106,6 +106,7 @@ class NPUMHATokenToKVPool(MHATokenToKVPool): **kwargs, ): self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False") + self.use_scatter_pa_kv_cache = envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get() self.use_triton_prefix_kv_cache_store = ( envs.SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE.get() ) @@ -286,7 +287,15 @@ class NPUMHATokenToKVPool(MHATokenToKVPool): cache_k = cache_k.view(self.store_dtype) cache_v = cache_v.view(self.store_dtype) - if self.use_fia: + # Both FIA and paged-attention buffers share the same physical layout. + # Opt in at pool initialization, keeping environment reads outside graph + # capture. Otherwise retain the original writers and their memory usage. + use_scatter_pa = ( + self.use_scatter_pa_kv_cache + and hasattr(torch_npu, "npu_scatter_pa_kv_cache") + and self.store_dtype in (torch.float16, torch.bfloat16, torch.int8) + ) + if self.use_fia or use_scatter_pa: k_buffer_layer = self.k_buffer[layer_id - self.start_layer] v_buffer_layer = self.v_buffer[layer_id - self.start_layer] num_rows = loc.numel() @@ -297,29 +306,43 @@ class NPUMHATokenToKVPool(MHATokenToKVPool): or cache_v.numel() != expected_v_numel ): raise ValueError( - "NPU FIA KV scatter row mismatch: " + "NPU KV scatter row mismatch: " f"loc_rows={num_rows}, cache_k_shape={tuple(cache_k.shape)}, " f"cache_v_shape={tuple(cache_v.shape)}, " f"head_num={self.head_num}, head_dim={self.head_dim}, " f"v_head_dim={self.v_head_dim}." ) - # aclnnScatterNdUpdate on the deployed CANN rejects the otherwise - # valid 4-D [slot, 1, head, dim] update during tiling. Flatten only - # the singleton FIA layout axis and scatter through an equivalent - # 3-D view; the underlying KV storage and attention layout stay - # unchanged. - loc_indices = loc.contiguous().view(-1, 1) - torch_npu.npu_scatter_nd_update_( - k_buffer_layer.view(-1, self.head_num, self.head_dim), - loc_indices, - cache_k.contiguous().view(num_rows, self.head_num, self.head_dim), - ) - torch_npu.npu_scatter_nd_update_( - v_buffer_layer.view(-1, self.head_num, self.v_head_dim), - loc_indices, - cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim), - ) + if use_scatter_pa: + torch_npu.npu_scatter_pa_kv_cache( + cache_k.contiguous().view(num_rows, self.head_num, self.head_dim), + cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim), + k_buffer_layer.view( + -1, self.page_size, self.head_num, self.head_dim + ), + v_buffer_layer.view( + -1, self.page_size, self.head_num, self.v_head_dim + ), + loc.reshape(-1).to(torch.int32).contiguous(), + cache_mode="Norm", + ) + else: + # aclnnScatterNdUpdate on the deployed CANN rejects the otherwise + # valid 4-D [slot, 1, head, dim] update during tiling. Flatten only + # the singleton FIA layout axis and scatter through an equivalent + # 3-D view; the underlying KV storage and attention layout stay + # unchanged. + loc_indices = loc.contiguous().view(-1, 1) + torch_npu.npu_scatter_nd_update_( + k_buffer_layer.view(-1, self.head_num, self.head_dim), + loc_indices, + cache_k.contiguous().view(num_rows, self.head_num, self.head_dim), + ) + torch_npu.npu_scatter_nd_update_( + v_buffer_layer.view(-1, self.head_num, self.v_head_dim), + loc_indices, + cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim), + ) else: loc = loc.to(torch.int32) torch_npu._npu_reshape_and_cache( diff --git a/test/registered/unit/npu/test_npu_scatter_pa_kv_cache.py b/test/registered/unit/npu/test_npu_scatter_pa_kv_cache.py new file mode 100644 index 0000000000..c18b1b8d89 --- /dev/null +++ b/test/registered/unit/npu/test_npu_scatter_pa_kv_cache.py @@ -0,0 +1,220 @@ +import os +import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import torch + +from sglang.test.ci.ci_register import register_npu_ci + +register_npu_ci(est_time=1, suite="stage-a-unit-test-npu") + +from sglang.srt.hardware_backend.npu import memory_pool_npu +from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMHATokenToKVPool +from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool + + +class TestNPUScatterPaKVCache(unittest.TestCase): + def setUp(self): + self.enterContext( + patch.dict(os.environ, {"SGLANG_NPU_USE_SCATTER_PA_KV_CACHE": "1"}) + ) + + @staticmethod + def _make_pool(*, use_fia=True, dtype=torch.bfloat16): + pool = object.__new__(NPUMHATokenToKVPool) + pool.use_fia = use_fia + pool.use_scatter_pa_kv_cache = ( + memory_pool_npu.envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get() + ) + pool.dtype = dtype + pool.store_dtype = dtype + pool.start_layer = 0 + pool.page_size = 4 + pool.head_num = 2 + pool.head_dim = 3 + pool.v_head_dim = 5 + if use_fia: + pool.k_buffer = [torch.zeros((8, 1, 2, 3), dtype=dtype)] + pool.v_buffer = [torch.zeros((8, 1, 2, 5), dtype=dtype)] + else: + pool.k_buffer = [torch.zeros((2, 4, 2, 3), dtype=dtype)] + pool.v_buffer = [torch.zeros((2, 4, 2, 5), dtype=dtype)] + return pool + + @staticmethod + def _inputs(dtype=torch.bfloat16): + layer = SimpleNamespace(layer_id=0) + loc = torch.tensor([6, 1], dtype=torch.int64) + cache_k = torch.arange(12, dtype=torch.float32).to(dtype).view(2, 2, 3) + cache_v = torch.arange(20, dtype=torch.float32).to(dtype).view(2, 2, 5) + return layer, loc, cache_k, cache_v + + def test_enabled_fia_write(self): + for dtype in (torch.float16, torch.bfloat16, torch.int8): + with self.subTest(dtype=dtype): + self._check_default_write(use_fia=True, dtype=dtype) + + def test_enabled_paged_write(self): + for dtype in (torch.float16, torch.bfloat16, torch.int8): + with self.subTest(dtype=dtype): + self._check_default_write(use_fia=False, dtype=dtype) + + def _check_default_write(self, *, use_fia, dtype): + pool = self._make_pool(use_fia=use_fia, dtype=dtype) + layer, loc, cache_k, cache_v = self._inputs(dtype=dtype) + # Projections can expose non-contiguous views of fused QKV outputs. + cache_k = cache_k.transpose(1, 2).contiguous().transpose(1, 2) + cache_v = cache_v.transpose(1, 2).contiguous().transpose(1, 2) + + def scatter_pa(key, value, key_cache, value_cache, slot_mapping, **kwargs): + self.assertEqual(kwargs, {"cache_mode": "Norm"}) + key_cache.view(-1, pool.head_num, pool.head_dim).index_copy_( + 0, slot_mapping.long(), key + ) + value_cache.view(-1, pool.head_num, pool.v_head_dim).index_copy_( + 0, slot_mapping.long(), value + ) + + fake_torch_npu = SimpleNamespace( + npu_scatter_pa_kv_cache=MagicMock(side_effect=scatter_pa), + npu_scatter_nd_update_=MagicMock(), + ) + + with patch.object(memory_pool_npu, "torch_npu", fake_torch_npu, create=True): + pool.set_kv_buffer( + layer, + loc, + cache_k, + cache_v, + ) + + fake_torch_npu.npu_scatter_nd_update_.assert_not_called() + call = fake_torch_npu.npu_scatter_pa_kv_cache.call_args + self.assertIsNotNone(call) + key, value, key_cache, value_cache, slot_mapping = call.args + self.assertEqual(key.shape, (2, 2, 3)) + self.assertEqual(value.shape, (2, 2, 5)) + self.assertEqual(key_cache.shape, (2, 4, 2, 3)) + self.assertEqual(value_cache.shape, (2, 4, 2, 5)) + self.assertEqual(slot_mapping.dtype, torch.int32) + self.assertTrue(all(tensor.is_contiguous() for tensor in (key, value))) + self.assertTrue(slot_mapping.is_contiguous()) + self.assertEqual(call.kwargs, {"cache_mode": "Norm"}) + torch.testing.assert_close( + pool.k_buffer[0].view(-1, pool.head_num, pool.head_dim)[loc], cache_k + ) + torch.testing.assert_close( + pool.v_buffer[0].view(-1, pool.head_num, pool.v_head_dim)[loc], cache_v + ) + + def test_missing_operator_uses_existing_writers(self): + for use_fia in (True, False): + with self.subTest(use_fia=use_fia): + pool = self._make_pool(use_fia=use_fia) + layer, loc, cache_k, cache_v = self._inputs() + fake = SimpleNamespace( + npu_scatter_nd_update_=MagicMock(), + _npu_reshape_and_cache=MagicMock(), + ) + with patch.object(memory_pool_npu, "torch_npu", fake, create=True): + pool.set_kv_buffer(layer, loc, cache_k, cache_v) + self.assertEqual( + fake.npu_scatter_nd_update_.call_count, 2 if use_fia else 0 + ) + self.assertEqual( + fake._npu_reshape_and_cache.call_count, 0 if use_fia else 1 + ) + + def test_unset_or_disabled_flag_uses_existing_writers(self): + for value in (None, "0"): + for use_fia in (True, False): + with self.subTest(value=value, use_fia=use_fia): + with patch.dict(os.environ): + os.environ.pop("SGLANG_NPU_USE_SCATTER_PA_KV_CACHE", None) + if value is not None: + os.environ["SGLANG_NPU_USE_SCATTER_PA_KV_CACHE"] = value + pool = self._make_pool(use_fia=use_fia) + self.assertFalse(pool.use_scatter_pa_kv_cache) + layer, loc, cache_k, cache_v = self._inputs() + fake = SimpleNamespace( + npu_scatter_pa_kv_cache=MagicMock(), + npu_scatter_nd_update_=MagicMock(), + _npu_reshape_and_cache=MagicMock(), + ) + with patch.object(memory_pool_npu, "torch_npu", fake, create=True): + pool.set_kv_buffer(layer, loc, cache_k, cache_v) + fake.npu_scatter_pa_kv_cache.assert_not_called() + self.assertEqual( + fake.npu_scatter_nd_update_.call_count, 2 if use_fia else 0 + ) + self.assertEqual( + fake._npu_reshape_and_cache.call_count, 0 if use_fia else 1 + ) + + def test_unsupported_storage_dtype_uses_existing_writers(self): + for use_fia in (True, False): + with self.subTest(use_fia=use_fia): + pool = self._make_pool(use_fia=use_fia, dtype=torch.uint8) + layer, loc, cache_k, cache_v = self._inputs(dtype=torch.uint8) + fake = SimpleNamespace( + npu_scatter_pa_kv_cache=MagicMock(), + npu_scatter_nd_update_=MagicMock(), + _npu_reshape_and_cache=MagicMock(), + ) + with patch.object(memory_pool_npu, "torch_npu", fake, create=True): + pool.set_kv_buffer(layer, loc, cache_k, cache_v) + fake.npu_scatter_pa_kv_cache.assert_not_called() + self.assertEqual( + fake.npu_scatter_nd_update_.call_count, 2 if use_fia else 0 + ) + self.assertEqual( + fake._npu_reshape_and_cache.call_count, 0 if use_fia else 1 + ) + + def test_row_mismatch_is_rejected(self): + pool = self._make_pool() + layer, loc, cache_k, cache_v = self._inputs() + fake = SimpleNamespace(npu_scatter_pa_kv_cache=MagicMock()) + with ( + patch.object(memory_pool_npu, "torch_npu", fake, create=True), + self.assertRaisesRegex(ValueError, "row mismatch"), + ): + pool.set_kv_buffer(layer, loc[:1], cache_k, cache_v) + fake.npu_scatter_pa_kv_cache.assert_not_called() + + def test_swa_pool_forwards_locations_to_selected_npu_pool(self): + inner_pool = SimpleNamespace( + set_kv_buffer=MagicMock(), + ) + pool = object.__new__(SWAKVPool) + pool.full_kv_pool = SimpleNamespace(set_kv_buffer=MagicMock()) + pool.swa_kv_pool = inner_pool + pool.layers_mapping = {3: (0, True)} + layer = SimpleNamespace(layer_id=3) + loc = torch.tensor([4, 5]) + swa_loc = torch.tensor([0, 1]) + cache_k = torch.zeros((2, 2, 3), dtype=torch.bfloat16) + cache_v = torch.zeros((2, 2, 5), dtype=torch.bfloat16) + + pool.set_kv_buffer( + layer, + KVWriteLoc(loc, swa_loc), + cache_k, + cache_v, + ) + + inner_pool.set_kv_buffer.assert_called_once_with( + None, + swa_loc, + cache_k, + cache_v, + 1.0, + 1.0, + layer_id_override=0, + ) + + +if __name__ == "__main__": + unittest.main()