[Perf] [NPU] mem_cache: Add optional ScatterPa KV cache writes (#40535)

Co-authored-by: sunguozhong <sun_gz2772@163.com>
This commit is contained in:
Anihc
2026-10-20 09:33:34 +08:00
committed by GitHub
co-authored by sunguozhong
parent 71ee2dc950
commit ebc71e650e
3 changed files with 264 additions and 18 deletions
+3
View File
@@ -1068,6 +1068,9 @@ class Envs:
SGLANG_USE_AG_AFTER_QLORA = EnvBool(False)
# Enable int4x2 weights loading
SGLANG_NPU_W4A4_NEW_PACKING = EnvBool(False)
# Opt in to fused MHA KV writes. Disabled by default because ScatterPa
# can increase memory usage during NPU compiled graph capture.
SGLANG_NPU_USE_SCATTER_PA_KV_CACHE = EnvBool(False)
# Use the graph-safe Triton-Ascend kernel for masked speculative KV commits.
SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE = EnvBool(False)
# Quantize x to int8 in the dispatch operator (vendor alias consumed by the
@@ -106,6 +106,7 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
**kwargs,
):
self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False")
self.use_scatter_pa_kv_cache = envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get()
self.use_triton_prefix_kv_cache_store = (
envs.SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE.get()
)
@@ -286,7 +287,15 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
cache_k = cache_k.view(self.store_dtype)
cache_v = cache_v.view(self.store_dtype)
if self.use_fia:
# Both FIA and paged-attention buffers share the same physical layout.
# Opt in at pool initialization, keeping environment reads outside graph
# capture. Otherwise retain the original writers and their memory usage.
use_scatter_pa = (
self.use_scatter_pa_kv_cache
and hasattr(torch_npu, "npu_scatter_pa_kv_cache")
and self.store_dtype in (torch.float16, torch.bfloat16, torch.int8)
)
if self.use_fia or use_scatter_pa:
k_buffer_layer = self.k_buffer[layer_id - self.start_layer]
v_buffer_layer = self.v_buffer[layer_id - self.start_layer]
num_rows = loc.numel()
@@ -297,29 +306,43 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
or cache_v.numel() != expected_v_numel
):
raise ValueError(
"NPU FIA KV scatter row mismatch: "
"NPU KV scatter row mismatch: "
f"loc_rows={num_rows}, cache_k_shape={tuple(cache_k.shape)}, "
f"cache_v_shape={tuple(cache_v.shape)}, "
f"head_num={self.head_num}, head_dim={self.head_dim}, "
f"v_head_dim={self.v_head_dim}."
)
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
# the singleton FIA layout axis and scatter through an equivalent
# 3-D view; the underlying KV storage and attention layout stay
# unchanged.
loc_indices = loc.contiguous().view(-1, 1)
torch_npu.npu_scatter_nd_update_(
k_buffer_layer.view(-1, self.head_num, self.head_dim),
loc_indices,
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
)
torch_npu.npu_scatter_nd_update_(
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
loc_indices,
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
)
if use_scatter_pa:
torch_npu.npu_scatter_pa_kv_cache(
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
k_buffer_layer.view(
-1, self.page_size, self.head_num, self.head_dim
),
v_buffer_layer.view(
-1, self.page_size, self.head_num, self.v_head_dim
),
loc.reshape(-1).to(torch.int32).contiguous(),
cache_mode="Norm",
)
else:
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
# the singleton FIA layout axis and scatter through an equivalent
# 3-D view; the underlying KV storage and attention layout stay
# unchanged.
loc_indices = loc.contiguous().view(-1, 1)
torch_npu.npu_scatter_nd_update_(
k_buffer_layer.view(-1, self.head_num, self.head_dim),
loc_indices,
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
)
torch_npu.npu_scatter_nd_update_(
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
loc_indices,
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
)
else:
loc = loc.to(torch.int32)
torch_npu._npu_reshape_and_cache(