[Perf] [NPU] mem_cache: Add optional ScatterPa KV cache writes (#40535)
Co-authored-by: sunguozhong <sun_gz2772@163.com>
This commit is contained in:
@@ -1068,6 +1068,9 @@ class Envs:
|
||||
SGLANG_USE_AG_AFTER_QLORA = EnvBool(False)
|
||||
# Enable int4x2 weights loading
|
||||
SGLANG_NPU_W4A4_NEW_PACKING = EnvBool(False)
|
||||
# Opt in to fused MHA KV writes. Disabled by default because ScatterPa
|
||||
# can increase memory usage during NPU compiled graph capture.
|
||||
SGLANG_NPU_USE_SCATTER_PA_KV_CACHE = EnvBool(False)
|
||||
# Use the graph-safe Triton-Ascend kernel for masked speculative KV commits.
|
||||
SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE = EnvBool(False)
|
||||
# Quantize x to int8 in the dispatch operator (vendor alias consumed by the
|
||||
|
||||
@@ -106,6 +106,7 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
|
||||
**kwargs,
|
||||
):
|
||||
self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False")
|
||||
self.use_scatter_pa_kv_cache = envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get()
|
||||
self.use_triton_prefix_kv_cache_store = (
|
||||
envs.SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE.get()
|
||||
)
|
||||
@@ -286,7 +287,15 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
|
||||
cache_k = cache_k.view(self.store_dtype)
|
||||
cache_v = cache_v.view(self.store_dtype)
|
||||
|
||||
if self.use_fia:
|
||||
# Both FIA and paged-attention buffers share the same physical layout.
|
||||
# Opt in at pool initialization, keeping environment reads outside graph
|
||||
# capture. Otherwise retain the original writers and their memory usage.
|
||||
use_scatter_pa = (
|
||||
self.use_scatter_pa_kv_cache
|
||||
and hasattr(torch_npu, "npu_scatter_pa_kv_cache")
|
||||
and self.store_dtype in (torch.float16, torch.bfloat16, torch.int8)
|
||||
)
|
||||
if self.use_fia or use_scatter_pa:
|
||||
k_buffer_layer = self.k_buffer[layer_id - self.start_layer]
|
||||
v_buffer_layer = self.v_buffer[layer_id - self.start_layer]
|
||||
num_rows = loc.numel()
|
||||
@@ -297,29 +306,43 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
|
||||
or cache_v.numel() != expected_v_numel
|
||||
):
|
||||
raise ValueError(
|
||||
"NPU FIA KV scatter row mismatch: "
|
||||
"NPU KV scatter row mismatch: "
|
||||
f"loc_rows={num_rows}, cache_k_shape={tuple(cache_k.shape)}, "
|
||||
f"cache_v_shape={tuple(cache_v.shape)}, "
|
||||
f"head_num={self.head_num}, head_dim={self.head_dim}, "
|
||||
f"v_head_dim={self.v_head_dim}."
|
||||
)
|
||||
|
||||
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
|
||||
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
|
||||
# the singleton FIA layout axis and scatter through an equivalent
|
||||
# 3-D view; the underlying KV storage and attention layout stay
|
||||
# unchanged.
|
||||
loc_indices = loc.contiguous().view(-1, 1)
|
||||
torch_npu.npu_scatter_nd_update_(
|
||||
k_buffer_layer.view(-1, self.head_num, self.head_dim),
|
||||
loc_indices,
|
||||
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
|
||||
)
|
||||
torch_npu.npu_scatter_nd_update_(
|
||||
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
|
||||
loc_indices,
|
||||
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
|
||||
)
|
||||
if use_scatter_pa:
|
||||
torch_npu.npu_scatter_pa_kv_cache(
|
||||
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
|
||||
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
|
||||
k_buffer_layer.view(
|
||||
-1, self.page_size, self.head_num, self.head_dim
|
||||
),
|
||||
v_buffer_layer.view(
|
||||
-1, self.page_size, self.head_num, self.v_head_dim
|
||||
),
|
||||
loc.reshape(-1).to(torch.int32).contiguous(),
|
||||
cache_mode="Norm",
|
||||
)
|
||||
else:
|
||||
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
|
||||
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
|
||||
# the singleton FIA layout axis and scatter through an equivalent
|
||||
# 3-D view; the underlying KV storage and attention layout stay
|
||||
# unchanged.
|
||||
loc_indices = loc.contiguous().view(-1, 1)
|
||||
torch_npu.npu_scatter_nd_update_(
|
||||
k_buffer_layer.view(-1, self.head_num, self.head_dim),
|
||||
loc_indices,
|
||||
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
|
||||
)
|
||||
torch_npu.npu_scatter_nd_update_(
|
||||
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
|
||||
loc_indices,
|
||||
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
|
||||
)
|
||||
else:
|
||||
loc = loc.to(torch.int32)
|
||||
torch_npu._npu_reshape_and_cache(
|
||||
|
||||
Reference in New Issue
Block a user