[NPU] Fuse FIA KV-cache K/V writes into one npu_scatter_pa_kv_cache call (#40445)

This commit is contained in:
iridiumine
2026-09-23 21:02:39 +08:00
committed by GitHub
parent abef3efb64
commit 3fdd63a562
2 changed files with 64 additions and 16 deletions
+4
View File
@@ -971,6 +971,10 @@ class Envs:
SGLANG_NPU_W4A4_NEW_PACKING = EnvBool(False)
# Use the graph-safe Triton-Ascend kernel for masked speculative KV commits.
SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE = EnvBool(False)
# Write K and V into the FIA paged KV cache with a single
# npu_scatter_pa_kv_cache call instead of two npu_scatter_nd_update_
# kernels (one write kernel launch instead of two per layer).
SGLANG_NPU_USE_SCATTER_PA_KV_CACHE = EnvBool(False)
# Quantize x to int8 in the dispatch operator (vendor alias consumed by the
# Ascend DeepEP library; the MTP draft-build scopes override it to False).
DEEP_NORMAL_MODE_USE_INT8_QUANT = EnvBool(False)
@@ -104,6 +104,7 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
**kwargs,
):
self.use_fia = get_bool_env_var("ASCEND_USE_FIA", "False")
self.use_scatter_pa_kv_cache = envs.SGLANG_NPU_USE_SCATTER_PA_KV_CACHE.get()
self.use_triton_prefix_kv_cache_store = (
envs.SGLANG_NPU_USE_TRITON_PREFIX_KV_CACHE_STORE.get()
)
@@ -275,22 +276,65 @@ class NPUMHATokenToKVPool(MHATokenToKVPool):
f"v_head_dim={self.v_head_dim}."
)
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
# the singleton FIA layout axis and scatter through an equivalent
# 3-D view; the underlying KV storage and attention layout stay
# unchanged.
loc_indices = loc.contiguous().view(-1, 1)
torch_npu.npu_scatter_nd_update_(
k_buffer_layer.view(-1, self.head_num, self.head_dim),
loc_indices,
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
)
torch_npu.npu_scatter_nd_update_(
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
loc_indices,
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
)
if self.use_scatter_pa_kv_cache:
# Single fused write for K and V: one kernel instead of the
# two aclnnScatterNdUpdate launches below. Bit-identical to
# the unfused path (same flat slot indexing, cross-page safe).
if not hasattr(torch_npu, "npu_scatter_pa_kv_cache"):
raise RuntimeError(
"The installed torch_npu does not provide "
"npu_scatter_pa_kv_cache. Please upgrade torch_npu/CANN "
"or unset SGLANG_NPU_USE_SCATTER_PA_KV_CACHE."
)
if self.store_dtype not in (
torch.float16,
torch.bfloat16,
torch.int8,
):
raise RuntimeError(
"npu_scatter_pa_kv_cache only supports fp16, bf16, and "
f"int8 KV cache, but got {self.store_dtype}."
)
key = cache_k.reshape(-1, self.head_num, self.head_dim).contiguous()
value = cache_v.reshape(-1, self.head_num, self.v_head_dim).contiguous()
key_cache = k_buffer_layer.view(
-1, self.page_size, self.head_num, self.head_dim
)
value_cache = v_buffer_layer.view(
-1, self.page_size, self.head_num, self.v_head_dim
)
# int32/int64 slot mappings are both accepted by the op; avoid
# a per-layer dtype-cast kernel (prefill/spec paths hand over
# int64 locs).
slot_mapping = loc.reshape(-1)
if slot_mapping.dtype not in (torch.int32, torch.int64):
slot_mapping = slot_mapping.to(torch.int32)
torch_npu.npu_scatter_pa_kv_cache(
key,
value,
key_cache,
value_cache,
slot_mapping,
cache_mode="Norm",
)
else:
# aclnnScatterNdUpdate on the deployed CANN rejects the otherwise
# valid 4-D [slot, 1, head, dim] update during tiling. Flatten only
# the singleton FIA layout axis and scatter through an equivalent
# 3-D view; the underlying KV storage and attention layout stay
# unchanged.
loc_indices = loc.contiguous().view(-1, 1)
torch_npu.npu_scatter_nd_update_(
k_buffer_layer.view(-1, self.head_num, self.head_dim),
loc_indices,
cache_k.contiguous().view(num_rows, self.head_num, self.head_dim),
)
torch_npu.npu_scatter_nd_update_(
v_buffer_layer.view(-1, self.head_num, self.v_head_dim),
loc_indices,
cache_v.contiguous().view(num_rows, self.head_num, self.v_head_dim),
)
else:
loc = loc.to(torch.int32)
torch_npu._npu_reshape_and_cache(