[Perf] Fuse BF16 writes to native HND KV cache storage (#43237)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Chunan Zeng
2026-10-11 12:41:18 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 58cb1a2be6
commit aa5c969dc3
2 changed files with 45 additions and 5 deletions
+31 -5
View File
@@ -163,6 +163,11 @@ def reshape_and_cache_flash(
k_scale_ptr,
v_scale_ptr,
block_stride,
value_block_stride,
cache_token_stride: tl.constexpr,
cache_head_stride: tl.constexpr,
value_cache_token_stride: tl.constexpr,
value_cache_head_stride: tl.constexpr,
key_stride,
value_stride,
num_heads,
@@ -181,6 +186,7 @@ def reshape_and_cache_flash(
Target cache layout:
cache: [num_blocks, block_size, num_heads, head_size]
Token and head strides may describe a view of physical HND storage.
Each Triton program instance handles:
- one token (program_id(0))
@@ -199,7 +205,12 @@ def reshape_and_cache_flash(
swa_slot_mapping_ptr: Optional second-stage slot remap for SWA mode.
k_scale_ptr: Optional key scaling factor pointer.
v_scale_ptr: Optional value scaling factor pointer.
block_stride: Stride between cache blocks.
block_stride: Stride between key cache blocks.
value_block_stride: Stride between value cache blocks.
cache_token_stride: Stride between key cache tokens.
cache_head_stride: Stride between key cache heads.
value_cache_token_stride: Stride between value cache tokens.
value_cache_head_stride: Stride between value cache heads.
key_stride: Stride between source key tokens.
value_stride: Stride between source value tokens.
num_heads: Number of attention heads.
@@ -270,10 +281,20 @@ def reshape_and_cache_flash(
# target layout
# [block_idx, block_offset, head, dim]
# ----------------------------------
tgt = block_idx * block_stride + block_offset * num_heads * head_size + offs
tl.store(key_cache_ptr + tgt, k, mask=mask)
tl.store(value_cache_ptr + tgt, v, mask=mask)
tgt_k = (
block_idx * block_stride
+ block_offset * cache_token_stride
+ head_idx[:, None] * cache_head_stride
+ dim_idx[None, :]
)
tgt_v = (
block_idx * value_block_stride
+ block_offset * value_cache_token_stride
+ head_idx[:, None] * value_cache_head_stride
+ dim_idx[None, :]
)
tl.store(key_cache_ptr + tgt_k, k, mask=mask)
tl.store(value_cache_ptr + tgt_v, v, mask=mask)
def launch_reshape_and_cache_flash(
@@ -326,6 +347,11 @@ def launch_reshape_and_cache_flash(
k_scale if k_scale is not None else key,
v_scale if v_scale is not None else key,
key_cache.stride(0),
value_cache.stride(0),
key_cache.stride(1),
key_cache.stride(2),
value_cache.stride(1),
value_cache.stride(2),
key.stride(0),
value.stride(0),
num_heads,
@@ -48,6 +48,7 @@ from sglang.kernels.ops.kvcache.cache_move import (
set_kv_buffer_prefix_valid_tiled,
set_kv_buffer_prefix_valid_tiled_fp8,
)
from sglang.kernels.ops.kvcache.cache_ops import launch_reshape_and_cache_flash
from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache
from sglang.kernels.ops.quantization.fp8_kernel import (
fp8_dtype,
@@ -2868,6 +2869,19 @@ class MHATokenToKVPool(KVCache):
# A slot is [page, :, off, :] (not a contiguous row), so scatter by (page, off).
k_buf = self.k_buffer[layer_id - self.start_layer]
v_buf = self.v_buffer[layer_id - self.start_layer]
if (
_is_cuda
and self.head_dim == self.v_head_dim
and cache_k.stride()[1:] == cache_v.stride()[1:] == (self.head_dim, 1)
):
launch_reshape_and_cache_flash(
cache_k,
cache_v,
k_buf.permute(0, 2, 1, 3),
v_buf.permute(0, 2, 1, 3),
loc,
)
return
pages = loc // self.page_size
offs = loc % self.page_size
k_buf[pages, :, offs, :] = cache_k