From aa5c969dc34e4b68e67c50a8528508e5909f4e33 Mon Sep 17 00:00:00 2001 From: Chunan Zeng Date: Sat, 10 Oct 2026 21:41:18 -0700 Subject: [PATCH] [Perf] Fuse BF16 writes to native HND KV cache storage (#43237) Co-authored-by: Claude Opus 5.5 --- .../sglang/kernels/ops/kvcache/cache_ops.py | 36 ++++++++++++++++--- python/sglang/srt/mem_cache/memory_pool.py | 14 ++++++++ 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/python/sglang/kernels/ops/kvcache/cache_ops.py b/python/sglang/kernels/ops/kvcache/cache_ops.py index bc4f71034e..9278641fd0 100644 --- a/python/sglang/kernels/ops/kvcache/cache_ops.py +++ b/python/sglang/kernels/ops/kvcache/cache_ops.py @@ -163,6 +163,11 @@ def reshape_and_cache_flash( k_scale_ptr, v_scale_ptr, block_stride, + value_block_stride, + cache_token_stride: tl.constexpr, + cache_head_stride: tl.constexpr, + value_cache_token_stride: tl.constexpr, + value_cache_head_stride: tl.constexpr, key_stride, value_stride, num_heads, @@ -181,6 +186,7 @@ def reshape_and_cache_flash( Target cache layout: cache: [num_blocks, block_size, num_heads, head_size] + Token and head strides may describe a view of physical HND storage. Each Triton program instance handles: - one token (program_id(0)) @@ -199,7 +205,12 @@ def reshape_and_cache_flash( swa_slot_mapping_ptr: Optional second-stage slot remap for SWA mode. k_scale_ptr: Optional key scaling factor pointer. v_scale_ptr: Optional value scaling factor pointer. - block_stride: Stride between cache blocks. + block_stride: Stride between key cache blocks. + value_block_stride: Stride between value cache blocks. + cache_token_stride: Stride between key cache tokens. + cache_head_stride: Stride between key cache heads. + value_cache_token_stride: Stride between value cache tokens. + value_cache_head_stride: Stride between value cache heads. key_stride: Stride between source key tokens. value_stride: Stride between source value tokens. num_heads: Number of attention heads. @@ -270,10 +281,20 @@ def reshape_and_cache_flash( # target layout # [block_idx, block_offset, head, dim] # ---------------------------------- - tgt = block_idx * block_stride + block_offset * num_heads * head_size + offs - - tl.store(key_cache_ptr + tgt, k, mask=mask) - tl.store(value_cache_ptr + tgt, v, mask=mask) + tgt_k = ( + block_idx * block_stride + + block_offset * cache_token_stride + + head_idx[:, None] * cache_head_stride + + dim_idx[None, :] + ) + tgt_v = ( + block_idx * value_block_stride + + block_offset * value_cache_token_stride + + head_idx[:, None] * value_cache_head_stride + + dim_idx[None, :] + ) + tl.store(key_cache_ptr + tgt_k, k, mask=mask) + tl.store(value_cache_ptr + tgt_v, v, mask=mask) def launch_reshape_and_cache_flash( @@ -326,6 +347,11 @@ def launch_reshape_and_cache_flash( k_scale if k_scale is not None else key, v_scale if v_scale is not None else key, key_cache.stride(0), + value_cache.stride(0), + key_cache.stride(1), + key_cache.stride(2), + value_cache.stride(1), + value_cache.stride(2), key.stride(0), value.stride(0), num_heads, diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 9025fedfa7..3aa56b5fad 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -48,6 +48,7 @@ from sglang.kernels.ops.kvcache.cache_move import ( set_kv_buffer_prefix_valid_tiled, set_kv_buffer_prefix_valid_tiled_fp8, ) +from sglang.kernels.ops.kvcache.cache_ops import launch_reshape_and_cache_flash from sglang.kernels.ops.kvcache.kvcache import can_use_store_cache, store_cache from sglang.kernels.ops.quantization.fp8_kernel import ( fp8_dtype, @@ -2868,6 +2869,19 @@ class MHATokenToKVPool(KVCache): # A slot is [page, :, off, :] (not a contiguous row), so scatter by (page, off). k_buf = self.k_buffer[layer_id - self.start_layer] v_buf = self.v_buffer[layer_id - self.start_layer] + if ( + _is_cuda + and self.head_dim == self.v_head_dim + and cache_k.stride()[1:] == cache_v.stride()[1:] == (self.head_dim, 1) + ): + launch_reshape_and_cache_flash( + cache_k, + cache_v, + k_buf.permute(0, 2, 1, 3), + v_buf.permute(0, 2, 1, 3), + loc, + ) + return pages = loc // self.page_size offs = loc % self.page_size k_buf[pages, :, offs, :] = cache_k