[unified-memory] Build paged KV views through one helper (2/7) (#40327)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-01 17:08:42 -07:00
committed by GitHub
co-authored by Cheng Wan
parent d3dbb71e5a
commit 5df52e6382
13 changed files with 704 additions and 47 deletions
@@ -33,6 +33,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLAMultiStepDraftBackend,
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.runtime_context import get_parallel
from sglang.srt.utils import is_flashinfer_available
@@ -226,7 +227,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
query = query.unsqueeze(1)
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
metadata = (
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
@@ -34,6 +34,7 @@ from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.kv_index_translator import KVReadTables
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
@@ -1305,20 +1306,33 @@ class FlashAttentionBackend(AttentionBackend):
head_group_num: int = 1,
) -> tuple[torch.Tensor, torch.Tensor]:
key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
return (
key_cache.view(
key_cache = paged_kv_view(
key_cache, self.page_size, layer.tp_k_head_num, layer.head_dim
)
value_cache = paged_kv_view(
value_cache, self.page_size, layer.tp_v_head_num, layer.v_head_dim
)
if head_group_num != 1:
# Reinterpret each page's heads as `head_group_num` pseudo-pages.
assert key_cache.is_contiguous() and value_cache.is_contiguous(), (
f"head_group_num={head_group_num} needs a contiguous paged KV "
"view; this one carries the pool's slot stride "
f"(key {key_cache.stride()}, value {value_cache.stride()}), "
"which no reshape of the head dimension can express."
)
key_cache = key_cache.view(
-1,
self.page_size,
layer.tp_k_head_num // head_group_num,
layer.head_dim,
),
value_cache.view(
)
value_cache = value_cache.view(
-1,
self.page_size,
layer.tp_v_head_num // head_group_num,
layer.v_head_dim,
),
)
)
return key_cache, value_cache
def prepare_paged_mha_query(
self,
@@ -1695,9 +1709,12 @@ class FlashAttentionBackend(AttentionBackend):
q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
# The suffix table stores physical token slots, so expose
# the paged cache as page-size-one blocks.
k_cache=key_cache.view(-1, 1, layer.tp_k_head_num, layer.head_dim),
v_cache=value_cache.view(
-1, 1, layer.tp_v_head_num, layer.v_head_dim
k_cache=paged_view(
key_cache.view(-1, layer.tp_k_head_num, layer.head_dim), 1
),
v_cache=paged_view(
value_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim),
1,
),
page_table=self.forward_metadata_spec_decode_expand.page_table,
cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32,
@@ -1797,15 +1814,8 @@ class FlashAttentionBackend(AttentionBackend):
)
k_rope = kv_cache[:, :, layer.v_head_dim :]
c_kv = kv_cache[:, :, : layer.v_head_dim]
k_rope_cache = k_rope.view(
-1,
self.page_size,
layer.tp_k_head_num,
layer.head_dim - layer.v_head_dim,
)
c_kv_cache = c_kv.view(
-1, self.page_size, layer.tp_v_head_num, layer.v_head_dim
)
k_rope_cache = paged_view(k_rope, self.page_size)
c_kv_cache = paged_view(c_kv, self.page_size)
if q_rope is not None:
q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim)
q_rope = q_rope.view(
@@ -2209,15 +2219,8 @@ class FlashAttentionBackend(AttentionBackend):
kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype)
k_rope = kv_cache[:, :, layer.v_head_dim :]
c_kv = kv_cache[:, :, : layer.v_head_dim]
k_rope_cache = k_rope.view(
-1,
self.page_size,
layer.tp_k_head_num,
layer.head_dim - layer.v_head_dim,
)
c_kv_cache = c_kv.view(
-1, self.page_size, layer.tp_v_head_num, layer.v_head_dim
)
k_rope_cache = paged_view(k_rope, self.page_size)
c_kv_cache = paged_view(c_kv, self.page_size)
if q_rope is not None:
q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim)
@@ -19,6 +19,7 @@ from sglang.kernels.ops.attention.utils import (
from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant
from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
from sglang.srt.mem_cache.layout.paged_view import paged_view
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.runtime_context import get_parallel, get_spec
@@ -461,7 +462,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
reshape_q_fp8 = reshape_q_fp8_2d.reshape(q_shape)
o, _ = flash_mla_with_kvcache(
q=reshape_q_fp8,
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
k_cache=paged_view(k_cache, PAGE_SIZE),
block_table=self.forward_metadata.block_kv_indices[:bs],
cache_seqlens=forward_batch.seq_lens.to(torch.int32),
head_dim_v=self.kv_lora_rank,
@@ -478,7 +479,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
# todo: need check all causal True or False?
o, lse = flash_mla_with_kvcache(
q=reshape_q,
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
k_cache=paged_view(k_cache, PAGE_SIZE),
block_table=self.forward_metadata.block_kv_indices[:bs],
cache_seqlens=forward_batch.seq_lens.to(torch.int32),
head_dim_v=self.kv_lora_rank,
@@ -552,7 +553,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
reshape_q_fp8 = reshape_q_fp8_2d.reshape(q_shape)
o, _ = flash_mla_with_kvcache(
q=reshape_q_fp8,
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
k_cache=paged_view(k_cache, PAGE_SIZE),
block_table=self.forward_metadata.block_kv_indices[:bs],
cache_seqlens=cache_seqlens,
head_dim_v=self.kv_lora_rank,
@@ -566,7 +567,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
else:
o, _ = flash_mla_with_kvcache(
q=reshape_q,
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
k_cache=paged_view(k_cache, PAGE_SIZE),
block_table=self.forward_metadata.block_kv_indices[:bs],
cache_seqlens=cache_seqlens,
head_dim_v=self.kv_lora_rank,
@@ -49,6 +49,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
TRTLLMMLAMultiStepDraftBackend,
)
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.runtime_context import (
get_parallel,
get_resources,
@@ -411,7 +412,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
query = query.unsqueeze(1)
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
metadata = (
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
or self.forward_decode_metadata
@@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
KVCacheAttentionAccessKind,
)
from sglang.srt.layers.radix_attention import AttentionType
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
@@ -1314,11 +1315,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
layer: RadixAttention,
head_dim: int,
) -> tuple[torch.Tensor, torch.Tensor]:
k_cache = k_cache.view(
-1, self.page_size, layer.tp_k_head_num, head_dim
k_cache = paged_kv_view(
k_cache, self.page_size, layer.tp_k_head_num, head_dim
).permute(0, 2, 1, 3)
v_cache = v_cache.view(
-1, self.page_size, layer.tp_v_head_num, head_dim
v_cache = paged_kv_view(
v_cache, self.page_size, layer.tp_v_head_num, head_dim
).permute(0, 2, 1, 3)
if layer.tp_k_head_num == 1:
k_cache = canonicalize_stride(k_cache)
@@ -1632,11 +1633,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
k_cache_raw, v_cache_raw, layer, layer.head_dim
)
else:
k_cache = k_cache_raw.view(
-1, self.page_size, layer.tp_k_head_num, layer.head_dim
k_cache = paged_kv_view(
k_cache_raw, self.page_size, layer.tp_k_head_num, layer.head_dim
)
v_cache = v_cache_raw.view(
-1, self.page_size, layer.tp_v_head_num, layer.head_dim
v_cache = paged_kv_view(
v_cache_raw, self.page_size, layer.tp_v_head_num, layer.head_dim
)
kv_cache = (k_cache, v_cache)
@@ -57,6 +57,7 @@ from sglang.srt.layers.attention.kv_shard_hooks import (
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
from sglang.srt.layers.dcp.layout import get_dcp_lens
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
is_in_breakable_cuda_graph,
@@ -1383,7 +1384,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
# Prepare KV cache inline
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
# Get metadata
metadata = (
@@ -1599,7 +1600,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
bs = forward_batch.batch_size
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
q = q.to(self.data_type)
@@ -0,0 +1,48 @@
"""Paged views over per-slot KV buffers.
Attention kernels take a paged ``[num_pages, page_size, ...]`` cache. These
helpers split a pool buffer's slot dim into pages from the buffer's own
strides, so a strided per-layer view keeps its slot stride at every page size.
"""
import torch
def paged_view(flat: torch.Tensor, page_size: int) -> torch.Tensor:
"""``[slots, ...]`` -> ``[slots // page_size, page_size, ...]``, as a view.
Built with ``as_strided``, not ``view``: dim 1 must carry the slot stride
at every page size, and ``view`` gives a size-1 dim (``page_size == 1``)
the row stride instead, which differs from the slot stride whenever the
per-layer view is strided.
"""
num_slots = int(flat.shape[0])
assert num_slots % page_size == 0, (num_slots, page_size)
slot_stride = flat.stride(0)
return flat.as_strided(
(num_slots // page_size, page_size, *flat.shape[1:]),
(slot_stride * page_size, slot_stride, *flat.stride()[1:]),
)
def paged_kv_view(
buf: torch.Tensor, page_size: int, head_num: int, head_dim: int
) -> torch.Tensor:
"""An MHA pool buffer as ``[num_pages, page_size, head_num, head_dim]``.
A slot-major buffer has its rows shaped and its pages split from its own
strides; an HND buffer (``[num_pages, head_num, page_size, head_dim]``) is
already page-split and only has its head and token dims swapped.
"""
if buf.dim() <= 3:
return paged_view(buf.view(buf.shape[0], head_num, head_dim), page_size)
assert tuple(buf.shape[1:]) == (head_num, page_size, head_dim), (
f"expected an HND buffer [pages, {head_num}, {page_size}, {head_dim}], "
f"got {tuple(buf.shape)}"
)
return buf.permute(0, 2, 1, 3)
def paged_row_view(flat: torch.Tensor, page_size: int) -> torch.Tensor:
"""``[slots, ...]`` -> ``[slots // page_size, page_size, row_elems]``, as a view."""
return paged_view(flat.view(int(flat.shape[0]), -1), page_size)
+5 -4
View File
@@ -32,6 +32,7 @@ from sglang.kernels.ops.layernorm.norm import (
from sglang.srt.environ import envs
from sglang.srt.layers.cp.utils import is_cp_active
from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
@@ -347,11 +348,11 @@ def create_fused_set_kv_buffer_arg(
key_cache = k_buffer
value_cache = v_buffer
else:
key_cache = k_buffer.view(
-1, page_size, layer.tp_k_head_num, layer.qk_head_dim
key_cache = paged_kv_view(
k_buffer, page_size, layer.tp_k_head_num, layer.qk_head_dim
)
value_cache = v_buffer.view(
-1, page_size, layer.tp_v_head_num, layer.v_head_dim
value_cache = paged_kv_view(
v_buffer, page_size, layer.tp_v_head_num, layer.v_head_dim
)
return {
"v": value.view(-1, layer.tp_v_head_num, layer.v_head_dim),