[unified-memory] Build paged KV views through one helper (2/7) (#40327)
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
@@ -33,6 +33,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
TRTLLMMLAMultiStepDraftBackend,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.utils import is_flashinfer_available
|
||||
|
||||
@@ -226,7 +227,7 @@ class CuteDslMLABackend(TRTLLMMLABackend):
|
||||
query = query.unsqueeze(1)
|
||||
|
||||
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
|
||||
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
|
||||
|
||||
metadata = (
|
||||
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
|
||||
|
||||
@@ -34,6 +34,7 @@ from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.kv_index_translator import KVReadTables
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view, paged_view
|
||||
from sglang.srt.mem_cache.memory_pool import KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
@@ -1305,20 +1306,33 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
head_group_num: int = 1,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
key_cache, value_cache = self.token_to_kv_pool.get_kv_buffer(layer.layer_id)
|
||||
return (
|
||||
key_cache.view(
|
||||
key_cache = paged_kv_view(
|
||||
key_cache, self.page_size, layer.tp_k_head_num, layer.head_dim
|
||||
)
|
||||
value_cache = paged_kv_view(
|
||||
value_cache, self.page_size, layer.tp_v_head_num, layer.v_head_dim
|
||||
)
|
||||
if head_group_num != 1:
|
||||
# Reinterpret each page's heads as `head_group_num` pseudo-pages.
|
||||
assert key_cache.is_contiguous() and value_cache.is_contiguous(), (
|
||||
f"head_group_num={head_group_num} needs a contiguous paged KV "
|
||||
"view; this one carries the pool's slot stride "
|
||||
f"(key {key_cache.stride()}, value {value_cache.stride()}), "
|
||||
"which no reshape of the head dimension can express."
|
||||
)
|
||||
key_cache = key_cache.view(
|
||||
-1,
|
||||
self.page_size,
|
||||
layer.tp_k_head_num // head_group_num,
|
||||
layer.head_dim,
|
||||
),
|
||||
value_cache.view(
|
||||
)
|
||||
value_cache = value_cache.view(
|
||||
-1,
|
||||
self.page_size,
|
||||
layer.tp_v_head_num // head_group_num,
|
||||
layer.v_head_dim,
|
||||
),
|
||||
)
|
||||
)
|
||||
return key_cache, value_cache
|
||||
|
||||
def prepare_paged_mha_query(
|
||||
self,
|
||||
@@ -1695,9 +1709,12 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
q=q.contiguous().view(-1, layer.tp_q_head_num, layer.head_dim),
|
||||
# The suffix table stores physical token slots, so expose
|
||||
# the paged cache as page-size-one blocks.
|
||||
k_cache=key_cache.view(-1, 1, layer.tp_k_head_num, layer.head_dim),
|
||||
v_cache=value_cache.view(
|
||||
-1, 1, layer.tp_v_head_num, layer.v_head_dim
|
||||
k_cache=paged_view(
|
||||
key_cache.view(-1, layer.tp_k_head_num, layer.head_dim), 1
|
||||
),
|
||||
v_cache=paged_view(
|
||||
value_cache.view(-1, layer.tp_v_head_num, layer.v_head_dim),
|
||||
1,
|
||||
),
|
||||
page_table=self.forward_metadata_spec_decode_expand.page_table,
|
||||
cache_seqlens=self.forward_metadata_spec_decode_expand.cache_seqlens_int32,
|
||||
@@ -1797,15 +1814,8 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
)
|
||||
k_rope = kv_cache[:, :, layer.v_head_dim :]
|
||||
c_kv = kv_cache[:, :, : layer.v_head_dim]
|
||||
k_rope_cache = k_rope.view(
|
||||
-1,
|
||||
self.page_size,
|
||||
layer.tp_k_head_num,
|
||||
layer.head_dim - layer.v_head_dim,
|
||||
)
|
||||
c_kv_cache = c_kv.view(
|
||||
-1, self.page_size, layer.tp_v_head_num, layer.v_head_dim
|
||||
)
|
||||
k_rope_cache = paged_view(k_rope, self.page_size)
|
||||
c_kv_cache = paged_view(c_kv, self.page_size)
|
||||
if q_rope is not None:
|
||||
q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim)
|
||||
q_rope = q_rope.view(
|
||||
@@ -2209,15 +2219,8 @@ class FlashAttentionBackend(AttentionBackend):
|
||||
kv_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id).to(q.dtype)
|
||||
k_rope = kv_cache[:, :, layer.v_head_dim :]
|
||||
c_kv = kv_cache[:, :, : layer.v_head_dim]
|
||||
k_rope_cache = k_rope.view(
|
||||
-1,
|
||||
self.page_size,
|
||||
layer.tp_k_head_num,
|
||||
layer.head_dim - layer.v_head_dim,
|
||||
)
|
||||
c_kv_cache = c_kv.view(
|
||||
-1, self.page_size, layer.tp_v_head_num, layer.v_head_dim
|
||||
)
|
||||
k_rope_cache = paged_view(k_rope, self.page_size)
|
||||
c_kv_cache = paged_view(c_kv, self.page_size)
|
||||
|
||||
if q_rope is not None:
|
||||
q_nope = q.view(-1, layer.tp_q_head_num, layer.v_head_dim)
|
||||
|
||||
@@ -19,6 +19,7 @@ from sglang.kernels.ops.attention.utils import (
|
||||
from sglang.kernels.ops.quantization.fp8_kernel import scaled_fp8_quant
|
||||
from sglang.srt.layers.attention.flashinfer_mla_backend import FlashInferMLAAttnBackend
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_view
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.runtime_context import get_parallel, get_spec
|
||||
|
||||
@@ -461,7 +462,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
reshape_q_fp8 = reshape_q_fp8_2d.reshape(q_shape)
|
||||
o, _ = flash_mla_with_kvcache(
|
||||
q=reshape_q_fp8,
|
||||
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
|
||||
k_cache=paged_view(k_cache, PAGE_SIZE),
|
||||
block_table=self.forward_metadata.block_kv_indices[:bs],
|
||||
cache_seqlens=forward_batch.seq_lens.to(torch.int32),
|
||||
head_dim_v=self.kv_lora_rank,
|
||||
@@ -478,7 +479,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
# todo: need check all causal True or False?
|
||||
o, lse = flash_mla_with_kvcache(
|
||||
q=reshape_q,
|
||||
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
|
||||
k_cache=paged_view(k_cache, PAGE_SIZE),
|
||||
block_table=self.forward_metadata.block_kv_indices[:bs],
|
||||
cache_seqlens=forward_batch.seq_lens.to(torch.int32),
|
||||
head_dim_v=self.kv_lora_rank,
|
||||
@@ -552,7 +553,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
reshape_q_fp8 = reshape_q_fp8_2d.reshape(q_shape)
|
||||
o, _ = flash_mla_with_kvcache(
|
||||
q=reshape_q_fp8,
|
||||
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
|
||||
k_cache=paged_view(k_cache, PAGE_SIZE),
|
||||
block_table=self.forward_metadata.block_kv_indices[:bs],
|
||||
cache_seqlens=cache_seqlens,
|
||||
head_dim_v=self.kv_lora_rank,
|
||||
@@ -566,7 +567,7 @@ class FlashMLABackend(FlashInferMLAAttnBackend):
|
||||
else:
|
||||
o, _ = flash_mla_with_kvcache(
|
||||
q=reshape_q,
|
||||
k_cache=k_cache.view(-1, PAGE_SIZE, 1, self.kv_cache_dim),
|
||||
k_cache=paged_view(k_cache, PAGE_SIZE),
|
||||
block_table=self.forward_metadata.block_kv_indices[:bs],
|
||||
cache_seqlens=cache_seqlens,
|
||||
head_dim_v=self.kv_lora_rank,
|
||||
|
||||
@@ -49,6 +49,7 @@ from sglang.srt.layers.attention.trtllm_mla_backend import (
|
||||
TRTLLMMLAMultiStepDraftBackend,
|
||||
)
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
|
||||
from sglang.srt.runtime_context import (
|
||||
get_parallel,
|
||||
get_resources,
|
||||
@@ -411,7 +412,7 @@ class TokenspeedMLABackend(TRTLLMMLABackend):
|
||||
query = query.unsqueeze(1)
|
||||
|
||||
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
|
||||
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
|
||||
metadata = (
|
||||
getattr(forward_batch, "decode_trtllm_mla_metadata", None)
|
||||
or self.forward_decode_metadata
|
||||
|
||||
@@ -42,6 +42,7 @@ from sglang.srt.layers.quantization.fp4_kv_cache_quant_method import (
|
||||
KVCacheAttentionAccessKind,
|
||||
)
|
||||
from sglang.srt.layers.radix_attention import AttentionType
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
|
||||
from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool, KVWriteLoc
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
@@ -1314,11 +1315,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
layer: RadixAttention,
|
||||
head_dim: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
k_cache = k_cache.view(
|
||||
-1, self.page_size, layer.tp_k_head_num, head_dim
|
||||
k_cache = paged_kv_view(
|
||||
k_cache, self.page_size, layer.tp_k_head_num, head_dim
|
||||
).permute(0, 2, 1, 3)
|
||||
v_cache = v_cache.view(
|
||||
-1, self.page_size, layer.tp_v_head_num, head_dim
|
||||
v_cache = paged_kv_view(
|
||||
v_cache, self.page_size, layer.tp_v_head_num, head_dim
|
||||
).permute(0, 2, 1, 3)
|
||||
if layer.tp_k_head_num == 1:
|
||||
k_cache = canonicalize_stride(k_cache)
|
||||
@@ -1632,11 +1633,11 @@ class TRTLLMHAAttnBackend(FlashInferAttnBackend):
|
||||
k_cache_raw, v_cache_raw, layer, layer.head_dim
|
||||
)
|
||||
else:
|
||||
k_cache = k_cache_raw.view(
|
||||
-1, self.page_size, layer.tp_k_head_num, layer.head_dim
|
||||
k_cache = paged_kv_view(
|
||||
k_cache_raw, self.page_size, layer.tp_k_head_num, layer.head_dim
|
||||
)
|
||||
v_cache = v_cache_raw.view(
|
||||
-1, self.page_size, layer.tp_v_head_num, layer.head_dim
|
||||
v_cache = paged_kv_view(
|
||||
v_cache_raw, self.page_size, layer.tp_v_head_num, layer.head_dim
|
||||
)
|
||||
|
||||
kv_cache = (k_cache, v_cache)
|
||||
|
||||
@@ -57,6 +57,7 @@ from sglang.srt.layers.attention.kv_shard_hooks import (
|
||||
from sglang.srt.layers.attention.verify_mask import VerifyMask, maybe_create_verify_mask
|
||||
from sglang.srt.layers.dcp.layout import get_dcp_lens
|
||||
from sglang.srt.layers.logits_processor import get_in_autotune_dummy_run
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_row_view
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode
|
||||
from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import (
|
||||
is_in_breakable_cuda_graph,
|
||||
@@ -1383,7 +1384,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
|
||||
# Prepare KV cache inline
|
||||
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
|
||||
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
|
||||
|
||||
# Get metadata
|
||||
metadata = (
|
||||
@@ -1599,7 +1600,7 @@ class TRTLLMMLABackend(FlashInferMLAAttnBackend):
|
||||
bs = forward_batch.batch_size
|
||||
|
||||
k_cache = self.token_to_kv_pool.get_key_buffer(layer.layer_id)
|
||||
kv_cache = k_cache.view(-1, self.page_size, self.kv_cache_dim).unsqueeze(1)
|
||||
kv_cache = paged_row_view(k_cache, self.page_size).unsqueeze(1)
|
||||
|
||||
q = q.to(self.data_type)
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Paged views over per-slot KV buffers.
|
||||
|
||||
Attention kernels take a paged ``[num_pages, page_size, ...]`` cache. These
|
||||
helpers split a pool buffer's slot dim into pages from the buffer's own
|
||||
strides, so a strided per-layer view keeps its slot stride at every page size.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def paged_view(flat: torch.Tensor, page_size: int) -> torch.Tensor:
|
||||
"""``[slots, ...]`` -> ``[slots // page_size, page_size, ...]``, as a view.
|
||||
|
||||
Built with ``as_strided``, not ``view``: dim 1 must carry the slot stride
|
||||
at every page size, and ``view`` gives a size-1 dim (``page_size == 1``)
|
||||
the row stride instead, which differs from the slot stride whenever the
|
||||
per-layer view is strided.
|
||||
"""
|
||||
num_slots = int(flat.shape[0])
|
||||
assert num_slots % page_size == 0, (num_slots, page_size)
|
||||
slot_stride = flat.stride(0)
|
||||
return flat.as_strided(
|
||||
(num_slots // page_size, page_size, *flat.shape[1:]),
|
||||
(slot_stride * page_size, slot_stride, *flat.stride()[1:]),
|
||||
)
|
||||
|
||||
|
||||
def paged_kv_view(
|
||||
buf: torch.Tensor, page_size: int, head_num: int, head_dim: int
|
||||
) -> torch.Tensor:
|
||||
"""An MHA pool buffer as ``[num_pages, page_size, head_num, head_dim]``.
|
||||
|
||||
A slot-major buffer has its rows shaped and its pages split from its own
|
||||
strides; an HND buffer (``[num_pages, head_num, page_size, head_dim]``) is
|
||||
already page-split and only has its head and token dims swapped.
|
||||
"""
|
||||
if buf.dim() <= 3:
|
||||
return paged_view(buf.view(buf.shape[0], head_num, head_dim), page_size)
|
||||
assert tuple(buf.shape[1:]) == (head_num, page_size, head_dim), (
|
||||
f"expected an HND buffer [pages, {head_num}, {page_size}, {head_dim}], "
|
||||
f"got {tuple(buf.shape)}"
|
||||
)
|
||||
return buf.permute(0, 2, 1, 3)
|
||||
|
||||
|
||||
def paged_row_view(flat: torch.Tensor, page_size: int) -> torch.Tensor:
|
||||
"""``[slots, ...]`` -> ``[slots // page_size, page_size, row_elems]``, as a view."""
|
||||
return paged_view(flat.view(int(flat.shape[0]), -1), page_size)
|
||||
@@ -32,6 +32,7 @@ from sglang.kernels.ops.layernorm.norm import (
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.cp.utils import is_cp_active
|
||||
from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.layout.paged_view import paged_kv_view
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.forward_context import get_token_to_kv_pool
|
||||
@@ -347,11 +348,11 @@ def create_fused_set_kv_buffer_arg(
|
||||
key_cache = k_buffer
|
||||
value_cache = v_buffer
|
||||
else:
|
||||
key_cache = k_buffer.view(
|
||||
-1, page_size, layer.tp_k_head_num, layer.qk_head_dim
|
||||
key_cache = paged_kv_view(
|
||||
k_buffer, page_size, layer.tp_k_head_num, layer.qk_head_dim
|
||||
)
|
||||
value_cache = v_buffer.view(
|
||||
-1, page_size, layer.tp_v_head_num, layer.v_head_dim
|
||||
value_cache = paged_kv_view(
|
||||
v_buffer, page_size, layer.tp_v_head_num, layer.v_head_dim
|
||||
)
|
||||
return {
|
||||
"v": value.view(-1, layer.tp_v_head_num, layer.v_head_dim),
|
||||
|
||||
Reference in New Issue
Block a user