[unified-memory] Token-major dense views for the unified memory pool (3/7) (#38592)

Co-authored-by: Caihua Li <caihua.li@bytedance.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
caihuali95
2026-10-01 17:09:38 -07:00
committed by GitHub
co-authored by Caihua Li Cheng Wan
parent 5df52e6382
commit cdadb1c79a
36 changed files with 1216 additions and 1109 deletions
@@ -70,19 +70,18 @@ class Memory(msgspec.Struct):
] = False
enable_page_major_kv_layout: A[
bool,
"Enable the page-major KV layout: lay out the Mamba state and full/SWA "
"KV caches in a page-granularity envelope (page is the outermost axis, "
"layer-major within a page) instead of the default per-layer "
"(layer-major) layout. Requires the Triton attention / linear-attn / "
"Mamba backends.",
"Store the Mamba state and full/SWA KV caches as one entry per token "
"slot, holding every layer's state, in a single page-granular buffer "
"instead of one buffer per layer. Implied by --enable-unified-memory, "
"and currently supported only with it.",
] = False
enable_unified_memory: A[
bool,
"Replace the statically-partitioned hybrid-model pools (full-attn KV + "
"SWA/Mamba state) with one byte buffer split dynamically between "
"sub-pools. Requires the Triton attention / linear-attn / Mamba "
"backends. Supported PD-disaggregation and speculative-decoding "
"configurations are validated at startup.",
"sub-pools. Requires stride-aware attention, linear-attention and Mamba "
"backends; the allowed backends and the supported PD-disaggregation and "
"speculative-decoding configurations are validated at startup.",
] = False
enable_session_radix_cache: A[
bool,
+25 -17
View File
@@ -608,6 +608,18 @@ def handle_unified_memory_pool(server_args: Any) -> None:
sorted(full_cg_backends),
sorted(backends),
)
# trtllm_mha refills its graph page table before replay from
# `cache_seqlens_int32`, which only the in-graph metadata kernel writes, so
# it uses the previous replay's lengths. Refuse until it uses the batch's.
_, decode_backend = attention_backends_of(resolved_view(server_args))
if decode_backend == "trtllm_mha":
assert _cg_cfg is None or _cg_cfg.decode.backend == Backend.DISABLED, (
"--enable-unified-memory does not yet support decode cuda graphs "
"with the trtllm_mha attention backend: its replay refills the page "
"table from the previous replay's sequence lengths. Pass "
"--disable-cuda-graph, or pick another decode attention backend "
"(fa3 / fa4 / flashinfer / triton)."
)
def _validate_unified_memory_dcp(server_args: Any) -> None:
@@ -656,9 +668,8 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
def handle_page_major_kv_layout(server_args: Any):
# The unified pool stores state in the page-major envelope-strided layout, so
# enabling it implies --enable-page-major-kv-layout — routing it through the
# single page-major path + stride-aware Triton asserts (set before the guard).
# --enable-unified-memory implies --enable-page-major-kv-layout, so the
# unified pool goes through this one gate (declared before the guard).
cfg = resolving_view(server_args)
if cfg.enable_unified_memory:
@@ -684,24 +695,24 @@ def handle_page_major_kv_layout(server_args: Any):
assert unified_memory_supported_for_model(
model_config, use_mla_backend=use_mla_backend(server_args)
), (
"--enable-unified-memory requires uniform K/V rows "
"(head_dim == v_head_dim); this model has "
"--enable-unified-memory does not yet admit asymmetric K/V rows "
"(head_dim != v_head_dim); this model has "
f"head_dim={model_config.head_dim}, "
f"v_head_dim={model_config.v_head_dim}, "
f"swa_head_dim={model_config.swa_head_dim}, "
f"swa_v_head_dim={model_config.swa_v_head_dim}. The unified "
"pool's per-layer views require a uniform row width; run "
"this model without --enable-unified-memory."
f"swa_v_head_dim={model_config.swa_v_head_dim}. The token-major "
"views can hold them, but the backends' write and read paths are "
"not audited for it; run this model without --enable-unified-memory."
)
# Allow-list. Every backend below reads through the translator, so what
# gates one is only whether its kernels can address the per-layer views:
# gates one is only whether its kernels address the per-layer views by
# their strides (the slot stride is the whole entry, not one row):
# * MLA models: the full paged MLA family, incl. flashmla (ps=64
# snap).
# * MHA/SWA models: fa3 / fa4 / flashinfer / trtllm_mha alongside
# Triton. fa4 is the fa3 class.
# * Without the unified pool, plain page-major stays Triton-only.
# Names are the RESOLVED ids from attention_backends_of.
if cfg.enable_unified_memory and use_mla_backend(server_args):
if use_mla_backend(server_args):
allowed_full = {
"triton",
"fa3",
@@ -711,7 +722,7 @@ def handle_page_major_kv_layout(server_args: Any):
"tokenspeed_mla",
"flashmla",
}
elif cfg.enable_unified_memory:
else:
allowed_full = {
"triton",
"fa3",
@@ -719,17 +730,14 @@ def handle_page_major_kv_layout(server_args: Any):
"flashinfer",
"trtllm_mha",
}
else:
allowed_full = {"triton"}
backends = set(attention_backends_of(resolved_view(server_args)))
backends.discard(None)
assert backends <= allowed_full, (
"--enable-page-major-kv-layout: the resolved attention backends "
f"{sorted(backends)} are not in the allowed set "
f"{sorted(allowed_full)} for this configuration (unified memory "
"allows the per-layer-view families; plain page-major keeps the "
"envelope-strided views only Triton reads). Pass a compatible "
"--attention-backend."
"allows the stride-aware per-layer-view families). Pass a "
"compatible --attention-backend."
)
# The Mamba/KDA state is stored in envelope-strided views; only
# stride-audited kernels may read it (Stage 4 audit, per slot):
@@ -50,6 +50,17 @@ logger = logging.getLogger(__name__)
device_module = get_device_module()
def storage_model_name(
model_name: Optional[str], host_pool: HostKVCache
) -> Optional[str]:
"""The model name storage backends key pages on, tagged with the host
pool's page format when it has one of its own."""
tag = host_pool.storage_format_tag
if tag is None:
return model_name
return f"{model_name}-{tag}" if model_name else tag
class LayerLoadingEvent:
def __init__(self, num_layers: int):
self._num_layers = num_layers
@@ -730,6 +741,7 @@ class HiCacheController:
)
attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
model_name = storage_model_name(model_name, self.storage_host_pool)
return HiCacheStorageConfig(
tp_rank=self.tp_rank,
@@ -316,13 +316,13 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.entry_bytes = spec.entry_bytes()
self.min_slot_index = unified_buffer.min_slot_index(sub_pool_name)
self.is_id_owner = is_id_owner
# Kernel-facing page-stride scale, from the spec that owns the layout;
# `kernel_page_multiplier=` overrides it only for tests.
self.kernel_page_multiplier = (
spec.blocks_per_page()
if kernel_page_multiplier is None
else kernel_page_multiplier
# Kernel-facing ids are the physical token ids: the token-major views
# step slots by the whole entry, so there is no per-page block scale.
assert kernel_page_multiplier in (None, 1), (
f"kernel_page_multiplier must be 1 (token-major views); got "
f"{kernel_page_multiplier}"
)
self.kernel_page_multiplier = 1
# Zero page envelopes on hand-out -- see _maybe_zero_pages.
self._zero_pages_on_alloc = isinstance(kvcache, UnifiedMLATokenToKVPool)
# Overlap mode: `free` drops a wait_stream(forward_stream) barrier so its
@@ -1060,14 +1060,15 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
*,
out: Optional[torch.Tensor] = None,
) -> torch.Tensor:
"""Virtual token ids -> kernel-facing ids:
"""Virtual token ids -> kernel-facing ids, which under the token-major
views are the physical token ids:
kernel_id(t) = (t // ps) * (ps * kernel_page_multiplier) + t % ps
kernel_id(t) = v2p[t // ps] * ps + t % ps
Internal machinery (compaction, in-flight write sets) MUST keep using
`translate_kv_loc`: kernel-facing ids are for kernels only. Tombstones (-1)
clamp to kernel-facing id 0, the page-0 sink. int64 out; a consumer whose
kernel ABI wants int32 narrows where it fills that buffer.
(`kernel_page_multiplier` is pinned to 1, so this equals
`translate_kv_loc`.) Tombstones (-1) clamp to id 0, the page-0 sink.
int64 out; a consumer whose kernel ABI wants int32 narrows where it
fills that buffer.
"""
with record_function("MultiEndedAlloc.translate_kv_loc_for_kernel"):
return self._translate_loc_fused(virt_tokens, dcp_size=1, out=out)
@@ -1,293 +0,0 @@
"""Page-granularity envelope (page-major, layer-major within a page) cache views.
A pool of this layout keeps all layers of all slots in one contiguous byte
buffer. The buffer is split into pages of ``page_size`` slots; within a page,
each layer's K and V (or each Mamba conv/temporal tensor) are grouped together:
page bytes = [L0_K * ps | L0_V * ps | L1_K * ps | L1_V * ps | ...]
Across pages the layout is envelope-major (one ``page_bytes`` block per page).
At ``page_size == 1`` a page is a single slot, so the within-page block is the
per-slot ``[L0_K | L0_V | L1_K | L1_V | ...]`` envelope (token-granularity).
These builders produce per-layer views into a raw ``uint8`` buffer; they hold
no allocator/ownership state. ``anchor_bytes`` is the byte offset of the
pool's region inside the raw buffer (0 for a standalone pool).
"""
from typing import List, Sequence, Tuple
import torch
def _prod(shape: Sequence[int]) -> int:
out = 1
for s in shape:
out *= int(s)
return out
def mha_entry_bytes(
*, layer_num: int, head_num: int, head_dim: int, v_head_dim: int, itemsize: int
) -> int:
"""Bytes occupied by one slot across all layers (K and V)."""
k_row_bytes = head_num * head_dim * itemsize
v_row_bytes = head_num * v_head_dim * itemsize
return layer_num * (k_row_bytes + v_row_bytes)
def build_mha_views(
raw: torch.Tensor,
*,
layer_num: int,
head_num: int,
head_dim: int,
v_head_dim: int,
store_dtype: torch.dtype,
page_size: int,
num_pages: int,
anchor_bytes: int = 0,
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
"""Per-layer K/V views over ``raw`` for uniform-row MHA.
The page envelope ``[L0_K*ps | L0_V*ps | L1_K*ps | ...]`` is a uniform
array of ``2*layer_num`` row-blocks when K and V rows are equally wide, so
it is a valid paged pool under
kernel_id(t) = (t // ps) * (ps * 2 * layer_num) + t % ps
with layer ``l``'s K at block ``2l`` and its V at block ``2l+1``. Each view
is a contiguous ``(num_pages * 2 * layer_num * ps, head_num, head_dim)``.
Views overlap by ``ps`` rows per block, safe because an id always resolves
inside its own block; the last view runs ``(2*layer_num - 1) * ps`` rows
past the envelope, so ``raw`` needs ``UnifiedKVPool.view_tail_pad_bytes``.
"""
assert head_dim == v_head_dim, (
f"build_mha_views requires uniform rows (head_dim == v_head_dim); "
f"got head_dim={head_dim}, v_head_dim={v_head_dim}. Asymmetric-KV "
"models cannot use the unified pool (screened out at startup)."
)
itemsize = store_dtype.itemsize
row_elems = head_num * head_dim
row_bytes = row_elems * itemsize
blocks = 2 * layer_num
page_bytes = page_size * blocks * row_bytes
n_rows = num_pages * blocks * page_size
assert anchor_bytes % itemsize == 0
last_view_end = (
anchor_bytes + (blocks - 1) * page_size * row_bytes + n_rows * row_bytes
)
assert last_view_end <= raw.numel() * raw.itemsize, (
f"build_mha_views: block {blocks - 1}'s view ends at byte "
f"{last_view_end} but the raw buffer holds only "
f"{raw.numel() * raw.itemsize} bytes; allocate the tail pad "
f"(one page envelope = {page_bytes} B) via view_tail_pad_bytes"
)
as_dtype_view = raw.view(store_dtype)
k_buffer: List[torch.Tensor] = []
v_buffer: List[torch.Tensor] = []
for layer in range(layer_num):
k_base_bytes = anchor_bytes + (2 * layer) * page_size * row_bytes
v_base_bytes = k_base_bytes + page_size * row_bytes
for base_bytes, out in ((k_base_bytes, k_buffer), (v_base_bytes, v_buffer)):
assert base_bytes % itemsize == 0
out.append(
torch.as_strided(
as_dtype_view,
size=(n_rows, head_num, head_dim),
stride=(row_elems, head_dim, 1),
storage_offset=base_bytes // itemsize,
)
)
return k_buffer, v_buffer
def mla_entry_bytes(*, layer_num: int, kv_cache_dim: int, itemsize: int) -> int:
"""Bytes occupied by one MLA slot across all layers (single latent row, no V)."""
return layer_num * kv_cache_dim * itemsize
def build_mla_views(
raw: torch.Tensor,
*,
layer_num: int,
kv_cache_dim: int,
store_dtype: torch.dtype,
page_size: int,
num_pages: int,
anchor_bytes: int = 0,
) -> List[torch.Tensor]:
"""Per-layer views over ``raw`` for MLA in the page-major layout.
The page envelope is ``[L0_latent * ps | L1_latent * ps | ...]``. Because all
MLA layers share one uniform row size (``kv_cache_dim``), the envelope is
itself a valid paged pool under a re-numbered index space: folding the
layer offset ``l * ps * kv_cache_dim`` into each view's storage_offset makes
every per-layer view a plain CONTIGUOUS ``(num_pages * layer_num * ps, 1,
kv_cache_dim)`` tensor, addressed by the layer-independent kernel-facing id
kernel_id(t) = (t // ps) * (ps * layer_num) + t % ps (t = physical token)
so one shared block table (entry = page * layer_num) serves every layer, and
kernels that require ``.view(-1, page_size, kv_cache_dim)`` (trtllm/cutlass/
flashmla) work on the views natively.
The views overlap each other (view ``l+1`` is view ``l`` shifted by ``ps``
rows); that is safe because layer ``l`` is only ever indexed at kernel-facing ids,
which always resolve to layer-``l`` bytes relative to view ``l``'s origin.
Layer ``layer_num-1``'s view extends ``(layer_num-1) * ps`` rows past the
last page envelope, so ``raw`` must carry at least one extra page envelope
of tail padding (``UnifiedKVPool``'s ``view_tail_pad_bytes``).
"""
itemsize = store_dtype.itemsize
row_bytes = kv_cache_dim * itemsize
page_bytes = page_size * layer_num * row_bytes
n_rows = num_pages * layer_num * page_size
assert anchor_bytes % itemsize == 0
last_view_end = (
anchor_bytes + (layer_num - 1) * page_size * row_bytes + (n_rows * row_bytes)
)
assert last_view_end <= raw.numel() * raw.itemsize, (
f"build_mla_views: layer {layer_num - 1}'s view ends at byte "
f"{last_view_end} but the raw buffer holds only "
f"{raw.numel() * raw.itemsize} bytes; allocate the tail pad "
f"(one page envelope = {page_bytes} B) via view_tail_pad_bytes"
)
as_dtype_view = raw.view(store_dtype)
views: List[torch.Tensor] = []
for layer in range(layer_num):
base_bytes = anchor_bytes + layer * page_size * row_bytes
assert base_bytes % itemsize == 0
views.append(
torch.as_strided(
as_dtype_view,
size=(n_rows, 1, kv_cache_dim),
stride=(kv_cache_dim, kv_cache_dim, 1),
storage_offset=base_bytes // itemsize,
)
)
return views
def mamba_entry_bytes(
*,
layer_num: int,
conv_state_shapes: Sequence[Sequence[int]],
conv_dtype: torch.dtype,
temporal_state_shape: Sequence[int],
temporal_dtype: torch.dtype,
) -> int:
"""Bytes occupied by one Mamba slot across all layers (conv + temporal)."""
total = 0
for shape in conv_state_shapes:
total += layer_num * _prod(shape) * conv_dtype.itemsize
total += layer_num * _prod(temporal_state_shape) * temporal_dtype.itemsize
return total
def build_page_major_mamba_views(
raw: torch.Tensor,
*,
layer_num: int,
conv_state_shapes: Sequence[Sequence[int]],
conv_dtype: torch.dtype,
temporal_state_shape: Sequence[int],
temporal_dtype: torch.dtype,
max_slots: int,
anchor_bytes: int = 0,
) -> Tuple[List[torch.Tensor], torch.Tensor]:
"""Per-slot envelope views over ``raw`` for Mamba state.
Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]...
[temporal rows × layers]``. Each returned view has shape
``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]``
/ ``.temporal``. Mamba state is always token-granular (page_size == 1).
"""
entry_bytes = mamba_entry_bytes(
layer_num=layer_num,
conv_state_shapes=conv_state_shapes,
conv_dtype=conv_dtype,
temporal_state_shape=temporal_state_shape,
temporal_dtype=temporal_dtype,
)
def contiguous_strides(shape: Sequence[int]) -> Tuple[int, ...]:
strides = []
acc = 1
for s in reversed(shape):
strides.append(acc)
acc *= int(s)
return tuple(reversed(strides))
conv_itemsize = conv_dtype.itemsize
assert entry_bytes % conv_itemsize == 0, (
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
f"multiple of the conv-state itemsize {conv_itemsize} B"
)
assert anchor_bytes % conv_itemsize == 0, (
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
f"of the conv-state itemsize {conv_itemsize} B"
)
as_conv_dtype = raw.view(conv_dtype)
conv_slot_stride_elems = entry_bytes // conv_itemsize
offset_bytes_within_entry = 0
conv_views: List[torch.Tensor] = []
for shape in conv_state_shapes:
inner_shape_bytes = _prod(shape) * conv_itemsize
assert inner_shape_bytes % conv_itemsize == 0
offset_elems = (anchor_bytes + offset_bytes_within_entry) // conv_itemsize
stride = (
inner_shape_bytes // conv_itemsize,
conv_slot_stride_elems,
) + contiguous_strides(shape)
conv_views.append(
torch.as_strided(
as_conv_dtype,
size=(layer_num, max_slots) + tuple(shape),
stride=stride,
storage_offset=offset_elems,
)
)
offset_bytes_within_entry += layer_num * inner_shape_bytes
# The temporal view's storage_offset is computed in temporal-dtype elements
# by integer-dividing a byte offset by itemsize, so every term of that byte
# offset (entry stride, anchor, the conv region) must be a whole multiple of
# itemsize or the offset truncates and mis-places the view.
itemsize = temporal_dtype.itemsize
assert entry_bytes % itemsize == 0, (
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
f"multiple of the temporal-state itemsize {itemsize} B; the temporal "
f"view's storage_offset would truncate and mis-place the state"
)
assert anchor_bytes % itemsize == 0, (
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
f"of the temporal-state itemsize {itemsize} B"
)
inner_shape_bytes = _prod(temporal_state_shape) * itemsize
assert inner_shape_bytes % itemsize == 0, (
f"misaligned mamba spec: temporal inner_shape_bytes={inner_shape_bytes} "
f"is not a multiple of the temporal-state itemsize {itemsize} B"
)
assert (anchor_bytes + offset_bytes_within_entry) % itemsize == 0, (
f"misaligned mamba spec: temporal region byte offset "
f"{anchor_bytes + offset_bytes_within_entry} is not a multiple of the "
f"temporal-state itemsize {itemsize} B"
)
offset_elems = (anchor_bytes + offset_bytes_within_entry) // itemsize
as_temporal_dtype = raw.view(temporal_dtype)
stride = (
inner_shape_bytes // itemsize,
entry_bytes // itemsize,
) + contiguous_strides(temporal_state_shape)
temporal_view = torch.as_strided(
as_temporal_dtype,
size=(layer_num, max_slots) + tuple(temporal_state_shape),
stride=stride,
storage_offset=offset_elems,
)
return conv_views, temporal_view
@@ -0,0 +1,261 @@
"""Token-major cache views over one raw byte buffer.
Each token slot owns one ENTRY holding every layer's state at fixed offsets,
and a page is ``page_size`` entries back to back:
entry bytes = [ K_0 | V_0 | K_1 | V_1 | ... ] (MHA)
[ lat_0 | lat_1 | ... ] (MLA)
byte(t, part, l) = anchor + t * entry_bytes + part offset + l * layer stride
so every per-layer view is ``(num_slots, *row_shape)`` with slot stride
``entry_bytes``, indexed by the physical token id. The Mamba state
(``build_mamba_entry_views``) uses the same per-slot entry. ``anchor_bytes``
is the pool region's offset inside the raw buffer.
"""
from typing import List, Sequence, Tuple
import msgspec
import torch
ENTRY_ALIGN_BYTES = 32
ROW_ALIGN_BYTES = 16
def _prod(shape: Sequence[int]) -> int:
out = 1
for s in shape:
out *= int(s)
return out
def _contiguous_strides(shape: Sequence[int]) -> Tuple[int, ...]:
strides = []
acc = 1
for s in reversed(shape):
strides.append(acc)
acc *= int(s)
return tuple(reversed(strides))
def align_entry_bytes(num_bytes: int) -> int:
"""Round a slot's byte sum up to the entry alignment."""
return -(-num_bytes // ENTRY_ALIGN_BYTES) * ENTRY_ALIGN_BYTES
class DensePart(msgspec.Struct, frozen=True, kw_only=True):
"""One row family inside the entry: ``layer_num`` rows of ``row_shape``,
layer ``l`` at ``offset_bytes + l * layer_stride_bytes``."""
name: str
offset_bytes: int
layer_stride_bytes: int
layer_num: int
row_shape: Tuple[int, ...]
dtype: torch.dtype
def row_bytes(self) -> int:
return _prod(self.row_shape) * self.dtype.itemsize
def layer_offset_bytes(self, layer: int) -> int:
return self.offset_bytes + layer * self.layer_stride_bytes
class DenseEntryLayout(msgspec.Struct, frozen=True, kw_only=True):
"""Byte layout of one slot's entry; ``entry_bytes`` is the slot stride of
every view built over it."""
entry_bytes: int
parts: Tuple[DensePart, ...]
def __post_init__(self):
self.validate()
def part(self, name: str) -> DensePart:
for p in self.parts:
if p.name == name:
return p
raise KeyError(f"no part {name!r} in {[p.name for p in self.parts]}")
def validate(self) -> None:
assert self.entry_bytes % ENTRY_ALIGN_BYTES == 0, (
f"entry_bytes={self.entry_bytes} is not a multiple of {ENTRY_ALIGN_BYTES}"
)
spans = []
for p in self.parts:
row = p.row_bytes()
assert (
p.offset_bytes % ROW_ALIGN_BYTES == 0
and p.layer_stride_bytes % ROW_ALIGN_BYTES == 0
and row % ROW_ALIGN_BYTES == 0
), (
f"part {p.name!r}: offset {p.offset_bytes}, layer stride "
f"{p.layer_stride_bytes} and row {row} B must all be multiples of "
f"{ROW_ALIGN_BYTES} (vector stores)"
)
for l in range(p.layer_num):
lo = p.layer_offset_bytes(l)
assert 0 <= lo and lo + row <= self.entry_bytes, (
f"part {p.name!r} layer {l} spans [{lo}, {lo + row}) outside "
f"the {self.entry_bytes}-byte entry"
)
spans.append((lo, lo + row, p.name, l))
spans.sort()
for a, b in zip(spans, spans[1:]):
assert a[1] <= b[0], (
f"parts overlap: {a[2]}[{a[3]}] [{a[0]}, {a[1]}) and "
f"{b[2]}[{b[3]}] [{b[0]}, {b[1]})"
)
def build_dense_views(
raw: torch.Tensor,
*,
layout: DenseEntryLayout,
part_name: str,
num_slots: int,
anchor_bytes: int = 0,
) -> List[torch.Tensor]:
"""Per-layer views of one part: ``(num_slots, *row_shape)`` with slot
stride ``layout.entry_bytes``, indexed by the physical token id
(``paged_view`` regroups them by page).
"""
part = layout.part(part_name)
itemsize = part.dtype.itemsize
assert layout.entry_bytes % itemsize == 0 and anchor_bytes % itemsize == 0, (
f"build_dense_views: entry {layout.entry_bytes} B and anchor "
f"{anchor_bytes} B must be multiples of the {part.dtype} itemsize"
)
end = anchor_bytes + num_slots * layout.entry_bytes
assert end <= raw.numel() * raw.itemsize, (
f"build_dense_views: {num_slots} slots of {layout.entry_bytes} B end at byte "
f"{end} but the raw buffer holds only {raw.numel() * raw.itemsize} bytes"
)
as_dtype_view = raw.view(part.dtype)
stride = (layout.entry_bytes // itemsize, *_contiguous_strides(part.row_shape))
views: List[torch.Tensor] = []
for layer in range(part.layer_num):
# Part offsets and layer strides are ROW_ALIGN_BYTES-aligned (validate()).
base_bytes = anchor_bytes + part.layer_offset_bytes(layer)
views.append(
torch.as_strided(
as_dtype_view,
size=(num_slots, *part.row_shape),
stride=stride,
storage_offset=base_bytes // itemsize,
)
)
return views
def mamba_entry_bytes(
*,
layer_num: int,
conv_state_shapes: Sequence[Sequence[int]],
conv_dtype: torch.dtype,
temporal_state_shape: Sequence[int],
temporal_dtype: torch.dtype,
) -> int:
"""Bytes occupied by one Mamba slot across all layers (conv + temporal)."""
total = 0
for shape in conv_state_shapes:
total += layer_num * _prod(shape) * conv_dtype.itemsize
total += layer_num * _prod(temporal_state_shape) * temporal_dtype.itemsize
return total
def build_mamba_entry_views(
raw: torch.Tensor,
*,
layer_num: int,
conv_state_shapes: Sequence[Sequence[int]],
conv_dtype: torch.dtype,
temporal_state_shape: Sequence[int],
temporal_dtype: torch.dtype,
max_slots: int,
anchor_bytes: int = 0,
) -> Tuple[List[torch.Tensor], torch.Tensor]:
"""Per-slot envelope views over ``raw`` for Mamba state.
Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]...
[temporal rows × layers]``. Each returned view has shape
``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]``
/ ``.temporal``. Mamba state is always token-granular (page_size == 1).
"""
entry_bytes = mamba_entry_bytes(
layer_num=layer_num,
conv_state_shapes=conv_state_shapes,
conv_dtype=conv_dtype,
temporal_state_shape=temporal_state_shape,
temporal_dtype=temporal_dtype,
)
conv_itemsize = conv_dtype.itemsize
assert entry_bytes % conv_itemsize == 0, (
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
f"multiple of the conv-state itemsize {conv_itemsize} B"
)
assert anchor_bytes % conv_itemsize == 0, (
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
f"of the conv-state itemsize {conv_itemsize} B"
)
as_conv_dtype = raw.view(conv_dtype)
conv_slot_stride_elems = entry_bytes // conv_itemsize
offset_bytes_within_entry = 0
conv_views: List[torch.Tensor] = []
for shape in conv_state_shapes:
inner_shape_bytes = _prod(shape) * conv_itemsize
assert inner_shape_bytes % conv_itemsize == 0
offset_elems = (anchor_bytes + offset_bytes_within_entry) // conv_itemsize
stride = (
inner_shape_bytes // conv_itemsize,
conv_slot_stride_elems,
) + _contiguous_strides(shape)
conv_views.append(
torch.as_strided(
as_conv_dtype,
size=(layer_num, max_slots) + tuple(shape),
stride=stride,
storage_offset=offset_elems,
)
)
offset_bytes_within_entry += layer_num * inner_shape_bytes
# The temporal view's storage_offset is computed in temporal-dtype elements
# by integer-dividing a byte offset by itemsize, so every term of that byte
# offset (entry stride, anchor, the conv region) must be a whole multiple of
# itemsize or the offset truncates and mis-places the view.
itemsize = temporal_dtype.itemsize
assert entry_bytes % itemsize == 0, (
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
f"multiple of the temporal-state itemsize {itemsize} B; the temporal "
f"view's storage_offset would truncate and mis-place the state"
)
assert anchor_bytes % itemsize == 0, (
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
f"of the temporal-state itemsize {itemsize} B"
)
inner_shape_bytes = _prod(temporal_state_shape) * itemsize
assert inner_shape_bytes % itemsize == 0, (
f"misaligned mamba spec: temporal inner_shape_bytes={inner_shape_bytes} "
f"is not a multiple of the temporal-state itemsize {itemsize} B"
)
assert (anchor_bytes + offset_bytes_within_entry) % itemsize == 0, (
f"misaligned mamba spec: temporal region byte offset "
f"{anchor_bytes + offset_bytes_within_entry} is not a multiple of the "
f"temporal-state itemsize {itemsize} B"
)
offset_elems = (anchor_bytes + offset_bytes_within_entry) // itemsize
as_temporal_dtype = raw.view(temporal_dtype)
stride = (
inner_shape_bytes // itemsize,
entry_bytes // itemsize,
) + _contiguous_strides(temporal_state_shape)
temporal_view = torch.as_strided(
as_temporal_dtype,
size=(layer_num, max_slots) + tuple(temporal_state_shape),
stride=stride,
storage_offset=offset_elems,
)
return conv_views, temporal_view
+5 -21
View File
@@ -60,8 +60,8 @@ from sglang.srt.layers.radix_attention import RadixAttention
from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
from sglang.srt.mem_cache.layout.page_major import (
build_page_major_mamba_views,
from sglang.srt.mem_cache.layout.token_major import (
build_mamba_entry_views,
mamba_entry_bytes,
)
from sglang.srt.mem_cache.utils import (
@@ -86,7 +86,6 @@ from sglang.srt.utils import (
next_power_of_2,
)
from sglang.srt.utils.async_probe import (
maybe_detect_kernel_facing_loc,
maybe_detect_oob,
)
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
@@ -588,7 +587,7 @@ class MambaPool:
# Page-granularity envelope layout (page_size==1 for state): all
# mamba layers/slots share one contiguous byte buffer; conv and
# temporal are strided views into it (see mem_cache/layout/
# page_major.py). Only the standard CUDA Triton path is supported.
# token_major.py). Only the standard CUDA Triton path is supported.
assert not _is_npu and not (_is_cpu and _cpu_has_amx_support), (
"envelope_layout mamba is only supported on the CUDA path"
)
@@ -603,7 +602,7 @@ class MambaPool:
self._raw = torch.zeros(
max_slots * entry_bytes, dtype=torch.uint8, device=device
)
conv_state, temporal_state = build_page_major_mamba_views(
conv_state, temporal_state = build_mamba_entry_views(
self._raw,
layer_num=num_mamba_layers,
conv_state_shapes=conv_state_shape,
@@ -1845,10 +1844,6 @@ class KVCache(abc.ABC):
):
self.size = size
self.page_size = page_size
# Row-blocks one page holds in this pool's kernel-facing id space; >1
# only for the unified pool's per-layer views, and then a write loc must
# have been translated into that space first.
self.kernel_page_blocks = 1
self.dtype = dtype
self.device = device
if dtype in (torch.float8_e5m2, torch.float8_e4m3fn, torch.float8_e4m3fnuz):
@@ -2615,9 +2610,6 @@ class MHATokenToKVPool(KVCache):
# Catch stale slot ids here instead of as illegal-addr / silent KV
# corruption in the store_kvcache write (gated on SGLANG_ENABLE_ASYNC_ASSERT).
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_kv_buffer (MHA)")
maybe_detect_kernel_facing_loc(
loc, self.page_size, self.kernel_page_blocks, "set_kv_buffer (MHA)"
)
layer_id = (
layer_id_override if layer_id_override is not None else layer.layer_id
)
@@ -4530,9 +4522,7 @@ class MLATokenToKVPool(KVCache):
# Has the WRITE loc arriving here already had the DCP owner rule resolved?
# False: this pool takes a WIDENED loc. The unified pool resolves it in
# `KVIndexTranslator.rebind_write_loc` and flips this. Not derivable from
# `kernel_page_blocks`: that is `layer_num`, so a rank owning one
# full-attention layer is translated with blocks_per_page 1.
# `KVIndexTranslator.rebind_write_loc` and flips this.
write_loc_is_dcp_resolved = False
@property
@@ -4564,9 +4554,6 @@ class MLATokenToKVPool(KVCache):
):
loc, _, _ = unwrap_write_loc(loc_info)
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_kv_buffer (MLA)")
maybe_detect_kernel_facing_loc(
loc, self.page_size, self.kernel_page_blocks, "set_kv_buffer (MLA)"
)
layer_id = (
layer_id_override if layer_id_override is not None else layer.layer_id
)
@@ -4651,9 +4638,6 @@ class MLATokenToKVPool(KVCache):
(self.size + self.page_size) * self._write_loc_dcp_span,
"set_mla_kv_buffer (MLA)",
)
maybe_detect_kernel_facing_loc(
loc, self.page_size, self.kernel_page_blocks, "set_mla_kv_buffer (MLA)"
)
layer_id = (
layer_id_override if layer_id_override is not None else layer.layer_id
)
@@ -157,6 +157,9 @@ class HostKVCache(abc.ABC):
dcp_size = 1
dcp_rank = 0
shared_allocation_domain = None
# Names this pool's page byte format in storage keys when it has one of its
# own, so pages persisted in another format miss instead of loading.
storage_format_tag: Optional[str] = None
def __init__(
self,
@@ -610,6 +610,8 @@ class _SharedPageEnvelopeHostBacking:
class UnifiedPageEnvelopeHostPool(HostKVCache):
"""Host mirror that transfers complete unified-memory page envelopes."""
storage_format_tag = "unified-token-major"
def __init__(
self,
device_pool,
@@ -788,13 +790,6 @@ class UnifiedPageEnvelopeHostPool(HostKVCache):
host_indices = self._shared_backing.translate_indices(
self.pool_label, host_indices
)
# The L2 engine resolves device IDs using the per-layer HiCache contract.
# This pool copies whole envelopes, so discard the kernel-view page stride.
kernel_page_size = self.page_size * self.device_pool.kernel_page_blocks
device_indices = (
device_indices // kernel_page_size * self.page_size
+ device_indices % self.page_size
)
if io_backend == "kernel":
if not host_indices.is_cuda:
host_indices = host_indices.to(
+129 -163
View File
@@ -36,10 +36,13 @@ from torch.profiler import record_function
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
from sglang.srt.mem_cache.layout.page_major import (
build_mha_views,
build_mla_views,
build_page_major_mamba_views,
from sglang.srt.mem_cache.layout.token_major import (
ROW_ALIGN_BYTES,
DenseEntryLayout,
DensePart,
align_entry_bytes,
build_dense_views,
build_mamba_entry_views,
)
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
@@ -64,6 +67,19 @@ def _prod(iterable) -> int:
return out
def _check_row_alignment(sub_pool: str, **row_bytes: int) -> None:
"""Entry parts are laid out in ROW_ALIGN_BYTES units; a model whose
per-rank rows are not cannot use the unified pool."""
for part, nbytes in row_bytes.items():
if nbytes % ROW_ALIGN_BYTES:
raise ValueError(
f"--enable-unified-memory needs every per-rank KV row to be a "
f"multiple of {ROW_ALIGN_BYTES} bytes; sub-pool {sub_pool!r} has a "
f"{nbytes}-byte {part} row. Run this model without "
"--enable-unified-memory."
)
def _store_dtype_for(kv_cache_dtype: torch.dtype) -> torch.dtype:
if kv_cache_dtype in (
torch.float8_e5m2,
@@ -110,19 +126,6 @@ class SubPoolSpec(ABC):
"""Storage dtype (informational). Multi-dtype subclasses return the dominant buffer's."""
raise NotImplementedError
def view_tail_pad_bytes(self, page_size: int) -> int:
"""Bytes this sub-pool's views reach PAST its last page envelope."""
return 0
def blocks_per_page(self) -> int:
"""Row-blocks one page holds in this sub-pool's kernel-facing id space.
The page envelope is a uniform array of equally wide row-blocks, so a
kernel-facing id is the physical page scaled by this count (see
`MultiEndedAllocator.translate_kv_loc_for_kernel`). 1 means the kernel-facing ids are the physical ones.
"""
return 1
@dataclass(frozen=True, kw_only=True)
class MHASubPoolSpec(SubPoolSpec):
@@ -159,20 +162,39 @@ class MHASubPoolSpec(SubPoolSpec):
return self.head_num * self.v_head_dim * self.store_dtype.itemsize
def entry_bytes(self) -> int:
return self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
# Page-major byte math: within a page block K/V group per layer
# [L0_K*ps | L0_V*ps | L1_K*ps | ...]; at ps==1 this collapses to the per-slot envelope.
return align_entry_bytes(
self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
)
def page_bytes(self, page_size: int) -> int:
return page_size * self.entry_bytes()
def view_tail_pad_bytes(self, page_size: int) -> int:
return page_size * self.entry_bytes()
def blocks_per_page(self) -> int:
"""Row-blocks per page in the kernel-facing id space (one K + one V per layer)."""
return 2 * self.layer_num
def layout(self) -> DenseEntryLayout:
"""Token-major entry ``[K_0 | V_0 | K_1 | V_1 | ...]`` per slot; a page
is ``page_size`` such entries back to back."""
_check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes())
layer_stride = self.k_row_bytes() + self.v_row_bytes()
return DenseEntryLayout(
entry_bytes=self.entry_bytes(),
parts=(
DensePart(
name="k",
offset_bytes=0,
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.head_dim),
dtype=self.store_dtype,
),
DensePart(
name="v",
offset_bytes=self.k_row_bytes(),
layer_stride_bytes=layer_stride,
layer_num=self.layer_num,
row_shape=(self.head_num, self.v_head_dim),
dtype=self.store_dtype,
),
),
)
def get_dtype(self) -> torch.dtype:
return self.store_dtype
@@ -205,16 +227,27 @@ class MLASubPoolSpec(SubPoolSpec):
def kv_cache_dim(self) -> int:
return self.kv_lora_rank + self.qk_rope_head_dim
def row_bytes(self) -> int:
return self.kv_cache_dim * self.store_dtype.itemsize
def entry_bytes(self) -> int:
return self.layer_num * self.kv_cache_dim * self.store_dtype.itemsize
return align_entry_bytes(self.layer_num * self.row_bytes())
def view_tail_pad_bytes(self, page_size: int) -> int:
return page_size * self.entry_bytes()
def blocks_per_page(self) -> int:
"""One latent row per layer, so L blocks per page (MHA has 2L: a K
block and a V block per layer)."""
return self.layer_num
def layout(self) -> DenseEntryLayout:
_check_row_alignment(self.name, latent=self.row_bytes())
return DenseEntryLayout(
entry_bytes=self.entry_bytes(),
parts=(
DensePart(
name="kv",
offset_bytes=0,
layer_stride_bytes=self.row_bytes(),
layer_num=self.layer_num,
row_shape=(1, self.kv_cache_dim),
dtype=self.store_dtype,
),
),
)
def get_dtype(self) -> torch.dtype:
return self.store_dtype
@@ -266,7 +299,7 @@ def _assert_kernel_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
assert n_rows < 2**31, (
f"sub-pool {sub_pool_name!r}: kernel-facing id space has {n_rows} rows, "
f"exceeding the int32 bound (2^31) that read-index buffers assume. "
"Reduce max_total_num_tokens or the layer count."
"Reduce max_total_num_tokens."
)
@@ -274,12 +307,12 @@ def _reserved_floor_bytes(sub_pool_specs: List[SubPoolSpec], page_size: int) ->
"""Bytes at the bottom of the buffer reserved as the slot-0 padding sink.
Slot-0 dummy writes for every sub-pool land here; each sub-pool's first
allocatable slot is chosen so real data starts past it. For a PAGE-AWARE
sub-pool the slot-0 write touches layer blocks spread across the whole
page-0 envelope (page_size * entry_bytes), not just one slot envelope --
but a mamba sub-pool is page_size=1, so its entry is charged ONCE. Charging
a mamba entry per page would reserve page_size * ~100 MB of buffer that the
sink never touches.
allocatable slot is chosen so real data starts past it. A slot-0 write
touches only slot 0's entry, but a PAGE-AWARE sub-pool allocates whole
pages, so its first allocatable page starts past all of page 0
(page_size * entry_bytes) -- while a mamba sub-pool is page_size=1, so its
entry is charged ONCE. Charging a mamba entry per page would reserve
page_size * ~100 MB of buffer that the sink never touches.
Single source of truth: `UnifiedKVPool` reserves exactly this, and the
factories' bs=1 feasibility floors charge exactly this.
@@ -296,10 +329,10 @@ def _reserved_floor_bytes(sub_pool_specs: List[SubPoolSpec], page_size: int) ->
class UnifiedKVPool:
"""One physical `uint8` byte buffer shared by N sub-pools, each exposing
per-layer views over its own byte range (contiguous per layer for KV,
strided for the Mamba state). Two END pools (one grow-up, one grow-down)
own the buffer's ends; optional "float" MIDDLE pools live between their
frontiers. Allocators keep byte ranges disjoint; no usage tracking here.
per-layer strided views over its own byte range. Two END pools (one
grow-up, one grow-down) own the buffer's ends; optional "float" MIDDLE
pools live between their frontiers. Allocators keep byte ranges disjoint;
no usage tracking here.
"""
def __init__(
@@ -347,13 +380,8 @@ class UnifiedKVPool:
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
enable=enable_memory_saver
)
self.view_tail_pad_bytes = max(
spec.view_tail_pad_bytes(page_size) for spec in sub_pool_specs
)
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
self._raw = torch.empty(
total_bytes + self.view_tail_pad_bytes, dtype=torch.uint8, device=device
)
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
if envs.SGLANG_DEBUG_POISON_POOL.get():
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
# fail deterministically.
@@ -368,7 +396,7 @@ class UnifiedKVPool:
self._max_slots: Dict[str, int] = {}
self._anchor_bytes: Dict[str, int] = {}
self._min_slot_index: Dict[str, int] = {}
# MHA: (k_buffer, v_buffer); MLA: [per-layer per-layer views];
# MHA: (k_buffer, v_buffer); MLA: [per-layer views];
# Mamba: (conv_state_list, temporal_state)
self._mha_views: Dict[str, Tuple[List[torch.Tensor], List[torch.Tensor]]] = {}
self._mla_views: Dict[str, List[torch.Tensor]] = {}
@@ -376,9 +404,9 @@ class UnifiedKVPool:
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
# each pool's first allocatable slot is chosen so real data starts past it.
# For a page-aware sub-pool the slot-0 write touches layer blocks spread
# across the WHOLE page-0 envelope (up to page_size * entry_bytes), not
# just one slot envelope — reserve the max of both.
# A page-aware sub-pool allocates whole pages, so the sink is all of page
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
# max of both.
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
for spec in self.sub_pool_specs:
@@ -490,22 +518,20 @@ class UnifiedKVPool:
max_slots: int,
page_size: int,
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
num_pages = max_slots // page_size
_assert_kernel_id_bound(
sub_pool_name=spec.name,
n_rows=num_pages * spec.blocks_per_page() * page_size,
)
return build_mha_views(
self._raw,
layer_num=spec.layer_num,
head_num=spec.head_num,
head_dim=spec.head_dim,
v_head_dim=spec.v_head_dim,
store_dtype=spec.store_dtype,
page_size=page_size,
num_pages=num_pages,
anchor_bytes=anchor_bytes,
num_slots = max_slots // page_size * page_size
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
layout = spec.layout()
k_views, v_views = (
build_dense_views(
self._raw,
layout=layout,
part_name=name,
num_slots=num_slots,
anchor_bytes=anchor_bytes,
)
for name in ("k", "v")
)
return k_views, v_views
def _build_mla_views(
self,
@@ -514,25 +540,21 @@ class UnifiedKVPool:
max_slots: int,
page_size: int,
) -> List[torch.Tensor]:
num_pages = max_slots // page_size
_assert_kernel_id_bound(
sub_pool_name=spec.name,
n_rows=num_pages * spec.blocks_per_page() * page_size,
)
return build_mla_views(
num_slots = max_slots // page_size * page_size
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
layout = spec.layout()
return build_dense_views(
self._raw,
layer_num=spec.layer_num,
kv_cache_dim=spec.kv_cache_dim,
store_dtype=spec.store_dtype,
page_size=page_size,
num_pages=num_pages,
layout=layout,
part_name="kv",
num_slots=num_slots,
anchor_bytes=anchor_bytes,
)
def _build_mamba_views(
self, spec: MambaSubPoolSpec, anchor_bytes: int, max_slots: int
) -> Tuple[List[torch.Tensor], torch.Tensor]:
return build_page_major_mamba_views(
return build_mamba_entry_views(
self._raw,
layer_num=spec.layer_num,
conv_state_shapes=spec.conv_state_shapes,
@@ -545,16 +567,10 @@ class UnifiedKVPool:
class UnifiedMHATokenToKVPool(MHATokenToKVPool):
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are `build_mha_views`
views into a `UnifiedKVPool` (requires uniform K/V rows).
Views are contiguous `(n_rows, head_num, head_dim)`; locs are
kernel_id(t) = (t // ps) * (ps * 2 * layer_num) + t % ps
which is layer- and K/V-independent, each view's storage_offset folding in
its block origin (layer l's K at block 2l, V at 2l+1). `move_kv_cache` is
the exception: compaction passes REAL physical token ids.
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are token-major views
into a `UnifiedKVPool`: `(num_pages * ps, head_num, head_dim)` with the whole
entry as slot stride, indexed by the physical token id. `move_kv_cache`
relocates whole page envelopes.
"""
def __init__(
@@ -577,7 +593,7 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
self._v_views = v_views
self._num_pages = max_slots // page_size
self._page_bytes = page_size * spec.entry_bytes()
view_rows = self._num_pages * spec.blocks_per_page() * page_size
view_rows = self._num_pages * page_size
super().__init__(
size=view_rows - page_size,
@@ -595,7 +611,6 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
enable_kv_cache_copy=False,
kv_cache_layout="page_major",
)
self.kernel_page_blocks = spec.blocks_per_page()
def _create_buffers(self):
self.k_buffer = self._k_views
@@ -611,9 +626,8 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
return 0, 0 # UnifiedKVPool logs the total; per-sub-pool would double-count
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
"""Relocate slots by whole page envelope.
`tgt_loc`/`src_loc` are REAL physical token ids, not kernel-facing ids.
"""
"""Relocate slots by whole page envelope (`tgt_loc`/`src_loc` are
physical token ids)."""
if tgt_loc.numel() == 0:
return
# The envelope view below starts at byte 0, so this sub-pool must be
@@ -654,20 +668,6 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
"""Physical growth direction used by this sub-pool's L1 allocator."""
return self._unified_buffer.spec(self._sub_pool_name).grow_direction
def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor:
return (indices // self.page_size) * (
self.page_size * self.kernel_page_blocks
) + indices % self.page_size
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
"""Translate physical host-pool ids for the page-major parent path."""
return super().get_cpu_copy(self._physical_to_kernel_indices(indices))
def load_cpu_copy(
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
):
super().load_cpu_copy(kv_cache_cpu, self._physical_to_kernel_indices(indices))
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
raise NotImplementedError(
"prefix-valid commit is unsupported under the unified layout "
@@ -676,21 +676,11 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
class UnifiedMLATokenToKVPool(MLATokenToKVPool):
"""MLA KV pool whose per-layer `kv_buffer` entries are kernel-facing views into a
`UnifiedKVPool` (see `build_mla_views`).
Loc-space contract: every loc this pool receives through the KVCache API
(`set_kv_buffer` / `set_mla_kv_buffer` / `get_mla_kv_buffer`, and the
kv_indices consumed by attention kernels reading `get_key_buffer` /
`get_value_buffer`) is a kernel-facing id — the `translate_kv_loc_for_kernel` output
kernel_id(t) = (t // ps) * (ps * layer_num) + t % ps
which is layer-independent (the layer offset is folded into each view's
storage_offset), so the stock `MLATokenToKVPool` read/write methods work on
the views unmodified. The ONE exception is `move_kv_cache`: the allocator's
compaction calls it with REAL physical token ids, and it is overridden to
relocate whole page envelopes on the raw buffer.
"""MLA KV pool whose per-layer `kv_buffer` entries are token-major views
into a `UnifiedKVPool`: `(num_pages * ps, 1, kv_cache_dim)` with the whole
entry as slot stride, indexed by the physical token id (`translate_kv_loc`
of the virtual id), so the stock `MLATokenToKVPool` read/write methods work
on them unmodified. `move_kv_cache` relocates whole page envelopes.
"""
def __init__(
@@ -714,10 +704,10 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
max_slots = unified_buffer.max_slots(sub_pool_name)
self._num_pages = max_slots // page_size
self._page_bytes = page_size * spec.entry_bytes()
self._view_rows = self._num_pages * spec.blocks_per_page() * page_size
self._view_rows = self._num_pages * page_size
super().__init__(
# OOB checks bound locs by `size + page_size`; kernel-facing ids run to
# OOB checks bound locs by `size + page_size`; physical ids run to
# `_view_rows` (page 0 is the reserved padding sink).
size=self._view_rows - page_size,
page_size=page_size,
@@ -728,7 +718,6 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
device=unified_buffer.device,
enable_memory_saver=False, # buffer owned by UnifiedKVPool
)
self.kernel_page_blocks = spec.blocks_per_page()
def _create_buffers(self):
self.kv_buffer = self._kv_views
@@ -749,36 +738,20 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
``raw_ptr + physical_page_id * page_envelope_bytes``.
The transfer item is the whole page envelope (all layers of one page)
rather than a per-layer region, because the per-layer per-layer views
overlap and index in kernel-facing ids. Both sides must therefore build the
pool with identical specs.
rather than a per-layer region: the layers interleave inside each
slot's entry. Both sides must therefore build the pool with identical
specs.
"""
# The address formula omits the anchor; a nonzero one would mis-address.
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
raw = self._unified_buffer._raw
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor:
"""Physical TOKEN ids -> the kernel-facing ids this class's `kv_buffer`
views are indexed by; the formula is the one in the class docstring."""
return (indices // self.page_size) * (
self.page_size * self.kernel_page_blocks
) + indices % self.page_size
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
"""Translate physical host-pool ids for the page-major parent path."""
return super().get_cpu_copy(self._physical_to_kernel_indices(indices))
def load_cpu_copy(
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
):
super().load_cpu_copy(kv_cache_cpu, self._physical_to_kernel_indices(indices))
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
"""Relocate whole page envelopes.
`tgt_loc`/`src_loc` are REAL physical token ids (NOT kernel-facing ids): both
compaction paths expand page ids into page-major-ordered token runs
`tgt_loc`/`src_loc` are physical token ids: both compaction paths
expand page ids into page-major-ordered token runs
(`pages[:, None] * ps + offsets`), relied on here to recover the page
lists. One contiguous envelope copy replaces the per-layer strided moves.
"""
@@ -1458,15 +1431,13 @@ def init_unified_mamba_pools(
if use_mla_backend:
logger.info(
"[unified-memory-pool] full_layers=%d, mamba_layers=%d, kv_lora_rank=%d, "
"qk_rope_head_dim=%d, page_size=%d (per-layer views, kernel_page_multiplier=%d, "
"view_tail_pad=%d B)",
"qk_rope_head_dim=%d, page_size=%d (token-major views, entry_bytes=%d B)",
len(full_attention_layer_ids),
len(mamba_layer_ids),
kv_lora_rank,
qk_rope_head_dim,
page_size,
len(full_attention_layer_ids),
shared_pool.view_tail_pad_bytes,
full_spec.entry_bytes(),
)
else:
logger.info(
@@ -1478,8 +1449,7 @@ def init_unified_mamba_pools(
head_dim,
page_size,
is_draft_worker,
"per-layer views, kernel_page_multiplier=%d, view_tail_pad=%d B"
% (full_spec.blocks_per_page(), shared_pool.view_tail_pad_bytes),
"token-major views, entry_bytes=%d B" % full_spec.entry_bytes(),
)
logger.info(
"[unified-memory-pool] total_bytes=%d, max_total_num_tokens=%d, max_mamba_cache_size=%d, "
@@ -1903,12 +1873,8 @@ def init_unified_swa_pools(
logger.info("[unified-memory-pool] UNIFIED MEMORY POOL ENABLED -- path=SWA hybrid")
logger.info(
"[unified-memory-pool] %s",
"per-layer views, kernel_page_multiplier full=%d swa=%d, view_tail_pad=%d B"
% (
full_spec.blocks_per_page(),
swa_spec.blocks_per_page(),
shared_pool.view_tail_pad_bytes,
),
"token-major views, entry_bytes full=%d swa=%d B"
% (full_spec.entry_bytes(), swa_spec.entry_bytes()),
)
logger.info(
"[unified-memory-pool] full_layers=%d, swa_layers=%d, head_num=%d, head_dim=%d, "
-21
View File
@@ -141,27 +141,6 @@ def maybe_detect_oob(indices: Optional[torch.Tensor], low: int, high: int, msg:
)
def maybe_detect_kernel_facing_loc(
indices: Optional[torch.Tensor], page_size: int, blocks_per_page: int, msg: str
):
"""Async check that a write loc is in the pool's KERNEL-FACING id space.
A kernel-facing id is `phys_page * (page_size * blocks_per_page) + offset`
with `offset < page_size`, so its remainder modulo the page stride is
below page_size; a VIRTUAL id satisfies that only in the first block.
Vacuous at blocks_per_page 1. Virtual ids are in range for the OOB probe,
so this is the only check that separates them.
"""
if blocks_per_page <= 1 or not envs.SGLANG_ENABLE_ASYNC_ASSERT.get():
return
if indices is None or indices.numel() == 0:
return
torch._assert_async(
(indices % (page_size * blocks_per_page) < page_size).all(),
f"write loc outside the kernel-facing id space (virtual ids?): {msg}",
)
def maybe_detect_page_aligned(
indices: Optional[torch.Tensor], page_size: int, msg: str
):