[unified-memory] Token-major dense views for the unified memory pool (3/7) (#38592)
Co-authored-by: Caihua Li <caihua.li@bytedance.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
Caihua Li
Cheng Wan
parent
5df52e6382
commit
cdadb1c79a
@@ -70,19 +70,18 @@ class Memory(msgspec.Struct):
|
||||
] = False
|
||||
enable_page_major_kv_layout: A[
|
||||
bool,
|
||||
"Enable the page-major KV layout: lay out the Mamba state and full/SWA "
|
||||
"KV caches in a page-granularity envelope (page is the outermost axis, "
|
||||
"layer-major within a page) instead of the default per-layer "
|
||||
"(layer-major) layout. Requires the Triton attention / linear-attn / "
|
||||
"Mamba backends.",
|
||||
"Store the Mamba state and full/SWA KV caches as one entry per token "
|
||||
"slot, holding every layer's state, in a single page-granular buffer "
|
||||
"instead of one buffer per layer. Implied by --enable-unified-memory, "
|
||||
"and currently supported only with it.",
|
||||
] = False
|
||||
enable_unified_memory: A[
|
||||
bool,
|
||||
"Replace the statically-partitioned hybrid-model pools (full-attn KV + "
|
||||
"SWA/Mamba state) with one byte buffer split dynamically between "
|
||||
"sub-pools. Requires the Triton attention / linear-attn / Mamba "
|
||||
"backends. Supported PD-disaggregation and speculative-decoding "
|
||||
"configurations are validated at startup.",
|
||||
"sub-pools. Requires stride-aware attention, linear-attention and Mamba "
|
||||
"backends; the allowed backends and the supported PD-disaggregation and "
|
||||
"speculative-decoding configurations are validated at startup.",
|
||||
] = False
|
||||
enable_session_radix_cache: A[
|
||||
bool,
|
||||
|
||||
@@ -608,6 +608,18 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
sorted(full_cg_backends),
|
||||
sorted(backends),
|
||||
)
|
||||
# trtllm_mha refills its graph page table before replay from
|
||||
# `cache_seqlens_int32`, which only the in-graph metadata kernel writes, so
|
||||
# it uses the previous replay's lengths. Refuse until it uses the batch's.
|
||||
_, decode_backend = attention_backends_of(resolved_view(server_args))
|
||||
if decode_backend == "trtllm_mha":
|
||||
assert _cg_cfg is None or _cg_cfg.decode.backend == Backend.DISABLED, (
|
||||
"--enable-unified-memory does not yet support decode cuda graphs "
|
||||
"with the trtllm_mha attention backend: its replay refills the page "
|
||||
"table from the previous replay's sequence lengths. Pass "
|
||||
"--disable-cuda-graph, or pick another decode attention backend "
|
||||
"(fa3 / fa4 / flashinfer / triton)."
|
||||
)
|
||||
|
||||
|
||||
def _validate_unified_memory_dcp(server_args: Any) -> None:
|
||||
@@ -656,9 +668,8 @@ def _validate_unified_memory_dcp(server_args: Any) -> None:
|
||||
|
||||
|
||||
def handle_page_major_kv_layout(server_args: Any):
|
||||
# The unified pool stores state in the page-major envelope-strided layout, so
|
||||
# enabling it implies --enable-page-major-kv-layout — routing it through the
|
||||
# single page-major path + stride-aware Triton asserts (set before the guard).
|
||||
# --enable-unified-memory implies --enable-page-major-kv-layout, so the
|
||||
# unified pool goes through this one gate (declared before the guard).
|
||||
|
||||
cfg = resolving_view(server_args)
|
||||
if cfg.enable_unified_memory:
|
||||
@@ -684,24 +695,24 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
assert unified_memory_supported_for_model(
|
||||
model_config, use_mla_backend=use_mla_backend(server_args)
|
||||
), (
|
||||
"--enable-unified-memory requires uniform K/V rows "
|
||||
"(head_dim == v_head_dim); this model has "
|
||||
"--enable-unified-memory does not yet admit asymmetric K/V rows "
|
||||
"(head_dim != v_head_dim); this model has "
|
||||
f"head_dim={model_config.head_dim}, "
|
||||
f"v_head_dim={model_config.v_head_dim}, "
|
||||
f"swa_head_dim={model_config.swa_head_dim}, "
|
||||
f"swa_v_head_dim={model_config.swa_v_head_dim}. The unified "
|
||||
"pool's per-layer views require a uniform row width; run "
|
||||
"this model without --enable-unified-memory."
|
||||
f"swa_v_head_dim={model_config.swa_v_head_dim}. The token-major "
|
||||
"views can hold them, but the backends' write and read paths are "
|
||||
"not audited for it; run this model without --enable-unified-memory."
|
||||
)
|
||||
# Allow-list. Every backend below reads through the translator, so what
|
||||
# gates one is only whether its kernels can address the per-layer views:
|
||||
# gates one is only whether its kernels address the per-layer views by
|
||||
# their strides (the slot stride is the whole entry, not one row):
|
||||
# * MLA models: the full paged MLA family, incl. flashmla (ps=64
|
||||
# snap).
|
||||
# * MHA/SWA models: fa3 / fa4 / flashinfer / trtllm_mha alongside
|
||||
# Triton. fa4 is the fa3 class.
|
||||
# * Without the unified pool, plain page-major stays Triton-only.
|
||||
# Names are the RESOLVED ids from attention_backends_of.
|
||||
if cfg.enable_unified_memory and use_mla_backend(server_args):
|
||||
if use_mla_backend(server_args):
|
||||
allowed_full = {
|
||||
"triton",
|
||||
"fa3",
|
||||
@@ -711,7 +722,7 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
"tokenspeed_mla",
|
||||
"flashmla",
|
||||
}
|
||||
elif cfg.enable_unified_memory:
|
||||
else:
|
||||
allowed_full = {
|
||||
"triton",
|
||||
"fa3",
|
||||
@@ -719,17 +730,14 @@ def handle_page_major_kv_layout(server_args: Any):
|
||||
"flashinfer",
|
||||
"trtllm_mha",
|
||||
}
|
||||
else:
|
||||
allowed_full = {"triton"}
|
||||
backends = set(attention_backends_of(resolved_view(server_args)))
|
||||
backends.discard(None)
|
||||
assert backends <= allowed_full, (
|
||||
"--enable-page-major-kv-layout: the resolved attention backends "
|
||||
f"{sorted(backends)} are not in the allowed set "
|
||||
f"{sorted(allowed_full)} for this configuration (unified memory "
|
||||
"allows the per-layer-view families; plain page-major keeps the "
|
||||
"envelope-strided views only Triton reads). Pass a compatible "
|
||||
"--attention-backend."
|
||||
"allows the stride-aware per-layer-view families). Pass a "
|
||||
"compatible --attention-backend."
|
||||
)
|
||||
# The Mamba/KDA state is stored in envelope-strided views; only
|
||||
# stride-audited kernels may read it (Stage 4 audit, per slot):
|
||||
|
||||
@@ -50,6 +50,17 @@ logger = logging.getLogger(__name__)
|
||||
device_module = get_device_module()
|
||||
|
||||
|
||||
def storage_model_name(
|
||||
model_name: Optional[str], host_pool: HostKVCache
|
||||
) -> Optional[str]:
|
||||
"""The model name storage backends key pages on, tagged with the host
|
||||
pool's page format when it has one of its own."""
|
||||
tag = host_pool.storage_format_tag
|
||||
if tag is None:
|
||||
return model_name
|
||||
return f"{model_name}-{tag}" if model_name else tag
|
||||
|
||||
|
||||
class LayerLoadingEvent:
|
||||
def __init__(self, num_layers: int):
|
||||
self._num_layers = num_layers
|
||||
@@ -730,6 +741,7 @@ class HiCacheController:
|
||||
)
|
||||
|
||||
attn_cp_rank, attn_cp_size = self.get_attn_cp_rank_and_size()
|
||||
model_name = storage_model_name(model_name, self.storage_host_pool)
|
||||
|
||||
return HiCacheStorageConfig(
|
||||
tp_rank=self.tp_rank,
|
||||
|
||||
@@ -316,13 +316,13 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self.entry_bytes = spec.entry_bytes()
|
||||
self.min_slot_index = unified_buffer.min_slot_index(sub_pool_name)
|
||||
self.is_id_owner = is_id_owner
|
||||
# Kernel-facing page-stride scale, from the spec that owns the layout;
|
||||
# `kernel_page_multiplier=` overrides it only for tests.
|
||||
self.kernel_page_multiplier = (
|
||||
spec.blocks_per_page()
|
||||
if kernel_page_multiplier is None
|
||||
else kernel_page_multiplier
|
||||
# Kernel-facing ids are the physical token ids: the token-major views
|
||||
# step slots by the whole entry, so there is no per-page block scale.
|
||||
assert kernel_page_multiplier in (None, 1), (
|
||||
f"kernel_page_multiplier must be 1 (token-major views); got "
|
||||
f"{kernel_page_multiplier}"
|
||||
)
|
||||
self.kernel_page_multiplier = 1
|
||||
# Zero page envelopes on hand-out -- see _maybe_zero_pages.
|
||||
self._zero_pages_on_alloc = isinstance(kvcache, UnifiedMLATokenToKVPool)
|
||||
# Overlap mode: `free` drops a wait_stream(forward_stream) barrier so its
|
||||
@@ -1060,14 +1060,15 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
*,
|
||||
out: Optional[torch.Tensor] = None,
|
||||
) -> torch.Tensor:
|
||||
"""Virtual token ids -> kernel-facing ids:
|
||||
"""Virtual token ids -> kernel-facing ids, which under the token-major
|
||||
views are the physical token ids:
|
||||
|
||||
kernel_id(t) = (t // ps) * (ps * kernel_page_multiplier) + t % ps
|
||||
kernel_id(t) = v2p[t // ps] * ps + t % ps
|
||||
|
||||
Internal machinery (compaction, in-flight write sets) MUST keep using
|
||||
`translate_kv_loc`: kernel-facing ids are for kernels only. Tombstones (-1)
|
||||
clamp to kernel-facing id 0, the page-0 sink. int64 out; a consumer whose
|
||||
kernel ABI wants int32 narrows where it fills that buffer.
|
||||
(`kernel_page_multiplier` is pinned to 1, so this equals
|
||||
`translate_kv_loc`.) Tombstones (-1) clamp to id 0, the page-0 sink.
|
||||
int64 out; a consumer whose kernel ABI wants int32 narrows where it
|
||||
fills that buffer.
|
||||
"""
|
||||
with record_function("MultiEndedAlloc.translate_kv_loc_for_kernel"):
|
||||
return self._translate_loc_fused(virt_tokens, dcp_size=1, out=out)
|
||||
|
||||
@@ -1,293 +0,0 @@
|
||||
"""Page-granularity envelope (page-major, layer-major within a page) cache views.
|
||||
|
||||
A pool of this layout keeps all layers of all slots in one contiguous byte
|
||||
buffer. The buffer is split into pages of ``page_size`` slots; within a page,
|
||||
each layer's K and V (or each Mamba conv/temporal tensor) are grouped together:
|
||||
|
||||
page bytes = [L0_K * ps | L0_V * ps | L1_K * ps | L1_V * ps | ...]
|
||||
|
||||
Across pages the layout is envelope-major (one ``page_bytes`` block per page).
|
||||
At ``page_size == 1`` a page is a single slot, so the within-page block is the
|
||||
per-slot ``[L0_K | L0_V | L1_K | L1_V | ...]`` envelope (token-granularity).
|
||||
|
||||
These builders produce per-layer views into a raw ``uint8`` buffer; they hold
|
||||
no allocator/ownership state. ``anchor_bytes`` is the byte offset of the
|
||||
pool's region inside the raw buffer (0 for a standalone pool).
|
||||
"""
|
||||
|
||||
from typing import List, Sequence, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
def _prod(shape: Sequence[int]) -> int:
|
||||
out = 1
|
||||
for s in shape:
|
||||
out *= int(s)
|
||||
return out
|
||||
|
||||
|
||||
def mha_entry_bytes(
|
||||
*, layer_num: int, head_num: int, head_dim: int, v_head_dim: int, itemsize: int
|
||||
) -> int:
|
||||
"""Bytes occupied by one slot across all layers (K and V)."""
|
||||
k_row_bytes = head_num * head_dim * itemsize
|
||||
v_row_bytes = head_num * v_head_dim * itemsize
|
||||
return layer_num * (k_row_bytes + v_row_bytes)
|
||||
|
||||
|
||||
def build_mha_views(
|
||||
raw: torch.Tensor,
|
||||
*,
|
||||
layer_num: int,
|
||||
head_num: int,
|
||||
head_dim: int,
|
||||
v_head_dim: int,
|
||||
store_dtype: torch.dtype,
|
||||
page_size: int,
|
||||
num_pages: int,
|
||||
anchor_bytes: int = 0,
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
"""Per-layer K/V views over ``raw`` for uniform-row MHA.
|
||||
|
||||
The page envelope ``[L0_K*ps | L0_V*ps | L1_K*ps | ...]`` is a uniform
|
||||
array of ``2*layer_num`` row-blocks when K and V rows are equally wide, so
|
||||
it is a valid paged pool under
|
||||
|
||||
kernel_id(t) = (t // ps) * (ps * 2 * layer_num) + t % ps
|
||||
|
||||
with layer ``l``'s K at block ``2l`` and its V at block ``2l+1``. Each view
|
||||
is a contiguous ``(num_pages * 2 * layer_num * ps, head_num, head_dim)``.
|
||||
|
||||
Views overlap by ``ps`` rows per block, safe because an id always resolves
|
||||
inside its own block; the last view runs ``(2*layer_num - 1) * ps`` rows
|
||||
past the envelope, so ``raw`` needs ``UnifiedKVPool.view_tail_pad_bytes``.
|
||||
"""
|
||||
assert head_dim == v_head_dim, (
|
||||
f"build_mha_views requires uniform rows (head_dim == v_head_dim); "
|
||||
f"got head_dim={head_dim}, v_head_dim={v_head_dim}. Asymmetric-KV "
|
||||
"models cannot use the unified pool (screened out at startup)."
|
||||
)
|
||||
itemsize = store_dtype.itemsize
|
||||
row_elems = head_num * head_dim
|
||||
row_bytes = row_elems * itemsize
|
||||
blocks = 2 * layer_num
|
||||
page_bytes = page_size * blocks * row_bytes
|
||||
n_rows = num_pages * blocks * page_size
|
||||
assert anchor_bytes % itemsize == 0
|
||||
last_view_end = (
|
||||
anchor_bytes + (blocks - 1) * page_size * row_bytes + n_rows * row_bytes
|
||||
)
|
||||
assert last_view_end <= raw.numel() * raw.itemsize, (
|
||||
f"build_mha_views: block {blocks - 1}'s view ends at byte "
|
||||
f"{last_view_end} but the raw buffer holds only "
|
||||
f"{raw.numel() * raw.itemsize} bytes; allocate the tail pad "
|
||||
f"(one page envelope = {page_bytes} B) via view_tail_pad_bytes"
|
||||
)
|
||||
|
||||
as_dtype_view = raw.view(store_dtype)
|
||||
k_buffer: List[torch.Tensor] = []
|
||||
v_buffer: List[torch.Tensor] = []
|
||||
for layer in range(layer_num):
|
||||
k_base_bytes = anchor_bytes + (2 * layer) * page_size * row_bytes
|
||||
v_base_bytes = k_base_bytes + page_size * row_bytes
|
||||
for base_bytes, out in ((k_base_bytes, k_buffer), (v_base_bytes, v_buffer)):
|
||||
assert base_bytes % itemsize == 0
|
||||
out.append(
|
||||
torch.as_strided(
|
||||
as_dtype_view,
|
||||
size=(n_rows, head_num, head_dim),
|
||||
stride=(row_elems, head_dim, 1),
|
||||
storage_offset=base_bytes // itemsize,
|
||||
)
|
||||
)
|
||||
return k_buffer, v_buffer
|
||||
|
||||
|
||||
def mla_entry_bytes(*, layer_num: int, kv_cache_dim: int, itemsize: int) -> int:
|
||||
"""Bytes occupied by one MLA slot across all layers (single latent row, no V)."""
|
||||
return layer_num * kv_cache_dim * itemsize
|
||||
|
||||
|
||||
def build_mla_views(
|
||||
raw: torch.Tensor,
|
||||
*,
|
||||
layer_num: int,
|
||||
kv_cache_dim: int,
|
||||
store_dtype: torch.dtype,
|
||||
page_size: int,
|
||||
num_pages: int,
|
||||
anchor_bytes: int = 0,
|
||||
) -> List[torch.Tensor]:
|
||||
"""Per-layer views over ``raw`` for MLA in the page-major layout.
|
||||
|
||||
The page envelope is ``[L0_latent * ps | L1_latent * ps | ...]``. Because all
|
||||
MLA layers share one uniform row size (``kv_cache_dim``), the envelope is
|
||||
itself a valid paged pool under a re-numbered index space: folding the
|
||||
layer offset ``l * ps * kv_cache_dim`` into each view's storage_offset makes
|
||||
every per-layer view a plain CONTIGUOUS ``(num_pages * layer_num * ps, 1,
|
||||
kv_cache_dim)`` tensor, addressed by the layer-independent kernel-facing id
|
||||
|
||||
kernel_id(t) = (t // ps) * (ps * layer_num) + t % ps (t = physical token)
|
||||
|
||||
so one shared block table (entry = page * layer_num) serves every layer, and
|
||||
kernels that require ``.view(-1, page_size, kv_cache_dim)`` (trtllm/cutlass/
|
||||
flashmla) work on the views natively.
|
||||
|
||||
The views overlap each other (view ``l+1`` is view ``l`` shifted by ``ps``
|
||||
rows); that is safe because layer ``l`` is only ever indexed at kernel-facing ids,
|
||||
which always resolve to layer-``l`` bytes relative to view ``l``'s origin.
|
||||
Layer ``layer_num-1``'s view extends ``(layer_num-1) * ps`` rows past the
|
||||
last page envelope, so ``raw`` must carry at least one extra page envelope
|
||||
of tail padding (``UnifiedKVPool``'s ``view_tail_pad_bytes``).
|
||||
"""
|
||||
itemsize = store_dtype.itemsize
|
||||
row_bytes = kv_cache_dim * itemsize
|
||||
page_bytes = page_size * layer_num * row_bytes
|
||||
n_rows = num_pages * layer_num * page_size
|
||||
assert anchor_bytes % itemsize == 0
|
||||
last_view_end = (
|
||||
anchor_bytes + (layer_num - 1) * page_size * row_bytes + (n_rows * row_bytes)
|
||||
)
|
||||
assert last_view_end <= raw.numel() * raw.itemsize, (
|
||||
f"build_mla_views: layer {layer_num - 1}'s view ends at byte "
|
||||
f"{last_view_end} but the raw buffer holds only "
|
||||
f"{raw.numel() * raw.itemsize} bytes; allocate the tail pad "
|
||||
f"(one page envelope = {page_bytes} B) via view_tail_pad_bytes"
|
||||
)
|
||||
|
||||
as_dtype_view = raw.view(store_dtype)
|
||||
views: List[torch.Tensor] = []
|
||||
for layer in range(layer_num):
|
||||
base_bytes = anchor_bytes + layer * page_size * row_bytes
|
||||
assert base_bytes % itemsize == 0
|
||||
views.append(
|
||||
torch.as_strided(
|
||||
as_dtype_view,
|
||||
size=(n_rows, 1, kv_cache_dim),
|
||||
stride=(kv_cache_dim, kv_cache_dim, 1),
|
||||
storage_offset=base_bytes // itemsize,
|
||||
)
|
||||
)
|
||||
return views
|
||||
|
||||
|
||||
def mamba_entry_bytes(
|
||||
*,
|
||||
layer_num: int,
|
||||
conv_state_shapes: Sequence[Sequence[int]],
|
||||
conv_dtype: torch.dtype,
|
||||
temporal_state_shape: Sequence[int],
|
||||
temporal_dtype: torch.dtype,
|
||||
) -> int:
|
||||
"""Bytes occupied by one Mamba slot across all layers (conv + temporal)."""
|
||||
total = 0
|
||||
for shape in conv_state_shapes:
|
||||
total += layer_num * _prod(shape) * conv_dtype.itemsize
|
||||
total += layer_num * _prod(temporal_state_shape) * temporal_dtype.itemsize
|
||||
return total
|
||||
|
||||
|
||||
def build_page_major_mamba_views(
|
||||
raw: torch.Tensor,
|
||||
*,
|
||||
layer_num: int,
|
||||
conv_state_shapes: Sequence[Sequence[int]],
|
||||
conv_dtype: torch.dtype,
|
||||
temporal_state_shape: Sequence[int],
|
||||
temporal_dtype: torch.dtype,
|
||||
max_slots: int,
|
||||
anchor_bytes: int = 0,
|
||||
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
||||
"""Per-slot envelope views over ``raw`` for Mamba state.
|
||||
|
||||
Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]...
|
||||
[temporal rows × layers]``. Each returned view has shape
|
||||
``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]``
|
||||
/ ``.temporal``. Mamba state is always token-granular (page_size == 1).
|
||||
"""
|
||||
entry_bytes = mamba_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
conv_state_shapes=conv_state_shapes,
|
||||
conv_dtype=conv_dtype,
|
||||
temporal_state_shape=temporal_state_shape,
|
||||
temporal_dtype=temporal_dtype,
|
||||
)
|
||||
|
||||
def contiguous_strides(shape: Sequence[int]) -> Tuple[int, ...]:
|
||||
strides = []
|
||||
acc = 1
|
||||
for s in reversed(shape):
|
||||
strides.append(acc)
|
||||
acc *= int(s)
|
||||
return tuple(reversed(strides))
|
||||
|
||||
conv_itemsize = conv_dtype.itemsize
|
||||
assert entry_bytes % conv_itemsize == 0, (
|
||||
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
|
||||
f"multiple of the conv-state itemsize {conv_itemsize} B"
|
||||
)
|
||||
assert anchor_bytes % conv_itemsize == 0, (
|
||||
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
|
||||
f"of the conv-state itemsize {conv_itemsize} B"
|
||||
)
|
||||
as_conv_dtype = raw.view(conv_dtype)
|
||||
conv_slot_stride_elems = entry_bytes // conv_itemsize
|
||||
|
||||
offset_bytes_within_entry = 0
|
||||
conv_views: List[torch.Tensor] = []
|
||||
for shape in conv_state_shapes:
|
||||
inner_shape_bytes = _prod(shape) * conv_itemsize
|
||||
assert inner_shape_bytes % conv_itemsize == 0
|
||||
offset_elems = (anchor_bytes + offset_bytes_within_entry) // conv_itemsize
|
||||
stride = (
|
||||
inner_shape_bytes // conv_itemsize,
|
||||
conv_slot_stride_elems,
|
||||
) + contiguous_strides(shape)
|
||||
conv_views.append(
|
||||
torch.as_strided(
|
||||
as_conv_dtype,
|
||||
size=(layer_num, max_slots) + tuple(shape),
|
||||
stride=stride,
|
||||
storage_offset=offset_elems,
|
||||
)
|
||||
)
|
||||
offset_bytes_within_entry += layer_num * inner_shape_bytes
|
||||
|
||||
# The temporal view's storage_offset is computed in temporal-dtype elements
|
||||
# by integer-dividing a byte offset by itemsize, so every term of that byte
|
||||
# offset (entry stride, anchor, the conv region) must be a whole multiple of
|
||||
# itemsize or the offset truncates and mis-places the view.
|
||||
itemsize = temporal_dtype.itemsize
|
||||
assert entry_bytes % itemsize == 0, (
|
||||
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
|
||||
f"multiple of the temporal-state itemsize {itemsize} B; the temporal "
|
||||
f"view's storage_offset would truncate and mis-place the state"
|
||||
)
|
||||
assert anchor_bytes % itemsize == 0, (
|
||||
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
|
||||
f"of the temporal-state itemsize {itemsize} B"
|
||||
)
|
||||
inner_shape_bytes = _prod(temporal_state_shape) * itemsize
|
||||
assert inner_shape_bytes % itemsize == 0, (
|
||||
f"misaligned mamba spec: temporal inner_shape_bytes={inner_shape_bytes} "
|
||||
f"is not a multiple of the temporal-state itemsize {itemsize} B"
|
||||
)
|
||||
assert (anchor_bytes + offset_bytes_within_entry) % itemsize == 0, (
|
||||
f"misaligned mamba spec: temporal region byte offset "
|
||||
f"{anchor_bytes + offset_bytes_within_entry} is not a multiple of the "
|
||||
f"temporal-state itemsize {itemsize} B"
|
||||
)
|
||||
offset_elems = (anchor_bytes + offset_bytes_within_entry) // itemsize
|
||||
as_temporal_dtype = raw.view(temporal_dtype)
|
||||
stride = (
|
||||
inner_shape_bytes // itemsize,
|
||||
entry_bytes // itemsize,
|
||||
) + contiguous_strides(temporal_state_shape)
|
||||
temporal_view = torch.as_strided(
|
||||
as_temporal_dtype,
|
||||
size=(layer_num, max_slots) + tuple(temporal_state_shape),
|
||||
stride=stride,
|
||||
storage_offset=offset_elems,
|
||||
)
|
||||
return conv_views, temporal_view
|
||||
@@ -0,0 +1,261 @@
|
||||
"""Token-major cache views over one raw byte buffer.
|
||||
|
||||
Each token slot owns one ENTRY holding every layer's state at fixed offsets,
|
||||
and a page is ``page_size`` entries back to back:
|
||||
|
||||
entry bytes = [ K_0 | V_0 | K_1 | V_1 | ... ] (MHA)
|
||||
[ lat_0 | lat_1 | ... ] (MLA)
|
||||
byte(t, part, l) = anchor + t * entry_bytes + part offset + l * layer stride
|
||||
|
||||
so every per-layer view is ``(num_slots, *row_shape)`` with slot stride
|
||||
``entry_bytes``, indexed by the physical token id. The Mamba state
|
||||
(``build_mamba_entry_views``) uses the same per-slot entry. ``anchor_bytes``
|
||||
is the pool region's offset inside the raw buffer.
|
||||
"""
|
||||
|
||||
from typing import List, Sequence, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
ENTRY_ALIGN_BYTES = 32
|
||||
ROW_ALIGN_BYTES = 16
|
||||
|
||||
|
||||
def _prod(shape: Sequence[int]) -> int:
|
||||
out = 1
|
||||
for s in shape:
|
||||
out *= int(s)
|
||||
return out
|
||||
|
||||
|
||||
def _contiguous_strides(shape: Sequence[int]) -> Tuple[int, ...]:
|
||||
strides = []
|
||||
acc = 1
|
||||
for s in reversed(shape):
|
||||
strides.append(acc)
|
||||
acc *= int(s)
|
||||
return tuple(reversed(strides))
|
||||
|
||||
|
||||
def align_entry_bytes(num_bytes: int) -> int:
|
||||
"""Round a slot's byte sum up to the entry alignment."""
|
||||
return -(-num_bytes // ENTRY_ALIGN_BYTES) * ENTRY_ALIGN_BYTES
|
||||
|
||||
|
||||
class DensePart(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""One row family inside the entry: ``layer_num`` rows of ``row_shape``,
|
||||
layer ``l`` at ``offset_bytes + l * layer_stride_bytes``."""
|
||||
|
||||
name: str
|
||||
offset_bytes: int
|
||||
layer_stride_bytes: int
|
||||
layer_num: int
|
||||
row_shape: Tuple[int, ...]
|
||||
dtype: torch.dtype
|
||||
|
||||
def row_bytes(self) -> int:
|
||||
return _prod(self.row_shape) * self.dtype.itemsize
|
||||
|
||||
def layer_offset_bytes(self, layer: int) -> int:
|
||||
return self.offset_bytes + layer * self.layer_stride_bytes
|
||||
|
||||
|
||||
class DenseEntryLayout(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Byte layout of one slot's entry; ``entry_bytes`` is the slot stride of
|
||||
every view built over it."""
|
||||
|
||||
entry_bytes: int
|
||||
parts: Tuple[DensePart, ...]
|
||||
|
||||
def __post_init__(self):
|
||||
self.validate()
|
||||
|
||||
def part(self, name: str) -> DensePart:
|
||||
for p in self.parts:
|
||||
if p.name == name:
|
||||
return p
|
||||
raise KeyError(f"no part {name!r} in {[p.name for p in self.parts]}")
|
||||
|
||||
def validate(self) -> None:
|
||||
assert self.entry_bytes % ENTRY_ALIGN_BYTES == 0, (
|
||||
f"entry_bytes={self.entry_bytes} is not a multiple of {ENTRY_ALIGN_BYTES}"
|
||||
)
|
||||
spans = []
|
||||
for p in self.parts:
|
||||
row = p.row_bytes()
|
||||
assert (
|
||||
p.offset_bytes % ROW_ALIGN_BYTES == 0
|
||||
and p.layer_stride_bytes % ROW_ALIGN_BYTES == 0
|
||||
and row % ROW_ALIGN_BYTES == 0
|
||||
), (
|
||||
f"part {p.name!r}: offset {p.offset_bytes}, layer stride "
|
||||
f"{p.layer_stride_bytes} and row {row} B must all be multiples of "
|
||||
f"{ROW_ALIGN_BYTES} (vector stores)"
|
||||
)
|
||||
for l in range(p.layer_num):
|
||||
lo = p.layer_offset_bytes(l)
|
||||
assert 0 <= lo and lo + row <= self.entry_bytes, (
|
||||
f"part {p.name!r} layer {l} spans [{lo}, {lo + row}) outside "
|
||||
f"the {self.entry_bytes}-byte entry"
|
||||
)
|
||||
spans.append((lo, lo + row, p.name, l))
|
||||
spans.sort()
|
||||
for a, b in zip(spans, spans[1:]):
|
||||
assert a[1] <= b[0], (
|
||||
f"parts overlap: {a[2]}[{a[3]}] [{a[0]}, {a[1]}) and "
|
||||
f"{b[2]}[{b[3]}] [{b[0]}, {b[1]})"
|
||||
)
|
||||
|
||||
|
||||
def build_dense_views(
|
||||
raw: torch.Tensor,
|
||||
*,
|
||||
layout: DenseEntryLayout,
|
||||
part_name: str,
|
||||
num_slots: int,
|
||||
anchor_bytes: int = 0,
|
||||
) -> List[torch.Tensor]:
|
||||
"""Per-layer views of one part: ``(num_slots, *row_shape)`` with slot
|
||||
stride ``layout.entry_bytes``, indexed by the physical token id
|
||||
(``paged_view`` regroups them by page).
|
||||
"""
|
||||
part = layout.part(part_name)
|
||||
itemsize = part.dtype.itemsize
|
||||
assert layout.entry_bytes % itemsize == 0 and anchor_bytes % itemsize == 0, (
|
||||
f"build_dense_views: entry {layout.entry_bytes} B and anchor "
|
||||
f"{anchor_bytes} B must be multiples of the {part.dtype} itemsize"
|
||||
)
|
||||
end = anchor_bytes + num_slots * layout.entry_bytes
|
||||
assert end <= raw.numel() * raw.itemsize, (
|
||||
f"build_dense_views: {num_slots} slots of {layout.entry_bytes} B end at byte "
|
||||
f"{end} but the raw buffer holds only {raw.numel() * raw.itemsize} bytes"
|
||||
)
|
||||
as_dtype_view = raw.view(part.dtype)
|
||||
stride = (layout.entry_bytes // itemsize, *_contiguous_strides(part.row_shape))
|
||||
views: List[torch.Tensor] = []
|
||||
for layer in range(part.layer_num):
|
||||
# Part offsets and layer strides are ROW_ALIGN_BYTES-aligned (validate()).
|
||||
base_bytes = anchor_bytes + part.layer_offset_bytes(layer)
|
||||
views.append(
|
||||
torch.as_strided(
|
||||
as_dtype_view,
|
||||
size=(num_slots, *part.row_shape),
|
||||
stride=stride,
|
||||
storage_offset=base_bytes // itemsize,
|
||||
)
|
||||
)
|
||||
return views
|
||||
|
||||
|
||||
def mamba_entry_bytes(
|
||||
*,
|
||||
layer_num: int,
|
||||
conv_state_shapes: Sequence[Sequence[int]],
|
||||
conv_dtype: torch.dtype,
|
||||
temporal_state_shape: Sequence[int],
|
||||
temporal_dtype: torch.dtype,
|
||||
) -> int:
|
||||
"""Bytes occupied by one Mamba slot across all layers (conv + temporal)."""
|
||||
total = 0
|
||||
for shape in conv_state_shapes:
|
||||
total += layer_num * _prod(shape) * conv_dtype.itemsize
|
||||
total += layer_num * _prod(temporal_state_shape) * temporal_dtype.itemsize
|
||||
return total
|
||||
|
||||
|
||||
def build_mamba_entry_views(
|
||||
raw: torch.Tensor,
|
||||
*,
|
||||
layer_num: int,
|
||||
conv_state_shapes: Sequence[Sequence[int]],
|
||||
conv_dtype: torch.dtype,
|
||||
temporal_state_shape: Sequence[int],
|
||||
temporal_dtype: torch.dtype,
|
||||
max_slots: int,
|
||||
anchor_bytes: int = 0,
|
||||
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
||||
"""Per-slot envelope views over ``raw`` for Mamba state.
|
||||
|
||||
Layout per slot: ``[conv[0] rows × layers][conv[1] rows × layers]...
|
||||
[temporal rows × layers]``. Each returned view has shape
|
||||
``(num_layers, max_slots, *inner_shape)`` matching ``MambaPool.State.conv[i]``
|
||||
/ ``.temporal``. Mamba state is always token-granular (page_size == 1).
|
||||
"""
|
||||
entry_bytes = mamba_entry_bytes(
|
||||
layer_num=layer_num,
|
||||
conv_state_shapes=conv_state_shapes,
|
||||
conv_dtype=conv_dtype,
|
||||
temporal_state_shape=temporal_state_shape,
|
||||
temporal_dtype=temporal_dtype,
|
||||
)
|
||||
|
||||
conv_itemsize = conv_dtype.itemsize
|
||||
assert entry_bytes % conv_itemsize == 0, (
|
||||
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
|
||||
f"multiple of the conv-state itemsize {conv_itemsize} B"
|
||||
)
|
||||
assert anchor_bytes % conv_itemsize == 0, (
|
||||
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
|
||||
f"of the conv-state itemsize {conv_itemsize} B"
|
||||
)
|
||||
as_conv_dtype = raw.view(conv_dtype)
|
||||
conv_slot_stride_elems = entry_bytes // conv_itemsize
|
||||
|
||||
offset_bytes_within_entry = 0
|
||||
conv_views: List[torch.Tensor] = []
|
||||
for shape in conv_state_shapes:
|
||||
inner_shape_bytes = _prod(shape) * conv_itemsize
|
||||
assert inner_shape_bytes % conv_itemsize == 0
|
||||
offset_elems = (anchor_bytes + offset_bytes_within_entry) // conv_itemsize
|
||||
stride = (
|
||||
inner_shape_bytes // conv_itemsize,
|
||||
conv_slot_stride_elems,
|
||||
) + _contiguous_strides(shape)
|
||||
conv_views.append(
|
||||
torch.as_strided(
|
||||
as_conv_dtype,
|
||||
size=(layer_num, max_slots) + tuple(shape),
|
||||
stride=stride,
|
||||
storage_offset=offset_elems,
|
||||
)
|
||||
)
|
||||
offset_bytes_within_entry += layer_num * inner_shape_bytes
|
||||
|
||||
# The temporal view's storage_offset is computed in temporal-dtype elements
|
||||
# by integer-dividing a byte offset by itemsize, so every term of that byte
|
||||
# offset (entry stride, anchor, the conv region) must be a whole multiple of
|
||||
# itemsize or the offset truncates and mis-places the view.
|
||||
itemsize = temporal_dtype.itemsize
|
||||
assert entry_bytes % itemsize == 0, (
|
||||
f"misaligned mamba spec: per-slot entry_bytes={entry_bytes} is not a "
|
||||
f"multiple of the temporal-state itemsize {itemsize} B; the temporal "
|
||||
f"view's storage_offset would truncate and mis-place the state"
|
||||
)
|
||||
assert anchor_bytes % itemsize == 0, (
|
||||
f"misaligned mamba spec: anchor_bytes={anchor_bytes} is not a multiple "
|
||||
f"of the temporal-state itemsize {itemsize} B"
|
||||
)
|
||||
inner_shape_bytes = _prod(temporal_state_shape) * itemsize
|
||||
assert inner_shape_bytes % itemsize == 0, (
|
||||
f"misaligned mamba spec: temporal inner_shape_bytes={inner_shape_bytes} "
|
||||
f"is not a multiple of the temporal-state itemsize {itemsize} B"
|
||||
)
|
||||
assert (anchor_bytes + offset_bytes_within_entry) % itemsize == 0, (
|
||||
f"misaligned mamba spec: temporal region byte offset "
|
||||
f"{anchor_bytes + offset_bytes_within_entry} is not a multiple of the "
|
||||
f"temporal-state itemsize {itemsize} B"
|
||||
)
|
||||
offset_elems = (anchor_bytes + offset_bytes_within_entry) // itemsize
|
||||
as_temporal_dtype = raw.view(temporal_dtype)
|
||||
stride = (
|
||||
inner_shape_bytes // itemsize,
|
||||
entry_bytes // itemsize,
|
||||
) + _contiguous_strides(temporal_state_shape)
|
||||
temporal_view = torch.as_strided(
|
||||
as_temporal_dtype,
|
||||
size=(layer_num, max_slots) + tuple(temporal_state_shape),
|
||||
stride=stride,
|
||||
storage_offset=offset_elems,
|
||||
)
|
||||
return conv_views, temporal_view
|
||||
@@ -60,8 +60,8 @@ from sglang.srt.layers.radix_attention import RadixAttention
|
||||
from sglang.srt.mem_cache.allocator.mamba import MambaSlotAllocator
|
||||
from sglang.srt.mem_cache.index_key_cache import IndexKeyCache
|
||||
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
|
||||
from sglang.srt.mem_cache.layout.page_major import (
|
||||
build_page_major_mamba_views,
|
||||
from sglang.srt.mem_cache.layout.token_major import (
|
||||
build_mamba_entry_views,
|
||||
mamba_entry_bytes,
|
||||
)
|
||||
from sglang.srt.mem_cache.utils import (
|
||||
@@ -86,7 +86,6 @@ from sglang.srt.utils import (
|
||||
next_power_of_2,
|
||||
)
|
||||
from sglang.srt.utils.async_probe import (
|
||||
maybe_detect_kernel_facing_loc,
|
||||
maybe_detect_oob,
|
||||
)
|
||||
from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter
|
||||
@@ -588,7 +587,7 @@ class MambaPool:
|
||||
# Page-granularity envelope layout (page_size==1 for state): all
|
||||
# mamba layers/slots share one contiguous byte buffer; conv and
|
||||
# temporal are strided views into it (see mem_cache/layout/
|
||||
# page_major.py). Only the standard CUDA Triton path is supported.
|
||||
# token_major.py). Only the standard CUDA Triton path is supported.
|
||||
assert not _is_npu and not (_is_cpu and _cpu_has_amx_support), (
|
||||
"envelope_layout mamba is only supported on the CUDA path"
|
||||
)
|
||||
@@ -603,7 +602,7 @@ class MambaPool:
|
||||
self._raw = torch.zeros(
|
||||
max_slots * entry_bytes, dtype=torch.uint8, device=device
|
||||
)
|
||||
conv_state, temporal_state = build_page_major_mamba_views(
|
||||
conv_state, temporal_state = build_mamba_entry_views(
|
||||
self._raw,
|
||||
layer_num=num_mamba_layers,
|
||||
conv_state_shapes=conv_state_shape,
|
||||
@@ -1845,10 +1844,6 @@ class KVCache(abc.ABC):
|
||||
):
|
||||
self.size = size
|
||||
self.page_size = page_size
|
||||
# Row-blocks one page holds in this pool's kernel-facing id space; >1
|
||||
# only for the unified pool's per-layer views, and then a write loc must
|
||||
# have been translated into that space first.
|
||||
self.kernel_page_blocks = 1
|
||||
self.dtype = dtype
|
||||
self.device = device
|
||||
if dtype in (torch.float8_e5m2, torch.float8_e4m3fn, torch.float8_e4m3fnuz):
|
||||
@@ -2615,9 +2610,6 @@ class MHATokenToKVPool(KVCache):
|
||||
# Catch stale slot ids here instead of as illegal-addr / silent KV
|
||||
# corruption in the store_kvcache write (gated on SGLANG_ENABLE_ASYNC_ASSERT).
|
||||
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_kv_buffer (MHA)")
|
||||
maybe_detect_kernel_facing_loc(
|
||||
loc, self.page_size, self.kernel_page_blocks, "set_kv_buffer (MHA)"
|
||||
)
|
||||
layer_id = (
|
||||
layer_id_override if layer_id_override is not None else layer.layer_id
|
||||
)
|
||||
@@ -4530,9 +4522,7 @@ class MLATokenToKVPool(KVCache):
|
||||
|
||||
# Has the WRITE loc arriving here already had the DCP owner rule resolved?
|
||||
# False: this pool takes a WIDENED loc. The unified pool resolves it in
|
||||
# `KVIndexTranslator.rebind_write_loc` and flips this. Not derivable from
|
||||
# `kernel_page_blocks`: that is `layer_num`, so a rank owning one
|
||||
# full-attention layer is translated with blocks_per_page 1.
|
||||
# `KVIndexTranslator.rebind_write_loc` and flips this.
|
||||
write_loc_is_dcp_resolved = False
|
||||
|
||||
@property
|
||||
@@ -4564,9 +4554,6 @@ class MLATokenToKVPool(KVCache):
|
||||
):
|
||||
loc, _, _ = unwrap_write_loc(loc_info)
|
||||
maybe_detect_oob(loc, 0, self.size + self.page_size, "set_kv_buffer (MLA)")
|
||||
maybe_detect_kernel_facing_loc(
|
||||
loc, self.page_size, self.kernel_page_blocks, "set_kv_buffer (MLA)"
|
||||
)
|
||||
layer_id = (
|
||||
layer_id_override if layer_id_override is not None else layer.layer_id
|
||||
)
|
||||
@@ -4651,9 +4638,6 @@ class MLATokenToKVPool(KVCache):
|
||||
(self.size + self.page_size) * self._write_loc_dcp_span,
|
||||
"set_mla_kv_buffer (MLA)",
|
||||
)
|
||||
maybe_detect_kernel_facing_loc(
|
||||
loc, self.page_size, self.kernel_page_blocks, "set_mla_kv_buffer (MLA)"
|
||||
)
|
||||
layer_id = (
|
||||
layer_id_override if layer_id_override is not None else layer.layer_id
|
||||
)
|
||||
|
||||
@@ -157,6 +157,9 @@ class HostKVCache(abc.ABC):
|
||||
dcp_size = 1
|
||||
dcp_rank = 0
|
||||
shared_allocation_domain = None
|
||||
# Names this pool's page byte format in storage keys when it has one of its
|
||||
# own, so pages persisted in another format miss instead of loading.
|
||||
storage_format_tag: Optional[str] = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -610,6 +610,8 @@ class _SharedPageEnvelopeHostBacking:
|
||||
class UnifiedPageEnvelopeHostPool(HostKVCache):
|
||||
"""Host mirror that transfers complete unified-memory page envelopes."""
|
||||
|
||||
storage_format_tag = "unified-token-major"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool,
|
||||
@@ -788,13 +790,6 @@ class UnifiedPageEnvelopeHostPool(HostKVCache):
|
||||
host_indices = self._shared_backing.translate_indices(
|
||||
self.pool_label, host_indices
|
||||
)
|
||||
# The L2 engine resolves device IDs using the per-layer HiCache contract.
|
||||
# This pool copies whole envelopes, so discard the kernel-view page stride.
|
||||
kernel_page_size = self.page_size * self.device_pool.kernel_page_blocks
|
||||
device_indices = (
|
||||
device_indices // kernel_page_size * self.page_size
|
||||
+ device_indices % self.page_size
|
||||
)
|
||||
if io_backend == "kernel":
|
||||
if not host_indices.is_cuda:
|
||||
host_indices = host_indices.to(
|
||||
|
||||
@@ -36,10 +36,13 @@ from torch.profiler import record_function
|
||||
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.layout.page_major import (
|
||||
build_mha_views,
|
||||
build_mla_views,
|
||||
build_page_major_mamba_views,
|
||||
from sglang.srt.mem_cache.layout.token_major import (
|
||||
ROW_ALIGN_BYTES,
|
||||
DenseEntryLayout,
|
||||
DensePart,
|
||||
align_entry_bytes,
|
||||
build_dense_views,
|
||||
build_mamba_entry_views,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
@@ -64,6 +67,19 @@ def _prod(iterable) -> int:
|
||||
return out
|
||||
|
||||
|
||||
def _check_row_alignment(sub_pool: str, **row_bytes: int) -> None:
|
||||
"""Entry parts are laid out in ROW_ALIGN_BYTES units; a model whose
|
||||
per-rank rows are not cannot use the unified pool."""
|
||||
for part, nbytes in row_bytes.items():
|
||||
if nbytes % ROW_ALIGN_BYTES:
|
||||
raise ValueError(
|
||||
f"--enable-unified-memory needs every per-rank KV row to be a "
|
||||
f"multiple of {ROW_ALIGN_BYTES} bytes; sub-pool {sub_pool!r} has a "
|
||||
f"{nbytes}-byte {part} row. Run this model without "
|
||||
"--enable-unified-memory."
|
||||
)
|
||||
|
||||
|
||||
def _store_dtype_for(kv_cache_dtype: torch.dtype) -> torch.dtype:
|
||||
if kv_cache_dtype in (
|
||||
torch.float8_e5m2,
|
||||
@@ -110,19 +126,6 @@ class SubPoolSpec(ABC):
|
||||
"""Storage dtype (informational). Multi-dtype subclasses return the dominant buffer's."""
|
||||
raise NotImplementedError
|
||||
|
||||
def view_tail_pad_bytes(self, page_size: int) -> int:
|
||||
"""Bytes this sub-pool's views reach PAST its last page envelope."""
|
||||
return 0
|
||||
|
||||
def blocks_per_page(self) -> int:
|
||||
"""Row-blocks one page holds in this sub-pool's kernel-facing id space.
|
||||
|
||||
The page envelope is a uniform array of equally wide row-blocks, so a
|
||||
kernel-facing id is the physical page scaled by this count (see
|
||||
`MultiEndedAllocator.translate_kv_loc_for_kernel`). 1 means the kernel-facing ids are the physical ones.
|
||||
"""
|
||||
return 1
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class MHASubPoolSpec(SubPoolSpec):
|
||||
@@ -159,20 +162,39 @@ class MHASubPoolSpec(SubPoolSpec):
|
||||
return self.head_num * self.v_head_dim * self.store_dtype.itemsize
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
return self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
|
||||
|
||||
# Page-major byte math: within a page block K/V group per layer
|
||||
# [L0_K*ps | L0_V*ps | L1_K*ps | ...]; at ps==1 this collapses to the per-slot envelope.
|
||||
return align_entry_bytes(
|
||||
self.layer_num * (self.k_row_bytes() + self.v_row_bytes())
|
||||
)
|
||||
|
||||
def page_bytes(self, page_size: int) -> int:
|
||||
return page_size * self.entry_bytes()
|
||||
|
||||
def view_tail_pad_bytes(self, page_size: int) -> int:
|
||||
return page_size * self.entry_bytes()
|
||||
|
||||
def blocks_per_page(self) -> int:
|
||||
"""Row-blocks per page in the kernel-facing id space (one K + one V per layer)."""
|
||||
return 2 * self.layer_num
|
||||
def layout(self) -> DenseEntryLayout:
|
||||
"""Token-major entry ``[K_0 | V_0 | K_1 | V_1 | ...]`` per slot; a page
|
||||
is ``page_size`` such entries back to back."""
|
||||
_check_row_alignment(self.name, K=self.k_row_bytes(), V=self.v_row_bytes())
|
||||
layer_stride = self.k_row_bytes() + self.v_row_bytes()
|
||||
return DenseEntryLayout(
|
||||
entry_bytes=self.entry_bytes(),
|
||||
parts=(
|
||||
DensePart(
|
||||
name="k",
|
||||
offset_bytes=0,
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(self.head_num, self.head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
DensePart(
|
||||
name="v",
|
||||
offset_bytes=self.k_row_bytes(),
|
||||
layer_stride_bytes=layer_stride,
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(self.head_num, self.v_head_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def get_dtype(self) -> torch.dtype:
|
||||
return self.store_dtype
|
||||
@@ -205,16 +227,27 @@ class MLASubPoolSpec(SubPoolSpec):
|
||||
def kv_cache_dim(self) -> int:
|
||||
return self.kv_lora_rank + self.qk_rope_head_dim
|
||||
|
||||
def row_bytes(self) -> int:
|
||||
return self.kv_cache_dim * self.store_dtype.itemsize
|
||||
|
||||
def entry_bytes(self) -> int:
|
||||
return self.layer_num * self.kv_cache_dim * self.store_dtype.itemsize
|
||||
return align_entry_bytes(self.layer_num * self.row_bytes())
|
||||
|
||||
def view_tail_pad_bytes(self, page_size: int) -> int:
|
||||
return page_size * self.entry_bytes()
|
||||
|
||||
def blocks_per_page(self) -> int:
|
||||
"""One latent row per layer, so L blocks per page (MHA has 2L: a K
|
||||
block and a V block per layer)."""
|
||||
return self.layer_num
|
||||
def layout(self) -> DenseEntryLayout:
|
||||
_check_row_alignment(self.name, latent=self.row_bytes())
|
||||
return DenseEntryLayout(
|
||||
entry_bytes=self.entry_bytes(),
|
||||
parts=(
|
||||
DensePart(
|
||||
name="kv",
|
||||
offset_bytes=0,
|
||||
layer_stride_bytes=self.row_bytes(),
|
||||
layer_num=self.layer_num,
|
||||
row_shape=(1, self.kv_cache_dim),
|
||||
dtype=self.store_dtype,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
def get_dtype(self) -> torch.dtype:
|
||||
return self.store_dtype
|
||||
@@ -266,7 +299,7 @@ def _assert_kernel_id_bound(*, sub_pool_name: str, n_rows: int) -> None:
|
||||
assert n_rows < 2**31, (
|
||||
f"sub-pool {sub_pool_name!r}: kernel-facing id space has {n_rows} rows, "
|
||||
f"exceeding the int32 bound (2^31) that read-index buffers assume. "
|
||||
"Reduce max_total_num_tokens or the layer count."
|
||||
"Reduce max_total_num_tokens."
|
||||
)
|
||||
|
||||
|
||||
@@ -274,12 +307,12 @@ def _reserved_floor_bytes(sub_pool_specs: List[SubPoolSpec], page_size: int) ->
|
||||
"""Bytes at the bottom of the buffer reserved as the slot-0 padding sink.
|
||||
|
||||
Slot-0 dummy writes for every sub-pool land here; each sub-pool's first
|
||||
allocatable slot is chosen so real data starts past it. For a PAGE-AWARE
|
||||
sub-pool the slot-0 write touches layer blocks spread across the whole
|
||||
page-0 envelope (page_size * entry_bytes), not just one slot envelope --
|
||||
but a mamba sub-pool is page_size=1, so its entry is charged ONCE. Charging
|
||||
a mamba entry per page would reserve page_size * ~100 MB of buffer that the
|
||||
sink never touches.
|
||||
allocatable slot is chosen so real data starts past it. A slot-0 write
|
||||
touches only slot 0's entry, but a PAGE-AWARE sub-pool allocates whole
|
||||
pages, so its first allocatable page starts past all of page 0
|
||||
(page_size * entry_bytes) -- while a mamba sub-pool is page_size=1, so its
|
||||
entry is charged ONCE. Charging a mamba entry per page would reserve
|
||||
page_size * ~100 MB of buffer that the sink never touches.
|
||||
|
||||
Single source of truth: `UnifiedKVPool` reserves exactly this, and the
|
||||
factories' bs=1 feasibility floors charge exactly this.
|
||||
@@ -296,10 +329,10 @@ def _reserved_floor_bytes(sub_pool_specs: List[SubPoolSpec], page_size: int) ->
|
||||
|
||||
class UnifiedKVPool:
|
||||
"""One physical `uint8` byte buffer shared by N sub-pools, each exposing
|
||||
per-layer views over its own byte range (contiguous per layer for KV,
|
||||
strided for the Mamba state). Two END pools (one grow-up, one grow-down)
|
||||
own the buffer's ends; optional "float" MIDDLE pools live between their
|
||||
frontiers. Allocators keep byte ranges disjoint; no usage tracking here.
|
||||
per-layer strided views over its own byte range. Two END pools (one
|
||||
grow-up, one grow-down) own the buffer's ends; optional "float" MIDDLE
|
||||
pools live between their frontiers. Allocators keep byte ranges disjoint;
|
||||
no usage tracking here.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -347,13 +380,8 @@ class UnifiedKVPool:
|
||||
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
enable=enable_memory_saver
|
||||
)
|
||||
self.view_tail_pad_bytes = max(
|
||||
spec.view_tail_pad_bytes(page_size) for spec in sub_pool_specs
|
||||
)
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
self._raw = torch.empty(
|
||||
total_bytes + self.view_tail_pad_bytes, dtype=torch.uint8, device=device
|
||||
)
|
||||
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
|
||||
if envs.SGLANG_DEBUG_POISON_POOL.get():
|
||||
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
|
||||
# fail deterministically.
|
||||
@@ -368,7 +396,7 @@ class UnifiedKVPool:
|
||||
self._max_slots: Dict[str, int] = {}
|
||||
self._anchor_bytes: Dict[str, int] = {}
|
||||
self._min_slot_index: Dict[str, int] = {}
|
||||
# MHA: (k_buffer, v_buffer); MLA: [per-layer per-layer views];
|
||||
# MHA: (k_buffer, v_buffer); MLA: [per-layer views];
|
||||
# Mamba: (conv_state_list, temporal_state)
|
||||
self._mha_views: Dict[str, Tuple[List[torch.Tensor], List[torch.Tensor]]] = {}
|
||||
self._mla_views: Dict[str, List[torch.Tensor]] = {}
|
||||
@@ -376,9 +404,9 @@ class UnifiedKVPool:
|
||||
|
||||
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
|
||||
# each pool's first allocatable slot is chosen so real data starts past it.
|
||||
# For a page-aware sub-pool the slot-0 write touches layer blocks spread
|
||||
# across the WHOLE page-0 envelope (up to page_size * entry_bytes), not
|
||||
# just one slot envelope — reserve the max of both.
|
||||
# A page-aware sub-pool allocates whole pages, so the sink is all of page
|
||||
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
|
||||
# max of both.
|
||||
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
|
||||
|
||||
for spec in self.sub_pool_specs:
|
||||
@@ -490,22 +518,20 @@ class UnifiedKVPool:
|
||||
max_slots: int,
|
||||
page_size: int,
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
num_pages = max_slots // page_size
|
||||
_assert_kernel_id_bound(
|
||||
sub_pool_name=spec.name,
|
||||
n_rows=num_pages * spec.blocks_per_page() * page_size,
|
||||
)
|
||||
return build_mha_views(
|
||||
self._raw,
|
||||
layer_num=spec.layer_num,
|
||||
head_num=spec.head_num,
|
||||
head_dim=spec.head_dim,
|
||||
v_head_dim=spec.v_head_dim,
|
||||
store_dtype=spec.store_dtype,
|
||||
page_size=page_size,
|
||||
num_pages=num_pages,
|
||||
anchor_bytes=anchor_bytes,
|
||||
num_slots = max_slots // page_size * page_size
|
||||
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
||||
layout = spec.layout()
|
||||
k_views, v_views = (
|
||||
build_dense_views(
|
||||
self._raw,
|
||||
layout=layout,
|
||||
part_name=name,
|
||||
num_slots=num_slots,
|
||||
anchor_bytes=anchor_bytes,
|
||||
)
|
||||
for name in ("k", "v")
|
||||
)
|
||||
return k_views, v_views
|
||||
|
||||
def _build_mla_views(
|
||||
self,
|
||||
@@ -514,25 +540,21 @@ class UnifiedKVPool:
|
||||
max_slots: int,
|
||||
page_size: int,
|
||||
) -> List[torch.Tensor]:
|
||||
num_pages = max_slots // page_size
|
||||
_assert_kernel_id_bound(
|
||||
sub_pool_name=spec.name,
|
||||
n_rows=num_pages * spec.blocks_per_page() * page_size,
|
||||
)
|
||||
return build_mla_views(
|
||||
num_slots = max_slots // page_size * page_size
|
||||
_assert_kernel_id_bound(sub_pool_name=spec.name, n_rows=num_slots)
|
||||
layout = spec.layout()
|
||||
return build_dense_views(
|
||||
self._raw,
|
||||
layer_num=spec.layer_num,
|
||||
kv_cache_dim=spec.kv_cache_dim,
|
||||
store_dtype=spec.store_dtype,
|
||||
page_size=page_size,
|
||||
num_pages=num_pages,
|
||||
layout=layout,
|
||||
part_name="kv",
|
||||
num_slots=num_slots,
|
||||
anchor_bytes=anchor_bytes,
|
||||
)
|
||||
|
||||
def _build_mamba_views(
|
||||
self, spec: MambaSubPoolSpec, anchor_bytes: int, max_slots: int
|
||||
) -> Tuple[List[torch.Tensor], torch.Tensor]:
|
||||
return build_page_major_mamba_views(
|
||||
return build_mamba_entry_views(
|
||||
self._raw,
|
||||
layer_num=spec.layer_num,
|
||||
conv_state_shapes=spec.conv_state_shapes,
|
||||
@@ -545,16 +567,10 @@ class UnifiedKVPool:
|
||||
|
||||
|
||||
class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are `build_mha_views`
|
||||
views into a `UnifiedKVPool` (requires uniform K/V rows).
|
||||
|
||||
Views are contiguous `(n_rows, head_num, head_dim)`; locs are
|
||||
|
||||
kernel_id(t) = (t // ps) * (ps * 2 * layer_num) + t % ps
|
||||
|
||||
which is layer- and K/V-independent, each view's storage_offset folding in
|
||||
its block origin (layer l's K at block 2l, V at 2l+1). `move_kv_cache` is
|
||||
the exception: compaction passes REAL physical token ids.
|
||||
"""MHA KV pool whose per-layer `k_buffer`/`v_buffer` are token-major views
|
||||
into a `UnifiedKVPool`: `(num_pages * ps, head_num, head_dim)` with the whole
|
||||
entry as slot stride, indexed by the physical token id. `move_kv_cache`
|
||||
relocates whole page envelopes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -577,7 +593,7 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
self._v_views = v_views
|
||||
self._num_pages = max_slots // page_size
|
||||
self._page_bytes = page_size * spec.entry_bytes()
|
||||
view_rows = self._num_pages * spec.blocks_per_page() * page_size
|
||||
view_rows = self._num_pages * page_size
|
||||
|
||||
super().__init__(
|
||||
size=view_rows - page_size,
|
||||
@@ -595,7 +611,6 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
enable_kv_cache_copy=False,
|
||||
kv_cache_layout="page_major",
|
||||
)
|
||||
self.kernel_page_blocks = spec.blocks_per_page()
|
||||
|
||||
def _create_buffers(self):
|
||||
self.k_buffer = self._k_views
|
||||
@@ -611,9 +626,8 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
return 0, 0 # UnifiedKVPool logs the total; per-sub-pool would double-count
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
"""Relocate slots by whole page envelope.
|
||||
`tgt_loc`/`src_loc` are REAL physical token ids, not kernel-facing ids.
|
||||
"""
|
||||
"""Relocate slots by whole page envelope (`tgt_loc`/`src_loc` are
|
||||
physical token ids)."""
|
||||
if tgt_loc.numel() == 0:
|
||||
return
|
||||
# The envelope view below starts at byte 0, so this sub-pool must be
|
||||
@@ -654,20 +668,6 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
"""Physical growth direction used by this sub-pool's L1 allocator."""
|
||||
return self._unified_buffer.spec(self._sub_pool_name).grow_direction
|
||||
|
||||
def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
return (indices // self.page_size) * (
|
||||
self.page_size * self.kernel_page_blocks
|
||||
) + indices % self.page_size
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
||||
"""Translate physical host-pool ids for the page-major parent path."""
|
||||
return super().get_cpu_copy(self._physical_to_kernel_indices(indices))
|
||||
|
||||
def load_cpu_copy(
|
||||
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
|
||||
):
|
||||
super().load_cpu_copy(kv_cache_cpu, self._physical_to_kernel_indices(indices))
|
||||
|
||||
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
|
||||
raise NotImplementedError(
|
||||
"prefix-valid commit is unsupported under the unified layout "
|
||||
@@ -676,21 +676,11 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
|
||||
|
||||
class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
"""MLA KV pool whose per-layer `kv_buffer` entries are kernel-facing views into a
|
||||
`UnifiedKVPool` (see `build_mla_views`).
|
||||
|
||||
Loc-space contract: every loc this pool receives through the KVCache API
|
||||
(`set_kv_buffer` / `set_mla_kv_buffer` / `get_mla_kv_buffer`, and the
|
||||
kv_indices consumed by attention kernels reading `get_key_buffer` /
|
||||
`get_value_buffer`) is a kernel-facing id — the `translate_kv_loc_for_kernel` output
|
||||
|
||||
kernel_id(t) = (t // ps) * (ps * layer_num) + t % ps
|
||||
|
||||
which is layer-independent (the layer offset is folded into each view's
|
||||
storage_offset), so the stock `MLATokenToKVPool` read/write methods work on
|
||||
the views unmodified. The ONE exception is `move_kv_cache`: the allocator's
|
||||
compaction calls it with REAL physical token ids, and it is overridden to
|
||||
relocate whole page envelopes on the raw buffer.
|
||||
"""MLA KV pool whose per-layer `kv_buffer` entries are token-major views
|
||||
into a `UnifiedKVPool`: `(num_pages * ps, 1, kv_cache_dim)` with the whole
|
||||
entry as slot stride, indexed by the physical token id (`translate_kv_loc`
|
||||
of the virtual id), so the stock `MLATokenToKVPool` read/write methods work
|
||||
on them unmodified. `move_kv_cache` relocates whole page envelopes.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -714,10 +704,10 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
max_slots = unified_buffer.max_slots(sub_pool_name)
|
||||
self._num_pages = max_slots // page_size
|
||||
self._page_bytes = page_size * spec.entry_bytes()
|
||||
self._view_rows = self._num_pages * spec.blocks_per_page() * page_size
|
||||
self._view_rows = self._num_pages * page_size
|
||||
|
||||
super().__init__(
|
||||
# OOB checks bound locs by `size + page_size`; kernel-facing ids run to
|
||||
# OOB checks bound locs by `size + page_size`; physical ids run to
|
||||
# `_view_rows` (page 0 is the reserved padding sink).
|
||||
size=self._view_rows - page_size,
|
||||
page_size=page_size,
|
||||
@@ -728,7 +718,6 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
device=unified_buffer.device,
|
||||
enable_memory_saver=False, # buffer owned by UnifiedKVPool
|
||||
)
|
||||
self.kernel_page_blocks = spec.blocks_per_page()
|
||||
|
||||
def _create_buffers(self):
|
||||
self.kv_buffer = self._kv_views
|
||||
@@ -749,36 +738,20 @@ class UnifiedMLATokenToKVPool(MLATokenToKVPool):
|
||||
``raw_ptr + physical_page_id * page_envelope_bytes``.
|
||||
|
||||
The transfer item is the whole page envelope (all layers of one page)
|
||||
rather than a per-layer region, because the per-layer per-layer views
|
||||
overlap and index in kernel-facing ids. Both sides must therefore build the
|
||||
pool with identical specs.
|
||||
rather than a per-layer region: the layers interleave inside each
|
||||
slot's entry. Both sides must therefore build the pool with identical
|
||||
specs.
|
||||
"""
|
||||
# The address formula omits the anchor; a nonzero one would mis-address.
|
||||
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
||||
raw = self._unified_buffer._raw
|
||||
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
|
||||
|
||||
def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
"""Physical TOKEN ids -> the kernel-facing ids this class's `kv_buffer`
|
||||
views are indexed by; the formula is the one in the class docstring."""
|
||||
return (indices // self.page_size) * (
|
||||
self.page_size * self.kernel_page_blocks
|
||||
) + indices % self.page_size
|
||||
|
||||
def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
|
||||
"""Translate physical host-pool ids for the page-major parent path."""
|
||||
return super().get_cpu_copy(self._physical_to_kernel_indices(indices))
|
||||
|
||||
def load_cpu_copy(
|
||||
self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None
|
||||
):
|
||||
super().load_cpu_copy(kv_cache_cpu, self._physical_to_kernel_indices(indices))
|
||||
|
||||
def move_kv_cache(self, tgt_loc: torch.Tensor, src_loc: torch.Tensor):
|
||||
"""Relocate whole page envelopes.
|
||||
|
||||
`tgt_loc`/`src_loc` are REAL physical token ids (NOT kernel-facing ids): both
|
||||
compaction paths expand page ids into page-major-ordered token runs
|
||||
`tgt_loc`/`src_loc` are physical token ids: both compaction paths
|
||||
expand page ids into page-major-ordered token runs
|
||||
(`pages[:, None] * ps + offsets`), relied on here to recover the page
|
||||
lists. One contiguous envelope copy replaces the per-layer strided moves.
|
||||
"""
|
||||
@@ -1458,15 +1431,13 @@ def init_unified_mamba_pools(
|
||||
if use_mla_backend:
|
||||
logger.info(
|
||||
"[unified-memory-pool] full_layers=%d, mamba_layers=%d, kv_lora_rank=%d, "
|
||||
"qk_rope_head_dim=%d, page_size=%d (per-layer views, kernel_page_multiplier=%d, "
|
||||
"view_tail_pad=%d B)",
|
||||
"qk_rope_head_dim=%d, page_size=%d (token-major views, entry_bytes=%d B)",
|
||||
len(full_attention_layer_ids),
|
||||
len(mamba_layer_ids),
|
||||
kv_lora_rank,
|
||||
qk_rope_head_dim,
|
||||
page_size,
|
||||
len(full_attention_layer_ids),
|
||||
shared_pool.view_tail_pad_bytes,
|
||||
full_spec.entry_bytes(),
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
@@ -1478,8 +1449,7 @@ def init_unified_mamba_pools(
|
||||
head_dim,
|
||||
page_size,
|
||||
is_draft_worker,
|
||||
"per-layer views, kernel_page_multiplier=%d, view_tail_pad=%d B"
|
||||
% (full_spec.blocks_per_page(), shared_pool.view_tail_pad_bytes),
|
||||
"token-major views, entry_bytes=%d B" % full_spec.entry_bytes(),
|
||||
)
|
||||
logger.info(
|
||||
"[unified-memory-pool] total_bytes=%d, max_total_num_tokens=%d, max_mamba_cache_size=%d, "
|
||||
@@ -1903,12 +1873,8 @@ def init_unified_swa_pools(
|
||||
logger.info("[unified-memory-pool] UNIFIED MEMORY POOL ENABLED -- path=SWA hybrid")
|
||||
logger.info(
|
||||
"[unified-memory-pool] %s",
|
||||
"per-layer views, kernel_page_multiplier full=%d swa=%d, view_tail_pad=%d B"
|
||||
% (
|
||||
full_spec.blocks_per_page(),
|
||||
swa_spec.blocks_per_page(),
|
||||
shared_pool.view_tail_pad_bytes,
|
||||
),
|
||||
"token-major views, entry_bytes full=%d swa=%d B"
|
||||
% (full_spec.entry_bytes(), swa_spec.entry_bytes()),
|
||||
)
|
||||
logger.info(
|
||||
"[unified-memory-pool] full_layers=%d, swa_layers=%d, head_num=%d, head_dim=%d, "
|
||||
|
||||
@@ -141,27 +141,6 @@ def maybe_detect_oob(indices: Optional[torch.Tensor], low: int, high: int, msg:
|
||||
)
|
||||
|
||||
|
||||
def maybe_detect_kernel_facing_loc(
|
||||
indices: Optional[torch.Tensor], page_size: int, blocks_per_page: int, msg: str
|
||||
):
|
||||
"""Async check that a write loc is in the pool's KERNEL-FACING id space.
|
||||
|
||||
A kernel-facing id is `phys_page * (page_size * blocks_per_page) + offset`
|
||||
with `offset < page_size`, so its remainder modulo the page stride is
|
||||
below page_size; a VIRTUAL id satisfies that only in the first block.
|
||||
Vacuous at blocks_per_page 1. Virtual ids are in range for the OOB probe,
|
||||
so this is the only check that separates them.
|
||||
"""
|
||||
if blocks_per_page <= 1 or not envs.SGLANG_ENABLE_ASYNC_ASSERT.get():
|
||||
return
|
||||
if indices is None or indices.numel() == 0:
|
||||
return
|
||||
torch._assert_async(
|
||||
(indices % (page_size * blocks_per_page) < page_size).all(),
|
||||
f"write loc outside the kernel-facing id space (virtual ids?): {msg}",
|
||||
)
|
||||
|
||||
|
||||
def maybe_detect_page_aligned(
|
||||
indices: Optional[torch.Tensor], page_size: int, msg: str
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user