[HiCache] Add the page_unified host KV layout and the unified L3 key (#39727)
Co-authored-by: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com> Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
co-authored by
huangtingwei
Zhangheng
parent
33618d4f00
commit
9ea5472833
@@ -0,0 +1,345 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
"""Unified L3 key scheme (``--hicache-storage-key-scheme unified``).
|
||||
|
||||
Replaces the rank/topology key suffixes (``_{tp_rank}_{tp_size}``,
|
||||
``_{pp_size}_{pp_rank}``, ``_cp{r}_{s}``) with one topology-free coordinate::
|
||||
|
||||
{page_hash}_{digest}_L{start}-{end}[_H{head_group}]
|
||||
|
||||
The coordinate names what an object HOLDS -- a layer-range x head-range
|
||||
rectangle of one page, both K and V -- never who wrote it, so any deployment
|
||||
whose shard tiles the grid derives the same keys for the same bytes. A TP2
|
||||
rank's objects are exactly the union of two TP4 ranks'; a PP1 rank's are the
|
||||
union of the stages'.
|
||||
|
||||
Two fleet-wide agreements set the grid: ``head_group`` (kv heads per chunk) and
|
||||
``layer_partition`` (layers per chunk). They are not local tuning knobs --
|
||||
they fix chunk boundaries, and boundaries are namespace identity. Left unset,
|
||||
each falls back to this rank's own shard, which shares only with its own
|
||||
topology.
|
||||
|
||||
The digest prefixes every key, so any identity mismatch misses rather than
|
||||
collides. dtype is the LOGICAL one: fp8_e4m3 and fp8_e5m2 never share a
|
||||
keyspace even though both store as uint8.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.mem_cache.pool_host.page_unified import PAGE_UNIFIED_OBJECT_LAYOUT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The digest covers the encoded struct, so any schema change changes every key.
|
||||
# Bump on any field change.
|
||||
_SCHEMA_VERSION = 1
|
||||
|
||||
|
||||
class KVCacheNamespace(
|
||||
msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=True
|
||||
):
|
||||
"""Immutable identity of one shared L3 KV keyspace.
|
||||
|
||||
Everything that must match for two deployments' KV bytes to be
|
||||
interchangeable. Field names and order are part of the encoding: append
|
||||
only, and bump ``schema_version``.
|
||||
"""
|
||||
|
||||
schema_version: int = _SCHEMA_VERSION
|
||||
model_id: str
|
||||
# Logical dtype, not the storage view: fp8 variants all store as uint8
|
||||
# and must not share a keyspace.
|
||||
dtype: str
|
||||
page_size: int
|
||||
# MLA-family pools: KV is replicated across attn-TP ranks, so there is no
|
||||
# head axis (total_kv_heads and head_group are 0).
|
||||
rank_replicated: bool
|
||||
total_kv_heads: int
|
||||
# Layers per chunk; 0 = this rank's own range (same-split sharing only).
|
||||
# A stage must START on a multiple of it; only the last PP stage may end
|
||||
# short, forming the model's trailing remainder chunk.
|
||||
layer_partition: int = 0
|
||||
head_group: int
|
||||
# Byte order of the stored objects. Layouts serialize a page in different
|
||||
# orders at EQUAL sizes, so omitting this would let two deployments
|
||||
# exchange byte-permuted KV under identical keys.
|
||||
object_layout: str
|
||||
|
||||
|
||||
def namespace_digest(namespace: KVCacheNamespace) -> str:
|
||||
"""Digest of the namespace encoding, used as the key prefix.
|
||||
|
||||
msgpack encoding of a Struct is deterministic given the class definition,
|
||||
which is why the schema versions the encoding.
|
||||
"""
|
||||
encoded = msgspec.msgpack.encode(namespace)
|
||||
return f"ukv{_SCHEMA_VERSION}-{hashlib.sha256(encoded).hexdigest()[:16]}"
|
||||
|
||||
|
||||
def derive_namespace(
|
||||
*,
|
||||
model_id: str,
|
||||
dtype: str,
|
||||
page_size: int,
|
||||
rank_replicated: bool,
|
||||
total_kv_heads: int,
|
||||
head_group: int,
|
||||
layer_partition: int = 0,
|
||||
object_layout: str = PAGE_UNIFIED_OBJECT_LAYOUT,
|
||||
) -> KVCacheNamespace:
|
||||
"""Derive the namespace from deployment facts plus the fleet agreements."""
|
||||
namespace = KVCacheNamespace(
|
||||
model_id=model_id,
|
||||
dtype=dtype,
|
||||
page_size=page_size,
|
||||
rank_replicated=rank_replicated,
|
||||
total_kv_heads=0 if rank_replicated else total_kv_heads,
|
||||
head_group=0 if rank_replicated else head_group,
|
||||
layer_partition=layer_partition,
|
||||
object_layout=object_layout,
|
||||
)
|
||||
_validate_grid(namespace)
|
||||
return namespace
|
||||
|
||||
|
||||
def _validate_grid(namespace: KVCacheNamespace) -> None:
|
||||
if namespace.layer_partition < 0:
|
||||
raise ValueError(f"layer_partition must be non-negative: {namespace}")
|
||||
if namespace.page_size <= 0:
|
||||
raise ValueError(f"page_size must be positive: {namespace}")
|
||||
if not namespace.model_id:
|
||||
raise ValueError(
|
||||
"the unified key scheme requires a non-empty model_id: an empty id "
|
||||
"would merge different models into one keyspace."
|
||||
)
|
||||
if namespace.rank_replicated:
|
||||
if namespace.head_group != 0 or namespace.total_kv_heads != 0:
|
||||
raise ValueError(
|
||||
f"rank_replicated namespaces have no head axis; set "
|
||||
f"total_kv_heads=0 and head_group=0: {namespace}"
|
||||
)
|
||||
return
|
||||
if namespace.head_group <= 0 or namespace.total_kv_heads <= 0:
|
||||
raise ValueError(
|
||||
f"sharded-KV namespaces need positive total_kv_heads/head_group: "
|
||||
f"{namespace}"
|
||||
)
|
||||
if namespace.total_kv_heads % namespace.head_group != 0:
|
||||
raise ValueError(
|
||||
f"head_group={namespace.head_group} must divide "
|
||||
f"total_kv_heads={namespace.total_kv_heads}."
|
||||
)
|
||||
|
||||
|
||||
def normalize_dtype(dtype: object) -> str:
|
||||
"""``torch.bfloat16`` -> ``"bfloat16"``."""
|
||||
return str(dtype).removeprefix("torch.")
|
||||
|
||||
|
||||
class UnifiedKVPlan(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""One rank's unified-key objects and where their bytes sit in a page.
|
||||
|
||||
``suffixes`` is layer-major / head-minor, and ``layer_ranges`` x
|
||||
``head_ranges`` is the same cross product in the same order: entry
|
||||
``i * len(head_ranges) + j`` of ``suffixes`` names
|
||||
``(layer_ranges[i], head_ranges[j])``. Everything downstream relies on that
|
||||
pairing, so the two are built together here rather than re-derived.
|
||||
|
||||
The ranges are LOCAL to this rank's pool, so they index the host pool
|
||||
directly; the suffixes carry the absolute, model-global coordinates.
|
||||
"""
|
||||
|
||||
namespace: KVCacheNamespace
|
||||
suffixes: list[str]
|
||||
layer_ranges: list[tuple[int, int]]
|
||||
# Head groups, or a single whole-pool range for rank-replicated pools.
|
||||
head_ranges: list[tuple[int, int]]
|
||||
|
||||
@property
|
||||
def head_group_num(self) -> int:
|
||||
return len(self.head_ranges)
|
||||
|
||||
|
||||
def plan_unified_kv(
|
||||
*,
|
||||
model_id: str,
|
||||
dtype: str,
|
||||
page_size: int,
|
||||
rank_replicated: bool,
|
||||
local_kv_heads: int,
|
||||
attn_tp_rank: int,
|
||||
attn_tp_size: int,
|
||||
attn_cp_size: int,
|
||||
start_layer: int,
|
||||
end_layer: int,
|
||||
is_final_stage: bool,
|
||||
head_group: int | None = None,
|
||||
layer_partition: int | None = None,
|
||||
) -> UnifiedKVPlan:
|
||||
"""Derive the namespace and this rank's chunk plan from deployment facts.
|
||||
|
||||
Raises rather than degrading: a rank whose shard does not tile the declared
|
||||
grid would write objects a reader cannot reassemble, and the symptom would
|
||||
be silently wrong KV rather than a miss.
|
||||
"""
|
||||
if attn_cp_size > 1:
|
||||
raise NotImplementedError(
|
||||
"the unified key scheme does not support attention context "
|
||||
"parallelism yet: NSA-CP ranks hold sub-page slices (needs the "
|
||||
"token-granule extension) and replicated-CP needs writer election. "
|
||||
"Use --hicache-storage-key-scheme rank-suffix."
|
||||
)
|
||||
if not 0 <= start_layer < end_layer:
|
||||
raise ValueError(f"invalid layer range [{start_layer}, {end_layer}).")
|
||||
|
||||
total_kv_heads = 0 if rank_replicated else local_kv_heads * attn_tp_size
|
||||
resolved_head_group = _resolve_head_group(
|
||||
rank_replicated=rank_replicated,
|
||||
local_kv_heads=local_kv_heads,
|
||||
attn_tp_size=attn_tp_size,
|
||||
total_kv_heads=total_kv_heads,
|
||||
head_group=head_group,
|
||||
)
|
||||
namespace = derive_namespace(
|
||||
model_id=model_id,
|
||||
dtype=dtype,
|
||||
page_size=page_size,
|
||||
rank_replicated=rank_replicated,
|
||||
total_kv_heads=total_kv_heads,
|
||||
head_group=resolved_head_group,
|
||||
layer_partition=layer_partition or 0,
|
||||
)
|
||||
|
||||
layer_coords, layer_ranges = _layer_chunks(
|
||||
namespace.layer_partition, start_layer, end_layer, is_final_stage
|
||||
)
|
||||
digest = namespace_digest(namespace)
|
||||
if rank_replicated:
|
||||
return UnifiedKVPlan(
|
||||
namespace=namespace,
|
||||
suffixes=[f"{digest}_{coord}" for coord in layer_coords],
|
||||
layer_ranges=layer_ranges,
|
||||
head_ranges=[(0, 1)],
|
||||
)
|
||||
|
||||
if local_kv_heads % resolved_head_group != 0:
|
||||
raise ValueError(
|
||||
f"this rank's {local_kv_heads} kv heads do not tile "
|
||||
f"head_group={resolved_head_group}."
|
||||
)
|
||||
groups_per_rank = local_kv_heads // resolved_head_group
|
||||
first_head_index = attn_tp_rank * groups_per_rank
|
||||
return UnifiedKVPlan(
|
||||
namespace=namespace,
|
||||
# Layer-major / head-minor, the order the page block packs them in.
|
||||
suffixes=[
|
||||
f"{digest}_{coord}_H{first_head_index + i}"
|
||||
for coord in layer_coords
|
||||
for i in range(groups_per_rank)
|
||||
],
|
||||
layer_ranges=layer_ranges,
|
||||
head_ranges=[
|
||||
(i * resolved_head_group, (i + 1) * resolved_head_group)
|
||||
for i in range(groups_per_rank)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _resolve_head_group(
|
||||
*,
|
||||
rank_replicated: bool,
|
||||
local_kv_heads: int,
|
||||
attn_tp_size: int,
|
||||
total_kv_heads: int,
|
||||
head_group: int | None,
|
||||
) -> int:
|
||||
if rank_replicated:
|
||||
return 0
|
||||
if head_group is None:
|
||||
if local_kv_heads == 1 and attn_tp_size > 1:
|
||||
# Ambiguous: 1 head/rank could equally mean kv-head replication,
|
||||
# where two ranks hold the same head and would race on one key.
|
||||
# An explicit head_group is the operator attesting it is sharding.
|
||||
raise NotImplementedError(
|
||||
"the unified key scheme cannot derive a namespace at 1 kv head "
|
||||
"per rank; set --hicache-storage-head-group, or use "
|
||||
"--hicache-storage-key-scheme rank-suffix."
|
||||
)
|
||||
if local_kv_heads != total_kv_heads:
|
||||
# Correct but unshareable: another attn-TP size derives a different
|
||||
# digest and misses everything. Nothing else reports this -- the
|
||||
# symptom is a 0% hit rate against a populated store.
|
||||
logger.warning(
|
||||
"unified key scheme: --hicache-storage-head-group is unset, so "
|
||||
"this namespace is keyed to this rank's %d of %d kv heads and "
|
||||
"can only share objects with deployments at attn-TP %d. Set it "
|
||||
"to total_kv_heads / lcm of the fleet's attn-TP sizes to share "
|
||||
"across TP sizes; a rank owning several groups simply owns "
|
||||
"several chunks.",
|
||||
local_kv_heads,
|
||||
total_kv_heads,
|
||||
attn_tp_size,
|
||||
)
|
||||
return local_kv_heads
|
||||
if head_group <= 0:
|
||||
raise ValueError(f"head_group must be positive: {head_group}")
|
||||
if local_kv_heads % head_group != 0 or head_group > local_kv_heads:
|
||||
raise ValueError(
|
||||
f"head_group={head_group} must divide this rank's {local_kv_heads} "
|
||||
f"kv heads. As a fleet grid it must divide every member's local "
|
||||
f"kv-head count, so pick total_kv_heads / lcm(the fleet's attn-TP "
|
||||
f"sizes) -- here total_kv_heads={total_kv_heads}."
|
||||
)
|
||||
return head_group
|
||||
|
||||
|
||||
def _layer_chunks(
|
||||
layer_partition: int, start_layer: int, end_layer: int, is_final_stage: bool
|
||||
) -> tuple[list[str], list[tuple[int, int]]]:
|
||||
"""Absolute layer coordinates and their pool-local ranges.
|
||||
|
||||
Coordinates are ABSOLUTE, so any PP split -- uneven stages included --
|
||||
yields collision-free names, and a differing split misses rather than
|
||||
colliding.
|
||||
"""
|
||||
if not layer_partition:
|
||||
return [f"L{start_layer}-{end_layer}"], [(0, end_layer - start_layer)]
|
||||
if start_layer % layer_partition != 0:
|
||||
raise ValueError(
|
||||
f"this rank's layer range [{start_layer}, {end_layer}) does not "
|
||||
f"start on a multiple of layer_partition={layer_partition}; stages "
|
||||
f"whose boundaries do not align to the layer unit cannot share a "
|
||||
f"partitioned namespace (drop layer_partition for per-stage ranges)."
|
||||
)
|
||||
if end_layer % layer_partition != 0 and not is_final_stage:
|
||||
raise ValueError(
|
||||
f"this rank's layer range [{start_layer}, {end_layer}) ends off the "
|
||||
f"layer_partition={layer_partition} grid; only the FINAL pipeline "
|
||||
f"stage may end short (the model's trailing remainder forms the "
|
||||
f"short last chunk)."
|
||||
)
|
||||
bounds = [
|
||||
(a, min(a + layer_partition, end_layer))
|
||||
for a in range(start_layer, end_layer, layer_partition)
|
||||
]
|
||||
return (
|
||||
[f"L{a}-{b}" for a, b in bounds],
|
||||
[(a - start_layer, b - start_layer) for a, b in bounds],
|
||||
)
|
||||
@@ -45,6 +45,7 @@ from sglang.srt.mem_cache.pool_host.common import (
|
||||
make_kernel_ptr_table,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin
|
||||
from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
|
||||
|
||||
@@ -141,9 +142,14 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
*,
|
||||
mtp_draft_device_pools: Sequence[MHATokenToKVPool] = (),
|
||||
pool_label: str = "kv",
|
||||
head_group_num: int = 1,
|
||||
):
|
||||
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
|
||||
self.target_layer_num = device_pool.layer_num
|
||||
# page_unified cuts the kv-head axis into this many groups, and the cut
|
||||
# is part of the byte order, so it has to be known before the buffer is
|
||||
# allocated in init_kv_buffer() below.
|
||||
self.head_group_num = head_group_num
|
||||
# Transfer buffers are captured below. The pool remains available for
|
||||
# NPU transfer APIs and device layer ownership.
|
||||
self.device_pool = device_pool
|
||||
@@ -195,7 +201,13 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
element_size=self.element_dim * self.dtype.itemsize,
|
||||
)
|
||||
|
||||
if self.layout == "page_first":
|
||||
if self.layout == "page_unified":
|
||||
# No per-component per-layer view exists: a layer's K and V are
|
||||
# interleaved inside each head group's block. The page_unified
|
||||
# transfer kernels address the fused buffer directly instead.
|
||||
self.k_data_refs = []
|
||||
self.v_data_refs = []
|
||||
elif self.layout == "page_first":
|
||||
# Transpose [page, layer, ...] -> [layer, page, ...] to get per-layer views
|
||||
# This swaps strides without copying data
|
||||
k_transposed = self.k_buffer.transpose(0, 1)
|
||||
@@ -205,15 +217,23 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
else:
|
||||
self.k_data_refs = [self.k_buffer[i] for i in range(self.layer_num)]
|
||||
self.v_data_refs = [self.v_buffer[i] for i in range(self.layer_num)]
|
||||
self.k_data_ptrs = make_kernel_ptr_table(
|
||||
self.k_data_refs,
|
||||
self._transfer_device,
|
||||
host_memory_registered=self.pin_memory,
|
||||
self.k_data_ptrs = (
|
||||
make_kernel_ptr_table(
|
||||
self.k_data_refs,
|
||||
self._transfer_device,
|
||||
host_memory_registered=self.pin_memory,
|
||||
)
|
||||
if self.k_data_refs
|
||||
else None
|
||||
)
|
||||
self.v_data_ptrs = make_kernel_ptr_table(
|
||||
self.v_data_refs,
|
||||
self._transfer_device,
|
||||
host_memory_registered=self.pin_memory,
|
||||
self.v_data_ptrs = (
|
||||
make_kernel_ptr_table(
|
||||
self.v_data_refs,
|
||||
self._transfer_device,
|
||||
host_memory_registered=self.pin_memory,
|
||||
)
|
||||
if self.v_data_refs
|
||||
else None
|
||||
)
|
||||
if not _is_npu:
|
||||
pools = (self.device_pool, *self.mtp_draft_device_pools)
|
||||
@@ -268,6 +288,16 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
self.layer_num,
|
||||
self.head_dim,
|
||||
)
|
||||
elif self.layout == "page_unified":
|
||||
self.page_unified_layout = PageUnifiedLayout(
|
||||
page_size=self.page_size,
|
||||
layer_num=self.layer_num,
|
||||
head_num=self.head_num,
|
||||
head_group_num=self.head_group_num,
|
||||
head_dim=self.head_dim,
|
||||
itemsize=self.dtype.itemsize,
|
||||
)
|
||||
dims = self.page_unified_layout.page_dims(self.page_num)
|
||||
else:
|
||||
raise ValueError(f"Unsupported layout: {self.layout}")
|
||||
self.token_stride_size = self.head_num * self.head_dim * self.dtype.itemsize
|
||||
@@ -281,7 +311,11 @@ class MHATokenToKVPoolHost(HostKVCache):
|
||||
pin_memory=self.pin_memory,
|
||||
allocator=self.allocator,
|
||||
registration_granularity_bytes=(
|
||||
self.page_size * self.layout_dim
|
||||
# page_unified's block carries both components, so its page is
|
||||
# twice page_size * layout_dim.
|
||||
self.page_unified_layout.bytes_per_page
|
||||
if self.layout == "page_unified"
|
||||
else self.page_size * self.layout_dim
|
||||
if self.layout in ("page_first", "page_first_direct")
|
||||
else None
|
||||
),
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.mem_cache.pool_host.npu_memfabric import (
|
||||
to_device_no_sync,
|
||||
track_pinned_staging,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
|
||||
|
||||
@@ -257,6 +258,21 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
1,
|
||||
self.kv_cache_dim,
|
||||
)
|
||||
elif self.layout == "page_unified":
|
||||
# Same bytes as page_first_direct -- MLA is rank-replicated, so the
|
||||
# grid never cuts a head axis and there is nothing to permute. It
|
||||
# is still its own layout so the L3 namespace, the transfer arms
|
||||
# and the MHA side all name one thing.
|
||||
self.page_unified_layout = PageUnifiedLayout(
|
||||
page_size=self.page_size,
|
||||
layer_num=self.layer_num,
|
||||
head_num=1,
|
||||
head_group_num=1,
|
||||
head_dim=self.kv_cache_dim,
|
||||
itemsize=self.dtype.itemsize,
|
||||
is_mla=True,
|
||||
)
|
||||
dims = self.page_unified_layout.page_dims(self.page_num)
|
||||
# Ascend-specific: Aligns with NPUMLATokenToKVPool layout
|
||||
# Separately allocate k_buffer and v_buffer for easier data transfer.
|
||||
elif self.layout == "page_first_kv_split":
|
||||
@@ -356,7 +372,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
|
||||
allocator=self.allocator,
|
||||
registration_granularity_bytes=(
|
||||
self.page_size * self.layout_dim
|
||||
if self.layout in ("page_first", "page_first_direct")
|
||||
if self.layout in ("page_first", "page_first_direct", "page_unified")
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
# Copyright 2023-2026 SGLang Team
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ==============================================================================
|
||||
|
||||
"""The ``page_unified`` host KV layout: one page block's byte order.
|
||||
|
||||
One page block, in this byte order::
|
||||
|
||||
MHA: (head_group, layer, 2, page_size, head_in_group, head_dim) K=0, V=1
|
||||
MLA: (layer, page_size, head_dim)
|
||||
|
||||
A PARTITION AXIS MUST BE OUTER. The unified L3 grid cuts a page along the kv-head
|
||||
axis and then along the layer axis, so both sit outside ``page_size``: a chunk is
|
||||
then an offset and a length, not a stride pattern. Put either one inside and
|
||||
naming a sub-range costs a descriptor per token.
|
||||
|
||||
The K/V axis sits INSIDE the layer axis rather than outside the pool, which is
|
||||
what makes a chunk a single run and lets one object carry both components.
|
||||
``layer_first`` and ``page_first_direct`` both hoist it outside, so under those
|
||||
a chunk is two objects whose runs interleave at token granularity.
|
||||
|
||||
The same order is what the write-back kernel produces and the load-back kernel
|
||||
reads, so a page is byte-identical whether it arrived from the device or from
|
||||
L3, and neither side has to ask which.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import msgspec
|
||||
|
||||
# Byte order identity, recorded in the L3 namespace digest. Bump when the axis
|
||||
# order changes, so old objects miss instead of deserializing permuted.
|
||||
PAGE_UNIFIED_OBJECT_LAYOUT = "page-unified-v1"
|
||||
|
||||
|
||||
class PageUnifiedLayout(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Byte layout of one ``page_unified`` page block.
|
||||
|
||||
Built directly from a host pool's own dimensions. ``head_group_num`` is the
|
||||
number of kv-head GROUPS the page is cut into, which is the unified grid's
|
||||
head partition; MLA is rank-replicated and has no head axis, so both it and
|
||||
``head_num`` are 1 there.
|
||||
"""
|
||||
|
||||
page_size: int
|
||||
layer_num: int
|
||||
head_num: int
|
||||
head_group_num: int
|
||||
head_dim: int
|
||||
itemsize: int
|
||||
is_mla: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for name in (
|
||||
"page_size",
|
||||
"layer_num",
|
||||
"head_num",
|
||||
"head_group_num",
|
||||
"head_dim",
|
||||
"itemsize",
|
||||
):
|
||||
if getattr(self, name) <= 0:
|
||||
raise ValueError(f"{name} must be positive: {self}")
|
||||
if self.head_num % self.head_group_num != 0:
|
||||
raise ValueError(
|
||||
f"head_group_num={self.head_group_num} must divide the pool's "
|
||||
f"{self.head_num} kv heads."
|
||||
)
|
||||
if self.is_mla and (self.head_group_num != 1 or self.head_num != 1):
|
||||
raise ValueError(f"MLA page blocks have no head axis: {self}")
|
||||
# The transfer kernels move a token's head-group run as 16-byte vectors.
|
||||
if self.group_bytes % 16 != 0:
|
||||
raise ValueError(
|
||||
f"one head group's token row is {self.group_bytes} bytes, which "
|
||||
f"the page_unified transfer kernels cannot move: they require a "
|
||||
f"multiple of 16. Raise head_dim, lower head_group_num, or use "
|
||||
f"another --hicache-mem-layout."
|
||||
)
|
||||
|
||||
@property
|
||||
def components(self) -> int:
|
||||
"""K and V, or the single latent cache for MLA."""
|
||||
return 1 if self.is_mla else 2
|
||||
|
||||
@property
|
||||
def heads_per_group(self) -> int:
|
||||
return self.head_num // self.head_group_num
|
||||
|
||||
@property
|
||||
def group_bytes(self) -> int:
|
||||
"""One head group's row for one token: the copy unit."""
|
||||
return self.heads_per_group * self.head_dim * self.itemsize
|
||||
|
||||
@property
|
||||
def layer_bytes(self) -> int:
|
||||
"""One (head group, layer) cell: both components, all tokens of a page."""
|
||||
return self.components * self.page_size * self.group_bytes
|
||||
|
||||
@property
|
||||
def head_group_bytes(self) -> int:
|
||||
return self.layer_num * self.layer_bytes
|
||||
|
||||
@property
|
||||
def bytes_per_page(self) -> int:
|
||||
return self.head_group_num * self.head_group_bytes
|
||||
|
||||
def page_dims(self, page_num: int) -> tuple[int, ...]:
|
||||
"""Buffer shape, whose C-contiguous order IS the byte order."""
|
||||
if self.is_mla:
|
||||
return (page_num, self.layer_num, self.page_size, self.head_dim)
|
||||
return (
|
||||
page_num,
|
||||
self.head_group_num,
|
||||
self.layer_num,
|
||||
self.components,
|
||||
self.page_size,
|
||||
self.heads_per_group,
|
||||
self.head_dim,
|
||||
)
|
||||
|
||||
def chunk_span(
|
||||
self, layer_range: tuple[int, int], head_group: int = 0
|
||||
) -> tuple[int, int]:
|
||||
"""Byte ``(offset, length)`` of one L3 chunk within a page block.
|
||||
|
||||
A chunk is a half-open LOCAL layer range within one head group. It is
|
||||
always exactly one run, including a ragged final layer window -- that
|
||||
is the property the axis order buys, and what lets the transport treat
|
||||
every chunk uniformly instead of switching on a descriptor count.
|
||||
"""
|
||||
start, end = layer_range
|
||||
if not 0 <= start < end <= self.layer_num:
|
||||
raise ValueError(
|
||||
f"layer range {layer_range} is not a non-empty sub-range of "
|
||||
f"[0, {self.layer_num})."
|
||||
)
|
||||
if not 0 <= head_group < self.head_group_num:
|
||||
raise ValueError(
|
||||
f"head group {head_group} is outside [0, {self.head_group_num})."
|
||||
)
|
||||
offset = head_group * self.head_group_bytes + start * self.layer_bytes
|
||||
return offset, (end - start) * self.layer_bytes
|
||||
Reference in New Issue
Block a user