[HiCache] Add the page_unified host KV layout and the unified L3 key (#39727)

Co-authored-by: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com>
Co-authored-by: Zhangheng <hzh0425@apache.org>
This commit is contained in:
Shunkangz
2026-10-15 15:46:13 +08:00
committed by GitHub
co-authored by huangtingwei Zhangheng
parent 33618d4f00
commit 9ea5472833
5 changed files with 968 additions and 11 deletions
@@ -0,0 +1,345 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Unified L3 key scheme (``--hicache-storage-key-scheme unified``).
Replaces the rank/topology key suffixes (``_{tp_rank}_{tp_size}``,
``_{pp_size}_{pp_rank}``, ``_cp{r}_{s}``) with one topology-free coordinate::
{page_hash}_{digest}_L{start}-{end}[_H{head_group}]
The coordinate names what an object HOLDS -- a layer-range x head-range
rectangle of one page, both K and V -- never who wrote it, so any deployment
whose shard tiles the grid derives the same keys for the same bytes. A TP2
rank's objects are exactly the union of two TP4 ranks'; a PP1 rank's are the
union of the stages'.
Two fleet-wide agreements set the grid: ``head_group`` (kv heads per chunk) and
``layer_partition`` (layers per chunk). They are not local tuning knobs --
they fix chunk boundaries, and boundaries are namespace identity. Left unset,
each falls back to this rank's own shard, which shares only with its own
topology.
The digest prefixes every key, so any identity mismatch misses rather than
collides. dtype is the LOGICAL one: fp8_e4m3 and fp8_e5m2 never share a
keyspace even though both store as uint8.
"""
from __future__ import annotations
import hashlib
import logging
import msgspec
from sglang.srt.mem_cache.pool_host.page_unified import PAGE_UNIFIED_OBJECT_LAYOUT
logger = logging.getLogger(__name__)
# The digest covers the encoded struct, so any schema change changes every key.
# Bump on any field change.
_SCHEMA_VERSION = 1
class KVCacheNamespace(
msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=True
):
"""Immutable identity of one shared L3 KV keyspace.
Everything that must match for two deployments' KV bytes to be
interchangeable. Field names and order are part of the encoding: append
only, and bump ``schema_version``.
"""
schema_version: int = _SCHEMA_VERSION
model_id: str
# Logical dtype, not the storage view: fp8 variants all store as uint8
# and must not share a keyspace.
dtype: str
page_size: int
# MLA-family pools: KV is replicated across attn-TP ranks, so there is no
# head axis (total_kv_heads and head_group are 0).
rank_replicated: bool
total_kv_heads: int
# Layers per chunk; 0 = this rank's own range (same-split sharing only).
# A stage must START on a multiple of it; only the last PP stage may end
# short, forming the model's trailing remainder chunk.
layer_partition: int = 0
head_group: int
# Byte order of the stored objects. Layouts serialize a page in different
# orders at EQUAL sizes, so omitting this would let two deployments
# exchange byte-permuted KV under identical keys.
object_layout: str
def namespace_digest(namespace: KVCacheNamespace) -> str:
"""Digest of the namespace encoding, used as the key prefix.
msgpack encoding of a Struct is deterministic given the class definition,
which is why the schema versions the encoding.
"""
encoded = msgspec.msgpack.encode(namespace)
return f"ukv{_SCHEMA_VERSION}-{hashlib.sha256(encoded).hexdigest()[:16]}"
def derive_namespace(
*,
model_id: str,
dtype: str,
page_size: int,
rank_replicated: bool,
total_kv_heads: int,
head_group: int,
layer_partition: int = 0,
object_layout: str = PAGE_UNIFIED_OBJECT_LAYOUT,
) -> KVCacheNamespace:
"""Derive the namespace from deployment facts plus the fleet agreements."""
namespace = KVCacheNamespace(
model_id=model_id,
dtype=dtype,
page_size=page_size,
rank_replicated=rank_replicated,
total_kv_heads=0 if rank_replicated else total_kv_heads,
head_group=0 if rank_replicated else head_group,
layer_partition=layer_partition,
object_layout=object_layout,
)
_validate_grid(namespace)
return namespace
def _validate_grid(namespace: KVCacheNamespace) -> None:
if namespace.layer_partition < 0:
raise ValueError(f"layer_partition must be non-negative: {namespace}")
if namespace.page_size <= 0:
raise ValueError(f"page_size must be positive: {namespace}")
if not namespace.model_id:
raise ValueError(
"the unified key scheme requires a non-empty model_id: an empty id "
"would merge different models into one keyspace."
)
if namespace.rank_replicated:
if namespace.head_group != 0 or namespace.total_kv_heads != 0:
raise ValueError(
f"rank_replicated namespaces have no head axis; set "
f"total_kv_heads=0 and head_group=0: {namespace}"
)
return
if namespace.head_group <= 0 or namespace.total_kv_heads <= 0:
raise ValueError(
f"sharded-KV namespaces need positive total_kv_heads/head_group: "
f"{namespace}"
)
if namespace.total_kv_heads % namespace.head_group != 0:
raise ValueError(
f"head_group={namespace.head_group} must divide "
f"total_kv_heads={namespace.total_kv_heads}."
)
def normalize_dtype(dtype: object) -> str:
"""``torch.bfloat16`` -> ``"bfloat16"``."""
return str(dtype).removeprefix("torch.")
class UnifiedKVPlan(msgspec.Struct, frozen=True, kw_only=True):
"""One rank's unified-key objects and where their bytes sit in a page.
``suffixes`` is layer-major / head-minor, and ``layer_ranges`` x
``head_ranges`` is the same cross product in the same order: entry
``i * len(head_ranges) + j`` of ``suffixes`` names
``(layer_ranges[i], head_ranges[j])``. Everything downstream relies on that
pairing, so the two are built together here rather than re-derived.
The ranges are LOCAL to this rank's pool, so they index the host pool
directly; the suffixes carry the absolute, model-global coordinates.
"""
namespace: KVCacheNamespace
suffixes: list[str]
layer_ranges: list[tuple[int, int]]
# Head groups, or a single whole-pool range for rank-replicated pools.
head_ranges: list[tuple[int, int]]
@property
def head_group_num(self) -> int:
return len(self.head_ranges)
def plan_unified_kv(
*,
model_id: str,
dtype: str,
page_size: int,
rank_replicated: bool,
local_kv_heads: int,
attn_tp_rank: int,
attn_tp_size: int,
attn_cp_size: int,
start_layer: int,
end_layer: int,
is_final_stage: bool,
head_group: int | None = None,
layer_partition: int | None = None,
) -> UnifiedKVPlan:
"""Derive the namespace and this rank's chunk plan from deployment facts.
Raises rather than degrading: a rank whose shard does not tile the declared
grid would write objects a reader cannot reassemble, and the symptom would
be silently wrong KV rather than a miss.
"""
if attn_cp_size > 1:
raise NotImplementedError(
"the unified key scheme does not support attention context "
"parallelism yet: NSA-CP ranks hold sub-page slices (needs the "
"token-granule extension) and replicated-CP needs writer election. "
"Use --hicache-storage-key-scheme rank-suffix."
)
if not 0 <= start_layer < end_layer:
raise ValueError(f"invalid layer range [{start_layer}, {end_layer}).")
total_kv_heads = 0 if rank_replicated else local_kv_heads * attn_tp_size
resolved_head_group = _resolve_head_group(
rank_replicated=rank_replicated,
local_kv_heads=local_kv_heads,
attn_tp_size=attn_tp_size,
total_kv_heads=total_kv_heads,
head_group=head_group,
)
namespace = derive_namespace(
model_id=model_id,
dtype=dtype,
page_size=page_size,
rank_replicated=rank_replicated,
total_kv_heads=total_kv_heads,
head_group=resolved_head_group,
layer_partition=layer_partition or 0,
)
layer_coords, layer_ranges = _layer_chunks(
namespace.layer_partition, start_layer, end_layer, is_final_stage
)
digest = namespace_digest(namespace)
if rank_replicated:
return UnifiedKVPlan(
namespace=namespace,
suffixes=[f"{digest}_{coord}" for coord in layer_coords],
layer_ranges=layer_ranges,
head_ranges=[(0, 1)],
)
if local_kv_heads % resolved_head_group != 0:
raise ValueError(
f"this rank's {local_kv_heads} kv heads do not tile "
f"head_group={resolved_head_group}."
)
groups_per_rank = local_kv_heads // resolved_head_group
first_head_index = attn_tp_rank * groups_per_rank
return UnifiedKVPlan(
namespace=namespace,
# Layer-major / head-minor, the order the page block packs them in.
suffixes=[
f"{digest}_{coord}_H{first_head_index + i}"
for coord in layer_coords
for i in range(groups_per_rank)
],
layer_ranges=layer_ranges,
head_ranges=[
(i * resolved_head_group, (i + 1) * resolved_head_group)
for i in range(groups_per_rank)
],
)
def _resolve_head_group(
*,
rank_replicated: bool,
local_kv_heads: int,
attn_tp_size: int,
total_kv_heads: int,
head_group: int | None,
) -> int:
if rank_replicated:
return 0
if head_group is None:
if local_kv_heads == 1 and attn_tp_size > 1:
# Ambiguous: 1 head/rank could equally mean kv-head replication,
# where two ranks hold the same head and would race on one key.
# An explicit head_group is the operator attesting it is sharding.
raise NotImplementedError(
"the unified key scheme cannot derive a namespace at 1 kv head "
"per rank; set --hicache-storage-head-group, or use "
"--hicache-storage-key-scheme rank-suffix."
)
if local_kv_heads != total_kv_heads:
# Correct but unshareable: another attn-TP size derives a different
# digest and misses everything. Nothing else reports this -- the
# symptom is a 0% hit rate against a populated store.
logger.warning(
"unified key scheme: --hicache-storage-head-group is unset, so "
"this namespace is keyed to this rank's %d of %d kv heads and "
"can only share objects with deployments at attn-TP %d. Set it "
"to total_kv_heads / lcm of the fleet's attn-TP sizes to share "
"across TP sizes; a rank owning several groups simply owns "
"several chunks.",
local_kv_heads,
total_kv_heads,
attn_tp_size,
)
return local_kv_heads
if head_group <= 0:
raise ValueError(f"head_group must be positive: {head_group}")
if local_kv_heads % head_group != 0 or head_group > local_kv_heads:
raise ValueError(
f"head_group={head_group} must divide this rank's {local_kv_heads} "
f"kv heads. As a fleet grid it must divide every member's local "
f"kv-head count, so pick total_kv_heads / lcm(the fleet's attn-TP "
f"sizes) -- here total_kv_heads={total_kv_heads}."
)
return head_group
def _layer_chunks(
layer_partition: int, start_layer: int, end_layer: int, is_final_stage: bool
) -> tuple[list[str], list[tuple[int, int]]]:
"""Absolute layer coordinates and their pool-local ranges.
Coordinates are ABSOLUTE, so any PP split -- uneven stages included --
yields collision-free names, and a differing split misses rather than
colliding.
"""
if not layer_partition:
return [f"L{start_layer}-{end_layer}"], [(0, end_layer - start_layer)]
if start_layer % layer_partition != 0:
raise ValueError(
f"this rank's layer range [{start_layer}, {end_layer}) does not "
f"start on a multiple of layer_partition={layer_partition}; stages "
f"whose boundaries do not align to the layer unit cannot share a "
f"partitioned namespace (drop layer_partition for per-stage ranges)."
)
if end_layer % layer_partition != 0 and not is_final_stage:
raise ValueError(
f"this rank's layer range [{start_layer}, {end_layer}) ends off the "
f"layer_partition={layer_partition} grid; only the FINAL pipeline "
f"stage may end short (the model's trailing remainder forms the "
f"short last chunk)."
)
bounds = [
(a, min(a + layer_partition, end_layer))
for a in range(start_layer, end_layer, layer_partition)
]
return (
[f"L{a}-{b}" for a, b in bounds],
[(a - start_layer, b - start_layer) for a, b in bounds],
)
+44 -10
View File
@@ -45,6 +45,7 @@ from sglang.srt.mem_cache.pool_host.common import (
make_kernel_ptr_table,
)
from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin
from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout
from sglang.srt.platforms import current_platform
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
@@ -141,9 +142,14 @@ class MHATokenToKVPoolHost(HostKVCache):
*,
mtp_draft_device_pools: Sequence[MHATokenToKVPool] = (),
pool_label: str = "kv",
head_group_num: int = 1,
):
self.mtp_draft_device_pools = tuple(mtp_draft_device_pools)
self.target_layer_num = device_pool.layer_num
# page_unified cuts the kv-head axis into this many groups, and the cut
# is part of the byte order, so it has to be known before the buffer is
# allocated in init_kv_buffer() below.
self.head_group_num = head_group_num
# Transfer buffers are captured below. The pool remains available for
# NPU transfer APIs and device layer ownership.
self.device_pool = device_pool
@@ -195,7 +201,13 @@ class MHATokenToKVPoolHost(HostKVCache):
element_size=self.element_dim * self.dtype.itemsize,
)
if self.layout == "page_first":
if self.layout == "page_unified":
# No per-component per-layer view exists: a layer's K and V are
# interleaved inside each head group's block. The page_unified
# transfer kernels address the fused buffer directly instead.
self.k_data_refs = []
self.v_data_refs = []
elif self.layout == "page_first":
# Transpose [page, layer, ...] -> [layer, page, ...] to get per-layer views
# This swaps strides without copying data
k_transposed = self.k_buffer.transpose(0, 1)
@@ -205,15 +217,23 @@ class MHATokenToKVPoolHost(HostKVCache):
else:
self.k_data_refs = [self.k_buffer[i] for i in range(self.layer_num)]
self.v_data_refs = [self.v_buffer[i] for i in range(self.layer_num)]
self.k_data_ptrs = make_kernel_ptr_table(
self.k_data_refs,
self._transfer_device,
host_memory_registered=self.pin_memory,
self.k_data_ptrs = (
make_kernel_ptr_table(
self.k_data_refs,
self._transfer_device,
host_memory_registered=self.pin_memory,
)
if self.k_data_refs
else None
)
self.v_data_ptrs = make_kernel_ptr_table(
self.v_data_refs,
self._transfer_device,
host_memory_registered=self.pin_memory,
self.v_data_ptrs = (
make_kernel_ptr_table(
self.v_data_refs,
self._transfer_device,
host_memory_registered=self.pin_memory,
)
if self.v_data_refs
else None
)
if not _is_npu:
pools = (self.device_pool, *self.mtp_draft_device_pools)
@@ -268,6 +288,16 @@ class MHATokenToKVPoolHost(HostKVCache):
self.layer_num,
self.head_dim,
)
elif self.layout == "page_unified":
self.page_unified_layout = PageUnifiedLayout(
page_size=self.page_size,
layer_num=self.layer_num,
head_num=self.head_num,
head_group_num=self.head_group_num,
head_dim=self.head_dim,
itemsize=self.dtype.itemsize,
)
dims = self.page_unified_layout.page_dims(self.page_num)
else:
raise ValueError(f"Unsupported layout: {self.layout}")
self.token_stride_size = self.head_num * self.head_dim * self.dtype.itemsize
@@ -281,7 +311,11 @@ class MHATokenToKVPoolHost(HostKVCache):
pin_memory=self.pin_memory,
allocator=self.allocator,
registration_granularity_bytes=(
self.page_size * self.layout_dim
# page_unified's block carries both components, so its page is
# twice page_size * layout_dim.
self.page_unified_layout.bytes_per_page
if self.layout == "page_unified"
else self.page_size * self.layout_dim
if self.layout in ("page_first", "page_first_direct")
else None
),
+17 -1
View File
@@ -37,6 +37,7 @@ from sglang.srt.mem_cache.pool_host.npu_memfabric import (
to_device_no_sync,
track_pinned_staging,
)
from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout
from sglang.srt.platforms import current_platform
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
@@ -257,6 +258,21 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
1,
self.kv_cache_dim,
)
elif self.layout == "page_unified":
# Same bytes as page_first_direct -- MLA is rank-replicated, so the
# grid never cuts a head axis and there is nothing to permute. It
# is still its own layout so the L3 namespace, the transfer arms
# and the MHA side all name one thing.
self.page_unified_layout = PageUnifiedLayout(
page_size=self.page_size,
layer_num=self.layer_num,
head_num=1,
head_group_num=1,
head_dim=self.kv_cache_dim,
itemsize=self.dtype.itemsize,
is_mla=True,
)
dims = self.page_unified_layout.page_dims(self.page_num)
# Ascend-specific: Aligns with NPUMLATokenToKVPool layout
# Separately allocate k_buffer and v_buffer for easier data transfer.
elif self.layout == "page_first_kv_split":
@@ -356,7 +372,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache):
allocator=self.allocator,
registration_granularity_bytes=(
self.page_size * self.layout_dim
if self.layout in ("page_first", "page_first_direct")
if self.layout in ("page_first", "page_first_direct", "page_unified")
else None
),
)
@@ -0,0 +1,152 @@
# Copyright 2023-2026 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""The ``page_unified`` host KV layout: one page block's byte order.
One page block, in this byte order::
MHA: (head_group, layer, 2, page_size, head_in_group, head_dim) K=0, V=1
MLA: (layer, page_size, head_dim)
A PARTITION AXIS MUST BE OUTER. The unified L3 grid cuts a page along the kv-head
axis and then along the layer axis, so both sit outside ``page_size``: a chunk is
then an offset and a length, not a stride pattern. Put either one inside and
naming a sub-range costs a descriptor per token.
The K/V axis sits INSIDE the layer axis rather than outside the pool, which is
what makes a chunk a single run and lets one object carry both components.
``layer_first`` and ``page_first_direct`` both hoist it outside, so under those
a chunk is two objects whose runs interleave at token granularity.
The same order is what the write-back kernel produces and the load-back kernel
reads, so a page is byte-identical whether it arrived from the device or from
L3, and neither side has to ask which.
"""
from __future__ import annotations
import msgspec
# Byte order identity, recorded in the L3 namespace digest. Bump when the axis
# order changes, so old objects miss instead of deserializing permuted.
PAGE_UNIFIED_OBJECT_LAYOUT = "page-unified-v1"
class PageUnifiedLayout(msgspec.Struct, frozen=True, kw_only=True):
"""Byte layout of one ``page_unified`` page block.
Built directly from a host pool's own dimensions. ``head_group_num`` is the
number of kv-head GROUPS the page is cut into, which is the unified grid's
head partition; MLA is rank-replicated and has no head axis, so both it and
``head_num`` are 1 there.
"""
page_size: int
layer_num: int
head_num: int
head_group_num: int
head_dim: int
itemsize: int
is_mla: bool = False
def __post_init__(self) -> None:
for name in (
"page_size",
"layer_num",
"head_num",
"head_group_num",
"head_dim",
"itemsize",
):
if getattr(self, name) <= 0:
raise ValueError(f"{name} must be positive: {self}")
if self.head_num % self.head_group_num != 0:
raise ValueError(
f"head_group_num={self.head_group_num} must divide the pool's "
f"{self.head_num} kv heads."
)
if self.is_mla and (self.head_group_num != 1 or self.head_num != 1):
raise ValueError(f"MLA page blocks have no head axis: {self}")
# The transfer kernels move a token's head-group run as 16-byte vectors.
if self.group_bytes % 16 != 0:
raise ValueError(
f"one head group's token row is {self.group_bytes} bytes, which "
f"the page_unified transfer kernels cannot move: they require a "
f"multiple of 16. Raise head_dim, lower head_group_num, or use "
f"another --hicache-mem-layout."
)
@property
def components(self) -> int:
"""K and V, or the single latent cache for MLA."""
return 1 if self.is_mla else 2
@property
def heads_per_group(self) -> int:
return self.head_num // self.head_group_num
@property
def group_bytes(self) -> int:
"""One head group's row for one token: the copy unit."""
return self.heads_per_group * self.head_dim * self.itemsize
@property
def layer_bytes(self) -> int:
"""One (head group, layer) cell: both components, all tokens of a page."""
return self.components * self.page_size * self.group_bytes
@property
def head_group_bytes(self) -> int:
return self.layer_num * self.layer_bytes
@property
def bytes_per_page(self) -> int:
return self.head_group_num * self.head_group_bytes
def page_dims(self, page_num: int) -> tuple[int, ...]:
"""Buffer shape, whose C-contiguous order IS the byte order."""
if self.is_mla:
return (page_num, self.layer_num, self.page_size, self.head_dim)
return (
page_num,
self.head_group_num,
self.layer_num,
self.components,
self.page_size,
self.heads_per_group,
self.head_dim,
)
def chunk_span(
self, layer_range: tuple[int, int], head_group: int = 0
) -> tuple[int, int]:
"""Byte ``(offset, length)`` of one L3 chunk within a page block.
A chunk is a half-open LOCAL layer range within one head group. It is
always exactly one run, including a ragged final layer window -- that
is the property the axis order buys, and what lets the transport treat
every chunk uniformly instead of switching on a descriptor count.
"""
start, end = layer_range
if not 0 <= start < end <= self.layer_num:
raise ValueError(
f"layer range {layer_range} is not a non-empty sub-range of "
f"[0, {self.layer_num})."
)
if not 0 <= head_group < self.head_group_num:
raise ValueError(
f"head group {head_group} is outside [0, {self.head_group_num})."
)
offset = head_group * self.head_group_bytes + start * self.layer_bytes
return offset, (end - start) * self.layer_bytes