From 9ea5472833296e97724f12df716936576d34ddbb Mon Sep 17 00:00:00 2001 From: Shunkangz <182541032+Shunkangz@users.noreply.github.com> Date: Thu, 15 Oct 2026 15:46:13 +0800 Subject: [PATCH] [HiCache] Add the page_unified host KV layout and the unified L3 key (#39727) Co-authored-by: huangtingwei <141888744+huangtingwei9988@users.noreply.github.com> Co-authored-by: Zhangheng --- .../srt/mem_cache/hicache_key_scheme.py | 345 +++++++++++++++ python/sglang/srt/mem_cache/pool_host/mha.py | 54 ++- python/sglang/srt/mem_cache/pool_host/mla.py | 18 +- .../srt/mem_cache/pool_host/page_unified.py | 152 +++++++ .../mem_cache/test_page_unified_layout.py | 410 ++++++++++++++++++ 5 files changed, 968 insertions(+), 11 deletions(-) create mode 100644 python/sglang/srt/mem_cache/hicache_key_scheme.py create mode 100644 python/sglang/srt/mem_cache/pool_host/page_unified.py create mode 100644 test/registered/unit/mem_cache/test_page_unified_layout.py diff --git a/python/sglang/srt/mem_cache/hicache_key_scheme.py b/python/sglang/srt/mem_cache/hicache_key_scheme.py new file mode 100644 index 0000000000..e73af96368 --- /dev/null +++ b/python/sglang/srt/mem_cache/hicache_key_scheme.py @@ -0,0 +1,345 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Unified L3 key scheme (``--hicache-storage-key-scheme unified``). + +Replaces the rank/topology key suffixes (``_{tp_rank}_{tp_size}``, +``_{pp_size}_{pp_rank}``, ``_cp{r}_{s}``) with one topology-free coordinate:: + + {page_hash}_{digest}_L{start}-{end}[_H{head_group}] + +The coordinate names what an object HOLDS -- a layer-range x head-range +rectangle of one page, both K and V -- never who wrote it, so any deployment +whose shard tiles the grid derives the same keys for the same bytes. A TP2 +rank's objects are exactly the union of two TP4 ranks'; a PP1 rank's are the +union of the stages'. + +Two fleet-wide agreements set the grid: ``head_group`` (kv heads per chunk) and +``layer_partition`` (layers per chunk). They are not local tuning knobs -- +they fix chunk boundaries, and boundaries are namespace identity. Left unset, +each falls back to this rank's own shard, which shares only with its own +topology. + +The digest prefixes every key, so any identity mismatch misses rather than +collides. dtype is the LOGICAL one: fp8_e4m3 and fp8_e5m2 never share a +keyspace even though both store as uint8. +""" + +from __future__ import annotations + +import hashlib +import logging + +import msgspec + +from sglang.srt.mem_cache.pool_host.page_unified import PAGE_UNIFIED_OBJECT_LAYOUT + +logger = logging.getLogger(__name__) + +# The digest covers the encoded struct, so any schema change changes every key. +# Bump on any field change. +_SCHEMA_VERSION = 1 + + +class KVCacheNamespace( + msgspec.Struct, frozen=True, kw_only=True, forbid_unknown_fields=True +): + """Immutable identity of one shared L3 KV keyspace. + + Everything that must match for two deployments' KV bytes to be + interchangeable. Field names and order are part of the encoding: append + only, and bump ``schema_version``. + """ + + schema_version: int = _SCHEMA_VERSION + model_id: str + # Logical dtype, not the storage view: fp8 variants all store as uint8 + # and must not share a keyspace. + dtype: str + page_size: int + # MLA-family pools: KV is replicated across attn-TP ranks, so there is no + # head axis (total_kv_heads and head_group are 0). + rank_replicated: bool + total_kv_heads: int + # Layers per chunk; 0 = this rank's own range (same-split sharing only). + # A stage must START on a multiple of it; only the last PP stage may end + # short, forming the model's trailing remainder chunk. + layer_partition: int = 0 + head_group: int + # Byte order of the stored objects. Layouts serialize a page in different + # orders at EQUAL sizes, so omitting this would let two deployments + # exchange byte-permuted KV under identical keys. + object_layout: str + + +def namespace_digest(namespace: KVCacheNamespace) -> str: + """Digest of the namespace encoding, used as the key prefix. + + msgpack encoding of a Struct is deterministic given the class definition, + which is why the schema versions the encoding. + """ + encoded = msgspec.msgpack.encode(namespace) + return f"ukv{_SCHEMA_VERSION}-{hashlib.sha256(encoded).hexdigest()[:16]}" + + +def derive_namespace( + *, + model_id: str, + dtype: str, + page_size: int, + rank_replicated: bool, + total_kv_heads: int, + head_group: int, + layer_partition: int = 0, + object_layout: str = PAGE_UNIFIED_OBJECT_LAYOUT, +) -> KVCacheNamespace: + """Derive the namespace from deployment facts plus the fleet agreements.""" + namespace = KVCacheNamespace( + model_id=model_id, + dtype=dtype, + page_size=page_size, + rank_replicated=rank_replicated, + total_kv_heads=0 if rank_replicated else total_kv_heads, + head_group=0 if rank_replicated else head_group, + layer_partition=layer_partition, + object_layout=object_layout, + ) + _validate_grid(namespace) + return namespace + + +def _validate_grid(namespace: KVCacheNamespace) -> None: + if namespace.layer_partition < 0: + raise ValueError(f"layer_partition must be non-negative: {namespace}") + if namespace.page_size <= 0: + raise ValueError(f"page_size must be positive: {namespace}") + if not namespace.model_id: + raise ValueError( + "the unified key scheme requires a non-empty model_id: an empty id " + "would merge different models into one keyspace." + ) + if namespace.rank_replicated: + if namespace.head_group != 0 or namespace.total_kv_heads != 0: + raise ValueError( + f"rank_replicated namespaces have no head axis; set " + f"total_kv_heads=0 and head_group=0: {namespace}" + ) + return + if namespace.head_group <= 0 or namespace.total_kv_heads <= 0: + raise ValueError( + f"sharded-KV namespaces need positive total_kv_heads/head_group: " + f"{namespace}" + ) + if namespace.total_kv_heads % namespace.head_group != 0: + raise ValueError( + f"head_group={namespace.head_group} must divide " + f"total_kv_heads={namespace.total_kv_heads}." + ) + + +def normalize_dtype(dtype: object) -> str: + """``torch.bfloat16`` -> ``"bfloat16"``.""" + return str(dtype).removeprefix("torch.") + + +class UnifiedKVPlan(msgspec.Struct, frozen=True, kw_only=True): + """One rank's unified-key objects and where their bytes sit in a page. + + ``suffixes`` is layer-major / head-minor, and ``layer_ranges`` x + ``head_ranges`` is the same cross product in the same order: entry + ``i * len(head_ranges) + j`` of ``suffixes`` names + ``(layer_ranges[i], head_ranges[j])``. Everything downstream relies on that + pairing, so the two are built together here rather than re-derived. + + The ranges are LOCAL to this rank's pool, so they index the host pool + directly; the suffixes carry the absolute, model-global coordinates. + """ + + namespace: KVCacheNamespace + suffixes: list[str] + layer_ranges: list[tuple[int, int]] + # Head groups, or a single whole-pool range for rank-replicated pools. + head_ranges: list[tuple[int, int]] + + @property + def head_group_num(self) -> int: + return len(self.head_ranges) + + +def plan_unified_kv( + *, + model_id: str, + dtype: str, + page_size: int, + rank_replicated: bool, + local_kv_heads: int, + attn_tp_rank: int, + attn_tp_size: int, + attn_cp_size: int, + start_layer: int, + end_layer: int, + is_final_stage: bool, + head_group: int | None = None, + layer_partition: int | None = None, +) -> UnifiedKVPlan: + """Derive the namespace and this rank's chunk plan from deployment facts. + + Raises rather than degrading: a rank whose shard does not tile the declared + grid would write objects a reader cannot reassemble, and the symptom would + be silently wrong KV rather than a miss. + """ + if attn_cp_size > 1: + raise NotImplementedError( + "the unified key scheme does not support attention context " + "parallelism yet: NSA-CP ranks hold sub-page slices (needs the " + "token-granule extension) and replicated-CP needs writer election. " + "Use --hicache-storage-key-scheme rank-suffix." + ) + if not 0 <= start_layer < end_layer: + raise ValueError(f"invalid layer range [{start_layer}, {end_layer}).") + + total_kv_heads = 0 if rank_replicated else local_kv_heads * attn_tp_size + resolved_head_group = _resolve_head_group( + rank_replicated=rank_replicated, + local_kv_heads=local_kv_heads, + attn_tp_size=attn_tp_size, + total_kv_heads=total_kv_heads, + head_group=head_group, + ) + namespace = derive_namespace( + model_id=model_id, + dtype=dtype, + page_size=page_size, + rank_replicated=rank_replicated, + total_kv_heads=total_kv_heads, + head_group=resolved_head_group, + layer_partition=layer_partition or 0, + ) + + layer_coords, layer_ranges = _layer_chunks( + namespace.layer_partition, start_layer, end_layer, is_final_stage + ) + digest = namespace_digest(namespace) + if rank_replicated: + return UnifiedKVPlan( + namespace=namespace, + suffixes=[f"{digest}_{coord}" for coord in layer_coords], + layer_ranges=layer_ranges, + head_ranges=[(0, 1)], + ) + + if local_kv_heads % resolved_head_group != 0: + raise ValueError( + f"this rank's {local_kv_heads} kv heads do not tile " + f"head_group={resolved_head_group}." + ) + groups_per_rank = local_kv_heads // resolved_head_group + first_head_index = attn_tp_rank * groups_per_rank + return UnifiedKVPlan( + namespace=namespace, + # Layer-major / head-minor, the order the page block packs them in. + suffixes=[ + f"{digest}_{coord}_H{first_head_index + i}" + for coord in layer_coords + for i in range(groups_per_rank) + ], + layer_ranges=layer_ranges, + head_ranges=[ + (i * resolved_head_group, (i + 1) * resolved_head_group) + for i in range(groups_per_rank) + ], + ) + + +def _resolve_head_group( + *, + rank_replicated: bool, + local_kv_heads: int, + attn_tp_size: int, + total_kv_heads: int, + head_group: int | None, +) -> int: + if rank_replicated: + return 0 + if head_group is None: + if local_kv_heads == 1 and attn_tp_size > 1: + # Ambiguous: 1 head/rank could equally mean kv-head replication, + # where two ranks hold the same head and would race on one key. + # An explicit head_group is the operator attesting it is sharding. + raise NotImplementedError( + "the unified key scheme cannot derive a namespace at 1 kv head " + "per rank; set --hicache-storage-head-group, or use " + "--hicache-storage-key-scheme rank-suffix." + ) + if local_kv_heads != total_kv_heads: + # Correct but unshareable: another attn-TP size derives a different + # digest and misses everything. Nothing else reports this -- the + # symptom is a 0% hit rate against a populated store. + logger.warning( + "unified key scheme: --hicache-storage-head-group is unset, so " + "this namespace is keyed to this rank's %d of %d kv heads and " + "can only share objects with deployments at attn-TP %d. Set it " + "to total_kv_heads / lcm of the fleet's attn-TP sizes to share " + "across TP sizes; a rank owning several groups simply owns " + "several chunks.", + local_kv_heads, + total_kv_heads, + attn_tp_size, + ) + return local_kv_heads + if head_group <= 0: + raise ValueError(f"head_group must be positive: {head_group}") + if local_kv_heads % head_group != 0 or head_group > local_kv_heads: + raise ValueError( + f"head_group={head_group} must divide this rank's {local_kv_heads} " + f"kv heads. As a fleet grid it must divide every member's local " + f"kv-head count, so pick total_kv_heads / lcm(the fleet's attn-TP " + f"sizes) -- here total_kv_heads={total_kv_heads}." + ) + return head_group + + +def _layer_chunks( + layer_partition: int, start_layer: int, end_layer: int, is_final_stage: bool +) -> tuple[list[str], list[tuple[int, int]]]: + """Absolute layer coordinates and their pool-local ranges. + + Coordinates are ABSOLUTE, so any PP split -- uneven stages included -- + yields collision-free names, and a differing split misses rather than + colliding. + """ + if not layer_partition: + return [f"L{start_layer}-{end_layer}"], [(0, end_layer - start_layer)] + if start_layer % layer_partition != 0: + raise ValueError( + f"this rank's layer range [{start_layer}, {end_layer}) does not " + f"start on a multiple of layer_partition={layer_partition}; stages " + f"whose boundaries do not align to the layer unit cannot share a " + f"partitioned namespace (drop layer_partition for per-stage ranges)." + ) + if end_layer % layer_partition != 0 and not is_final_stage: + raise ValueError( + f"this rank's layer range [{start_layer}, {end_layer}) ends off the " + f"layer_partition={layer_partition} grid; only the FINAL pipeline " + f"stage may end short (the model's trailing remainder forms the " + f"short last chunk)." + ) + bounds = [ + (a, min(a + layer_partition, end_layer)) + for a in range(start_layer, end_layer, layer_partition) + ] + return ( + [f"L{a}-{b}" for a, b in bounds], + [(a - start_layer, b - start_layer) for a, b in bounds], + ) diff --git a/python/sglang/srt/mem_cache/pool_host/mha.py b/python/sglang/srt/mem_cache/pool_host/mha.py index 51ab281630..aa4ffd387b 100644 --- a/python/sglang/srt/mem_cache/pool_host/mha.py +++ b/python/sglang/srt/mem_cache/pool_host/mha.py @@ -45,6 +45,7 @@ from sglang.srt.mem_cache.pool_host.common import ( make_kernel_ptr_table, ) from sglang.srt.mem_cache.pool_host.hisparse import HiSparseHostPoolMixin +from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout from sglang.srt.platforms import current_platform from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu @@ -141,9 +142,14 @@ class MHATokenToKVPoolHost(HostKVCache): *, mtp_draft_device_pools: Sequence[MHATokenToKVPool] = (), pool_label: str = "kv", + head_group_num: int = 1, ): self.mtp_draft_device_pools = tuple(mtp_draft_device_pools) self.target_layer_num = device_pool.layer_num + # page_unified cuts the kv-head axis into this many groups, and the cut + # is part of the byte order, so it has to be known before the buffer is + # allocated in init_kv_buffer() below. + self.head_group_num = head_group_num # Transfer buffers are captured below. The pool remains available for # NPU transfer APIs and device layer ownership. self.device_pool = device_pool @@ -195,7 +201,13 @@ class MHATokenToKVPoolHost(HostKVCache): element_size=self.element_dim * self.dtype.itemsize, ) - if self.layout == "page_first": + if self.layout == "page_unified": + # No per-component per-layer view exists: a layer's K and V are + # interleaved inside each head group's block. The page_unified + # transfer kernels address the fused buffer directly instead. + self.k_data_refs = [] + self.v_data_refs = [] + elif self.layout == "page_first": # Transpose [page, layer, ...] -> [layer, page, ...] to get per-layer views # This swaps strides without copying data k_transposed = self.k_buffer.transpose(0, 1) @@ -205,15 +217,23 @@ class MHATokenToKVPoolHost(HostKVCache): else: self.k_data_refs = [self.k_buffer[i] for i in range(self.layer_num)] self.v_data_refs = [self.v_buffer[i] for i in range(self.layer_num)] - self.k_data_ptrs = make_kernel_ptr_table( - self.k_data_refs, - self._transfer_device, - host_memory_registered=self.pin_memory, + self.k_data_ptrs = ( + make_kernel_ptr_table( + self.k_data_refs, + self._transfer_device, + host_memory_registered=self.pin_memory, + ) + if self.k_data_refs + else None ) - self.v_data_ptrs = make_kernel_ptr_table( - self.v_data_refs, - self._transfer_device, - host_memory_registered=self.pin_memory, + self.v_data_ptrs = ( + make_kernel_ptr_table( + self.v_data_refs, + self._transfer_device, + host_memory_registered=self.pin_memory, + ) + if self.v_data_refs + else None ) if not _is_npu: pools = (self.device_pool, *self.mtp_draft_device_pools) @@ -268,6 +288,16 @@ class MHATokenToKVPoolHost(HostKVCache): self.layer_num, self.head_dim, ) + elif self.layout == "page_unified": + self.page_unified_layout = PageUnifiedLayout( + page_size=self.page_size, + layer_num=self.layer_num, + head_num=self.head_num, + head_group_num=self.head_group_num, + head_dim=self.head_dim, + itemsize=self.dtype.itemsize, + ) + dims = self.page_unified_layout.page_dims(self.page_num) else: raise ValueError(f"Unsupported layout: {self.layout}") self.token_stride_size = self.head_num * self.head_dim * self.dtype.itemsize @@ -281,7 +311,11 @@ class MHATokenToKVPoolHost(HostKVCache): pin_memory=self.pin_memory, allocator=self.allocator, registration_granularity_bytes=( - self.page_size * self.layout_dim + # page_unified's block carries both components, so its page is + # twice page_size * layout_dim. + self.page_unified_layout.bytes_per_page + if self.layout == "page_unified" + else self.page_size * self.layout_dim if self.layout in ("page_first", "page_first_direct") else None ), diff --git a/python/sglang/srt/mem_cache/pool_host/mla.py b/python/sglang/srt/mem_cache/pool_host/mla.py index 1b12b559dc..cf545d516f 100644 --- a/python/sglang/srt/mem_cache/pool_host/mla.py +++ b/python/sglang/srt/mem_cache/pool_host/mla.py @@ -37,6 +37,7 @@ from sglang.srt.mem_cache.pool_host.npu_memfabric import ( to_device_no_sync, track_pinned_staging, ) +from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout from sglang.srt.platforms import current_platform from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu @@ -257,6 +258,21 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): 1, self.kv_cache_dim, ) + elif self.layout == "page_unified": + # Same bytes as page_first_direct -- MLA is rank-replicated, so the + # grid never cuts a head axis and there is nothing to permute. It + # is still its own layout so the L3 namespace, the transfer arms + # and the MHA side all name one thing. + self.page_unified_layout = PageUnifiedLayout( + page_size=self.page_size, + layer_num=self.layer_num, + head_num=1, + head_group_num=1, + head_dim=self.kv_cache_dim, + itemsize=self.dtype.itemsize, + is_mla=True, + ) + dims = self.page_unified_layout.page_dims(self.page_num) # Ascend-specific: Aligns with NPUMLATokenToKVPool layout # Separately allocate k_buffer and v_buffer for easier data transfer. elif self.layout == "page_first_kv_split": @@ -356,7 +372,7 @@ class MLATokenToKVPoolHost(HiSparseHostPoolMixin, HostKVCache): allocator=self.allocator, registration_granularity_bytes=( self.page_size * self.layout_dim - if self.layout in ("page_first", "page_first_direct") + if self.layout in ("page_first", "page_first_direct", "page_unified") else None ), ) diff --git a/python/sglang/srt/mem_cache/pool_host/page_unified.py b/python/sglang/srt/mem_cache/pool_host/page_unified.py new file mode 100644 index 0000000000..14bb96bcf8 --- /dev/null +++ b/python/sglang/srt/mem_cache/pool_host/page_unified.py @@ -0,0 +1,152 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""The ``page_unified`` host KV layout: one page block's byte order. + +One page block, in this byte order:: + + MHA: (head_group, layer, 2, page_size, head_in_group, head_dim) K=0, V=1 + MLA: (layer, page_size, head_dim) + +A PARTITION AXIS MUST BE OUTER. The unified L3 grid cuts a page along the kv-head +axis and then along the layer axis, so both sit outside ``page_size``: a chunk is +then an offset and a length, not a stride pattern. Put either one inside and +naming a sub-range costs a descriptor per token. + +The K/V axis sits INSIDE the layer axis rather than outside the pool, which is +what makes a chunk a single run and lets one object carry both components. +``layer_first`` and ``page_first_direct`` both hoist it outside, so under those +a chunk is two objects whose runs interleave at token granularity. + +The same order is what the write-back kernel produces and the load-back kernel +reads, so a page is byte-identical whether it arrived from the device or from +L3, and neither side has to ask which. +""" + +from __future__ import annotations + +import msgspec + +# Byte order identity, recorded in the L3 namespace digest. Bump when the axis +# order changes, so old objects miss instead of deserializing permuted. +PAGE_UNIFIED_OBJECT_LAYOUT = "page-unified-v1" + + +class PageUnifiedLayout(msgspec.Struct, frozen=True, kw_only=True): + """Byte layout of one ``page_unified`` page block. + + Built directly from a host pool's own dimensions. ``head_group_num`` is the + number of kv-head GROUPS the page is cut into, which is the unified grid's + head partition; MLA is rank-replicated and has no head axis, so both it and + ``head_num`` are 1 there. + """ + + page_size: int + layer_num: int + head_num: int + head_group_num: int + head_dim: int + itemsize: int + is_mla: bool = False + + def __post_init__(self) -> None: + for name in ( + "page_size", + "layer_num", + "head_num", + "head_group_num", + "head_dim", + "itemsize", + ): + if getattr(self, name) <= 0: + raise ValueError(f"{name} must be positive: {self}") + if self.head_num % self.head_group_num != 0: + raise ValueError( + f"head_group_num={self.head_group_num} must divide the pool's " + f"{self.head_num} kv heads." + ) + if self.is_mla and (self.head_group_num != 1 or self.head_num != 1): + raise ValueError(f"MLA page blocks have no head axis: {self}") + # The transfer kernels move a token's head-group run as 16-byte vectors. + if self.group_bytes % 16 != 0: + raise ValueError( + f"one head group's token row is {self.group_bytes} bytes, which " + f"the page_unified transfer kernels cannot move: they require a " + f"multiple of 16. Raise head_dim, lower head_group_num, or use " + f"another --hicache-mem-layout." + ) + + @property + def components(self) -> int: + """K and V, or the single latent cache for MLA.""" + return 1 if self.is_mla else 2 + + @property + def heads_per_group(self) -> int: + return self.head_num // self.head_group_num + + @property + def group_bytes(self) -> int: + """One head group's row for one token: the copy unit.""" + return self.heads_per_group * self.head_dim * self.itemsize + + @property + def layer_bytes(self) -> int: + """One (head group, layer) cell: both components, all tokens of a page.""" + return self.components * self.page_size * self.group_bytes + + @property + def head_group_bytes(self) -> int: + return self.layer_num * self.layer_bytes + + @property + def bytes_per_page(self) -> int: + return self.head_group_num * self.head_group_bytes + + def page_dims(self, page_num: int) -> tuple[int, ...]: + """Buffer shape, whose C-contiguous order IS the byte order.""" + if self.is_mla: + return (page_num, self.layer_num, self.page_size, self.head_dim) + return ( + page_num, + self.head_group_num, + self.layer_num, + self.components, + self.page_size, + self.heads_per_group, + self.head_dim, + ) + + def chunk_span( + self, layer_range: tuple[int, int], head_group: int = 0 + ) -> tuple[int, int]: + """Byte ``(offset, length)`` of one L3 chunk within a page block. + + A chunk is a half-open LOCAL layer range within one head group. It is + always exactly one run, including a ragged final layer window -- that + is the property the axis order buys, and what lets the transport treat + every chunk uniformly instead of switching on a descriptor count. + """ + start, end = layer_range + if not 0 <= start < end <= self.layer_num: + raise ValueError( + f"layer range {layer_range} is not a non-empty sub-range of " + f"[0, {self.layer_num})." + ) + if not 0 <= head_group < self.head_group_num: + raise ValueError( + f"head group {head_group} is outside [0, {self.head_group_num})." + ) + offset = head_group * self.head_group_bytes + start * self.layer_bytes + return offset, (end - start) * self.layer_bytes diff --git a/test/registered/unit/mem_cache/test_page_unified_layout.py b/test/registered/unit/mem_cache/test_page_unified_layout.py new file mode 100644 index 0000000000..a337c08b71 --- /dev/null +++ b/test/registered/unit/mem_cache/test_page_unified_layout.py @@ -0,0 +1,410 @@ +"""The ``page_unified`` page layout and the unified L3 key scheme that names it. + +Two halves of one thing: the layout fixes where a chunk's bytes sit inside a +page, and the key scheme names that same chunk as a layer-range x head-range +coordinate. They share a file because they have to agree on the grid. +""" + +import unittest + +import torch + +from sglang.srt.mem_cache.hicache_key_scheme import ( + derive_namespace, + namespace_digest, + normalize_dtype, + plan_unified_kv, +) +from sglang.srt.mem_cache.pool_host.page_unified import PageUnifiedLayout +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def _mha(**overrides): + kwargs = dict( + page_size=4, + layer_num=6, + head_num=4, + head_group_num=2, + head_dim=8, + itemsize=2, + ) + kwargs.update(overrides) + return PageUnifiedLayout(**kwargs) + + +class TestPageUnifiedLayout(CustomTestCase): + def test_page_dims_are_the_byte_order(self): + """The buffer's C-contiguous strides must equal the declared strides. + + Everything else here -- and both transfer kernels -- computes offsets + from the layout rather than from the tensor, so a disagreement between + the two would move correct-looking bytes to wrong addresses. + """ + layout = _mha() + buffer = torch.empty(layout.page_dims(3), dtype=torch.bfloat16) + strides = [s * buffer.element_size() for s in buffer.stride()] + self.assertEqual( + strides, + [ + layout.bytes_per_page, + layout.head_group_bytes, + layout.layer_bytes, + layout.page_size * layout.group_bytes, + layout.group_bytes, + layout.head_dim * layout.itemsize, + layout.itemsize, + ], + ) + + def test_chunks_tile_the_page_exactly_once(self): + """The grid's chunks must partition a page: no gap, no overlap. + + A gap is KV that is never stored; an overlap is two keys naming the + same bytes, which diverge as soon as one is evicted. + """ + layout = _mha(layer_num=6, head_num=3, head_group_num=3) + covered = bytearray(layout.bytes_per_page) + for head_group in range(layout.head_group_num): + for start in range(0, layout.layer_num, 2): + offset, length = layout.chunk_span((start, start + 2), head_group) + for byte in range(offset, offset + length): + covered[byte] += 1 + self.assertEqual(set(covered), {1}) + + def test_ragged_layer_window_is_still_one_run(self): + """A short final window is one run, which is the point of the axis order. + + DeepSeek-V3's 61 layers at layer_partition=30 leave a width-1 tail. If + that tail were several runs the transport would have to switch + descriptor forms mid-plan. + """ + layout = _mha(layer_num=61, head_num=4, head_group_num=1) + spans = [layout.chunk_span(r) for r in [(0, 30), (30, 60), (60, 61)]] + self.assertEqual( + spans, + [ + (0, 30 * layout.layer_bytes), + (30 * layout.layer_bytes, 30 * layout.layer_bytes), + (60 * layout.layer_bytes, layout.layer_bytes), + ], + ) + self.assertEqual(sum(length for _, length in spans), layout.bytes_per_page) + + def test_head_group_count_does_not_change_page_size(self): + """Cutting the head axis permutes a page; it must not resize it.""" + sizes = { + PageUnifiedLayout( + page_size=4, + layer_num=6, + head_num=8, + head_group_num=groups, + head_dim=16, + itemsize=2, + ).bytes_per_page + for groups in (1, 2, 4, 8) + } + self.assertEqual(len(sizes), 1) + + def test_mla_page_has_no_head_or_component_axis(self): + layout = PageUnifiedLayout( + page_size=64, + layer_num=61, + head_num=1, + head_group_num=1, + head_dim=576, + itemsize=2, + is_mla=True, + ) + self.assertEqual(layout.page_dims(7), (7, 61, 64, 576)) + self.assertEqual(layout.components, 1) + self.assertEqual(layout.chunk_span((0, 61)), (0, layout.bytes_per_page)) + + def test_unaligned_group_row_is_rejected(self): + """The transfer kernels move 16-byte vectors and cannot express a tail.""" + with self.assertRaisesRegex(ValueError, "multiple of 16"): + _mha(head_num=2, head_group_num=2, head_dim=7, itemsize=1) + + def test_head_group_must_divide_the_pools_heads(self): + with self.assertRaisesRegex(ValueError, "must divide"): + PageUnifiedLayout( + page_size=4, + layer_num=2, + head_num=6, + head_group_num=4, + head_dim=16, + itemsize=2, + ) + + def test_chunk_span_rejects_out_of_range_coordinates(self): + layout = _mha() + with self.assertRaisesRegex(ValueError, "layer range"): + layout.chunk_span((0, layout.layer_num + 1)) + with self.assertRaisesRegex(ValueError, "layer range"): + layout.chunk_span((3, 3)) + with self.assertRaisesRegex(ValueError, "head group"): + layout.chunk_span((0, 1), layout.head_group_num) + + +# GQA-70B shape: 8 kv heads, 80 layers. +BASE = dict( + model_id="meta-llama/Llama-3.1-70B", + dtype="bfloat16", + page_size=64, + rank_replicated=False, + attn_cp_size=1, + start_layer=0, + end_layer=80, + is_final_stage=True, +) + + +def _plan(**overrides): + kwargs = dict(BASE) + kwargs.update(overrides) + return plan_unified_kv(**kwargs) + + +class TestUnifiedKeyGrid(CustomTestCase): + def test_tp_shards_of_one_grid_cover_each_other(self): + """TP2's objects are exactly the union of TP4's, in order. + + This is the whole point of the scheme: a cache written by one topology + must be readable by another. Nothing else in the stack checks it, and + the failure mode is a silent 0% hit rate. + """ + tp2 = [ + _plan(local_kv_heads=4, attn_tp_rank=r, attn_tp_size=2, head_group=2) + for r in range(2) + ] + tp4 = [ + _plan(local_kv_heads=2, attn_tp_rank=r, attn_tp_size=4, head_group=2) + for r in range(4) + ] + self.assertEqual( + [s for p in tp2 for s in p.suffixes], + [s for p in tp4 for s in p.suffixes], + ) + + def test_pp_stages_cover_the_unsplit_range(self): + """Same property along the layer axis, including a ragged tail.""" + whole = _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + layer_partition=30, + end_layer=61, + ) + stage0 = _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + layer_partition=30, + start_layer=0, + end_layer=30, + is_final_stage=False, + ) + stage1 = _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + layer_partition=30, + start_layer=30, + end_layer=61, + ) + self.assertEqual(whole.suffixes, stage0.suffixes + stage1.suffixes) + # The 61st layer forms a short trailing chunk rather than being dropped. + self.assertTrue(whole.suffixes[-1].endswith("_L60-61_H0")) + + def test_suffix_order_matches_the_range_cross_product(self): + """Layer-major / head-minor, and the ranges pair with it positionally. + + The storage layer zips suffixes against layer_ranges x head_ranges to + find each object's bytes; a reordering here writes every chunk under + its neighbour's key. + """ + plan = _plan( + local_kv_heads=4, + attn_tp_rank=0, + attn_tp_size=2, + head_group=2, + layer_partition=40, + ) + self.assertEqual(len(plan.layer_ranges), 2) + self.assertEqual(plan.head_ranges, [(0, 2), (2, 4)]) + digest = namespace_digest(plan.namespace) + self.assertEqual( + plan.suffixes, + [ + f"{digest}_L0-40_H0", + f"{digest}_L0-40_H1", + f"{digest}_L40-80_H0", + f"{digest}_L40-80_H1", + ], + ) + + def test_head_index_is_global_not_rank_local(self): + """Rank 1's groups must not collide with rank 0's.""" + plans = [ + _plan(local_kv_heads=2, attn_tp_rank=r, attn_tp_size=4, head_group=2) + for r in range(4) + ] + self.assertEqual( + [p.suffixes[0].rsplit("_", 1)[1] for p in plans], + ["H0", "H1", "H2", "H3"], + ) + + def test_local_ranges_are_pool_relative(self): + """A PP stage indexes its own pool, so its ranges start at 0.""" + stage1 = _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + layer_partition=20, + start_layer=40, + end_layer=80, + ) + self.assertEqual(stage1.layer_ranges, [(0, 20), (20, 40)]) + self.assertTrue(stage1.suffixes[0].endswith("_L40-60_H0")) + + +class TestNamespaceIdentity(CustomTestCase): + def test_identity_fields_partition_the_keyspace(self): + """Every identity field must change the digest. + + A field that does not reach the digest lets two deployments with + different bytes collide on one key, which is corruption rather than a + miss. + """ + base = dict( + model_id="m", + dtype="bfloat16", + page_size=64, + rank_replicated=False, + total_kv_heads=8, + head_group=2, + ) + digests = {namespace_digest(derive_namespace(**base))} + for field, value in [ + ("model_id", "other"), + ("dtype", "float16"), + ("page_size", 32), + ("total_kv_heads", 16), + ("head_group", 4), + ("layer_partition", 30), + ("object_layout", "something-else"), + ]: + digests.add(namespace_digest(derive_namespace(**{**base, field: value}))) + self.assertEqual(len(digests), 8) + + def test_fp8_variants_do_not_share_a_keyspace(self): + """Both store as uint8, so only the logical dtype separates them.""" + self.assertNotEqual( + namespace_digest( + _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + dtype="float8_e4m3fn", + ).namespace + ), + namespace_digest( + _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + dtype="float8_e5m2", + ).namespace + ), + ) + + def test_rank_replicated_drops_the_head_axis(self): + plan = _plan( + rank_replicated=True, + local_kv_heads=0, + attn_tp_rank=3, + attn_tp_size=8, + ) + self.assertEqual(plan.head_ranges, [(0, 1)]) + self.assertNotIn("_H", plan.suffixes[0]) + # Every rank derives the same keys, which is what makes one writer enough. + other = _plan( + rank_replicated=True, local_kv_heads=0, attn_tp_rank=0, attn_tp_size=8 + ) + self.assertEqual(plan.suffixes, other.suffixes) + + def test_normalize_dtype_strips_the_torch_prefix(self): + self.assertEqual(normalize_dtype(torch.bfloat16), "bfloat16") + + +class TestUnifiedKeySchemeRejections(CustomTestCase): + def test_head_group_must_tile_this_ranks_heads(self): + with self.assertRaisesRegex(ValueError, "must divide this rank's"): + _plan(local_kv_heads=3, attn_tp_rank=0, attn_tp_size=2, head_group=2) + + def test_misaligned_stage_start_is_rejected(self): + """A stage starting off-grid would name chunks no reader can assemble.""" + with self.assertRaisesRegex(ValueError, "does not start on a multiple"): + _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + layer_partition=30, + start_layer=20, + end_layer=50, + ) + + def test_non_final_stage_may_not_end_short(self): + with self.assertRaisesRegex(ValueError, "only the FINAL pipeline stage"): + _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + layer_partition=30, + start_layer=0, + end_layer=50, + is_final_stage=False, + ) + + def test_one_kv_head_per_rank_needs_an_explicit_head_group(self): + """1 head per rank is ambiguous: sharded, or replicated across ranks? + + Replication would have several ranks racing to write one key, so the + operator has to attest which it is. + """ + with self.assertRaises(NotImplementedError): + _plan(local_kv_heads=1, attn_tp_rank=0, attn_tp_size=8) + + def test_attention_cp_is_refused(self): + with self.assertRaises(NotImplementedError): + _plan( + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + attn_cp_size=2, + ) + + def test_empty_model_id_is_refused(self): + with self.assertRaisesRegex(ValueError, "non-empty model_id"): + _plan( + model_id="", + local_kv_heads=8, + attn_tp_rank=0, + attn_tp_size=1, + head_group=8, + ) + + +if __name__ == "__main__": + unittest.main()