From 5733bcaad32a063e2e7f557693f5ae72f1ad66e3 Mon Sep 17 00:00:00 2001 From: Ke Bao Date: Thu, 1 Oct 2026 13:08:26 +0800 Subject: [PATCH] Declare HiCache host pools for DSA indexer (#40913) --- .../hybrid_cache/host_pool_config.py | 270 +++++++ .../hybrid_cache/hybrid_pool_assembler.py | 342 +++++---- python/sglang/srt/mem_cache/memory_pool.py | 28 + python/sglang/srt/mem_cache/pool_host/dsa.py | 161 +++- .../srt/mem_cache/pool_host/host_pool_decl.py | 112 +++ .../unit/mem_cache/test_dsa_pool_host_unit.py | 8 +- .../mem_cache/test_hicache_host_register.py | 1 + ...test_hicache_staged_write_back_dispatch.py | 2 + .../mem_cache/test_hybrid_pool_assembler.py | 695 +++++++++++++++++- .../unit/mem_cache/test_mem_pool_host.py | 102 +++ .../test_mla_host_dedup_primitives.py | 11 +- 11 files changed, 1520 insertions(+), 212 deletions(-) create mode 100644 python/sglang/srt/mem_cache/hybrid_cache/host_pool_config.py create mode 100644 python/sglang/srt/mem_cache/pool_host/host_pool_decl.py diff --git a/python/sglang/srt/mem_cache/hybrid_cache/host_pool_config.py b/python/sglang/srt/mem_cache/hybrid_cache/host_pool_config.py new file mode 100644 index 0000000000..3e6251047c --- /dev/null +++ b/python/sglang/srt/mem_cache/hybrid_cache/host_pool_config.py @@ -0,0 +1,270 @@ +"""Pure HiCache declaration validation and transfer-layer binding.""" + +from __future__ import annotations + +from typing import Any, Optional + +import msgspec + +from sglang.srt.mem_cache.hicache_storage import PoolName +from sglang.srt.mem_cache.pool_host.host_pool_decl import HostPoolDecl + + +class HostPoolConfig(msgspec.Struct, frozen=True, kw_only=True): + """Validated device owners and transfer layers for one host pool.""" + + decl: HostPoolDecl + layer_mapping: dict[int, int] + # Exclusive upper bound, including packed draft tails. + transfer_layer_id_max: int + # Draft pools whose same-named buffers are appended as tail layers, in depth order. + packed_draft_device_pools: tuple[Any, ...] = () + + +class HostPoolGroupConfig(msgspec.Struct, frozen=True, kw_only=True): + """Validated host pools and their shared transfer granularity.""" + + # Original-token slots moved together. Individual buffers may use fewer rows. + transfer_page_size: int + # Dependency order with the unique primary KV root first. + pools: tuple[HostPoolConfig, ...] + + +def with_packed_draft_layer_mapping( + layer_mapping: dict[int, int], + *, + transfer_layer_start: int, + target_device_layer_num: int, + draft_layer_num: int, +) -> dict[int, int]: + return layer_mapping | { + transfer_layer_start + depth: target_device_layer_num + depth + for depth in range(draft_layer_num) + } + + +def _validate_packed_draft_pools( + *, target_decls: tuple[HostPoolDecl, ...], draft_pools: tuple[Any, ...] +) -> tuple[tuple[HostPoolDecl, ...], ...]: + """Collect draft declarations once and check they can be appended to target host + pools: it declares the same pools with the same storage info and every + packed layer owns its buffer. The draft plan already chose packing, so a + draft that cannot be packed is an error, not a silent skip.""" + targets = {d.pool_name: d for d in target_decls} + collected = [] + for pool in draft_pools: + declarations = pool.host_pool_decls() + collected.append(declarations) + drafts = {d.pool_name: d for d in declarations} + if len(drafts) != len(declarations): + raise ValueError("packed draft declares duplicate host pool names") + if set(drafts) != set(targets): + raise ValueError( + f"packed draft {type(pool).__name__} declares " + f"{sorted(d.value for d in drafts)} but the target declares " + f"{sorted(d.value for d in targets)}; every target pool needs a " + "draft counterpart or the draft buffers are not restored" + ) + for name, target in targets.items(): + draft = drafts[name] + if draft.storage_info != target.storage_info: + raise ValueError( + f"packed draft {name.value} storage {draft.storage_info} " + f"differs from target {target.storage_info}" + ) + owned = draft.owned_device_layers + if owned is not None and ( + len(set(owned)) != draft.device_pool.layer_num + or set(owned) != set(range(draft.device_pool.layer_num)) + ): + raise ValueError( + f"packed draft {name.value} owns buffers on {len(owned)} of " + f"{draft.device_pool.layer_num} layers; every packed layer " + "must own its buffer" + ) + if layout_root(declarations).device_pool.layer_num != 1: + raise ValueError( + "packed draft requires exactly one KV layer per draft pool" + ) + return tuple(collected) + + +def layout_root(decls: tuple[HostPoolDecl, ...]) -> HostPoolDecl: + """The one declaration whose host pool decides the group's capacity.""" + roots = [d for d in decls if d.is_layout_root] + if len(roots) != 1: + raise ValueError( + f"expected exactly one layout root, got {[d.pool_name for d in roots]}" + ) + return roots[0] + + +def _find_pool_decl(*, decls: tuple[HostPoolDecl, ...], name: PoolName) -> HostPoolDecl: + return next(d for d in decls if d.pool_name == name) + + +def _validate_and_order_decls( + decls: tuple[HostPoolDecl, ...], +) -> tuple[HostPoolDecl, ...]: + names = [decl.pool_name for decl in decls] + if len(set(names)) != len(names): + raise ValueError(f"duplicate host pool names: {names}") + root = layout_root(decls) + if not root.is_primary or root.pool_name != PoolName.KV: + raise ValueError( + f"expected the layout root to be the primary KV pool, got {root.pool_name}" + ) + for decl in decls: + if not decl.is_layout_root and ( + decl.host_pool_builder is None or decl.storage_info is None + ): + raise ValueError( + f"{decl.pool_name}: dependent pool needs builder and storage_info" + ) + owned = decl.owned_device_layers + if owned is not None and ( + len(set(owned)) != len(owned) + or any(layer < 0 or layer >= decl.device_pool.layer_num for layer in owned) + ): + raise ValueError(f"{decl.pool_name}: invalid owned_device_layers {owned}") + if decl.is_primary: + continue + if decl.indices_from_pool != root.pool_name: + raise ValueError( + f"{decl.pool_name}.indices_from_pool must be {root.pool_name}, " + f"got {decl.indices_from_pool}" + ) + if decl.is_layout_root: + continue + if decl.layout_source == decl.pool_name: + raise ValueError(f"{decl.pool_name}.layout_source must name another pool") + if decl.layout_source not in names: + raise ValueError( + f"{decl.pool_name} references undeclared pool {decl.layout_source}" + ) + # Topologically sort by layout_source so dependencies are built first. + ordered = [] + pending = list(decls) + built = set() + while pending: + ready = [ + decl + for decl in pending + if decl.is_layout_root or decl.layout_source in built + ] + if not ready: + raise ValueError( + f"cyclic layout_source dependencies: {[decl.pool_name for decl in pending]}" + ) + for decl in ready: + ordered.append(decl) + built.add(decl.pool_name) + pending.remove(decl) + return tuple(ordered) + + +def prepare_host_pool_config( + *, + decls: tuple[HostPoolDecl, ...], + full_layer_mapping: dict[int, int], + transfer_layer_id_max: int, + transfer_page_size: int, + packed_draft_device_pools: tuple[Any, ...] = (), +) -> HostPoolGroupConfig: + """Validate declarations and buffers, then append packed draft transfer layers. + + The input mapping and limit describe target layers only. No host memory is + allocated here. config.pools are in host construction order. + """ + decls = _validate_and_order_decls(decls) + root = decls[0] + target_layers = root.device_pool.layer_num + for transfer_id, device_id in full_layer_mapping.items(): + if not 0 <= transfer_id < transfer_layer_id_max: + raise ValueError( + f"transfer layer {transfer_id} outside [0, {transfer_layer_id_max})" + ) + if not 0 <= device_id < target_layers: + raise ValueError(f"device layer {device_id} outside [0, {target_layers})") + packed_draft_decls = _validate_packed_draft_pools( + target_decls=decls, draft_pools=packed_draft_device_pools + ) + mapping = with_packed_draft_layer_mapping( + full_layer_mapping, + transfer_layer_start=transfer_layer_id_max, + target_device_layer_num=target_layers, + draft_layer_num=len(packed_draft_decls), + ) + configs = tuple( + HostPoolConfig( + decl=d, + layer_mapping=_filter_owned_layer_mapping( + mapping=mapping, + owned_device_layers=d.owned_device_layers, + target_layer_num=target_layers, + ), + transfer_layer_id_max=transfer_layer_id_max + len(packed_draft_decls), + packed_draft_device_pools=tuple( + _find_pool_decl(decls=decls_of_draft, name=d.pool_name).device_pool + for decls_of_draft in packed_draft_decls + ), + ) + for d in decls + ) + check_packed_kv_rows( + kv_pool=root.device_pool, + drafts=configs[0].packed_draft_device_pools, + ) + for config in configs[1:]: + config.decl.host_pool_builder.validate( + decl=config.decl, + transfer_page_size=transfer_page_size, + packed_draft_device_pools=config.packed_draft_device_pools, + ) + return HostPoolGroupConfig(transfer_page_size=transfer_page_size, pools=configs) + + +def _filter_owned_layer_mapping( + *, + mapping: dict[int, int], + owned_device_layers: Optional[tuple[int, ...]], + target_layer_num: int, +) -> dict[int, int]: + """Drop transfer layers whose device layer owns no buffer for this pool. + Packed draft tails (device index >= target layer count) are kept.""" + if owned_device_layers is None: + return mapping + owned = set(owned_device_layers) + return { + t: dev for t, dev in mapping.items() if dev >= target_layer_num or dev in owned + } + + +def is_mla_pool(pool: Any) -> bool: + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + + return isinstance(pool, MLATokenToKVPool) + + +def _kv_row_signature(pool: Any, *, use_mla: bool) -> tuple: + if use_mla: + return (pool.kv_cache_dim, pool.store_dtype) + return (pool.head_num, pool.head_dim, pool.v_head_dim, pool.store_dtype) + + +def check_packed_kv_rows( + *, kv_pool: Any, drafts: tuple[Any, ...], use_mla: Optional[bool] = None +): + """Packed draft KV layers share the target's host row, so their device + rows must have the same shape and dtype.""" + if not drafts: + return + if use_mla is None: + use_mla = is_mla_pool(kv_pool) + target = _kv_row_signature(kv_pool, use_mla=use_mla) + for draft in drafts: + row = _kv_row_signature(draft, use_mla=use_mla) + if row != target: + raise ValueError( + f"packed draft KV row {row} differs from target KV row {target}" + ) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index a9f26c6ac6..33bb67cdef 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -9,6 +9,13 @@ from sglang.srt.mem_cache.hicache_storage import ( PoolName, SidecarPoolSpec, ) +from sglang.srt.mem_cache.hybrid_cache.host_pool_config import ( + HostPoolGroupConfig, + check_packed_kv_rows, + is_mla_pool, + prepare_host_pool_config, + with_packed_draft_layer_mapping, +) from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( HybridCacheController, ) @@ -19,7 +26,7 @@ from sglang.srt.mem_cache.memory_pool_host import ( ) from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry from sglang.srt.mem_cache.pool_host.common import get_allocator_type -from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost +from sglang.srt.mem_cache.pool_host.host_pool_decl import HostPoolDecl from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.srt.mem_cache.pool_host.mha import ( MHATokenToKOnlyPoolHost, @@ -87,19 +94,6 @@ def _stage_local_layer_mapping( } -def _with_mtp_layer_mapping( - layer_mapping: dict[int, int], - *, - transfer_layer_start: int, - target_device_layer_num: int, - draft_layer_num: int, -) -> dict[int, int]: - return layer_mapping | { - transfer_layer_start + depth: target_device_layer_num + depth - for depth in range(draft_layer_num) - } - - class _DeepSeekV4LayerMappings(NamedTuple): transfer_layer_id_max: int full: dict[int, int] @@ -142,15 +136,24 @@ def build_kv_host_pool( *, kv_pool: Any, page_size: int, - use_mla: bool, + use_mla: Optional[bool] = None, override_kv_cache_dim: Optional[int] = None, host_size: Optional[float] = None, mtp_draft_device_pools: tuple[Any, ...] = (), pool_label: str = "kv", ): + if use_mla is None: + use_mla = is_mla_pool(kv_pool) kv_host_pool_cls = ( MLATokenToKVPoolHost if use_mla else get_mha_host_pool_cls(kv_pool) ) + if use_mla and override_kv_cache_dim is None: + # An fp8 DSA store is wider than kv_lora_rank + qk_rope_head_dim. The + # device pool already resolved the row width. + override_kv_cache_dim = kv_pool.kv_cache_dim + check_packed_kv_rows( + kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla + ) kwargs = {} if override_kv_cache_dim is not None: kwargs["override_kv_cache_dim"] = override_kv_cache_dim @@ -245,7 +248,7 @@ def build_kv_only_group( mtp_draft_device_pools=mtp_draft_device_pools, ) if mtp_draft_device_pools: - full_layer_mapping = _with_mtp_layer_mapping( + full_layer_mapping = with_packed_draft_layer_mapping( full_layer_mapping, transfer_layer_start=transfer_layer_id_max, target_device_layer_num=kv_pool.layer_num, @@ -356,7 +359,7 @@ def build_hybrid_swa_group( pool_label="swa", ) if mtp_swa_device_pools: - swa_layer_mapping = _with_mtp_layer_mapping( + swa_layer_mapping = with_packed_draft_layer_mapping( swa_layer_mapping, transfer_layer_start=transfer_layer_id_max, target_device_layer_num=swa_kv_pool.layer_num, @@ -856,7 +859,7 @@ def build_deepseek_v4_hicache_stack( for pool in params.mtp_draft_device_pools for buffer in pool.swa_kv_pool.kv_buffer ] - swa_layer_mapping = _with_mtp_layer_mapping( + swa_layer_mapping = with_packed_draft_layer_mapping( swa_layer_mapping, transfer_layer_start=transfer_layer_id_max, target_device_layer_num=transfer_layer_id_max, @@ -1155,7 +1158,7 @@ def build_hybrid_mamba_stack( mtp_draft_device_pools=mtp_draft_device_pools, ) if mtp_draft_device_pools: - full_layer_mapping = _with_mtp_layer_mapping( + full_layer_mapping = with_packed_draft_layer_mapping( full_layer_mapping, transfer_layer_start=transfer_layer_id_max, target_device_layer_num=kv_pool.layer_num, @@ -1345,82 +1348,41 @@ def build_hybrid_mamba_swa_stack( return host_pool_group, cache_controller -def build_anchor_sidecar_stack( +def build_host_pool_group( *, - params: CacheInitParams, - kv_pool: Any, - sidecar_pool_name: PoolName, - full_layer_mapping: dict[int, int], - load_cache_event, - storage_backend: Optional[str], - use_mla: bool, - override_kv_cache_dim: Optional[int] = None, - sidecar_host_pool_factory: Callable[[Any], Any], - prefetch_threshold: int = 256, - model_name: Optional[str] = None, - storage_backend_extra_config: Optional[dict] = None, - enable_storage_metrics: bool = False, -) -> tuple[HostPoolGroup, HybridCacheController]: - transfer_layer_id_max = len(full_layer_mapping) - mtp_draft_device_pools = tuple( - pool for pool in params.mtp_draft_device_pools if pool.index_k_with_scale_buffer - ) - kv_host_pool = build_kv_host_pool( - kv_pool=kv_pool, - page_size=params.page_size, - use_mla=use_mla, - override_kv_cache_dim=override_kv_cache_dim, - mtp_draft_device_pools=mtp_draft_device_pools, - ) - sidecar_host_pool = sidecar_host_pool_factory(kv_host_pool) - # Expose packed MTP tail layers to the controller's flat transfer builder. - if mtp_draft_device_pools: - full_layer_mapping = _with_mtp_layer_mapping( - full_layer_mapping, - transfer_layer_start=transfer_layer_id_max, - target_device_layer_num=kv_pool.layer_num, - draft_layer_num=len(mtp_draft_device_pools), + config: HostPoolGroupConfig, +) -> HostPoolGroup: + """Allocate host pools and transfer entries from prepared configs.""" + root = config.pools[0] + host_pools = { + root.decl.pool_name: build_kv_host_pool( + kv_pool=root.decl.device_pool, + page_size=config.transfer_page_size, + mtp_draft_device_pools=root.packed_draft_device_pools, ) - entries = [ - build_pool_entry( - name=PoolName.KV, - host_pool=kv_host_pool, - device_pool=kv_pool, - layer_mapping=full_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools), - is_anchor=True, - packed_draft_device_pools=mtp_draft_device_pools, - ), - build_pool_entry( - name=sidecar_pool_name, - host_pool=sidecar_host_pool, - device_pool=kv_pool, - layer_mapping=full_layer_mapping, - transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools), - packed_draft_device_pools=mtp_draft_device_pools, - ), - ] - host_pool_group = HostPoolGroup(entries) - cache_controller = HybridCacheController( - params.token_to_kv_pool_allocator, - host_pool_group, - params.page_size, - params.tp_cache_group, - load_cache_event=load_cache_event, - attn_cp_group=params.attn_cp_cache_group, - attn_tp_group=params.attn_tp_cache_group, - pp_group=params.pp_cache_group, - write_policy=get_memory().hicache_write_policy, - io_backend=get_memory().hicache_io_backend, - storage_backend=storage_backend, - prefetch_threshold=prefetch_threshold, - model_name=model_name, - storage_backend_extra_config=storage_backend_extra_config, - transfer_layer_id_max=transfer_layer_id_max, - enable_storage_metrics=enable_storage_metrics, - host_memory_mode=get_memory().hicache_host_memory_mode, - ) - return host_pool_group, cache_controller + } + entries = [] + for pool_config in config.pools: + decl = pool_config.decl + if not decl.is_layout_root: + host_pools[decl.pool_name] = decl.host_pool_builder.build( + decl=decl, + anchor_host=host_pools[decl.layout_source], + allocator_type=_get_allocator_type(), + packed_draft_device_pools=pool_config.packed_draft_device_pools, + ) + entries.append( + build_pool_entry( + name=decl.pool_name, + host_pool=host_pools[decl.pool_name], + device_pool=decl.device_pool, + layer_mapping=pool_config.layer_mapping, + transfer_layer_id_max=pool_config.transfer_layer_id_max, + is_anchor=decl.is_primary, + packed_draft_device_pools=pool_config.packed_draft_device_pools, + ) + ) + return HostPoolGroup(entries) def _build_mha_mla_host_pool( @@ -1504,18 +1466,20 @@ def build_full_draft_pools( ] if isinstance(pool, DSATokenToKVPool) and pool.index_k_with_scale_buffer: + from sglang.srt.mem_cache.pool_host.dsa import ( + DSAIndexerPoolHost, + make_dsa_indexer_pool_decl, + ) + + # Separate draft indexer: its own host pool laid out on the draft KV + # host pool, but transfer indices still follow the target KV anchor. + indexer_decl = make_dsa_indexer_pool_decl(pool, name=PoolName.DRAFT_INDEXER) indexer_host_pool = DSAIndexerPoolHost( - pool, - draft_host_pool, - get_memory().hicache_mem_layout, + decl=indexer_decl, + anchor_host=draft_host_pool, allocator_type=_get_allocator_type(), ) - specs.append( - SidecarPoolSpec( - pool_name=PoolName.DRAFT_INDEXER, - indices_from_pool=PoolName.KV, - ) - ) + specs.append(indexer_decl.sidecar_spec()) entries.append( build_pool_entry( name=PoolName.DRAFT_INDEXER, @@ -1626,6 +1590,7 @@ class StackBuildResult: # layer_transfer_counter has to be wired separately. register_req_to_token_counter: bool = False pools_desc: str = "" + pool_declarations: Optional[tuple[HostPoolDecl, ...]] = None class StackStrategy: @@ -1976,78 +1941,6 @@ class _MambaSwaStrategy(StackStrategy): ) -class _DsaStrategy(StackStrategy): - def matches(self, kvcache, components): - from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool - - return isinstance(kvcache, DSATokenToKVPool) and components == { - ComponentType.FULL - } - - def build_direct_linker_pool_group(self, *, kvcache, params, page_size): - from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import ( - _build_dsa_device_pool_group, - ) - - return _build_dsa_device_pool_group( - kvcache, page_size, params.mtp_draft_device_pools - ) - - def build( - self, - *, - cache, - kvcache, - params, - server_args, - load_cache_event, - storage_backend=None, - storage_backend_extra_config=None, - prefetch_threshold=256, - model_name=None, - enable_storage_metrics=False, - ): - from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool - - full_kv_pool = kvcache - use_mla = isinstance(kvcache, MLATokenToKVPool) - full_layer_mapping = {i: i for i in range(full_kv_pool.layer_num)} - host_pool_group, cache_controller = build_anchor_sidecar_stack( - params=params, - kv_pool=full_kv_pool, - sidecar_pool_name=PoolName.INDEXER, - full_layer_mapping=full_layer_mapping, - load_cache_event=load_cache_event, - storage_backend=storage_backend, - use_mla=use_mla, - override_kv_cache_dim=full_kv_pool.kv_cache_dim, - sidecar_host_pool_factory=lambda kv_host_pool: DSAIndexerPoolHost( - full_kv_pool, - kv_host_pool, - get_memory().hicache_mem_layout, - allocator_type=_get_allocator_type(), - ), - prefetch_threshold=prefetch_threshold, - model_name=model_name, - storage_backend_extra_config=storage_backend_extra_config, - enable_storage_metrics=enable_storage_metrics, - ) - return StackBuildResult( - host_pool_group=host_pool_group, - cache_controller=cache_controller, - component_host_pools={ - ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), - }, - sidecars=[ - SidecarPoolSpec( - pool_name=PoolName.INDEXER, - indices_from_pool=PoolName.KV, - ), - ], - pools_desc="KV + INDEXER", - ) - - class _MiniMaxSparseStrategy(StackStrategy): def matches(self, kvcache, components): from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool @@ -2165,6 +2058,79 @@ class _PlainKvStrategy(StackStrategy): ) +class _DsaStrategy(StackStrategy): + def matches(self, kvcache, components): + from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool + + return isinstance(kvcache, DSATokenToKVPool) and components == { + ComponentType.FULL + } + + def build_direct_linker_pool_group(self, *, kvcache, params, page_size): + from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import ( + _build_dsa_device_pool_group, + ) + + return _build_dsa_device_pool_group( + kvcache, page_size, params.mtp_draft_device_pools + ) + + def build( + self, + *, + cache, + kvcache, + params, + server_args, + load_cache_event, + storage_backend=None, + storage_backend_extra_config=None, + prefetch_threshold=256, + model_name=None, + enable_storage_metrics=False, + ): + config = prepare_host_pool_config( + decls=kvcache.host_pool_decls(), + full_layer_mapping={i: i for i in range(kvcache.layer_num)}, + transfer_layer_id_max=kvcache.layer_num, + transfer_page_size=params.page_size, + packed_draft_device_pools=params.mtp_draft_device_pools, + ) + host_pool_group = build_host_pool_group(config=config) + cache_controller = HybridCacheController( + params.token_to_kv_pool_allocator, + host_pool_group, + config.transfer_page_size, + params.tp_cache_group, + load_cache_event=load_cache_event, + attn_cp_group=params.attn_cp_cache_group, + attn_tp_group=params.attn_tp_cache_group, + pp_group=params.pp_cache_group, + write_policy=get_memory().hicache_write_policy, + io_backend=get_memory().hicache_io_backend, + storage_backend=storage_backend, + prefetch_threshold=prefetch_threshold, + model_name=model_name, + storage_backend_extra_config=storage_backend_extra_config, + # The controller adds packed draft transfers after target layers. + transfer_layer_id_max=kvcache.layer_num, + enable_storage_metrics=enable_storage_metrics, + host_memory_mode=get_memory().hicache_host_memory_mode, + ) + return StackBuildResult( + host_pool_group=host_pool_group, + cache_controller=cache_controller, + component_host_pools={ + ComponentType.FULL: host_pool_group.get_pool(PoolName.KV), + }, + sidecars=[ + c.decl.sidecar_spec() for c in config.pools if not c.decl.is_primary + ], + pools_desc=" + ".join(c.decl.pool_name.value.upper() for c in config.pools), + pool_declarations=tuple(c.decl for c in config.pools), + ) + + # Resolved first-to-last; _PlainKvStrategy is the catch-all fallback. _STRATEGIES: list[StackStrategy] = [ _DeepSeekV4Strategy(), @@ -2193,6 +2159,33 @@ def _select_strategy(kvcache: Any, components: set[ComponentType]) -> StackStrat ) +# Strategies that assemble from host_pool_decls(), so every declared pool has an +# entry by construction. A miss here is a bug, not an unsupported combination. +_DECLARATION_VERIFIED_STRATEGIES: tuple[type, ...] = (_DsaStrategy,) + + +def _check_declared_pools_present( + *, kvcache: Any, result: StackBuildResult, strategy: StackStrategy +) -> None: + entries = result.host_pool_group.entry_map + decls = result.pool_declarations + if decls is None: + decls = kvcache.host_pool_decls() + missing = [d.pool_name.value for d in decls if d.pool_name not in entries] + if not missing: + return + msg = ( + f"{type(kvcache).__name__} declares host pools {missing} that " + f"{type(strategy).__name__} did not assemble" + ) + if isinstance(strategy, _DECLARATION_VERIFIED_STRATEGIES): + raise ValueError(msg) + # Pre-declaration strategies: restoring KV without these pools (e.g. the + # indexer's key buffers) corrupts sparse attention after a host hit. + # Loud until the path is migrated. + logger.error("%s; host restore of these pools is unsupported on this path", msg) + + def _apply_stack_result( cache: UnifiedRadixCache, kvcache: Any, @@ -2250,6 +2243,7 @@ def attach_hybrid_pool_to_unified_cache( model_name=get_serving().served_model_name, enable_storage_metrics=cache._enable_metrics_flag, ) + _check_declared_pools_present(kvcache=kvcache, result=result, strategy=strategy) _apply_stack_result(cache, kvcache, params, result) except Exception: logger.exception("attach_hybrid_pool_to_unified_cache failed") diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 7e6151c890..a39d8a62d7 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -1874,6 +1874,17 @@ class KVCache(abc.ABC): maybe_init_custom_mem_pool(device=self.device) ) + def host_pool_decls(self): + """Host pools HiCache keeps for the buffers this device pool (and its + sub-pools) owns: the KV pool itself, plus dependent pools such as sparse + index keys in subclasses. Not a description of the whole model: buffers + owned elsewhere (e.g. Mamba state in req_to_token_pool) are declared by + their owner or assembled by the existing strategy paths.""" + # pool_host imports this module. Resolve the declaration types lazily. + from sglang.srt.mem_cache.pool_host.host_pool_decl import make_kv_pool_decl + + return (make_kv_pool_decl(self),) + def _finalize_allocation_log(self, num_tokens: int): """Common logging and mem_usage computation for KV cache allocation. Supports both tuple (K, V) size returns and single KV size returns. @@ -4014,6 +4025,12 @@ class HybridLinearKVPool(KVCache): k_size, v_size = self.get_kv_size_bytes() self.mem_usage = (k_size + v_size) / GB + def host_pool_decls(self): + # Only the full-attention sub-pool owns HiCache-addressable buffers here. + # Mamba state belongs to req_to_token_pool's MambaPool and is assembled + # by _MambaStrategy, not declared through this pool. + return self.full_kv_pool.host_pool_decls() + @property def post_capture_active(self) -> bool: return self.full_kv_pool.post_capture_active @@ -4966,6 +4983,17 @@ class DSATokenToKVPool(MLATokenToKVPool): def _should_allocate_index_layer(self, local_layer_idx: int) -> bool: return not self.skip_topk_layers[local_layer_idx] + def host_pool_decls(self): + # pool_host imports this module. Resolve the mirror side lazily. + from sglang.srt.mem_cache.pool_host.dsa import make_dsa_indexer_pool_decl + + kv_decls = super().host_pool_decls() + # Shared-topk layers own a 0-row placeholder, so a non-empty buffer list + # is not enough: some layer must actually hold index keys. + if not self.index_k_with_scale_buffer or all(self.skip_topk_layers): + return kv_decls + return (*kv_decls, make_dsa_indexer_pool_decl(self)) + @property def index_k_with_scale_buffer(self): # Preserve direct HiCache access while storage lives behind the facade. diff --git a/python/sglang/srt/mem_cache/pool_host/dsa.py b/python/sglang/srt/mem_cache/pool_host/dsa.py index e6d3d62413..05a733f5bb 100644 --- a/python/sglang/srt/mem_cache/pool_host/dsa.py +++ b/python/sglang/srt/mem_cache/pool_host/dsa.py @@ -15,6 +15,7 @@ from sglang.kernels.ops.kvcache.hicache import ( from sglang.kernels.ops.kvcache.hicache import ( transfer_hicache_all_layer_mla_staged_lf_pf as jit_transfer_hicache_all_layer_mla_staged_lf_pf, ) +from sglang.srt.mem_cache.hicache_storage import PoolName from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool from sglang.srt.mem_cache.pool_host.base import ( _WRITE_BACK_STAGING_PAGE_CHUNK, @@ -26,6 +27,10 @@ from sglang.srt.mem_cache.pool_host.common import ( get_allocator_from_storage, make_kernel_ptr_table, ) +from sglang.srt.mem_cache.pool_host.host_pool_decl import ( + HostPoolDecl, + HostPoolStorageInfo, +) from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu _is_cuda = is_cuda() @@ -47,6 +52,73 @@ if _is_cuda or _is_hip: logger = logging.getLogger(__name__) +def dsa_indexer_bytes_per_token_per_layer( + *, index_head_dim: int, quant_block_size: int +) -> int: + # packed index keys plus one fp32 scale per quant block, stored as uint8 + elems = index_head_dim + index_head_dim // quant_block_size * 4 + return elems * DSATokenToKVPool.index_k_with_scale_buffer_dtype.itemsize + + +class DSAIndexerHostPoolBuilder: + def validate( + self, + *, + decl: HostPoolDecl, + transfer_page_size: int, + packed_draft_device_pools: tuple[DSATokenToKVPool, ...], + ) -> None: + target = decl.device_pool + target_format = (target.index_head_dim, target.quant_block_size) + for draft in packed_draft_device_pools: + draft_format = (draft.index_head_dim, draft.quant_block_size) + if draft_format != target_format: + raise ValueError( + f"{decl.pool_name}: packed index key format {draft_format} " + f"differs from target {target_format}" + ) + + def build( + self, + *, + decl: HostPoolDecl, + anchor_host: MLATokenToKVPoolHost, + allocator_type: str, + packed_draft_device_pools: tuple[DSATokenToKVPool, ...], + ) -> DSAIndexerPoolHost: + return DSAIndexerPoolHost( + decl=decl, + anchor_host=anchor_host, + packed_draft_device_pools=packed_draft_device_pools, + allocator_type=allocator_type, + ) + + +def make_dsa_indexer_pool_decl( + pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER +) -> HostPoolDecl: + """Index key buffers riding on the full-KV pages: indices and layout both follow KV.""" + return HostPoolDecl( + pool_name=name, + device_pool=pool, + indices_from_pool=PoolName.KV, + layout_source=PoolName.KV, + storage_info=HostPoolStorageInfo( + bytes_per_token_per_layer=dsa_indexer_bytes_per_token_per_layer( + index_head_dim=pool.index_head_dim, + quant_block_size=pool.quant_block_size, + ), + dtype=DSATokenToKVPool.index_k_with_scale_buffer_dtype, + ), + host_pool_builder=DSAIndexerHostPoolBuilder(), + # Shared-topk layers own a 0-row placeholder buffer. They get no host + # layer and must not reach the transfer kernels. + owned_device_layers=tuple( + i for i, skip in enumerate(pool.skip_topk_layers) if not skip + ), + ) + + class DSAIndexerPoolHost(HostKVCache): """Host-side DSA index buffers only. Slot layout matches the anchor MLA host pool.""" @@ -54,46 +126,53 @@ class DSAIndexerPoolHost(HostKVCache): def __init__( self, - device_pool: DSATokenToKVPool, + decl: HostPoolDecl, anchor_host: MLATokenToKVPoolHost, - layout: str, + *, + packed_draft_device_pools: tuple[DSATokenToKVPool, ...] = (), pin_memory: bool = True, device: str = "cpu", allocator_type: str = "default", is_dummy: bool = False, ): self._is_dummy = is_dummy + self.decl = decl + storage_info = decl.storage_info + device_pool = decl.device_pool self.device_pool = device_pool self.page_size = anchor_host.page_size - self.layout = layout + self.layout = anchor_host.layout self.pin_memory = pin_memory self.device = device self.allocator = get_allocator_from_storage(allocator_type) self.dtype = device_pool.store_dtype self.start_layer = device_pool.start_layer self.end_layer = device_pool.end_layer - self.target_layer_num = self._effective_host_layer_num() - self.mtp_draft_device_pools = anchor_host.mtp_draft_device_pools + # Host layers are compact: only owned device layers that hold index + # buffers, then one tail layer per packed draft pool. + owned_start, owned_end = self._device_owned_layer_range() + declared = decl.owned_device_layers + self._live_target_layers = [ + layer + for layer in range(owned_start, owned_end) + if declared is None or layer in declared + ] + self._device_to_host_layer = { + layer: i for i, layer in enumerate(self._live_target_layers) + } + self.target_layer_num = len(self._live_target_layers) + self.mtp_draft_device_pools = tuple(packed_draft_device_pools) self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools) - self.index_head_dim = device_pool.index_head_dim - self.indexer_quant_block_size = device_pool.quant_block_size - self.indexer_dtype = DSATokenToKVPool.index_k_with_scale_buffer_dtype - self.indexer_size_per_token = ( - self.index_head_dim - + self.index_head_dim // self.indexer_quant_block_size * 4 - ) + self.indexer_dtype = storage_info.dtype self.size = anchor_host.size self.page_num = anchor_host.page_num - self.indexer_page_stride_size = ( - self.indexer_size_per_token * self.page_size * self.indexer_dtype.itemsize - ) + # uint8 storage, so element counts below are byte counts + self.indexer_page_stride_size = storage_info.page_bytes(self.page_size) self.indexer_layout_dim = self.indexer_page_stride_size * self.layer_num self.indexer_page_num = (self.size + self.page_size + 1) // self.page_size - self.size_per_token = ( - self.indexer_size_per_token * self.layer_num * self.indexer_dtype.itemsize - ) + self.size_per_token = storage_info.bytes_per_token_per_layer * self.layer_num self.can_use_jit = False self.can_use_write_back_jit = False @@ -109,8 +188,9 @@ class DSAIndexerPoolHost(HostKVCache): self.clear() return - buf_elem_size = self.page_num * self.layer_num * self.indexer_page_stride_size - requested_bytes = buf_elem_size * self.indexer_dtype.itemsize + requested_bytes = storage_info.host_bytes( + page_num=self.page_num, layer_num=self.layer_num, page_size=self.page_size + ) available_bytes = host_memory_budget_bytes(requested_bytes) if requested_bytes > available_bytes: raise ValueError( @@ -125,7 +205,7 @@ class DSAIndexerPoolHost(HostKVCache): "packed MTP layers: " "target_layers=%d, draft_layers=%d, total_layers=%d.", requested_bytes / 1e9, - layout, + self.layout, self.target_layer_num, draft_layer_num, self.layer_num, @@ -134,7 +214,7 @@ class DSAIndexerPoolHost(HostKVCache): logger.info( "Allocating %.2f GB host memory for DSA indexer (layout=%s).", requested_bytes / 1e9, - layout, + self.layout, ) self.init_kv_buffer() self._init_write_back_staging_buffers() @@ -142,18 +222,33 @@ class DSAIndexerPoolHost(HostKVCache): self.clear() def get_size_per_token(self): - return ( - self.indexer_size_per_token * self.layer_num * self.indexer_dtype.itemsize - ) + return self.decl.storage_info.bytes_per_token_per_layer * self.layer_num def get_ksize_per_token(self): return self.get_size_per_token() + def _is_device_layer_owned(self, device_pool, layer_id: int) -> bool: + return layer_id in self._device_to_host_layer + + def _host_layer_index(self, layer_id: int, device_pool=None) -> int: + return self._device_to_host_layer[layer_id] + + def _owned_device_layer_ids(self, device_pool) -> list[int]: + return list(self._live_target_layers) + + def _draft_host_layer(self, layer_id: int) -> int: + # The controller hands packed drafts ``target_device_layer_num + depth``. + return self.target_layer_num + (layer_id - self.device_pool.layer_num) + def init_kv_buffer(self): alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device] - device_pools = (self.device_pool, *self.mtp_draft_device_pools) self.packed_device_index_buffers = [ - buffer for pool in device_pools for buffer in pool.index_k_with_scale_buffer + self.device_pool.index_k_with_scale_buffer[layer] + for layer in self._live_target_layers + ] + [ + buffer + for pool in self.mtp_draft_device_pools + for buffer in pool.index_k_with_scale_buffer ] self.index_k_device_ptrs = torch.tensor( [x.data_ptr() for x in self.packed_device_index_buffers], @@ -249,7 +344,11 @@ class DSAIndexerPoolHost(HostKVCache): "load on a dummy (non-src DSA) host pool" ) # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + host_layer_id = ( + self._draft_host_layer(layer_id) + if is_draft + else self._host_layer_index(layer_id) + ) device_layer_id = 0 if is_draft else layer_id host_page_indices, device_page_indices = self._get_indexer_page_indices( @@ -314,7 +413,11 @@ class DSAIndexerPoolHost(HostKVCache): "backup on a dummy (non-src DSA) host pool" ) # MTP draft layers do not participate in CP layer sharding. - host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id) + host_layer_id = ( + self._draft_host_layer(layer_id) + if is_draft + else self._host_layer_index(layer_id) + ) device_layer_id = 0 if is_draft else layer_id host_page_indices, device_page_indices = self._get_indexer_page_indices( diff --git a/python/sglang/srt/mem_cache/pool_host/host_pool_decl.py b/python/sglang/srt/mem_cache/pool_host/host_pool_decl.py new file mode 100644 index 0000000000..85d9f63299 --- /dev/null +++ b/python/sglang/srt/mem_cache/pool_host/host_pool_decl.py @@ -0,0 +1,112 @@ +"""Declarations of the host pools a device pool needs HiCache to keep. + +A device pool declares them (``HostPoolDecl``) through ``host_pool_decls()``. +The assembler (hybrid_pool_assembler) binds each declaration to transfer layers +and builds one entry per declaration. This module holds only the declaration +types and their basic constructors. +""" + +from __future__ import annotations + +from typing import Any, Optional, Protocol + +import msgspec +import torch + +from sglang.srt.mem_cache.hicache_storage import ( + PoolHitPolicy, + PoolName, + SidecarPoolSpec, +) + + +class HostPoolStorageInfo(msgspec.Struct, frozen=True, kw_only=True): + """Bytes one host pool stores per token per layer, and their dtype. + + Two equal values mean equal byte counts and dtype only. Page geometry, + scale placement and kernel layout are checked by the host pool builder. + """ + + bytes_per_token_per_layer: int + dtype: torch.dtype + + def page_bytes(self, page_size: int) -> int: + """Bytes of one page of one layer.""" + return self.bytes_per_token_per_layer * page_size + + def host_bytes(self, *, page_num: int, layer_num: int, page_size: int) -> int: + return page_num * layer_num * self.page_bytes(page_size) + + +class HostPoolBuilder(Protocol): + def validate( + self, + *, + decl: HostPoolDecl, + # Original-token slots, not the number of physical rows in this pool. + transfer_page_size: int, + packed_draft_device_pools: tuple[Any, ...], + ) -> None: ... + + def build( + self, + *, + decl: HostPoolDecl, + anchor_host: Any, + allocator_type: str, + packed_draft_device_pools: tuple[Any, ...], + ) -> Any: ... + + +class HostPoolDecl(msgspec.Struct, frozen=True, kw_only=True): + """One host pool a device pool asks HiCache to keep. Pool-intrinsic: no layer binding.""" + + pool_name: PoolName + # The pool object that owns the device buffers behind this host pool. A + # composite pool keeps its KV buffers on a sub-pool and, e.g., its + # compressed index keys on itself, so each declaration names its owner. + device_pool: Any + # Whose host page indices this pool reuses when transferred. None: primary. + indices_from_pool: Optional[PoolName] + # Whose host pool decides this pool's capacity and memory layout. None: self. + layout_source: Optional[PoolName] + # None for a layout root (KV, DRAFT): the assembler builds its host pool + # from the device pool, so no byte facts are declared for it. + storage_info: Optional[HostPoolStorageInfo] + host_pool_builder: Optional[HostPoolBuilder] + hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES + # Local device layers that own buffers for this pool. None means every + # layer. Layers outside it get no host layer and no transfer. + owned_device_layers: Optional[tuple[int, ...]] = None + + @property + def is_primary(self) -> bool: + return self.indices_from_pool is None + + @property + def is_layout_root(self) -> bool: + return self.layout_source is None + + def sidecar_spec(self) -> SidecarPoolSpec: + if self.indices_from_pool is None: + raise ValueError( + f"{self.pool_name} is the primary pool and has no sidecar spec" + ) + return SidecarPoolSpec( + pool_name=self.pool_name, + indices_from_pool=self.indices_from_pool, + hit_policy=self.hit_policy, + ) + + +def make_kv_pool_decl(pool: Any) -> HostPoolDecl: + """The primary KV pool every device pool declares. The assembler builds its + host pool, so no storage info is declared here.""" + return HostPoolDecl( + pool_name=PoolName.KV, + device_pool=pool, + indices_from_pool=None, + layout_source=None, + storage_info=None, + host_pool_builder=None, + ) diff --git a/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py b/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py index cb8d5b0f0e..1b025f21bb 100644 --- a/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py +++ b/test/registered/unit/mem_cache/test_dsa_pool_host_unit.py @@ -8,7 +8,10 @@ from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, alloc_with_pin_memory, ) -from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost +from sglang.srt.mem_cache.pool_host.dsa import ( + DSAIndexerPoolHost, + make_dsa_indexer_pool_decl, +) from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -81,9 +84,8 @@ class TestDSAHiCacheTransfer(unittest.TestCase): override_kv_cache_dim=device_pool.kv_cache_dim, ) indexer_host = DSAIndexerPoolHost( - device_pool=device_pool, + decl=make_dsa_indexer_pool_decl(device_pool), anchor_host=mla_host, - layout="layer_first", pin_memory=pin_memory, device="cpu", allocator_type="default", diff --git a/test/registered/unit/mem_cache/test_hicache_host_register.py b/test/registered/unit/mem_cache/test_hicache_host_register.py index e12f30cd19..a7f74cbc13 100644 --- a/test/registered/unit/mem_cache/test_hicache_host_register.py +++ b/test/registered/unit/mem_cache/test_hicache_host_register.py @@ -80,6 +80,7 @@ class TestHiCacheHostRegister(unittest.TestCase): SimpleNamespace(index_k_with_scale_buffer=[draft_buffer]) ] host.layout = layout + host._live_target_layers = [0, 1, 2] host.layer_num = 4 host.indexer_page_num = 3 host.indexer_page_stride_size = 512 diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index 505c10939a..01fb49a4fa 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -1031,6 +1031,8 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): host.layout = "page_first" host.page_size = page_size host.layer_num = layer_num + host._live_target_layers = list(range(layer_num)) + host._device_to_host_layer = {layer: layer for layer in range(layer_num)} host.indexer_page_stride_size = indexer_page_stride_size host.indexer_layout_dim = host.layer_num * host.indexer_page_stride_size host.index_k_device_ptrs = torch.tensor( diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index eafba69166..7e970d0d9d 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -4,11 +4,19 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch +import msgspec import torch from sglang.srt.mem_cache.base_prefix_cache import EvictParams +from sglang.srt.mem_cache.hicache_storage import PoolName from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler +from sglang.srt.mem_cache.hybrid_cache.host_pool_config import ( + prepare_host_pool_config, +) from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( + StackBuildResult, + _check_declared_pools_present, + _DsaStrategy, _evict_mamba_for_device_alloc, _evict_swa_for_device_alloc, _MambaStrategy, @@ -17,9 +25,15 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( _split_hicache_size, _SwaStrategy, build_full_draft_pools, + build_host_pool_group, build_hybrid_swa_group, ) -from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool +from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool, HybridLinearKVPool +from sglang.srt.mem_cache.pool_host import dsa as pool_host_dsa +from sglang.srt.mem_cache.pool_host.host_pool_decl import ( + HostPoolDecl, + make_kv_pool_decl, +) from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools from sglang.test.ci.ci_register import register_cpu_ci @@ -145,6 +159,7 @@ class TestHybridStageLayerMappings(CustomTestCase): full_kv_pool=object(), swa_kv_pool=object(), use_mla=False, + host_pool_decls=lambda: (), ) req_pool = SimpleNamespace( mamba_map=global_maps.get("mamba", {}).copy(), @@ -188,7 +203,9 @@ class TestHybridStageLayerMappings(CustomTestCase): class TestDraftSidecarPoolDispatch(CustomTestCase): def test_full_builder_unwraps_empty_hybrid_linear_pool(self): draft_kv_pool = object.__new__(HybridLinearKVPool) - draft_kv_pool.full_kv_pool = SimpleNamespace(layer_num=0) + full = SimpleNamespace(layer_num=0) + full.host_pool_decls = lambda: (make_kv_pool_decl(full),) + draft_kv_pool.full_kv_pool = full specs, entries = build_full_draft_pools( draft_kv_pool=draft_kv_pool, @@ -200,6 +217,7 @@ class TestDraftSidecarPoolDispatch(CustomTestCase): def test_full_builder_sizes_sidecar_for_anchor_logical_space(self): draft_kv_pool = SimpleNamespace(layer_num=1, size=800) + draft_kv_pool.host_pool_decls = lambda: (make_kv_pool_decl(draft_kv_pool),) draft_host_pool = SimpleNamespace(layer_num=1) tree_cache = SimpleNamespace( cache_controller=SimpleNamespace( @@ -236,6 +254,679 @@ class TestDraftSidecarPoolDispatch(CustomTestCase): self.assertEqual(len(specs), 1) self.assertIs(entries[0].host_pool, draft_host_pool) + def test_full_builder_registers_separate_dsa_draft_indexer(self): + """The separate-draft DSA branch must build its indexer mirror from a + DRAFT_INDEXER decl. A constructor change that skips this call site + breaks only here, not on the target path.""" + draft_kv_pool = object.__new__(DSATokenToKVPool) + draft_kv_pool.layer_num = 1 + draft_kv_pool.size = 800 + draft_kv_pool.index_head_dim = 128 + draft_kv_pool.skip_topk_layers = [False] + draft_kv_pool.index_key_cache = SimpleNamespace(buffer=[object()]) + draft_host_pool = SimpleNamespace(layer_num=1) + tree_cache = SimpleNamespace( + cache_controller=SimpleNamespace( + mem_pool_host=SimpleNamespace(size=100, logical_size=800), + page_size=512, + ) + ) + from sglang.srt.runtime_context import publish, reset_context + from sglang.srt.server_args import ServerArgs + + publish( + ServerArgs(model_path="dummy", hicache_mem_layout="page_first"), + role="scheduler", + ) + self.addCleanup(reset_context) + + seen = {} + + def fake_indexer_host( + decl, anchor_host, *, allocator_type, packed_draft_device_pools=() + ): + self.assertIsInstance(decl, HostPoolDecl) + self.assertIs(decl.device_pool, draft_kv_pool) + self.assertIs(anchor_host, draft_host_pool) + self.assertEqual(packed_draft_device_pools, ()) + seen["decl"] = decl + return SimpleNamespace(layer_num=1) + + with ( + patch.object( + hybrid_pool_assembler, + "_build_mha_mla_host_pool", + return_value=draft_host_pool, + ), + patch.object( + hybrid_pool_assembler, "_get_allocator_type", return_value="default" + ), + patch.object(pool_host_dsa, "DSAIndexerPoolHost", fake_indexer_host), + ): + specs, entries = build_full_draft_pools( + draft_kv_pool=draft_kv_pool, + tree_cache=tree_cache, + ) + + self.assertEqual(seen["decl"].pool_name, PoolName.DRAFT_INDEXER) + self.assertEqual(seen["decl"].indices_from_pool, PoolName.KV) + self.assertEqual(specs[1], seen["decl"].sidecar_spec()) + self.assertEqual(entries[1].name, PoolName.DRAFT_INDEXER) + + +def _dsa_pool_stub(*, layer_num: int, size: int = 4096, shard: tuple | None = None): + """DSATokenToKVPool shape without CUDA. ``shard=(rank, size)`` marks the pool + layer-sharded with rank owning a contiguous local layer range.""" + pool = object.__new__(DSATokenToKVPool) + pool.layer_num = layer_num + pool.size = size + pool.start_layer = 0 + pool.end_layer = layer_num - 1 + pool.store_dtype = torch.bfloat16 + pool.kv_lora_rank = 512 + pool.qk_rope_head_dim = 64 + pool.kv_cache_dim = 576 + pool.index_head_dim = 128 + pool.skip_topk_layers = [False] * layer_num + pool.index_key_cache = SimpleNamespace(buffer=[object()] * layer_num) + pool.layer_shard_enabled = shard is not None + if shard is not None: + from sglang.srt.layers.cp.utils import get_layer_shard_range + + rank, shard_size = shard + pool.layer_shard_size = shard_size + pool._owned_local_layer_range = lambda: get_layer_shard_range( + rank, shard_size, layer_num + ) + return pool + + +def _kv_pool_stub(*, layer_num: int, size: int = 4096): + """A plain MLA KV pool without CUDA: declares KV only.""" + from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool + + pool = object.__new__(MLATokenToKVPool) + pool.layer_num = layer_num + pool.size = size + pool.start_layer = 0 + pool.end_layer = layer_num - 1 + pool.store_dtype = torch.bfloat16 + pool.kv_lora_rank = 512 + pool.qk_rope_head_dim = 64 + pool.kv_cache_dim = 576 + pool.layer_shard_enabled = False + return pool + + +def _mirror_shape(host): + shape = { + "size": host.size, + "page_num": host.page_num, + "layer_num": host.layer_num, + "size_per_token": host.size_per_token, + "layout": host.layout, + } + if isinstance(host, pool_host_dsa.DSAIndexerPoolHost): + shape["indexer_page_stride_size"] = host.indexer_page_stride_size + shape["indexer_layout_dim"] = host.indexer_layout_dim + return shape + + +def _entry_shape(group, transfer_layer_num): + """Everything the controller reads from a HostPoolGroup, in comparable form.""" + return [ + ( + entry.name, + entry.is_primary_index_anchor, + id(entry.device_pool), + tuple(id(p) for p in entry.packed_draft_device_pools), + tuple(entry.layer_mapper(i) for i in range(-1, transfer_layer_num + 2)), + _mirror_shape(entry.host_pool), + ) + for entry in group.entries + ] + + +def _target_params(drafts=(), page_size=64): + return SimpleNamespace( + page_size=page_size, + mtp_draft_device_pools=tuple(drafts), + token_to_kv_pool_allocator=None, + tp_cache_group=None, + attn_cp_cache_group=None, + attn_tp_cache_group=None, + pp_cache_group=None, + ) + + +# Mirror geometry for the 4096-token stubs at page 64 and host ratio 2: +# 8256 host tokens (2 x 4096 plus one reserve page) = 129 pages. The MLA row is +# kv_cache_dim 576 x bf16 = 1152 B per layer. The DSA index row is 132 B per +# token, 8448 B per page, per layer. +_HOST_SIZE, _HOST_PAGES, _KV_ROW, _INDEX_ROW, _INDEX_PAGE = 8256, 129, 1152, 132, 8448 + + +def _kv_shape(layers): + return { + "size": _HOST_SIZE, + "page_num": _HOST_PAGES, + "layer_num": layers, + "size_per_token": _KV_ROW * layers, + "layout": "page_first", + } + + +def _indexer_shape(layers): + return { + **_kv_shape(layers), + "size_per_token": _INDEX_ROW * layers, + "indexer_page_stride_size": _INDEX_PAGE, + "indexer_layout_dim": _INDEX_PAGE * layers, + } + + +class TestDeclaredStackStructure(CustomTestCase): + """Everything the controller reads from a declared target stack, pinned + against the pre-declaration DSA and plain-KV assembly it replaced: entry + order, anchor, device owners, layer mapper over every transfer layer, + packed drafts, mirror geometry, controller arguments, sidecars.""" + + def _run(self, *, pool, drafts, full_layer_mapping, page_size=64): + from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost + + real_indexer_host = pool_host_dsa.DSAIndexerPoolHost + + def dummy_kv_host(**kwargs): + return MLATokenToKVPoolHost( + kwargs["kv_pool"], + host_to_device_ratio=2, + host_size=0, + page_size=kwargs["page_size"], + layout="page_first", + pin_memory=False, + is_dummy=True, + override_kv_cache_dim=kwargs["kv_pool"].kv_cache_dim, + mtp_draft_device_pools=kwargs["mtp_draft_device_pools"], + ) + + def dummy_indexer_host( + decl, anchor_host, *, allocator_type, packed_draft_device_pools=() + ): + return real_indexer_host( + decl, + anchor_host, + packed_draft_device_pools=packed_draft_device_pools, + allocator_type=allocator_type, + pin_memory=False, + is_dummy=True, + ) + + with ( + patch.object(hybrid_pool_assembler, "build_kv_host_pool", dummy_kv_host), + patch.object(pool_host_dsa, "DSAIndexerPoolHost", dummy_indexer_host), + patch.object( + hybrid_pool_assembler, "_get_allocator_type", return_value="default" + ), + ): + config = prepare_host_pool_config( + decls=pool.host_pool_decls(), + full_layer_mapping=dict(full_layer_mapping), + transfer_layer_id_max=max(full_layer_mapping) + 1, + transfer_page_size=page_size, + packed_draft_device_pools=drafts, + ) + group = build_host_pool_group(config=config) + return SimpleNamespace( + host_pool_group=group, + sidecars=[ + c.decl.sidecar_spec() for c in config.pools if not c.decl.is_primary + ], + ) + + def test_dsa_target(self): + pool = _dsa_pool_stub(layer_num=3) + stack = self._run(pool=pool, drafts=(), full_layer_mapping={0: 0, 1: 1, 2: 2}) + mapper = (None, 0, 1, 2, None, None) + self.assertEqual( + _entry_shape(stack.host_pool_group, 3), + [ + (PoolName.KV, True, id(pool), (), mapper, _kv_shape(3)), + (PoolName.INDEXER, False, id(pool), (), mapper, _indexer_shape(3)), + ], + ) + self.assertEqual( + stack.sidecars, + [pool_host_dsa.make_dsa_indexer_pool_decl(pool).sidecar_spec()], + ) + + def test_group_uses_prepared_page_size(self): + pool = _dsa_pool_stub(layer_num=1) + stack = self._run( + pool=pool, drafts=(), full_layer_mapping={0: 0}, page_size=128 + ) + for entry in stack.host_pool_group.entries: + self.assertEqual(entry.host_pool.page_size, 128) + indexer = stack.host_pool_group.get_pool(PoolName.INDEXER) + self.assertEqual(indexer.indexer_page_stride_size, 128 * _INDEX_ROW) + + def test_dsa_target_with_packed_draft(self): + pool = _dsa_pool_stub(layer_num=3) + draft = _dsa_pool_stub(layer_num=1) + stack = self._run( + pool=pool, drafts=(draft,), full_layer_mapping={0: 0, 1: 1, 2: 2} + ) + # transfer layer 3 is the draft's tail layer, device layer 3 = target_layer_num + depth + mapper = (None, 0, 1, 2, 3, None, None) + self.assertEqual( + _entry_shape(stack.host_pool_group, 4), + [ + (PoolName.KV, True, id(pool), (id(draft),), mapper, _kv_shape(4)), + ( + PoolName.INDEXER, + False, + id(pool), + (id(draft),), + mapper, + _indexer_shape(4), + ), + ], + ) + + def test_strategy_keeps_target_controller_range_with_packed_draft(self): + pool = _dsa_pool_stub(layer_num=3) + draft = _dsa_pool_stub(layer_num=1) + stack = self._run( + pool=pool, drafts=(draft,), full_layer_mapping={0: 0, 1: 1, 2: 2} + ) + with ( + patch.object( + hybrid_pool_assembler, + "build_host_pool_group", + return_value=stack.host_pool_group, + ) as build, + patch.object(hybrid_pool_assembler, "HybridCacheController") as controller, + patch.object( + hybrid_pool_assembler, + "get_memory", + return_value=SimpleNamespace( + hicache_write_policy="write_through", + hicache_io_backend="kernel", + hicache_host_memory_mode="cache", + ), + ), + ): + result = _DsaStrategy().build( + cache=None, + kvcache=pool, + params=_target_params((draft,)), + server_args=None, + load_cache_event=None, + ) + self.assertEqual(controller.call_args.kwargs["transfer_layer_id_max"], 3) + self.assertEqual(controller.call_args.args[2], 64) + for config in build.call_args.kwargs["config"].pools: + self.assertEqual(config.transfer_layer_id_max, 4) + self.assertEqual(config.layer_mapping, {0: 0, 1: 1, 2: 2, 3: 3}) + self.assertEqual(config.packed_draft_device_pools, (draft,)) + self.assertIs(result.host_pool_group, stack.host_pool_group) + self.assertEqual(result.sidecars, stack.sidecars) + + def test_dsa_target_layer_sharded(self): + # rank 0 of 2 owns local layers 0 and 1 of 3. The permuted stage mapping + # is passed through untouched and the host_pools hold only the owned layers. + pool = _dsa_pool_stub(layer_num=3, shard=(0, 2)) + stack = self._run(pool=pool, drafts=(), full_layer_mapping={0: 1, 1: 2, 2: 0}) + mapper = (None, 1, 2, 0, None, None) + self.assertEqual( + _entry_shape(stack.host_pool_group, 3), + [ + (PoolName.KV, True, id(pool), (), mapper, _kv_shape(2)), + (PoolName.INDEXER, False, id(pool), (), mapper, _indexer_shape(2)), + ], + ) + + def test_plain_mla_target(self): + # A pool that declares KV only builds the anchor-only group the removed + # build_kv_only_stack built, packed draft included. + for drafts in (0, 1): + with self.subTest(drafts=drafts): + pool = _kv_pool_stub(layer_num=3) + ds = tuple(_kv_pool_stub(layer_num=1) for _ in range(drafts)) + stack = self._run( + pool=pool, drafts=ds, full_layer_mapping={0: 0, 1: 1, 2: 2} + ) + mapper = (None, 0, 1, 2, *([3] if drafts else []), None, None) + self.assertEqual( + _entry_shape(stack.host_pool_group, 3 + drafts), + [ + ( + PoolName.KV, + True, + id(pool), + tuple(id(d) for d in ds), + mapper, + _kv_shape(3 + drafts), + ) + ], + ) + self.assertEqual(stack.sidecars, []) + + +class TestPackedDraftPairing(CustomTestCase): + """Packing appends draft layers to the target host_pools, so a draft must + declare every target pool with an identical per-layer layout.""" + + def test_draft_without_indexer_is_rejected(self): + # The pre-declaration filter on ``index_k_with_scale_buffer`` dropped + # such a draft silently, leaving its state without any host mirror. + target = _dsa_pool_stub(layer_num=2) + no_index = _dsa_pool_stub(layer_num=1) + no_index.index_key_cache = SimpleNamespace(buffer=[]) + with self.assertRaisesRegex(ValueError, "draft counterpart"): + prepare_host_pool_config( + decls=target.host_pool_decls(), + full_layer_mapping={i: i for i in range(target.layer_num)}, + transfer_layer_id_max=target.layer_num, + transfer_page_size=64, + packed_draft_device_pools=(no_index,), + ) + + def test_draft_with_empty_index_layers_is_rejected(self): + # A 0-row placeholder layer would be packed as a null device pointer. + target = _dsa_pool_stub(layer_num=2) + shared = _dsa_pool_stub(layer_num=1) + shared.skip_topk_layers = [True] + shared.index_key_cache = SimpleNamespace(buffer=[object()]) + with self.assertRaisesRegex(ValueError, "draft counterpart"): + prepare_host_pool_config( + decls=target.host_pool_decls(), + full_layer_mapping={i: i for i in range(target.layer_num)}, + transfer_layer_id_max=target.layer_num, + transfer_page_size=64, + packed_draft_device_pools=(shared,), + ) + partial = _dsa_pool_stub(layer_num=2) + partial.skip_topk_layers = [False, True] + with self.assertRaisesRegex(ValueError, "owns buffers on 1 of 2"): + prepare_host_pool_config( + decls=target.host_pool_decls(), + full_layer_mapping={i: i for i in range(target.layer_num)}, + transfer_layer_id_max=target.layer_num, + transfer_page_size=64, + packed_draft_device_pools=(partial,), + ) + + def test_storage_info_mismatch_is_rejected(self): + target = _dsa_pool_stub(layer_num=2) + wide = _dsa_pool_stub(layer_num=1) + wide.index_head_dim = 256 + with self.assertRaisesRegex(ValueError, "storage"): + prepare_host_pool_config( + decls=target.host_pool_decls(), + full_layer_mapping={i: i for i in range(target.layer_num)}, + transfer_layer_id_max=target.layer_num, + transfer_page_size=64, + packed_draft_device_pools=(wide,), + ) + + def test_pool_with_only_shared_topk_layers_declares_kv_only(self): + pool = _dsa_pool_stub(layer_num=2) + pool.skip_topk_layers = [True, True] + self.assertEqual([d.pool_name for d in pool.host_pool_decls()], [PoolName.KV]) + + +class TestKvHostPoolRow(CustomTestCase): + """The KV mirror takes its row width from the MLA device pool (an fp8 DSA + store is wider than kv_lora_rank + qk_rope_head_dim) and refuses packed + drafts whose KV rows differ from the target's.""" + + def _build(self, pool, drafts=()): + seen = {} + + def fake_host(kv_pool, ratio, size, page_size, layout, **kwargs): + seen.update(kwargs) + return SimpleNamespace(layer_num=kv_pool.layer_num) + + with ( + patch.object(hybrid_pool_assembler, "MLATokenToKVPoolHost", fake_host), + patch.object( + hybrid_pool_assembler, + "get_parallel", + return_value=SimpleNamespace(dcp_enabled=False), + ), + patch.object( + hybrid_pool_assembler, + "get_memory", + return_value=SimpleNamespace( + hicache_ratio=2, hicache_size=0, hicache_mem_layout="page_first" + ), + ), + patch.object( + hybrid_pool_assembler, "_get_allocator_type", return_value="default" + ), + ): + hybrid_pool_assembler.build_kv_host_pool( + kv_pool=pool, + page_size=64, + mtp_draft_device_pools=drafts, + ) + return seen + + def test_mla_row_width_comes_from_the_device_pool(self): + pool = _dsa_pool_stub(layer_num=2) + pool.kv_cache_dim = 656 # fp8 DSA store: 576 + fp32 scales + self.assertEqual(self._build(pool)["override_kv_cache_dim"], 656) + + def test_packed_draft_with_a_different_kv_row_is_rejected(self): + pool = _dsa_pool_stub(layer_num=2) + draft = _dsa_pool_stub(layer_num=1) + draft.store_dtype = torch.float8_e4m3fn + with self.assertRaisesRegex(ValueError, "KV row"): + self._build(pool, (draft,)) + self._build(pool, (_dsa_pool_stub(layer_num=1),)) + + +class TestDeclaredPoolPlanning(CustomTestCase): + """Sidecar indices resolve from one primary source in HostPoolGroup, so the + planner must reject self-references and sidecar chains up front.""" + + def _prepare(self, decls): + return prepare_host_pool_config( + decls=decls, + full_layer_mapping={0: 0}, + transfer_layer_id_max=1, + transfer_page_size=64, + ) + + def test_rejects_self_referencing_index_source(self): + import msgspec + + kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls() + bad = msgspec.structs.replace(indexer, indices_from_pool=PoolName.INDEXER) + with self.assertRaisesRegex(ValueError, "indices_from_pool"): + self._prepare((kv, bad)) + + def test_rejects_self_referencing_layout_source(self): + import msgspec + + kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls() + bad = msgspec.structs.replace(indexer, layout_source=PoolName.INDEXER) + with self.assertRaisesRegex(ValueError, "layout_source"): + self._prepare((kv, bad)) + + def test_rejects_primary_that_is_not_kv(self): + import msgspec + + kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls() + swa_primary = msgspec.structs.replace(kv, pool_name=PoolName.SWA) + follower = msgspec.structs.replace( + indexer, indices_from_pool=PoolName.SWA, layout_source=PoolName.SWA + ) + with self.assertRaisesRegex(ValueError, "primary KV pool"): + self._prepare((swa_primary, follower)) + + def test_accepts_dsa_declaration(self): + config = self._prepare(_dsa_pool_stub(layer_num=1).host_pool_decls()) + self.assertEqual( + [c.decl.pool_name for c in config.pools], [PoolName.KV, PoolName.INDEXER] + ) + + +class TestHostPoolPreflight(CustomTestCase): + """Malformed declarations must fail before reserving host memory.""" + + def test_invalid_dependencies_do_not_allocate(self): + kv, indexer = _dsa_pool_stub(layer_num=2).host_pool_decls() + cyclic_indexer = msgspec.structs.replace(indexer, layout_source=PoolName.SWA) + cyclic_swa = msgspec.structs.replace( + indexer, pool_name=PoolName.SWA, layout_source=PoolName.INDEXER + ) + cases = [ + ((kv, msgspec.structs.replace(indexer, host_pool_builder=None)), "builder"), + ((kv, msgspec.structs.replace(indexer, storage_info=None)), "storage_info"), + ((kv, cyclic_indexer, cyclic_swa), "cyclic"), + ( + (kv, msgspec.structs.replace(indexer, owned_device_layers=(0, 0))), + "owned_device_layers", + ), + ] + for decls, error in cases: + with ( + self.subTest(error=error), + patch.object(hybrid_pool_assembler, "build_kv_host_pool") as allocate, + ): + with self.assertRaisesRegex(ValueError, error): + config = prepare_host_pool_config( + decls=decls, + full_layer_mapping={0: 0, 1: 1}, + transfer_layer_id_max=2, + transfer_page_size=64, + ) + build_host_pool_group(config=config) + allocate.assert_not_called() + + def test_incompatible_packed_buffers_fail_before_allocation(self): + target = _dsa_pool_stub(layer_num=2) + bad_kv = _dsa_pool_stub(layer_num=1) + bad_kv.store_dtype = torch.float8_e4m3fn + bad_index = _dsa_pool_stub(layer_num=1) + # Same 132 bytes per row, different key/scale format. + bad_index.index_head_dim = 132 + bad_index.quant_block_size = 256 + for draft, error in ((bad_kv, "KV row"), (bad_index, "index key format")): + with ( + self.subTest(error=error), + patch.object(hybrid_pool_assembler, "build_kv_host_pool") as allocate, + ): + with self.assertRaisesRegex(ValueError, error): + _DsaStrategy().build( + cache=None, + kvcache=target, + params=_target_params((draft,)), + server_args=None, + load_cache_event=None, + ) + allocate.assert_not_called() + + def test_draft_declarations_are_collected_once(self): + target = _dsa_pool_stub(layer_num=2) + draft = _dsa_pool_stub(layer_num=1) + draft.host_pool_decls = MagicMock(wraps=draft.host_pool_decls) + decls = target.host_pool_decls() + target_mapping = {0: 0, 4: 1} + config = prepare_host_pool_config( + decls=decls, + full_layer_mapping=target_mapping, + transfer_layer_id_max=5, + transfer_page_size=64, + packed_draft_device_pools=(draft,), + ) + self.assertEqual(target_mapping, {0: 0, 4: 1}) + draft.host_pool_decls.assert_called_once() + self.assertIs(config.pools[1].packed_draft_device_pools[0], draft) + self.assertEqual(config.pools[1].layer_mapping, {0: 0, 4: 1, 5: 2}) + + def test_packed_draft_rejects_duplicate_pool_names(self): + target = _dsa_pool_stub(layer_num=2) + draft = _dsa_pool_stub(layer_num=1) + declarations = draft.host_pool_decls() + draft.host_pool_decls = lambda: (*declarations, declarations[1]) + with self.assertRaisesRegex(ValueError, "duplicate"): + prepare_host_pool_config( + decls=target.host_pool_decls(), + full_layer_mapping={i: i for i in range(target.layer_num)}, + transfer_layer_id_max=target.layer_num, + transfer_page_size=64, + packed_draft_device_pools=(draft,), + ) + + def test_transfer_range_is_not_the_number_of_entries(self): + decls = _dsa_pool_stub(layer_num=2).host_pool_decls() + with self.assertRaisesRegex(ValueError, "transfer layer 4"): + prepare_host_pool_config( + decls=decls, + full_layer_mapping={0: 0, 4: 1}, + transfer_layer_id_max=2, + transfer_page_size=64, + ) + + def test_build_order_resolves_dependencies_before_allocation(self): + kv, indexer = _dsa_pool_stub(layer_num=1).host_pool_decls() + config = prepare_host_pool_config( + decls=(indexer, kv), + full_layer_mapping={0: 0}, + transfer_layer_id_max=1, + transfer_page_size=64, + ) + self.assertEqual( + [c.decl.pool_name for c in config.pools], [PoolName.KV, PoolName.INDEXER] + ) + + +class TestDeclaredPoolVerification(CustomTestCase): + def _result_with(self, *names): + group = SimpleNamespace(entry_map={n: object() for n in names}) + return StackBuildResult( + host_pool_group=group, cache_controller=None, component_host_pools={} + ) + + def test_unmigrated_strategy_logs_missing_indexer(self): + pool = _dsa_pool_stub(layer_num=2) + with self.assertLogs(hybrid_pool_assembler.logger, level="ERROR") as logs: + _check_declared_pools_present( + kvcache=pool, + result=self._result_with(PoolName.KV), + strategy=_SwaStrategy(), + ) + self.assertIn("indexer", logs.output[0]) + + def test_migrated_strategy_raises_on_missing_indexer(self): + pool = _dsa_pool_stub(layer_num=2) + with self.assertRaisesRegex(ValueError, "indexer"): + _check_declared_pools_present( + kvcache=pool, + result=self._result_with(PoolName.KV), + strategy=_DsaStrategy(), + ) + + def test_complete_stack_passes_silently(self): + pool = _dsa_pool_stub(layer_num=2) + _check_declared_pools_present( + kvcache=pool, + result=self._result_with(PoolName.KV, PoolName.INDEXER), + strategy=_DsaStrategy(), + ) + + def test_hybrid_pool_declares_through_its_sub_pool(self): + # HybridLinearKVPool once exposed host_pool_decls as a property, so the + # polymorphic call every strategy makes raised on hybrid models. + pool = object.__new__(HybridLinearKVPool) + pool.full_kv_pool = _dsa_pool_stub(layer_num=2) + self.assertEqual( + [(d.pool_name, d.device_pool) for d in pool.host_pool_decls()], + [(PoolName.KV, pool.full_kv_pool), (PoolName.INDEXER, pool.full_kv_pool)], + ) + _ASSEMBLER = "sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler." diff --git a/test/registered/unit/mem_cache/test_mem_pool_host.py b/test/registered/unit/mem_cache/test_mem_pool_host.py index 360899c703..c39f67753c 100644 --- a/test/registered/unit/mem_cache/test_mem_pool_host.py +++ b/test/registered/unit/mem_cache/test_mem_pool_host.py @@ -4,6 +4,7 @@ import threading import unittest import unittest.mock from functools import partial +from types import SimpleNamespace import torch @@ -14,8 +15,13 @@ from sglang.srt.mem_cache.memory_pool_host import ( LogicalHostPool, ) from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry, base +from sglang.srt.mem_cache.pool_host.dsa import ( + DSAIndexerPoolHost, + make_dsa_indexer_pool_decl, +) from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost +from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.srt.runtime_context import get_context from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -424,5 +430,101 @@ class TestHostPoolGroup(CustomTestCase): self.assertEqual(group.available_size(PoolName.MAMBA), 2) +class TestDSAIndexerPoolDecl(CustomTestCase): + """The declaration is the single source of indexer host bytes. The mirror + must not re-derive them.""" + + def _stub(self): + return SimpleNamespace( + layer_num=5, + layer_shard_enabled=False, + store_dtype=torch.bfloat16, + size=64 * 8, + start_layer=0, + end_layer=5, + kv_lora_rank=512, + qk_rope_head_dim=64, + index_head_dim=128, + quant_block_size=128, + skip_topk_layers=[False] * 5, + ) + + def test_host_bytes_match_observed_allocation(self): + # GLM-5.2 DSA, page 64, 5 layers, host 18192320 tokens: the server + # allocated 12006973440 bytes (12.01 GB) for the indexer mirror. + storage_info = make_dsa_indexer_pool_decl(self._stub()).storage_info + self.assertEqual(storage_info.bytes_per_token_per_layer, 132) + self.assertEqual(storage_info.page_bytes(64), 8448) + self.assertEqual( + storage_info.host_bytes(page_num=284256, layer_num=5, page_size=64), + 12006973440, + ) + + def test_mirror_consumes_decl(self): + stub = self._stub() + decl = make_dsa_indexer_pool_decl(stub) + storage_info = decl.storage_info + anchor = MLATokenToKVPoolHost( + stub, + host_to_device_ratio=2, + host_size=0, + page_size=64, + layout="page_first", + pin_memory=False, + is_dummy=True, + ) + mirror = DSAIndexerPoolHost( + decl=decl, + anchor_host=anchor, + pin_memory=False, + is_dummy=True, + ) + self.assertEqual(mirror.layout, anchor.layout) + self.assertEqual(mirror.indexer_page_stride_size, storage_info.page_bytes(64)) + self.assertEqual( + mirror.get_size_per_token(), storage_info.bytes_per_token_per_layer * 5 + ) + self.assertEqual( + storage_info.host_bytes( + page_num=anchor.page_num, layer_num=5, page_size=64 + ), + anchor.page_num * mirror.indexer_layout_dim, + ) + + def test_mirror_is_compact_over_layers_that_own_index_buffers(self): + """Shared-topk layers have 0-row device buffers. Mirroring or + transferring them dereferences a null pointer. The mirror must cover + only the declared layers and translate packed-draft layer ids + relative to that compact count.""" + stub = self._stub() + stub.skip_topk_layers = [False, True, True, False, True] + decl = make_dsa_indexer_pool_decl(stub) + self.assertEqual(decl.owned_device_layers, (0, 3)) + anchor = MLATokenToKVPoolHost( + stub, + host_to_device_ratio=2, + host_size=0, + page_size=64, + layout="page_first", + pin_memory=False, + is_dummy=True, + ) + mirror = DSAIndexerPoolHost( + decl=decl, + anchor_host=anchor, + pin_memory=False, + is_dummy=True, + ) + self.assertEqual(mirror.layer_num, 2) + self.assertEqual( + mirror.get_size_per_token(), decl.storage_info.bytes_per_token_per_layer * 2 + ) + self.assertEqual(mirror._owned_device_layer_ids(stub), [0, 3]) + self.assertEqual(mirror._host_layer_index(3), 1) + self.assertFalse(mirror._is_device_layer_owned(stub, 1)) + # packed draft depth 0 arrives as device layer_num + 0 and lands after the live layers + self.assertEqual(mirror._draft_host_layer(stub.layer_num), 2) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py index 57c6905d86..aa94fde703 100644 --- a/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py +++ b/test/registered/unit/mem_cache/test_mla_host_dedup_primitives.py @@ -10,7 +10,10 @@ from sglang.srt.mem_cache.mla_host_dedup import ( MLAHostDedupContext, maybe_create_mla_host_dedup_context, ) -from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost +from sglang.srt.mem_cache.pool_host.dsa import ( + DSAIndexerPoolHost, + make_dsa_indexer_pool_decl, +) from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost from sglang.test.ci.ci_register import register_cpu_ci @@ -80,11 +83,11 @@ class TestMLAHostDedupPrimitives(unittest.TestCase): end_layer=2, index_head_dim=8, quant_block_size=4, + skip_topk_layers=[False] * 2, ) indexer_host = DSAIndexerPoolHost( - dsa_device_pool, - mla_host, - layout="page_first", + decl=make_dsa_indexer_pool_decl(dsa_device_pool), + anchor_host=mla_host, pin_memory=False, is_dummy=True, )