Declare HiCache host pools for DSA indexer (#40913)

This commit is contained in:
Ke Bao
2026-10-01 13:08:26 +08:00
committed by GitHub
parent f02828ea39
commit 5733bcaad3
11 changed files with 1520 additions and 212 deletions
@@ -0,0 +1,270 @@
"""Pure HiCache declaration validation and transfer-layer binding."""
from __future__ import annotations
from typing import Any, Optional
import msgspec
from sglang.srt.mem_cache.hicache_storage import PoolName
from sglang.srt.mem_cache.pool_host.host_pool_decl import HostPoolDecl
class HostPoolConfig(msgspec.Struct, frozen=True, kw_only=True):
"""Validated device owners and transfer layers for one host pool."""
decl: HostPoolDecl
layer_mapping: dict[int, int]
# Exclusive upper bound, including packed draft tails.
transfer_layer_id_max: int
# Draft pools whose same-named buffers are appended as tail layers, in depth order.
packed_draft_device_pools: tuple[Any, ...] = ()
class HostPoolGroupConfig(msgspec.Struct, frozen=True, kw_only=True):
"""Validated host pools and their shared transfer granularity."""
# Original-token slots moved together. Individual buffers may use fewer rows.
transfer_page_size: int
# Dependency order with the unique primary KV root first.
pools: tuple[HostPoolConfig, ...]
def with_packed_draft_layer_mapping(
layer_mapping: dict[int, int],
*,
transfer_layer_start: int,
target_device_layer_num: int,
draft_layer_num: int,
) -> dict[int, int]:
return layer_mapping | {
transfer_layer_start + depth: target_device_layer_num + depth
for depth in range(draft_layer_num)
}
def _validate_packed_draft_pools(
*, target_decls: tuple[HostPoolDecl, ...], draft_pools: tuple[Any, ...]
) -> tuple[tuple[HostPoolDecl, ...], ...]:
"""Collect draft declarations once and check they can be appended to target host
pools: it declares the same pools with the same storage info and every
packed layer owns its buffer. The draft plan already chose packing, so a
draft that cannot be packed is an error, not a silent skip."""
targets = {d.pool_name: d for d in target_decls}
collected = []
for pool in draft_pools:
declarations = pool.host_pool_decls()
collected.append(declarations)
drafts = {d.pool_name: d for d in declarations}
if len(drafts) != len(declarations):
raise ValueError("packed draft declares duplicate host pool names")
if set(drafts) != set(targets):
raise ValueError(
f"packed draft {type(pool).__name__} declares "
f"{sorted(d.value for d in drafts)} but the target declares "
f"{sorted(d.value for d in targets)}; every target pool needs a "
"draft counterpart or the draft buffers are not restored"
)
for name, target in targets.items():
draft = drafts[name]
if draft.storage_info != target.storage_info:
raise ValueError(
f"packed draft {name.value} storage {draft.storage_info} "
f"differs from target {target.storage_info}"
)
owned = draft.owned_device_layers
if owned is not None and (
len(set(owned)) != draft.device_pool.layer_num
or set(owned) != set(range(draft.device_pool.layer_num))
):
raise ValueError(
f"packed draft {name.value} owns buffers on {len(owned)} of "
f"{draft.device_pool.layer_num} layers; every packed layer "
"must own its buffer"
)
if layout_root(declarations).device_pool.layer_num != 1:
raise ValueError(
"packed draft requires exactly one KV layer per draft pool"
)
return tuple(collected)
def layout_root(decls: tuple[HostPoolDecl, ...]) -> HostPoolDecl:
"""The one declaration whose host pool decides the group's capacity."""
roots = [d for d in decls if d.is_layout_root]
if len(roots) != 1:
raise ValueError(
f"expected exactly one layout root, got {[d.pool_name for d in roots]}"
)
return roots[0]
def _find_pool_decl(*, decls: tuple[HostPoolDecl, ...], name: PoolName) -> HostPoolDecl:
return next(d for d in decls if d.pool_name == name)
def _validate_and_order_decls(
decls: tuple[HostPoolDecl, ...],
) -> tuple[HostPoolDecl, ...]:
names = [decl.pool_name for decl in decls]
if len(set(names)) != len(names):
raise ValueError(f"duplicate host pool names: {names}")
root = layout_root(decls)
if not root.is_primary or root.pool_name != PoolName.KV:
raise ValueError(
f"expected the layout root to be the primary KV pool, got {root.pool_name}"
)
for decl in decls:
if not decl.is_layout_root and (
decl.host_pool_builder is None or decl.storage_info is None
):
raise ValueError(
f"{decl.pool_name}: dependent pool needs builder and storage_info"
)
owned = decl.owned_device_layers
if owned is not None and (
len(set(owned)) != len(owned)
or any(layer < 0 or layer >= decl.device_pool.layer_num for layer in owned)
):
raise ValueError(f"{decl.pool_name}: invalid owned_device_layers {owned}")
if decl.is_primary:
continue
if decl.indices_from_pool != root.pool_name:
raise ValueError(
f"{decl.pool_name}.indices_from_pool must be {root.pool_name}, "
f"got {decl.indices_from_pool}"
)
if decl.is_layout_root:
continue
if decl.layout_source == decl.pool_name:
raise ValueError(f"{decl.pool_name}.layout_source must name another pool")
if decl.layout_source not in names:
raise ValueError(
f"{decl.pool_name} references undeclared pool {decl.layout_source}"
)
# Topologically sort by layout_source so dependencies are built first.
ordered = []
pending = list(decls)
built = set()
while pending:
ready = [
decl
for decl in pending
if decl.is_layout_root or decl.layout_source in built
]
if not ready:
raise ValueError(
f"cyclic layout_source dependencies: {[decl.pool_name for decl in pending]}"
)
for decl in ready:
ordered.append(decl)
built.add(decl.pool_name)
pending.remove(decl)
return tuple(ordered)
def prepare_host_pool_config(
*,
decls: tuple[HostPoolDecl, ...],
full_layer_mapping: dict[int, int],
transfer_layer_id_max: int,
transfer_page_size: int,
packed_draft_device_pools: tuple[Any, ...] = (),
) -> HostPoolGroupConfig:
"""Validate declarations and buffers, then append packed draft transfer layers.
The input mapping and limit describe target layers only. No host memory is
allocated here. config.pools are in host construction order.
"""
decls = _validate_and_order_decls(decls)
root = decls[0]
target_layers = root.device_pool.layer_num
for transfer_id, device_id in full_layer_mapping.items():
if not 0 <= transfer_id < transfer_layer_id_max:
raise ValueError(
f"transfer layer {transfer_id} outside [0, {transfer_layer_id_max})"
)
if not 0 <= device_id < target_layers:
raise ValueError(f"device layer {device_id} outside [0, {target_layers})")
packed_draft_decls = _validate_packed_draft_pools(
target_decls=decls, draft_pools=packed_draft_device_pools
)
mapping = with_packed_draft_layer_mapping(
full_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=target_layers,
draft_layer_num=len(packed_draft_decls),
)
configs = tuple(
HostPoolConfig(
decl=d,
layer_mapping=_filter_owned_layer_mapping(
mapping=mapping,
owned_device_layers=d.owned_device_layers,
target_layer_num=target_layers,
),
transfer_layer_id_max=transfer_layer_id_max + len(packed_draft_decls),
packed_draft_device_pools=tuple(
_find_pool_decl(decls=decls_of_draft, name=d.pool_name).device_pool
for decls_of_draft in packed_draft_decls
),
)
for d in decls
)
check_packed_kv_rows(
kv_pool=root.device_pool,
drafts=configs[0].packed_draft_device_pools,
)
for config in configs[1:]:
config.decl.host_pool_builder.validate(
decl=config.decl,
transfer_page_size=transfer_page_size,
packed_draft_device_pools=config.packed_draft_device_pools,
)
return HostPoolGroupConfig(transfer_page_size=transfer_page_size, pools=configs)
def _filter_owned_layer_mapping(
*,
mapping: dict[int, int],
owned_device_layers: Optional[tuple[int, ...]],
target_layer_num: int,
) -> dict[int, int]:
"""Drop transfer layers whose device layer owns no buffer for this pool.
Packed draft tails (device index >= target layer count) are kept."""
if owned_device_layers is None:
return mapping
owned = set(owned_device_layers)
return {
t: dev for t, dev in mapping.items() if dev >= target_layer_num or dev in owned
}
def is_mla_pool(pool: Any) -> bool:
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
return isinstance(pool, MLATokenToKVPool)
def _kv_row_signature(pool: Any, *, use_mla: bool) -> tuple:
if use_mla:
return (pool.kv_cache_dim, pool.store_dtype)
return (pool.head_num, pool.head_dim, pool.v_head_dim, pool.store_dtype)
def check_packed_kv_rows(
*, kv_pool: Any, drafts: tuple[Any, ...], use_mla: Optional[bool] = None
):
"""Packed draft KV layers share the target's host row, so their device
rows must have the same shape and dtype."""
if not drafts:
return
if use_mla is None:
use_mla = is_mla_pool(kv_pool)
target = _kv_row_signature(kv_pool, use_mla=use_mla)
for draft in drafts:
row = _kv_row_signature(draft, use_mla=use_mla)
if row != target:
raise ValueError(
f"packed draft KV row {row} differs from target KV row {target}"
)
@@ -9,6 +9,13 @@ from sglang.srt.mem_cache.hicache_storage import (
PoolName,
SidecarPoolSpec,
)
from sglang.srt.mem_cache.hybrid_cache.host_pool_config import (
HostPoolGroupConfig,
check_packed_kv_rows,
is_mla_pool,
prepare_host_pool_config,
with_packed_draft_layer_mapping,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
HybridCacheController,
)
@@ -19,7 +26,7 @@ from sglang.srt.mem_cache.memory_pool_host import (
)
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.pool_host.common import get_allocator_type
from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost
from sglang.srt.mem_cache.pool_host.host_pool_decl import HostPoolDecl
from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost
from sglang.srt.mem_cache.pool_host.mha import (
MHATokenToKOnlyPoolHost,
@@ -87,19 +94,6 @@ def _stage_local_layer_mapping(
}
def _with_mtp_layer_mapping(
layer_mapping: dict[int, int],
*,
transfer_layer_start: int,
target_device_layer_num: int,
draft_layer_num: int,
) -> dict[int, int]:
return layer_mapping | {
transfer_layer_start + depth: target_device_layer_num + depth
for depth in range(draft_layer_num)
}
class _DeepSeekV4LayerMappings(NamedTuple):
transfer_layer_id_max: int
full: dict[int, int]
@@ -142,15 +136,24 @@ def build_kv_host_pool(
*,
kv_pool: Any,
page_size: int,
use_mla: bool,
use_mla: Optional[bool] = None,
override_kv_cache_dim: Optional[int] = None,
host_size: Optional[float] = None,
mtp_draft_device_pools: tuple[Any, ...] = (),
pool_label: str = "kv",
):
if use_mla is None:
use_mla = is_mla_pool(kv_pool)
kv_host_pool_cls = (
MLATokenToKVPoolHost if use_mla else get_mha_host_pool_cls(kv_pool)
)
if use_mla and override_kv_cache_dim is None:
# An fp8 DSA store is wider than kv_lora_rank + qk_rope_head_dim. The
# device pool already resolved the row width.
override_kv_cache_dim = kv_pool.kv_cache_dim
check_packed_kv_rows(
kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla
)
kwargs = {}
if override_kv_cache_dim is not None:
kwargs["override_kv_cache_dim"] = override_kv_cache_dim
@@ -245,7 +248,7 @@ def build_kv_only_group(
mtp_draft_device_pools=mtp_draft_device_pools,
)
if mtp_draft_device_pools:
full_layer_mapping = _with_mtp_layer_mapping(
full_layer_mapping = with_packed_draft_layer_mapping(
full_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=kv_pool.layer_num,
@@ -356,7 +359,7 @@ def build_hybrid_swa_group(
pool_label="swa",
)
if mtp_swa_device_pools:
swa_layer_mapping = _with_mtp_layer_mapping(
swa_layer_mapping = with_packed_draft_layer_mapping(
swa_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=swa_kv_pool.layer_num,
@@ -856,7 +859,7 @@ def build_deepseek_v4_hicache_stack(
for pool in params.mtp_draft_device_pools
for buffer in pool.swa_kv_pool.kv_buffer
]
swa_layer_mapping = _with_mtp_layer_mapping(
swa_layer_mapping = with_packed_draft_layer_mapping(
swa_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=transfer_layer_id_max,
@@ -1155,7 +1158,7 @@ def build_hybrid_mamba_stack(
mtp_draft_device_pools=mtp_draft_device_pools,
)
if mtp_draft_device_pools:
full_layer_mapping = _with_mtp_layer_mapping(
full_layer_mapping = with_packed_draft_layer_mapping(
full_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=kv_pool.layer_num,
@@ -1345,82 +1348,41 @@ def build_hybrid_mamba_swa_stack(
return host_pool_group, cache_controller
def build_anchor_sidecar_stack(
def build_host_pool_group(
*,
params: CacheInitParams,
kv_pool: Any,
sidecar_pool_name: PoolName,
full_layer_mapping: dict[int, int],
load_cache_event,
storage_backend: Optional[str],
use_mla: bool,
override_kv_cache_dim: Optional[int] = None,
sidecar_host_pool_factory: Callable[[Any], Any],
prefetch_threshold: int = 256,
model_name: Optional[str] = None,
storage_backend_extra_config: Optional[dict] = None,
enable_storage_metrics: bool = False,
) -> tuple[HostPoolGroup, HybridCacheController]:
transfer_layer_id_max = len(full_layer_mapping)
mtp_draft_device_pools = tuple(
pool for pool in params.mtp_draft_device_pools if pool.index_k_with_scale_buffer
)
kv_host_pool = build_kv_host_pool(
kv_pool=kv_pool,
page_size=params.page_size,
use_mla=use_mla,
override_kv_cache_dim=override_kv_cache_dim,
mtp_draft_device_pools=mtp_draft_device_pools,
)
sidecar_host_pool = sidecar_host_pool_factory(kv_host_pool)
# Expose packed MTP tail layers to the controller's flat transfer builder.
if mtp_draft_device_pools:
full_layer_mapping = _with_mtp_layer_mapping(
full_layer_mapping,
transfer_layer_start=transfer_layer_id_max,
target_device_layer_num=kv_pool.layer_num,
draft_layer_num=len(mtp_draft_device_pools),
config: HostPoolGroupConfig,
) -> HostPoolGroup:
"""Allocate host pools and transfer entries from prepared configs."""
root = config.pools[0]
host_pools = {
root.decl.pool_name: build_kv_host_pool(
kv_pool=root.decl.device_pool,
page_size=config.transfer_page_size,
mtp_draft_device_pools=root.packed_draft_device_pools,
)
entries = [
build_pool_entry(
name=PoolName.KV,
host_pool=kv_host_pool,
device_pool=kv_pool,
layer_mapping=full_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools),
is_anchor=True,
packed_draft_device_pools=mtp_draft_device_pools,
),
build_pool_entry(
name=sidecar_pool_name,
host_pool=sidecar_host_pool,
device_pool=kv_pool,
layer_mapping=full_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools),
packed_draft_device_pools=mtp_draft_device_pools,
),
]
host_pool_group = HostPoolGroup(entries)
cache_controller = HybridCacheController(
params.token_to_kv_pool_allocator,
host_pool_group,
params.page_size,
params.tp_cache_group,
load_cache_event=load_cache_event,
attn_cp_group=params.attn_cp_cache_group,
attn_tp_group=params.attn_tp_cache_group,
pp_group=params.pp_cache_group,
write_policy=get_memory().hicache_write_policy,
io_backend=get_memory().hicache_io_backend,
storage_backend=storage_backend,
prefetch_threshold=prefetch_threshold,
model_name=model_name,
storage_backend_extra_config=storage_backend_extra_config,
transfer_layer_id_max=transfer_layer_id_max,
enable_storage_metrics=enable_storage_metrics,
host_memory_mode=get_memory().hicache_host_memory_mode,
)
return host_pool_group, cache_controller
}
entries = []
for pool_config in config.pools:
decl = pool_config.decl
if not decl.is_layout_root:
host_pools[decl.pool_name] = decl.host_pool_builder.build(
decl=decl,
anchor_host=host_pools[decl.layout_source],
allocator_type=_get_allocator_type(),
packed_draft_device_pools=pool_config.packed_draft_device_pools,
)
entries.append(
build_pool_entry(
name=decl.pool_name,
host_pool=host_pools[decl.pool_name],
device_pool=decl.device_pool,
layer_mapping=pool_config.layer_mapping,
transfer_layer_id_max=pool_config.transfer_layer_id_max,
is_anchor=decl.is_primary,
packed_draft_device_pools=pool_config.packed_draft_device_pools,
)
)
return HostPoolGroup(entries)
def _build_mha_mla_host_pool(
@@ -1504,18 +1466,20 @@ def build_full_draft_pools(
]
if isinstance(pool, DSATokenToKVPool) and pool.index_k_with_scale_buffer:
from sglang.srt.mem_cache.pool_host.dsa import (
DSAIndexerPoolHost,
make_dsa_indexer_pool_decl,
)
# Separate draft indexer: its own host pool laid out on the draft KV
# host pool, but transfer indices still follow the target KV anchor.
indexer_decl = make_dsa_indexer_pool_decl(pool, name=PoolName.DRAFT_INDEXER)
indexer_host_pool = DSAIndexerPoolHost(
pool,
draft_host_pool,
get_memory().hicache_mem_layout,
decl=indexer_decl,
anchor_host=draft_host_pool,
allocator_type=_get_allocator_type(),
)
specs.append(
SidecarPoolSpec(
pool_name=PoolName.DRAFT_INDEXER,
indices_from_pool=PoolName.KV,
)
)
specs.append(indexer_decl.sidecar_spec())
entries.append(
build_pool_entry(
name=PoolName.DRAFT_INDEXER,
@@ -1626,6 +1590,7 @@ class StackBuildResult:
# layer_transfer_counter has to be wired separately.
register_req_to_token_counter: bool = False
pools_desc: str = ""
pool_declarations: Optional[tuple[HostPoolDecl, ...]] = None
class StackStrategy:
@@ -1976,78 +1941,6 @@ class _MambaSwaStrategy(StackStrategy):
)
class _DsaStrategy(StackStrategy):
def matches(self, kvcache, components):
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
return isinstance(kvcache, DSATokenToKVPool) and components == {
ComponentType.FULL
}
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
_build_dsa_device_pool_group,
)
return _build_dsa_device_pool_group(
kvcache, page_size, params.mtp_draft_device_pools
)
def build(
self,
*,
cache,
kvcache,
params,
server_args,
load_cache_event,
storage_backend=None,
storage_backend_extra_config=None,
prefetch_threshold=256,
model_name=None,
enable_storage_metrics=False,
):
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
full_kv_pool = kvcache
use_mla = isinstance(kvcache, MLATokenToKVPool)
full_layer_mapping = {i: i for i in range(full_kv_pool.layer_num)}
host_pool_group, cache_controller = build_anchor_sidecar_stack(
params=params,
kv_pool=full_kv_pool,
sidecar_pool_name=PoolName.INDEXER,
full_layer_mapping=full_layer_mapping,
load_cache_event=load_cache_event,
storage_backend=storage_backend,
use_mla=use_mla,
override_kv_cache_dim=full_kv_pool.kv_cache_dim,
sidecar_host_pool_factory=lambda kv_host_pool: DSAIndexerPoolHost(
full_kv_pool,
kv_host_pool,
get_memory().hicache_mem_layout,
allocator_type=_get_allocator_type(),
),
prefetch_threshold=prefetch_threshold,
model_name=model_name,
storage_backend_extra_config=storage_backend_extra_config,
enable_storage_metrics=enable_storage_metrics,
)
return StackBuildResult(
host_pool_group=host_pool_group,
cache_controller=cache_controller,
component_host_pools={
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
},
sidecars=[
SidecarPoolSpec(
pool_name=PoolName.INDEXER,
indices_from_pool=PoolName.KV,
),
],
pools_desc="KV + INDEXER",
)
class _MiniMaxSparseStrategy(StackStrategy):
def matches(self, kvcache, components):
from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool
@@ -2165,6 +2058,79 @@ class _PlainKvStrategy(StackStrategy):
)
class _DsaStrategy(StackStrategy):
def matches(self, kvcache, components):
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
return isinstance(kvcache, DSATokenToKVPool) and components == {
ComponentType.FULL
}
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
_build_dsa_device_pool_group,
)
return _build_dsa_device_pool_group(
kvcache, page_size, params.mtp_draft_device_pools
)
def build(
self,
*,
cache,
kvcache,
params,
server_args,
load_cache_event,
storage_backend=None,
storage_backend_extra_config=None,
prefetch_threshold=256,
model_name=None,
enable_storage_metrics=False,
):
config = prepare_host_pool_config(
decls=kvcache.host_pool_decls(),
full_layer_mapping={i: i for i in range(kvcache.layer_num)},
transfer_layer_id_max=kvcache.layer_num,
transfer_page_size=params.page_size,
packed_draft_device_pools=params.mtp_draft_device_pools,
)
host_pool_group = build_host_pool_group(config=config)
cache_controller = HybridCacheController(
params.token_to_kv_pool_allocator,
host_pool_group,
config.transfer_page_size,
params.tp_cache_group,
load_cache_event=load_cache_event,
attn_cp_group=params.attn_cp_cache_group,
attn_tp_group=params.attn_tp_cache_group,
pp_group=params.pp_cache_group,
write_policy=get_memory().hicache_write_policy,
io_backend=get_memory().hicache_io_backend,
storage_backend=storage_backend,
prefetch_threshold=prefetch_threshold,
model_name=model_name,
storage_backend_extra_config=storage_backend_extra_config,
# The controller adds packed draft transfers after target layers.
transfer_layer_id_max=kvcache.layer_num,
enable_storage_metrics=enable_storage_metrics,
host_memory_mode=get_memory().hicache_host_memory_mode,
)
return StackBuildResult(
host_pool_group=host_pool_group,
cache_controller=cache_controller,
component_host_pools={
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
},
sidecars=[
c.decl.sidecar_spec() for c in config.pools if not c.decl.is_primary
],
pools_desc=" + ".join(c.decl.pool_name.value.upper() for c in config.pools),
pool_declarations=tuple(c.decl for c in config.pools),
)
# Resolved first-to-last; _PlainKvStrategy is the catch-all fallback.
_STRATEGIES: list[StackStrategy] = [
_DeepSeekV4Strategy(),
@@ -2193,6 +2159,33 @@ def _select_strategy(kvcache: Any, components: set[ComponentType]) -> StackStrat
)
# Strategies that assemble from host_pool_decls(), so every declared pool has an
# entry by construction. A miss here is a bug, not an unsupported combination.
_DECLARATION_VERIFIED_STRATEGIES: tuple[type, ...] = (_DsaStrategy,)
def _check_declared_pools_present(
*, kvcache: Any, result: StackBuildResult, strategy: StackStrategy
) -> None:
entries = result.host_pool_group.entry_map
decls = result.pool_declarations
if decls is None:
decls = kvcache.host_pool_decls()
missing = [d.pool_name.value for d in decls if d.pool_name not in entries]
if not missing:
return
msg = (
f"{type(kvcache).__name__} declares host pools {missing} that "
f"{type(strategy).__name__} did not assemble"
)
if isinstance(strategy, _DECLARATION_VERIFIED_STRATEGIES):
raise ValueError(msg)
# Pre-declaration strategies: restoring KV without these pools (e.g. the
# indexer's key buffers) corrupts sparse attention after a host hit.
# Loud until the path is migrated.
logger.error("%s; host restore of these pools is unsupported on this path", msg)
def _apply_stack_result(
cache: UnifiedRadixCache,
kvcache: Any,
@@ -2250,6 +2243,7 @@ def attach_hybrid_pool_to_unified_cache(
model_name=get_serving().served_model_name,
enable_storage_metrics=cache._enable_metrics_flag,
)
_check_declared_pools_present(kvcache=kvcache, result=result, strategy=strategy)
_apply_stack_result(cache, kvcache, params, result)
except Exception:
logger.exception("attach_hybrid_pool_to_unified_cache failed")
@@ -1874,6 +1874,17 @@ class KVCache(abc.ABC):
maybe_init_custom_mem_pool(device=self.device)
)
def host_pool_decls(self):
"""Host pools HiCache keeps for the buffers this device pool (and its
sub-pools) owns: the KV pool itself, plus dependent pools such as sparse
index keys in subclasses. Not a description of the whole model: buffers
owned elsewhere (e.g. Mamba state in req_to_token_pool) are declared by
their owner or assembled by the existing strategy paths."""
# pool_host imports this module. Resolve the declaration types lazily.
from sglang.srt.mem_cache.pool_host.host_pool_decl import make_kv_pool_decl
return (make_kv_pool_decl(self),)
def _finalize_allocation_log(self, num_tokens: int):
"""Common logging and mem_usage computation for KV cache allocation.
Supports both tuple (K, V) size returns and single KV size returns.
@@ -4014,6 +4025,12 @@ class HybridLinearKVPool(KVCache):
k_size, v_size = self.get_kv_size_bytes()
self.mem_usage = (k_size + v_size) / GB
def host_pool_decls(self):
# Only the full-attention sub-pool owns HiCache-addressable buffers here.
# Mamba state belongs to req_to_token_pool's MambaPool and is assembled
# by _MambaStrategy, not declared through this pool.
return self.full_kv_pool.host_pool_decls()
@property
def post_capture_active(self) -> bool:
return self.full_kv_pool.post_capture_active
@@ -4966,6 +4983,17 @@ class DSATokenToKVPool(MLATokenToKVPool):
def _should_allocate_index_layer(self, local_layer_idx: int) -> bool:
return not self.skip_topk_layers[local_layer_idx]
def host_pool_decls(self):
# pool_host imports this module. Resolve the mirror side lazily.
from sglang.srt.mem_cache.pool_host.dsa import make_dsa_indexer_pool_decl
kv_decls = super().host_pool_decls()
# Shared-topk layers own a 0-row placeholder, so a non-empty buffer list
# is not enough: some layer must actually hold index keys.
if not self.index_k_with_scale_buffer or all(self.skip_topk_layers):
return kv_decls
return (*kv_decls, make_dsa_indexer_pool_decl(self))
@property
def index_k_with_scale_buffer(self):
# Preserve direct HiCache access while storage lives behind the facade.
+132 -29
View File
@@ -15,6 +15,7 @@ from sglang.kernels.ops.kvcache.hicache import (
from sglang.kernels.ops.kvcache.hicache import (
transfer_hicache_all_layer_mla_staged_lf_pf as jit_transfer_hicache_all_layer_mla_staged_lf_pf,
)
from sglang.srt.mem_cache.hicache_storage import PoolName
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import (
_WRITE_BACK_STAGING_PAGE_CHUNK,
@@ -26,6 +27,10 @@ from sglang.srt.mem_cache.pool_host.common import (
get_allocator_from_storage,
make_kernel_ptr_table,
)
from sglang.srt.mem_cache.pool_host.host_pool_decl import (
HostPoolDecl,
HostPoolStorageInfo,
)
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
_is_cuda = is_cuda()
@@ -47,6 +52,73 @@ if _is_cuda or _is_hip:
logger = logging.getLogger(__name__)
def dsa_indexer_bytes_per_token_per_layer(
*, index_head_dim: int, quant_block_size: int
) -> int:
# packed index keys plus one fp32 scale per quant block, stored as uint8
elems = index_head_dim + index_head_dim // quant_block_size * 4
return elems * DSATokenToKVPool.index_k_with_scale_buffer_dtype.itemsize
class DSAIndexerHostPoolBuilder:
def validate(
self,
*,
decl: HostPoolDecl,
transfer_page_size: int,
packed_draft_device_pools: tuple[DSATokenToKVPool, ...],
) -> None:
target = decl.device_pool
target_format = (target.index_head_dim, target.quant_block_size)
for draft in packed_draft_device_pools:
draft_format = (draft.index_head_dim, draft.quant_block_size)
if draft_format != target_format:
raise ValueError(
f"{decl.pool_name}: packed index key format {draft_format} "
f"differs from target {target_format}"
)
def build(
self,
*,
decl: HostPoolDecl,
anchor_host: MLATokenToKVPoolHost,
allocator_type: str,
packed_draft_device_pools: tuple[DSATokenToKVPool, ...],
) -> DSAIndexerPoolHost:
return DSAIndexerPoolHost(
decl=decl,
anchor_host=anchor_host,
packed_draft_device_pools=packed_draft_device_pools,
allocator_type=allocator_type,
)
def make_dsa_indexer_pool_decl(
pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER
) -> HostPoolDecl:
"""Index key buffers riding on the full-KV pages: indices and layout both follow KV."""
return HostPoolDecl(
pool_name=name,
device_pool=pool,
indices_from_pool=PoolName.KV,
layout_source=PoolName.KV,
storage_info=HostPoolStorageInfo(
bytes_per_token_per_layer=dsa_indexer_bytes_per_token_per_layer(
index_head_dim=pool.index_head_dim,
quant_block_size=pool.quant_block_size,
),
dtype=DSATokenToKVPool.index_k_with_scale_buffer_dtype,
),
host_pool_builder=DSAIndexerHostPoolBuilder(),
# Shared-topk layers own a 0-row placeholder buffer. They get no host
# layer and must not reach the transfer kernels.
owned_device_layers=tuple(
i for i, skip in enumerate(pool.skip_topk_layers) if not skip
),
)
class DSAIndexerPoolHost(HostKVCache):
"""Host-side DSA index buffers only. Slot layout matches the anchor MLA host pool."""
@@ -54,46 +126,53 @@ class DSAIndexerPoolHost(HostKVCache):
def __init__(
self,
device_pool: DSATokenToKVPool,
decl: HostPoolDecl,
anchor_host: MLATokenToKVPoolHost,
layout: str,
*,
packed_draft_device_pools: tuple[DSATokenToKVPool, ...] = (),
pin_memory: bool = True,
device: str = "cpu",
allocator_type: str = "default",
is_dummy: bool = False,
):
self._is_dummy = is_dummy
self.decl = decl
storage_info = decl.storage_info
device_pool = decl.device_pool
self.device_pool = device_pool
self.page_size = anchor_host.page_size
self.layout = layout
self.layout = anchor_host.layout
self.pin_memory = pin_memory
self.device = device
self.allocator = get_allocator_from_storage(allocator_type)
self.dtype = device_pool.store_dtype
self.start_layer = device_pool.start_layer
self.end_layer = device_pool.end_layer
self.target_layer_num = self._effective_host_layer_num()
self.mtp_draft_device_pools = anchor_host.mtp_draft_device_pools
# Host layers are compact: only owned device layers that hold index
# buffers, then one tail layer per packed draft pool.
owned_start, owned_end = self._device_owned_layer_range()
declared = decl.owned_device_layers
self._live_target_layers = [
layer
for layer in range(owned_start, owned_end)
if declared is None or layer in declared
]
self._device_to_host_layer = {
layer: i for i, layer in enumerate(self._live_target_layers)
}
self.target_layer_num = len(self._live_target_layers)
self.mtp_draft_device_pools = tuple(packed_draft_device_pools)
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
self.index_head_dim = device_pool.index_head_dim
self.indexer_quant_block_size = device_pool.quant_block_size
self.indexer_dtype = DSATokenToKVPool.index_k_with_scale_buffer_dtype
self.indexer_size_per_token = (
self.index_head_dim
+ self.index_head_dim // self.indexer_quant_block_size * 4
)
self.indexer_dtype = storage_info.dtype
self.size = anchor_host.size
self.page_num = anchor_host.page_num
self.indexer_page_stride_size = (
self.indexer_size_per_token * self.page_size * self.indexer_dtype.itemsize
)
# uint8 storage, so element counts below are byte counts
self.indexer_page_stride_size = storage_info.page_bytes(self.page_size)
self.indexer_layout_dim = self.indexer_page_stride_size * self.layer_num
self.indexer_page_num = (self.size + self.page_size + 1) // self.page_size
self.size_per_token = (
self.indexer_size_per_token * self.layer_num * self.indexer_dtype.itemsize
)
self.size_per_token = storage_info.bytes_per_token_per_layer * self.layer_num
self.can_use_jit = False
self.can_use_write_back_jit = False
@@ -109,8 +188,9 @@ class DSAIndexerPoolHost(HostKVCache):
self.clear()
return
buf_elem_size = self.page_num * self.layer_num * self.indexer_page_stride_size
requested_bytes = buf_elem_size * self.indexer_dtype.itemsize
requested_bytes = storage_info.host_bytes(
page_num=self.page_num, layer_num=self.layer_num, page_size=self.page_size
)
available_bytes = host_memory_budget_bytes(requested_bytes)
if requested_bytes > available_bytes:
raise ValueError(
@@ -125,7 +205,7 @@ class DSAIndexerPoolHost(HostKVCache):
"packed MTP layers: "
"target_layers=%d, draft_layers=%d, total_layers=%d.",
requested_bytes / 1e9,
layout,
self.layout,
self.target_layer_num,
draft_layer_num,
self.layer_num,
@@ -134,7 +214,7 @@ class DSAIndexerPoolHost(HostKVCache):
logger.info(
"Allocating %.2f GB host memory for DSA indexer (layout=%s).",
requested_bytes / 1e9,
layout,
self.layout,
)
self.init_kv_buffer()
self._init_write_back_staging_buffers()
@@ -142,18 +222,33 @@ class DSAIndexerPoolHost(HostKVCache):
self.clear()
def get_size_per_token(self):
return (
self.indexer_size_per_token * self.layer_num * self.indexer_dtype.itemsize
)
return self.decl.storage_info.bytes_per_token_per_layer * self.layer_num
def get_ksize_per_token(self):
return self.get_size_per_token()
def _is_device_layer_owned(self, device_pool, layer_id: int) -> bool:
return layer_id in self._device_to_host_layer
def _host_layer_index(self, layer_id: int, device_pool=None) -> int:
return self._device_to_host_layer[layer_id]
def _owned_device_layer_ids(self, device_pool) -> list[int]:
return list(self._live_target_layers)
def _draft_host_layer(self, layer_id: int) -> int:
# The controller hands packed drafts ``target_device_layer_num + depth``.
return self.target_layer_num + (layer_id - self.device_pool.layer_num)
def init_kv_buffer(self):
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
device_pools = (self.device_pool, *self.mtp_draft_device_pools)
self.packed_device_index_buffers = [
buffer for pool in device_pools for buffer in pool.index_k_with_scale_buffer
self.device_pool.index_k_with_scale_buffer[layer]
for layer in self._live_target_layers
] + [
buffer
for pool in self.mtp_draft_device_pools
for buffer in pool.index_k_with_scale_buffer
]
self.index_k_device_ptrs = torch.tensor(
[x.data_ptr() for x in self.packed_device_index_buffers],
@@ -249,7 +344,11 @@ class DSAIndexerPoolHost(HostKVCache):
"load on a dummy (non-src DSA) host pool"
)
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
host_layer_id = (
self._draft_host_layer(layer_id)
if is_draft
else self._host_layer_index(layer_id)
)
device_layer_id = 0 if is_draft else layer_id
host_page_indices, device_page_indices = self._get_indexer_page_indices(
@@ -314,7 +413,11 @@ class DSAIndexerPoolHost(HostKVCache):
"backup on a dummy (non-src DSA) host pool"
)
# MTP draft layers do not participate in CP layer sharding.
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
host_layer_id = (
self._draft_host_layer(layer_id)
if is_draft
else self._host_layer_index(layer_id)
)
device_layer_id = 0 if is_draft else layer_id
host_page_indices, device_page_indices = self._get_indexer_page_indices(
@@ -0,0 +1,112 @@
"""Declarations of the host pools a device pool needs HiCache to keep.
A device pool declares them (``HostPoolDecl``) through ``host_pool_decls()``.
The assembler (hybrid_pool_assembler) binds each declaration to transfer layers
and builds one entry per declaration. This module holds only the declaration
types and their basic constructors.
"""
from __future__ import annotations
from typing import Any, Optional, Protocol
import msgspec
import torch
from sglang.srt.mem_cache.hicache_storage import (
PoolHitPolicy,
PoolName,
SidecarPoolSpec,
)
class HostPoolStorageInfo(msgspec.Struct, frozen=True, kw_only=True):
"""Bytes one host pool stores per token per layer, and their dtype.
Two equal values mean equal byte counts and dtype only. Page geometry,
scale placement and kernel layout are checked by the host pool builder.
"""
bytes_per_token_per_layer: int
dtype: torch.dtype
def page_bytes(self, page_size: int) -> int:
"""Bytes of one page of one layer."""
return self.bytes_per_token_per_layer * page_size
def host_bytes(self, *, page_num: int, layer_num: int, page_size: int) -> int:
return page_num * layer_num * self.page_bytes(page_size)
class HostPoolBuilder(Protocol):
def validate(
self,
*,
decl: HostPoolDecl,
# Original-token slots, not the number of physical rows in this pool.
transfer_page_size: int,
packed_draft_device_pools: tuple[Any, ...],
) -> None: ...
def build(
self,
*,
decl: HostPoolDecl,
anchor_host: Any,
allocator_type: str,
packed_draft_device_pools: tuple[Any, ...],
) -> Any: ...
class HostPoolDecl(msgspec.Struct, frozen=True, kw_only=True):
"""One host pool a device pool asks HiCache to keep. Pool-intrinsic: no layer binding."""
pool_name: PoolName
# The pool object that owns the device buffers behind this host pool. A
# composite pool keeps its KV buffers on a sub-pool and, e.g., its
# compressed index keys on itself, so each declaration names its owner.
device_pool: Any
# Whose host page indices this pool reuses when transferred. None: primary.
indices_from_pool: Optional[PoolName]
# Whose host pool decides this pool's capacity and memory layout. None: self.
layout_source: Optional[PoolName]
# None for a layout root (KV, DRAFT): the assembler builds its host pool
# from the device pool, so no byte facts are declared for it.
storage_info: Optional[HostPoolStorageInfo]
host_pool_builder: Optional[HostPoolBuilder]
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
# Local device layers that own buffers for this pool. None means every
# layer. Layers outside it get no host layer and no transfer.
owned_device_layers: Optional[tuple[int, ...]] = None
@property
def is_primary(self) -> bool:
return self.indices_from_pool is None
@property
def is_layout_root(self) -> bool:
return self.layout_source is None
def sidecar_spec(self) -> SidecarPoolSpec:
if self.indices_from_pool is None:
raise ValueError(
f"{self.pool_name} is the primary pool and has no sidecar spec"
)
return SidecarPoolSpec(
pool_name=self.pool_name,
indices_from_pool=self.indices_from_pool,
hit_policy=self.hit_policy,
)
def make_kv_pool_decl(pool: Any) -> HostPoolDecl:
"""The primary KV pool every device pool declares. The assembler builds its
host pool, so no storage info is declared here."""
return HostPoolDecl(
pool_name=PoolName.KV,
device_pool=pool,
indices_from_pool=None,
layout_source=None,
storage_info=None,
host_pool_builder=None,
)