Declare HiCache host pools for DSA indexer (#40913)
This commit is contained in:
@@ -0,0 +1,270 @@
|
||||
"""Pure HiCache declaration validation and transfer-layer binding."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
from sglang.srt.mem_cache.pool_host.host_pool_decl import HostPoolDecl
|
||||
|
||||
|
||||
class HostPoolConfig(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Validated device owners and transfer layers for one host pool."""
|
||||
|
||||
decl: HostPoolDecl
|
||||
layer_mapping: dict[int, int]
|
||||
# Exclusive upper bound, including packed draft tails.
|
||||
transfer_layer_id_max: int
|
||||
# Draft pools whose same-named buffers are appended as tail layers, in depth order.
|
||||
packed_draft_device_pools: tuple[Any, ...] = ()
|
||||
|
||||
|
||||
class HostPoolGroupConfig(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Validated host pools and their shared transfer granularity."""
|
||||
|
||||
# Original-token slots moved together. Individual buffers may use fewer rows.
|
||||
transfer_page_size: int
|
||||
# Dependency order with the unique primary KV root first.
|
||||
pools: tuple[HostPoolConfig, ...]
|
||||
|
||||
|
||||
def with_packed_draft_layer_mapping(
|
||||
layer_mapping: dict[int, int],
|
||||
*,
|
||||
transfer_layer_start: int,
|
||||
target_device_layer_num: int,
|
||||
draft_layer_num: int,
|
||||
) -> dict[int, int]:
|
||||
return layer_mapping | {
|
||||
transfer_layer_start + depth: target_device_layer_num + depth
|
||||
for depth in range(draft_layer_num)
|
||||
}
|
||||
|
||||
|
||||
def _validate_packed_draft_pools(
|
||||
*, target_decls: tuple[HostPoolDecl, ...], draft_pools: tuple[Any, ...]
|
||||
) -> tuple[tuple[HostPoolDecl, ...], ...]:
|
||||
"""Collect draft declarations once and check they can be appended to target host
|
||||
pools: it declares the same pools with the same storage info and every
|
||||
packed layer owns its buffer. The draft plan already chose packing, so a
|
||||
draft that cannot be packed is an error, not a silent skip."""
|
||||
targets = {d.pool_name: d for d in target_decls}
|
||||
collected = []
|
||||
for pool in draft_pools:
|
||||
declarations = pool.host_pool_decls()
|
||||
collected.append(declarations)
|
||||
drafts = {d.pool_name: d for d in declarations}
|
||||
if len(drafts) != len(declarations):
|
||||
raise ValueError("packed draft declares duplicate host pool names")
|
||||
if set(drafts) != set(targets):
|
||||
raise ValueError(
|
||||
f"packed draft {type(pool).__name__} declares "
|
||||
f"{sorted(d.value for d in drafts)} but the target declares "
|
||||
f"{sorted(d.value for d in targets)}; every target pool needs a "
|
||||
"draft counterpart or the draft buffers are not restored"
|
||||
)
|
||||
for name, target in targets.items():
|
||||
draft = drafts[name]
|
||||
if draft.storage_info != target.storage_info:
|
||||
raise ValueError(
|
||||
f"packed draft {name.value} storage {draft.storage_info} "
|
||||
f"differs from target {target.storage_info}"
|
||||
)
|
||||
owned = draft.owned_device_layers
|
||||
if owned is not None and (
|
||||
len(set(owned)) != draft.device_pool.layer_num
|
||||
or set(owned) != set(range(draft.device_pool.layer_num))
|
||||
):
|
||||
raise ValueError(
|
||||
f"packed draft {name.value} owns buffers on {len(owned)} of "
|
||||
f"{draft.device_pool.layer_num} layers; every packed layer "
|
||||
"must own its buffer"
|
||||
)
|
||||
if layout_root(declarations).device_pool.layer_num != 1:
|
||||
raise ValueError(
|
||||
"packed draft requires exactly one KV layer per draft pool"
|
||||
)
|
||||
return tuple(collected)
|
||||
|
||||
|
||||
def layout_root(decls: tuple[HostPoolDecl, ...]) -> HostPoolDecl:
|
||||
"""The one declaration whose host pool decides the group's capacity."""
|
||||
roots = [d for d in decls if d.is_layout_root]
|
||||
if len(roots) != 1:
|
||||
raise ValueError(
|
||||
f"expected exactly one layout root, got {[d.pool_name for d in roots]}"
|
||||
)
|
||||
return roots[0]
|
||||
|
||||
|
||||
def _find_pool_decl(*, decls: tuple[HostPoolDecl, ...], name: PoolName) -> HostPoolDecl:
|
||||
return next(d for d in decls if d.pool_name == name)
|
||||
|
||||
|
||||
def _validate_and_order_decls(
|
||||
decls: tuple[HostPoolDecl, ...],
|
||||
) -> tuple[HostPoolDecl, ...]:
|
||||
names = [decl.pool_name for decl in decls]
|
||||
if len(set(names)) != len(names):
|
||||
raise ValueError(f"duplicate host pool names: {names}")
|
||||
root = layout_root(decls)
|
||||
if not root.is_primary or root.pool_name != PoolName.KV:
|
||||
raise ValueError(
|
||||
f"expected the layout root to be the primary KV pool, got {root.pool_name}"
|
||||
)
|
||||
for decl in decls:
|
||||
if not decl.is_layout_root and (
|
||||
decl.host_pool_builder is None or decl.storage_info is None
|
||||
):
|
||||
raise ValueError(
|
||||
f"{decl.pool_name}: dependent pool needs builder and storage_info"
|
||||
)
|
||||
owned = decl.owned_device_layers
|
||||
if owned is not None and (
|
||||
len(set(owned)) != len(owned)
|
||||
or any(layer < 0 or layer >= decl.device_pool.layer_num for layer in owned)
|
||||
):
|
||||
raise ValueError(f"{decl.pool_name}: invalid owned_device_layers {owned}")
|
||||
if decl.is_primary:
|
||||
continue
|
||||
if decl.indices_from_pool != root.pool_name:
|
||||
raise ValueError(
|
||||
f"{decl.pool_name}.indices_from_pool must be {root.pool_name}, "
|
||||
f"got {decl.indices_from_pool}"
|
||||
)
|
||||
if decl.is_layout_root:
|
||||
continue
|
||||
if decl.layout_source == decl.pool_name:
|
||||
raise ValueError(f"{decl.pool_name}.layout_source must name another pool")
|
||||
if decl.layout_source not in names:
|
||||
raise ValueError(
|
||||
f"{decl.pool_name} references undeclared pool {decl.layout_source}"
|
||||
)
|
||||
# Topologically sort by layout_source so dependencies are built first.
|
||||
ordered = []
|
||||
pending = list(decls)
|
||||
built = set()
|
||||
while pending:
|
||||
ready = [
|
||||
decl
|
||||
for decl in pending
|
||||
if decl.is_layout_root or decl.layout_source in built
|
||||
]
|
||||
if not ready:
|
||||
raise ValueError(
|
||||
f"cyclic layout_source dependencies: {[decl.pool_name for decl in pending]}"
|
||||
)
|
||||
for decl in ready:
|
||||
ordered.append(decl)
|
||||
built.add(decl.pool_name)
|
||||
pending.remove(decl)
|
||||
return tuple(ordered)
|
||||
|
||||
|
||||
def prepare_host_pool_config(
|
||||
*,
|
||||
decls: tuple[HostPoolDecl, ...],
|
||||
full_layer_mapping: dict[int, int],
|
||||
transfer_layer_id_max: int,
|
||||
transfer_page_size: int,
|
||||
packed_draft_device_pools: tuple[Any, ...] = (),
|
||||
) -> HostPoolGroupConfig:
|
||||
"""Validate declarations and buffers, then append packed draft transfer layers.
|
||||
|
||||
The input mapping and limit describe target layers only. No host memory is
|
||||
allocated here. config.pools are in host construction order.
|
||||
"""
|
||||
decls = _validate_and_order_decls(decls)
|
||||
root = decls[0]
|
||||
target_layers = root.device_pool.layer_num
|
||||
for transfer_id, device_id in full_layer_mapping.items():
|
||||
if not 0 <= transfer_id < transfer_layer_id_max:
|
||||
raise ValueError(
|
||||
f"transfer layer {transfer_id} outside [0, {transfer_layer_id_max})"
|
||||
)
|
||||
if not 0 <= device_id < target_layers:
|
||||
raise ValueError(f"device layer {device_id} outside [0, {target_layers})")
|
||||
packed_draft_decls = _validate_packed_draft_pools(
|
||||
target_decls=decls, draft_pools=packed_draft_device_pools
|
||||
)
|
||||
mapping = with_packed_draft_layer_mapping(
|
||||
full_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=target_layers,
|
||||
draft_layer_num=len(packed_draft_decls),
|
||||
)
|
||||
configs = tuple(
|
||||
HostPoolConfig(
|
||||
decl=d,
|
||||
layer_mapping=_filter_owned_layer_mapping(
|
||||
mapping=mapping,
|
||||
owned_device_layers=d.owned_device_layers,
|
||||
target_layer_num=target_layers,
|
||||
),
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(packed_draft_decls),
|
||||
packed_draft_device_pools=tuple(
|
||||
_find_pool_decl(decls=decls_of_draft, name=d.pool_name).device_pool
|
||||
for decls_of_draft in packed_draft_decls
|
||||
),
|
||||
)
|
||||
for d in decls
|
||||
)
|
||||
check_packed_kv_rows(
|
||||
kv_pool=root.device_pool,
|
||||
drafts=configs[0].packed_draft_device_pools,
|
||||
)
|
||||
for config in configs[1:]:
|
||||
config.decl.host_pool_builder.validate(
|
||||
decl=config.decl,
|
||||
transfer_page_size=transfer_page_size,
|
||||
packed_draft_device_pools=config.packed_draft_device_pools,
|
||||
)
|
||||
return HostPoolGroupConfig(transfer_page_size=transfer_page_size, pools=configs)
|
||||
|
||||
|
||||
def _filter_owned_layer_mapping(
|
||||
*,
|
||||
mapping: dict[int, int],
|
||||
owned_device_layers: Optional[tuple[int, ...]],
|
||||
target_layer_num: int,
|
||||
) -> dict[int, int]:
|
||||
"""Drop transfer layers whose device layer owns no buffer for this pool.
|
||||
Packed draft tails (device index >= target layer count) are kept."""
|
||||
if owned_device_layers is None:
|
||||
return mapping
|
||||
owned = set(owned_device_layers)
|
||||
return {
|
||||
t: dev for t, dev in mapping.items() if dev >= target_layer_num or dev in owned
|
||||
}
|
||||
|
||||
|
||||
def is_mla_pool(pool: Any) -> bool:
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
|
||||
return isinstance(pool, MLATokenToKVPool)
|
||||
|
||||
|
||||
def _kv_row_signature(pool: Any, *, use_mla: bool) -> tuple:
|
||||
if use_mla:
|
||||
return (pool.kv_cache_dim, pool.store_dtype)
|
||||
return (pool.head_num, pool.head_dim, pool.v_head_dim, pool.store_dtype)
|
||||
|
||||
|
||||
def check_packed_kv_rows(
|
||||
*, kv_pool: Any, drafts: tuple[Any, ...], use_mla: Optional[bool] = None
|
||||
):
|
||||
"""Packed draft KV layers share the target's host row, so their device
|
||||
rows must have the same shape and dtype."""
|
||||
if not drafts:
|
||||
return
|
||||
if use_mla is None:
|
||||
use_mla = is_mla_pool(kv_pool)
|
||||
target = _kv_row_signature(kv_pool, use_mla=use_mla)
|
||||
for draft in drafts:
|
||||
row = _kv_row_signature(draft, use_mla=use_mla)
|
||||
if row != target:
|
||||
raise ValueError(
|
||||
f"packed draft KV row {row} differs from target KV row {target}"
|
||||
)
|
||||
@@ -9,6 +9,13 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolName,
|
||||
SidecarPoolSpec,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.host_pool_config import (
|
||||
HostPoolGroupConfig,
|
||||
check_packed_kv_rows,
|
||||
is_mla_pool,
|
||||
prepare_host_pool_config,
|
||||
with_packed_draft_layer_mapping,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
HybridCacheController,
|
||||
)
|
||||
@@ -19,7 +26,7 @@ from sglang.srt.mem_cache.memory_pool_host import (
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
|
||||
from sglang.srt.mem_cache.pool_host.common import get_allocator_type
|
||||
from sglang.srt.mem_cache.pool_host.dsa import DSAIndexerPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.host_pool_decl import HostPoolDecl
|
||||
from sglang.srt.mem_cache.pool_host.mamba import MambaPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.mha import (
|
||||
MHATokenToKOnlyPoolHost,
|
||||
@@ -87,19 +94,6 @@ def _stage_local_layer_mapping(
|
||||
}
|
||||
|
||||
|
||||
def _with_mtp_layer_mapping(
|
||||
layer_mapping: dict[int, int],
|
||||
*,
|
||||
transfer_layer_start: int,
|
||||
target_device_layer_num: int,
|
||||
draft_layer_num: int,
|
||||
) -> dict[int, int]:
|
||||
return layer_mapping | {
|
||||
transfer_layer_start + depth: target_device_layer_num + depth
|
||||
for depth in range(draft_layer_num)
|
||||
}
|
||||
|
||||
|
||||
class _DeepSeekV4LayerMappings(NamedTuple):
|
||||
transfer_layer_id_max: int
|
||||
full: dict[int, int]
|
||||
@@ -142,15 +136,24 @@ def build_kv_host_pool(
|
||||
*,
|
||||
kv_pool: Any,
|
||||
page_size: int,
|
||||
use_mla: bool,
|
||||
use_mla: Optional[bool] = None,
|
||||
override_kv_cache_dim: Optional[int] = None,
|
||||
host_size: Optional[float] = None,
|
||||
mtp_draft_device_pools: tuple[Any, ...] = (),
|
||||
pool_label: str = "kv",
|
||||
):
|
||||
if use_mla is None:
|
||||
use_mla = is_mla_pool(kv_pool)
|
||||
kv_host_pool_cls = (
|
||||
MLATokenToKVPoolHost if use_mla else get_mha_host_pool_cls(kv_pool)
|
||||
)
|
||||
if use_mla and override_kv_cache_dim is None:
|
||||
# An fp8 DSA store is wider than kv_lora_rank + qk_rope_head_dim. The
|
||||
# device pool already resolved the row width.
|
||||
override_kv_cache_dim = kv_pool.kv_cache_dim
|
||||
check_packed_kv_rows(
|
||||
kv_pool=kv_pool, drafts=mtp_draft_device_pools, use_mla=use_mla
|
||||
)
|
||||
kwargs = {}
|
||||
if override_kv_cache_dim is not None:
|
||||
kwargs["override_kv_cache_dim"] = override_kv_cache_dim
|
||||
@@ -245,7 +248,7 @@ def build_kv_only_group(
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
)
|
||||
if mtp_draft_device_pools:
|
||||
full_layer_mapping = _with_mtp_layer_mapping(
|
||||
full_layer_mapping = with_packed_draft_layer_mapping(
|
||||
full_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=kv_pool.layer_num,
|
||||
@@ -356,7 +359,7 @@ def build_hybrid_swa_group(
|
||||
pool_label="swa",
|
||||
)
|
||||
if mtp_swa_device_pools:
|
||||
swa_layer_mapping = _with_mtp_layer_mapping(
|
||||
swa_layer_mapping = with_packed_draft_layer_mapping(
|
||||
swa_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=swa_kv_pool.layer_num,
|
||||
@@ -856,7 +859,7 @@ def build_deepseek_v4_hicache_stack(
|
||||
for pool in params.mtp_draft_device_pools
|
||||
for buffer in pool.swa_kv_pool.kv_buffer
|
||||
]
|
||||
swa_layer_mapping = _with_mtp_layer_mapping(
|
||||
swa_layer_mapping = with_packed_draft_layer_mapping(
|
||||
swa_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=transfer_layer_id_max,
|
||||
@@ -1155,7 +1158,7 @@ def build_hybrid_mamba_stack(
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
)
|
||||
if mtp_draft_device_pools:
|
||||
full_layer_mapping = _with_mtp_layer_mapping(
|
||||
full_layer_mapping = with_packed_draft_layer_mapping(
|
||||
full_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=kv_pool.layer_num,
|
||||
@@ -1345,82 +1348,41 @@ def build_hybrid_mamba_swa_stack(
|
||||
return host_pool_group, cache_controller
|
||||
|
||||
|
||||
def build_anchor_sidecar_stack(
|
||||
def build_host_pool_group(
|
||||
*,
|
||||
params: CacheInitParams,
|
||||
kv_pool: Any,
|
||||
sidecar_pool_name: PoolName,
|
||||
full_layer_mapping: dict[int, int],
|
||||
load_cache_event,
|
||||
storage_backend: Optional[str],
|
||||
use_mla: bool,
|
||||
override_kv_cache_dim: Optional[int] = None,
|
||||
sidecar_host_pool_factory: Callable[[Any], Any],
|
||||
prefetch_threshold: int = 256,
|
||||
model_name: Optional[str] = None,
|
||||
storage_backend_extra_config: Optional[dict] = None,
|
||||
enable_storage_metrics: bool = False,
|
||||
) -> tuple[HostPoolGroup, HybridCacheController]:
|
||||
transfer_layer_id_max = len(full_layer_mapping)
|
||||
mtp_draft_device_pools = tuple(
|
||||
pool for pool in params.mtp_draft_device_pools if pool.index_k_with_scale_buffer
|
||||
)
|
||||
kv_host_pool = build_kv_host_pool(
|
||||
kv_pool=kv_pool,
|
||||
page_size=params.page_size,
|
||||
use_mla=use_mla,
|
||||
override_kv_cache_dim=override_kv_cache_dim,
|
||||
mtp_draft_device_pools=mtp_draft_device_pools,
|
||||
)
|
||||
sidecar_host_pool = sidecar_host_pool_factory(kv_host_pool)
|
||||
# Expose packed MTP tail layers to the controller's flat transfer builder.
|
||||
if mtp_draft_device_pools:
|
||||
full_layer_mapping = _with_mtp_layer_mapping(
|
||||
full_layer_mapping,
|
||||
transfer_layer_start=transfer_layer_id_max,
|
||||
target_device_layer_num=kv_pool.layer_num,
|
||||
draft_layer_num=len(mtp_draft_device_pools),
|
||||
config: HostPoolGroupConfig,
|
||||
) -> HostPoolGroup:
|
||||
"""Allocate host pools and transfer entries from prepared configs."""
|
||||
root = config.pools[0]
|
||||
host_pools = {
|
||||
root.decl.pool_name: build_kv_host_pool(
|
||||
kv_pool=root.decl.device_pool,
|
||||
page_size=config.transfer_page_size,
|
||||
mtp_draft_device_pools=root.packed_draft_device_pools,
|
||||
)
|
||||
entries = [
|
||||
build_pool_entry(
|
||||
name=PoolName.KV,
|
||||
host_pool=kv_host_pool,
|
||||
device_pool=kv_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools),
|
||||
is_anchor=True,
|
||||
packed_draft_device_pools=mtp_draft_device_pools,
|
||||
),
|
||||
build_pool_entry(
|
||||
name=sidecar_pool_name,
|
||||
host_pool=sidecar_host_pool,
|
||||
device_pool=kv_pool,
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max + len(mtp_draft_device_pools),
|
||||
packed_draft_device_pools=mtp_draft_device_pools,
|
||||
),
|
||||
]
|
||||
host_pool_group = HostPoolGroup(entries)
|
||||
cache_controller = HybridCacheController(
|
||||
params.token_to_kv_pool_allocator,
|
||||
host_pool_group,
|
||||
params.page_size,
|
||||
params.tp_cache_group,
|
||||
load_cache_event=load_cache_event,
|
||||
attn_cp_group=params.attn_cp_cache_group,
|
||||
attn_tp_group=params.attn_tp_cache_group,
|
||||
pp_group=params.pp_cache_group,
|
||||
write_policy=get_memory().hicache_write_policy,
|
||||
io_backend=get_memory().hicache_io_backend,
|
||||
storage_backend=storage_backend,
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
model_name=model_name,
|
||||
storage_backend_extra_config=storage_backend_extra_config,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
enable_storage_metrics=enable_storage_metrics,
|
||||
host_memory_mode=get_memory().hicache_host_memory_mode,
|
||||
)
|
||||
return host_pool_group, cache_controller
|
||||
}
|
||||
entries = []
|
||||
for pool_config in config.pools:
|
||||
decl = pool_config.decl
|
||||
if not decl.is_layout_root:
|
||||
host_pools[decl.pool_name] = decl.host_pool_builder.build(
|
||||
decl=decl,
|
||||
anchor_host=host_pools[decl.layout_source],
|
||||
allocator_type=_get_allocator_type(),
|
||||
packed_draft_device_pools=pool_config.packed_draft_device_pools,
|
||||
)
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=decl.pool_name,
|
||||
host_pool=host_pools[decl.pool_name],
|
||||
device_pool=decl.device_pool,
|
||||
layer_mapping=pool_config.layer_mapping,
|
||||
transfer_layer_id_max=pool_config.transfer_layer_id_max,
|
||||
is_anchor=decl.is_primary,
|
||||
packed_draft_device_pools=pool_config.packed_draft_device_pools,
|
||||
)
|
||||
)
|
||||
return HostPoolGroup(entries)
|
||||
|
||||
|
||||
def _build_mha_mla_host_pool(
|
||||
@@ -1504,18 +1466,20 @@ def build_full_draft_pools(
|
||||
]
|
||||
|
||||
if isinstance(pool, DSATokenToKVPool) and pool.index_k_with_scale_buffer:
|
||||
from sglang.srt.mem_cache.pool_host.dsa import (
|
||||
DSAIndexerPoolHost,
|
||||
make_dsa_indexer_pool_decl,
|
||||
)
|
||||
|
||||
# Separate draft indexer: its own host pool laid out on the draft KV
|
||||
# host pool, but transfer indices still follow the target KV anchor.
|
||||
indexer_decl = make_dsa_indexer_pool_decl(pool, name=PoolName.DRAFT_INDEXER)
|
||||
indexer_host_pool = DSAIndexerPoolHost(
|
||||
pool,
|
||||
draft_host_pool,
|
||||
get_memory().hicache_mem_layout,
|
||||
decl=indexer_decl,
|
||||
anchor_host=draft_host_pool,
|
||||
allocator_type=_get_allocator_type(),
|
||||
)
|
||||
specs.append(
|
||||
SidecarPoolSpec(
|
||||
pool_name=PoolName.DRAFT_INDEXER,
|
||||
indices_from_pool=PoolName.KV,
|
||||
)
|
||||
)
|
||||
specs.append(indexer_decl.sidecar_spec())
|
||||
entries.append(
|
||||
build_pool_entry(
|
||||
name=PoolName.DRAFT_INDEXER,
|
||||
@@ -1626,6 +1590,7 @@ class StackBuildResult:
|
||||
# layer_transfer_counter has to be wired separately.
|
||||
register_req_to_token_counter: bool = False
|
||||
pools_desc: str = ""
|
||||
pool_declarations: Optional[tuple[HostPoolDecl, ...]] = None
|
||||
|
||||
|
||||
class StackStrategy:
|
||||
@@ -1976,78 +1941,6 @@ class _MambaSwaStrategy(StackStrategy):
|
||||
)
|
||||
|
||||
|
||||
class _DsaStrategy(StackStrategy):
|
||||
def matches(self, kvcache, components):
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
|
||||
return isinstance(kvcache, DSATokenToKVPool) and components == {
|
||||
ComponentType.FULL
|
||||
}
|
||||
|
||||
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
|
||||
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
|
||||
_build_dsa_device_pool_group,
|
||||
)
|
||||
|
||||
return _build_dsa_device_pool_group(
|
||||
kvcache, page_size, params.mtp_draft_device_pools
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
cache,
|
||||
kvcache,
|
||||
params,
|
||||
server_args,
|
||||
load_cache_event,
|
||||
storage_backend=None,
|
||||
storage_backend_extra_config=None,
|
||||
prefetch_threshold=256,
|
||||
model_name=None,
|
||||
enable_storage_metrics=False,
|
||||
):
|
||||
from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool
|
||||
|
||||
full_kv_pool = kvcache
|
||||
use_mla = isinstance(kvcache, MLATokenToKVPool)
|
||||
full_layer_mapping = {i: i for i in range(full_kv_pool.layer_num)}
|
||||
host_pool_group, cache_controller = build_anchor_sidecar_stack(
|
||||
params=params,
|
||||
kv_pool=full_kv_pool,
|
||||
sidecar_pool_name=PoolName.INDEXER,
|
||||
full_layer_mapping=full_layer_mapping,
|
||||
load_cache_event=load_cache_event,
|
||||
storage_backend=storage_backend,
|
||||
use_mla=use_mla,
|
||||
override_kv_cache_dim=full_kv_pool.kv_cache_dim,
|
||||
sidecar_host_pool_factory=lambda kv_host_pool: DSAIndexerPoolHost(
|
||||
full_kv_pool,
|
||||
kv_host_pool,
|
||||
get_memory().hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(),
|
||||
),
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
model_name=model_name,
|
||||
storage_backend_extra_config=storage_backend_extra_config,
|
||||
enable_storage_metrics=enable_storage_metrics,
|
||||
)
|
||||
return StackBuildResult(
|
||||
host_pool_group=host_pool_group,
|
||||
cache_controller=cache_controller,
|
||||
component_host_pools={
|
||||
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
|
||||
},
|
||||
sidecars=[
|
||||
SidecarPoolSpec(
|
||||
pool_name=PoolName.INDEXER,
|
||||
indices_from_pool=PoolName.KV,
|
||||
),
|
||||
],
|
||||
pools_desc="KV + INDEXER",
|
||||
)
|
||||
|
||||
|
||||
class _MiniMaxSparseStrategy(StackStrategy):
|
||||
def matches(self, kvcache, components):
|
||||
from sglang.srt.mem_cache.memory_pool import MiniMaxSparseKVPool
|
||||
@@ -2165,6 +2058,79 @@ class _PlainKvStrategy(StackStrategy):
|
||||
)
|
||||
|
||||
|
||||
class _DsaStrategy(StackStrategy):
|
||||
def matches(self, kvcache, components):
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
|
||||
return isinstance(kvcache, DSATokenToKVPool) and components == {
|
||||
ComponentType.FULL
|
||||
}
|
||||
|
||||
def build_direct_linker_pool_group(self, *, kvcache, params, page_size):
|
||||
from sglang.srt.mem_cache.hybrid_cache.linker_pool_assembler import (
|
||||
_build_dsa_device_pool_group,
|
||||
)
|
||||
|
||||
return _build_dsa_device_pool_group(
|
||||
kvcache, page_size, params.mtp_draft_device_pools
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
cache,
|
||||
kvcache,
|
||||
params,
|
||||
server_args,
|
||||
load_cache_event,
|
||||
storage_backend=None,
|
||||
storage_backend_extra_config=None,
|
||||
prefetch_threshold=256,
|
||||
model_name=None,
|
||||
enable_storage_metrics=False,
|
||||
):
|
||||
config = prepare_host_pool_config(
|
||||
decls=kvcache.host_pool_decls(),
|
||||
full_layer_mapping={i: i for i in range(kvcache.layer_num)},
|
||||
transfer_layer_id_max=kvcache.layer_num,
|
||||
transfer_page_size=params.page_size,
|
||||
packed_draft_device_pools=params.mtp_draft_device_pools,
|
||||
)
|
||||
host_pool_group = build_host_pool_group(config=config)
|
||||
cache_controller = HybridCacheController(
|
||||
params.token_to_kv_pool_allocator,
|
||||
host_pool_group,
|
||||
config.transfer_page_size,
|
||||
params.tp_cache_group,
|
||||
load_cache_event=load_cache_event,
|
||||
attn_cp_group=params.attn_cp_cache_group,
|
||||
attn_tp_group=params.attn_tp_cache_group,
|
||||
pp_group=params.pp_cache_group,
|
||||
write_policy=get_memory().hicache_write_policy,
|
||||
io_backend=get_memory().hicache_io_backend,
|
||||
storage_backend=storage_backend,
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
model_name=model_name,
|
||||
storage_backend_extra_config=storage_backend_extra_config,
|
||||
# The controller adds packed draft transfers after target layers.
|
||||
transfer_layer_id_max=kvcache.layer_num,
|
||||
enable_storage_metrics=enable_storage_metrics,
|
||||
host_memory_mode=get_memory().hicache_host_memory_mode,
|
||||
)
|
||||
return StackBuildResult(
|
||||
host_pool_group=host_pool_group,
|
||||
cache_controller=cache_controller,
|
||||
component_host_pools={
|
||||
ComponentType.FULL: host_pool_group.get_pool(PoolName.KV),
|
||||
},
|
||||
sidecars=[
|
||||
c.decl.sidecar_spec() for c in config.pools if not c.decl.is_primary
|
||||
],
|
||||
pools_desc=" + ".join(c.decl.pool_name.value.upper() for c in config.pools),
|
||||
pool_declarations=tuple(c.decl for c in config.pools),
|
||||
)
|
||||
|
||||
|
||||
# Resolved first-to-last; _PlainKvStrategy is the catch-all fallback.
|
||||
_STRATEGIES: list[StackStrategy] = [
|
||||
_DeepSeekV4Strategy(),
|
||||
@@ -2193,6 +2159,33 @@ def _select_strategy(kvcache: Any, components: set[ComponentType]) -> StackStrat
|
||||
)
|
||||
|
||||
|
||||
# Strategies that assemble from host_pool_decls(), so every declared pool has an
|
||||
# entry by construction. A miss here is a bug, not an unsupported combination.
|
||||
_DECLARATION_VERIFIED_STRATEGIES: tuple[type, ...] = (_DsaStrategy,)
|
||||
|
||||
|
||||
def _check_declared_pools_present(
|
||||
*, kvcache: Any, result: StackBuildResult, strategy: StackStrategy
|
||||
) -> None:
|
||||
entries = result.host_pool_group.entry_map
|
||||
decls = result.pool_declarations
|
||||
if decls is None:
|
||||
decls = kvcache.host_pool_decls()
|
||||
missing = [d.pool_name.value for d in decls if d.pool_name not in entries]
|
||||
if not missing:
|
||||
return
|
||||
msg = (
|
||||
f"{type(kvcache).__name__} declares host pools {missing} that "
|
||||
f"{type(strategy).__name__} did not assemble"
|
||||
)
|
||||
if isinstance(strategy, _DECLARATION_VERIFIED_STRATEGIES):
|
||||
raise ValueError(msg)
|
||||
# Pre-declaration strategies: restoring KV without these pools (e.g. the
|
||||
# indexer's key buffers) corrupts sparse attention after a host hit.
|
||||
# Loud until the path is migrated.
|
||||
logger.error("%s; host restore of these pools is unsupported on this path", msg)
|
||||
|
||||
|
||||
def _apply_stack_result(
|
||||
cache: UnifiedRadixCache,
|
||||
kvcache: Any,
|
||||
@@ -2250,6 +2243,7 @@ def attach_hybrid_pool_to_unified_cache(
|
||||
model_name=get_serving().served_model_name,
|
||||
enable_storage_metrics=cache._enable_metrics_flag,
|
||||
)
|
||||
_check_declared_pools_present(kvcache=kvcache, result=result, strategy=strategy)
|
||||
_apply_stack_result(cache, kvcache, params, result)
|
||||
except Exception:
|
||||
logger.exception("attach_hybrid_pool_to_unified_cache failed")
|
||||
|
||||
@@ -1874,6 +1874,17 @@ class KVCache(abc.ABC):
|
||||
maybe_init_custom_mem_pool(device=self.device)
|
||||
)
|
||||
|
||||
def host_pool_decls(self):
|
||||
"""Host pools HiCache keeps for the buffers this device pool (and its
|
||||
sub-pools) owns: the KV pool itself, plus dependent pools such as sparse
|
||||
index keys in subclasses. Not a description of the whole model: buffers
|
||||
owned elsewhere (e.g. Mamba state in req_to_token_pool) are declared by
|
||||
their owner or assembled by the existing strategy paths."""
|
||||
# pool_host imports this module. Resolve the declaration types lazily.
|
||||
from sglang.srt.mem_cache.pool_host.host_pool_decl import make_kv_pool_decl
|
||||
|
||||
return (make_kv_pool_decl(self),)
|
||||
|
||||
def _finalize_allocation_log(self, num_tokens: int):
|
||||
"""Common logging and mem_usage computation for KV cache allocation.
|
||||
Supports both tuple (K, V) size returns and single KV size returns.
|
||||
@@ -4014,6 +4025,12 @@ class HybridLinearKVPool(KVCache):
|
||||
k_size, v_size = self.get_kv_size_bytes()
|
||||
self.mem_usage = (k_size + v_size) / GB
|
||||
|
||||
def host_pool_decls(self):
|
||||
# Only the full-attention sub-pool owns HiCache-addressable buffers here.
|
||||
# Mamba state belongs to req_to_token_pool's MambaPool and is assembled
|
||||
# by _MambaStrategy, not declared through this pool.
|
||||
return self.full_kv_pool.host_pool_decls()
|
||||
|
||||
@property
|
||||
def post_capture_active(self) -> bool:
|
||||
return self.full_kv_pool.post_capture_active
|
||||
@@ -4966,6 +4983,17 @@ class DSATokenToKVPool(MLATokenToKVPool):
|
||||
def _should_allocate_index_layer(self, local_layer_idx: int) -> bool:
|
||||
return not self.skip_topk_layers[local_layer_idx]
|
||||
|
||||
def host_pool_decls(self):
|
||||
# pool_host imports this module. Resolve the mirror side lazily.
|
||||
from sglang.srt.mem_cache.pool_host.dsa import make_dsa_indexer_pool_decl
|
||||
|
||||
kv_decls = super().host_pool_decls()
|
||||
# Shared-topk layers own a 0-row placeholder, so a non-empty buffer list
|
||||
# is not enough: some layer must actually hold index keys.
|
||||
if not self.index_k_with_scale_buffer or all(self.skip_topk_layers):
|
||||
return kv_decls
|
||||
return (*kv_decls, make_dsa_indexer_pool_decl(self))
|
||||
|
||||
@property
|
||||
def index_k_with_scale_buffer(self):
|
||||
# Preserve direct HiCache access while storage lives behind the facade.
|
||||
|
||||
@@ -15,6 +15,7 @@ from sglang.kernels.ops.kvcache.hicache import (
|
||||
from sglang.kernels.ops.kvcache.hicache import (
|
||||
transfer_hicache_all_layer_mla_staged_lf_pf as jit_transfer_hicache_all_layer_mla_staged_lf_pf,
|
||||
)
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName
|
||||
from sglang.srt.mem_cache.memory_pool import DSATokenToKVPool
|
||||
from sglang.srt.mem_cache.pool_host.base import (
|
||||
_WRITE_BACK_STAGING_PAGE_CHUNK,
|
||||
@@ -26,6 +27,10 @@ from sglang.srt.mem_cache.pool_host.common import (
|
||||
get_allocator_from_storage,
|
||||
make_kernel_ptr_table,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.host_pool_decl import (
|
||||
HostPoolDecl,
|
||||
HostPoolStorageInfo,
|
||||
)
|
||||
from sglang.srt.utils import is_cuda, is_hip, is_mps, is_npu, is_xpu
|
||||
|
||||
_is_cuda = is_cuda()
|
||||
@@ -47,6 +52,73 @@ if _is_cuda or _is_hip:
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def dsa_indexer_bytes_per_token_per_layer(
|
||||
*, index_head_dim: int, quant_block_size: int
|
||||
) -> int:
|
||||
# packed index keys plus one fp32 scale per quant block, stored as uint8
|
||||
elems = index_head_dim + index_head_dim // quant_block_size * 4
|
||||
return elems * DSATokenToKVPool.index_k_with_scale_buffer_dtype.itemsize
|
||||
|
||||
|
||||
class DSAIndexerHostPoolBuilder:
|
||||
def validate(
|
||||
self,
|
||||
*,
|
||||
decl: HostPoolDecl,
|
||||
transfer_page_size: int,
|
||||
packed_draft_device_pools: tuple[DSATokenToKVPool, ...],
|
||||
) -> None:
|
||||
target = decl.device_pool
|
||||
target_format = (target.index_head_dim, target.quant_block_size)
|
||||
for draft in packed_draft_device_pools:
|
||||
draft_format = (draft.index_head_dim, draft.quant_block_size)
|
||||
if draft_format != target_format:
|
||||
raise ValueError(
|
||||
f"{decl.pool_name}: packed index key format {draft_format} "
|
||||
f"differs from target {target_format}"
|
||||
)
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
decl: HostPoolDecl,
|
||||
anchor_host: MLATokenToKVPoolHost,
|
||||
allocator_type: str,
|
||||
packed_draft_device_pools: tuple[DSATokenToKVPool, ...],
|
||||
) -> DSAIndexerPoolHost:
|
||||
return DSAIndexerPoolHost(
|
||||
decl=decl,
|
||||
anchor_host=anchor_host,
|
||||
packed_draft_device_pools=packed_draft_device_pools,
|
||||
allocator_type=allocator_type,
|
||||
)
|
||||
|
||||
|
||||
def make_dsa_indexer_pool_decl(
|
||||
pool: DSATokenToKVPool, *, name: PoolName = PoolName.INDEXER
|
||||
) -> HostPoolDecl:
|
||||
"""Index key buffers riding on the full-KV pages: indices and layout both follow KV."""
|
||||
return HostPoolDecl(
|
||||
pool_name=name,
|
||||
device_pool=pool,
|
||||
indices_from_pool=PoolName.KV,
|
||||
layout_source=PoolName.KV,
|
||||
storage_info=HostPoolStorageInfo(
|
||||
bytes_per_token_per_layer=dsa_indexer_bytes_per_token_per_layer(
|
||||
index_head_dim=pool.index_head_dim,
|
||||
quant_block_size=pool.quant_block_size,
|
||||
),
|
||||
dtype=DSATokenToKVPool.index_k_with_scale_buffer_dtype,
|
||||
),
|
||||
host_pool_builder=DSAIndexerHostPoolBuilder(),
|
||||
# Shared-topk layers own a 0-row placeholder buffer. They get no host
|
||||
# layer and must not reach the transfer kernels.
|
||||
owned_device_layers=tuple(
|
||||
i for i, skip in enumerate(pool.skip_topk_layers) if not skip
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class DSAIndexerPoolHost(HostKVCache):
|
||||
"""Host-side DSA index buffers only. Slot layout matches the anchor MLA host pool."""
|
||||
|
||||
@@ -54,46 +126,53 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_pool: DSATokenToKVPool,
|
||||
decl: HostPoolDecl,
|
||||
anchor_host: MLATokenToKVPoolHost,
|
||||
layout: str,
|
||||
*,
|
||||
packed_draft_device_pools: tuple[DSATokenToKVPool, ...] = (),
|
||||
pin_memory: bool = True,
|
||||
device: str = "cpu",
|
||||
allocator_type: str = "default",
|
||||
is_dummy: bool = False,
|
||||
):
|
||||
self._is_dummy = is_dummy
|
||||
self.decl = decl
|
||||
storage_info = decl.storage_info
|
||||
device_pool = decl.device_pool
|
||||
self.device_pool = device_pool
|
||||
self.page_size = anchor_host.page_size
|
||||
self.layout = layout
|
||||
self.layout = anchor_host.layout
|
||||
self.pin_memory = pin_memory
|
||||
self.device = device
|
||||
self.allocator = get_allocator_from_storage(allocator_type)
|
||||
self.dtype = device_pool.store_dtype
|
||||
self.start_layer = device_pool.start_layer
|
||||
self.end_layer = device_pool.end_layer
|
||||
self.target_layer_num = self._effective_host_layer_num()
|
||||
self.mtp_draft_device_pools = anchor_host.mtp_draft_device_pools
|
||||
# Host layers are compact: only owned device layers that hold index
|
||||
# buffers, then one tail layer per packed draft pool.
|
||||
owned_start, owned_end = self._device_owned_layer_range()
|
||||
declared = decl.owned_device_layers
|
||||
self._live_target_layers = [
|
||||
layer
|
||||
for layer in range(owned_start, owned_end)
|
||||
if declared is None or layer in declared
|
||||
]
|
||||
self._device_to_host_layer = {
|
||||
layer: i for i, layer in enumerate(self._live_target_layers)
|
||||
}
|
||||
self.target_layer_num = len(self._live_target_layers)
|
||||
self.mtp_draft_device_pools = tuple(packed_draft_device_pools)
|
||||
self.layer_num = self.target_layer_num + len(self.mtp_draft_device_pools)
|
||||
|
||||
self.index_head_dim = device_pool.index_head_dim
|
||||
self.indexer_quant_block_size = device_pool.quant_block_size
|
||||
self.indexer_dtype = DSATokenToKVPool.index_k_with_scale_buffer_dtype
|
||||
self.indexer_size_per_token = (
|
||||
self.index_head_dim
|
||||
+ self.index_head_dim // self.indexer_quant_block_size * 4
|
||||
)
|
||||
self.indexer_dtype = storage_info.dtype
|
||||
self.size = anchor_host.size
|
||||
self.page_num = anchor_host.page_num
|
||||
|
||||
self.indexer_page_stride_size = (
|
||||
self.indexer_size_per_token * self.page_size * self.indexer_dtype.itemsize
|
||||
)
|
||||
# uint8 storage, so element counts below are byte counts
|
||||
self.indexer_page_stride_size = storage_info.page_bytes(self.page_size)
|
||||
self.indexer_layout_dim = self.indexer_page_stride_size * self.layer_num
|
||||
self.indexer_page_num = (self.size + self.page_size + 1) // self.page_size
|
||||
self.size_per_token = (
|
||||
self.indexer_size_per_token * self.layer_num * self.indexer_dtype.itemsize
|
||||
)
|
||||
self.size_per_token = storage_info.bytes_per_token_per_layer * self.layer_num
|
||||
|
||||
self.can_use_jit = False
|
||||
self.can_use_write_back_jit = False
|
||||
@@ -109,8 +188,9 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
self.clear()
|
||||
return
|
||||
|
||||
buf_elem_size = self.page_num * self.layer_num * self.indexer_page_stride_size
|
||||
requested_bytes = buf_elem_size * self.indexer_dtype.itemsize
|
||||
requested_bytes = storage_info.host_bytes(
|
||||
page_num=self.page_num, layer_num=self.layer_num, page_size=self.page_size
|
||||
)
|
||||
available_bytes = host_memory_budget_bytes(requested_bytes)
|
||||
if requested_bytes > available_bytes:
|
||||
raise ValueError(
|
||||
@@ -125,7 +205,7 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
"packed MTP layers: "
|
||||
"target_layers=%d, draft_layers=%d, total_layers=%d.",
|
||||
requested_bytes / 1e9,
|
||||
layout,
|
||||
self.layout,
|
||||
self.target_layer_num,
|
||||
draft_layer_num,
|
||||
self.layer_num,
|
||||
@@ -134,7 +214,7 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
logger.info(
|
||||
"Allocating %.2f GB host memory for DSA indexer (layout=%s).",
|
||||
requested_bytes / 1e9,
|
||||
layout,
|
||||
self.layout,
|
||||
)
|
||||
self.init_kv_buffer()
|
||||
self._init_write_back_staging_buffers()
|
||||
@@ -142,18 +222,33 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
self.clear()
|
||||
|
||||
def get_size_per_token(self):
|
||||
return (
|
||||
self.indexer_size_per_token * self.layer_num * self.indexer_dtype.itemsize
|
||||
)
|
||||
return self.decl.storage_info.bytes_per_token_per_layer * self.layer_num
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return self.get_size_per_token()
|
||||
|
||||
def _is_device_layer_owned(self, device_pool, layer_id: int) -> bool:
|
||||
return layer_id in self._device_to_host_layer
|
||||
|
||||
def _host_layer_index(self, layer_id: int, device_pool=None) -> int:
|
||||
return self._device_to_host_layer[layer_id]
|
||||
|
||||
def _owned_device_layer_ids(self, device_pool) -> list[int]:
|
||||
return list(self._live_target_layers)
|
||||
|
||||
def _draft_host_layer(self, layer_id: int) -> int:
|
||||
# The controller hands packed drafts ``target_device_layer_num + depth``.
|
||||
return self.target_layer_num + (layer_id - self.device_pool.layer_num)
|
||||
|
||||
def init_kv_buffer(self):
|
||||
alloc_func = ALLOC_MEMORY_FUNCS[self.device_pool.device]
|
||||
device_pools = (self.device_pool, *self.mtp_draft_device_pools)
|
||||
self.packed_device_index_buffers = [
|
||||
buffer for pool in device_pools for buffer in pool.index_k_with_scale_buffer
|
||||
self.device_pool.index_k_with_scale_buffer[layer]
|
||||
for layer in self._live_target_layers
|
||||
] + [
|
||||
buffer
|
||||
for pool in self.mtp_draft_device_pools
|
||||
for buffer in pool.index_k_with_scale_buffer
|
||||
]
|
||||
self.index_k_device_ptrs = torch.tensor(
|
||||
[x.data_ptr() for x in self.packed_device_index_buffers],
|
||||
@@ -249,7 +344,11 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
"load on a dummy (non-src DSA) host pool"
|
||||
)
|
||||
# MTP draft layers do not participate in CP layer sharding.
|
||||
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
|
||||
host_layer_id = (
|
||||
self._draft_host_layer(layer_id)
|
||||
if is_draft
|
||||
else self._host_layer_index(layer_id)
|
||||
)
|
||||
device_layer_id = 0 if is_draft else layer_id
|
||||
|
||||
host_page_indices, device_page_indices = self._get_indexer_page_indices(
|
||||
@@ -314,7 +413,11 @@ class DSAIndexerPoolHost(HostKVCache):
|
||||
"backup on a dummy (non-src DSA) host pool"
|
||||
)
|
||||
# MTP draft layers do not participate in CP layer sharding.
|
||||
host_layer_id = layer_id if is_draft else self._host_layer_index(layer_id)
|
||||
host_layer_id = (
|
||||
self._draft_host_layer(layer_id)
|
||||
if is_draft
|
||||
else self._host_layer_index(layer_id)
|
||||
)
|
||||
device_layer_id = 0 if is_draft else layer_id
|
||||
|
||||
host_page_indices, device_page_indices = self._get_indexer_page_indices(
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Declarations of the host pools a device pool needs HiCache to keep.
|
||||
|
||||
A device pool declares them (``HostPoolDecl``) through ``host_pool_decls()``.
|
||||
The assembler (hybrid_pool_assembler) binds each declaration to transfer layers
|
||||
and builds one entry per declaration. This module holds only the declaration
|
||||
types and their basic constructors.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional, Protocol
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolHitPolicy,
|
||||
PoolName,
|
||||
SidecarPoolSpec,
|
||||
)
|
||||
|
||||
|
||||
class HostPoolStorageInfo(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Bytes one host pool stores per token per layer, and their dtype.
|
||||
|
||||
Two equal values mean equal byte counts and dtype only. Page geometry,
|
||||
scale placement and kernel layout are checked by the host pool builder.
|
||||
"""
|
||||
|
||||
bytes_per_token_per_layer: int
|
||||
dtype: torch.dtype
|
||||
|
||||
def page_bytes(self, page_size: int) -> int:
|
||||
"""Bytes of one page of one layer."""
|
||||
return self.bytes_per_token_per_layer * page_size
|
||||
|
||||
def host_bytes(self, *, page_num: int, layer_num: int, page_size: int) -> int:
|
||||
return page_num * layer_num * self.page_bytes(page_size)
|
||||
|
||||
|
||||
class HostPoolBuilder(Protocol):
|
||||
def validate(
|
||||
self,
|
||||
*,
|
||||
decl: HostPoolDecl,
|
||||
# Original-token slots, not the number of physical rows in this pool.
|
||||
transfer_page_size: int,
|
||||
packed_draft_device_pools: tuple[Any, ...],
|
||||
) -> None: ...
|
||||
|
||||
def build(
|
||||
self,
|
||||
*,
|
||||
decl: HostPoolDecl,
|
||||
anchor_host: Any,
|
||||
allocator_type: str,
|
||||
packed_draft_device_pools: tuple[Any, ...],
|
||||
) -> Any: ...
|
||||
|
||||
|
||||
class HostPoolDecl(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""One host pool a device pool asks HiCache to keep. Pool-intrinsic: no layer binding."""
|
||||
|
||||
pool_name: PoolName
|
||||
# The pool object that owns the device buffers behind this host pool. A
|
||||
# composite pool keeps its KV buffers on a sub-pool and, e.g., its
|
||||
# compressed index keys on itself, so each declaration names its owner.
|
||||
device_pool: Any
|
||||
# Whose host page indices this pool reuses when transferred. None: primary.
|
||||
indices_from_pool: Optional[PoolName]
|
||||
# Whose host pool decides this pool's capacity and memory layout. None: self.
|
||||
layout_source: Optional[PoolName]
|
||||
# None for a layout root (KV, DRAFT): the assembler builds its host pool
|
||||
# from the device pool, so no byte facts are declared for it.
|
||||
storage_info: Optional[HostPoolStorageInfo]
|
||||
host_pool_builder: Optional[HostPoolBuilder]
|
||||
hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES
|
||||
# Local device layers that own buffers for this pool. None means every
|
||||
# layer. Layers outside it get no host layer and no transfer.
|
||||
owned_device_layers: Optional[tuple[int, ...]] = None
|
||||
|
||||
@property
|
||||
def is_primary(self) -> bool:
|
||||
return self.indices_from_pool is None
|
||||
|
||||
@property
|
||||
def is_layout_root(self) -> bool:
|
||||
return self.layout_source is None
|
||||
|
||||
def sidecar_spec(self) -> SidecarPoolSpec:
|
||||
if self.indices_from_pool is None:
|
||||
raise ValueError(
|
||||
f"{self.pool_name} is the primary pool and has no sidecar spec"
|
||||
)
|
||||
return SidecarPoolSpec(
|
||||
pool_name=self.pool_name,
|
||||
indices_from_pool=self.indices_from_pool,
|
||||
hit_policy=self.hit_policy,
|
||||
)
|
||||
|
||||
|
||||
def make_kv_pool_decl(pool: Any) -> HostPoolDecl:
|
||||
"""The primary KV pool every device pool declares. The assembler builds its
|
||||
host pool, so no storage info is declared here."""
|
||||
return HostPoolDecl(
|
||||
pool_name=PoolName.KV,
|
||||
device_pool=pool,
|
||||
indices_from_pool=None,
|
||||
layout_source=None,
|
||||
storage_info=None,
|
||||
host_pool_builder=None,
|
||||
)
|
||||
Reference in New Issue
Block a user