Support unified memory decode host pools (#39478)

Co-authored-by: yhzhuang <yhzhuang@fb.com>
Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com>
Co-authored-by: Yonghao Zhuang <yhzhuang@users.noreply.github.com>
Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
Yonghao Zhuang
2026-09-24 11:02:15 -07:00
committed by GitHub
co-authored by yhzhuang Lianmin Zheng Yonghao Zhuang Cheng Wan
parent 984994e3aa
commit 78b382b1ad
24 changed files with 1945 additions and 186 deletions
+33 -10
View File
@@ -15,6 +15,7 @@ from sglang.srt.arg_groups.overrides import (
resolving_view,
use_mla_backend,
)
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.runtime_context import get_platform
@@ -371,7 +372,6 @@ def handle_cache_compatibility(server_args: Any) -> None:
"--disaggregation-decode-retraction-backup=host_pool requires "
"--disable-priority-preemption when priority scheduling is enabled."
)
if cfg.radix_eviction_policy == "tlru":
tlru_config = cfg.radix_eviction_policy_config or {}
threshold = tlru_config.get("threshold", 0)
@@ -467,15 +467,38 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"ships host/C4 rows straight from the allocator, bypassing the "
"virtual->physical translation the unified pool needs."
)
assert cfg.disaggregation_decode_retraction_backup != "host_pool", (
"--enable-unified-memory with PD disaggregation does not support "
"--disaggregation-decode-retraction-backup=host_pool; use "
"cpu_tensor (the automatic default for unified pools)."
)
assert not cfg.disaggregation_decode_enable_offload_kvcache, (
"--enable-unified-memory with PD disaggregation does not yet support "
"--disaggregation-decode-enable-offload-kvcache."
)
if cfg.disaggregation_decode_enable_offload_kvcache:
assert cfg.hicache_storage_backend == "file", (
"--enable-unified-memory with decode KV offload currently "
"supports --hicache-storage-backend=file only."
)
model_config = model_config_of(server_args)
assert not use_mla_backend(server_args), (
"--enable-unified-memory decode KV offload does not support "
"MLA models yet."
)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory decode KV offload does not support "
"hybrid-Mamba models."
)
assert not model_config.is_hybrid_swa, (
"--enable-unified-memory decode KV offload does not support "
"hybrid-SWA H2D/D2H transfers yet."
)
if cfg.disaggregation_decode_retraction_backup == "host_pool":
model_config = model_config_of(server_args)
assert not cfg.disaggregation_decode_enable_radix_cache, (
"--enable-unified-memory host-pool decode retraction does not "
"support decode radix-cache H2D/D2H transfers yet."
)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory host-pool decode retraction does not "
"support hybrid-Mamba models."
)
assert not model_config.is_hybrid_swa, (
"--enable-unified-memory host-pool decode retraction does not "
"support hybrid-SWA H2D/D2H transfers yet."
)
assert cfg.speculative_algorithm in (None, "DSPARK"), (
"--enable-unified-memory only supports --speculative-algorithm "
"DSPARK (chain draft); other speculative algorithms are not yet "
@@ -14,7 +14,14 @@ from sglang.srt.environ import envs
from sglang.srt.managers.cache_controller import HiCacheController
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.hicache_storage import (
PoolHitPolicy,
PoolName,
PoolTransfer,
)
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
build_hybrid_swa_stack,
build_kv_host_pool,
)
from sglang.srt.mem_cache.memory_pool import (
@@ -22,6 +29,7 @@ from sglang.srt.mem_cache.memory_pool import (
MLATokenToKVPool,
ReqToTokenPool,
)
from sglang.srt.mem_cache.unified_memory_pool import UnifiedSWAKVPool
from sglang.srt.mem_cache.utils import storage_namespace_seed
from sglang.srt.runtime_context import (
get_memory,
@@ -57,15 +65,6 @@ class DecodeKVCacheOffloadManager:
self.offload_stride = max(
self.page_size, (env_stride // self.page_size) * self.page_size
)
kv_cache = self.token_to_kv_pool_allocator.get_kvcache()
if not isinstance(kv_cache, (MHATokenToKVPool, MLATokenToKVPool)):
raise ValueError("Unsupported KV cache type for decode offload")
self.decode_host_mem_pool = build_kv_host_pool(
kv_pool=kv_cache,
page_size=self.page_size,
use_mla=isinstance(kv_cache, MLATokenToKVPool),
)
self.tp_group = tp_group
self.tp_world_size = torch.distributed.get_world_size(group=self.tp_group)
@@ -80,17 +79,56 @@ class DecodeKVCacheOffloadManager:
f"Invalid hicache storage backend extra config JSON: {e}"
)
self.cache_controller = HiCacheController(
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
mem_pool_host=self.decode_host_mem_pool,
page_size=self.page_size,
tp_group=tp_group,
io_backend=get_memory().hicache_io_backend,
load_cache_event=threading.Event(),
storage_backend=get_memory().hicache_storage_backend,
model_name=get_serving().served_model_name,
storage_backend_extra_config=hicache_storage_backend_extra_config,
)
self.kv_cache = self.token_to_kv_pool_allocator.get_kvcache()
if isinstance(self.kv_cache, UnifiedSWAKVPool):
full_mapping = {
gid: lid
for gid, (lid, is_swa) in self.kv_cache.layers_mapping.items()
if not is_swa
}
swa_mapping = {
gid: lid
for gid, (lid, is_swa) in self.kv_cache.layers_mapping.items()
if is_swa
}
params = CacheInitParams(
disable=False,
req_to_token_pool=self.req_to_token_pool,
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
page_size=self.page_size,
tp_cache_group=tp_group,
)
self.decode_host_mem_pool, self.cache_controller = build_hybrid_swa_stack(
params=params,
full_kv_pool=self.kv_cache.full_kv_pool,
swa_kv_pool=self.kv_cache.swa_kv_pool,
full_layer_mapping=full_mapping,
swa_layer_mapping=swa_mapping,
load_cache_event=threading.Event(),
storage_backend=get_memory().hicache_storage_backend,
use_mla=False,
model_name=get_serving().served_model_name,
storage_backend_extra_config=hicache_storage_backend_extra_config,
)
else:
if not isinstance(self.kv_cache, (MHATokenToKVPool, MLATokenToKVPool)):
raise ValueError("Unsupported KV cache type for decode offload")
self.decode_host_mem_pool = build_kv_host_pool(
kv_pool=self.kv_cache,
page_size=self.page_size,
use_mla=isinstance(self.kv_cache, MLATokenToKVPool),
)
self.cache_controller = HiCacheController(
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
mem_pool_host=self.decode_host_mem_pool,
page_size=self.page_size,
tp_group=tp_group,
io_backend=get_memory().hicache_io_backend,
load_cache_event=threading.Event(),
storage_backend=get_memory().hicache_storage_backend,
model_name=get_serving().served_model_name,
storage_backend_extra_config=hicache_storage_backend_extra_config,
)
self.ongoing_offload = {}
self.ongoing_backup = {}
@@ -120,6 +158,32 @@ class DecodeKVCacheOffloadManager:
# Page-aligned prompt length; the prefill instance offloaded this part.
return len(req.origin_input_ids) // self.page_size * self.page_size
def has_inflight_device_transfer(self) -> bool:
"""Whether a device-to-host copy may still read unified KV pages."""
return bool(self.ongoing_offload)
def _resolve_offload_transfers(
self, virtual_indices: torch.Tensor
) -> tuple[torch.Tensor, list[PoolTransfer]]:
full_indices = virtual_indices
if not isinstance(self.kv_cache, UnifiedSWAKVPool):
return full_indices, []
swa_indices = self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa(
virtual_indices
)
# Page-aligned chunks have whole-page bindings; zero denotes the sink page.
live_swa_indices = swa_indices[swa_indices > 0].to(torch.int64)
if live_swa_indices.numel() == 0:
return full_indices, []
return full_indices, [
PoolTransfer(
name=PoolName.SWA,
device_indices=live_swa_indices,
hit_policy=PoolHitPolicy.TRAILING_PAGES,
)
]
def offload_kv_cache(self, req) -> bool:
"""Offload incremental KV cache for decode side."""
@@ -159,7 +223,10 @@ class DecodeKVCacheOffloadManager:
start = prefill_offloaded_len + state.inc_len
end = start + incremental_aligned_len
incremental_tokens = all_tokens[start:end]
incremental_indices = token_indices[start:end]
incremental_indices = token_indices[start:end].to(torch.int64)
device_indices, pool_transfers = self._resolve_offload_transfers(
incremental_indices
)
# Prefill-aligned GPU slots are freed at request finish in
# _release_finished_req, NOT here. The decoding request
@@ -171,8 +238,9 @@ class DecodeKVCacheOffloadManager:
self.request_counter += 1
ack_id = self.request_counter
host_indices = self.cache_controller.write(
device_indices=incremental_indices.long(),
device_indices=device_indices,
node_id=ack_id,
extra_pools=pool_transfers,
)
if host_indices is None:
logger.error(f"Not enough host memory for request {req.rid}")
@@ -184,6 +252,7 @@ class DecodeKVCacheOffloadManager:
host_indices,
incremental_tokens,
time.time(),
pool_transfers,
)
state.inc_len += incremental_aligned_len
return True
@@ -219,6 +288,7 @@ class DecodeKVCacheOffloadManager:
host_indices,
incremental_tokens,
start_time,
pool_transfers,
) = self.ongoing_offload.pop(ack_id)
self._mark_offload_finished(req)
@@ -228,7 +298,12 @@ class DecodeKVCacheOffloadManager:
else None
)
last_hash = self._trigger_backup(
req, host_indices, incremental_tokens, start_time, prior_hash
req,
host_indices,
incremental_tokens,
start_time,
prior_hash,
pool_transfers,
)
if req in self.offloaded_state:
self.offloaded_state[req].last_hash = last_hash
@@ -259,9 +334,15 @@ class DecodeKVCacheOffloadManager:
for _ in range(finish_count):
storage_operation = self.cache_controller.ack_backup_queue.get()
ack_id = storage_operation.id
req_id, host_indices, start_time = self.ongoing_backup.pop(ack_id)
req_id, host_indices, start_time, pool_transfers = self.ongoing_backup.pop(
ack_id
)
# Release host memory
for transfer in pool_transfers:
self.decode_host_mem_pool.get_pool(transfer.name).free(
transfer.host_indices
)
self.decode_host_mem_pool.free(host_indices)
logger.debug(
@@ -269,16 +350,31 @@ class DecodeKVCacheOffloadManager:
)
def _trigger_backup(
self, req, host_indices, incremental_tokens, start_time, prior_hash
self,
req,
host_indices,
incremental_tokens,
start_time,
prior_hash,
pool_transfers=(),
):
"""Trigger async backup from host to storage."""
page_hashes = self._compute_prefix_hash(req, incremental_tokens, prior_hash)
for transfer in pool_transfers:
page_count = transfer.device_indices.numel() // self.page_size
transfer.keys = page_hashes[-page_count:]
ack_id = self.cache_controller.write_storage(
host_indices,
incremental_tokens,
hash_value=page_hashes,
**({"extra_pools": pool_transfers} if pool_transfers else {}),
)
self.ongoing_backup[ack_id] = (
req.rid,
host_indices,
start_time,
pool_transfers,
)
self.ongoing_backup[ack_id] = (req.rid, host_indices, start_time)
return page_hashes[-1] if len(page_hashes) > 0 else prior_hash
def _compute_prefix_hash(self, req: Req, tokens, prior_hash=""):
@@ -131,9 +131,14 @@ def unified_memory_disagg_move_gate(scheduler):
if scheduler.disaggregation_mode == DisaggregationMode.DECODE:
def decode_gate() -> bool:
decode_offload_manager = scheduler.decode_offload_manager
return not (
scheduler.disagg_decode_transfer_queue.queue
or scheduler.disagg_decode_prealloc_queue.has_published_destinations
or (
decode_offload_manager is not None
and decode_offload_manager.has_inflight_device_transfer()
)
)
return decode_gate
@@ -803,10 +803,13 @@ class HiCacheController:
device_indices: torch.Tensor,
priority: Optional[int] = None,
node_id: int = -1,
extra_pools: Optional[List[PoolTransfer]] = None,
) -> Optional[torch.Tensor]:
"""
Back up KV caches from device memory to host memory.
"""
if extra_pools:
raise ValueError("Side-pool transfers require HybridCacheController.")
host_indices = self.mem_pool_host.alloc(len(device_indices))
if host_indices is None:
return None
@@ -37,6 +37,7 @@ from sglang.srt.mem_cache.hicache_storage import (
)
from sglang.srt.mem_cache.l2_transfer import L2Transfer
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
from sglang.srt.mem_cache.radix_cache import RadixKey
@@ -452,17 +453,30 @@ class HybridCacheController(BaseHiCacheController):
release_queue.queue.clear()
self.prefetch_tokens_occupied = 0
def write(
def allocate_host_transfers(
self,
device_indices: torch.Tensor,
priority: Optional[int] = None,
node_id: int = -1,
extra_pools: Optional[list[PoolTransfer]] = None,
flush: bool = True,
) -> Optional[torch.Tensor]:
"""Queue a D2H backup; flush=False leaves it queued so the caller can
merge several nodes into one start_writing() submit."""
host_indices = self.mem_pool_host.alloc(len(device_indices))
*,
reclaim: Optional[Callable[[int], Any]] = None,
) -> Optional[tuple[torch.Tensor, Optional[list[PoolTransfer]]]]:
if self._uses_shared_host_domain(extra_pools):
allocation = self.allocate_shared_host_transfers(
device_indices, extra_pools
)
anchor = self.mem_pool_host.anchor_entry
if (
allocation is None
and reclaim is not None
and anchor.host_evict_fn is None
):
reclaim(len(device_indices))
allocation = self.allocate_shared_host_transfers(
device_indices, extra_pools
)
return allocation
host_indices = self.mem_pool_host.alloc(len(device_indices), reclaim=reclaim)
if host_indices is None:
return None
pool_transfers = self.mem_pool_host.resolve_host_transfers(
@@ -473,6 +487,20 @@ class HybridCacheController(BaseHiCacheController):
if pool_transfers is None and extra_pools:
self.mem_pool_host.free(host_indices)
return None
return host_indices, pool_transfers
def write(
self,
device_indices: torch.Tensor,
priority: Optional[int] = None,
node_id: int = -1,
extra_pools: Optional[list[PoolTransfer]] = None,
flush: bool = True,
) -> Optional[torch.Tensor]:
allocation = self.allocate_host_transfers(device_indices, extra_pools)
if allocation is None:
return None
host_indices, pool_transfers = allocation
self.write_queue.append(
CacheOperation(
@@ -487,6 +515,135 @@ class HybridCacheController(BaseHiCacheController):
self.start_writing()
return host_indices
def _uses_shared_host_domain(
self, extra_pools: Optional[list[PoolTransfer]]
) -> bool:
anchor_pool = self.mem_pool_host.anchor_entry.host_pool
if not uses_shared_host_layout(anchor_pool):
return False
domain = anchor_pool.shared_allocation_domain
for transfer in extra_pools or []:
if (
transfer.indices_from_pool is not None
or transfer.host_indices is not None
or transfer.device_indices is None
):
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
if entry is not None and (
not uses_shared_host_layout(entry.host_pool)
or entry.host_pool.shared_allocation_domain is not domain
):
return False
return True
def _alloc_shared_host_requests_with_reclaim(
self, requests: list[tuple[PoolName, int]]
) -> Optional[list[torch.Tensor]]:
entries = [self.mem_pool_host.entry_map[name] for name, _ in requests]
domain = entries[0].host_pool.shared_allocation_domain
if domain is None or any(
entry.host_pool.shared_allocation_domain is not domain for entry in entries
):
raise ValueError("Host pools do not share one allocation domain.")
domain_requests = [
(entry.host_pool.pool_label, need_size)
for entry, (_, need_size) in zip(entries, requests, strict=True)
]
allocated = domain.alloc_many(domain_requests)
if allocated is not None:
return allocated
pools = [entry.host_pool for entry in entries]
requested_bytes = sum(
need_size * pool.size_per_token
for pool, (_, need_size) in zip(pools, requests, strict=True)
)
requested_names = list(dict.fromkeys(name for name, _ in reversed(requests)))
candidates = [self.mem_pool_host.entry_map[name] for name in requested_names]
candidates.extend(
entry
for entry in self.mem_pool_host.entries
if entry.name not in requested_names
and uses_shared_host_layout(entry.host_pool)
and entry.host_pool.shared_allocation_domain is domain
)
# alloc_many compacts the shared layout before reporting failure. Reclaim
# only the remaining byte or per-side logical-slot shortfall, retrying
# after every successful eviction. The callbacks release only evictable
# tree entries; protected retraction backups remain safe.
while True:
made_progress = False
for entry in candidates:
if entry.host_evict_fn is None:
continue
pool = entry.host_pool
shortfall_bytes = max(
pool.page_size * pool.size_per_token,
requested_bytes - domain.free_bytes(),
)
tokens = (
(
(shortfall_bytes + pool.size_per_token - 1)
// pool.size_per_token
+ pool.page_size
- 1
)
// pool.page_size
* pool.page_size
)
if entry.host_evict_fn(tokens) <= 0:
continue
made_progress = True
allocated = domain.alloc_many(domain_requests)
if allocated is not None:
return allocated
if not made_progress:
return None
def allocate_shared_host_transfers(
self,
kv_device_indices: torch.Tensor,
extra_pools: Optional[list[PoolTransfer]] = None,
) -> Optional[tuple[torch.Tensor, Optional[list[PoolTransfer]]]]:
"""Atomically reserve the anchor and all independent host side pools."""
requests = [(self.mem_pool_host.anchor_entry.name, len(kv_device_indices))]
request_transfers: list[PoolTransfer] = []
for transfer in extra_pools or []:
if transfer.indices_from_pool is not None:
continue
entry = self.mem_pool_host.entry_map.get(transfer.name)
if (
entry is None
or transfer.host_indices is not None
or transfer.device_indices is None
):
continue
requests.append((transfer.name, len(transfer.device_indices)))
request_transfers.append(transfer)
allocated = self._alloc_shared_host_requests_with_reclaim(requests)
if allocated is None:
return None
host_indices = allocated[0]
for transfer, indices in zip(request_transfers, allocated[1:], strict=True):
transfer.host_indices = indices
resolved = self.mem_pool_host.resolve_host_transfers(
extra_pools,
primary_device_indices=kv_device_indices,
primary_host_indices=host_indices,
)
if resolved is None and extra_pools:
for (name, _), indices in zip(requests, allocated, strict=True):
self.mem_pool_host.free(indices, pool=name)
for transfer in request_transfers:
transfer.host_indices = None
return None
return host_indices, resolved
def _move_op_indices(
self, op: CacheOperation
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
@@ -498,39 +655,12 @@ class HybridCacheController(BaseHiCacheController):
host_group = self.mem_pool_host
if self.io_backend != "kernel" or host_group.layout != "page_first":
return self.move_hybrid_indices(op)
if not getattr(host_group, "supports_per_pool_backup_indices", False):
if not getattr(host_group, "can_use_write_back_jit", False):
if not host_group.supports_per_pool_backup_indices:
if not host_group.can_use_write_back_jit:
return self.move_hybrid_indices(op)
return op.host_indices, op.device_indices, op.pool_transfers
def move_for_pool(host_pool, host_indices, device_indices):
if getattr(host_pool, "can_use_write_back_jit", False):
if host_indices.is_cuda:
host_indices = host_indices.cpu()
return host_indices, device_indices
return self.move_indices(host_indices, device_indices)
host_indices, device_indices = move_for_pool(
host_group.anchor_entry.host_pool,
op.host_indices,
op.device_indices,
)
pool_transfers = []
for transfer in op.pool_transfers or []:
entry = host_group.entry_map[transfer.name]
transfer_host_indices, transfer_device_indices = move_for_pool(
entry.host_pool,
transfer.host_indices,
transfer.device_indices,
)
pool_transfers.append(
replace(
transfer,
host_indices=transfer_host_indices,
device_indices=transfer_device_indices,
)
)
return host_indices, device_indices, pool_transfers or None
return self.move_hybrid_indices(op, write_back_jit=True)
def _l2_transfers(
self,
@@ -1075,35 +1205,53 @@ class HybridCacheController(BaseHiCacheController):
kv_hit_pages * self.page_size,
)
def _move_pool_indices(
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
) -> tuple[torch.Tensor, torch.Tensor]:
if uses_shared_host_layout(host_pool):
return host_indices, device_indices
if write_back_jit and host_pool.can_use_write_back_jit:
if host_indices.is_cuda:
host_indices = host_indices.cpu()
return host_indices, device_indices
return self.move_indices(host_indices, device_indices)
def move_hybrid_indices(
self, operation: CacheOperation
self, operation: CacheOperation, *, write_back_jit: bool = False
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
host_indices, device_indices = self.move_indices(
operation.host_indices, operation.device_indices
host_indices, device_indices = self._move_pool_indices(
self.mem_pool_host.anchor_entry.host_pool,
operation.host_indices,
operation.device_indices,
write_back_jit=write_back_jit,
)
resolved_pool_transfers = None
if operation.pool_transfers:
resolved_pool_transfers = []
for transfer in operation.pool_transfers:
transfer_host_indices, transfer_device_indices = self.move_indices(
transfer.host_indices, transfer.device_indices
transfer_host_indices, transfer_device_indices = (
self._move_pool_indices(
self.mem_pool_host.entry_map[transfer.name].host_pool,
transfer.host_indices,
transfer.device_indices,
write_back_jit=write_back_jit,
)
)
# Keep the original PoolTransfer unchanged because tree-owned
# transfers may still reference radix-tree host state. The
# controller only needs a normalized execution-time copy.
# Tree-owned transfers keep their original radix-tree indices.
resolved_pool_transfers.append(
PoolTransfer(
name=transfer.name,
replace(
transfer,
host_indices=transfer_host_indices,
device_indices=transfer_device_indices,
keys=transfer.keys,
hit_policy=transfer.hit_policy,
indices_from_pool=transfer.indices_from_pool,
)
)
return host_indices, device_indices, resolved_pool_transfers
def _page_transfer(self, operation: PrefetchOperation) -> bool:
with self.mem_pool_host.layout_lease():
return self._page_transfer_with_stable_layout(operation)
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
# KV pools and KV-derived pools first — determines actual completed page count
kv_completed_pages = super()._page_transfer(operation)
@@ -1172,6 +1320,10 @@ class HybridCacheController(BaseHiCacheController):
operation.storage_start += trim_tokens
def _page_backup(self, operation):
with self.mem_pool_host.layout_lease():
return self._page_backup_with_stable_layout(operation)
def _page_backup_with_stable_layout(self, operation):
# MLA KV is replicated across TP ranks and should still be written only
# by TP0. Rank-sharded sidecars still need every TP rank.
backup_transfers = [
@@ -1401,7 +1553,8 @@ class HybridCacheController(BaseHiCacheController):
for pool in derived_transfers:
if pool.indices_from_pool == PoolName.KV:
pool.host_indices = kv_host_indices
pool.device_indices = kv_device_indices
if pool.device_indices is None:
pool.device_indices = kv_device_indices
continue
source = next(
@@ -1417,7 +1570,8 @@ class HybridCacheController(BaseHiCacheController):
rollback_allocated()
return None
pool.host_indices = source.host_indices
pool.device_indices = source.device_indices
if pool.device_indices is None:
pool.device_indices = source.device_indices
return extra_pools
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.pool_host.mha import (
get_mha_host_pool_cls,
)
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
@@ -279,6 +280,26 @@ def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free)
def _uses_unified_page_envelope_host(
full_kv_pool: Any,
swa_kv_pool: Any,
use_mla: bool,
mtp_swa_device_pools: tuple[Any, ...] = (),
) -> bool:
from sglang.srt.mem_cache.unified_memory_pool import UnifiedMHATokenToKVPool
return (
not use_mla
and not mtp_swa_device_pools
and all(
isinstance(pool, UnifiedMHATokenToKVPool)
for pool in (full_kv_pool, swa_kv_pool)
)
and {full_kv_pool.grow_direction, swa_kv_pool.grow_direction} == {"up", "down"}
and get_memory().hicache_host_memory_mode != "buffer_only"
)
def build_hybrid_swa_group(
*,
page_size: int,
@@ -289,6 +310,7 @@ def build_hybrid_swa_group(
use_mla: bool,
kv_host_size: Optional[float] = None,
swa_host_size: Optional[float] = None,
host_full_evict_fn: Optional[Callable[[int], Any]] = None,
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
swa_attn_allocator: Any = None,
@@ -300,21 +322,39 @@ def build_hybrid_swa_group(
transfer_layer_id_max = (
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
)
kv_host_pool = build_kv_host_pool(
kv_pool=full_kv_pool,
page_size=page_size,
use_mla=use_mla,
host_size=kv_host_size,
pool_label="full",
)
swa_host_pool = build_kv_host_pool(
kv_pool=swa_kv_pool,
page_size=page_size,
use_mla=use_mla,
host_size=swa_host_size,
mtp_draft_device_pools=mtp_swa_device_pools,
pool_label="swa",
)
if _uses_unified_page_envelope_host(
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
):
memory = get_memory()
kv_host_pool, swa_host_pool = (
UnifiedPageEnvelopeHostPool.build_hybrid_swa_pool_pair(
device_pools=(full_kv_pool, swa_kv_pool),
host_to_device_ratio=memory.hicache_ratio,
host_size=memory.hicache_size,
page_size=page_size,
layout=memory.hicache_mem_layout,
allocator_type=_get_allocator_type(),
mtp_draft_device_pools=mtp_swa_device_pools,
)
)
has_shared_arena = True
else:
has_shared_arena = False
kv_host_pool = build_kv_host_pool(
kv_pool=full_kv_pool,
page_size=page_size,
use_mla=use_mla,
host_size=kv_host_size,
pool_label="full",
)
swa_host_pool = build_kv_host_pool(
kv_pool=swa_kv_pool,
page_size=page_size,
use_mla=use_mla,
host_size=swa_host_size,
mtp_draft_device_pools=mtp_swa_device_pools,
pool_label="swa",
)
if mtp_swa_device_pools:
swa_layer_mapping = _with_mtp_layer_mapping(
swa_layer_mapping,
@@ -331,6 +371,7 @@ def build_hybrid_swa_group(
layer_mapping=full_layer_mapping,
transfer_layer_id_max=transfer_layer_id_max,
is_anchor=True,
host_evict_fn=host_full_evict_fn if has_shared_arena else None,
),
build_pool_entry(
name=PoolName.SWA,
@@ -406,6 +447,7 @@ def build_hybrid_swa_stack(
load_cache_event,
storage_backend: Optional[str],
use_mla: bool,
host_full_evict_fn: Optional[Callable[[int], Any]] = None,
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
prefetch_threshold: int = 256,
@@ -422,9 +464,12 @@ def build_hybrid_swa_stack(
)
kv_host_size = swa_host_size = None
if get_memory().hicache_size > 0:
memory = get_memory()
if memory.hicache_size > 0 and not _uses_unified_page_envelope_host(
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
):
kv_host_size, swa_host_size = _split_hicache_size(
get_memory().hicache_size, (full_kv_pool, swa_kv_pool)
memory.hicache_size, (full_kv_pool, swa_kv_pool)
)
host_pool_group = build_hybrid_swa_group(
@@ -436,6 +481,7 @@ def build_hybrid_swa_stack(
use_mla=use_mla,
kv_host_size=kv_host_size,
swa_host_size=swa_host_size,
host_full_evict_fn=host_full_evict_fn,
host_swa_evict_fn=host_swa_evict_fn,
device_swa_evict_fn=device_swa_evict_fn,
# For SWA hybrid, device allocation goes through the inner allocator.
@@ -1842,6 +1888,7 @@ class _SwaStrategy(StackStrategy):
load_cache_event=load_cache_event,
storage_backend=storage_backend,
use_mla=False,
host_full_evict_fn=lambda n: cache.evict_host(n, ComponentType.FULL),
host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA),
device_swa_evict_fn=lambda n: _evict_swa_for_device_alloc(cache, n),
prefetch_threshold=prefetch_threshold,
@@ -177,6 +177,20 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
fields = {}
backend = disagg.disaggregation_decode_retraction_backup
unified_hybrid_swa = memory.enable_unified_memory and tp_worker.is_hybrid_swa
unified_decode_radix = (
memory.enable_unified_memory and disagg.disaggregation_decode_enable_radix_cache
)
if backend == "host_pool" and unified_decode_radix:
raise ValueError(
"Unified-memory H2D/D2H does not support host-pool decode "
"retraction with decode radix cache yet."
)
if backend == "host_pool" and unified_hybrid_swa:
raise ValueError(
"Unified-memory hybrid-SWA H2D/D2H does not support host-pool "
"decode retraction yet."
)
if backend is None:
kv_cache = tp_worker.get_memory_pool()[1].get_kvcache()
full_tokens_per_layer = (
@@ -184,10 +198,17 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
if tp_worker.is_hybrid_swa
else None
)
# Host-pool retraction does not address unified page envelopes or
# recurrent state, so those configurations stay on cpu_tensor.
supports_host_pool = (
draft_pools = tp_worker.model_runner.mtp_draft_device_pools
unified_draft_host_pool_supported = (
not memory.enable_unified_memory
or not draft_pools
or tp_worker.model_runner.spec_algorithm.is_dspark()
)
# Host-pool retraction has no recurrent-state sidecar, so a model with
# recurrent state stays on cpu_tensor.
supports_host_pool = (
unified_draft_host_pool_supported
and not unified_hybrid_swa
and not uses_ssm_state(tp_worker.model_runner.model_config)
and (
isinstance(kv_cache, MHATokenToKVPool)
+62 -25
View File
@@ -1,11 +1,13 @@
from __future__ import annotations
import logging
from contextlib import contextmanager
from functools import cache
from typing import Any, Callable, NamedTuple, Optional
import torch
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
from sglang.srt.utils import get_device_module
logger = logging.getLogger(__name__)
@@ -81,23 +83,63 @@ class L2TransferEngine:
resolved = translate(indices)
return resolved.to(original_device)
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
start_event = self._start_event(None)
def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]:
prepared = []
for transfer in transfers:
host_indices, device_indices = transfer.host_pool.prepare_transfer_indices(
transfer.host_indices,
self._resolve_device_indices(transfer),
self.io_backend,
)
prepared.append(
transfer._replace(
host_indices=host_indices,
device_indices=device_indices,
)
)
return prepared
@contextmanager
def _submission(self, transfers, stream, transfer_key, start_event=None):
start_event = self._start_event(start_event)
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
with device_module.stream(self.device_to_host_stream):
start_event.wait(self.device_to_host_stream)
device_indices = [self._resolve_device_indices(t) for t in transfers]
ack_start.record()
for transfer, dev_idx in zip(transfers, device_indices):
transfer.host_pool.backup_from_device_all_layer(
completion = TransferCompletion(ack_start, ack_finish, timing_enabled)
domains = shared_host_layout_domains(t.host_pool for t in transfers)
for domain in domains:
domain.acquire_layout()
finish_recorded = False
try:
with device_module.stream(stream):
# Index preparation can read producer-owned device indices.
start_event.wait(stream)
prepared = self._prepare_transfers(transfers)
ack_start.record()
yield prepared, completion
ack_finish.record()
finish_recorded = True
self._record_stream(transfers + prepared, stream)
except Exception:
stream.synchronize()
raise
finally:
for domain in reversed(domains):
domain.release_layout(
ack_finish if finish_recorded else None,
(id(self), transfer_key),
)
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
with self._submission(
transfers, self.device_to_host_stream, "device_to_host"
) as (transfers, completion):
for transfer in transfers:
transfer.host_pool.backup_from_device_all_layer_physical(
transfer.device_pool,
transfer.host_indices,
dev_idx,
transfer.device_indices,
self.io_backend,
)
ack_finish.record()
self._record_stream(transfers, self.device_to_host_stream, device_indices)
return TransferCompletion(ack_start, ack_finish, timing_enabled)
return completion
def submit_host_to_device(
self,
@@ -107,15 +149,12 @@ class L2TransferEngine:
start_event=None,
on_layer_done=None,
) -> TransferCompletion:
start_event = self._start_event(start_event)
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
primary = transfers[0] if transfers else None
with device_module.stream(self.host_to_device_stream):
start_event.wait(self.host_to_device_stream)
device_indices = [self._resolve_device_indices(t) for t in transfers]
ack_start.record()
with self._submission(
transfers, self.host_to_device_stream, "host_to_device", start_event
) as (transfers, completion):
primary = transfers[0] if transfers else None
for layer_id in range(transfer_layer_id_max):
for transfer, dev_idx in zip(transfers, device_indices):
for transfer in transfers:
local_layer_id = (
transfer.layer_mapper(layer_id)
if transfer.layer_mapper is not None
@@ -127,19 +166,17 @@ class L2TransferEngine:
and layer_id >= transfer.host_pool.layer_num
):
continue
transfer.host_pool.load_to_device_per_layer(
transfer.host_pool.load_to_device_per_layer_physical(
transfer.device_pool,
transfer.host_indices,
dev_idx,
transfer.device_indices,
local_layer_id,
self.io_backend,
is_draft=transfer.is_draft,
)
if on_layer_done is not None:
on_layer_done(layer_id)
ack_finish.record()
self._record_stream(transfers, self.host_to_device_stream, device_indices)
return TransferCompletion(ack_start, ack_finish, timing_enabled)
return completion
@staticmethod
def _start_event(start_event):
@@ -63,6 +63,8 @@ class LogicalHostPool:
compressed side pools use these logical FULL indices as stable page anchors.
"""
shared_allocation_domain = None
def __init__(self, size: int, page_size: int, layout: str = "layer_first"):
if size % page_size != 0:
raise ValueError(
@@ -150,6 +152,26 @@ class LogicalHostPool:
):
pass
def prepare_transfer_indices(self, host_indices, device_indices, io_backend):
return host_indices, device_indices
def backup_from_device_all_layer_physical(
self, device_pool, host_indices, device_indices, io_backend
):
pass
def load_to_device_per_layer_physical(
self,
device_pool,
host_indices,
device_indices,
layer_id,
io_backend,
*,
is_draft: bool = False,
):
pass
def load_to_device_per_layer(
self,
device_pool,
@@ -174,6 +196,9 @@ class LogicalHostPool:
def get_page_buffer_meta(self, indices):
return None
def get_page_buffer_element_size(self, split_factor: int = 1):
return None
def get_ksize_per_token(self):
return 0
+72 -1
View File
@@ -3,10 +3,11 @@ from __future__ import annotations
import abc
import logging
import threading
from collections.abc import Iterable
from contextlib import contextmanager
from contextvars import ContextVar
from functools import wraps
from typing import Optional
from typing import Optional, TypeGuard
import torch
@@ -35,6 +36,26 @@ _host_memory_budget: ContextVar[Optional[int]] = ContextVar(
)
def uses_shared_host_layout(host_pool: object) -> TypeGuard[HostKVCache]:
return (
isinstance(host_pool, HostKVCache)
and host_pool.shared_allocation_domain is not None
)
def shared_host_layout_domains(host_pools: Iterable[object]) -> list:
domains = []
seen = set()
for pool in host_pools:
if not uses_shared_host_layout(pool):
continue
domain = pool.shared_allocation_domain
if id(domain) not in seen:
seen.add(id(domain))
domains.append(domain)
return domains
@contextmanager
def host_memory_budget_scope(budget_bytes: int):
"""Book every pool built inside against one snapshot, not re-sampled psutil."""
@@ -135,6 +156,7 @@ def synchronized(func):
class HostKVCache(abc.ABC):
dcp_size = 1
dcp_rank = 0
shared_allocation_domain = None
def __init__(
self,
@@ -323,6 +345,55 @@ class HostKVCache(abc.ABC):
"""
raise NotImplementedError()
def get_page_buffer_element_size(self, split_factor: int = 1) -> Optional[int]:
"""Byte size of one storage element, or None for a logical anchor."""
indices = torch.zeros(self.page_size, dtype=torch.int64)
meta = (
self.get_split_heads_page_buffer_meta(indices, split_factor)
if split_factor != 1
else self.get_page_buffer_meta(indices)
)
sizes = meta[1] if meta else None
return int(sizes[0]) if sizes else None
def prepare_transfer_indices(
self, host_indices, device_indices, io_backend
) -> tuple[torch.Tensor, torch.Tensor]:
"""Resolve indices that must remain stable through one L2 submission.
Normal host pools expose physical indices directly and controllers have
already moved them to the backend-required device. Shared compacting
pools override this hook and resolve their logical ids while the transfer
engine holds a layout lease.
"""
return host_indices, device_indices
def backup_from_device_all_layer_physical(
self, device_pool, host_indices, device_indices, io_backend
) -> None:
self.backup_from_device_all_layer(
device_pool, host_indices, device_indices, io_backend
)
def load_to_device_per_layer_physical(
self,
device_pool,
host_indices,
device_indices,
layer_id,
io_backend,
*,
is_draft: bool = False,
) -> None:
self.load_to_device_per_layer(
device_pool,
host_indices,
device_indices,
layer_id,
io_backend,
is_draft=is_draft,
)
@abc.abstractmethod
def get_data_page(self, index, flat: bool = True) -> torch.Tensor:
"""
+33 -2
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from collections.abc import Callable
from contextlib import contextmanager
from dataclasses import dataclass
from typing import Any
@@ -8,6 +9,7 @@ import torch
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
from sglang.srt.mem_cache.pool_host.base import HostKVCache, shared_host_layout_domains
@dataclass
@@ -76,6 +78,33 @@ class HostPoolGroup:
def get_pool(self, name: PoolName):
return self.get_entry(name).host_pool
@contextmanager
def layout_lease(self):
domains = shared_host_layout_domains(entry.host_pool for entry in self.entries)
for domain in domains:
domain.acquire_layout()
try:
yield
finally:
for domain in reversed(domains):
domain.release_layout()
def get_page_buffer_element_size(self, split_factor: int = 1):
anchor_pool = self.anchor_entry.host_pool
if (
not isinstance(anchor_pool, HostKVCache)
or anchor_pool.shared_allocation_domain is None
):
return anchor_pool.get_page_buffer_element_size(split_factor)
element_sizes = set()
for entry in self.entries:
element_size = entry.host_pool.get_page_buffer_element_size(split_factor)
if element_size is None:
return None
element_sizes.add(element_size)
return element_sizes.pop() if len(element_sizes) == 1 else None
def get_contiguous_buf_infos(self):
"""Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes)."""
host_by_device_ptr = {}
@@ -199,7 +228,8 @@ class HostPoolGroup:
for transfer in derived_transfers:
if transfer.indices_from_pool == self.anchor_entry.name:
transfer.host_indices = primary_host_indices
transfer.device_indices = primary_device_indices
if transfer.device_indices is None:
transfer.device_indices = primary_device_indices
continue
source = next(
@@ -215,7 +245,8 @@ class HostPoolGroup:
rollback()
return None
transfer.host_indices = source.host_indices
transfer.device_indices = source.device_indices
if transfer.device_indices is None:
transfer.device_indices = source.device_indices
return transfers
def release_transfers(self, transfers: list[PoolTransfer] | None) -> int:
File diff suppressed because it is too large Load Diff
@@ -648,25 +648,16 @@ class UMBPStore(HiCacheStorage):
if "dram_page_size" in extra:
page_byte_size = int(extra["dram_page_size"])
elif mem_pool_host is not None:
# Probe element_size from the same buffer-meta helper that
# batch_preprocess will actually use; this matches per-call
# Put/Get size byte-for-byte for MHA / MHA-split / MLA / NSA
# without per-case formulas (NSA in particular: get_ksize_per_token
# would over-count by the indexer buffer that is never put to UMBP).
dummy = torch.zeros(mem_pool_host.page_size, dtype=torch.int64)
if self.is_mla_backend:
meta = mem_pool_host.get_page_buffer_meta(dummy)
elif storage_config is not None and getattr(
storage_config, "should_split_heads", False
split_factor = 1
if (
not self.is_mla_backend
and storage_config is not None
and storage_config.should_split_heads
):
sf = storage_config.tp_lcm_size // storage_config.tp_size
meta = mem_pool_host.get_split_heads_page_buffer_meta(dummy, sf)
else:
meta = mem_pool_host.get_page_buffer_meta(dummy)
# A hybrid logical anchor returns None here by design; leave
# dram_page_size at 0 and let the per-pool v2 sizes handle it.
esz = meta[1] if meta else None
page_byte_size = int(esz[0]) if esz else 0
split_factor = storage_config.tp_lcm_size // storage_config.tp_size
page_byte_size = mem_pool_host.get_page_buffer_element_size(
split_factor
)
if (
page_byte_size is not None
@@ -689,7 +680,7 @@ class UMBPStore(HiCacheStorage):
if (
mem_pool_host is not None
and storage_config is not None
and getattr(storage_config, "should_split_heads", False)
and storage_config.should_split_heads
)
else ""
),
@@ -627,6 +627,20 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
raw = self._unified_buffer._raw
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
def get_page_envelope_buffer(self) -> torch.Tensor:
"""Return this sub-pool's page-strided view of the shared allocation."""
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
raw = self._unified_buffer._raw
page_count = (
self._unified_buffer.max_slots(self._sub_pool_name) // self.page_size
)
return raw[: page_count * self._page_bytes].view(page_count, self._page_bytes)
@property
def grow_direction(self) -> str:
"""Physical growth direction used by this sub-pool's L1 allocator."""
return self._unified_buffer.spec(self._sub_pool_name).grow_direction
def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor:
return (indices // self.page_size) * (
self.page_size * self.kernel_page_blocks
@@ -1474,21 +1474,14 @@ class UnifiedRadixCache(BasePrefixCache):
assert req.seqlen > 1
device_indices, extra_transfers = self._retraction_device_transfers(req)
host_indices = self.host_pool_group.alloc(len(device_indices))
if host_indices is None:
self._reclaim_retraction_host(len(device_indices))
host_indices = self.host_pool_group.alloc(len(device_indices))
if host_indices is None:
return None
resolved = self.host_pool_group.resolve_host_transfers(
allocation = self.cache_controller.allocate_host_transfers(
device_indices,
extra_transfers or None,
primary_device_indices=device_indices,
primary_host_indices=host_indices,
reclaim=self._reclaim_retraction_host,
)
if resolved is None and extra_transfers:
self.host_pool_group.free(host_indices)
if allocation is None:
return None
host_indices, resolved = allocation
backup = RetractionBackup(
host_indices=host_indices,
@@ -1642,11 +1635,16 @@ class UnifiedRadixCache(BasePrefixCache):
def _execute_kv_backup(self, node_id, device_value, comp_xfers, sidecar_xfers):
"""Execute Backup action."""
kv_tokens = len(device_value)
host_avail = self.cache_controller.mem_pool_host.available_size()
if host_avail < kv_tokens:
needed = kv_tokens - host_avail
if self.evict_host(needed) < needed:
return None
anchor_entry = self.cache_controller.mem_pool_host.anchor_entry
if (
anchor_entry.host_pool.shared_allocation_domain is None
or anchor_entry.host_evict_fn is None
):
host_avail = self.cache_controller.mem_pool_host.available_size()
if host_avail < kv_tokens:
needed = kv_tokens - host_avail
if self.evict_host(needed) < needed:
return None
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
aux_xfers.extend(sidecar_xfers)
# Defer submission so the next flush can merge pending node backups.