Support unified memory decode host pools (#39478)
Co-authored-by: yhzhuang <yhzhuang@fb.com> Co-authored-by: Lianmin Zheng <lianminzheng@gmail.com> Co-authored-by: Yonghao Zhuang <yhzhuang@users.noreply.github.com> Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
co-authored by
yhzhuang
Lianmin Zheng
Yonghao Zhuang
Cheng Wan
parent
984994e3aa
commit
78b382b1ad
@@ -15,6 +15,7 @@ from sglang.srt.arg_groups.overrides import (
|
||||
resolving_view,
|
||||
use_mla_backend,
|
||||
)
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.runtime_context import get_platform
|
||||
@@ -371,7 +372,6 @@ def handle_cache_compatibility(server_args: Any) -> None:
|
||||
"--disaggregation-decode-retraction-backup=host_pool requires "
|
||||
"--disable-priority-preemption when priority scheduling is enabled."
|
||||
)
|
||||
|
||||
if cfg.radix_eviction_policy == "tlru":
|
||||
tlru_config = cfg.radix_eviction_policy_config or {}
|
||||
threshold = tlru_config.get("threshold", 0)
|
||||
@@ -467,15 +467,38 @@ def handle_unified_memory_pool(server_args: Any) -> None:
|
||||
"ships host/C4 rows straight from the allocator, bypassing the "
|
||||
"virtual->physical translation the unified pool needs."
|
||||
)
|
||||
assert cfg.disaggregation_decode_retraction_backup != "host_pool", (
|
||||
"--enable-unified-memory with PD disaggregation does not support "
|
||||
"--disaggregation-decode-retraction-backup=host_pool; use "
|
||||
"cpu_tensor (the automatic default for unified pools)."
|
||||
)
|
||||
assert not cfg.disaggregation_decode_enable_offload_kvcache, (
|
||||
"--enable-unified-memory with PD disaggregation does not yet support "
|
||||
"--disaggregation-decode-enable-offload-kvcache."
|
||||
)
|
||||
if cfg.disaggregation_decode_enable_offload_kvcache:
|
||||
assert cfg.hicache_storage_backend == "file", (
|
||||
"--enable-unified-memory with decode KV offload currently "
|
||||
"supports --hicache-storage-backend=file only."
|
||||
)
|
||||
model_config = model_config_of(server_args)
|
||||
assert not use_mla_backend(server_args), (
|
||||
"--enable-unified-memory decode KV offload does not support "
|
||||
"MLA models yet."
|
||||
)
|
||||
assert mambaish_config(model_config) is None, (
|
||||
"--enable-unified-memory decode KV offload does not support "
|
||||
"hybrid-Mamba models."
|
||||
)
|
||||
assert not model_config.is_hybrid_swa, (
|
||||
"--enable-unified-memory decode KV offload does not support "
|
||||
"hybrid-SWA H2D/D2H transfers yet."
|
||||
)
|
||||
if cfg.disaggregation_decode_retraction_backup == "host_pool":
|
||||
model_config = model_config_of(server_args)
|
||||
assert not cfg.disaggregation_decode_enable_radix_cache, (
|
||||
"--enable-unified-memory host-pool decode retraction does not "
|
||||
"support decode radix-cache H2D/D2H transfers yet."
|
||||
)
|
||||
assert mambaish_config(model_config) is None, (
|
||||
"--enable-unified-memory host-pool decode retraction does not "
|
||||
"support hybrid-Mamba models."
|
||||
)
|
||||
assert not model_config.is_hybrid_swa, (
|
||||
"--enable-unified-memory host-pool decode retraction does not "
|
||||
"support hybrid-SWA H2D/D2H transfers yet."
|
||||
)
|
||||
assert cfg.speculative_algorithm in (None, "DSPARK"), (
|
||||
"--enable-unified-memory only supports --speculative-algorithm "
|
||||
"DSPARK (chain draft); other speculative algorithms are not yet "
|
||||
|
||||
@@ -14,7 +14,14 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.cache_controller import HiCacheController
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.hicache_storage import (
|
||||
PoolHitPolicy,
|
||||
PoolName,
|
||||
PoolTransfer,
|
||||
)
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import (
|
||||
build_hybrid_swa_stack,
|
||||
build_kv_host_pool,
|
||||
)
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
@@ -22,6 +29,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
MLATokenToKVPool,
|
||||
ReqToTokenPool,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedSWAKVPool
|
||||
from sglang.srt.mem_cache.utils import storage_namespace_seed
|
||||
from sglang.srt.runtime_context import (
|
||||
get_memory,
|
||||
@@ -57,15 +65,6 @@ class DecodeKVCacheOffloadManager:
|
||||
self.offload_stride = max(
|
||||
self.page_size, (env_stride // self.page_size) * self.page_size
|
||||
)
|
||||
kv_cache = self.token_to_kv_pool_allocator.get_kvcache()
|
||||
if not isinstance(kv_cache, (MHATokenToKVPool, MLATokenToKVPool)):
|
||||
raise ValueError("Unsupported KV cache type for decode offload")
|
||||
self.decode_host_mem_pool = build_kv_host_pool(
|
||||
kv_pool=kv_cache,
|
||||
page_size=self.page_size,
|
||||
use_mla=isinstance(kv_cache, MLATokenToKVPool),
|
||||
)
|
||||
|
||||
self.tp_group = tp_group
|
||||
self.tp_world_size = torch.distributed.get_world_size(group=self.tp_group)
|
||||
|
||||
@@ -80,17 +79,56 @@ class DecodeKVCacheOffloadManager:
|
||||
f"Invalid hicache storage backend extra config JSON: {e}"
|
||||
)
|
||||
|
||||
self.cache_controller = HiCacheController(
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
mem_pool_host=self.decode_host_mem_pool,
|
||||
page_size=self.page_size,
|
||||
tp_group=tp_group,
|
||||
io_backend=get_memory().hicache_io_backend,
|
||||
load_cache_event=threading.Event(),
|
||||
storage_backend=get_memory().hicache_storage_backend,
|
||||
model_name=get_serving().served_model_name,
|
||||
storage_backend_extra_config=hicache_storage_backend_extra_config,
|
||||
)
|
||||
self.kv_cache = self.token_to_kv_pool_allocator.get_kvcache()
|
||||
if isinstance(self.kv_cache, UnifiedSWAKVPool):
|
||||
full_mapping = {
|
||||
gid: lid
|
||||
for gid, (lid, is_swa) in self.kv_cache.layers_mapping.items()
|
||||
if not is_swa
|
||||
}
|
||||
swa_mapping = {
|
||||
gid: lid
|
||||
for gid, (lid, is_swa) in self.kv_cache.layers_mapping.items()
|
||||
if is_swa
|
||||
}
|
||||
params = CacheInitParams(
|
||||
disable=False,
|
||||
req_to_token_pool=self.req_to_token_pool,
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
page_size=self.page_size,
|
||||
tp_cache_group=tp_group,
|
||||
)
|
||||
self.decode_host_mem_pool, self.cache_controller = build_hybrid_swa_stack(
|
||||
params=params,
|
||||
full_kv_pool=self.kv_cache.full_kv_pool,
|
||||
swa_kv_pool=self.kv_cache.swa_kv_pool,
|
||||
full_layer_mapping=full_mapping,
|
||||
swa_layer_mapping=swa_mapping,
|
||||
load_cache_event=threading.Event(),
|
||||
storage_backend=get_memory().hicache_storage_backend,
|
||||
use_mla=False,
|
||||
model_name=get_serving().served_model_name,
|
||||
storage_backend_extra_config=hicache_storage_backend_extra_config,
|
||||
)
|
||||
else:
|
||||
if not isinstance(self.kv_cache, (MHATokenToKVPool, MLATokenToKVPool)):
|
||||
raise ValueError("Unsupported KV cache type for decode offload")
|
||||
self.decode_host_mem_pool = build_kv_host_pool(
|
||||
kv_pool=self.kv_cache,
|
||||
page_size=self.page_size,
|
||||
use_mla=isinstance(self.kv_cache, MLATokenToKVPool),
|
||||
)
|
||||
self.cache_controller = HiCacheController(
|
||||
token_to_kv_pool_allocator=self.token_to_kv_pool_allocator,
|
||||
mem_pool_host=self.decode_host_mem_pool,
|
||||
page_size=self.page_size,
|
||||
tp_group=tp_group,
|
||||
io_backend=get_memory().hicache_io_backend,
|
||||
load_cache_event=threading.Event(),
|
||||
storage_backend=get_memory().hicache_storage_backend,
|
||||
model_name=get_serving().served_model_name,
|
||||
storage_backend_extra_config=hicache_storage_backend_extra_config,
|
||||
)
|
||||
|
||||
self.ongoing_offload = {}
|
||||
self.ongoing_backup = {}
|
||||
@@ -120,6 +158,32 @@ class DecodeKVCacheOffloadManager:
|
||||
# Page-aligned prompt length; the prefill instance offloaded this part.
|
||||
return len(req.origin_input_ids) // self.page_size * self.page_size
|
||||
|
||||
def has_inflight_device_transfer(self) -> bool:
|
||||
"""Whether a device-to-host copy may still read unified KV pages."""
|
||||
return bool(self.ongoing_offload)
|
||||
|
||||
def _resolve_offload_transfers(
|
||||
self, virtual_indices: torch.Tensor
|
||||
) -> tuple[torch.Tensor, list[PoolTransfer]]:
|
||||
full_indices = virtual_indices
|
||||
if not isinstance(self.kv_cache, UnifiedSWAKVPool):
|
||||
return full_indices, []
|
||||
|
||||
swa_indices = self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa(
|
||||
virtual_indices
|
||||
)
|
||||
# Page-aligned chunks have whole-page bindings; zero denotes the sink page.
|
||||
live_swa_indices = swa_indices[swa_indices > 0].to(torch.int64)
|
||||
if live_swa_indices.numel() == 0:
|
||||
return full_indices, []
|
||||
return full_indices, [
|
||||
PoolTransfer(
|
||||
name=PoolName.SWA,
|
||||
device_indices=live_swa_indices,
|
||||
hit_policy=PoolHitPolicy.TRAILING_PAGES,
|
||||
)
|
||||
]
|
||||
|
||||
def offload_kv_cache(self, req) -> bool:
|
||||
"""Offload incremental KV cache for decode side."""
|
||||
|
||||
@@ -159,7 +223,10 @@ class DecodeKVCacheOffloadManager:
|
||||
start = prefill_offloaded_len + state.inc_len
|
||||
end = start + incremental_aligned_len
|
||||
incremental_tokens = all_tokens[start:end]
|
||||
incremental_indices = token_indices[start:end]
|
||||
incremental_indices = token_indices[start:end].to(torch.int64)
|
||||
device_indices, pool_transfers = self._resolve_offload_transfers(
|
||||
incremental_indices
|
||||
)
|
||||
|
||||
# Prefill-aligned GPU slots are freed at request finish in
|
||||
# _release_finished_req, NOT here. The decoding request
|
||||
@@ -171,8 +238,9 @@ class DecodeKVCacheOffloadManager:
|
||||
self.request_counter += 1
|
||||
ack_id = self.request_counter
|
||||
host_indices = self.cache_controller.write(
|
||||
device_indices=incremental_indices.long(),
|
||||
device_indices=device_indices,
|
||||
node_id=ack_id,
|
||||
extra_pools=pool_transfers,
|
||||
)
|
||||
if host_indices is None:
|
||||
logger.error(f"Not enough host memory for request {req.rid}")
|
||||
@@ -184,6 +252,7 @@ class DecodeKVCacheOffloadManager:
|
||||
host_indices,
|
||||
incremental_tokens,
|
||||
time.time(),
|
||||
pool_transfers,
|
||||
)
|
||||
state.inc_len += incremental_aligned_len
|
||||
return True
|
||||
@@ -219,6 +288,7 @@ class DecodeKVCacheOffloadManager:
|
||||
host_indices,
|
||||
incremental_tokens,
|
||||
start_time,
|
||||
pool_transfers,
|
||||
) = self.ongoing_offload.pop(ack_id)
|
||||
|
||||
self._mark_offload_finished(req)
|
||||
@@ -228,7 +298,12 @@ class DecodeKVCacheOffloadManager:
|
||||
else None
|
||||
)
|
||||
last_hash = self._trigger_backup(
|
||||
req, host_indices, incremental_tokens, start_time, prior_hash
|
||||
req,
|
||||
host_indices,
|
||||
incremental_tokens,
|
||||
start_time,
|
||||
prior_hash,
|
||||
pool_transfers,
|
||||
)
|
||||
if req in self.offloaded_state:
|
||||
self.offloaded_state[req].last_hash = last_hash
|
||||
@@ -259,9 +334,15 @@ class DecodeKVCacheOffloadManager:
|
||||
for _ in range(finish_count):
|
||||
storage_operation = self.cache_controller.ack_backup_queue.get()
|
||||
ack_id = storage_operation.id
|
||||
req_id, host_indices, start_time = self.ongoing_backup.pop(ack_id)
|
||||
req_id, host_indices, start_time, pool_transfers = self.ongoing_backup.pop(
|
||||
ack_id
|
||||
)
|
||||
|
||||
# Release host memory
|
||||
for transfer in pool_transfers:
|
||||
self.decode_host_mem_pool.get_pool(transfer.name).free(
|
||||
transfer.host_indices
|
||||
)
|
||||
self.decode_host_mem_pool.free(host_indices)
|
||||
|
||||
logger.debug(
|
||||
@@ -269,16 +350,31 @@ class DecodeKVCacheOffloadManager:
|
||||
)
|
||||
|
||||
def _trigger_backup(
|
||||
self, req, host_indices, incremental_tokens, start_time, prior_hash
|
||||
self,
|
||||
req,
|
||||
host_indices,
|
||||
incremental_tokens,
|
||||
start_time,
|
||||
prior_hash,
|
||||
pool_transfers=(),
|
||||
):
|
||||
"""Trigger async backup from host to storage."""
|
||||
page_hashes = self._compute_prefix_hash(req, incremental_tokens, prior_hash)
|
||||
for transfer in pool_transfers:
|
||||
page_count = transfer.device_indices.numel() // self.page_size
|
||||
transfer.keys = page_hashes[-page_count:]
|
||||
ack_id = self.cache_controller.write_storage(
|
||||
host_indices,
|
||||
incremental_tokens,
|
||||
hash_value=page_hashes,
|
||||
**({"extra_pools": pool_transfers} if pool_transfers else {}),
|
||||
)
|
||||
self.ongoing_backup[ack_id] = (
|
||||
req.rid,
|
||||
host_indices,
|
||||
start_time,
|
||||
pool_transfers,
|
||||
)
|
||||
self.ongoing_backup[ack_id] = (req.rid, host_indices, start_time)
|
||||
return page_hashes[-1] if len(page_hashes) > 0 else prior_hash
|
||||
|
||||
def _compute_prefix_hash(self, req: Req, tokens, prior_hash=""):
|
||||
|
||||
@@ -131,9 +131,14 @@ def unified_memory_disagg_move_gate(scheduler):
|
||||
if scheduler.disaggregation_mode == DisaggregationMode.DECODE:
|
||||
|
||||
def decode_gate() -> bool:
|
||||
decode_offload_manager = scheduler.decode_offload_manager
|
||||
return not (
|
||||
scheduler.disagg_decode_transfer_queue.queue
|
||||
or scheduler.disagg_decode_prealloc_queue.has_published_destinations
|
||||
or (
|
||||
decode_offload_manager is not None
|
||||
and decode_offload_manager.has_inflight_device_transfer()
|
||||
)
|
||||
)
|
||||
|
||||
return decode_gate
|
||||
|
||||
@@ -803,10 +803,13 @@ class HiCacheController:
|
||||
device_indices: torch.Tensor,
|
||||
priority: Optional[int] = None,
|
||||
node_id: int = -1,
|
||||
extra_pools: Optional[List[PoolTransfer]] = None,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""
|
||||
Back up KV caches from device memory to host memory.
|
||||
"""
|
||||
if extra_pools:
|
||||
raise ValueError("Side-pool transfers require HybridCacheController.")
|
||||
host_indices = self.mem_pool_host.alloc(len(device_indices))
|
||||
if host_indices is None:
|
||||
return None
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.srt.mem_cache.hicache_storage import (
|
||||
)
|
||||
from sglang.srt.mem_cache.l2_transfer import L2Transfer
|
||||
from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry
|
||||
from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout
|
||||
from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
|
||||
@@ -452,17 +453,30 @@ class HybridCacheController(BaseHiCacheController):
|
||||
release_queue.queue.clear()
|
||||
self.prefetch_tokens_occupied = 0
|
||||
|
||||
def write(
|
||||
def allocate_host_transfers(
|
||||
self,
|
||||
device_indices: torch.Tensor,
|
||||
priority: Optional[int] = None,
|
||||
node_id: int = -1,
|
||||
extra_pools: Optional[list[PoolTransfer]] = None,
|
||||
flush: bool = True,
|
||||
) -> Optional[torch.Tensor]:
|
||||
"""Queue a D2H backup; flush=False leaves it queued so the caller can
|
||||
merge several nodes into one start_writing() submit."""
|
||||
host_indices = self.mem_pool_host.alloc(len(device_indices))
|
||||
*,
|
||||
reclaim: Optional[Callable[[int], Any]] = None,
|
||||
) -> Optional[tuple[torch.Tensor, Optional[list[PoolTransfer]]]]:
|
||||
if self._uses_shared_host_domain(extra_pools):
|
||||
allocation = self.allocate_shared_host_transfers(
|
||||
device_indices, extra_pools
|
||||
)
|
||||
anchor = self.mem_pool_host.anchor_entry
|
||||
if (
|
||||
allocation is None
|
||||
and reclaim is not None
|
||||
and anchor.host_evict_fn is None
|
||||
):
|
||||
reclaim(len(device_indices))
|
||||
allocation = self.allocate_shared_host_transfers(
|
||||
device_indices, extra_pools
|
||||
)
|
||||
return allocation
|
||||
|
||||
host_indices = self.mem_pool_host.alloc(len(device_indices), reclaim=reclaim)
|
||||
if host_indices is None:
|
||||
return None
|
||||
pool_transfers = self.mem_pool_host.resolve_host_transfers(
|
||||
@@ -473,6 +487,20 @@ class HybridCacheController(BaseHiCacheController):
|
||||
if pool_transfers is None and extra_pools:
|
||||
self.mem_pool_host.free(host_indices)
|
||||
return None
|
||||
return host_indices, pool_transfers
|
||||
|
||||
def write(
|
||||
self,
|
||||
device_indices: torch.Tensor,
|
||||
priority: Optional[int] = None,
|
||||
node_id: int = -1,
|
||||
extra_pools: Optional[list[PoolTransfer]] = None,
|
||||
flush: bool = True,
|
||||
) -> Optional[torch.Tensor]:
|
||||
allocation = self.allocate_host_transfers(device_indices, extra_pools)
|
||||
if allocation is None:
|
||||
return None
|
||||
host_indices, pool_transfers = allocation
|
||||
|
||||
self.write_queue.append(
|
||||
CacheOperation(
|
||||
@@ -487,6 +515,135 @@ class HybridCacheController(BaseHiCacheController):
|
||||
self.start_writing()
|
||||
return host_indices
|
||||
|
||||
def _uses_shared_host_domain(
|
||||
self, extra_pools: Optional[list[PoolTransfer]]
|
||||
) -> bool:
|
||||
anchor_pool = self.mem_pool_host.anchor_entry.host_pool
|
||||
if not uses_shared_host_layout(anchor_pool):
|
||||
return False
|
||||
domain = anchor_pool.shared_allocation_domain
|
||||
for transfer in extra_pools or []:
|
||||
if (
|
||||
transfer.indices_from_pool is not None
|
||||
or transfer.host_indices is not None
|
||||
or transfer.device_indices is None
|
||||
):
|
||||
continue
|
||||
entry = self.mem_pool_host.entry_map.get(transfer.name)
|
||||
if entry is not None and (
|
||||
not uses_shared_host_layout(entry.host_pool)
|
||||
or entry.host_pool.shared_allocation_domain is not domain
|
||||
):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _alloc_shared_host_requests_with_reclaim(
|
||||
self, requests: list[tuple[PoolName, int]]
|
||||
) -> Optional[list[torch.Tensor]]:
|
||||
entries = [self.mem_pool_host.entry_map[name] for name, _ in requests]
|
||||
domain = entries[0].host_pool.shared_allocation_domain
|
||||
if domain is None or any(
|
||||
entry.host_pool.shared_allocation_domain is not domain for entry in entries
|
||||
):
|
||||
raise ValueError("Host pools do not share one allocation domain.")
|
||||
domain_requests = [
|
||||
(entry.host_pool.pool_label, need_size)
|
||||
for entry, (_, need_size) in zip(entries, requests, strict=True)
|
||||
]
|
||||
|
||||
allocated = domain.alloc_many(domain_requests)
|
||||
if allocated is not None:
|
||||
return allocated
|
||||
|
||||
pools = [entry.host_pool for entry in entries]
|
||||
requested_bytes = sum(
|
||||
need_size * pool.size_per_token
|
||||
for pool, (_, need_size) in zip(pools, requests, strict=True)
|
||||
)
|
||||
requested_names = list(dict.fromkeys(name for name, _ in reversed(requests)))
|
||||
candidates = [self.mem_pool_host.entry_map[name] for name in requested_names]
|
||||
candidates.extend(
|
||||
entry
|
||||
for entry in self.mem_pool_host.entries
|
||||
if entry.name not in requested_names
|
||||
and uses_shared_host_layout(entry.host_pool)
|
||||
and entry.host_pool.shared_allocation_domain is domain
|
||||
)
|
||||
|
||||
# alloc_many compacts the shared layout before reporting failure. Reclaim
|
||||
# only the remaining byte or per-side logical-slot shortfall, retrying
|
||||
# after every successful eviction. The callbacks release only evictable
|
||||
# tree entries; protected retraction backups remain safe.
|
||||
while True:
|
||||
made_progress = False
|
||||
for entry in candidates:
|
||||
if entry.host_evict_fn is None:
|
||||
continue
|
||||
pool = entry.host_pool
|
||||
shortfall_bytes = max(
|
||||
pool.page_size * pool.size_per_token,
|
||||
requested_bytes - domain.free_bytes(),
|
||||
)
|
||||
tokens = (
|
||||
(
|
||||
(shortfall_bytes + pool.size_per_token - 1)
|
||||
// pool.size_per_token
|
||||
+ pool.page_size
|
||||
- 1
|
||||
)
|
||||
// pool.page_size
|
||||
* pool.page_size
|
||||
)
|
||||
if entry.host_evict_fn(tokens) <= 0:
|
||||
continue
|
||||
made_progress = True
|
||||
allocated = domain.alloc_many(domain_requests)
|
||||
if allocated is not None:
|
||||
return allocated
|
||||
if not made_progress:
|
||||
return None
|
||||
|
||||
def allocate_shared_host_transfers(
|
||||
self,
|
||||
kv_device_indices: torch.Tensor,
|
||||
extra_pools: Optional[list[PoolTransfer]] = None,
|
||||
) -> Optional[tuple[torch.Tensor, Optional[list[PoolTransfer]]]]:
|
||||
"""Atomically reserve the anchor and all independent host side pools."""
|
||||
requests = [(self.mem_pool_host.anchor_entry.name, len(kv_device_indices))]
|
||||
request_transfers: list[PoolTransfer] = []
|
||||
for transfer in extra_pools or []:
|
||||
if transfer.indices_from_pool is not None:
|
||||
continue
|
||||
entry = self.mem_pool_host.entry_map.get(transfer.name)
|
||||
if (
|
||||
entry is None
|
||||
or transfer.host_indices is not None
|
||||
or transfer.device_indices is None
|
||||
):
|
||||
continue
|
||||
requests.append((transfer.name, len(transfer.device_indices)))
|
||||
request_transfers.append(transfer)
|
||||
|
||||
allocated = self._alloc_shared_host_requests_with_reclaim(requests)
|
||||
if allocated is None:
|
||||
return None
|
||||
host_indices = allocated[0]
|
||||
for transfer, indices in zip(request_transfers, allocated[1:], strict=True):
|
||||
transfer.host_indices = indices
|
||||
|
||||
resolved = self.mem_pool_host.resolve_host_transfers(
|
||||
extra_pools,
|
||||
primary_device_indices=kv_device_indices,
|
||||
primary_host_indices=host_indices,
|
||||
)
|
||||
if resolved is None and extra_pools:
|
||||
for (name, _), indices in zip(requests, allocated, strict=True):
|
||||
self.mem_pool_host.free(indices, pool=name)
|
||||
for transfer in request_transfers:
|
||||
transfer.host_indices = None
|
||||
return None
|
||||
return host_indices, resolved
|
||||
|
||||
def _move_op_indices(
|
||||
self, op: CacheOperation
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
|
||||
@@ -498,39 +655,12 @@ class HybridCacheController(BaseHiCacheController):
|
||||
host_group = self.mem_pool_host
|
||||
if self.io_backend != "kernel" or host_group.layout != "page_first":
|
||||
return self.move_hybrid_indices(op)
|
||||
if not getattr(host_group, "supports_per_pool_backup_indices", False):
|
||||
if not getattr(host_group, "can_use_write_back_jit", False):
|
||||
if not host_group.supports_per_pool_backup_indices:
|
||||
if not host_group.can_use_write_back_jit:
|
||||
return self.move_hybrid_indices(op)
|
||||
return op.host_indices, op.device_indices, op.pool_transfers
|
||||
|
||||
def move_for_pool(host_pool, host_indices, device_indices):
|
||||
if getattr(host_pool, "can_use_write_back_jit", False):
|
||||
if host_indices.is_cuda:
|
||||
host_indices = host_indices.cpu()
|
||||
return host_indices, device_indices
|
||||
return self.move_indices(host_indices, device_indices)
|
||||
|
||||
host_indices, device_indices = move_for_pool(
|
||||
host_group.anchor_entry.host_pool,
|
||||
op.host_indices,
|
||||
op.device_indices,
|
||||
)
|
||||
pool_transfers = []
|
||||
for transfer in op.pool_transfers or []:
|
||||
entry = host_group.entry_map[transfer.name]
|
||||
transfer_host_indices, transfer_device_indices = move_for_pool(
|
||||
entry.host_pool,
|
||||
transfer.host_indices,
|
||||
transfer.device_indices,
|
||||
)
|
||||
pool_transfers.append(
|
||||
replace(
|
||||
transfer,
|
||||
host_indices=transfer_host_indices,
|
||||
device_indices=transfer_device_indices,
|
||||
)
|
||||
)
|
||||
return host_indices, device_indices, pool_transfers or None
|
||||
return self.move_hybrid_indices(op, write_back_jit=True)
|
||||
|
||||
def _l2_transfers(
|
||||
self,
|
||||
@@ -1075,35 +1205,53 @@ class HybridCacheController(BaseHiCacheController):
|
||||
kv_hit_pages * self.page_size,
|
||||
)
|
||||
|
||||
def _move_pool_indices(
|
||||
self, host_pool, host_indices, device_indices, *, write_back_jit: bool
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if uses_shared_host_layout(host_pool):
|
||||
return host_indices, device_indices
|
||||
if write_back_jit and host_pool.can_use_write_back_jit:
|
||||
if host_indices.is_cuda:
|
||||
host_indices = host_indices.cpu()
|
||||
return host_indices, device_indices
|
||||
return self.move_indices(host_indices, device_indices)
|
||||
|
||||
def move_hybrid_indices(
|
||||
self, operation: CacheOperation
|
||||
self, operation: CacheOperation, *, write_back_jit: bool = False
|
||||
) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]:
|
||||
host_indices, device_indices = self.move_indices(
|
||||
operation.host_indices, operation.device_indices
|
||||
host_indices, device_indices = self._move_pool_indices(
|
||||
self.mem_pool_host.anchor_entry.host_pool,
|
||||
operation.host_indices,
|
||||
operation.device_indices,
|
||||
write_back_jit=write_back_jit,
|
||||
)
|
||||
resolved_pool_transfers = None
|
||||
if operation.pool_transfers:
|
||||
resolved_pool_transfers = []
|
||||
for transfer in operation.pool_transfers:
|
||||
transfer_host_indices, transfer_device_indices = self.move_indices(
|
||||
transfer.host_indices, transfer.device_indices
|
||||
transfer_host_indices, transfer_device_indices = (
|
||||
self._move_pool_indices(
|
||||
self.mem_pool_host.entry_map[transfer.name].host_pool,
|
||||
transfer.host_indices,
|
||||
transfer.device_indices,
|
||||
write_back_jit=write_back_jit,
|
||||
)
|
||||
)
|
||||
# Keep the original PoolTransfer unchanged because tree-owned
|
||||
# transfers may still reference radix-tree host state. The
|
||||
# controller only needs a normalized execution-time copy.
|
||||
# Tree-owned transfers keep their original radix-tree indices.
|
||||
resolved_pool_transfers.append(
|
||||
PoolTransfer(
|
||||
name=transfer.name,
|
||||
replace(
|
||||
transfer,
|
||||
host_indices=transfer_host_indices,
|
||||
device_indices=transfer_device_indices,
|
||||
keys=transfer.keys,
|
||||
hit_policy=transfer.hit_policy,
|
||||
indices_from_pool=transfer.indices_from_pool,
|
||||
)
|
||||
)
|
||||
return host_indices, device_indices, resolved_pool_transfers
|
||||
|
||||
def _page_transfer(self, operation: PrefetchOperation) -> bool:
|
||||
with self.mem_pool_host.layout_lease():
|
||||
return self._page_transfer_with_stable_layout(operation)
|
||||
|
||||
def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None:
|
||||
# KV pools and KV-derived pools first — determines actual completed page count
|
||||
kv_completed_pages = super()._page_transfer(operation)
|
||||
|
||||
@@ -1172,6 +1320,10 @@ class HybridCacheController(BaseHiCacheController):
|
||||
operation.storage_start += trim_tokens
|
||||
|
||||
def _page_backup(self, operation):
|
||||
with self.mem_pool_host.layout_lease():
|
||||
return self._page_backup_with_stable_layout(operation)
|
||||
|
||||
def _page_backup_with_stable_layout(self, operation):
|
||||
# MLA KV is replicated across TP ranks and should still be written only
|
||||
# by TP0. Rank-sharded sidecars still need every TP rank.
|
||||
backup_transfers = [
|
||||
@@ -1401,7 +1553,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
for pool in derived_transfers:
|
||||
if pool.indices_from_pool == PoolName.KV:
|
||||
pool.host_indices = kv_host_indices
|
||||
pool.device_indices = kv_device_indices
|
||||
if pool.device_indices is None:
|
||||
pool.device_indices = kv_device_indices
|
||||
continue
|
||||
|
||||
source = next(
|
||||
@@ -1417,7 +1570,8 @@ class HybridCacheController(BaseHiCacheController):
|
||||
rollback_allocated()
|
||||
return None
|
||||
pool.host_indices = source.host_indices
|
||||
pool.device_indices = source.device_indices
|
||||
if pool.device_indices is None:
|
||||
pool.device_indices = source.device_indices
|
||||
return extra_pools
|
||||
|
||||
def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None:
|
||||
|
||||
@@ -26,6 +26,7 @@ from sglang.srt.mem_cache.pool_host.mha import (
|
||||
get_mha_host_pool_cls,
|
||||
)
|
||||
from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost
|
||||
from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.runtime_context import get_memory, get_parallel, get_serving
|
||||
|
||||
@@ -279,6 +280,26 @@ def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict:
|
||||
return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free)
|
||||
|
||||
|
||||
def _uses_unified_page_envelope_host(
|
||||
full_kv_pool: Any,
|
||||
swa_kv_pool: Any,
|
||||
use_mla: bool,
|
||||
mtp_swa_device_pools: tuple[Any, ...] = (),
|
||||
) -> bool:
|
||||
from sglang.srt.mem_cache.unified_memory_pool import UnifiedMHATokenToKVPool
|
||||
|
||||
return (
|
||||
not use_mla
|
||||
and not mtp_swa_device_pools
|
||||
and all(
|
||||
isinstance(pool, UnifiedMHATokenToKVPool)
|
||||
for pool in (full_kv_pool, swa_kv_pool)
|
||||
)
|
||||
and {full_kv_pool.grow_direction, swa_kv_pool.grow_direction} == {"up", "down"}
|
||||
and get_memory().hicache_host_memory_mode != "buffer_only"
|
||||
)
|
||||
|
||||
|
||||
def build_hybrid_swa_group(
|
||||
*,
|
||||
page_size: int,
|
||||
@@ -289,6 +310,7 @@ def build_hybrid_swa_group(
|
||||
use_mla: bool,
|
||||
kv_host_size: Optional[float] = None,
|
||||
swa_host_size: Optional[float] = None,
|
||||
host_full_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
swa_attn_allocator: Any = None,
|
||||
@@ -300,21 +322,39 @@ def build_hybrid_swa_group(
|
||||
transfer_layer_id_max = (
|
||||
max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1
|
||||
)
|
||||
kv_host_pool = build_kv_host_pool(
|
||||
kv_pool=full_kv_pool,
|
||||
page_size=page_size,
|
||||
use_mla=use_mla,
|
||||
host_size=kv_host_size,
|
||||
pool_label="full",
|
||||
)
|
||||
swa_host_pool = build_kv_host_pool(
|
||||
kv_pool=swa_kv_pool,
|
||||
page_size=page_size,
|
||||
use_mla=use_mla,
|
||||
host_size=swa_host_size,
|
||||
mtp_draft_device_pools=mtp_swa_device_pools,
|
||||
pool_label="swa",
|
||||
)
|
||||
if _uses_unified_page_envelope_host(
|
||||
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
|
||||
):
|
||||
memory = get_memory()
|
||||
kv_host_pool, swa_host_pool = (
|
||||
UnifiedPageEnvelopeHostPool.build_hybrid_swa_pool_pair(
|
||||
device_pools=(full_kv_pool, swa_kv_pool),
|
||||
host_to_device_ratio=memory.hicache_ratio,
|
||||
host_size=memory.hicache_size,
|
||||
page_size=page_size,
|
||||
layout=memory.hicache_mem_layout,
|
||||
allocator_type=_get_allocator_type(),
|
||||
mtp_draft_device_pools=mtp_swa_device_pools,
|
||||
)
|
||||
)
|
||||
has_shared_arena = True
|
||||
else:
|
||||
has_shared_arena = False
|
||||
kv_host_pool = build_kv_host_pool(
|
||||
kv_pool=full_kv_pool,
|
||||
page_size=page_size,
|
||||
use_mla=use_mla,
|
||||
host_size=kv_host_size,
|
||||
pool_label="full",
|
||||
)
|
||||
swa_host_pool = build_kv_host_pool(
|
||||
kv_pool=swa_kv_pool,
|
||||
page_size=page_size,
|
||||
use_mla=use_mla,
|
||||
host_size=swa_host_size,
|
||||
mtp_draft_device_pools=mtp_swa_device_pools,
|
||||
pool_label="swa",
|
||||
)
|
||||
if mtp_swa_device_pools:
|
||||
swa_layer_mapping = _with_mtp_layer_mapping(
|
||||
swa_layer_mapping,
|
||||
@@ -331,6 +371,7 @@ def build_hybrid_swa_group(
|
||||
layer_mapping=full_layer_mapping,
|
||||
transfer_layer_id_max=transfer_layer_id_max,
|
||||
is_anchor=True,
|
||||
host_evict_fn=host_full_evict_fn if has_shared_arena else None,
|
||||
),
|
||||
build_pool_entry(
|
||||
name=PoolName.SWA,
|
||||
@@ -406,6 +447,7 @@ def build_hybrid_swa_stack(
|
||||
load_cache_event,
|
||||
storage_backend: Optional[str],
|
||||
use_mla: bool,
|
||||
host_full_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
host_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
device_swa_evict_fn: Optional[Callable[[int], Any]] = None,
|
||||
prefetch_threshold: int = 256,
|
||||
@@ -422,9 +464,12 @@ def build_hybrid_swa_stack(
|
||||
)
|
||||
|
||||
kv_host_size = swa_host_size = None
|
||||
if get_memory().hicache_size > 0:
|
||||
memory = get_memory()
|
||||
if memory.hicache_size > 0 and not _uses_unified_page_envelope_host(
|
||||
full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools
|
||||
):
|
||||
kv_host_size, swa_host_size = _split_hicache_size(
|
||||
get_memory().hicache_size, (full_kv_pool, swa_kv_pool)
|
||||
memory.hicache_size, (full_kv_pool, swa_kv_pool)
|
||||
)
|
||||
|
||||
host_pool_group = build_hybrid_swa_group(
|
||||
@@ -436,6 +481,7 @@ def build_hybrid_swa_stack(
|
||||
use_mla=use_mla,
|
||||
kv_host_size=kv_host_size,
|
||||
swa_host_size=swa_host_size,
|
||||
host_full_evict_fn=host_full_evict_fn,
|
||||
host_swa_evict_fn=host_swa_evict_fn,
|
||||
device_swa_evict_fn=device_swa_evict_fn,
|
||||
# For SWA hybrid, device allocation goes through the inner allocator.
|
||||
@@ -1842,6 +1888,7 @@ class _SwaStrategy(StackStrategy):
|
||||
load_cache_event=load_cache_event,
|
||||
storage_backend=storage_backend,
|
||||
use_mla=False,
|
||||
host_full_evict_fn=lambda n: cache.evict_host(n, ComponentType.FULL),
|
||||
host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA),
|
||||
device_swa_evict_fn=lambda n: _evict_swa_for_device_alloc(cache, n),
|
||||
prefetch_threshold=prefetch_threshold,
|
||||
|
||||
@@ -177,6 +177,20 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
|
||||
fields = {}
|
||||
|
||||
backend = disagg.disaggregation_decode_retraction_backup
|
||||
unified_hybrid_swa = memory.enable_unified_memory and tp_worker.is_hybrid_swa
|
||||
unified_decode_radix = (
|
||||
memory.enable_unified_memory and disagg.disaggregation_decode_enable_radix_cache
|
||||
)
|
||||
if backend == "host_pool" and unified_decode_radix:
|
||||
raise ValueError(
|
||||
"Unified-memory H2D/D2H does not support host-pool decode "
|
||||
"retraction with decode radix cache yet."
|
||||
)
|
||||
if backend == "host_pool" and unified_hybrid_swa:
|
||||
raise ValueError(
|
||||
"Unified-memory hybrid-SWA H2D/D2H does not support host-pool "
|
||||
"decode retraction yet."
|
||||
)
|
||||
if backend is None:
|
||||
kv_cache = tp_worker.get_memory_pool()[1].get_kvcache()
|
||||
full_tokens_per_layer = (
|
||||
@@ -184,10 +198,17 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str:
|
||||
if tp_worker.is_hybrid_swa
|
||||
else None
|
||||
)
|
||||
# Host-pool retraction does not address unified page envelopes or
|
||||
# recurrent state, so those configurations stay on cpu_tensor.
|
||||
supports_host_pool = (
|
||||
draft_pools = tp_worker.model_runner.mtp_draft_device_pools
|
||||
unified_draft_host_pool_supported = (
|
||||
not memory.enable_unified_memory
|
||||
or not draft_pools
|
||||
or tp_worker.model_runner.spec_algorithm.is_dspark()
|
||||
)
|
||||
# Host-pool retraction has no recurrent-state sidecar, so a model with
|
||||
# recurrent state stays on cpu_tensor.
|
||||
supports_host_pool = (
|
||||
unified_draft_host_pool_supported
|
||||
and not unified_hybrid_swa
|
||||
and not uses_ssm_state(tp_worker.model_runner.model_config)
|
||||
and (
|
||||
isinstance(kv_cache, MHATokenToKVPool)
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from functools import cache
|
||||
from typing import Any, Callable, NamedTuple, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains
|
||||
from sglang.srt.utils import get_device_module
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -81,23 +83,63 @@ class L2TransferEngine:
|
||||
resolved = translate(indices)
|
||||
return resolved.to(original_device)
|
||||
|
||||
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
|
||||
start_event = self._start_event(None)
|
||||
def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]:
|
||||
prepared = []
|
||||
for transfer in transfers:
|
||||
host_indices, device_indices = transfer.host_pool.prepare_transfer_indices(
|
||||
transfer.host_indices,
|
||||
self._resolve_device_indices(transfer),
|
||||
self.io_backend,
|
||||
)
|
||||
prepared.append(
|
||||
transfer._replace(
|
||||
host_indices=host_indices,
|
||||
device_indices=device_indices,
|
||||
)
|
||||
)
|
||||
return prepared
|
||||
|
||||
@contextmanager
|
||||
def _submission(self, transfers, stream, transfer_key, start_event=None):
|
||||
start_event = self._start_event(start_event)
|
||||
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
|
||||
with device_module.stream(self.device_to_host_stream):
|
||||
start_event.wait(self.device_to_host_stream)
|
||||
device_indices = [self._resolve_device_indices(t) for t in transfers]
|
||||
ack_start.record()
|
||||
for transfer, dev_idx in zip(transfers, device_indices):
|
||||
transfer.host_pool.backup_from_device_all_layer(
|
||||
completion = TransferCompletion(ack_start, ack_finish, timing_enabled)
|
||||
domains = shared_host_layout_domains(t.host_pool for t in transfers)
|
||||
for domain in domains:
|
||||
domain.acquire_layout()
|
||||
finish_recorded = False
|
||||
try:
|
||||
with device_module.stream(stream):
|
||||
# Index preparation can read producer-owned device indices.
|
||||
start_event.wait(stream)
|
||||
prepared = self._prepare_transfers(transfers)
|
||||
ack_start.record()
|
||||
yield prepared, completion
|
||||
ack_finish.record()
|
||||
finish_recorded = True
|
||||
self._record_stream(transfers + prepared, stream)
|
||||
except Exception:
|
||||
stream.synchronize()
|
||||
raise
|
||||
finally:
|
||||
for domain in reversed(domains):
|
||||
domain.release_layout(
|
||||
ack_finish if finish_recorded else None,
|
||||
(id(self), transfer_key),
|
||||
)
|
||||
|
||||
def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion:
|
||||
with self._submission(
|
||||
transfers, self.device_to_host_stream, "device_to_host"
|
||||
) as (transfers, completion):
|
||||
for transfer in transfers:
|
||||
transfer.host_pool.backup_from_device_all_layer_physical(
|
||||
transfer.device_pool,
|
||||
transfer.host_indices,
|
||||
dev_idx,
|
||||
transfer.device_indices,
|
||||
self.io_backend,
|
||||
)
|
||||
ack_finish.record()
|
||||
self._record_stream(transfers, self.device_to_host_stream, device_indices)
|
||||
return TransferCompletion(ack_start, ack_finish, timing_enabled)
|
||||
return completion
|
||||
|
||||
def submit_host_to_device(
|
||||
self,
|
||||
@@ -107,15 +149,12 @@ class L2TransferEngine:
|
||||
start_event=None,
|
||||
on_layer_done=None,
|
||||
) -> TransferCompletion:
|
||||
start_event = self._start_event(start_event)
|
||||
ack_start, ack_finish, timing_enabled = make_timing_event_pair()
|
||||
primary = transfers[0] if transfers else None
|
||||
with device_module.stream(self.host_to_device_stream):
|
||||
start_event.wait(self.host_to_device_stream)
|
||||
device_indices = [self._resolve_device_indices(t) for t in transfers]
|
||||
ack_start.record()
|
||||
with self._submission(
|
||||
transfers, self.host_to_device_stream, "host_to_device", start_event
|
||||
) as (transfers, completion):
|
||||
primary = transfers[0] if transfers else None
|
||||
for layer_id in range(transfer_layer_id_max):
|
||||
for transfer, dev_idx in zip(transfers, device_indices):
|
||||
for transfer in transfers:
|
||||
local_layer_id = (
|
||||
transfer.layer_mapper(layer_id)
|
||||
if transfer.layer_mapper is not None
|
||||
@@ -127,19 +166,17 @@ class L2TransferEngine:
|
||||
and layer_id >= transfer.host_pool.layer_num
|
||||
):
|
||||
continue
|
||||
transfer.host_pool.load_to_device_per_layer(
|
||||
transfer.host_pool.load_to_device_per_layer_physical(
|
||||
transfer.device_pool,
|
||||
transfer.host_indices,
|
||||
dev_idx,
|
||||
transfer.device_indices,
|
||||
local_layer_id,
|
||||
self.io_backend,
|
||||
is_draft=transfer.is_draft,
|
||||
)
|
||||
if on_layer_done is not None:
|
||||
on_layer_done(layer_id)
|
||||
ack_finish.record()
|
||||
self._record_stream(transfers, self.host_to_device_stream, device_indices)
|
||||
return TransferCompletion(ack_start, ack_finish, timing_enabled)
|
||||
return completion
|
||||
|
||||
@staticmethod
|
||||
def _start_event(start_event):
|
||||
|
||||
@@ -63,6 +63,8 @@ class LogicalHostPool:
|
||||
compressed side pools use these logical FULL indices as stable page anchors.
|
||||
"""
|
||||
|
||||
shared_allocation_domain = None
|
||||
|
||||
def __init__(self, size: int, page_size: int, layout: str = "layer_first"):
|
||||
if size % page_size != 0:
|
||||
raise ValueError(
|
||||
@@ -150,6 +152,26 @@ class LogicalHostPool:
|
||||
):
|
||||
pass
|
||||
|
||||
def prepare_transfer_indices(self, host_indices, device_indices, io_backend):
|
||||
return host_indices, device_indices
|
||||
|
||||
def backup_from_device_all_layer_physical(
|
||||
self, device_pool, host_indices, device_indices, io_backend
|
||||
):
|
||||
pass
|
||||
|
||||
def load_to_device_per_layer_physical(
|
||||
self,
|
||||
device_pool,
|
||||
host_indices,
|
||||
device_indices,
|
||||
layer_id,
|
||||
io_backend,
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
):
|
||||
pass
|
||||
|
||||
def load_to_device_per_layer(
|
||||
self,
|
||||
device_pool,
|
||||
@@ -174,6 +196,9 @@ class LogicalHostPool:
|
||||
def get_page_buffer_meta(self, indices):
|
||||
return None
|
||||
|
||||
def get_page_buffer_element_size(self, split_factor: int = 1):
|
||||
return None
|
||||
|
||||
def get_ksize_per_token(self):
|
||||
return 0
|
||||
|
||||
|
||||
@@ -3,10 +3,11 @@ from __future__ import annotations
|
||||
import abc
|
||||
import logging
|
||||
import threading
|
||||
from collections.abc import Iterable
|
||||
from contextlib import contextmanager
|
||||
from contextvars import ContextVar
|
||||
from functools import wraps
|
||||
from typing import Optional
|
||||
from typing import Optional, TypeGuard
|
||||
|
||||
import torch
|
||||
|
||||
@@ -35,6 +36,26 @@ _host_memory_budget: ContextVar[Optional[int]] = ContextVar(
|
||||
)
|
||||
|
||||
|
||||
def uses_shared_host_layout(host_pool: object) -> TypeGuard[HostKVCache]:
|
||||
return (
|
||||
isinstance(host_pool, HostKVCache)
|
||||
and host_pool.shared_allocation_domain is not None
|
||||
)
|
||||
|
||||
|
||||
def shared_host_layout_domains(host_pools: Iterable[object]) -> list:
|
||||
domains = []
|
||||
seen = set()
|
||||
for pool in host_pools:
|
||||
if not uses_shared_host_layout(pool):
|
||||
continue
|
||||
domain = pool.shared_allocation_domain
|
||||
if id(domain) not in seen:
|
||||
seen.add(id(domain))
|
||||
domains.append(domain)
|
||||
return domains
|
||||
|
||||
|
||||
@contextmanager
|
||||
def host_memory_budget_scope(budget_bytes: int):
|
||||
"""Book every pool built inside against one snapshot, not re-sampled psutil."""
|
||||
@@ -135,6 +156,7 @@ def synchronized(func):
|
||||
class HostKVCache(abc.ABC):
|
||||
dcp_size = 1
|
||||
dcp_rank = 0
|
||||
shared_allocation_domain = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -323,6 +345,55 @@ class HostKVCache(abc.ABC):
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def get_page_buffer_element_size(self, split_factor: int = 1) -> Optional[int]:
|
||||
"""Byte size of one storage element, or None for a logical anchor."""
|
||||
indices = torch.zeros(self.page_size, dtype=torch.int64)
|
||||
meta = (
|
||||
self.get_split_heads_page_buffer_meta(indices, split_factor)
|
||||
if split_factor != 1
|
||||
else self.get_page_buffer_meta(indices)
|
||||
)
|
||||
sizes = meta[1] if meta else None
|
||||
return int(sizes[0]) if sizes else None
|
||||
|
||||
def prepare_transfer_indices(
|
||||
self, host_indices, device_indices, io_backend
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Resolve indices that must remain stable through one L2 submission.
|
||||
|
||||
Normal host pools expose physical indices directly and controllers have
|
||||
already moved them to the backend-required device. Shared compacting
|
||||
pools override this hook and resolve their logical ids while the transfer
|
||||
engine holds a layout lease.
|
||||
"""
|
||||
return host_indices, device_indices
|
||||
|
||||
def backup_from_device_all_layer_physical(
|
||||
self, device_pool, host_indices, device_indices, io_backend
|
||||
) -> None:
|
||||
self.backup_from_device_all_layer(
|
||||
device_pool, host_indices, device_indices, io_backend
|
||||
)
|
||||
|
||||
def load_to_device_per_layer_physical(
|
||||
self,
|
||||
device_pool,
|
||||
host_indices,
|
||||
device_indices,
|
||||
layer_id,
|
||||
io_backend,
|
||||
*,
|
||||
is_draft: bool = False,
|
||||
) -> None:
|
||||
self.load_to_device_per_layer(
|
||||
device_pool,
|
||||
host_indices,
|
||||
device_indices,
|
||||
layer_id,
|
||||
io_backend,
|
||||
is_draft=is_draft,
|
||||
)
|
||||
|
||||
@abc.abstractmethod
|
||||
def get_data_page(self, index, flat: bool = True) -> torch.Tensor:
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
@@ -8,6 +9,7 @@ import torch
|
||||
|
||||
from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer
|
||||
from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool
|
||||
from sglang.srt.mem_cache.pool_host.base import HostKVCache, shared_host_layout_domains
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -76,6 +78,33 @@ class HostPoolGroup:
|
||||
def get_pool(self, name: PoolName):
|
||||
return self.get_entry(name).host_pool
|
||||
|
||||
@contextmanager
|
||||
def layout_lease(self):
|
||||
domains = shared_host_layout_domains(entry.host_pool for entry in self.entries)
|
||||
for domain in domains:
|
||||
domain.acquire_layout()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for domain in reversed(domains):
|
||||
domain.release_layout()
|
||||
|
||||
def get_page_buffer_element_size(self, split_factor: int = 1):
|
||||
anchor_pool = self.anchor_entry.host_pool
|
||||
if (
|
||||
not isinstance(anchor_pool, HostKVCache)
|
||||
or anchor_pool.shared_allocation_domain is None
|
||||
):
|
||||
return anchor_pool.get_page_buffer_element_size(split_factor)
|
||||
|
||||
element_sizes = set()
|
||||
for entry in self.entries:
|
||||
element_size = entry.host_pool.get_page_buffer_element_size(split_factor)
|
||||
if element_size is None:
|
||||
return None
|
||||
element_sizes.add(element_size)
|
||||
return element_sizes.pop() if len(element_sizes) == 1 else None
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
"""Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes)."""
|
||||
host_by_device_ptr = {}
|
||||
@@ -199,7 +228,8 @@ class HostPoolGroup:
|
||||
for transfer in derived_transfers:
|
||||
if transfer.indices_from_pool == self.anchor_entry.name:
|
||||
transfer.host_indices = primary_host_indices
|
||||
transfer.device_indices = primary_device_indices
|
||||
if transfer.device_indices is None:
|
||||
transfer.device_indices = primary_device_indices
|
||||
continue
|
||||
|
||||
source = next(
|
||||
@@ -215,7 +245,8 @@ class HostPoolGroup:
|
||||
rollback()
|
||||
return None
|
||||
transfer.host_indices = source.host_indices
|
||||
transfer.device_indices = source.device_indices
|
||||
if transfer.device_indices is None:
|
||||
transfer.device_indices = source.device_indices
|
||||
return transfers
|
||||
|
||||
def release_transfers(self, transfers: list[PoolTransfer] | None) -> int:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -648,25 +648,16 @@ class UMBPStore(HiCacheStorage):
|
||||
if "dram_page_size" in extra:
|
||||
page_byte_size = int(extra["dram_page_size"])
|
||||
elif mem_pool_host is not None:
|
||||
# Probe element_size from the same buffer-meta helper that
|
||||
# batch_preprocess will actually use; this matches per-call
|
||||
# Put/Get size byte-for-byte for MHA / MHA-split / MLA / NSA
|
||||
# without per-case formulas (NSA in particular: get_ksize_per_token
|
||||
# would over-count by the indexer buffer that is never put to UMBP).
|
||||
dummy = torch.zeros(mem_pool_host.page_size, dtype=torch.int64)
|
||||
if self.is_mla_backend:
|
||||
meta = mem_pool_host.get_page_buffer_meta(dummy)
|
||||
elif storage_config is not None and getattr(
|
||||
storage_config, "should_split_heads", False
|
||||
split_factor = 1
|
||||
if (
|
||||
not self.is_mla_backend
|
||||
and storage_config is not None
|
||||
and storage_config.should_split_heads
|
||||
):
|
||||
sf = storage_config.tp_lcm_size // storage_config.tp_size
|
||||
meta = mem_pool_host.get_split_heads_page_buffer_meta(dummy, sf)
|
||||
else:
|
||||
meta = mem_pool_host.get_page_buffer_meta(dummy)
|
||||
# A hybrid logical anchor returns None here by design; leave
|
||||
# dram_page_size at 0 and let the per-pool v2 sizes handle it.
|
||||
esz = meta[1] if meta else None
|
||||
page_byte_size = int(esz[0]) if esz else 0
|
||||
split_factor = storage_config.tp_lcm_size // storage_config.tp_size
|
||||
page_byte_size = mem_pool_host.get_page_buffer_element_size(
|
||||
split_factor
|
||||
)
|
||||
|
||||
if (
|
||||
page_byte_size is not None
|
||||
@@ -689,7 +680,7 @@ class UMBPStore(HiCacheStorage):
|
||||
if (
|
||||
mem_pool_host is not None
|
||||
and storage_config is not None
|
||||
and getattr(storage_config, "should_split_heads", False)
|
||||
and storage_config.should_split_heads
|
||||
)
|
||||
else ""
|
||||
),
|
||||
|
||||
@@ -627,6 +627,20 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
raw = self._unified_buffer._raw
|
||||
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
|
||||
|
||||
def get_page_envelope_buffer(self) -> torch.Tensor:
|
||||
"""Return this sub-pool's page-strided view of the shared allocation."""
|
||||
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
||||
raw = self._unified_buffer._raw
|
||||
page_count = (
|
||||
self._unified_buffer.max_slots(self._sub_pool_name) // self.page_size
|
||||
)
|
||||
return raw[: page_count * self._page_bytes].view(page_count, self._page_bytes)
|
||||
|
||||
@property
|
||||
def grow_direction(self) -> str:
|
||||
"""Physical growth direction used by this sub-pool's L1 allocator."""
|
||||
return self._unified_buffer.spec(self._sub_pool_name).grow_direction
|
||||
|
||||
def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor:
|
||||
return (indices // self.page_size) * (
|
||||
self.page_size * self.kernel_page_blocks
|
||||
|
||||
@@ -1474,21 +1474,14 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
assert req.seqlen > 1
|
||||
|
||||
device_indices, extra_transfers = self._retraction_device_transfers(req)
|
||||
host_indices = self.host_pool_group.alloc(len(device_indices))
|
||||
if host_indices is None:
|
||||
self._reclaim_retraction_host(len(device_indices))
|
||||
host_indices = self.host_pool_group.alloc(len(device_indices))
|
||||
if host_indices is None:
|
||||
return None
|
||||
|
||||
resolved = self.host_pool_group.resolve_host_transfers(
|
||||
allocation = self.cache_controller.allocate_host_transfers(
|
||||
device_indices,
|
||||
extra_transfers or None,
|
||||
primary_device_indices=device_indices,
|
||||
primary_host_indices=host_indices,
|
||||
reclaim=self._reclaim_retraction_host,
|
||||
)
|
||||
if resolved is None and extra_transfers:
|
||||
self.host_pool_group.free(host_indices)
|
||||
if allocation is None:
|
||||
return None
|
||||
host_indices, resolved = allocation
|
||||
|
||||
backup = RetractionBackup(
|
||||
host_indices=host_indices,
|
||||
@@ -1642,11 +1635,16 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def _execute_kv_backup(self, node_id, device_value, comp_xfers, sidecar_xfers):
|
||||
"""Execute Backup action."""
|
||||
kv_tokens = len(device_value)
|
||||
host_avail = self.cache_controller.mem_pool_host.available_size()
|
||||
if host_avail < kv_tokens:
|
||||
needed = kv_tokens - host_avail
|
||||
if self.evict_host(needed) < needed:
|
||||
return None
|
||||
anchor_entry = self.cache_controller.mem_pool_host.anchor_entry
|
||||
if (
|
||||
anchor_entry.host_pool.shared_allocation_domain is None
|
||||
or anchor_entry.host_evict_fn is None
|
||||
):
|
||||
host_avail = self.cache_controller.mem_pool_host.available_size()
|
||||
if host_avail < kv_tokens:
|
||||
needed = kv_tokens - host_avail
|
||||
if self.evict_host(needed) < needed:
|
||||
return None
|
||||
aux_xfers = [x for xfers in comp_xfers.values() for x in xfers]
|
||||
aux_xfers.extend(sidecar_xfers)
|
||||
# Defer submission so the next flush can merge pending node backups.
|
||||
|
||||
Reference in New Issue
Block a user