diff --git a/python/sglang/srt/arg_groups/kv_cache_hook.py b/python/sglang/srt/arg_groups/kv_cache_hook.py index 8b7219e80d..8ac8e63130 100644 --- a/python/sglang/srt/arg_groups/kv_cache_hook.py +++ b/python/sglang/srt/arg_groups/kv_cache_hook.py @@ -15,6 +15,7 @@ from sglang.srt.arg_groups.overrides import ( resolving_view, use_mla_backend, ) +from sglang.srt.configs.hybrid_arch import mambaish_config from sglang.srt.environ import envs from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.runtime_context import get_platform @@ -371,7 +372,6 @@ def handle_cache_compatibility(server_args: Any) -> None: "--disaggregation-decode-retraction-backup=host_pool requires " "--disable-priority-preemption when priority scheduling is enabled." ) - if cfg.radix_eviction_policy == "tlru": tlru_config = cfg.radix_eviction_policy_config or {} threshold = tlru_config.get("threshold", 0) @@ -467,15 +467,38 @@ def handle_unified_memory_pool(server_args: Any) -> None: "ships host/C4 rows straight from the allocator, bypassing the " "virtual->physical translation the unified pool needs." ) - assert cfg.disaggregation_decode_retraction_backup != "host_pool", ( - "--enable-unified-memory with PD disaggregation does not support " - "--disaggregation-decode-retraction-backup=host_pool; use " - "cpu_tensor (the automatic default for unified pools)." - ) - assert not cfg.disaggregation_decode_enable_offload_kvcache, ( - "--enable-unified-memory with PD disaggregation does not yet support " - "--disaggregation-decode-enable-offload-kvcache." - ) + if cfg.disaggregation_decode_enable_offload_kvcache: + assert cfg.hicache_storage_backend == "file", ( + "--enable-unified-memory with decode KV offload currently " + "supports --hicache-storage-backend=file only." + ) + model_config = model_config_of(server_args) + assert not use_mla_backend(server_args), ( + "--enable-unified-memory decode KV offload does not support " + "MLA models yet." + ) + assert mambaish_config(model_config) is None, ( + "--enable-unified-memory decode KV offload does not support " + "hybrid-Mamba models." + ) + assert not model_config.is_hybrid_swa, ( + "--enable-unified-memory decode KV offload does not support " + "hybrid-SWA H2D/D2H transfers yet." + ) + if cfg.disaggregation_decode_retraction_backup == "host_pool": + model_config = model_config_of(server_args) + assert not cfg.disaggregation_decode_enable_radix_cache, ( + "--enable-unified-memory host-pool decode retraction does not " + "support decode radix-cache H2D/D2H transfers yet." + ) + assert mambaish_config(model_config) is None, ( + "--enable-unified-memory host-pool decode retraction does not " + "support hybrid-Mamba models." + ) + assert not model_config.is_hybrid_swa, ( + "--enable-unified-memory host-pool decode retraction does not " + "support hybrid-SWA H2D/D2H transfers yet." + ) assert cfg.speculative_algorithm in (None, "DSPARK"), ( "--enable-unified-memory only supports --speculative-algorithm " "DSPARK (chain draft); other speculative algorithms are not yet " diff --git a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py index 7e9f85aef6..c6baa81fc4 100644 --- a/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py +++ b/python/sglang/srt/disaggregation/decode_kvcache_offload_manager.py @@ -14,7 +14,14 @@ from sglang.srt.environ import envs from sglang.srt.managers.cache_controller import HiCacheController from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.hicache_storage import ( + PoolHitPolicy, + PoolName, + PoolTransfer, +) from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( + build_hybrid_swa_stack, build_kv_host_pool, ) from sglang.srt.mem_cache.memory_pool import ( @@ -22,6 +29,7 @@ from sglang.srt.mem_cache.memory_pool import ( MLATokenToKVPool, ReqToTokenPool, ) +from sglang.srt.mem_cache.unified_memory_pool import UnifiedSWAKVPool from sglang.srt.mem_cache.utils import storage_namespace_seed from sglang.srt.runtime_context import ( get_memory, @@ -57,15 +65,6 @@ class DecodeKVCacheOffloadManager: self.offload_stride = max( self.page_size, (env_stride // self.page_size) * self.page_size ) - kv_cache = self.token_to_kv_pool_allocator.get_kvcache() - if not isinstance(kv_cache, (MHATokenToKVPool, MLATokenToKVPool)): - raise ValueError("Unsupported KV cache type for decode offload") - self.decode_host_mem_pool = build_kv_host_pool( - kv_pool=kv_cache, - page_size=self.page_size, - use_mla=isinstance(kv_cache, MLATokenToKVPool), - ) - self.tp_group = tp_group self.tp_world_size = torch.distributed.get_world_size(group=self.tp_group) @@ -80,17 +79,56 @@ class DecodeKVCacheOffloadManager: f"Invalid hicache storage backend extra config JSON: {e}" ) - self.cache_controller = HiCacheController( - token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, - mem_pool_host=self.decode_host_mem_pool, - page_size=self.page_size, - tp_group=tp_group, - io_backend=get_memory().hicache_io_backend, - load_cache_event=threading.Event(), - storage_backend=get_memory().hicache_storage_backend, - model_name=get_serving().served_model_name, - storage_backend_extra_config=hicache_storage_backend_extra_config, - ) + self.kv_cache = self.token_to_kv_pool_allocator.get_kvcache() + if isinstance(self.kv_cache, UnifiedSWAKVPool): + full_mapping = { + gid: lid + for gid, (lid, is_swa) in self.kv_cache.layers_mapping.items() + if not is_swa + } + swa_mapping = { + gid: lid + for gid, (lid, is_swa) in self.kv_cache.layers_mapping.items() + if is_swa + } + params = CacheInitParams( + disable=False, + req_to_token_pool=self.req_to_token_pool, + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + page_size=self.page_size, + tp_cache_group=tp_group, + ) + self.decode_host_mem_pool, self.cache_controller = build_hybrid_swa_stack( + params=params, + full_kv_pool=self.kv_cache.full_kv_pool, + swa_kv_pool=self.kv_cache.swa_kv_pool, + full_layer_mapping=full_mapping, + swa_layer_mapping=swa_mapping, + load_cache_event=threading.Event(), + storage_backend=get_memory().hicache_storage_backend, + use_mla=False, + model_name=get_serving().served_model_name, + storage_backend_extra_config=hicache_storage_backend_extra_config, + ) + else: + if not isinstance(self.kv_cache, (MHATokenToKVPool, MLATokenToKVPool)): + raise ValueError("Unsupported KV cache type for decode offload") + self.decode_host_mem_pool = build_kv_host_pool( + kv_pool=self.kv_cache, + page_size=self.page_size, + use_mla=isinstance(self.kv_cache, MLATokenToKVPool), + ) + self.cache_controller = HiCacheController( + token_to_kv_pool_allocator=self.token_to_kv_pool_allocator, + mem_pool_host=self.decode_host_mem_pool, + page_size=self.page_size, + tp_group=tp_group, + io_backend=get_memory().hicache_io_backend, + load_cache_event=threading.Event(), + storage_backend=get_memory().hicache_storage_backend, + model_name=get_serving().served_model_name, + storage_backend_extra_config=hicache_storage_backend_extra_config, + ) self.ongoing_offload = {} self.ongoing_backup = {} @@ -120,6 +158,32 @@ class DecodeKVCacheOffloadManager: # Page-aligned prompt length; the prefill instance offloaded this part. return len(req.origin_input_ids) // self.page_size * self.page_size + def has_inflight_device_transfer(self) -> bool: + """Whether a device-to-host copy may still read unified KV pages.""" + return bool(self.ongoing_offload) + + def _resolve_offload_transfers( + self, virtual_indices: torch.Tensor + ) -> tuple[torch.Tensor, list[PoolTransfer]]: + full_indices = virtual_indices + if not isinstance(self.kv_cache, UnifiedSWAKVPool): + return full_indices, [] + + swa_indices = self.token_to_kv_pool_allocator.translate_loc_from_full_to_swa( + virtual_indices + ) + # Page-aligned chunks have whole-page bindings; zero denotes the sink page. + live_swa_indices = swa_indices[swa_indices > 0].to(torch.int64) + if live_swa_indices.numel() == 0: + return full_indices, [] + return full_indices, [ + PoolTransfer( + name=PoolName.SWA, + device_indices=live_swa_indices, + hit_policy=PoolHitPolicy.TRAILING_PAGES, + ) + ] + def offload_kv_cache(self, req) -> bool: """Offload incremental KV cache for decode side.""" @@ -159,7 +223,10 @@ class DecodeKVCacheOffloadManager: start = prefill_offloaded_len + state.inc_len end = start + incremental_aligned_len incremental_tokens = all_tokens[start:end] - incremental_indices = token_indices[start:end] + incremental_indices = token_indices[start:end].to(torch.int64) + device_indices, pool_transfers = self._resolve_offload_transfers( + incremental_indices + ) # Prefill-aligned GPU slots are freed at request finish in # _release_finished_req, NOT here. The decoding request @@ -171,8 +238,9 @@ class DecodeKVCacheOffloadManager: self.request_counter += 1 ack_id = self.request_counter host_indices = self.cache_controller.write( - device_indices=incremental_indices.long(), + device_indices=device_indices, node_id=ack_id, + extra_pools=pool_transfers, ) if host_indices is None: logger.error(f"Not enough host memory for request {req.rid}") @@ -184,6 +252,7 @@ class DecodeKVCacheOffloadManager: host_indices, incremental_tokens, time.time(), + pool_transfers, ) state.inc_len += incremental_aligned_len return True @@ -219,6 +288,7 @@ class DecodeKVCacheOffloadManager: host_indices, incremental_tokens, start_time, + pool_transfers, ) = self.ongoing_offload.pop(ack_id) self._mark_offload_finished(req) @@ -228,7 +298,12 @@ class DecodeKVCacheOffloadManager: else None ) last_hash = self._trigger_backup( - req, host_indices, incremental_tokens, start_time, prior_hash + req, + host_indices, + incremental_tokens, + start_time, + prior_hash, + pool_transfers, ) if req in self.offloaded_state: self.offloaded_state[req].last_hash = last_hash @@ -259,9 +334,15 @@ class DecodeKVCacheOffloadManager: for _ in range(finish_count): storage_operation = self.cache_controller.ack_backup_queue.get() ack_id = storage_operation.id - req_id, host_indices, start_time = self.ongoing_backup.pop(ack_id) + req_id, host_indices, start_time, pool_transfers = self.ongoing_backup.pop( + ack_id + ) # Release host memory + for transfer in pool_transfers: + self.decode_host_mem_pool.get_pool(transfer.name).free( + transfer.host_indices + ) self.decode_host_mem_pool.free(host_indices) logger.debug( @@ -269,16 +350,31 @@ class DecodeKVCacheOffloadManager: ) def _trigger_backup( - self, req, host_indices, incremental_tokens, start_time, prior_hash + self, + req, + host_indices, + incremental_tokens, + start_time, + prior_hash, + pool_transfers=(), ): """Trigger async backup from host to storage.""" page_hashes = self._compute_prefix_hash(req, incremental_tokens, prior_hash) + for transfer in pool_transfers: + page_count = transfer.device_indices.numel() // self.page_size + transfer.keys = page_hashes[-page_count:] ack_id = self.cache_controller.write_storage( host_indices, incremental_tokens, hash_value=page_hashes, + **({"extra_pools": pool_transfers} if pool_transfers else {}), + ) + self.ongoing_backup[ack_id] = ( + req.rid, + host_indices, + start_time, + pool_transfers, ) - self.ongoing_backup[ack_id] = (req.rid, host_indices, start_time) return page_hashes[-1] if len(page_hashes) > 0 else prior_hash def _compute_prefix_hash(self, req: Req, tokens, prior_hash=""): diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index a8b9115ed9..2876f1d814 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -131,9 +131,14 @@ def unified_memory_disagg_move_gate(scheduler): if scheduler.disaggregation_mode == DisaggregationMode.DECODE: def decode_gate() -> bool: + decode_offload_manager = scheduler.decode_offload_manager return not ( scheduler.disagg_decode_transfer_queue.queue or scheduler.disagg_decode_prealloc_queue.has_published_destinations + or ( + decode_offload_manager is not None + and decode_offload_manager.has_inflight_device_transfer() + ) ) return decode_gate diff --git a/python/sglang/srt/managers/cache_controller.py b/python/sglang/srt/managers/cache_controller.py index 474623a6f7..115720b06f 100644 --- a/python/sglang/srt/managers/cache_controller.py +++ b/python/sglang/srt/managers/cache_controller.py @@ -803,10 +803,13 @@ class HiCacheController: device_indices: torch.Tensor, priority: Optional[int] = None, node_id: int = -1, + extra_pools: Optional[List[PoolTransfer]] = None, ) -> Optional[torch.Tensor]: """ Back up KV caches from device memory to host memory. """ + if extra_pools: + raise ValueError("Side-pool transfers require HybridCacheController.") host_indices = self.mem_pool_host.alloc(len(device_indices)) if host_indices is None: return None diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index f592b69739..c48ff62d5a 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -37,6 +37,7 @@ from sglang.srt.mem_cache.hicache_storage import ( ) from sglang.srt.mem_cache.l2_transfer import L2Transfer from sglang.srt.mem_cache.pool_host import HostPoolGroup, PoolEntry +from sglang.srt.mem_cache.pool_host.base import uses_shared_host_layout from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.srt.mem_cache.radix_cache import RadixKey @@ -452,17 +453,30 @@ class HybridCacheController(BaseHiCacheController): release_queue.queue.clear() self.prefetch_tokens_occupied = 0 - def write( + def allocate_host_transfers( self, device_indices: torch.Tensor, - priority: Optional[int] = None, - node_id: int = -1, extra_pools: Optional[list[PoolTransfer]] = None, - flush: bool = True, - ) -> Optional[torch.Tensor]: - """Queue a D2H backup; flush=False leaves it queued so the caller can - merge several nodes into one start_writing() submit.""" - host_indices = self.mem_pool_host.alloc(len(device_indices)) + *, + reclaim: Optional[Callable[[int], Any]] = None, + ) -> Optional[tuple[torch.Tensor, Optional[list[PoolTransfer]]]]: + if self._uses_shared_host_domain(extra_pools): + allocation = self.allocate_shared_host_transfers( + device_indices, extra_pools + ) + anchor = self.mem_pool_host.anchor_entry + if ( + allocation is None + and reclaim is not None + and anchor.host_evict_fn is None + ): + reclaim(len(device_indices)) + allocation = self.allocate_shared_host_transfers( + device_indices, extra_pools + ) + return allocation + + host_indices = self.mem_pool_host.alloc(len(device_indices), reclaim=reclaim) if host_indices is None: return None pool_transfers = self.mem_pool_host.resolve_host_transfers( @@ -473,6 +487,20 @@ class HybridCacheController(BaseHiCacheController): if pool_transfers is None and extra_pools: self.mem_pool_host.free(host_indices) return None + return host_indices, pool_transfers + + def write( + self, + device_indices: torch.Tensor, + priority: Optional[int] = None, + node_id: int = -1, + extra_pools: Optional[list[PoolTransfer]] = None, + flush: bool = True, + ) -> Optional[torch.Tensor]: + allocation = self.allocate_host_transfers(device_indices, extra_pools) + if allocation is None: + return None + host_indices, pool_transfers = allocation self.write_queue.append( CacheOperation( @@ -487,6 +515,135 @@ class HybridCacheController(BaseHiCacheController): self.start_writing() return host_indices + def _uses_shared_host_domain( + self, extra_pools: Optional[list[PoolTransfer]] + ) -> bool: + anchor_pool = self.mem_pool_host.anchor_entry.host_pool + if not uses_shared_host_layout(anchor_pool): + return False + domain = anchor_pool.shared_allocation_domain + for transfer in extra_pools or []: + if ( + transfer.indices_from_pool is not None + or transfer.host_indices is not None + or transfer.device_indices is None + ): + continue + entry = self.mem_pool_host.entry_map.get(transfer.name) + if entry is not None and ( + not uses_shared_host_layout(entry.host_pool) + or entry.host_pool.shared_allocation_domain is not domain + ): + return False + return True + + def _alloc_shared_host_requests_with_reclaim( + self, requests: list[tuple[PoolName, int]] + ) -> Optional[list[torch.Tensor]]: + entries = [self.mem_pool_host.entry_map[name] for name, _ in requests] + domain = entries[0].host_pool.shared_allocation_domain + if domain is None or any( + entry.host_pool.shared_allocation_domain is not domain for entry in entries + ): + raise ValueError("Host pools do not share one allocation domain.") + domain_requests = [ + (entry.host_pool.pool_label, need_size) + for entry, (_, need_size) in zip(entries, requests, strict=True) + ] + + allocated = domain.alloc_many(domain_requests) + if allocated is not None: + return allocated + + pools = [entry.host_pool for entry in entries] + requested_bytes = sum( + need_size * pool.size_per_token + for pool, (_, need_size) in zip(pools, requests, strict=True) + ) + requested_names = list(dict.fromkeys(name for name, _ in reversed(requests))) + candidates = [self.mem_pool_host.entry_map[name] for name in requested_names] + candidates.extend( + entry + for entry in self.mem_pool_host.entries + if entry.name not in requested_names + and uses_shared_host_layout(entry.host_pool) + and entry.host_pool.shared_allocation_domain is domain + ) + + # alloc_many compacts the shared layout before reporting failure. Reclaim + # only the remaining byte or per-side logical-slot shortfall, retrying + # after every successful eviction. The callbacks release only evictable + # tree entries; protected retraction backups remain safe. + while True: + made_progress = False + for entry in candidates: + if entry.host_evict_fn is None: + continue + pool = entry.host_pool + shortfall_bytes = max( + pool.page_size * pool.size_per_token, + requested_bytes - domain.free_bytes(), + ) + tokens = ( + ( + (shortfall_bytes + pool.size_per_token - 1) + // pool.size_per_token + + pool.page_size + - 1 + ) + // pool.page_size + * pool.page_size + ) + if entry.host_evict_fn(tokens) <= 0: + continue + made_progress = True + allocated = domain.alloc_many(domain_requests) + if allocated is not None: + return allocated + if not made_progress: + return None + + def allocate_shared_host_transfers( + self, + kv_device_indices: torch.Tensor, + extra_pools: Optional[list[PoolTransfer]] = None, + ) -> Optional[tuple[torch.Tensor, Optional[list[PoolTransfer]]]]: + """Atomically reserve the anchor and all independent host side pools.""" + requests = [(self.mem_pool_host.anchor_entry.name, len(kv_device_indices))] + request_transfers: list[PoolTransfer] = [] + for transfer in extra_pools or []: + if transfer.indices_from_pool is not None: + continue + entry = self.mem_pool_host.entry_map.get(transfer.name) + if ( + entry is None + or transfer.host_indices is not None + or transfer.device_indices is None + ): + continue + requests.append((transfer.name, len(transfer.device_indices))) + request_transfers.append(transfer) + + allocated = self._alloc_shared_host_requests_with_reclaim(requests) + if allocated is None: + return None + host_indices = allocated[0] + for transfer, indices in zip(request_transfers, allocated[1:], strict=True): + transfer.host_indices = indices + + resolved = self.mem_pool_host.resolve_host_transfers( + extra_pools, + primary_device_indices=kv_device_indices, + primary_host_indices=host_indices, + ) + if resolved is None and extra_pools: + for (name, _), indices in zip(requests, allocated, strict=True): + self.mem_pool_host.free(indices, pool=name) + for transfer in request_transfers: + transfer.host_indices = None + return None + return host_indices, resolved + def _move_op_indices( self, op: CacheOperation ) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]: @@ -498,39 +655,12 @@ class HybridCacheController(BaseHiCacheController): host_group = self.mem_pool_host if self.io_backend != "kernel" or host_group.layout != "page_first": return self.move_hybrid_indices(op) - if not getattr(host_group, "supports_per_pool_backup_indices", False): - if not getattr(host_group, "can_use_write_back_jit", False): + if not host_group.supports_per_pool_backup_indices: + if not host_group.can_use_write_back_jit: return self.move_hybrid_indices(op) return op.host_indices, op.device_indices, op.pool_transfers - def move_for_pool(host_pool, host_indices, device_indices): - if getattr(host_pool, "can_use_write_back_jit", False): - if host_indices.is_cuda: - host_indices = host_indices.cpu() - return host_indices, device_indices - return self.move_indices(host_indices, device_indices) - - host_indices, device_indices = move_for_pool( - host_group.anchor_entry.host_pool, - op.host_indices, - op.device_indices, - ) - pool_transfers = [] - for transfer in op.pool_transfers or []: - entry = host_group.entry_map[transfer.name] - transfer_host_indices, transfer_device_indices = move_for_pool( - entry.host_pool, - transfer.host_indices, - transfer.device_indices, - ) - pool_transfers.append( - replace( - transfer, - host_indices=transfer_host_indices, - device_indices=transfer_device_indices, - ) - ) - return host_indices, device_indices, pool_transfers or None + return self.move_hybrid_indices(op, write_back_jit=True) def _l2_transfers( self, @@ -1075,35 +1205,53 @@ class HybridCacheController(BaseHiCacheController): kv_hit_pages * self.page_size, ) + def _move_pool_indices( + self, host_pool, host_indices, device_indices, *, write_back_jit: bool + ) -> tuple[torch.Tensor, torch.Tensor]: + if uses_shared_host_layout(host_pool): + return host_indices, device_indices + if write_back_jit and host_pool.can_use_write_back_jit: + if host_indices.is_cuda: + host_indices = host_indices.cpu() + return host_indices, device_indices + return self.move_indices(host_indices, device_indices) + def move_hybrid_indices( - self, operation: CacheOperation + self, operation: CacheOperation, *, write_back_jit: bool = False ) -> tuple[torch.Tensor, torch.Tensor, Optional[list[PoolTransfer]]]: - host_indices, device_indices = self.move_indices( - operation.host_indices, operation.device_indices + host_indices, device_indices = self._move_pool_indices( + self.mem_pool_host.anchor_entry.host_pool, + operation.host_indices, + operation.device_indices, + write_back_jit=write_back_jit, ) resolved_pool_transfers = None if operation.pool_transfers: resolved_pool_transfers = [] for transfer in operation.pool_transfers: - transfer_host_indices, transfer_device_indices = self.move_indices( - transfer.host_indices, transfer.device_indices + transfer_host_indices, transfer_device_indices = ( + self._move_pool_indices( + self.mem_pool_host.entry_map[transfer.name].host_pool, + transfer.host_indices, + transfer.device_indices, + write_back_jit=write_back_jit, + ) ) - # Keep the original PoolTransfer unchanged because tree-owned - # transfers may still reference radix-tree host state. The - # controller only needs a normalized execution-time copy. + # Tree-owned transfers keep their original radix-tree indices. resolved_pool_transfers.append( - PoolTransfer( - name=transfer.name, + replace( + transfer, host_indices=transfer_host_indices, device_indices=transfer_device_indices, - keys=transfer.keys, - hit_policy=transfer.hit_policy, - indices_from_pool=transfer.indices_from_pool, ) ) return host_indices, device_indices, resolved_pool_transfers def _page_transfer(self, operation: PrefetchOperation) -> bool: + with self.mem_pool_host.layout_lease(): + return self._page_transfer_with_stable_layout(operation) + + def _page_transfer_with_stable_layout(self, operation: PrefetchOperation) -> None: # KV pools and KV-derived pools first — determines actual completed page count kv_completed_pages = super()._page_transfer(operation) @@ -1172,6 +1320,10 @@ class HybridCacheController(BaseHiCacheController): operation.storage_start += trim_tokens def _page_backup(self, operation): + with self.mem_pool_host.layout_lease(): + return self._page_backup_with_stable_layout(operation) + + def _page_backup_with_stable_layout(self, operation): # MLA KV is replicated across TP ranks and should still be written only # by TP0. Rank-sharded sidecars still need every TP rank. backup_transfers = [ @@ -1401,7 +1553,8 @@ class HybridCacheController(BaseHiCacheController): for pool in derived_transfers: if pool.indices_from_pool == PoolName.KV: pool.host_indices = kv_host_indices - pool.device_indices = kv_device_indices + if pool.device_indices is None: + pool.device_indices = kv_device_indices continue source = next( @@ -1417,7 +1570,8 @@ class HybridCacheController(BaseHiCacheController): rollback_allocated() return None pool.host_indices = source.host_indices - pool.device_indices = source.device_indices + if pool.device_indices is None: + pool.device_indices = source.device_indices return extra_pools def _reduce_prefetch_ack(self, ack: PrefetchAck) -> None: diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 9cee653925..a9f26c6ac6 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -26,6 +26,7 @@ from sglang.srt.mem_cache.pool_host.mha import ( get_mha_host_pool_cls, ) from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.runtime_context import get_memory, get_parallel, get_serving @@ -279,6 +280,26 @@ def _swa_allocation_callbacks(allocator, bind=None, free_bound=None) -> dict: return dict(device_alloc_fn=allocator.alloc, device_free_fn=allocator.free) +def _uses_unified_page_envelope_host( + full_kv_pool: Any, + swa_kv_pool: Any, + use_mla: bool, + mtp_swa_device_pools: tuple[Any, ...] = (), +) -> bool: + from sglang.srt.mem_cache.unified_memory_pool import UnifiedMHATokenToKVPool + + return ( + not use_mla + and not mtp_swa_device_pools + and all( + isinstance(pool, UnifiedMHATokenToKVPool) + for pool in (full_kv_pool, swa_kv_pool) + ) + and {full_kv_pool.grow_direction, swa_kv_pool.grow_direction} == {"up", "down"} + and get_memory().hicache_host_memory_mode != "buffer_only" + ) + + def build_hybrid_swa_group( *, page_size: int, @@ -289,6 +310,7 @@ def build_hybrid_swa_group( use_mla: bool, kv_host_size: Optional[float] = None, swa_host_size: Optional[float] = None, + host_full_evict_fn: Optional[Callable[[int], Any]] = None, host_swa_evict_fn: Optional[Callable[[int], Any]] = None, device_swa_evict_fn: Optional[Callable[[int], Any]] = None, swa_attn_allocator: Any = None, @@ -300,21 +322,39 @@ def build_hybrid_swa_group( transfer_layer_id_max = ( max(full_layer_mapping.keys() | swa_layer_mapping.keys()) + 1 ) - kv_host_pool = build_kv_host_pool( - kv_pool=full_kv_pool, - page_size=page_size, - use_mla=use_mla, - host_size=kv_host_size, - pool_label="full", - ) - swa_host_pool = build_kv_host_pool( - kv_pool=swa_kv_pool, - page_size=page_size, - use_mla=use_mla, - host_size=swa_host_size, - mtp_draft_device_pools=mtp_swa_device_pools, - pool_label="swa", - ) + if _uses_unified_page_envelope_host( + full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools + ): + memory = get_memory() + kv_host_pool, swa_host_pool = ( + UnifiedPageEnvelopeHostPool.build_hybrid_swa_pool_pair( + device_pools=(full_kv_pool, swa_kv_pool), + host_to_device_ratio=memory.hicache_ratio, + host_size=memory.hicache_size, + page_size=page_size, + layout=memory.hicache_mem_layout, + allocator_type=_get_allocator_type(), + mtp_draft_device_pools=mtp_swa_device_pools, + ) + ) + has_shared_arena = True + else: + has_shared_arena = False + kv_host_pool = build_kv_host_pool( + kv_pool=full_kv_pool, + page_size=page_size, + use_mla=use_mla, + host_size=kv_host_size, + pool_label="full", + ) + swa_host_pool = build_kv_host_pool( + kv_pool=swa_kv_pool, + page_size=page_size, + use_mla=use_mla, + host_size=swa_host_size, + mtp_draft_device_pools=mtp_swa_device_pools, + pool_label="swa", + ) if mtp_swa_device_pools: swa_layer_mapping = _with_mtp_layer_mapping( swa_layer_mapping, @@ -331,6 +371,7 @@ def build_hybrid_swa_group( layer_mapping=full_layer_mapping, transfer_layer_id_max=transfer_layer_id_max, is_anchor=True, + host_evict_fn=host_full_evict_fn if has_shared_arena else None, ), build_pool_entry( name=PoolName.SWA, @@ -406,6 +447,7 @@ def build_hybrid_swa_stack( load_cache_event, storage_backend: Optional[str], use_mla: bool, + host_full_evict_fn: Optional[Callable[[int], Any]] = None, host_swa_evict_fn: Optional[Callable[[int], Any]] = None, device_swa_evict_fn: Optional[Callable[[int], Any]] = None, prefetch_threshold: int = 256, @@ -422,9 +464,12 @@ def build_hybrid_swa_stack( ) kv_host_size = swa_host_size = None - if get_memory().hicache_size > 0: + memory = get_memory() + if memory.hicache_size > 0 and not _uses_unified_page_envelope_host( + full_kv_pool, swa_kv_pool, use_mla, mtp_swa_device_pools + ): kv_host_size, swa_host_size = _split_hicache_size( - get_memory().hicache_size, (full_kv_pool, swa_kv_pool) + memory.hicache_size, (full_kv_pool, swa_kv_pool) ) host_pool_group = build_hybrid_swa_group( @@ -436,6 +481,7 @@ def build_hybrid_swa_stack( use_mla=use_mla, kv_host_size=kv_host_size, swa_host_size=swa_host_size, + host_full_evict_fn=host_full_evict_fn, host_swa_evict_fn=host_swa_evict_fn, device_swa_evict_fn=device_swa_evict_fn, # For SWA hybrid, device allocation goes through the inner allocator. @@ -1842,6 +1888,7 @@ class _SwaStrategy(StackStrategy): load_cache_event=load_cache_event, storage_backend=storage_backend, use_mla=False, + host_full_evict_fn=lambda n: cache.evict_host(n, ComponentType.FULL), host_swa_evict_fn=lambda n: cache.evict_host(n, ComponentType.SWA), device_swa_evict_fn=lambda n: _evict_swa_for_device_alloc(cache, n), prefetch_threshold=prefetch_threshold, diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py index 788f6bff15..ec75dd81de 100644 --- a/python/sglang/srt/mem_cache/kv_cache_builder.py +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py @@ -177,6 +177,20 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str: fields = {} backend = disagg.disaggregation_decode_retraction_backup + unified_hybrid_swa = memory.enable_unified_memory and tp_worker.is_hybrid_swa + unified_decode_radix = ( + memory.enable_unified_memory and disagg.disaggregation_decode_enable_radix_cache + ) + if backend == "host_pool" and unified_decode_radix: + raise ValueError( + "Unified-memory H2D/D2H does not support host-pool decode " + "retraction with decode radix cache yet." + ) + if backend == "host_pool" and unified_hybrid_swa: + raise ValueError( + "Unified-memory hybrid-SWA H2D/D2H does not support host-pool " + "decode retraction yet." + ) if backend is None: kv_cache = tp_worker.get_memory_pool()[1].get_kvcache() full_tokens_per_layer = ( @@ -184,10 +198,17 @@ def resolve_decode_retraction_backup(*, tp_worker: BaseTpWorker) -> str: if tp_worker.is_hybrid_swa else None ) - # Host-pool retraction does not address unified page envelopes or - # recurrent state, so those configurations stay on cpu_tensor. - supports_host_pool = ( + draft_pools = tp_worker.model_runner.mtp_draft_device_pools + unified_draft_host_pool_supported = ( not memory.enable_unified_memory + or not draft_pools + or tp_worker.model_runner.spec_algorithm.is_dspark() + ) + # Host-pool retraction has no recurrent-state sidecar, so a model with + # recurrent state stays on cpu_tensor. + supports_host_pool = ( + unified_draft_host_pool_supported + and not unified_hybrid_swa and not uses_ssm_state(tp_worker.model_runner.model_config) and ( isinstance(kv_cache, MHATokenToKVPool) diff --git a/python/sglang/srt/mem_cache/l2_transfer.py b/python/sglang/srt/mem_cache/l2_transfer.py index 0cd7e5e30f..7c0eae564c 100644 --- a/python/sglang/srt/mem_cache/l2_transfer.py +++ b/python/sglang/srt/mem_cache/l2_transfer.py @@ -1,11 +1,13 @@ from __future__ import annotations import logging +from contextlib import contextmanager from functools import cache from typing import Any, Callable, NamedTuple, Optional import torch +from sglang.srt.mem_cache.pool_host.base import shared_host_layout_domains from sglang.srt.utils import get_device_module logger = logging.getLogger(__name__) @@ -81,23 +83,63 @@ class L2TransferEngine: resolved = translate(indices) return resolved.to(original_device) - def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion: - start_event = self._start_event(None) + def _prepare_transfers(self, transfers: list[L2Transfer]) -> list[L2Transfer]: + prepared = [] + for transfer in transfers: + host_indices, device_indices = transfer.host_pool.prepare_transfer_indices( + transfer.host_indices, + self._resolve_device_indices(transfer), + self.io_backend, + ) + prepared.append( + transfer._replace( + host_indices=host_indices, + device_indices=device_indices, + ) + ) + return prepared + + @contextmanager + def _submission(self, transfers, stream, transfer_key, start_event=None): + start_event = self._start_event(start_event) ack_start, ack_finish, timing_enabled = make_timing_event_pair() - with device_module.stream(self.device_to_host_stream): - start_event.wait(self.device_to_host_stream) - device_indices = [self._resolve_device_indices(t) for t in transfers] - ack_start.record() - for transfer, dev_idx in zip(transfers, device_indices): - transfer.host_pool.backup_from_device_all_layer( + completion = TransferCompletion(ack_start, ack_finish, timing_enabled) + domains = shared_host_layout_domains(t.host_pool for t in transfers) + for domain in domains: + domain.acquire_layout() + finish_recorded = False + try: + with device_module.stream(stream): + # Index preparation can read producer-owned device indices. + start_event.wait(stream) + prepared = self._prepare_transfers(transfers) + ack_start.record() + yield prepared, completion + ack_finish.record() + finish_recorded = True + self._record_stream(transfers + prepared, stream) + except Exception: + stream.synchronize() + raise + finally: + for domain in reversed(domains): + domain.release_layout( + ack_finish if finish_recorded else None, + (id(self), transfer_key), + ) + + def submit_device_to_host(self, transfers: list[L2Transfer]) -> TransferCompletion: + with self._submission( + transfers, self.device_to_host_stream, "device_to_host" + ) as (transfers, completion): + for transfer in transfers: + transfer.host_pool.backup_from_device_all_layer_physical( transfer.device_pool, transfer.host_indices, - dev_idx, + transfer.device_indices, self.io_backend, ) - ack_finish.record() - self._record_stream(transfers, self.device_to_host_stream, device_indices) - return TransferCompletion(ack_start, ack_finish, timing_enabled) + return completion def submit_host_to_device( self, @@ -107,15 +149,12 @@ class L2TransferEngine: start_event=None, on_layer_done=None, ) -> TransferCompletion: - start_event = self._start_event(start_event) - ack_start, ack_finish, timing_enabled = make_timing_event_pair() - primary = transfers[0] if transfers else None - with device_module.stream(self.host_to_device_stream): - start_event.wait(self.host_to_device_stream) - device_indices = [self._resolve_device_indices(t) for t in transfers] - ack_start.record() + with self._submission( + transfers, self.host_to_device_stream, "host_to_device", start_event + ) as (transfers, completion): + primary = transfers[0] if transfers else None for layer_id in range(transfer_layer_id_max): - for transfer, dev_idx in zip(transfers, device_indices): + for transfer in transfers: local_layer_id = ( transfer.layer_mapper(layer_id) if transfer.layer_mapper is not None @@ -127,19 +166,17 @@ class L2TransferEngine: and layer_id >= transfer.host_pool.layer_num ): continue - transfer.host_pool.load_to_device_per_layer( + transfer.host_pool.load_to_device_per_layer_physical( transfer.device_pool, transfer.host_indices, - dev_idx, + transfer.device_indices, local_layer_id, self.io_backend, is_draft=transfer.is_draft, ) if on_layer_done is not None: on_layer_done(layer_id) - ack_finish.record() - self._record_stream(transfers, self.host_to_device_stream, device_indices) - return TransferCompletion(ack_start, ack_finish, timing_enabled) + return completion @staticmethod def _start_event(start_event): diff --git a/python/sglang/srt/mem_cache/memory_pool_host.py b/python/sglang/srt/mem_cache/memory_pool_host.py index d7ce8f7e2e..476712b364 100644 --- a/python/sglang/srt/mem_cache/memory_pool_host.py +++ b/python/sglang/srt/mem_cache/memory_pool_host.py @@ -63,6 +63,8 @@ class LogicalHostPool: compressed side pools use these logical FULL indices as stable page anchors. """ + shared_allocation_domain = None + def __init__(self, size: int, page_size: int, layout: str = "layer_first"): if size % page_size != 0: raise ValueError( @@ -150,6 +152,26 @@ class LogicalHostPool: ): pass + def prepare_transfer_indices(self, host_indices, device_indices, io_backend): + return host_indices, device_indices + + def backup_from_device_all_layer_physical( + self, device_pool, host_indices, device_indices, io_backend + ): + pass + + def load_to_device_per_layer_physical( + self, + device_pool, + host_indices, + device_indices, + layer_id, + io_backend, + *, + is_draft: bool = False, + ): + pass + def load_to_device_per_layer( self, device_pool, @@ -174,6 +196,9 @@ class LogicalHostPool: def get_page_buffer_meta(self, indices): return None + def get_page_buffer_element_size(self, split_factor: int = 1): + return None + def get_ksize_per_token(self): return 0 diff --git a/python/sglang/srt/mem_cache/pool_host/base.py b/python/sglang/srt/mem_cache/pool_host/base.py index c2d4c239e4..972b1cb6a1 100644 --- a/python/sglang/srt/mem_cache/pool_host/base.py +++ b/python/sglang/srt/mem_cache/pool_host/base.py @@ -3,10 +3,11 @@ from __future__ import annotations import abc import logging import threading +from collections.abc import Iterable from contextlib import contextmanager from contextvars import ContextVar from functools import wraps -from typing import Optional +from typing import Optional, TypeGuard import torch @@ -35,6 +36,26 @@ _host_memory_budget: ContextVar[Optional[int]] = ContextVar( ) +def uses_shared_host_layout(host_pool: object) -> TypeGuard[HostKVCache]: + return ( + isinstance(host_pool, HostKVCache) + and host_pool.shared_allocation_domain is not None + ) + + +def shared_host_layout_domains(host_pools: Iterable[object]) -> list: + domains = [] + seen = set() + for pool in host_pools: + if not uses_shared_host_layout(pool): + continue + domain = pool.shared_allocation_domain + if id(domain) not in seen: + seen.add(id(domain)) + domains.append(domain) + return domains + + @contextmanager def host_memory_budget_scope(budget_bytes: int): """Book every pool built inside against one snapshot, not re-sampled psutil.""" @@ -135,6 +156,7 @@ def synchronized(func): class HostKVCache(abc.ABC): dcp_size = 1 dcp_rank = 0 + shared_allocation_domain = None def __init__( self, @@ -323,6 +345,55 @@ class HostKVCache(abc.ABC): """ raise NotImplementedError() + def get_page_buffer_element_size(self, split_factor: int = 1) -> Optional[int]: + """Byte size of one storage element, or None for a logical anchor.""" + indices = torch.zeros(self.page_size, dtype=torch.int64) + meta = ( + self.get_split_heads_page_buffer_meta(indices, split_factor) + if split_factor != 1 + else self.get_page_buffer_meta(indices) + ) + sizes = meta[1] if meta else None + return int(sizes[0]) if sizes else None + + def prepare_transfer_indices( + self, host_indices, device_indices, io_backend + ) -> tuple[torch.Tensor, torch.Tensor]: + """Resolve indices that must remain stable through one L2 submission. + + Normal host pools expose physical indices directly and controllers have + already moved them to the backend-required device. Shared compacting + pools override this hook and resolve their logical ids while the transfer + engine holds a layout lease. + """ + return host_indices, device_indices + + def backup_from_device_all_layer_physical( + self, device_pool, host_indices, device_indices, io_backend + ) -> None: + self.backup_from_device_all_layer( + device_pool, host_indices, device_indices, io_backend + ) + + def load_to_device_per_layer_physical( + self, + device_pool, + host_indices, + device_indices, + layer_id, + io_backend, + *, + is_draft: bool = False, + ) -> None: + self.load_to_device_per_layer( + device_pool, + host_indices, + device_indices, + layer_id, + io_backend, + is_draft=is_draft, + ) + @abc.abstractmethod def get_data_page(self, index, flat: bool = True) -> torch.Tensor: """ diff --git a/python/sglang/srt/mem_cache/pool_host/group.py b/python/sglang/srt/mem_cache/pool_host/group.py index 8a56f92df3..4abe9b3e0b 100644 --- a/python/sglang/srt/mem_cache/pool_host/group.py +++ b/python/sglang/srt/mem_cache/pool_host/group.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from contextlib import contextmanager from dataclasses import dataclass from typing import Any @@ -8,6 +9,7 @@ import torch from sglang.srt.mem_cache.hicache_storage import PoolName, PoolTransfer from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool +from sglang.srt.mem_cache.pool_host.base import HostKVCache, shared_host_layout_domains @dataclass @@ -76,6 +78,33 @@ class HostPoolGroup: def get_pool(self, name: PoolName): return self.get_entry(name).host_pool + @contextmanager + def layout_lease(self): + domains = shared_host_layout_domains(entry.host_pool for entry in self.entries) + for domain in domains: + domain.acquire_layout() + try: + yield + finally: + for domain in reversed(domains): + domain.release_layout() + + def get_page_buffer_element_size(self, split_factor: int = 1): + anchor_pool = self.anchor_entry.host_pool + if ( + not isinstance(anchor_pool, HostKVCache) + or anchor_pool.shared_allocation_domain is None + ): + return anchor_pool.get_page_buffer_element_size(split_factor) + + element_sizes = set() + for entry in self.entries: + element_size = entry.host_pool.get_page_buffer_element_size(split_factor) + if element_size is None: + return None + element_sizes.add(element_size) + return element_sizes.pop() if len(element_sizes) == 1 else None + def get_contiguous_buf_infos(self): """Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes).""" host_by_device_ptr = {} @@ -199,7 +228,8 @@ class HostPoolGroup: for transfer in derived_transfers: if transfer.indices_from_pool == self.anchor_entry.name: transfer.host_indices = primary_host_indices - transfer.device_indices = primary_device_indices + if transfer.device_indices is None: + transfer.device_indices = primary_device_indices continue source = next( @@ -215,7 +245,8 @@ class HostPoolGroup: rollback() return None transfer.host_indices = source.host_indices - transfer.device_indices = source.device_indices + if transfer.device_indices is None: + transfer.device_indices = source.device_indices return transfers def release_transfers(self, transfers: list[PoolTransfer] | None) -> int: diff --git a/python/sglang/srt/mem_cache/pool_host/unified.py b/python/sglang/srt/mem_cache/pool_host/unified.py new file mode 100644 index 0000000000..f3aa017274 --- /dev/null +++ b/python/sglang/srt/mem_cache/pool_host/unified.py @@ -0,0 +1,1040 @@ +from __future__ import annotations + +import threading +from collections.abc import Sequence +from contextlib import contextmanager + +import torch + +from sglang.srt.mem_cache.pool_host.base import HostKVCache, host_memory_budget_bytes +from sglang.srt.mem_cache.pool_host.common import ( + ALLOC_MEMORY_FUNCS, + _cuda_host_unregister, + get_allocator_from_storage, +) +from sglang.srt.utils import is_cuda, is_hip + +_is_cuda = is_cuda() +_is_hip = is_hip() +if _is_cuda or _is_hip: + from sgl_kernel.kvcacheio import transfer_kv_direct, transfer_kv_per_layer_mla + + +class _SharedPageEnvelopeHostBacking: + class _LayoutLeaseState(threading.local): + def __init__(self): + self.depth = 0 + + class _Side: + def __init__( + self, + *, + page_size: int, + page_bytes: int, + page_num: int, + grow_direction: str, + ): + if grow_direction not in {"up", "down"}: + raise ValueError( + f"Unsupported unified host-pool grow direction: {grow_direction}" + ) + self.page_size = page_size + self.page_bytes = page_bytes + self.page_num = page_num + self.grow_direction = grow_direction + + # The cache tree owns logical page ids. Physical page ids are private + # to the backing and may change whenever the two sides are compacted. + self.free_logical_extents = [(0, page_num)] + self.logical_to_physical: dict[int, int] = {} + self.physical_to_logical: dict[int, int] = {} + self.free_physical_pages: set[int] = set() + self.watermark = 0 if grow_direction == "up" else page_num - 1 + + @property + def live_page_count(self) -> int: + return len(self.logical_to_physical) + + def __init__( + self, + *, + device_pools: Sequence, + pool_names: Sequence[str], + host_to_device_ratio: float, + host_size: float, + pin_memory: bool, + device: str, + allocator_type: str, + ): + if len(device_pools) != len(pool_names) or not device_pools: + raise ValueError("Unified host pool devices and names must be non-empty.") + if len(device_pools) > 2: + raise ValueError("Unified host backing supports at most two pools.") + if len(set(pool_names)) != len(pool_names): + raise ValueError(f"Unified host pool names must be unique: {pool_names}.") + + device_buffers = [pool.get_page_envelope_buffer() for pool in device_pools] + registrations = [pool.get_contiguous_buf_infos() for pool in device_pools] + for ptrs, lens, item_lens in registrations: + if len(ptrs) != len(lens) or len(ptrs) != len(item_lens) or len(ptrs) != 1: + raise ValueError( + "Unified page-envelope pools must expose one memory region." + ) + shared_regions = {(ptrs[0], lens[0]) for ptrs, lens, _ in registrations} + if len(shared_regions) != 1: + raise ValueError("Unified full and SWA pools must share one device region.") + + page_bytes = [int(buffer.shape[1]) for buffer in device_buffers] + if host_size > 0: + token_capacities = [ + int(host_size * 1e9 * pool.page_size // item_bytes) + for pool, item_bytes in zip(device_pools, page_bytes, strict=True) + ] + else: + token_capacities = [ + int( + ( + pool.host_capacity_tokens + if pool.host_capacity_tokens is not None + else pool.size + ) + * host_to_device_ratio + ) + for pool in device_pools + ] + + nominal_view_bytes = [ + (capacity // pool.page_size + 1) * item_bytes + for pool, item_bytes, capacity in zip( + device_pools, page_bytes, token_capacities, strict=True + ) + ] + + if host_size == 0 and host_to_device_ratio == 0: + total_bytes = host_memory_budget_bytes() + elif host_size > 0: + total_bytes = max(nominal_view_bytes) + else: + device_total_bytes = next(iter(shared_regions))[1] + total_bytes = max( + int(device_total_bytes * host_to_device_ratio) + max(page_bytes), + *nominal_view_bytes, + ) + if total_bytes <= 0: + raise ValueError( + f"Unified host backing size must be positive: {total_bytes}" + ) + + available_bytes = host_memory_budget_bytes(total_bytes) + if total_bytes > available_bytes: + raise ValueError( + "Not enough host memory for unified page-envelope backing. " + f"Requesting {total_bytes / 1e9:.2f} GB but only have " + f"{available_bytes / 1e9:.2f} GB free." + ) + + grow_directions = [pool.grow_direction for pool in device_pools] + if len(grow_directions) == 2 and set(grow_directions) != {"up", "down"}: + raise ValueError( + "Unified full and SWA host pools must grow from opposite ends; " + f"got {grow_directions}." + ) + self.sides = {} + for pool, name, item_bytes, grow_direction in zip( + device_pools, + pool_names, + page_bytes, + grow_directions, + strict=True, + ): + self.sides[name] = self._Side( + page_size=pool.page_size, + page_bytes=item_bytes, + page_num=total_bytes // item_bytes, + grow_direction=grow_direction, + ) + if any(side.page_num == 0 for side in self.sides.values()): + raise ValueError( + "Unified host backing must fit at least one page from every pool." + ) + + down_sides = [ + side for side in self.sides.values() if side.grow_direction == "down" + ] + self.capacity_bytes = total_bytes + if down_sides: + # Match L1's page-indexed view: bytes after the grow-down side's final + # complete page are tail padding, not shared allocatable capacity. + self._allocatable_bytes = down_sides[0].page_num * down_sides[0].page_bytes + else: + only_side = next(iter(self.sides.values())) + self._allocatable_bytes = only_side.page_num * only_side.page_bytes + + self.pin_memory = pin_memory + self.allocator = get_allocator_from_storage(allocator_type) + self.lock = threading.RLock() + alloc_func = ALLOC_MEMORY_FUNCS[device_buffers[0].device] + self.raw = alloc_func( + (total_bytes,), + dtype=torch.uint8, + device=device, + pin_memory=pin_memory, + allocator=self.allocator, + ) + self.fd = getattr(self.allocator, "fd", None) + self.registration_owner = pool_names[0] + self._users: set[str] = set() + self._destroyed = False + + # A layout lease pins logical->physical mappings. CUDA transfers hand + # the lease off to their finish event; synchronous L3 I/O holds it until + # the backend returns. Compaction blocks new leases and drains both. + self._layout_condition = threading.Condition(self.lock) + self._layout_users = 0 + self._layout_lease_state = self._LayoutLeaseState() + self._compacting = False + self._pending_transfer_events = {} + + def retain(self, name: str) -> None: + if name in self._users: + raise RuntimeError(f"Unified host pool {name!r} is already attached.") + self._users.add(name) + + def release(self, name: str) -> None: + with self.lock: + self._users.discard(name) + self.destroy_if_unused() + + def destroy_if_unused(self) -> None: + with self.lock: + if self._users or self._destroyed: + return + self._destroyed = True + events = self._begin_compaction() + try: + for event in events: + event.synchronize() + if self.pin_memory and (_is_cuda or _is_hip): + _cuda_host_unregister(self.raw) + self.raw = None + finally: + self._finish_compaction() + + def page_buffer(self, name: str) -> torch.Tensor: + side = self.sides[name] + return self.raw[: side.page_num * side.page_bytes].view( + side.page_num, side.page_bytes + ) + + @staticmethod + def _reserve_logical_pages( + extents: list[tuple[int, int]], page_count: int + ) -> list[int] | None: + if page_count == 0: + return [] + remaining = page_count + pages = [] + new_extents = [] + for start, end in extents: + capacity = end - start + take = min(remaining, capacity) + pages.extend(range(start, start + take)) + if start + take < end: + new_extents.append((start + take, end)) + remaining -= take + if remaining == 0: + extents[:] = new_extents + return pages + + return None + + def _request_page_counts( + self, requests: Sequence[tuple[str, int]] + ) -> tuple[list[int], dict[str, int]]: + request_page_counts = [] + page_counts = {name: 0 for name in self.sides} + for name, need_size in requests: + if name not in self.sides: + raise ValueError(f"Unknown unified host pool {name!r}.") + side = self.sides[name] + if need_size % side.page_size != 0: + raise AssertionError( + "The requested size should be a multiple of the page size." + ) + page_count = need_size // side.page_size + request_page_counts.append(page_count) + page_counts[name] += page_count + return request_page_counts, page_counts + + @staticmethod + def _merge_extents(extents: list[tuple[int, int]]) -> list[tuple[int, int]]: + merged = [] + for start, end in sorted(extents): + if merged and start <= merged[-1][1]: + merged[-1] = (merged[-1][0], max(merged[-1][1], end)) + else: + merged.append((start, end)) + return merged + + @staticmethod + def _logical_free_page_count(side: _Side) -> int: + return sum(end - start for start, end in side.free_logical_extents) + + def _current_gap_bytes(self) -> int: + up_sides = [side for side in self.sides.values() if side.grow_direction == "up"] + down_sides = [ + side for side in self.sides.values() if side.grow_direction == "down" + ] + low = up_sides[0].watermark * up_sides[0].page_bytes if up_sides else 0 + high = ( + (down_sides[0].watermark + 1) * down_sides[0].page_bytes + if down_sides + else self._allocatable_bytes + ) + return max(0, high - low) + + def _can_reserve_without_compaction(self, page_counts: dict[str, int]) -> bool: + extension_bytes = 0 + for name, page_count in page_counts.items(): + side = self.sides[name] + if page_count > self._logical_free_page_count(side): + return False + extension_pages = max(0, page_count - len(side.free_physical_pages)) + extension_bytes += extension_pages * side.page_bytes + return extension_bytes <= self._current_gap_bytes() + + def _can_fit_packed(self, page_counts: dict[str, int]) -> bool: + used_bytes = 0 + for name, page_count in page_counts.items(): + side = self.sides[name] + if page_count > self._logical_free_page_count(side): + return False + used_bytes += (side.live_page_count + page_count) * side.page_bytes + return used_bytes <= self._allocatable_bytes + + def _begin_compaction(self) -> list: + if self._layout_lease_state.depth: + raise RuntimeError( + "Cannot compact a unified host pool while the current thread " + "holds a layout lease." + ) + self._compacting = True + while self._layout_users: + self._layout_condition.wait() + events = list(self._pending_transfer_events.values()) + self._pending_transfer_events = {} + return events + + def _finish_compaction(self) -> None: + self._compacting = False + self._layout_condition.notify_all() + + def acquire_layout(self) -> None: + with self._layout_condition: + if self._layout_lease_state.depth: + self._layout_lease_state.depth += 1 + return + self._prune_completed_transfer_events_locked() + while self._compacting: + self._layout_condition.wait() + self._layout_users += 1 + self._layout_lease_state.depth = 1 + + def _prune_completed_transfer_events_locked(self) -> None: + completed = [] + for key, event in self._pending_transfer_events.items(): + try: + if event.query(): + completed.append(key) + except Exception: + # A failed query is not proof of completion. Keep the event so + # compaction will synchronize it before moving its pages. + continue + for key in completed: + del self._pending_transfer_events[key] + + def release_layout(self, finish_event=None, transfer_key=None) -> None: + with self._layout_condition: + if self._layout_lease_state.depth <= 0: + raise RuntimeError( + "Unified host layout lease released without acquire." + ) + self._prune_completed_transfer_events_locked() + if finish_event is not None: + if transfer_key is None: + transfer_key = id(finish_event) + self._pending_transfer_events[transfer_key] = finish_event + self._layout_lease_state.depth -= 1 + if self._layout_lease_state.depth: + return + self._layout_users -= 1 + if self._layout_users == 0: + self._layout_condition.notify_all() + + @contextmanager + def layout_lease(self): + self.acquire_layout() + try: + yield + finally: + self.release_layout() + + def _compact_side(self, side: _Side) -> None: + reverse = side.grow_direction == "down" + live = sorted(side.physical_to_logical.items(), reverse=reverse) + if reverse: + targets = range(side.page_num - 1, side.page_num - 1 - len(live), -1) + else: + targets = range(len(live)) + + new_logical_to_physical = {} + new_physical_to_logical = {} + for (source, logical), target in zip(live, targets, strict=True): + if source != target: + source_offset = source * side.page_bytes + target_offset = target * side.page_bytes + self.raw[target_offset : target_offset + side.page_bytes].copy_( + self.raw[source_offset : source_offset + side.page_bytes].clone() + ) + new_logical_to_physical[logical] = target + new_physical_to_logical[target] = logical + + side.logical_to_physical = new_logical_to_physical + side.physical_to_logical = new_physical_to_logical + side.free_physical_pages.clear() + side.watermark = ( + len(live) if side.grow_direction == "up" else side.page_num - 1 - len(live) + ) + + def _compact(self) -> None: + events = self._begin_compaction() + try: + for event in events: + event.synchronize() + for side in self.sides.values(): + self._compact_side(side) + finally: + self._finish_compaction() + + def _take_physical_pages(self, side: _Side, page_count: int) -> list[int]: + reverse = side.grow_direction == "down" + holes = sorted(side.free_physical_pages, reverse=reverse) + pages = holes[:page_count] + side.free_physical_pages.difference_update(pages) + remaining = page_count - len(pages) + if remaining == 0: + return pages + + if side.grow_direction == "up": + pages.extend(range(side.watermark, side.watermark + remaining)) + side.watermark += remaining + else: + pages.extend(range(side.watermark, side.watermark - remaining, -1)) + side.watermark -= remaining + return pages + + @staticmethod + def _release_center_holes(side: _Side) -> None: + if side.grow_direction == "up": + while side.watermark > 0 and side.watermark - 1 in side.free_physical_pages: + side.watermark -= 1 + side.free_physical_pages.remove(side.watermark) + else: + while ( + side.watermark + 1 < side.page_num + and side.watermark + 1 in side.free_physical_pages + ): + side.watermark += 1 + side.free_physical_pages.remove(side.watermark) + + @staticmethod + def _page_ids(indices: torch.Tensor, page_size: int) -> list[int]: + indices = indices.to(dtype=torch.int64, device="cpu").flatten() + if indices.numel() % page_size != 0: + raise AssertionError("Host indices must contain whole pages.") + groups = indices.view(-1, page_size) + offsets = torch.arange(page_size, dtype=torch.int64) + if groups.numel() and ( + bool(torch.any(groups[:, 0] % page_size != 0)) + or not torch.equal(groups, groups[:, :1] + offsets) + ): + raise AssertionError("Host indices must contain contiguous pages.") + return (groups[:, 0] // page_size).tolist() + + def available_size(self, name: str) -> int: + with self.lock: + side = self.sides[name] + page_count = min( + self._logical_free_page_count(side), + self.free_bytes() // side.page_bytes, + ) + return page_count * side.page_size + + def free_bytes(self) -> int: + with self.lock: + used_bytes = sum( + side.live_page_count * side.page_bytes for side in self.sides.values() + ) + return self._allocatable_bytes - used_bytes + + @staticmethod + def _expand_pages(pages: Sequence[int], page_size: int) -> torch.Tensor: + if not pages: + return torch.empty(0, dtype=torch.int64) + page_ids = torch.tensor(pages, dtype=torch.int64) + offsets = torch.arange(page_size, dtype=torch.int64) + return (page_ids[:, None] * page_size + offsets).reshape(-1) + + def alloc(self, name: str, need_size: int) -> torch.Tensor | None: + result = self.alloc_many(((name, need_size),)) + return None if result is None else result[0] + + def alloc_many( + self, requests: Sequence[tuple[str, int]] + ) -> list[torch.Tensor] | None: + with self.lock: + request_page_counts, page_counts = self._request_page_counts(requests) + if not self._can_reserve_without_compaction(page_counts): + if not self._can_fit_packed(page_counts): + return None + self._compact() + if not self._can_reserve_without_compaction(page_counts): + raise RuntimeError( + "Unified host-pool compaction did not create the planned capacity." + ) + + logical_extents = { + name: list(side.free_logical_extents) + for name, side in self.sides.items() + } + logical_pages = [] + for (name, _), page_count in zip( + requests, request_page_counts, strict=True + ): + pages = self._reserve_logical_pages(logical_extents[name], page_count) + if pages is None: + return None + logical_pages.append(pages) + + physical_pages_by_name = { + name: self._take_physical_pages(self.sides[name], page_count) + for name, page_count in page_counts.items() + } + physical_offsets = {name: 0 for name in self.sides} + results = [] + for (name, _), request_logical_pages in zip( + requests, logical_pages, strict=True + ): + side = self.sides[name] + start = physical_offsets[name] + end = start + len(request_logical_pages) + request_physical_pages = physical_pages_by_name[name][start:end] + physical_offsets[name] = end + for logical_page, physical_page in zip( + request_logical_pages, request_physical_pages, strict=True + ): + side.logical_to_physical[logical_page] = physical_page + side.physical_to_logical[physical_page] = logical_page + results.append( + self._expand_pages(request_logical_pages, side.page_size) + ) + + for name, extents in logical_extents.items(): + self.sides[name].free_logical_extents = extents + return results + + def free(self, name: str, indices: torch.Tensor) -> int: + with self.lock: + side = self.sides[name] + logical_pages = self._page_ids(indices, side.page_size) + if len(logical_pages) != len(set(logical_pages)): + raise AssertionError("Host pages cannot be freed more than once.") + if any(page not in side.logical_to_physical for page in logical_pages): + raise AssertionError(f"Host pages are not allocated in {name!r}.") + for logical_page in logical_pages: + physical_page = side.logical_to_physical.pop(logical_page) + del side.physical_to_logical[physical_page] + side.free_physical_pages.add(physical_page) + side.free_logical_extents = self._merge_extents( + [ + *side.free_logical_extents, + *((page, page + 1) for page in logical_pages), + ] + ) + self._release_center_holes(side) + return len(indices) + + def clear(self, name: str) -> None: + with self.lock: + events = self._begin_compaction() + try: + for event in events: + event.synchronize() + side = self.sides[name] + side.free_logical_extents = [(0, side.page_num)] + side.logical_to_physical.clear() + side.physical_to_logical.clear() + side.free_physical_pages.clear() + side.watermark = 0 if side.grow_direction == "up" else side.page_num - 1 + finally: + self._finish_compaction() + + def translate_indices(self, name: str, indices: torch.Tensor) -> torch.Tensor: + with self.lock: + side = self.sides[name] + logical_pages = self._page_ids(indices, side.page_size) + try: + physical_pages = [ + side.logical_to_physical[page] for page in logical_pages + ] + except KeyError as error: + raise AssertionError( + f"Host page {error.args[0]} is not allocated in {name!r}." + ) from error + translated = self._expand_pages(physical_pages, side.page_size) + return translated.to(device=indices.device) + + def translate_index(self, name: str, index: int) -> int: + with self.lock: + side = self.sides[name] + logical_page, offset = divmod(index, side.page_size) + try: + physical_page = side.logical_to_physical[logical_page] + except KeyError as error: + raise AssertionError( + f"Host page {logical_page} is not allocated in {name!r}." + ) from error + return physical_page * side.page_size + offset + + +class UnifiedPageEnvelopeHostPool(HostKVCache): + """Host mirror that transfers complete unified-memory page envelopes.""" + + def __init__( + self, + device_pool, + host_to_device_ratio: float, + host_size: int, + page_size: int, + layout: str, + pin_memory: bool = True, + device: str = "cpu", + allocator_type: str = "default", + *, + mtp_draft_device_pools: Sequence = (), + pool_label: str = "kv", + _shared_backing: _SharedPageEnvelopeHostBacking | None = None, + ): + if mtp_draft_device_pools: + raise NotImplementedError( + "Unified page-envelope host pools do not support packed MTP draft pools." + ) + + device_page_buffer = device_pool.get_page_envelope_buffer() + if device_page_buffer.ndim != 2: + raise ValueError( + "Unified page-envelope buffers must have shape [pages, bytes]; " + f"got {tuple(device_page_buffer.shape)}." + ) + page_bytes = int(device_page_buffer.shape[1]) + if page_bytes % page_size != 0: + raise ValueError( + "Unified page-envelope bytes must be divisible by page_size; " + f"got page_bytes={page_bytes}, page_size={page_size}." + ) + if device_pool.page_size != page_size: + raise ValueError( + "Unified device and host page sizes must match; " + f"got device={device_pool.page_size}, host={page_size}." + ) + if layout not in {"layer_first", "page_first", "page_first_direct"}: + raise ValueError(f"Unsupported layout: {layout}") + + if _shared_backing is None: + _shared_backing = _SharedPageEnvelopeHostBacking( + device_pools=(device_pool,), + pool_names=(pool_label,), + host_to_device_ratio=host_to_device_ratio, + host_size=host_size, + pin_memory=pin_memory, + device=device, + allocator_type=allocator_type, + ) + + self.device_pool = device_pool + self.pool_label = pool_label + self._shared_backing = _shared_backing + self._device_page_buffer = device_page_buffer + self._host_page_buffer_view = _shared_backing.page_buffer(pool_label) + self.size_per_token = page_bytes // page_size + self.item_bytes = page_bytes + self.page_size = page_size + self.page_num = _shared_backing.sides[pool_label].page_num + self.device_capacity_tokens = ( + device_pool.host_capacity_tokens + if device_pool.host_capacity_tokens is not None + else device_pool.size + ) + self.size = self.page_num * page_size + self.dcp_size = 1 + self.dcp_rank = 0 + self.layout = layout + self.pin_memory = pin_memory + self.device = device + self.dtype = torch.uint8 + self.allocator = _shared_backing.allocator + self.layer_num = 1 + self.can_use_write_back_jit = False + self.start_layer = device_pool.start_layer + self.end_layer = device_pool.end_layer + self.kv_buffer = ( + _shared_backing.raw + if pool_label == _shared_backing.registration_owner + else None + ) + self.fd = _shared_backing.fd + self.lock = _shared_backing.lock + self._destroyed = False + _shared_backing.retain(pool_label) + + @classmethod + def build_hybrid_swa_pool_pair( + cls, + *, + device_pools: Sequence, + host_to_device_ratio: float, + host_size: float, + page_size: int, + layout: str, + allocator_type: str, + mtp_draft_device_pools: Sequence = (), + pin_memory: bool = True, + device: str = "cpu", + ) -> tuple[HostKVCache, HostKVCache]: + if mtp_draft_device_pools: + raise NotImplementedError( + "Unified page-envelope host pools do not support packed MTP draft pools." + ) + if len(device_pools) != 2: + raise ValueError("A hybrid-SWA host pool pair requires exactly two pools.") + + pool_names = ("full", "swa") + backing = _SharedPageEnvelopeHostBacking( + device_pools=device_pools, + pool_names=pool_names, + host_to_device_ratio=host_to_device_ratio, + host_size=host_size, + pin_memory=pin_memory, + device=device, + allocator_type=allocator_type, + ) + host_pools = [] + try: + for device_pool, pool_name in zip(device_pools, pool_names, strict=True): + host_pools.append( + cls( + device_pool, + host_to_device_ratio, + 0, + page_size, + layout, + pin_memory=pin_memory, + device=device, + allocator_type=allocator_type, + pool_label=pool_name, + _shared_backing=backing, + ) + ) + except Exception: + for host_pool in host_pools: + host_pool.destroy() + backing.destroy_if_unused() + raise + return tuple(host_pools) + + def get_size_per_token(self): + return self.size_per_token + + def get_ksize_per_token(self): + return self.size_per_token + + @property + def shared_allocation_domain(self): + return self._shared_backing + + def init_kv_buffer(self): + return self._host_page_buffer_view + + def get_hybrid_pool_buffer(self): + if self.pool_label == self._shared_backing.registration_owner: + return [self._shared_backing.raw] + return [] + + def clear(self) -> None: + self._shared_backing.clear(self.pool_label) + + def available_size(self): + return self._shared_backing.available_size(self.pool_label) + + def alloc(self, need_size: int) -> torch.Tensor | None: + return self._shared_backing.alloc(self.pool_label, need_size) + + def free(self, indices: torch.Tensor) -> int: + return self._shared_backing.free(self.pool_label, indices) + + def prepare_transfer_indices( + self, host_indices, device_indices, io_backend + ) -> tuple[torch.Tensor, torch.Tensor]: + host_indices = self._shared_backing.translate_indices( + self.pool_label, host_indices + ) + # The L2 engine resolves device IDs using the per-layer HiCache contract. + # This pool copies whole envelopes, so discard the kernel-view page stride. + kernel_page_size = self.page_size * self.device_pool.kernel_page_blocks + device_indices = ( + device_indices // kernel_page_size * self.page_size + + device_indices % self.page_size + ) + if io_backend == "kernel": + if not host_indices.is_cuda: + host_indices = host_indices.to( + self._device_page_buffer.device, non_blocking=True + ) + return host_indices, device_indices + if io_backend == "direct": + if self.layout == "layer_first": + device_indices = device_indices.cpu() + host_indices, order = host_indices.sort() + return host_indices, device_indices.index_select(0, order) + if self.layout == "page_first_direct": + return host_indices, device_indices.cpu() + raise ValueError( + f"Unsupported layout {self.layout!r} for io backend 'direct'" + ) + if io_backend == "kernel_ascend": + return host_indices, device_indices.cpu() + raise ValueError(f"Unsupported io backend: {io_backend}") + + def destroy(self) -> None: + if self._destroyed: + return + self._destroyed = True + self.kv_buffer = None + self._host_page_buffer_view = None + self._shared_backing.release(self.pool_label) + + def _to_page_indices(self, indices: torch.Tensor) -> torch.Tensor: + indices = indices.to(dtype=torch.int64) + if indices.numel() == 0: + return indices + if indices.numel() % self.page_size != 0: + raise ValueError( + "Unified page-envelope transfer index count must be page-aligned; " + f"got {indices.numel()} indices for page_size={self.page_size}." + ) + grouped = indices.reshape(-1, self.page_size) + # Device indices are produced by the page allocator. Validating every + # value here would turn both reductions into host-device synchronizations + # on every offload. Keep the full check for CPU/debug inputs while the + # hot CUDA/ROCm path relies on that allocator invariant. + if indices.device.type == "cpu": + offsets = torch.arange(self.page_size, dtype=torch.int64) + if bool(torch.any(grouped[:, 0] % self.page_size != 0)) or not torch.equal( + grouped, grouped[:, :1] + offsets + ): + raise ValueError("Unified page-envelope transfers require whole pages.") + return grouped[:, 0] // self.page_size + + def _has_transfer_indices( + self, host_indices: torch.Tensor | None, device_indices: torch.Tensor | None + ) -> bool: + if host_indices is None or device_indices is None: + return False + if host_indices.numel() != device_indices.numel(): + raise ValueError( + "Unified page-envelope transfer index size mismatch: " + f"host={host_indices.numel()}, device={device_indices.numel()}." + ) + return host_indices.numel() > 0 + + def _transfer_pages( + self, + src: torch.Tensor, + dst: torch.Tensor, + src_pages: torch.Tensor, + dst_pages: torch.Tensor, + io_backend: str, + ) -> None: + if io_backend not in {"kernel", "direct"}: + raise ValueError(f"Unsupported io_backend: {io_backend}") + if self._device_page_buffer.device.type == "cpu": + dst.index_copy_(0, dst_pages, src.index_select(0, src_pages)) + return + + if not (_is_cuda or _is_hip): + raise RuntimeError( + "Unified page-envelope transfers require a CUDA or ROCm device." + ) + if io_backend == "kernel": + transfer_kv_per_layer_mla( + src=src, + dst=dst, + src_indices=src_pages, + dst_indices=dst_pages, + item_size=self.item_bytes, + ) + else: + transfer_kv_direct( + src_layers=[src], + dst_layers=[dst], + src_indices=src_pages, + dst_indices=dst_pages, + page_size=1, + ) + + @contextmanager + def _direct_transfer(self, host_indices, device_indices, io_backend): + self._shared_backing.acquire_layout() + finish_event = None + transfer_key = None + try: + yield self.prepare_transfer_indices( + host_indices, device_indices, io_backend + ) + if self._device_page_buffer.device.type != "cpu": + finish_event = torch.cuda.Event() + finish_event.record() + transfer_key = ( + "direct", + int(torch.cuda.current_stream().cuda_stream), + ) + except Exception: + if self._device_page_buffer.device.type != "cpu": + torch.cuda.current_stream().synchronize() + raise + finally: + self._shared_backing.release_layout(finish_event, transfer_key) + + def backup_from_device_all_layer( + self, device_pool, host_indices, device_indices, io_backend + ): + if not self._has_transfer_indices(host_indices, device_indices): + return + with self._direct_transfer(host_indices, device_indices, io_backend) as ( + host_indices, + device_indices, + ): + self.backup_from_device_all_layer_physical( + device_pool, host_indices, device_indices, io_backend + ) + + def backup_from_device_all_layer_physical( + self, device_pool, host_indices, device_indices, io_backend + ): + if not self._has_transfer_indices(host_indices, device_indices): + return + self._transfer_pages( + self._device_page_buffer, + self._host_page_buffer_view, + self._to_page_indices(device_indices), + self._to_page_indices(host_indices), + io_backend, + ) + + def load_to_device_per_layer( + self, + device_pool, + host_indices, + device_indices, + layer_id, + io_backend, + *, + is_draft: bool = False, + ): + if is_draft: + raise NotImplementedError( + "Unified page-envelope host pools do not contain draft KV." + ) + if layer_id != 0 or not self._has_transfer_indices( + host_indices, device_indices + ): + return + with self._direct_transfer(host_indices, device_indices, io_backend) as ( + host_indices, + device_indices, + ): + self.load_to_device_per_layer_physical( + device_pool, + host_indices, + device_indices, + layer_id, + io_backend, + is_draft=is_draft, + ) + + def load_to_device_per_layer_physical( + self, + device_pool, + host_indices, + device_indices, + layer_id, + io_backend, + *, + is_draft: bool = False, + ): + if is_draft: + raise NotImplementedError( + "Unified page-envelope host pools do not contain draft KV." + ) + if layer_id != 0 or not self._has_transfer_indices( + host_indices, device_indices + ): + return + self._transfer_pages( + self._host_page_buffer_view, + self._device_page_buffer, + self._to_page_indices(host_indices), + self._to_page_indices(device_indices), + io_backend, + ) + + def get_data_page(self, index, flat: bool = True) -> torch.Tensor: + with self._shared_backing.layout_lease(): + physical = self._shared_backing.translate_index(self.pool_label, int(index)) + page = self._host_page_buffer_view[physical // self.page_size] + return page.flatten() if flat else page.view(1, self.item_bytes) + + def get_dummy_flat_data_page(self) -> torch.Tensor: + return torch.zeros( + self.item_bytes, + dtype=self.dtype, + device=self.device, + pin_memory=self.pin_memory, + ) + + def get_page_buffer_element_size(self, split_factor: int = 1) -> int: + if split_factor != 1: + raise ValueError( + "Unified page-envelope host pools do not support split-head metadata." + ) + return self.item_bytes + + def set_from_flat_data_page(self, index: int, data_page: torch.Tensor) -> None: + with self._shared_backing.layout_lease(): + physical = self._shared_backing.translate_index(self.pool_label, int(index)) + page = data_page.view(self.dtype).reshape(self.item_bytes) + self._host_page_buffer_view[physical // self.page_size].copy_(page) + + def get_page_buffer_meta(self, indices): + physical = self._shared_backing.translate_indices(self.pool_label, indices) + pages = self._to_page_indices(physical).tolist() + base_ptr = self._host_page_buffer_view.data_ptr() + ptrs = [base_ptr + int(page) * self.item_bytes for page in pages] + return ptrs, [self.item_bytes] * len(ptrs) + + def is_stride_page_aligned(self, page_size_bytes: int = 4096) -> bool: + buffer = self._host_page_buffer_view + return ( + buffer.data_ptr() % page_size_bytes == 0 + and self.item_bytes % page_size_bytes == 0 + ) diff --git a/python/sglang/srt/mem_cache/storage/umbp/umbp_store.py b/python/sglang/srt/mem_cache/storage/umbp/umbp_store.py index 10ec5105ec..019fad9585 100644 --- a/python/sglang/srt/mem_cache/storage/umbp/umbp_store.py +++ b/python/sglang/srt/mem_cache/storage/umbp/umbp_store.py @@ -648,25 +648,16 @@ class UMBPStore(HiCacheStorage): if "dram_page_size" in extra: page_byte_size = int(extra["dram_page_size"]) elif mem_pool_host is not None: - # Probe element_size from the same buffer-meta helper that - # batch_preprocess will actually use; this matches per-call - # Put/Get size byte-for-byte for MHA / MHA-split / MLA / NSA - # without per-case formulas (NSA in particular: get_ksize_per_token - # would over-count by the indexer buffer that is never put to UMBP). - dummy = torch.zeros(mem_pool_host.page_size, dtype=torch.int64) - if self.is_mla_backend: - meta = mem_pool_host.get_page_buffer_meta(dummy) - elif storage_config is not None and getattr( - storage_config, "should_split_heads", False + split_factor = 1 + if ( + not self.is_mla_backend + and storage_config is not None + and storage_config.should_split_heads ): - sf = storage_config.tp_lcm_size // storage_config.tp_size - meta = mem_pool_host.get_split_heads_page_buffer_meta(dummy, sf) - else: - meta = mem_pool_host.get_page_buffer_meta(dummy) - # A hybrid logical anchor returns None here by design; leave - # dram_page_size at 0 and let the per-pool v2 sizes handle it. - esz = meta[1] if meta else None - page_byte_size = int(esz[0]) if esz else 0 + split_factor = storage_config.tp_lcm_size // storage_config.tp_size + page_byte_size = mem_pool_host.get_page_buffer_element_size( + split_factor + ) if ( page_byte_size is not None @@ -689,7 +680,7 @@ class UMBPStore(HiCacheStorage): if ( mem_pool_host is not None and storage_config is not None - and getattr(storage_config, "should_split_heads", False) + and storage_config.should_split_heads ) else "" ), diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index ad3cc0642b..d8bf163e49 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -627,6 +627,20 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool): raw = self._unified_buffer._raw return [raw.data_ptr()], [raw.numel()], [self._page_bytes] + def get_page_envelope_buffer(self) -> torch.Tensor: + """Return this sub-pool's page-strided view of the shared allocation.""" + assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0 + raw = self._unified_buffer._raw + page_count = ( + self._unified_buffer.max_slots(self._sub_pool_name) // self.page_size + ) + return raw[: page_count * self._page_bytes].view(page_count, self._page_bytes) + + @property + def grow_direction(self) -> str: + """Physical growth direction used by this sub-pool's L1 allocator.""" + return self._unified_buffer.spec(self._sub_pool_name).grow_direction + def _physical_to_kernel_indices(self, indices: torch.Tensor) -> torch.Tensor: return (indices // self.page_size) * ( self.page_size * self.kernel_page_blocks diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 54c300544a..b13c268e26 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -1474,21 +1474,14 @@ class UnifiedRadixCache(BasePrefixCache): assert req.seqlen > 1 device_indices, extra_transfers = self._retraction_device_transfers(req) - host_indices = self.host_pool_group.alloc(len(device_indices)) - if host_indices is None: - self._reclaim_retraction_host(len(device_indices)) - host_indices = self.host_pool_group.alloc(len(device_indices)) - if host_indices is None: - return None - - resolved = self.host_pool_group.resolve_host_transfers( + allocation = self.cache_controller.allocate_host_transfers( + device_indices, extra_transfers or None, - primary_device_indices=device_indices, - primary_host_indices=host_indices, + reclaim=self._reclaim_retraction_host, ) - if resolved is None and extra_transfers: - self.host_pool_group.free(host_indices) + if allocation is None: return None + host_indices, resolved = allocation backup = RetractionBackup( host_indices=host_indices, @@ -1642,11 +1635,16 @@ class UnifiedRadixCache(BasePrefixCache): def _execute_kv_backup(self, node_id, device_value, comp_xfers, sidecar_xfers): """Execute Backup action.""" kv_tokens = len(device_value) - host_avail = self.cache_controller.mem_pool_host.available_size() - if host_avail < kv_tokens: - needed = kv_tokens - host_avail - if self.evict_host(needed) < needed: - return None + anchor_entry = self.cache_controller.mem_pool_host.anchor_entry + if ( + anchor_entry.host_pool.shared_allocation_domain is None + or anchor_entry.host_evict_fn is None + ): + host_avail = self.cache_controller.mem_pool_host.available_size() + if host_avail < kv_tokens: + needed = kv_tokens - host_avail + if self.evict_host(needed) < needed: + return None aux_xfers = [x for xfers in comp_xfers.values() for x in xfers] aux_xfers.extend(sidecar_xfers) # Defer submission so the next flush can merge pending node backups. diff --git a/test/registered/kernels/ops/kvcache/test_hicache.py b/test/registered/kernels/ops/kvcache/test_hicache.py index 2bc89d94f9..4f2fe71c4c 100644 --- a/test/registered/kernels/ops/kvcache/test_hicache.py +++ b/test/registered/kernels/ops/kvcache/test_hicache.py @@ -4,6 +4,7 @@ import pytest import torch from sglang.kernels.ops.kvcache.hicache import can_use_write_back_jit_kernel +from sglang.srt.mem_cache.l2_transfer import L2Transfer, L2TransferEngine from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, MLATokenToKVPool from sglang.srt.mem_cache.pool_host.common import ( ALLOC_MEMORY_FUNCS, @@ -11,6 +12,10 @@ from sglang.srt.mem_cache.pool_host.common import ( ) from sglang.srt.mem_cache.pool_host.mha import MHATokenToKVPoolHost from sglang.srt.mem_cache.pool_host.mla import MLATokenToKVPoolHost +from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool +from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs from sglang.srt.utils import is_cuda, is_hip, is_npu, is_xpu from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci @@ -35,6 +40,83 @@ LAYOUTS = ["layer_first", "page_first"] STAGED_WRITE_BACK_PAGE_COUNTS = [1, 63, 64, 65, 67, 128, 129] +@pytest.mark.parametrize("direction", ["d2h", "h2d"]) +@pytest.mark.parametrize("backend", ["direct", "kernel"]) +@pytest.mark.skipif(not is_cuda(), reason="Uses a delayed CUDA producer stream") +def test_unified_l2_waits_for_index_producer(direction, backend): + publish(ServerArgs(model_path="dummy"), role="tokenizer") + bundle = init_unified_swa_pools( + device=DEVICE, + kv_cache_dtype=torch.float16, + head_num=1, + head_dim=4, + v_head_dim=4, + swa_head_num=1, + swa_head_dim=4, + swa_v_head_dim=4, + page_size=1, + start_layer=0, + end_layer=2, + swa_attention_layer_ids=[1], + full_attention_layer_ids=[0], + total_bytes=4096, + enable_memory_saver=False, + need_sort=False, + lazy_compaction=True, + ) + device_pool = bundle.token_to_kv_pool.full_kv_pool + host_pool = UnifiedPageEnvelopeHostPool( + device_pool, + 1.0, + 0, + 1, + "layer_first", + pin_memory=True, + allocator_type="default", + ) + engine = L2TransferEngine(backend) + producer = torch.cuda.Stream() + allocator = bundle.token_to_kv_pool_allocator + virtual_indices = allocator.alloc(5) + device_indices = virtual_indices[:1].clone() + host_indices = host_pool.alloc(1) + transfer = L2Transfer(host_pool, device_pool, host_indices, device_indices) + device_pages = device_pool.get_page_envelope_buffer() + try: + # Warm both copy paths before creating the delayed producer dependency. + engine.submit_device_to_host([transfer]).finish_event.synchronize() + engine.submit_host_to_device( + [transfer], transfer_layer_id_max=1 + ).finish_event.synchronize() + device_pages.zero_() + physical_pages = allocator.translate_kv_indices_for_transfer(virtual_indices) + first_page, last_page = int(physical_pages[0]), int(physical_pages[-1]) + device_pages[last_page].fill_(77) + host_pool.get_data_page(int(host_indices[0])).fill_(33) + producer.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(producer): + torch.cuda._sleep(400_000_000) + device_indices.copy_(virtual_indices[-1:]) + if direction == "d2h": + completion = engine.submit_device_to_host([transfer]) + else: + ready = torch.cuda.Event() + ready.record() + completion = engine.submit_host_to_device( + [transfer], transfer_layer_id_max=1, start_event=ready + ) + completion.finish_event.synchronize() + if direction == "d2h": + assert torch.all(host_pool.get_data_page(int(host_indices[0])) == 77) + else: + assert torch.all(device_pages[last_page] == 33) + assert torch.all(device_pages[first_page] == 0) + finally: + producer.synchronize() + host_pool.destroy() + reset_context() + + def _token_indices_for_pages( pages: torch.Tensor, page_size: int = PAGE_SIZE, diff --git a/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py b/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py index 89d094418c..53c10c36c4 100644 --- a/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py +++ b/test/registered/unit/disaggregation/test_specv2_kvcache_offloading.py @@ -107,6 +107,7 @@ def _make_manager(pool_size: int, page_size: int = 1): manager = object.__new__(DecodeKVCacheOffloadManager) manager.req_to_token_pool = req_to_token_pool manager.token_to_kv_pool_allocator = allocator + manager.kv_cache = MagicMock() manager.page_size = page_size manager.tree_cache = tree_cache manager.offloaded_state = WeakKeyDict() @@ -287,6 +288,7 @@ class TestReleaseFinishedReq(unittest.TestCase): torch.arange(4, 8, dtype=torch.int64), [10, 11, 12, 13], 0.0, + [], ) manager.cache_controller = MagicMock() manager.cache_controller.ack_write_queue = [ @@ -409,6 +411,7 @@ class TestReleaseFinishedReq(unittest.TestCase): torch.arange(4, 8, dtype=torch.int64), [10, 11, 12, 13], 0.0, + [], ) manager.cache_controller = MagicMock() manager.cache_controller.ack_write_queue = [ @@ -441,6 +444,7 @@ class TestReleaseFinishedReq(unittest.TestCase): torch.arange(8, 12, dtype=torch.int64), [14, 15, 16, 17], 0.0, + [], ) manager.cache_controller = MagicMock() manager.cache_controller.ack_write_queue = [ @@ -486,7 +490,7 @@ class TestSamplingMaskAbortOffload(CustomTestCase): processor = SimpleNamespace(decode_offload_manager=manager) if inflight: manager.offload_inflight[req] = 1 - manager.ongoing_offload[1] = (req, torch.arange(4), [1], 0.0) + manager.ongoing_offload[1] = (req, torch.arange(4), [1], 0.0, []) manager.cache_controller = MagicMock() manager.cache_controller.ack_write_queue = [ HiCacheAck(None, _FinishedEvent(), [1]) diff --git a/test/registered/unit/disaggregation/test_unified_memory_move_gate.py b/test/registered/unit/disaggregation/test_unified_memory_move_gate.py index e50cd4ff85..e5b7a5d727 100644 --- a/test/registered/unit/disaggregation/test_unified_memory_move_gate.py +++ b/test/registered/unit/disaggregation/test_unified_memory_move_gate.py @@ -60,6 +60,7 @@ class _FakeScheduler: self.disagg_prefill_pending_chunk_rids: Set[str] = set() self.disagg_decode_transfer_queue = _FakeTransferQueue() self.disagg_decode_prealloc_queue = _FakePreallocQueue() + self.decode_offload_manager = None class TestDecodeMoveGate(CustomTestCase): diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 911cf56dd5..9745851c2d 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -239,10 +239,19 @@ class TestDecodeRetractionBackup(CustomTestCase): cache = env.cache req, source_indices = self._admit_req(env, self.num_tokens) + virtual_to_physical = torch.roll( + torch.arange(self.pool_size, device=self.device), + shifts=self.pool_size // 2, + ) + target_pool.host_transfer_translate = lambda indices: virtual_to_physical[ + indices + ] + source_physical_indices = target_pool.host_transfer_translate(source_indices) + self.assertFalse(torch.equal(source_indices, source_physical_indices)) - self._seed_pool(target_pool, source_indices, base=1000) + self._seed_pool(target_pool, source_physical_indices, base=1000) self._seed_pool(draft_pool, source_indices, base=3000) - target_expected = self._snapshot_pool(target_pool, source_indices) + target_expected = self._snapshot_pool(target_pool, source_physical_indices) draft_expected = self._snapshot_pool(draft_pool, source_indices) host_free_before = cache.host_pool_group.available_size() @@ -267,10 +276,15 @@ class TestDecodeRetractionBackup(CustomTestCase): req_to_token_pool.write( (req.kv.req_pool_idx, slice(0, self.num_tokens)), destination_indices ) + destination_physical_indices = target_pool.host_transfer_translate( + destination_indices + ) cache.restore_kv_cache(req, backup) - self._assert_pool_equal(target_pool, destination_indices, target_expected) + self._assert_pool_equal( + target_pool, destination_physical_indices, target_expected + ) self._assert_pool_equal(draft_pool, destination_indices, draft_expected) self.assertEqual(cache.host_pool_group.available_size(), host_free_before) diff --git a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py index df1c6c74ec..505c10939a 100644 --- a/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py +++ b/test/registered/unit/mem_cache/test_hicache_staged_write_back_dispatch.py @@ -64,7 +64,7 @@ def _device_pool_stub(*, layer_num: int, **fields) -> SimpleNamespace: def _host_group_stub(captured, *, can_use_write_back_jit: bool) -> SimpleNamespace: - class FakeHostPool: + class FakeHostPool(MHATokenToKVPoolHost): size_per_token = 2 def backup_from_device_all_layer( @@ -75,7 +75,7 @@ def _host_group_stub(captured, *, can_use_write_back_jit: bool) -> SimpleNamespa entries = [ PoolEntry( name=name, - host_pool=FakeHostPool(), + host_pool=FakeHostPool.__new__(FakeHostPool), device_pool=None, layer_mapper=lambda layer_id: layer_id, is_primary_index_anchor=name == PoolName.KV, @@ -85,6 +85,7 @@ def _host_group_stub(captured, *, can_use_write_back_jit: bool) -> SimpleNamespa return SimpleNamespace( layout="page_first", can_use_write_back_jit=can_use_write_back_jit, + supports_per_pool_backup_indices=False, anchor_entry=entries[0], entry_map={entry.name: entry for entry in entries}, ) @@ -335,7 +336,8 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): pipeline.release_staged_hold.assert_not_called() def test_l2_transfer_maps_global_layers(self): - host_pool = mock.Mock() + host_pool = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost) + host_pool.load_to_device_per_layer = mock.Mock() transfer = L2Transfer( host_pool=host_pool, device_pool=mock.sentinel.device_pool, @@ -357,7 +359,8 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): ) def test_packed_draft_load_is_flattened_into_l2_transfers(self): - host_pool = mock.Mock() + host_pool = MHATokenToKVPoolHost.__new__(MHATokenToKVPoolHost) + host_pool.load_to_device_per_layer = mock.Mock() controller = HybridCacheController.__new__(HybridCacheController) entry = PoolEntry( name=PoolName.KV, @@ -1183,6 +1186,12 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): every backup it receives.""" class ChainHostPool: + def prepare_transfer_indices(self, host, device, backend): + return host, device + + def backup_from_device_all_layer_physical(self, *args): + self.backup_from_device_all_layer(*args) + layout = "page_first" page_size = 4 device = "cpu" @@ -1264,7 +1273,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): def test_write_back_jit_cache_controller_keeps_host_indices_on_cpu(self): captured = {} - class FakeHostPool: + class FakeHostPool(MHATokenToKVPoolHost): layout = "page_first" can_use_write_back_jit = True size_per_token = 2 @@ -1283,7 +1292,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): ) ] controller.io_backend = "kernel" - controller.mem_pool_host = FakeHostPool() + controller.mem_pool_host = FakeHostPool.__new__(FakeHostPool) controller.mem_pool_device = None controller.device = "cuda" controller.ack_write_queue = [] @@ -1301,7 +1310,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): def test_cache_controller_moves_indices_without_write_back_jit(self): captured = {} - class FakeHostPool: + class FakeHostPool(MHATokenToKVPoolHost): layout = "page_first" can_use_write_back_jit = False size_per_token = 2 @@ -1319,7 +1328,7 @@ class TestHiCacheStagedWriteBackDispatch(CustomTestCase): controller = HiCacheController.__new__(HiCacheController) controller.write_queue = [op] controller.io_backend = "kernel" - controller.mem_pool_host = FakeHostPool() + controller.mem_pool_host = FakeHostPool.__new__(FakeHostPool) controller.mem_pool_device = None controller.device = "cuda" controller.ack_write_queue = [] diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 4181e6f27f..eafba69166 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -4,6 +4,8 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch +import torch + from sglang.srt.mem_cache.base_prefix_cache import EvictParams from sglang.srt.mem_cache.hybrid_cache import hybrid_pool_assembler from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( @@ -18,6 +20,8 @@ from sglang.srt.mem_cache.hybrid_cache.hybrid_pool_assembler import ( build_hybrid_swa_group, ) from sglang.srt.mem_cache.memory_pool import HybridLinearKVPool +from sglang.srt.mem_cache.pool_host.unified import UnifiedPageEnvelopeHostPool +from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -272,5 +276,58 @@ class TestTransferLayerSpan(CustomTestCase): ) +def _build_unified_swa_pool(page_size: int = 4): + return init_unified_swa_pools( + device="cpu", + kv_cache_dtype=torch.float16, + head_num=2, + head_dim=4, + v_head_dim=4, + swa_head_num=2, + swa_head_dim=4, + swa_v_head_dim=4, + page_size=page_size, + start_layer=0, + end_layer=4, + swa_attention_layer_ids=[3], + full_attention_layer_ids=[0, 1, 2], + total_bytes=4096, + enable_memory_saver=False, + need_sort=False, + lazy_compaction=True, + ) + + +def _build_unified_host_pair(bundle): + pool = bundle.token_to_kv_pool + return UnifiedPageEnvelopeHostPool.build_hybrid_swa_pool_pair( + device_pools=(pool.full_kv_pool, pool.swa_kv_pool), + host_to_device_ratio=1.0, + host_size=0, + page_size=pool.page_size, + layout="layer_first", + pin_memory=False, + allocator_type="default", + ) + + +class TestUnifiedPageEnvelopeHostPool(CustomTestCase): + def test_shared_arena_can_reuse_bytes_across_sides(self): + page_size = 4 + full_pool, swa_pool = _build_unified_host_pair( + _build_unified_swa_pool(page_size) + ) + self.addCleanup(full_pool.destroy) + self.addCleanup(swa_pool.destroy) + + full_indices = full_pool.alloc(full_pool.available_size()) + self.assertIsNotNone(full_indices) + self.assertIsNotNone(swa_pool.alloc(swa_pool.available_size())) + self.assertIsNone(swa_pool.alloc(page_size)) + + full_pool.free(full_indices[page_size:]) + self.assertIsNotNone(swa_pool.alloc(page_size)) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py index 6802af04e6..1f46b17464 100644 --- a/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py +++ b/test/registered/unit/mem_cache/test_pp_prefetch_ticket.py @@ -4,6 +4,7 @@ import pickle import threading import unittest from array import array +from contextlib import nullcontext from queue import Empty, Queue from unittest.mock import Mock, call, patch @@ -49,6 +50,7 @@ class TestPPPrefetchTicket(unittest.TestCase): c.storage_stop_event = threading.Event() c.prefetch_tokens_occupied = 0 c.mem_pool_host = Mock(page_size=4) + c.mem_pool_host.layout_lease.side_effect = nullcontext c.mem_pool_host.alloc.side_effect = lambda size, **_: torch.arange(size) c._storage_hit_query = Mock(return_value=(["h0", "h1"], 8)) c._all_reduce = Mock() diff --git a/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py b/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py index 4b4a82bd7f..bfe0e7412c 100644 --- a/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py +++ b/test/registered/unit/mem_cache/test_swa_cpu_copy_filter.py @@ -61,6 +61,39 @@ class TestSWACpuCopyFilter(unittest.TestCase): class TestUnifiedSWATransfers(CustomTestCase): + def test_offload_uses_physical_swa_pages(self): + from sglang.srt.disaggregation.decode_kvcache_offload_manager import ( + DecodeKVCacheOffloadManager, + ) + + for tombstone_tokens in (0, 4, 12): + with self.subTest(tombstone_tokens=tombstone_tokens): + bundle = _swa_factory(page_size=4) + allocator = bundle.token_to_kv_pool_allocator + indices = allocator.alloc(12) + allocator.free_swa(indices[:tombstone_tokens]) + raw = bundle.unified_memory_pool._raw + raw.copy_(torch.arange(raw.numel()).remainder(251).to(torch.uint8)) + manager = object.__new__(DecodeKVCacheOffloadManager) + manager.token_to_kv_pool_allocator = allocator + manager.kv_cache = bundle.token_to_kv_pool + full, transfers = manager._resolve_offload_transfers(indices) + self.assertTrue(torch.equal(full, indices)) + if tombstone_tokens == len(indices): + self.assertEqual(transfers, []) + continue + expected_pages = allocator.swa_v2p_page_table[ + indices[tombstone_tokens::4] // 4 + ] + pages = transfers[0].device_indices[::4] // ( + 4 * manager.kv_cache.swa_kv_pool.kernel_page_blocks + ) + buffer = manager.kv_cache.swa_kv_pool.get_page_envelope_buffer() + self.assertTrue( + torch.equal(buffer[pages].clone(), buffer[expected_pages]) + ) + self.assertTrue(torch.equal(pages, expected_pages)) + def test_cpu_copy_round_trip_with_request_index_and_swa_tombstone(self): bundle = _swa_factory(page_size=4) allocator = bundle.token_to_kv_pool_allocator diff --git a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py index 4bd2da3a9d..73fd3219fb 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_regressions.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_regressions.py @@ -219,7 +219,8 @@ class TestTransferStreamOrdering(unittest.TestCase): host = SimpleNamespace( layer_num=1, - load_to_device_per_layer=lambda pool, h, d, layer, backend, **kw: ( + prepare_transfer_indices=lambda host, device, backend: (host, device), + load_to_device_per_layer_physical=lambda pool, h, d, layer, backend, **kw: ( output.copy_(d) ), )