diff --git a/python/sglang/srt/arg_groups/fields/disagg.py b/python/sglang/srt/arg_groups/fields/disagg.py index da764f851c..04b187d5ac 100644 --- a/python/sglang/srt/arg_groups/fields/disagg.py +++ b/python/sglang/srt/arg_groups/fields/disagg.py @@ -178,5 +178,5 @@ class Disagg(msgspec.Struct): sm_group_num: A[int, "Number of sm partition groups."] = 8 disaggregation_decode_host_receive_threshold: A[ float, - "Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense MHA and a transfer backend that supports host destinations. No built-in backend currently supports this.", + "Device token usage fraction at which incoming KV is received in the decode retraction host pool, excluding evictable cache pages. Range [0, 1]; 1 disables host receive (default), 0 always uses host receive. Size with --hicache-size or --hicache-ratio; requires dense NHD MHA buffers (including separate paged SWA pools) and a transfer backend that supports host destinations. No built-in backend currently supports this.", ] = 1.0 diff --git a/python/sglang/srt/disaggregation/base/conn.py b/python/sglang/srt/disaggregation/base/conn.py index a3272299d5..e9a057cf44 100644 --- a/python/sglang/srt/disaggregation/base/conn.py +++ b/python/sglang/srt/disaggregation/base/conn.py @@ -61,6 +61,9 @@ class KVArgs: host_kv_data_ptrs: Optional[List[int]] = None host_kv_data_lens: Optional[List[int]] = None host_kv_item_lens: Optional[List[int]] = None + host_state_data_ptrs: Optional[List[List[int]]] = None + host_state_data_lens: Optional[List[List[int]]] = None + host_state_item_lens: Optional[List[List[int]]] = None aux_data_ptrs: List[int] aux_data_lens: List[int] aux_item_lens: List[int] @@ -232,6 +235,14 @@ class BaseKVReceiver(ABC): """Whether this receiver's peer and layout support host KV destinations.""" return False + def is_abort_release_safe(self) -> bool: + """Whether all writes to an aborted host destination have drained. + + Host-capable backends must retain this state until ``clear()``. A failed + poll or elapsed timeout alone does not prove the buffers can be reused. + """ + raise NotImplementedError + @abstractmethod def __init__( self, diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index ff1561f175..50e9329eca 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -659,12 +659,12 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): if get_disagg().disaggregation_decode_host_receive_threshold < 1: pool = self.token_to_kv_pool group = self.tree_cache.host_pool_group - if kv_args.state_types or any( - spec.indices_from_pool != PoolName.KV + if (kv_args.state_types and kv_args.state_types != [StateType.SWA]) or any( + spec.indices_from_pool not in (PoolName.KV, PoolName.SWA) for spec in self.tree_cache.sidecar_pool_specs ): raise ValueError( - "Host receive requires KV pools sharing the primary indices" + "Host receive supports full KV and separate paged SWA pools" ) self.host_pool = group.get_pool(PoolName.KV) self.host_reserved_tokens = decode_retraction_max_tokens( @@ -675,12 +675,66 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): "Host pool must hold a retraction and a receive page; " "increase --hicache-size or --hicache-ratio" ) - _, host_buffers = group.get_contiguous_buf_infos() + + def pool_names(source): + return {source} | { + spec.pool_name + for spec in self.tree_cache.sidecar_pool_specs + if spec.indices_from_pool == source + } + + device_buffers, host_buffers = group.get_contiguous_buf_infos( + pool_names(PoolName.KV) + ) + if device_buffers != ( + kv_args.kv_data_ptrs, + kv_args.kv_data_lens, + kv_args.kv_item_lens, + ): + raise ValueError( + "Host receive buffers must match the KV transfer layout" + ) ( kv_args.host_kv_data_ptrs, kv_args.host_kv_data_lens, kv_args.host_kv_item_lens, ) = host_buffers + self.host_swa_pool = None + self.host_swa_reserved_tokens = 0 + if kv_args.state_types: + self.host_swa_pool = group.get_pool(PoolName.SWA) + self.host_swa_reserved_tokens = min( + decode_retraction_max_tokens( + self.req_to_token_pool, pool.swa_kv_pool + ), + ceil_align( + self.scheduler.sliding_window_size + pool.page_size - 1, + pool.page_size, + ), + ) + if self.host_swa_pool.logical_size < ( + self.host_swa_reserved_tokens + pool.page_size + ): + raise ValueError( + "SWA host pool must hold a retraction and a receive page; " + "increase --hicache-size or --hicache-ratio" + ) + device_state, host_state = group.get_contiguous_buf_infos( + pool_names(PoolName.SWA) + ) + if device_state != ( + kv_args.state_data_ptrs[0], + kv_args.state_data_lens[0], + kv_args.state_item_lens[0], + ): + raise ValueError( + "Host receive buffers must match the SWA transfer layout" + ) + ( + kv_args.host_state_data_ptrs, + kv_args.host_state_data_lens, + kv_args.host_state_item_lens, + ) = ([values] for values in host_state) kv_args.ib_device = get_disagg().disaggregation_ib_device kv_args.gpu_id = get_device().gpu_id @@ -1755,9 +1809,9 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): decode_req.req.time_stats.set_decode_transfer_queue_entry_time() def _pre_alloc_host(self, decode_req: DecodeRequest) -> bool: - num_tokens = ceil_align( - len(decode_req.req.origin_input_ids), self.host_pool.page_size - ) + seq_len = len(decode_req.req.origin_input_ids) + page_size = self.host_pool.page_size + num_tokens = ceil_align(seq_len, page_size) required_tokens = num_tokens + self.host_reserved_tokens if ( self.host_pool.available_size() < required_tokens @@ -1766,17 +1820,43 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): self.tree_cache.evict_host(required_tokens) if self.host_pool.available_size() < required_tokens: return False + swa_num_tokens = 0 + if self.host_swa_pool is not None: + window_start = page_align_floor( + max(0, seq_len - self.scheduler.sliding_window_size), page_size + ) + swa_num_tokens = num_tokens - window_start + required_swa_tokens = swa_num_tokens + self.host_swa_reserved_tokens + if self.host_swa_pool.available_size() < required_swa_tokens: + entry = self.tree_cache.host_pool_group.get_entry(PoolName.SWA) + if entry.host_evict_fn is not None: + entry.host_evict_fn(required_swa_tokens) + if self.host_swa_pool.available_size() < required_swa_tokens: + return False host_indices = self.host_pool.alloc(num_tokens) if host_indices is None: return False + indices_by_pool = {PoolName.KV: host_indices} + transfers = [] + state_indices = None + if self.host_swa_pool is not None: + swa_host_indices = self.host_swa_pool.alloc(swa_num_tokens) + if swa_host_indices is None: + self.host_pool.free(host_indices) + return False + indices_by_pool[PoolName.SWA] = swa_host_indices + transfers.append( + PoolTransfer(name=PoolName.SWA, host_indices=swa_host_indices) + ) + state_indices = [kv_to_page_indices(swa_host_indices, page_size)] assert decode_req.req.kv.retraction_backup is None - # All sidecars share the primary KV indices. decode_req.req.kv.retraction_backup = RetractionBackup( host_indices=host_indices, - pool_transfers=[ + pool_transfers=transfers + + [ PoolTransfer( name=spec.pool_name, - host_indices=host_indices, + host_indices=indices_by_pool[spec.indices_from_pool], indices_from_pool=spec.indices_from_pool, hit_policy=spec.hit_policy, ) @@ -1791,6 +1871,7 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): decode_req, host_indices, self.token_to_kv_pool_allocator.page_size, + state_indices, decode_prefix_len=0, destination=KVTransferDestination.HOST, ) @@ -1815,13 +1896,29 @@ class DecodePreallocQueue(DecodeHiCachePreallocMixin): 0, retractable_tokens, ) + swa_required_tokens = 0 + swa_allocatable_tokens = None + if self._uses_swa_reservation(): + _, swa_required_tokens = self._prealloc_required_tokens(req) + full_allocatable_tokens, swa_allocatable_tokens = ( + self._swa_aware_allocatable_token_budgets( + retractable_tokens=retractable_tokens, + retractable_swa_tokens=sum( + self._swa_retractable_len(r) + for r in self.scheduler.running_batch.reqs + ), + count_retracted=True, + ) + ) + else: + full_allocatable_tokens = self._allocatable_token_budgets( + retractable_tokens=retractable_tokens, count_retracted=True + ) if not self._prealloc_reservation_fits( required_tokens, - 0, - full_allocatable_tokens=self._allocatable_token_budgets( - retractable_tokens=retractable_tokens, count_retracted=True - ), - swa_allocatable_tokens=None, + swa_required_tokens, + full_allocatable_tokens=full_allocatable_tokens, + swa_allocatable_tokens=swa_allocatable_tokens, ): return False self._pre_alloc(req) @@ -2611,6 +2708,14 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): return release_kv_cache(decode_req.req, self.tree_cache, checkpoint=False) + def _record_transfer_failure(self, failure_reason: str) -> None: + if self.scheduler.metrics_reporter.enable_metrics: + self.scheduler.metrics_collector.increment_transfer_failed_reqs() + + def _abort_failed_transfer(self, decode_req: DecodeRequest) -> None: + if self.enable_host_receive and decode_req.host_staged: + decode_req.kv_receiver.abort() + def pop_transferred(self, rids_to_check: Optional[List[str]] = None) -> List[Req]: if not self.queue: return [] @@ -2643,17 +2748,20 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): poll == KVPoll.Failed or hicache_restore_status == HiCacheRestoreResult.FAILED ): + self._abort_failed_transfer(decode_req) error_message = ( f"Decode transfer failed for request rank={self.tp_rank} " f"{decode_req.req.rid=} {decode_req.req.bootstrap_room=}" ) is_propagated = False + failure_reason = str(hicache_restore_status) if poll == KVPoll.Failed: try: decode_req.kv_receiver.failure_exception() except Exception as e: error_message += f" with exception {e}" is_propagated = getattr(e, "is_from_another_rank", False) + failure_reason = getattr(e, "failure_reason", str(e)) self._clean_hicache_prefetch_resources(decode_req) # Mute error message for propagated exceptions to avoid duplicate logging if is_propagated: @@ -2674,8 +2782,6 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): requires_host_drain = ( self.enable_host_receive and decode_req.host_staged ) - if requires_host_drain: - decode_req.kv_receiver.abort() deferrable = ( self.enable_deferred_kv_release and decode_req.kv_receiver.kv_mgr.enable_deferred_decode_kv_release @@ -2704,8 +2810,7 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): decode_req.kv_receiver.clear() decode_req.kv_receiver = None indices_to_remove.add(i) - if self.scheduler.metrics_reporter.enable_metrics: - self.scheduler.metrics_collector.increment_transfer_failed_reqs() + self._record_transfer_failure(failure_reason) continue elif poll == KVPoll.Success: if self.enable_host_receive and decode_req.host_staged: @@ -2741,8 +2846,9 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): ) self._clean_hicache_prefetch_resources(decode_req) self._release_request(decode_req) - if self.scheduler.metrics_reporter.enable_metrics: - self.scheduler.metrics_collector.increment_transfer_failed_reqs() + self._record_transfer_failure( + decode_req.req.finished_reason.message + ) else: transferred_reqs.append(decode_req.req) elif poll in [ @@ -2780,9 +2886,13 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): def _defer_release(self, decode_req: DecodeRequest) -> None: start_time = time.monotonic() deadline = start_time + self.deferred_kv_release_timeout - # Require an ack from every notified prefill rank (dummy-proof). Snapshot - # now -- the receiver may be cleared by resolve time. - required_acks = len(decode_req.kv_receiver.bootstrap_infos) + # Host-capable receivers own their drain tracking. Device destinations + # snapshot the number of prefill acknowledgements required by the manager. + required_acks = ( + 0 + if self.enable_host_receive and decode_req.host_staged + else len(decode_req.kv_receiver.bootstrap_infos) + ) self._deferred_releases.append( ( decode_req, @@ -2801,7 +2911,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): self._release_request(decode_req) self.metadata_buffers.bootstrap_room[idx] = 0 self.req_to_metadata_buffer_idx_allocator.free(idx) - decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room) + if not (self.enable_host_receive and decode_req.host_staged): + decode_req.kv_receiver.kv_mgr.clear_deferred_abort_state(room) decode_req.kv_receiver.clear() decode_req.kv_receiver = None @@ -2826,10 +2937,8 @@ class DecodeTransferQueue(DecodeHiCacheTransferMixin): return host_ready = torch.tensor( [ - req.kv_receiver.kv_mgr.is_abort_release_safe( - req.req.bootstrap_room, acks - ) - for req, _, _, _, acks in self._deferred_releases + req.kv_receiver.is_abort_release_safe() + for req, _, _, _, _ in self._deferred_releases if self.enable_host_receive and req.host_staged ], dtype=torch.int32, diff --git a/python/sglang/srt/mem_cache/pool_host/group.py b/python/sglang/srt/mem_cache/pool_host/group.py index 68d7a24639..cac407ae70 100644 --- a/python/sglang/srt/mem_cache/pool_host/group.py +++ b/python/sglang/srt/mem_cache/pool_host/group.py @@ -105,16 +105,18 @@ class HostPoolGroup: element_sizes.add(element_size) return element_sizes.pop() if len(element_sizes) == 1 else None - def get_contiguous_buf_infos(self): + def get_contiguous_buf_infos(self, pool_names: set[PoolName] | None = None): """Return (device_buffers, host_buffers), each (ptrs, sizes, item_sizes).""" host_by_device_ptr = {} device_infos = ([], [], []) for entry in self.entries: + if pool_names is not None and entry.name not in pool_names: + continue host = entry.host_pool pools = (entry.device_pool, *entry.packed_draft_device_pools) for pool in pools: dense_mha = ( - type(pool) is MHATokenToKVPool + isinstance(pool, MHATokenToKVPool) and pool.kv_cache_layout == "nhd" and pool.v_head_dim == pool.head_dim ) @@ -127,9 +129,21 @@ class HostPoolGroup: "Host receive requires dense NHD MHA target and " "draft KV with matching page sizes" ) - for combined, values in zip( - device_infos, pool.get_contiguous_buf_infos(), strict=True + buffers = pool.k_buffer + pool.v_buffer + infos = pool.get_contiguous_buf_infos() + expected = ( + [buffer.data_ptr() for buffer in buffers], + [buffer.nbytes for buffer in buffers], + [buffer[0].nbytes * self.page_size for buffer in buffers], + ) + if infos != expected or any( + not buffer.is_contiguous() for buffer in buffers ): + raise ValueError( + "Host receive requires contiguous per-layer KV buffers " + "without embedded state" + ) + for combined, values in zip(device_infos, infos, strict=True): combined.extend(values) # Packed MHA stores target/draft K followed by target/draft V, diff --git a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py index bec48bc1d8..be1f3f641e 100644 --- a/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py +++ b/test/registered/unit/disaggregation/test_deferred_decode_kv_release.py @@ -667,6 +667,7 @@ def _make_decode_req(room, idx, mgr, n_prefill_ranks=1): # One entry per prefill rank the decode notified of the abort; its length # is the required drain-ack count (see DecodeTransferQueue._defer_release). bootstrap_infos=[{"rank": r} for r in range(n_prefill_ranks)], + is_abort_release_safe=lambda: mgr.is_abort_release_safe(room, n_prefill_ranks), clear=lambda: None, ) return SimpleNamespace( diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 8290850d50..972508f72f 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -21,6 +21,7 @@ from sglang.srt.mem_cache.allocator import ( PagedTokenToKVPoolAllocator, TokenToKVPoolAllocator, ) +from sglang.srt.mem_cache.allocator.swa import SWATokenToKVPoolAllocator from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.common import ( backup_kv_cache, @@ -34,6 +35,7 @@ from sglang.srt.mem_cache.memory_pool import ( MLATokenToKVPool, ReqToTokenPool, ) +from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.srt.runtime_context import get_parallel @@ -55,6 +57,24 @@ class TestDecodeRetractionBackup(CustomTestCase): dtype = torch.bfloat16 device = "cuda" + def test_host_receive_dense_subclass_checks_actual_buffer_geometry(self): + class DensePool(MHATokenToKVPool): + pass + + with patch(f"{__name__}.MHATokenToKVPool", DensePool): + env = self._build_cache(hicache_ratio=2.0, shared_receive=True) + group = env.cache.host_pool_group + device, host = group.get_contiguous_buf_infos() + self.assertEqual(device, env.target_pool.get_contiguous_buf_infos()) + self.assertEqual(device[2], host[2]) + + wrong_stride = (device[0], device[1], [n + 4 for n in device[2]]) + with patch.object( + env.target_pool, "get_contiguous_buf_infos", return_value=wrong_stride + ): + with self.assertRaisesRegex(ValueError, "without embedded state"): + group.get_contiguous_buf_infos() + def _make_pool(self, layer_num: int, *, page_size=1) -> MHATokenToKVPool: return MHATokenToKVPool( size=self.pool_size, @@ -110,6 +130,7 @@ class TestDecodeRetractionBackup(CustomTestCase): draft_mode=None, radix_cache=False, host_receive_threshold=0.5, + hybrid_swa=False, ): """Bring up a UnifiedRadixCache over fresh pools, optionally with draft KV.""" server_args = ServerArgs( @@ -138,7 +159,21 @@ class TestDecodeRetractionBackup(CustomTestCase): device=self.device, enable_memory_saver=False, ) - target_pool = self._make_pool(layer_num=2, page_size=page_size) + target_pool = ( + SWAKVPool( + size=self.pool_size, + size_swa=self.pool_size, + page_size=page_size, + dtype=self.dtype, + head_num=2, + head_dim=64, + swa_attention_layer_ids=[0], + full_attention_layer_ids=[1], + device=self.device, + ) + if hybrid_swa + else self._make_pool(layer_num=2, page_size=page_size) + ) if draft_mode is None: draft_mode = ( HiCacheDraftMode.NONE if shared_receive else HiCacheDraftMode.SIDECAR @@ -151,6 +186,8 @@ class TestDecodeRetractionBackup(CustomTestCase): allocator_cls = ( PagedTokenToKVPoolAllocator if page_size > 1 else TokenToKVPoolAllocator ) + if hybrid_swa: + allocator_cls = SWATokenToKVPoolAllocator allocator = allocator_cls( size=self.pool_size, dtype=self.dtype, @@ -158,6 +195,7 @@ class TestDecodeRetractionBackup(CustomTestCase): kvcache=target_pool, need_sort=False, **({"page_size": page_size} if page_size > 1 else {}), + **({"size_swa": self.pool_size} if hybrid_swa else {}), ) params = CacheInitParams( disable=not radix_cache, @@ -165,10 +203,15 @@ class TestDecodeRetractionBackup(CustomTestCase): token_to_kv_pool_allocator=allocator, page_size=page_size, is_eagle=draft_pool is not None, - tree_components=(ComponentType.FULL,), - mtp_draft_device_pools=(draft_pool,) - if draft_mode == HiCacheDraftMode.PACKED - else (), + tree_components=( + (ComponentType.FULL, ComponentType.SWA) + if hybrid_swa + else (ComponentType.FULL,) + ), + sliding_window_size=8 if hybrid_swa else None, + mtp_draft_device_pools=( + (draft_pool,) if draft_mode == HiCacheDraftMode.PACKED else () + ), ) cache = UnifiedRadixCache(params) cache.init_hicache(server_args, params) @@ -293,6 +336,7 @@ class TestDecodeRetractionBackup(CustomTestCase): req_to_token_pool.free(req) def _receive_queue(self, env): + hybrid_swa = isinstance(env.target_pool, SWAKVPool) queue = object.__new__(DecodePreallocQueue) queue.__dict__.update( tree_cache=env.cache, @@ -325,6 +369,7 @@ class TestDecodeRetractionBackup(CustomTestCase): running_batch=SimpleNamespace(reqs=[]), waiting_queue=[], last_batch=None, + sliding_window_size=8 if hybrid_swa else None, enable_hisparse=False, enable_decode_hicache=False, enable_lora=False, @@ -333,7 +378,7 @@ class TestDecodeRetractionBackup(CustomTestCase): current_scheduler_metrics_enabled=False ), tp_worker=SimpleNamespace( - is_hybrid_swa=False, + is_hybrid_swa=hybrid_swa, model_runner=SimpleNamespace(kv_cache_dtype_str="bfloat16"), ), ), @@ -349,15 +394,106 @@ class TestDecodeRetractionBackup(CustomTestCase): req_to_token_pool=env.req_to_token_pool, session_controller=None, hisparse_coordinator=None, - is_hybrid_swa=False, + is_hybrid_swa=hybrid_swa, is_hybrid_ssm=False, enable_hisparse=False, full_tokens_per_layer=self.pool_size, - swa_tokens_per_layer=None, + swa_tokens_per_layer=self.pool_size if hybrid_swa else None, max_total_num_tokens=self.pool_size, ) return queue, queue.kv_manager.kv_args + @patch("torch.distributed.get_world_size", return_value=1) + def test_host_receive_swa_tail_capacity_restore_and_release(self, _world_size): + env = self._build_cache( + hicache_ratio=2.0, + shared_receive=True, + page_size=4, + hybrid_swa=True, + host_receive_threshold=0, + ) + queue, _ = self._receive_queue(env) + group = env.cache.host_pool_group + full_host = group.get_pool(PoolName.KV) + swa_host = group.get_pool(PoolName.SWA) + capacities = (full_host.available_size(), swa_host.available_size()) + req = Req( + rid="swa-host-receive", + origin_input_text="", + bootstrap_host="localhost", + origin_input_ids=array("q", [1] * 23), + sampling_params=SamplingParams(max_new_tokens=1), + ) + receiver = Mock() + decode_req = DecodeRequest(req=req, kv_receiver=receiver) + blocker = swa_host.alloc( + swa_host.available_size() - queue.host_swa_reserved_tokens + ) + self.assertFalse(queue._pre_alloc_host(decode_req)) + self.assertEqual(full_host.available_size(), capacities[0]) + receiver.send_metadata.assert_not_called() + swa_host.free(blocker) + self.assertTrue(queue._pre_alloc_host(decode_req)) + backup = req.kv.retraction_backup + self.assertEqual(len(backup.host_indices), 24) + self.assertEqual(len(backup.pool_transfers[0].host_indices), 12) + self.assertEqual(len(receiver.send_metadata.call_args.args[2][0]), 3) + + # Independent SWA pressure must delay device admission even when full KV fits. + swa_blocker = env.allocator.swa_attn_allocator.alloc(self.pool_size) + self.assertFalse(queue.allocate_host_staged(decode_req)) + self.assertIsNone(req.kv.req_pool_idx) + self.assertIs(req.kv.retraction_backup, backup) + env.allocator.swa_attn_allocator.free(swa_blocker) + + expected = [] + for pool_name, host_indices, num_tokens in ( + (PoolName.KV, backup.host_indices, 23), + (PoolName.SWA, backup.pool_transfers[0].host_indices, 11), + ): + entry = group.get_entry(pool_name) + pool = entry.device_pool + for index, (buffer, host_buffer) in enumerate( + zip( + pool.k_buffer + pool.v_buffer, + entry.host_pool.host_kv_data_refs, + strict=True, + ) + ): + values = torch.arange(len(host_indices) * buffer[0].numel()).reshape( + len(host_indices), *buffer.shape[1:] + ) + values = ((values + index * 37) % 251).to(self.dtype) + host_buffer[host_indices] = values + expected.append((pool_name, buffer, values[:num_tokens])) + buffer.fill_(-1) + + self.assertTrue(queue.allocate_host_staged(decode_req)) + req.output_ids.append(99) + env.cache.restore_kv_cache(req, backup) + full_indices = env.req_to_token_pool.req_to_token[ + req.kv.req_pool_idx, :23 + ].long() + swa_indices = env.allocator.translate_swa_indices_for_transfer( + full_indices[12:] + ) + for pool_name, buffer, values in expected: + indices = full_indices if pool_name == PoolName.KV else swa_indices + self.assertTrue(torch.equal(buffer[indices].cpu(), values)) + self.assertEqual( + (full_host.available_size(), swa_host.available_size()), capacities + ) + req.kv.retraction_backup = None + env.allocator.free(full_indices) + env.req_to_token_pool.free(req) + + # An abort before device admission returns both independent allocations. + self.assertTrue(queue._pre_alloc_host(decode_req)) + env.cache.discard_kv_cache_backup(req.kv.retraction_backup) + self.assertEqual( + (full_host.available_size(), swa_host.available_size()), capacities + ) + @patch("torch.distributed.get_world_size", return_value=1) def test_host_receive_threshold_controls_device_allocation(self, _world_size): """Zero always stages, one disables, and intermediate values gate on usage."""