diff --git a/python/sglang/srt/mem_cache/hicache_storage.py b/python/sglang/srt/mem_cache/hicache_storage.py index e623d71b6b..a226069588 100644 --- a/python/sglang/srt/mem_cache/hicache_storage.py +++ b/python/sglang/srt/mem_cache/hicache_storage.py @@ -93,10 +93,15 @@ class PoolHitPolicy(str, Enum): ALL_PAGES : every page in [0, kv_hit) must exist (e.g. DSA). TRAILING_PAGES : only the last N pages must exist (e.g. Mamba/SWA states). + BLOCK_GROUPS : whole key groups at block boundaries from the span start + (a second form of the FULL pages, e.g. compressed blocks). + The pool reports the pages its leading complete groups + cover as its own verdict; it never gates the KV hit. """ ALL_PAGES = "all_pages" TRAILING_PAGES = "trailing_pages" + BLOCK_GROUPS = "block_groups" @dataclass @@ -115,6 +120,9 @@ class PoolTransfer: hit_policy: PoolHitPolicy = PoolHitPolicy.ALL_PAGES nodes_to_load: Optional[List[Any]] = None indices_from_pool: Optional[PoolName] = None + # BLOCK_GROUPS geometry: FULL pages per block and keys (pages) per group. + block_pages: int = 0 + group_pages: int = 0 @dataclass(frozen=True) diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py index b2d571fb72..11cb913517 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_cache_controller.py @@ -156,6 +156,10 @@ class PrefetchOperation(StorageOperation): self._lock = threading.Lock() self._terminated_flag = False self.storage_hit_count = 0 + # KV pages the hit-time plan skips in front of the KV transfer: the + # device trim, then the span head a component's block groups serve. + self.prefetch_kv_page_offset = 0 + self.group_skip_pages = 0 self.start_time = time.monotonic() # Take the whole span as present instead of querying for it; the read # itself is fail-soft, so a wrong guess shortens the fetch. @@ -963,15 +967,25 @@ class HybridCacheController(BaseHiCacheController): priority: Optional[int] = None, node_id: int = -1, extra_pools: Optional[list[PoolTransfer]] = None, + *, + device_indices: Optional[torch.Tensor] = None, + anchor_indices: Optional[torch.Tensor] = None, ) -> Optional[torch.Tensor]: + """Queue an H2D of ``host_indices`` into freshly allocated FULL rows, + or into caller-owned ``device_indices`` (kept on failure); side pools + bind their device slots against ``anchor_indices`` (default: the KV + destination rows).""" need_load_kv = host_indices.numel() > 0 + caller_owned = device_indices is not None full_allocator = getattr( self.mem_pool_device_allocator, "full_attn_allocator", self.mem_pool_device_allocator, ) - if not need_load_kv: + if caller_owned: + assert len(device_indices) == len(host_indices) + elif not need_load_kv: device_indices = torch.empty((0,), dtype=torch.int64, device=self.device) else: device_indices = full_allocator.alloc(len(host_indices)) @@ -980,11 +994,13 @@ class HybridCacheController(BaseHiCacheController): pool_transfers = self._resolve_device_transfers( extra_pools, - kv_device_indices=device_indices, + kv_device_indices=( + anchor_indices if anchor_indices is not None else device_indices + ), kv_host_indices=host_indices, ) if pool_transfers is None and extra_pools: - if need_load_kv: + if need_load_kv and not caller_owned: full_allocator.free(device_indices) return None @@ -1543,6 +1559,7 @@ class HybridCacheController(BaseHiCacheController): operation.sidecar_hash_values = original_hashes operation.sidecar_hit_pages = len(original_hashes) operation.hash_value = original_hashes[trim_pages:] + operation.prefetch_kv_page_offset += trim_pages operation.storage_hit_count -= trim_tokens operation.storage_start += trim_tokens diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 9e36908e8c..8cabad6d87 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -912,11 +912,29 @@ class UnifiedRadixCache(BasePrefixCache): for ct in self.tree_components: request_cnt = request_by_type[ct] + # An allocation-driven FULL request names the allocator's + # shortfall, not what the walk must free: an allocator can hide + # headroom behind a clamp (a reserve it refills before raw + # allocation sees a page again) or page rounding, so freeing + # exactly the shortfall can leave its target unmet. Such a walk + # runs to the target; the check after every victim still stops it + # at the first sufficient one. + to_target = ( + ct == ComponentType.FULL + and request_cnt > 0 + and available_size_targets is not None + and ct in available_size_targets + ) # A preceding component may have cascade-evicted this component or, # on a shared pool, released enough bytes to satisfy its allocation. - if tracker[ct] >= request_cnt or target_reached(ct): + if target_reached(ct) or (not to_target and tracker[ct] >= request_cnt): continue - self.tree_core.evict_device_start(ct, request_cnt) + walk_quota = request_cnt + if to_target: + walk_quota = max( + request_cnt, tracker[ct] + self.tree_core.full_evictable_size() + ) + self.tree_core.evict_device_start(ct, walk_quota) try: while True: node_id, made_progress = self._evict_device_next_node(ct, tracker) diff --git a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py index 94622c624e..70006ca298 100644 --- a/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py +++ b/test/registered/unit/mem_cache/test_hybrid_pool_assembler.py @@ -1250,7 +1250,12 @@ class TestUnifiedPageEnvelopeHostPool(CustomTestCase): operation.hash_value = list(hashes) operation.storage_hit_count = hit_tokens operation.sidecar_hash_values = None + operation.prefetch_kv_page_offset = 0 cc.trim_prefetch_full_head(operation, hit_tokens - full_tokens) + self.assertEqual( + operation.prefetch_kv_page_offset, + (hit_tokens - full_tokens) // page_size, + ) self.assertTrue( cc.can_fit_prefetch_host_buffers(operation, full_tokens) ) diff --git a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py index ce16486d07..974113068c 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py @@ -281,6 +281,63 @@ class TestFloatSwaReservation(_ConfigCase): self.assertIn(("move", "swa"), moves) +class TestCallerOwnedLoadDestination(_ConfigCase): + """A load into rows the caller already owns allocates nothing, keeps those + rows when it rolls back, and binds KV-derived side pools to the caller's + anchor rows rather than to the destination.""" + + def test_load_into_caller_owned_rows_allocates_nothing_and_keeps_them(self): + allocator, _ = _unified_swa() + full = allocator.full_attn_allocator + owned = allocator.alloc(PAGE) + anchor = allocator.alloc(PAGE) + allocated = full.allocated_count() + host_indices = torch.arange(PAGE) + + controller = object.__new__(HybridCacheController) + controller.mem_pool_device_allocator = allocator + controller.device = "cpu" + controller.load_queue, controller.ack_load_queue = [], [] + controller.mem_pool_host = SimpleNamespace( + entry_map={ + PoolName.SWA: SimpleNamespace( + device_alloc_fn=lambda size: None, + device_free_fn=lambda indices: None, + device_evict_fn=None, + ) + } + ) + + # A side pool that cannot allocate cancels the load; the destination + # rows were never the load's to free. + self.assertIsNone( + controller.load( + host_indices=host_indices, + extra_pools=[ + PoolTransfer(name=PoolName.SWA, host_indices=torch.arange(PAGE)) + ], + device_indices=owned, + anchor_indices=anchor, + ) + ) + self.assertEqual(controller.load_queue, []) + self.assertEqual(full.allocated_count(), allocated) + + derived = PoolTransfer(name=PoolName.SWA, indices_from_pool=PoolName.KV) + self.assertIs( + controller.load( + host_indices=host_indices, + extra_pools=[derived], + device_indices=owned, + anchor_indices=anchor, + ), + owned, + ) + self.assertEqual(full.allocated_count(), allocated) + self.assertIs(controller.load_queue[-1].device_indices, owned) + self.assertIs(derived.device_indices, anchor) + + class TestAllocationAndTransferOrder(_ConfigCase): """Allocation reuses free pages without waiting on HiCache copies.""" diff --git a/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py b/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py index da017ca902..1f8756e754 100644 --- a/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py +++ b/test/registered/unit/mem_cache/test_unified_radix_allocation_eviction.py @@ -31,6 +31,7 @@ class TestUnifiedRadixAllocationEviction(CustomTestCase): cache.cache_controller = None cache.metrics_collector = None cache.tree_core = MagicMock() + cache.tree_core.full_evictable_size.return_value = 80 capacity = {"available": 30} allocator = MagicMock() @@ -112,6 +113,32 @@ class TestUnifiedRadixAllocationEviction(CustomTestCase): self.assertEqual(result.num_tokens_evicted, 20) self.assertEqual(result.mamba_num_evicted, 1) + def test_allocation_eviction_runs_to_the_target_when_frees_stay_hidden(self): + """An allocator can hide freed pages behind a clamp or page rounding, so + freeing exactly the shortfall may leave its availability target unmet; + an allocation-driven FULL walk must keep going until the target shows.""" + cache, capacity, leaf_count = self._build_cache(collateral_capacity_gain=0) + cache.tree_core.full_evictable_size.return_value = 60 + quota = {} + cache.tree_core.evict_device_start.side_effect = quota.__setitem__ + + def next_node(component_type, tracker): + # The tree yields victims only up to the quota the walk opened with. + if tracker[component_type] >= quota[component_type]: + return None, False + return leaf_count["value"] + 1, True + + cache._evict_device_next_node.side_effect = next_node + + result = cache.evict_for_alloc(EvictParams(num_tokens=20)) + + self.assertEqual(leaf_count["value"], 2) + self.assertEqual(capacity["available"], 50) + self.assertEqual(result.num_tokens_evicted, 40) + cache.tree_core.evict_device_start.assert_called_once_with( + ComponentType.FULL, 60 + ) + def test_explicit_evict_preserves_component_count_semantics(self): cache, _, leaf_count = self._build_cache(collateral_capacity_gain=70)