From 671000aee669c5af5bc8b40c70867c311050522e Mon Sep 17 00:00:00 2001 From: metamergebot Date: Fri, 25 Sep 2026 12:17:28 -0700 Subject: [PATCH] [unified-memory] Honor move gates in float relocation and size auto HiCache from host capacity (#41248) Co-authored-by: Yonghao Zhuang --- .../mem_cache/allocator/unified_sub_pool.py | 8 +++- .../sglang/srt/mem_cache/hicache_auto_size.py | 4 +- .../unit/mem_cache/test_hicache_auto_size.py | 41 ++++++++++++++++++- .../mem_cache/test_multi_ended_allocator.py | 35 ++++++++++++++++ .../test_unified_hicache_strided_state.py | 8 ++-- 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 5e924d376f..2594572d72 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -2385,7 +2385,11 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): lo, hi = self._region_bounds_pages() gap_low, gap_high = self._gap_pages() gap_side_bytes = (gap_low if side == "low" else gap_high) * epp - if gap_side_bytes >= min_bytes or self._is_frontier_transparent(): + if ( + self.moves_blocked() + or gap_side_bytes >= min_bytes + or self._is_frontier_transparent() + ): return gap_side_bytes # Capacity: even packing every live page flush against the far side @@ -2560,7 +2564,7 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): OPPOSITE ``retreat_side`` (order-preserving), shrinking the span on ``retreat_side`` by the hole count. Returns pages moved.""" assert retreat_side in ("low", "high") - if self._hole_pages() == 0: + if self.moves_blocked() or self._hole_pages() == 0: return 0 # Settle before the first copy -- see `make_room`. self._settle_inflight_forward() diff --git a/python/sglang/srt/mem_cache/hicache_auto_size.py b/python/sglang/srt/mem_cache/hicache_auto_size.py index 91d613c254..094f8020d4 100644 --- a/python/sglang/srt/mem_cache/hicache_auto_size.py +++ b/python/sglang/srt/mem_cache/hicache_auto_size.py @@ -46,7 +46,9 @@ def _pool_bytes(pool) -> int: return _pool_bytes(pool.full_kv_pool) + _pool_bytes(pool.swa_kv_pool) if isinstance(pool, HybridLinearKVPool): return _pool_bytes(pool.full_kv_pool) - sizes = pool.get_kv_size_bytes() + sizes = getattr(pool, "host_capacity_bytes", None) + if sizes is None: + sizes = pool.get_kv_size_bytes() return sum(sizes) if isinstance(sizes, tuple) else sizes diff --git a/test/registered/unit/mem_cache/test_hicache_auto_size.py b/test/registered/unit/mem_cache/test_hicache_auto_size.py index f0ed470d52..a29029b729 100644 --- a/test/registered/unit/mem_cache/test_hicache_auto_size.py +++ b/test/registered/unit/mem_cache/test_hicache_auto_size.py @@ -26,12 +26,21 @@ class TestHiCacheAutoSize(CustomTestCase): target = Mock( spec=SWAKVPool, size=128, - full_kv_pool=Mock(size=128, get_kv_size_bytes=Mock(return_value=4096)), - swa_kv_pool=Mock(size=32, get_kv_size_bytes=Mock(return_value=1024)), + full_kv_pool=Mock( + size=128, + host_capacity_bytes=None, + get_kv_size_bytes=Mock(return_value=4096), + ), + swa_kv_pool=Mock( + size=32, + host_capacity_bytes=None, + get_kv_size_bytes=Mock(return_value=1024), + ), ) draft_mha = Mock( spec=MHATokenToKVPool, size=16, + host_capacity_bytes=None, get_kv_size_bytes=Mock(return_value=(128, 128)), ) params = CacheInitParams( @@ -53,6 +62,34 @@ class TestHiCacheAutoSize(CustomTestCase): 4096 + 1024 + expected_sidecar_bytes, ) + def test_unified_views_are_sized_from_host_capacity(self): + """Unified sub-pools answer get_kv_size_bytes with zero (UnifiedKVPool + logs the shared buffer once) and publish host_capacity_bytes instead, + the weight the explicit host-size split already uses. An estimate taken + from get_kv_size_bytes is zero device bytes, and the default ratio then + divides by it.""" + target = Mock( + spec=SWAKVPool, + size=128, + full_kv_pool=Mock( + size=128, + host_capacity_bytes=4096, + get_kv_size_bytes=Mock(return_value=(0, 0)), + ), + swa_kv_pool=Mock( + size=32, + host_capacity_bytes=1024, + get_kv_size_bytes=Mock(return_value=(0, 0)), + ), + ) + params = CacheInitParams( + disable=False, + req_to_token_pool=None, + token_to_kv_pool_allocator=Mock(get_kvcache=Mock(return_value=target)), + page_size=2, + ) + self.assertEqual(sizing._estimate_hicache_bytes(params, None), 4096 + 1024) + def test_default_ratio_fits_host_budget_and_pools_book_it(self): """With only --enable-hierarchical-cache the default ratio shrinks to the per-rank budget, pools book one snapshot, and an explicit ratio opts out.""" diff --git a/test/registered/unit/mem_cache/test_multi_ended_allocator.py b/test/registered/unit/mem_cache/test_multi_ended_allocator.py index 1f6b874cb5..2f82ba87d2 100644 --- a/test/registered/unit/mem_cache/test_multi_ended_allocator.py +++ b/test/registered/unit/mem_cache/test_multi_ended_allocator.py @@ -3096,6 +3096,41 @@ class TestFloatMultiEndedAllocator(unittest.TestCase): self.assertGreater(moved, 0) self._check_float_state(fla, kv) + def test_on_demand_movers_honor_move_gate(self): + """`make_room` / `compact_holes` relocate live pages just as `_flush` + does, so a closed move gate must stop them too: the shortfall ladder + reaches `make_room` while a host transfer or RDMA still addresses the + float's physical pages, and an ungated mover copies KV out from under + it. A blocked ask reports what is open now and leaves state untouched. + """ + _, _, fla, _, kv = self._build_tri() + vs = [fla.alloc(2) for _ in range(4)] + for v in vs: + self._stamp(fla, kv, v) + fla.free(vs[1]) # an interior hole `compact_holes` would pack + epp = fla.entry_bytes_per_page + _, gap_high = fla._gap_pages() + ask = (gap_high + 2) * epp + + def snapshot(): + return ( + fla.low_wm_page, + fla.high_wm_page, + fla._hole_pages(), + len(fla._inverse_history), + ) + + fla.host_transfer_move_gate = lambda: False + before = snapshot() + self.assertLess(fla.make_room(side="high", min_bytes=ask), ask) + self.assertEqual(fla.compact_holes(retreat_side="high"), 0) + self.assertEqual(snapshot(), before) + self._check_float_state(fla, kv) + + fla.host_transfer_move_gate = lambda: True + self.assertGreaterEqual(fla.make_room(side="high", min_bytes=ask), ask) + self._check_float_state(fla, kv) + class TestDcpWidening(unittest.TestCase): """`dcp_size > 1`: the alloc surface speaks a widened virtual id space while diff --git a/test/registered/unit/mem_cache/test_unified_hicache_strided_state.py b/test/registered/unit/mem_cache/test_unified_hicache_strided_state.py index 476f65801b..93c661faa2 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_strided_state.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_strided_state.py @@ -53,10 +53,6 @@ class TestStridedStateDetection(CustomTestCase): self.assertTrue(torch.all(dst_view[slot] == 0)) -if __name__ == "__main__": - unittest.main() - - class TestMambaSlotWiringIsShared(CustomTestCase): """Both Mamba factories must install slot allocation and transfer translation. @@ -113,3 +109,7 @@ class TestMambaSlotWiringIsShared(CustomTestCase): and node.func.id == "_wire_mamba_slot_allocator" } self.assertEqual(expected, expected & callers, f"missing: {expected - callers}") + + +if __name__ == "__main__": + unittest.main()