From bab4a808d81fd47a039889a3f71514679d799136 Mon Sep 17 00:00:00 2001 From: metamergebot Date: Tue, 20 Oct 2026 13:24:09 -0700 Subject: [PATCH] [Perf] unified-memory: Allocate free pages from each sub-pool's own end first (#45222) Co-authored-by: metamergebot Co-authored-by: ZYHowell --- .../mem_cache/allocator/unified_sub_pool.py | 57 +++++++++++++++++-- ...test_unified_hicache_transfer_lifecycle.py | 5 +- 2 files changed, 56 insertions(+), 6 deletions(-) diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 86c8ac2ac2..90bd1fcc84 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -280,7 +280,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): """Allocator for one sub-pool over a `UnifiedKVPool`. ``need_sort`` applies to transfer-facing physical ids, not virtual ids. - Physical free pages are sorted during compaction. + Physical free pages are sorted during compaction, and allocation hands out + the free pages nearest the pool's own end first (`take_physical`). """ # Capacity-bearing state: any rebind bumps `_capacity_epoch`, invalidating @@ -396,6 +397,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self._free_phys_pages: torch.Tensor = torch.empty( 0, dtype=torch.int64, device=device ) + # The `_free_phys_pages` tensor last known to be ascending. Frees and + # pending-reuse returns rebind the list unsorted, so identity tells + # `_sorted_free_phys_pages` whether a sort is due. + self._free_phys_sorted: Optional[torch.Tensor] = None # One entry per event, shared by its pending batches: `cpu_list` drives # the Set update (no sync); batch tensors stay alive so drain cats them # without an H2D. @@ -836,8 +841,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): return None if n_drain > 0: - drained_t = self._free_phys_pages[:n_drain] - self._free_phys_pages = self._free_phys_pages[n_drain:] + drained_t = self._take_free_pages_from_end( + n_drain, "low" if self.grow_direction == "up" else "high" + ) else: drained_t = None @@ -943,6 +949,39 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): with record_function("MultiEndedAlloc.take_physical_pages"): return self.take_physical(num_pages * self.page_size) + def _sorted_free_phys_pages(self) -> torch.Tensor: + """`_free_phys_pages` in ascending order. Sorts on the device, and only + when the list was rebound since it was last known sorted; there is no host + sync, because the length is tensor metadata. + """ + free = self._free_phys_pages + if free is not self._free_phys_sorted: + if free.numel() > 1: + free, _ = torch.sort(free) + self._free_phys_pages = free + self._free_phys_sorted = free + return free + + def _take_free_pages_from_end(self, n: int, end: str) -> torch.Tensor: + """Remove and return ``n`` free pages from the ``end`` ("low" or "high") + of the ascending free list; the rest stays ascending. Taking the whole + list needs no sort. ``0 < n <= len(_free_phys_pages)``. + """ + n_free = int(self._free_phys_pages.shape[0]) + assert 0 < n <= n_free, f"take {n} of {n_free} free pages" + if n == n_free: + taken = self._free_phys_pages + rest = taken[:0] + else: + free = self._sorted_free_phys_pages() + if end == "low": + taken, rest = free[:n], free[n:] + else: + taken, rest = free[n_free - n :], free[: n_free - n] + self._free_phys_pages = rest + self._free_phys_sorted = rest + return taken + def bind(self, virtual_ids: torch.Tensor, physical_ids: torch.Tensor) -> None: """Bind page-granular virtual ids to physical ids.""" with record_function("MultiEndedAlloc.bind"): @@ -2189,6 +2228,12 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): # boundaries are live and the deferred absorb can skip its D2H. _holes_dirty: bool = False + # A middle pool has no origin of its own, so its requests take holes from the + # two ends of the ascending hole list in turn: the whole request from one end, + # the next request from the other, whatever their sizes. Only a request that + # drains holes and succeeds flips it. + _next_hole_end: str = "low" + def __init__(self, **kwargs): assert not kwargs.get("lazy_compaction", False), ( "FloatMultiEndedAllocator is holes-first; the lazy event pipeline " @@ -2361,8 +2406,10 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): ) if n_drain > 0: - drained = self._free_phys_pages[:n_drain].clone() - self._free_phys_pages = self._free_phys_pages[n_drain:] + drained = self._take_free_pages_from_end( + n_drain, self._next_hole_end + ).clone() + self._next_hole_end = "high" if self._next_hole_end == "low" else "low" else: drained = None diff --git a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py index 0d23dc5ce6..3c3013ea33 100644 --- a/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py +++ b/test/registered/unit/mem_cache/test_unified_hicache_transfer_lifecycle.py @@ -198,12 +198,15 @@ class TestPhysicalReservation(_ConfigCase): # The node being loaded: its FULL rows are resident, SWA is tombstoned. target = allocator.alloc(2 * PAGE) allocator.free_swa(target) - allocator.free(gap) + # The reservation takes the only free SWA rows, at the band's end. reserved = swa.alloc_physical(2 * PAGE) # Stand in for the H2D copy into the reserved rows. rows.swa[reserved] = rows.markers(300, 2 * PAGE) allocator.set_full_to_swa_mapping(target, reserved) + # An interior hole left after the reservation: once the load is queued, + # compaction relocates the reserved rows into it. + allocator.free(gap) moves = [] _record_moves(allocator, moves)