[Perf] unified-memory: Allocate free pages from each sub-pool's own end first (#45222)

Co-authored-by: metamergebot <metamergebot@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-20 13:24:09 -07:00
committed by GitHub
co-authored by metamergebot ZYHowell
parent 0d7638b035
commit bab4a808d8
2 changed files with 56 additions and 6 deletions
@@ -280,7 +280,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
"""Allocator for one sub-pool over a `UnifiedKVPool`.
``need_sort`` applies to transfer-facing physical ids, not virtual ids.
Physical free pages are sorted during compaction.
Physical free pages are sorted during compaction, and allocation hands out
the free pages nearest the pool's own end first (`take_physical`).
"""
# Capacity-bearing state: any rebind bumps `_capacity_epoch`, invalidating
@@ -396,6 +397,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._free_phys_pages: torch.Tensor = torch.empty(
0, dtype=torch.int64, device=device
)
# The `_free_phys_pages` tensor last known to be ascending. Frees and
# pending-reuse returns rebind the list unsorted, so identity tells
# `_sorted_free_phys_pages` whether a sort is due.
self._free_phys_sorted: Optional[torch.Tensor] = None
# One entry per event, shared by its pending batches: `cpu_list` drives
# the Set update (no sync); batch tensors stay alive so drain cats them
# without an H2D.
@@ -836,8 +841,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
return None
if n_drain > 0:
drained_t = self._free_phys_pages[:n_drain]
self._free_phys_pages = self._free_phys_pages[n_drain:]
drained_t = self._take_free_pages_from_end(
n_drain, "low" if self.grow_direction == "up" else "high"
)
else:
drained_t = None
@@ -943,6 +949,39 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
with record_function("MultiEndedAlloc.take_physical_pages"):
return self.take_physical(num_pages * self.page_size)
def _sorted_free_phys_pages(self) -> torch.Tensor:
"""`_free_phys_pages` in ascending order. Sorts on the device, and only
when the list was rebound since it was last known sorted; there is no host
sync, because the length is tensor metadata.
"""
free = self._free_phys_pages
if free is not self._free_phys_sorted:
if free.numel() > 1:
free, _ = torch.sort(free)
self._free_phys_pages = free
self._free_phys_sorted = free
return free
def _take_free_pages_from_end(self, n: int, end: str) -> torch.Tensor:
"""Remove and return ``n`` free pages from the ``end`` ("low" or "high")
of the ascending free list; the rest stays ascending. Taking the whole
list needs no sort. ``0 < n <= len(_free_phys_pages)``.
"""
n_free = int(self._free_phys_pages.shape[0])
assert 0 < n <= n_free, f"take {n} of {n_free} free pages"
if n == n_free:
taken = self._free_phys_pages
rest = taken[:0]
else:
free = self._sorted_free_phys_pages()
if end == "low":
taken, rest = free[:n], free[n:]
else:
taken, rest = free[n_free - n :], free[: n_free - n]
self._free_phys_pages = rest
self._free_phys_sorted = rest
return taken
def bind(self, virtual_ids: torch.Tensor, physical_ids: torch.Tensor) -> None:
"""Bind page-granular virtual ids to physical ids."""
with record_function("MultiEndedAlloc.bind"):
@@ -2189,6 +2228,12 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
# boundaries are live and the deferred absorb can skip its D2H.
_holes_dirty: bool = False
# A middle pool has no origin of its own, so its requests take holes from the
# two ends of the ascending hole list in turn: the whole request from one end,
# the next request from the other, whatever their sizes. Only a request that
# drains holes and succeeds flips it.
_next_hole_end: str = "low"
def __init__(self, **kwargs):
assert not kwargs.get("lazy_compaction", False), (
"FloatMultiEndedAllocator is holes-first; the lazy event pipeline "
@@ -2361,8 +2406,10 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
)
if n_drain > 0:
drained = self._free_phys_pages[:n_drain].clone()
self._free_phys_pages = self._free_phys_pages[n_drain:]
drained = self._take_free_pages_from_end(
n_drain, self._next_hole_end
).clone()
self._next_hole_end = "high" if self._next_hole_end == "low" else "low"
else:
drained = None