[Perf] unified-memory: Allocate free pages from each sub-pool's own end first (#45222)
Co-authored-by: metamergebot <metamergebot@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
metamergebot
ZYHowell
parent
0d7638b035
commit
bab4a808d8
@@ -280,7 +280,8 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
"""Allocator for one sub-pool over a `UnifiedKVPool`.
|
||||
|
||||
``need_sort`` applies to transfer-facing physical ids, not virtual ids.
|
||||
Physical free pages are sorted during compaction.
|
||||
Physical free pages are sorted during compaction, and allocation hands out
|
||||
the free pages nearest the pool's own end first (`take_physical`).
|
||||
"""
|
||||
|
||||
# Capacity-bearing state: any rebind bumps `_capacity_epoch`, invalidating
|
||||
@@ -396,6 +397,10 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
self._free_phys_pages: torch.Tensor = torch.empty(
|
||||
0, dtype=torch.int64, device=device
|
||||
)
|
||||
# The `_free_phys_pages` tensor last known to be ascending. Frees and
|
||||
# pending-reuse returns rebind the list unsorted, so identity tells
|
||||
# `_sorted_free_phys_pages` whether a sort is due.
|
||||
self._free_phys_sorted: Optional[torch.Tensor] = None
|
||||
# One entry per event, shared by its pending batches: `cpu_list` drives
|
||||
# the Set update (no sync); batch tensors stay alive so drain cats them
|
||||
# without an H2D.
|
||||
@@ -836,8 +841,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
return None
|
||||
|
||||
if n_drain > 0:
|
||||
drained_t = self._free_phys_pages[:n_drain]
|
||||
self._free_phys_pages = self._free_phys_pages[n_drain:]
|
||||
drained_t = self._take_free_pages_from_end(
|
||||
n_drain, "low" if self.grow_direction == "up" else "high"
|
||||
)
|
||||
else:
|
||||
drained_t = None
|
||||
|
||||
@@ -943,6 +949,39 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
with record_function("MultiEndedAlloc.take_physical_pages"):
|
||||
return self.take_physical(num_pages * self.page_size)
|
||||
|
||||
def _sorted_free_phys_pages(self) -> torch.Tensor:
|
||||
"""`_free_phys_pages` in ascending order. Sorts on the device, and only
|
||||
when the list was rebound since it was last known sorted; there is no host
|
||||
sync, because the length is tensor metadata.
|
||||
"""
|
||||
free = self._free_phys_pages
|
||||
if free is not self._free_phys_sorted:
|
||||
if free.numel() > 1:
|
||||
free, _ = torch.sort(free)
|
||||
self._free_phys_pages = free
|
||||
self._free_phys_sorted = free
|
||||
return free
|
||||
|
||||
def _take_free_pages_from_end(self, n: int, end: str) -> torch.Tensor:
|
||||
"""Remove and return ``n`` free pages from the ``end`` ("low" or "high")
|
||||
of the ascending free list; the rest stays ascending. Taking the whole
|
||||
list needs no sort. ``0 < n <= len(_free_phys_pages)``.
|
||||
"""
|
||||
n_free = int(self._free_phys_pages.shape[0])
|
||||
assert 0 < n <= n_free, f"take {n} of {n_free} free pages"
|
||||
if n == n_free:
|
||||
taken = self._free_phys_pages
|
||||
rest = taken[:0]
|
||||
else:
|
||||
free = self._sorted_free_phys_pages()
|
||||
if end == "low":
|
||||
taken, rest = free[:n], free[n:]
|
||||
else:
|
||||
taken, rest = free[n_free - n :], free[: n_free - n]
|
||||
self._free_phys_pages = rest
|
||||
self._free_phys_sorted = rest
|
||||
return taken
|
||||
|
||||
def bind(self, virtual_ids: torch.Tensor, physical_ids: torch.Tensor) -> None:
|
||||
"""Bind page-granular virtual ids to physical ids."""
|
||||
with record_function("MultiEndedAlloc.bind"):
|
||||
@@ -2189,6 +2228,12 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
|
||||
# boundaries are live and the deferred absorb can skip its D2H.
|
||||
_holes_dirty: bool = False
|
||||
|
||||
# A middle pool has no origin of its own, so its requests take holes from the
|
||||
# two ends of the ascending hole list in turn: the whole request from one end,
|
||||
# the next request from the other, whatever their sizes. Only a request that
|
||||
# drains holes and succeeds flips it.
|
||||
_next_hole_end: str = "low"
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
assert not kwargs.get("lazy_compaction", False), (
|
||||
"FloatMultiEndedAllocator is holes-first; the lazy event pipeline "
|
||||
@@ -2361,8 +2406,10 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
|
||||
)
|
||||
|
||||
if n_drain > 0:
|
||||
drained = self._free_phys_pages[:n_drain].clone()
|
||||
self._free_phys_pages = self._free_phys_pages[n_drain:]
|
||||
drained = self._take_free_pages_from_end(
|
||||
n_drain, self._next_hole_end
|
||||
).clone()
|
||||
self._next_hole_end = "high" if self._next_hole_end == "low" else "low"
|
||||
else:
|
||||
drained = None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user