[Bugfix] Fix unified-memory compaction gates and pending page reuse (#39982)

Co-authored-by: Cheng Wan <cheng.wan@radixark.ai>
This commit is contained in:
SuperSong
2026-10-04 02:48:08 -07:00
committed by GitHub
co-authored by Cheng Wan
parent 1d02a36bb7
commit 6cec8f98ce
6 changed files with 1018 additions and 119 deletions
@@ -208,6 +208,16 @@ def _relieve_for_alloc(short_pool, need_tokens: int) -> bool:
return need_tokens <= short_pool.available_size()
def _drains_on_urgent_flush(neighbor) -> bool:
"""Whether an urgent flush of ``neighbor`` would release its holes; a blocked
neighbor cannot, so its holes earn no credit."""
return (
neighbor is not None
and neighbor.lazy_compaction
and not neighbor.moves_blocked()
)
def _flush_deferred_free_group(
allocator: BaseTokenToKVPoolAllocator,
pending_groups: Sequence[Optional[Sequence[torch.Tensor]]],
@@ -383,11 +393,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._free_phys_pages: torch.Tensor = torch.empty(
0, dtype=torch.int64, device=device
)
# ONE entry per BATCH, keyed by Event: `cpu_list` drives the Set update
# (no sync); `gpu_tensor` is kept alive so drain cats it without an H2D.
# One entry per event, shared by its pending batches: `cpu_list` drives
# the Set update (no sync); batch tensors stay alive so drain cats them
# without an H2D.
self._pending_reuse: Dict[
torch.cuda.Event,
Tuple[List[int], torch.Tensor],
Tuple[List[int], List[torch.Tensor]],
] = {}
# CPU mirror of `_pending_reuse` for O(1) membership in the survivor walk.
self._pending_reuse_pages_cpu: Set[int] = set()
@@ -439,6 +450,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self._avail_memo_tokens: int = 0
self._sched_avail_memo_key: Optional[tuple] = None
self._sched_avail_memo_tokens: int = 0
self._credit_neighbors_memo: Optional[tuple] = None
self.clear()
@@ -726,10 +738,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
awaits an event -- so the credit is realizable.
"""
neighbor = self._growth_side_neighbor()
if neighbor is None or not neighbor.lazy_compaction:
return 0
if neighbor.moves_blocked():
# A blocked neighbor cannot reclaim holes to satisfy an allocation.
if not _drains_on_urgent_flush(neighbor):
return 0
return len(neighbor._free_phys_pages) * neighbor.entry_bytes_per_page
@@ -743,14 +752,20 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
return True
return False
def _credit_neighbors(self) -> tuple:
"""Chain members whose holes `schedulable_available_size` credits. Which
members these are depends only on epoch-tracked state."""
return (self._growth_side_neighbor(),)
def _schedulable_capacity_key(self) -> tuple:
gates = [self.moves_blocked()]
for direction in ("low_peer", "high_peer"):
neighbor = getattr(self, direction)
while neighbor is not None:
gates.append(neighbor.moves_blocked())
neighbor = getattr(neighbor, direction)
return self._chain_capacity_epoch(), tuple(gates)
"""Chain epoch plus each credited neighbor's drain state: gates flip with
no allocator write, so the epoch alone misses them."""
epoch = self._chain_capacity_epoch()
m = self._credit_neighbors_memo
if m is None or m[0] != epoch:
m = self._credit_neighbors_memo = (epoch, self._credit_neighbors())
(neighbor,) = m[1]
return epoch, _drains_on_urgent_flush(neighbor)
def schedulable_available_size(self) -> int:
"""Tokens allocatable after flushing a neighbor, including reclaimable holes.
@@ -1444,6 +1459,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
virtuals, record inverse history. Does NOT clear p2v[src] -- callers own
vacated-region clearing. Returns the moved virtual page ids.
"""
assert not self.moves_blocked(), (
f"[{self.sub_pool_name}] page copy while a move gate is closed"
)
v_moved = self.physical_to_virtual[src_pages].clone() # read pre-wipe
# Expand to PHYSICAL token granularity (the move kernel is
@@ -1591,8 +1609,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
def _drain_pending_reuse(self, *, urgent: bool) -> None:
"""Move ready `_pending_reuse` entries back into `_free_phys_pages`.
Urgent uses `stream.wait_event` on unfired events -- a stream-side
dependency, not a host block. ONE dict entry per BATCH, keyed by Event;
no watermark / `live_page_count` change.
dependency, not a host block. No watermark / `live_page_count` change.
"""
self._stats_n_drain_calls += 1
if not self._pending_reuse:
@@ -1600,13 +1617,13 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
with record_function("MultiEndedAlloc._drain_pending_reuse"):
ready_tensors: List[torch.Tensor] = []
ready_entries: List[Tuple[torch.cuda.Event, List[int]]] = []
for event, (cpu_list, gpu_tensor) in self._pending_reuse.items():
for event, (cpu_list, gpu_batches) in self._pending_reuse.items():
if event is None or event.query():
ready_tensors.append(gpu_tensor)
ready_tensors.extend(gpu_batches)
ready_entries.append((event, cpu_list))
elif urgent:
torch.cuda.current_stream().wait_event(event)
ready_tensors.append(gpu_tensor)
ready_tensors.extend(gpu_batches)
ready_entries.append((event, cpu_list))
for event, cpu_list in ready_entries:
@@ -1722,7 +1739,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
if not self.lazy_compaction:
return 0
if self.moves_blocked():
# Holes stay in the free list; the next flush picks them up.
# Holes stay in the free list; the next flush picks them up. Fired
# pending sources rejoin it now, since that copies nothing.
self._drain_pending_reuse(urgent=False)
return 0
self._wait_hicache_transfers()
self._stats_n_flush_calls += 1
@@ -1887,6 +1906,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
"""
if not srcs:
return
assert not self.moves_blocked(), (
f"[{self.sub_pool_name}] page copy while a move gate is closed"
)
with record_function("MultiEndedAlloc._commit_move_batch"):
src_pages_t = torch.tensor(srcs, dtype=torch.int64, device=self.device)
dst_pages_t = torch.tensor(dsts, dtype=torch.int64, device=self.device)
@@ -1910,15 +1932,18 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
self.physical_to_virtual[dst_pages_t] = v_moveds_t
self.physical_to_virtual.index_fill_(0, src_pages_t, -1)
self._inverse_history.append((src_pages_t, dst_pages_t, v_moveds_t))
# Src disposition -- ONE entry per batch. `src_pages_t` is reused as the
# `_pending_reuse` GPU tensor (no second H2D at drain).
# Src disposition. `src_pages_t` is reused as the `_pending_reuse` GPU
# tensor (no second H2D at drain).
event_fired = latest_event is None or latest_event.query()
if event_fired:
released_fired.append(src_pages_t)
else:
srcs_copy: List[int] = list(srcs) # caller mutates `srcs`
self._pending_reuse[latest_event] = (srcs_copy, src_pages_t)
self._pending_reuse_pages_cpu.update(srcs_copy)
cpu_list, gpu_batches = self._pending_reuse.setdefault(
latest_event, ([], [])
)
cpu_list.extend(srcs) # copies: the caller mutates `srcs`
gpu_batches.append(src_pages_t)
self._pending_reuse_pages_cpu.update(srcs)
def flush_for_allocation(self) -> int:
"""Public urgent flush used by peer allocation-pressure recovery."""
@@ -2203,16 +2228,19 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
# -- availability --
def _side_neighbor(self, side: str) -> Optional[MultiEndedAllocator]:
"""Nearest non-transparent member on ``side``."""
p = self.low_peer if side == "low" else self.high_peer
while p is not None and p._is_frontier_transparent():
p = p.low_peer if side == "low" else p.high_peer
return p
def _side_drainable_hole_bytes(self, side: str) -> int:
"""Realizable gap bytes an urgent flush of the neighbour on ``side``
would release, walking past transparent members like the frontier walk.
"""
p = self.low_peer if side == "low" else self.high_peer
while p is not None and p._is_frontier_transparent():
p = p.low_peer if side == "low" else p.high_peer
if p is None or not p.lazy_compaction:
return 0
if p.moves_blocked():
p = self._side_neighbor(side)
if not _drains_on_urgent_flush(p):
return 0
return len(p._free_phys_pages) * p.entry_bytes_per_page
@@ -2226,6 +2254,18 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
self._side_drainable_hole_bytes("high"),
)
def _credit_neighbors(self) -> tuple:
return (self._side_neighbor("low"), self._side_neighbor("high"))
def _schedulable_capacity_key(self) -> tuple:
# One drain bit per side: `_available_tokens` credits each side on its own.
epoch = self._chain_capacity_epoch()
m = self._credit_neighbors_memo
if m is None or m[0] != epoch:
m = self._credit_neighbors_memo = (epoch, self._credit_neighbors())
low, high = m[1]
return epoch, _drains_on_urgent_flush(low), _drains_on_urgent_flush(high)
def _available_tokens(self, extra_gap_bytes: int = 0) -> int:
gap_low, gap_high = self._gap_pages()
if extra_gap_bytes > 0:
@@ -2429,24 +2469,23 @@ class FloatMultiEndedAllocator(MultiEndedAllocator):
``side`` boundary and the region bound on that side, relocating the
minimum set of live boundary pages (holes-first destinations, then the far
gap). Returns the bytes now open on ``side``; a result < ``min_bytes``
means the ask is impossible now, and state is then unchanged.
means the ask is impossible now or movement is gated, and state is then
unchanged.
Scheduler-phase only; stream safety is owned HERE, not by the caller --
the entry settles the in-flight forward before the first copy. Moves at
most min(L_live, G) pages: every live page when the ask exceeds them.
"""
assert side in ("low", "high"), f"side must be 'low'|'high'; got {side!r}"
epp = self.entry_bytes_per_page
gap_low, gap_high = self._gap_pages()
gap_side_bytes = (gap_low if side == "low" else gap_high) * epp
if self.moves_blocked():
return gap_side_bytes
# Order the copies after the in-flight forward, or they carry pre-write
# bytes; one wait covers read AND write (the event is post-forward).
self._settle_inflight_forward()
epp = self.entry_bytes_per_page
lo, hi = self._region_bounds_pages()
gap_low, gap_high = self._gap_pages()
gap_side_bytes = (gap_low if side == "low" else gap_high) * epp
if (
self.moves_blocked()
or gap_side_bytes >= min_bytes
or self._is_frontier_transparent()
):
if gap_side_bytes >= min_bytes or self._is_frontier_transparent():
return gap_side_bytes
# Capacity: even packing every live page flush against the far side
@@ -0,0 +1,146 @@
"""Small unified-pool builders and fakes shared by allocator regression tests."""
from types import SimpleNamespace
import torch
from sglang.srt.mem_cache.unified_memory_pool import (
MambaSubPoolSpec,
MHASubPoolSpec,
UnifiedKVPool,
init_unified_mamba_swa_pools,
init_unified_swa_pools,
)
from sglang.srt.runtime_context import publish, reset_context
from sglang.srt.server_args import ServerArgs
def setup_allocator_context():
reset_context()
publish(ServerArgs(model_path="dummy", device="cpu"), role="tokenizer")
def build_swa_pool(*, page_size=1):
"""Lazy full + SWA composite over real CPU pools, 100 pages per side."""
bundle = init_unified_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
head_num=1,
head_dim=8,
v_head_dim=8,
swa_head_num=1,
swa_head_dim=8,
swa_v_head_dim=8,
page_size=page_size,
start_layer=0,
end_layer=2,
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
full_max_total_num_tokens=100 * page_size,
swa_max_total_num_tokens=100 * page_size,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
)
return bundle.token_to_kv_pool_allocator
def build_tri_pool(*, page_size=1):
"""Lazy full + SWA + mamba composite over real CPU pools; returns the pool
bundle and its allocator."""
config = SimpleNamespace(
shape=SimpleNamespace(conv=[(3, 8)], temporal=(1, 4, 8)),
dtype=SimpleNamespace(conv=torch.bfloat16, temporal=torch.float32),
layers=[0],
)
bundle = init_unified_mamba_swa_pools(
device="cpu",
kv_cache_dtype=torch.float16,
head_num=1,
head_dim=8,
v_head_dim=8,
swa_head_num=1,
swa_head_dim=8,
swa_v_head_dim=8,
page_size=page_size,
start_layer=0,
end_layer=2,
swa_attention_layer_ids=[1],
full_attention_layer_ids=[0],
full_max_total_num_tokens=100 * page_size,
swa_max_total_num_tokens=100 * page_size,
enable_memory_saver=False,
need_sort=False,
lazy_compaction=True,
mamba_layer_ids=[0],
mamba2_cache_params=config,
max_mamba_cache_size=8,
model_context_len=256 * page_size,
extra_max_context_len=1,
max_num_reqs=4,
enable_mamba_extra_buffer=False,
disable_overlap_schedule=True,
sliding_window_size=32,
)
return bundle, bundle.token_to_kv_pool_allocator
class FakeKVCache:
"""buf[p] == virtual id stored at physical slot p (-1 free); moves copy it."""
def __init__(self, max_slots: int):
self.buf = torch.full((max_slots,), -1, dtype=torch.int64)
def move_kv_cache(self, dst_loc: torch.Tensor, src_loc: torch.Tensor):
self.buf[dst_loc] = self.buf[src_loc].clone()
class FakeUnifiedSWAKVPool:
class _SubKV(FakeKVCache):
def __init__(self, max_slots):
super().__init__(max_slots)
self.allocator = None
def attach_allocator(self, allocator):
self.allocator = allocator
def __init__(self, shared_pool: UnifiedKVPool):
self.full_kv_pool = self._SubKV(shared_pool.max_slots("full"))
self.swa_kv_pool = self._SubKV(shared_pool.max_slots("swa"))
self._full_allocator = None
self._swa_allocator = None
def attach_allocators(self, *, full_allocator, swa_allocator):
self._full_allocator = full_allocator
self._swa_allocator = swa_allocator
def tri_sub_pool_specs(
full_layer_num=4, swa_layer_num=2, state_layer_num=2, head_num=2, head_dim=4
):
full = MHASubPoolSpec(
name="full",
layer_num=full_layer_num,
head_num=head_num,
head_dim=head_dim,
store_dtype=torch.float16,
grow_direction="down",
)
swa = MHASubPoolSpec(
name="swa",
layer_num=swa_layer_num,
head_num=head_num,
head_dim=head_dim,
store_dtype=torch.float16,
grow_direction="float",
)
mamba = MambaSubPoolSpec(
name="mamba",
layer_num=state_layer_num,
conv_state_shapes=((3, 8),),
conv_dtype=torch.bfloat16,
temporal_state_shape=(0, 0, 0), # Inkling: conv-only, no SSM state
temporal_dtype=torch.float32,
grow_direction="up",
)
return full, swa, mamba