diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 45c6404b17..a99d1163c2 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -208,6 +208,16 @@ def _relieve_for_alloc(short_pool, need_tokens: int) -> bool: return need_tokens <= short_pool.available_size() +def _drains_on_urgent_flush(neighbor) -> bool: + """Whether an urgent flush of ``neighbor`` would release its holes; a blocked + neighbor cannot, so its holes earn no credit.""" + return ( + neighbor is not None + and neighbor.lazy_compaction + and not neighbor.moves_blocked() + ) + + def _flush_deferred_free_group( allocator: BaseTokenToKVPoolAllocator, pending_groups: Sequence[Optional[Sequence[torch.Tensor]]], @@ -383,11 +393,12 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self._free_phys_pages: torch.Tensor = torch.empty( 0, dtype=torch.int64, device=device ) - # ONE entry per BATCH, keyed by Event: `cpu_list` drives the Set update - # (no sync); `gpu_tensor` is kept alive so drain cats it without an H2D. + # One entry per event, shared by its pending batches: `cpu_list` drives + # the Set update (no sync); batch tensors stay alive so drain cats them + # without an H2D. self._pending_reuse: Dict[ torch.cuda.Event, - Tuple[List[int], torch.Tensor], + Tuple[List[int], List[torch.Tensor]], ] = {} # CPU mirror of `_pending_reuse` for O(1) membership in the survivor walk. self._pending_reuse_pages_cpu: Set[int] = set() @@ -439,6 +450,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self._avail_memo_tokens: int = 0 self._sched_avail_memo_key: Optional[tuple] = None self._sched_avail_memo_tokens: int = 0 + self._credit_neighbors_memo: Optional[tuple] = None self.clear() @@ -726,10 +738,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): awaits an event -- so the credit is realizable. """ neighbor = self._growth_side_neighbor() - if neighbor is None or not neighbor.lazy_compaction: - return 0 - if neighbor.moves_blocked(): - # A blocked neighbor cannot reclaim holes to satisfy an allocation. + if not _drains_on_urgent_flush(neighbor): return 0 return len(neighbor._free_phys_pages) * neighbor.entry_bytes_per_page @@ -743,14 +752,20 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): return True return False + def _credit_neighbors(self) -> tuple: + """Chain members whose holes `schedulable_available_size` credits. Which + members these are depends only on epoch-tracked state.""" + return (self._growth_side_neighbor(),) + def _schedulable_capacity_key(self) -> tuple: - gates = [self.moves_blocked()] - for direction in ("low_peer", "high_peer"): - neighbor = getattr(self, direction) - while neighbor is not None: - gates.append(neighbor.moves_blocked()) - neighbor = getattr(neighbor, direction) - return self._chain_capacity_epoch(), tuple(gates) + """Chain epoch plus each credited neighbor's drain state: gates flip with + no allocator write, so the epoch alone misses them.""" + epoch = self._chain_capacity_epoch() + m = self._credit_neighbors_memo + if m is None or m[0] != epoch: + m = self._credit_neighbors_memo = (epoch, self._credit_neighbors()) + (neighbor,) = m[1] + return epoch, _drains_on_urgent_flush(neighbor) def schedulable_available_size(self) -> int: """Tokens allocatable after flushing a neighbor, including reclaimable holes. @@ -1444,6 +1459,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): virtuals, record inverse history. Does NOT clear p2v[src] -- callers own vacated-region clearing. Returns the moved virtual page ids. """ + assert not self.moves_blocked(), ( + f"[{self.sub_pool_name}] page copy while a move gate is closed" + ) v_moved = self.physical_to_virtual[src_pages].clone() # read pre-wipe # Expand to PHYSICAL token granularity (the move kernel is @@ -1591,8 +1609,7 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): def _drain_pending_reuse(self, *, urgent: bool) -> None: """Move ready `_pending_reuse` entries back into `_free_phys_pages`. Urgent uses `stream.wait_event` on unfired events -- a stream-side - dependency, not a host block. ONE dict entry per BATCH, keyed by Event; - no watermark / `live_page_count` change. + dependency, not a host block. No watermark / `live_page_count` change. """ self._stats_n_drain_calls += 1 if not self._pending_reuse: @@ -1600,13 +1617,13 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): with record_function("MultiEndedAlloc._drain_pending_reuse"): ready_tensors: List[torch.Tensor] = [] ready_entries: List[Tuple[torch.cuda.Event, List[int]]] = [] - for event, (cpu_list, gpu_tensor) in self._pending_reuse.items(): + for event, (cpu_list, gpu_batches) in self._pending_reuse.items(): if event is None or event.query(): - ready_tensors.append(gpu_tensor) + ready_tensors.extend(gpu_batches) ready_entries.append((event, cpu_list)) elif urgent: torch.cuda.current_stream().wait_event(event) - ready_tensors.append(gpu_tensor) + ready_tensors.extend(gpu_batches) ready_entries.append((event, cpu_list)) for event, cpu_list in ready_entries: @@ -1722,7 +1739,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): if not self.lazy_compaction: return 0 if self.moves_blocked(): - # Holes stay in the free list; the next flush picks them up. + # Holes stay in the free list; the next flush picks them up. Fired + # pending sources rejoin it now, since that copies nothing. + self._drain_pending_reuse(urgent=False) return 0 self._wait_hicache_transfers() self._stats_n_flush_calls += 1 @@ -1887,6 +1906,9 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): """ if not srcs: return + assert not self.moves_blocked(), ( + f"[{self.sub_pool_name}] page copy while a move gate is closed" + ) with record_function("MultiEndedAlloc._commit_move_batch"): src_pages_t = torch.tensor(srcs, dtype=torch.int64, device=self.device) dst_pages_t = torch.tensor(dsts, dtype=torch.int64, device=self.device) @@ -1910,15 +1932,18 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): self.physical_to_virtual[dst_pages_t] = v_moveds_t self.physical_to_virtual.index_fill_(0, src_pages_t, -1) self._inverse_history.append((src_pages_t, dst_pages_t, v_moveds_t)) - # Src disposition -- ONE entry per batch. `src_pages_t` is reused as the - # `_pending_reuse` GPU tensor (no second H2D at drain). + # Src disposition. `src_pages_t` is reused as the `_pending_reuse` GPU + # tensor (no second H2D at drain). event_fired = latest_event is None or latest_event.query() if event_fired: released_fired.append(src_pages_t) else: - srcs_copy: List[int] = list(srcs) # caller mutates `srcs` - self._pending_reuse[latest_event] = (srcs_copy, src_pages_t) - self._pending_reuse_pages_cpu.update(srcs_copy) + cpu_list, gpu_batches = self._pending_reuse.setdefault( + latest_event, ([], []) + ) + cpu_list.extend(srcs) # copies: the caller mutates `srcs` + gpu_batches.append(src_pages_t) + self._pending_reuse_pages_cpu.update(srcs) def flush_for_allocation(self) -> int: """Public urgent flush used by peer allocation-pressure recovery.""" @@ -2203,16 +2228,19 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): # -- availability -- + def _side_neighbor(self, side: str) -> Optional[MultiEndedAllocator]: + """Nearest non-transparent member on ``side``.""" + p = self.low_peer if side == "low" else self.high_peer + while p is not None and p._is_frontier_transparent(): + p = p.low_peer if side == "low" else p.high_peer + return p + def _side_drainable_hole_bytes(self, side: str) -> int: """Realizable gap bytes an urgent flush of the neighbour on ``side`` would release, walking past transparent members like the frontier walk. """ - p = self.low_peer if side == "low" else self.high_peer - while p is not None and p._is_frontier_transparent(): - p = p.low_peer if side == "low" else p.high_peer - if p is None or not p.lazy_compaction: - return 0 - if p.moves_blocked(): + p = self._side_neighbor(side) + if not _drains_on_urgent_flush(p): return 0 return len(p._free_phys_pages) * p.entry_bytes_per_page @@ -2226,6 +2254,18 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): self._side_drainable_hole_bytes("high"), ) + def _credit_neighbors(self) -> tuple: + return (self._side_neighbor("low"), self._side_neighbor("high")) + + def _schedulable_capacity_key(self) -> tuple: + # One drain bit per side: `_available_tokens` credits each side on its own. + epoch = self._chain_capacity_epoch() + m = self._credit_neighbors_memo + if m is None or m[0] != epoch: + m = self._credit_neighbors_memo = (epoch, self._credit_neighbors()) + low, high = m[1] + return epoch, _drains_on_urgent_flush(low), _drains_on_urgent_flush(high) + def _available_tokens(self, extra_gap_bytes: int = 0) -> int: gap_low, gap_high = self._gap_pages() if extra_gap_bytes > 0: @@ -2429,24 +2469,23 @@ class FloatMultiEndedAllocator(MultiEndedAllocator): ``side`` boundary and the region bound on that side, relocating the minimum set of live boundary pages (holes-first destinations, then the far gap). Returns the bytes now open on ``side``; a result < ``min_bytes`` - means the ask is impossible now, and state is then unchanged. + means the ask is impossible now or movement is gated, and state is then + unchanged. Scheduler-phase only; stream safety is owned HERE, not by the caller -- the entry settles the in-flight forward before the first copy. Moves at most min(L_live, G) pages: every live page when the ask exceeds them. """ assert side in ("low", "high"), f"side must be 'low'|'high'; got {side!r}" + epp = self.entry_bytes_per_page + gap_low, gap_high = self._gap_pages() + gap_side_bytes = (gap_low if side == "low" else gap_high) * epp + if self.moves_blocked(): + return gap_side_bytes # Order the copies after the in-flight forward, or they carry pre-write # bytes; one wait covers read AND write (the event is post-forward). self._settle_inflight_forward() - epp = self.entry_bytes_per_page lo, hi = self._region_bounds_pages() - gap_low, gap_high = self._gap_pages() - gap_side_bytes = (gap_low if side == "low" else gap_high) * epp - if ( - self.moves_blocked() - or gap_side_bytes >= min_bytes - or self._is_frontier_transparent() - ): + if gap_side_bytes >= min_bytes or self._is_frontier_transparent(): return gap_side_bytes # Capacity: even packing every live page flush against the far side diff --git a/python/sglang/test/unified_allocator_fixtures.py b/python/sglang/test/unified_allocator_fixtures.py new file mode 100644 index 0000000000..71fd492dd2 --- /dev/null +++ b/python/sglang/test/unified_allocator_fixtures.py @@ -0,0 +1,146 @@ +"""Small unified-pool builders and fakes shared by allocator regression tests.""" + +from types import SimpleNamespace + +import torch + +from sglang.srt.mem_cache.unified_memory_pool import ( + MambaSubPoolSpec, + MHASubPoolSpec, + UnifiedKVPool, + init_unified_mamba_swa_pools, + init_unified_swa_pools, +) +from sglang.srt.runtime_context import publish, reset_context +from sglang.srt.server_args import ServerArgs + + +def setup_allocator_context(): + reset_context() + publish(ServerArgs(model_path="dummy", device="cpu"), role="tokenizer") + + +def build_swa_pool(*, page_size=1): + """Lazy full + SWA composite over real CPU pools, 100 pages per side.""" + bundle = init_unified_swa_pools( + device="cpu", + kv_cache_dtype=torch.float16, + head_num=1, + head_dim=8, + v_head_dim=8, + swa_head_num=1, + swa_head_dim=8, + swa_v_head_dim=8, + page_size=page_size, + start_layer=0, + end_layer=2, + swa_attention_layer_ids=[1], + full_attention_layer_ids=[0], + full_max_total_num_tokens=100 * page_size, + swa_max_total_num_tokens=100 * page_size, + enable_memory_saver=False, + need_sort=False, + lazy_compaction=True, + ) + return bundle.token_to_kv_pool_allocator + + +def build_tri_pool(*, page_size=1): + """Lazy full + SWA + mamba composite over real CPU pools; returns the pool + bundle and its allocator.""" + config = SimpleNamespace( + shape=SimpleNamespace(conv=[(3, 8)], temporal=(1, 4, 8)), + dtype=SimpleNamespace(conv=torch.bfloat16, temporal=torch.float32), + layers=[0], + ) + bundle = init_unified_mamba_swa_pools( + device="cpu", + kv_cache_dtype=torch.float16, + head_num=1, + head_dim=8, + v_head_dim=8, + swa_head_num=1, + swa_head_dim=8, + swa_v_head_dim=8, + page_size=page_size, + start_layer=0, + end_layer=2, + swa_attention_layer_ids=[1], + full_attention_layer_ids=[0], + full_max_total_num_tokens=100 * page_size, + swa_max_total_num_tokens=100 * page_size, + enable_memory_saver=False, + need_sort=False, + lazy_compaction=True, + mamba_layer_ids=[0], + mamba2_cache_params=config, + max_mamba_cache_size=8, + model_context_len=256 * page_size, + extra_max_context_len=1, + max_num_reqs=4, + enable_mamba_extra_buffer=False, + disable_overlap_schedule=True, + sliding_window_size=32, + ) + return bundle, bundle.token_to_kv_pool_allocator + + +class FakeKVCache: + """buf[p] == virtual id stored at physical slot p (-1 free); moves copy it.""" + + def __init__(self, max_slots: int): + self.buf = torch.full((max_slots,), -1, dtype=torch.int64) + + def move_kv_cache(self, dst_loc: torch.Tensor, src_loc: torch.Tensor): + self.buf[dst_loc] = self.buf[src_loc].clone() + + +class FakeUnifiedSWAKVPool: + class _SubKV(FakeKVCache): + def __init__(self, max_slots): + super().__init__(max_slots) + self.allocator = None + + def attach_allocator(self, allocator): + self.allocator = allocator + + def __init__(self, shared_pool: UnifiedKVPool): + self.full_kv_pool = self._SubKV(shared_pool.max_slots("full")) + self.swa_kv_pool = self._SubKV(shared_pool.max_slots("swa")) + self._full_allocator = None + self._swa_allocator = None + + def attach_allocators(self, *, full_allocator, swa_allocator): + self._full_allocator = full_allocator + self._swa_allocator = swa_allocator + + +def tri_sub_pool_specs( + full_layer_num=4, swa_layer_num=2, state_layer_num=2, head_num=2, head_dim=4 +): + full = MHASubPoolSpec( + name="full", + layer_num=full_layer_num, + head_num=head_num, + head_dim=head_dim, + store_dtype=torch.float16, + grow_direction="down", + ) + swa = MHASubPoolSpec( + name="swa", + layer_num=swa_layer_num, + head_num=head_num, + head_dim=head_dim, + store_dtype=torch.float16, + grow_direction="float", + ) + mamba = MambaSubPoolSpec( + name="mamba", + layer_num=state_layer_num, + conv_state_shapes=((3, 8),), + conv_dtype=torch.bfloat16, + temporal_state_shape=(0, 0, 0), # Inkling: conv-only, no SSM state + temporal_dtype=torch.float32, + grow_direction="up", + ) + return full, swa, mamba diff --git a/test/registered/unit/mem_cache/test_unified_dynamic_gate_capacity.py b/test/registered/unit/mem_cache/test_unified_dynamic_gate_capacity.py new file mode 100644 index 0000000000..dc9a200a0d --- /dev/null +++ b/test/registered/unit/mem_cache/test_unified_dynamic_gate_capacity.py @@ -0,0 +1,274 @@ +"""Dynamic PD gates invalidate peer-compaction credit without allocator writes.""" + +import contextlib +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.unified_allocator_fixtures import ( + build_swa_pool, + build_tri_pool, + reset_context, + setup_allocator_context, +) + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class TestDynamicGateCapacity(CustomTestCase): + def setUp(self): + self.addCleanup(reset_context) + setup_allocator_context() + + def assert_dynamic_views(self, allocator, state): + members = allocator._move_gate_targets() + immediate = [x.available_size() for x in members] + epochs = [x._chain_capacity_epoch() for x in members] + observed = [] + for opened in (True, False, True, False, True): + state["open"] = opened + self.assertEqual(allocator.verify_byte_accounting(), []) + values = [] + with contextlib.ExitStack() as stack: + # Every page copy goes through a member's `move_kv_cache`. + for kv in {id(x._kvcache): x._kvcache for x in members}.values(): + stack.enter_context( + patch.object( + kv, + "move_kv_cache", + side_effect=AssertionError("query moved"), + ) + ) + for x, original in zip(members, immediate): + actual = x.schedulable_available_size() + expected = x._available_tokens( + extra_gap_bytes=x._peer_drainable_hole_bytes() + ) + self.assertEqual(actual, expected) + self.assertEqual(x.available_size(), original) + values.append(actual) + self.assertEqual(allocator.verify_byte_accounting(), []) + self.assertEqual(epochs, [x._chain_capacity_epoch() for x in members]) + observed.append(values) + self.assertEqual(observed[0], observed[2]) + self.assertEqual(observed[1], observed[3]) + self.assertNotEqual(observed[0], observed[1]) + + def test_two_pool_public_gate_transitions(self): + for ps in (1, 4, 16): + for initially_open in (False, True): + with self.subTest(page_size=ps, initially_open=initially_open): + a = build_swa_pool(page_size=ps) + slots = a.alloc(96 * ps) + a.free_swa(slots[: 40 * ps]) + # Prime ungated memo, then install the public gate. + a.full_attn_allocator.schedulable_available_size() + state = {"open": initially_open} + a.set_disagg_move_gate(lambda: state["open"]) + x = a.full_attn_allocator + self.assertEqual( + x.schedulable_available_size(), + x._available_tokens(x._peer_drainable_hole_bytes()), + ) + self.assert_dynamic_views(a, state) + a.set_disagg_move_gate(lambda: False) + self.assertEqual( + x.schedulable_available_size(), x._available_tokens() + ) + + def test_three_pool_end_float_and_transparent_peer(self): + for ps in (1, 4, 16): + for empty_float in (False, True): + with self.subTest(page_size=ps, empty_float=empty_float): + b, a = build_tri_pool(page_size=ps) + states = a.mamba_allocator.alloc(8) + slots = a.alloc(80 * ps) + a.mamba_allocator.free(states[1:4]) + if empty_float: + a.free_swa(slots) + else: + a.free(slots[10 * ps : 11 * ps]) + state = {"open": True} + a.set_disagg_move_gate(lambda: state["open"]) + self.assert_dynamic_views(a, state) + + def test_real_prefill_queue_gate(self): + from sglang.srt.disaggregation.utils import ( + DisaggregationMode, + unified_memory_disagg_move_gate, + ) + + a = build_swa_pool() + slots = a.alloc(96) + a.free_swa(slots[:40]) + scheduler = SimpleNamespace( + disaggregation_mode=DisaggregationMode.PREFILL, + disagg_prefill_inflight_queue=[], + disagg_prefill_pending_chunk_rids=set(), + ) + a.set_disagg_move_gate(unified_memory_disagg_move_gate(scheduler)) + x = a.full_attn_allocator + opened = x.schedulable_available_size() + scheduler.disagg_prefill_pending_chunk_rids.add("request") + closed = x.schedulable_available_size() + self.assertLess(closed, opened) + self.assertEqual(closed, x._available_tokens()) + scheduler.disagg_prefill_pending_chunk_rids.clear() + self.assertEqual(x.schedulable_available_size(), opened) + self.assertEqual(a.verify_byte_accounting(), []) + + def test_ungated_view_keeps_its_memo(self): + a = build_swa_pool() + slots = a.alloc(96) + a.free_swa(slots[:40]) + x = a.full_attn_allocator + expected = x._available_tokens(x._peer_drainable_hole_bytes()) + with patch.object(x, "_available_tokens", wraps=x._available_tokens) as compute: + self.assertEqual(x.schedulable_available_size(), expected) + self.assertEqual(x.schedulable_available_size(), expected) + self.assertEqual(compute.call_count, 1) + + def test_stable_gate_reuses_memo_and_flips_refresh_it(self): + """Stable PD gates must not force a fresh capacity calculation per query.""" + a = build_swa_pool() + slots = a.alloc(96) + a.free_swa(slots[:40]) + state = {"open": True} + a.set_disagg_move_gate(lambda: state["open"]) + x = a.full_attn_allocator + with patch.object(x, "_available_tokens", wraps=x._available_tokens) as compute: + opened = x.schedulable_available_size() + for _ in range(1999): + self.assertEqual(x.schedulable_available_size(), opened) + self.assertEqual(compute.call_count, 1) + epoch = x._chain_capacity_epoch() + for flip in range(100): + state["open"] = bool(flip % 2) + expected = x._available_tokens(x._peer_drainable_hole_bytes()) + with patch.object( + x, "_available_tokens", wraps=x._available_tokens + ) as compute: + self.assertEqual(x.schedulable_available_size(), expected) + self.assertEqual(x.schedulable_available_size(), expected) + self.assertEqual(compute.call_count, 1) + self.assertEqual(x._chain_capacity_epoch(), epoch) + self.assertEqual(x._capacity_memo_violations(), []) + + def test_gated_memo_diagnostic_detects_untracked_write(self): + """The gate-key cache must retain the missed-epoch diagnostic under PD.""" + a = build_swa_pool() + slots = a.alloc(96) + a.free_swa(slots[:40]) + a.set_disagg_move_gate(lambda: True) + x = a.full_attn_allocator + peer = a.swa_attn_allocator + cached = x.schedulable_available_size() + epoch = x._chain_capacity_epoch() + # Inject a write bypassing the descriptor, the failure this check guards. + peer.__dict__["_free_phys_pages"] = peer._free_phys_pages[:0] + self.assertEqual(x._chain_capacity_epoch(), epoch) + self.assertNotEqual(cached, x._available_tokens(x._peer_drainable_hole_bytes())) + self.assertTrue( + any( + "stale schedulable_available_size" in e + for e in x._capacity_memo_violations() + ) + ) + + def test_float_signature_distinguishes_asymmetric_gates(self): + """Closing opposite sides must not alias a single combined gate boolean.""" + _, a = build_tri_pool() + states = a.mamba_allocator.alloc(8) + slots = a.alloc(80) + a.mamba_allocator.free(states[1:4]) + a.free_full(slots[10:11]) + state = {"low": True, "high": True} + x = a.swa_attn_allocator + x.low_peer.disagg_move_gate = lambda: state["low"] + x.high_peer.disagg_move_gate = lambda: state["high"] + epoch = x._chain_capacity_epoch() + observed = {} + for low, high in ((False, True), (True, False), (False, False), (True, True)): + state.update(low=low, high=high) + expected = x._available_tokens(x._peer_drainable_hole_bytes()) + observed[low, high] = x.schedulable_available_size() + self.assertEqual(observed[low, high], expected) + self.assertEqual(x._chain_capacity_epoch(), epoch) + self.assertEqual(x._capacity_memo_violations(), []) + self.assertNotEqual(observed[False, True], observed[True, False]) + + def test_neighbor_memo_tracks_float_becoming_transparent(self): + """Freeing the float must invalidate the end's cached blocking neighbor.""" + _, a = build_tri_pool() + states = a.mamba_allocator.alloc(8) + slots = a.alloc(80) + a.mamba_allocator.free(states[1:4]) + a.set_disagg_move_gate(lambda: True) + x = a.full_attn_allocator + self.assertIs(x._growth_side_neighbor(), a.swa_attn_allocator) + x.schedulable_available_size() + a.free_swa(slots) + self.assertIs(x._growth_side_neighbor(), a.mamba_allocator) + self.assertEqual( + x.schedulable_available_size(), + x._available_tokens(x._peer_drainable_hole_bytes()), + ) + self.assertEqual(a.verify_byte_accounting(), []) + + def test_independent_pd_and_host_gates_refresh_memo(self): + for ps in (1, 4, 16): + with self.subTest(page_size=ps): + a = build_swa_pool(page_size=ps) + slots = a.alloc(96 * ps) + a.free_swa(slots[: 40 * ps]) + state = {"pd": True, "host": True} + a.set_disagg_move_gate(lambda: state["pd"]) + a.set_host_transfer_move_gate(lambda: state["host"]) + x = a.full_attn_allocator + epoch = x._chain_capacity_epoch() + opened = x.schedulable_available_size() + for pd, host in ( + (True, False), + (False, False), + (False, True), + (True, True), + ): + state.update(pd=pd, host=host) + expected = x._available_tokens(x._peer_drainable_hole_bytes()) + self.assertEqual(x.schedulable_available_size(), expected) + self.assertEqual(expected == opened, pd and host) + with patch.object( + x, + "_available_tokens", + side_effect=AssertionError("stable gate recomputed"), + ): + self.assertEqual(x.schedulable_available_size(), expected) + self.assertEqual(x._chain_capacity_epoch(), epoch) + self.assertEqual(x._capacity_memo_violations(), []) + + def test_float_opposite_gate_owners_do_not_alias(self): + _, a = build_tri_pool() + states = a.mamba_allocator.alloc(8) + slots = a.alloc(80) + a.mamba_allocator.free(states[1:4]) + a.free_full(slots[10:11]) + x = a.swa_attn_allocator + state = {"low": True, "high": True} + x.low_peer.host_transfer_move_gate = lambda: state["low"] + x.high_peer.disagg_move_gate = lambda: state["high"] + epoch = x._chain_capacity_epoch() + observed = {} + for low, high in ((False, True), (True, False), (False, False), (True, True)): + state.update(low=low, high=high) + expected = x._available_tokens(x._peer_drainable_hole_bytes()) + observed[low, high] = x.schedulable_available_size() + self.assertEqual(observed[low, high], expected) + self.assertEqual(x._chain_capacity_epoch(), epoch) + self.assertEqual(x._capacity_memo_violations(), []) + self.assertNotEqual(observed[False, True], observed[True, False]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_float_move_gate.py b/test/registered/unit/mem_cache/test_unified_float_move_gate.py new file mode 100644 index 0000000000..aaae2a7cd2 --- /dev/null +++ b/test/registered/unit/mem_cache/test_unified_float_move_gate.py @@ -0,0 +1,246 @@ +"""FLOAT movement gates protect transfer addresses while allowing hole reuse.""" + +import itertools +import unittest +from unittest.mock import patch + +import torch + +from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( + UnifiedMambaSWATokenToKVPoolAllocator, +) +from sglang.srt.mem_cache.unified_memory_pool import UnifiedKVPool +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.unified_allocator_fixtures import ( + FakeKVCache, + FakeUnifiedSWAKVPool, + tri_sub_pool_specs, +) + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +def build_geometry(ps, ratio, lazy, pattern, gate): + fs, ss, ms = tri_sub_pool_specs( + full_layer_num=ratio[0], + swa_layer_num=ratio[1], + state_layer_num=1, + head_num=1, + head_dim=8, + ) + total = (fs.entry_bytes() + ss.entry_bytes()) * ps * 32 + ms.entry_bytes() * 9 + pool = UnifiedKVPool( + total_bytes=total, + sub_pool_specs=[fs, ss, ms], + device="cpu", + enable_memory_saver=False, + page_size=ps, + ) + kv = FakeUnifiedSWAKVPool(pool) + mk = FakeKVCache(pool.max_slots("mamba")) + a = UnifiedMambaSWATokenToKVPoolAllocator( + unified_buffer=pool, + kvcache=kv, + mamba_kvcache=mk, + device="cpu", + full_max_total_num_tokens=ps * 1000, + swa_max_total_num_tokens=ps * 1000, + page_size=ps, + lazy_compaction=lazy, + ) + states = a.mamba_allocator.alloc(8) + assert states is not None + slots = a.alloc(26 * ps) + assert slots is not None + for member, buf in [ + (a.full_attn_allocator, kv.full_kv_pool.buf), + (a.swa_attn_allocator, kv.swa_kv_pool.buf), + ]: + ids = member.translate_kv_loc(slots) + buf[ids] = slots + mk.buf[a.mamba_allocator.virtual_to_physical[states]] = states + 1000 + victims = { + "none": [], + "paired": [0, 4, 25], + "swa_edges": [0, 25], + "swa_internal": [2, 4, 8], + "empty": list(range(26)), + "mixed": [0, 2, 25], + }[pattern] + ix = ( + torch.cat([slots[i * ps : (i + 1) * ps] for i in victims]) + if victims + else slots[:0] + ) + if pattern == "paired": + a.free(ix) + else: + a.free_swa(ix) + if pattern == "mixed": + a.free_full(slots[2 * ps : 3 * ps]) + a.mamba_allocator.free(states[1:3]) + if pattern == "swa_edges": + a.mamba_allocator.free(states[1:4]) + if gate: + for x in a._flush_targets(): + x.disagg_move_gate = lambda: False + return (a, kv, mk) + + +def snapshot(a): + return [ + ( + x._byte_low_frontier(), + x._byte_high_frontier(), + x._free_phys_pages.tolist(), + x.virtual_to_physical.tolist(), + x.physical_to_virtual.tolist(), + x.live_page_count, + [(id(e), v[0]) for e, v in x._pending_reuse.items()], + ) + for x in a._flush_targets() + ] + + +def payload(a, kv, mk): + result = [] + for member, buf, offset in [ + (a.full_attn_allocator, kv.full_kv_pool.buf, 0), + (a.swa_attn_allocator, kv.swa_kv_pool.buf, 0), + (a.mamba_allocator, mk.buf, 1000), + ]: + ids = torch.where(member.virtual_to_physical >= member.min_page_index)[0] + ids = ids[ids >= member.min_page_index] + tokens = ( + ids[:, None] * member.page_size + torch.arange(member.page_size)[None, :] + ).flatten() + result.append( + (member, buf, tokens.clone(), buf[member.translate_kv_loc(tokens)].clone()) + ) + return result + + +class TestFloatMovementGate(CustomTestCase): + def check_saved(self, a, saved): + for member, buf, ids, values in saved: + self.assertTrue( + torch.all( + member.virtual_to_physical[ids // member.page_size] + >= member.min_page_index + ) + ) + torch.testing.assert_close(buf[member.translate_kv_loc(ids)], values) + self.assertFalse(a.verify_byte_accounting()) + + def test_float_movement_owners_respect_gate(self): + for side, method, gates in itertools.product( + ("low", "high"), + ("make_room", "compact_holes"), + ((False, True), (True, False), (False, False)), + ): + with self.subTest(side=side, method=method, gates=gates): + a, kv, mk = build_geometry(1, (1, 1), True, "swa_internal", False) + sa = a.swa_attn_allocator + saved = payload(a, kv, mk) + sa.disagg_move_gate = lambda: gates[0] + sa.host_transfer_move_gate = lambda: gates[1] + before = snapshot(a) + low, high = sa._gap_pages() + gap = (low if side == "low" else high) * sa.entry_bytes_per_page + with ( + patch.object( + sa, + "_settle_inflight_forward", + wraps=sa._settle_inflight_forward, + ) as settle, + patch.object( + kv.swa_kv_pool, + "move_kv_cache", + wraps=kv.swa_kv_pool.move_kv_cache, + ) as copies, + ): + if method == "make_room": + self.assertEqual( + sa.make_room( + side=side, min_bytes=gap + sa.entry_bytes_per_page + ), + gap, + ) + else: + self.assertEqual(sa.compact_holes(retreat_side=side), 0) + self.assertEqual(settle.call_count, 0) + self.assertEqual(copies.call_count, 0) + self.assertEqual(snapshot(a), before) + sa.disagg_move_gate = lambda: True + sa.host_transfer_move_gate = lambda: True + with patch.object( + kv.swa_kv_pool, "move_kv_cache", wraps=kv.swa_kv_pool.move_kv_cache + ) as copies: + if method == "make_room": + self.assertGreaterEqual( + sa.make_room( + side=side, min_bytes=gap + sa.entry_bytes_per_page + ), + gap + sa.entry_bytes_per_page, + ) + else: + self.assertGreater(sa.compact_holes(retreat_side=side), 0) + self.assertGreater(copies.call_count, 0) + self.check_saved(a, saved) + + def test_gated_token_recovery_and_hole_reuse(self): + for ratio, pattern, demand in ( + ((1, 4), "swa_edges", 5), + ((4, 1), "paired", 7), + ((1, 1), "swa_internal", 7), + ): + with self.subTest(ratio=ratio, pattern=pattern): + a, kv, mk = build_geometry(1, ratio, True, pattern, True) + saved = payload(a, kv, mk) + with patch.object( + kv.swa_kv_pool, "move_kv_cache", wraps=kv.swa_kv_pool.move_kv_cache + ) as copies: + a.ensure_capacity(demand, demand) + got = a.alloc(demand) + self.assertEqual(copies.call_count, 0) + self.check_saved(a, saved) + if got is None: + for member in a._flush_targets(): + member.disagg_move_gate = None + a.ensure_capacity(demand, demand) + self.assertIsNotNone(a.alloc(demand)) + self.check_saved(a, saved) + a, kv, mk = build_geometry(1, (1, 1), True, "paired", True) + saved = payload(a, kv, mk) + with patch.object( + kv.swa_kv_pool, "move_kv_cache", wraps=kv.swa_kv_pool.move_kv_cache + ) as copies: + self.assertIsNotNone(a.alloc(2)) + self.assertEqual(copies.call_count, 0) + self.check_saved(a, saved) + + def test_state_recovery_obeys_float_gate(self): + a, kv, mk = build_geometry(1, (4, 1), True, "none", False) + ma = a.mamba_allocator + sa = a.swa_attn_allocator + saved = payload(a, kv, mk) + demand = ma.available_size() + 1 + sa.disagg_move_gate = lambda: False + with patch.object( + kv.swa_kv_pool, "move_kv_cache", wraps=kv.swa_kv_pool.move_kv_cache + ) as copies: + self.assertIsNone(ma.alloc(demand)) + self.assertEqual(copies.call_count, 0) + self.check_saved(a, saved) + sa.disagg_move_gate = lambda: True + with patch.object( + kv.swa_kv_pool, "move_kv_cache", wraps=kv.swa_kv_pool.move_kv_cache + ) as copies: + self.assertIsNotNone(ma.alloc(demand)) + self.assertGreater(copies.call_count, 0) + self.check_saved(a, saved) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_pending_event_batches.py b/test/registered/unit/mem_cache/test_unified_pending_event_batches.py new file mode 100644 index 0000000000..6e9ad069dc --- /dev/null +++ b/test/registered/unit/mem_cache/test_unified_pending_event_batches.py @@ -0,0 +1,252 @@ +"""Preserve all moved sources until their forward event can release them.""" + +import itertools +import unittest +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase +from sglang.test.unified_allocator_fixtures import ( + build_swa_pool, + build_tri_pool, + reset_context, + setup_allocator_context, +) + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class _Event: + def __init__(self): + self.fired = False + + def query(self): + return self.fired + + +def _views(bundle, allocator, slots, states): + result = [] + if slots is not None: + kv = bundle.token_to_kv_pool + for layer, member in enumerate( + (allocator.full_attn_allocator, allocator.swa_attn_allocator) + ): + for value in (False, True): + buf = kv.get_value_buffer(layer) if value else kv.get_key_buffer(layer) + result.append((member, buf, slots, layer * 2 + int(value))) + if states is not None: + cache = bundle.req_to_token_pool.mamba_pool.mamba_cache + result.extend( + (allocator.mamba_allocator, view[0], states, 4 + index) + for index, view in enumerate([*cache.conv, cache.temporal]) + ) + return result + + +def _payload(ids, buffer, tag, generation=0): + width = buffer[0].numel() + digits = min(width, 8) + ordinal = (generation * 4096 + ids.long()) * 16 + tag + assert ordinal.max() < 16**digits + data = torch.arange(width).expand(len(ids), width).clone() + 32 + data[:, :digits] = ordinal[:, None] // (16 ** torch.arange(digits)) % 16 + encoded = data.to(buffer.dtype) + assert torch.equal(encoded.long(), data) + return encoded.reshape(len(ids), *buffer.shape[1:]) + + +class TestUnifiedPendingEventBatches(CustomTestCase): + def test_multiple_flushes_preserve_event_sources(self): + # Events model reader completion only; these CPU checks make no CUDA + # stream-ordering claim. + for (layout, owner), page_size, mode in itertools.product( + ((2, "full"), (3, "full"), (3, "mamba")), + (1, 4, 16), + ("same", "different", "fired_between"), + ): + with self.subTest( + layout=layout, owner=owner, page_size=page_size, mode=mode + ): + try: + setup_allocator_context() + if layout == 3: + bundle, allocator = build_tri_pool(page_size=page_size) + states = allocator.mamba_allocator.alloc(8) + self.assertIsNotNone(states) + else: + allocator = build_swa_pool(page_size=page_size) + bundle = SimpleNamespace( + token_to_kv_pool=allocator.get_kvcache() + ) + states = None + slots = allocator.alloc(80 * page_size) + self.assertIsNotNone(slots) + retained_slots = slots + retained_states = states + for member, buf, ids, tag in _views( + bundle, allocator, slots, states + ): + buf[member.translate_kv_loc(ids)] = _payload(ids, buf, tag) + member = ( + allocator.full_attn_allocator + if owner == "full" + else allocator.mamba_allocator + ) + first, second = _Event(), _Event() + expected = {} + all_sources = set() + + def assert_pending(): + self.assertEqual(set(member._pending_reuse), set(expected)) + union = set().union(*expected.values()) if expected else set() + self.assertEqual(member._pending_reuse_pages_cpu, union) + free = member._free_phys_pages.tolist() + self.assertTrue(union.isdisjoint(free)) + self.assertEqual(len(free), len(set(free))) + for event, sources in expected.items(): + cpu, device = member._pending_reuse[event] + self.assertEqual(set(cpu), sources) + self.assertEqual(len(cpu), len(sources)) + self.assertEqual(torch.cat(device).tolist(), cpu) + + def assert_payload(): + for current, buf, ids, tag in _views( + bundle, allocator, retained_slots, retained_states + ): + self.assertTrue( + torch.equal( + buf[current.translate_kv_loc(ids)], + _payload(ids, buf, tag), + ), + (current.sub_pool_name, tag), + ) + self.assertEqual(allocator.verify_byte_accounting(), []) + + for index in (0, 1): + event = second if mode == "different" and index else first + if index and mode == "fired_between": + first.fired = True + expected.clear() + member.set_latest_forward_done_event(event) + member.set_inflight_forward(event, None) + if owner == "full": + drop = slots[ + (10 + index * 10) * page_size : (11 + index * 10) + * page_size + ] + allocator.free(drop) + retained_slots = retained_slots[ + ~torch.isin(retained_slots, drop) + ] + else: + drop = states[1 + index * 2 : 2 + index * 2] + member.free(drop) + retained_states = retained_states[ + ~torch.isin(retained_states, drop) + ] + before = member.virtual_to_physical.clone() + self.assertGreater(member._flush(urgent=False), 0) + after = member.virtual_to_physical + moved = ( + (before >= member.min_page_index) + & (after >= member.min_page_index) + & (before != after) + ) + # Derive sources from retained virtual mappings, independently + # of either pending representation being checked. + sources = set(before[moved].tolist()) + self.assertTrue(sources) + all_sources.update(sources) + if not event.fired: + expected.setdefault(event, set()).update(sources) + assert_pending() + assert_payload() + + member._drain_pending_reuse(urgent=False) + assert_pending() + for event in (second, first): + event.fired = True + expected.pop(event, None) + member._drain_pending_reuse(urgent=False) + assert_pending() + assert_payload() + self.assertFalse(member._pending_reuse) + self.assertFalse(member._pending_reuse_pages_cpu) + if mode != "fired_between": + self.assertEqual( + set(member._free_phys_pages.tolist()), all_sources + ) + if owner == "full": + fresh = allocator.alloc(2 * page_size) + self.assertIsNotNone(fresh) + fresh_views = _views(bundle, allocator, fresh, None) + self.assertFalse(torch.isin(fresh, retained_slots).any()) + else: + fresh = member.alloc(2) + self.assertIsNotNone(fresh) + fresh_views = _views(bundle, allocator, None, fresh) + self.assertFalse(torch.isin(fresh, retained_states).any()) + self.assertIsNotNone(fresh) + self.assertEqual( + set( + member.virtual_to_physical[ + fresh // member.pool_page_size + ].tolist() + ), + all_sources, + ) + for current, buf, ids, tag in fresh_views: + buf[current.translate_kv_loc(ids)] = _payload( + ids, buf, tag, generation=1 + ) + assert_payload() + finally: + reset_context() + + def test_gated_flush_still_returns_fired_sources(self): + """A closed move gate stops copies, not the release of fired sources.""" + for page_size in (1, 4): + with self.subTest(page_size=page_size): + try: + setup_allocator_context() + allocator = build_swa_pool(page_size=page_size) + slots = allocator.alloc(80 * page_size) + self.assertIsNotNone(slots) + member = allocator.full_attn_allocator + event = _Event() + member.set_latest_forward_done_event(event) + member.set_inflight_forward(event, None) + allocator.free(slots[10 * page_size : 11 * page_size]) + self.assertGreater(member._flush(urgent=False), 0) + sources = set(member._pending_reuse_pages_cpu) + self.assertTrue(sources) + before = member.available_size() + + member.disagg_move_gate = lambda: False + with patch.object( + member._kvcache, + "move_kv_cache", + side_effect=AssertionError("gated flush copied"), + ): + # Unfired: even an urgent flush leaves them pending. + self.assertEqual(member._flush(urgent=True), 0) + self.assertEqual(member._pending_reuse_pages_cpu, sources) + event.fired = True + self.assertEqual(member.flush_opportunistic(), 0) + + self.assertFalse(member._pending_reuse) + self.assertFalse(member._pending_reuse_pages_cpu) + self.assertLessEqual(sources, set(member._free_phys_pages.tolist())) + self.assertEqual( + member.available_size(), before + len(sources) * page_size + ) + self.assertEqual(allocator.verify_byte_accounting(), []) + finally: + reset_context() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/mem_cache/test_unified_tri_pool.py b/test/registered/unit/mem_cache/test_unified_tri_pool.py index db1f31b066..6bf8c7986c 100644 --- a/test/registered/unit/mem_cache/test_unified_tri_pool.py +++ b/test/registered/unit/mem_cache/test_unified_tri_pool.py @@ -34,14 +34,17 @@ from sglang.srt.mem_cache.allocator.unified_sub_pool import ( from sglang.srt.mem_cache.base_prefix_cache import EvictParams from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_memory_pool import ( - MambaSubPoolSpec, - MHASubPoolSpec, UnifiedKVPool, UnifiedMambaSlotAllocator, init_unified_mamba_swa_pools, ) from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.unified_allocator_fixtures import ( + FakeKVCache, + FakeUnifiedSWAKVPool, + tri_sub_pool_specs, +) # Hermetic convention of this directory's pool tests: plain unittest.TestCase, # only ci_register imported (no heavy sglang.test.test_utils chain). @@ -50,67 +53,6 @@ register_cpu_ci(est_time=8, suite="base-a-test-cpu") _DEV = "cpu" -class _FakeKVCache: - """buf[p] == virtual id stored at physical slot p (-1 free); moves copy it.""" - - def __init__(self, max_slots: int): - self.buf = torch.full((max_slots,), -1, dtype=torch.int64) - - def move_kv_cache(self, dst_loc: torch.Tensor, src_loc: torch.Tensor): - self.buf[dst_loc] = self.buf[src_loc].clone() - - -class _FakeUnifiedSWAKVPool: - class _SubKV(_FakeKVCache): - def __init__(self, max_slots): - super().__init__(max_slots) - self.allocator = None - - def attach_allocator(self, allocator): - self.allocator = allocator - - def __init__(self, shared_pool: UnifiedKVPool): - self.full_kv_pool = self._SubKV(shared_pool.max_slots("full")) - self.swa_kv_pool = self._SubKV(shared_pool.max_slots("swa")) - self._full_allocator = None - self._swa_allocator = None - - def attach_allocators(self, *, full_allocator, swa_allocator): - self._full_allocator = full_allocator - self._swa_allocator = swa_allocator - - -def _tri_specs( - full_layer_num=4, swa_layer_num=2, state_layer_num=2, head_num=2, head_dim=4 -): - full = MHASubPoolSpec( - name="full", - layer_num=full_layer_num, - head_num=head_num, - head_dim=head_dim, - store_dtype=torch.float16, - grow_direction="down", - ) - swa = MHASubPoolSpec( - name="swa", - layer_num=swa_layer_num, - head_num=head_num, - head_dim=head_dim, - store_dtype=torch.float16, - grow_direction="float", - ) - mamba = MambaSubPoolSpec( - name="mamba", - layer_num=state_layer_num, - conv_state_shapes=((3, 8),), - conv_dtype=torch.bfloat16, - temporal_state_shape=(0, 0, 0), # Inkling: conv-only, no SSM state - temporal_dtype=torch.float32, - grow_direction="up", - ) - return full, swa, mamba - - class TestUnifiedTriPool(unittest.TestCase): def _build( self, @@ -120,7 +62,7 @@ class TestUnifiedTriPool(unittest.TestCase): page_size=1, lazy_compaction=False, ): - full, swa, mamba = _tri_specs() + full, swa, mamba = tri_sub_pool_specs() total = ( n_full * full.entry_bytes() + n_swa * swa.entry_bytes() @@ -133,8 +75,8 @@ class TestUnifiedTriPool(unittest.TestCase): enable_memory_saver=False, page_size=page_size, ) - kvcache = _FakeUnifiedSWAKVPool(pool) - mamba_kv = _FakeKVCache(pool.max_slots("mamba")) + kvcache = FakeUnifiedSWAKVPool(pool) + mamba_kv = FakeKVCache(pool.max_slots("mamba")) allocator = UnifiedMambaSWATokenToKVPoolAllocator( unified_buffer=pool, kvcache=kvcache, @@ -496,7 +438,7 @@ class TestTriPagedFreeGroup(unittest.TestCase): """ def _build_paged(self, page_size=4, n_full=64, n_swa=32, n_state=8): - full, swa, mamba = _tri_specs() + full, swa, mamba = tri_sub_pool_specs() total = ( n_full * full.entry_bytes() + n_swa * swa.entry_bytes() @@ -509,8 +451,8 @@ class TestTriPagedFreeGroup(unittest.TestCase): enable_memory_saver=False, page_size=page_size, ) - kvcache = _FakeUnifiedSWAKVPool(pool) - mamba_kv = _FakeKVCache(pool.max_slots("mamba")) + kvcache = FakeUnifiedSWAKVPool(pool) + mamba_kv = FakeKVCache(pool.max_slots("mamba")) allocator = UnifiedMambaSWATokenToKVPoolAllocator( unified_buffer=pool, kvcache=kvcache, @@ -1211,11 +1153,11 @@ class TestJointCapacityIsHonoured(unittest.TestCase): enable_memory_saver=False, page_size=page_size, ) - kvcache = _FakeUnifiedSWAKVPool(pool) + kvcache = FakeUnifiedSWAKVPool(pool) return pool, UnifiedMambaSWATokenToKVPoolAllocator( unified_buffer=pool, kvcache=kvcache, - mamba_kvcache=_FakeKVCache(pool.max_slots("mamba")), + mamba_kvcache=FakeKVCache(pool.max_slots("mamba")), device=_DEV, full_max_total_num_tokens=n_full, swa_max_total_num_tokens=n_swa, @@ -1231,7 +1173,7 @@ class TestJointCapacityIsHonoured(unittest.TestCase): for fl, sl, ml in ((4, 3, 1), (4, 2, 2), (6, 3, 1), (3, 5, 2)): for n_full, n_swa, n_state in ((24, 16, 4), (32, 16, 8), (20, 12, 6)): for lazy in (False, True): - specs = _tri_specs( + specs = tri_sub_pool_specs( full_layer_num=fl, swa_layer_num=sl, state_layer_num=ml, @@ -1277,7 +1219,7 @@ class TestJointCapacityIsHonoured(unittest.TestCase): """Direct form: the joint answer, converted to float pages, must fit inside what `_region_bounds_pages` actually offers.""" for page_size in (1, 4): - specs = _tri_specs( + specs = tri_sub_pool_specs( full_layer_num=4, swa_layer_num=3, state_layer_num=1, @@ -1315,7 +1257,7 @@ class TestFloatRelocationIsOrderedAgainstTheForward(unittest.TestCase): """ def _tri(self, lazy=True): - full, swa, mamba = _tri_specs(head_num=1, head_dim=8) + full, swa, mamba = tri_sub_pool_specs(head_num=1, head_dim=8) total = ( 48 * full.entry_bytes() + 32 * swa.entry_bytes() + 8 * mamba.entry_bytes() ) @@ -1325,11 +1267,11 @@ class TestFloatRelocationIsOrderedAgainstTheForward(unittest.TestCase): device=_DEV, enable_memory_saver=False, ) - kvcache = _FakeUnifiedSWAKVPool(pool) + kvcache = FakeUnifiedSWAKVPool(pool) alloc = UnifiedMambaSWATokenToKVPoolAllocator( unified_buffer=pool, kvcache=kvcache, - mamba_kvcache=_FakeKVCache(pool.max_slots("mamba")), + mamba_kvcache=FakeKVCache(pool.max_slots("mamba")), device=_DEV, full_max_total_num_tokens=48, swa_max_total_num_tokens=32, @@ -1408,7 +1350,7 @@ class TestFloatHoleCreditIsPerSide(unittest.TestCase): """ def _float(self): - full, swa, mamba = _tri_specs(head_num=1, head_dim=8) + full, swa, mamba = tri_sub_pool_specs(head_num=1, head_dim=8) total = ( 48 * full.entry_bytes() + 32 * swa.entry_bytes() + 8 * mamba.entry_bytes() ) @@ -1420,8 +1362,8 @@ class TestFloatHoleCreditIsPerSide(unittest.TestCase): ) alloc = UnifiedMambaSWATokenToKVPoolAllocator( unified_buffer=pool, - kvcache=_FakeUnifiedSWAKVPool(pool), - mamba_kvcache=_FakeKVCache(pool.max_slots("mamba")), + kvcache=FakeUnifiedSWAKVPool(pool), + mamba_kvcache=FakeKVCache(pool.max_slots("mamba")), device=_DEV, full_max_total_num_tokens=48, swa_max_total_num_tokens=32,