diff --git a/python/sglang/srt/managers/scheduler.py b/python/sglang/srt/managers/scheduler.py index b6a3aa45c9..be1f64ec9a 100644 --- a/python/sglang/srt/managers/scheduler.py +++ b/python/sglang/srt/managers/scheduler.py @@ -2356,8 +2356,6 @@ class Scheduler( swa_tokens_per_layer=self.swa_tokens_per_layer, # Match the allocator and radix counters' logical units. max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening, - get_last_batch=lambda: self.last_batch, - get_running_batch=lambda: self.running_batch, ) def init_invariant_checker(self) -> None: diff --git a/python/sglang/srt/managers/scheduler_components/invariant_checker.py b/python/sglang/srt/managers/scheduler_components/invariant_checker.py index 8c0678c826..d2ae577144 100644 --- a/python/sglang/srt/managers/scheduler_components/invariant_checker.py +++ b/python/sglang/srt/managers/scheduler_components/invariant_checker.py @@ -19,13 +19,14 @@ from sglang.srt.environ import envs from sglang.srt.managers.scheduler_components.pool_stats_observer import ( PoolStats, SchedulerPoolStatsObserver, + kv_private_swa_tokens, + kv_private_tokens, ) from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool -from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.observability.scheduler_stage_metrics import ( SCHEDULER_STAGE_SANITY_CHECK_CACHE, SchedulerStageMetricsRecorder, @@ -39,6 +40,7 @@ from sglang.srt.utils.common import ( from sglang.srt.utils.watchdog import WatchdogRaw if TYPE_CHECKING: + from sglang.srt.managers.schedule_batch import Req from sglang.srt.managers.scheduler import Scheduler @@ -65,7 +67,7 @@ class SchedulerInvariantChecker: get_running_batch: Callable scheduler_stage_metrics: SchedulerStageMetricsRecorder # The chunked-prefill request parked between chunks is in neither batch; - # its uncached tokens must still be counted. + # it still owns its KV row. get_chunked_req: Callable = field(default=lambda: None) count_req_pool_leak_warnings: int = 0 count_memory_leak_warnings: int = 0 @@ -98,7 +100,7 @@ class SchedulerInvariantChecker: return False, "" if self.is_hybrid_swa: protected = self.tree_cache.full_protected_size() - session_held = self.pool_stats_observer.session_held_full_tokens() + session_held = self.pool_stats_observer.session_held_tokens() total = ps.full_capacity elif self.is_hybrid_ssm: # `total` is the allocator's `.size`: static for non-unified pools, @@ -269,55 +271,38 @@ class SchedulerInvariantChecker: ) return active_leak or int8_leak, active_msg + "\n" + int8_msg - def _get_total_uncached_sizes( - self, - ) -> Tuple[int, int]: - """Sum uncached tokens for full and SWA pools across all active batches. - - Returns (full_uncached, swa_uncached). For non-SWA models, swa_uncached is 0. - - For full pool: uncached = allocated - cache_protected_len - For SWA pool: uncached = allocated - max(cache_protected_len, swa_evicted_seqlen) - """ - # After decode: running_batch IS last_batch (same object), count once. - # After prefill: they differ, both hold uncached tokens. - # Use identity (is / is not), not membership or ==: ScheduleBatch's - # dataclass __eq__ compares tensor fields and raises on ambiguous bools. + def _requests_owning_rows(self) -> List[Req]: last_batch = self.get_last_batch() running_batch = self.get_running_batch() - batches = [last_batch] - if ( - running_batch is not None - and running_batch is not last_batch - and not running_batch.is_empty() - ): - batches.append(running_batch) - - full_uncached = 0 - swa_uncached = self.tree_cache.swa_transient_size() - counted: set[int] = set() - reqs = [req for batch in batches for req in batch.reqs] + reqs = [] if last_batch is None else list(last_batch.reqs) + # After decode running_batch IS last_batch. Compare by identity: + # ScheduleBatch's dataclass __eq__ compares tensors and raises. + if running_batch is not None and running_batch is not last_batch: + reqs += running_batch.reqs chunked_req = self.get_chunked_req() if chunked_req is not None: reqs.append(chunked_req) + + # A request running on a session's record does not own that row. + session_kvs = {id(kv) for kv in self.tree_cache.session_records().values()} + owners, seen = [], set() for req in reqs: - if id(req) in counted: - continue - counted.add(id(req)) - if not req.kv.holds_kv: + if id(req) in seen: continue + seen.add(id(req)) + if req.kv.holds_kv and id(req.kv) not in session_kvs: + owners.append(req) + return owners - allocated_len = req.kv.kv_allocated_len - if self.page_size > 1: - allocated_len = ceil_align(allocated_len, self.page_size) - assert req.kv.cache_protected_len % self.page_size == 0 - - full_uncached += allocated_len - req.kv.cache_protected_len + def _get_total_uncached_sizes(self) -> Tuple[int, int]: + """(full, swa) footprint of the KV rows requests own; 0 swa off hybrid-SWA.""" + full_uncached = 0 + swa_uncached = self.tree_cache.swa_transient_size() + for req in self._requests_owning_rows(): + assert req.kv.cache_protected_len % self.page_size == 0 + full_uncached += kv_private_tokens(req.kv, self.page_size) if self.is_hybrid_swa: - swa_uncached += allocated_len - max( - req.kv.cache_protected_len, - req.kv.get_evicted_seqlen(ComponentType.SWA), - ) + swa_uncached += kv_private_swa_tokens(req.kv, self.page_size) if req.beam_group is not None: full_uncached += req.beam_group.extra_uncached_tokens() @@ -370,34 +355,16 @@ class SchedulerInvariantChecker: rtt = self.req_to_token_pool.req_to_token row_width = rtt.shape[1] - def _add_owner(req_or_slot, label, rpi, committed, allocated): - assert 0 <= committed <= allocated <= row_width - owners.append((label, rpi, allocated)) + def _add_owner(label, kv): + assert 0 <= kv.kv_committed_len <= kv.kv_allocated_len <= row_width + owners.append((label, kv.req_pool_idx, kv.kv_allocated_len)) owners: list[tuple[str, Optional[int], int]] = [] - batch = self.get_last_batch() - if batch is not None: - for req in batch.reqs: - if not req.kv.holds_kv: - continue - _add_owner( - req, - f"req {req.rid}", - req.kv.req_pool_idx, - req.kv.kv_committed_len, - req.kv.kv_allocated_len, - ) - sess = getattr(self.tree_cache, "slots", None) - if sess: - for sid, slot in sess.items(): - if slot.kv.holds_kv: - _add_owner( - slot, - f"slot {sid[:8]}", - slot.kv.req_pool_idx, - slot.kv.kv_committed_len, - slot.kv.kv_allocated_len, - ) + for req in self._requests_owning_rows(): + _add_owner(f"req {req.rid}", req.kv) + for sid, kv in self.tree_cache.session_records().items(): + if kv.holds_kv: + _add_owner(f"slot {sid[:8]}", kv) active = [ (label, rpi, al) for label, rpi, al in owners if rpi is not None and al > 0 diff --git a/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py b/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py index 3c6fc964c1..422c50aa5d 100644 --- a/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py +++ b/python/sglang/srt/managers/scheduler_components/pool_stats_observer.py @@ -5,15 +5,17 @@ from dataclasses import dataclass from typing import ( TYPE_CHECKING, Any, - Callable, List, Optional, Tuple, ) from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring +from sglang.srt.mem_cache.unified_cache.component_type import ComponentType +from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: + from sglang.srt.managers.schedule_batch import ReqKvInfo from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool @@ -22,6 +24,28 @@ if TYPE_CHECKING: class SchedulerStats: ... # type: ignore[no-redef] +# A KV record's footprint outside the tree, in whole pages. Each row is summed +# once, under its owner: uncached for a request's own, session-held for a session's. + + +def kv_private_tokens(kv: ReqKvInfo, page_size: int) -> int: + return ceil_align(kv.kv_allocated_len, page_size) - kv.cache_protected_len + + +def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int: + allocated = ceil_align(kv.kv_allocated_len, page_size) + return allocated - max( + kv.cache_protected_len, kv.get_evicted_seqlen(ComponentType.SWA) + ) + + +def kv_mamba_slots(kv: ReqKvInfo) -> int: + total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0 + if kv.mamba_ping_pong_track_buffer is not None: + total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item()) + return total + + @dataclasses.dataclass class PoolStats: # For full pools (required) @@ -155,8 +179,6 @@ class SchedulerPoolStatsObserver: full_tokens_per_layer: Any swa_tokens_per_layer: Any max_total_num_tokens: int - get_last_batch: Callable - get_running_batch: Callable def streaming_session_count(self) -> int: return sum( @@ -165,35 +187,25 @@ class SchedulerPoolStatsObserver: if session.streaming ) - def active_pool_idxs(self) -> set: - """Pool idxs currently owned by reqs in last_batch / running_batch. - - Used to decide which session slots' KV is owned by batch reqs - (and thus counted via uncached_size, not session_held). - """ - idxs = set() - for batch in [self.get_last_batch(), self.get_running_batch()]: - if batch is None or batch.is_empty(): - continue - for req in batch.reqs: - if req.kv.holds_kv: - idxs.add(req.kv.req_pool_idx) - return idxs + def _session_kv_rows(self) -> List[ReqKvInfo]: + records = self.tree_cache.session_records().values() + return [kv for kv in records if kv.holds_kv] def session_held_tokens(self) -> int: - return self.tree_cache.session_held_tokens(self.active_pool_idxs()) - - def session_held_full_tokens(self) -> int: - return self.tree_cache.session_held_full_tokens(self.active_pool_idxs()) + page_size = self.tree_cache.page_size + return sum(kv_private_tokens(kv, page_size) for kv in self._session_kv_rows()) def session_held_swa_tokens(self) -> int: - return self.tree_cache.session_held_swa_tokens(self.active_pool_idxs()) + page_size = self.tree_cache.page_size + return sum( + kv_private_swa_tokens(kv, page_size) for kv in self._session_kv_rows() + ) def session_held_req_count(self) -> int: - return self.tree_cache.session_held_req_count() + return len(self._session_kv_rows()) def session_held_mamba_slots(self) -> int: - return self.tree_cache.session_held_mamba_slots(self.active_pool_idxs()) + return sum(kv_mamba_slots(kv) for kv in self._session_kv_rows()) def get_pool_stats(self) -> PoolStats: if self.is_hybrid_swa: diff --git a/python/sglang/srt/mem_cache/base_prefix_cache.py b/python/sglang/srt/mem_cache/base_prefix_cache.py index eff3b57bda..81f4ceb037 100644 --- a/python/sglang/srt/mem_cache/base_prefix_cache.py +++ b/python/sglang/srt/mem_cache/base_prefix_cache.py @@ -32,7 +32,7 @@ from sglang.srt.runtime_context import get_observability, get_parallel if TYPE_CHECKING: from sglang.srt.managers.cache_controller import HiCacheController - from sglang.srt.managers.schedule_batch import Req + from sglang.srt.managers.schedule_batch import Req, ReqKvInfo from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries @@ -651,20 +651,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait): def release_radix_session(self, session_id: str) -> None: pass - def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return 0 - - def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return 0 - - def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return 0 - - def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int: - return 0 - - def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int: - return 0 + def session_records(self) -> dict[str, ReqKvInfo]: + """The KV records sessions own, by session id. Pool accounting counts them + as session-held, including while a request runs on one.""" + return {} def supports_prefix_sharing(self) -> bool: """Whether a request's prefix stays in the cache for other requests to diff --git a/python/sglang/srt/mem_cache/unified_radix_cache.py b/python/sglang/srt/mem_cache/unified_radix_cache.py index 5b1475d90d..8515fd4890 100644 --- a/python/sglang/srt/mem_cache/unified_radix_cache.py +++ b/python/sglang/srt/mem_cache/unified_radix_cache.py @@ -98,7 +98,7 @@ from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: from sglang.srt.managers.cache_controller import HiCacheAck - from sglang.srt.managers.schedule_batch import Req + from sglang.srt.managers.schedule_batch import Req, ReqKvInfo from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import ( PrefetchOperation, @@ -3559,20 +3559,8 @@ class UnifiedRadixCache(BasePrefixCache): def release_session(self, session_id: str) -> None: self.session.release_session(session_id) - def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return self.session.session_held_tokens(active_pool_idxs) - - def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return self.session.session_held_full_tokens(active_pool_idxs) - - def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return self.session.session_held_swa_tokens(active_pool_idxs) - - def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int: - return self.session.session_held_req_count(active_pool_idxs) - - def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int: - return self.session.session_held_mamba_slots(active_pool_idxs) + def session_records(self) -> dict[str, ReqKvInfo]: + return {sid: slot.kv for sid, slot in self.session.slots.items()} def evictable_size(self) -> int: return self.tree_core.evictable_size() diff --git a/python/sglang/srt/session/streaming_session.py b/python/sglang/srt/session/streaming_session.py index 508682d0ac..2a55ebea91 100644 --- a/python/sglang/srt/session/streaming_session.py +++ b/python/sglang/srt/session/streaming_session.py @@ -14,7 +14,6 @@ from sglang.srt.mem_cache.base_prefix_cache import ( MatchPrefixParams, MatchResult, ) -from sglang.srt.mem_cache.unified_cache.component_type import ComponentType from sglang.srt.utils.common import ceil_align, is_npu if TYPE_CHECKING: @@ -265,67 +264,6 @@ class StreamingSession: self._free_slot_mamba(slot) - def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - """KV tokens held by idle session slots; a slot whose pool idx is in - ``active_pool_idxs`` is counted via uncached_size instead.""" - total = 0 - for slot in self.slots.values(): - in_batch = ( - active_pool_idxs is not None - and slot.kv.req_pool_idx in active_pool_idxs - ) - if slot.kv.holds_kv and not in_batch: - allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size) - total += allocated - slot.kv.cache_protected_len - return total - - def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - return self.session_held_tokens(active_pool_idxs) - - def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int: - """Total SWA tokens held by session slots, not tracked by the tree.""" - total = 0 - for slot in self.slots.values(): - in_batch = ( - active_pool_idxs is not None - and slot.kv.req_pool_idx in active_pool_idxs - ) - if slot.kv.holds_kv and not in_batch: - allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size) - total += allocated - max( - slot.kv.cache_protected_len, - slot.kv.get_evicted_seqlen(ComponentType.SWA), - ) - return total - - def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int: - """Number of req pool slots held by session slots.""" - - def _owned(s): - in_batch = ( - active_pool_idxs is not None and s.kv.req_pool_idx in active_pool_idxs - ) - return s.kv.holds_kv and not in_batch - - return sum(_owned(s) for s in self.slots.values()) - - def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int: - """mamba_pool entries held by idle session slots (same exclusion as - ``session_held_tokens``).""" - total = 0 - for slot in self.slots.values(): - in_batch = ( - active_pool_idxs is not None - and slot.kv.req_pool_idx in active_pool_idxs - ) - if in_batch: - continue - if slot.kv.holds_mamba: - total += slot.kv.mamba_pool_idx.numel() - if slot.kv.mamba_ping_pong_track_buffer is not None: - total += int((slot.kv.mamba_ping_pong_track_buffer != -1).sum().item()) - return total - def _free_slot_mamba(self, slot: SessionSlot) -> None: """Return a session slot's mamba pool state to the allocator.""" mamba_allocator = getattr(self.cache.req_to_token_pool, "mamba_allocator", None) diff --git a/test/registered/unit/layers/test_minicpm_sparse_cache.py b/test/registered/unit/layers/test_minicpm_sparse_cache.py index f15d94d034..640bc47c83 100644 --- a/test/registered/unit/layers/test_minicpm_sparse_cache.py +++ b/test/registered/unit/layers/test_minicpm_sparse_cache.py @@ -227,8 +227,6 @@ def test_hybrid_pool_stats_exclude_reserved_slots(): full_tokens_per_layer=None, swa_tokens_per_layer=None, max_total_num_tokens=42, - get_last_batch=lambda: None, - get_running_batch=lambda: None, ) stats = observer._get_mamba_token_info() diff --git a/test/registered/unit/managers/scheduler_components/test_invariant_checker.py b/test/registered/unit/managers/scheduler_components/test_invariant_checker.py index ff61894a76..4a60d4ee44 100644 --- a/test/registered/unit/managers/scheduler_components/test_invariant_checker.py +++ b/test/registered/unit/managers/scheduler_components/test_invariant_checker.py @@ -28,6 +28,7 @@ from sglang.srt.mem_cache.memory_pool import ReqToTokenPool from sglang.srt.mem_cache.radix_cache import RadixKey from sglang.srt.mem_cache.unified_cache.components import ComponentType from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache +from sglang.srt.session.streaming_session import SessionSlot register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -156,8 +157,6 @@ class TestShardedFullPoolInvariant(CustomTestCase): full_tokens_per_layer=None, swa_tokens_per_layer=None, max_total_num_tokens=self.allocator.size, - get_last_batch=lambda: self.last_batch, - get_running_batch=lambda: self.running_batch, ) return SchedulerInvariantChecker( is_hybrid_swa=False, @@ -288,6 +287,25 @@ class TestShardedFullPoolInvariant(CustomTestCase): with envs.SGLANG_CHECK_KV_PAGE_INVARIANTS.override(False): checker.self_check_during_busy() + def test_session_record_is_counted_once_by_its_owner(self): + checker = self._make_checker() + req = self._active_partial_page(checker) + # The session owns the row: session-held whether the turn is in a batch + # or parked between prefill chunks, never also uncached. + self.cache.session.slots["s"] = SessionSlot(kv=req.kv) + for parked in (False, True): + with self.subTest(parked=parked): + self.chunked_req = req + if parked: + self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True) + self.running_batch = self.last_batch + self.assertEqual(checker._get_total_uncached_sizes(), (0, 0)) + self.assertEqual( + checker.pool_stats_observer.session_held_tokens(), self.PAGE_SIZE + ) + with envs.SGLANG_CHECK_KV_PAGE_INVARIANTS.override(False): + checker.self_check_during_busy() + def test_misaligned_evictable_counter_is_not_rounded_up(self): checker = self._make_checker() self._cache_prefix(self.PAGE_SIZE) diff --git a/test/registered/unit/managers/test_kv_page_invariants.py b/test/registered/unit/managers/test_kv_page_invariants.py index 685eb7f26d..4dc1a53922 100644 --- a/test/registered/unit/managers/test_kv_page_invariants.py +++ b/test/registered/unit/managers/test_kv_page_invariants.py @@ -25,7 +25,8 @@ def _make_checker(page_size=_PAGE_SIZE, row_width=4096, num_reqs=8, free_pages=N free_pages=free_pages, get_all_free_pages=lambda: free_pages, ) - tc = SimpleNamespace(slots={}) + records = {} + tc = SimpleNamespace(session_records=lambda: records) _ps, _rtp, _alloc, _tc = page_size, rtp, alloc, tc class _FakeChecker: @@ -34,6 +35,8 @@ def _make_checker(page_size=_PAGE_SIZE, row_width=4096, num_reqs=8, free_pages=N token_to_kv_pool_allocator = _alloc tree_cache = _tc get_last_batch = lambda self: None + get_running_batch = lambda self: None + get_chunked_req = lambda self: None count_memory_leak_warnings = 0 from sglang.srt.managers.scheduler_components.invariant_checker import ( @@ -41,6 +44,7 @@ def _make_checker(page_size=_PAGE_SIZE, row_width=4096, num_reqs=8, free_pages=N ) _check_kv_page_invariants = _RIC._check_kv_page_invariants + _requests_owning_rows = _RIC._requests_owning_rows return _FakeChecker(), rtt, tc, alloc @@ -80,7 +84,7 @@ class TestKVPageInvariants(CustomTestCase): def test_slot_committed_gt_allocated_raises(self): chk, rtt, tc, alloc = _make_checker() chk.get_last_batch = lambda: None - tc.slots = {"s1": _FakeOwner(0, 145, 144)} + tc.session_records()["s1"] = _FakeOwner(0, 145, 144).kv with self.assertRaises(AssertionError): chk._check_kv_page_invariants() @@ -96,6 +100,22 @@ class TestKVPageInvariants(CustomTestCase): with self.assertRaises(ValueError): chk._check_kv_page_invariants() + def test_requests_outside_last_batch_are_checked(self): + # A request still owns its row from running_batch or parked between chunks. + for where in ("running", "chunked"): + with self.subTest(where=where): + chk, rtt, tc, alloc = _make_checker(free_pages=torch.tensor([5, 6, 7])) + rtt[0, :3] = torch.tensor( + [5 * _PAGE_SIZE, 5 * _PAGE_SIZE + 1, 5 * _PAGE_SIZE + 2] + ) + owner = _FakeOwner(0, 3, 3, rid="a") + if where == "running": + chk.get_running_batch = lambda: SimpleNamespace(reqs=[owner]) + else: + chk.get_chunked_req = lambda: owner + with self.assertRaises(ValueError): + chk._check_kv_page_invariants() + def test_classed_allocator_without_flat_free_list_is_checked(self): chk, rtt, _tc, alloc = _make_checker(page_size=4, row_width=8) alloc.free_pages = None diff --git a/test/registered/unit/mem_cache/test_decode_retraction_backup.py b/test/registered/unit/mem_cache/test_decode_retraction_backup.py index 8732bde51a..c9cc5b6536 100644 --- a/test/registered/unit/mem_cache/test_decode_retraction_backup.py +++ b/test/registered/unit/mem_cache/test_decode_retraction_backup.py @@ -355,8 +355,6 @@ class TestDecodeRetractionBackup(CustomTestCase): full_tokens_per_layer=self.pool_size, swa_tokens_per_layer=None, max_total_num_tokens=self.pool_size, - get_last_batch=lambda: queue.scheduler.last_batch, - get_running_batch=lambda: queue.scheduler.running_batch, ) return queue, queue.kv_manager.kv_args diff --git a/test/registered/unit/mem_cache/test_unified_tri_pool.py b/test/registered/unit/mem_cache/test_unified_tri_pool.py index 61b45b4ded..f8e482c57b 100644 --- a/test/registered/unit/mem_cache/test_unified_tri_pool.py +++ b/test/registered/unit/mem_cache/test_unified_tri_pool.py @@ -24,6 +24,9 @@ from unittest.mock import MagicMock, patch import torch +from sglang.srt.managers.scheduler_components.pool_stats_observer import ( + kv_mamba_slots, +) from sglang.srt.mem_cache.allocator.unified_hybrid_swa import ( UnifiedMambaSWATokenToKVPoolAllocator, ) @@ -1021,10 +1024,9 @@ class TestTriFactorySizing(unittest.TestCase): ) session = StreamingSession(SimpleNamespace(req_to_token_pool=pool)) session.slots = {"test": slot} - self.assertEqual(session.session_held_mamba_slots(), count + 1) - self.assertEqual(session.session_held_mamba_slots({1}), 0) + self.assertEqual(kv_mamba_slots(slot.kv), count + 1) session._free_slot_mamba(slot) - self.assertEqual(session.session_held_mamba_slots(), 0) + self.assertEqual(kv_mamba_slots(slot.kv), 0) self.assertEqual(allocator.available_size(), available) session._free_slot_mamba(slot) self.assertEqual(allocator.available_size(), available)