[Session] Count streaming session KV by owner in pool accounting (#42539)

This commit is contained in:
Liangsheng Yin
2026-10-04 22:15:57 -07:00
committed by GitHub
parent 456d533e61
commit 503b898540
11 changed files with 127 additions and 198 deletions
-2
View File
@@ -2356,8 +2356,6 @@ class Scheduler(
swa_tokens_per_layer=self.swa_tokens_per_layer,
# Match the allocator and radix counters' logical units.
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
get_last_batch=lambda: self.last_batch,
get_running_batch=lambda: self.running_batch,
)
def init_invariant_checker(self) -> None:
@@ -19,13 +19,14 @@ from sglang.srt.environ import envs
from sglang.srt.managers.scheduler_components.pool_stats_observer import (
PoolStats,
SchedulerPoolStatsObserver,
kv_private_swa_tokens,
kv_private_tokens,
)
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.observability.scheduler_stage_metrics import (
SCHEDULER_STAGE_SANITY_CHECK_CACHE,
SchedulerStageMetricsRecorder,
@@ -39,6 +40,7 @@ from sglang.srt.utils.common import (
from sglang.srt.utils.watchdog import WatchdogRaw
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.managers.scheduler import Scheduler
@@ -65,7 +67,7 @@ class SchedulerInvariantChecker:
get_running_batch: Callable
scheduler_stage_metrics: SchedulerStageMetricsRecorder
# The chunked-prefill request parked between chunks is in neither batch;
# its uncached tokens must still be counted.
# it still owns its KV row.
get_chunked_req: Callable = field(default=lambda: None)
count_req_pool_leak_warnings: int = 0
count_memory_leak_warnings: int = 0
@@ -98,7 +100,7 @@ class SchedulerInvariantChecker:
return False, ""
if self.is_hybrid_swa:
protected = self.tree_cache.full_protected_size()
session_held = self.pool_stats_observer.session_held_full_tokens()
session_held = self.pool_stats_observer.session_held_tokens()
total = ps.full_capacity
elif self.is_hybrid_ssm:
# `total` is the allocator's `.size`: static for non-unified pools,
@@ -269,55 +271,38 @@ class SchedulerInvariantChecker:
)
return active_leak or int8_leak, active_msg + "\n" + int8_msg
def _get_total_uncached_sizes(
self,
) -> Tuple[int, int]:
"""Sum uncached tokens for full and SWA pools across all active batches.
Returns (full_uncached, swa_uncached). For non-SWA models, swa_uncached is 0.
For full pool: uncached = allocated - cache_protected_len
For SWA pool: uncached = allocated - max(cache_protected_len, swa_evicted_seqlen)
"""
# After decode: running_batch IS last_batch (same object), count once.
# After prefill: they differ, both hold uncached tokens.
# Use identity (is / is not), not membership or ==: ScheduleBatch's
# dataclass __eq__ compares tensor fields and raises on ambiguous bools.
def _requests_owning_rows(self) -> List[Req]:
last_batch = self.get_last_batch()
running_batch = self.get_running_batch()
batches = [last_batch]
if (
running_batch is not None
and running_batch is not last_batch
and not running_batch.is_empty()
):
batches.append(running_batch)
full_uncached = 0
swa_uncached = self.tree_cache.swa_transient_size()
counted: set[int] = set()
reqs = [req for batch in batches for req in batch.reqs]
reqs = [] if last_batch is None else list(last_batch.reqs)
# After decode running_batch IS last_batch. Compare by identity:
# ScheduleBatch's dataclass __eq__ compares tensors and raises.
if running_batch is not None and running_batch is not last_batch:
reqs += running_batch.reqs
chunked_req = self.get_chunked_req()
if chunked_req is not None:
reqs.append(chunked_req)
# A request running on a session's record does not own that row.
session_kvs = {id(kv) for kv in self.tree_cache.session_records().values()}
owners, seen = [], set()
for req in reqs:
if id(req) in counted:
continue
counted.add(id(req))
if not req.kv.holds_kv:
if id(req) in seen:
continue
seen.add(id(req))
if req.kv.holds_kv and id(req.kv) not in session_kvs:
owners.append(req)
return owners
allocated_len = req.kv.kv_allocated_len
if self.page_size > 1:
allocated_len = ceil_align(allocated_len, self.page_size)
assert req.kv.cache_protected_len % self.page_size == 0
full_uncached += allocated_len - req.kv.cache_protected_len
def _get_total_uncached_sizes(self) -> Tuple[int, int]:
"""(full, swa) footprint of the KV rows requests own; 0 swa off hybrid-SWA."""
full_uncached = 0
swa_uncached = self.tree_cache.swa_transient_size()
for req in self._requests_owning_rows():
assert req.kv.cache_protected_len % self.page_size == 0
full_uncached += kv_private_tokens(req.kv, self.page_size)
if self.is_hybrid_swa:
swa_uncached += allocated_len - max(
req.kv.cache_protected_len,
req.kv.get_evicted_seqlen(ComponentType.SWA),
)
swa_uncached += kv_private_swa_tokens(req.kv, self.page_size)
if req.beam_group is not None:
full_uncached += req.beam_group.extra_uncached_tokens()
@@ -370,34 +355,16 @@ class SchedulerInvariantChecker:
rtt = self.req_to_token_pool.req_to_token
row_width = rtt.shape[1]
def _add_owner(req_or_slot, label, rpi, committed, allocated):
assert 0 <= committed <= allocated <= row_width
owners.append((label, rpi, allocated))
def _add_owner(label, kv):
assert 0 <= kv.kv_committed_len <= kv.kv_allocated_len <= row_width
owners.append((label, kv.req_pool_idx, kv.kv_allocated_len))
owners: list[tuple[str, Optional[int], int]] = []
batch = self.get_last_batch()
if batch is not None:
for req in batch.reqs:
if not req.kv.holds_kv:
continue
_add_owner(
req,
f"req {req.rid}",
req.kv.req_pool_idx,
req.kv.kv_committed_len,
req.kv.kv_allocated_len,
)
sess = getattr(self.tree_cache, "slots", None)
if sess:
for sid, slot in sess.items():
if slot.kv.holds_kv:
_add_owner(
slot,
f"slot {sid[:8]}",
slot.kv.req_pool_idx,
slot.kv.kv_committed_len,
slot.kv.kv_allocated_len,
)
for req in self._requests_owning_rows():
_add_owner(f"req {req.rid}", req.kv)
for sid, kv in self.tree_cache.session_records().items():
if kv.holds_kv:
_add_owner(f"slot {sid[:8]}", kv)
active = [
(label, rpi, al) for label, rpi, al in owners if rpi is not None and al > 0
@@ -5,15 +5,17 @@ from dataclasses import dataclass
from typing import (
TYPE_CHECKING,
Any,
Callable,
List,
Optional,
Tuple,
)
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.managers.schedule_batch import ReqKvInfo
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
@@ -22,6 +24,28 @@ if TYPE_CHECKING:
class SchedulerStats: ... # type: ignore[no-redef]
# A KV record's footprint outside the tree, in whole pages. Each row is summed
# once, under its owner: uncached for a request's own, session-held for a session's.
def kv_private_tokens(kv: ReqKvInfo, page_size: int) -> int:
return ceil_align(kv.kv_allocated_len, page_size) - kv.cache_protected_len
def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int:
allocated = ceil_align(kv.kv_allocated_len, page_size)
return allocated - max(
kv.cache_protected_len, kv.get_evicted_seqlen(ComponentType.SWA)
)
def kv_mamba_slots(kv: ReqKvInfo) -> int:
total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0
if kv.mamba_ping_pong_track_buffer is not None:
total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item())
return total
@dataclasses.dataclass
class PoolStats:
# For full pools (required)
@@ -155,8 +179,6 @@ class SchedulerPoolStatsObserver:
full_tokens_per_layer: Any
swa_tokens_per_layer: Any
max_total_num_tokens: int
get_last_batch: Callable
get_running_batch: Callable
def streaming_session_count(self) -> int:
return sum(
@@ -165,35 +187,25 @@ class SchedulerPoolStatsObserver:
if session.streaming
)
def active_pool_idxs(self) -> set:
"""Pool idxs currently owned by reqs in last_batch / running_batch.
Used to decide which session slots' KV is owned by batch reqs
(and thus counted via uncached_size, not session_held).
"""
idxs = set()
for batch in [self.get_last_batch(), self.get_running_batch()]:
if batch is None or batch.is_empty():
continue
for req in batch.reqs:
if req.kv.holds_kv:
idxs.add(req.kv.req_pool_idx)
return idxs
def _session_kv_rows(self) -> List[ReqKvInfo]:
records = self.tree_cache.session_records().values()
return [kv for kv in records if kv.holds_kv]
def session_held_tokens(self) -> int:
return self.tree_cache.session_held_tokens(self.active_pool_idxs())
def session_held_full_tokens(self) -> int:
return self.tree_cache.session_held_full_tokens(self.active_pool_idxs())
page_size = self.tree_cache.page_size
return sum(kv_private_tokens(kv, page_size) for kv in self._session_kv_rows())
def session_held_swa_tokens(self) -> int:
return self.tree_cache.session_held_swa_tokens(self.active_pool_idxs())
page_size = self.tree_cache.page_size
return sum(
kv_private_swa_tokens(kv, page_size) for kv in self._session_kv_rows()
)
def session_held_req_count(self) -> int:
return self.tree_cache.session_held_req_count()
return len(self._session_kv_rows())
def session_held_mamba_slots(self) -> int:
return self.tree_cache.session_held_mamba_slots(self.active_pool_idxs())
return sum(kv_mamba_slots(kv) for kv in self._session_kv_rows())
def get_pool_stats(self) -> PoolStats:
if self.is_hybrid_swa:
@@ -32,7 +32,7 @@ from sglang.srt.runtime_context import get_observability, get_parallel
if TYPE_CHECKING:
from sglang.srt.managers.cache_controller import HiCacheController
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
@@ -651,20 +651,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
def release_radix_session(self, session_id: str) -> None:
pass
def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
return 0
def session_records(self) -> dict[str, ReqKvInfo]:
"""The KV records sessions own, by session id. Pool accounting counts them
as session-held, including while a request runs on one."""
return {}
def supports_prefix_sharing(self) -> bool:
"""Whether a request's prefix stays in the cache for other requests to
@@ -98,7 +98,7 @@ from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.managers.cache_controller import HiCacheAck
from sglang.srt.managers.schedule_batch import Req
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
PrefetchOperation,
@@ -3559,20 +3559,8 @@ class UnifiedRadixCache(BasePrefixCache):
def release_session(self, session_id: str) -> None:
self.session.release_session(session_id)
def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return self.session.session_held_tokens(active_pool_idxs)
def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return self.session.session_held_full_tokens(active_pool_idxs)
def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return self.session.session_held_swa_tokens(active_pool_idxs)
def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int:
return self.session.session_held_req_count(active_pool_idxs)
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
return self.session.session_held_mamba_slots(active_pool_idxs)
def session_records(self) -> dict[str, ReqKvInfo]:
return {sid: slot.kv for sid, slot in self.session.slots.items()}
def evictable_size(self) -> int:
return self.tree_core.evictable_size()
@@ -14,7 +14,6 @@ from sglang.srt.mem_cache.base_prefix_cache import (
MatchPrefixParams,
MatchResult,
)
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
from sglang.srt.utils.common import ceil_align, is_npu
if TYPE_CHECKING:
@@ -265,67 +264,6 @@ class StreamingSession:
self._free_slot_mamba(slot)
def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
"""KV tokens held by idle session slots; a slot whose pool idx is in
``active_pool_idxs`` is counted via uncached_size instead."""
total = 0
for slot in self.slots.values():
in_batch = (
active_pool_idxs is not None
and slot.kv.req_pool_idx in active_pool_idxs
)
if slot.kv.holds_kv and not in_batch:
allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size)
total += allocated - slot.kv.cache_protected_len
return total
def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
return self.session_held_tokens(active_pool_idxs)
def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
"""Total SWA tokens held by session slots, not tracked by the tree."""
total = 0
for slot in self.slots.values():
in_batch = (
active_pool_idxs is not None
and slot.kv.req_pool_idx in active_pool_idxs
)
if slot.kv.holds_kv and not in_batch:
allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size)
total += allocated - max(
slot.kv.cache_protected_len,
slot.kv.get_evicted_seqlen(ComponentType.SWA),
)
return total
def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int:
"""Number of req pool slots held by session slots."""
def _owned(s):
in_batch = (
active_pool_idxs is not None and s.kv.req_pool_idx in active_pool_idxs
)
return s.kv.holds_kv and not in_batch
return sum(_owned(s) for s in self.slots.values())
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
"""mamba_pool entries held by idle session slots (same exclusion as
``session_held_tokens``)."""
total = 0
for slot in self.slots.values():
in_batch = (
active_pool_idxs is not None
and slot.kv.req_pool_idx in active_pool_idxs
)
if in_batch:
continue
if slot.kv.holds_mamba:
total += slot.kv.mamba_pool_idx.numel()
if slot.kv.mamba_ping_pong_track_buffer is not None:
total += int((slot.kv.mamba_ping_pong_track_buffer != -1).sum().item())
return total
def _free_slot_mamba(self, slot: SessionSlot) -> None:
"""Return a session slot's mamba pool state to the allocator."""
mamba_allocator = getattr(self.cache.req_to_token_pool, "mamba_allocator", None)