[Session] Count streaming session KV by owner in pool accounting (#42539)
This commit is contained in:
@@ -2356,8 +2356,6 @@ class Scheduler(
|
||||
swa_tokens_per_layer=self.swa_tokens_per_layer,
|
||||
# Match the allocator and radix counters' logical units.
|
||||
max_total_num_tokens=self.max_total_num_tokens * self.kv_shard_widening,
|
||||
get_last_batch=lambda: self.last_batch,
|
||||
get_running_batch=lambda: self.running_batch,
|
||||
)
|
||||
|
||||
def init_invariant_checker(self) -> None:
|
||||
|
||||
@@ -19,13 +19,14 @@ from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.scheduler_components.pool_stats_observer import (
|
||||
PoolStats,
|
||||
SchedulerPoolStatsObserver,
|
||||
kv_private_swa_tokens,
|
||||
kv_private_tokens,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.observability.scheduler_stage_metrics import (
|
||||
SCHEDULER_STAGE_SANITY_CHECK_CACHE,
|
||||
SchedulerStageMetricsRecorder,
|
||||
@@ -39,6 +40,7 @@ from sglang.srt.utils.common import (
|
||||
from sglang.srt.utils.watchdog import WatchdogRaw
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.managers.scheduler import Scheduler
|
||||
|
||||
|
||||
@@ -65,7 +67,7 @@ class SchedulerInvariantChecker:
|
||||
get_running_batch: Callable
|
||||
scheduler_stage_metrics: SchedulerStageMetricsRecorder
|
||||
# The chunked-prefill request parked between chunks is in neither batch;
|
||||
# its uncached tokens must still be counted.
|
||||
# it still owns its KV row.
|
||||
get_chunked_req: Callable = field(default=lambda: None)
|
||||
count_req_pool_leak_warnings: int = 0
|
||||
count_memory_leak_warnings: int = 0
|
||||
@@ -98,7 +100,7 @@ class SchedulerInvariantChecker:
|
||||
return False, ""
|
||||
if self.is_hybrid_swa:
|
||||
protected = self.tree_cache.full_protected_size()
|
||||
session_held = self.pool_stats_observer.session_held_full_tokens()
|
||||
session_held = self.pool_stats_observer.session_held_tokens()
|
||||
total = ps.full_capacity
|
||||
elif self.is_hybrid_ssm:
|
||||
# `total` is the allocator's `.size`: static for non-unified pools,
|
||||
@@ -269,55 +271,38 @@ class SchedulerInvariantChecker:
|
||||
)
|
||||
return active_leak or int8_leak, active_msg + "\n" + int8_msg
|
||||
|
||||
def _get_total_uncached_sizes(
|
||||
self,
|
||||
) -> Tuple[int, int]:
|
||||
"""Sum uncached tokens for full and SWA pools across all active batches.
|
||||
|
||||
Returns (full_uncached, swa_uncached). For non-SWA models, swa_uncached is 0.
|
||||
|
||||
For full pool: uncached = allocated - cache_protected_len
|
||||
For SWA pool: uncached = allocated - max(cache_protected_len, swa_evicted_seqlen)
|
||||
"""
|
||||
# After decode: running_batch IS last_batch (same object), count once.
|
||||
# After prefill: they differ, both hold uncached tokens.
|
||||
# Use identity (is / is not), not membership or ==: ScheduleBatch's
|
||||
# dataclass __eq__ compares tensor fields and raises on ambiguous bools.
|
||||
def _requests_owning_rows(self) -> List[Req]:
|
||||
last_batch = self.get_last_batch()
|
||||
running_batch = self.get_running_batch()
|
||||
batches = [last_batch]
|
||||
if (
|
||||
running_batch is not None
|
||||
and running_batch is not last_batch
|
||||
and not running_batch.is_empty()
|
||||
):
|
||||
batches.append(running_batch)
|
||||
|
||||
full_uncached = 0
|
||||
swa_uncached = self.tree_cache.swa_transient_size()
|
||||
counted: set[int] = set()
|
||||
reqs = [req for batch in batches for req in batch.reqs]
|
||||
reqs = [] if last_batch is None else list(last_batch.reqs)
|
||||
# After decode running_batch IS last_batch. Compare by identity:
|
||||
# ScheduleBatch's dataclass __eq__ compares tensors and raises.
|
||||
if running_batch is not None and running_batch is not last_batch:
|
||||
reqs += running_batch.reqs
|
||||
chunked_req = self.get_chunked_req()
|
||||
if chunked_req is not None:
|
||||
reqs.append(chunked_req)
|
||||
|
||||
# A request running on a session's record does not own that row.
|
||||
session_kvs = {id(kv) for kv in self.tree_cache.session_records().values()}
|
||||
owners, seen = [], set()
|
||||
for req in reqs:
|
||||
if id(req) in counted:
|
||||
continue
|
||||
counted.add(id(req))
|
||||
if not req.kv.holds_kv:
|
||||
if id(req) in seen:
|
||||
continue
|
||||
seen.add(id(req))
|
||||
if req.kv.holds_kv and id(req.kv) not in session_kvs:
|
||||
owners.append(req)
|
||||
return owners
|
||||
|
||||
allocated_len = req.kv.kv_allocated_len
|
||||
if self.page_size > 1:
|
||||
allocated_len = ceil_align(allocated_len, self.page_size)
|
||||
assert req.kv.cache_protected_len % self.page_size == 0
|
||||
|
||||
full_uncached += allocated_len - req.kv.cache_protected_len
|
||||
def _get_total_uncached_sizes(self) -> Tuple[int, int]:
|
||||
"""(full, swa) footprint of the KV rows requests own; 0 swa off hybrid-SWA."""
|
||||
full_uncached = 0
|
||||
swa_uncached = self.tree_cache.swa_transient_size()
|
||||
for req in self._requests_owning_rows():
|
||||
assert req.kv.cache_protected_len % self.page_size == 0
|
||||
full_uncached += kv_private_tokens(req.kv, self.page_size)
|
||||
if self.is_hybrid_swa:
|
||||
swa_uncached += allocated_len - max(
|
||||
req.kv.cache_protected_len,
|
||||
req.kv.get_evicted_seqlen(ComponentType.SWA),
|
||||
)
|
||||
swa_uncached += kv_private_swa_tokens(req.kv, self.page_size)
|
||||
|
||||
if req.beam_group is not None:
|
||||
full_uncached += req.beam_group.extra_uncached_tokens()
|
||||
@@ -370,34 +355,16 @@ class SchedulerInvariantChecker:
|
||||
rtt = self.req_to_token_pool.req_to_token
|
||||
row_width = rtt.shape[1]
|
||||
|
||||
def _add_owner(req_or_slot, label, rpi, committed, allocated):
|
||||
assert 0 <= committed <= allocated <= row_width
|
||||
owners.append((label, rpi, allocated))
|
||||
def _add_owner(label, kv):
|
||||
assert 0 <= kv.kv_committed_len <= kv.kv_allocated_len <= row_width
|
||||
owners.append((label, kv.req_pool_idx, kv.kv_allocated_len))
|
||||
|
||||
owners: list[tuple[str, Optional[int], int]] = []
|
||||
batch = self.get_last_batch()
|
||||
if batch is not None:
|
||||
for req in batch.reqs:
|
||||
if not req.kv.holds_kv:
|
||||
continue
|
||||
_add_owner(
|
||||
req,
|
||||
f"req {req.rid}",
|
||||
req.kv.req_pool_idx,
|
||||
req.kv.kv_committed_len,
|
||||
req.kv.kv_allocated_len,
|
||||
)
|
||||
sess = getattr(self.tree_cache, "slots", None)
|
||||
if sess:
|
||||
for sid, slot in sess.items():
|
||||
if slot.kv.holds_kv:
|
||||
_add_owner(
|
||||
slot,
|
||||
f"slot {sid[:8]}",
|
||||
slot.kv.req_pool_idx,
|
||||
slot.kv.kv_committed_len,
|
||||
slot.kv.kv_allocated_len,
|
||||
)
|
||||
for req in self._requests_owning_rows():
|
||||
_add_owner(f"req {req.rid}", req.kv)
|
||||
for sid, kv in self.tree_cache.session_records().items():
|
||||
if kv.holds_kv:
|
||||
_add_owner(f"slot {sid[:8]}", kv)
|
||||
|
||||
active = [
|
||||
(label, rpi, al) for label, rpi, al in owners if rpi is not None and al > 0
|
||||
|
||||
@@ -5,15 +5,17 @@ from dataclasses import dataclass
|
||||
from typing import (
|
||||
TYPE_CHECKING,
|
||||
Any,
|
||||
Callable,
|
||||
List,
|
||||
Optional,
|
||||
Tuple,
|
||||
)
|
||||
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.schedule_batch import ReqKvInfo
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
@@ -22,6 +24,28 @@ if TYPE_CHECKING:
|
||||
class SchedulerStats: ... # type: ignore[no-redef]
|
||||
|
||||
|
||||
# A KV record's footprint outside the tree, in whole pages. Each row is summed
|
||||
# once, under its owner: uncached for a request's own, session-held for a session's.
|
||||
|
||||
|
||||
def kv_private_tokens(kv: ReqKvInfo, page_size: int) -> int:
|
||||
return ceil_align(kv.kv_allocated_len, page_size) - kv.cache_protected_len
|
||||
|
||||
|
||||
def kv_private_swa_tokens(kv: ReqKvInfo, page_size: int) -> int:
|
||||
allocated = ceil_align(kv.kv_allocated_len, page_size)
|
||||
return allocated - max(
|
||||
kv.cache_protected_len, kv.get_evicted_seqlen(ComponentType.SWA)
|
||||
)
|
||||
|
||||
|
||||
def kv_mamba_slots(kv: ReqKvInfo) -> int:
|
||||
total = kv.mamba_pool_idx.numel() if kv.holds_mamba else 0
|
||||
if kv.mamba_ping_pong_track_buffer is not None:
|
||||
total += int((kv.mamba_ping_pong_track_buffer != -1).sum().item())
|
||||
return total
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class PoolStats:
|
||||
# For full pools (required)
|
||||
@@ -155,8 +179,6 @@ class SchedulerPoolStatsObserver:
|
||||
full_tokens_per_layer: Any
|
||||
swa_tokens_per_layer: Any
|
||||
max_total_num_tokens: int
|
||||
get_last_batch: Callable
|
||||
get_running_batch: Callable
|
||||
|
||||
def streaming_session_count(self) -> int:
|
||||
return sum(
|
||||
@@ -165,35 +187,25 @@ class SchedulerPoolStatsObserver:
|
||||
if session.streaming
|
||||
)
|
||||
|
||||
def active_pool_idxs(self) -> set:
|
||||
"""Pool idxs currently owned by reqs in last_batch / running_batch.
|
||||
|
||||
Used to decide which session slots' KV is owned by batch reqs
|
||||
(and thus counted via uncached_size, not session_held).
|
||||
"""
|
||||
idxs = set()
|
||||
for batch in [self.get_last_batch(), self.get_running_batch()]:
|
||||
if batch is None or batch.is_empty():
|
||||
continue
|
||||
for req in batch.reqs:
|
||||
if req.kv.holds_kv:
|
||||
idxs.add(req.kv.req_pool_idx)
|
||||
return idxs
|
||||
def _session_kv_rows(self) -> List[ReqKvInfo]:
|
||||
records = self.tree_cache.session_records().values()
|
||||
return [kv for kv in records if kv.holds_kv]
|
||||
|
||||
def session_held_tokens(self) -> int:
|
||||
return self.tree_cache.session_held_tokens(self.active_pool_idxs())
|
||||
|
||||
def session_held_full_tokens(self) -> int:
|
||||
return self.tree_cache.session_held_full_tokens(self.active_pool_idxs())
|
||||
page_size = self.tree_cache.page_size
|
||||
return sum(kv_private_tokens(kv, page_size) for kv in self._session_kv_rows())
|
||||
|
||||
def session_held_swa_tokens(self) -> int:
|
||||
return self.tree_cache.session_held_swa_tokens(self.active_pool_idxs())
|
||||
page_size = self.tree_cache.page_size
|
||||
return sum(
|
||||
kv_private_swa_tokens(kv, page_size) for kv in self._session_kv_rows()
|
||||
)
|
||||
|
||||
def session_held_req_count(self) -> int:
|
||||
return self.tree_cache.session_held_req_count()
|
||||
return len(self._session_kv_rows())
|
||||
|
||||
def session_held_mamba_slots(self) -> int:
|
||||
return self.tree_cache.session_held_mamba_slots(self.active_pool_idxs())
|
||||
return sum(kv_mamba_slots(kv) for kv in self._session_kv_rows())
|
||||
|
||||
def get_pool_stats(self) -> PoolStats:
|
||||
if self.is_hybrid_swa:
|
||||
|
||||
@@ -32,7 +32,7 @@ from sglang.srt.runtime_context import get_observability, get_parallel
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import HiCacheController
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
|
||||
from sglang.srt.mem_cache.buffer_mode.pipeline import BufferModePipeline
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.storage_prefetch import StoragePrefetchRetries
|
||||
@@ -651,20 +651,10 @@ class BasePrefixCache(ABC, PrefixCacheTrait):
|
||||
def release_radix_session(self, session_id: str) -> None:
|
||||
pass
|
||||
|
||||
def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return 0
|
||||
|
||||
def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return 0
|
||||
|
||||
def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return 0
|
||||
|
||||
def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return 0
|
||||
|
||||
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return 0
|
||||
def session_records(self) -> dict[str, ReqKvInfo]:
|
||||
"""The KV records sessions own, by session id. Pool accounting counts them
|
||||
as session-held, including while a request runs on one."""
|
||||
return {}
|
||||
|
||||
def supports_prefix_sharing(self) -> bool:
|
||||
"""Whether a request's prefix stays in the cache for other requests to
|
||||
|
||||
@@ -98,7 +98,7 @@ from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.managers.cache_controller import HiCacheAck
|
||||
from sglang.srt.managers.schedule_batch import Req
|
||||
from sglang.srt.managers.schedule_batch import Req, ReqKvInfo
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.hybrid_cache.hybrid_cache_controller import (
|
||||
PrefetchOperation,
|
||||
@@ -3559,20 +3559,8 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
def release_session(self, session_id: str) -> None:
|
||||
self.session.release_session(session_id)
|
||||
|
||||
def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return self.session.session_held_tokens(active_pool_idxs)
|
||||
|
||||
def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return self.session.session_held_full_tokens(active_pool_idxs)
|
||||
|
||||
def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return self.session.session_held_swa_tokens(active_pool_idxs)
|
||||
|
||||
def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return self.session.session_held_req_count(active_pool_idxs)
|
||||
|
||||
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return self.session.session_held_mamba_slots(active_pool_idxs)
|
||||
def session_records(self) -> dict[str, ReqKvInfo]:
|
||||
return {sid: slot.kv for sid, slot in self.session.slots.items()}
|
||||
|
||||
def evictable_size(self) -> int:
|
||||
return self.tree_core.evictable_size()
|
||||
|
||||
@@ -14,7 +14,6 @@ from sglang.srt.mem_cache.base_prefix_cache import (
|
||||
MatchPrefixParams,
|
||||
MatchResult,
|
||||
)
|
||||
from sglang.srt.mem_cache.unified_cache.component_type import ComponentType
|
||||
from sglang.srt.utils.common import ceil_align, is_npu
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -265,67 +264,6 @@ class StreamingSession:
|
||||
|
||||
self._free_slot_mamba(slot)
|
||||
|
||||
def session_held_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
"""KV tokens held by idle session slots; a slot whose pool idx is in
|
||||
``active_pool_idxs`` is counted via uncached_size instead."""
|
||||
total = 0
|
||||
for slot in self.slots.values():
|
||||
in_batch = (
|
||||
active_pool_idxs is not None
|
||||
and slot.kv.req_pool_idx in active_pool_idxs
|
||||
)
|
||||
if slot.kv.holds_kv and not in_batch:
|
||||
allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size)
|
||||
total += allocated - slot.kv.cache_protected_len
|
||||
return total
|
||||
|
||||
def session_held_full_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
return self.session_held_tokens(active_pool_idxs)
|
||||
|
||||
def session_held_swa_tokens(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
"""Total SWA tokens held by session slots, not tracked by the tree."""
|
||||
total = 0
|
||||
for slot in self.slots.values():
|
||||
in_batch = (
|
||||
active_pool_idxs is not None
|
||||
and slot.kv.req_pool_idx in active_pool_idxs
|
||||
)
|
||||
if slot.kv.holds_kv and not in_batch:
|
||||
allocated = ceil_align(slot.kv.kv_allocated_len, self.cache.page_size)
|
||||
total += allocated - max(
|
||||
slot.kv.cache_protected_len,
|
||||
slot.kv.get_evicted_seqlen(ComponentType.SWA),
|
||||
)
|
||||
return total
|
||||
|
||||
def session_held_req_count(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
"""Number of req pool slots held by session slots."""
|
||||
|
||||
def _owned(s):
|
||||
in_batch = (
|
||||
active_pool_idxs is not None and s.kv.req_pool_idx in active_pool_idxs
|
||||
)
|
||||
return s.kv.holds_kv and not in_batch
|
||||
|
||||
return sum(_owned(s) for s in self.slots.values())
|
||||
|
||||
def session_held_mamba_slots(self, active_pool_idxs: Optional[set] = None) -> int:
|
||||
"""mamba_pool entries held by idle session slots (same exclusion as
|
||||
``session_held_tokens``)."""
|
||||
total = 0
|
||||
for slot in self.slots.values():
|
||||
in_batch = (
|
||||
active_pool_idxs is not None
|
||||
and slot.kv.req_pool_idx in active_pool_idxs
|
||||
)
|
||||
if in_batch:
|
||||
continue
|
||||
if slot.kv.holds_mamba:
|
||||
total += slot.kv.mamba_pool_idx.numel()
|
||||
if slot.kv.mamba_ping_pong_track_buffer is not None:
|
||||
total += int((slot.kv.mamba_ping_pong_track_buffer != -1).sum().item())
|
||||
return total
|
||||
|
||||
def _free_slot_mamba(self, slot: SessionSlot) -> None:
|
||||
"""Return a session slot's mamba pool state to the allocator."""
|
||||
mamba_allocator = getattr(self.cache.req_to_token_pool, "mamba_allocator", None)
|
||||
|
||||
Reference in New Issue
Block a user