[kv-shard 3/4] Enable Control plane (#38468)
This commit is contained in:
@@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import (
|
||||
SchedulerPoolStatsObserver,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
@@ -117,29 +118,60 @@ class SchedulerInvariantChecker:
|
||||
session_held = self.pool_stats_observer.session_held_tokens()
|
||||
total = self.max_total_num_tokens
|
||||
full_evictable_size = ps.full_evictable_size
|
||||
if get_parallel().dcp_enabled and allocator.page_size > 1:
|
||||
# DCP stores logical tokens in widened physical pages. Prefix cache
|
||||
# counters are logical-token based, while the allocator frees whole
|
||||
# physical pages, so round cached tokens up to physical page units.
|
||||
full_evictable_size = (
|
||||
(full_evictable_size + allocator.page_size - 1)
|
||||
// allocator.page_size
|
||||
* allocator.page_size
|
||||
full_available = ps.full_available_size
|
||||
class_watermark_msg = ""
|
||||
kv_shard_size = page_interleave_shard_size(allocator)
|
||||
dcp_page_alloc = (
|
||||
kv_shard_size == 1
|
||||
and get_parallel().dcp_enabled
|
||||
and allocator.page_size > 1
|
||||
)
|
||||
if kv_shard_size > 1:
|
||||
# Conservation needs every free page, not the min-class capacity
|
||||
# floor used for admission. Sharding keeps the tree and allocator
|
||||
# page sizes equal, so cached counts must already be page-aligned.
|
||||
full_available = allocator.aggregate_free_size()
|
||||
class_watermark_msg = (
|
||||
f", class_free_pages={allocator.class_free_page_counts()}"
|
||||
)
|
||||
elif dcp_page_alloc:
|
||||
# Preserve the existing DCP-only accounting adjustment. DCP widens
|
||||
# allocator pages; its scheduler counters still use per-rank units.
|
||||
full_evictable_size = ceil_align(full_evictable_size, allocator.page_size)
|
||||
leak, msg = self._check_pool_invariant(
|
||||
"full",
|
||||
ps.full_available_size,
|
||||
full_available,
|
||||
full_evictable_size,
|
||||
protected,
|
||||
session_held,
|
||||
total,
|
||||
uncached,
|
||||
)
|
||||
if leak and get_parallel().dcp_enabled and allocator.page_size > 1:
|
||||
# Radix/Mamba cache accounting is logical-token based while DCP full
|
||||
# KV allocation is physical-page based. Partial physical pages can
|
||||
# leave a small page-level slack even when all pages are owned by
|
||||
# either the allocator or the prefix cache.
|
||||
msg += class_watermark_msg
|
||||
if kv_shard_size > 1:
|
||||
# Partial active pages are already rounded in
|
||||
# _get_total_uncached_sizes. Rounding cached counts or accepting
|
||||
# arbitrary slack here would hide corruption and lost pages.
|
||||
unaligned = {
|
||||
name: value
|
||||
for name, value in (
|
||||
("available", full_available),
|
||||
("evictable", full_evictable_size),
|
||||
("protected", protected),
|
||||
("session_held", session_held),
|
||||
("uncached", uncached),
|
||||
("total", total),
|
||||
)
|
||||
if value % allocator.page_size != 0
|
||||
}
|
||||
if unaligned:
|
||||
return True, (
|
||||
f"{msg}, unaligned_sharded_counts={unaligned}, "
|
||||
f"page_size={allocator.page_size}"
|
||||
)
|
||||
if leak and dcp_page_alloc:
|
||||
# Legacy DCP counters use different page/capacity units. Keep that
|
||||
# compatibility path separate from exact KV-shard conservation.
|
||||
return False, f"{msg}, dcp_physical_page_slack_allowed=True"
|
||||
return leak, msg
|
||||
|
||||
@@ -380,14 +412,17 @@ class SchedulerInvariantChecker:
|
||||
idx = torch.as_tensor([rpi for _, rpi, _ in active], device=rtt.device)
|
||||
allocs = torch.as_tensor([al for _, _, al in active], device=rtt.device)
|
||||
mask = torch.arange(row_width, device=rtt.device)[None, :] < allocs[:, None]
|
||||
owner_pages = rtt[idx][mask] // self.page_size
|
||||
owner_locs = rtt[idx][mask]
|
||||
|
||||
# Sub-allocators to check: a flat allocator is its own single sub; a
|
||||
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator.
|
||||
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator;
|
||||
# the classed sharding allocator keeps per-class lists instead of a
|
||||
# flat free_pages and exposes them through get_all_free_pages().
|
||||
alloc = self.token_to_kv_pool_allocator
|
||||
sub_allocs = (
|
||||
[alloc]
|
||||
if getattr(alloc, "free_pages", None) is not None
|
||||
or page_interleave_shard_size(alloc) > 1
|
||||
else [
|
||||
sub
|
||||
for n in ("full_attn_allocator", "swa_attn_allocator")
|
||||
@@ -398,6 +433,10 @@ class SchedulerInvariantChecker:
|
||||
if not sub_allocs:
|
||||
return
|
||||
|
||||
# Page ids in the ALLOCATOR's page units (== the physical page for
|
||||
# the classed sharding allocator, whose free lists hold logical page
|
||||
# ids in the same unit).
|
||||
owner_pages = owner_locs // sub_allocs[0].page_size
|
||||
# Check B: every sub-pool's free set has no duplicate pages.
|
||||
for i, sub in enumerate(sub_allocs):
|
||||
free = sub.get_all_free_pages()
|
||||
|
||||
@@ -32,18 +32,25 @@ therefore can be striped without extra compute-time communication:
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keep enough assembled prefix KV for eight long-context requests. The
|
||||
# new-token compute budget is independent: eight 1K extensions can fill an
|
||||
# 8K prefill chunk even when each request has a nearly full-context prefix.
|
||||
_KV_SHARD_SCRATCH_PREFIX_CONTEXTS = 8
|
||||
|
||||
|
||||
class PageShardSpec(msgspec.Struct, frozen=True):
|
||||
"""Everything both sides of any transfer need to reproduce the layout."""
|
||||
@@ -52,7 +59,12 @@ class PageShardSpec(msgspec.Struct, frozen=True):
|
||||
shard_size: int
|
||||
page_size: int # physical page size (kernel-visible)
|
||||
max_prefix_tokens: int # scratch prefix-region capacity, granule-aligned
|
||||
chunk_tokens: int # scratch chunk-region capacity, granule-aligned
|
||||
chunk_tokens: int # scratch chunk-region capacity, physical-page-aligned
|
||||
|
||||
@property
|
||||
def scratch_rows(self) -> int:
|
||||
"""Rows in each slot: assembled prefix, current batch chunk, trash page."""
|
||||
return self.max_prefix_tokens + self.chunk_tokens + self.page_size
|
||||
|
||||
@property
|
||||
def logical_page_size(self) -> int:
|
||||
@@ -111,3 +123,67 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
|
||||
if use_mla_backend:
|
||||
return get_parallel().attn_tp_group
|
||||
return cp_group
|
||||
|
||||
|
||||
def get_kv_shard_group_info(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> Tuple[Optional[int], int]:
|
||||
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
|
||||
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
|
||||
return None, 1
|
||||
group = get_kv_shard_group(kvc.use_mla_backend)
|
||||
if group.world_size <= 1:
|
||||
return None, 1
|
||||
return group.rank_in_group, group.world_size
|
||||
|
||||
|
||||
def _page_shard_row_bytes(kvc: KVCacheConfigurator) -> int:
|
||||
model_config = kvc.model_config
|
||||
kv_size = torch._utils._element_size(kvc.kv_cache_dtype)
|
||||
if kvc.use_mla_backend:
|
||||
return (model_config.kv_lora_rank + model_config.qk_rope_head_dim) * kv_size
|
||||
return (
|
||||
model_config.get_num_kv_heads(get_parallel().attn_tp_size)
|
||||
* (model_config.head_dim + model_config.v_head_dim)
|
||||
* kv_size
|
||||
)
|
||||
|
||||
|
||||
def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
|
||||
"""Provision eight full-context prefixes plus the batch's new-token budget.
|
||||
|
||||
Prefix capacity must not shrink with the number of new tokens: eight
|
||||
long cached prefixes can accompany just 1K new tokens each. Round each
|
||||
context to a shard-group gather span before multiplying, so per-request
|
||||
padding cannot overflow the region even for an unaligned context length.
|
||||
|
||||
This is a fixed eight-context prefix budget, independent of request limits
|
||||
or chunk size. Both scratch slots are charged against HBM before sizing
|
||||
persistent KV; large model contexts therefore reduce persistent capacity.
|
||||
Admission still checks the actual batch's padded prefix and chunk spans.
|
||||
"""
|
||||
shard_rank, shard_size = get_kv_shard_group_info(kvc)
|
||||
if shard_rank is None:
|
||||
return None
|
||||
|
||||
page_size = kvc.page_size
|
||||
granule = shard_size * page_size
|
||||
return PageShardSpec(
|
||||
shard_rank=shard_rank,
|
||||
shard_size=shard_size,
|
||||
page_size=page_size,
|
||||
max_prefix_tokens=_KV_SHARD_SCRATCH_PREFIX_CONTEXTS
|
||||
* ceil_align(kvc.model_config.context_len, granule),
|
||||
# chunked_prefill_size is the TOTAL new-token budget for the batch.
|
||||
chunk_tokens=ceil_align(get_schedule().chunked_prefill_size, page_size),
|
||||
)
|
||||
|
||||
|
||||
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
|
||||
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
|
||||
the KV budget before pool sizing. Use the same spec as the pool builders:
|
||||
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
|
||||
spec = make_page_shard_spec(kvc)
|
||||
if spec is None:
|
||||
return 0
|
||||
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
|
||||
|
||||
@@ -113,9 +113,9 @@ class PageInterleaveKVPoolMixin:
|
||||
assert spec.shard_size == shard_group.world_size
|
||||
assert spec.shard_rank == shard_group.rank_in_group
|
||||
assert spec.page_size == self.page_size
|
||||
# The prefix region must fit N * ceil(prefix_pages / N) pages for any
|
||||
# prefix, i.e. be a multiple of the full-group span; the chunk region
|
||||
# is per-page.
|
||||
# The batch's prefix region is split into equal per-rank gather
|
||||
# blocks, so it must be a multiple of the full-group span. The chunk
|
||||
# region is per-page; admission checks both regions against this spec.
|
||||
assert spec.max_prefix_tokens % spec.logical_page_size == 0
|
||||
assert spec.chunk_tokens % spec.page_size == 0
|
||||
|
||||
@@ -133,7 +133,7 @@ class PageInterleaveKVPoolMixin:
|
||||
self.kv_gather_stream = self.device_module.Stream()
|
||||
|
||||
# Scratch: [prefix | chunk | trash page], double-buffered.
|
||||
scratch_rows = spec.max_prefix_tokens + spec.chunk_tokens + spec.page_size
|
||||
scratch_rows = spec.scratch_rows
|
||||
self._chunk_base = spec.max_prefix_tokens
|
||||
self._trash_base = spec.max_prefix_tokens + spec.chunk_tokens
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
@@ -171,13 +171,23 @@ class PageInterleaveKVPoolMixin:
|
||||
self._write_plan: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
||||
self._translate_cache: Dict[Tuple[int, int], torch.Tensor] = {}
|
||||
|
||||
scratch_bytes = sum(
|
||||
tensor.numel() * tensor.element_size()
|
||||
for slot in self._slots
|
||||
for tensor in slot.tensors.values()
|
||||
)
|
||||
logger.info(
|
||||
"Page-interleave KV sharding enabled: shard_rank=%d shard_size=%d "
|
||||
"page_size=%d scratch_rows=%d x2",
|
||||
"page_size=%d prefix_tokens=%d chunk_tokens=%d scratch_rows=%d x2 "
|
||||
"(%.2f MiB per rank, charged against the KV budget). "
|
||||
"Batch admission is limited by this scratch capacity.",
|
||||
self.shard_rank,
|
||||
self.shard_size,
|
||||
spec.page_size,
|
||||
spec.max_prefix_tokens,
|
||||
spec.chunk_tokens,
|
||||
scratch_rows,
|
||||
scratch_bytes / (1024 * 1024),
|
||||
)
|
||||
|
||||
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
|
||||
@@ -186,6 +196,28 @@ class PageInterleaveKVPoolMixin:
|
||||
"(it writes pool rows directly, bypassing the ownership filter)"
|
||||
)
|
||||
|
||||
def register_layer_transfer_counter(self, layer_transfer_counter):
|
||||
# `None` is the in-tree "disable" signal from the SWA/hybrid wrappers,
|
||||
# so it has to stay a no-op.
|
||||
if layer_transfer_counter is None:
|
||||
super().register_layer_transfer_counter(None)
|
||||
return
|
||||
# Layer-wise KV load-back is unsupported here, and refusing it at
|
||||
# registration is the only safe answer. The gather in `_prefetch_layer`
|
||||
# reads pool rows directly (`_gather_pairs`), so it never passes
|
||||
# through the base getters' `wait_until` hook -- and the first gather
|
||||
# is kicked from `begin_shard_extend`, before any getter runs. Adding
|
||||
# the wait would not make the combination work: the loader writes whole
|
||||
# pool rows at LOGICAL indices with no ownership filter, while this
|
||||
# pool's rows are local physical ones, so ordering the read would only
|
||||
# turn a race into a silent wrong-row read. Same reason as
|
||||
# `set_kv_buffer_prefix_valid` above.
|
||||
raise NotImplementedError(
|
||||
"layer-wise KV load-back (HiCache / external linker) is "
|
||||
"unsupported under logical-page KV sharding (the gather reads "
|
||||
"pool rows directly and the load-back writes them unfiltered)"
|
||||
)
|
||||
|
||||
# ---- subclass hooks -------------------------------------------------------
|
||||
|
||||
def _scratch_tensor_specs(self, rows: int) -> Dict[str, torch.Tensor]:
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import os
|
||||
import unittest
|
||||
from array import array
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from sglang.test.ci.ci_register import register_cpu_ci
|
||||
@@ -8,11 +10,24 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
|
||||
|
||||
maybe_stub_sgl_kernel()
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.srt.disaggregation.utils import DisaggregationMode
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.managers.scheduler_components import invariant_checker
|
||||
from sglang.srt.managers.scheduler_components.invariant_checker import (
|
||||
SchedulerInvariantChecker,
|
||||
)
|
||||
from sglang.srt.managers.scheduler_components.pool_stats_observer import (
|
||||
SchedulerPoolStatsObserver,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
|
||||
from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams
|
||||
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
from sglang.srt.mem_cache.radix_cache import RadixKey
|
||||
from sglang.srt.mem_cache.unified_cache.components import ComponentType
|
||||
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
|
||||
|
||||
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||
|
||||
@@ -86,5 +101,220 @@ class TestCheckTreeCacheGate(CustomTestCase):
|
||||
checker.tree_cache.sanity_check.assert_called_once()
|
||||
|
||||
|
||||
class TestShardedFullPoolInvariant(CustomTestCase):
|
||||
PAGE_SIZE = 16
|
||||
SHARD_SIZE = 4
|
||||
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
parallel = patch.object(
|
||||
invariant_checker,
|
||||
"get_parallel",
|
||||
return_value=SimpleNamespace(dcp_enabled=False),
|
||||
)
|
||||
parallel.start()
|
||||
self.addCleanup(parallel.stop)
|
||||
|
||||
def _make_checker(self):
|
||||
self.allocator = PageInterleavePoolAllocator(
|
||||
size=4 * self.PAGE_SIZE,
|
||||
physical_page_size=self.PAGE_SIZE,
|
||||
shard_size=self.SHARD_SIZE,
|
||||
dtype=torch.bfloat16,
|
||||
device="cpu",
|
||||
kvcache=None,
|
||||
need_sort=False,
|
||||
)
|
||||
req_pool = ReqToTokenPool(
|
||||
size=4,
|
||||
max_context_len=128,
|
||||
device="cpu",
|
||||
enable_memory_saver=False,
|
||||
)
|
||||
with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("python"):
|
||||
self.cache = UnifiedRadixCache(
|
||||
CacheInitParams(
|
||||
disable=False,
|
||||
req_to_token_pool=req_pool,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
page_size=self.PAGE_SIZE,
|
||||
tree_components=(ComponentType.FULL,),
|
||||
)
|
||||
)
|
||||
self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True)
|
||||
self.running_batch = self.last_batch
|
||||
self.chunked_req = None
|
||||
observer = SchedulerPoolStatsObserver(
|
||||
tree_cache=self.cache,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
req_to_token_pool=req_pool,
|
||||
session_controller=None,
|
||||
hisparse_coordinator=None,
|
||||
is_hybrid_swa=False,
|
||||
is_hybrid_ssm=False,
|
||||
enable_hisparse=False,
|
||||
full_tokens_per_layer=None,
|
||||
swa_tokens_per_layer=None,
|
||||
max_total_num_tokens=self.allocator.size,
|
||||
get_last_batch=lambda: self.last_batch,
|
||||
get_running_batch=lambda: self.running_batch,
|
||||
)
|
||||
return SchedulerInvariantChecker(
|
||||
is_hybrid_swa=False,
|
||||
is_hybrid_ssm=False,
|
||||
disaggregation_mode=DisaggregationMode.PREFILL,
|
||||
page_size=self.PAGE_SIZE,
|
||||
full_tokens_per_layer=None,
|
||||
swa_tokens_per_layer=None,
|
||||
max_total_num_tokens=self.allocator.size,
|
||||
tree_cache=self.cache,
|
||||
token_to_kv_pool_allocator=self.allocator,
|
||||
req_to_token_pool=req_pool,
|
||||
pool_stats_observer=observer,
|
||||
get_last_batch=lambda: self.last_batch,
|
||||
get_running_batch=lambda: self.running_batch,
|
||||
get_chunked_req=lambda: self.chunked_req,
|
||||
scheduler_stage_metrics=None,
|
||||
)
|
||||
|
||||
def _allocate(self, tokens, *, prefix=0, last_loc=-1, rotation_base=None):
|
||||
prefix_lens = torch.tensor([prefix], dtype=torch.int64)
|
||||
seq_lens = torch.tensor([prefix + tokens], dtype=torch.int64)
|
||||
bases = [rotation_base]
|
||||
locs = self.allocator.alloc_extend(
|
||||
prefix_lens=prefix_lens,
|
||||
prefix_lens_cpu=prefix_lens,
|
||||
seq_lens=seq_lens,
|
||||
seq_lens_cpu=seq_lens,
|
||||
last_loc=torch.tensor([last_loc], dtype=torch.int64),
|
||||
extend_num_tokens=tokens,
|
||||
rotation_bases=bases,
|
||||
)
|
||||
self.assertIsNotNone(locs)
|
||||
return locs, bases[0]
|
||||
|
||||
def _cache_prefix(self, tokens, *, protected=False):
|
||||
locs, base = self._allocate(tokens)
|
||||
key = RadixKey(array("q", range(tokens)))
|
||||
self.cache.insert(InsertParams(key=key, value=locs, rotation_base=base))
|
||||
if protected:
|
||||
matched = self.cache.match_prefix(MatchPrefixParams(key=key))
|
||||
self.cache.inc_lock_ref(matched.last_device_node)
|
||||
return locs, base
|
||||
|
||||
def _active_partial_page(self, checker):
|
||||
prefix_locs, base = self._cache_prefix(self.PAGE_SIZE, protected=True)
|
||||
tail_locs, _ = self._allocate(
|
||||
3,
|
||||
prefix=self.PAGE_SIZE,
|
||||
last_loc=prefix_locs[-1].item(),
|
||||
rotation_base=base,
|
||||
)
|
||||
locs = torch.cat((prefix_locs, tail_locs))
|
||||
checker.req_to_token_pool.req_to_token[0, : len(locs)] = locs
|
||||
req = SimpleNamespace(
|
||||
kv=SimpleNamespace(
|
||||
holds_kv=True,
|
||||
req_pool_idx=0,
|
||||
kv_allocated_len=len(locs),
|
||||
cache_protected_len=self.PAGE_SIZE,
|
||||
),
|
||||
beam_group=None,
|
||||
)
|
||||
self.last_batch = SimpleNamespace(reqs=[req], is_empty=lambda: False)
|
||||
self.running_batch = self.last_batch
|
||||
return req
|
||||
|
||||
def test_balanced_and_skewed_cache_ownership_conserve_full_pool(self):
|
||||
for cached_pages in (1, self.SHARD_SIZE):
|
||||
for protected in (False, True):
|
||||
with self.subTest(cached_pages=cached_pages, protected=protected):
|
||||
checker = self._make_checker()
|
||||
cached_tokens = cached_pages * self.PAGE_SIZE
|
||||
self._cache_prefix(cached_tokens, protected=protected)
|
||||
self.assertEqual(
|
||||
self.cache.protected_size() + self.cache.evictable_size(),
|
||||
cached_tokens,
|
||||
)
|
||||
stats = checker.pool_stats_observer.get_pool_stats()
|
||||
if cached_pages == 1:
|
||||
self.assertLess(
|
||||
stats.full_available_size,
|
||||
self.allocator.aggregate_free_size(),
|
||||
)
|
||||
else:
|
||||
self.assertEqual(
|
||||
stats.full_available_size,
|
||||
self.allocator.aggregate_free_size(),
|
||||
)
|
||||
leak, msg = checker._check_full_pool(stats)
|
||||
self.assertFalse(leak, msg)
|
||||
self.assertIn("class_free_pages=", msg)
|
||||
|
||||
def test_idle_check_detects_one_unowned_page(self):
|
||||
checker = self._make_checker()
|
||||
self._cache_prefix(self.PAGE_SIZE)
|
||||
self._allocate(self.PAGE_SIZE) # Deliberately no request or cache owner.
|
||||
leak, messages = checker._check_all_pools(
|
||||
checker.pool_stats_observer.get_pool_stats()
|
||||
)
|
||||
self.assertTrue(leak, messages)
|
||||
|
||||
def test_busy_check_detects_one_unowned_page(self):
|
||||
checker = self._make_checker()
|
||||
self._active_partial_page(checker)
|
||||
self._allocate(self.PAGE_SIZE) # Leak in addition to the active tail.
|
||||
self.assertEqual(checker._get_total_uncached_sizes(), (self.PAGE_SIZE, 0))
|
||||
with self.assertRaisesRegex(AssertionError, "Full Pool Mem Leak Detected"):
|
||||
checker.self_check_during_busy()
|
||||
|
||||
def test_partial_active_page_is_accounted_without_slack_exemption(self):
|
||||
checker = self._make_checker()
|
||||
req = self._active_partial_page(checker)
|
||||
# Count the same request only once, including when it is parked
|
||||
# between prefill chunks instead of appearing in either batch.
|
||||
for parked in (False, True):
|
||||
with self.subTest(parked=parked):
|
||||
self.chunked_req = req
|
||||
if parked:
|
||||
self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True)
|
||||
self.running_batch = self.last_batch
|
||||
uncached, swa_uncached = checker._get_total_uncached_sizes()
|
||||
self.assertEqual((uncached, swa_uncached), (self.PAGE_SIZE, 0))
|
||||
stats = checker.pool_stats_observer.get_pool_stats()
|
||||
leak, msg = checker._check_full_pool(stats, uncached=uncached)
|
||||
self.assertFalse(leak, msg)
|
||||
self.assertNotIn("slack_allowed", msg)
|
||||
with envs.SGLANG_CHECK_KV_PAGE_INVARIANTS.override(False):
|
||||
checker.self_check_during_busy()
|
||||
|
||||
def test_misaligned_evictable_counter_is_not_rounded_up(self):
|
||||
checker = self._make_checker()
|
||||
self._cache_prefix(self.PAGE_SIZE)
|
||||
self.cache.tree_core.component_evictable_size_[ComponentType.FULL] -= 1
|
||||
leak, msg = checker._check_full_pool(
|
||||
checker.pool_stats_observer.get_pool_stats()
|
||||
)
|
||||
self.assertTrue(leak, msg)
|
||||
self.assertIn(f"evictable={self.PAGE_SIZE - 1}", msg)
|
||||
|
||||
def test_compensating_misaligned_counters_still_report_alignment_error(self):
|
||||
checker = self._make_checker()
|
||||
self._cache_prefix(self.PAGE_SIZE)
|
||||
core = self.cache.tree_core
|
||||
core.component_evictable_size_[ComponentType.FULL] -= 1
|
||||
core.component_protected_size_[ComponentType.FULL] += 1
|
||||
# Their sum still matches the allocated page; neither individual
|
||||
# counter can legitimately contain a fraction of a physical page.
|
||||
self.assertEqual(
|
||||
self.cache.evictable_size() + self.cache.protected_size(), self.PAGE_SIZE
|
||||
)
|
||||
leak, msg = checker._check_full_pool(
|
||||
checker.pool_stats_observer.get_pool_stats()
|
||||
)
|
||||
self.assertTrue(leak, msg)
|
||||
self.assertIn("align", msg.lower())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import torch
|
||||
|
||||
@@ -95,6 +96,26 @@ class TestKVPageInvariants(CustomTestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
chk._check_kv_page_invariants()
|
||||
|
||||
def test_classed_allocator_without_flat_free_list_is_checked(self):
|
||||
chk, rtt, _tc, alloc = _make_checker(page_size=4, row_width=8)
|
||||
alloc.free_pages = None
|
||||
alloc.get_all_free_pages = MagicMock(
|
||||
return_value=torch.tensor([5, 6], dtype=torch.int64)
|
||||
)
|
||||
rtt[0, :1] = torch.tensor([5 * alloc.page_size])
|
||||
chk.get_last_batch = lambda: SimpleNamespace(
|
||||
reqs=[_FakeOwner(0, 1, 1, rid="classed")]
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.srt.managers.scheduler_components.invariant_checker.page_interleave_shard_size",
|
||||
return_value=2,
|
||||
):
|
||||
with self.assertRaisesRegex(ValueError, "use-after-free"):
|
||||
chk._check_kv_page_invariants()
|
||||
|
||||
alloc.get_all_free_pages.assert_called()
|
||||
|
||||
def test_free_pool_duplicate_raises(self):
|
||||
chk, rtt, tc, alloc = _make_checker(free_pages=torch.tensor([3, 3, 4]))
|
||||
rtt[0, :1] = torch.tensor([10 * _PAGE_SIZE]) # owner page 10, not in free
|
||||
|
||||
@@ -43,6 +43,7 @@ import os
|
||||
import unittest
|
||||
import unittest.mock
|
||||
from array import array
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import torch
|
||||
@@ -52,6 +53,7 @@ from sglang.srt.distributed import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from sglang.srt.mem_cache import page_interleave
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import (
|
||||
PageInterleavePoolAllocator,
|
||||
page_interleave_shard_size,
|
||||
@@ -69,7 +71,9 @@ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, ReqToTokenPool
|
||||
from sglang.srt.mem_cache.page_interleave import (
|
||||
PageInterleavePlacement,
|
||||
PageShardSpec,
|
||||
compute_page_shard_scratch_bytes,
|
||||
get_kv_shard_group,
|
||||
make_page_shard_spec,
|
||||
)
|
||||
from sglang.srt.mem_cache.page_interleave_pool import (
|
||||
PageInterleaveKVPoolMixin,
|
||||
@@ -93,6 +97,147 @@ PS = 16 # physical page size
|
||||
GS = N * PS # full-group span (N physical pages)
|
||||
|
||||
|
||||
class TestPageShardScratchSizing(CustomTestCase):
|
||||
@contextmanager
|
||||
def _fixture(
|
||||
self,
|
||||
*,
|
||||
enabled=True,
|
||||
draft=False,
|
||||
shard_size=4,
|
||||
use_mla=True,
|
||||
context_len=4096,
|
||||
chunk_tokens=256,
|
||||
prefill_max_requests=None,
|
||||
max_running_requests=None,
|
||||
dp_size=1,
|
||||
tp_size=4,
|
||||
num_kv_heads=8,
|
||||
head_dim=96,
|
||||
v_head_dim=64,
|
||||
):
|
||||
group = SimpleNamespace(world_size=shard_size, rank_in_group=0)
|
||||
parallel = SimpleNamespace(
|
||||
enable_kv_cache_sharding=enabled,
|
||||
attn_cp_group=(SimpleNamespace(world_size=1) if use_mla else group),
|
||||
attn_tp_group=group,
|
||||
attn_tp_size=tp_size,
|
||||
attn_dp_size=dp_size,
|
||||
)
|
||||
schedule = SimpleNamespace(
|
||||
chunked_prefill_size=chunk_tokens,
|
||||
prefill_max_requests=prefill_max_requests,
|
||||
max_running_requests=max_running_requests,
|
||||
)
|
||||
kvc = SimpleNamespace(
|
||||
is_draft_worker=draft,
|
||||
use_mla_backend=use_mla,
|
||||
page_size=16,
|
||||
kv_cache_dtype=torch.bfloat16,
|
||||
model_config=SimpleNamespace(
|
||||
context_len=context_len,
|
||||
kv_lora_rank=16,
|
||||
qk_rope_head_dim=8,
|
||||
head_dim=head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
get_num_kv_heads=lambda tp: num_kv_heads // tp,
|
||||
),
|
||||
)
|
||||
with (
|
||||
unittest.mock.patch.object(
|
||||
page_interleave, "get_parallel", return_value=parallel
|
||||
),
|
||||
unittest.mock.patch.object(
|
||||
page_interleave, "get_schedule", return_value=schedule
|
||||
),
|
||||
):
|
||||
yield kvc
|
||||
|
||||
def test_disabled_draft_and_trivial_group_need_no_scratch(self):
|
||||
for options in (
|
||||
{"enabled": False},
|
||||
{"draft": True},
|
||||
{"shard_size": 1},
|
||||
):
|
||||
with self.subTest(options=options), self._fixture(**options) as kvc:
|
||||
self.assertIsNone(make_page_shard_spec(kvc))
|
||||
self.assertEqual(compute_page_shard_scratch_bytes(kvc), 0)
|
||||
|
||||
def test_eight_contexts_are_reserved_independently_of_batch_limits(self):
|
||||
for options in (
|
||||
{},
|
||||
{"prefill_max_requests": 1},
|
||||
{"max_running_requests": 1},
|
||||
{"max_running_requests": 4, "dp_size": 4},
|
||||
{"chunk_tokens": 16},
|
||||
{"chunk_tokens": 8192, "prefill_max_requests": 32},
|
||||
):
|
||||
with self.subTest(options=options), self._fixture(**options) as kvc:
|
||||
spec = make_page_shard_spec(kvc)
|
||||
self.assertEqual(spec.max_prefix_tokens, 8 * 4096)
|
||||
self.assertEqual(spec.chunk_tokens, options.get("chunk_tokens", 256))
|
||||
self.assertEqual(
|
||||
spec.scratch_rows, spec.max_prefix_tokens + spec.chunk_tokens + 16
|
||||
)
|
||||
|
||||
def test_each_context_is_aligned_before_multiplying_by_eight(self):
|
||||
with self._fixture(context_len=4097, chunk_tokens=33) as kvc:
|
||||
spec = make_page_shard_spec(kvc)
|
||||
# Each request's prefix gather pads independently to 64 tokens.
|
||||
# Aligning only 8 * context_len would reserve too few rows.
|
||||
self.assertEqual(spec.max_prefix_tokens, 8 * 4160)
|
||||
self.assertGreater(spec.max_prefix_tokens, 32832)
|
||||
self.assertEqual(spec.chunk_tokens, 48)
|
||||
self.assertEqual(spec.max_prefix_tokens % spec.logical_page_size, 0)
|
||||
|
||||
def test_wide_gqa_keeps_eight_contexts_without_a_byte_cap(self):
|
||||
with self._fixture(
|
||||
use_mla=False,
|
||||
context_len=65536,
|
||||
chunk_tokens=4096,
|
||||
tp_size=1,
|
||||
num_kv_heads=64,
|
||||
head_dim=128,
|
||||
v_head_dim=128,
|
||||
) as kvc:
|
||||
spec = make_page_shard_spec(kvc)
|
||||
estimated = compute_page_shard_scratch_bytes(kvc)
|
||||
# Numerical sizing only: do not allocate this large scratch on CPU.
|
||||
# 64 heads * (128 K + 128 V) * two-byte BF16 = 32 KiB per row.
|
||||
self.assertEqual(spec.max_prefix_tokens, 8 * 65536)
|
||||
self.assertEqual(spec.chunk_tokens, 4096)
|
||||
self.assertEqual(estimated, 2 * (8 * 65536 + 4096 + 16) * 32768)
|
||||
self.assertGreater(estimated, 64 << 20)
|
||||
|
||||
def test_estimate_matches_actual_mla_and_gqa_scratch_tensors(self):
|
||||
for use_mla in (True, False):
|
||||
with (
|
||||
self.subTest(use_mla=use_mla),
|
||||
self._fixture(use_mla=use_mla, context_len=128, tp_size=2) as kvc,
|
||||
):
|
||||
spec = make_page_shard_spec(kvc)
|
||||
estimated = compute_page_shard_scratch_bytes(kvc)
|
||||
pool = SimpleNamespace(
|
||||
device="cpu",
|
||||
store_dtype=kvc.kv_cache_dtype,
|
||||
kv_cache_dim=24,
|
||||
head_num=4,
|
||||
head_dim=96,
|
||||
v_head_dim=64,
|
||||
)
|
||||
pool_cls = (
|
||||
PageInterleaveMLATokenToKVPool
|
||||
if use_mla
|
||||
else PageInterleaveMHATokenToKVPool
|
||||
)
|
||||
tensors = pool_cls._scratch_tensor_specs(pool, spec.scratch_rows)
|
||||
actual = 2 * sum(
|
||||
tensor.numel() * tensor.element_size()
|
||||
for tensor in tensors.values()
|
||||
)
|
||||
self.assertEqual(estimated, actual)
|
||||
|
||||
|
||||
def _make_spec(shard_rank=0, max_prefix_groups=64, chunk_pages=32):
|
||||
return PageShardSpec(
|
||||
shard_rank=shard_rank,
|
||||
@@ -1016,6 +1161,35 @@ class TestBeginShardExtendPlan(CustomTestCase):
|
||||
self.assertIn("cyclic", str(ctx.exception))
|
||||
|
||||
|
||||
class TestLayerTransferCounterRefused(CustomTestCase):
|
||||
"""Layer-wise KV load-back must fail loud at registration.
|
||||
|
||||
The gather reads pool rows directly (`_gather_pairs`), so it never passes
|
||||
through the base getters' `layer_transfer_counter.wait_until` hook, and
|
||||
`begin_shard_extend` kicks the first gather before any getter runs.
|
||||
"""
|
||||
|
||||
class _Base:
|
||||
def register_layer_transfer_counter(self, counter):
|
||||
self.counter = counter
|
||||
|
||||
class _Pool(PageInterleaveKVPoolMixin, _Base):
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
def test_a_real_counter_is_refused(self):
|
||||
pool = self._Pool()
|
||||
with self.assertRaises(NotImplementedError) as cm:
|
||||
pool.register_layer_transfer_counter(object())
|
||||
self.assertIn("logical-page KV sharding", str(cm.exception))
|
||||
|
||||
def test_none_stays_a_no_op(self):
|
||||
"""The SWA/hybrid wrappers disable the counter by passing None."""
|
||||
pool = self._Pool()
|
||||
pool.register_layer_transfer_counter(None)
|
||||
self.assertIsNone(pool.counter)
|
||||
|
||||
|
||||
class TestScratchTranslation(CustomTestCase):
|
||||
def _plan(self, base=2, n_prefix=7, n_chunk=9, rank=1):
|
||||
pages = _chain_pages(base=base, n_pages=n_prefix + n_chunk)
|
||||
|
||||
Reference in New Issue
Block a user