From ddebc52f237a1dbb56533469ab2ec2a7b856c4ab Mon Sep 17 00:00:00 2001 From: Shunkangz <182541032+Shunkangz@users.noreply.github.com> Date: Tue, 22 Sep 2026 19:57:44 +0800 Subject: [PATCH] [kv-shard 3/4] Enable Control plane (#38468) --- .../scheduler_components/invariant_checker.py | 71 ++++-- .../sglang/srt/mem_cache/page_interleave.py | 82 ++++++- .../srt/mem_cache/page_interleave_pool.py | 42 +++- .../test_invariant_checker.py | 230 ++++++++++++++++++ .../unit/managers/test_kv_page_invariants.py | 21 ++ .../mem_cache/test_page_interleave_shard.py | 174 +++++++++++++ 6 files changed, 596 insertions(+), 24 deletions(-) diff --git a/python/sglang/srt/managers/scheduler_components/invariant_checker.py b/python/sglang/srt/managers/scheduler_components/invariant_checker.py index b10f18d5e8..5f2a020012 100644 --- a/python/sglang/srt/managers/scheduler_components/invariant_checker.py +++ b/python/sglang/srt/managers/scheduler_components/invariant_checker.py @@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import ( SchedulerPoolStatsObserver, ) from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator +from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache from sglang.srt.mem_cache.memory_pool import ReqToTokenPool @@ -117,29 +118,60 @@ class SchedulerInvariantChecker: session_held = self.pool_stats_observer.session_held_tokens() total = self.max_total_num_tokens full_evictable_size = ps.full_evictable_size - if get_parallel().dcp_enabled and allocator.page_size > 1: - # DCP stores logical tokens in widened physical pages. Prefix cache - # counters are logical-token based, while the allocator frees whole - # physical pages, so round cached tokens up to physical page units. - full_evictable_size = ( - (full_evictable_size + allocator.page_size - 1) - // allocator.page_size - * allocator.page_size + full_available = ps.full_available_size + class_watermark_msg = "" + kv_shard_size = page_interleave_shard_size(allocator) + dcp_page_alloc = ( + kv_shard_size == 1 + and get_parallel().dcp_enabled + and allocator.page_size > 1 + ) + if kv_shard_size > 1: + # Conservation needs every free page, not the min-class capacity + # floor used for admission. Sharding keeps the tree and allocator + # page sizes equal, so cached counts must already be page-aligned. + full_available = allocator.aggregate_free_size() + class_watermark_msg = ( + f", class_free_pages={allocator.class_free_page_counts()}" ) + elif dcp_page_alloc: + # Preserve the existing DCP-only accounting adjustment. DCP widens + # allocator pages; its scheduler counters still use per-rank units. + full_evictable_size = ceil_align(full_evictable_size, allocator.page_size) leak, msg = self._check_pool_invariant( "full", - ps.full_available_size, + full_available, full_evictable_size, protected, session_held, total, uncached, ) - if leak and get_parallel().dcp_enabled and allocator.page_size > 1: - # Radix/Mamba cache accounting is logical-token based while DCP full - # KV allocation is physical-page based. Partial physical pages can - # leave a small page-level slack even when all pages are owned by - # either the allocator or the prefix cache. + msg += class_watermark_msg + if kv_shard_size > 1: + # Partial active pages are already rounded in + # _get_total_uncached_sizes. Rounding cached counts or accepting + # arbitrary slack here would hide corruption and lost pages. + unaligned = { + name: value + for name, value in ( + ("available", full_available), + ("evictable", full_evictable_size), + ("protected", protected), + ("session_held", session_held), + ("uncached", uncached), + ("total", total), + ) + if value % allocator.page_size != 0 + } + if unaligned: + return True, ( + f"{msg}, unaligned_sharded_counts={unaligned}, " + f"page_size={allocator.page_size}" + ) + if leak and dcp_page_alloc: + # Legacy DCP counters use different page/capacity units. Keep that + # compatibility path separate from exact KV-shard conservation. return False, f"{msg}, dcp_physical_page_slack_allowed=True" return leak, msg @@ -380,14 +412,17 @@ class SchedulerInvariantChecker: idx = torch.as_tensor([rpi for _, rpi, _ in active], device=rtt.device) allocs = torch.as_tensor([al for _, _, al in active], device=rtt.device) mask = torch.arange(row_width, device=rtt.device)[None, :] < allocs[:, None] - owner_pages = rtt[idx][mask] // self.page_size + owner_locs = rtt[idx][mask] # Sub-allocators to check: a flat allocator is its own single sub; a - # hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator. + # hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator; + # the classed sharding allocator keeps per-class lists instead of a + # flat free_pages and exposes them through get_all_free_pages(). alloc = self.token_to_kv_pool_allocator sub_allocs = ( [alloc] if getattr(alloc, "free_pages", None) is not None + or page_interleave_shard_size(alloc) > 1 else [ sub for n in ("full_attn_allocator", "swa_attn_allocator") @@ -398,6 +433,10 @@ class SchedulerInvariantChecker: if not sub_allocs: return + # Page ids in the ALLOCATOR's page units (== the physical page for + # the classed sharding allocator, whose free lists hold logical page + # ids in the same unit). + owner_pages = owner_locs // sub_allocs[0].page_size # Check B: every sub-pool's free set has no duplicate pages. for i, sub in enumerate(sub_allocs): free = sub.get_all_free_pages() diff --git a/python/sglang/srt/mem_cache/page_interleave.py b/python/sglang/srt/mem_cache/page_interleave.py index 2ef3c8fb95..a9e5c837dd 100644 --- a/python/sglang/srt/mem_cache/page_interleave.py +++ b/python/sglang/srt/mem_cache/page_interleave.py @@ -32,18 +32,25 @@ therefore can be striped without extra compute-time communication: from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Optional, Tuple import msgspec import torch -from sglang.srt.runtime_context import get_parallel +from sglang.srt.runtime_context import get_parallel, get_schedule +from sglang.srt.utils.common import ceil_align if TYPE_CHECKING: from sglang.srt.distributed.parallel_state import GroupCoordinator + from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator logger = logging.getLogger(__name__) +# Keep enough assembled prefix KV for eight long-context requests. The +# new-token compute budget is independent: eight 1K extensions can fill an +# 8K prefill chunk even when each request has a nearly full-context prefix. +_KV_SHARD_SCRATCH_PREFIX_CONTEXTS = 8 + class PageShardSpec(msgspec.Struct, frozen=True): """Everything both sides of any transfer need to reproduce the layout.""" @@ -52,7 +59,12 @@ class PageShardSpec(msgspec.Struct, frozen=True): shard_size: int page_size: int # physical page size (kernel-visible) max_prefix_tokens: int # scratch prefix-region capacity, granule-aligned - chunk_tokens: int # scratch chunk-region capacity, granule-aligned + chunk_tokens: int # scratch chunk-region capacity, physical-page-aligned + + @property + def scratch_rows(self) -> int: + """Rows in each slot: assembled prefix, current batch chunk, trash page.""" + return self.max_prefix_tokens + self.chunk_tokens + self.page_size @property def logical_page_size(self) -> int: @@ -111,3 +123,67 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator: if use_mla_backend: return get_parallel().attn_tp_group return cp_group + + +def get_kv_shard_group_info( + kvc: KVCacheConfigurator, +) -> Tuple[Optional[int], int]: + """``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables.""" + if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding: + return None, 1 + group = get_kv_shard_group(kvc.use_mla_backend) + if group.world_size <= 1: + return None, 1 + return group.rank_in_group, group.world_size + + +def _page_shard_row_bytes(kvc: KVCacheConfigurator) -> int: + model_config = kvc.model_config + kv_size = torch._utils._element_size(kvc.kv_cache_dtype) + if kvc.use_mla_backend: + return (model_config.kv_lora_rank + model_config.qk_rope_head_dim) * kv_size + return ( + model_config.get_num_kv_heads(get_parallel().attn_tp_size) + * (model_config.head_dim + model_config.v_head_dim) + * kv_size + ) + + +def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]: + """Provision eight full-context prefixes plus the batch's new-token budget. + + Prefix capacity must not shrink with the number of new tokens: eight + long cached prefixes can accompany just 1K new tokens each. Round each + context to a shard-group gather span before multiplying, so per-request + padding cannot overflow the region even for an unaligned context length. + + This is a fixed eight-context prefix budget, independent of request limits + or chunk size. Both scratch slots are charged against HBM before sizing + persistent KV; large model contexts therefore reduce persistent capacity. + Admission still checks the actual batch's padded prefix and chunk spans. + """ + shard_rank, shard_size = get_kv_shard_group_info(kvc) + if shard_rank is None: + return None + + page_size = kvc.page_size + granule = shard_size * page_size + return PageShardSpec( + shard_rank=shard_rank, + shard_size=shard_size, + page_size=page_size, + max_prefix_tokens=_KV_SHARD_SCRATCH_PREFIX_CONTEXTS + * ceil_align(kvc.model_config.context_len, granule), + # chunked_prefill_size is the TOTAL new-token budget for the batch. + chunk_tokens=ceil_align(get_schedule().chunked_prefill_size, page_size), + ) + + +def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int: + """Fixed HBM cost of the double-buffered assembly scratch, charged against + the KV budget before pool sizing. Use the same spec as the pool builders: + two slots, each holding ONE layer's ``[prefix | chunk | trash page]``.""" + spec = make_page_shard_spec(kvc) + if spec is None: + return 0 + return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc) diff --git a/python/sglang/srt/mem_cache/page_interleave_pool.py b/python/sglang/srt/mem_cache/page_interleave_pool.py index f4aab3973e..3952bb12f5 100644 --- a/python/sglang/srt/mem_cache/page_interleave_pool.py +++ b/python/sglang/srt/mem_cache/page_interleave_pool.py @@ -113,9 +113,9 @@ class PageInterleaveKVPoolMixin: assert spec.shard_size == shard_group.world_size assert spec.shard_rank == shard_group.rank_in_group assert spec.page_size == self.page_size - # The prefix region must fit N * ceil(prefix_pages / N) pages for any - # prefix, i.e. be a multiple of the full-group span; the chunk region - # is per-page. + # The batch's prefix region is split into equal per-rank gather + # blocks, so it must be a multiple of the full-group span. The chunk + # region is per-page; admission checks both regions against this spec. assert spec.max_prefix_tokens % spec.logical_page_size == 0 assert spec.chunk_tokens % spec.page_size == 0 @@ -133,7 +133,7 @@ class PageInterleaveKVPoolMixin: self.kv_gather_stream = self.device_module.Stream() # Scratch: [prefix | chunk | trash page], double-buffered. - scratch_rows = spec.max_prefix_tokens + spec.chunk_tokens + spec.page_size + scratch_rows = spec.scratch_rows self._chunk_base = spec.max_prefix_tokens self._trash_base = spec.max_prefix_tokens + spec.chunk_tokens with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE): @@ -171,13 +171,23 @@ class PageInterleaveKVPoolMixin: self._write_plan: Optional[Tuple[torch.Tensor, torch.Tensor]] = None self._translate_cache: Dict[Tuple[int, int], torch.Tensor] = {} + scratch_bytes = sum( + tensor.numel() * tensor.element_size() + for slot in self._slots + for tensor in slot.tensors.values() + ) logger.info( "Page-interleave KV sharding enabled: shard_rank=%d shard_size=%d " - "page_size=%d scratch_rows=%d x2", + "page_size=%d prefix_tokens=%d chunk_tokens=%d scratch_rows=%d x2 " + "(%.2f MiB per rank, charged against the KV budget). " + "Batch admission is limited by this scratch capacity.", self.shard_rank, self.shard_size, spec.page_size, + spec.max_prefix_tokens, + spec.chunk_tokens, scratch_rows, + scratch_bytes / (1024 * 1024), ) def set_kv_buffer_prefix_valid(self, *args, **kwargs): @@ -186,6 +196,28 @@ class PageInterleaveKVPoolMixin: "(it writes pool rows directly, bypassing the ownership filter)" ) + def register_layer_transfer_counter(self, layer_transfer_counter): + # `None` is the in-tree "disable" signal from the SWA/hybrid wrappers, + # so it has to stay a no-op. + if layer_transfer_counter is None: + super().register_layer_transfer_counter(None) + return + # Layer-wise KV load-back is unsupported here, and refusing it at + # registration is the only safe answer. The gather in `_prefetch_layer` + # reads pool rows directly (`_gather_pairs`), so it never passes + # through the base getters' `wait_until` hook -- and the first gather + # is kicked from `begin_shard_extend`, before any getter runs. Adding + # the wait would not make the combination work: the loader writes whole + # pool rows at LOGICAL indices with no ownership filter, while this + # pool's rows are local physical ones, so ordering the read would only + # turn a race into a silent wrong-row read. Same reason as + # `set_kv_buffer_prefix_valid` above. + raise NotImplementedError( + "layer-wise KV load-back (HiCache / external linker) is " + "unsupported under logical-page KV sharding (the gather reads " + "pool rows directly and the load-back writes them unfiltered)" + ) + # ---- subclass hooks ------------------------------------------------------- def _scratch_tensor_specs(self, rows: int) -> Dict[str, torch.Tensor]: diff --git a/test/registered/unit/managers/scheduler_components/test_invariant_checker.py b/test/registered/unit/managers/scheduler_components/test_invariant_checker.py index 97eeb06f67..0981b8f3e3 100644 --- a/test/registered/unit/managers/scheduler_components/test_invariant_checker.py +++ b/test/registered/unit/managers/scheduler_components/test_invariant_checker.py @@ -1,6 +1,8 @@ import os import unittest +from array import array from contextlib import contextmanager +from types import SimpleNamespace from unittest.mock import MagicMock, patch from sglang.test.ci.ci_register import register_cpu_ci @@ -8,11 +10,24 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel maybe_stub_sgl_kernel() +import torch + from sglang.srt.disaggregation.utils import DisaggregationMode from sglang.srt.environ import envs +from sglang.srt.managers.scheduler_components import invariant_checker from sglang.srt.managers.scheduler_components.invariant_checker import ( SchedulerInvariantChecker, ) +from sglang.srt.managers.scheduler_components.pool_stats_observer import ( + SchedulerPoolStatsObserver, +) +from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator +from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams +from sglang.srt.mem_cache.cache_init_params import CacheInitParams +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.radix_cache import RadixKey +from sglang.srt.mem_cache.unified_cache.components import ComponentType +from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache register_cpu_ci(est_time=10, suite="base-a-test-cpu") @@ -86,5 +101,220 @@ class TestCheckTreeCacheGate(CustomTestCase): checker.tree_cache.sanity_check.assert_called_once() +class TestShardedFullPoolInvariant(CustomTestCase): + PAGE_SIZE = 16 + SHARD_SIZE = 4 + + def setUp(self): + super().setUp() + parallel = patch.object( + invariant_checker, + "get_parallel", + return_value=SimpleNamespace(dcp_enabled=False), + ) + parallel.start() + self.addCleanup(parallel.stop) + + def _make_checker(self): + self.allocator = PageInterleavePoolAllocator( + size=4 * self.PAGE_SIZE, + physical_page_size=self.PAGE_SIZE, + shard_size=self.SHARD_SIZE, + dtype=torch.bfloat16, + device="cpu", + kvcache=None, + need_sort=False, + ) + req_pool = ReqToTokenPool( + size=4, + max_context_len=128, + device="cpu", + enable_memory_saver=False, + ) + with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("python"): + self.cache = UnifiedRadixCache( + CacheInitParams( + disable=False, + req_to_token_pool=req_pool, + token_to_kv_pool_allocator=self.allocator, + page_size=self.PAGE_SIZE, + tree_components=(ComponentType.FULL,), + ) + ) + self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True) + self.running_batch = self.last_batch + self.chunked_req = None + observer = SchedulerPoolStatsObserver( + tree_cache=self.cache, + token_to_kv_pool_allocator=self.allocator, + req_to_token_pool=req_pool, + session_controller=None, + hisparse_coordinator=None, + is_hybrid_swa=False, + is_hybrid_ssm=False, + enable_hisparse=False, + full_tokens_per_layer=None, + swa_tokens_per_layer=None, + max_total_num_tokens=self.allocator.size, + get_last_batch=lambda: self.last_batch, + get_running_batch=lambda: self.running_batch, + ) + return SchedulerInvariantChecker( + is_hybrid_swa=False, + is_hybrid_ssm=False, + disaggregation_mode=DisaggregationMode.PREFILL, + page_size=self.PAGE_SIZE, + full_tokens_per_layer=None, + swa_tokens_per_layer=None, + max_total_num_tokens=self.allocator.size, + tree_cache=self.cache, + token_to_kv_pool_allocator=self.allocator, + req_to_token_pool=req_pool, + pool_stats_observer=observer, + get_last_batch=lambda: self.last_batch, + get_running_batch=lambda: self.running_batch, + get_chunked_req=lambda: self.chunked_req, + scheduler_stage_metrics=None, + ) + + def _allocate(self, tokens, *, prefix=0, last_loc=-1, rotation_base=None): + prefix_lens = torch.tensor([prefix], dtype=torch.int64) + seq_lens = torch.tensor([prefix + tokens], dtype=torch.int64) + bases = [rotation_base] + locs = self.allocator.alloc_extend( + prefix_lens=prefix_lens, + prefix_lens_cpu=prefix_lens, + seq_lens=seq_lens, + seq_lens_cpu=seq_lens, + last_loc=torch.tensor([last_loc], dtype=torch.int64), + extend_num_tokens=tokens, + rotation_bases=bases, + ) + self.assertIsNotNone(locs) + return locs, bases[0] + + def _cache_prefix(self, tokens, *, protected=False): + locs, base = self._allocate(tokens) + key = RadixKey(array("q", range(tokens))) + self.cache.insert(InsertParams(key=key, value=locs, rotation_base=base)) + if protected: + matched = self.cache.match_prefix(MatchPrefixParams(key=key)) + self.cache.inc_lock_ref(matched.last_device_node) + return locs, base + + def _active_partial_page(self, checker): + prefix_locs, base = self._cache_prefix(self.PAGE_SIZE, protected=True) + tail_locs, _ = self._allocate( + 3, + prefix=self.PAGE_SIZE, + last_loc=prefix_locs[-1].item(), + rotation_base=base, + ) + locs = torch.cat((prefix_locs, tail_locs)) + checker.req_to_token_pool.req_to_token[0, : len(locs)] = locs + req = SimpleNamespace( + kv=SimpleNamespace( + holds_kv=True, + req_pool_idx=0, + kv_allocated_len=len(locs), + cache_protected_len=self.PAGE_SIZE, + ), + beam_group=None, + ) + self.last_batch = SimpleNamespace(reqs=[req], is_empty=lambda: False) + self.running_batch = self.last_batch + return req + + def test_balanced_and_skewed_cache_ownership_conserve_full_pool(self): + for cached_pages in (1, self.SHARD_SIZE): + for protected in (False, True): + with self.subTest(cached_pages=cached_pages, protected=protected): + checker = self._make_checker() + cached_tokens = cached_pages * self.PAGE_SIZE + self._cache_prefix(cached_tokens, protected=protected) + self.assertEqual( + self.cache.protected_size() + self.cache.evictable_size(), + cached_tokens, + ) + stats = checker.pool_stats_observer.get_pool_stats() + if cached_pages == 1: + self.assertLess( + stats.full_available_size, + self.allocator.aggregate_free_size(), + ) + else: + self.assertEqual( + stats.full_available_size, + self.allocator.aggregate_free_size(), + ) + leak, msg = checker._check_full_pool(stats) + self.assertFalse(leak, msg) + self.assertIn("class_free_pages=", msg) + + def test_idle_check_detects_one_unowned_page(self): + checker = self._make_checker() + self._cache_prefix(self.PAGE_SIZE) + self._allocate(self.PAGE_SIZE) # Deliberately no request or cache owner. + leak, messages = checker._check_all_pools( + checker.pool_stats_observer.get_pool_stats() + ) + self.assertTrue(leak, messages) + + def test_busy_check_detects_one_unowned_page(self): + checker = self._make_checker() + self._active_partial_page(checker) + self._allocate(self.PAGE_SIZE) # Leak in addition to the active tail. + self.assertEqual(checker._get_total_uncached_sizes(), (self.PAGE_SIZE, 0)) + with self.assertRaisesRegex(AssertionError, "Full Pool Mem Leak Detected"): + checker.self_check_during_busy() + + def test_partial_active_page_is_accounted_without_slack_exemption(self): + checker = self._make_checker() + req = self._active_partial_page(checker) + # Count the same request only once, including when it is parked + # between prefill chunks instead of appearing in either batch. + for parked in (False, True): + with self.subTest(parked=parked): + self.chunked_req = req + if parked: + self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True) + self.running_batch = self.last_batch + uncached, swa_uncached = checker._get_total_uncached_sizes() + self.assertEqual((uncached, swa_uncached), (self.PAGE_SIZE, 0)) + stats = checker.pool_stats_observer.get_pool_stats() + leak, msg = checker._check_full_pool(stats, uncached=uncached) + self.assertFalse(leak, msg) + self.assertNotIn("slack_allowed", msg) + with envs.SGLANG_CHECK_KV_PAGE_INVARIANTS.override(False): + checker.self_check_during_busy() + + def test_misaligned_evictable_counter_is_not_rounded_up(self): + checker = self._make_checker() + self._cache_prefix(self.PAGE_SIZE) + self.cache.tree_core.component_evictable_size_[ComponentType.FULL] -= 1 + leak, msg = checker._check_full_pool( + checker.pool_stats_observer.get_pool_stats() + ) + self.assertTrue(leak, msg) + self.assertIn(f"evictable={self.PAGE_SIZE - 1}", msg) + + def test_compensating_misaligned_counters_still_report_alignment_error(self): + checker = self._make_checker() + self._cache_prefix(self.PAGE_SIZE) + core = self.cache.tree_core + core.component_evictable_size_[ComponentType.FULL] -= 1 + core.component_protected_size_[ComponentType.FULL] += 1 + # Their sum still matches the allocated page; neither individual + # counter can legitimately contain a fraction of a physical page. + self.assertEqual( + self.cache.evictable_size() + self.cache.protected_size(), self.PAGE_SIZE + ) + leak, msg = checker._check_full_pool( + checker.pool_stats_observer.get_pool_stats() + ) + self.assertTrue(leak, msg) + self.assertIn("align", msg.lower()) + + if __name__ == "__main__": unittest.main() diff --git a/test/registered/unit/managers/test_kv_page_invariants.py b/test/registered/unit/managers/test_kv_page_invariants.py index 9ba86262b2..685eb7f26d 100644 --- a/test/registered/unit/managers/test_kv_page_invariants.py +++ b/test/registered/unit/managers/test_kv_page_invariants.py @@ -2,6 +2,7 @@ import unittest from types import SimpleNamespace +from unittest.mock import MagicMock, patch import torch @@ -95,6 +96,26 @@ class TestKVPageInvariants(CustomTestCase): with self.assertRaises(ValueError): chk._check_kv_page_invariants() + def test_classed_allocator_without_flat_free_list_is_checked(self): + chk, rtt, _tc, alloc = _make_checker(page_size=4, row_width=8) + alloc.free_pages = None + alloc.get_all_free_pages = MagicMock( + return_value=torch.tensor([5, 6], dtype=torch.int64) + ) + rtt[0, :1] = torch.tensor([5 * alloc.page_size]) + chk.get_last_batch = lambda: SimpleNamespace( + reqs=[_FakeOwner(0, 1, 1, rid="classed")] + ) + + with patch( + "sglang.srt.managers.scheduler_components.invariant_checker.page_interleave_shard_size", + return_value=2, + ): + with self.assertRaisesRegex(ValueError, "use-after-free"): + chk._check_kv_page_invariants() + + alloc.get_all_free_pages.assert_called() + def test_free_pool_duplicate_raises(self): chk, rtt, tc, alloc = _make_checker(free_pages=torch.tensor([3, 3, 4])) rtt[0, :1] = torch.tensor([10 * _PAGE_SIZE]) # owner page 10, not in free diff --git a/test/registered/unit/mem_cache/test_page_interleave_shard.py b/test/registered/unit/mem_cache/test_page_interleave_shard.py index 8ddcd37d43..90729d6c49 100644 --- a/test/registered/unit/mem_cache/test_page_interleave_shard.py +++ b/test/registered/unit/mem_cache/test_page_interleave_shard.py @@ -43,6 +43,7 @@ import os import unittest import unittest.mock from array import array +from contextlib import contextmanager from types import SimpleNamespace import torch @@ -52,6 +53,7 @@ from sglang.srt.distributed import ( init_distributed_environment, initialize_model_parallel, ) +from sglang.srt.mem_cache import page_interleave from sglang.srt.mem_cache.allocator.page_interleave import ( PageInterleavePoolAllocator, page_interleave_shard_size, @@ -69,7 +71,9 @@ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, ReqToTokenPool from sglang.srt.mem_cache.page_interleave import ( PageInterleavePlacement, PageShardSpec, + compute_page_shard_scratch_bytes, get_kv_shard_group, + make_page_shard_spec, ) from sglang.srt.mem_cache.page_interleave_pool import ( PageInterleaveKVPoolMixin, @@ -93,6 +97,147 @@ PS = 16 # physical page size GS = N * PS # full-group span (N physical pages) +class TestPageShardScratchSizing(CustomTestCase): + @contextmanager + def _fixture( + self, + *, + enabled=True, + draft=False, + shard_size=4, + use_mla=True, + context_len=4096, + chunk_tokens=256, + prefill_max_requests=None, + max_running_requests=None, + dp_size=1, + tp_size=4, + num_kv_heads=8, + head_dim=96, + v_head_dim=64, + ): + group = SimpleNamespace(world_size=shard_size, rank_in_group=0) + parallel = SimpleNamespace( + enable_kv_cache_sharding=enabled, + attn_cp_group=(SimpleNamespace(world_size=1) if use_mla else group), + attn_tp_group=group, + attn_tp_size=tp_size, + attn_dp_size=dp_size, + ) + schedule = SimpleNamespace( + chunked_prefill_size=chunk_tokens, + prefill_max_requests=prefill_max_requests, + max_running_requests=max_running_requests, + ) + kvc = SimpleNamespace( + is_draft_worker=draft, + use_mla_backend=use_mla, + page_size=16, + kv_cache_dtype=torch.bfloat16, + model_config=SimpleNamespace( + context_len=context_len, + kv_lora_rank=16, + qk_rope_head_dim=8, + head_dim=head_dim, + v_head_dim=v_head_dim, + get_num_kv_heads=lambda tp: num_kv_heads // tp, + ), + ) + with ( + unittest.mock.patch.object( + page_interleave, "get_parallel", return_value=parallel + ), + unittest.mock.patch.object( + page_interleave, "get_schedule", return_value=schedule + ), + ): + yield kvc + + def test_disabled_draft_and_trivial_group_need_no_scratch(self): + for options in ( + {"enabled": False}, + {"draft": True}, + {"shard_size": 1}, + ): + with self.subTest(options=options), self._fixture(**options) as kvc: + self.assertIsNone(make_page_shard_spec(kvc)) + self.assertEqual(compute_page_shard_scratch_bytes(kvc), 0) + + def test_eight_contexts_are_reserved_independently_of_batch_limits(self): + for options in ( + {}, + {"prefill_max_requests": 1}, + {"max_running_requests": 1}, + {"max_running_requests": 4, "dp_size": 4}, + {"chunk_tokens": 16}, + {"chunk_tokens": 8192, "prefill_max_requests": 32}, + ): + with self.subTest(options=options), self._fixture(**options) as kvc: + spec = make_page_shard_spec(kvc) + self.assertEqual(spec.max_prefix_tokens, 8 * 4096) + self.assertEqual(spec.chunk_tokens, options.get("chunk_tokens", 256)) + self.assertEqual( + spec.scratch_rows, spec.max_prefix_tokens + spec.chunk_tokens + 16 + ) + + def test_each_context_is_aligned_before_multiplying_by_eight(self): + with self._fixture(context_len=4097, chunk_tokens=33) as kvc: + spec = make_page_shard_spec(kvc) + # Each request's prefix gather pads independently to 64 tokens. + # Aligning only 8 * context_len would reserve too few rows. + self.assertEqual(spec.max_prefix_tokens, 8 * 4160) + self.assertGreater(spec.max_prefix_tokens, 32832) + self.assertEqual(spec.chunk_tokens, 48) + self.assertEqual(spec.max_prefix_tokens % spec.logical_page_size, 0) + + def test_wide_gqa_keeps_eight_contexts_without_a_byte_cap(self): + with self._fixture( + use_mla=False, + context_len=65536, + chunk_tokens=4096, + tp_size=1, + num_kv_heads=64, + head_dim=128, + v_head_dim=128, + ) as kvc: + spec = make_page_shard_spec(kvc) + estimated = compute_page_shard_scratch_bytes(kvc) + # Numerical sizing only: do not allocate this large scratch on CPU. + # 64 heads * (128 K + 128 V) * two-byte BF16 = 32 KiB per row. + self.assertEqual(spec.max_prefix_tokens, 8 * 65536) + self.assertEqual(spec.chunk_tokens, 4096) + self.assertEqual(estimated, 2 * (8 * 65536 + 4096 + 16) * 32768) + self.assertGreater(estimated, 64 << 20) + + def test_estimate_matches_actual_mla_and_gqa_scratch_tensors(self): + for use_mla in (True, False): + with ( + self.subTest(use_mla=use_mla), + self._fixture(use_mla=use_mla, context_len=128, tp_size=2) as kvc, + ): + spec = make_page_shard_spec(kvc) + estimated = compute_page_shard_scratch_bytes(kvc) + pool = SimpleNamespace( + device="cpu", + store_dtype=kvc.kv_cache_dtype, + kv_cache_dim=24, + head_num=4, + head_dim=96, + v_head_dim=64, + ) + pool_cls = ( + PageInterleaveMLATokenToKVPool + if use_mla + else PageInterleaveMHATokenToKVPool + ) + tensors = pool_cls._scratch_tensor_specs(pool, spec.scratch_rows) + actual = 2 * sum( + tensor.numel() * tensor.element_size() + for tensor in tensors.values() + ) + self.assertEqual(estimated, actual) + + def _make_spec(shard_rank=0, max_prefix_groups=64, chunk_pages=32): return PageShardSpec( shard_rank=shard_rank, @@ -1016,6 +1161,35 @@ class TestBeginShardExtendPlan(CustomTestCase): self.assertIn("cyclic", str(ctx.exception)) +class TestLayerTransferCounterRefused(CustomTestCase): + """Layer-wise KV load-back must fail loud at registration. + + The gather reads pool rows directly (`_gather_pairs`), so it never passes + through the base getters' `layer_transfer_counter.wait_until` hook, and + `begin_shard_extend` kicks the first gather before any getter runs. + """ + + class _Base: + def register_layer_transfer_counter(self, counter): + self.counter = counter + + class _Pool(PageInterleaveKVPoolMixin, _Base): + def __init__(self): + pass + + def test_a_real_counter_is_refused(self): + pool = self._Pool() + with self.assertRaises(NotImplementedError) as cm: + pool.register_layer_transfer_counter(object()) + self.assertIn("logical-page KV sharding", str(cm.exception)) + + def test_none_stays_a_no_op(self): + """The SWA/hybrid wrappers disable the counter by passing None.""" + pool = self._Pool() + pool.register_layer_transfer_counter(None) + self.assertIsNone(pool.counter) + + class TestScratchTranslation(CustomTestCase): def _plan(self, base=2, n_prefix=7, n_chunk=9, rank=1): pages = _chain_pages(base=base, n_pages=n_prefix + n_chunk)