[kv-shard 3/4] Enable Control plane (#38468)

This commit is contained in:
Shunkangz
2026-09-22 19:57:44 +08:00
committed by GitHub
parent 2524b61f9b
commit ddebc52f23
6 changed files with 596 additions and 24 deletions
@@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import (
SchedulerPoolStatsObserver,
)
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
@@ -117,29 +118,60 @@ class SchedulerInvariantChecker:
session_held = self.pool_stats_observer.session_held_tokens()
total = self.max_total_num_tokens
full_evictable_size = ps.full_evictable_size
if get_parallel().dcp_enabled and allocator.page_size > 1:
# DCP stores logical tokens in widened physical pages. Prefix cache
# counters are logical-token based, while the allocator frees whole
# physical pages, so round cached tokens up to physical page units.
full_evictable_size = (
(full_evictable_size + allocator.page_size - 1)
// allocator.page_size
* allocator.page_size
full_available = ps.full_available_size
class_watermark_msg = ""
kv_shard_size = page_interleave_shard_size(allocator)
dcp_page_alloc = (
kv_shard_size == 1
and get_parallel().dcp_enabled
and allocator.page_size > 1
)
if kv_shard_size > 1:
# Conservation needs every free page, not the min-class capacity
# floor used for admission. Sharding keeps the tree and allocator
# page sizes equal, so cached counts must already be page-aligned.
full_available = allocator.aggregate_free_size()
class_watermark_msg = (
f", class_free_pages={allocator.class_free_page_counts()}"
)
elif dcp_page_alloc:
# Preserve the existing DCP-only accounting adjustment. DCP widens
# allocator pages; its scheduler counters still use per-rank units.
full_evictable_size = ceil_align(full_evictable_size, allocator.page_size)
leak, msg = self._check_pool_invariant(
"full",
ps.full_available_size,
full_available,
full_evictable_size,
protected,
session_held,
total,
uncached,
)
if leak and get_parallel().dcp_enabled and allocator.page_size > 1:
# Radix/Mamba cache accounting is logical-token based while DCP full
# KV allocation is physical-page based. Partial physical pages can
# leave a small page-level slack even when all pages are owned by
# either the allocator or the prefix cache.
msg += class_watermark_msg
if kv_shard_size > 1:
# Partial active pages are already rounded in
# _get_total_uncached_sizes. Rounding cached counts or accepting
# arbitrary slack here would hide corruption and lost pages.
unaligned = {
name: value
for name, value in (
("available", full_available),
("evictable", full_evictable_size),
("protected", protected),
("session_held", session_held),
("uncached", uncached),
("total", total),
)
if value % allocator.page_size != 0
}
if unaligned:
return True, (
f"{msg}, unaligned_sharded_counts={unaligned}, "
f"page_size={allocator.page_size}"
)
if leak and dcp_page_alloc:
# Legacy DCP counters use different page/capacity units. Keep that
# compatibility path separate from exact KV-shard conservation.
return False, f"{msg}, dcp_physical_page_slack_allowed=True"
return leak, msg
@@ -380,14 +412,17 @@ class SchedulerInvariantChecker:
idx = torch.as_tensor([rpi for _, rpi, _ in active], device=rtt.device)
allocs = torch.as_tensor([al for _, _, al in active], device=rtt.device)
mask = torch.arange(row_width, device=rtt.device)[None, :] < allocs[:, None]
owner_pages = rtt[idx][mask] // self.page_size
owner_locs = rtt[idx][mask]
# Sub-allocators to check: a flat allocator is its own single sub; a
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator.
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator;
# the classed sharding allocator keeps per-class lists instead of a
# flat free_pages and exposes them through get_all_free_pages().
alloc = self.token_to_kv_pool_allocator
sub_allocs = (
[alloc]
if getattr(alloc, "free_pages", None) is not None
or page_interleave_shard_size(alloc) > 1
else [
sub
for n in ("full_attn_allocator", "swa_attn_allocator")
@@ -398,6 +433,10 @@ class SchedulerInvariantChecker:
if not sub_allocs:
return
# Page ids in the ALLOCATOR's page units (== the physical page for
# the classed sharding allocator, whose free lists hold logical page
# ids in the same unit).
owner_pages = owner_locs // sub_allocs[0].page_size
# Check B: every sub-pool's free set has no duplicate pages.
for i, sub in enumerate(sub_allocs):
free = sub.get_all_free_pages()
+79 -3
View File
@@ -32,18 +32,25 @@ therefore can be striped without extra compute-time communication:
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Optional, Tuple
import msgspec
import torch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_parallel, get_schedule
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
logger = logging.getLogger(__name__)
# Keep enough assembled prefix KV for eight long-context requests. The
# new-token compute budget is independent: eight 1K extensions can fill an
# 8K prefill chunk even when each request has a nearly full-context prefix.
_KV_SHARD_SCRATCH_PREFIX_CONTEXTS = 8
class PageShardSpec(msgspec.Struct, frozen=True):
"""Everything both sides of any transfer need to reproduce the layout."""
@@ -52,7 +59,12 @@ class PageShardSpec(msgspec.Struct, frozen=True):
shard_size: int
page_size: int # physical page size (kernel-visible)
max_prefix_tokens: int # scratch prefix-region capacity, granule-aligned
chunk_tokens: int # scratch chunk-region capacity, granule-aligned
chunk_tokens: int # scratch chunk-region capacity, physical-page-aligned
@property
def scratch_rows(self) -> int:
"""Rows in each slot: assembled prefix, current batch chunk, trash page."""
return self.max_prefix_tokens + self.chunk_tokens + self.page_size
@property
def logical_page_size(self) -> int:
@@ -111,3 +123,67 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
if use_mla_backend:
return get_parallel().attn_tp_group
return cp_group
def get_kv_shard_group_info(
kvc: KVCacheConfigurator,
) -> Tuple[Optional[int], int]:
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
return None, 1
group = get_kv_shard_group(kvc.use_mla_backend)
if group.world_size <= 1:
return None, 1
return group.rank_in_group, group.world_size
def _page_shard_row_bytes(kvc: KVCacheConfigurator) -> int:
model_config = kvc.model_config
kv_size = torch._utils._element_size(kvc.kv_cache_dtype)
if kvc.use_mla_backend:
return (model_config.kv_lora_rank + model_config.qk_rope_head_dim) * kv_size
return (
model_config.get_num_kv_heads(get_parallel().attn_tp_size)
* (model_config.head_dim + model_config.v_head_dim)
* kv_size
)
def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
"""Provision eight full-context prefixes plus the batch's new-token budget.
Prefix capacity must not shrink with the number of new tokens: eight
long cached prefixes can accompany just 1K new tokens each. Round each
context to a shard-group gather span before multiplying, so per-request
padding cannot overflow the region even for an unaligned context length.
This is a fixed eight-context prefix budget, independent of request limits
or chunk size. Both scratch slots are charged against HBM before sizing
persistent KV; large model contexts therefore reduce persistent capacity.
Admission still checks the actual batch's padded prefix and chunk spans.
"""
shard_rank, shard_size = get_kv_shard_group_info(kvc)
if shard_rank is None:
return None
page_size = kvc.page_size
granule = shard_size * page_size
return PageShardSpec(
shard_rank=shard_rank,
shard_size=shard_size,
page_size=page_size,
max_prefix_tokens=_KV_SHARD_SCRATCH_PREFIX_CONTEXTS
* ceil_align(kvc.model_config.context_len, granule),
# chunked_prefill_size is the TOTAL new-token budget for the batch.
chunk_tokens=ceil_align(get_schedule().chunked_prefill_size, page_size),
)
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
the KV budget before pool sizing. Use the same spec as the pool builders:
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
spec = make_page_shard_spec(kvc)
if spec is None:
return 0
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
@@ -113,9 +113,9 @@ class PageInterleaveKVPoolMixin:
assert spec.shard_size == shard_group.world_size
assert spec.shard_rank == shard_group.rank_in_group
assert spec.page_size == self.page_size
# The prefix region must fit N * ceil(prefix_pages / N) pages for any
# prefix, i.e. be a multiple of the full-group span; the chunk region
# is per-page.
# The batch's prefix region is split into equal per-rank gather
# blocks, so it must be a multiple of the full-group span. The chunk
# region is per-page; admission checks both regions against this spec.
assert spec.max_prefix_tokens % spec.logical_page_size == 0
assert spec.chunk_tokens % spec.page_size == 0
@@ -133,7 +133,7 @@ class PageInterleaveKVPoolMixin:
self.kv_gather_stream = self.device_module.Stream()
# Scratch: [prefix | chunk | trash page], double-buffered.
scratch_rows = spec.max_prefix_tokens + spec.chunk_tokens + spec.page_size
scratch_rows = spec.scratch_rows
self._chunk_base = spec.max_prefix_tokens
self._trash_base = spec.max_prefix_tokens + spec.chunk_tokens
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
@@ -171,13 +171,23 @@ class PageInterleaveKVPoolMixin:
self._write_plan: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
self._translate_cache: Dict[Tuple[int, int], torch.Tensor] = {}
scratch_bytes = sum(
tensor.numel() * tensor.element_size()
for slot in self._slots
for tensor in slot.tensors.values()
)
logger.info(
"Page-interleave KV sharding enabled: shard_rank=%d shard_size=%d "
"page_size=%d scratch_rows=%d x2",
"page_size=%d prefix_tokens=%d chunk_tokens=%d scratch_rows=%d x2 "
"(%.2f MiB per rank, charged against the KV budget). "
"Batch admission is limited by this scratch capacity.",
self.shard_rank,
self.shard_size,
spec.page_size,
spec.max_prefix_tokens,
spec.chunk_tokens,
scratch_rows,
scratch_bytes / (1024 * 1024),
)
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
@@ -186,6 +196,28 @@ class PageInterleaveKVPoolMixin:
"(it writes pool rows directly, bypassing the ownership filter)"
)
def register_layer_transfer_counter(self, layer_transfer_counter):
# `None` is the in-tree "disable" signal from the SWA/hybrid wrappers,
# so it has to stay a no-op.
if layer_transfer_counter is None:
super().register_layer_transfer_counter(None)
return
# Layer-wise KV load-back is unsupported here, and refusing it at
# registration is the only safe answer. The gather in `_prefetch_layer`
# reads pool rows directly (`_gather_pairs`), so it never passes
# through the base getters' `wait_until` hook -- and the first gather
# is kicked from `begin_shard_extend`, before any getter runs. Adding
# the wait would not make the combination work: the loader writes whole
# pool rows at LOGICAL indices with no ownership filter, while this
# pool's rows are local physical ones, so ordering the read would only
# turn a race into a silent wrong-row read. Same reason as
# `set_kv_buffer_prefix_valid` above.
raise NotImplementedError(
"layer-wise KV load-back (HiCache / external linker) is "
"unsupported under logical-page KV sharding (the gather reads "
"pool rows directly and the load-back writes them unfiltered)"
)
# ---- subclass hooks -------------------------------------------------------
def _scratch_tensor_specs(self, rows: int) -> Dict[str, torch.Tensor]:
@@ -1,6 +1,8 @@
import os
import unittest
from array import array
from contextlib import contextmanager
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from sglang.test.ci.ci_register import register_cpu_ci
@@ -8,11 +10,24 @@ from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel()
import torch
from sglang.srt.disaggregation.utils import DisaggregationMode
from sglang.srt.environ import envs
from sglang.srt.managers.scheduler_components import invariant_checker
from sglang.srt.managers.scheduler_components.invariant_checker import (
SchedulerInvariantChecker,
)
from sglang.srt.managers.scheduler_components.pool_stats_observer import (
SchedulerPoolStatsObserver,
)
from sglang.srt.mem_cache.allocator.page_interleave import PageInterleavePoolAllocator
from sglang.srt.mem_cache.base_prefix_cache import InsertParams, MatchPrefixParams
from sglang.srt.mem_cache.cache_init_params import CacheInitParams
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
from sglang.srt.mem_cache.radix_cache import RadixKey
from sglang.srt.mem_cache.unified_cache.components import ComponentType
from sglang.srt.mem_cache.unified_radix_cache import UnifiedRadixCache
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
@@ -86,5 +101,220 @@ class TestCheckTreeCacheGate(CustomTestCase):
checker.tree_cache.sanity_check.assert_called_once()
class TestShardedFullPoolInvariant(CustomTestCase):
PAGE_SIZE = 16
SHARD_SIZE = 4
def setUp(self):
super().setUp()
parallel = patch.object(
invariant_checker,
"get_parallel",
return_value=SimpleNamespace(dcp_enabled=False),
)
parallel.start()
self.addCleanup(parallel.stop)
def _make_checker(self):
self.allocator = PageInterleavePoolAllocator(
size=4 * self.PAGE_SIZE,
physical_page_size=self.PAGE_SIZE,
shard_size=self.SHARD_SIZE,
dtype=torch.bfloat16,
device="cpu",
kvcache=None,
need_sort=False,
)
req_pool = ReqToTokenPool(
size=4,
max_context_len=128,
device="cpu",
enable_memory_saver=False,
)
with envs.SGLANG_UNIFIED_RADIX_TREE_CORE_BACKEND.override("python"):
self.cache = UnifiedRadixCache(
CacheInitParams(
disable=False,
req_to_token_pool=req_pool,
token_to_kv_pool_allocator=self.allocator,
page_size=self.PAGE_SIZE,
tree_components=(ComponentType.FULL,),
)
)
self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True)
self.running_batch = self.last_batch
self.chunked_req = None
observer = SchedulerPoolStatsObserver(
tree_cache=self.cache,
token_to_kv_pool_allocator=self.allocator,
req_to_token_pool=req_pool,
session_controller=None,
hisparse_coordinator=None,
is_hybrid_swa=False,
is_hybrid_ssm=False,
enable_hisparse=False,
full_tokens_per_layer=None,
swa_tokens_per_layer=None,
max_total_num_tokens=self.allocator.size,
get_last_batch=lambda: self.last_batch,
get_running_batch=lambda: self.running_batch,
)
return SchedulerInvariantChecker(
is_hybrid_swa=False,
is_hybrid_ssm=False,
disaggregation_mode=DisaggregationMode.PREFILL,
page_size=self.PAGE_SIZE,
full_tokens_per_layer=None,
swa_tokens_per_layer=None,
max_total_num_tokens=self.allocator.size,
tree_cache=self.cache,
token_to_kv_pool_allocator=self.allocator,
req_to_token_pool=req_pool,
pool_stats_observer=observer,
get_last_batch=lambda: self.last_batch,
get_running_batch=lambda: self.running_batch,
get_chunked_req=lambda: self.chunked_req,
scheduler_stage_metrics=None,
)
def _allocate(self, tokens, *, prefix=0, last_loc=-1, rotation_base=None):
prefix_lens = torch.tensor([prefix], dtype=torch.int64)
seq_lens = torch.tensor([prefix + tokens], dtype=torch.int64)
bases = [rotation_base]
locs = self.allocator.alloc_extend(
prefix_lens=prefix_lens,
prefix_lens_cpu=prefix_lens,
seq_lens=seq_lens,
seq_lens_cpu=seq_lens,
last_loc=torch.tensor([last_loc], dtype=torch.int64),
extend_num_tokens=tokens,
rotation_bases=bases,
)
self.assertIsNotNone(locs)
return locs, bases[0]
def _cache_prefix(self, tokens, *, protected=False):
locs, base = self._allocate(tokens)
key = RadixKey(array("q", range(tokens)))
self.cache.insert(InsertParams(key=key, value=locs, rotation_base=base))
if protected:
matched = self.cache.match_prefix(MatchPrefixParams(key=key))
self.cache.inc_lock_ref(matched.last_device_node)
return locs, base
def _active_partial_page(self, checker):
prefix_locs, base = self._cache_prefix(self.PAGE_SIZE, protected=True)
tail_locs, _ = self._allocate(
3,
prefix=self.PAGE_SIZE,
last_loc=prefix_locs[-1].item(),
rotation_base=base,
)
locs = torch.cat((prefix_locs, tail_locs))
checker.req_to_token_pool.req_to_token[0, : len(locs)] = locs
req = SimpleNamespace(
kv=SimpleNamespace(
holds_kv=True,
req_pool_idx=0,
kv_allocated_len=len(locs),
cache_protected_len=self.PAGE_SIZE,
),
beam_group=None,
)
self.last_batch = SimpleNamespace(reqs=[req], is_empty=lambda: False)
self.running_batch = self.last_batch
return req
def test_balanced_and_skewed_cache_ownership_conserve_full_pool(self):
for cached_pages in (1, self.SHARD_SIZE):
for protected in (False, True):
with self.subTest(cached_pages=cached_pages, protected=protected):
checker = self._make_checker()
cached_tokens = cached_pages * self.PAGE_SIZE
self._cache_prefix(cached_tokens, protected=protected)
self.assertEqual(
self.cache.protected_size() + self.cache.evictable_size(),
cached_tokens,
)
stats = checker.pool_stats_observer.get_pool_stats()
if cached_pages == 1:
self.assertLess(
stats.full_available_size,
self.allocator.aggregate_free_size(),
)
else:
self.assertEqual(
stats.full_available_size,
self.allocator.aggregate_free_size(),
)
leak, msg = checker._check_full_pool(stats)
self.assertFalse(leak, msg)
self.assertIn("class_free_pages=", msg)
def test_idle_check_detects_one_unowned_page(self):
checker = self._make_checker()
self._cache_prefix(self.PAGE_SIZE)
self._allocate(self.PAGE_SIZE) # Deliberately no request or cache owner.
leak, messages = checker._check_all_pools(
checker.pool_stats_observer.get_pool_stats()
)
self.assertTrue(leak, messages)
def test_busy_check_detects_one_unowned_page(self):
checker = self._make_checker()
self._active_partial_page(checker)
self._allocate(self.PAGE_SIZE) # Leak in addition to the active tail.
self.assertEqual(checker._get_total_uncached_sizes(), (self.PAGE_SIZE, 0))
with self.assertRaisesRegex(AssertionError, "Full Pool Mem Leak Detected"):
checker.self_check_during_busy()
def test_partial_active_page_is_accounted_without_slack_exemption(self):
checker = self._make_checker()
req = self._active_partial_page(checker)
# Count the same request only once, including when it is parked
# between prefill chunks instead of appearing in either batch.
for parked in (False, True):
with self.subTest(parked=parked):
self.chunked_req = req
if parked:
self.last_batch = SimpleNamespace(reqs=[], is_empty=lambda: True)
self.running_batch = self.last_batch
uncached, swa_uncached = checker._get_total_uncached_sizes()
self.assertEqual((uncached, swa_uncached), (self.PAGE_SIZE, 0))
stats = checker.pool_stats_observer.get_pool_stats()
leak, msg = checker._check_full_pool(stats, uncached=uncached)
self.assertFalse(leak, msg)
self.assertNotIn("slack_allowed", msg)
with envs.SGLANG_CHECK_KV_PAGE_INVARIANTS.override(False):
checker.self_check_during_busy()
def test_misaligned_evictable_counter_is_not_rounded_up(self):
checker = self._make_checker()
self._cache_prefix(self.PAGE_SIZE)
self.cache.tree_core.component_evictable_size_[ComponentType.FULL] -= 1
leak, msg = checker._check_full_pool(
checker.pool_stats_observer.get_pool_stats()
)
self.assertTrue(leak, msg)
self.assertIn(f"evictable={self.PAGE_SIZE - 1}", msg)
def test_compensating_misaligned_counters_still_report_alignment_error(self):
checker = self._make_checker()
self._cache_prefix(self.PAGE_SIZE)
core = self.cache.tree_core
core.component_evictable_size_[ComponentType.FULL] -= 1
core.component_protected_size_[ComponentType.FULL] += 1
# Their sum still matches the allocated page; neither individual
# counter can legitimately contain a fraction of a physical page.
self.assertEqual(
self.cache.evictable_size() + self.cache.protected_size(), self.PAGE_SIZE
)
leak, msg = checker._check_full_pool(
checker.pool_stats_observer.get_pool_stats()
)
self.assertTrue(leak, msg)
self.assertIn("align", msg.lower())
if __name__ == "__main__":
unittest.main()
@@ -2,6 +2,7 @@
import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
import torch
@@ -95,6 +96,26 @@ class TestKVPageInvariants(CustomTestCase):
with self.assertRaises(ValueError):
chk._check_kv_page_invariants()
def test_classed_allocator_without_flat_free_list_is_checked(self):
chk, rtt, _tc, alloc = _make_checker(page_size=4, row_width=8)
alloc.free_pages = None
alloc.get_all_free_pages = MagicMock(
return_value=torch.tensor([5, 6], dtype=torch.int64)
)
rtt[0, :1] = torch.tensor([5 * alloc.page_size])
chk.get_last_batch = lambda: SimpleNamespace(
reqs=[_FakeOwner(0, 1, 1, rid="classed")]
)
with patch(
"sglang.srt.managers.scheduler_components.invariant_checker.page_interleave_shard_size",
return_value=2,
):
with self.assertRaisesRegex(ValueError, "use-after-free"):
chk._check_kv_page_invariants()
alloc.get_all_free_pages.assert_called()
def test_free_pool_duplicate_raises(self):
chk, rtt, tc, alloc = _make_checker(free_pages=torch.tensor([3, 3, 4]))
rtt[0, :1] = torch.tensor([10 * _PAGE_SIZE]) # owner page 10, not in free
@@ -43,6 +43,7 @@ import os
import unittest
import unittest.mock
from array import array
from contextlib import contextmanager
from types import SimpleNamespace
import torch
@@ -52,6 +53,7 @@ from sglang.srt.distributed import (
init_distributed_environment,
initialize_model_parallel,
)
from sglang.srt.mem_cache import page_interleave
from sglang.srt.mem_cache.allocator.page_interleave import (
PageInterleavePoolAllocator,
page_interleave_shard_size,
@@ -69,7 +71,9 @@ from sglang.srt.mem_cache.memory_pool import MHATokenToKVPool, ReqToTokenPool
from sglang.srt.mem_cache.page_interleave import (
PageInterleavePlacement,
PageShardSpec,
compute_page_shard_scratch_bytes,
get_kv_shard_group,
make_page_shard_spec,
)
from sglang.srt.mem_cache.page_interleave_pool import (
PageInterleaveKVPoolMixin,
@@ -93,6 +97,147 @@ PS = 16 # physical page size
GS = N * PS # full-group span (N physical pages)
class TestPageShardScratchSizing(CustomTestCase):
@contextmanager
def _fixture(
self,
*,
enabled=True,
draft=False,
shard_size=4,
use_mla=True,
context_len=4096,
chunk_tokens=256,
prefill_max_requests=None,
max_running_requests=None,
dp_size=1,
tp_size=4,
num_kv_heads=8,
head_dim=96,
v_head_dim=64,
):
group = SimpleNamespace(world_size=shard_size, rank_in_group=0)
parallel = SimpleNamespace(
enable_kv_cache_sharding=enabled,
attn_cp_group=(SimpleNamespace(world_size=1) if use_mla else group),
attn_tp_group=group,
attn_tp_size=tp_size,
attn_dp_size=dp_size,
)
schedule = SimpleNamespace(
chunked_prefill_size=chunk_tokens,
prefill_max_requests=prefill_max_requests,
max_running_requests=max_running_requests,
)
kvc = SimpleNamespace(
is_draft_worker=draft,
use_mla_backend=use_mla,
page_size=16,
kv_cache_dtype=torch.bfloat16,
model_config=SimpleNamespace(
context_len=context_len,
kv_lora_rank=16,
qk_rope_head_dim=8,
head_dim=head_dim,
v_head_dim=v_head_dim,
get_num_kv_heads=lambda tp: num_kv_heads // tp,
),
)
with (
unittest.mock.patch.object(
page_interleave, "get_parallel", return_value=parallel
),
unittest.mock.patch.object(
page_interleave, "get_schedule", return_value=schedule
),
):
yield kvc
def test_disabled_draft_and_trivial_group_need_no_scratch(self):
for options in (
{"enabled": False},
{"draft": True},
{"shard_size": 1},
):
with self.subTest(options=options), self._fixture(**options) as kvc:
self.assertIsNone(make_page_shard_spec(kvc))
self.assertEqual(compute_page_shard_scratch_bytes(kvc), 0)
def test_eight_contexts_are_reserved_independently_of_batch_limits(self):
for options in (
{},
{"prefill_max_requests": 1},
{"max_running_requests": 1},
{"max_running_requests": 4, "dp_size": 4},
{"chunk_tokens": 16},
{"chunk_tokens": 8192, "prefill_max_requests": 32},
):
with self.subTest(options=options), self._fixture(**options) as kvc:
spec = make_page_shard_spec(kvc)
self.assertEqual(spec.max_prefix_tokens, 8 * 4096)
self.assertEqual(spec.chunk_tokens, options.get("chunk_tokens", 256))
self.assertEqual(
spec.scratch_rows, spec.max_prefix_tokens + spec.chunk_tokens + 16
)
def test_each_context_is_aligned_before_multiplying_by_eight(self):
with self._fixture(context_len=4097, chunk_tokens=33) as kvc:
spec = make_page_shard_spec(kvc)
# Each request's prefix gather pads independently to 64 tokens.
# Aligning only 8 * context_len would reserve too few rows.
self.assertEqual(spec.max_prefix_tokens, 8 * 4160)
self.assertGreater(spec.max_prefix_tokens, 32832)
self.assertEqual(spec.chunk_tokens, 48)
self.assertEqual(spec.max_prefix_tokens % spec.logical_page_size, 0)
def test_wide_gqa_keeps_eight_contexts_without_a_byte_cap(self):
with self._fixture(
use_mla=False,
context_len=65536,
chunk_tokens=4096,
tp_size=1,
num_kv_heads=64,
head_dim=128,
v_head_dim=128,
) as kvc:
spec = make_page_shard_spec(kvc)
estimated = compute_page_shard_scratch_bytes(kvc)
# Numerical sizing only: do not allocate this large scratch on CPU.
# 64 heads * (128 K + 128 V) * two-byte BF16 = 32 KiB per row.
self.assertEqual(spec.max_prefix_tokens, 8 * 65536)
self.assertEqual(spec.chunk_tokens, 4096)
self.assertEqual(estimated, 2 * (8 * 65536 + 4096 + 16) * 32768)
self.assertGreater(estimated, 64 << 20)
def test_estimate_matches_actual_mla_and_gqa_scratch_tensors(self):
for use_mla in (True, False):
with (
self.subTest(use_mla=use_mla),
self._fixture(use_mla=use_mla, context_len=128, tp_size=2) as kvc,
):
spec = make_page_shard_spec(kvc)
estimated = compute_page_shard_scratch_bytes(kvc)
pool = SimpleNamespace(
device="cpu",
store_dtype=kvc.kv_cache_dtype,
kv_cache_dim=24,
head_num=4,
head_dim=96,
v_head_dim=64,
)
pool_cls = (
PageInterleaveMLATokenToKVPool
if use_mla
else PageInterleaveMHATokenToKVPool
)
tensors = pool_cls._scratch_tensor_specs(pool, spec.scratch_rows)
actual = 2 * sum(
tensor.numel() * tensor.element_size()
for tensor in tensors.values()
)
self.assertEqual(estimated, actual)
def _make_spec(shard_rank=0, max_prefix_groups=64, chunk_pages=32):
return PageShardSpec(
shard_rank=shard_rank,
@@ -1016,6 +1161,35 @@ class TestBeginShardExtendPlan(CustomTestCase):
self.assertIn("cyclic", str(ctx.exception))
class TestLayerTransferCounterRefused(CustomTestCase):
"""Layer-wise KV load-back must fail loud at registration.
The gather reads pool rows directly (`_gather_pairs`), so it never passes
through the base getters' `layer_transfer_counter.wait_until` hook, and
`begin_shard_extend` kicks the first gather before any getter runs.
"""
class _Base:
def register_layer_transfer_counter(self, counter):
self.counter = counter
class _Pool(PageInterleaveKVPoolMixin, _Base):
def __init__(self):
pass
def test_a_real_counter_is_refused(self):
pool = self._Pool()
with self.assertRaises(NotImplementedError) as cm:
pool.register_layer_transfer_counter(object())
self.assertIn("logical-page KV sharding", str(cm.exception))
def test_none_stays_a_no_op(self):
"""The SWA/hybrid wrappers disable the counter by passing None."""
pool = self._Pool()
pool.register_layer_transfer_counter(None)
self.assertIsNone(pool.counter)
class TestScratchTranslation(CustomTestCase):
def _plan(self, base=2, n_prefix=7, n_chunk=9, rank=1):
pages = _chain_pages(base=base, n_pages=n_prefix + n_chunk)