[kv-shard 3/4] Enable Control plane (#38468)

This commit is contained in:
Shunkangz
2026-09-22 19:57:44 +08:00
committed by GitHub
parent 2524b61f9b
commit ddebc52f23
6 changed files with 596 additions and 24 deletions
@@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import (
SchedulerPoolStatsObserver,
)
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
@@ -117,29 +118,60 @@ class SchedulerInvariantChecker:
session_held = self.pool_stats_observer.session_held_tokens()
total = self.max_total_num_tokens
full_evictable_size = ps.full_evictable_size
if get_parallel().dcp_enabled and allocator.page_size > 1:
# DCP stores logical tokens in widened physical pages. Prefix cache
# counters are logical-token based, while the allocator frees whole
# physical pages, so round cached tokens up to physical page units.
full_evictable_size = (
(full_evictable_size + allocator.page_size - 1)
// allocator.page_size
* allocator.page_size
full_available = ps.full_available_size
class_watermark_msg = ""
kv_shard_size = page_interleave_shard_size(allocator)
dcp_page_alloc = (
kv_shard_size == 1
and get_parallel().dcp_enabled
and allocator.page_size > 1
)
if kv_shard_size > 1:
# Conservation needs every free page, not the min-class capacity
# floor used for admission. Sharding keeps the tree and allocator
# page sizes equal, so cached counts must already be page-aligned.
full_available = allocator.aggregate_free_size()
class_watermark_msg = (
f", class_free_pages={allocator.class_free_page_counts()}"
)
elif dcp_page_alloc:
# Preserve the existing DCP-only accounting adjustment. DCP widens
# allocator pages; its scheduler counters still use per-rank units.
full_evictable_size = ceil_align(full_evictable_size, allocator.page_size)
leak, msg = self._check_pool_invariant(
"full",
ps.full_available_size,
full_available,
full_evictable_size,
protected,
session_held,
total,
uncached,
)
if leak and get_parallel().dcp_enabled and allocator.page_size > 1:
# Radix/Mamba cache accounting is logical-token based while DCP full
# KV allocation is physical-page based. Partial physical pages can
# leave a small page-level slack even when all pages are owned by
# either the allocator or the prefix cache.
msg += class_watermark_msg
if kv_shard_size > 1:
# Partial active pages are already rounded in
# _get_total_uncached_sizes. Rounding cached counts or accepting
# arbitrary slack here would hide corruption and lost pages.
unaligned = {
name: value
for name, value in (
("available", full_available),
("evictable", full_evictable_size),
("protected", protected),
("session_held", session_held),
("uncached", uncached),
("total", total),
)
if value % allocator.page_size != 0
}
if unaligned:
return True, (
f"{msg}, unaligned_sharded_counts={unaligned}, "
f"page_size={allocator.page_size}"
)
if leak and dcp_page_alloc:
# Legacy DCP counters use different page/capacity units. Keep that
# compatibility path separate from exact KV-shard conservation.
return False, f"{msg}, dcp_physical_page_slack_allowed=True"
return leak, msg
@@ -380,14 +412,17 @@ class SchedulerInvariantChecker:
idx = torch.as_tensor([rpi for _, rpi, _ in active], device=rtt.device)
allocs = torch.as_tensor([al for _, _, al in active], device=rtt.device)
mask = torch.arange(row_width, device=rtt.device)[None, :] < allocs[:, None]
owner_pages = rtt[idx][mask] // self.page_size
owner_locs = rtt[idx][mask]
# Sub-allocators to check: a flat allocator is its own single sub; a
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator.
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator;
# the classed sharding allocator keeps per-class lists instead of a
# flat free_pages and exposes them through get_all_free_pages().
alloc = self.token_to_kv_pool_allocator
sub_allocs = (
[alloc]
if getattr(alloc, "free_pages", None) is not None
or page_interleave_shard_size(alloc) > 1
else [
sub
for n in ("full_attn_allocator", "swa_attn_allocator")
@@ -398,6 +433,10 @@ class SchedulerInvariantChecker:
if not sub_allocs:
return
# Page ids in the ALLOCATOR's page units (== the physical page for
# the classed sharding allocator, whose free lists hold logical page
# ids in the same unit).
owner_pages = owner_locs // sub_allocs[0].page_size
# Check B: every sub-pool's free set has no duplicate pages.
for i, sub in enumerate(sub_allocs):
free = sub.get_all_free_pages()
+79 -3
View File
@@ -32,18 +32,25 @@ therefore can be striped without extra compute-time communication:
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Optional, Tuple
import msgspec
import torch
from sglang.srt.runtime_context import get_parallel
from sglang.srt.runtime_context import get_parallel, get_schedule
from sglang.srt.utils.common import ceil_align
if TYPE_CHECKING:
from sglang.srt.distributed.parallel_state import GroupCoordinator
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
logger = logging.getLogger(__name__)
# Keep enough assembled prefix KV for eight long-context requests. The
# new-token compute budget is independent: eight 1K extensions can fill an
# 8K prefill chunk even when each request has a nearly full-context prefix.
_KV_SHARD_SCRATCH_PREFIX_CONTEXTS = 8
class PageShardSpec(msgspec.Struct, frozen=True):
"""Everything both sides of any transfer need to reproduce the layout."""
@@ -52,7 +59,12 @@ class PageShardSpec(msgspec.Struct, frozen=True):
shard_size: int
page_size: int # physical page size (kernel-visible)
max_prefix_tokens: int # scratch prefix-region capacity, granule-aligned
chunk_tokens: int # scratch chunk-region capacity, granule-aligned
chunk_tokens: int # scratch chunk-region capacity, physical-page-aligned
@property
def scratch_rows(self) -> int:
"""Rows in each slot: assembled prefix, current batch chunk, trash page."""
return self.max_prefix_tokens + self.chunk_tokens + self.page_size
@property
def logical_page_size(self) -> int:
@@ -111,3 +123,67 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
if use_mla_backend:
return get_parallel().attn_tp_group
return cp_group
def get_kv_shard_group_info(
kvc: KVCacheConfigurator,
) -> Tuple[Optional[int], int]:
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
return None, 1
group = get_kv_shard_group(kvc.use_mla_backend)
if group.world_size <= 1:
return None, 1
return group.rank_in_group, group.world_size
def _page_shard_row_bytes(kvc: KVCacheConfigurator) -> int:
model_config = kvc.model_config
kv_size = torch._utils._element_size(kvc.kv_cache_dtype)
if kvc.use_mla_backend:
return (model_config.kv_lora_rank + model_config.qk_rope_head_dim) * kv_size
return (
model_config.get_num_kv_heads(get_parallel().attn_tp_size)
* (model_config.head_dim + model_config.v_head_dim)
* kv_size
)
def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
"""Provision eight full-context prefixes plus the batch's new-token budget.
Prefix capacity must not shrink with the number of new tokens: eight
long cached prefixes can accompany just 1K new tokens each. Round each
context to a shard-group gather span before multiplying, so per-request
padding cannot overflow the region even for an unaligned context length.
This is a fixed eight-context prefix budget, independent of request limits
or chunk size. Both scratch slots are charged against HBM before sizing
persistent KV; large model contexts therefore reduce persistent capacity.
Admission still checks the actual batch's padded prefix and chunk spans.
"""
shard_rank, shard_size = get_kv_shard_group_info(kvc)
if shard_rank is None:
return None
page_size = kvc.page_size
granule = shard_size * page_size
return PageShardSpec(
shard_rank=shard_rank,
shard_size=shard_size,
page_size=page_size,
max_prefix_tokens=_KV_SHARD_SCRATCH_PREFIX_CONTEXTS
* ceil_align(kvc.model_config.context_len, granule),
# chunked_prefill_size is the TOTAL new-token budget for the batch.
chunk_tokens=ceil_align(get_schedule().chunked_prefill_size, page_size),
)
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
the KV budget before pool sizing. Use the same spec as the pool builders:
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
spec = make_page_shard_spec(kvc)
if spec is None:
return 0
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
@@ -113,9 +113,9 @@ class PageInterleaveKVPoolMixin:
assert spec.shard_size == shard_group.world_size
assert spec.shard_rank == shard_group.rank_in_group
assert spec.page_size == self.page_size
# The prefix region must fit N * ceil(prefix_pages / N) pages for any
# prefix, i.e. be a multiple of the full-group span; the chunk region
# is per-page.
# The batch's prefix region is split into equal per-rank gather
# blocks, so it must be a multiple of the full-group span. The chunk
# region is per-page; admission checks both regions against this spec.
assert spec.max_prefix_tokens % spec.logical_page_size == 0
assert spec.chunk_tokens % spec.page_size == 0
@@ -133,7 +133,7 @@ class PageInterleaveKVPoolMixin:
self.kv_gather_stream = self.device_module.Stream()
# Scratch: [prefix | chunk | trash page], double-buffered.
scratch_rows = spec.max_prefix_tokens + spec.chunk_tokens + spec.page_size
scratch_rows = spec.scratch_rows
self._chunk_base = spec.max_prefix_tokens
self._trash_base = spec.max_prefix_tokens + spec.chunk_tokens
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
@@ -171,13 +171,23 @@ class PageInterleaveKVPoolMixin:
self._write_plan: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
self._translate_cache: Dict[Tuple[int, int], torch.Tensor] = {}
scratch_bytes = sum(
tensor.numel() * tensor.element_size()
for slot in self._slots
for tensor in slot.tensors.values()
)
logger.info(
"Page-interleave KV sharding enabled: shard_rank=%d shard_size=%d "
"page_size=%d scratch_rows=%d x2",
"page_size=%d prefix_tokens=%d chunk_tokens=%d scratch_rows=%d x2 "
"(%.2f MiB per rank, charged against the KV budget). "
"Batch admission is limited by this scratch capacity.",
self.shard_rank,
self.shard_size,
spec.page_size,
spec.max_prefix_tokens,
spec.chunk_tokens,
scratch_rows,
scratch_bytes / (1024 * 1024),
)
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
@@ -186,6 +196,28 @@ class PageInterleaveKVPoolMixin:
"(it writes pool rows directly, bypassing the ownership filter)"
)
def register_layer_transfer_counter(self, layer_transfer_counter):
# `None` is the in-tree "disable" signal from the SWA/hybrid wrappers,
# so it has to stay a no-op.
if layer_transfer_counter is None:
super().register_layer_transfer_counter(None)
return
# Layer-wise KV load-back is unsupported here, and refusing it at
# registration is the only safe answer. The gather in `_prefetch_layer`
# reads pool rows directly (`_gather_pairs`), so it never passes
# through the base getters' `wait_until` hook -- and the first gather
# is kicked from `begin_shard_extend`, before any getter runs. Adding
# the wait would not make the combination work: the loader writes whole
# pool rows at LOGICAL indices with no ownership filter, while this
# pool's rows are local physical ones, so ordering the read would only
# turn a race into a silent wrong-row read. Same reason as
# `set_kv_buffer_prefix_valid` above.
raise NotImplementedError(
"layer-wise KV load-back (HiCache / external linker) is "
"unsupported under logical-page KV sharding (the gather reads "
"pool rows directly and the load-back writes them unfiltered)"
)
# ---- subclass hooks -------------------------------------------------------
def _scratch_tensor_specs(self, rows: int) -> Dict[str, torch.Tensor]: