[kv-shard 3/4] Enable Control plane (#38468)
This commit is contained in:
@@ -21,6 +21,7 @@ from sglang.srt.managers.scheduler_components.pool_stats_observer import (
|
||||
SchedulerPoolStatsObserver,
|
||||
)
|
||||
from sglang.srt.mem_cache.allocator import BaseTokenToKVPoolAllocator
|
||||
from sglang.srt.mem_cache.allocator.page_interleave import page_interleave_shard_size
|
||||
from sglang.srt.mem_cache.allocator.swa import is_swa_req_ring
|
||||
from sglang.srt.mem_cache.base_prefix_cache import BasePrefixCache
|
||||
from sglang.srt.mem_cache.memory_pool import ReqToTokenPool
|
||||
@@ -117,29 +118,60 @@ class SchedulerInvariantChecker:
|
||||
session_held = self.pool_stats_observer.session_held_tokens()
|
||||
total = self.max_total_num_tokens
|
||||
full_evictable_size = ps.full_evictable_size
|
||||
if get_parallel().dcp_enabled and allocator.page_size > 1:
|
||||
# DCP stores logical tokens in widened physical pages. Prefix cache
|
||||
# counters are logical-token based, while the allocator frees whole
|
||||
# physical pages, so round cached tokens up to physical page units.
|
||||
full_evictable_size = (
|
||||
(full_evictable_size + allocator.page_size - 1)
|
||||
// allocator.page_size
|
||||
* allocator.page_size
|
||||
full_available = ps.full_available_size
|
||||
class_watermark_msg = ""
|
||||
kv_shard_size = page_interleave_shard_size(allocator)
|
||||
dcp_page_alloc = (
|
||||
kv_shard_size == 1
|
||||
and get_parallel().dcp_enabled
|
||||
and allocator.page_size > 1
|
||||
)
|
||||
if kv_shard_size > 1:
|
||||
# Conservation needs every free page, not the min-class capacity
|
||||
# floor used for admission. Sharding keeps the tree and allocator
|
||||
# page sizes equal, so cached counts must already be page-aligned.
|
||||
full_available = allocator.aggregate_free_size()
|
||||
class_watermark_msg = (
|
||||
f", class_free_pages={allocator.class_free_page_counts()}"
|
||||
)
|
||||
elif dcp_page_alloc:
|
||||
# Preserve the existing DCP-only accounting adjustment. DCP widens
|
||||
# allocator pages; its scheduler counters still use per-rank units.
|
||||
full_evictable_size = ceil_align(full_evictable_size, allocator.page_size)
|
||||
leak, msg = self._check_pool_invariant(
|
||||
"full",
|
||||
ps.full_available_size,
|
||||
full_available,
|
||||
full_evictable_size,
|
||||
protected,
|
||||
session_held,
|
||||
total,
|
||||
uncached,
|
||||
)
|
||||
if leak and get_parallel().dcp_enabled and allocator.page_size > 1:
|
||||
# Radix/Mamba cache accounting is logical-token based while DCP full
|
||||
# KV allocation is physical-page based. Partial physical pages can
|
||||
# leave a small page-level slack even when all pages are owned by
|
||||
# either the allocator or the prefix cache.
|
||||
msg += class_watermark_msg
|
||||
if kv_shard_size > 1:
|
||||
# Partial active pages are already rounded in
|
||||
# _get_total_uncached_sizes. Rounding cached counts or accepting
|
||||
# arbitrary slack here would hide corruption and lost pages.
|
||||
unaligned = {
|
||||
name: value
|
||||
for name, value in (
|
||||
("available", full_available),
|
||||
("evictable", full_evictable_size),
|
||||
("protected", protected),
|
||||
("session_held", session_held),
|
||||
("uncached", uncached),
|
||||
("total", total),
|
||||
)
|
||||
if value % allocator.page_size != 0
|
||||
}
|
||||
if unaligned:
|
||||
return True, (
|
||||
f"{msg}, unaligned_sharded_counts={unaligned}, "
|
||||
f"page_size={allocator.page_size}"
|
||||
)
|
||||
if leak and dcp_page_alloc:
|
||||
# Legacy DCP counters use different page/capacity units. Keep that
|
||||
# compatibility path separate from exact KV-shard conservation.
|
||||
return False, f"{msg}, dcp_physical_page_slack_allowed=True"
|
||||
return leak, msg
|
||||
|
||||
@@ -380,14 +412,17 @@ class SchedulerInvariantChecker:
|
||||
idx = torch.as_tensor([rpi for _, rpi, _ in active], device=rtt.device)
|
||||
allocs = torch.as_tensor([al for _, _, al in active], device=rtt.device)
|
||||
mask = torch.arange(row_width, device=rtt.device)[None, :] < allocs[:, None]
|
||||
owner_pages = rtt[idx][mask] // self.page_size
|
||||
owner_locs = rtt[idx][mask]
|
||||
|
||||
# Sub-allocators to check: a flat allocator is its own single sub; a
|
||||
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator.
|
||||
# hybrid-SWA wrapper exposes full_attn_allocator + swa_attn_allocator;
|
||||
# the classed sharding allocator keeps per-class lists instead of a
|
||||
# flat free_pages and exposes them through get_all_free_pages().
|
||||
alloc = self.token_to_kv_pool_allocator
|
||||
sub_allocs = (
|
||||
[alloc]
|
||||
if getattr(alloc, "free_pages", None) is not None
|
||||
or page_interleave_shard_size(alloc) > 1
|
||||
else [
|
||||
sub
|
||||
for n in ("full_attn_allocator", "swa_attn_allocator")
|
||||
@@ -398,6 +433,10 @@ class SchedulerInvariantChecker:
|
||||
if not sub_allocs:
|
||||
return
|
||||
|
||||
# Page ids in the ALLOCATOR's page units (== the physical page for
|
||||
# the classed sharding allocator, whose free lists hold logical page
|
||||
# ids in the same unit).
|
||||
owner_pages = owner_locs // sub_allocs[0].page_size
|
||||
# Check B: every sub-pool's free set has no duplicate pages.
|
||||
for i, sub in enumerate(sub_allocs):
|
||||
free = sub.get_all_free_pages()
|
||||
|
||||
@@ -32,18 +32,25 @@ therefore can be striped without extra compute-time communication:
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, Optional, Tuple
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
from sglang.srt.runtime_context import get_parallel, get_schedule
|
||||
from sglang.srt.utils.common import ceil_align
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.distributed.parallel_state import GroupCoordinator
|
||||
from sglang.srt.mem_cache.kv_cache_configurator import KVCacheConfigurator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Keep enough assembled prefix KV for eight long-context requests. The
|
||||
# new-token compute budget is independent: eight 1K extensions can fill an
|
||||
# 8K prefill chunk even when each request has a nearly full-context prefix.
|
||||
_KV_SHARD_SCRATCH_PREFIX_CONTEXTS = 8
|
||||
|
||||
|
||||
class PageShardSpec(msgspec.Struct, frozen=True):
|
||||
"""Everything both sides of any transfer need to reproduce the layout."""
|
||||
@@ -52,7 +59,12 @@ class PageShardSpec(msgspec.Struct, frozen=True):
|
||||
shard_size: int
|
||||
page_size: int # physical page size (kernel-visible)
|
||||
max_prefix_tokens: int # scratch prefix-region capacity, granule-aligned
|
||||
chunk_tokens: int # scratch chunk-region capacity, granule-aligned
|
||||
chunk_tokens: int # scratch chunk-region capacity, physical-page-aligned
|
||||
|
||||
@property
|
||||
def scratch_rows(self) -> int:
|
||||
"""Rows in each slot: assembled prefix, current batch chunk, trash page."""
|
||||
return self.max_prefix_tokens + self.chunk_tokens + self.page_size
|
||||
|
||||
@property
|
||||
def logical_page_size(self) -> int:
|
||||
@@ -111,3 +123,67 @@ def get_kv_shard_group(use_mla_backend: bool) -> GroupCoordinator:
|
||||
if use_mla_backend:
|
||||
return get_parallel().attn_tp_group
|
||||
return cp_group
|
||||
|
||||
|
||||
def get_kv_shard_group_info(
|
||||
kvc: KVCacheConfigurator,
|
||||
) -> Tuple[Optional[int], int]:
|
||||
"""``(shard_rank, shard_size)`` for the KV pool; ``(None, 1)`` disables."""
|
||||
if kvc.is_draft_worker or not get_parallel().enable_kv_cache_sharding:
|
||||
return None, 1
|
||||
group = get_kv_shard_group(kvc.use_mla_backend)
|
||||
if group.world_size <= 1:
|
||||
return None, 1
|
||||
return group.rank_in_group, group.world_size
|
||||
|
||||
|
||||
def _page_shard_row_bytes(kvc: KVCacheConfigurator) -> int:
|
||||
model_config = kvc.model_config
|
||||
kv_size = torch._utils._element_size(kvc.kv_cache_dtype)
|
||||
if kvc.use_mla_backend:
|
||||
return (model_config.kv_lora_rank + model_config.qk_rope_head_dim) * kv_size
|
||||
return (
|
||||
model_config.get_num_kv_heads(get_parallel().attn_tp_size)
|
||||
* (model_config.head_dim + model_config.v_head_dim)
|
||||
* kv_size
|
||||
)
|
||||
|
||||
|
||||
def make_page_shard_spec(kvc: KVCacheConfigurator) -> Optional[PageShardSpec]:
|
||||
"""Provision eight full-context prefixes plus the batch's new-token budget.
|
||||
|
||||
Prefix capacity must not shrink with the number of new tokens: eight
|
||||
long cached prefixes can accompany just 1K new tokens each. Round each
|
||||
context to a shard-group gather span before multiplying, so per-request
|
||||
padding cannot overflow the region even for an unaligned context length.
|
||||
|
||||
This is a fixed eight-context prefix budget, independent of request limits
|
||||
or chunk size. Both scratch slots are charged against HBM before sizing
|
||||
persistent KV; large model contexts therefore reduce persistent capacity.
|
||||
Admission still checks the actual batch's padded prefix and chunk spans.
|
||||
"""
|
||||
shard_rank, shard_size = get_kv_shard_group_info(kvc)
|
||||
if shard_rank is None:
|
||||
return None
|
||||
|
||||
page_size = kvc.page_size
|
||||
granule = shard_size * page_size
|
||||
return PageShardSpec(
|
||||
shard_rank=shard_rank,
|
||||
shard_size=shard_size,
|
||||
page_size=page_size,
|
||||
max_prefix_tokens=_KV_SHARD_SCRATCH_PREFIX_CONTEXTS
|
||||
* ceil_align(kvc.model_config.context_len, granule),
|
||||
# chunked_prefill_size is the TOTAL new-token budget for the batch.
|
||||
chunk_tokens=ceil_align(get_schedule().chunked_prefill_size, page_size),
|
||||
)
|
||||
|
||||
|
||||
def compute_page_shard_scratch_bytes(kvc: KVCacheConfigurator) -> int:
|
||||
"""Fixed HBM cost of the double-buffered assembly scratch, charged against
|
||||
the KV budget before pool sizing. Use the same spec as the pool builders:
|
||||
two slots, each holding ONE layer's ``[prefix | chunk | trash page]``."""
|
||||
spec = make_page_shard_spec(kvc)
|
||||
if spec is None:
|
||||
return 0
|
||||
return 2 * spec.scratch_rows * _page_shard_row_bytes(kvc)
|
||||
|
||||
@@ -113,9 +113,9 @@ class PageInterleaveKVPoolMixin:
|
||||
assert spec.shard_size == shard_group.world_size
|
||||
assert spec.shard_rank == shard_group.rank_in_group
|
||||
assert spec.page_size == self.page_size
|
||||
# The prefix region must fit N * ceil(prefix_pages / N) pages for any
|
||||
# prefix, i.e. be a multiple of the full-group span; the chunk region
|
||||
# is per-page.
|
||||
# The batch's prefix region is split into equal per-rank gather
|
||||
# blocks, so it must be a multiple of the full-group span. The chunk
|
||||
# region is per-page; admission checks both regions against this spec.
|
||||
assert spec.max_prefix_tokens % spec.logical_page_size == 0
|
||||
assert spec.chunk_tokens % spec.page_size == 0
|
||||
|
||||
@@ -133,7 +133,7 @@ class PageInterleaveKVPoolMixin:
|
||||
self.kv_gather_stream = self.device_module.Stream()
|
||||
|
||||
# Scratch: [prefix | chunk | trash page], double-buffered.
|
||||
scratch_rows = spec.max_prefix_tokens + spec.chunk_tokens + spec.page_size
|
||||
scratch_rows = spec.scratch_rows
|
||||
self._chunk_base = spec.max_prefix_tokens
|
||||
self._trash_base = spec.max_prefix_tokens + spec.chunk_tokens
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
@@ -171,13 +171,23 @@ class PageInterleaveKVPoolMixin:
|
||||
self._write_plan: Optional[Tuple[torch.Tensor, torch.Tensor]] = None
|
||||
self._translate_cache: Dict[Tuple[int, int], torch.Tensor] = {}
|
||||
|
||||
scratch_bytes = sum(
|
||||
tensor.numel() * tensor.element_size()
|
||||
for slot in self._slots
|
||||
for tensor in slot.tensors.values()
|
||||
)
|
||||
logger.info(
|
||||
"Page-interleave KV sharding enabled: shard_rank=%d shard_size=%d "
|
||||
"page_size=%d scratch_rows=%d x2",
|
||||
"page_size=%d prefix_tokens=%d chunk_tokens=%d scratch_rows=%d x2 "
|
||||
"(%.2f MiB per rank, charged against the KV budget). "
|
||||
"Batch admission is limited by this scratch capacity.",
|
||||
self.shard_rank,
|
||||
self.shard_size,
|
||||
spec.page_size,
|
||||
spec.max_prefix_tokens,
|
||||
spec.chunk_tokens,
|
||||
scratch_rows,
|
||||
scratch_bytes / (1024 * 1024),
|
||||
)
|
||||
|
||||
def set_kv_buffer_prefix_valid(self, *args, **kwargs):
|
||||
@@ -186,6 +196,28 @@ class PageInterleaveKVPoolMixin:
|
||||
"(it writes pool rows directly, bypassing the ownership filter)"
|
||||
)
|
||||
|
||||
def register_layer_transfer_counter(self, layer_transfer_counter):
|
||||
# `None` is the in-tree "disable" signal from the SWA/hybrid wrappers,
|
||||
# so it has to stay a no-op.
|
||||
if layer_transfer_counter is None:
|
||||
super().register_layer_transfer_counter(None)
|
||||
return
|
||||
# Layer-wise KV load-back is unsupported here, and refusing it at
|
||||
# registration is the only safe answer. The gather in `_prefetch_layer`
|
||||
# reads pool rows directly (`_gather_pairs`), so it never passes
|
||||
# through the base getters' `wait_until` hook -- and the first gather
|
||||
# is kicked from `begin_shard_extend`, before any getter runs. Adding
|
||||
# the wait would not make the combination work: the loader writes whole
|
||||
# pool rows at LOGICAL indices with no ownership filter, while this
|
||||
# pool's rows are local physical ones, so ordering the read would only
|
||||
# turn a race into a silent wrong-row read. Same reason as
|
||||
# `set_kv_buffer_prefix_valid` above.
|
||||
raise NotImplementedError(
|
||||
"layer-wise KV load-back (HiCache / external linker) is "
|
||||
"unsupported under logical-page KV sharding (the gather reads "
|
||||
"pool rows directly and the load-back writes them unfiltered)"
|
||||
)
|
||||
|
||||
# ---- subclass hooks -------------------------------------------------------
|
||||
|
||||
def _scratch_tensor_specs(self, rows: int) -> Dict[str, torch.Tensor]:
|
||||
|
||||
Reference in New Issue
Block a user