Support post-capture KV sizing for the unified hybrid-SWA pool (#41961)

Co-authored-by: merrymercy <merrymercy@users.noreply.github.com>
Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-03 09:54:20 -07:00
committed by GitHub
co-authored by merrymercy ZYHowell
parent 2bae12b9b3
commit b016ca406f
9 changed files with 461 additions and 28 deletions
+14 -4
View File
@@ -1702,9 +1702,6 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
mla_enabled = use_mla_backend(server_args)
if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get():
return False
# Unified arenas are fully backed before capture and cannot resize afterward.
if cfg.enable_unified_memory:
return False
if cfg.device != "cuda":
return False
if cfg.dcp_size != 1:
@@ -1725,6 +1722,19 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
):
return False
# Only the hybrid-SWA byte pool has a matching post-capture resize path.
model_config = model_config_of(server_args)
if cfg.enable_unified_memory:
from sglang.srt.configs.hybrid_arch import mambaish_config
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
if not model_config.is_hybrid_swa or mambaish_config(model_config) is not None:
return False
# The solver budgets these independent draft pools again after capture.
spec = SpeculativeAlgorithm.from_string(cfg.speculative_algorithm)
if spec.is_eagle() or spec.is_standalone() or spec.is_dflash_family():
return False
if (
cfg.disaggregation_mode != "prefill"
and cfg.cuda_graph_config.decode.backend == Backend.DISABLED
@@ -1745,7 +1755,7 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
from sglang.srt.configs.model_config import is_deepseek_v4, is_minimax_sparse
hf_config = model_config_of(server_args).hf_config
hf_config = model_config.hf_config
if is_deepseek_v4(hf_config) or is_minimax_sparse(hf_config):
return False
@@ -1232,6 +1232,57 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
return 0
return fa.flush_opportunistic() + sa.flush_opportunistic()
def resize(self, config) -> None:
"""Synchronize allocator capacity after the shared pool is finalized."""
if not self.unified_buffer.post_capture_active:
raise RuntimeError(
"UnifiedSWATokenToKVPoolAllocator.resize requires post-capture backing"
)
final_pool_bytes = config.unified_memory_pool_bytes
if final_pool_bytes is None:
raise ValueError(
"UnifiedSWATokenToKVPoolAllocator.resize requires "
"config.unified_memory_pool_bytes"
)
if int(final_pool_bytes) != self.unified_buffer.total_bytes:
raise RuntimeError(
"UnifiedKVPool.finalize_backing must run before allocator.resize: "
f"config has {int(final_pool_bytes)} bytes but the pool exposes "
f"{self.unified_buffer.total_bytes} bytes"
)
for allocator in (self.full_attn_allocator, self.swa_attn_allocator):
if allocator.allocated_count() != 0 or allocator._pending_reuse:
raise RuntimeError(
f"cannot resize non-empty unified allocator "
f"{allocator.sub_pool_name!r}"
)
self.full_attn_allocator._set_capacity(self.unified_buffer.max_slots("full"))
self.swa_attn_allocator._set_capacity(
self.unified_buffer.max_slots("swa"),
virtual_num_pages=self.full_attn_allocator.num_virtual_ids,
)
self._empty_shared_gap_bytes = self.full_attn_allocator._current_gap_bytes()
self._size_full = self.full_attn_allocator.available_size()
self._size_swa = min(
self.swa_attn_allocator.available_size(),
len(self.full_attn_allocator.free_virtual_ids) * self.page_size,
)
self._full_max_total_num_tokens = self._size_full
self._swa_max_total_num_tokens = self._size_swa
self._kvcache.size = self.full_attn_allocator.max_slots - 1
self._kvcache.size_swa = self.swa_attn_allocator.max_slots - 1
for allocator, pool, host_capacity in (
(self.full_attn_allocator, self._kvcache.full_kv_pool, self._size_full),
(self.swa_attn_allocator, self._kvcache.swa_kv_pool, self._size_swa),
):
pool._num_pages = allocator.num_pages
pool.size = allocator.num_pages * self.page_size - self.page_size
pool.host_capacity_tokens = host_capacity
pool.host_capacity_bytes = (
host_capacity
* self.unified_buffer.spec(allocator.sub_pool_name).entry_bytes()
)
class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
"""Tri-pool composite for models with full KV + SWA KV + mamba/conv state
@@ -1951,6 +1951,20 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
reps = torch.cat(pending)
self.free(reps, _pages=reps // self.page_size)
def _set_capacity(
self, max_slots: int, *, virtual_num_pages: Optional[int] = None
) -> None:
"""Set active ranges while retaining the captured v2p/p2v storage."""
num_pages = int(max_slots) // self.page_size
num_virtual_ids = (
num_pages if virtual_num_pages is None else int(virtual_num_pages)
)
self.num_pages = num_pages
self.num_virtual_ids = num_virtual_ids
self.max_slots = num_pages * self.page_size
self.size = self.max_slots
self.clear()
def _chain_byte_accounting_violations(
chain: List[MultiEndedAllocator],
@@ -927,6 +927,7 @@ class KVCacheConfigurator:
swa_max_total_num_tokens=swa_max_total_num_tokens,
total_bytes=total_bytes,
enable_memory_saver=get_exec().features.enable_memory_saver,
post_capture_active=self.post_capture_kv_active,
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
# Overlap mode: same wait_stream(forward_stream) rationale as
# `_init_unified_mamba_pools`.
@@ -37,6 +37,7 @@ from sglang.kernels.ops.kvcache.copy_pages import copy_pages
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
from sglang.srt.environ import envs
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
from sglang.srt.mem_cache.layout.token_major import (
ROW_ALIGN_BYTES,
DenseEntryLayout,
@@ -48,6 +49,7 @@ from sglang.srt.mem_cache.layout.token_major import (
from sglang.srt.mem_cache.memory_pool import (
HybridLinearKVPool,
HybridReqToTokenPool,
KvBufferDesc,
KVWriteLoc,
MambaPool,
MHATokenToKVPool,
@@ -346,6 +348,8 @@ class UnifiedKVPool:
device: str,
enable_memory_saver: bool,
page_size: int = 1,
post_capture_active: bool = False,
bs1_floor_terms: Optional[List[Tuple[str, int]]] = None,
):
assert page_size >= 1, f"page_size must be >= 1; got {page_size}"
assert len(sub_pool_specs) >= 2, (
@@ -379,22 +383,57 @@ class UnifiedKVPool:
self._specs_by_name: Dict[str, SubPoolSpec] = {
s.name: s for s in sub_pool_specs
}
self.post_capture_active = post_capture_active
self._post_capture_owner: Optional[KvVmmBufferOwner] = None
self._bs1_floor_terms = bs1_floor_terms or []
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
enable=enable_memory_saver
)
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
# each pool's first allocatable slot is chosen so real data starts past it.
# A page-aware sub-pool allocates whole pages, so the sink is all of page
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
# max of both.
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
if envs.SGLANG_DEBUG_POISON_POOL.get():
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
# fail deterministically.
self._raw.view(torch.int16).fill_(0x7FC1)
logger.warning(
"[unified-memory-pool] POISONED: pool filled with bf16-NaN "
"patterns (SGLANG_DEBUG_POISON_POOL)"
)
else:
self._raw.zero_() # unset slots must read as zeros (matches non-shared)
if post_capture_active:
dev = torch.device(device)
if dev.type != "cuda":
raise ValueError(
"UnifiedKVPool post-capture backing requires a CUDA device"
)
device_id = (
dev.index if dev.index is not None else torch.cuda.current_device()
)
self._post_capture_owner = KvVmmBufferOwner(
device=device,
device_id=device_id,
store_dtype=torch.uint8,
page_size=1,
reserved_num_tokens=total_bytes,
buffer_descs=[
KvBufferDesc(
"unified_raw",
(total_bytes,),
row_bytes=1,
tokens_per_row=1,
)
],
)
self._raw = self._post_capture_owner.tensors[0]
# Capture routes every dummy KV write to slot 0, whose page
# envelope is the whole reserved sink: back it before any graph
# is captured.
self._post_capture_owner.ensure_prefix(reserved_floor)
accessible_raw = self._raw[:reserved_floor]
self._accessible_bytes = reserved_floor
else:
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
accessible_raw = self._raw
self._accessible_bytes = total_bytes
self._initialize_raw(accessible_raw)
self._max_slots: Dict[str, int] = {}
self._anchor_bytes: Dict[str, int] = {}
@@ -405,13 +444,6 @@ class UnifiedKVPool:
self._mla_views: Dict[str, List[torch.Tensor]] = {}
self._mamba_views: Dict[str, Tuple[List[torch.Tensor], torch.Tensor]] = {}
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
# each pool's first allocatable slot is chosen so real data starts past it.
# A page-aware sub-pool allocates whole pages, so the sink is all of page
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
# max of both.
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
for spec in self.sub_pool_specs:
entry_bytes = spec.entry_bytes()
max_slots = total_bytes // entry_bytes
@@ -448,8 +480,9 @@ class UnifiedKVPool:
raise TypeError(f"unsupported SubPoolSpec type: {type(spec)}")
logger.info(
"[unified-memory-pool] UnifiedKVPool allocated: total_bytes=%.2f GB (=%d B), "
"[unified-memory-pool] UnifiedKVPool %s: total_bytes=%.2f GB (=%d B), "
"%d sub-pool(s)",
"reserved VA upper bound" if post_capture_active else "allocated",
total_bytes / GB,
total_bytes,
len(self.sub_pool_specs),
@@ -468,6 +501,67 @@ class UnifiedKVPool:
self._min_slot_index[s.name],
)
def _initialize_raw(self, raw: torch.Tensor) -> None:
if envs.SGLANG_DEBUG_POISON_POOL.get():
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
# fail deterministically.
raw.view(torch.int16).fill_(0x7FC1)
logger.warning(
"[unified-memory-pool] POISONED: backed pool bytes filled with "
"bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)"
)
else:
raw.zero_() # unset slots must read as zeros (matches non-shared)
def _max_slots_for_total_bytes(self, total_bytes: int) -> Dict[str, int]:
max_slots_by_name = {}
for spec in self.sub_pool_specs:
max_slots = total_bytes // spec.entry_bytes()
min_page_index = (
self._min_slot_index[spec.name] + self._page_size - 1
) // self._page_size
if max_slots // self._page_size <= min_page_index:
raise ValueError(
f"unified pool size {total_bytes} leaves no allocatable page "
f"for sub-pool {spec.name!r} after its reserved slot-0 sink"
)
max_slots_by_name[spec.name] = max_slots
return max_slots_by_name
@property
def post_capture_backed_bytes(self) -> int:
if self._post_capture_owner is None:
return 0
return self._post_capture_owner.backed_bytes
@property
def active_allocation_bytes(self) -> int:
"""Bytes that are safe to access/register in the shared raw tensor."""
return self._accessible_bytes
def finalize_backing(self, config) -> None:
"""Back the final shared byte budget without changing any tensor address."""
if self._post_capture_owner is None:
raise RuntimeError("UnifiedKVPool post-capture backing is not active")
final_total_bytes = config.unified_memory_pool_bytes
if final_total_bytes is None:
raise ValueError(
"UnifiedKVPool post-capture sizing requires "
"config.unified_memory_pool_bytes"
)
final_total_bytes = int(final_total_bytes)
_check_bs1_feasibility_floor(
total_bytes=final_total_bytes,
floor_terms=self._bs1_floor_terms,
factory="UnifiedKVPool.finalize_backing",
)
final_max_slots = self._max_slots_for_total_bytes(final_total_bytes)
self._post_capture_owner.ensure_prefix(final_total_bytes)
self._initialize_raw(self._raw[:final_total_bytes])
self.total_bytes = final_total_bytes
self._accessible_bytes = final_total_bytes
self._max_slots = final_max_slots
# -- introspection --
def spec(self, name: str) -> SubPoolSpec:
@@ -660,7 +754,11 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
"""
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
raw = self._unified_buffer._raw
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
return (
[raw.data_ptr()],
[self._unified_buffer.active_allocation_bytes],
[self._page_bytes],
)
def get_page_envelope_buffer(self) -> torch.Tensor:
"""Return this sub-pool's page-strided view of the shared allocation."""
@@ -1617,6 +1715,17 @@ class UnifiedSWAKVPool(SWAKVPool):
def get_kv_size_bytes(self):
return 0, 0 # UnifiedKVPool logs the total; per-side would double-count
@property
def post_capture_active(self) -> bool:
return self.unified_buffer.post_capture_active
@property
def post_capture_backed_bytes(self) -> int:
return self.unified_buffer.post_capture_backed_bytes
def finalize_backing(self, config) -> None:
self.unified_buffer.finalize_backing(config)
def get_contiguous_buf_infos(self):
return self.full_kv_pool.get_contiguous_buf_infos()
@@ -1777,6 +1886,7 @@ def init_unified_swa_pools(
swa_max_total_num_tokens: Optional[int] = None,
total_bytes: Optional[int] = None,
enable_memory_saver: bool,
post_capture_active: bool = False,
need_sort: bool,
forward_stream: Optional[torch.cuda.Stream] = None,
lazy_compaction: bool = False,
@@ -1834,6 +1944,7 @@ def init_unified_swa_pools(
"full_max_total_num_tokens": full_max_total_num_tokens,
"swa_max_total_num_tokens": swa_max_total_num_tokens,
}
bs1_floor_terms = []
if model_context_len is not None:
# bs=1 floor: ONE sliding window of swa KV (+ a page of slack for the
# page-granular walk) + the slot-0 sink. The full side is not charged
@@ -1844,12 +1955,13 @@ def init_unified_swa_pools(
if sliding_window_size is not None
else model_context_len
)
bs1_floor_terms = [
("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()),
("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)),
]
_check_bs1_feasibility_floor(
total_bytes=total_bytes,
floor_terms=[
("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()),
("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)),
],
floor_terms=bs1_floor_terms,
factory="init_unified_swa_pools",
)
if total_bytes <= 0:
@@ -1860,6 +1972,8 @@ def init_unified_swa_pools(
device=device,
enable_memory_saver=enable_memory_saver,
page_size=page_size,
post_capture_active=post_capture_active,
bs1_floor_terms=bs1_floor_terms,
)
token_to_kv_pool = UnifiedSWAKVPool(
unified_buffer=shared_pool,
@@ -990,6 +990,9 @@ class ModelRunner:
self.memory_pool_config.swa_max_total_num_tokens = (
resize.swa_max_total_num_tokens
)
self.memory_pool_config.unified_memory_pool_bytes = (
resize.unified_memory_pool_bytes
)
if resize.capped_max_running_requests is not None:
self.max_running_requests = resize.capped_max_running_requests
if self.memory_pool_config is not None:
@@ -43,6 +43,7 @@ class PostCaptureKVResize(msgspec.Struct, frozen=True, kw_only=True):
max_total_num_tokens: int
full_max_total_num_tokens: Optional[int]
swa_max_total_num_tokens: Optional[int]
unified_memory_pool_bytes: Optional[int]
capped_max_running_requests: Optional[int]
@@ -169,5 +170,6 @@ def compute_post_capture_kv_resize(
max_total_num_tokens=config.max_total_num_tokens,
full_max_total_num_tokens=config.full_max_total_num_tokens,
swa_max_total_num_tokens=config.swa_max_total_num_tokens,
unified_memory_pool_bytes=config.unified_memory_pool_bytes,
capped_max_running_requests=capped_max_running_requests,
)