Support post-capture KV sizing for the unified hybrid-SWA pool (#41961)
Co-authored-by: merrymercy <merrymercy@users.noreply.github.com> Co-authored-by: ZYHowell <ZYHowell@users.noreply.github.com>
This commit is contained in:
co-authored by
merrymercy
ZYHowell
parent
2bae12b9b3
commit
b016ca406f
@@ -1702,9 +1702,6 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
|
||||
mla_enabled = use_mla_backend(server_args)
|
||||
if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get():
|
||||
return False
|
||||
# Unified arenas are fully backed before capture and cannot resize afterward.
|
||||
if cfg.enable_unified_memory:
|
||||
return False
|
||||
if cfg.device != "cuda":
|
||||
return False
|
||||
if cfg.dcp_size != 1:
|
||||
@@ -1725,6 +1722,19 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
|
||||
):
|
||||
return False
|
||||
|
||||
# Only the hybrid-SWA byte pool has a matching post-capture resize path.
|
||||
model_config = model_config_of(server_args)
|
||||
if cfg.enable_unified_memory:
|
||||
from sglang.srt.configs.hybrid_arch import mambaish_config
|
||||
from sglang.srt.speculative.spec_info import SpeculativeAlgorithm
|
||||
|
||||
if not model_config.is_hybrid_swa or mambaish_config(model_config) is not None:
|
||||
return False
|
||||
# The solver budgets these independent draft pools again after capture.
|
||||
spec = SpeculativeAlgorithm.from_string(cfg.speculative_algorithm)
|
||||
if spec.is_eagle() or spec.is_standalone() or spec.is_dflash_family():
|
||||
return False
|
||||
|
||||
if (
|
||||
cfg.disaggregation_mode != "prefill"
|
||||
and cfg.cuda_graph_config.decode.backend == Backend.DISABLED
|
||||
@@ -1745,7 +1755,7 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool:
|
||||
|
||||
from sglang.srt.configs.model_config import is_deepseek_v4, is_minimax_sparse
|
||||
|
||||
hf_config = model_config_of(server_args).hf_config
|
||||
hf_config = model_config.hf_config
|
||||
if is_deepseek_v4(hf_config) or is_minimax_sparse(hf_config):
|
||||
return False
|
||||
|
||||
|
||||
@@ -1232,6 +1232,57 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
return 0
|
||||
return fa.flush_opportunistic() + sa.flush_opportunistic()
|
||||
|
||||
def resize(self, config) -> None:
|
||||
"""Synchronize allocator capacity after the shared pool is finalized."""
|
||||
if not self.unified_buffer.post_capture_active:
|
||||
raise RuntimeError(
|
||||
"UnifiedSWATokenToKVPoolAllocator.resize requires post-capture backing"
|
||||
)
|
||||
final_pool_bytes = config.unified_memory_pool_bytes
|
||||
if final_pool_bytes is None:
|
||||
raise ValueError(
|
||||
"UnifiedSWATokenToKVPoolAllocator.resize requires "
|
||||
"config.unified_memory_pool_bytes"
|
||||
)
|
||||
if int(final_pool_bytes) != self.unified_buffer.total_bytes:
|
||||
raise RuntimeError(
|
||||
"UnifiedKVPool.finalize_backing must run before allocator.resize: "
|
||||
f"config has {int(final_pool_bytes)} bytes but the pool exposes "
|
||||
f"{self.unified_buffer.total_bytes} bytes"
|
||||
)
|
||||
for allocator in (self.full_attn_allocator, self.swa_attn_allocator):
|
||||
if allocator.allocated_count() != 0 or allocator._pending_reuse:
|
||||
raise RuntimeError(
|
||||
f"cannot resize non-empty unified allocator "
|
||||
f"{allocator.sub_pool_name!r}"
|
||||
)
|
||||
self.full_attn_allocator._set_capacity(self.unified_buffer.max_slots("full"))
|
||||
self.swa_attn_allocator._set_capacity(
|
||||
self.unified_buffer.max_slots("swa"),
|
||||
virtual_num_pages=self.full_attn_allocator.num_virtual_ids,
|
||||
)
|
||||
self._empty_shared_gap_bytes = self.full_attn_allocator._current_gap_bytes()
|
||||
self._size_full = self.full_attn_allocator.available_size()
|
||||
self._size_swa = min(
|
||||
self.swa_attn_allocator.available_size(),
|
||||
len(self.full_attn_allocator.free_virtual_ids) * self.page_size,
|
||||
)
|
||||
self._full_max_total_num_tokens = self._size_full
|
||||
self._swa_max_total_num_tokens = self._size_swa
|
||||
self._kvcache.size = self.full_attn_allocator.max_slots - 1
|
||||
self._kvcache.size_swa = self.swa_attn_allocator.max_slots - 1
|
||||
for allocator, pool, host_capacity in (
|
||||
(self.full_attn_allocator, self._kvcache.full_kv_pool, self._size_full),
|
||||
(self.swa_attn_allocator, self._kvcache.swa_kv_pool, self._size_swa),
|
||||
):
|
||||
pool._num_pages = allocator.num_pages
|
||||
pool.size = allocator.num_pages * self.page_size - self.page_size
|
||||
pool.host_capacity_tokens = host_capacity
|
||||
pool.host_capacity_bytes = (
|
||||
host_capacity
|
||||
* self.unified_buffer.spec(allocator.sub_pool_name).entry_bytes()
|
||||
)
|
||||
|
||||
|
||||
class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase):
|
||||
"""Tri-pool composite for models with full KV + SWA KV + mamba/conv state
|
||||
|
||||
@@ -1951,6 +1951,20 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator):
|
||||
reps = torch.cat(pending)
|
||||
self.free(reps, _pages=reps // self.page_size)
|
||||
|
||||
def _set_capacity(
|
||||
self, max_slots: int, *, virtual_num_pages: Optional[int] = None
|
||||
) -> None:
|
||||
"""Set active ranges while retaining the captured v2p/p2v storage."""
|
||||
num_pages = int(max_slots) // self.page_size
|
||||
num_virtual_ids = (
|
||||
num_pages if virtual_num_pages is None else int(virtual_num_pages)
|
||||
)
|
||||
self.num_pages = num_pages
|
||||
self.num_virtual_ids = num_virtual_ids
|
||||
self.max_slots = num_pages * self.page_size
|
||||
self.size = self.max_slots
|
||||
self.clear()
|
||||
|
||||
|
||||
def _chain_byte_accounting_violations(
|
||||
chain: List[MultiEndedAllocator],
|
||||
|
||||
@@ -927,6 +927,7 @@ class KVCacheConfigurator:
|
||||
swa_max_total_num_tokens=swa_max_total_num_tokens,
|
||||
total_bytes=total_bytes,
|
||||
enable_memory_saver=get_exec().features.enable_memory_saver,
|
||||
post_capture_active=self.post_capture_kv_active,
|
||||
need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"),
|
||||
# Overlap mode: same wait_stream(forward_stream) rationale as
|
||||
# `_init_unified_mamba_pools`.
|
||||
|
||||
@@ -37,6 +37,7 @@ from sglang.kernels.ops.kvcache.copy_pages import copy_pages
|
||||
from sglang.kernels.ops.kvcache.zero_pages import zero_pages
|
||||
from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner
|
||||
from sglang.srt.mem_cache.layout.token_major import (
|
||||
ROW_ALIGN_BYTES,
|
||||
DenseEntryLayout,
|
||||
@@ -48,6 +49,7 @@ from sglang.srt.mem_cache.layout.token_major import (
|
||||
from sglang.srt.mem_cache.memory_pool import (
|
||||
HybridLinearKVPool,
|
||||
HybridReqToTokenPool,
|
||||
KvBufferDesc,
|
||||
KVWriteLoc,
|
||||
MambaPool,
|
||||
MHATokenToKVPool,
|
||||
@@ -346,6 +348,8 @@ class UnifiedKVPool:
|
||||
device: str,
|
||||
enable_memory_saver: bool,
|
||||
page_size: int = 1,
|
||||
post_capture_active: bool = False,
|
||||
bs1_floor_terms: Optional[List[Tuple[str, int]]] = None,
|
||||
):
|
||||
assert page_size >= 1, f"page_size must be >= 1; got {page_size}"
|
||||
assert len(sub_pool_specs) >= 2, (
|
||||
@@ -379,22 +383,57 @@ class UnifiedKVPool:
|
||||
self._specs_by_name: Dict[str, SubPoolSpec] = {
|
||||
s.name: s for s in sub_pool_specs
|
||||
}
|
||||
self.post_capture_active = post_capture_active
|
||||
self._post_capture_owner: Optional[KvVmmBufferOwner] = None
|
||||
self._bs1_floor_terms = bs1_floor_terms or []
|
||||
|
||||
self.memory_saver_adapter = TorchMemorySaverAdapter.create(
|
||||
enable=enable_memory_saver
|
||||
)
|
||||
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
|
||||
# each pool's first allocatable slot is chosen so real data starts past it.
|
||||
# A page-aware sub-pool allocates whole pages, so the sink is all of page
|
||||
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
|
||||
# max of both.
|
||||
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
|
||||
|
||||
with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE):
|
||||
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
|
||||
if envs.SGLANG_DEBUG_POISON_POOL.get():
|
||||
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
|
||||
# fail deterministically.
|
||||
self._raw.view(torch.int16).fill_(0x7FC1)
|
||||
logger.warning(
|
||||
"[unified-memory-pool] POISONED: pool filled with bf16-NaN "
|
||||
"patterns (SGLANG_DEBUG_POISON_POOL)"
|
||||
)
|
||||
else:
|
||||
self._raw.zero_() # unset slots must read as zeros (matches non-shared)
|
||||
if post_capture_active:
|
||||
dev = torch.device(device)
|
||||
if dev.type != "cuda":
|
||||
raise ValueError(
|
||||
"UnifiedKVPool post-capture backing requires a CUDA device"
|
||||
)
|
||||
device_id = (
|
||||
dev.index if dev.index is not None else torch.cuda.current_device()
|
||||
)
|
||||
self._post_capture_owner = KvVmmBufferOwner(
|
||||
device=device,
|
||||
device_id=device_id,
|
||||
store_dtype=torch.uint8,
|
||||
page_size=1,
|
||||
reserved_num_tokens=total_bytes,
|
||||
buffer_descs=[
|
||||
KvBufferDesc(
|
||||
"unified_raw",
|
||||
(total_bytes,),
|
||||
row_bytes=1,
|
||||
tokens_per_row=1,
|
||||
)
|
||||
],
|
||||
)
|
||||
self._raw = self._post_capture_owner.tensors[0]
|
||||
# Capture routes every dummy KV write to slot 0, whose page
|
||||
# envelope is the whole reserved sink: back it before any graph
|
||||
# is captured.
|
||||
self._post_capture_owner.ensure_prefix(reserved_floor)
|
||||
accessible_raw = self._raw[:reserved_floor]
|
||||
self._accessible_bytes = reserved_floor
|
||||
else:
|
||||
self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device)
|
||||
accessible_raw = self._raw
|
||||
self._accessible_bytes = total_bytes
|
||||
self._initialize_raw(accessible_raw)
|
||||
|
||||
self._max_slots: Dict[str, int] = {}
|
||||
self._anchor_bytes: Dict[str, int] = {}
|
||||
@@ -405,13 +444,6 @@ class UnifiedKVPool:
|
||||
self._mla_views: Dict[str, List[torch.Tensor]] = {}
|
||||
self._mamba_views: Dict[str, Tuple[List[torch.Tensor], torch.Tensor]] = {}
|
||||
|
||||
# Slot-0 dummy writes for both pools land in the reserved low-byte sink;
|
||||
# each pool's first allocatable slot is chosen so real data starts past it.
|
||||
# A page-aware sub-pool allocates whole pages, so the sink is all of page
|
||||
# 0 (up to page_size * entry_bytes), not one slot entry -- reserve the
|
||||
# max of both.
|
||||
reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size)
|
||||
|
||||
for spec in self.sub_pool_specs:
|
||||
entry_bytes = spec.entry_bytes()
|
||||
max_slots = total_bytes // entry_bytes
|
||||
@@ -448,8 +480,9 @@ class UnifiedKVPool:
|
||||
raise TypeError(f"unsupported SubPoolSpec type: {type(spec)}")
|
||||
|
||||
logger.info(
|
||||
"[unified-memory-pool] UnifiedKVPool allocated: total_bytes=%.2f GB (=%d B), "
|
||||
"[unified-memory-pool] UnifiedKVPool %s: total_bytes=%.2f GB (=%d B), "
|
||||
"%d sub-pool(s)",
|
||||
"reserved VA upper bound" if post_capture_active else "allocated",
|
||||
total_bytes / GB,
|
||||
total_bytes,
|
||||
len(self.sub_pool_specs),
|
||||
@@ -468,6 +501,67 @@ class UnifiedKVPool:
|
||||
self._min_slot_index[s.name],
|
||||
)
|
||||
|
||||
def _initialize_raw(self, raw: torch.Tensor) -> None:
|
||||
if envs.SGLANG_DEBUG_POISON_POOL.get():
|
||||
# Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes
|
||||
# fail deterministically.
|
||||
raw.view(torch.int16).fill_(0x7FC1)
|
||||
logger.warning(
|
||||
"[unified-memory-pool] POISONED: backed pool bytes filled with "
|
||||
"bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)"
|
||||
)
|
||||
else:
|
||||
raw.zero_() # unset slots must read as zeros (matches non-shared)
|
||||
|
||||
def _max_slots_for_total_bytes(self, total_bytes: int) -> Dict[str, int]:
|
||||
max_slots_by_name = {}
|
||||
for spec in self.sub_pool_specs:
|
||||
max_slots = total_bytes // spec.entry_bytes()
|
||||
min_page_index = (
|
||||
self._min_slot_index[spec.name] + self._page_size - 1
|
||||
) // self._page_size
|
||||
if max_slots // self._page_size <= min_page_index:
|
||||
raise ValueError(
|
||||
f"unified pool size {total_bytes} leaves no allocatable page "
|
||||
f"for sub-pool {spec.name!r} after its reserved slot-0 sink"
|
||||
)
|
||||
max_slots_by_name[spec.name] = max_slots
|
||||
return max_slots_by_name
|
||||
|
||||
@property
|
||||
def post_capture_backed_bytes(self) -> int:
|
||||
if self._post_capture_owner is None:
|
||||
return 0
|
||||
return self._post_capture_owner.backed_bytes
|
||||
|
||||
@property
|
||||
def active_allocation_bytes(self) -> int:
|
||||
"""Bytes that are safe to access/register in the shared raw tensor."""
|
||||
return self._accessible_bytes
|
||||
|
||||
def finalize_backing(self, config) -> None:
|
||||
"""Back the final shared byte budget without changing any tensor address."""
|
||||
if self._post_capture_owner is None:
|
||||
raise RuntimeError("UnifiedKVPool post-capture backing is not active")
|
||||
final_total_bytes = config.unified_memory_pool_bytes
|
||||
if final_total_bytes is None:
|
||||
raise ValueError(
|
||||
"UnifiedKVPool post-capture sizing requires "
|
||||
"config.unified_memory_pool_bytes"
|
||||
)
|
||||
final_total_bytes = int(final_total_bytes)
|
||||
_check_bs1_feasibility_floor(
|
||||
total_bytes=final_total_bytes,
|
||||
floor_terms=self._bs1_floor_terms,
|
||||
factory="UnifiedKVPool.finalize_backing",
|
||||
)
|
||||
final_max_slots = self._max_slots_for_total_bytes(final_total_bytes)
|
||||
self._post_capture_owner.ensure_prefix(final_total_bytes)
|
||||
self._initialize_raw(self._raw[:final_total_bytes])
|
||||
self.total_bytes = final_total_bytes
|
||||
self._accessible_bytes = final_total_bytes
|
||||
self._max_slots = final_max_slots
|
||||
|
||||
# -- introspection --
|
||||
|
||||
def spec(self, name: str) -> SubPoolSpec:
|
||||
@@ -660,7 +754,11 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool):
|
||||
"""
|
||||
assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0
|
||||
raw = self._unified_buffer._raw
|
||||
return [raw.data_ptr()], [raw.numel()], [self._page_bytes]
|
||||
return (
|
||||
[raw.data_ptr()],
|
||||
[self._unified_buffer.active_allocation_bytes],
|
||||
[self._page_bytes],
|
||||
)
|
||||
|
||||
def get_page_envelope_buffer(self) -> torch.Tensor:
|
||||
"""Return this sub-pool's page-strided view of the shared allocation."""
|
||||
@@ -1617,6 +1715,17 @@ class UnifiedSWAKVPool(SWAKVPool):
|
||||
def get_kv_size_bytes(self):
|
||||
return 0, 0 # UnifiedKVPool logs the total; per-side would double-count
|
||||
|
||||
@property
|
||||
def post_capture_active(self) -> bool:
|
||||
return self.unified_buffer.post_capture_active
|
||||
|
||||
@property
|
||||
def post_capture_backed_bytes(self) -> int:
|
||||
return self.unified_buffer.post_capture_backed_bytes
|
||||
|
||||
def finalize_backing(self, config) -> None:
|
||||
self.unified_buffer.finalize_backing(config)
|
||||
|
||||
def get_contiguous_buf_infos(self):
|
||||
return self.full_kv_pool.get_contiguous_buf_infos()
|
||||
|
||||
@@ -1777,6 +1886,7 @@ def init_unified_swa_pools(
|
||||
swa_max_total_num_tokens: Optional[int] = None,
|
||||
total_bytes: Optional[int] = None,
|
||||
enable_memory_saver: bool,
|
||||
post_capture_active: bool = False,
|
||||
need_sort: bool,
|
||||
forward_stream: Optional[torch.cuda.Stream] = None,
|
||||
lazy_compaction: bool = False,
|
||||
@@ -1834,6 +1944,7 @@ def init_unified_swa_pools(
|
||||
"full_max_total_num_tokens": full_max_total_num_tokens,
|
||||
"swa_max_total_num_tokens": swa_max_total_num_tokens,
|
||||
}
|
||||
bs1_floor_terms = []
|
||||
if model_context_len is not None:
|
||||
# bs=1 floor: ONE sliding window of swa KV (+ a page of slack for the
|
||||
# page-granular walk) + the slot-0 sink. The full side is not charged
|
||||
@@ -1844,12 +1955,13 @@ def init_unified_swa_pools(
|
||||
if sliding_window_size is not None
|
||||
else model_context_len
|
||||
)
|
||||
bs1_floor_terms = [
|
||||
("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()),
|
||||
("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)),
|
||||
]
|
||||
_check_bs1_feasibility_floor(
|
||||
total_bytes=total_bytes,
|
||||
floor_terms=[
|
||||
("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()),
|
||||
("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)),
|
||||
],
|
||||
floor_terms=bs1_floor_terms,
|
||||
factory="init_unified_swa_pools",
|
||||
)
|
||||
if total_bytes <= 0:
|
||||
@@ -1860,6 +1972,8 @@ def init_unified_swa_pools(
|
||||
device=device,
|
||||
enable_memory_saver=enable_memory_saver,
|
||||
page_size=page_size,
|
||||
post_capture_active=post_capture_active,
|
||||
bs1_floor_terms=bs1_floor_terms,
|
||||
)
|
||||
token_to_kv_pool = UnifiedSWAKVPool(
|
||||
unified_buffer=shared_pool,
|
||||
|
||||
@@ -990,6 +990,9 @@ class ModelRunner:
|
||||
self.memory_pool_config.swa_max_total_num_tokens = (
|
||||
resize.swa_max_total_num_tokens
|
||||
)
|
||||
self.memory_pool_config.unified_memory_pool_bytes = (
|
||||
resize.unified_memory_pool_bytes
|
||||
)
|
||||
if resize.capped_max_running_requests is not None:
|
||||
self.max_running_requests = resize.capped_max_running_requests
|
||||
if self.memory_pool_config is not None:
|
||||
|
||||
@@ -43,6 +43,7 @@ class PostCaptureKVResize(msgspec.Struct, frozen=True, kw_only=True):
|
||||
max_total_num_tokens: int
|
||||
full_max_total_num_tokens: Optional[int]
|
||||
swa_max_total_num_tokens: Optional[int]
|
||||
unified_memory_pool_bytes: Optional[int]
|
||||
capped_max_running_requests: Optional[int]
|
||||
|
||||
|
||||
@@ -169,5 +170,6 @@ def compute_post_capture_kv_resize(
|
||||
max_total_num_tokens=config.max_total_num_tokens,
|
||||
full_max_total_num_tokens=config.full_max_total_num_tokens,
|
||||
swa_max_total_num_tokens=config.swa_max_total_num_tokens,
|
||||
unified_memory_pool_bytes=config.unified_memory_pool_bytes,
|
||||
capped_max_running_requests=capped_max_running_requests,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user