From b016ca406fa7079625525f99d0fe638b1dcb7f5a Mon Sep 17 00:00:00 2001 From: metamergebot Date: Sat, 3 Oct 2026 09:54:20 -0700 Subject: [PATCH] Support post-capture KV sizing for the unified hybrid-SWA pool (#41961) Co-authored-by: merrymercy Co-authored-by: ZYHowell --- python/sglang/srt/arg_groups/overrides.py | 18 +- .../mem_cache/allocator/unified_hybrid_swa.py | 51 +++++ .../mem_cache/allocator/unified_sub_pool.py | 14 ++ .../srt/mem_cache/kv_cache_configurator.py | 1 + .../srt/mem_cache/unified_memory_pool.py | 162 ++++++++++++-- .../sglang/srt/model_executor/model_runner.py | 3 + .../kv_pool_runtime.py | 2 + .../mem_cache/test_multi_ended_allocator.py | 33 +++ .../test_unified_post_capture_sizing.py | 205 ++++++++++++++++++ 9 files changed, 461 insertions(+), 28 deletions(-) create mode 100644 test/registered/unit/mem_cache/test_unified_post_capture_sizing.py diff --git a/python/sglang/srt/arg_groups/overrides.py b/python/sglang/srt/arg_groups/overrides.py index 05eaa3693e..fee2c4c53c 100644 --- a/python/sglang/srt/arg_groups/overrides.py +++ b/python/sglang/srt/arg_groups/overrides.py @@ -1702,9 +1702,6 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool: mla_enabled = use_mla_backend(server_args) if not envs.SGLANG_ENABLE_POST_CAPTURE_KV_SIZING.get(): return False - # Unified arenas are fully backed before capture and cannot resize afterward. - if cfg.enable_unified_memory: - return False if cfg.device != "cuda": return False if cfg.dcp_size != 1: @@ -1725,6 +1722,19 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool: ): return False + # Only the hybrid-SWA byte pool has a matching post-capture resize path. + model_config = model_config_of(server_args) + if cfg.enable_unified_memory: + from sglang.srt.configs.hybrid_arch import mambaish_config + from sglang.srt.speculative.spec_info import SpeculativeAlgorithm + + if not model_config.is_hybrid_swa or mambaish_config(model_config) is not None: + return False + # The solver budgets these independent draft pools again after capture. + spec = SpeculativeAlgorithm.from_string(cfg.speculative_algorithm) + if spec.is_eagle() or spec.is_standalone() or spec.is_dflash_family(): + return False + if ( cfg.disaggregation_mode != "prefill" and cfg.cuda_graph_config.decode.backend == Backend.DISABLED @@ -1745,7 +1755,7 @@ def post_capture_kv_sizing_planned(server_args: Any) -> bool: from sglang.srt.configs.model_config import is_deepseek_v4, is_minimax_sparse - hf_config = model_config_of(server_args).hf_config + hf_config = model_config.hf_config if is_deepseek_v4(hf_config) or is_minimax_sparse(hf_config): return False diff --git a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py index e7747f4988..069ab94c71 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py +++ b/python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py @@ -1232,6 +1232,57 @@ class UnifiedSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): return 0 return fa.flush_opportunistic() + sa.flush_opportunistic() + def resize(self, config) -> None: + """Synchronize allocator capacity after the shared pool is finalized.""" + if not self.unified_buffer.post_capture_active: + raise RuntimeError( + "UnifiedSWATokenToKVPoolAllocator.resize requires post-capture backing" + ) + final_pool_bytes = config.unified_memory_pool_bytes + if final_pool_bytes is None: + raise ValueError( + "UnifiedSWATokenToKVPoolAllocator.resize requires " + "config.unified_memory_pool_bytes" + ) + if int(final_pool_bytes) != self.unified_buffer.total_bytes: + raise RuntimeError( + "UnifiedKVPool.finalize_backing must run before allocator.resize: " + f"config has {int(final_pool_bytes)} bytes but the pool exposes " + f"{self.unified_buffer.total_bytes} bytes" + ) + for allocator in (self.full_attn_allocator, self.swa_attn_allocator): + if allocator.allocated_count() != 0 or allocator._pending_reuse: + raise RuntimeError( + f"cannot resize non-empty unified allocator " + f"{allocator.sub_pool_name!r}" + ) + self.full_attn_allocator._set_capacity(self.unified_buffer.max_slots("full")) + self.swa_attn_allocator._set_capacity( + self.unified_buffer.max_slots("swa"), + virtual_num_pages=self.full_attn_allocator.num_virtual_ids, + ) + self._empty_shared_gap_bytes = self.full_attn_allocator._current_gap_bytes() + self._size_full = self.full_attn_allocator.available_size() + self._size_swa = min( + self.swa_attn_allocator.available_size(), + len(self.full_attn_allocator.free_virtual_ids) * self.page_size, + ) + self._full_max_total_num_tokens = self._size_full + self._swa_max_total_num_tokens = self._size_swa + self._kvcache.size = self.full_attn_allocator.max_slots - 1 + self._kvcache.size_swa = self.swa_attn_allocator.max_slots - 1 + for allocator, pool, host_capacity in ( + (self.full_attn_allocator, self._kvcache.full_kv_pool, self._size_full), + (self.swa_attn_allocator, self._kvcache.swa_kv_pool, self._size_swa), + ): + pool._num_pages = allocator.num_pages + pool.size = allocator.num_pages * self.page_size - self.page_size + pool.host_capacity_tokens = host_capacity + pool.host_capacity_bytes = ( + host_capacity + * self.unified_buffer.spec(allocator.sub_pool_name).entry_bytes() + ) + class UnifiedMambaSWATokenToKVPoolAllocator(UnifiedSWAAllocatorBase): """Tri-pool composite for models with full KV + SWA KV + mamba/conv state diff --git a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py index 86e4184410..f24ee2ad0f 100644 --- a/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py +++ b/python/sglang/srt/mem_cache/allocator/unified_sub_pool.py @@ -1951,6 +1951,20 @@ class MultiEndedAllocator(BaseTokenToKVPoolAllocator): reps = torch.cat(pending) self.free(reps, _pages=reps // self.page_size) + def _set_capacity( + self, max_slots: int, *, virtual_num_pages: Optional[int] = None + ) -> None: + """Set active ranges while retaining the captured v2p/p2v storage.""" + num_pages = int(max_slots) // self.page_size + num_virtual_ids = ( + num_pages if virtual_num_pages is None else int(virtual_num_pages) + ) + self.num_pages = num_pages + self.num_virtual_ids = num_virtual_ids + self.max_slots = num_pages * self.page_size + self.size = self.max_slots + self.clear() + def _chain_byte_accounting_violations( chain: List[MultiEndedAllocator], diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index da2b357e62..405715bdb4 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -927,6 +927,7 @@ class KVCacheConfigurator: swa_max_total_num_tokens=swa_max_total_num_tokens, total_bytes=total_bytes, enable_memory_saver=get_exec().features.enable_memory_saver, + post_capture_active=self.post_capture_kv_active, need_sort=get_disagg().disaggregation_mode in ("decode", "prefill"), # Overlap mode: same wait_stream(forward_stream) rationale as # `_init_unified_mamba_pools`. diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 24274fb7ef..3dba9051f4 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -37,6 +37,7 @@ from sglang.kernels.ops.kvcache.copy_pages import copy_pages from sglang.kernels.ops.kvcache.zero_pages import zero_pages from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs +from sglang.srt.mem_cache.kv_vmm_backing import KvVmmBufferOwner from sglang.srt.mem_cache.layout.token_major import ( ROW_ALIGN_BYTES, DenseEntryLayout, @@ -48,6 +49,7 @@ from sglang.srt.mem_cache.layout.token_major import ( from sglang.srt.mem_cache.memory_pool import ( HybridLinearKVPool, HybridReqToTokenPool, + KvBufferDesc, KVWriteLoc, MambaPool, MHATokenToKVPool, @@ -346,6 +348,8 @@ class UnifiedKVPool: device: str, enable_memory_saver: bool, page_size: int = 1, + post_capture_active: bool = False, + bs1_floor_terms: Optional[List[Tuple[str, int]]] = None, ): assert page_size >= 1, f"page_size must be >= 1; got {page_size}" assert len(sub_pool_specs) >= 2, ( @@ -379,22 +383,57 @@ class UnifiedKVPool: self._specs_by_name: Dict[str, SubPoolSpec] = { s.name: s for s in sub_pool_specs } + self.post_capture_active = post_capture_active + self._post_capture_owner: Optional[KvVmmBufferOwner] = None + self._bs1_floor_terms = bs1_floor_terms or [] self.memory_saver_adapter = TorchMemorySaverAdapter.create( enable=enable_memory_saver ) + # Slot-0 dummy writes for both pools land in the reserved low-byte sink; + # each pool's first allocatable slot is chosen so real data starts past it. + # A page-aware sub-pool allocates whole pages, so the sink is all of page + # 0 (up to page_size * entry_bytes), not one slot entry -- reserve the + # max of both. + reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size) + with self.memory_saver_adapter.region(GPU_MEMORY_TYPE_KV_CACHE): - self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device) - if envs.SGLANG_DEBUG_POISON_POOL.get(): - # Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes - # fail deterministically. - self._raw.view(torch.int16).fill_(0x7FC1) - logger.warning( - "[unified-memory-pool] POISONED: pool filled with bf16-NaN " - "patterns (SGLANG_DEBUG_POISON_POOL)" - ) - else: - self._raw.zero_() # unset slots must read as zeros (matches non-shared) + if post_capture_active: + dev = torch.device(device) + if dev.type != "cuda": + raise ValueError( + "UnifiedKVPool post-capture backing requires a CUDA device" + ) + device_id = ( + dev.index if dev.index is not None else torch.cuda.current_device() + ) + self._post_capture_owner = KvVmmBufferOwner( + device=device, + device_id=device_id, + store_dtype=torch.uint8, + page_size=1, + reserved_num_tokens=total_bytes, + buffer_descs=[ + KvBufferDesc( + "unified_raw", + (total_bytes,), + row_bytes=1, + tokens_per_row=1, + ) + ], + ) + self._raw = self._post_capture_owner.tensors[0] + # Capture routes every dummy KV write to slot 0, whose page + # envelope is the whole reserved sink: back it before any graph + # is captured. + self._post_capture_owner.ensure_prefix(reserved_floor) + accessible_raw = self._raw[:reserved_floor] + self._accessible_bytes = reserved_floor + else: + self._raw = torch.empty(total_bytes, dtype=torch.uint8, device=device) + accessible_raw = self._raw + self._accessible_bytes = total_bytes + self._initialize_raw(accessible_raw) self._max_slots: Dict[str, int] = {} self._anchor_bytes: Dict[str, int] = {} @@ -405,13 +444,6 @@ class UnifiedKVPool: self._mla_views: Dict[str, List[torch.Tensor]] = {} self._mamba_views: Dict[str, Tuple[List[torch.Tensor], torch.Tensor]] = {} - # Slot-0 dummy writes for both pools land in the reserved low-byte sink; - # each pool's first allocatable slot is chosen so real data starts past it. - # A page-aware sub-pool allocates whole pages, so the sink is all of page - # 0 (up to page_size * entry_bytes), not one slot entry -- reserve the - # max of both. - reserved_floor = _reserved_floor_bytes(self.sub_pool_specs, page_size) - for spec in self.sub_pool_specs: entry_bytes = spec.entry_bytes() max_slots = total_bytes // entry_bytes @@ -448,8 +480,9 @@ class UnifiedKVPool: raise TypeError(f"unsupported SubPoolSpec type: {type(spec)}") logger.info( - "[unified-memory-pool] UnifiedKVPool allocated: total_bytes=%.2f GB (=%d B), " + "[unified-memory-pool] UnifiedKVPool %s: total_bytes=%.2f GB (=%d B), " "%d sub-pool(s)", + "reserved VA upper bound" if post_capture_active else "allocated", total_bytes / GB, total_bytes, len(self.sub_pool_specs), @@ -468,6 +501,67 @@ class UnifiedKVPool: self._min_slot_index[s.name], ) + def _initialize_raw(self, raw: torch.Tensor) -> None: + if envs.SGLANG_DEBUG_POISON_POOL.get(): + # Debug: bf16-NaN-fill so NaN-unsafe reads of never-written bytes + # fail deterministically. + raw.view(torch.int16).fill_(0x7FC1) + logger.warning( + "[unified-memory-pool] POISONED: backed pool bytes filled with " + "bf16-NaN patterns (SGLANG_DEBUG_POISON_POOL)" + ) + else: + raw.zero_() # unset slots must read as zeros (matches non-shared) + + def _max_slots_for_total_bytes(self, total_bytes: int) -> Dict[str, int]: + max_slots_by_name = {} + for spec in self.sub_pool_specs: + max_slots = total_bytes // spec.entry_bytes() + min_page_index = ( + self._min_slot_index[spec.name] + self._page_size - 1 + ) // self._page_size + if max_slots // self._page_size <= min_page_index: + raise ValueError( + f"unified pool size {total_bytes} leaves no allocatable page " + f"for sub-pool {spec.name!r} after its reserved slot-0 sink" + ) + max_slots_by_name[spec.name] = max_slots + return max_slots_by_name + + @property + def post_capture_backed_bytes(self) -> int: + if self._post_capture_owner is None: + return 0 + return self._post_capture_owner.backed_bytes + + @property + def active_allocation_bytes(self) -> int: + """Bytes that are safe to access/register in the shared raw tensor.""" + return self._accessible_bytes + + def finalize_backing(self, config) -> None: + """Back the final shared byte budget without changing any tensor address.""" + if self._post_capture_owner is None: + raise RuntimeError("UnifiedKVPool post-capture backing is not active") + final_total_bytes = config.unified_memory_pool_bytes + if final_total_bytes is None: + raise ValueError( + "UnifiedKVPool post-capture sizing requires " + "config.unified_memory_pool_bytes" + ) + final_total_bytes = int(final_total_bytes) + _check_bs1_feasibility_floor( + total_bytes=final_total_bytes, + floor_terms=self._bs1_floor_terms, + factory="UnifiedKVPool.finalize_backing", + ) + final_max_slots = self._max_slots_for_total_bytes(final_total_bytes) + self._post_capture_owner.ensure_prefix(final_total_bytes) + self._initialize_raw(self._raw[:final_total_bytes]) + self.total_bytes = final_total_bytes + self._accessible_bytes = final_total_bytes + self._max_slots = final_max_slots + # -- introspection -- def spec(self, name: str) -> SubPoolSpec: @@ -660,7 +754,11 @@ class UnifiedMHATokenToKVPool(MHATokenToKVPool): """ assert self._unified_buffer.anchor_bytes(self._sub_pool_name) == 0 raw = self._unified_buffer._raw - return [raw.data_ptr()], [raw.numel()], [self._page_bytes] + return ( + [raw.data_ptr()], + [self._unified_buffer.active_allocation_bytes], + [self._page_bytes], + ) def get_page_envelope_buffer(self) -> torch.Tensor: """Return this sub-pool's page-strided view of the shared allocation.""" @@ -1617,6 +1715,17 @@ class UnifiedSWAKVPool(SWAKVPool): def get_kv_size_bytes(self): return 0, 0 # UnifiedKVPool logs the total; per-side would double-count + @property + def post_capture_active(self) -> bool: + return self.unified_buffer.post_capture_active + + @property + def post_capture_backed_bytes(self) -> int: + return self.unified_buffer.post_capture_backed_bytes + + def finalize_backing(self, config) -> None: + self.unified_buffer.finalize_backing(config) + def get_contiguous_buf_infos(self): return self.full_kv_pool.get_contiguous_buf_infos() @@ -1777,6 +1886,7 @@ def init_unified_swa_pools( swa_max_total_num_tokens: Optional[int] = None, total_bytes: Optional[int] = None, enable_memory_saver: bool, + post_capture_active: bool = False, need_sort: bool, forward_stream: Optional[torch.cuda.Stream] = None, lazy_compaction: bool = False, @@ -1834,6 +1944,7 @@ def init_unified_swa_pools( "full_max_total_num_tokens": full_max_total_num_tokens, "swa_max_total_num_tokens": swa_max_total_num_tokens, } + bs1_floor_terms = [] if model_context_len is not None: # bs=1 floor: ONE sliding window of swa KV (+ a page of slack for the # page-granular walk) + the slot-0 sink. The full side is not charged @@ -1844,12 +1955,13 @@ def init_unified_swa_pools( if sliding_window_size is not None else model_context_len ) + bs1_floor_terms = [ + ("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()), + ("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)), + ] _check_bs1_feasibility_floor( total_bytes=total_bytes, - floor_terms=[ - ("swa_window_kv", swa_bs1_tokens * swa_spec.entry_bytes()), - ("sink", _reserved_floor_bytes([full_spec, swa_spec], page_size)), - ], + floor_terms=bs1_floor_terms, factory="init_unified_swa_pools", ) if total_bytes <= 0: @@ -1860,6 +1972,8 @@ def init_unified_swa_pools( device=device, enable_memory_saver=enable_memory_saver, page_size=page_size, + post_capture_active=post_capture_active, + bs1_floor_terms=bs1_floor_terms, ) token_to_kv_pool = UnifiedSWAKVPool( unified_buffer=shared_pool, diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py index 937cd160ae..48813ff721 100644 --- a/python/sglang/srt/model_executor/model_runner.py +++ b/python/sglang/srt/model_executor/model_runner.py @@ -990,6 +990,9 @@ class ModelRunner: self.memory_pool_config.swa_max_total_num_tokens = ( resize.swa_max_total_num_tokens ) + self.memory_pool_config.unified_memory_pool_bytes = ( + resize.unified_memory_pool_bytes + ) if resize.capped_max_running_requests is not None: self.max_running_requests = resize.capped_max_running_requests if self.memory_pool_config is not None: diff --git a/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py b/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py index 321e686a67..f66bd539d9 100644 --- a/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py +++ b/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py @@ -43,6 +43,7 @@ class PostCaptureKVResize(msgspec.Struct, frozen=True, kw_only=True): max_total_num_tokens: int full_max_total_num_tokens: Optional[int] swa_max_total_num_tokens: Optional[int] + unified_memory_pool_bytes: Optional[int] capped_max_running_requests: Optional[int] @@ -169,5 +170,6 @@ def compute_post_capture_kv_resize( max_total_num_tokens=config.max_total_num_tokens, full_max_total_num_tokens=config.full_max_total_num_tokens, swa_max_total_num_tokens=config.swa_max_total_num_tokens, + unified_memory_pool_bytes=config.unified_memory_pool_bytes, capped_max_running_requests=capped_max_running_requests, ) diff --git a/test/registered/unit/mem_cache/test_multi_ended_allocator.py b/test/registered/unit/mem_cache/test_multi_ended_allocator.py index c91f0289e5..f2e631815b 100644 --- a/test/registered/unit/mem_cache/test_multi_ended_allocator.py +++ b/test/registered/unit/mem_cache/test_multi_ended_allocator.py @@ -513,6 +513,39 @@ class TestMultiEndedAllocator(unittest.TestCase): self.assertEqual(int(full_alloc.virtual_to_physical[0].item()), 0) self.assertTrue(torch.equal(full_alloc.translate_kv_loc(zeros), zeros)) + def test_set_capacity_keeps_tables_and_shrinks_the_active_range(self): + """Post-capture sizing shrinks a captured allocator in place: the + v2p/p2v tables the graphs captured keep their storage, and every + capacity-derived range follows the new slot count on both ends.""" + pool, full_alloc, mamba_alloc, full_kv, mamba_kv = self._build_pair() + for alloc, kv in ((full_alloc, full_kv), (mamba_alloc, mamba_kv)): + with self.subTest(sub_pool=alloc.sub_pool_name): + v2p, p2v = alloc.virtual_to_physical, alloc.physical_to_virtual + new_slots = pool.max_slots(alloc.sub_pool_name) // 2 + alloc._set_capacity(new_slots) + self.assertIs(alloc.virtual_to_physical, v2p) + self.assertIs(alloc.physical_to_virtual, p2v) + self.assertEqual(alloc.max_slots, new_slots) + self.assertEqual(alloc.num_pages, new_slots) + self.assertEqual(alloc.num_virtual_ids, new_slots) + self.assertEqual( + alloc.free_virtual_ids.tolist(), + list(range(alloc.min_page_index, new_slots)), + ) + # Each side still fills its whole range, and only inside the new bounds. + for alloc, kv in ((full_alloc, full_kv), (mamba_alloc, mamba_kv)): + with self.subTest(sub_pool=alloc.sub_pool_name): + n = alloc.available_size() + self.assertGreater(n, 0) + v = self._alloc(alloc, kv, n) + p = alloc.virtual_to_physical[v] + self.assertGreaterEqual(int(p.min()), alloc.min_slot_index) + self.assertLess(int(p.max()), alloc.max_slots) + self.assertIsNone(alloc.alloc(1)) + self._check_invariants(alloc, kv) + self._free(alloc, kv, v) + self._check_invariants(alloc, kv) + # --------------------------------------------------------------------------- # Shared SWA composite -- unit tests diff --git a/test/registered/unit/mem_cache/test_unified_post_capture_sizing.py b/test/registered/unit/mem_cache/test_unified_post_capture_sizing.py new file mode 100644 index 0000000000..d94a114598 --- /dev/null +++ b/test/registered/unit/mem_cache/test_unified_post_capture_sizing.py @@ -0,0 +1,205 @@ +# Copyright 2023-2026 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Post-capture KV sizing on the unified hybrid-SWA byte pool. + +With post-capture sizing the shared arena is reserved as CUDA virtual memory +before graph capture: only the slot-0 sink is physically backed, the graphs +capture the final tensor address, and ``finalize_backing`` backs the measured +byte budget in place. ``UnifiedSWATokenToKVPoolAllocator.resize`` then +re-derives every capacity the sub-allocators and sub-pools expose. + +The oracle is the eagerly-allocated pool of the same final size: a resized pool +must hand out exactly the capacity it would have had if it had been built at +that size, without moving the tensor the graphs captured. +""" + +import unittest +from types import SimpleNamespace + +import torch + +from sglang.srt.mem_cache.unified_memory_pool import init_unified_swa_pools +from sglang.test.ci.ci_register import register_cuda_ci + +register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small") + +_RESERVED_BYTES = 64 << 20 +_FINAL_BYTES = 16 << 20 +_PAGE_SIZE = 4 + + +def _bundle(*, total_bytes: int, post_capture_active: bool): + return init_unified_swa_pools( + device="cuda", + kv_cache_dtype=torch.float16, + head_num=2, + head_dim=8, + v_head_dim=8, + swa_head_num=2, + swa_head_dim=8, + swa_v_head_dim=8, + page_size=_PAGE_SIZE, + start_layer=0, + end_layer=4, + swa_attention_layer_ids=[1, 3], + full_attention_layer_ids=[0, 2], + total_bytes=total_bytes, + enable_memory_saver=False, + post_capture_active=post_capture_active, + need_sort=False, + model_context_len=256, + sliding_window_size=64, + ) + + +def _capacities(bundle) -> dict: + """Every capacity-derived value a resize must keep consistent.""" + shared = bundle.unified_memory_pool + kvcache = bundle.token_to_kv_pool + allocator = bundle.token_to_kv_pool_allocator + out = { + "total_bytes": shared.total_bytes, + "active_allocation_bytes": shared.active_allocation_bytes, + "buf_lens": kvcache.get_contiguous_buf_infos()[1], + "kvcache.size": kvcache.size, + "kvcache.size_swa": kvcache.size_swa, + "allocator.size": allocator.size, + "allocator.available_size": allocator.available_size(), + "allocator.full_available_size": allocator.full_available_size(), + "allocator.swa_available_size": allocator.swa_available_size(), + } + for name, sub, pool in ( + ("full", allocator.full_attn_allocator, kvcache.full_kv_pool), + ("swa", allocator.swa_attn_allocator, kvcache.swa_kv_pool), + ): + out[f"{name}.max_slots"] = shared.max_slots(name) + out[f"{name}.sub.max_slots"] = sub.max_slots + out[f"{name}.sub.num_pages"] = sub.num_pages + out[f"{name}.sub.num_virtual_ids"] = sub.num_virtual_ids + out[f"{name}.sub.available_size"] = sub.available_size() + out[f"{name}.pool.size"] = pool.size + out[f"{name}.pool.host_capacity_tokens"] = pool.host_capacity_tokens + out[f"{name}.pool.host_capacity_bytes"] = pool.host_capacity_bytes + return out + + +@unittest.skipUnless(torch.cuda.is_available(), "requires CUDA") +class TestUnifiedPostCaptureSizing(unittest.TestCase): + def _post_capture_bundle(self, total_bytes=_RESERVED_BYTES): + return _bundle(total_bytes=total_bytes, post_capture_active=True) + + def test_reservation_backs_only_the_sink_before_capture(self): + """Before sizing, the arena is address space: the sink is resident and + only it may be handed to transfer engines, not the reserved bound.""" + bundle = self._post_capture_bundle() + shared = bundle.unified_memory_pool + self.assertTrue(bundle.token_to_kv_pool.post_capture_active) + self.assertEqual(shared.total_bytes, _RESERVED_BYTES) + self.assertGreater(shared.active_allocation_bytes, 0) + self.assertLessEqual( + shared.active_allocation_bytes, shared.post_capture_backed_bytes + ) + self.assertLess(shared.post_capture_backed_bytes, _RESERVED_BYTES) + _, buf_lens, _ = bundle.token_to_kv_pool.get_contiguous_buf_infos() + self.assertEqual(buf_lens, [shared.active_allocation_bytes]) + + def test_finalize_and_resize_match_the_eager_pool_of_the_final_size(self): + bundle = self._post_capture_bundle() + shared = bundle.unified_memory_pool + allocator = bundle.token_to_kv_pool_allocator + captured_ptr = shared._raw.data_ptr() + tables = [ + table + for sub in (allocator.full_attn_allocator, allocator.swa_attn_allocator) + for table in (sub.virtual_to_physical, sub.physical_to_virtual) + ] + table_ptrs = [table.data_ptr() for table in tables] + + config = SimpleNamespace(unified_memory_pool_bytes=_FINAL_BYTES) + bundle.token_to_kv_pool.finalize_backing(config) + allocator.resize(config) + + # Graphs captured these addresses; sizing may not move them. + self.assertEqual(shared._raw.data_ptr(), captured_ptr) + self.assertEqual([table.data_ptr() for table in tables], table_ptrs) + self.assertGreaterEqual( + shared.post_capture_backed_bytes, shared.active_allocation_bytes + ) + # The whole advertised span is resident, not just the sink. + last = shared._raw[shared.active_allocation_bytes - 1] + last.fill_(7) + torch.cuda.synchronize() + self.assertEqual(int(last.item()), 7) + + eager = _bundle(total_bytes=_FINAL_BYTES, post_capture_active=False) + self.assertEqual(_capacities(bundle), _capacities(eager)) + self.assertEqual(allocator.verify_byte_accounting(), []) + + # The resized capacity is fully allocatable and accounted. + n = allocator.available_size() + self.assertGreater(n, 0) + self.assertIsNotNone(allocator.alloc(n)) + self.assertEqual(allocator.available_size(), 0) + + def test_sizing_contract_is_enforced(self): + bundle = self._post_capture_bundle() + kvcache = bundle.token_to_kv_pool + allocator = bundle.token_to_kv_pool_allocator + with self.subTest("resize before finalize_backing"): + with self.assertRaisesRegex(RuntimeError, "finalize_backing must run"): + allocator.resize( + SimpleNamespace(unified_memory_pool_bytes=_FINAL_BYTES) + ) + with self.subTest("budget without a byte count"): + with self.assertRaises(ValueError): + kvcache.finalize_backing( + SimpleNamespace(unified_memory_pool_bytes=None) + ) + with self.subTest("budget above the reservation"): + with self.assertRaises(ValueError): + kvcache.finalize_backing( + SimpleNamespace(unified_memory_pool_bytes=2 * _RESERVED_BYTES) + ) + with self.subTest("budget below the bs=1 floor"): + with self.assertRaisesRegex(RuntimeError, "bs=1 floor"): + kvcache.finalize_backing( + SimpleNamespace(unified_memory_pool_bytes=4096) + ) + with self.subTest("resize with live allocations"): + kvcache.finalize_backing( + SimpleNamespace(unified_memory_pool_bytes=_FINAL_BYTES) + ) + handed = allocator.alloc(_PAGE_SIZE) + self.assertIsNotNone(handed) + with self.assertRaisesRegex(RuntimeError, "non-empty"): + allocator.resize( + SimpleNamespace(unified_memory_pool_bytes=_FINAL_BYTES) + ) + allocator.free(handed) + + eager = _bundle(total_bytes=_FINAL_BYTES, post_capture_active=False) + with self.subTest("eager pool cannot be resized"): + self.assertFalse(eager.token_to_kv_pool.post_capture_active) + with self.assertRaises(RuntimeError): + eager.token_to_kv_pool.finalize_backing( + SimpleNamespace(unified_memory_pool_bytes=_FINAL_BYTES) + ) + with self.assertRaises(RuntimeError): + eager.token_to_kv_pool_allocator.resize( + SimpleNamespace(unified_memory_pool_bytes=_FINAL_BYTES) + ) + + +if __name__ == "__main__": + unittest.main()