diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 21975537ff..4da8bc98e3 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -81,6 +81,7 @@ from sglang.srt.mem_cache.memory_pool import ( get_minimax_sparse_index_dtype, ) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool +from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.platforms import current_platform from sglang.srt.platforms.interface import KVPoolKind, reject_out_of_tree_path from sglang.srt.runtime_context import ( @@ -2579,7 +2580,24 @@ class KVCacheConfigurator: ) slack_gb = pre_model_load_memory * (1 - get_schedule().mem_fraction_static) - if self.mambaish_config is not None and self.post_capture_kv_active: + # Every prefill batch fits a captured bucket when batches are capped at the + # request slots; the exemption also needs an explicit Mamba size: a larger + # rest_memory would otherwise auto-size a bigger physical Mamba pool below. + prefill_max_requests = get_schedule().prefill_max_requests + request_slots = get_exec().graph.cuda_graph_config.prefill.full_prefill_max_req + capped_full_prefill_role = ( + get_disagg().disaggregation_mode == "prefill" + and get_exec().graph.cuda_graph_config.prefill.backend == Backend.FULL + and get_schedule().max_mamba_cache_size is not None + and prefill_max_requests is not None + and request_slots is not None + and prefill_max_requests <= request_slots + ) + if ( + self.mambaish_config is not None + and self.post_capture_kv_active + and not capped_full_prefill_role + ): # Mamba state is a fixed pre-capture allocation, so it can't ride the ~0 post-capture slack. slack_gb = max( slack_gb, diff --git a/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py b/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py index a41743987d..5e32362184 100644 --- a/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py +++ b/python/sglang/srt/model_executor/model_runner_components/kv_pool_runtime.py @@ -17,6 +17,7 @@ from sglang.srt.runtime_context import ( get_exec, get_mm, get_parallel, + get_schedule, pre_capture_activation_reserve_mb, ) from sglang.srt.utils.common import get_available_gpu_memory, get_device_memory_capacity @@ -90,13 +91,26 @@ def compute_post_capture_kv_resize( running_requests, ) if eager_decode_gap or mambaish_config(model_runner.model_config) is not None: - headroom_gb = max( - headroom_gb, - pre_capture_activation_reserve_mb( - get_device_memory_capacity(model_runner.device) - ) - / 1024, + prefill_max_requests = get_schedule().prefill_max_requests + request_slots = get_exec().graph.cuda_graph_config.prefill.full_prefill_max_req + # Every prefill batch fits a captured bucket when batches are capped at the + # request slots; the configured headroom then covers the eager tail. + capped_full_prefill_role = ( + get_disagg().disaggregation_mode == "prefill" + and get_exec().graph.cuda_graph_config.prefill.backend == Backend.FULL + and get_schedule().max_mamba_cache_size is not None + and prefill_max_requests is not None + and request_slots is not None + and prefill_max_requests <= request_slots ) + if not capped_full_prefill_role: + headroom_gb = max( + headroom_gb, + pre_capture_activation_reserve_mb( + get_device_memory_capacity(model_runner.device) + ) + / 1024, + ) if not graph_pool_borrow_enabled(): # Borrowing serves the sampling temporaries out of idle graph storage; # without it they need real headroom the KV pool must not claim.