Skip the eager activation reserve in KV sizing on a capped Full prefill role (#43435)

Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com>
Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
metamergebot
2026-10-09 19:36:48 -07:00
committed by GitHub
co-authored by Hanming Lu 842974287
parent a338b9858a
commit 55a8dffc87
2 changed files with 39 additions and 7 deletions
@@ -81,6 +81,7 @@ from sglang.srt.mem_cache.memory_pool import (
get_minimax_sparse_index_dtype,
)
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
from sglang.srt.model_executor.cuda_graph_config import Backend
from sglang.srt.platforms import current_platform
from sglang.srt.platforms.interface import KVPoolKind, reject_out_of_tree_path
from sglang.srt.runtime_context import (
@@ -2579,7 +2580,24 @@ class KVCacheConfigurator:
)
slack_gb = pre_model_load_memory * (1 - get_schedule().mem_fraction_static)
if self.mambaish_config is not None and self.post_capture_kv_active:
# Every prefill batch fits a captured bucket when batches are capped at the
# request slots; the exemption also needs an explicit Mamba size: a larger
# rest_memory would otherwise auto-size a bigger physical Mamba pool below.
prefill_max_requests = get_schedule().prefill_max_requests
request_slots = get_exec().graph.cuda_graph_config.prefill.full_prefill_max_req
capped_full_prefill_role = (
get_disagg().disaggregation_mode == "prefill"
and get_exec().graph.cuda_graph_config.prefill.backend == Backend.FULL
and get_schedule().max_mamba_cache_size is not None
and prefill_max_requests is not None
and request_slots is not None
and prefill_max_requests <= request_slots
)
if (
self.mambaish_config is not None
and self.post_capture_kv_active
and not capped_full_prefill_role
):
# Mamba state is a fixed pre-capture allocation, so it can't ride the ~0 post-capture slack.
slack_gb = max(
slack_gb,
@@ -17,6 +17,7 @@ from sglang.srt.runtime_context import (
get_exec,
get_mm,
get_parallel,
get_schedule,
pre_capture_activation_reserve_mb,
)
from sglang.srt.utils.common import get_available_gpu_memory, get_device_memory_capacity
@@ -90,13 +91,26 @@ def compute_post_capture_kv_resize(
running_requests,
)
if eager_decode_gap or mambaish_config(model_runner.model_config) is not None:
headroom_gb = max(
headroom_gb,
pre_capture_activation_reserve_mb(
get_device_memory_capacity(model_runner.device)
)
/ 1024,
prefill_max_requests = get_schedule().prefill_max_requests
request_slots = get_exec().graph.cuda_graph_config.prefill.full_prefill_max_req
# Every prefill batch fits a captured bucket when batches are capped at the
# request slots; the configured headroom then covers the eager tail.
capped_full_prefill_role = (
get_disagg().disaggregation_mode == "prefill"
and get_exec().graph.cuda_graph_config.prefill.backend == Backend.FULL
and get_schedule().max_mamba_cache_size is not None
and prefill_max_requests is not None
and request_slots is not None
and prefill_max_requests <= request_slots
)
if not capped_full_prefill_role:
headroom_gb = max(
headroom_gb,
pre_capture_activation_reserve_mb(
get_device_memory_capacity(model_runner.device)
)
/ 1024,
)
if not graph_pool_borrow_enabled():
# Borrowing serves the sampling temporaries out of idle graph storage;
# without it they need real headroom the KV pool must not claim.