Skip the eager activation reserve in KV sizing on a capped Full prefill role (#43435)
Co-authored-by: Hanming Lu <69857889+hanming-lu@users.noreply.github.com> Co-authored-by: 842974287 <842974287@users.noreply.github.com>
This commit is contained in:
co-authored by
Hanming Lu
842974287
parent
a338b9858a
commit
55a8dffc87
@@ -81,6 +81,7 @@ from sglang.srt.mem_cache.memory_pool import (
|
||||
get_minimax_sparse_index_dtype,
|
||||
)
|
||||
from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
|
||||
from sglang.srt.model_executor.cuda_graph_config import Backend
|
||||
from sglang.srt.platforms import current_platform
|
||||
from sglang.srt.platforms.interface import KVPoolKind, reject_out_of_tree_path
|
||||
from sglang.srt.runtime_context import (
|
||||
@@ -2579,7 +2580,24 @@ class KVCacheConfigurator:
|
||||
)
|
||||
|
||||
slack_gb = pre_model_load_memory * (1 - get_schedule().mem_fraction_static)
|
||||
if self.mambaish_config is not None and self.post_capture_kv_active:
|
||||
# Every prefill batch fits a captured bucket when batches are capped at the
|
||||
# request slots; the exemption also needs an explicit Mamba size: a larger
|
||||
# rest_memory would otherwise auto-size a bigger physical Mamba pool below.
|
||||
prefill_max_requests = get_schedule().prefill_max_requests
|
||||
request_slots = get_exec().graph.cuda_graph_config.prefill.full_prefill_max_req
|
||||
capped_full_prefill_role = (
|
||||
get_disagg().disaggregation_mode == "prefill"
|
||||
and get_exec().graph.cuda_graph_config.prefill.backend == Backend.FULL
|
||||
and get_schedule().max_mamba_cache_size is not None
|
||||
and prefill_max_requests is not None
|
||||
and request_slots is not None
|
||||
and prefill_max_requests <= request_slots
|
||||
)
|
||||
if (
|
||||
self.mambaish_config is not None
|
||||
and self.post_capture_kv_active
|
||||
and not capped_full_prefill_role
|
||||
):
|
||||
# Mamba state is a fixed pre-capture allocation, so it can't ride the ~0 post-capture slack.
|
||||
slack_gb = max(
|
||||
slack_gb,
|
||||
|
||||
@@ -17,6 +17,7 @@ from sglang.srt.runtime_context import (
|
||||
get_exec,
|
||||
get_mm,
|
||||
get_parallel,
|
||||
get_schedule,
|
||||
pre_capture_activation_reserve_mb,
|
||||
)
|
||||
from sglang.srt.utils.common import get_available_gpu_memory, get_device_memory_capacity
|
||||
@@ -90,13 +91,26 @@ def compute_post_capture_kv_resize(
|
||||
running_requests,
|
||||
)
|
||||
if eager_decode_gap or mambaish_config(model_runner.model_config) is not None:
|
||||
headroom_gb = max(
|
||||
headroom_gb,
|
||||
pre_capture_activation_reserve_mb(
|
||||
get_device_memory_capacity(model_runner.device)
|
||||
)
|
||||
/ 1024,
|
||||
prefill_max_requests = get_schedule().prefill_max_requests
|
||||
request_slots = get_exec().graph.cuda_graph_config.prefill.full_prefill_max_req
|
||||
# Every prefill batch fits a captured bucket when batches are capped at the
|
||||
# request slots; the configured headroom then covers the eager tail.
|
||||
capped_full_prefill_role = (
|
||||
get_disagg().disaggregation_mode == "prefill"
|
||||
and get_exec().graph.cuda_graph_config.prefill.backend == Backend.FULL
|
||||
and get_schedule().max_mamba_cache_size is not None
|
||||
and prefill_max_requests is not None
|
||||
and request_slots is not None
|
||||
and prefill_max_requests <= request_slots
|
||||
)
|
||||
if not capped_full_prefill_role:
|
||||
headroom_gb = max(
|
||||
headroom_gb,
|
||||
pre_capture_activation_reserve_mb(
|
||||
get_device_memory_capacity(model_runner.device)
|
||||
)
|
||||
/ 1024,
|
||||
)
|
||||
if not graph_pool_borrow_enabled():
|
||||
# Borrowing serves the sampling temporaries out of idle graph storage;
|
||||
# without it they need real headroom the KV pool must not claim.
|
||||
|
||||
Reference in New Issue
Block a user