minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Respect explicit --max-running-requests instead of clamping to heuristic (#27399)

Browse Source 
Co-authored-by: Vedant Jhaveri <vjhaveri@linkedin.com>
This commit is contained in:
Vedant V Jhaveri
2026-06-12 12:30:47 -07:00
committed by GitHub
co-authored by Vedant Jhaveri
parent 6e0fa5afe1
commit 3be5a7ec89
1 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py
+10 -1
View File
@@ -885,8 +885,10 @@ class ModelRunnerKVCacheMixin:
max_num_reqs = self.server_args.max_running_requests max_num_reqs = self.server_args.max_running_requests
if max_num_reqs is not None: if max_num_reqs is not None:
max_num_reqs = min(max_num_reqs // self.dp_size, estimated) requested_per_worker = max_num_reqs // self.dp_size
max_num_reqs = min(requested_per_worker, token_capacity // 2)
else: else:
requested_per_worker = None
max_num_reqs = min(estimated, token_capacity // 2) max_num_reqs = min(estimated, token_capacity // 2)
if self.mambaish_config is not None: if self.mambaish_config is not None:
@@ -904,6 +906,13 @@ class ModelRunnerKVCacheMixin:
f"(2) increase --mem-fraction-static, or " f"(2) increase --mem-fraction-static, or "
f"(3) use GPUs with more memory." f"(3) use GPUs with more memory."
) )
if requested_per_worker is not None and max_num_reqs < requested_per_worker:
logger.warning(
"max_running_requests was reduced from the requested %d to %d "
"(per dp worker) due to the available KV cache capacity.",
requested_per_worker,
max_num_reqs,
)
logger.info( logger.info(
f"Max concurrent requests (per dp worker) from the finalized token capacity: " f"Max concurrent requests (per dp worker) from the finalized token capacity: "
f"max_num_reqs={max_num_reqs}." f"max_num_reqs={max_num_reqs}."