From 3be5a7ec8976278196e7022a15671cfed83e8782 Mon Sep 17 00:00:00 2001 From: Vedant V Jhaveri Date: Fri, 12 Jun 2026 12:30:47 -0700 Subject: [PATCH] Respect explicit --max-running-requests instead of clamping to heuristic (#27399) Co-authored-by: Vedant Jhaveri --- .../srt/model_executor/model_runner_kv_cache_mixin.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py index fcba8ad6c..0b308a257 100644 --- a/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py +++ b/python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py @@ -885,8 +885,10 @@ class ModelRunnerKVCacheMixin: max_num_reqs = self.server_args.max_running_requests if max_num_reqs is not None: - max_num_reqs = min(max_num_reqs // self.dp_size, estimated) + requested_per_worker = max_num_reqs // self.dp_size + max_num_reqs = min(requested_per_worker, token_capacity // 2) else: + requested_per_worker = None max_num_reqs = min(estimated, token_capacity // 2) if self.mambaish_config is not None: @@ -904,6 +906,13 @@ class ModelRunnerKVCacheMixin: f"(2) increase --mem-fraction-static, or " f"(3) use GPUs with more memory." ) + if requested_per_worker is not None and max_num_reqs < requested_per_worker: + logger.warning( + "max_running_requests was reduced from the requested %d to %d " + "(per dp worker) due to the available KV cache capacity.", + requested_per_worker, + max_num_reqs, + ) logger.info( f"Max concurrent requests (per dp worker) from the finalized token capacity: " f"max_num_reqs={max_num_reqs}."