minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Core] Unify max_num_reqs dp_size division for pool sizing (#20063)

Browse Source 
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Liangsheng Yin
2026-03-06 16:12:59 -08:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 7a12255b6e
commit ddcecdea49
1 changed files with 8 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py
+8 -19
View File
@@ -676,20 +676,18 @@ class ModelRunnerKVCacheMixin:
) )
def init_memory_pool(self: ModelRunner, pre_model_load_memory: int): def init_memory_pool(self: ModelRunner, pre_model_load_memory: int):
max_num_reqs = self.server_args.max_running_requests
max_total_tokens = self.server_args.max_total_tokens max_total_tokens = self.server_args.max_total_tokens
self.max_total_num_tokens = self.profile_max_num_token(pre_model_load_memory) self.max_total_num_tokens = self.profile_max_num_token(pre_model_load_memory)
if max_num_reqs is None: # Resolve max_num_reqs (per dp worker)
max_num_reqs = min( max_num_reqs = self.server_args.max_running_requests
max( if max_num_reqs is not None:
int( max_num_reqs = max_num_reqs // self.dp_size
self.max_total_num_tokens / self.model_config.context_len * 512 else:
), estimated = int(
2048, self.max_total_num_tokens / self.model_config.context_len * 512
),
4096,
) )
max_num_reqs = max(min(estimated, 4096), 2048)
if self.mambaish_config is not None: if self.mambaish_config is not None:
ratio = self._calculate_mamba_ratio() ratio = self._calculate_mamba_ratio()
@@ -699,15 +697,6 @@ class ModelRunnerKVCacheMixin:
max_num_reqs, self.server_args.max_mamba_cache_size // ratio max_num_reqs, self.server_args.max_mamba_cache_size // ratio
) )
# for dp attention, we need control the max_num_reqs for speculative decoding mamba space
if (
not self.spec_algorithm.is_none()
and self.server_args.enable_dp_attention
):
max_num_reqs = min(
max_num_reqs, self.server_args.max_running_requests // self.dp_size
)
if max_total_tokens is not None: if max_total_tokens is not None:
if max_total_tokens > self.max_total_num_tokens: if max_total_tokens > self.max_total_num_tokens:
logging.warning( logging.warning(