minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Tiny remove additional args in init_memory_pool (#16158)

Browse Source
This commit is contained in:
Ke Bao
2025-12-30 21:38:06 +08:00
committed by GitHub
parent 5d200dd8d9
commit 059428bd8a
2 changed files with 4 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/model_executor/model_runner.py
+1 -1
View File
@@ -518,7 +518,7 @@ class ModelRunner(ModelRunnerKVCacheMixin):
self.configure_kv_cache_dtype() self.configure_kv_cache_dtype()
# Init memory pool and attention backends # Init memory pool and attention backends
self.init_memory_pool(min_per_gpu_memory, server_args) self.init_memory_pool(min_per_gpu_memory)
# Init max running requests # Init max running requests
self.max_running_requests = min( self.max_running_requests = min(
python/sglang/srt/model_executor/model_runner_kv_cache_mixin.py
+3 -6
View File
@@ -33,7 +33,6 @@ from sglang.srt.utils.common import (
if TYPE_CHECKING: if TYPE_CHECKING:
from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.model_executor.model_runner import ModelRunner
from sglang.srt.server_args import ServerArgs
# the ratio of mamba cache pool size to max_running_requests # the ratio of mamba cache pool size to max_running_requests
MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO = 3 MAMBA_CACHE_SIZE_MAX_RUNNING_REQUESTS_RATIO = 3
@@ -268,11 +267,9 @@ class ModelRunnerKVCacheMixin:
f"Use sliding window memory pool. full_layer_tokens={self.full_max_total_num_tokens}, swa_layer_tokens={self.swa_max_total_num_tokens}" f"Use sliding window memory pool. full_layer_tokens={self.full_max_total_num_tokens}, swa_layer_tokens={self.swa_max_total_num_tokens}"
) )
def init_memory_pool( def init_memory_pool(self: ModelRunner, total_gpu_memory: int):
self: ModelRunner, total_gpu_memory: int, server_args: ServerArgs max_num_reqs = self.server_args.max_running_requests
): max_total_tokens = self.server_args.max_total_tokens
max_num_reqs = server_args.max_running_requests
max_total_tokens = server_args.max_total_tokens
self.max_total_num_tokens = self.profile_max_num_token(total_gpu_memory) self.max_total_num_tokens = self.profile_max_num_token(total_gpu_memory)
if max_num_reqs is None: if max_num_reqs is None: