From f3b2725609d9a733cad13eb3bd4ee3d802d4c167 Mon Sep 17 00:00:00 2001 From: Xiaoyu Zhang <1182563586@qq.com> Date: Fri, 4 Sep 2026 18:47:26 +0800 Subject: [PATCH] sm120 32GB mem-tier: raise decode cuda-graph max_bs 24->48 + chunked_prefill 2k->4k (#37898) --- python/sglang/srt/arg_groups/memory_hook.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/arg_groups/memory_hook.py b/python/sglang/srt/arg_groups/memory_hook.py index 76606b6f2..af06d3bd7 100644 --- a/python/sglang/srt/arg_groups/memory_hook.py +++ b/python/sglang/srt/arg_groups/memory_hook.py @@ -77,18 +77,21 @@ def handle_gpu_memory_settings(server_args: Any, gpu_mem): decode_cuda_graph_config.max_bs = 8 elif gpu_mem < 35 * 1024: # A10, 4090, 5090 - # (chunked_prefill_size 2k, max_bs 24 if tp < 4 else 80) + # (chunked_prefill_size 4k, max_bs 48 if tp < 4 else 160) + # 32GB Blackwell (RTX 5090) can hold decode cuda graphs well past + # bs=24; the previous cap forced eager decode at bs>=32 and + # collapsed high-concurrency throughput vs vLLM. if cfg.chunked_prefill_size is None: declare_resolution( server_args, "_handle_gpu_memory_settings", - chunked_prefill_size=2048, + chunked_prefill_size=4096, ) if decode_cuda_graph_config.max_bs is None: if cfg.tp_size < 4: - decode_cuda_graph_config.max_bs = 24 + decode_cuda_graph_config.max_bs = 48 else: - decode_cuda_graph_config.max_bs = 80 + decode_cuda_graph_config.max_bs = 160 elif gpu_mem < 60 * 1024: # A100 (40GB), L40, # (chunked_prefill_size 4k, max_bs 32 if tp < 4 else 160)