[Perf] cuda-graph: Raise MLA prefill capture default to chunked size capped at 8192 (#45565)

This commit is contained in:
Baizhou Zhang
2026-10-21 23:49:25 -07:00
committed by GitHub
parent f68e82e7bc
commit 779adb4b82
+2 -4
View File
@@ -231,13 +231,11 @@ def handle_gpu_memory_settings(server_args: Any):
decode_cuda_graph_config.max_bs = cfg.torch_compile_max_bs
if prefill_cuda_graph_config.max_bs is None:
# Refer to pr #15927, by default we set the prefill max_bs to the chunked prefill size.
# For MLA backend, the introduction of piecewise cuda graph will influence the kernel dispatch difference compared to the original mode.
# To avoid the performance regression, we set max_bs to 2048 by default.
# Default to the chunked prefill size; cap MLA capture at 8192 tokens.
if not use_mla_backend(server_args):
prefill_cuda_graph_config.max_bs = cfg.chunked_prefill_size
else:
prefill_cuda_graph_config.max_bs = 2048
prefill_cuda_graph_config.max_bs = min(8192, cfg.chunked_prefill_size)
# For opt-in PP breakable graphs, capture small aggregate-token
# buckets by default and leave larger forwards on the eager path.