[Perf] cuda-graph: Raise MLA prefill capture default to chunked size capped at 8192 (#45565)
This commit is contained in:
@@ -231,13 +231,11 @@ def handle_gpu_memory_settings(server_args: Any):
|
||||
decode_cuda_graph_config.max_bs = cfg.torch_compile_max_bs
|
||||
|
||||
if prefill_cuda_graph_config.max_bs is None:
|
||||
# Refer to pr #15927, by default we set the prefill max_bs to the chunked prefill size.
|
||||
# For MLA backend, the introduction of piecewise cuda graph will influence the kernel dispatch difference compared to the original mode.
|
||||
# To avoid the performance regression, we set max_bs to 2048 by default.
|
||||
# Default to the chunked prefill size; cap MLA capture at 8192 tokens.
|
||||
if not use_mla_backend(server_args):
|
||||
prefill_cuda_graph_config.max_bs = cfg.chunked_prefill_size
|
||||
else:
|
||||
prefill_cuda_graph_config.max_bs = 2048
|
||||
prefill_cuda_graph_config.max_bs = min(8192, cfg.chunked_prefill_size)
|
||||
|
||||
# For opt-in PP breakable graphs, capture small aggregate-token
|
||||
# buckets by default and leave larger forwards on the eager path.
|
||||
|
||||
Reference in New Issue
Block a user