Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: rdxa <rdxa@rdxa-int-spark-01.yvb.moe> Co-authored-by: Yangmin Li <yangminl@nvidia.com> Co-authored-by: Manrique <nanomlm@gmail.com> Co-authored-by: yhyang201 <yhyang201@gmail.com>
456 lines
19 KiB
Python
456 lines
19 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
"""Server-argument resolution for the GPU memory budget."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import logging
|
|
from typing import Any
|
|
|
|
from sglang.srt.arg_groups.overrides import (
|
|
declare_resolution,
|
|
model_config_of,
|
|
post_capture_kv_sizing_planned,
|
|
resolved_view,
|
|
resolving_view,
|
|
use_mla_backend,
|
|
)
|
|
from sglang.srt.environ import envs
|
|
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase
|
|
from sglang.srt.runtime_context import get_platform
|
|
from sglang.srt.utils.common import get_device_memory_capacity
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_DEFAULT_PP_PREFILL_CUDA_GRAPH_MAX_TOKENS = 8192
|
|
|
|
|
|
def handle_offload_compatibility(server_args: Any) -> None:
|
|
"""Flag-only check; re-run after the model overrides fill in the PLE default."""
|
|
cfg = resolving_view(server_args)
|
|
if cfg.ple_offload_embedding and (
|
|
cfg.cpu_offload_gb > 0 or cfg.offload_group_size > 0
|
|
):
|
|
raise ValueError(
|
|
"--ple-offload-embedding cannot be combined with "
|
|
"--cpu-offload-gb or --offload-group-size: generic layer offload "
|
|
"would stage the pinned PLE embedding back to the device."
|
|
)
|
|
|
|
if cfg.ple_offload_backend == "file" and cfg.ple_offload_embedding is False:
|
|
raise ValueError(
|
|
"--ple-offload-backend file requires --ple-offload-embedding: "
|
|
"the file-backed table is the offloaded table."
|
|
)
|
|
|
|
|
|
def handle_gpu_memory_settings(server_args: Any):
|
|
"""
|
|
Configure GPU memory-dependent settings including
|
|
chunked_prefill_size, cuda_graph_config[decode].max_bs, and mem_fraction_static.
|
|
|
|
Here are our heuristics:
|
|
- Set chunked_prefill_size and cuda_graph_config[decode].max_bs based on the GPU memory capacity.
|
|
This is because GPUs with more memory are generally more powerful, we need to use a larger
|
|
chunked_prefill_size and a larger decode max_bs to fully utilize the GPU.
|
|
- Then set mem_fraction_static based on chunked_prefill_size and decode max_bs.
|
|
|
|
GPU memory capacity = model weights + KV cache pool + activations + cuda graph buffers
|
|
|
|
The argument mem_fraction_static is defined as (model weights + KV cache pool) / GPU memory capacity,
|
|
or equivalently, mem_fraction_static = (GPU memory capacity - activations - cuda graph buffers) / GPU memory capacity.
|
|
|
|
In order to compute mem_fraction_static, we need to estimate the size of activations and cuda graph buffers.
|
|
The activation memory is proportional to the chunked_prefill_size.
|
|
The cuda graph memory is proportional to the decode max_bs.
|
|
We use reserved_mem = chunked_prefill_size * 1.5 + max_bs * 2 to estimate the size of activations and cuda graph buffers in GB,
|
|
and set mem_fraction_static = (GPU memory capacity - reserved_mem) / GPU memory capacity.
|
|
|
|
The coefficient 1.5 is a heuristic value, in the future, we can do better estimation by looking at the model types, hidden sizes or even do a dummy run.
|
|
"""
|
|
from sglang.srt.arg_groups.cuda_graph_hook import (
|
|
generate_cpu_graph_batch_sizes,
|
|
generate_decode_cuda_graph_batch_sizes,
|
|
generate_prefill_cuda_graph_batch_sizes,
|
|
)
|
|
|
|
cfg = resolving_view(server_args)
|
|
gpu_mem = get_device_memory_capacity(cfg.device)
|
|
# A copy, so an earlier declaration keeps the value it recorded.
|
|
cuda_graph_config = copy.deepcopy(cfg.cuda_graph_config)
|
|
decode_cuda_graph_config = cuda_graph_config.decode
|
|
prefill_cuda_graph_config = cuda_graph_config.prefill
|
|
|
|
# ------------------------------------------------------------------
|
|
# GPU-dependent capacity defaults
|
|
# ------------------------------------------------------------------
|
|
|
|
if gpu_mem is not None:
|
|
if gpu_mem < 20 * 1024:
|
|
# T4, 4080
|
|
# (chunked_prefill_size 2k, max_bs 8)
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=2048,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
decode_cuda_graph_config.max_bs = 8
|
|
elif gpu_mem < 35 * 1024:
|
|
# A10, 4090, 5090
|
|
# (chunked_prefill_size 4k, max_bs 48 if tp < 4 else 160)
|
|
# 32GB Blackwell (RTX 5090) can hold decode cuda graphs well past
|
|
# bs=24; the previous cap forced eager decode at bs>=32 and
|
|
# collapsed high-concurrency throughput vs vLLM.
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=4096,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
if cfg.tp_size < 4:
|
|
decode_cuda_graph_config.max_bs = 48
|
|
else:
|
|
decode_cuda_graph_config.max_bs = 160
|
|
elif gpu_mem < 60 * 1024:
|
|
# A100 (40GB), L40,
|
|
# (chunked_prefill_size 4k, max_bs 32 if tp < 4 else 160)
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=4096,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
if cfg.tp_size < 4:
|
|
decode_cuda_graph_config.max_bs = 32
|
|
else:
|
|
decode_cuda_graph_config.max_bs = 160
|
|
elif gpu_mem < 90 * 1024:
|
|
# H100, A100
|
|
# (chunked_prefill_size 8k, max_bs 256 if tp < 4 else 512)
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=8192,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
if cfg.tp_size < 4:
|
|
decode_cuda_graph_config.max_bs = 256
|
|
else:
|
|
decode_cuda_graph_config.max_bs = 512
|
|
elif gpu_mem < 160 * 1024:
|
|
# H20, H200
|
|
# (chunked_prefill_size 8k, max_bs 256 if tp < 4 else 512)
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=8192,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
if cfg.tp_size < 4:
|
|
decode_cuda_graph_config.max_bs = 256
|
|
else:
|
|
decode_cuda_graph_config.max_bs = 512
|
|
else:
|
|
# B200, MI300
|
|
# (chunked_prefill_size 16k, max_bs 512)
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=16384,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
decode_cuda_graph_config.max_bs = 512
|
|
else:
|
|
# Fallback defaults when gpu_mem is None
|
|
if cfg.chunked_prefill_size is None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
chunked_prefill_size=4096,
|
|
)
|
|
if decode_cuda_graph_config.max_bs is None:
|
|
decode_cuda_graph_config.max_bs = 160
|
|
|
|
from sglang.srt.arg_groups.model_overrides.qwen3_vl import (
|
|
expand_multimodal_decode_graph_to_running_limit,
|
|
)
|
|
|
|
expand_multimodal_decode_graph_to_running_limit(
|
|
server_args, decode_cuda_graph_config, gpu_mem
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# CUDA graph batch-size materialization
|
|
# ------------------------------------------------------------------
|
|
if cfg.device != "cpu":
|
|
if decode_cuda_graph_config.bs is None:
|
|
decode_cuda_graph_config.bs = generate_decode_cuda_graph_batch_sizes(
|
|
server_args, decode_cuda_graph_config.max_bs
|
|
)
|
|
else:
|
|
decode_cuda_graph_config.max_bs = max(decode_cuda_graph_config.bs)
|
|
else:
|
|
# Reuse decode_cuda_graph_config.bs for cpu graph and use torch_compile_max_bs for cpu graph batch size limit,
|
|
# as cpu graph is based on torch.compile
|
|
if decode_cuda_graph_config.bs is not None:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
torch_compile_max_bs=max(decode_cuda_graph_config.bs),
|
|
)
|
|
else:
|
|
# If decode_cuda_graph_config.bs is not set, we will preferentially use torch_compile_max_bs
|
|
# to generate decode_cuda_graph_config.bs
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
torch_compile_max_bs=cfg.torch_compile_max_bs
|
|
or decode_cuda_graph_config.max_bs,
|
|
)
|
|
decode_cuda_graph_config.bs = generate_cpu_graph_batch_sizes(server_args)
|
|
|
|
assert cfg.torch_compile_max_bs > 0, (
|
|
"cuda_graph_config[decode].bs should contain positive batch sizes"
|
|
)
|
|
decode_cuda_graph_config.max_bs = cfg.torch_compile_max_bs
|
|
|
|
if prefill_cuda_graph_config.max_bs is None:
|
|
# Refer to pr #15927, by default we set the prefill max_bs to the chunked prefill size.
|
|
# For MLA backend, the introduction of piecewise cuda graph will influence the kernel dispatch difference compared to the original mode.
|
|
# To avoid the performance regression, we set max_bs to 2048 by default.
|
|
if not use_mla_backend(server_args):
|
|
prefill_cuda_graph_config.max_bs = cfg.chunked_prefill_size
|
|
else:
|
|
prefill_cuda_graph_config.max_bs = 2048
|
|
|
|
# For opt-in PP breakable graphs, capture small aggregate-token
|
|
# buckets by default and leave larger forwards on the eager path.
|
|
# Explicit max_bs or bs settings retain their existing semantics.
|
|
if (
|
|
cfg.pp_size > 1
|
|
and prefill_cuda_graph_config.backend == Backend.BREAKABLE
|
|
and (Phase.PREFILL, "bs") not in server_args._cuda_graph_config_locked
|
|
and prefill_cuda_graph_config.max_bs
|
|
> _DEFAULT_PP_PREFILL_CUDA_GRAPH_MAX_TOKENS
|
|
):
|
|
prefill_cuda_graph_config.max_bs = _DEFAULT_PP_PREFILL_CUDA_GRAPH_MAX_TOKENS
|
|
|
|
# If max_total_tokens is set, cap prefill max_bs to not exceed max_total_tokens.
|
|
if cfg.max_total_tokens is not None:
|
|
prefill_cuda_graph_config.max_bs = min(
|
|
prefill_cuda_graph_config.max_bs, cfg.max_total_tokens
|
|
)
|
|
|
|
# For Llama2 series models, max_bs is limited to 4096.
|
|
# TODO(yuwei): remove this after the issue is fixed
|
|
if "llama-2" in cfg.model_path.lower():
|
|
prefill_cuda_graph_config.max_bs = min(
|
|
prefill_cuda_graph_config.max_bs, 4096
|
|
)
|
|
|
|
if prefill_cuda_graph_config.bs is None:
|
|
prefill_cuda_graph_config.bs = generate_prefill_cuda_graph_batch_sizes(
|
|
prefill_cuda_graph_config.max_bs
|
|
)
|
|
|
|
if cuda_graph_config != cfg.cuda_graph_config:
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
cuda_graph_config=cuda_graph_config,
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Static memory and runtime headroom
|
|
# ------------------------------------------------------------------
|
|
|
|
if cfg.mem_fraction_static is None:
|
|
model_config = model_config_of(server_args)
|
|
is_vlm = (
|
|
model_config.is_multimodal
|
|
and not cfg.language_only
|
|
and not cfg.language_model_only
|
|
and cfg.disaggregation_mode != "decode"
|
|
)
|
|
post_capture_kv_sizing = post_capture_kv_sizing_planned(server_args)
|
|
|
|
if post_capture_kv_sizing:
|
|
# Post-capture sizing measures free memory after graph capture, so
|
|
# skip the graph/activation reserve; keep only the floor + parallel slack.
|
|
reserved_mem = 1536
|
|
reserved_mem += cfg.tp_size * cfg.pp_size / 8 * 1024
|
|
else:
|
|
# Tokens the activation working set scales with (per serving mode).
|
|
if cfg.disaggregation_mode == "decode":
|
|
running_requests = (
|
|
cfg.max_running_requests or decode_cuda_graph_config.max_bs or 1
|
|
)
|
|
draft_tokens = cfg.speculative_num_draft_tokens or 1
|
|
activation_tokens = max(running_requests * draft_tokens, 2048)
|
|
elif cfg.chunked_prefill_size > 0:
|
|
activation_tokens = max(cfg.chunked_prefill_size, 2048)
|
|
else:
|
|
activation_tokens = max(cfg.max_prefill_tokens, 2048)
|
|
# Constant meta data (e.g., from attention backend) + activation slack.
|
|
reserved_mem = 512
|
|
reserved_mem += activation_tokens * 1.5
|
|
# Some adjustments for large parallel size
|
|
reserved_mem += cfg.tp_size * cfg.pp_size / 8 * 1024
|
|
reserved_mem += reserve_for_graph_mb(server_args)
|
|
if gpu_mem is not None and gpu_mem > 60 * 1024:
|
|
reserved_mem = max(reserved_mem, 10 * 1024)
|
|
# Reserve headroom for DeepEP all-to-all buffers on top of the floor.
|
|
reserved_mem += reserve_for_deepep_a2a_mb(server_args)
|
|
# XPU: oneDNN allocates scratch space for matmul when
|
|
# M is not a power-of-2-aligned value (e.g. M=2100). Reserve extra
|
|
# headroom so non-aligned prefill lengths don't hit OOM.
|
|
if get_platform().is_xpu:
|
|
reserved_mem += 2 * 1024
|
|
|
|
mem_fraction_static = (
|
|
round((gpu_mem - reserved_mem) / gpu_mem, 3)
|
|
if gpu_mem is not None
|
|
else 0.95
|
|
)
|
|
|
|
# Multimodal models need more memory for the image processing.
|
|
if is_vlm:
|
|
mem_fraction_static = adjust_mem_fraction_for_vlm(
|
|
mem_fraction_static,
|
|
model_config,
|
|
post_capture_kv_sizing,
|
|
gpu_mem,
|
|
)
|
|
|
|
declare_resolution(
|
|
server_args,
|
|
"_handle_gpu_memory_settings",
|
|
mem_fraction_static=mem_fraction_static,
|
|
)
|
|
|
|
# ------------------------------------------------------------------
|
|
# Symmetric-memory preallocation
|
|
# ------------------------------------------------------------------
|
|
|
|
if cfg.enable_symm_mem and not envs.SGLANG_SYMM_MEM_PREALLOC_GB_SIZE.is_set():
|
|
envs.SGLANG_SYMM_MEM_PREALLOC_GB_SIZE.set(4)
|
|
logger.warning(
|
|
"Symmetric memory is enabled, setting symmetric memory prealloc size to 4GB as default."
|
|
"Use environment variable SGLANG_SYMM_MEM_PREALLOC_GB_SIZE to change the prealloc size."
|
|
)
|
|
|
|
|
|
def reserve_for_graph_mb(server_args: Any) -> float:
|
|
|
|
cfg = resolving_view(server_args)
|
|
decode_cuda_graph_config = cfg.cuda_graph_config.decode
|
|
prefill_cuda_graph_config = cfg.cuda_graph_config.prefill
|
|
|
|
reserved_mem = 0.0
|
|
if (
|
|
cfg.disaggregation_mode != "prefill"
|
|
and decode_cuda_graph_config.backend != Backend.DISABLED
|
|
):
|
|
reserved_mem += decode_cuda_graph_config.max_bs * 2
|
|
|
|
if (
|
|
resolved_view(server_args).enable_dp_attention
|
|
and cfg.disaggregation_mode != "prefill"
|
|
):
|
|
# DP attention needs more padding for some operations, and much more for large
|
|
# cuda graph max bs (torch allocator / implementation inefficiencies).
|
|
reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 3
|
|
if decode_cuda_graph_config.max_bs > 300:
|
|
reserved_mem += decode_cuda_graph_config.max_bs * cfg.dp_size * 1.5
|
|
|
|
if (
|
|
cfg.disaggregation_mode != "decode"
|
|
and prefill_cuda_graph_config.backend != Backend.DISABLED
|
|
):
|
|
if not use_mla_backend(server_args):
|
|
# Only non-torch memory is counted; torch memory is reused by cuda graph capture.
|
|
reserved_mem += len(prefill_cuda_graph_config.bs) * 8
|
|
else:
|
|
# MLA backend overhead is much higher than expected with fa3.
|
|
reserved_mem += 1.5 * 1024
|
|
|
|
if (
|
|
prefill_cuda_graph_config.backend == Backend.BREAKABLE
|
|
and resolved_view(server_args).moe_a2a_backend == "deepep"
|
|
):
|
|
# Prefill-BCG DeepEP delta (bridge pool + NVL first-touch
|
|
# during capture); decode-side DeepEP is a baseline cost.
|
|
reserved_mem += 1 * 1024
|
|
|
|
return reserved_mem
|
|
|
|
|
|
def reserve_for_deepep_a2a_mb(server_args: Any) -> float:
|
|
# DeepEP all-to-all buffers captured in the decode graph are real extra
|
|
# allocations, reserved on top of the floor.
|
|
|
|
cfg = resolving_view(server_args)
|
|
decode_cuda_graph_config = cfg.cuda_graph_config.decode
|
|
if (
|
|
cfg.disaggregation_mode != "prefill"
|
|
and decode_cuda_graph_config.backend != Backend.DISABLED
|
|
and resolved_view(server_args).moe_a2a_backend == "deepep"
|
|
):
|
|
return 2 * 1024
|
|
return 0.0
|
|
|
|
|
|
def adjust_mem_fraction_for_vlm(
|
|
mem_fraction_static: float,
|
|
model_config,
|
|
post_capture_kv_sizing: bool,
|
|
gpu_mem,
|
|
) -> float:
|
|
if post_capture_kv_sizing:
|
|
# Graph and activation memory will be measured after capture, so only
|
|
# reserve a fixed 8 GiB of VLM headroom for image-encoder execution here.
|
|
return (
|
|
mem_fraction_static - 8 * 1024 / gpu_mem
|
|
if gpu_mem is not None
|
|
else mem_fraction_static
|
|
)
|
|
|
|
vision_config = getattr(model_config.hf_config, "vision_config", None)
|
|
if vision_config is None:
|
|
return mem_fraction_static
|
|
|
|
# roughly reduce the mem_fraction_static base on params of Vit
|
|
# a base mem_fraction_static factor for regular Vit
|
|
base_mem_fraction_reduction_ratio = 0.95
|
|
|
|
vit_num_layers = getattr(vision_config, "num_hidden_layers", 24)
|
|
vit_hidden_size = getattr(vision_config, "hidden_size", 1024)
|
|
|
|
# baseline ViT params (ViT-L/14)
|
|
baseline_vit_layers = 24
|
|
baseline_vit_hidden_size = 1024
|
|
|
|
# weight params count
|
|
current_complexity_score = vit_num_layers * (vit_hidden_size**2)
|
|
baseline_complexity_score = baseline_vit_layers * (baseline_vit_hidden_size**2)
|
|
complexity_ratio = (
|
|
current_complexity_score / baseline_complexity_score
|
|
if baseline_complexity_score > 0
|
|
else 1.0
|
|
)
|
|
|
|
# every time the complexity grows 100%, adjust final factor for 10%
|
|
sensitivity_scale = 0.1
|
|
dynamic_adjustment_factor = 1.0 - sensitivity_scale * (complexity_ratio - 1.0)
|
|
dynamic_adjustment_factor = max(0.8, min(1.05, dynamic_adjustment_factor))
|
|
|
|
final_overall_factor = base_mem_fraction_reduction_ratio * dynamic_adjustment_factor
|
|
return mem_fraction_static * final_overall_factor
|