Files
sglang/python/sglang/srt/arg_groups/serving_hook.py
T
5b5d721239 [PD] Add opt-in prefill-complete decode KV allocation (#40703)
Co-authored-by: Po-Han Huang (NVIDIA) <53919306+nvpohanh@users.noreply.github.com>
Co-authored-by: Po-Han Huang <pohanh@nvidia.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Shangming Cai <csmthu@gmail.com>
2026-10-03 21:03:09 +08:00

930 lines
36 KiB
Python

# SPDX-License-Identifier: Apache-2.0
"""Server-argument resolution for serving-surface and multimodal entry validation."""
from __future__ import annotations
import json
import logging
import os
import random
import socket
from typing import Any
from sglang.srt.arg_groups.overrides import (
declare_resolution,
model_config_of,
resolving_view,
)
from sglang.srt.environ import envs
from sglang.srt.model_executor.cuda_graph_config import Backend, Phase, with_phase
from sglang.srt.runtime_context import get_platform
from sglang.srt.utils.common import (
configure_media_url_security,
get_device,
is_gfx95_supported,
is_mnnvl_fabric_device,
)
from sglang.utils import is_in_ci
logger = logging.getLogger(__name__)
def handle_ssl_validation(server_args: Any):
"""Ensure SSL arguments are consistent and referenced files exist."""
cfg = resolving_view(server_args)
if cfg.ssl_keyfile and not cfg.ssl_certfile:
raise ValueError(
"--ssl-keyfile requires --ssl-certfile to be specified as well."
)
if cfg.ssl_certfile and not cfg.ssl_keyfile:
raise ValueError(
"--ssl-certfile requires --ssl-keyfile to be specified as well."
)
if not cfg.ssl_certfile and not cfg.ssl_keyfile:
if cfg.ssl_ca_certs:
raise ValueError(
"--ssl-ca-certs has no effect without --ssl-certfile and --ssl-keyfile."
)
if cfg.ssl_keyfile_password:
raise ValueError(
"--ssl-keyfile-password has no effect without --ssl-certfile and --ssl-keyfile."
)
# Validate files exist early to avoid late failures after model loading.
if cfg.ssl_keyfile and not os.path.isfile(cfg.ssl_keyfile):
raise ValueError(
f"SSL key file not found: '{cfg.ssl_keyfile}'. "
f"Please check the --ssl-keyfile path."
)
if cfg.ssl_certfile and not os.path.isfile(cfg.ssl_certfile):
raise ValueError(
f"SSL certificate file not found: '{cfg.ssl_certfile}'. "
f"Please check the --ssl-certfile path."
)
if cfg.ssl_ca_certs and not os.path.isfile(cfg.ssl_ca_certs):
raise ValueError(
f"SSL CA certificates file not found: '{cfg.ssl_ca_certs}'. "
f"Please check the --ssl-ca-certs path."
)
if cfg.enable_ssl_refresh and not (cfg.ssl_certfile and cfg.ssl_keyfile):
raise ValueError(
"--enable-ssl-refresh requires --ssl-certfile and --ssl-keyfile "
"to be specified."
)
if cfg.enable_http2:
if not 0 < cfg.http2_max_concurrent_streams < 2**32:
raise ValueError(
"--http2-max-concurrent-streams must be between 1 and 4294967295."
)
if not 1024 <= cfg.http2_initial_connection_window_size < 2**31:
raise ValueError(
"--http2-initial-connection-window-size must be between "
"1024 and 2147483647."
)
try:
import granian # noqa: F401
except ImportError:
raise ValueError(
"--enable-http2 requires the 'granian' package. "
'Install it with: pip install "sglang[http2]"'
)
if cfg.enable_ssl_refresh:
raise ValueError(
"--enable-ssl-refresh is not supported with --enable-http2. "
"Granian does not support SSL certificate hot-reloading. "
"Use Uvicorn (the default) or handle certificate rotation externally."
)
def handle_asr_validation(server_args: Any):
"""Validate transcription/ASR-specific server args."""
cfg = resolving_view(server_args)
if cfg.asr_max_buffer_seconds <= 0:
raise ValueError(
f"--asr-max-buffer-seconds must be positive "
f"(got {cfg.asr_max_buffer_seconds})."
)
if cfg.asr_max_concurrent_sessions <= 0:
raise ValueError(
f"--asr-max-concurrent-sessions must be positive "
f"(got {cfg.asr_max_concurrent_sessions})."
)
def handle_multimodal(server_args: Any):
"""Validate mm_process_config structure before model loading."""
cfg = resolving_view(server_args)
if (
cfg.mm_preprocess_cache_size_mb is not None
and cfg.mm_preprocess_cache_size_mb < 0
):
raise ValueError("mm_preprocess_cache_size_mb must be non-negative")
if cfg.mm_process_config is not None:
if not isinstance(cfg.mm_process_config, dict):
raise TypeError(
f"mm_process_config must be a dict, "
f"but got {type(cfg.mm_process_config)}"
)
for key in ("image", "video", "audio"):
if key in cfg.mm_process_config and not isinstance(
cfg.mm_process_config[key], dict
):
raise TypeError(
f"mm_process_config['{key}'] must be a dict, "
f"but got {type(cfg.mm_process_config[key])}"
)
def handle_crash_dump_env(server_args: Any):
cfg = resolving_view(server_args)
if not cfg.crash_dump_folder:
return
_CUDA_COREDUMP_DEFAULTS = {
"CUDA_ENABLE_COREDUMP_ON_EXCEPTION": "1",
"CUDA_ENABLE_USER_TRIGGERED_COREDUMP": "1",
"CUDA_COREDUMP_SHOW_PROGRESS": "1",
"CUDA_COREDUMP_GENERATION_FLAGS": (
"skip_nonrelocated_elf_images,skip_global_memory,"
"skip_shared_memory,skip_local_memory,skip_constbank_memory"
),
"CUDA_COREDUMP_FILE": f"{cfg.crash_dump_folder}/%h/core.cuda.%t.%p",
"CUDA_COREDUMP_PIPE": "/tmp/corepipe.cuda.%h.%p",
}
for key, value in _CUDA_COREDUMP_DEFAULTS.items():
if key not in os.environ:
os.environ[key] = value
logger.info("Auto-set %s=%s (from --crash-dump-folder)", key, value)
coredump_dir = os.path.dirname(
os.environ["CUDA_COREDUMP_FILE"].replace("%h", socket.gethostname())
)
if "%" in coredump_dir:
logger.warning(
"Cannot pre-create CUDA coredump directory %s: only %%h is "
"supported in the directory part of CUDA_COREDUMP_FILE; "
"coredumps may fail to write.",
coredump_dir,
)
elif coredump_dir:
try:
os.makedirs(coredump_dir, exist_ok=True)
except OSError as e:
logger.warning(
"Failed to create CUDA coredump directory %s: %s; "
"coredumps may fail to write.",
coredump_dir,
e,
)
def handle_media_url_security(server_args: Any):
"""Normalize and publish the media URL policy before workers start."""
cfg = resolving_view(server_args)
declare_resolution(
server_args,
"_handle_media_url_security",
allowed_media_domains=configure_media_url_security(
cfg.allowed_media_domains,
cfg.media_url_max_file_size_mb,
),
)
def handle_load_balance_method(server_args: Any):
cfg = resolving_view(server_args)
if cfg.disaggregation_mode not in ("null", "prefill", "decode"):
raise ValueError(f"Invalid disaggregation_mode={cfg.disaggregation_mode!r}")
if cfg.load_balance_method == "auto":
# Default behavior:
# - non-PD: round_robin
# - PD prefill: follow_bootstrap_room
# - PD decode: round_robin
declare_resolution(
server_args,
"_handle_load_balance_method",
load_balance_method=(
"follow_bootstrap_room"
if cfg.disaggregation_mode == "prefill"
else "round_robin"
),
)
return
def handle_grammar_backend(server_args: Any):
cfg = resolving_view(server_args)
if cfg.grammar_backend is None:
declare_resolution(
server_args, "_handle_grammar_backend", grammar_backend="xgrammar"
)
def handle_debug_utils(server_args: Any):
cfg = resolving_view(server_args)
if is_in_ci() and cfg.soft_watchdog_timeout is None:
logger.info("Set soft_watchdog_timeout since in CI")
declare_resolution(
server_args, "_handle_debug_utils", soft_watchdog_timeout=300
)
def handle_deprecated_args(server_args: Any):
cfg = resolving_view(server_args)
if cfg.disable_fast_image_processor:
if cfg.image_processor_backend not in {"auto", "pil"}:
raise ValueError(
"--disable-fast-image-processor conflicts with "
f"--image-processor-backend={cfg.image_processor_backend}."
)
logger.warning(
"--disable-fast-image-processor is deprecated; use "
"--image-processor-backend=pil instead."
)
declare_resolution(
server_args, "_handle_deprecated_args", image_processor_backend="pil"
)
# Handle deprecated tool call parsers
deprecated_tool_call_parsers = {"qwen25": "qwen", "glm45": "glm"}
if cfg.tool_call_parser in deprecated_tool_call_parsers:
logger.warning(
f"The tool_call_parser '{cfg.tool_call_parser}' is deprecated. Please use '{deprecated_tool_call_parsers[cfg.tool_call_parser]}' instead."
)
declare_resolution(
server_args,
"_handle_deprecated_args",
tool_call_parser=deprecated_tool_call_parsers[cfg.tool_call_parser],
)
# When user passes --enable-flashinfer-allreduce-fusion, enable with auto backend
if (
cfg.enable_flashinfer_allreduce_fusion
and cfg.flashinfer_allreduce_fusion_backend is None
):
logger.warning(
"--enable-flashinfer-allreduce-fusion is deprecated. "
"Please use --flashinfer-allreduce-fusion-backend=auto instead."
)
declare_resolution(
server_args,
"_handle_deprecated_args",
flashinfer_allreduce_fusion_backend="auto",
)
declare_resolution(
server_args,
"_handle_deprecated_args",
enable_flashinfer_allreduce_fusion=False,
)
# Deprecated attention-backend alias: "compressed" -> "dsv4".
renamed = {}
for attr in (
"attention_backend",
"decode_attention_backend",
"prefill_attention_backend",
"speculative_draft_attention_backend",
):
if getattr(server_args, attr, None) == "compressed":
logger.warning(
"--%s=compressed is deprecated; use 'dsv4' instead.",
attr.replace("_", "-"),
)
renamed[attr] = "dsv4"
if renamed:
declare_resolution(server_args, "_handle_deprecated_args", **renamed)
# --grpc-mode is a deprecated alias for --smg-grpc-mode.
if cfg.grpc_mode and not cfg.smg_grpc_mode:
logger.warning(
"--grpc-mode is deprecated and will be removed in a future "
"version. Use --smg-grpc-mode for the legacy SMG gRPC server, "
"or --grpc-port for the native gRPC server."
)
declare_resolution(
server_args,
"_handle_deprecated_args",
smg_grpc_mode=True,
)
# Native gRPC tuning knob is env-only; --grpc-port (CLI) enables the
# native server, falling back to SGLANG_GRPC_PORT.
declare_resolution(
server_args,
"_handle_deprecated_args",
grpc_worker_threads=envs.SGLANG_GRPC_WORKER_THREADS.get(),
)
grpc_port_env = envs.SGLANG_GRPC_PORT.get()
if cfg.grpc_port is None and grpc_port_env is not None:
declare_resolution(
server_args,
"_handle_deprecated_args",
grpc_port=grpc_port_env,
)
# Legacy SMG defaults its port to --port + 10000. Derive/validate only
# when gRPC is in use, so HTTP-only high ports don't fail validation.
legacy_grpc = cfg.smg_grpc_mode or cfg.grpc_mode
if legacy_grpc and cfg.grpc_port is None:
declare_resolution(
server_args,
"_handle_deprecated_args",
grpc_port=cfg.port + 10000,
)
if cfg.grpc_response_timeout_secs <= 0:
raise ValueError("--grpc-response-timeout-secs must be positive")
if cfg.grpc_port is not None:
if not (1 <= cfg.grpc_port <= 65535):
raise ValueError(
"--grpc-port / SGLANG_GRPC_PORT "
f"({cfg.grpc_port}) must be between 1 and 65535"
)
if cfg.grpc_worker_threads is not None and cfg.grpc_worker_threads < 1:
raise ValueError(
f"SGLANG_GRPC_WORKER_THREADS ({cfg.grpc_worker_threads}) must be >= 1"
)
# Native gRPC is incompatible with launch paths it doesn't wire into.
# Legacy takes precedence over grpc_port, keeping re-runs idempotent.
native_grpc = cfg.grpc_port is not None and not legacy_grpc
if cfg.sidecar_args is not None:
if cfg.sidecar is None:
raise ValueError("--sidecar-args requires --sidecar.")
if not isinstance(cfg.sidecar_args, list) or not all(
isinstance(arg, str) for arg in cfg.sidecar_args
):
raise ValueError("--sidecar-args must be a JSON array of strings.")
if cfg.sidecar is not None:
if not cfg.sidecar.strip():
raise ValueError("--sidecar must not be empty.")
if legacy_grpc:
raise ValueError(
"--sidecar requires SGLang's native gRPC server; "
"it cannot be combined with --smg-grpc-mode/--grpc-mode."
)
if cfg.grpc_port is None:
raise ValueError("--sidecar requires --grpc-port or SGLANG_GRPC_PORT.")
if envs.SGLANG_RUST_SERVER.get():
raise ValueError(
"--sidecar is not yet supported with SGLANG_RUST_SERVER: "
"Rust-server mode does not run the Python sidecar lifecycle."
)
if native_grpc:
if cfg.use_ray:
raise ValueError(
"--grpc-port is not supported with --use-ray: the Ray "
"serve launch path does not start the native gRPC server."
)
if cfg.encoder_only:
raise ValueError(
"--grpc-port is not supported with --encoder-only: "
"encoder disaggregation uses its own server."
)
if cfg.tokenizer_worker_num > 1:
raise ValueError(
"Native gRPC does not yet support --tokenizer-worker-num > 1. "
"Unset --grpc-port or set --tokenizer-worker-num 1."
)
if cfg.api_key or cfg.admin_api_key:
raise ValueError(
"--grpc-port is incompatible with --api-key/--admin-api-key: "
"the native gRPC listener bypasses HTTP auth middleware."
)
def handle_environment_variables(server_args: Any):
cfg = resolving_view(server_args)
handle_multimodal_feature_transport(server_args)
envs.SGLANG_ENABLE_TORCH_COMPILE.set("1" if cfg.enable_torch_compile else "0")
if cfg.mamba_ssm_dtype is not None:
envs.SGLANG_MAMBA_SSM_DTYPE.set(cfg.mamba_ssm_dtype)
envs.SGLANG_DISABLE_OUTLINES_DISK_CACHE.set(
"1" if cfg.disable_outlines_disk_cache else "0"
)
envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.set(
"1" if cfg.enable_deterministic_inference else "0"
)
if cfg.enable_deterministic_inference:
envs.SGLANG_FLASHINFER_MOE_FUSED_FINALIZE.set("0")
if cfg.debug_cuda_graph:
if not (get_platform().is_cuda or get_platform().is_hip):
logger.warning(
"--debug-cuda-graph is not supported on non CUDA/HIP devices. "
"Disabling breakable CUDA graph."
)
declare_resolution(
server_args, "_handle_environment_variables", debug_cuda_graph=False
)
else:
envs.SGLANG_USE_BREAKABLE_CUDA_GRAPH.set("1")
logger.warning(
"Debug mode for CUDA graph is enabled via breakable CUDA graph. "
"All operations will run eagerly through the graph capture/replay path."
)
if cfg.enable_deepseek_v4_fp4_indexer and not (
get_platform().is_sm100 or get_platform().is_sm120 or is_gfx95_supported()
):
raise ValueError(
"--enable-deepseek-v4-fp4-indexer requires SM100, SM120, or gfx95 GPUs "
"with FP4 indexer support."
)
# FP8 W_o GEMM needs DeepGEMM JIT. Enable exactly where the runtime can
# run it, mirroring the forward scale split: the default sm100 UE8M0
# path, or explicit opt-in on sm90 (FP32 scales) and sm120 (UE8M0).
# SM120 API compatibility is centralized in deep_gemm_wrapper.configurer.
if get_platform().is_cuda and envs.SGLANG_OPT_FP8_WO_A_GEMM.get():
from sglang.srt.layers import deep_gemm_wrapper
sm = get_platform().device_sm
explicit = envs.SGLANG_OPT_FP8_WO_A_GEMM.is_set()
supported = deep_gemm_wrapper.DEEPGEMM_SCALE_UE8M0 or (
deep_gemm_wrapper.ENABLE_JIT_DEEPGEMM
and get_platform().is_sm90
and explicit
)
if not supported and explicit:
logger.warning(
"Disabling SGLANG_OPT_FP8_WO_A_GEMM: requires DeepGEMM JIT "
"and a compatible sm100/sm120 build, or explicit opt-in on sm90; "
"detected sm%d.",
sm,
)
if not supported:
envs.SGLANG_OPT_FP8_WO_A_GEMM.set(False)
def handle_other_validations(server_args: Any):
cfg = resolving_view(server_args)
if cfg.default_chat_template_kwargs is not None and not isinstance(
cfg.default_chat_template_kwargs, dict
):
raise ValueError("--default-chat-template-kwargs must decode to a JSON object")
# Handle optimistic prefill validation
if cfg.optimistic_prefill_attempts > 0 and cfg.disaggregation_mode == "prefill":
if cfg.pp_size > 1:
logger.warning("Optimistic prefill does not support pp_size > 1")
declare_resolution(
server_args,
"_handle_other_validations",
optimistic_prefill_attempts=0,
)
elif cfg.enable_hierarchical_cache and not (
(
cfg.hicache_storage_backend is None
and cfg.hicache_write_policy in ("write_back", "write_through")
)
or (
cfg.hicache_storage_backend is not None
and cfg.hicache_host_memory_mode == "buffer_only"
and cfg.hicache_write_policy == "write_through"
)
):
logger.warning(
"Optimistic prefill supports L2 write-back/write-through or "
"L3 buffer-only write-through hierarchical cache"
)
declare_resolution(
server_args,
"_handle_other_validations",
optimistic_prefill_attempts=0,
)
if (
cfg.disaggregation_decode_allocation_policy == "prefill_complete"
and cfg.disaggregation_mode == "prefill"
and resolving_view(server_args).optimistic_prefill_attempts <= 0
):
raise ValueError(
"prefill_complete requires optimistic prefill, but the resolved "
"model/cache configuration disabled it."
)
# Handle model inference tensor dump.
if cfg.debug_tensor_dump_output_folder is not None:
logger.warning(
"Cuda graph and server warmup are disabled because of using tensor dump mode"
)
declare_resolution(
server_args,
"_handle_other_validations",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.DECODE, backend=Backend.DISABLED
),
)
declare_resolution(
server_args,
"_handle_other_validations",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
declare_resolution(
server_args, "_handle_other_validations", skip_server_warmup=True
)
if cfg.msprobe_dump_config is not None:
logger.warning(
"When msProbe is enabled, "
"cuda graph is disabled because msProbe only supports dump in eager mode, "
"warmup is disabled(skip_server_warmup=True) because there is no need to dump data for this stage."
)
declare_resolution(
server_args,
"_handle_other_validations",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.DECODE, backend=Backend.DISABLED
),
)
declare_resolution(
server_args,
"_handle_other_validations",
cuda_graph_config=with_phase(
cfg.cuda_graph_config, Phase.PREFILL, backend=Backend.DISABLED
),
)
declare_resolution(
server_args, "_handle_other_validations", skip_server_warmup=True
)
# Validate limit_mm_per_prompt modalities
if cfg.limit_mm_data_per_request:
if isinstance(cfg.limit_mm_data_per_request, str):
declare_resolution(
server_args,
"_handle_other_validations",
limit_mm_data_per_request=json.loads(cfg.limit_mm_data_per_request),
)
if isinstance(cfg.limit_mm_data_per_request, dict):
allowed_modalities = {"image", "video", "audio"}
for modality in cfg.limit_mm_data_per_request.keys():
if modality not in allowed_modalities:
raise ValueError(
f"Invalid modality '{modality}' in --limit-mm-data-per-request."
f"Allowed modalities are: {list(allowed_modalities)}"
)
# Validate preferred_sampling_params
if cfg.preferred_sampling_params:
if isinstance(cfg.preferred_sampling_params, str):
declare_resolution(
server_args,
"_handle_other_validations",
preferred_sampling_params=json.loads(cfg.preferred_sampling_params),
)
# Validate preferred_sampling_params doesn't use tokenizer-dependent features
if cfg.skip_tokenizer_init:
from sglang.srt.sampling.sampling_params import SamplingParams
test_params = SamplingParams(**cfg.preferred_sampling_params)
# raises if tokenizer-dependent features used
test_params.normalize(None)
def handle_missing_default_values(server_args: Any):
from sglang.srt.arg_groups.model_path_hook import handle_modelscope_paths
cfg = resolving_view(server_args)
if cfg.tokenizer_path is None:
declare_resolution(
server_args,
"_handle_missing_default_values",
tokenizer_path=cfg.model_path,
)
if cfg.served_model_name is None:
declare_resolution(
server_args,
"_handle_missing_default_values",
served_model_name=cfg.model_path,
)
if cfg.device is None:
declare_resolution(
server_args,
"_handle_missing_default_values",
device=get_device(),
)
# strip device index from user if any (e.g. "cuda:0" -> "cuda")
declare_resolution(
server_args,
"_handle_missing_default_values",
device=cfg.device.split(":")[0],
)
if cfg.random_seed is None:
declare_resolution(
server_args,
"_handle_missing_default_values",
random_seed=random.randint(0, 1 << 30),
)
if cfg.mm_process_config is None:
declare_resolution(
server_args, "_handle_missing_default_values", mm_process_config={}
)
# Handle ModelScope model downloads
if envs.SGLANG_USE_MODELSCOPE.get():
handle_modelscope_paths(server_args)
# In speculative scenario:
# - If `speculative_draft_model_quantization` is specified, the draft model uses this quantization method.
# - Otherwise, the draft model defaults to the same quantization as the target model.
if cfg._speculative_draft_quantization_explicitly_set is None:
declare_resolution(
server_args,
"_handle_missing_default_values",
_speculative_draft_quantization_explicitly_set=cfg.speculative_draft_model_quantization
is not None,
)
if cfg.speculative_draft_model_quantization is None:
declare_resolution(
server_args,
"_handle_missing_default_values",
speculative_draft_model_quantization=cfg.quantization,
)
# Resolve --quantization unquant before model config validation. Record
# the explicit opt-out so later auto-detection does not re-enable
# quantization.
if cfg.quantization == "unquant":
declare_resolution(
server_args,
"_handle_missing_default_values",
quantization=None,
)
server_args._quantization_explicitly_unset = True
else:
server_args._quantization_explicitly_unset = False
if cfg.speculative_draft_model_quantization == "unquant":
declare_resolution(
server_args,
"_handle_missing_default_values",
speculative_draft_model_quantization=None,
)
def handle_return_hidden_states_mode(server_args: Any):
cfg = resolving_view(server_args)
if cfg.return_hidden_states_mode not in (None, "last", "full"):
raise ValueError(
"return_hidden_states_mode must be one of: None, 'last', or 'full'."
)
if cfg.return_hidden_states_mode is None:
if cfg.enable_return_hidden_states:
declare_resolution(
server_args,
"_handle_return_hidden_states_mode",
return_hidden_states_mode="full",
)
else:
declare_resolution(
server_args,
"_handle_return_hidden_states_mode",
enable_return_hidden_states=True,
)
def handle_prefill_delayer_env_compat(server_args: Any):
if envs.SGLANG_SCHEDULER_DECREASE_PREFILL_IDLE.get():
declare_resolution(
server_args,
"_handle_prefill_delayer_env_compat",
enable_prefill_delayer=True,
)
if x := envs.SGLANG_PREFILL_DELAYER_MAX_DELAY_PASSES.get():
declare_resolution(
server_args,
"_handle_prefill_delayer_env_compat",
prefill_delayer_max_delay_passes=x,
)
if x := envs.SGLANG_PREFILL_DELAYER_TOKEN_USAGE_LOW_WATERMARK.get():
declare_resolution(
server_args,
"_handle_prefill_delayer_env_compat",
prefill_delayer_token_usage_low_watermark=x,
)
def handle_tokenizer_batching(server_args: Any):
cfg = resolving_view(server_args)
if cfg.enable_tokenizer_batch_encode and cfg.enable_dynamic_batch_tokenizer:
raise ValueError(
"Cannot enable both --enable-tokenizer-batch-encode and --enable-dynamic-batch-tokenizer. "
"Please choose one tokenizer batching approach."
)
if cfg.skip_tokenizer_init and not envs.SGLANG_RUST_SERVER.get():
# Tokenizer workers still serve HTTP / state / output work, so
# their fanout is preserved; detokenizer workers only decode.
if cfg.detokenizer_worker_num != 1:
logger.warning(
"skip_tokenizer_init=True leaves no decode work for detokenizer workers; "
f"forcing detokenizer_worker_num=1 (requested {cfg.detokenizer_worker_num})."
)
declare_resolution(
server_args, "_handle_tokenizer_batching", detokenizer_worker_num=1
)
if cfg.enable_tokenizer_batch_encode:
logger.warning(
"skip_tokenizer_init=True ignores --enable-tokenizer-batch-encode; disabling it."
)
declare_resolution(
server_args,
"_handle_tokenizer_batching",
enable_tokenizer_batch_encode=False,
)
if cfg.enable_dynamic_batch_tokenizer:
logger.warning(
"skip_tokenizer_init=True ignores --enable-dynamic-batch-tokenizer; disabling it."
)
declare_resolution(
server_args,
"_handle_tokenizer_batching",
enable_dynamic_batch_tokenizer=False,
)
logger.info(
"skip_tokenizer_init=True: string-based stop conditions (stop, stop_regex) "
"and min_new_tokens are unavailable."
)
def handle_multimodal_feature_transport(server_args: Any):
"""Resolve multimodal feature transport before tokenizer workers start.
CUDA IPC is opt-in because its fixed pool on ``base_gpu_id`` reduces the
memory left for model/KV-cache allocations. Multi-node MNNVL deployments
may still auto-select CUDA VMM. The legacy CUDA IPC environment variable
remains supported so existing deployments map to this policy.
"""
cfg = resolving_view(server_args)
requested_transport = cfg.mm_feature_transport
legacy_ipc_is_set = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.is_set()
legacy_ipc_enabled = envs.SGLANG_USE_CUDA_IPC_TRANSPORT.get()
if requested_transport is None:
if legacy_ipc_is_set:
requested_transport = "cuda_ipc" if legacy_ipc_enabled else "cpu"
logger.warning(
"SGLANG_USE_CUDA_IPC_TRANSPORT is deprecated; use "
"--mm-feature-transport=%s instead.",
requested_transport,
)
elif cfg.encoder_only:
requested_transport = "cpu"
logger.info(
"Multimodal feature transport auto-resolved to cpu for "
"encoder-only serving; encoder outputs use "
"--encoder-transfer-backend instead."
)
elif (
model_config_of(server_args).is_multimodal
and get_platform().is_cuda
and cfg.disaggregation_mode == "null"
):
# A full GPU pool always degrades to CPU transport per tensor.
# Keep CUDA IPC opt-in because even an idle pool consumes HBM
# that would otherwise back the KV cache. Multi-node
# auto-selection is limited to GB200/GB300 systems where the
# runtime already enables the MNNVL/IMEX communication stack.
if cfg.nnodes == 1:
requested_transport = "cpu"
elif is_mnnvl_fabric_device() and os.path.exists(
"/dev/nvidia-caps-imex-channels/channel0"
):
from sglang.srt.model_loader.utils import (
supports_cuda_vmm_feature_transport,
)
if supports_cuda_vmm_feature_transport(model_config_of(server_args)):
requested_transport = "cuda_vmm"
logger.info(
"Multimodal feature transport auto-resolved to "
"cuda_vmm (multi-node GB200/GB300 MNNVL). Pass "
"--mm-feature-transport=cpu to opt out."
)
else:
requested_transport = "cpu"
logger.info(
"Multimodal feature transport auto-resolved to cpu: "
"the model has not opted into CUDA VMM transport."
)
else:
requested_transport = "cpu"
if is_mnnvl_fabric_device():
logger.info(
"Multimodal feature transport auto-resolved to cpu: "
"GB200/GB300 was detected but no IMEX channel is "
"mounted. Configure the MNNVL compute domain or pass "
"--mm-feature-transport=cuda_vmm after doing so."
)
else:
requested_transport = "cpu"
elif legacy_ipc_is_set and legacy_ipc_enabled != (
requested_transport == "cuda_ipc"
):
logger.warning(
"--mm-feature-transport=%s overrides the conflicting legacy "
"SGLANG_USE_CUDA_IPC_TRANSPORT=%s setting.",
requested_transport,
int(legacy_ipc_enabled),
)
if cfg.encoder_only and requested_transport in ("cuda_ipc", "cuda_vmm"):
logger.warning(
"--mm-feature-transport=%s does not control encoder-only "
"output transfer; using cpu for this inactive transport. Select "
"--encoder-transfer-backend for encoder outputs.",
requested_transport,
)
requested_transport = "cpu"
if requested_transport == "cuda_vmm":
if not get_platform().is_cuda:
raise ValueError("--mm-feature-transport=cuda_vmm requires NVIDIA CUDA.")
if cfg.pp_size != 1:
raise ValueError(
"--mm-feature-transport=cuda_vmm does not support pipeline parallelism."
)
pool_budget_mb = envs.SGLANG_MM_FEATURE_CACHE_MB.get()
handle_kind = "CUDA FABRIC" if cfg.nnodes > 1 else "POSIX FD"
logger.info(
"Using CUDA VMM for multimodal features with %s sharing: "
"reserving up to %d MiB on base GPU %d across %d tokenizer "
"worker(s). This reduces KV cache headroom; a full pool falls "
"back to inline CPU transport.",
handle_kind,
pool_budget_mb,
cfg.base_gpu_id,
cfg.tokenizer_worker_num,
)
if requested_transport == "cuda_ipc":
if not get_platform().is_cuda:
raise ValueError("--mm-feature-transport=cuda_ipc requires NVIDIA CUDA.")
if cfg.nnodes != 1:
raise ValueError(
"--mm-feature-transport=cuda_ipc only supports a single node."
)
pool_budget_mb = envs.SGLANG_MM_FEATURE_CACHE_MB.get()
logger.info(
"Using CUDA IPC for multimodal features: reserving up to %d MiB "
"on base GPU %d across %d tokenizer worker(s). This reduces KV "
"cache headroom; a full pool falls back to CPU transport.",
pool_budget_mb,
cfg.base_gpu_id,
cfg.tokenizer_worker_num,
)
logger.info(
"CUDA IPC pool-handle caching is %s. It reuses mappings to the "
"existing bounded pool without reserving another pool; set "
"SGLANG_USE_IPC_POOL_HANDLE_CACHE=0 to disable it.",
("enabled" if envs.SGLANG_USE_IPC_POOL_HANDLE_CACHE.get() else "disabled"),
)
declare_resolution(
server_args,
"_handle_multimodal_feature_transport",
mm_feature_transport=requested_transport,
)
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.set(
"1" if requested_transport == "cuda_ipc" else "0"
)
_ssl_verify_warned = False
def ssl_verify_of(cfg: Any):
"""What to pass as the requests library's ``verify=``.
A CA file means validate against it. SSL configured without one means
verification off -- self-signed certificates in development -- and that is
worth saying out loud, once. No SSL means the system CA bundle.
The warning is once per process: the message is about how this process was
configured, and a second engine repeating it says nothing new.
"""
global _ssl_verify_warned
if cfg.ssl_ca_certs:
return cfg.ssl_ca_certs
if cfg.ssl_certfile:
if not _ssl_verify_warned:
logger.warning(
"SSL is enabled but --ssl-ca-certs was not provided. Certificate "
"verification is DISABLED for internal health checks. For "
"production deployments, provide --ssl-ca-certs or use CA-signed "
"certificates."
)
_ssl_verify_warned = True
return False
return True