minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

config: the forwarding slots go; the dispatcher calls the family directly (#36792)

Browse Source 
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Cheng Wan
2026-08-28 10:26:29 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 7bc3204117
commit ef20fab38a
23 changed files with 617 additions and 800 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/arg_groups/cuda_graph_hook.py
+7 -7
View File
@@ -138,11 +138,11 @@ def apply_cuda_graph_compatibility(server_args: Any):
)
if cfg.cuda_graph_config.prefill.backend == Backend.TC_PIECEWISE:
server_args._disable_tc_piecewise_cudagraph_if_incompatible()
disable_tc_piecewise_cudagraph_if_incompatible(server_args)
elif cfg.cuda_graph_config.prefill.backend == Backend.BREAKABLE:
server_args._disable_breakable_cudagraph_if_incompatible()
disable_breakable_cudagraph_if_incompatible(server_args)
elif cfg.cuda_graph_config.prefill.backend == Backend.FULL:
server_args._disable_full_prefill_cudagraph_if_incompatible()
disable_full_prefill_cudagraph_if_incompatible(server_args)
def disable_tc_piecewise_cudagraph_if_incompatible(server_args: Any):
@@ -427,11 +427,11 @@ def apply_muse_glimmer_prefill_cuda_graph_max_bs_default(server_args: Any):
def handle_cuda_graph_config(server_args: Any):
cfg = resolving_view(server_args)
server_args._parse_cuda_graph_config()
server_args._apply_cuda_graph_compatibility()
server_args._apply_deepep_adjustments()
parse_cuda_graph_config(server_args)
apply_cuda_graph_compatibility(server_args)
apply_deepep_adjustments(server_args)
server_args._apply_cuda_graph_disaggregation_roles()
server_args._validate_cuda_graph_config()
validate_cuda_graph_config(server_args)
# Warn on the final resolved config (not inside the compat cascade —
# that path is skipped when the user explicitly sets the backend,
# which is the only way to get 'full' for prefill today).
python/sglang/srt/arg_groups/hicache_hook.py
+4 -4
View File
@@ -33,16 +33,16 @@ def handle_hicache(server_args: Any):
):
return
server_args._validate_hicache_host_memory_mode()
validate_hicache_host_memory_mode(server_args)
# Step 1: Initial layout-io compatibility normalization.
server_args._resolve_layout_io_compatibility()
resolve_layout_io_compatibility(server_args)
# Step 2: Storage-layout normalization without changing io backend.
server_args._resolve_storage_layout_compatibility()
resolve_storage_layout_compatibility(server_args)
# Step 3: DCP compatibility for the L2 (device<->host) path.
server_args._resolve_hicache_dcp_compatibility()
resolve_hicache_dcp_compatibility(server_args)
def handle_hicache_ratio_default(server_args: Any):
python/sglang/srt/arg_groups/lora_hook.py
+1 -1
View File
@@ -50,7 +50,7 @@ def check_lora_server_args(server_args: Any):
)
# Validate compatibility with speculative decoding
server_args._check_lora_speculative_compatibility()
check_lora_speculative_compatibility(server_args)
# Parse lora_paths
if isinstance(cfg.lora_paths, list):
python/sglang/srt/arg_groups/model_hook.py
+17 -5
View File
@@ -93,7 +93,7 @@ def handle_model_specific_adjustments(server_args: Any):
_hybrid_spec = get_linear_attn_spec_by_arch(model_arch)
if _hybrid_spec is not None and _hybrid_spec.uses_mamba_radix_cache:
server_args._handle_mamba_radix_cache(model_arch=model_arch)
handle_mamba_radix_cache(server_args, model_arch)
# Collect the declarative model overrides (registry) on the
# pristine config and stash them for publish-time flags resolution;
@@ -561,7 +561,7 @@ def handle_model_specific_adjustments(server_args: Any):
# for them this re-invocation is an idempotent no-op plus validation.
# Kept ahead of the sparse-head pass: the legacy per-branch calls
# resolved before that tail write of disable_overlap_schedule.
server_args._handle_mamba_radix_cache(model_arch=model_arch)
handle_mamba_radix_cache(server_args, model_arch)
from sglang.srt.arg_groups.overrides import (
_sparse_head_overlap_disable,
@@ -585,6 +585,10 @@ def handle_model_specific_adjustments(server_args: Any):
def handle_model_capability_adjustments(server_args: Any):
from sglang.srt.arg_groups.kv_cache_hook import (
validate_prefill_only_disable_kv_cache_args,
)
cfg = resolving_view(server_args)
if parse_connector_type(cfg.model_path) == ConnectorType.INSTANCE:
return
@@ -725,7 +729,7 @@ def handle_model_capability_adjustments(server_args: Any):
"_handle_model_capability_adjustments",
prefill_only_disable_kv_cache=True,
)
server_args._validate_prefill_only_disable_kv_cache_args()
validate_prefill_only_disable_kv_cache_args(server_args)
declare_resolution(
server_args,
"_handle_model_capability_adjustments",
@@ -808,6 +812,10 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
# _mamba_radix_cache_resolution), invoked here at each legacy call
# slot; this handler keeps the validation.
from sglang.srt.arg_groups.mamba_hook import (
validate_mamba_extra_buffer,
validate_mamba_no_buffer,
)
from sglang.srt.arg_groups.overrides import (
_mamba_radix_cache_resolution,
mamba_extra_buffer_of,
@@ -820,9 +828,13 @@ def handle_mamba_radix_cache(server_args: Any, model_arch: str):
return
if mamba_extra_buffer_of(view):
server_args._validate_mamba_extra_buffer(view, model_arch)
validate_mamba_extra_buffer(
view,
model_arch,
mamba_cache_chunk_size_of=lambda: server_args.mamba_cache_chunk_size,
)
else:
server_args._validate_mamba_no_buffer(view, model_arch)
validate_mamba_no_buffer(view, model_arch)
def handle_language_model_only(server_args: Any):
python/sglang/srt/arg_groups/model_path_hook.py
+5 -3
View File
@@ -22,7 +22,7 @@ logger = logging.getLogger(__name__)
def handle_model_source_paths(server_args: Any):
"""Prepare metadata for model paths backed by remote object stores."""
cfg = resolving_view(server_args)
server_args._resolve_hf_gguf_model_path()
resolve_hf_gguf_model_path(server_args)
seen_paths = set()
for model_path in (
@@ -241,7 +241,7 @@ def handle_load_format(server_args: Any):
)
elif (
cfg.remote_instance_weight_loader_backend == "transfer_engine"
and not server_args.validate_transfer_engine()
and not validate_transfer_engine(server_args)
):
logger.warning(
"Fallback load_format to 'auto' due to 'transfer_engine' backend is not supported."
@@ -257,7 +257,9 @@ def handle_load_format(server_args: Any):
declare_resolution(
server_args,
"_handle_load_format",
remote_instance_weight_loader_start_seed_via_transfer_engine=server_args.validate_transfer_engine(),
remote_instance_weight_loader_start_seed_via_transfer_engine=validate_transfer_engine(
server_args
),
)
# "ipc_cache" is an internal-only load format: ModelRunner sets it
python/sglang/srt/arg_groups/moe_hook.py
+1 -1
View File
@@ -178,7 +178,7 @@ def handle_a2a_moe(server_args: Any):
)
if a2a_backend == "deepep_v2":
server_args._validate_deepep_v2_model_architecture()
validate_deepep_v2_model_architecture(server_args)
if resolved_view(server_args).enable_deterministic_inference:
raise ValueError(
"DeepEP v2 does not forward deterministic=True to "
python/sglang/srt/arg_groups/parallel_hook.py
+4 -2
View File
@@ -326,6 +326,8 @@ def handle_dwdp(server_args: Any):
def handle_elastic_ep(server_args: Any):
from sglang.srt.arg_groups.validation_hook import validate_ib_devices
cfg = resolving_view(server_args)
if cfg.elastic_ep_rejoin:
if cfg.ep_join_mode is None:
@@ -361,8 +363,8 @@ def handle_elastic_ep(server_args: Any):
declare_resolution(
server_args,
"_handle_elastic_ep",
mooncake_ib_device=server_args._validate_ib_devices(
cfg.mooncake_ib_device
mooncake_ib_device=validate_ib_devices(
server_args, cfg.mooncake_ib_device
),
)
if cfg.ep_join_mode is not None:
python/sglang/srt/arg_groups/pd_disaggregation_hook.py
+5 -3
View File
@@ -185,10 +185,12 @@ def _alias_bootstrap_port_to_api_port(server_args: ServerArgs) -> None:
def handle_encoder_disaggregation(server_args: Any):
from sglang.srt.arg_groups.model_hook import handle_language_model_only
from sglang.srt.arg_groups.validation_hook import validate_ib_devices
from sglang.srt.server_args import resolve_encoder_transfer_backend
cfg = resolving_view(server_args)
server_args._handle_language_model_only()
handle_language_model_only(server_args)
if cfg.enable_prefix_mm_cache and not cfg.encoder_only:
raise ValueError(
"--enable-prefix-mm-cache requires --encoder-only to be enabled"
@@ -215,8 +217,8 @@ def handle_encoder_disaggregation(server_args: Any):
declare_resolution(
server_args,
"_handle_encoder_disaggregation",
disaggregation_ib_device=server_args._validate_ib_devices(
cfg.disaggregation_ib_device
disaggregation_ib_device=validate_ib_devices(
server_args, cfg.disaggregation_ib_device
),
)
python/sglang/srt/arg_groups/serving_hook.py
+4 -2
View File
@@ -390,7 +390,7 @@ def handle_deprecated_args(server_args: Any):
def handle_environment_variables(server_args: Any):
cfg = resolving_view(server_args)
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
envs.SGLANG_ENABLE_TORCH_COMPILE.set("1" if cfg.enable_torch_compile else "0")
if cfg.mamba_ssm_dtype is not None:
envs.SGLANG_MAMBA_SSM_DTYPE.set(cfg.mamba_ssm_dtype)
@@ -571,6 +571,8 @@ def handle_other_validations(server_args: Any):
def handle_missing_default_values(server_args: Any):
from sglang.srt.arg_groups.model_path_hook import handle_modelscope_paths
cfg = resolving_view(server_args)
if cfg.tokenizer_path is None:
declare_resolution(
@@ -609,7 +611,7 @@ def handle_missing_default_values(server_args: Any):
# Handle ModelScope model downloads
if envs.SGLANG_USE_MODELSCOPE.get():
server_args._handle_modelscope_paths()
handle_modelscope_paths(server_args)
# In speculative scenario:
# - If `speculative_draft_model_quantization` is specified, the draft model uses this quantization method.
python/sglang/srt/arg_groups/validation_hook.py
+9 -7
View File
@@ -21,6 +21,8 @@ logger = logging.getLogger(__name__)
def check_server_args(server_args: Any):
from sglang.srt.arg_groups.lora_hook import check_lora_server_args
cfg = resolving_view(server_args)
# Check parallel size constraints
@@ -74,7 +76,7 @@ def check_server_args(server_args: Any):
)
# Check LoRA
server_args.check_lora_server_args()
check_lora_server_args(server_args)
# Check speculative decoding
if cfg.speculative_algorithm is not None:
@@ -119,11 +121,11 @@ def check_server_args(server_args: Any):
assert cfg.detokenizer_worker_num > 0, "Detokenizer worker num must >= 1"
assert cfg.mm_processor_worker_num >= 0, "Multimodal processor worker num must >= 0"
assert cfg.mm_io_worker_num >= 0, "Multimodal I/O worker num must >= 0"
server_args.validate_buckets_rule(
"--prompt-tokens-buckets", cfg.prompt_tokens_buckets
validate_buckets_rule(
server_args, "--prompt-tokens-buckets", cfg.prompt_tokens_buckets
)
server_args.validate_buckets_rule(
"--generation-tokens-buckets", cfg.generation_tokens_buckets
validate_buckets_rule(
server_args, "--generation-tokens-buckets", cfg.generation_tokens_buckets
)
# Check scheduling policy
@@ -185,7 +187,7 @@ def check_server_args(server_args: Any):
)
# Check two batch overlap backend requirement.
server_args._check_two_batch_overlap()
check_two_batch_overlap(server_args)
# Check communications compression
if cfg.enable_quant_communications and cfg.tp_size == 1:
@@ -216,7 +218,7 @@ def check_server_args(server_args: Any):
"--kv-canary-sweep-interval requires --kv-canary in {log, raise}"
)
server_args.check_load_publish_args()
check_load_publish_args(server_args)
def validate_buckets_rule(server_args: Any, arg_name: str, buckets_rule: List[str]):
python/sglang/srt/server_args.py
+179 -583
View File
@@ -3802,10 +3802,38 @@ class ServerArgs:
from sglang.srt.arg_groups.mega_moe_hook import handle_mega_moe
handle_mega_moe(self)
self._handle_return_hidden_states_mode()
self._handle_media_url_security()
self._handle_hicache_ratio_default()
self._validate_prefill_decode_interval()
from sglang.srt.arg_groups.serving_hook import (
handle_asr_validation,
handle_crash_dump_env,
handle_debug_utils,
handle_deprecated_args,
handle_environment_variables,
handle_grammar_backend,
handle_load_balance_method,
handle_media_url_security,
handle_missing_default_values,
handle_multimodal,
handle_other_validations,
handle_prefill_delayer_env_compat,
handle_return_hidden_states_mode,
handle_ssl_validation,
handle_tokenizer_batching,
)
handle_return_hidden_states_mode(self)
handle_media_url_security(self)
from sglang.srt.arg_groups.hicache_hook import (
handle_hicache,
handle_hicache_ratio_default,
)
handle_hicache_ratio_default(self)
from sglang.srt.arg_groups.validation_hook import (
validate_experimental_sgl_marlin,
validate_prefill_decode_interval,
)
validate_prefill_decode_interval(self)
# Reject an explicitly enabled but incompatible hardware runtime before
# model path resolution, downloads, or the dummy-model short circuit.
@@ -3813,55 +3841,105 @@ class ServerArgs:
if cfg.model_path.lower() in ["none", "dummy"]:
return
self._handle_model_source_paths()
from sglang.srt.arg_groups.model_path_hook import (
handle_load_format,
handle_model_source_paths,
)
handle_model_source_paths(self)
# Validate mm_process_config.
self._handle_multimodal()
handle_multimodal(self)
# Validate SSL arguments early.
self._handle_ssl_validation()
handle_ssl_validation(self)
# Validate transcription/ASR-specific server args.
self._handle_asr_validation()
handle_asr_validation(self)
# Handle deprecated arguments.
self._handle_deprecated_args()
handle_deprecated_args(self)
# Handle deprecated environment variables for prefill delayer.
self._handle_prefill_delayer_env_compat()
handle_prefill_delayer_env_compat(self)
# Set missing default values.
self._handle_missing_default_values()
handle_missing_default_values(self)
# expert_pack may replace a raw GGUF input with its generated local
# model metadata before any model-specific handler calls get_model_config.
# It also establishes eager-only invariants before CUDA graph parsing.
self._handle_expert_pack()
from sglang.srt.arg_groups.expert_pack_hook import handle_expert_pack
handle_expert_pack(self)
# Validate PD disaggregation flags before CUDA graph config.
self._handle_pd_disaggregation()
from sglang.srt.arg_groups.pd_disaggregation_hook import (
handle_encoder_disaggregation,
handle_pd_disaggregation,
)
handle_pd_disaggregation(self)
# Normalize deprecated CP aliases before validations or model-specific
# defaults inspect enable_prefill_cp/cp_strategy.
self._handle_legacy_cp_arguments()
self._validate_prefill_only_disable_kv_cache_args()
self._handle_dcp_validation()
from sglang.srt.arg_groups.parallel_hook import (
handle_context_parallelism,
handle_data_parallelism,
handle_dcp_validation,
handle_dwdp,
handle_elastic_ep,
handle_eplb_and_dispatch,
handle_expert_distribution_metrics,
handle_legacy_cp_arguments,
)
handle_legacy_cp_arguments(self)
from sglang.srt.arg_groups.kv_cache_hook import (
handle_cache_compatibility,
handle_kv4_compatibility,
handle_mxfp8_kv_cache_compatibility,
handle_page_major_kv_layout,
handle_prefill_only_disable_kv_cache,
handle_unified_memory_pool,
validate_prefill_only_disable_kv_cache_args,
)
validate_prefill_only_disable_kv_cache_args(self)
handle_dcp_validation(self)
# Model-arch prefill CUDA-graph default must land before cuda-graph
# resolution (the declarative registry materializes too late to affect
# it). Inkling opts into full-graph prefill capture here.
self._apply_inkling_prefill_cuda_graph_default()
self._apply_muse_glimmer_prefill_cuda_graph_max_bs_default()
from sglang.srt.arg_groups.cuda_graph_hook import (
apply_inkling_prefill_cuda_graph_default,
apply_muse_glimmer_prefill_cuda_graph_max_bs_default,
disable_prefill_cuda_graph_for_deepseek_trtllm_mla,
handle_cuda_graph_config,
)
apply_inkling_prefill_cuda_graph_default(self)
apply_muse_glimmer_prefill_cuda_graph_max_bs_default(self)
# must run before _handle_cuda_graph_config and _handle_data_parallelism
self._handle_dwdp()
handle_dwdp(self)
self._handle_cuda_graph_config()
handle_cuda_graph_config(self)
# Handle device-specific backends.
self._handle_hpu_backends()
self._handle_cpu_backends()
self._handle_npu_backends()
self._handle_mps_backends()
self._handle_xpu_backends()
from sglang.srt.arg_groups.platform_hook import (
handle_amd_specifics,
handle_cpu_backends,
handle_hpu_backends,
handle_mps_backends,
handle_nccl_pre_warm,
handle_npu_backends,
handle_xpu_backends,
)
handle_hpu_backends(self)
handle_cpu_backends(self)
handle_npu_backends(self)
handle_mps_backends(self)
handle_xpu_backends(self)
# OOT platform plugins set fields directly (an interface this tree
# does not own); the diff records what they applied.
@@ -3874,193 +3952,146 @@ class ServerArgs:
gpu_mem = get_device_memory_capacity(cfg.device)
# Handle memory-related, chunked prefill, and CUDA graph batch size configurations.
self._handle_gpu_memory_settings(gpu_mem)
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
handle_gpu_memory_settings(self, gpu_mem)
# Apply model-specific adjustments.
self._handle_model_specific_adjustments()
from sglang.srt.arg_groups.model_hook import (
handle_model_capability_adjustments,
handle_model_specific_adjustments,
)
handle_model_specific_adjustments(self)
# Set kernel backends.
self._handle_sampling_backend()
# Must run before _handle_attention_backend_compatibility so the
# deterministic backend is set before auto-detection fills it in.
self._handle_deterministic_inference()
self._handle_attention_backend_compatibility()
from sglang.srt.arg_groups.attention_hook import (
handle_attention_backend_compatibility,
handle_deterministic_inference,
handle_linear_attn_backend,
handle_multi_item_scoring,
)
handle_deterministic_inference(self)
handle_attention_backend_compatibility(self)
# Must run after the attention backend is resolved so the trtllm_mla
# default (auto-selected for DeepseekV3ForCausalLM on sm100) is visible.
self._disable_prefill_cuda_graph_for_deepseek_trtllm_mla()
self._handle_mamba_backend()
self._handle_int8_mamba_checkpoint()
self._handle_linear_attn_backend()
self._handle_kv4_compatibility()
self._handle_mxfp8_kv_cache_compatibility()
disable_prefill_cuda_graph_for_deepseek_trtllm_mla(self)
from sglang.srt.arg_groups.mamba_hook import (
handle_int8_mamba_checkpoint,
handle_mamba_backend,
)
handle_mamba_backend(self)
handle_int8_mamba_checkpoint(self)
handle_linear_attn_backend(self)
handle_kv4_compatibility(self)
handle_mxfp8_kv_cache_compatibility(self)
self._handle_page_size()
self._handle_amd_specifics()
self._handle_nccl_pre_warm()
self._handle_grammar_backend()
handle_amd_specifics(self)
handle_nccl_pre_warm(self)
handle_grammar_backend(self)
# Handle multi-item scoring constraints. Must run after the above so
# the final attention backend and chunked_prefill_size are in effect.
self._handle_multi_item_scoring()
handle_multi_item_scoring(self)
# Backend-dependent half of --prefill-only-disable-kv-cache validation.
# Must stay after _handle_attention_backend_compatibility() (above) and
# _handle_multi_item_scoring() so the resolved prefill backend is final;
# the flag/precondition half runs earlier in
# _validate_prefill_only_disable_kv_cache_args().
self._handle_prefill_only_disable_kv_cache()
handle_prefill_only_disable_kv_cache(self)
# Handle Hicache settings.
self._handle_hicache()
handle_hicache(self)
# Handle data parallelism.
self._handle_data_parallelism()
handle_data_parallelism(self)
# Normalize load balancing defaults.
self._handle_load_balance_method()
handle_load_balance_method(self)
# Re-apply after model-specific defaults resolve attention_backend so
# canonical CP mirrors to the right legacy runtime aliases.
self._handle_legacy_cp_arguments()
handle_legacy_cp_arguments(self)
# Handle context parallelism.
self._handle_context_parallelism()
handle_context_parallelism(self)
# Handle MoE configurations.
self._handle_moe_kernel_config()
self._handle_a2a_moe()
self._handle_eplb_and_dispatch()
self._handle_expert_distribution_metrics()
self._handle_elastic_ep()
self._validate_experimental_sgl_marlin()
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
handle_moe_kernel_config,
validate_cutedsl_a2a_token_budget,
validate_deepep_v2_dispatch_token_budget,
validate_deepep_v2_speculative_draft,
)
handle_moe_kernel_config(self)
handle_a2a_moe(self)
handle_eplb_and_dispatch(self)
handle_expert_distribution_metrics(self)
handle_elastic_ep(self)
validate_experimental_sgl_marlin(self)
# Handle pipeline parallelism.
self._handle_pipeline_parallelism()
# Handle speculative decoding logic.
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
handle_speculative_decoding(self)
# Validate the CuteDSL A2A token budget now that num_tokens_per_req is final.
self._validate_cutedsl_a2a_token_budget()
validate_cutedsl_a2a_token_budget(self)
# Handle model loading format.
self._handle_load_format()
handle_load_format(self)
# Handle Encoder disaggregation.
self._handle_encoder_disaggregation()
handle_encoder_disaggregation(self)
# Validate tokenizer settings.
self._handle_tokenizer_batching()
handle_tokenizer_batching(self)
# Propagate environment variables.
self._handle_environment_variables()
handle_environment_variables(self)
# Validate cache settings.
self._handle_cache_compatibility()
handle_cache_compatibility(self)
self._handle_page_major_kv_layout()
handle_page_major_kv_layout(self)
self._handle_unified_memory_pool()
handle_unified_memory_pool(self)
# Handle diffusion LLM inference.
self._handle_dllm_inference()
from sglang.srt.arg_groups.dllm_hook import handle_dllm_inference
handle_dllm_inference(self)
# Handle crash dump environment variables (must run before CUDA init).
self._handle_crash_dump_env()
handle_crash_dump_env(self)
# Handle debug utilities.
self._handle_debug_utils()
handle_debug_utils(self)
# Handle any other necessary validations.
self._handle_other_validations()
handle_other_validations(self)
# Model-capability adjustments that legacy code applied at model-load
# time; last declarations of the resolution, mirroring that order.
self._handle_model_capability_adjustments()
# Validate after all batch-size declarations are visible.
self._validate_deepep_v2_speculative_draft()
self._validate_deepep_v2_dispatch_token_budget()
self._resolution_finished = True
def _handle_return_hidden_states_mode(self):
from sglang.srt.arg_groups.serving_hook import handle_return_hidden_states_mode
handle_return_hidden_states_mode(self)
def _handle_model_capability_adjustments(self):
from sglang.srt.arg_groups.model_hook import handle_model_capability_adjustments
handle_model_capability_adjustments(self)
def _handle_model_source_paths(self):
from sglang.srt.arg_groups.model_path_hook import handle_model_source_paths
# Validate after all batch-size declarations are visible.
validate_deepep_v2_speculative_draft(self)
validate_deepep_v2_dispatch_token_budget(self)
handle_model_source_paths(self)
def _handle_pd_disaggregation(self):
from sglang.srt.arg_groups.pd_disaggregation_hook import (
handle_pd_disaggregation,
)
handle_pd_disaggregation(self)
def _handle_dcp_validation(self):
from sglang.srt.arg_groups.parallel_hook import handle_dcp_validation
handle_dcp_validation(self)
def _handle_load_balance_method(self):
from sglang.srt.arg_groups.serving_hook import handle_load_balance_method
handle_load_balance_method(self)
def _handle_ssl_validation(self):
from sglang.srt.arg_groups.serving_hook import handle_ssl_validation
handle_ssl_validation(self)
def _handle_multimodal(self):
from sglang.srt.arg_groups.serving_hook import handle_multimodal
handle_multimodal(self)
def _handle_media_url_security(self):
from sglang.srt.arg_groups.serving_hook import handle_media_url_security
handle_media_url_security(self)
def _handle_deprecated_args(self):
from sglang.srt.arg_groups.serving_hook import handle_deprecated_args
handle_deprecated_args(self)
def _handle_prefill_delayer_env_compat(self):
from sglang.srt.arg_groups.serving_hook import handle_prefill_delayer_env_compat
handle_prefill_delayer_env_compat(self)
def _handle_missing_default_values(self):
from sglang.srt.arg_groups.serving_hook import handle_missing_default_values
handle_missing_default_values(self)
def _handle_modelscope_paths(self):
from sglang.srt.arg_groups.model_path_hook import handle_modelscope_paths
handle_modelscope_paths(self)
def _handle_hpu_backends(self):
from sglang.srt.arg_groups.platform_hook import handle_hpu_backends
handle_hpu_backends(self)
def _handle_cpu_backends(self):
from sglang.srt.arg_groups.platform_hook import handle_cpu_backends
handle_cpu_backends(self)
self._resolution_finished = True
def _handle_hardware_runtime_validation(self):
# This is intentionally independent of self.device: setting
@@ -4069,57 +4100,9 @@ class ServerArgs:
# use_mlx() remains lazy and does not import MLX.
use_mlx()
def _handle_npu_backends(self):
from sglang.srt.arg_groups.platform_hook import handle_npu_backends
handle_npu_backends(self)
def _handle_mps_backends(self):
from sglang.srt.arg_groups.platform_hook import handle_mps_backends
handle_mps_backends(self)
def _handle_xpu_backends(self):
from sglang.srt.arg_groups.platform_hook import handle_xpu_backends
handle_xpu_backends(self)
# ------------------------------------------------------------------
# CUDA graph configuration resolution
# ------------------------------------------------------------------
def _apply_inkling_prefill_cuda_graph_default(self):
from sglang.srt.arg_groups.cuda_graph_hook import (
apply_inkling_prefill_cuda_graph_default,
)
apply_inkling_prefill_cuda_graph_default(self)
def _apply_muse_glimmer_prefill_cuda_graph_max_bs_default(self):
from sglang.srt.arg_groups.cuda_graph_hook import (
apply_muse_glimmer_prefill_cuda_graph_max_bs_default,
)
apply_muse_glimmer_prefill_cuda_graph_max_bs_default(self)
def _handle_cuda_graph_config(self):
from sglang.srt.arg_groups.cuda_graph_hook import handle_cuda_graph_config
handle_cuda_graph_config(self)
def _apply_deepep_adjustments(self):
from sglang.srt.arg_groups.cuda_graph_hook import apply_deepep_adjustments
apply_deepep_adjustments(self)
def _parse_cuda_graph_config(self):
from sglang.srt.arg_groups.cuda_graph_hook import parse_cuda_graph_config
parse_cuda_graph_config(self)
def _apply_cuda_graph_compatibility(self):
from sglang.srt.arg_groups.cuda_graph_hook import apply_cuda_graph_compatibility
apply_cuda_graph_compatibility(self)
def _apply_cuda_graph_disaggregation_roles(self):
cfg = resolving_view(self)
@@ -4140,49 +4123,6 @@ class ServerArgs:
),
)
def _disable_tc_piecewise_cudagraph_if_incompatible(self):
from sglang.srt.arg_groups.cuda_graph_hook import (
disable_tc_piecewise_cudagraph_if_incompatible,
)
disable_tc_piecewise_cudagraph_if_incompatible(self)
def _disable_breakable_cudagraph_if_incompatible(self):
from sglang.srt.arg_groups.cuda_graph_hook import (
disable_breakable_cudagraph_if_incompatible,
)
disable_breakable_cudagraph_if_incompatible(self)
def _disable_full_prefill_cudagraph_if_incompatible(self):
from sglang.srt.arg_groups.cuda_graph_hook import (
disable_full_prefill_cudagraph_if_incompatible,
)
disable_full_prefill_cudagraph_if_incompatible(self)
def _disable_prefill_cuda_graph_for_deepseek_trtllm_mla(self):
from sglang.srt.arg_groups.cuda_graph_hook import (
disable_prefill_cuda_graph_for_deepseek_trtllm_mla,
)
disable_prefill_cuda_graph_for_deepseek_trtllm_mla(self)
def _validate_cuda_graph_config(self):
from sglang.srt.arg_groups.cuda_graph_hook import validate_cuda_graph_config
validate_cuda_graph_config(self)
def _handle_multi_item_scoring(self):
from sglang.srt.arg_groups.attention_hook import handle_multi_item_scoring
handle_multi_item_scoring(self)
def _handle_gpu_memory_settings(self, gpu_mem):
from sglang.srt.arg_groups.memory_hook import handle_gpu_memory_settings
handle_gpu_memory_settings(self, gpu_mem)
def post_capture_kv_sizing_planned(self) -> bool:
"""Whether the mem_fraction heuristic may skip the graph reserve; must be
False for any config the runtime won't post-capture-size, else it gets an
@@ -4408,48 +4348,11 @@ class ServerArgs:
run_post_process_pass(self, _dsa_split_backend_resolution)
def _validate_hisparse_dsa_backend(self, attr: str, label: str):
from sglang.srt.arg_groups.hisparse_hook import validate_hisparse_dsa_backend
validate_hisparse_dsa_backend(self, attr, label)
def _validate_hisparse_kv_cache_dtype(self):
from sglang.srt.arg_groups.hisparse_hook import validate_hisparse_kv_cache_dtype
validate_hisparse_kv_cache_dtype(self)
def _handle_model_specific_adjustments(self):
from sglang.srt.arg_groups.model_hook import handle_model_specific_adjustments
handle_model_specific_adjustments(self)
def _support_mamba_cache_extra_buffer(self, model_arch: str):
from sglang.srt.arg_groups.overrides import supports_mamba_cache_extra_buffer
return supports_mamba_cache_extra_buffer(self, model_arch)
def _validate_mamba_no_buffer(self, view, model_arch: str):
from sglang.srt.arg_groups.mamba_hook import validate_mamba_no_buffer
validate_mamba_no_buffer(view, model_arch)
def _validate_mamba_extra_buffer(self, view, model_arch: str):
from sglang.srt.arg_groups.mamba_hook import validate_mamba_extra_buffer
validate_mamba_extra_buffer(
view,
model_arch,
mamba_cache_chunk_size_of=lambda: self.mamba_cache_chunk_size,
)
def _handle_mamba_radix_cache(self, model_arch: str):
# Resolution moved to the resolution pipeline (arg_groups/overrides.py:
# _mamba_radix_cache_resolution), invoked here at each legacy call
# slot; this handler keeps the validation.
from sglang.srt.arg_groups.model_hook import handle_mamba_radix_cache
handle_mamba_radix_cache(self, model_arch)
def _handle_sampling_backend(self):
# Moved to the resolution pipeline (arg_groups/overrides.py:
# _sampling_backend_default), invoked here at its legacy slot.
@@ -4535,25 +4438,6 @@ class ServerArgs:
else:
return "triton"
def _handle_attention_backend_compatibility(self):
from sglang.srt.arg_groups.attention_hook import (
handle_attention_backend_compatibility,
)
handle_attention_backend_compatibility(self)
def _handle_mxfp8_kv_cache_compatibility(self):
from sglang.srt.arg_groups.kv_cache_hook import (
handle_mxfp8_kv_cache_compatibility,
)
handle_mxfp8_kv_cache_compatibility(self)
def _handle_kv4_compatibility(self):
from sglang.srt.arg_groups.kv_cache_hook import handle_kv4_compatibility
handle_kv4_compatibility(self)
def _handle_page_size(self):
# Moved to the resolution pipeline (arg_groups/overrides.py:
# _page_size_default), invoked here at its legacy slot.
@@ -4564,61 +4448,6 @@ class ServerArgs:
run_post_process_pass(self, _page_size_default)
def _handle_amd_specifics(self):
from sglang.srt.arg_groups.platform_hook import handle_amd_specifics
handle_amd_specifics(self)
def _handle_nccl_pre_warm(self):
from sglang.srt.arg_groups.platform_hook import handle_nccl_pre_warm
handle_nccl_pre_warm(self)
def _handle_grammar_backend(self):
from sglang.srt.arg_groups.serving_hook import handle_grammar_backend
handle_grammar_backend(self)
def _handle_mamba_backend(self):
from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend
handle_mamba_backend(self)
def _handle_int8_mamba_checkpoint(self):
from sglang.srt.arg_groups.mamba_hook import handle_int8_mamba_checkpoint
handle_int8_mamba_checkpoint(self)
def _handle_linear_attn_backend(self):
from sglang.srt.arg_groups.attention_hook import handle_linear_attn_backend
handle_linear_attn_backend(self)
def _handle_legacy_cp_arguments(self):
from sglang.srt.arg_groups.parallel_hook import handle_legacy_cp_arguments
handle_legacy_cp_arguments(self)
def _handle_context_parallelism(self):
from sglang.srt.arg_groups.parallel_hook import handle_context_parallelism
handle_context_parallelism(self)
def _handle_dwdp(self):
from sglang.srt.arg_groups.parallel_hook import handle_dwdp
handle_dwdp(self)
def _handle_data_parallelism(self):
from sglang.srt.arg_groups.parallel_hook import handle_data_parallelism
handle_data_parallelism(self)
def _handle_moe_kernel_config(self):
from sglang.srt.arg_groups.moe_hook import handle_moe_kernel_config
handle_moe_kernel_config(self)
def cutedsl_moe_max_num_tokens(self) -> int:
"""Largest number of tokens a single forward routes through a CuteDSL
MoE layer on one (DP) rank. Single source of truth for both the
@@ -4654,33 +4483,6 @@ class ServerArgs:
tokens = max(tokens, cfg.max_prefill_tokens or 0, math.ceil(chunked * 1.25))
return tokens
def _validate_cutedsl_a2a_token_budget(self):
from sglang.srt.arg_groups.moe_hook import validate_cutedsl_a2a_token_budget
validate_cutedsl_a2a_token_budget(self)
def _validate_deepep_v2_dispatch_token_budget(self) -> None:
from sglang.srt.arg_groups.moe_hook import (
validate_deepep_v2_dispatch_token_budget,
)
validate_deepep_v2_dispatch_token_budget(self)
def _validate_deepep_v2_model_architecture(self) -> None:
from sglang.srt.arg_groups.moe_hook import validate_deepep_v2_model_architecture
validate_deepep_v2_model_architecture(self)
def _validate_deepep_v2_speculative_draft(self) -> None:
from sglang.srt.arg_groups.moe_hook import validate_deepep_v2_speculative_draft
validate_deepep_v2_speculative_draft(self)
def _handle_a2a_moe(self):
from sglang.srt.arg_groups.moe_hook import handle_a2a_moe
handle_a2a_moe(self)
def _required_mori_dispatch_tokens_per_rank(self) -> int:
"""Max tokens a single rank dispatches through MoRI in one forward."""
cfg = resolving_view(self)
@@ -4694,31 +4496,8 @@ class ServerArgs:
required = max(required, cfg.cuda_graph_max_bs_decode)
return required
def _handle_eplb_and_dispatch(self):
from sglang.srt.arg_groups.parallel_hook import handle_eplb_and_dispatch
handle_eplb_and_dispatch(self)
def _handle_elastic_ep(self):
from sglang.srt.arg_groups.parallel_hook import handle_elastic_ep
handle_elastic_ep(self)
def _validate_experimental_sgl_marlin(self):
from sglang.srt.arg_groups.validation_hook import (
validate_experimental_sgl_marlin,
)
validate_experimental_sgl_marlin(self)
# ===== END TO BE REFACTORED ====
def _handle_expert_distribution_metrics(self):
from sglang.srt.arg_groups.parallel_hook import (
handle_expert_distribution_metrics,
)
handle_expert_distribution_metrics(self)
def _handle_pipeline_parallelism(self):
# Moved to the resolution pipeline (arg_groups/overrides.py:
# _pipeline_parallel_overlap_disable), invoked here at its legacy slot.
@@ -4729,70 +4508,6 @@ class ServerArgs:
run_post_process_pass(self, _pipeline_parallel_overlap_disable)
def _validate_prefill_only_disable_kv_cache_args(self):
from sglang.srt.arg_groups.kv_cache_hook import (
validate_prefill_only_disable_kv_cache_args,
)
validate_prefill_only_disable_kv_cache_args(self)
def _handle_prefill_only_disable_kv_cache(self):
from sglang.srt.arg_groups.kv_cache_hook import (
handle_prefill_only_disable_kv_cache,
)
handle_prefill_only_disable_kv_cache(self)
def _handle_hicache_ratio_default(self):
from sglang.srt.arg_groups.hicache_hook import handle_hicache_ratio_default
handle_hicache_ratio_default(self)
def _handle_hicache(self):
from sglang.srt.arg_groups.hicache_hook import handle_hicache
handle_hicache(self)
def _validate_hicache_host_memory_mode(self):
from sglang.srt.arg_groups.hicache_hook import validate_hicache_host_memory_mode
validate_hicache_host_memory_mode(self)
def _resolve_hicache_dcp_compatibility(self):
from sglang.srt.arg_groups.hicache_hook import resolve_hicache_dcp_compatibility
resolve_hicache_dcp_compatibility(self)
def _resolve_layout_io_compatibility(self):
from sglang.srt.arg_groups.hicache_hook import resolve_layout_io_compatibility
resolve_layout_io_compatibility(self)
def _resolve_storage_layout_compatibility(self):
from sglang.srt.arg_groups.hicache_hook import (
resolve_storage_layout_compatibility,
)
resolve_storage_layout_compatibility(self)
def _resolve_hf_gguf_model_path(self):
from sglang.srt.arg_groups.model_path_hook import resolve_hf_gguf_model_path
resolve_hf_gguf_model_path(self)
def _handle_expert_pack(self):
from sglang.srt.arg_groups.expert_pack_hook import handle_expert_pack
handle_expert_pack(self)
def _handle_load_format(self):
# The quantization side of the gguf coupling moved to the pipeline
# (arg_groups/overrides.py: _gguf_quantization); load_format itself is
# genuine config (runtime user updates write it) and stays imperative.
from sglang.srt.arg_groups.model_path_hook import handle_load_format
handle_load_format(self)
def _is_mistral_native_format(self) -> bool:
"""True iff the checkpoint requires load_format=mistral.
@@ -4855,95 +4570,10 @@ class ServerArgs:
LANGUAGE_MODEL_ONLY_ARCHITECTURES = ("MuseGlimmerForConditionalGeneration",)
def _handle_language_model_only(self):
from sglang.srt.arg_groups.model_hook import handle_language_model_only
handle_language_model_only(self)
def _handle_encoder_disaggregation(self):
from sglang.srt.arg_groups.pd_disaggregation_hook import (
handle_encoder_disaggregation,
)
handle_encoder_disaggregation(self)
def _validate_ib_devices(self, device_str: Optional[str]) -> Optional[str]:
from sglang.srt.arg_groups.validation_hook import validate_ib_devices
return validate_ib_devices(self, device_str)
def _handle_tokenizer_batching(self):
from sglang.srt.arg_groups.serving_hook import handle_tokenizer_batching
handle_tokenizer_batching(self)
def _handle_multimodal_feature_transport(self):
from sglang.srt.arg_groups.serving_hook import (
handle_multimodal_feature_transport,
)
handle_multimodal_feature_transport(self)
def _handle_environment_variables(self):
from sglang.srt.arg_groups.serving_hook import handle_environment_variables
handle_environment_variables(self)
def _handle_cache_compatibility(self):
from sglang.srt.arg_groups.kv_cache_hook import handle_cache_compatibility
handle_cache_compatibility(self)
def _handle_deterministic_inference(self):
from sglang.srt.arg_groups.attention_hook import handle_deterministic_inference
handle_deterministic_inference(self)
def _handle_unified_memory_pool(self):
from sglang.srt.arg_groups.kv_cache_hook import handle_unified_memory_pool
handle_unified_memory_pool(self)
# The strided-layout Triton requirement is enforced via
# --enable-page-major-kv-layout (implied by the unified pool in
# _handle_page_major_kv_layout); the model-family gate is enforced at pool
# construction in model_runner_kv_cache_mixin._init_pools.
def _handle_page_major_kv_layout(self):
from sglang.srt.arg_groups.kv_cache_hook import handle_page_major_kv_layout
handle_page_major_kv_layout(self)
def _handle_dllm_inference(self):
from sglang.srt.arg_groups.dllm_hook import handle_dllm_inference
handle_dllm_inference(self)
def _handle_asr_validation(self):
from sglang.srt.arg_groups.serving_hook import handle_asr_validation
handle_asr_validation(self)
def _validate_prefill_decode_interval(self):
from sglang.srt.arg_groups.validation_hook import (
validate_prefill_decode_interval,
)
validate_prefill_decode_interval(self)
def _handle_other_validations(self):
from sglang.srt.arg_groups.serving_hook import handle_other_validations
handle_other_validations(self)
def _handle_crash_dump_env(self):
from sglang.srt.arg_groups.serving_hook import handle_crash_dump_env
handle_crash_dump_env(self)
def _handle_debug_utils(self):
from sglang.srt.arg_groups.serving_hook import handle_debug_utils
handle_debug_utils(self)
# The strided-layout Triton requirement is enforced via
# --enable-page-major-kv-layout (implied by the unified pool in
# _handle_page_major_kv_layout); the model-family gate is enforced at pool
# construction in model_runner_kv_cache_mixin._init_pools.
@staticmethod
def add_cli_args(parser: argparse.ArgumentParser):
@@ -5480,40 +5110,11 @@ class ServerArgs:
self._mamba_cache_chunk_size = max(chunk_size, page_size)
return self._mamba_cache_chunk_size
def _check_two_batch_overlap(self):
# With no EP a2a backend, two-batch-overlap is only valid on the non-EP
# DP TP-MoE path (overlapping the DP all_gatherv / reduce_scatterv with
# the other ubatch's compute), which requires DP attention. Enabling it
# there needs no extra opt-in env flag.
from sglang.srt.arg_groups.validation_hook import check_two_batch_overlap
check_two_batch_overlap(self)
def check_server_args(self):
from sglang.srt.arg_groups.validation_hook import check_server_args
check_server_args(self)
def check_load_publish_args(self):
from sglang.srt.arg_groups.validation_hook import check_load_publish_args
check_load_publish_args(self)
def check_lora_server_args(self):
from sglang.srt.arg_groups.lora_hook import check_lora_server_args
check_lora_server_args(self)
def _check_lora_speculative_compatibility(self):
from sglang.srt.arg_groups.lora_hook import check_lora_speculative_compatibility
check_lora_speculative_compatibility(self)
def validate_buckets_rule(self, arg_name: str, buckets_rule: List[str]):
from sglang.srt.arg_groups.validation_hook import validate_buckets_rule
validate_buckets_rule(self, arg_name, buckets_rule)
def adjust_mem_fraction_for_vlm(self, model_config):
cfg = resolving_view(self)
vision_config = getattr(model_config.hf_config, "vision_config", None)
@@ -5554,11 +5155,6 @@ class ServerArgs:
mem_fraction_static=original_server_arg_mem_fraction * final_overall_factor,
)
def validate_transfer_engine(self):
from sglang.srt.arg_groups.model_path_hook import validate_transfer_engine
return validate_transfer_engine(self)
@property
def _parsed_modelexpress_config(self) -> dict:
cache = getattr(self, "_mx_config_cache", None)
test/registered/cpu/test_server_args_backend.py
+5 -3
View File
@@ -5,6 +5,8 @@ import unittest
from unittest.mock import patch
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.platform_hook import handle_cpu_backends
from sglang.srt.arg_groups.validation_hook import validate_ib_devices
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
@@ -24,7 +26,7 @@ class TestServerArgsCPUBackend(unittest.TestCase):
def test_arm_cpu_defaults_to_torch_native(self, _mock_is_arm64):
server_args = self._make_server_args()
ServerArgs._handle_cpu_backends(server_args)
handle_cpu_backends(server_args)
self.assertEqual(
resolution_result(server_args, "attention_backend"), "torch_native"
@@ -35,7 +37,7 @@ class TestServerArgsCPUBackend(unittest.TestCase):
def test_x86_cpu_defaults_to_intel_amx(self, _mock_is_arm64):
server_args = self._make_server_args()
ServerArgs._handle_cpu_backends(server_args)
handle_cpu_backends(server_args)
self.assertEqual(
resolution_result(server_args, "attention_backend"), "intel_amx"
@@ -68,7 +70,7 @@ class TestServerArgsIBDeviceValidation(unittest.TestCase):
else real_listdir(path)
),
):
return ServerArgs._validate_ib_devices(server_args, device_str)
return validate_ib_devices(server_args, device_str)
def test_validate_ib_devices_accepts_comma_separated(self):
self.assertEqual(
test/registered/unit/configs/test_multimodal_piecewise_cuda_graph.py
+10 -7
View File
@@ -4,6 +4,8 @@ import unittest
from types import SimpleNamespace
from unittest.mock import patch
from sglang.srt.arg_groups.cuda_graph_hook import apply_cuda_graph_compatibility
from sglang.srt.arg_groups.model_hook import handle_model_capability_adjustments
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.configs.embedding_model_spec import resolve_embedding_model_spec
from sglang.srt.configs.model_config import (
@@ -82,14 +84,15 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
args._cuda_graph_config_locked = set()
with (
patch.object(
ServerArgs, "_disable_tc_piecewise_cudagraph_if_incompatible"
patch(
"sglang.srt.arg_groups.cuda_graph_hook"
".disable_tc_piecewise_cudagraph_if_incompatible"
) as disable_if_incompatible,
patch.object(
args, "_resolved_attention_backends", return_value=("fa3", "fa3")
),
):
args._apply_cuda_graph_compatibility()
apply_cuda_graph_compatibility(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
@@ -120,7 +123,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
),
patch.object(args, "use_mla_backend", return_value=True),
):
args._apply_cuda_graph_compatibility()
apply_cuda_graph_compatibility(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
@@ -139,7 +142,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
"_resolved_attention_backends",
return_value=("trtllm_mla", "trtllm_mla"),
):
args._apply_cuda_graph_compatibility()
apply_cuda_graph_compatibility(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
@@ -186,7 +189,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
patch.object(args, "get_model_config", return_value=args._model_config),
patch("sglang.srt.arg_groups.model_hook.is_cuda", return_value=True),
):
args._handle_model_capability_adjustments()
handle_model_capability_adjustments(args)
self.assertTrue(resolution_result(args, "disable_radix_cache"))
self.assertEqual(resolution_result(args, "chunked_prefill_size"), -1)
@@ -213,7 +216,7 @@ class TestMultimodalPiecewiseCudaGraph(CustomTestCase):
)
with patch.object(args, "get_model_config", return_value=args._model_config):
args._handle_model_capability_adjustments()
handle_model_capability_adjustments(args)
self.assertTrue(resolution_result(args, "is_embedding"))
test/registered/unit/entrypoints/test_server_info.py
+7 -6
View File
@@ -26,6 +26,7 @@ import json
import unittest
from types import SimpleNamespace
from sglang.srt.arg_groups.validation_hook import check_load_publish_args
from sglang.srt.entrypoints import http_server
from sglang.srt.lora.lora_registry import LoRARef
from sglang.srt.managers.tokenizer_manager import TokenizerManager
@@ -530,7 +531,7 @@ class TestLoadPublishEndpointValidation(CustomTestCase):
def test_requires_kv_events_config(self):
args = ServerArgs(model_path="dummy", load_publish_endpoint="tcp://*:6000")
with self.assertRaisesRegex(ValueError, "kv-events"):
args.check_load_publish_args()
check_load_publish_args(args)
def test_rejects_non_bindable_endpoint(self):
args = ServerArgs(
@@ -539,7 +540,7 @@ class TestLoadPublishEndpointValidation(CustomTestCase):
load_publish_endpoint="tcp://10.0.0.5:6000",
)
with self.assertRaisesRegex(ValueError, "bindable"):
args.check_load_publish_args()
check_load_publish_args(args)
def test_rejects_endpoint_overlapping_the_kv_range(self):
args = ServerArgs(
@@ -549,7 +550,7 @@ class TestLoadPublishEndpointValidation(CustomTestCase):
load_publish_endpoint="tcp://*:5558",
)
with self.assertRaisesRegex(ValueError, "overlaps"):
args.check_load_publish_args()
check_load_publish_args(args)
def test_rejects_null_publisher(self):
# publisher='null' disables KV events, so there is nothing to advertise
@@ -560,7 +561,7 @@ class TestLoadPublishEndpointValidation(CustomTestCase):
load_publish_endpoint="auto",
)
with self.assertRaisesRegex(ValueError, "null"):
args.check_load_publish_args()
check_load_publish_args(args)
def test_rejects_unparseable_kv_events_config(self):
args = ServerArgs(
@@ -569,7 +570,7 @@ class TestLoadPublishEndpointValidation(CustomTestCase):
load_publish_endpoint="auto",
)
with self.assertRaisesRegex(ValueError, "not parseable"):
args.check_load_publish_args()
check_load_publish_args(args)
def test_off_and_valid_endpoint_pass(self):
for endpoint in (None, "off", "OFF", "auto", "tcp://*:6000"):
@@ -581,7 +582,7 @@ class TestLoadPublishEndpointValidation(CustomTestCase):
),
load_publish_endpoint=endpoint,
)
args.check_load_publish_args() # must not raise
check_load_publish_args(args) # must not raise
if __name__ == "__main__":
test/registered/unit/layers/attention/test_kda_helion_dispatcher.py
+4 -3
View File
@@ -3,6 +3,7 @@ from unittest.mock import ANY, MagicMock, patch
import torch
from sglang.srt.arg_groups.attention_hook import handle_linear_attn_backend
from sglang.srt.layers.attention.linear.kda_backend import KDAKernelDispatcher
from sglang.srt.layers.attention.linear.kernels.kda_helion import HelionKDAKernel
from sglang.srt.layers.attention.linear.kernels.kda_triton import TritonKDAKernel
@@ -170,7 +171,7 @@ class TestHelionKDADispatcher(unittest.TestCase):
linear_attn_decode_backend="helion",
enable_linear_replayssm=True,
)
helion_args._handle_linear_attn_backend()
handle_linear_attn_backend(helion_args)
flashinfer_args = ServerArgs(
model_path="dummy",
@@ -178,7 +179,7 @@ class TestHelionKDADispatcher(unittest.TestCase):
enable_linear_replayssm=True,
)
with self.assertRaisesRegex(ValueError, "Triton, or Helion"):
flashinfer_args._handle_linear_attn_backend()
handle_linear_attn_backend(flashinfer_args)
def test_explicit_base_backend_is_not_replaced_by_flashinfer(self):
args = ServerArgs(
@@ -193,7 +194,7 @@ class TestHelionKDADispatcher(unittest.TestCase):
),
patch("sglang.srt.arg_groups.attention_hook.is_cuda", return_value=False),
):
args._handle_linear_attn_backend()
handle_linear_attn_backend(args)
self.assertIsNone(args.linear_attn_decode_backend)
self.assertEqual(args.linear_attn_backend, "helion")
test/registered/unit/managers/test_mm_process_config.py
+2 -1
View File
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
import torch
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.serving_hook import handle_multimodal
from sglang.srt.environ import envs
from sglang.srt.runtime_context import get_context
from sglang.srt.server_args import ServerArgs
@@ -21,7 +22,7 @@ class TestMmProcessConfigValidation(CustomTestCase):
def _validate_config(self, mm_process_config):
args = ServerArgs(model_path="dummy", mm_process_config=mm_process_config)
args._handle_multimodal()
handle_multimodal(args)
return args
def test_valid_config_accepted(self):
test/registered/unit/mem_cache/test_mamba_path_state_cap.py
+2 -1
View File
@@ -1,5 +1,6 @@
"""CPU-only unit tests for the per-path Mamba checkpoint cap."""
from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend
from sglang.test.ci.ci_register import register_cpu_ci, register_cuda_ci
register_cpu_ci(est_time=2, suite="base-a-test-cpu")
@@ -108,7 +109,7 @@ class TestMambaPathStateCap(unittest.TestCase):
ValueError,
"must be -1 \\(unlimited\\) or a positive integer",
):
args._handle_mamba_backend()
handle_mamba_backend(args)
def test_unified_cache_removes_only_shallow_mamba_state(self):
component, nodes, core, cache = _build_unified_chain(cap=2)
test/registered/unit/server_args/test_model_config_reads_resolved_input.py
+27 -30
View File
@@ -337,38 +337,33 @@ def _pipeline():
methods = {
node.name: node for node in record.body if isinstance(node, ast.FunctionDef)
}
# The dispatcher calls its hooks by bare name, so the walk resolves those
# against `arg_groups/` alongside the record's own methods.
hooks = _hook_functions()
# Follow exactly one edge: the slot's own `from arg_groups.X import f` /
# `f(self)`. Merging every hook function by bare name would let the walk
# wander into families the slot never calls.
slot_target = {}
for name, node in methods.items():
imported = {
alias.asname or alias.name
for child in ast.walk(node)
if isinstance(child, ast.ImportFrom)
and child.module
and child.module.startswith("sglang.srt.arg_groups")
for alias in child.names
}
called = {
child.func.id
for child in ast.walk(node)
if isinstance(child, ast.Call) and isinstance(child.func, ast.Name)
}
for target in sorted(imported & called & set(hooks)):
slot_target.setdefault(name, target)
methods.update({name: hooks[name] for name in slot_target.values()})
methods.update({name: node for name, node in hooks.items() if name not in methods})
dispatch = methods["_run_resolution_pipeline"]
# A step is either a record method (`self._x()`) or a bare-name hook call.
steps = [
name
for _line, name in sorted(
(node.lineno, node.func.attr)
(
node.lineno,
(
node.func.attr
if isinstance(node.func, ast.Attribute)
else node.func.id
),
)
for node in ast.walk(dispatch)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == "self"
and (
(
isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == "self"
)
or (isinstance(node.func, ast.Name) and node.func.id in hooks)
)
)
]
@@ -387,20 +382,22 @@ def _pipeline():
and node.func.attr in methods
):
reaches(node.func.attr, seen)
target = slot_target.get(name)
if target is not None:
reaches(target, seen)
elif isinstance(node.func, ast.Name) and node.func.id in hooks:
reaches(node.func.id, seen)
return seen
step_lines = {}
for node in ast.walk(dispatch):
if not isinstance(node, ast.Call):
continue
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == "self"
):
step_lines.setdefault(node.func.attr, node.lineno)
elif isinstance(node.func, ast.Name) and node.func.id in hooks:
step_lines.setdefault(node.func.id, node.lineno)
return steps, methods, {name: reaches(name) for name in steps}, step_lines
test/registered/unit/server_args/test_page_major_backend_allowlist.py
+2 -1
View File
@@ -31,6 +31,7 @@ under its own default configuration.
import unittest
from sglang.srt.arg_groups.kv_cache_hook import handle_page_major_kv_layout
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
@@ -68,7 +69,7 @@ def _accepts(
sa.use_mla_backend = lambda: use_mla
sa._resolved_attention_backends = lambda: [backend]
try:
ServerArgs._handle_page_major_kv_layout(sa)
handle_page_major_kv_layout(sa)
return True
except AssertionError:
return False
test/registered/unit/server_args/test_resolution_is_reproducible.py
+132
View File
@@ -1084,5 +1084,137 @@ class TestTheResolutionSeamHasOneCaller(CustomTestCase):
)
class TestResolutionStaysLazy(CustomTestCase):
"""Resolving a dummy model must not load the families it never reaches.
The forwarding slots imported their hook only when the step ran, so a
`ServerArgs(model_path="dummy")` resolution touched four hook modules. With
the slots gone the imports are function-local for the same reason, and a
module-level one costs every caller of the dummy boundary -- which is every
`override_server_args` in the test suite.
"""
def test_no_hook_module_imports_another_at_module_scope(self):
import ast
import sglang
srt = pathlib.Path(next(iter(sglang.__path__))).resolve() / "srt"
offenders = []
for path in sorted((srt / "arg_groups").glob("*.py")):
for node in ast.parse(path.read_text(encoding="utf-8-sig")).body:
if (
isinstance(node, ast.ImportFrom)
and node.module
and node.module.startswith("sglang.srt.arg_groups")
and node.module.endswith("_hook")
):
offenders.append(f"{path.name}:{node.lineno} -> {node.module}")
self.assertEqual(
offenders,
[],
"a hook module imports another at module scope, so loading one "
"family drags in a family it may never call. Import it inside the "
"function that calls it:\n " + "\n ".join(offenders),
)
def test_no_family_is_imported_before_the_step_that_calls_it(self):
"""Source-level, so it holds whatever else the process has imported.
Every hook import inside the dispatcher must come after the imports of
the families reached earlier and before its own first call -- what an
eager block at the top of the function breaks, and what a `sys.modules`
diff cannot see once another test has loaded those modules.
"""
import ast
import sglang
srt = pathlib.Path(next(iter(sglang.__path__))).resolve() / "srt"
tree = ast.parse((srt / "server_args.py").read_text(encoding="utf-8-sig"))
dispatch = next(
node
for node in ast.walk(tree)
if isinstance(node, ast.FunctionDef)
and node.name == "_run_resolution_pipeline"
)
early_return = min(
(
n.lineno
for n in ast.walk(dispatch)
if isinstance(n, ast.Return) and n.value is None
),
default=None,
)
self.assertIsNotNone(early_return, "the dummy short circuit is gone")
imported_early, called_early = set(), set()
for node in ast.walk(dispatch):
if (
isinstance(node, ast.ImportFrom)
and node.module
and node.module.endswith("_hook")
and node.lineno < early_return
):
imported_early.add(node.module.rsplit(".", 1)[-1])
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.lineno < early_return
):
called_early.add(node.func.id)
hooks = {}
for path in sorted((srt / "arg_groups").glob("*_hook.py")):
for node in ast.parse(path.read_text(encoding="utf-8-sig")).body:
if isinstance(node, ast.FunctionDef):
hooks[node.name] = path.stem
needed_early = {hooks[name] for name in called_early if name in hooks}
self.assertEqual(
imported_early - needed_early,
set(),
"the dispatcher imports a hook family before the dummy short "
"circuit without calling it there, so every dummy resolution pays "
"for a family it never reaches",
)
def test_a_dummy_resolution_loads_only_what_it_reaches(self):
"""The same claim measured, in an interpreter of its own.
In-process this would be vacuous: another test that resolved a real
model has already imported the late families, and the `sys.modules`
diff comes back empty.
"""
import subprocess
import sys
import sglang
probe = (
"import sys\n"
"from sglang.srt.server_args import ServerArgs\n"
"before = set(sys.modules)\n"
"ServerArgs(model_path='dummy').resolve_once()\n"
"print(','.join(sorted(m.rsplit('.', 1)[-1] for m in set(sys.modules) - before"
" if '.arg_groups.' in m and m.endswith('_hook'))))\n"
)
env = dict(os.environ)
env["PYTHONPATH"] = str(
pathlib.Path(next(iter(sglang.__path__))).resolve().parent
)
out = subprocess.run(
[sys.executable, "-c", probe],
capture_output=True,
text=True,
timeout=300,
env=env,
)
self.assertEqual(out.returncode, 0, out.stderr[-2000:])
loaded = [name for name in out.stdout.strip().split(",") if name]
self.assertTrue(loaded, f"the probe reported nothing:\n{out.stdout}")
for late in ("model_hook", "cuda_graph_hook", "attention_hook", "lora_hook"):
self.assertNotIn(late, loaded)
if __name__ == "__main__":
unittest.main()
test/registered/unit/server_args/test_resolution_reads_the_declarations.py
+19 -3
View File
@@ -471,8 +471,22 @@ class TestResolutionReadsTheDeclarations(CustomTestCase):
def test_no_handler_reads_a_field_off_self(self):
handlers = _resolution_handlers()
self.assertGreater(
len(handlers), 50, f"only {len(handlers)} handlers were reached"
# What the dispatcher reaches inside the class is these read wrappers;
# the package side is covered by
# `test_no_hook_reads_a_field_off_the_record`. Pinned rather than
# counted: a walk that collapsed to the wrappers would clear any floor
# low enough to admit them.
self.assertEqual(
set(handlers),
{
"_run_resolution_pipeline",
"_handle_hardware_runtime_validation",
"_handle_page_size",
"_handle_pipeline_parallelism",
"_handle_sampling_backend",
},
f"the walk reached {sorted(handlers)}; if the dispatcher grew or "
"lost a handler, add it here after checking it reads the view",
)
offenders = []
for name, fn in sorted(handlers.items()):
@@ -492,7 +506,9 @@ class TestResolutionReadsTheDeclarations(CustomTestCase):
len(decided), 100, f"the declaration set derived only {len(decided)} fields"
)
members = _record_members()
self.assertGreater(len(members), 100, f"only {len(members)} members were found")
# The floor is here to catch the scan collapsing, not to pin the
# class's size.
self.assertGreater(len(members), 40, f"only {len(members)} members were found")
offenders = []
for name, fn in sorted(members.items()):
holders = _holders(fn) | {"self"}
test/registered/unit/server_args/test_server_args.py
+166 -123
View File
@@ -10,8 +10,51 @@ from unittest.mock import MagicMock, patch
import sglang.srt.server_args as server_args_module
from sglang.srt.arg_groups import parallel_hook, pd_disaggregation_hook, serving_hook
from sglang.srt.arg_groups.attention_hook import (
handle_attention_backend_compatibility,
handle_deterministic_inference,
)
from sglang.srt.arg_groups.cuda_graph_hook import (
disable_tc_piecewise_cudagraph_if_incompatible,
handle_cuda_graph_config,
)
from sglang.srt.arg_groups.hicache_hook import (
handle_hicache,
handle_hicache_ratio_default,
)
from sglang.srt.arg_groups.hisparse_hook import (
validate_hisparse_dsa_backend,
validate_hisparse_kv_cache_dtype,
)
from sglang.srt.arg_groups.kv_cache_hook import (
handle_cache_compatibility,
validate_prefill_only_disable_kv_cache_args,
)
from sglang.srt.arg_groups.mamba_hook import handle_mamba_backend
from sglang.srt.arg_groups.model_path_hook import handle_load_format
from sglang.srt.arg_groups.moe_hook import (
handle_a2a_moe,
validate_deepep_v2_dispatch_token_budget,
validate_deepep_v2_speculative_draft,
)
from sglang.srt.arg_groups.overrides import resolution_result
from sglang.srt.arg_groups.parallel_hook import (
handle_context_parallelism,
handle_data_parallelism,
handle_legacy_cp_arguments,
)
from sglang.srt.arg_groups.pd_disaggregation_hook import handle_pd_disaggregation
from sglang.srt.arg_groups.serving_hook import (
handle_crash_dump_env,
handle_deprecated_args,
handle_load_balance_method,
handle_missing_default_values,
handle_multimodal_feature_transport,
handle_ssl_validation,
handle_tokenizer_batching,
)
from sglang.srt.arg_groups.speculative_hook import handle_speculative_decoding
from sglang.srt.arg_groups.validation_hook import check_two_batch_overlap
from sglang.srt.entrypoints.sidecar import (
SGLANG_GRPC_ENDPOINT_ENV,
Sidecar,
@@ -56,7 +99,7 @@ class TestPrepareServerArgs(CustomTestCase):
# This validation runs before model construction and should allow the
# daemon to build the same static EPLB layout as the engine.
args._handle_load_format()
handle_load_format(args)
def test_enable_w4a4_mxfp4_megamoe_sets_deepgemm_env(self):
deepgemm_env = {
@@ -174,7 +217,7 @@ class TestPrepareServerArgs(CustomTestCase):
def test_draft_quantization_explicitness_survives_asdict_round_trip(self):
inherited = ServerArgs(model_path="dummy", quantization="modelopt_fp4")
inherited._handle_missing_default_values()
handle_missing_default_values(inherited)
self.assertEqual(
resolution_result(inherited, "speculative_draft_model_quantization"),
"modelopt_fp4",
@@ -186,7 +229,7 @@ class TestPrepareServerArgs(CustomTestCase):
)
reconstructed = ServerArgs(**dataclasses.asdict(inherited))
reconstructed._handle_missing_default_values()
handle_missing_default_values(reconstructed)
self.assertFalse(
resolution_result(
@@ -226,7 +269,7 @@ class TestMmEncoderDataParallelLogging(CustomTestCase):
)
with self.assertLogs(parallel_hook.logger, level="WARNING") as logs:
server_args._handle_data_parallelism()
handle_data_parallelism(server_args)
self.assertIn("TP=1", logs.output[0])
self.assertIn("no data-parallel work", logs.output[0])
@@ -237,7 +280,7 @@ class TestMmEncoderDataParallelLogging(CustomTestCase):
)
with self.assertLogs(parallel_hook.logger, level="INFO") as logs:
server_args._handle_data_parallelism()
handle_data_parallelism(server_args)
self.assertIn("TP=4", logs.output[0])
self.assertIn("high-resolution or multi-image", logs.output[0])
@@ -247,7 +290,7 @@ class TestImageProcessorBackend(CustomTestCase):
def test_new_backend_does_not_set_legacy_flag(self):
server_args = ServerArgs(model_path="dummy", image_processor_backend="pil")
server_args._handle_deprecated_args()
handle_deprecated_args(server_args)
self.assertEqual(
resolution_result(server_args, "image_processor_backend"), "pil"
@@ -258,7 +301,7 @@ class TestImageProcessorBackend(CustomTestCase):
server_args = ServerArgs(model_path="dummy", disable_fast_image_processor=True)
with self.assertLogs(serving_hook.logger, level="WARNING") as logs:
server_args._handle_deprecated_args()
handle_deprecated_args(server_args)
self.assertEqual(
resolution_result(server_args, "image_processor_backend"), "pil"
@@ -279,7 +322,7 @@ class TestImageProcessorBackend(CustomTestCase):
)
with self.assertRaisesRegex(ValueError, "conflicts.*torchvision"):
server_args._handle_deprecated_args()
handle_deprecated_args(server_args)
class TestMultimodalFeatureTransport(CustomTestCase):
@@ -298,7 +341,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}):
with self.assertLogs(serving_hook.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_ipc"
@@ -315,7 +358,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}):
with self.assertLogs(serving_hook.logger, level="WARNING") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_ipc"
@@ -335,7 +378,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
with self.assertRaisesRegex(ValueError, "conflicts.*cuda_vmm"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_explicit_cpu_overrides_legacy_environment(self, _mock_is_cuda):
@@ -343,7 +386,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "1"}):
with self.assertLogs(serving_hook.logger, level="WARNING") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -356,7 +399,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
server_args = ServerArgs(model_path="dummy")
with patch.dict(os.environ, {"SGLANG_USE_CUDA_IPC_TRANSPORT": "0"}):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -371,7 +414,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
with self.assertNoLogs(server_args_module.logger, level="INFO"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -386,7 +429,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
with self.assertNoLogs(server_args_module.logger, level="INFO"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -415,7 +458,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
with self.assertLogs(serving_hook.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_vmm"
@@ -446,7 +489,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
self._set_model_type(server_args, is_multimodal=True)
with self.assertLogs(serving_hook.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(resolution_result(server_args, "mm_feature_transport"), "cpu")
self.assertIn("has not opted into CUDA VMM", "\n".join(logs.output))
@@ -465,7 +508,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
with self.assertLogs(serving_hook.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -485,7 +528,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -499,7 +542,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
with patch.dict(os.environ, {}, clear=False):
envs.SGLANG_USE_CUDA_IPC_TRANSPORT.clear()
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cpu"
@@ -511,7 +554,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_ipc")
with self.assertRaisesRegex(ValueError, "requires NVIDIA CUDA"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_ipc_rejects_multi_node(self, _mock_is_cuda):
@@ -520,7 +563,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
with self.assertRaisesRegex(ValueError, "single node"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_vmm_is_explicit_and_uses_shared_budget(self, _mock_is_cuda):
@@ -536,7 +579,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
envs.SGLANG_MM_FEATURE_CACHE_MB.override(256),
):
with self.assertLogs(serving_hook.logger, level="INFO") as logs:
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
self.assertEqual(
resolution_result(server_args, "mm_feature_transport"), "cuda_vmm"
@@ -554,7 +597,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
server_args = ServerArgs(model_path="dummy", mm_feature_transport="cuda_vmm")
with self.assertRaisesRegex(ValueError, "requires NVIDIA CUDA"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_vmm_rejects_rust_server(self, _mock_is_cuda):
@@ -564,7 +607,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
envs.SGLANG_RUST_SERVER.override(True),
self.assertRaisesRegex(ValueError, "SGLANG_RUST_SERVER"),
):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
@patch("sglang.srt.arg_groups.serving_hook.is_cuda", return_value=True)
def test_cuda_vmm_rejects_pipeline_parallelism(self, _mock_is_cuda):
@@ -573,7 +616,7 @@ class TestMultimodalFeatureTransport(CustomTestCase):
)
with self.assertRaisesRegex(ValueError, "pipeline parallelism"):
server_args._handle_multimodal_feature_transport()
handle_multimodal_feature_transport(server_args)
class TestMambaCacheStochasticRounding(unittest.TestCase):
@@ -585,7 +628,7 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "--mamba-ssm-dtype float16"):
server_args._handle_mamba_backend()
handle_mamba_backend(server_args)
@patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=False)
def test_rejects_non_cuda(self, _mock_is_cuda):
@@ -596,7 +639,7 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "NVIDIA CUDA"):
server_args._handle_mamba_backend()
handle_mamba_backend(server_args)
@patch("sglang.srt.arg_groups.mamba_hook.is_cuda", return_value=True)
@patch("sglang.srt.arg_groups.mamba_hook.is_sm100_supported", return_value=False)
@@ -609,14 +652,14 @@ class TestMambaCacheStochasticRounding(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "requires SM100"):
server_args._handle_mamba_backend()
handle_mamba_backend(server_args)
class TestLoadBalanceMethod(unittest.TestCase):
def _load_balance_args(self, **kwargs):
server_args = ServerArgs(model_path="dummy", **kwargs)
server_args._handle_pd_disaggregation()
server_args._handle_load_balance_method()
handle_pd_disaggregation(server_args)
handle_load_balance_method(server_args)
return server_args
def test_non_pd_defaults_to_round_robin(self):
@@ -645,7 +688,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
dcp_size=4,
)
with self.assertLogs(pd_disaggregation_hook.logger, level="WARNING") as logs:
server_args._handle_pd_disaggregation()
handle_pd_disaggregation(server_args)
self.assertIn("without improving prefill performance", "\n".join(logs.output))
def test_pd_decode_dcp_forces_chunk_cache(self):
@@ -666,7 +709,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
with self.assertRaisesRegex(
ValueError, "mooncake, nixl, or fake for synthetic benchmarking"
):
server_args._handle_pd_disaggregation()
handle_pd_disaggregation(server_args)
def test_pd_decode_dcp_allows_fake_transfer_backend(self):
server_args = self._load_balance_args(
@@ -685,7 +728,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
dcp_size=4,
)
with self.assertRaisesRegex(ValueError, "currently requires chunk cache"):
server_args._handle_pd_disaggregation()
handle_pd_disaggregation(server_args)
def test_pd_decode_dcp_rejects_hierarchical_cache(self):
server_args = ServerArgs(
@@ -696,7 +739,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
dcp_size=4,
)
with self.assertRaisesRegex(ValueError, "--enable-hierarchical-cache"):
server_args._handle_pd_disaggregation()
handle_pd_disaggregation(server_args)
def test_pd_decode_radix_cache_rejects_hisparse(self):
server_args = ServerArgs(
@@ -707,7 +750,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
enable_hisparse=True,
)
with self.assertRaises(ValueError) as context:
server_args._handle_pd_disaggregation()
handle_pd_disaggregation(server_args)
self.assertIn(
"--disaggregation-decode-enable-radix-cache is incompatible with "
@@ -723,7 +766,7 @@ class TestLoadBalanceMethod(unittest.TestCase):
disaggregation_transfer_backend="fake",
)
with self.assertRaises(ValueError) as context:
server_args._handle_pd_disaggregation()
handle_pd_disaggregation(server_args)
self.assertIn(
"--disaggregation-decode-enable-radix-cache is incompatible "
@@ -754,7 +797,7 @@ class TestSkipTokenizerInit(unittest.TestCase):
detokenizer_worker_num=3,
)
server_args._handle_tokenizer_batching()
handle_tokenizer_batching(server_args)
# Tokenizer fanout preserved; detokenizer coerced to 1 (no decode work).
self.assertEqual(resolution_result(server_args, "tokenizer_worker_num"), 4)
@@ -830,8 +873,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
dsa_decode_backend="flashinfer_sparse_mla",
)
server_args._validate_hisparse_dsa_backend("dsa_prefill_backend", "prefill")
server_args._validate_hisparse_dsa_backend("dsa_decode_backend", "decode")
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.server_args.is_hip", return_value=True)
def test_hisparse_defaults_to_tilelang_on_rocm(self, _mock_is_hip):
@@ -859,8 +902,8 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
dsa_decode_backend="aiter",
)
server_args._validate_hisparse_dsa_backend("dsa_prefill_backend", "prefill")
server_args._validate_hisparse_dsa_backend("dsa_decode_backend", "decode")
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
@patch("sglang.srt.server_args.is_hip", return_value=True)
def test_hisparse_rejects_cuda_backend_on_rocm(self, _mock_is_hip):
@@ -872,7 +915,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "tilelang"):
server_args._validate_hisparse_dsa_backend("dsa_prefill_backend", "prefill")
validate_hisparse_dsa_backend(server_args, "dsa_prefill_backend", "prefill")
@patch("sglang.srt.server_args.is_hip", return_value=False)
def test_hisparse_rejects_rocm_backend_on_cuda(self, _mock_is_hip):
@@ -884,7 +927,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "flashmla_sparse"):
server_args._validate_hisparse_dsa_backend("dsa_decode_backend", "decode")
validate_hisparse_dsa_backend(server_args, "dsa_decode_backend", "decode")
def test_hisparse_accepts_bfloat16_kv_cache_dtype(self):
server_args = ServerArgs(
@@ -893,7 +936,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
kv_cache_dtype="bfloat16",
)
server_args._validate_hisparse_kv_cache_dtype()
validate_hisparse_kv_cache_dtype(server_args)
def test_hisparse_accepts_fp8_e4m3_kv_cache_dtype(self):
server_args = ServerArgs(
@@ -902,7 +945,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
kv_cache_dtype="fp8_e4m3",
)
server_args._validate_hisparse_kv_cache_dtype()
validate_hisparse_kv_cache_dtype(server_args)
def test_hisparse_rejects_unsupported_kv_cache_dtype(self):
server_args = ServerArgs(
@@ -912,7 +955,7 @@ class TestHiSparseDsaBackendPolicy(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, r"fp8_e4m3"):
server_args._validate_hisparse_kv_cache_dtype()
validate_hisparse_kv_cache_dtype(server_args)
class TestFa4PageSizeAutoForce(CustomTestCase):
@@ -942,7 +985,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
# `--attention-backend fa4` (combined): prefill/decode fields stay None.
args = self._make_args(attention_backend="fa4")
args._handle_attention_backend_compatibility()
handle_attention_backend_compatibility(args)
from sglang.srt.arg_groups.overrides import resolved_view
@@ -955,7 +998,7 @@ class TestFa4PageSizeAutoForce(CustomTestCase):
# `--prefill-attention-backend fa4`: the previously-covered path.
args = self._make_args(attention_backend=None, prefill="fa4", page_size=1)
args._handle_attention_backend_compatibility()
handle_attention_backend_compatibility(args)
from sglang.srt.arg_groups.overrides import resolved_view
@@ -1002,7 +1045,7 @@ class TestContextParallelServerArgs(CustomTestCase):
cp_strategy=resolution_result(args, "cp_strategy"),
)
with self.assertRaisesRegex(ValueError, "--cp-strategy"):
server_args._handle_context_parallelism()
handle_context_parallelism(server_args)
def test_deprecated_dsa_cp_mode_maps_to_unified_strategy(self):
args = self.parser.parse_args(
@@ -1021,7 +1064,7 @@ class TestContextParallelServerArgs(CustomTestCase):
dsa_prefill_cp_mode=resolution_result(args, "dsa_prefill_cp_mode"),
)
server_args._handle_legacy_cp_arguments()
handle_legacy_cp_arguments(server_args)
self.assertTrue(resolution_result(server_args, "enable_prefill_cp"))
self.assertEqual(resolution_result(server_args, "cp_strategy"), "interleave")
@@ -1036,8 +1079,8 @@ class TestContextParallelServerArgs(CustomTestCase):
attention_backend="dsa",
)
server_args._handle_legacy_cp_arguments()
server_args._handle_context_parallelism()
handle_legacy_cp_arguments(server_args)
handle_context_parallelism(server_args)
self.assertTrue(
resolution_result(server_args, "enable_dsa_prefill_context_parallel")
@@ -1060,7 +1103,7 @@ class TestContextParallelServerArgs(CustomTestCase):
tp_size=2,
)
server_args._handle_context_parallelism()
handle_context_parallelism(server_args)
self.assertTrue(is_cp_enabled())
self.assertTrue(is_interleave())
@@ -1132,8 +1175,8 @@ class TestContextParallelServerArgs(CustomTestCase):
with self.subTest(name=name):
server_args = self._new_cp_args(**overrides)
server_args._handle_legacy_cp_arguments()
server_args._handle_context_parallelism()
handle_legacy_cp_arguments(server_args)
handle_context_parallelism(server_args)
self.assertTrue(resolution_result(server_args, "enable_prefill_cp"))
self.assertEqual(
@@ -1303,7 +1346,7 @@ class TestPortArgs(unittest.TestCase):
class TestSSLArgs(unittest.TestCase):
def _validate_ssl(self, **kwargs):
server_args = ServerArgs(model_path="dummy", **kwargs)
server_args._handle_ssl_validation()
handle_ssl_validation(server_args)
return server_args
def test_ssl_keyfile_without_certfile_raises(self):
@@ -1413,7 +1456,7 @@ class TestHiCacheArgs(unittest.TestCase):
# so `_handle_hicache` would never run. Its one prerequisite (the
# host/device ratio default) is run by hand.
args = ServerArgs(model_path="dummy", **overrides)
args._handle_hicache_ratio_default()
handle_hicache_ratio_default(args)
return args
def _assert_hicache_fields(
@@ -1495,7 +1538,7 @@ class TestHiCacheArgs(unittest.TestCase):
for case in cases:
with self.subTest(case=case["name"]):
args = self._make_args(**case["overrides"])
args._handle_hicache()
handle_hicache(args)
self._assert_hicache_fields(
args,
expected_io_backend=case["expected_io_backend"],
@@ -1510,7 +1553,7 @@ class TestHiCacheArgs(unittest.TestCase):
attention_backend="fa3",
decode_attention_backend=None,
)
args._handle_hicache()
handle_hicache(args)
self.assertEqual(resolution_result(args, "hicache_io_backend"), "kernel")
self.assertEqual(resolution_result(args, "hicache_mem_layout"), "page_first")
@@ -1525,7 +1568,7 @@ class TestHiCacheArgs(unittest.TestCase):
)
with self.assertRaisesRegex(ValueError, "mutually exclusive"):
args._handle_cache_compatibility()
handle_cache_compatibility(args)
def test_decode_offload_allows_cpu_tensor_retraction(self):
args = self._make_args(
@@ -1535,7 +1578,7 @@ class TestHiCacheArgs(unittest.TestCase):
disaggregation_decode_retraction_backup="cpu_tensor",
)
args._handle_cache_compatibility()
handle_cache_compatibility(args)
class TestNgramExternalSamArgs(CustomTestCase):
@@ -1659,7 +1702,7 @@ class TestWaterfillArgs(CustomTestCase):
disable_shared_experts_fusion=True,
)
# dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe()
handle_a2a_moe(server_args)
from sglang.srt.arg_groups.overrides import resolved_view
@@ -1674,7 +1717,7 @@ class TestWaterfillArgs(CustomTestCase):
enable_waterfill=True,
)
# dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe()
handle_a2a_moe(server_args)
from sglang.srt.arg_groups.overrides import resolved_view
@@ -1690,7 +1733,7 @@ class TestWaterfillArgs(CustomTestCase):
disable_shared_experts_fusion=True,
)
# dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe()
handle_a2a_moe(server_args)
from sglang.srt.arg_groups.overrides import resolved_view
@@ -1706,7 +1749,7 @@ class TestWaterfillArgs(CustomTestCase):
deepep_mode="low_latency",
)
# dummy-model path short-circuits __post_init__; invoke the handler directly.
server_args._handle_a2a_moe()
handle_a2a_moe(server_args)
self.assertEqual(resolution_result(server_args, "deepep_mode"), "low_latency")
self.assertFalse(resolution_result(server_args, "disable_cuda_graph"))
@@ -1739,8 +1782,8 @@ class TestPrefillOnlyDisableKvCache(unittest.TestCase):
def _validate_prefill_only_args(self, **overrides):
sa = ServerArgs(**self._base_kwargs(**overrides))
sa._handle_legacy_cp_arguments()
sa._validate_prefill_only_disable_kv_cache_args()
handle_legacy_cp_arguments(sa)
validate_prefill_only_disable_kv_cache_args(sa)
return sa
def test_valid_minimal_config_constructs(self):
@@ -1829,7 +1872,7 @@ class TestCudaGraphDisaggregationRoles(CustomTestCase):
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
):
args._handle_cuda_graph_config()
handle_cuda_graph_config(args)
return args
def test_cuda_graph_prefill_role_defaults_disable_decode_graph(self):
@@ -1902,7 +1945,7 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
):
args._handle_cuda_graph_config()
handle_cuda_graph_config(args)
return args
def test_enable_lora_keeps_breakable_prefill_graph(self):
@@ -1941,7 +1984,7 @@ class TestPrefillCudaGraphLoRACompatibility(CustomTestCase):
patch("sglang.srt.arg_groups.cuda_graph_hook.is_mps", return_value=False),
patch("sglang.srt.arg_groups.cuda_graph_hook.is_xpu", return_value=False),
):
args._disable_tc_piecewise_cudagraph_if_incompatible()
disable_tc_piecewise_cudagraph_if_incompatible(args)
self.assertEqual(
resolution_result(args, "cuda_graph_config").prefill.backend,
@@ -1966,7 +2009,7 @@ class TestBreakableCudaGraphMultimodalAllowlist(CustomTestCase):
patch("sglang.srt.utils.is_cuda", return_value=True),
patch.object(ServerArgs, "use_mla_backend", return_value=False),
):
args._handle_cuda_graph_config()
handle_cuda_graph_config(args)
return args
def test_multimodal_arch_disables_prefill_breakable(self):
@@ -2163,7 +2206,7 @@ class TestDeepEPv2Args(CustomTestCase):
):
args = self._args(moe_runner_backend="deep_gemm")
args._model_config.hf_config.architectures = [architecture]
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_unvalidated_and_missing_architectures_rejected(self):
for architectures in (
@@ -2175,7 +2218,7 @@ class TestDeepEPv2Args(CustomTestCase):
args = self._args(moe_runner_backend="deep_gemm")
args._model_config.hf_config.architectures = architectures
with self.assertRaisesRegex(ValueError, "not validated"):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_instance_connector_rejected(self):
args = self._args(
@@ -2183,7 +2226,7 @@ class TestDeepEPv2Args(CustomTestCase):
moe_runner_backend="deep_gemm",
)
with self.assertRaisesRegex(ValueError, "instance connector"):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_deterministic_inference_rejected(self):
args = self._args(
@@ -2191,7 +2234,7 @@ class TestDeepEPv2Args(CustomTestCase):
enable_deterministic_inference=True,
)
with self.assertRaisesRegex(ValueError, "deterministic sorting"):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_rl_on_policy_deterministic_inference_rejected(self):
args = self._args(
@@ -2205,9 +2248,9 @@ class TestDeepEPv2Args(CustomTestCase):
envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get()
),
):
args._handle_deterministic_inference()
handle_deterministic_inference(args)
with self.assertRaisesRegex(ValueError, "deterministic sorting"):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_deterministic_inference_does_not_affect_legacy_deepep(self):
args = self._args(
@@ -2215,7 +2258,7 @@ class TestDeepEPv2Args(CustomTestCase):
moe_runner_backend="deep_gemm",
enable_deterministic_inference=True,
)
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_runner_restored_by_declaration_fails_fast(self):
# Validate the declaration-resolved runner rather than the raw field.
@@ -2224,13 +2267,13 @@ class TestDeepEPv2Args(CustomTestCase):
("test_mxfp8", {"moe_runner_backend": "flashinfer_trtllm"})
]
with self.assertRaises(ValueError):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_declarations_resolve_ep_size_and_fusion(self):
from sglang.srt.arg_groups.overrides import resolved_view
args = self._args(moe_runner_backend="auto", tp_size=2)
args._handle_a2a_moe()
handle_a2a_moe(args)
self.assertEqual(resolved_view(args).ep_size, args.tp_size)
self.assertTrue(resolved_view(args).disable_shared_experts_fusion)
@@ -2238,23 +2281,23 @@ class TestDeepEPv2Args(CustomTestCase):
from sglang.srt.arg_groups.overrides import resolved_view
args = self._args(moe_runner_backend="auto")
args._handle_a2a_moe()
handle_a2a_moe(args)
self.assertEqual(resolved_view(args).moe_runner_backend, "deep_gemm")
def test_unsupported_runner_rejected(self):
args = self._args(moe_runner_backend="flashinfer_trtllm")
with self.assertRaises(ValueError):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_triton_runner_rejected(self):
args = self._args(moe_runner_backend="triton")
with self.assertRaises(ValueError):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_decode_graph_stays_enabled_in_both_comm_modes(self):
for mode in ("direct", "hybrid"):
args = self._args(moe_runner_backend="deep_gemm", deepep_v2_mode=mode)
args._handle_a2a_moe()
handle_a2a_moe(args)
declared = resolution_result(args, "cuda_graph_config")
self.assertEqual(declared.decode.backend, Backend.FULL)
self.assertEqual(declared.prefill.backend, Backend.DISABLED)
@@ -2262,7 +2305,7 @@ class TestDeepEPv2Args(CustomTestCase):
def test_two_batch_overlap_rejected(self):
args = self._args(moe_runner_backend="deep_gemm", enable_two_batch_overlap=True)
with self.assertRaises(ValueError):
args._handle_a2a_moe()
handle_a2a_moe(args)
def test_speculative_draft_backend_rejected(self):
for main_backend in ("none", "deepep", "deepep_v2"):
@@ -2272,7 +2315,7 @@ class TestDeepEPv2Args(CustomTestCase):
speculative_moe_a2a_backend="deepep_v2",
)
with self.assertRaisesRegex(ValueError, "speculative draft backend"):
args._validate_deepep_v2_speculative_draft()
validate_deepep_v2_speculative_draft(args)
def test_inherited_speculative_draft_backend_rejected(self):
args = self._args(
@@ -2280,14 +2323,14 @@ class TestDeepEPv2Args(CustomTestCase):
speculative_algorithm="EAGLE",
)
with self.assertRaisesRegex(ValueError, "speculative draft backend"):
args._validate_deepep_v2_speculative_draft()
validate_deepep_v2_speculative_draft(args)
def test_ngram_does_not_inherit_a_draft_backend(self):
args = self._args(
moe_runner_backend="deep_gemm",
speculative_algorithm="NGRAM",
)
args._validate_deepep_v2_speculative_draft()
validate_deepep_v2_speculative_draft(args)
def test_explicit_legacy_speculative_backend_allowed(self):
args = self._args(
@@ -2295,7 +2338,7 @@ class TestDeepEPv2Args(CustomTestCase):
speculative_algorithm="EAGLE",
speculative_moe_a2a_backend="deepep",
)
args._validate_deepep_v2_speculative_draft()
validate_deepep_v2_speculative_draft(args)
def test_resolved_legacy_speculative_backend_allowed(self):
args = self._args(
@@ -2308,18 +2351,18 @@ class TestDeepEPv2Args(CustomTestCase):
{"speculative_moe_a2a_backend": "deepep"},
)
]
args._validate_deepep_v2_speculative_draft()
validate_deepep_v2_speculative_draft(args)
def test_prefill_chunk_exceeding_cap_rejected(self):
args = self._args(moe_runner_backend="deep_gemm", chunked_prefill_size=2048)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(1024):
with self.assertRaisesRegex(ValueError, "NUM_MAX_DISPATCH_TOKENS_PER_RANK"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_prefill_chunk_at_cap_boundary_accepted(self):
args = self._args(moe_runner_backend="deep_gemm", chunked_prefill_size=1024)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(1024):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_dynamic_chunking_probe_is_included(self):
args = self._args(
@@ -2331,7 +2374,7 @@ class TestDeepEPv2Args(CustomTestCase):
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(1024):
with self.assertRaisesRegex(ValueError, "required=1280"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_disabled_chunking_uses_max_prefill_tokens(self):
for disabled in (None, 0, -1):
@@ -2342,7 +2385,7 @@ class TestDeepEPv2Args(CustomTestCase):
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
with self.assertRaisesRegex(ValueError, "required=1024"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_decode_role_skips_prefill_capacity(self):
args = self._args(
@@ -2352,7 +2395,7 @@ class TestDeepEPv2Args(CustomTestCase):
dp_size=1,
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_decode_graph_capacity_boundaries(self):
for max_bs, raises in ((128, False), (129, True)):
@@ -2364,9 +2407,9 @@ class TestDeepEPv2Args(CustomTestCase):
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
if raises:
with self.assertRaisesRegex(ValueError, "decode CUDA graph"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
else:
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_dp_attention_divides_max_running_requests_per_rank(self):
args = self._args(
@@ -2377,7 +2420,7 @@ class TestDeepEPv2Args(CustomTestCase):
enable_dp_attention=True,
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_tp_only_max_running_requests_is_not_divided(self):
args = self._args(
@@ -2389,7 +2432,7 @@ class TestDeepEPv2Args(CustomTestCase):
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
with self.assertRaisesRegex(ValueError, "decode CUDA graph"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_memory_derived_eager_pool_remains_runtime_validated(self):
args = self._args(
@@ -2398,7 +2441,7 @@ class TestDeepEPv2Args(CustomTestCase):
)
args.cuda_graph_config.decode.backend = Backend.DISABLED
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(1):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_speculative_decode_width_is_included(self):
args = self._args(
@@ -2411,7 +2454,7 @@ class TestDeepEPv2Args(CustomTestCase):
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
with self.assertRaisesRegex(ValueError, "tokens/request=8"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_adaptive_speculative_uses_widest_candidate(self):
args = self._args(
@@ -2430,7 +2473,7 @@ class TestDeepEPv2Args(CustomTestCase):
):
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
with self.assertRaisesRegex(ValueError, "tokens/request=16"):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_prefill_role_skips_decode_capacity(self):
args = self._args(
@@ -2439,7 +2482,7 @@ class TestDeepEPv2Args(CustomTestCase):
max_running_requests=8192,
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(128):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_other_backend_skips_capacity_validation(self):
args = self._args(
@@ -2448,13 +2491,13 @@ class TestDeepEPv2Args(CustomTestCase):
max_running_requests=4096,
)
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(1):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
def test_capacity_validation_uses_resolved_backend(self):
args = self._args(chunked_prefill_size=4096)
args._resolved_overrides = [("test", {"moe_a2a_backend": "deepep"})]
with envs.SGLANG_DEEPEP_V2_NUM_MAX_DISPATCH_TOKENS_PER_RANK.override(1):
args._validate_deepep_v2_dispatch_token_budget()
validate_deepep_v2_dispatch_token_budget(args)
class TestHandleCrashDumpEnv(CustomTestCase):
@@ -2474,7 +2517,7 @@ class TestHandleCrashDumpEnv(CustomTestCase):
for key in self._COREDUMP_ENV_KEYS:
if key not in (preset_env or {}):
os.environ.pop(key, None)
ServerArgs._handle_crash_dump_env(server_args)
handle_crash_dump_env(server_args)
def test_creates_coredump_dir_when_auto_set(self):
with tempfile.TemporaryDirectory() as tmp:
@@ -2504,7 +2547,7 @@ class TestGrpcServerArgs(CustomTestCase):
alongside HTTP; --smg-grpc-mode (and the deprecated --grpc-mode) select the
legacy SMG server. Worker-threads / max-prefill-tokens are env-only knobs.
The gRPC setup lives in ServerArgs._handle_deprecated_args, which
The gRPC setup lives in `serving_hook.handle_deprecated_args`, which
__post_init__ skips for dummy models, so these tests build a dummy
ServerArgs and invoke that handler directly (mirroring the real flow for a
concrete model path).
@@ -2516,20 +2559,20 @@ class TestGrpcServerArgs(CustomTestCase):
def test_http_only_high_port_does_not_derive_grpc_port(self):
sa = self._args(port=56000)
sa._handle_deprecated_args()
handle_deprecated_args(sa)
self.assertIsNone(resolution_result(sa, "grpc_port"))
def test_grpc_port_enables_native_and_env_knobs(self):
sa = self._args(grpc_port=50051)
with envs.SGLANG_GRPC_WORKER_THREADS.override(8):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
self.assertEqual(resolution_result(sa, "grpc_port"), 50051)
self.assertEqual(resolution_result(sa, "grpc_worker_threads"), 8)
def test_env_grpc_port_enables_native(self):
sa = self._args(port=30000)
with envs.SGLANG_GRPC_PORT.override(45000):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
self.assertEqual(resolution_result(sa, "grpc_port"), 45000)
@staticmethod
@@ -2602,17 +2645,17 @@ class TestGrpcServerArgs(CustomTestCase):
def test_sidecar_requires_native_grpc(self):
sa = self._args(sidecar="example.sidecar")
with self.assertRaisesRegex(ValueError, "requires --grpc-port"):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
def test_sidecar_rejects_legacy_grpc(self):
sa = self._args(sidecar="example.sidecar", smg_grpc_mode=True)
with self.assertRaisesRegex(ValueError, "native gRPC server"):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
def test_sidecar_rejects_empty_value(self):
sa = self._args(sidecar="", grpc_port=50051)
with self.assertRaisesRegex(ValueError, "must not be empty"):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
def test_sidecar_sets_endpoint_env_before_import_and_calls_main(self):
main = MagicMock()
@@ -2666,37 +2709,37 @@ class TestGrpcServerArgs(CustomTestCase):
def test_legacy_smg_derives_grpc_port_from_http_port(self):
sa = self._args(port=30000, smg_grpc_mode=True)
sa._handle_deprecated_args()
handle_deprecated_args(sa)
self.assertEqual(resolution_result(sa, "grpc_port"), 40000)
def test_grpc_mode_is_deprecated_alias_for_smg_grpc_mode(self):
sa = self._args(grpc_mode=True)
with self.assertLogs(serving_hook.logger, level="WARNING") as cm:
sa._handle_deprecated_args()
handle_deprecated_args(sa)
self.assertTrue(resolution_result(sa, "smg_grpc_mode"))
self.assertTrue(any("--grpc-mode is deprecated" in line for line in cm.output))
def test_legacy_smg_takes_precedence_over_grpc_port(self):
sa = self._args(grpc_port=50051, smg_grpc_mode=True)
sa._handle_deprecated_args()
handle_deprecated_args(sa)
self.assertTrue(resolution_result(sa, "smg_grpc_mode"))
self.assertEqual(resolution_result(sa, "grpc_port"), 50051)
def test_native_grpc_rejects_multi_tokenizer(self):
sa = self._args(grpc_port=40000, tokenizer_worker_num=2)
with self.assertRaises(ValueError):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
def test_native_grpc_rejects_http_auth(self):
sa = self._args(grpc_port=40000, api_key="secret")
with self.assertRaises(ValueError):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
def test_invalid_grpc_worker_threads_rejected(self):
sa = self._args(grpc_port=40000)
with envs.SGLANG_GRPC_WORKER_THREADS.override(0):
with self.assertRaises(ValueError):
sa._handle_deprecated_args()
handle_deprecated_args(sa)
def test_start_server_call_site_matches_native_signature(self):
"""Regression for the startup blocker: the native start_server binding
@@ -2764,19 +2807,19 @@ class TestTwoBatchOverlapBackend(CustomTestCase):
def test_no_a2a_without_dp_attention_raises(self):
args = self._args(enable_dp_attention=False)
with self.assertRaisesRegex(ValueError, "enable-dp-attention"):
args._check_two_batch_overlap()
check_two_batch_overlap(args)
def test_no_a2a_with_dp_attention_ok(self):
# DP TBO path is valid: --enable-dp-attention + --enable-two-batch-overlap
# with a2a backend 'none' must NOT raise (no SGLANG_ENABLE_DP_TBO needed).
args = self._args(enable_dp_attention=True)
args._check_two_batch_overlap()
check_two_batch_overlap(args)
def test_ep_a2a_backend_ok_without_dp_attention(self):
# EP a2a path (e.g. deepep) overlaps dispatch/combine; the guard does not
# require dp-attention there.
args = self._args(moe_a2a_backend="deepep", enable_dp_attention=False)
args._check_two_batch_overlap()
check_two_batch_overlap(args)
class TestDcpKvEventContract(CustomTestCase):
test/registered/unit/spec/test_dflash_extra_buffer_lazy.py
+5 -4
View File
@@ -9,13 +9,13 @@ from unittest import mock
import torch
from sglang.srt.arg_groups.mamba_hook import validate_mamba_extra_buffer
from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.test_utils import CustomTestCase
register_cuda_ci(est_time=10, stage="base-b", runner_config="1-gpu-small")
from sglang.srt.model_executor.forward_batch_info import ForwardMode
from sglang.srt.server_args import ServerArgs
from sglang.srt.speculative import dflash_info
from sglang.srt.speculative.dflash_info import DFlashVerifyInput
@@ -39,7 +39,6 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase):
"""The DFLASH rejection is gone; the neighboring invariants still hold."""
def _validate(self, view):
fake_self = SimpleNamespace(mamba_cache_chunk_size=64)
with mock.patch(
"sglang.srt.arg_groups.overrides.supports_mamba_cache_extra_buffer",
return_value=True,
@@ -49,8 +48,10 @@ class TestValidateMambaExtraBufferLazyDflash(CustomTestCase):
"sglang.srt.arg_groups.mamba_hook.is_cuda",
return_value=True,
):
ServerArgs._validate_mamba_extra_buffer(
fake_self, view, "Qwen3NextForCausalLM"
validate_mamba_extra_buffer(
view,
"Qwen3NextForCausalLM",
mamba_cache_chunk_size_of=lambda: 64,
)
def test_dflash_with_extra_buffer_lazy_is_accepted(self):