ae7a516ba7
Co-authored-by: yuchuan <yuchuan.7streams@gmail.com> Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
512 lines
15 KiB
Python
512 lines
15 KiB
Python
"""The order ``ServerArgs`` presents its fields in, frozen.
|
|
|
|
A dataclass turns field order into a positional constructor signature, so
|
|
``ServerArgs(model_path, tokenizer_path)`` has to keep meaning what it means.
|
|
Grouping the declarations by namespace would move the second argument onto
|
|
another field, silently.
|
|
|
|
A compatibility record and nothing else -- a field's namespace is the module it
|
|
is declared in, and only ``collect_input_fields`` reads this. A name that is not
|
|
here sorts after every name that is, which is the only backward-compatible
|
|
position for a new field anyway.
|
|
"""
|
|
|
|
# fmt: off
|
|
POSITIONAL_FIELD_ORDER = (
|
|
"model_path",
|
|
"tokenizer_path",
|
|
"tokenizer_mode",
|
|
"tokenizer_backend",
|
|
"tokenizer_worker_num",
|
|
"detokenizer_worker_num",
|
|
"skip_tokenizer_init",
|
|
"load_format",
|
|
"model_loader_extra_config",
|
|
"trust_remote_code",
|
|
"context_length",
|
|
"is_embedding",
|
|
"enable_multimodal",
|
|
"revision",
|
|
"model_impl",
|
|
"model_config_parser",
|
|
"json_model_override_args",
|
|
"dtype",
|
|
"quantization",
|
|
"quantization_param_path",
|
|
"kv_cache_dtype",
|
|
"enable_fp32_lm_head",
|
|
"modelopt_quant",
|
|
"modelopt_checkpoint_restore_path",
|
|
"modelopt_checkpoint_save_path",
|
|
"modelopt_export_path",
|
|
"quantize_and_serve",
|
|
"rl_quant_profile",
|
|
"enable_tf32_matmul",
|
|
"mem_fraction_static",
|
|
"max_running_requests",
|
|
"max_queued_requests",
|
|
"max_total_tokens",
|
|
"chunked_prefill_size",
|
|
"prefill_decode_interval",
|
|
"enable_dynamic_chunking",
|
|
"max_prefill_tokens",
|
|
"prefill_max_requests",
|
|
"schedule_policy",
|
|
"enable_priority_scheduling",
|
|
"disable_priority_preemption",
|
|
"default_priority_value",
|
|
"abort_on_priority_when_disabled",
|
|
"schedule_low_priority_values_first",
|
|
"priority_scheduling_preemption_threshold",
|
|
"retraction_policy",
|
|
"schedule_conservativeness",
|
|
"page_size",
|
|
"c128_page_size",
|
|
"swa_full_tokens_ratio",
|
|
"disable_hybrid_swa_memory",
|
|
"radix_eviction_policy",
|
|
"radix_eviction_policy_config",
|
|
"prefill_only_disable_kv_cache",
|
|
"disable_radix_cache",
|
|
"enable_page_major_kv_layout",
|
|
"enable_unified_memory",
|
|
"disable_chunked_prefix_cache",
|
|
"disable_overlap_schedule",
|
|
"num_continuous_decode_steps",
|
|
"scheduler_recv_interval",
|
|
"enable_mixed_chunk",
|
|
"nccl_port",
|
|
"dist_timeout",
|
|
"dist_init_addr",
|
|
"gated_launch_port",
|
|
"nnodes",
|
|
"node_rank",
|
|
"tp_size",
|
|
"dcp_size",
|
|
"pp_size",
|
|
"pp_max_micro_batch_size",
|
|
"pp_async_batch_depth",
|
|
"dp_size",
|
|
"load_balance_method",
|
|
"attn_cp_size",
|
|
"moe_dp_size",
|
|
"dwdp_size",
|
|
"dcp_comm_backend",
|
|
"dcp_replicate_q_proj",
|
|
"enable_prefill_cp",
|
|
"cp_strategy",
|
|
"enable_dsa_cache_layer_split",
|
|
"enable_cp_decode_attn_tp",
|
|
"enable_dp_attention",
|
|
"enable_dp_attention_local_control_broadcast",
|
|
"enable_dp_lm_head",
|
|
"enable_tp_lm_head_all_to_all",
|
|
"enable_attn_tp_input_scattered",
|
|
"enable_shared_experts_attn_tp",
|
|
"shared_experts_tp_size",
|
|
"enable_dense_mlp_attn_tp",
|
|
"enable_layernorm_sp",
|
|
"disable_attn_tp_gather",
|
|
"enable_p2p_check",
|
|
"device",
|
|
"base_gpu_id",
|
|
"gpu_id_step",
|
|
"random_seed",
|
|
"mlx_enable_sampling",
|
|
"watchdog_timeout",
|
|
"soft_watchdog_timeout",
|
|
"sleep_on_idle",
|
|
"use_ray",
|
|
"custom_sigquit_handler",
|
|
"numa_node",
|
|
"gc_threshold",
|
|
"host",
|
|
"port",
|
|
"fastapi_root_path",
|
|
"smg_grpc_mode",
|
|
"grpc_mode",
|
|
"grpc_port",
|
|
"grpc_worker_threads",
|
|
"sidecar",
|
|
"sidecar_args",
|
|
"skip_server_warmup",
|
|
"warmups",
|
|
"enable_http2",
|
|
"http2_max_concurrent_streams",
|
|
"http2_initial_connection_window_size",
|
|
"ssl_keyfile",
|
|
"ssl_certfile",
|
|
"ssl_ca_certs",
|
|
"ssl_keyfile_password",
|
|
"enable_ssl_refresh",
|
|
"api_key",
|
|
"admin_api_key",
|
|
"served_model_name",
|
|
"weight_version",
|
|
"chat_template",
|
|
"hf_chat_template_name",
|
|
"completion_template",
|
|
"file_storage_path",
|
|
"enable_cache_report",
|
|
"return_input_ids",
|
|
"return_output_ids",
|
|
"reasoning_parser",
|
|
"default_chat_template_kwargs",
|
|
"strip_thinking_cache",
|
|
"enable_strict_thinking",
|
|
"tool_call_parser",
|
|
"tool_server",
|
|
"sampling_defaults",
|
|
"asr_max_buffer_seconds",
|
|
"asr_max_concurrent_sessions",
|
|
"preferred_sampling_params",
|
|
"allow_auto_truncate",
|
|
"stream_interval",
|
|
"batch_notify_size",
|
|
"stream_response_default_include_usage",
|
|
"incremental_streaming_output",
|
|
"enable_streaming_session",
|
|
"enable_session_radix_cache",
|
|
"log_level",
|
|
"log_level_http",
|
|
"log_requests",
|
|
"log_requests_level",
|
|
"log_requests_format",
|
|
"log_requests_target",
|
|
"uvicorn_access_log_exclude_prefixes",
|
|
"crash_dump_folder",
|
|
"show_time_cost",
|
|
"enable_metrics",
|
|
"smg_http_sidecar_port",
|
|
"enable_mfu_metrics",
|
|
"enable_metrics_for_all_schedulers",
|
|
"load_snapshot_publish_interval",
|
|
"tokenizer_metrics_custom_labels_header",
|
|
"tokenizer_metrics_allowed_custom_labels",
|
|
"extra_metric_labels",
|
|
"bucket_time_to_first_token",
|
|
"bucket_inter_token_latency",
|
|
"bucket_e2e_request_latency",
|
|
"prompt_tokens_buckets",
|
|
"generation_tokens_buckets",
|
|
"gc_warning_threshold_secs",
|
|
"decode_log_interval",
|
|
"enable_request_time_stats_logging",
|
|
"kv_events_config",
|
|
"load_publish_endpoint",
|
|
"enable_forward_pass_metrics",
|
|
"forward_pass_metrics_worker_id",
|
|
"forward_pass_metrics_ipc_name",
|
|
"enable_trace",
|
|
"trace_modules",
|
|
"otlp_traces_endpoint",
|
|
"export_metrics_to_file",
|
|
"export_metrics_to_file_dir",
|
|
"stat_loggers",
|
|
"constrained_json_whitespace_pattern",
|
|
"constrained_json_disable_any_whitespace",
|
|
"constrained_json_max_whitespace_cnt",
|
|
"attention_backend",
|
|
"decode_attention_backend",
|
|
"enable_lean_attention",
|
|
"prefill_attention_backend",
|
|
"sampling_backend",
|
|
"grammar_backend",
|
|
"radix_cache_backend",
|
|
"mm_attention_backend",
|
|
"fp8_gemm_runner_backend",
|
|
"fp4_gemm_runner_backend",
|
|
"bf16_gemm_backend",
|
|
"dsa_prefill_backend",
|
|
"dsv4_prefill_backend",
|
|
"dsa_decode_backend",
|
|
"dsa_paged_mqa_logits_backend",
|
|
"dsa_topk_backend",
|
|
"disable_flashinfer_autotune",
|
|
"flashinfer_autotune_skip_ops",
|
|
"mamba_backend",
|
|
"cuda_graph_config",
|
|
"cuda_graph_backend_decode",
|
|
"cuda_graph_backend_prefill",
|
|
"cuda_graph_max_bs_decode",
|
|
"cuda_graph_max_bs_prefill",
|
|
"cuda_graph_bs_decode",
|
|
"cuda_graph_bs_prefill",
|
|
"cuda_graph_prefill_max_context",
|
|
"cuda_graph_tc_compiler",
|
|
"disable_prefill_cuda_graph",
|
|
"disable_decode_cuda_graph",
|
|
"disable_cuda_graph",
|
|
"disable_cuda_graph_padding",
|
|
"enable_profile_cuda_graph",
|
|
"enable_cudagraph_gc",
|
|
"debug_cuda_graph",
|
|
"enable_layerwise_nvtx_marker",
|
|
"enable_nccl_nvls",
|
|
"enable_symm_mem",
|
|
"triton_attention_reduce_in_fp32",
|
|
"triton_attention_num_kv_splits",
|
|
"triton_attention_split_tile_size",
|
|
"flashinfer_mla_disable_ragged",
|
|
"enable_fused_qk_norm_rope",
|
|
"enable_precise_embedding_interpolation",
|
|
"enable_fused_moe_sum_all_reduce",
|
|
"enable_deepseek_v4_fp4_indexer",
|
|
"disable_custom_all_reduce",
|
|
"enable_mscclpp",
|
|
"enable_torch_symm_mem",
|
|
"enable_scattered_sconv",
|
|
"pre_warm_nccl",
|
|
"enable_quant_communications",
|
|
"enable_flashinfer_allreduce_fusion",
|
|
"enforce_disable_flashinfer_allreduce_fusion",
|
|
"flashinfer_allreduce_fusion_backend",
|
|
"enable_aiter_allreduce_fusion",
|
|
"enable_torch_compile",
|
|
"enable_torch_compile_debug_mode",
|
|
"torch_compile_max_bs",
|
|
"speculative_algorithm",
|
|
"uno_lora_path",
|
|
"speculative_draft_model_path",
|
|
"speculative_draft_model_revision",
|
|
"speculative_draft_load_format",
|
|
"speculative_num_steps",
|
|
"speculative_eagle_topk",
|
|
"speculative_num_draft_tokens",
|
|
"speculative_dflash_block_size",
|
|
"speculative_dspark_block_size",
|
|
"speculative_dspark_sps_table_path",
|
|
"speculative_dspark_confidence_sts_path",
|
|
"speculative_dspark_align_verify_tokens_to_graph_tier",
|
|
"speculative_accept_threshold_single",
|
|
"speculative_accept_threshold_acc",
|
|
"speculative_use_rejection_sampling",
|
|
"speculative_token_map",
|
|
"speculative_attention_mode",
|
|
"speculative_draft_attention_backend",
|
|
"speculative_dsa_topk_backend",
|
|
"speculative_draft_kv_cache_dtype",
|
|
"speculative_draft_window_size",
|
|
"speculative_moe_runner_backend",
|
|
"speculative_moe_a2a_backend",
|
|
"speculative_draft_model_quantization",
|
|
"_speculative_draft_quantization_explicitly_set",
|
|
"speculative_skip_dp_mlp_sync",
|
|
"enable_multi_layer_eagle",
|
|
"speculative_adaptive",
|
|
"speculative_adaptive_config",
|
|
"decoupled_spec_bind_endpoint",
|
|
"decoupled_spec_connect_endpoints",
|
|
"decoupled_spec_rank",
|
|
"decoupled_spec_role",
|
|
"spec_trace_dir",
|
|
"speculative_ngram_min_bfs_breadth",
|
|
"speculative_ngram_max_bfs_breadth",
|
|
"speculative_ngram_match_type",
|
|
"speculative_ngram_max_trie_depth",
|
|
"speculative_ngram_capacity",
|
|
"speculative_ngram_external_corpus_path",
|
|
"speculative_ngram_external_sam_budget",
|
|
"speculative_ngram_external_corpus_max_tokens",
|
|
"ep_size",
|
|
"moe_a2a_backend",
|
|
"enable_w4a4_mxfp4_megamoe",
|
|
"deepep_v2_mode",
|
|
"moe_runner_backend",
|
|
"flashinfer_mxfp4_moe_precision",
|
|
"deepep_mode",
|
|
"fuseep_mode",
|
|
"deepep_dispatcher_output_dtype",
|
|
"ep_num_redundant_experts",
|
|
"ep_dispatch_algorithm",
|
|
"init_expert_location",
|
|
"enable_eplb",
|
|
"eplb_algorithm",
|
|
"eplb_rebalance_num_iterations",
|
|
"eplb_rebalance_layers_per_chunk",
|
|
"eplb_min_rebalancing_utilization_threshold",
|
|
"expert_distribution_recorder_mode",
|
|
"expert_distribution_recorder_buffer_size",
|
|
"expert_balancedness_report_mode",
|
|
"deepep_config",
|
|
"moe_dense_tp_size",
|
|
"elastic_ep_backend",
|
|
"enable_elastic_expert_backup",
|
|
"mooncake_ib_device",
|
|
"enable_waterfill",
|
|
"ep_join_mode",
|
|
"ep_join_rank_offset",
|
|
"elastic_ep_initial_size",
|
|
"max_ep_size",
|
|
"elastic_ep_scale_timeout",
|
|
"elastic_ep_rejoin",
|
|
"disable_flashinfer_cutlass_moe_fp4_allgather",
|
|
"disable_shared_experts_fusion",
|
|
"enforce_shared_experts_fusion",
|
|
"max_mamba_cache_size",
|
|
"mamba_ssm_dtype",
|
|
"mamba_max_states_per_path",
|
|
"enable_mamba_cache_stochastic_rounding",
|
|
"mamba_cache_philox_rounds",
|
|
"mamba_full_memory_ratio",
|
|
"mamba_radix_cache_strategy",
|
|
"uses_mamba_radix_cache",
|
|
"mamba_track_interval",
|
|
"enable_int8_mamba_checkpoint",
|
|
"int8_mamba_ckpt_size",
|
|
"linear_attn_backend",
|
|
"linear_attn_decode_backend",
|
|
"linear_attn_prefill_backend",
|
|
"linear_attn_verify_backend",
|
|
"enable_linear_replayssm",
|
|
"linear_replayssm_cache_len",
|
|
"enable_linear_replayssm_spec",
|
|
"enable_hierarchical_cache",
|
|
"hicache_host_memory_mode",
|
|
"hicache_ratio",
|
|
"hicache_size",
|
|
"hicache_write_policy",
|
|
"hicache_io_backend",
|
|
"hicache_mem_layout",
|
|
"hicache_storage_backend",
|
|
"hicache_storage_prefetch_policy",
|
|
"hicache_storage_backend_extra_config",
|
|
"hicache_storage_prefetch_retry_poll_interval",
|
|
"hicache_storage_prefetch_retry_max_attempts",
|
|
"enable_unified_cache_external_linker",
|
|
"unified_cache_external_linker_backend",
|
|
"enable_hisparse",
|
|
"hisparse_config",
|
|
"enable_broadcast_mm_inputs_process",
|
|
"enable_prefix_mm_cache",
|
|
"mm_enable_dp_encoder",
|
|
"mm_process_config",
|
|
"mm_processor_worker_num",
|
|
"mm_io_worker_num",
|
|
"allowed_media_domains",
|
|
"media_url_max_file_size_mb",
|
|
"mm_preprocess_cache_size_mb",
|
|
"trust_mm_content_hashes",
|
|
"limit_mm_data_per_request",
|
|
"enable_mm_global_cache",
|
|
"image_processor_backend",
|
|
"mm_global_cache_backend",
|
|
"disable_fast_image_processor",
|
|
"mm_feature_transport",
|
|
"keep_mm_feature_on_device",
|
|
"enable_lora",
|
|
"enable_lora_overlap_loading",
|
|
"max_lora_rank",
|
|
"lora_target_modules",
|
|
"lora_paths",
|
|
"max_loaded_loras",
|
|
"max_loras_per_batch",
|
|
"lora_eviction_policy",
|
|
"lora_backend",
|
|
"max_lora_chunk_size",
|
|
"experts_shared_outer_loras",
|
|
"lora_use_virtual_experts",
|
|
"lora_strict_loading",
|
|
"lora_drain_wait_threshold",
|
|
"enable_two_batch_overlap",
|
|
"enable_single_batch_overlap",
|
|
"tbo_token_distribution_threshold",
|
|
"cpu_offload_gb",
|
|
"offload_group_size",
|
|
"offload_num_in_group",
|
|
"offload_prefetch_step",
|
|
"offload_mode",
|
|
"enable_lmcache",
|
|
"lmcache_config_file",
|
|
"enable_flexkv",
|
|
"flexkv_config_file",
|
|
"kt_weight_path",
|
|
"kt_method",
|
|
"kt_cpuinfer",
|
|
"kt_threadpool_count",
|
|
"kt_num_gpu_experts",
|
|
"kt_max_deferred_experts_per_token",
|
|
"dllm_algorithm",
|
|
"dllm_algorithm_config",
|
|
"dllm_fdfo",
|
|
"disaggregation_mode",
|
|
"disaggregation_transfer_backend",
|
|
"disaggregation_bootstrap_port",
|
|
"disaggregation_ib_device",
|
|
"disaggregation_decode_enable_radix_cache",
|
|
"disaggregation_decode_enable_offload_kvcache",
|
|
"disaggregation_decode_retraction_backup",
|
|
"num_reserved_decode_tokens",
|
|
"disaggregation_decode_extra_slots",
|
|
"disaggregation_decode_polling_interval",
|
|
"optimistic_prefill_attempts",
|
|
"encoder_only",
|
|
"language_only",
|
|
"language_model_only",
|
|
"encoder_transfer_backend",
|
|
"encoder_urls",
|
|
"encoder_bootstrap_port",
|
|
"encoder_register_urls",
|
|
"enable_adaptive_dispatch_to_encoder",
|
|
"enable_pdmux",
|
|
"pdmux_config_path",
|
|
"sm_group_num",
|
|
"startup_weight_load_mode",
|
|
"custom_weight_loader",
|
|
"weight_loader_disable_mmap",
|
|
"weight_loader_prefetch_checkpoints",
|
|
"weight_loader_prefetch_num_threads",
|
|
"weight_loader_drop_cache_after_load",
|
|
"remote_instance_weight_loader_seed_instance_ip",
|
|
"remote_instance_weight_loader_seed_instance_service_port",
|
|
"remote_instance_weight_loader_send_weights_group_ports",
|
|
"remote_instance_weight_loader_backend",
|
|
"remote_instance_weight_loader_start_seed_via_transfer_engine",
|
|
"engine_info_bootstrap_port",
|
|
"modelexpress_config",
|
|
"download_dir",
|
|
"model_checksum",
|
|
"delete_ckpt_after_loading",
|
|
"decrypted_config_file",
|
|
"decrypted_draft_config_file",
|
|
"checkpoint_engine_wait_weights_before_ready",
|
|
"enable_prefill_delayer",
|
|
"prefill_delayer_max_delay_passes",
|
|
"prefill_delayer_token_usage_low_watermark",
|
|
"prefill_delayer_forward_passes_buckets",
|
|
"prefill_delayer_wait_seconds_buckets",
|
|
"prefill_delayer_queue_min_ratio",
|
|
"prefill_delayer_max_delay_ms",
|
|
"min_free_slots_delay",
|
|
"enable_deterministic_inference",
|
|
"rl_on_policy_target",
|
|
"kv_canary",
|
|
"kv_canary_real_data",
|
|
"kv_canary_sweep_interval",
|
|
"enable_dynamic_batch_tokenizer",
|
|
"dynamic_batch_tokenizer_batch_size",
|
|
"dynamic_batch_tokenizer_batch_timeout",
|
|
"enable_tokenizer_batch_encode",
|
|
"disable_tokenizer_batch_decode",
|
|
"debug_tensor_dump_output_folder",
|
|
"debug_tensor_dump_layers",
|
|
"debug_tensor_dump_input_file",
|
|
"enable_memory_saver",
|
|
"enable_weights_cpu_backup",
|
|
"enable_draft_weights_cpu_backup",
|
|
"enable_custom_logit_processor",
|
|
"enable_return_hidden_states",
|
|
"return_hidden_states_mode",
|
|
"enable_return_routed_experts",
|
|
"enable_return_indexer_topk",
|
|
"sampling_mask_max_tokens",
|
|
"disable_outlines_disk_cache",
|
|
"enable_mis",
|
|
"weight_cache_mode",
|
|
"weight_cache_socket",
|
|
"weight_cache_timeout",
|
|
"forward_hooks",
|
|
"msprobe_dump_config",
|
|
)
|
|
# fmt: on
|