diff --git a/python/sglang/srt/server_args.py b/python/sglang/srt/server_args.py index 9e2e6b646..3cf1d3410 100644 --- a/python/sglang/srt/server_args.py +++ b/python/sglang/srt/server_args.py @@ -132,30 +132,6 @@ LOAD_FORMAT_CHOICES = [ # TODO: this list should likely contain only methods that support online quantization, or that support using custom quantization classes compatible with a given `quant_method` in config.json. # Some of the choices here do NOT support online quantization. -# Attention backends whose kernels read the chunked prefix-cache layout. -# Out-of-tree platforms may extend this list (via -# add_chunked_prefix_cache_attention_backend) before ServerArgs construction; -# the chunked-prefix gate is evaluated during resolution. -CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [ - "flashinfer", - "fa3", - "fa4", - "flashmla", - "cutedsl_mla", - "cutlass_mla", - "trtllm_mla", - "tokenspeed_mla", -] - - -def add_chunked_prefix_cache_attention_backend(backend_name): - if backend_name not in CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS: - CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append(backend_name) - logger.info( - f"Added {backend_name} to CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS." - ) - - QUANTIZATION_CHOICES = [ "awq", "fp8", # MOE + linear online quantization. @@ -194,9 +170,6 @@ QUANTIZATION_CHOICES = [ "humming", ] - -SPECULATIVE_DRAFT_MODEL_QUANTIZATION_CHOICES = QUANTIZATION_CHOICES - ATTENTION_BACKEND_CHOICES = [ # Common "triton", @@ -227,6 +200,21 @@ ATTENTION_BACKEND_CHOICES = [ "intel_xpu", ] +# Attention backends whose kernels read the chunked prefix-cache layout. +# Out-of-tree platforms may extend this list (via +# add_chunked_prefix_cache_attention_backend) before ServerArgs construction; +# the chunked-prefix gate is evaluated during resolution. +CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [ + "flashinfer", + "fa3", + "fa4", + "flashmla", + "cutedsl_mla", + "cutlass_mla", + "trtllm_mla", + "tokenspeed_mla", +] + DETERMINISTIC_ATTENTION_BACKEND_CHOICES = [ "ascend", "fa3", @@ -372,6 +360,10 @@ def add_attention_backend_choices(choices): ATTENTION_BACKEND_CHOICES.extend(choices) +def add_chunked_prefix_cache_attention_backend(backend_name): + CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append(backend_name) + + def add_deterministic_attention_backend_choices(choices): DETERMINISTIC_ATTENTION_BACKEND_CHOICES.extend(choices) @@ -2140,7 +2132,7 @@ class ServerArgs: Optional[str], Arg( help="The quantization method for speculative model.", - choices=SPECULATIVE_DRAFT_MODEL_QUANTIZATION_CHOICES, + choices=QUANTIZATION_CHOICES, ), NS("spec"), ] = None @@ -2203,6 +2195,7 @@ class ServerArgs: NS("spec"), ] = None + # ------------------------------------------------------------------------- # Speculative decoding (ngram) # ------------------------------------------------------------------------- speculative_ngram_min_bfs_breadth: A[ @@ -3336,6 +3329,9 @@ class ServerArgs: NS("exec.features"), ] = False + # ------------------------------------------------------------------------- + # Weight cache + # ------------------------------------------------------------------------- weight_cache_mode: A[ str, Arg(