[Refactor] Tidy server_args.py section grouping and drop unused alias (#32496)
This commit is contained in:
@@ -132,30 +132,6 @@ LOAD_FORMAT_CHOICES = [
|
|||||||
|
|
||||||
# TODO: this list should likely contain only methods that support online quantization, or that support using custom quantization classes compatible with a given `quant_method` in config.json.
|
# TODO: this list should likely contain only methods that support online quantization, or that support using custom quantization classes compatible with a given `quant_method` in config.json.
|
||||||
# Some of the choices here do NOT support online quantization.
|
# Some of the choices here do NOT support online quantization.
|
||||||
# Attention backends whose kernels read the chunked prefix-cache layout.
|
|
||||||
# Out-of-tree platforms may extend this list (via
|
|
||||||
# add_chunked_prefix_cache_attention_backend) before ServerArgs construction;
|
|
||||||
# the chunked-prefix gate is evaluated during resolution.
|
|
||||||
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [
|
|
||||||
"flashinfer",
|
|
||||||
"fa3",
|
|
||||||
"fa4",
|
|
||||||
"flashmla",
|
|
||||||
"cutedsl_mla",
|
|
||||||
"cutlass_mla",
|
|
||||||
"trtllm_mla",
|
|
||||||
"tokenspeed_mla",
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def add_chunked_prefix_cache_attention_backend(backend_name):
|
|
||||||
if backend_name not in CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS:
|
|
||||||
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append(backend_name)
|
|
||||||
logger.info(
|
|
||||||
f"Added {backend_name} to CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS."
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
QUANTIZATION_CHOICES = [
|
QUANTIZATION_CHOICES = [
|
||||||
"awq",
|
"awq",
|
||||||
"fp8", # MOE + linear online quantization.
|
"fp8", # MOE + linear online quantization.
|
||||||
@@ -194,9 +170,6 @@ QUANTIZATION_CHOICES = [
|
|||||||
"humming",
|
"humming",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
SPECULATIVE_DRAFT_MODEL_QUANTIZATION_CHOICES = QUANTIZATION_CHOICES
|
|
||||||
|
|
||||||
ATTENTION_BACKEND_CHOICES = [
|
ATTENTION_BACKEND_CHOICES = [
|
||||||
# Common
|
# Common
|
||||||
"triton",
|
"triton",
|
||||||
@@ -227,6 +200,21 @@ ATTENTION_BACKEND_CHOICES = [
|
|||||||
"intel_xpu",
|
"intel_xpu",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
# Attention backends whose kernels read the chunked prefix-cache layout.
|
||||||
|
# Out-of-tree platforms may extend this list (via
|
||||||
|
# add_chunked_prefix_cache_attention_backend) before ServerArgs construction;
|
||||||
|
# the chunked-prefix gate is evaluated during resolution.
|
||||||
|
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS = [
|
||||||
|
"flashinfer",
|
||||||
|
"fa3",
|
||||||
|
"fa4",
|
||||||
|
"flashmla",
|
||||||
|
"cutedsl_mla",
|
||||||
|
"cutlass_mla",
|
||||||
|
"trtllm_mla",
|
||||||
|
"tokenspeed_mla",
|
||||||
|
]
|
||||||
|
|
||||||
DETERMINISTIC_ATTENTION_BACKEND_CHOICES = [
|
DETERMINISTIC_ATTENTION_BACKEND_CHOICES = [
|
||||||
"ascend",
|
"ascend",
|
||||||
"fa3",
|
"fa3",
|
||||||
@@ -372,6 +360,10 @@ def add_attention_backend_choices(choices):
|
|||||||
ATTENTION_BACKEND_CHOICES.extend(choices)
|
ATTENTION_BACKEND_CHOICES.extend(choices)
|
||||||
|
|
||||||
|
|
||||||
|
def add_chunked_prefix_cache_attention_backend(backend_name):
|
||||||
|
CHUNKED_PREFIX_CACHE_SUPPORTED_ATTENTION_BACKENDS.append(backend_name)
|
||||||
|
|
||||||
|
|
||||||
def add_deterministic_attention_backend_choices(choices):
|
def add_deterministic_attention_backend_choices(choices):
|
||||||
DETERMINISTIC_ATTENTION_BACKEND_CHOICES.extend(choices)
|
DETERMINISTIC_ATTENTION_BACKEND_CHOICES.extend(choices)
|
||||||
|
|
||||||
@@ -2140,7 +2132,7 @@ class ServerArgs:
|
|||||||
Optional[str],
|
Optional[str],
|
||||||
Arg(
|
Arg(
|
||||||
help="The quantization method for speculative model.",
|
help="The quantization method for speculative model.",
|
||||||
choices=SPECULATIVE_DRAFT_MODEL_QUANTIZATION_CHOICES,
|
choices=QUANTIZATION_CHOICES,
|
||||||
),
|
),
|
||||||
NS("spec"),
|
NS("spec"),
|
||||||
] = None
|
] = None
|
||||||
@@ -2203,6 +2195,7 @@ class ServerArgs:
|
|||||||
NS("spec"),
|
NS("spec"),
|
||||||
] = None
|
] = None
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------------
|
||||||
# Speculative decoding (ngram)
|
# Speculative decoding (ngram)
|
||||||
# -------------------------------------------------------------------------
|
# -------------------------------------------------------------------------
|
||||||
speculative_ngram_min_bfs_breadth: A[
|
speculative_ngram_min_bfs_breadth: A[
|
||||||
@@ -3336,6 +3329,9 @@ class ServerArgs:
|
|||||||
NS("exec.features"),
|
NS("exec.features"),
|
||||||
] = False
|
] = False
|
||||||
|
|
||||||
|
# -------------------------------------------------------------------------
|
||||||
|
# Weight cache
|
||||||
|
# -------------------------------------------------------------------------
|
||||||
weight_cache_mode: A[
|
weight_cache_mode: A[
|
||||||
str,
|
str,
|
||||||
Arg(
|
Arg(
|
||||||
|
|||||||
Reference in New Issue
Block a user