Cleanup server args (#14027)
This commit is contained in:
@@ -240,10 +240,6 @@ class ServerArgs:
|
|||||||
revision: Optional[str] = None
|
revision: Optional[str] = None
|
||||||
model_impl: str = "auto"
|
model_impl: str = "auto"
|
||||||
|
|
||||||
# Diffusion LLM
|
|
||||||
dllm_algorithm: Optional[str] = None
|
|
||||||
dllm_block_size: Optional[int] = None
|
|
||||||
|
|
||||||
# HTTP server
|
# HTTP server
|
||||||
host: str = "127.0.0.1"
|
host: str = "127.0.0.1"
|
||||||
port: int = 30000
|
port: int = 30000
|
||||||
@@ -397,7 +393,7 @@ class ServerArgs:
|
|||||||
speculative_attention_mode: str = "prefill"
|
speculative_attention_mode: str = "prefill"
|
||||||
speculative_moe_runner_backend: Optional[str] = None
|
speculative_moe_runner_backend: Optional[str] = None
|
||||||
|
|
||||||
# For ngram only
|
# Speculative decoding (ngram)
|
||||||
speculative_ngram_min_match_window_size: int = 1
|
speculative_ngram_min_match_window_size: int = 1
|
||||||
speculative_ngram_max_match_window_size: int = 12
|
speculative_ngram_max_match_window_size: int = 12
|
||||||
speculative_ngram_min_bfs_breadth: int = 1
|
speculative_ngram_min_bfs_breadth: int = 1
|
||||||
@@ -457,6 +453,10 @@ class ServerArgs:
|
|||||||
kt_num_gpu_experts: Optional[int] = None
|
kt_num_gpu_experts: Optional[int] = None
|
||||||
kt_max_deferred_experts_per_token: Optional[int] = None
|
kt_max_deferred_experts_per_token: Optional[int] = None
|
||||||
|
|
||||||
|
# Diffusion LLM
|
||||||
|
dllm_algorithm: Optional[str] = None
|
||||||
|
dllm_block_size: Optional[int] = None
|
||||||
|
|
||||||
# Double Sparsity
|
# Double Sparsity
|
||||||
enable_double_sparsity: bool = False
|
enable_double_sparsity: bool = False
|
||||||
ds_channel_config_path: Optional[str] = None
|
ds_channel_config_path: Optional[str] = None
|
||||||
@@ -2124,20 +2124,6 @@ class ServerArgs:
|
|||||||
"implementation.\n",
|
"implementation.\n",
|
||||||
)
|
)
|
||||||
|
|
||||||
# Diffusion LLM
|
|
||||||
parser.add_argument(
|
|
||||||
"--dllm-algorithm",
|
|
||||||
type=str,
|
|
||||||
default=ServerArgs.dllm_algorithm,
|
|
||||||
help="The diffusion LLM algorithm.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"--dllm-block-size",
|
|
||||||
type=int,
|
|
||||||
default=ServerArgs.dllm_block_size,
|
|
||||||
help="The number of tokens processed in each iteration of the block diffusion LLM.",
|
|
||||||
)
|
|
||||||
|
|
||||||
# HTTP server
|
# HTTP server
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--host",
|
"--host",
|
||||||
@@ -2973,7 +2959,8 @@ class ServerArgs:
|
|||||||
default=ServerArgs.speculative_moe_runner_backend,
|
default=ServerArgs.speculative_moe_runner_backend,
|
||||||
help="Choose the runner backend for MoE in speculative decoding.",
|
help="Choose the runner backend for MoE in speculative decoding.",
|
||||||
)
|
)
|
||||||
# Ngram speculative decoding
|
|
||||||
|
# Speculative decoding (ngram)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--speculative-ngram-min-match-window-size",
|
"--speculative-ngram-min-match-window-size",
|
||||||
type=int,
|
type=int,
|
||||||
@@ -3282,6 +3269,21 @@ class ServerArgs:
|
|||||||
default=ServerArgs.kt_max_deferred_experts_per_token,
|
default=ServerArgs.kt_max_deferred_experts_per_token,
|
||||||
help="[ktransformers parameter] Maximum number of experts deferred to CPU per token. All MoE layers except the final one use this value; the final layer always uses 0.",
|
help="[ktransformers parameter] Maximum number of experts deferred to CPU per token. All MoE layers except the final one use this value; the final layer always uses 0.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Diffusion LLM
|
||||||
|
parser.add_argument(
|
||||||
|
"--dllm-algorithm",
|
||||||
|
type=str,
|
||||||
|
default=ServerArgs.dllm_algorithm,
|
||||||
|
help="The diffusion LLM algorithm.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--dllm-block-size",
|
||||||
|
type=int,
|
||||||
|
default=ServerArgs.dllm_block_size,
|
||||||
|
help="The number of tokens processed in each iteration of the block diffusion LLM.",
|
||||||
|
)
|
||||||
|
|
||||||
# Double Sparsity
|
# Double Sparsity
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--enable-double-sparsity",
|
"--enable-double-sparsity",
|
||||||
|
|||||||
Reference in New Issue
Block a user