[2/n] lora - Shared outer experts and support qwen3_30b_a3b_instruct (#21466)
Co-authored-by: Baizhou Zhang <sobereddiezhang@gmail.com>
This commit is contained in:
co-authored by
Baizhou Zhang
parent
f4505e2ee3
commit
3c91ebdf55
@@ -464,6 +464,7 @@ class ServerArgs:
|
||||
lora_eviction_policy: str = "lru"
|
||||
lora_backend: str = "csgmv"
|
||||
max_lora_chunk_size: Optional[int] = 16
|
||||
experts_shared_outer_loras: Optional[bool] = None
|
||||
|
||||
# Kernel backend
|
||||
attention_backend: Optional[str] = None
|
||||
@@ -4595,6 +4596,14 @@ class ServerArgs:
|
||||
choices=[16, 32, 64, 128],
|
||||
help="Maximum chunk size for the ChunkedSGMV LoRA backend. Only used when --lora-backend is 'csgmv'. Choosing a larger value might improve performance.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--experts-shared-outer-loras",
|
||||
default=ServerArgs.experts_shared_outer_loras,
|
||||
action="store_true",
|
||||
help="Force shared outer LoRA mode for MoE models. "
|
||||
"When set, w1/w3 lora_A and w2 lora_B are shared across experts "
|
||||
"(expert_dim=1). By default this is auto-detected from adapter weights.",
|
||||
)
|
||||
|
||||
# Kernel backend
|
||||
parser.add_argument(
|
||||
|
||||
Reference in New Issue
Block a user