[Feature] Enable return routed experts (#12162)
Co-authored-by: yizhang2077 <1109276519@qq.com> Co-authored-by: Liangsheng Yin <lsyincs@gmail.com>
This commit is contained in:
co-authored by
yizhang2077
Liangsheng Yin
parent
8fe3e37468
commit
bed301a5ac
@@ -573,6 +573,7 @@ class ServerArgs:
|
||||
disable_fast_image_processor: bool = False
|
||||
keep_mm_feature_on_device: bool = False
|
||||
enable_return_hidden_states: bool = False
|
||||
enable_return_routed_experts: bool = False
|
||||
scheduler_recv_interval: int = 1
|
||||
numa_node: Optional[List[int]] = None
|
||||
enable_deterministic_inference: bool = False
|
||||
@@ -4101,6 +4102,11 @@ class ServerArgs:
|
||||
action="store_true",
|
||||
help="Enable returning hidden states with responses.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-return-routed-experts",
|
||||
action="store_true",
|
||||
help="Enable returning routed experts of each layer with responses.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--scheduler-recv-interval",
|
||||
type=int,
|
||||
|
||||
Reference in New Issue
Block a user