feat: expose consistent_hashing policy in Python router CLI args (#17972)
This commit is contained in:
@@ -246,7 +246,15 @@ class RouterArgs:
|
|||||||
f"--{prefix}policy",
|
f"--{prefix}policy",
|
||||||
type=str,
|
type=str,
|
||||||
default=RouterArgs.policy,
|
default=RouterArgs.policy,
|
||||||
choices=["random", "round_robin", "cache_aware", "power_of_two", "manual"],
|
choices=[
|
||||||
|
"random",
|
||||||
|
"round_robin",
|
||||||
|
"cache_aware",
|
||||||
|
"power_of_two",
|
||||||
|
"manual",
|
||||||
|
"consistent_hashing",
|
||||||
|
"prefix_hash",
|
||||||
|
],
|
||||||
help="Load balancing policy to use. In PD mode, this is used for both prefill and decode unless overridden",
|
help="Load balancing policy to use. In PD mode, this is used for both prefill and decode unless overridden",
|
||||||
)
|
)
|
||||||
routing_group.add_argument(
|
routing_group.add_argument(
|
||||||
@@ -260,6 +268,8 @@ class RouterArgs:
|
|||||||
"power_of_two",
|
"power_of_two",
|
||||||
"manual",
|
"manual",
|
||||||
"bucket",
|
"bucket",
|
||||||
|
"consistent_hashing",
|
||||||
|
"prefix_hash",
|
||||||
],
|
],
|
||||||
help="Specific policy for prefill nodes in PD mode. If not specified, uses the main policy",
|
help="Specific policy for prefill nodes in PD mode. If not specified, uses the main policy",
|
||||||
)
|
)
|
||||||
@@ -267,7 +277,15 @@ class RouterArgs:
|
|||||||
f"--{prefix}decode-policy",
|
f"--{prefix}decode-policy",
|
||||||
type=str,
|
type=str,
|
||||||
default=None,
|
default=None,
|
||||||
choices=["random", "round_robin", "cache_aware", "power_of_two", "manual"],
|
choices=[
|
||||||
|
"random",
|
||||||
|
"round_robin",
|
||||||
|
"cache_aware",
|
||||||
|
"power_of_two",
|
||||||
|
"manual",
|
||||||
|
"consistent_hashing",
|
||||||
|
"prefix_hash",
|
||||||
|
],
|
||||||
help="Specific policy for decode nodes in PD mode. If not specified, uses the main policy",
|
help="Specific policy for decode nodes in PD mode. If not specified, uses the main policy",
|
||||||
)
|
)
|
||||||
routing_group.add_argument(
|
routing_group.add_argument(
|
||||||
|
|||||||
Reference in New Issue
Block a user