[Feat] sgl-router: Add --dp-rank-policy round_robin for --dp-aware (#43511)
This commit is contained in:
@@ -283,9 +283,10 @@ client sent. The router picks the first of these that applies:
|
||||
1. A hash of the sticky routing key or session id, so a conversation keeps
|
||||
its rank and router replicas agree.
|
||||
2. The rank with the deepest cached prefix in the local KV tree.
|
||||
3. The rank with the fewest requests this router has in flight on it.
|
||||
3. The rank with the fewest requests this router has in flight on it, or with
|
||||
`--dp-rank-policy round_robin`, the worker's ranks in turn.
|
||||
|
||||
In PD mode, decode is ranked by load only. The bootstrap room satisfies
|
||||
In PD mode, decode is ranked by step 3 only. The bootstrap room satisfies
|
||||
`room % prefill_dp_size == prefill_rank`, which is how a decode engine finds
|
||||
the prefill rank.
|
||||
|
||||
|
||||
@@ -14,12 +14,12 @@ use crate::config::{
|
||||
default_cb_cool_down, default_host, default_port, default_proxy_request_timeout_secs,
|
||||
default_shutdown_drain_secs, default_stale_request_timeout_secs, resolve_mode, AffinityConfig,
|
||||
AffinityMode, BalancedBy, CacheAwareConfig, CachePrefixProvider, ChatRoutingKind,
|
||||
CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, EligibilityConfig,
|
||||
FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig, K8sDiscoveryMode,
|
||||
KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig, PolicyKind, ProxyConfig,
|
||||
ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind,
|
||||
TokenizerBackend, TokenizerConfig, DEFAULT_FUSE, DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS,
|
||||
DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
|
||||
CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, DpRankPolicy,
|
||||
EligibilityConfig, FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig,
|
||||
K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig,
|
||||
PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig,
|
||||
StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, DEFAULT_FUSE,
|
||||
DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
|
||||
};
|
||||
use crate::policies_reorg::admission::AdmissionLimits;
|
||||
use crate::policies_reorg::factory::AffinitySpec;
|
||||
@@ -245,6 +245,11 @@ pub struct RoutingArgs {
|
||||
#[arg(long)]
|
||||
pub dp_aware: bool,
|
||||
|
||||
/// How --dp-aware picks a rank that no routing key or cached prefix decides,
|
||||
/// which in PD mode is every decode rank. Defaults to least_in_flight.
|
||||
#[arg(long, value_enum, requires = "dp_aware")]
|
||||
pub dp_rank_policy: Option<DpRankPolicy>,
|
||||
|
||||
/// Static bucket configuration JSON; the reorg schema with --chat-routing reorg.
|
||||
/// Omit to use the global candidate domain (default buckets with reorg).
|
||||
#[arg(long)]
|
||||
@@ -616,6 +621,7 @@ impl Cli {
|
||||
policy: self.routing.policy,
|
||||
decode_policy: self.routing.decode_policy,
|
||||
dp_aware: self.routing.dp_aware,
|
||||
dp_rank_policy: self.routing.dp_rank_policy.unwrap_or_default(),
|
||||
bucket_config,
|
||||
reorg_buckets,
|
||||
reorg_admission,
|
||||
|
||||
@@ -262,6 +262,7 @@ mod tests {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: DecodePolicyKind::PowerOfTwo,
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -152,6 +152,19 @@ pub enum DecodePolicyKind {
|
||||
LegacyHostAffinity,
|
||||
}
|
||||
|
||||
/// How `--dp-aware` picks a DP rank that no routing key or cached prefix decides,
|
||||
/// which in PD mode is every decode rank.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)]
|
||||
pub enum DpRankPolicy {
|
||||
/// The rank with the fewest requests this router has in flight on it.
|
||||
#[default]
|
||||
#[value(name = "least_in_flight")]
|
||||
LeastInFlight,
|
||||
/// The worker's ranks in turn.
|
||||
#[value(name = "round_robin")]
|
||||
RoundRobin,
|
||||
}
|
||||
|
||||
/// Role served by a static bucket.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
@@ -372,6 +385,8 @@ pub struct ModelConfig {
|
||||
pub decode_policy: DecodePolicyKind,
|
||||
/// Send a DP rank as `X-Data-Parallel-Rank`; see [`crate::policies::dp_rank`].
|
||||
pub dp_aware: bool,
|
||||
/// Rank choice under `dp_aware` when no routing key or cached prefix decides it.
|
||||
pub dp_rank_policy: DpRankPolicy,
|
||||
/// Optional static bucket configuration. `None` uses the global domain.
|
||||
pub bucket_config: Option<BucketConfig>,
|
||||
/// Reorg `--bucket-config`; `None` builds the default plain and P/D buckets.
|
||||
|
||||
@@ -5,8 +5,10 @@
|
||||
//!
|
||||
//! First match wins: a sticky/session key hashes to a fixed rank, so router
|
||||
//! replicas agree without shared state; else the rank with the deepest cached
|
||||
//! prefix; else the rank with the fewest router in-flight requests.
|
||||
//! prefix; else [`DpRankPolicy`] picks: the fewest router in-flight requests,
|
||||
//! or the worker's ranks in turn.
|
||||
|
||||
use crate::config::DpRankPolicy;
|
||||
use crate::workers::Worker;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
@@ -15,6 +17,7 @@ pub fn select_dp_rank(
|
||||
worker: &Worker,
|
||||
affinity_key: Option<&str>,
|
||||
prefix_depths: &[(u32, usize)],
|
||||
policy: DpRankPolicy,
|
||||
) -> Option<u32> {
|
||||
let ranks = worker.dp_ranks();
|
||||
if ranks <= 1 {
|
||||
@@ -37,6 +40,9 @@ pub fn select_dp_rank(
|
||||
.collect(),
|
||||
None => (0..ranks).collect(),
|
||||
};
|
||||
if policy == DpRankPolicy::RoundRobin {
|
||||
return Some(candidates[worker.next_dp_rank_turn() % candidates.len()]);
|
||||
}
|
||||
// Random start so equally loaded ranks share traffic.
|
||||
let start = rand::random::<usize>() % candidates.len();
|
||||
candidates[start..]
|
||||
@@ -70,14 +76,22 @@ mod tests {
|
||||
Worker::with_cb_config(spec, None, profile)
|
||||
}
|
||||
|
||||
const LEAST: DpRankPolicy = DpRankPolicy::LeastInFlight;
|
||||
|
||||
#[test]
|
||||
fn affinity_then_prefix_then_load() {
|
||||
let w = worker(4);
|
||||
assert_eq!(select_dp_rank(&worker(1), Some("k"), &[]), None);
|
||||
assert_eq!(select_dp_rank(&worker(1), Some("k"), &[], LEAST), None);
|
||||
// Pinned: replicas and releases must agree on the mapping.
|
||||
assert_eq!(select_dp_rank(&w, Some("session-42"), &[(0, 9)]), Some(1));
|
||||
assert_eq!(select_dp_rank(&w, None, &[(1, 2), (3, 5), (9, 9)]), Some(3));
|
||||
assert_eq!(
|
||||
select_dp_rank(&w, Some("session-42"), &[(0, 9)], LEAST),
|
||||
Some(1)
|
||||
);
|
||||
assert_eq!(
|
||||
select_dp_rank(&w, None, &[(1, 2), (3, 5), (9, 9)], LEAST),
|
||||
Some(3)
|
||||
);
|
||||
let _busy = [0, 1, 3].map(|rank| w.dp_rank_guard(rank));
|
||||
assert_eq!(select_dp_rank(&w, None, &[]), Some(2));
|
||||
assert_eq!(select_dp_rank(&w, None, &[], LEAST), Some(2));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -342,6 +342,7 @@ mod tests {
|
||||
policy,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -217,6 +217,7 @@ impl AppContext {
|
||||
policy: crate::config::PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -80,7 +80,7 @@ pub(super) async fn forward_request(
|
||||
let decode_rank = decode
|
||||
.as_deref()
|
||||
.filter(|_| dp_aware)
|
||||
.and_then(|decode| select_dp_rank(decode, None, &[]));
|
||||
.and_then(|decode| select_dp_rank(decode, None, &[], ctx.config.model.dp_rank_policy));
|
||||
|
||||
// Track worker occupancy and the prompt's contribution to active load.
|
||||
// A retry keeps the request's stale deadline rather than starting a new one.
|
||||
@@ -271,7 +271,7 @@ fn prompt_dp_rank(
|
||||
}
|
||||
_ => Vec::new(),
|
||||
};
|
||||
select_dp_rank(worker, key, &prefix_depths)
|
||||
select_dp_rank(worker, key, &prefix_depths, model.dp_rank_policy)
|
||||
}
|
||||
|
||||
/// Under `--dp-aware` the router owns the rank: the header pins it for chat, and
|
||||
|
||||
@@ -55,6 +55,7 @@ mod tests {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -129,6 +129,7 @@ mod tests {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -326,6 +326,7 @@ mod tests {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -637,6 +637,7 @@ mod tests {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -228,6 +228,8 @@ pub struct Worker {
|
||||
services: RwLock<BTreeSet<String>>,
|
||||
/// Router in-flight requests per DP rank; one slot per rank.
|
||||
dp_rank_inflight: Arc<[AtomicUsize]>,
|
||||
/// Round-robin position over this worker's DP ranks.
|
||||
dp_rank_turn: AtomicUsize,
|
||||
openai: Option<Arc<OpenAiSettings>>,
|
||||
}
|
||||
|
||||
@@ -271,6 +273,7 @@ impl Worker {
|
||||
version_group: spec.version_group,
|
||||
services: RwLock::new(spec.services),
|
||||
dp_rank_inflight: (0..dp_ranks.max(1)).map(|_| AtomicUsize::new(0)).collect(),
|
||||
dp_rank_turn: AtomicUsize::new(0),
|
||||
openai,
|
||||
}
|
||||
}
|
||||
@@ -364,6 +367,11 @@ impl Worker {
|
||||
self.dp_rank_inflight[rank as usize].load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
/// Advances the round-robin DP-rank position and returns the previous one.
|
||||
pub fn next_dp_rank_turn(&self) -> usize {
|
||||
self.dp_rank_turn.fetch_add(1, Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn dp_rank_guard(&self, rank: u32) -> DpRankGuard {
|
||||
let rank = rank as usize;
|
||||
self.dp_rank_inflight[rank].fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
@@ -136,6 +136,7 @@ async fn static_urls_pd_role_resolved_end_to_end() {
|
||||
policy: sgl_router::config::PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -117,6 +117,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -66,6 +66,7 @@ fn build_app_context(
|
||||
policy,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: Some(bucket_config),
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -45,6 +45,7 @@ fn config_for(_worker_url: &str) -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -50,6 +50,7 @@ pub fn config() -> Config {
|
||||
policy: PolicyKind::CacheAware,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -9,7 +9,7 @@ use std::time::{Duration, Instant};
|
||||
use axum::body::Body;
|
||||
use axum::http::Request;
|
||||
use serde_json::json;
|
||||
use sgl_router::config::{Config, PolicyKind, StickyConfig, StickyFallbackKind};
|
||||
use sgl_router::config::{Config, DpRankPolicy, PolicyKind, StickyConfig, StickyFallbackKind};
|
||||
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
|
||||
use sgl_router::policies::factory::build_registry;
|
||||
use sgl_router::policies::request_tokens_for;
|
||||
@@ -167,6 +167,26 @@ async fn pd_room_maps_decode_to_the_prefill_rank() {
|
||||
assert_eq!(room % 4, rank.parse::<u64>().unwrap());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn round_robin_gives_decode_ranks_turns() {
|
||||
let (prefill, decode) = (
|
||||
MockWorker::start(vec![]).await,
|
||||
MockWorker::start(vec![]).await,
|
||||
);
|
||||
let workers = [
|
||||
(&prefill, WorkerMode::Prefill, 4),
|
||||
(&decode, WorkerMode::Decode, 4),
|
||||
];
|
||||
let mut cfg = sticky_config();
|
||||
cfg.model.dp_rank_policy = DpRankPolicy::RoundRobin;
|
||||
let app = router(cfg, &workers, Default::default());
|
||||
let mut ranks = Vec::new();
|
||||
for _ in 0..8 {
|
||||
ranks.push(send(&app, &decode, &[(KEY, "conv-pd")]).await.unwrap());
|
||||
}
|
||||
assert_eq!(ranks, ["0", "1", "2", "3", "0", "1", "2", "3"]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn any_policy_picks_the_rank_with_the_deepest_prefix() {
|
||||
let worker = MockWorker::start(vec![]).await;
|
||||
|
||||
@@ -54,6 +54,7 @@ async fn errors_with_one_dead_worker(max_attempts: u32) -> usize {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -58,6 +58,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc<AppContext> {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -36,6 +36,7 @@ async fn forwards_whitelisted_headers_strips_others() {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -51,6 +51,7 @@ fn config() -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -50,6 +50,7 @@ fn config() -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -138,6 +138,7 @@ fn config() -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -44,6 +44,7 @@ fn config(max_attempts: u32) -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -49,6 +49,7 @@ fn config() -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -154,6 +154,7 @@ fn config(policy: PolicyKind) -> Config {
|
||||
policy,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -59,6 +59,7 @@ fn config() -> Config {
|
||||
policy: PolicyKind::Sticky,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -47,6 +47,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc<AppContext
|
||||
policy: PolicyKind::Sticky,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
@@ -44,6 +44,7 @@ fn config(_worker_url: &str) -> Config {
|
||||
policy: PolicyKind::RoundRobin,
|
||||
decode_policy: Default::default(),
|
||||
dp_aware: false,
|
||||
dp_rank_policy: Default::default(),
|
||||
bucket_config: None,
|
||||
reorg_buckets: None,
|
||||
reorg_admission: Default::default(),
|
||||
|
||||
Reference in New Issue
Block a user