[Feat] sgl-router: Add --dp-rank-policy round_robin for --dp-aware (#43511)

This commit is contained in:
AMD-yanfeiwang
2026-10-15 12:03:01 +08:00
committed by GitHub
parent c92108bb93
commit 08f15e76bb
31 changed files with 104 additions and 16 deletions
+3 -2
View File
@@ -283,9 +283,10 @@ client sent. The router picks the first of these that applies:
1. A hash of the sticky routing key or session id, so a conversation keeps
its rank and router replicas agree.
2. The rank with the deepest cached prefix in the local KV tree.
3. The rank with the fewest requests this router has in flight on it.
3. The rank with the fewest requests this router has in flight on it, or with
`--dp-rank-policy round_robin`, the worker's ranks in turn.
In PD mode, decode is ranked by load only. The bootstrap room satisfies
In PD mode, decode is ranked by step 3 only. The bootstrap room satisfies
`room % prefill_dp_size == prefill_rank`, which is how a decode engine finds
the prefill rank.
+12 -6
View File
@@ -14,12 +14,12 @@ use crate::config::{
default_cb_cool_down, default_host, default_port, default_proxy_request_timeout_secs,
default_shutdown_drain_secs, default_stale_request_timeout_secs, resolve_mode, AffinityConfig,
AffinityMode, BalancedBy, CacheAwareConfig, CachePrefixProvider, ChatRoutingKind,
CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, EligibilityConfig,
FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig, K8sDiscoveryMode,
KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig, PolicyKind, ProxyConfig,
ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind,
TokenizerBackend, TokenizerConfig, DEFAULT_FUSE, DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS,
DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, DpRankPolicy,
EligibilityConfig, FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig,
K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig,
PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig,
StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, DEFAULT_FUSE,
DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS,
};
use crate::policies_reorg::admission::AdmissionLimits;
use crate::policies_reorg::factory::AffinitySpec;
@@ -245,6 +245,11 @@ pub struct RoutingArgs {
#[arg(long)]
pub dp_aware: bool,
/// How --dp-aware picks a rank that no routing key or cached prefix decides,
/// which in PD mode is every decode rank. Defaults to least_in_flight.
#[arg(long, value_enum, requires = "dp_aware")]
pub dp_rank_policy: Option<DpRankPolicy>,
/// Static bucket configuration JSON; the reorg schema with --chat-routing reorg.
/// Omit to use the global candidate domain (default buckets with reorg).
#[arg(long)]
@@ -616,6 +621,7 @@ impl Cli {
policy: self.routing.policy,
decode_policy: self.routing.decode_policy,
dp_aware: self.routing.dp_aware,
dp_rank_policy: self.routing.dp_rank_policy.unwrap_or_default(),
bucket_config,
reorg_buckets,
reorg_admission,
@@ -262,6 +262,7 @@ mod tests {
policy: PolicyKind::RoundRobin,
decode_policy: DecodePolicyKind::PowerOfTwo,
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -152,6 +152,19 @@ pub enum DecodePolicyKind {
LegacyHostAffinity,
}
/// How `--dp-aware` picks a DP rank that no routing key or cached prefix decides,
/// which in PD mode is every decode rank.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)]
pub enum DpRankPolicy {
/// The rank with the fewest requests this router has in flight on it.
#[default]
#[value(name = "least_in_flight")]
LeastInFlight,
/// The worker's ranks in turn.
#[value(name = "round_robin")]
RoundRobin,
}
/// Role served by a static bucket.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)]
#[serde(rename_all = "snake_case")]
@@ -372,6 +385,8 @@ pub struct ModelConfig {
pub decode_policy: DecodePolicyKind,
/// Send a DP rank as `X-Data-Parallel-Rank`; see [`crate::policies::dp_rank`].
pub dp_aware: bool,
/// Rank choice under `dp_aware` when no routing key or cached prefix decides it.
pub dp_rank_policy: DpRankPolicy,
/// Optional static bucket configuration. `None` uses the global domain.
pub bucket_config: Option<BucketConfig>,
/// Reorg `--bucket-config`; `None` builds the default plain and P/D buckets.
@@ -5,8 +5,10 @@
//!
//! First match wins: a sticky/session key hashes to a fixed rank, so router
//! replicas agree without shared state; else the rank with the deepest cached
//! prefix; else the rank with the fewest router in-flight requests.
//! prefix; else [`DpRankPolicy`] picks: the fewest router in-flight requests,
//! or the worker's ranks in turn.
use crate::config::DpRankPolicy;
use crate::workers::Worker;
use sha2::{Digest, Sha256};
@@ -15,6 +17,7 @@ pub fn select_dp_rank(
worker: &Worker,
affinity_key: Option<&str>,
prefix_depths: &[(u32, usize)],
policy: DpRankPolicy,
) -> Option<u32> {
let ranks = worker.dp_ranks();
if ranks <= 1 {
@@ -37,6 +40,9 @@ pub fn select_dp_rank(
.collect(),
None => (0..ranks).collect(),
};
if policy == DpRankPolicy::RoundRobin {
return Some(candidates[worker.next_dp_rank_turn() % candidates.len()]);
}
// Random start so equally loaded ranks share traffic.
let start = rand::random::<usize>() % candidates.len();
candidates[start..]
@@ -70,14 +76,22 @@ mod tests {
Worker::with_cb_config(spec, None, profile)
}
const LEAST: DpRankPolicy = DpRankPolicy::LeastInFlight;
#[test]
fn affinity_then_prefix_then_load() {
let w = worker(4);
assert_eq!(select_dp_rank(&worker(1), Some("k"), &[]), None);
assert_eq!(select_dp_rank(&worker(1), Some("k"), &[], LEAST), None);
// Pinned: replicas and releases must agree on the mapping.
assert_eq!(select_dp_rank(&w, Some("session-42"), &[(0, 9)]), Some(1));
assert_eq!(select_dp_rank(&w, None, &[(1, 2), (3, 5), (9, 9)]), Some(3));
assert_eq!(
select_dp_rank(&w, Some("session-42"), &[(0, 9)], LEAST),
Some(1)
);
assert_eq!(
select_dp_rank(&w, None, &[(1, 2), (3, 5), (9, 9)], LEAST),
Some(3)
);
let _busy = [0, 1, 3].map(|rank| w.dp_rank_guard(rank));
assert_eq!(select_dp_rank(&w, None, &[]), Some(2));
assert_eq!(select_dp_rank(&w, None, &[], LEAST), Some(2));
}
}
@@ -342,6 +342,7 @@ mod tests {
policy,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -217,6 +217,7 @@ impl AppContext {
policy: crate::config::PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -80,7 +80,7 @@ pub(super) async fn forward_request(
let decode_rank = decode
.as_deref()
.filter(|_| dp_aware)
.and_then(|decode| select_dp_rank(decode, None, &[]));
.and_then(|decode| select_dp_rank(decode, None, &[], ctx.config.model.dp_rank_policy));
// Track worker occupancy and the prompt's contribution to active load.
// A retry keeps the request's stale deadline rather than starting a new one.
@@ -271,7 +271,7 @@ fn prompt_dp_rank(
}
_ => Vec::new(),
};
select_dp_rank(worker, key, &prefix_depths)
select_dp_rank(worker, key, &prefix_depths, model.dp_rank_policy)
}
/// Under `--dp-aware` the router owns the rank: the header pins it for chat, and
@@ -55,6 +55,7 @@ mod tests {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -129,6 +129,7 @@ mod tests {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -326,6 +326,7 @@ mod tests {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -637,6 +637,7 @@ mod tests {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -228,6 +228,8 @@ pub struct Worker {
services: RwLock<BTreeSet<String>>,
/// Router in-flight requests per DP rank; one slot per rank.
dp_rank_inflight: Arc<[AtomicUsize]>,
/// Round-robin position over this worker's DP ranks.
dp_rank_turn: AtomicUsize,
openai: Option<Arc<OpenAiSettings>>,
}
@@ -271,6 +273,7 @@ impl Worker {
version_group: spec.version_group,
services: RwLock::new(spec.services),
dp_rank_inflight: (0..dp_ranks.max(1)).map(|_| AtomicUsize::new(0)).collect(),
dp_rank_turn: AtomicUsize::new(0),
openai,
}
}
@@ -364,6 +367,11 @@ impl Worker {
self.dp_rank_inflight[rank as usize].load(Ordering::Relaxed)
}
/// Advances the round-robin DP-rank position and returns the previous one.
pub fn next_dp_rank_turn(&self) -> usize {
self.dp_rank_turn.fetch_add(1, Ordering::Relaxed)
}
pub fn dp_rank_guard(&self, rank: u32) -> DpRankGuard {
let rank = rank as usize;
self.dp_rank_inflight[rank].fetch_add(1, Ordering::Relaxed);
@@ -136,6 +136,7 @@ async fn static_urls_pd_role_resolved_end_to_end() {
policy: sgl_router::config::PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -117,6 +117,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -66,6 +66,7 @@ fn build_app_context(
policy,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: Some(bucket_config),
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -45,6 +45,7 @@ fn config_for(_worker_url: &str) -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -50,6 +50,7 @@ pub fn config() -> Config {
policy: PolicyKind::CacheAware,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -9,7 +9,7 @@ use std::time::{Duration, Instant};
use axum::body::Body;
use axum::http::Request;
use serde_json::json;
use sgl_router::config::{Config, PolicyKind, StickyConfig, StickyFallbackKind};
use sgl_router::config::{Config, DpRankPolicy, PolicyKind, StickyConfig, StickyFallbackKind};
use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec};
use sgl_router::policies::factory::build_registry;
use sgl_router::policies::request_tokens_for;
@@ -167,6 +167,26 @@ async fn pd_room_maps_decode_to_the_prefill_rank() {
assert_eq!(room % 4, rank.parse::<u64>().unwrap());
}
#[tokio::test]
async fn round_robin_gives_decode_ranks_turns() {
let (prefill, decode) = (
MockWorker::start(vec![]).await,
MockWorker::start(vec![]).await,
);
let workers = [
(&prefill, WorkerMode::Prefill, 4),
(&decode, WorkerMode::Decode, 4),
];
let mut cfg = sticky_config();
cfg.model.dp_rank_policy = DpRankPolicy::RoundRobin;
let app = router(cfg, &workers, Default::default());
let mut ranks = Vec::new();
for _ in 0..8 {
ranks.push(send(&app, &decode, &[(KEY, "conv-pd")]).await.unwrap());
}
assert_eq!(ranks, ["0", "1", "2", "3", "0", "1", "2", "3"]);
}
#[tokio::test]
async fn any_policy_picks_the_rank_with_the_deepest_prefix() {
let worker = MockWorker::start(vec![]).await;
@@ -54,6 +54,7 @@ async fn errors_with_one_dead_worker(max_attempts: u32) -> usize {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -58,6 +58,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc<AppContext> {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -36,6 +36,7 @@ async fn forwards_whitelisted_headers_strips_others() {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -51,6 +51,7 @@ fn config() -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -50,6 +50,7 @@ fn config() -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -138,6 +138,7 @@ fn config() -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -44,6 +44,7 @@ fn config(max_attempts: u32) -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -49,6 +49,7 @@ fn config() -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -154,6 +154,7 @@ fn config(policy: PolicyKind) -> Config {
policy,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -59,6 +59,7 @@ fn config() -> Config {
policy: PolicyKind::Sticky,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -47,6 +47,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc<AppContext
policy: PolicyKind::Sticky,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),
@@ -44,6 +44,7 @@ fn config(_worker_url: &str) -> Config {
policy: PolicyKind::RoundRobin,
decode_policy: Default::default(),
dp_aware: false,
dp_rank_policy: Default::default(),
bucket_config: None,
reorg_buckets: None,
reorg_admission: Default::default(),