From 08f15e76bb466cfc6dfaa80bff439f4bf1940ca5 Mon Sep 17 00:00:00 2001 From: AMD-yanfeiwang Date: Thu, 15 Oct 2026 12:03:01 +0800 Subject: [PATCH] [Feat] sgl-router: Add --dp-rank-policy round_robin for --dp-aware (#43511) --- experimental/sgl-router/README.md | 5 ++-- experimental/sgl-router/src/config/cli.rs | 18 +++++++++----- experimental/sgl-router/src/config/mod.rs | 1 + experimental/sgl-router/src/config/types.rs | 15 ++++++++++++ .../sgl-router/src/policies/dp_rank.rs | 24 +++++++++++++++---- .../sgl-router/src/policies/factory.rs | 1 + .../sgl-router/src/server/app_context.rs | 1 + .../src/server/routes/chat/forward.rs | 4 ++-- .../sgl-router/src/server/routes/models.rs | 1 + .../sgl-router/src/server/routes/tokenize.rs | 1 + experimental/sgl-router/src/tokenizer/mod.rs | 1 + .../sgl-router/src/workers/manager.rs | 1 + experimental/sgl-router/src/workers/worker.rs | 8 +++++++ .../tests/component/discovery/static_urls.rs | 1 + .../component/tokenizer/render_parity.rs | 1 + .../sgl-router/tests/proxy/bucket_routing.rs | 1 + .../sgl-router/tests/proxy/chat_routing.rs | 1 + .../tests/proxy/common/cache_aware_fixture.rs | 1 + .../sgl-router/tests/proxy/dp_rank_routing.rs | 22 ++++++++++++++++- .../sgl-router/tests/proxy/failover.rs | 1 + .../tests/proxy/graceful_shutdown.rs | 1 + .../tests/proxy/header_forwarding.rs | 1 + .../tests/proxy/pd_bootstrap_injection.rs | 1 + .../tests/proxy/pd_pool_isolation.rs | 1 + .../tests/proxy/pd_protocol_binding.rs | 1 + experimental/sgl-router/tests/proxy/retry.rs | 1 + .../tests/proxy/roundrobin_input_ids.rs | 1 + .../tests/proxy/shared_prefill_admission.rs | 1 + .../tests/proxy/sticky_input_ids.rs | 1 + .../sgl-router/tests/proxy/sticky_routing.rs | 1 + .../sgl-router/tests/proxy/timeout.rs | 1 + 31 files changed, 104 insertions(+), 16 deletions(-) diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md index 45592a04bd..3de09b4762 100644 --- a/experimental/sgl-router/README.md +++ b/experimental/sgl-router/README.md @@ -283,9 +283,10 @@ client sent. The router picks the first of these that applies: 1. A hash of the sticky routing key or session id, so a conversation keeps its rank and router replicas agree. 2. The rank with the deepest cached prefix in the local KV tree. -3. The rank with the fewest requests this router has in flight on it. +3. The rank with the fewest requests this router has in flight on it, or with + `--dp-rank-policy round_robin`, the worker's ranks in turn. -In PD mode, decode is ranked by load only. The bootstrap room satisfies +In PD mode, decode is ranked by step 3 only. The bootstrap room satisfies `room % prefill_dp_size == prefill_rank`, which is how a decode engine finds the prefill rank. diff --git a/experimental/sgl-router/src/config/cli.rs b/experimental/sgl-router/src/config/cli.rs index b51595ae37..b779dbd7bb 100644 --- a/experimental/sgl-router/src/config/cli.rs +++ b/experimental/sgl-router/src/config/cli.rs @@ -14,12 +14,12 @@ use crate::config::{ default_cb_cool_down, default_host, default_port, default_proxy_request_timeout_secs, default_shutdown_drain_secs, default_stale_request_timeout_secs, resolve_mode, AffinityConfig, AffinityMode, BalancedBy, CacheAwareConfig, CachePrefixProvider, ChatRoutingKind, - CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, EligibilityConfig, - FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig, K8sDiscoveryMode, - KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig, PolicyKind, ProxyConfig, - ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig, StickyConfig, StickyFallbackKind, - TokenizerBackend, TokenizerConfig, DEFAULT_FUSE, DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, - DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS, + CircuitBreakerConfig, Config, DecodePolicyKind, DiscoveryBackend, DpRankPolicy, + EligibilityConfig, FilterKind, FusedTerm, InflightLoadConfig, K8sDiscoveryConfig, + K8sDiscoveryMode, KvIndexerEndpointConfig, LogFormat, ModelConfig, ObservabilityConfig, + PolicyKind, ProxyConfig, ServerConfig, SessionAffinityMode, StaticUrlsDiscoveryConfig, + StickyConfig, StickyFallbackKind, TokenizerBackend, TokenizerConfig, DEFAULT_FUSE, + DEFAULT_KV_BOOTSTRAP_FETCH_TIMEOUT_CAP_MS, DEFAULT_KV_BOOTSTRAP_TIMEOUT_MS, }; use crate::policies_reorg::admission::AdmissionLimits; use crate::policies_reorg::factory::AffinitySpec; @@ -245,6 +245,11 @@ pub struct RoutingArgs { #[arg(long)] pub dp_aware: bool, + /// How --dp-aware picks a rank that no routing key or cached prefix decides, + /// which in PD mode is every decode rank. Defaults to least_in_flight. + #[arg(long, value_enum, requires = "dp_aware")] + pub dp_rank_policy: Option, + /// Static bucket configuration JSON; the reorg schema with --chat-routing reorg. /// Omit to use the global candidate domain (default buckets with reorg). #[arg(long)] @@ -616,6 +621,7 @@ impl Cli { policy: self.routing.policy, decode_policy: self.routing.decode_policy, dp_aware: self.routing.dp_aware, + dp_rank_policy: self.routing.dp_rank_policy.unwrap_or_default(), bucket_config, reorg_buckets, reorg_admission, diff --git a/experimental/sgl-router/src/config/mod.rs b/experimental/sgl-router/src/config/mod.rs index ad8f3f74d3..906e00fbe2 100644 --- a/experimental/sgl-router/src/config/mod.rs +++ b/experimental/sgl-router/src/config/mod.rs @@ -262,6 +262,7 @@ mod tests { policy: PolicyKind::RoundRobin, decode_policy: DecodePolicyKind::PowerOfTwo, dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/config/types.rs b/experimental/sgl-router/src/config/types.rs index 7ff589bd4e..a57beb88c1 100644 --- a/experimental/sgl-router/src/config/types.rs +++ b/experimental/sgl-router/src/config/types.rs @@ -152,6 +152,19 @@ pub enum DecodePolicyKind { LegacyHostAffinity, } +/// How `--dp-aware` picks a DP rank that no routing key or cached prefix decides, +/// which in PD mode is every decode rank. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, clap::ValueEnum)] +pub enum DpRankPolicy { + /// The rank with the fewest requests this router has in flight on it. + #[default] + #[value(name = "least_in_flight")] + LeastInFlight, + /// The worker's ranks in turn. + #[value(name = "round_robin")] + RoundRobin, +} + /// Role served by a static bucket. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize)] #[serde(rename_all = "snake_case")] @@ -372,6 +385,8 @@ pub struct ModelConfig { pub decode_policy: DecodePolicyKind, /// Send a DP rank as `X-Data-Parallel-Rank`; see [`crate::policies::dp_rank`]. pub dp_aware: bool, + /// Rank choice under `dp_aware` when no routing key or cached prefix decides it. + pub dp_rank_policy: DpRankPolicy, /// Optional static bucket configuration. `None` uses the global domain. pub bucket_config: Option, /// Reorg `--bucket-config`; `None` builds the default plain and P/D buckets. diff --git a/experimental/sgl-router/src/policies/dp_rank.rs b/experimental/sgl-router/src/policies/dp_rank.rs index 004fcf7527..537581bbd4 100644 --- a/experimental/sgl-router/src/policies/dp_rank.rs +++ b/experimental/sgl-router/src/policies/dp_rank.rs @@ -5,8 +5,10 @@ //! //! First match wins: a sticky/session key hashes to a fixed rank, so router //! replicas agree without shared state; else the rank with the deepest cached -//! prefix; else the rank with the fewest router in-flight requests. +//! prefix; else [`DpRankPolicy`] picks: the fewest router in-flight requests, +//! or the worker's ranks in turn. +use crate::config::DpRankPolicy; use crate::workers::Worker; use sha2::{Digest, Sha256}; @@ -15,6 +17,7 @@ pub fn select_dp_rank( worker: &Worker, affinity_key: Option<&str>, prefix_depths: &[(u32, usize)], + policy: DpRankPolicy, ) -> Option { let ranks = worker.dp_ranks(); if ranks <= 1 { @@ -37,6 +40,9 @@ pub fn select_dp_rank( .collect(), None => (0..ranks).collect(), }; + if policy == DpRankPolicy::RoundRobin { + return Some(candidates[worker.next_dp_rank_turn() % candidates.len()]); + } // Random start so equally loaded ranks share traffic. let start = rand::random::() % candidates.len(); candidates[start..] @@ -70,14 +76,22 @@ mod tests { Worker::with_cb_config(spec, None, profile) } + const LEAST: DpRankPolicy = DpRankPolicy::LeastInFlight; + #[test] fn affinity_then_prefix_then_load() { let w = worker(4); - assert_eq!(select_dp_rank(&worker(1), Some("k"), &[]), None); + assert_eq!(select_dp_rank(&worker(1), Some("k"), &[], LEAST), None); // Pinned: replicas and releases must agree on the mapping. - assert_eq!(select_dp_rank(&w, Some("session-42"), &[(0, 9)]), Some(1)); - assert_eq!(select_dp_rank(&w, None, &[(1, 2), (3, 5), (9, 9)]), Some(3)); + assert_eq!( + select_dp_rank(&w, Some("session-42"), &[(0, 9)], LEAST), + Some(1) + ); + assert_eq!( + select_dp_rank(&w, None, &[(1, 2), (3, 5), (9, 9)], LEAST), + Some(3) + ); let _busy = [0, 1, 3].map(|rank| w.dp_rank_guard(rank)); - assert_eq!(select_dp_rank(&w, None, &[]), Some(2)); + assert_eq!(select_dp_rank(&w, None, &[], LEAST), Some(2)); } } diff --git a/experimental/sgl-router/src/policies/factory.rs b/experimental/sgl-router/src/policies/factory.rs index 1392bee831..9426bee6af 100644 --- a/experimental/sgl-router/src/policies/factory.rs +++ b/experimental/sgl-router/src/policies/factory.rs @@ -342,6 +342,7 @@ mod tests { policy, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/server/app_context.rs b/experimental/sgl-router/src/server/app_context.rs index 94fbaedc6b..f221959d46 100644 --- a/experimental/sgl-router/src/server/app_context.rs +++ b/experimental/sgl-router/src/server/app_context.rs @@ -217,6 +217,7 @@ impl AppContext { policy: crate::config::PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/server/routes/chat/forward.rs b/experimental/sgl-router/src/server/routes/chat/forward.rs index 04eab898bc..7bd92f2b4a 100644 --- a/experimental/sgl-router/src/server/routes/chat/forward.rs +++ b/experimental/sgl-router/src/server/routes/chat/forward.rs @@ -80,7 +80,7 @@ pub(super) async fn forward_request( let decode_rank = decode .as_deref() .filter(|_| dp_aware) - .and_then(|decode| select_dp_rank(decode, None, &[])); + .and_then(|decode| select_dp_rank(decode, None, &[], ctx.config.model.dp_rank_policy)); // Track worker occupancy and the prompt's contribution to active load. // A retry keeps the request's stale deadline rather than starting a new one. @@ -271,7 +271,7 @@ fn prompt_dp_rank( } _ => Vec::new(), }; - select_dp_rank(worker, key, &prefix_depths) + select_dp_rank(worker, key, &prefix_depths, model.dp_rank_policy) } /// Under `--dp-aware` the router owns the rank: the header pins it for chat, and diff --git a/experimental/sgl-router/src/server/routes/models.rs b/experimental/sgl-router/src/server/routes/models.rs index 291e5881eb..ca18c87d59 100644 --- a/experimental/sgl-router/src/server/routes/models.rs +++ b/experimental/sgl-router/src/server/routes/models.rs @@ -55,6 +55,7 @@ mod tests { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/server/routes/tokenize.rs b/experimental/sgl-router/src/server/routes/tokenize.rs index 7051b0cee1..36215aa28e 100644 --- a/experimental/sgl-router/src/server/routes/tokenize.rs +++ b/experimental/sgl-router/src/server/routes/tokenize.rs @@ -129,6 +129,7 @@ mod tests { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/tokenizer/mod.rs b/experimental/sgl-router/src/tokenizer/mod.rs index 38995e6e89..e0df162211 100644 --- a/experimental/sgl-router/src/tokenizer/mod.rs +++ b/experimental/sgl-router/src/tokenizer/mod.rs @@ -326,6 +326,7 @@ mod tests { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/workers/manager.rs b/experimental/sgl-router/src/workers/manager.rs index a074d594c1..8314e58367 100644 --- a/experimental/sgl-router/src/workers/manager.rs +++ b/experimental/sgl-router/src/workers/manager.rs @@ -637,6 +637,7 @@ mod tests { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/src/workers/worker.rs b/experimental/sgl-router/src/workers/worker.rs index c7f6184b30..8c6cf6c657 100644 --- a/experimental/sgl-router/src/workers/worker.rs +++ b/experimental/sgl-router/src/workers/worker.rs @@ -228,6 +228,8 @@ pub struct Worker { services: RwLock>, /// Router in-flight requests per DP rank; one slot per rank. dp_rank_inflight: Arc<[AtomicUsize]>, + /// Round-robin position over this worker's DP ranks. + dp_rank_turn: AtomicUsize, openai: Option>, } @@ -271,6 +273,7 @@ impl Worker { version_group: spec.version_group, services: RwLock::new(spec.services), dp_rank_inflight: (0..dp_ranks.max(1)).map(|_| AtomicUsize::new(0)).collect(), + dp_rank_turn: AtomicUsize::new(0), openai, } } @@ -364,6 +367,11 @@ impl Worker { self.dp_rank_inflight[rank as usize].load(Ordering::Relaxed) } + /// Advances the round-robin DP-rank position and returns the previous one. + pub fn next_dp_rank_turn(&self) -> usize { + self.dp_rank_turn.fetch_add(1, Ordering::Relaxed) + } + pub fn dp_rank_guard(&self, rank: u32) -> DpRankGuard { let rank = rank as usize; self.dp_rank_inflight[rank].fetch_add(1, Ordering::Relaxed); diff --git a/experimental/sgl-router/tests/component/discovery/static_urls.rs b/experimental/sgl-router/tests/component/discovery/static_urls.rs index f67b2a8d87..1beec760dd 100644 --- a/experimental/sgl-router/tests/component/discovery/static_urls.rs +++ b/experimental/sgl-router/tests/component/discovery/static_urls.rs @@ -136,6 +136,7 @@ async fn static_urls_pd_role_resolved_end_to_end() { policy: sgl_router::config::PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs index 747c1c0388..b20bb90fd8 100644 --- a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs +++ b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs @@ -117,6 +117,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/bucket_routing.rs b/experimental/sgl-router/tests/proxy/bucket_routing.rs index 706625b999..a1a4ce7c5e 100644 --- a/experimental/sgl-router/tests/proxy/bucket_routing.rs +++ b/experimental/sgl-router/tests/proxy/bucket_routing.rs @@ -66,6 +66,7 @@ fn build_app_context( policy, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: Some(bucket_config), reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/chat_routing.rs b/experimental/sgl-router/tests/proxy/chat_routing.rs index 5d3742cf0d..5bb7f25c8c 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing.rs @@ -45,6 +45,7 @@ fn config_for(_worker_url: &str) -> Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs index 439102bcf0..fb96011256 100644 --- a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs +++ b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs @@ -50,6 +50,7 @@ pub fn config() -> Config { policy: PolicyKind::CacheAware, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs index d117f479a7..4578ce7652 100644 --- a/experimental/sgl-router/tests/proxy/dp_rank_routing.rs +++ b/experimental/sgl-router/tests/proxy/dp_rank_routing.rs @@ -9,7 +9,7 @@ use std::time::{Duration, Instant}; use axum::body::Body; use axum::http::Request; use serde_json::json; -use sgl_router::config::{Config, PolicyKind, StickyConfig, StickyFallbackKind}; +use sgl_router::config::{Config, DpRankPolicy, PolicyKind, StickyConfig, StickyFallbackKind}; use sgl_router::discovery::{ModelId, WorkerId, WorkerMode, WorkerSpec}; use sgl_router::policies::factory::build_registry; use sgl_router::policies::request_tokens_for; @@ -167,6 +167,26 @@ async fn pd_room_maps_decode_to_the_prefill_rank() { assert_eq!(room % 4, rank.parse::().unwrap()); } +#[tokio::test] +async fn round_robin_gives_decode_ranks_turns() { + let (prefill, decode) = ( + MockWorker::start(vec![]).await, + MockWorker::start(vec![]).await, + ); + let workers = [ + (&prefill, WorkerMode::Prefill, 4), + (&decode, WorkerMode::Decode, 4), + ]; + let mut cfg = sticky_config(); + cfg.model.dp_rank_policy = DpRankPolicy::RoundRobin; + let app = router(cfg, &workers, Default::default()); + let mut ranks = Vec::new(); + for _ in 0..8 { + ranks.push(send(&app, &decode, &[(KEY, "conv-pd")]).await.unwrap()); + } + assert_eq!(ranks, ["0", "1", "2", "3", "0", "1", "2", "3"]); +} + #[tokio::test] async fn any_policy_picks_the_rank_with_the_deepest_prefix() { let worker = MockWorker::start(vec![]).await; diff --git a/experimental/sgl-router/tests/proxy/failover.rs b/experimental/sgl-router/tests/proxy/failover.rs index 3fb33e052d..a55381feb4 100644 --- a/experimental/sgl-router/tests/proxy/failover.rs +++ b/experimental/sgl-router/tests/proxy/failover.rs @@ -54,6 +54,7 @@ async fn errors_with_one_dead_worker(max_attempts: u32) -> usize { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs index 9a4214950b..4f791d79df 100644 --- a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs +++ b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs @@ -58,6 +58,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/header_forwarding.rs b/experimental/sgl-router/tests/proxy/header_forwarding.rs index 014b25dfed..a5be471175 100644 --- a/experimental/sgl-router/tests/proxy/header_forwarding.rs +++ b/experimental/sgl-router/tests/proxy/header_forwarding.rs @@ -36,6 +36,7 @@ async fn forwards_whitelisted_headers_strips_others() { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs index 8c3d60d443..99fd20330d 100644 --- a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs +++ b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs @@ -51,6 +51,7 @@ fn config() -> Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs index d777d40c6f..7b52008988 100644 --- a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs +++ b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs @@ -50,6 +50,7 @@ fn config() -> Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs b/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs index 7918e6850f..d937634ea0 100644 --- a/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs +++ b/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs @@ -138,6 +138,7 @@ fn config() -> Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/retry.rs b/experimental/sgl-router/tests/proxy/retry.rs index 34fe1d675f..0c3432b9b6 100644 --- a/experimental/sgl-router/tests/proxy/retry.rs +++ b/experimental/sgl-router/tests/proxy/retry.rs @@ -44,6 +44,7 @@ fn config(max_attempts: u32) -> Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs b/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs index 64a2b73a1c..eea7a2931f 100644 --- a/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs @@ -49,6 +49,7 @@ fn config() -> Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs b/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs index 447fdb37b9..ca88f0497c 100644 --- a/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs +++ b/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs @@ -154,6 +154,7 @@ fn config(policy: PolicyKind) -> Config { policy, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/sticky_input_ids.rs b/experimental/sgl-router/tests/proxy/sticky_input_ids.rs index eed1968e63..758a4ec90b 100644 --- a/experimental/sgl-router/tests/proxy/sticky_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/sticky_input_ids.rs @@ -59,6 +59,7 @@ fn config() -> Config { policy: PolicyKind::Sticky, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(), diff --git a/experimental/sgl-router/tests/proxy/sticky_routing.rs b/experimental/sgl-router/tests/proxy/sticky_routing.rs index 5babaeae9e..b924267821 100644 --- a/experimental/sgl-router/tests/proxy/sticky_routing.rs +++ b/experimental/sgl-router/tests/proxy/sticky_routing.rs @@ -47,6 +47,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc Config { policy: PolicyKind::RoundRobin, decode_policy: Default::default(), dp_aware: false, + dp_rank_policy: Default::default(), bucket_config: None, reorg_buckets: None, reorg_admission: Default::default(),