diff --git a/docs/docs/advanced_features/sgl_model_gateway.mdx b/docs/docs/advanced_features/sgl_model_gateway.mdx index 049e6d4081..489ed8eb61 100644 --- a/docs/docs/advanced_features/sgl_model_gateway.mdx +++ b/docs/docs/advanced_features/sgl_model_gateway.mdx @@ -2670,6 +2670,12 @@ groups: None Override policy for decode nodes + + `--decode-dispatch-limit` + int + None + Requests per prefill worker sent to decode before decode answers or prefill finishes; the rest pick their decode worker when sent. Needs prefill `--optimistic-prefill-attempts` above 0 + `--worker-startup-timeout-secs` int diff --git a/sgl-model-gateway/README.md b/sgl-model-gateway/README.md index bffeae39f1..89a3c8e1d2 100644 --- a/sgl-model-gateway/README.md +++ b/sgl-model-gateway/README.md @@ -160,6 +160,8 @@ The `--version` (or `-V`) flag displays the version string. Use `--version-verbo ``` Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client. +A decode worker picked on arrival only starts decoding once prefill has queued, computed and sent the KV, so with long prefill queues, decode workers (with `--dp-aware`, DP ranks) that hold equal numbers of requests can run unequal batches. `--decode-dispatch-limit N` still sends every request to prefill on arrival, but sends at most N per prefill worker on to decode before decode answers or prefill finishes; the rest wait in arrival order and pick their decode worker when they are sent. Prefill must start a request before decode connects, so prefill engines need `--optimistic-prefill-attempts` above 0. + ### Multi-Model Inference Gateway Enable IGW mode to route multiple models through a single router while applying per-model policies: ```bash diff --git a/sgl-model-gateway/bindings/python/src/lib.rs b/sgl-model-gateway/bindings/python/src/lib.rs index f18bdf4ec6..447e4f87ca 100644 --- a/sgl-model-gateway/bindings/python/src/lib.rs +++ b/sgl-model-gateway/bindings/python/src/lib.rs @@ -412,6 +412,7 @@ struct Router { queue_size: usize, queue_timeout_secs: u64, rate_limit_tokens_per_second: Option, + decode_dispatch_limit: Option, connection_mode: core::ConnectionMode, model_path: Option, tokenizer_path: Option, @@ -637,6 +638,7 @@ impl Router { .maybe_log_level(self.log_level.as_ref()) .maybe_request_id_headers(self.request_id_headers.clone()) .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second) + .maybe_decode_dispatch_limit(self.decode_dispatch_limit) .maybe_model_path(self.model_path.as_ref()) .maybe_tokenizer_path(self.tokenizer_path.as_ref()) .maybe_chat_template(self.chat_template.as_ref()) @@ -733,6 +735,7 @@ impl Router { queue_size = 100, queue_timeout_secs = 60, rate_limit_tokens_per_second = None, + decode_dispatch_limit = None, model_path = None, tokenizer_path = None, chat_template = None, @@ -825,6 +828,7 @@ impl Router { queue_size: usize, queue_timeout_secs: u64, rate_limit_tokens_per_second: Option, + decode_dispatch_limit: Option, model_path: Option, tokenizer_path: Option, chat_template: Option, @@ -930,6 +934,7 @@ impl Router { queue_size, queue_timeout_secs, rate_limit_tokens_per_second, + decode_dispatch_limit, connection_mode, model_path, tokenizer_path, diff --git a/sgl-model-gateway/bindings/python/src/sglang_router/router.py b/sgl-model-gateway/bindings/python/src/sglang_router/router.py index fa051b5765..0679bf192e 100644 --- a/sgl-model-gateway/bindings/python/src/sglang_router/router.py +++ b/sgl-model-gateway/bindings/python/src/sglang_router/router.py @@ -211,6 +211,7 @@ class Router: queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100 queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60 rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None + decode_dispatch_limit: Max requests per prefill worker in flight to decode (PD mode only). Default: None cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: [] health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3 health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2 diff --git a/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py b/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py index a2ea4f2eab..5e7e0e7511 100644 --- a/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py +++ b/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py @@ -98,6 +98,7 @@ class RouterArgs: queue_timeout_secs: int = 60 # Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests rate_limit_tokens_per_second: Optional[int] = None + decode_dispatch_limit: Optional[int] = None # CORS allowed origins cors_allowed_origins: List[str] = dataclasses.field(default_factory=list) # Retry configuration @@ -398,6 +399,13 @@ class RouterArgs: metavar=("URL",), help="Decode server URL. Can be specified multiple times.", ) + pd_group.add_argument( + f"--{prefix}decode-dispatch-limit", + type=int, + default=RouterArgs.decode_dispatch_limit, + help="Max requests per prefill worker in flight to decode; the rest pick their " + "decode worker when sent (PD mode only)", + ) pd_group.add_argument( f"--{prefix}worker-startup-timeout-secs", type=int, diff --git a/sgl-model-gateway/src/config/builder.rs b/sgl-model-gateway/src/config/builder.rs index 5d4891ca60..ad24d408d5 100644 --- a/sgl-model-gateway/src/config/builder.rs +++ b/sgl-model-gateway/src/config/builder.rs @@ -229,6 +229,11 @@ impl RouterConfigBuilder { self } + pub fn decode_dispatch_limit(mut self, limit: usize) -> Self { + self.config.decode_dispatch_limit = Some(limit); + self + } + pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self { self.config.rate_limit_tokens_per_second = Some(tokens); self @@ -531,6 +536,11 @@ impl RouterConfigBuilder { self } + pub fn maybe_decode_dispatch_limit(mut self, limit: Option) -> Self { + self.config.decode_dispatch_limit = limit; + self + } + pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option) -> Self { self.config.rate_limit_tokens_per_second = tokens; self diff --git a/sgl-model-gateway/src/config/types.rs b/sgl-model-gateway/src/config/types.rs index b19802642c..231047b6a2 100644 --- a/sgl-model-gateway/src/config/types.rs +++ b/sgl-model-gateway/src/config/types.rs @@ -26,6 +26,8 @@ pub struct RouterConfig { pub worker_startup_timeout_secs: u64, pub worker_startup_check_interval_secs: u64, pub dp_aware: bool, + #[serde(default)] + pub decode_dispatch_limit: Option, pub api_key: Option, pub discovery: Option, pub metrics: Option, @@ -514,6 +516,7 @@ impl Default for RouterConfig { worker_startup_timeout_secs: 1800, // 30 minutes for large model loading worker_startup_check_interval_secs: 30, dp_aware: false, + decode_dispatch_limit: None, api_key: None, discovery: None, metrics: None, diff --git a/sgl-model-gateway/src/config/validation.rs b/sgl-model-gateway/src/config/validation.rs index bd3d38ab8f..47be1f2933 100644 --- a/sgl-model-gateway/src/config/validation.rs +++ b/sgl-model-gateway/src/config/validation.rs @@ -286,6 +286,14 @@ impl ConfigValidator { }); } + if config.decode_dispatch_limit == Some(0) { + return Err(ConfigError::InvalidValue { + field: "decode_dispatch_limit".to_string(), + value: "0".to_string(), + reason: "Must be > 0 when specified".to_string(), + }); + } + if let Some(tokens_per_second) = config.rate_limit_tokens_per_second { // Allow 0 for pure concurrency limiting (semaphore behavior) if tokens_per_second < 0 { diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs index b389bdad19..15296826b7 100644 --- a/sgl-model-gateway/src/main.rs +++ b/sgl-model-gateway/src/main.rs @@ -211,6 +211,10 @@ struct CliArgs { #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")] decode_policy: Option, + /// Max requests per prefill worker in flight to decode; the rest pick their decode worker when sent + #[arg(long, help_heading = "PD Disaggregation")] + decode_dispatch_limit: Option, + /// Timeout in seconds for worker startup and registration #[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")] worker_startup_timeout_secs: u64, @@ -1056,6 +1060,7 @@ impl CliArgs { (!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()), ) .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second) + .maybe_decode_dispatch_limit(self.decode_dispatch_limit) .maybe_model_path(self.model_path.as_ref()) .maybe_tokenizer_path(self.tokenizer_path.as_ref()) .maybe_chat_template(self.chat_template.as_ref()) diff --git a/sgl-model-gateway/src/routers/http/decode_gate.rs b/sgl-model-gateway/src/routers/http/decode_gate.rs new file mode 100644 index 0000000000..27431e1f28 --- /dev/null +++ b/sgl-model-gateway/src/routers/http/decode_gate.rs @@ -0,0 +1,39 @@ +use std::sync::{Arc, Mutex}; + +use dashmap::DashMap; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +#[derive(Debug)] +pub struct DecodeGate { + limit: usize, + gates: DashMap>, +} + +impl DecodeGate { + pub fn new(limit: usize) -> Self { + Self { + limit, + gates: DashMap::new(), + } + } + + pub async fn admit(&self, prefill_url: &str, slot: &DecodeSlot) { + let gate = Arc::clone( + &self + .gates + .entry(prefill_url.to_string()) + .or_insert_with(|| Arc::new(Semaphore::new(self.limit))), + ); + let permit = gate.acquire_owned().await.expect("gates are never closed"); + *slot.0.lock().unwrap() = Some(permit); + } +} + +#[derive(Debug, Clone, Default)] +pub struct DecodeSlot(Arc>>); + +impl DecodeSlot { + pub fn release(&self) { + self.0.lock().unwrap().take(); + } +} diff --git a/sgl-model-gateway/src/routers/http/mod.rs b/sgl-model-gateway/src/routers/http/mod.rs index 3f31b6f869..de8268eaeb 100644 --- a/sgl-model-gateway/src/routers/http/mod.rs +++ b/sgl-model-gateway/src/routers/http/mod.rs @@ -1,5 +1,6 @@ //! HTTP router implementations +pub mod decode_gate; pub mod pd_router; pub mod pd_types; pub mod router; diff --git a/sgl-model-gateway/src/routers/http/pd_router.rs b/sgl-model-gateway/src/routers/http/pd_router.rs index a0a196db64..ea60c3f299 100644 --- a/sgl-model-gateway/src/routers/http/pd_router.rs +++ b/sgl-model-gateway/src/routers/http/pd_router.rs @@ -1,4 +1,8 @@ -use std::{borrow::Cow, sync::Arc, time::Instant}; +use std::{ + borrow::Cow, + sync::Arc, + time::{Duration, Instant}, +}; use async_trait::async_trait; use axum::{ @@ -18,7 +22,10 @@ use serde_json::{json, Value}; use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, error, warn}; -use super::pd_types::api_path; +use super::{ + decode_gate::{DecodeGate, DecodeSlot}, + pd_types::api_path, +}; use crate::{ config::types::RetryConfig, core::{ @@ -58,6 +65,8 @@ pub struct PDRouter { pub retry_config: RetryConfig, pub api_key: Option, pub enable_igw: bool, + pub decode_gate: Option>, + pub request_timeout: Duration, } struct PreparedWorkerRequest<'a> { @@ -189,6 +198,11 @@ impl PDRouter { retry_config: ctx.router_config.effective_retry_config(), api_key: ctx.router_config.api_key.clone(), enable_igw: ctx.router_config.enable_igw, + decode_gate: ctx + .router_config + .decode_dispatch_limit + .map(|limit| Arc::new(DecodeGate::new(limit))), + request_timeout: Duration::from_secs(ctx.router_config.request_timeout_secs), }) } @@ -350,6 +364,7 @@ impl PDRouter { }) } + #[cfg(test)] async fn prepare_pd_worker_requests<'a>( route: &'static str, json_request: &'a Value, @@ -398,14 +413,26 @@ impl PDRouter { let shared_request = Arc::clone(&shared_request); let context = context.clone(); async move { - let (prefill, decode) = match self - .select_pd_pair( - context.request_text.as_deref(), + let request_text = context.request_text.as_deref(); + let selected = if self.decode_gate.is_some() { + self.select_worker( + request_text, + context.model_id, + context.headers.as_ref(), + false, + ) + .await + .map(|prefill| (prefill, None)) + } else { + self.select_pd_pair( + request_text, context.model_id, context.headers.as_ref(), ) .await - { + .map(|(prefill, decode)| (prefill, Some(decode))) + }; + let (prefill, decode) = match selected { Ok(pair) => pair, Err(e) => { return Self::handle_server_selection_error(e); @@ -413,10 +440,10 @@ impl PDRouter { }; debug!( - "PD retry attempt {} using prefill={} decode={}", + "PD retry attempt {} using prefill={} decode={:?}", attempt, prefill.url(), - decode.url() + decode.as_ref().map(|d| d.url()) ); let mut json_request = match serde_json::to_value(shared_request.as_ref()) { @@ -438,13 +465,13 @@ impl PDRouter { }; let ctx_is_stream = context.is_stream; - let response = self + let (response, decode) = self .execute_dual_dispatch_internal( headers, json_request, context, Arc::clone(&prefill), - Arc::clone(&decode), + decode, start_time, ) .await; @@ -464,7 +491,7 @@ impl PDRouter { // its byte stream records the outcome on drop. Skip the // eager success record to avoid masking "200-then-broken" // decode workers. - if !ctx_is_stream { + if let (false, Some(decode)) = (ctx_is_stream, &decode) { decode.record_outcome(not_error); } } @@ -649,40 +676,40 @@ impl PDRouter { } // Internal method that performs the actual dual dispatch (without retry logic) + #[allow(clippy::too_many_arguments)] async fn execute_dual_dispatch_internal( &self, headers: Option<&HeaderMap>, json_request: Value, context: PDRequestContext<'_>, prefill: Arc, - decode: Arc, + decode: Option>, _start_time: Instant, - ) -> Response { + ) -> (Response, Option>) { // Count the request from dispatch so load-based policies see it before // upstream headers arrive; streaming responses attach their own guards. let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers); - let _decode_guard = WorkerLoadGuard::new(decode.clone(), headers); let mut headers_with_trace = headers.cloned().unwrap_or_default(); inject_trace_context_http(&mut headers_with_trace); let headers = Some(&headers_with_trace); - let (prepared_prefill, prepared_decode) = match Self::prepare_pd_worker_requests( + let prepared_prefill = match Self::prepare_worker_request( context.route, - &json_request, prefill.as_ref(), - decode.as_ref(), + Cow::Borrowed(&json_request), ) .await { - Ok(requests) => requests, + Ok(request) => request, Err(e) => { error!("Failed to prepare PD worker requests: {}", e); - return error::internal_error("pd_request_preparation_failed", e); + return ( + error::internal_error("pd_request_preparation_failed", e), + None, + ); } }; - - // Build both requests let prefill_request = self.build_post_with_headers( &self.client, &prepared_prefill.endpoint_url, @@ -690,32 +717,77 @@ impl PDRouter { headers, false, ); - let decode_request = self.build_post_with_headers( - &self.client, - &prepared_decode.endpoint_url, - &prepared_decode.body, - headers, - false, - ); + + let slot = decode.is_none().then(DecodeSlot::default); + let decode_leg = async { + if let (Some(gate), Some(slot)) = (&self.decode_gate, &slot) { + let admit = gate.admit(prefill.url(), slot); + if tokio::time::timeout(self.request_timeout, admit) + .await + .is_err() + { + return Err(error::create_error( + StatusCode::GATEWAY_TIMEOUT, + "decode_dispatch_timeout", + "Timed out waiting to dispatch the decode request", + )); + } + } + let decode = match decode { + Some(decode) => decode, + None => self + .select_worker( + context.request_text.as_deref(), + context.model_id, + context.headers.as_ref(), + true, + ) + .await + .map_err(Self::handle_server_selection_error)?, + }; + let decode_guard = WorkerLoadGuard::new(decode.clone(), headers); + let prepared_decode = async { + let body = Self::inject_prefill_dp_rank_for_decode( + Cow::Borrowed(&json_request), + prefill.as_ref(), + )?; + Self::prepare_worker_request(context.route, decode.as_ref(), body).await + } + .await + .map_err(|e| { + error!("Failed to prepare PD worker requests: {}", e); + error::internal_error("pd_request_preparation_failed", e) + })?; + let decode_request = self.build_post_with_headers( + &self.client, + &prepared_decode.endpoint_url, + &prepared_decode.body, + headers, + false, + ); + events::RequestPDSentEvent { + prefill_url: prefill.url(), + decode_url: decode.url(), + } + .emit(); + let result = decode_request.send().await; + if let Some(slot) = &slot { + slot.release(); + } + Ok((decode, decode_guard, result)) + }; // Run both in this handler task (not a detached tokio::spawn) so a client // disconnect cancels the pending decode request too, keeping the // upstream-cancel behavior from #19524. - events::RequestPDSentEvent { - prefill_url: prefill.url(), - decode_url: decode.url(), - } - .emit(); - let prefill_fut = prefill_request.send(); - let decode_fut = decode_request.send(); tokio::pin!(prefill_fut); - tokio::pin!(decode_fut); + tokio::pin!(decode_leg); // Poll both until prefill resolves; decode normally resolves later, but // may resolve first if it rejects the request outright. let prefill_result; - let mut decode_early: Option> = None; + let mut decode_early = None; loop { tokio::select! { biased; @@ -723,11 +795,14 @@ impl PDRouter { prefill_result = pr; break; } - dr = &mut decode_fut, if decode_early.is_none() => { + dr = &mut decode_leg, if decode_early.is_none() => { decode_early = Some(dr); } } } + if let Some(slot) = &slot { + slot.release(); + } // Decode can't generate without prefill's KV, so any prefill failure // (non-2xx / transport error) dooms the paired decode request, which would @@ -741,8 +816,7 @@ impl PDRouter { if prefill_failed { warn!( - "Prefill failed, aborting paired decode request decode_url={} prefill_url={}", - decode.url(), + "Prefill failed, aborting paired decode request prefill_url={}", prefill.url() ); @@ -767,17 +841,41 @@ impl PDRouter { ), }; response.extensions_mut().insert(BreakerOutcomesRecorded); - return response; + return (response, None); } // Prefill ok: take decode's result, awaiting it if still pending. - let decode_result = match decode_early { - Some(dr) => dr, - None => (&mut decode_fut).await, + let leg = match decode_early { + Some(leg) => leg, + None => (&mut decode_leg).await, + }; + let (decode, _decode_guard, decode_result) = match leg { + Ok(leg) => leg, + Err(response) => return (response, None), }; events::RequestReceivedEvent {}.emit(); + let response = self + .process_pd_responses( + &context, + Arc::clone(&prefill), + Arc::clone(&decode), + prefill_result, + decode_result, + ) + .await; + (response, Some(decode)) + } + + async fn process_pd_responses( + &self, + context: &PDRequestContext<'_>, + prefill: Arc, + decode: Arc, + prefill_result: Result, + decode_result: Result, + ) -> Response { // Process decode response match decode_result { Ok(res) => { @@ -821,7 +919,7 @@ impl PDRouter { } let mut response = self - .handle_decode_error_response(res, &context, prefill, decode) + .handle_decode_error_response(res, context, prefill, decode) .await; response.extensions_mut().insert(BreakerOutcomesRecorded); return response; @@ -979,79 +1077,83 @@ impl PDRouter { model_id: Option<&str>, headers: Option<&HeaderMap>, ) -> Result<(Arc, Arc), String> { + let prefill = self + .select_worker(request_text, model_id, headers, false) + .await?; + let decode = self + .select_worker(request_text, model_id, headers, true) + .await?; + Ok((prefill, decode)) + } + + async fn select_worker( + &self, + request_text: Option<&str>, + model_id: Option<&str>, + headers: Option<&HeaderMap>, + decode: bool, + ) -> Result, String> { let effective_model_id = if !self.enable_igw { None } else { model_id }; debug!( - "Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}", - self.enable_igw, model_id, effective_model_id + "Selecting PD worker: decode={}, enable_igw={}, model_id={:?}, effective_model_id={:?}", + decode, self.enable_igw, model_id, effective_model_id ); - let prefill_workers = if let Some(model) = effective_model_id { - self.worker_registry + let workers = match effective_model_id { + Some(model) => self + .worker_registry .get_by_model(model) .iter() - .filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. })) + .filter(|w| match w.worker_type() { + WorkerType::Decode => decode, + WorkerType::Prefill { .. } => !decode, + _ => false, + }) .cloned() - .collect() - } else { - self.worker_registry.get_prefill_workers() + .collect(), + None if decode => self.worker_registry.get_decode_workers(), + None => self.worker_registry.get_prefill_workers(), }; - let decode_workers = if let Some(model) = effective_model_id { - self.worker_registry - .get_by_model(model) - .iter() - .filter(|w| matches!(w.worker_type(), WorkerType::Decode)) - .cloned() - .collect() + let (policy, role, label) = if decode { + ( + self.policy_registry.get_decode_policy(), + "decode", + metrics_labels::WORKER_DECODE, + ) } else { - self.worker_registry.get_decode_workers() + ( + self.policy_registry.get_prefill_policy(), + "prefill", + metrics_labels::WORKER_PREFILL, + ) }; - let prefill_policy = self.policy_registry.get_prefill_policy(); - let decode_policy = self.policy_registry.get_decode_policy(); - // Get cached hash ring for consistent hashing let hash_ring = self .worker_registry .get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID)); - let prefill = Self::pick_worker_by_policy_arc( - &prefill_workers, - &*prefill_policy, - request_text, - headers, - hash_ring.clone(), - "prefill", - ) - .await?; - - let decode = Self::pick_worker_by_policy_arc( - &decode_workers, - &*decode_policy, + let worker = Self::pick_worker_by_policy_arc( + &workers, + &*policy, request_text, headers, hash_ring, - "decode", + role, ) .await?; // Record worker selection metrics (Layer 3) - let model = model_id.unwrap_or(UNKNOWN_MODEL_ID); Metrics::record_worker_selection( - metrics_labels::WORKER_PREFILL, + label, metrics_labels::CONNECTION_HTTP, - model, - prefill_policy.name(), - ); - Metrics::record_worker_selection( - metrics_labels::WORKER_DECODE, - metrics_labels::CONNECTION_HTTP, - model, - decode_policy.name(), + model_id.unwrap_or(UNKNOWN_MODEL_ID), + policy.name(), ); - Ok((prefill, decode)) + Ok(worker) } async fn pick_worker_by_policy_arc( @@ -1778,6 +1880,8 @@ mod tests { retry_config: RetryConfig::default(), api_key: Some("test_api_key".to_string()), enable_igw: false, + decode_gate: None, + request_timeout: Duration::from_secs(600), } } diff --git a/sgl-model-gateway/tests/common/mock_worker.rs b/sgl-model-gateway/tests/common/mock_worker.rs index 131a5bbb63..148e623270 100755 --- a/sgl-model-gateway/tests/common/mock_worker.rs +++ b/sgl-model-gateway/tests/common/mock_worker.rs @@ -308,6 +308,7 @@ async fn generate_handler( Json(payload): Json, ) -> Response { let config = config.read().await; + record_request_arrival(config.port); let worker_id = format!("worker-{}", config.port); if should_fail(&config).await { @@ -1470,6 +1471,27 @@ async fn responses_cancel_handler( // can enable slow streaming WITHOUT relying on the request payload // (the gateway deserializes/re-serializes the body, dropping unknown fields). +static REQUEST_ARRIVALS: OnceLock>>> = OnceLock::new(); + +fn record_request_arrival(port: u16) { + REQUEST_ARRIVALS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap() + .entry(port) + .or_default() + .push(std::time::Instant::now()); +} + +pub fn take_request_arrivals(port: u16) -> Vec { + REQUEST_ARRIVALS + .get_or_init(|| Mutex::new(HashMap::new())) + .lock() + .unwrap() + .remove(&port) + .unwrap_or_default() +} + static SLOW_STREAM_CONFIG: OnceLock>> = OnceLock::new(); fn get_slow_stream_config() -> &'static Mutex> { diff --git a/sgl-model-gateway/tests/routing/pd_routing_test.rs b/sgl-model-gateway/tests/routing/pd_routing_test.rs index 6ad31453d7..423c0c1d4d 100644 --- a/sgl-model-gateway/tests/routing/pd_routing_test.rs +++ b/sgl-model-gateway/tests/routing/pd_routing_test.rs @@ -219,6 +219,87 @@ mod pd_routing_tests { ctx.shutdown().await; } + + /// Prefill gets both requests at once; the second decode waits for a slot. + #[tokio::test] + async fn test_pd_decode_dispatch_limit_holds_decode_not_prefill() { + use std::time::{Duration, Instant}; + + use crate::common::mock_worker::take_request_arrivals; + + let (prefill_port, decode_port) = (19858, 19859); + let config = RouterConfig::builder() + .prefill_decode_mode( + vec![(format!("http://127.0.0.1:{prefill_port}"), None)], + vec![format!("http://127.0.0.1:{decode_port}")], + ) + .round_robin_policy() + .decode_dispatch_limit(1) + .host("127.0.0.1") + .port(3858) + .max_payload_size(256 * 1024 * 1024) + .request_timeout_secs(600) + .worker_startup_timeout_secs(5) + .worker_startup_check_interval_secs(1) + .max_concurrent_requests(64) + .queue_timeout_secs(60) + .build_unchecked(); + let slow = |mut worker: MockWorkerConfig| { + worker.response_delay_ms = 600; + worker + }; + let ctx = AppTestContext::new_with_config( + config, + vec![ + slow(TestWorkerConfig::prefill(prefill_port)), + slow(TestWorkerConfig::decode(decode_port)), + ], + ) + .await; + let app = ctx.create_app().await; + take_request_arrivals(prefill_port); + take_request_arrivals(decode_port); + + let send = || { + let app = app.clone(); + async move { + let request = Request::builder() + .method("POST") + .uri("/generate") + .header(CONTENT_TYPE, "application/json") + .body(Body::from( + json!({"text": "hi", "stream": false}).to_string(), + )) + .unwrap(); + app.oneshot(request).await.unwrap().status() + } + }; + let start = Instant::now(); + let statuses = tokio::join!(send(), send()); + assert_eq!(statuses, (StatusCode::OK, StatusCode::OK)); + + let since_start = |port| { + let mut arrivals: Vec<_> = take_request_arrivals(port) + .into_iter() + .map(|t| t - start) + .collect(); + arrivals.sort(); + arrivals + }; + let prefill = since_start(prefill_port); + let decode = since_start(decode_port); + assert!( + prefill.len() == 2 && prefill[1] < Duration::from_millis(300), + "{prefill:?}" + ); + assert!( + decode.len() == 2 + && decode[0] < Duration::from_millis(300) + && decode[1] >= Duration::from_millis(500), + "{decode:?}" + ); + ctx.shutdown().await; + } } #[cfg(test)] @@ -321,6 +402,8 @@ mod pd_responses_routing_tests { retry_config: RetryConfig::default(), api_key: None, enable_igw: false, + decode_gate: None, + request_timeout: Duration::from_secs(600), } }