[Feat] sgl-model-gateway: Add --decode-dispatch-limit to pick PD decode workers near KV arrival (#44689)

This commit is contained in:
AMD-yanfeiwang
2026-10-20 02:16:21 +08:00
committed by GitHub
parent 80b86dbecb
commit 678913a0f0
14 changed files with 386 additions and 89 deletions
@@ -2670,6 +2670,12 @@ groups:
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td> <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Override policy for decode nodes</td> <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Override policy for decode nodes</td>
</tr> </tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--decode-dispatch-limit`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Requests per prefill worker sent to decode before decode answers or prefill finishes; the rest pick their decode worker when sent. Needs prefill `--optimistic-prefill-attempts` above 0</td>
</tr>
<tr> <tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--worker-startup-timeout-secs`</td> <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--worker-startup-timeout-secs`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td> <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
+2
View File
@@ -160,6 +160,8 @@ The `--version` (or `-V`) flag displays the version string. Use `--version-verbo
``` ```
Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client. Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client.
A decode worker picked on arrival only starts decoding once prefill has queued, computed and sent the KV, so with long prefill queues, decode workers (with `--dp-aware`, DP ranks) that hold equal numbers of requests can run unequal batches. `--decode-dispatch-limit N` still sends every request to prefill on arrival, but sends at most N per prefill worker on to decode before decode answers or prefill finishes; the rest wait in arrival order and pick their decode worker when they are sent. Prefill must start a request before decode connects, so prefill engines need `--optimistic-prefill-attempts` above 0.
### Multi-Model Inference Gateway ### Multi-Model Inference Gateway
Enable IGW mode to route multiple models through a single router while applying per-model policies: Enable IGW mode to route multiple models through a single router while applying per-model policies:
```bash ```bash
@@ -412,6 +412,7 @@ struct Router {
queue_size: usize, queue_size: usize,
queue_timeout_secs: u64, queue_timeout_secs: u64,
rate_limit_tokens_per_second: Option<i32>, rate_limit_tokens_per_second: Option<i32>,
decode_dispatch_limit: Option<usize>,
connection_mode: core::ConnectionMode, connection_mode: core::ConnectionMode,
model_path: Option<String>, model_path: Option<String>,
tokenizer_path: Option<String>, tokenizer_path: Option<String>,
@@ -637,6 +638,7 @@ impl Router {
.maybe_log_level(self.log_level.as_ref()) .maybe_log_level(self.log_level.as_ref())
.maybe_request_id_headers(self.request_id_headers.clone()) .maybe_request_id_headers(self.request_id_headers.clone())
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second) .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
.maybe_model_path(self.model_path.as_ref()) .maybe_model_path(self.model_path.as_ref())
.maybe_tokenizer_path(self.tokenizer_path.as_ref()) .maybe_tokenizer_path(self.tokenizer_path.as_ref())
.maybe_chat_template(self.chat_template.as_ref()) .maybe_chat_template(self.chat_template.as_ref())
@@ -733,6 +735,7 @@ impl Router {
queue_size = 100, queue_size = 100,
queue_timeout_secs = 60, queue_timeout_secs = 60,
rate_limit_tokens_per_second = None, rate_limit_tokens_per_second = None,
decode_dispatch_limit = None,
model_path = None, model_path = None,
tokenizer_path = None, tokenizer_path = None,
chat_template = None, chat_template = None,
@@ -825,6 +828,7 @@ impl Router {
queue_size: usize, queue_size: usize,
queue_timeout_secs: u64, queue_timeout_secs: u64,
rate_limit_tokens_per_second: Option<i32>, rate_limit_tokens_per_second: Option<i32>,
decode_dispatch_limit: Option<usize>,
model_path: Option<String>, model_path: Option<String>,
tokenizer_path: Option<String>, tokenizer_path: Option<String>,
chat_template: Option<String>, chat_template: Option<String>,
@@ -930,6 +934,7 @@ impl Router {
queue_size, queue_size,
queue_timeout_secs, queue_timeout_secs,
rate_limit_tokens_per_second, rate_limit_tokens_per_second,
decode_dispatch_limit,
connection_mode, connection_mode,
model_path, model_path,
tokenizer_path, tokenizer_path,
@@ -211,6 +211,7 @@ class Router:
queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100 queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60 queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
decode_dispatch_limit: Max requests per prefill worker in flight to decode (PD mode only). Default: None
cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: [] cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3 health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2 health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
@@ -98,6 +98,7 @@ class RouterArgs:
queue_timeout_secs: int = 60 queue_timeout_secs: int = 60
# Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests # Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
rate_limit_tokens_per_second: Optional[int] = None rate_limit_tokens_per_second: Optional[int] = None
decode_dispatch_limit: Optional[int] = None
# CORS allowed origins # CORS allowed origins
cors_allowed_origins: List[str] = dataclasses.field(default_factory=list) cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
# Retry configuration # Retry configuration
@@ -398,6 +399,13 @@ class RouterArgs:
metavar=("URL",), metavar=("URL",),
help="Decode server URL. Can be specified multiple times.", help="Decode server URL. Can be specified multiple times.",
) )
pd_group.add_argument(
f"--{prefix}decode-dispatch-limit",
type=int,
default=RouterArgs.decode_dispatch_limit,
help="Max requests per prefill worker in flight to decode; the rest pick their "
"decode worker when sent (PD mode only)",
)
pd_group.add_argument( pd_group.add_argument(
f"--{prefix}worker-startup-timeout-secs", f"--{prefix}worker-startup-timeout-secs",
type=int, type=int,
+10
View File
@@ -229,6 +229,11 @@ impl RouterConfigBuilder {
self self
} }
pub fn decode_dispatch_limit(mut self, limit: usize) -> Self {
self.config.decode_dispatch_limit = Some(limit);
self
}
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self { pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
self.config.rate_limit_tokens_per_second = Some(tokens); self.config.rate_limit_tokens_per_second = Some(tokens);
self self
@@ -531,6 +536,11 @@ impl RouterConfigBuilder {
self self
} }
pub fn maybe_decode_dispatch_limit(mut self, limit: Option<usize>) -> Self {
self.config.decode_dispatch_limit = limit;
self
}
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self { pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self {
self.config.rate_limit_tokens_per_second = tokens; self.config.rate_limit_tokens_per_second = tokens;
self self
+3
View File
@@ -26,6 +26,8 @@ pub struct RouterConfig {
pub worker_startup_timeout_secs: u64, pub worker_startup_timeout_secs: u64,
pub worker_startup_check_interval_secs: u64, pub worker_startup_check_interval_secs: u64,
pub dp_aware: bool, pub dp_aware: bool,
#[serde(default)]
pub decode_dispatch_limit: Option<usize>,
pub api_key: Option<String>, pub api_key: Option<String>,
pub discovery: Option<DiscoveryConfig>, pub discovery: Option<DiscoveryConfig>,
pub metrics: Option<MetricsConfig>, pub metrics: Option<MetricsConfig>,
@@ -514,6 +516,7 @@ impl Default for RouterConfig {
worker_startup_timeout_secs: 1800, // 30 minutes for large model loading worker_startup_timeout_secs: 1800, // 30 minutes for large model loading
worker_startup_check_interval_secs: 30, worker_startup_check_interval_secs: 30,
dp_aware: false, dp_aware: false,
decode_dispatch_limit: None,
api_key: None, api_key: None,
discovery: None, discovery: None,
metrics: None, metrics: None,
@@ -286,6 +286,14 @@ impl ConfigValidator {
}); });
} }
if config.decode_dispatch_limit == Some(0) {
return Err(ConfigError::InvalidValue {
field: "decode_dispatch_limit".to_string(),
value: "0".to_string(),
reason: "Must be > 0 when specified".to_string(),
});
}
if let Some(tokens_per_second) = config.rate_limit_tokens_per_second { if let Some(tokens_per_second) = config.rate_limit_tokens_per_second {
// Allow 0 for pure concurrency limiting (semaphore behavior) // Allow 0 for pure concurrency limiting (semaphore behavior)
if tokens_per_second < 0 { if tokens_per_second < 0 {
+5
View File
@@ -211,6 +211,10 @@ struct CliArgs {
#[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")] #[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")]
decode_policy: Option<String>, decode_policy: Option<String>,
/// Max requests per prefill worker in flight to decode; the rest pick their decode worker when sent
#[arg(long, help_heading = "PD Disaggregation")]
decode_dispatch_limit: Option<usize>,
/// Timeout in seconds for worker startup and registration /// Timeout in seconds for worker startup and registration
#[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")] #[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")]
worker_startup_timeout_secs: u64, worker_startup_timeout_secs: u64,
@@ -1056,6 +1060,7 @@ impl CliArgs {
(!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()), (!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
) )
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second) .maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
.maybe_model_path(self.model_path.as_ref()) .maybe_model_path(self.model_path.as_ref())
.maybe_tokenizer_path(self.tokenizer_path.as_ref()) .maybe_tokenizer_path(self.tokenizer_path.as_ref())
.maybe_chat_template(self.chat_template.as_ref()) .maybe_chat_template(self.chat_template.as_ref())
@@ -0,0 +1,39 @@
use std::sync::{Arc, Mutex};
use dashmap::DashMap;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
#[derive(Debug)]
pub struct DecodeGate {
limit: usize,
gates: DashMap<String, Arc<Semaphore>>,
}
impl DecodeGate {
pub fn new(limit: usize) -> Self {
Self {
limit,
gates: DashMap::new(),
}
}
pub async fn admit(&self, prefill_url: &str, slot: &DecodeSlot) {
let gate = Arc::clone(
&self
.gates
.entry(prefill_url.to_string())
.or_insert_with(|| Arc::new(Semaphore::new(self.limit))),
);
let permit = gate.acquire_owned().await.expect("gates are never closed");
*slot.0.lock().unwrap() = Some(permit);
}
}
#[derive(Debug, Clone, Default)]
pub struct DecodeSlot(Arc<Mutex<Option<OwnedSemaphorePermit>>>);
impl DecodeSlot {
pub fn release(&self) {
self.0.lock().unwrap().take();
}
}
@@ -1,5 +1,6 @@
//! HTTP router implementations //! HTTP router implementations
pub mod decode_gate;
pub mod pd_router; pub mod pd_router;
pub mod pd_types; pub mod pd_types;
pub mod router; pub mod router;
+193 -89
View File
@@ -1,4 +1,8 @@
use std::{borrow::Cow, sync::Arc, time::Instant}; use std::{
borrow::Cow,
sync::Arc,
time::{Duration, Instant},
};
use async_trait::async_trait; use async_trait::async_trait;
use axum::{ use axum::{
@@ -18,7 +22,10 @@ use serde_json::{json, Value};
use tokio_stream::wrappers::UnboundedReceiverStream; use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{debug, error, warn}; use tracing::{debug, error, warn};
use super::pd_types::api_path; use super::{
decode_gate::{DecodeGate, DecodeSlot},
pd_types::api_path,
};
use crate::{ use crate::{
config::types::RetryConfig, config::types::RetryConfig,
core::{ core::{
@@ -58,6 +65,8 @@ pub struct PDRouter {
pub retry_config: RetryConfig, pub retry_config: RetryConfig,
pub api_key: Option<String>, pub api_key: Option<String>,
pub enable_igw: bool, pub enable_igw: bool,
pub decode_gate: Option<Arc<DecodeGate>>,
pub request_timeout: Duration,
} }
struct PreparedWorkerRequest<'a> { struct PreparedWorkerRequest<'a> {
@@ -189,6 +198,11 @@ impl PDRouter {
retry_config: ctx.router_config.effective_retry_config(), retry_config: ctx.router_config.effective_retry_config(),
api_key: ctx.router_config.api_key.clone(), api_key: ctx.router_config.api_key.clone(),
enable_igw: ctx.router_config.enable_igw, enable_igw: ctx.router_config.enable_igw,
decode_gate: ctx
.router_config
.decode_dispatch_limit
.map(|limit| Arc::new(DecodeGate::new(limit))),
request_timeout: Duration::from_secs(ctx.router_config.request_timeout_secs),
}) })
} }
@@ -350,6 +364,7 @@ impl PDRouter {
}) })
} }
#[cfg(test)]
async fn prepare_pd_worker_requests<'a>( async fn prepare_pd_worker_requests<'a>(
route: &'static str, route: &'static str,
json_request: &'a Value, json_request: &'a Value,
@@ -398,14 +413,26 @@ impl PDRouter {
let shared_request = Arc::clone(&shared_request); let shared_request = Arc::clone(&shared_request);
let context = context.clone(); let context = context.clone();
async move { async move {
let (prefill, decode) = match self let request_text = context.request_text.as_deref();
.select_pd_pair( let selected = if self.decode_gate.is_some() {
context.request_text.as_deref(), self.select_worker(
request_text,
context.model_id,
context.headers.as_ref(),
false,
)
.await
.map(|prefill| (prefill, None))
} else {
self.select_pd_pair(
request_text,
context.model_id, context.model_id,
context.headers.as_ref(), context.headers.as_ref(),
) )
.await .await
{ .map(|(prefill, decode)| (prefill, Some(decode)))
};
let (prefill, decode) = match selected {
Ok(pair) => pair, Ok(pair) => pair,
Err(e) => { Err(e) => {
return Self::handle_server_selection_error(e); return Self::handle_server_selection_error(e);
@@ -413,10 +440,10 @@ impl PDRouter {
}; };
debug!( debug!(
"PD retry attempt {} using prefill={} decode={}", "PD retry attempt {} using prefill={} decode={:?}",
attempt, attempt,
prefill.url(), prefill.url(),
decode.url() decode.as_ref().map(|d| d.url())
); );
let mut json_request = match serde_json::to_value(shared_request.as_ref()) { let mut json_request = match serde_json::to_value(shared_request.as_ref()) {
@@ -438,13 +465,13 @@ impl PDRouter {
}; };
let ctx_is_stream = context.is_stream; let ctx_is_stream = context.is_stream;
let response = self let (response, decode) = self
.execute_dual_dispatch_internal( .execute_dual_dispatch_internal(
headers, headers,
json_request, json_request,
context, context,
Arc::clone(&prefill), Arc::clone(&prefill),
Arc::clone(&decode), decode,
start_time, start_time,
) )
.await; .await;
@@ -464,7 +491,7 @@ impl PDRouter {
// its byte stream records the outcome on drop. Skip the // its byte stream records the outcome on drop. Skip the
// eager success record to avoid masking "200-then-broken" // eager success record to avoid masking "200-then-broken"
// decode workers. // decode workers.
if !ctx_is_stream { if let (false, Some(decode)) = (ctx_is_stream, &decode) {
decode.record_outcome(not_error); decode.record_outcome(not_error);
} }
} }
@@ -649,40 +676,40 @@ impl PDRouter {
} }
// Internal method that performs the actual dual dispatch (without retry logic) // Internal method that performs the actual dual dispatch (without retry logic)
#[allow(clippy::too_many_arguments)]
async fn execute_dual_dispatch_internal( async fn execute_dual_dispatch_internal(
&self, &self,
headers: Option<&HeaderMap>, headers: Option<&HeaderMap>,
json_request: Value, json_request: Value,
context: PDRequestContext<'_>, context: PDRequestContext<'_>,
prefill: Arc<dyn Worker>, prefill: Arc<dyn Worker>,
decode: Arc<dyn Worker>, decode: Option<Arc<dyn Worker>>,
_start_time: Instant, _start_time: Instant,
) -> Response { ) -> (Response, Option<Arc<dyn Worker>>) {
// Count the request from dispatch so load-based policies see it before // Count the request from dispatch so load-based policies see it before
// upstream headers arrive; streaming responses attach their own guards. // upstream headers arrive; streaming responses attach their own guards.
let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers); let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers);
let _decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
let mut headers_with_trace = headers.cloned().unwrap_or_default(); let mut headers_with_trace = headers.cloned().unwrap_or_default();
inject_trace_context_http(&mut headers_with_trace); inject_trace_context_http(&mut headers_with_trace);
let headers = Some(&headers_with_trace); let headers = Some(&headers_with_trace);
let (prepared_prefill, prepared_decode) = match Self::prepare_pd_worker_requests( let prepared_prefill = match Self::prepare_worker_request(
context.route, context.route,
&json_request,
prefill.as_ref(), prefill.as_ref(),
decode.as_ref(), Cow::Borrowed(&json_request),
) )
.await .await
{ {
Ok(requests) => requests, Ok(request) => request,
Err(e) => { Err(e) => {
error!("Failed to prepare PD worker requests: {}", e); error!("Failed to prepare PD worker requests: {}", e);
return error::internal_error("pd_request_preparation_failed", e); return (
error::internal_error("pd_request_preparation_failed", e),
None,
);
} }
}; };
// Build both requests
let prefill_request = self.build_post_with_headers( let prefill_request = self.build_post_with_headers(
&self.client, &self.client,
&prepared_prefill.endpoint_url, &prepared_prefill.endpoint_url,
@@ -690,32 +717,77 @@ impl PDRouter {
headers, headers,
false, false,
); );
let decode_request = self.build_post_with_headers(
&self.client, let slot = decode.is_none().then(DecodeSlot::default);
&prepared_decode.endpoint_url, let decode_leg = async {
&prepared_decode.body, if let (Some(gate), Some(slot)) = (&self.decode_gate, &slot) {
headers, let admit = gate.admit(prefill.url(), slot);
false, if tokio::time::timeout(self.request_timeout, admit)
); .await
.is_err()
{
return Err(error::create_error(
StatusCode::GATEWAY_TIMEOUT,
"decode_dispatch_timeout",
"Timed out waiting to dispatch the decode request",
));
}
}
let decode = match decode {
Some(decode) => decode,
None => self
.select_worker(
context.request_text.as_deref(),
context.model_id,
context.headers.as_ref(),
true,
)
.await
.map_err(Self::handle_server_selection_error)?,
};
let decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
let prepared_decode = async {
let body = Self::inject_prefill_dp_rank_for_decode(
Cow::Borrowed(&json_request),
prefill.as_ref(),
)?;
Self::prepare_worker_request(context.route, decode.as_ref(), body).await
}
.await
.map_err(|e| {
error!("Failed to prepare PD worker requests: {}", e);
error::internal_error("pd_request_preparation_failed", e)
})?;
let decode_request = self.build_post_with_headers(
&self.client,
&prepared_decode.endpoint_url,
&prepared_decode.body,
headers,
false,
);
events::RequestPDSentEvent {
prefill_url: prefill.url(),
decode_url: decode.url(),
}
.emit();
let result = decode_request.send().await;
if let Some(slot) = &slot {
slot.release();
}
Ok((decode, decode_guard, result))
};
// Run both in this handler task (not a detached tokio::spawn) so a client // Run both in this handler task (not a detached tokio::spawn) so a client
// disconnect cancels the pending decode request too, keeping the // disconnect cancels the pending decode request too, keeping the
// upstream-cancel behavior from #19524. // upstream-cancel behavior from #19524.
events::RequestPDSentEvent {
prefill_url: prefill.url(),
decode_url: decode.url(),
}
.emit();
let prefill_fut = prefill_request.send(); let prefill_fut = prefill_request.send();
let decode_fut = decode_request.send();
tokio::pin!(prefill_fut); tokio::pin!(prefill_fut);
tokio::pin!(decode_fut); tokio::pin!(decode_leg);
// Poll both until prefill resolves; decode normally resolves later, but // Poll both until prefill resolves; decode normally resolves later, but
// may resolve first if it rejects the request outright. // may resolve first if it rejects the request outright.
let prefill_result; let prefill_result;
let mut decode_early: Option<Result<reqwest::Response, reqwest::Error>> = None; let mut decode_early = None;
loop { loop {
tokio::select! { tokio::select! {
biased; biased;
@@ -723,11 +795,14 @@ impl PDRouter {
prefill_result = pr; prefill_result = pr;
break; break;
} }
dr = &mut decode_fut, if decode_early.is_none() => { dr = &mut decode_leg, if decode_early.is_none() => {
decode_early = Some(dr); decode_early = Some(dr);
} }
} }
} }
if let Some(slot) = &slot {
slot.release();
}
// Decode can't generate without prefill's KV, so any prefill failure // Decode can't generate without prefill's KV, so any prefill failure
// (non-2xx / transport error) dooms the paired decode request, which would // (non-2xx / transport error) dooms the paired decode request, which would
@@ -741,8 +816,7 @@ impl PDRouter {
if prefill_failed { if prefill_failed {
warn!( warn!(
"Prefill failed, aborting paired decode request decode_url={} prefill_url={}", "Prefill failed, aborting paired decode request prefill_url={}",
decode.url(),
prefill.url() prefill.url()
); );
@@ -767,17 +841,41 @@ impl PDRouter {
), ),
}; };
response.extensions_mut().insert(BreakerOutcomesRecorded); response.extensions_mut().insert(BreakerOutcomesRecorded);
return response; return (response, None);
} }
// Prefill ok: take decode's result, awaiting it if still pending. // Prefill ok: take decode's result, awaiting it if still pending.
let decode_result = match decode_early { let leg = match decode_early {
Some(dr) => dr, Some(leg) => leg,
None => (&mut decode_fut).await, None => (&mut decode_leg).await,
};
let (decode, _decode_guard, decode_result) = match leg {
Ok(leg) => leg,
Err(response) => return (response, None),
}; };
events::RequestReceivedEvent {}.emit(); events::RequestReceivedEvent {}.emit();
let response = self
.process_pd_responses(
&context,
Arc::clone(&prefill),
Arc::clone(&decode),
prefill_result,
decode_result,
)
.await;
(response, Some(decode))
}
async fn process_pd_responses(
&self,
context: &PDRequestContext<'_>,
prefill: Arc<dyn Worker>,
decode: Arc<dyn Worker>,
prefill_result: Result<reqwest::Response, reqwest::Error>,
decode_result: Result<reqwest::Response, reqwest::Error>,
) -> Response {
// Process decode response // Process decode response
match decode_result { match decode_result {
Ok(res) => { Ok(res) => {
@@ -821,7 +919,7 @@ impl PDRouter {
} }
let mut response = self let mut response = self
.handle_decode_error_response(res, &context, prefill, decode) .handle_decode_error_response(res, context, prefill, decode)
.await; .await;
response.extensions_mut().insert(BreakerOutcomesRecorded); response.extensions_mut().insert(BreakerOutcomesRecorded);
return response; return response;
@@ -979,79 +1077,83 @@ impl PDRouter {
model_id: Option<&str>, model_id: Option<&str>,
headers: Option<&HeaderMap>, headers: Option<&HeaderMap>,
) -> Result<(Arc<dyn Worker>, Arc<dyn Worker>), String> { ) -> Result<(Arc<dyn Worker>, Arc<dyn Worker>), String> {
let prefill = self
.select_worker(request_text, model_id, headers, false)
.await?;
let decode = self
.select_worker(request_text, model_id, headers, true)
.await?;
Ok((prefill, decode))
}
async fn select_worker(
&self,
request_text: Option<&str>,
model_id: Option<&str>,
headers: Option<&HeaderMap>,
decode: bool,
) -> Result<Arc<dyn Worker>, String> {
let effective_model_id = if !self.enable_igw { None } else { model_id }; let effective_model_id = if !self.enable_igw { None } else { model_id };
debug!( debug!(
"Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}", "Selecting PD worker: decode={}, enable_igw={}, model_id={:?}, effective_model_id={:?}",
self.enable_igw, model_id, effective_model_id decode, self.enable_igw, model_id, effective_model_id
); );
let prefill_workers = if let Some(model) = effective_model_id { let workers = match effective_model_id {
self.worker_registry Some(model) => self
.worker_registry
.get_by_model(model) .get_by_model(model)
.iter() .iter()
.filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. })) .filter(|w| match w.worker_type() {
WorkerType::Decode => decode,
WorkerType::Prefill { .. } => !decode,
_ => false,
})
.cloned() .cloned()
.collect() .collect(),
} else { None if decode => self.worker_registry.get_decode_workers(),
self.worker_registry.get_prefill_workers() None => self.worker_registry.get_prefill_workers(),
}; };
let decode_workers = if let Some(model) = effective_model_id { let (policy, role, label) = if decode {
self.worker_registry (
.get_by_model(model) self.policy_registry.get_decode_policy(),
.iter() "decode",
.filter(|w| matches!(w.worker_type(), WorkerType::Decode)) metrics_labels::WORKER_DECODE,
.cloned() )
.collect()
} else { } else {
self.worker_registry.get_decode_workers() (
self.policy_registry.get_prefill_policy(),
"prefill",
metrics_labels::WORKER_PREFILL,
)
}; };
let prefill_policy = self.policy_registry.get_prefill_policy();
let decode_policy = self.policy_registry.get_decode_policy();
// Get cached hash ring for consistent hashing // Get cached hash ring for consistent hashing
let hash_ring = self let hash_ring = self
.worker_registry .worker_registry
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID)); .get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
let prefill = Self::pick_worker_by_policy_arc( let worker = Self::pick_worker_by_policy_arc(
&prefill_workers, &workers,
&*prefill_policy, &*policy,
request_text,
headers,
hash_ring.clone(),
"prefill",
)
.await?;
let decode = Self::pick_worker_by_policy_arc(
&decode_workers,
&*decode_policy,
request_text, request_text,
headers, headers,
hash_ring, hash_ring,
"decode", role,
) )
.await?; .await?;
// Record worker selection metrics (Layer 3) // Record worker selection metrics (Layer 3)
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
Metrics::record_worker_selection( Metrics::record_worker_selection(
metrics_labels::WORKER_PREFILL, label,
metrics_labels::CONNECTION_HTTP, metrics_labels::CONNECTION_HTTP,
model, model_id.unwrap_or(UNKNOWN_MODEL_ID),
prefill_policy.name(), policy.name(),
);
Metrics::record_worker_selection(
metrics_labels::WORKER_DECODE,
metrics_labels::CONNECTION_HTTP,
model,
decode_policy.name(),
); );
Ok((prefill, decode)) Ok(worker)
} }
async fn pick_worker_by_policy_arc( async fn pick_worker_by_policy_arc(
@@ -1778,6 +1880,8 @@ mod tests {
retry_config: RetryConfig::default(), retry_config: RetryConfig::default(),
api_key: Some("test_api_key".to_string()), api_key: Some("test_api_key".to_string()),
enable_igw: false, enable_igw: false,
decode_gate: None,
request_timeout: Duration::from_secs(600),
} }
} }
@@ -308,6 +308,7 @@ async fn generate_handler(
Json(payload): Json<serde_json::Value>, Json(payload): Json<serde_json::Value>,
) -> Response { ) -> Response {
let config = config.read().await; let config = config.read().await;
record_request_arrival(config.port);
let worker_id = format!("worker-{}", config.port); let worker_id = format!("worker-{}", config.port);
if should_fail(&config).await { if should_fail(&config).await {
@@ -1470,6 +1471,27 @@ async fn responses_cancel_handler(
// can enable slow streaming WITHOUT relying on the request payload // can enable slow streaming WITHOUT relying on the request payload
// (the gateway deserializes/re-serializes the body, dropping unknown fields). // (the gateway deserializes/re-serializes the body, dropping unknown fields).
static REQUEST_ARRIVALS: OnceLock<Mutex<HashMap<u16, Vec<std::time::Instant>>>> = OnceLock::new();
fn record_request_arrival(port: u16) {
REQUEST_ARRIVALS
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap()
.entry(port)
.or_default()
.push(std::time::Instant::now());
}
pub fn take_request_arrivals(port: u16) -> Vec<std::time::Instant> {
REQUEST_ARRIVALS
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap()
.remove(&port)
.unwrap_or_default()
}
static SLOW_STREAM_CONFIG: OnceLock<Mutex<HashMap<u16, usize>>> = OnceLock::new(); static SLOW_STREAM_CONFIG: OnceLock<Mutex<HashMap<u16, usize>>> = OnceLock::new();
fn get_slow_stream_config() -> &'static Mutex<HashMap<u16, usize>> { fn get_slow_stream_config() -> &'static Mutex<HashMap<u16, usize>> {
@@ -219,6 +219,87 @@ mod pd_routing_tests {
ctx.shutdown().await; ctx.shutdown().await;
} }
/// Prefill gets both requests at once; the second decode waits for a slot.
#[tokio::test]
async fn test_pd_decode_dispatch_limit_holds_decode_not_prefill() {
use std::time::{Duration, Instant};
use crate::common::mock_worker::take_request_arrivals;
let (prefill_port, decode_port) = (19858, 19859);
let config = RouterConfig::builder()
.prefill_decode_mode(
vec![(format!("http://127.0.0.1:{prefill_port}"), None)],
vec![format!("http://127.0.0.1:{decode_port}")],
)
.round_robin_policy()
.decode_dispatch_limit(1)
.host("127.0.0.1")
.port(3858)
.max_payload_size(256 * 1024 * 1024)
.request_timeout_secs(600)
.worker_startup_timeout_secs(5)
.worker_startup_check_interval_secs(1)
.max_concurrent_requests(64)
.queue_timeout_secs(60)
.build_unchecked();
let slow = |mut worker: MockWorkerConfig| {
worker.response_delay_ms = 600;
worker
};
let ctx = AppTestContext::new_with_config(
config,
vec![
slow(TestWorkerConfig::prefill(prefill_port)),
slow(TestWorkerConfig::decode(decode_port)),
],
)
.await;
let app = ctx.create_app().await;
take_request_arrivals(prefill_port);
take_request_arrivals(decode_port);
let send = || {
let app = app.clone();
async move {
let request = Request::builder()
.method("POST")
.uri("/generate")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(
json!({"text": "hi", "stream": false}).to_string(),
))
.unwrap();
app.oneshot(request).await.unwrap().status()
}
};
let start = Instant::now();
let statuses = tokio::join!(send(), send());
assert_eq!(statuses, (StatusCode::OK, StatusCode::OK));
let since_start = |port| {
let mut arrivals: Vec<_> = take_request_arrivals(port)
.into_iter()
.map(|t| t - start)
.collect();
arrivals.sort();
arrivals
};
let prefill = since_start(prefill_port);
let decode = since_start(decode_port);
assert!(
prefill.len() == 2 && prefill[1] < Duration::from_millis(300),
"{prefill:?}"
);
assert!(
decode.len() == 2
&& decode[0] < Duration::from_millis(300)
&& decode[1] >= Duration::from_millis(500),
"{decode:?}"
);
ctx.shutdown().await;
}
} }
#[cfg(test)] #[cfg(test)]
@@ -321,6 +402,8 @@ mod pd_responses_routing_tests {
retry_config: RetryConfig::default(), retry_config: RetryConfig::default(),
api_key: None, api_key: None,
enable_igw: false, enable_igw: false,
decode_gate: None,
request_timeout: Duration::from_secs(600),
} }
} }