[Feat] sgl-model-gateway: Add --decode-dispatch-limit to pick PD decode workers near KV arrival (#44689)
This commit is contained in:
@@ -2670,6 +2670,12 @@ groups:
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Override policy for decode nodes</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--decode-dispatch-limit`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Requests per prefill worker sent to decode before decode answers or prefill finishes; the rest pick their decode worker when sent. Needs prefill `--optimistic-prefill-attempts` above 0</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--worker-startup-timeout-secs`</td>
|
||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
|
||||
|
||||
@@ -160,6 +160,8 @@ The `--version` (or `-V`) flag displays the version string. Use `--version-verbo
|
||||
```
|
||||
Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client.
|
||||
|
||||
A decode worker picked on arrival only starts decoding once prefill has queued, computed and sent the KV, so with long prefill queues, decode workers (with `--dp-aware`, DP ranks) that hold equal numbers of requests can run unequal batches. `--decode-dispatch-limit N` still sends every request to prefill on arrival, but sends at most N per prefill worker on to decode before decode answers or prefill finishes; the rest wait in arrival order and pick their decode worker when they are sent. Prefill must start a request before decode connects, so prefill engines need `--optimistic-prefill-attempts` above 0.
|
||||
|
||||
### Multi-Model Inference Gateway
|
||||
Enable IGW mode to route multiple models through a single router while applying per-model policies:
|
||||
```bash
|
||||
|
||||
@@ -412,6 +412,7 @@ struct Router {
|
||||
queue_size: usize,
|
||||
queue_timeout_secs: u64,
|
||||
rate_limit_tokens_per_second: Option<i32>,
|
||||
decode_dispatch_limit: Option<usize>,
|
||||
connection_mode: core::ConnectionMode,
|
||||
model_path: Option<String>,
|
||||
tokenizer_path: Option<String>,
|
||||
@@ -637,6 +638,7 @@ impl Router {
|
||||
.maybe_log_level(self.log_level.as_ref())
|
||||
.maybe_request_id_headers(self.request_id_headers.clone())
|
||||
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
|
||||
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
|
||||
.maybe_model_path(self.model_path.as_ref())
|
||||
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
|
||||
.maybe_chat_template(self.chat_template.as_ref())
|
||||
@@ -733,6 +735,7 @@ impl Router {
|
||||
queue_size = 100,
|
||||
queue_timeout_secs = 60,
|
||||
rate_limit_tokens_per_second = None,
|
||||
decode_dispatch_limit = None,
|
||||
model_path = None,
|
||||
tokenizer_path = None,
|
||||
chat_template = None,
|
||||
@@ -825,6 +828,7 @@ impl Router {
|
||||
queue_size: usize,
|
||||
queue_timeout_secs: u64,
|
||||
rate_limit_tokens_per_second: Option<i32>,
|
||||
decode_dispatch_limit: Option<usize>,
|
||||
model_path: Option<String>,
|
||||
tokenizer_path: Option<String>,
|
||||
chat_template: Option<String>,
|
||||
@@ -930,6 +934,7 @@ impl Router {
|
||||
queue_size,
|
||||
queue_timeout_secs,
|
||||
rate_limit_tokens_per_second,
|
||||
decode_dispatch_limit,
|
||||
connection_mode,
|
||||
model_path,
|
||||
tokenizer_path,
|
||||
|
||||
@@ -211,6 +211,7 @@ class Router:
|
||||
queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
|
||||
queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
|
||||
rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
|
||||
decode_dispatch_limit: Max requests per prefill worker in flight to decode (PD mode only). Default: None
|
||||
cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
|
||||
health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
|
||||
health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
|
||||
|
||||
@@ -98,6 +98,7 @@ class RouterArgs:
|
||||
queue_timeout_secs: int = 60
|
||||
# Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
|
||||
rate_limit_tokens_per_second: Optional[int] = None
|
||||
decode_dispatch_limit: Optional[int] = None
|
||||
# CORS allowed origins
|
||||
cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
|
||||
# Retry configuration
|
||||
@@ -398,6 +399,13 @@ class RouterArgs:
|
||||
metavar=("URL",),
|
||||
help="Decode server URL. Can be specified multiple times.",
|
||||
)
|
||||
pd_group.add_argument(
|
||||
f"--{prefix}decode-dispatch-limit",
|
||||
type=int,
|
||||
default=RouterArgs.decode_dispatch_limit,
|
||||
help="Max requests per prefill worker in flight to decode; the rest pick their "
|
||||
"decode worker when sent (PD mode only)",
|
||||
)
|
||||
pd_group.add_argument(
|
||||
f"--{prefix}worker-startup-timeout-secs",
|
||||
type=int,
|
||||
|
||||
@@ -229,6 +229,11 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn decode_dispatch_limit(mut self, limit: usize) -> Self {
|
||||
self.config.decode_dispatch_limit = Some(limit);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
|
||||
self.config.rate_limit_tokens_per_second = Some(tokens);
|
||||
self
|
||||
@@ -531,6 +536,11 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn maybe_decode_dispatch_limit(mut self, limit: Option<usize>) -> Self {
|
||||
self.config.decode_dispatch_limit = limit;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self {
|
||||
self.config.rate_limit_tokens_per_second = tokens;
|
||||
self
|
||||
|
||||
@@ -26,6 +26,8 @@ pub struct RouterConfig {
|
||||
pub worker_startup_timeout_secs: u64,
|
||||
pub worker_startup_check_interval_secs: u64,
|
||||
pub dp_aware: bool,
|
||||
#[serde(default)]
|
||||
pub decode_dispatch_limit: Option<usize>,
|
||||
pub api_key: Option<String>,
|
||||
pub discovery: Option<DiscoveryConfig>,
|
||||
pub metrics: Option<MetricsConfig>,
|
||||
@@ -514,6 +516,7 @@ impl Default for RouterConfig {
|
||||
worker_startup_timeout_secs: 1800, // 30 minutes for large model loading
|
||||
worker_startup_check_interval_secs: 30,
|
||||
dp_aware: false,
|
||||
decode_dispatch_limit: None,
|
||||
api_key: None,
|
||||
discovery: None,
|
||||
metrics: None,
|
||||
|
||||
@@ -286,6 +286,14 @@ impl ConfigValidator {
|
||||
});
|
||||
}
|
||||
|
||||
if config.decode_dispatch_limit == Some(0) {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
field: "decode_dispatch_limit".to_string(),
|
||||
value: "0".to_string(),
|
||||
reason: "Must be > 0 when specified".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(tokens_per_second) = config.rate_limit_tokens_per_second {
|
||||
// Allow 0 for pure concurrency limiting (semaphore behavior)
|
||||
if tokens_per_second < 0 {
|
||||
|
||||
@@ -211,6 +211,10 @@ struct CliArgs {
|
||||
#[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")]
|
||||
decode_policy: Option<String>,
|
||||
|
||||
/// Max requests per prefill worker in flight to decode; the rest pick their decode worker when sent
|
||||
#[arg(long, help_heading = "PD Disaggregation")]
|
||||
decode_dispatch_limit: Option<usize>,
|
||||
|
||||
/// Timeout in seconds for worker startup and registration
|
||||
#[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")]
|
||||
worker_startup_timeout_secs: u64,
|
||||
@@ -1056,6 +1060,7 @@ impl CliArgs {
|
||||
(!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
|
||||
)
|
||||
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
|
||||
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
|
||||
.maybe_model_path(self.model_path.as_ref())
|
||||
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
|
||||
.maybe_chat_template(self.chat_template.as_ref())
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use dashmap::DashMap;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DecodeGate {
|
||||
limit: usize,
|
||||
gates: DashMap<String, Arc<Semaphore>>,
|
||||
}
|
||||
|
||||
impl DecodeGate {
|
||||
pub fn new(limit: usize) -> Self {
|
||||
Self {
|
||||
limit,
|
||||
gates: DashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn admit(&self, prefill_url: &str, slot: &DecodeSlot) {
|
||||
let gate = Arc::clone(
|
||||
&self
|
||||
.gates
|
||||
.entry(prefill_url.to_string())
|
||||
.or_insert_with(|| Arc::new(Semaphore::new(self.limit))),
|
||||
);
|
||||
let permit = gate.acquire_owned().await.expect("gates are never closed");
|
||||
*slot.0.lock().unwrap() = Some(permit);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct DecodeSlot(Arc<Mutex<Option<OwnedSemaphorePermit>>>);
|
||||
|
||||
impl DecodeSlot {
|
||||
pub fn release(&self) {
|
||||
self.0.lock().unwrap().take();
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
//! HTTP router implementations
|
||||
|
||||
pub mod decode_gate;
|
||||
pub mod pd_router;
|
||||
pub mod pd_types;
|
||||
pub mod router;
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
use std::{borrow::Cow, sync::Arc, time::Instant};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use async_trait::async_trait;
|
||||
use axum::{
|
||||
@@ -18,7 +22,10 @@ use serde_json::{json, Value};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::pd_types::api_path;
|
||||
use super::{
|
||||
decode_gate::{DecodeGate, DecodeSlot},
|
||||
pd_types::api_path,
|
||||
};
|
||||
use crate::{
|
||||
config::types::RetryConfig,
|
||||
core::{
|
||||
@@ -58,6 +65,8 @@ pub struct PDRouter {
|
||||
pub retry_config: RetryConfig,
|
||||
pub api_key: Option<String>,
|
||||
pub enable_igw: bool,
|
||||
pub decode_gate: Option<Arc<DecodeGate>>,
|
||||
pub request_timeout: Duration,
|
||||
}
|
||||
|
||||
struct PreparedWorkerRequest<'a> {
|
||||
@@ -189,6 +198,11 @@ impl PDRouter {
|
||||
retry_config: ctx.router_config.effective_retry_config(),
|
||||
api_key: ctx.router_config.api_key.clone(),
|
||||
enable_igw: ctx.router_config.enable_igw,
|
||||
decode_gate: ctx
|
||||
.router_config
|
||||
.decode_dispatch_limit
|
||||
.map(|limit| Arc::new(DecodeGate::new(limit))),
|
||||
request_timeout: Duration::from_secs(ctx.router_config.request_timeout_secs),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -350,6 +364,7 @@ impl PDRouter {
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn prepare_pd_worker_requests<'a>(
|
||||
route: &'static str,
|
||||
json_request: &'a Value,
|
||||
@@ -398,14 +413,26 @@ impl PDRouter {
|
||||
let shared_request = Arc::clone(&shared_request);
|
||||
let context = context.clone();
|
||||
async move {
|
||||
let (prefill, decode) = match self
|
||||
.select_pd_pair(
|
||||
context.request_text.as_deref(),
|
||||
let request_text = context.request_text.as_deref();
|
||||
let selected = if self.decode_gate.is_some() {
|
||||
self.select_worker(
|
||||
request_text,
|
||||
context.model_id,
|
||||
context.headers.as_ref(),
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.map(|prefill| (prefill, None))
|
||||
} else {
|
||||
self.select_pd_pair(
|
||||
request_text,
|
||||
context.model_id,
|
||||
context.headers.as_ref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
.map(|(prefill, decode)| (prefill, Some(decode)))
|
||||
};
|
||||
let (prefill, decode) = match selected {
|
||||
Ok(pair) => pair,
|
||||
Err(e) => {
|
||||
return Self::handle_server_selection_error(e);
|
||||
@@ -413,10 +440,10 @@ impl PDRouter {
|
||||
};
|
||||
|
||||
debug!(
|
||||
"PD retry attempt {} using prefill={} decode={}",
|
||||
"PD retry attempt {} using prefill={} decode={:?}",
|
||||
attempt,
|
||||
prefill.url(),
|
||||
decode.url()
|
||||
decode.as_ref().map(|d| d.url())
|
||||
);
|
||||
|
||||
let mut json_request = match serde_json::to_value(shared_request.as_ref()) {
|
||||
@@ -438,13 +465,13 @@ impl PDRouter {
|
||||
};
|
||||
|
||||
let ctx_is_stream = context.is_stream;
|
||||
let response = self
|
||||
let (response, decode) = self
|
||||
.execute_dual_dispatch_internal(
|
||||
headers,
|
||||
json_request,
|
||||
context,
|
||||
Arc::clone(&prefill),
|
||||
Arc::clone(&decode),
|
||||
decode,
|
||||
start_time,
|
||||
)
|
||||
.await;
|
||||
@@ -464,7 +491,7 @@ impl PDRouter {
|
||||
// its byte stream records the outcome on drop. Skip the
|
||||
// eager success record to avoid masking "200-then-broken"
|
||||
// decode workers.
|
||||
if !ctx_is_stream {
|
||||
if let (false, Some(decode)) = (ctx_is_stream, &decode) {
|
||||
decode.record_outcome(not_error);
|
||||
}
|
||||
}
|
||||
@@ -649,40 +676,40 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
// Internal method that performs the actual dual dispatch (without retry logic)
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_dual_dispatch_internal(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
json_request: Value,
|
||||
context: PDRequestContext<'_>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
decode: Option<Arc<dyn Worker>>,
|
||||
_start_time: Instant,
|
||||
) -> Response {
|
||||
) -> (Response, Option<Arc<dyn Worker>>) {
|
||||
// Count the request from dispatch so load-based policies see it before
|
||||
// upstream headers arrive; streaming responses attach their own guards.
|
||||
let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers);
|
||||
let _decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
|
||||
|
||||
let mut headers_with_trace = headers.cloned().unwrap_or_default();
|
||||
inject_trace_context_http(&mut headers_with_trace);
|
||||
let headers = Some(&headers_with_trace);
|
||||
|
||||
let (prepared_prefill, prepared_decode) = match Self::prepare_pd_worker_requests(
|
||||
let prepared_prefill = match Self::prepare_worker_request(
|
||||
context.route,
|
||||
&json_request,
|
||||
prefill.as_ref(),
|
||||
decode.as_ref(),
|
||||
Cow::Borrowed(&json_request),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(requests) => requests,
|
||||
Ok(request) => request,
|
||||
Err(e) => {
|
||||
error!("Failed to prepare PD worker requests: {}", e);
|
||||
return error::internal_error("pd_request_preparation_failed", e);
|
||||
return (
|
||||
error::internal_error("pd_request_preparation_failed", e),
|
||||
None,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Build both requests
|
||||
let prefill_request = self.build_post_with_headers(
|
||||
&self.client,
|
||||
&prepared_prefill.endpoint_url,
|
||||
@@ -690,32 +717,77 @@ impl PDRouter {
|
||||
headers,
|
||||
false,
|
||||
);
|
||||
let decode_request = self.build_post_with_headers(
|
||||
&self.client,
|
||||
&prepared_decode.endpoint_url,
|
||||
&prepared_decode.body,
|
||||
headers,
|
||||
false,
|
||||
);
|
||||
|
||||
let slot = decode.is_none().then(DecodeSlot::default);
|
||||
let decode_leg = async {
|
||||
if let (Some(gate), Some(slot)) = (&self.decode_gate, &slot) {
|
||||
let admit = gate.admit(prefill.url(), slot);
|
||||
if tokio::time::timeout(self.request_timeout, admit)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
return Err(error::create_error(
|
||||
StatusCode::GATEWAY_TIMEOUT,
|
||||
"decode_dispatch_timeout",
|
||||
"Timed out waiting to dispatch the decode request",
|
||||
));
|
||||
}
|
||||
}
|
||||
let decode = match decode {
|
||||
Some(decode) => decode,
|
||||
None => self
|
||||
.select_worker(
|
||||
context.request_text.as_deref(),
|
||||
context.model_id,
|
||||
context.headers.as_ref(),
|
||||
true,
|
||||
)
|
||||
.await
|
||||
.map_err(Self::handle_server_selection_error)?,
|
||||
};
|
||||
let decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
|
||||
let prepared_decode = async {
|
||||
let body = Self::inject_prefill_dp_rank_for_decode(
|
||||
Cow::Borrowed(&json_request),
|
||||
prefill.as_ref(),
|
||||
)?;
|
||||
Self::prepare_worker_request(context.route, decode.as_ref(), body).await
|
||||
}
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("Failed to prepare PD worker requests: {}", e);
|
||||
error::internal_error("pd_request_preparation_failed", e)
|
||||
})?;
|
||||
let decode_request = self.build_post_with_headers(
|
||||
&self.client,
|
||||
&prepared_decode.endpoint_url,
|
||||
&prepared_decode.body,
|
||||
headers,
|
||||
false,
|
||||
);
|
||||
events::RequestPDSentEvent {
|
||||
prefill_url: prefill.url(),
|
||||
decode_url: decode.url(),
|
||||
}
|
||||
.emit();
|
||||
let result = decode_request.send().await;
|
||||
if let Some(slot) = &slot {
|
||||
slot.release();
|
||||
}
|
||||
Ok((decode, decode_guard, result))
|
||||
};
|
||||
|
||||
// Run both in this handler task (not a detached tokio::spawn) so a client
|
||||
// disconnect cancels the pending decode request too, keeping the
|
||||
// upstream-cancel behavior from #19524.
|
||||
events::RequestPDSentEvent {
|
||||
prefill_url: prefill.url(),
|
||||
decode_url: decode.url(),
|
||||
}
|
||||
.emit();
|
||||
|
||||
let prefill_fut = prefill_request.send();
|
||||
let decode_fut = decode_request.send();
|
||||
tokio::pin!(prefill_fut);
|
||||
tokio::pin!(decode_fut);
|
||||
tokio::pin!(decode_leg);
|
||||
|
||||
// Poll both until prefill resolves; decode normally resolves later, but
|
||||
// may resolve first if it rejects the request outright.
|
||||
let prefill_result;
|
||||
let mut decode_early: Option<Result<reqwest::Response, reqwest::Error>> = None;
|
||||
let mut decode_early = None;
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
@@ -723,11 +795,14 @@ impl PDRouter {
|
||||
prefill_result = pr;
|
||||
break;
|
||||
}
|
||||
dr = &mut decode_fut, if decode_early.is_none() => {
|
||||
dr = &mut decode_leg, if decode_early.is_none() => {
|
||||
decode_early = Some(dr);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(slot) = &slot {
|
||||
slot.release();
|
||||
}
|
||||
|
||||
// Decode can't generate without prefill's KV, so any prefill failure
|
||||
// (non-2xx / transport error) dooms the paired decode request, which would
|
||||
@@ -741,8 +816,7 @@ impl PDRouter {
|
||||
|
||||
if prefill_failed {
|
||||
warn!(
|
||||
"Prefill failed, aborting paired decode request decode_url={} prefill_url={}",
|
||||
decode.url(),
|
||||
"Prefill failed, aborting paired decode request prefill_url={}",
|
||||
prefill.url()
|
||||
);
|
||||
|
||||
@@ -767,17 +841,41 @@ impl PDRouter {
|
||||
),
|
||||
};
|
||||
response.extensions_mut().insert(BreakerOutcomesRecorded);
|
||||
return response;
|
||||
return (response, None);
|
||||
}
|
||||
|
||||
// Prefill ok: take decode's result, awaiting it if still pending.
|
||||
let decode_result = match decode_early {
|
||||
Some(dr) => dr,
|
||||
None => (&mut decode_fut).await,
|
||||
let leg = match decode_early {
|
||||
Some(leg) => leg,
|
||||
None => (&mut decode_leg).await,
|
||||
};
|
||||
let (decode, _decode_guard, decode_result) = match leg {
|
||||
Ok(leg) => leg,
|
||||
Err(response) => return (response, None),
|
||||
};
|
||||
|
||||
events::RequestReceivedEvent {}.emit();
|
||||
|
||||
let response = self
|
||||
.process_pd_responses(
|
||||
&context,
|
||||
Arc::clone(&prefill),
|
||||
Arc::clone(&decode),
|
||||
prefill_result,
|
||||
decode_result,
|
||||
)
|
||||
.await;
|
||||
(response, Some(decode))
|
||||
}
|
||||
|
||||
async fn process_pd_responses(
|
||||
&self,
|
||||
context: &PDRequestContext<'_>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
prefill_result: Result<reqwest::Response, reqwest::Error>,
|
||||
decode_result: Result<reqwest::Response, reqwest::Error>,
|
||||
) -> Response {
|
||||
// Process decode response
|
||||
match decode_result {
|
||||
Ok(res) => {
|
||||
@@ -821,7 +919,7 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
let mut response = self
|
||||
.handle_decode_error_response(res, &context, prefill, decode)
|
||||
.handle_decode_error_response(res, context, prefill, decode)
|
||||
.await;
|
||||
response.extensions_mut().insert(BreakerOutcomesRecorded);
|
||||
return response;
|
||||
@@ -979,79 +1077,83 @@ impl PDRouter {
|
||||
model_id: Option<&str>,
|
||||
headers: Option<&HeaderMap>,
|
||||
) -> Result<(Arc<dyn Worker>, Arc<dyn Worker>), String> {
|
||||
let prefill = self
|
||||
.select_worker(request_text, model_id, headers, false)
|
||||
.await?;
|
||||
let decode = self
|
||||
.select_worker(request_text, model_id, headers, true)
|
||||
.await?;
|
||||
Ok((prefill, decode))
|
||||
}
|
||||
|
||||
async fn select_worker(
|
||||
&self,
|
||||
request_text: Option<&str>,
|
||||
model_id: Option<&str>,
|
||||
headers: Option<&HeaderMap>,
|
||||
decode: bool,
|
||||
) -> Result<Arc<dyn Worker>, String> {
|
||||
let effective_model_id = if !self.enable_igw { None } else { model_id };
|
||||
|
||||
debug!(
|
||||
"Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}",
|
||||
self.enable_igw, model_id, effective_model_id
|
||||
"Selecting PD worker: decode={}, enable_igw={}, model_id={:?}, effective_model_id={:?}",
|
||||
decode, self.enable_igw, model_id, effective_model_id
|
||||
);
|
||||
|
||||
let prefill_workers = if let Some(model) = effective_model_id {
|
||||
self.worker_registry
|
||||
let workers = match effective_model_id {
|
||||
Some(model) => self
|
||||
.worker_registry
|
||||
.get_by_model(model)
|
||||
.iter()
|
||||
.filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. }))
|
||||
.filter(|w| match w.worker_type() {
|
||||
WorkerType::Decode => decode,
|
||||
WorkerType::Prefill { .. } => !decode,
|
||||
_ => false,
|
||||
})
|
||||
.cloned()
|
||||
.collect()
|
||||
} else {
|
||||
self.worker_registry.get_prefill_workers()
|
||||
.collect(),
|
||||
None if decode => self.worker_registry.get_decode_workers(),
|
||||
None => self.worker_registry.get_prefill_workers(),
|
||||
};
|
||||
|
||||
let decode_workers = if let Some(model) = effective_model_id {
|
||||
self.worker_registry
|
||||
.get_by_model(model)
|
||||
.iter()
|
||||
.filter(|w| matches!(w.worker_type(), WorkerType::Decode))
|
||||
.cloned()
|
||||
.collect()
|
||||
let (policy, role, label) = if decode {
|
||||
(
|
||||
self.policy_registry.get_decode_policy(),
|
||||
"decode",
|
||||
metrics_labels::WORKER_DECODE,
|
||||
)
|
||||
} else {
|
||||
self.worker_registry.get_decode_workers()
|
||||
(
|
||||
self.policy_registry.get_prefill_policy(),
|
||||
"prefill",
|
||||
metrics_labels::WORKER_PREFILL,
|
||||
)
|
||||
};
|
||||
|
||||
let prefill_policy = self.policy_registry.get_prefill_policy();
|
||||
let decode_policy = self.policy_registry.get_decode_policy();
|
||||
|
||||
// Get cached hash ring for consistent hashing
|
||||
let hash_ring = self
|
||||
.worker_registry
|
||||
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
|
||||
|
||||
let prefill = Self::pick_worker_by_policy_arc(
|
||||
&prefill_workers,
|
||||
&*prefill_policy,
|
||||
request_text,
|
||||
headers,
|
||||
hash_ring.clone(),
|
||||
"prefill",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let decode = Self::pick_worker_by_policy_arc(
|
||||
&decode_workers,
|
||||
&*decode_policy,
|
||||
let worker = Self::pick_worker_by_policy_arc(
|
||||
&workers,
|
||||
&*policy,
|
||||
request_text,
|
||||
headers,
|
||||
hash_ring,
|
||||
"decode",
|
||||
role,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Record worker selection metrics (Layer 3)
|
||||
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
|
||||
Metrics::record_worker_selection(
|
||||
metrics_labels::WORKER_PREFILL,
|
||||
label,
|
||||
metrics_labels::CONNECTION_HTTP,
|
||||
model,
|
||||
prefill_policy.name(),
|
||||
);
|
||||
Metrics::record_worker_selection(
|
||||
metrics_labels::WORKER_DECODE,
|
||||
metrics_labels::CONNECTION_HTTP,
|
||||
model,
|
||||
decode_policy.name(),
|
||||
model_id.unwrap_or(UNKNOWN_MODEL_ID),
|
||||
policy.name(),
|
||||
);
|
||||
|
||||
Ok((prefill, decode))
|
||||
Ok(worker)
|
||||
}
|
||||
|
||||
async fn pick_worker_by_policy_arc(
|
||||
@@ -1778,6 +1880,8 @@ mod tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: Some("test_api_key".to_string()),
|
||||
enable_igw: false,
|
||||
decode_gate: None,
|
||||
request_timeout: Duration::from_secs(600),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -308,6 +308,7 @@ async fn generate_handler(
|
||||
Json(payload): Json<serde_json::Value>,
|
||||
) -> Response {
|
||||
let config = config.read().await;
|
||||
record_request_arrival(config.port);
|
||||
let worker_id = format!("worker-{}", config.port);
|
||||
|
||||
if should_fail(&config).await {
|
||||
@@ -1470,6 +1471,27 @@ async fn responses_cancel_handler(
|
||||
// can enable slow streaming WITHOUT relying on the request payload
|
||||
// (the gateway deserializes/re-serializes the body, dropping unknown fields).
|
||||
|
||||
static REQUEST_ARRIVALS: OnceLock<Mutex<HashMap<u16, Vec<std::time::Instant>>>> = OnceLock::new();
|
||||
|
||||
fn record_request_arrival(port: u16) {
|
||||
REQUEST_ARRIVALS
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.unwrap()
|
||||
.entry(port)
|
||||
.or_default()
|
||||
.push(std::time::Instant::now());
|
||||
}
|
||||
|
||||
pub fn take_request_arrivals(port: u16) -> Vec<std::time::Instant> {
|
||||
REQUEST_ARRIVALS
|
||||
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||
.lock()
|
||||
.unwrap()
|
||||
.remove(&port)
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
static SLOW_STREAM_CONFIG: OnceLock<Mutex<HashMap<u16, usize>>> = OnceLock::new();
|
||||
|
||||
fn get_slow_stream_config() -> &'static Mutex<HashMap<u16, usize>> {
|
||||
|
||||
@@ -219,6 +219,87 @@ mod pd_routing_tests {
|
||||
|
||||
ctx.shutdown().await;
|
||||
}
|
||||
|
||||
/// Prefill gets both requests at once; the second decode waits for a slot.
|
||||
#[tokio::test]
|
||||
async fn test_pd_decode_dispatch_limit_holds_decode_not_prefill() {
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::common::mock_worker::take_request_arrivals;
|
||||
|
||||
let (prefill_port, decode_port) = (19858, 19859);
|
||||
let config = RouterConfig::builder()
|
||||
.prefill_decode_mode(
|
||||
vec![(format!("http://127.0.0.1:{prefill_port}"), None)],
|
||||
vec![format!("http://127.0.0.1:{decode_port}")],
|
||||
)
|
||||
.round_robin_policy()
|
||||
.decode_dispatch_limit(1)
|
||||
.host("127.0.0.1")
|
||||
.port(3858)
|
||||
.max_payload_size(256 * 1024 * 1024)
|
||||
.request_timeout_secs(600)
|
||||
.worker_startup_timeout_secs(5)
|
||||
.worker_startup_check_interval_secs(1)
|
||||
.max_concurrent_requests(64)
|
||||
.queue_timeout_secs(60)
|
||||
.build_unchecked();
|
||||
let slow = |mut worker: MockWorkerConfig| {
|
||||
worker.response_delay_ms = 600;
|
||||
worker
|
||||
};
|
||||
let ctx = AppTestContext::new_with_config(
|
||||
config,
|
||||
vec![
|
||||
slow(TestWorkerConfig::prefill(prefill_port)),
|
||||
slow(TestWorkerConfig::decode(decode_port)),
|
||||
],
|
||||
)
|
||||
.await;
|
||||
let app = ctx.create_app().await;
|
||||
take_request_arrivals(prefill_port);
|
||||
take_request_arrivals(decode_port);
|
||||
|
||||
let send = || {
|
||||
let app = app.clone();
|
||||
async move {
|
||||
let request = Request::builder()
|
||||
.method("POST")
|
||||
.uri("/generate")
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(Body::from(
|
||||
json!({"text": "hi", "stream": false}).to_string(),
|
||||
))
|
||||
.unwrap();
|
||||
app.oneshot(request).await.unwrap().status()
|
||||
}
|
||||
};
|
||||
let start = Instant::now();
|
||||
let statuses = tokio::join!(send(), send());
|
||||
assert_eq!(statuses, (StatusCode::OK, StatusCode::OK));
|
||||
|
||||
let since_start = |port| {
|
||||
let mut arrivals: Vec<_> = take_request_arrivals(port)
|
||||
.into_iter()
|
||||
.map(|t| t - start)
|
||||
.collect();
|
||||
arrivals.sort();
|
||||
arrivals
|
||||
};
|
||||
let prefill = since_start(prefill_port);
|
||||
let decode = since_start(decode_port);
|
||||
assert!(
|
||||
prefill.len() == 2 && prefill[1] < Duration::from_millis(300),
|
||||
"{prefill:?}"
|
||||
);
|
||||
assert!(
|
||||
decode.len() == 2
|
||||
&& decode[0] < Duration::from_millis(300)
|
||||
&& decode[1] >= Duration::from_millis(500),
|
||||
"{decode:?}"
|
||||
);
|
||||
ctx.shutdown().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -321,6 +402,8 @@ mod pd_responses_routing_tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: None,
|
||||
enable_igw: false,
|
||||
decode_gate: None,
|
||||
request_timeout: Duration::from_secs(600),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user