[Feat] sgl-model-gateway: Add --decode-dispatch-limit to pick PD decode workers near KV arrival (#44689)

This commit is contained in:
AMD-yanfeiwang
2026-10-20 02:16:21 +08:00
committed by GitHub
parent 80b86dbecb
commit 678913a0f0
14 changed files with 386 additions and 89 deletions
@@ -2670,6 +2670,12 @@ groups:
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Override policy for decode nodes</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--decode-dispatch-limit`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Requests per prefill worker sent to decode before decode answers or prefill finishes; the rest pick their decode worker when sent. Needs prefill `--optimistic-prefill-attempts` above 0</td>
</tr>
<tr>
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--worker-startup-timeout-secs`</td>
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
+2
View File
@@ -160,6 +160,8 @@ The `--version` (or `-V`) flag displays the version string. Use `--version-verbo
```
Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client.
A decode worker picked on arrival only starts decoding once prefill has queued, computed and sent the KV, so with long prefill queues, decode workers (with `--dp-aware`, DP ranks) that hold equal numbers of requests can run unequal batches. `--decode-dispatch-limit N` still sends every request to prefill on arrival, but sends at most N per prefill worker on to decode before decode answers or prefill finishes; the rest wait in arrival order and pick their decode worker when they are sent. Prefill must start a request before decode connects, so prefill engines need `--optimistic-prefill-attempts` above 0.
### Multi-Model Inference Gateway
Enable IGW mode to route multiple models through a single router while applying per-model policies:
```bash
@@ -412,6 +412,7 @@ struct Router {
queue_size: usize,
queue_timeout_secs: u64,
rate_limit_tokens_per_second: Option<i32>,
decode_dispatch_limit: Option<usize>,
connection_mode: core::ConnectionMode,
model_path: Option<String>,
tokenizer_path: Option<String>,
@@ -637,6 +638,7 @@ impl Router {
.maybe_log_level(self.log_level.as_ref())
.maybe_request_id_headers(self.request_id_headers.clone())
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
.maybe_model_path(self.model_path.as_ref())
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
.maybe_chat_template(self.chat_template.as_ref())
@@ -733,6 +735,7 @@ impl Router {
queue_size = 100,
queue_timeout_secs = 60,
rate_limit_tokens_per_second = None,
decode_dispatch_limit = None,
model_path = None,
tokenizer_path = None,
chat_template = None,
@@ -825,6 +828,7 @@ impl Router {
queue_size: usize,
queue_timeout_secs: u64,
rate_limit_tokens_per_second: Option<i32>,
decode_dispatch_limit: Option<usize>,
model_path: Option<String>,
tokenizer_path: Option<String>,
chat_template: Option<String>,
@@ -930,6 +934,7 @@ impl Router {
queue_size,
queue_timeout_secs,
rate_limit_tokens_per_second,
decode_dispatch_limit,
connection_mode,
model_path,
tokenizer_path,
@@ -211,6 +211,7 @@ class Router:
queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
decode_dispatch_limit: Max requests per prefill worker in flight to decode (PD mode only). Default: None
cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
@@ -98,6 +98,7 @@ class RouterArgs:
queue_timeout_secs: int = 60
# Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
rate_limit_tokens_per_second: Optional[int] = None
decode_dispatch_limit: Optional[int] = None
# CORS allowed origins
cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
# Retry configuration
@@ -398,6 +399,13 @@ class RouterArgs:
metavar=("URL",),
help="Decode server URL. Can be specified multiple times.",
)
pd_group.add_argument(
f"--{prefix}decode-dispatch-limit",
type=int,
default=RouterArgs.decode_dispatch_limit,
help="Max requests per prefill worker in flight to decode; the rest pick their "
"decode worker when sent (PD mode only)",
)
pd_group.add_argument(
f"--{prefix}worker-startup-timeout-secs",
type=int,
+10
View File
@@ -229,6 +229,11 @@ impl RouterConfigBuilder {
self
}
pub fn decode_dispatch_limit(mut self, limit: usize) -> Self {
self.config.decode_dispatch_limit = Some(limit);
self
}
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
self.config.rate_limit_tokens_per_second = Some(tokens);
self
@@ -531,6 +536,11 @@ impl RouterConfigBuilder {
self
}
pub fn maybe_decode_dispatch_limit(mut self, limit: Option<usize>) -> Self {
self.config.decode_dispatch_limit = limit;
self
}
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self {
self.config.rate_limit_tokens_per_second = tokens;
self
+3
View File
@@ -26,6 +26,8 @@ pub struct RouterConfig {
pub worker_startup_timeout_secs: u64,
pub worker_startup_check_interval_secs: u64,
pub dp_aware: bool,
#[serde(default)]
pub decode_dispatch_limit: Option<usize>,
pub api_key: Option<String>,
pub discovery: Option<DiscoveryConfig>,
pub metrics: Option<MetricsConfig>,
@@ -514,6 +516,7 @@ impl Default for RouterConfig {
worker_startup_timeout_secs: 1800, // 30 minutes for large model loading
worker_startup_check_interval_secs: 30,
dp_aware: false,
decode_dispatch_limit: None,
api_key: None,
discovery: None,
metrics: None,
@@ -286,6 +286,14 @@ impl ConfigValidator {
});
}
if config.decode_dispatch_limit == Some(0) {
return Err(ConfigError::InvalidValue {
field: "decode_dispatch_limit".to_string(),
value: "0".to_string(),
reason: "Must be > 0 when specified".to_string(),
});
}
if let Some(tokens_per_second) = config.rate_limit_tokens_per_second {
// Allow 0 for pure concurrency limiting (semaphore behavior)
if tokens_per_second < 0 {
+5
View File
@@ -211,6 +211,10 @@ struct CliArgs {
#[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")]
decode_policy: Option<String>,
/// Max requests per prefill worker in flight to decode; the rest pick their decode worker when sent
#[arg(long, help_heading = "PD Disaggregation")]
decode_dispatch_limit: Option<usize>,
/// Timeout in seconds for worker startup and registration
#[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")]
worker_startup_timeout_secs: u64,
@@ -1056,6 +1060,7 @@ impl CliArgs {
(!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
)
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
.maybe_model_path(self.model_path.as_ref())
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
.maybe_chat_template(self.chat_template.as_ref())
@@ -0,0 +1,39 @@
use std::sync::{Arc, Mutex};
use dashmap::DashMap;
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
#[derive(Debug)]
pub struct DecodeGate {
limit: usize,
gates: DashMap<String, Arc<Semaphore>>,
}
impl DecodeGate {
pub fn new(limit: usize) -> Self {
Self {
limit,
gates: DashMap::new(),
}
}
pub async fn admit(&self, prefill_url: &str, slot: &DecodeSlot) {
let gate = Arc::clone(
&self
.gates
.entry(prefill_url.to_string())
.or_insert_with(|| Arc::new(Semaphore::new(self.limit))),
);
let permit = gate.acquire_owned().await.expect("gates are never closed");
*slot.0.lock().unwrap() = Some(permit);
}
}
#[derive(Debug, Clone, Default)]
pub struct DecodeSlot(Arc<Mutex<Option<OwnedSemaphorePermit>>>);
impl DecodeSlot {
pub fn release(&self) {
self.0.lock().unwrap().take();
}
}
@@ -1,5 +1,6 @@
//! HTTP router implementations
pub mod decode_gate;
pub mod pd_router;
pub mod pd_types;
pub mod router;
+193 -89
View File
@@ -1,4 +1,8 @@
use std::{borrow::Cow, sync::Arc, time::Instant};
use std::{
borrow::Cow,
sync::Arc,
time::{Duration, Instant},
};
use async_trait::async_trait;
use axum::{
@@ -18,7 +22,10 @@ use serde_json::{json, Value};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{debug, error, warn};
use super::pd_types::api_path;
use super::{
decode_gate::{DecodeGate, DecodeSlot},
pd_types::api_path,
};
use crate::{
config::types::RetryConfig,
core::{
@@ -58,6 +65,8 @@ pub struct PDRouter {
pub retry_config: RetryConfig,
pub api_key: Option<String>,
pub enable_igw: bool,
pub decode_gate: Option<Arc<DecodeGate>>,
pub request_timeout: Duration,
}
struct PreparedWorkerRequest<'a> {
@@ -189,6 +198,11 @@ impl PDRouter {
retry_config: ctx.router_config.effective_retry_config(),
api_key: ctx.router_config.api_key.clone(),
enable_igw: ctx.router_config.enable_igw,
decode_gate: ctx
.router_config
.decode_dispatch_limit
.map(|limit| Arc::new(DecodeGate::new(limit))),
request_timeout: Duration::from_secs(ctx.router_config.request_timeout_secs),
})
}
@@ -350,6 +364,7 @@ impl PDRouter {
})
}
#[cfg(test)]
async fn prepare_pd_worker_requests<'a>(
route: &'static str,
json_request: &'a Value,
@@ -398,14 +413,26 @@ impl PDRouter {
let shared_request = Arc::clone(&shared_request);
let context = context.clone();
async move {
let (prefill, decode) = match self
.select_pd_pair(
context.request_text.as_deref(),
let request_text = context.request_text.as_deref();
let selected = if self.decode_gate.is_some() {
self.select_worker(
request_text,
context.model_id,
context.headers.as_ref(),
false,
)
.await
.map(|prefill| (prefill, None))
} else {
self.select_pd_pair(
request_text,
context.model_id,
context.headers.as_ref(),
)
.await
{
.map(|(prefill, decode)| (prefill, Some(decode)))
};
let (prefill, decode) = match selected {
Ok(pair) => pair,
Err(e) => {
return Self::handle_server_selection_error(e);
@@ -413,10 +440,10 @@ impl PDRouter {
};
debug!(
"PD retry attempt {} using prefill={} decode={}",
"PD retry attempt {} using prefill={} decode={:?}",
attempt,
prefill.url(),
decode.url()
decode.as_ref().map(|d| d.url())
);
let mut json_request = match serde_json::to_value(shared_request.as_ref()) {
@@ -438,13 +465,13 @@ impl PDRouter {
};
let ctx_is_stream = context.is_stream;
let response = self
let (response, decode) = self
.execute_dual_dispatch_internal(
headers,
json_request,
context,
Arc::clone(&prefill),
Arc::clone(&decode),
decode,
start_time,
)
.await;
@@ -464,7 +491,7 @@ impl PDRouter {
// its byte stream records the outcome on drop. Skip the
// eager success record to avoid masking "200-then-broken"
// decode workers.
if !ctx_is_stream {
if let (false, Some(decode)) = (ctx_is_stream, &decode) {
decode.record_outcome(not_error);
}
}
@@ -649,40 +676,40 @@ impl PDRouter {
}
// Internal method that performs the actual dual dispatch (without retry logic)
#[allow(clippy::too_many_arguments)]
async fn execute_dual_dispatch_internal(
&self,
headers: Option<&HeaderMap>,
json_request: Value,
context: PDRequestContext<'_>,
prefill: Arc<dyn Worker>,
decode: Arc<dyn Worker>,
decode: Option<Arc<dyn Worker>>,
_start_time: Instant,
) -> Response {
) -> (Response, Option<Arc<dyn Worker>>) {
// Count the request from dispatch so load-based policies see it before
// upstream headers arrive; streaming responses attach their own guards.
let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers);
let _decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
let mut headers_with_trace = headers.cloned().unwrap_or_default();
inject_trace_context_http(&mut headers_with_trace);
let headers = Some(&headers_with_trace);
let (prepared_prefill, prepared_decode) = match Self::prepare_pd_worker_requests(
let prepared_prefill = match Self::prepare_worker_request(
context.route,
&json_request,
prefill.as_ref(),
decode.as_ref(),
Cow::Borrowed(&json_request),
)
.await
{
Ok(requests) => requests,
Ok(request) => request,
Err(e) => {
error!("Failed to prepare PD worker requests: {}", e);
return error::internal_error("pd_request_preparation_failed", e);
return (
error::internal_error("pd_request_preparation_failed", e),
None,
);
}
};
// Build both requests
let prefill_request = self.build_post_with_headers(
&self.client,
&prepared_prefill.endpoint_url,
@@ -690,32 +717,77 @@ impl PDRouter {
headers,
false,
);
let decode_request = self.build_post_with_headers(
&self.client,
&prepared_decode.endpoint_url,
&prepared_decode.body,
headers,
false,
);
let slot = decode.is_none().then(DecodeSlot::default);
let decode_leg = async {
if let (Some(gate), Some(slot)) = (&self.decode_gate, &slot) {
let admit = gate.admit(prefill.url(), slot);
if tokio::time::timeout(self.request_timeout, admit)
.await
.is_err()
{
return Err(error::create_error(
StatusCode::GATEWAY_TIMEOUT,
"decode_dispatch_timeout",
"Timed out waiting to dispatch the decode request",
));
}
}
let decode = match decode {
Some(decode) => decode,
None => self
.select_worker(
context.request_text.as_deref(),
context.model_id,
context.headers.as_ref(),
true,
)
.await
.map_err(Self::handle_server_selection_error)?,
};
let decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
let prepared_decode = async {
let body = Self::inject_prefill_dp_rank_for_decode(
Cow::Borrowed(&json_request),
prefill.as_ref(),
)?;
Self::prepare_worker_request(context.route, decode.as_ref(), body).await
}
.await
.map_err(|e| {
error!("Failed to prepare PD worker requests: {}", e);
error::internal_error("pd_request_preparation_failed", e)
})?;
let decode_request = self.build_post_with_headers(
&self.client,
&prepared_decode.endpoint_url,
&prepared_decode.body,
headers,
false,
);
events::RequestPDSentEvent {
prefill_url: prefill.url(),
decode_url: decode.url(),
}
.emit();
let result = decode_request.send().await;
if let Some(slot) = &slot {
slot.release();
}
Ok((decode, decode_guard, result))
};
// Run both in this handler task (not a detached tokio::spawn) so a client
// disconnect cancels the pending decode request too, keeping the
// upstream-cancel behavior from #19524.
events::RequestPDSentEvent {
prefill_url: prefill.url(),
decode_url: decode.url(),
}
.emit();
let prefill_fut = prefill_request.send();
let decode_fut = decode_request.send();
tokio::pin!(prefill_fut);
tokio::pin!(decode_fut);
tokio::pin!(decode_leg);
// Poll both until prefill resolves; decode normally resolves later, but
// may resolve first if it rejects the request outright.
let prefill_result;
let mut decode_early: Option<Result<reqwest::Response, reqwest::Error>> = None;
let mut decode_early = None;
loop {
tokio::select! {
biased;
@@ -723,11 +795,14 @@ impl PDRouter {
prefill_result = pr;
break;
}
dr = &mut decode_fut, if decode_early.is_none() => {
dr = &mut decode_leg, if decode_early.is_none() => {
decode_early = Some(dr);
}
}
}
if let Some(slot) = &slot {
slot.release();
}
// Decode can't generate without prefill's KV, so any prefill failure
// (non-2xx / transport error) dooms the paired decode request, which would
@@ -741,8 +816,7 @@ impl PDRouter {
if prefill_failed {
warn!(
"Prefill failed, aborting paired decode request decode_url={} prefill_url={}",
decode.url(),
"Prefill failed, aborting paired decode request prefill_url={}",
prefill.url()
);
@@ -767,17 +841,41 @@ impl PDRouter {
),
};
response.extensions_mut().insert(BreakerOutcomesRecorded);
return response;
return (response, None);
}
// Prefill ok: take decode's result, awaiting it if still pending.
let decode_result = match decode_early {
Some(dr) => dr,
None => (&mut decode_fut).await,
let leg = match decode_early {
Some(leg) => leg,
None => (&mut decode_leg).await,
};
let (decode, _decode_guard, decode_result) = match leg {
Ok(leg) => leg,
Err(response) => return (response, None),
};
events::RequestReceivedEvent {}.emit();
let response = self
.process_pd_responses(
&context,
Arc::clone(&prefill),
Arc::clone(&decode),
prefill_result,
decode_result,
)
.await;
(response, Some(decode))
}
async fn process_pd_responses(
&self,
context: &PDRequestContext<'_>,
prefill: Arc<dyn Worker>,
decode: Arc<dyn Worker>,
prefill_result: Result<reqwest::Response, reqwest::Error>,
decode_result: Result<reqwest::Response, reqwest::Error>,
) -> Response {
// Process decode response
match decode_result {
Ok(res) => {
@@ -821,7 +919,7 @@ impl PDRouter {
}
let mut response = self
.handle_decode_error_response(res, &context, prefill, decode)
.handle_decode_error_response(res, context, prefill, decode)
.await;
response.extensions_mut().insert(BreakerOutcomesRecorded);
return response;
@@ -979,79 +1077,83 @@ impl PDRouter {
model_id: Option<&str>,
headers: Option<&HeaderMap>,
) -> Result<(Arc<dyn Worker>, Arc<dyn Worker>), String> {
let prefill = self
.select_worker(request_text, model_id, headers, false)
.await?;
let decode = self
.select_worker(request_text, model_id, headers, true)
.await?;
Ok((prefill, decode))
}
async fn select_worker(
&self,
request_text: Option<&str>,
model_id: Option<&str>,
headers: Option<&HeaderMap>,
decode: bool,
) -> Result<Arc<dyn Worker>, String> {
let effective_model_id = if !self.enable_igw { None } else { model_id };
debug!(
"Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}",
self.enable_igw, model_id, effective_model_id
"Selecting PD worker: decode={}, enable_igw={}, model_id={:?}, effective_model_id={:?}",
decode, self.enable_igw, model_id, effective_model_id
);
let prefill_workers = if let Some(model) = effective_model_id {
self.worker_registry
let workers = match effective_model_id {
Some(model) => self
.worker_registry
.get_by_model(model)
.iter()
.filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. }))
.filter(|w| match w.worker_type() {
WorkerType::Decode => decode,
WorkerType::Prefill { .. } => !decode,
_ => false,
})
.cloned()
.collect()
} else {
self.worker_registry.get_prefill_workers()
.collect(),
None if decode => self.worker_registry.get_decode_workers(),
None => self.worker_registry.get_prefill_workers(),
};
let decode_workers = if let Some(model) = effective_model_id {
self.worker_registry
.get_by_model(model)
.iter()
.filter(|w| matches!(w.worker_type(), WorkerType::Decode))
.cloned()
.collect()
let (policy, role, label) = if decode {
(
self.policy_registry.get_decode_policy(),
"decode",
metrics_labels::WORKER_DECODE,
)
} else {
self.worker_registry.get_decode_workers()
(
self.policy_registry.get_prefill_policy(),
"prefill",
metrics_labels::WORKER_PREFILL,
)
};
let prefill_policy = self.policy_registry.get_prefill_policy();
let decode_policy = self.policy_registry.get_decode_policy();
// Get cached hash ring for consistent hashing
let hash_ring = self
.worker_registry
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
let prefill = Self::pick_worker_by_policy_arc(
&prefill_workers,
&*prefill_policy,
request_text,
headers,
hash_ring.clone(),
"prefill",
)
.await?;
let decode = Self::pick_worker_by_policy_arc(
&decode_workers,
&*decode_policy,
let worker = Self::pick_worker_by_policy_arc(
&workers,
&*policy,
request_text,
headers,
hash_ring,
"decode",
role,
)
.await?;
// Record worker selection metrics (Layer 3)
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
Metrics::record_worker_selection(
metrics_labels::WORKER_PREFILL,
label,
metrics_labels::CONNECTION_HTTP,
model,
prefill_policy.name(),
);
Metrics::record_worker_selection(
metrics_labels::WORKER_DECODE,
metrics_labels::CONNECTION_HTTP,
model,
decode_policy.name(),
model_id.unwrap_or(UNKNOWN_MODEL_ID),
policy.name(),
);
Ok((prefill, decode))
Ok(worker)
}
async fn pick_worker_by_policy_arc(
@@ -1778,6 +1880,8 @@ mod tests {
retry_config: RetryConfig::default(),
api_key: Some("test_api_key".to_string()),
enable_igw: false,
decode_gate: None,
request_timeout: Duration::from_secs(600),
}
}
@@ -308,6 +308,7 @@ async fn generate_handler(
Json(payload): Json<serde_json::Value>,
) -> Response {
let config = config.read().await;
record_request_arrival(config.port);
let worker_id = format!("worker-{}", config.port);
if should_fail(&config).await {
@@ -1470,6 +1471,27 @@ async fn responses_cancel_handler(
// can enable slow streaming WITHOUT relying on the request payload
// (the gateway deserializes/re-serializes the body, dropping unknown fields).
static REQUEST_ARRIVALS: OnceLock<Mutex<HashMap<u16, Vec<std::time::Instant>>>> = OnceLock::new();
fn record_request_arrival(port: u16) {
REQUEST_ARRIVALS
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap()
.entry(port)
.or_default()
.push(std::time::Instant::now());
}
pub fn take_request_arrivals(port: u16) -> Vec<std::time::Instant> {
REQUEST_ARRIVALS
.get_or_init(|| Mutex::new(HashMap::new()))
.lock()
.unwrap()
.remove(&port)
.unwrap_or_default()
}
static SLOW_STREAM_CONFIG: OnceLock<Mutex<HashMap<u16, usize>>> = OnceLock::new();
fn get_slow_stream_config() -> &'static Mutex<HashMap<u16, usize>> {
@@ -219,6 +219,87 @@ mod pd_routing_tests {
ctx.shutdown().await;
}
/// Prefill gets both requests at once; the second decode waits for a slot.
#[tokio::test]
async fn test_pd_decode_dispatch_limit_holds_decode_not_prefill() {
use std::time::{Duration, Instant};
use crate::common::mock_worker::take_request_arrivals;
let (prefill_port, decode_port) = (19858, 19859);
let config = RouterConfig::builder()
.prefill_decode_mode(
vec![(format!("http://127.0.0.1:{prefill_port}"), None)],
vec![format!("http://127.0.0.1:{decode_port}")],
)
.round_robin_policy()
.decode_dispatch_limit(1)
.host("127.0.0.1")
.port(3858)
.max_payload_size(256 * 1024 * 1024)
.request_timeout_secs(600)
.worker_startup_timeout_secs(5)
.worker_startup_check_interval_secs(1)
.max_concurrent_requests(64)
.queue_timeout_secs(60)
.build_unchecked();
let slow = |mut worker: MockWorkerConfig| {
worker.response_delay_ms = 600;
worker
};
let ctx = AppTestContext::new_with_config(
config,
vec![
slow(TestWorkerConfig::prefill(prefill_port)),
slow(TestWorkerConfig::decode(decode_port)),
],
)
.await;
let app = ctx.create_app().await;
take_request_arrivals(prefill_port);
take_request_arrivals(decode_port);
let send = || {
let app = app.clone();
async move {
let request = Request::builder()
.method("POST")
.uri("/generate")
.header(CONTENT_TYPE, "application/json")
.body(Body::from(
json!({"text": "hi", "stream": false}).to_string(),
))
.unwrap();
app.oneshot(request).await.unwrap().status()
}
};
let start = Instant::now();
let statuses = tokio::join!(send(), send());
assert_eq!(statuses, (StatusCode::OK, StatusCode::OK));
let since_start = |port| {
let mut arrivals: Vec<_> = take_request_arrivals(port)
.into_iter()
.map(|t| t - start)
.collect();
arrivals.sort();
arrivals
};
let prefill = since_start(prefill_port);
let decode = since_start(decode_port);
assert!(
prefill.len() == 2 && prefill[1] < Duration::from_millis(300),
"{prefill:?}"
);
assert!(
decode.len() == 2
&& decode[0] < Duration::from_millis(300)
&& decode[1] >= Duration::from_millis(500),
"{decode:?}"
);
ctx.shutdown().await;
}
}
#[cfg(test)]
@@ -321,6 +402,8 @@ mod pd_responses_routing_tests {
retry_config: RetryConfig::default(),
api_key: None,
enable_igw: false,
decode_gate: None,
request_timeout: Duration::from_secs(600),
}
}