[Feat] sgl-model-gateway: Add --decode-dispatch-limit to pick PD decode workers near KV arrival (#44689)
This commit is contained in:
@@ -2670,6 +2670,12 @@ groups:
|
|||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Override policy for decode nodes</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Override policy for decode nodes</td>
|
||||||
</tr>
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--decode-dispatch-limit`</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>None</td>
|
||||||
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Requests per prefill worker sent to decode before decode answers or prefill finishes; the rest pick their decode worker when sent. Needs prefill `--optimistic-prefill-attempts` above 0</td>
|
||||||
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--worker-startup-timeout-secs`</td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--worker-startup-timeout-secs`</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>int</td>
|
||||||
|
|||||||
@@ -160,6 +160,8 @@ The `--version` (or `-V`) flag displays the version string. Use `--version-verbo
|
|||||||
```
|
```
|
||||||
Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client.
|
Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client.
|
||||||
|
|
||||||
|
A decode worker picked on arrival only starts decoding once prefill has queued, computed and sent the KV, so with long prefill queues, decode workers (with `--dp-aware`, DP ranks) that hold equal numbers of requests can run unequal batches. `--decode-dispatch-limit N` still sends every request to prefill on arrival, but sends at most N per prefill worker on to decode before decode answers or prefill finishes; the rest wait in arrival order and pick their decode worker when they are sent. Prefill must start a request before decode connects, so prefill engines need `--optimistic-prefill-attempts` above 0.
|
||||||
|
|
||||||
### Multi-Model Inference Gateway
|
### Multi-Model Inference Gateway
|
||||||
Enable IGW mode to route multiple models through a single router while applying per-model policies:
|
Enable IGW mode to route multiple models through a single router while applying per-model policies:
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -412,6 +412,7 @@ struct Router {
|
|||||||
queue_size: usize,
|
queue_size: usize,
|
||||||
queue_timeout_secs: u64,
|
queue_timeout_secs: u64,
|
||||||
rate_limit_tokens_per_second: Option<i32>,
|
rate_limit_tokens_per_second: Option<i32>,
|
||||||
|
decode_dispatch_limit: Option<usize>,
|
||||||
connection_mode: core::ConnectionMode,
|
connection_mode: core::ConnectionMode,
|
||||||
model_path: Option<String>,
|
model_path: Option<String>,
|
||||||
tokenizer_path: Option<String>,
|
tokenizer_path: Option<String>,
|
||||||
@@ -637,6 +638,7 @@ impl Router {
|
|||||||
.maybe_log_level(self.log_level.as_ref())
|
.maybe_log_level(self.log_level.as_ref())
|
||||||
.maybe_request_id_headers(self.request_id_headers.clone())
|
.maybe_request_id_headers(self.request_id_headers.clone())
|
||||||
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
|
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
|
||||||
|
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
|
||||||
.maybe_model_path(self.model_path.as_ref())
|
.maybe_model_path(self.model_path.as_ref())
|
||||||
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
|
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
|
||||||
.maybe_chat_template(self.chat_template.as_ref())
|
.maybe_chat_template(self.chat_template.as_ref())
|
||||||
@@ -733,6 +735,7 @@ impl Router {
|
|||||||
queue_size = 100,
|
queue_size = 100,
|
||||||
queue_timeout_secs = 60,
|
queue_timeout_secs = 60,
|
||||||
rate_limit_tokens_per_second = None,
|
rate_limit_tokens_per_second = None,
|
||||||
|
decode_dispatch_limit = None,
|
||||||
model_path = None,
|
model_path = None,
|
||||||
tokenizer_path = None,
|
tokenizer_path = None,
|
||||||
chat_template = None,
|
chat_template = None,
|
||||||
@@ -825,6 +828,7 @@ impl Router {
|
|||||||
queue_size: usize,
|
queue_size: usize,
|
||||||
queue_timeout_secs: u64,
|
queue_timeout_secs: u64,
|
||||||
rate_limit_tokens_per_second: Option<i32>,
|
rate_limit_tokens_per_second: Option<i32>,
|
||||||
|
decode_dispatch_limit: Option<usize>,
|
||||||
model_path: Option<String>,
|
model_path: Option<String>,
|
||||||
tokenizer_path: Option<String>,
|
tokenizer_path: Option<String>,
|
||||||
chat_template: Option<String>,
|
chat_template: Option<String>,
|
||||||
@@ -930,6 +934,7 @@ impl Router {
|
|||||||
queue_size,
|
queue_size,
|
||||||
queue_timeout_secs,
|
queue_timeout_secs,
|
||||||
rate_limit_tokens_per_second,
|
rate_limit_tokens_per_second,
|
||||||
|
decode_dispatch_limit,
|
||||||
connection_mode,
|
connection_mode,
|
||||||
model_path,
|
model_path,
|
||||||
tokenizer_path,
|
tokenizer_path,
|
||||||
|
|||||||
@@ -211,6 +211,7 @@ class Router:
|
|||||||
queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
|
queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
|
||||||
queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
|
queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
|
||||||
rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
|
rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
|
||||||
|
decode_dispatch_limit: Max requests per prefill worker in flight to decode (PD mode only). Default: None
|
||||||
cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
|
cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
|
||||||
health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
|
health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
|
||||||
health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
|
health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
|
||||||
|
|||||||
@@ -98,6 +98,7 @@ class RouterArgs:
|
|||||||
queue_timeout_secs: int = 60
|
queue_timeout_secs: int = 60
|
||||||
# Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
|
# Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
|
||||||
rate_limit_tokens_per_second: Optional[int] = None
|
rate_limit_tokens_per_second: Optional[int] = None
|
||||||
|
decode_dispatch_limit: Optional[int] = None
|
||||||
# CORS allowed origins
|
# CORS allowed origins
|
||||||
cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
|
cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
|
||||||
# Retry configuration
|
# Retry configuration
|
||||||
@@ -398,6 +399,13 @@ class RouterArgs:
|
|||||||
metavar=("URL",),
|
metavar=("URL",),
|
||||||
help="Decode server URL. Can be specified multiple times.",
|
help="Decode server URL. Can be specified multiple times.",
|
||||||
)
|
)
|
||||||
|
pd_group.add_argument(
|
||||||
|
f"--{prefix}decode-dispatch-limit",
|
||||||
|
type=int,
|
||||||
|
default=RouterArgs.decode_dispatch_limit,
|
||||||
|
help="Max requests per prefill worker in flight to decode; the rest pick their "
|
||||||
|
"decode worker when sent (PD mode only)",
|
||||||
|
)
|
||||||
pd_group.add_argument(
|
pd_group.add_argument(
|
||||||
f"--{prefix}worker-startup-timeout-secs",
|
f"--{prefix}worker-startup-timeout-secs",
|
||||||
type=int,
|
type=int,
|
||||||
|
|||||||
@@ -229,6 +229,11 @@ impl RouterConfigBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn decode_dispatch_limit(mut self, limit: usize) -> Self {
|
||||||
|
self.config.decode_dispatch_limit = Some(limit);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
|
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
|
||||||
self.config.rate_limit_tokens_per_second = Some(tokens);
|
self.config.rate_limit_tokens_per_second = Some(tokens);
|
||||||
self
|
self
|
||||||
@@ -531,6 +536,11 @@ impl RouterConfigBuilder {
|
|||||||
self
|
self
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn maybe_decode_dispatch_limit(mut self, limit: Option<usize>) -> Self {
|
||||||
|
self.config.decode_dispatch_limit = limit;
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self {
|
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option<i32>) -> Self {
|
||||||
self.config.rate_limit_tokens_per_second = tokens;
|
self.config.rate_limit_tokens_per_second = tokens;
|
||||||
self
|
self
|
||||||
|
|||||||
@@ -26,6 +26,8 @@ pub struct RouterConfig {
|
|||||||
pub worker_startup_timeout_secs: u64,
|
pub worker_startup_timeout_secs: u64,
|
||||||
pub worker_startup_check_interval_secs: u64,
|
pub worker_startup_check_interval_secs: u64,
|
||||||
pub dp_aware: bool,
|
pub dp_aware: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
pub decode_dispatch_limit: Option<usize>,
|
||||||
pub api_key: Option<String>,
|
pub api_key: Option<String>,
|
||||||
pub discovery: Option<DiscoveryConfig>,
|
pub discovery: Option<DiscoveryConfig>,
|
||||||
pub metrics: Option<MetricsConfig>,
|
pub metrics: Option<MetricsConfig>,
|
||||||
@@ -514,6 +516,7 @@ impl Default for RouterConfig {
|
|||||||
worker_startup_timeout_secs: 1800, // 30 minutes for large model loading
|
worker_startup_timeout_secs: 1800, // 30 minutes for large model loading
|
||||||
worker_startup_check_interval_secs: 30,
|
worker_startup_check_interval_secs: 30,
|
||||||
dp_aware: false,
|
dp_aware: false,
|
||||||
|
decode_dispatch_limit: None,
|
||||||
api_key: None,
|
api_key: None,
|
||||||
discovery: None,
|
discovery: None,
|
||||||
metrics: None,
|
metrics: None,
|
||||||
|
|||||||
@@ -286,6 +286,14 @@ impl ConfigValidator {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config.decode_dispatch_limit == Some(0) {
|
||||||
|
return Err(ConfigError::InvalidValue {
|
||||||
|
field: "decode_dispatch_limit".to_string(),
|
||||||
|
value: "0".to_string(),
|
||||||
|
reason: "Must be > 0 when specified".to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
if let Some(tokens_per_second) = config.rate_limit_tokens_per_second {
|
if let Some(tokens_per_second) = config.rate_limit_tokens_per_second {
|
||||||
// Allow 0 for pure concurrency limiting (semaphore behavior)
|
// Allow 0 for pure concurrency limiting (semaphore behavior)
|
||||||
if tokens_per_second < 0 {
|
if tokens_per_second < 0 {
|
||||||
|
|||||||
@@ -211,6 +211,10 @@ struct CliArgs {
|
|||||||
#[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")]
|
#[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")]
|
||||||
decode_policy: Option<String>,
|
decode_policy: Option<String>,
|
||||||
|
|
||||||
|
/// Max requests per prefill worker in flight to decode; the rest pick their decode worker when sent
|
||||||
|
#[arg(long, help_heading = "PD Disaggregation")]
|
||||||
|
decode_dispatch_limit: Option<usize>,
|
||||||
|
|
||||||
/// Timeout in seconds for worker startup and registration
|
/// Timeout in seconds for worker startup and registration
|
||||||
#[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")]
|
#[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")]
|
||||||
worker_startup_timeout_secs: u64,
|
worker_startup_timeout_secs: u64,
|
||||||
@@ -1056,6 +1060,7 @@ impl CliArgs {
|
|||||||
(!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
|
(!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
|
||||||
)
|
)
|
||||||
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
|
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
|
||||||
|
.maybe_decode_dispatch_limit(self.decode_dispatch_limit)
|
||||||
.maybe_model_path(self.model_path.as_ref())
|
.maybe_model_path(self.model_path.as_ref())
|
||||||
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
|
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
|
||||||
.maybe_chat_template(self.chat_template.as_ref())
|
.maybe_chat_template(self.chat_template.as_ref())
|
||||||
|
|||||||
@@ -0,0 +1,39 @@
|
|||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
|
use dashmap::DashMap;
|
||||||
|
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub struct DecodeGate {
|
||||||
|
limit: usize,
|
||||||
|
gates: DashMap<String, Arc<Semaphore>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl DecodeGate {
|
||||||
|
pub fn new(limit: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
limit,
|
||||||
|
gates: DashMap::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn admit(&self, prefill_url: &str, slot: &DecodeSlot) {
|
||||||
|
let gate = Arc::clone(
|
||||||
|
&self
|
||||||
|
.gates
|
||||||
|
.entry(prefill_url.to_string())
|
||||||
|
.or_insert_with(|| Arc::new(Semaphore::new(self.limit))),
|
||||||
|
);
|
||||||
|
let permit = gate.acquire_owned().await.expect("gates are never closed");
|
||||||
|
*slot.0.lock().unwrap() = Some(permit);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Default)]
|
||||||
|
pub struct DecodeSlot(Arc<Mutex<Option<OwnedSemaphorePermit>>>);
|
||||||
|
|
||||||
|
impl DecodeSlot {
|
||||||
|
pub fn release(&self) {
|
||||||
|
self.0.lock().unwrap().take();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
//! HTTP router implementations
|
//! HTTP router implementations
|
||||||
|
|
||||||
|
pub mod decode_gate;
|
||||||
pub mod pd_router;
|
pub mod pd_router;
|
||||||
pub mod pd_types;
|
pub mod pd_types;
|
||||||
pub mod router;
|
pub mod router;
|
||||||
|
|||||||
@@ -1,4 +1,8 @@
|
|||||||
use std::{borrow::Cow, sync::Arc, time::Instant};
|
use std::{
|
||||||
|
borrow::Cow,
|
||||||
|
sync::Arc,
|
||||||
|
time::{Duration, Instant},
|
||||||
|
};
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use axum::{
|
use axum::{
|
||||||
@@ -18,7 +22,10 @@ use serde_json::{json, Value};
|
|||||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||||
use tracing::{debug, error, warn};
|
use tracing::{debug, error, warn};
|
||||||
|
|
||||||
use super::pd_types::api_path;
|
use super::{
|
||||||
|
decode_gate::{DecodeGate, DecodeSlot},
|
||||||
|
pd_types::api_path,
|
||||||
|
};
|
||||||
use crate::{
|
use crate::{
|
||||||
config::types::RetryConfig,
|
config::types::RetryConfig,
|
||||||
core::{
|
core::{
|
||||||
@@ -58,6 +65,8 @@ pub struct PDRouter {
|
|||||||
pub retry_config: RetryConfig,
|
pub retry_config: RetryConfig,
|
||||||
pub api_key: Option<String>,
|
pub api_key: Option<String>,
|
||||||
pub enable_igw: bool,
|
pub enable_igw: bool,
|
||||||
|
pub decode_gate: Option<Arc<DecodeGate>>,
|
||||||
|
pub request_timeout: Duration,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct PreparedWorkerRequest<'a> {
|
struct PreparedWorkerRequest<'a> {
|
||||||
@@ -189,6 +198,11 @@ impl PDRouter {
|
|||||||
retry_config: ctx.router_config.effective_retry_config(),
|
retry_config: ctx.router_config.effective_retry_config(),
|
||||||
api_key: ctx.router_config.api_key.clone(),
|
api_key: ctx.router_config.api_key.clone(),
|
||||||
enable_igw: ctx.router_config.enable_igw,
|
enable_igw: ctx.router_config.enable_igw,
|
||||||
|
decode_gate: ctx
|
||||||
|
.router_config
|
||||||
|
.decode_dispatch_limit
|
||||||
|
.map(|limit| Arc::new(DecodeGate::new(limit))),
|
||||||
|
request_timeout: Duration::from_secs(ctx.router_config.request_timeout_secs),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -350,6 +364,7 @@ impl PDRouter {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
async fn prepare_pd_worker_requests<'a>(
|
async fn prepare_pd_worker_requests<'a>(
|
||||||
route: &'static str,
|
route: &'static str,
|
||||||
json_request: &'a Value,
|
json_request: &'a Value,
|
||||||
@@ -398,14 +413,26 @@ impl PDRouter {
|
|||||||
let shared_request = Arc::clone(&shared_request);
|
let shared_request = Arc::clone(&shared_request);
|
||||||
let context = context.clone();
|
let context = context.clone();
|
||||||
async move {
|
async move {
|
||||||
let (prefill, decode) = match self
|
let request_text = context.request_text.as_deref();
|
||||||
.select_pd_pair(
|
let selected = if self.decode_gate.is_some() {
|
||||||
context.request_text.as_deref(),
|
self.select_worker(
|
||||||
|
request_text,
|
||||||
|
context.model_id,
|
||||||
|
context.headers.as_ref(),
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map(|prefill| (prefill, None))
|
||||||
|
} else {
|
||||||
|
self.select_pd_pair(
|
||||||
|
request_text,
|
||||||
context.model_id,
|
context.model_id,
|
||||||
context.headers.as_ref(),
|
context.headers.as_ref(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
.map(|(prefill, decode)| (prefill, Some(decode)))
|
||||||
|
};
|
||||||
|
let (prefill, decode) = match selected {
|
||||||
Ok(pair) => pair,
|
Ok(pair) => pair,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
return Self::handle_server_selection_error(e);
|
return Self::handle_server_selection_error(e);
|
||||||
@@ -413,10 +440,10 @@ impl PDRouter {
|
|||||||
};
|
};
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"PD retry attempt {} using prefill={} decode={}",
|
"PD retry attempt {} using prefill={} decode={:?}",
|
||||||
attempt,
|
attempt,
|
||||||
prefill.url(),
|
prefill.url(),
|
||||||
decode.url()
|
decode.as_ref().map(|d| d.url())
|
||||||
);
|
);
|
||||||
|
|
||||||
let mut json_request = match serde_json::to_value(shared_request.as_ref()) {
|
let mut json_request = match serde_json::to_value(shared_request.as_ref()) {
|
||||||
@@ -438,13 +465,13 @@ impl PDRouter {
|
|||||||
};
|
};
|
||||||
|
|
||||||
let ctx_is_stream = context.is_stream;
|
let ctx_is_stream = context.is_stream;
|
||||||
let response = self
|
let (response, decode) = self
|
||||||
.execute_dual_dispatch_internal(
|
.execute_dual_dispatch_internal(
|
||||||
headers,
|
headers,
|
||||||
json_request,
|
json_request,
|
||||||
context,
|
context,
|
||||||
Arc::clone(&prefill),
|
Arc::clone(&prefill),
|
||||||
Arc::clone(&decode),
|
decode,
|
||||||
start_time,
|
start_time,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -464,7 +491,7 @@ impl PDRouter {
|
|||||||
// its byte stream records the outcome on drop. Skip the
|
// its byte stream records the outcome on drop. Skip the
|
||||||
// eager success record to avoid masking "200-then-broken"
|
// eager success record to avoid masking "200-then-broken"
|
||||||
// decode workers.
|
// decode workers.
|
||||||
if !ctx_is_stream {
|
if let (false, Some(decode)) = (ctx_is_stream, &decode) {
|
||||||
decode.record_outcome(not_error);
|
decode.record_outcome(not_error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -649,40 +676,40 @@ impl PDRouter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Internal method that performs the actual dual dispatch (without retry logic)
|
// Internal method that performs the actual dual dispatch (without retry logic)
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
async fn execute_dual_dispatch_internal(
|
async fn execute_dual_dispatch_internal(
|
||||||
&self,
|
&self,
|
||||||
headers: Option<&HeaderMap>,
|
headers: Option<&HeaderMap>,
|
||||||
json_request: Value,
|
json_request: Value,
|
||||||
context: PDRequestContext<'_>,
|
context: PDRequestContext<'_>,
|
||||||
prefill: Arc<dyn Worker>,
|
prefill: Arc<dyn Worker>,
|
||||||
decode: Arc<dyn Worker>,
|
decode: Option<Arc<dyn Worker>>,
|
||||||
_start_time: Instant,
|
_start_time: Instant,
|
||||||
) -> Response {
|
) -> (Response, Option<Arc<dyn Worker>>) {
|
||||||
// Count the request from dispatch so load-based policies see it before
|
// Count the request from dispatch so load-based policies see it before
|
||||||
// upstream headers arrive; streaming responses attach their own guards.
|
// upstream headers arrive; streaming responses attach their own guards.
|
||||||
let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers);
|
let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers);
|
||||||
let _decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
|
|
||||||
|
|
||||||
let mut headers_with_trace = headers.cloned().unwrap_or_default();
|
let mut headers_with_trace = headers.cloned().unwrap_or_default();
|
||||||
inject_trace_context_http(&mut headers_with_trace);
|
inject_trace_context_http(&mut headers_with_trace);
|
||||||
let headers = Some(&headers_with_trace);
|
let headers = Some(&headers_with_trace);
|
||||||
|
|
||||||
let (prepared_prefill, prepared_decode) = match Self::prepare_pd_worker_requests(
|
let prepared_prefill = match Self::prepare_worker_request(
|
||||||
context.route,
|
context.route,
|
||||||
&json_request,
|
|
||||||
prefill.as_ref(),
|
prefill.as_ref(),
|
||||||
decode.as_ref(),
|
Cow::Borrowed(&json_request),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(requests) => requests,
|
Ok(request) => request,
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
error!("Failed to prepare PD worker requests: {}", e);
|
error!("Failed to prepare PD worker requests: {}", e);
|
||||||
return error::internal_error("pd_request_preparation_failed", e);
|
return (
|
||||||
|
error::internal_error("pd_request_preparation_failed", e),
|
||||||
|
None,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
// Build both requests
|
|
||||||
let prefill_request = self.build_post_with_headers(
|
let prefill_request = self.build_post_with_headers(
|
||||||
&self.client,
|
&self.client,
|
||||||
&prepared_prefill.endpoint_url,
|
&prepared_prefill.endpoint_url,
|
||||||
@@ -690,32 +717,77 @@ impl PDRouter {
|
|||||||
headers,
|
headers,
|
||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
let decode_request = self.build_post_with_headers(
|
|
||||||
&self.client,
|
let slot = decode.is_none().then(DecodeSlot::default);
|
||||||
&prepared_decode.endpoint_url,
|
let decode_leg = async {
|
||||||
&prepared_decode.body,
|
if let (Some(gate), Some(slot)) = (&self.decode_gate, &slot) {
|
||||||
headers,
|
let admit = gate.admit(prefill.url(), slot);
|
||||||
false,
|
if tokio::time::timeout(self.request_timeout, admit)
|
||||||
);
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
return Err(error::create_error(
|
||||||
|
StatusCode::GATEWAY_TIMEOUT,
|
||||||
|
"decode_dispatch_timeout",
|
||||||
|
"Timed out waiting to dispatch the decode request",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let decode = match decode {
|
||||||
|
Some(decode) => decode,
|
||||||
|
None => self
|
||||||
|
.select_worker(
|
||||||
|
context.request_text.as_deref(),
|
||||||
|
context.model_id,
|
||||||
|
context.headers.as_ref(),
|
||||||
|
true,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(Self::handle_server_selection_error)?,
|
||||||
|
};
|
||||||
|
let decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
|
||||||
|
let prepared_decode = async {
|
||||||
|
let body = Self::inject_prefill_dp_rank_for_decode(
|
||||||
|
Cow::Borrowed(&json_request),
|
||||||
|
prefill.as_ref(),
|
||||||
|
)?;
|
||||||
|
Self::prepare_worker_request(context.route, decode.as_ref(), body).await
|
||||||
|
}
|
||||||
|
.await
|
||||||
|
.map_err(|e| {
|
||||||
|
error!("Failed to prepare PD worker requests: {}", e);
|
||||||
|
error::internal_error("pd_request_preparation_failed", e)
|
||||||
|
})?;
|
||||||
|
let decode_request = self.build_post_with_headers(
|
||||||
|
&self.client,
|
||||||
|
&prepared_decode.endpoint_url,
|
||||||
|
&prepared_decode.body,
|
||||||
|
headers,
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
events::RequestPDSentEvent {
|
||||||
|
prefill_url: prefill.url(),
|
||||||
|
decode_url: decode.url(),
|
||||||
|
}
|
||||||
|
.emit();
|
||||||
|
let result = decode_request.send().await;
|
||||||
|
if let Some(slot) = &slot {
|
||||||
|
slot.release();
|
||||||
|
}
|
||||||
|
Ok((decode, decode_guard, result))
|
||||||
|
};
|
||||||
|
|
||||||
// Run both in this handler task (not a detached tokio::spawn) so a client
|
// Run both in this handler task (not a detached tokio::spawn) so a client
|
||||||
// disconnect cancels the pending decode request too, keeping the
|
// disconnect cancels the pending decode request too, keeping the
|
||||||
// upstream-cancel behavior from #19524.
|
// upstream-cancel behavior from #19524.
|
||||||
events::RequestPDSentEvent {
|
|
||||||
prefill_url: prefill.url(),
|
|
||||||
decode_url: decode.url(),
|
|
||||||
}
|
|
||||||
.emit();
|
|
||||||
|
|
||||||
let prefill_fut = prefill_request.send();
|
let prefill_fut = prefill_request.send();
|
||||||
let decode_fut = decode_request.send();
|
|
||||||
tokio::pin!(prefill_fut);
|
tokio::pin!(prefill_fut);
|
||||||
tokio::pin!(decode_fut);
|
tokio::pin!(decode_leg);
|
||||||
|
|
||||||
// Poll both until prefill resolves; decode normally resolves later, but
|
// Poll both until prefill resolves; decode normally resolves later, but
|
||||||
// may resolve first if it rejects the request outright.
|
// may resolve first if it rejects the request outright.
|
||||||
let prefill_result;
|
let prefill_result;
|
||||||
let mut decode_early: Option<Result<reqwest::Response, reqwest::Error>> = None;
|
let mut decode_early = None;
|
||||||
loop {
|
loop {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
biased;
|
biased;
|
||||||
@@ -723,11 +795,14 @@ impl PDRouter {
|
|||||||
prefill_result = pr;
|
prefill_result = pr;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
dr = &mut decode_fut, if decode_early.is_none() => {
|
dr = &mut decode_leg, if decode_early.is_none() => {
|
||||||
decode_early = Some(dr);
|
decode_early = Some(dr);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if let Some(slot) = &slot {
|
||||||
|
slot.release();
|
||||||
|
}
|
||||||
|
|
||||||
// Decode can't generate without prefill's KV, so any prefill failure
|
// Decode can't generate without prefill's KV, so any prefill failure
|
||||||
// (non-2xx / transport error) dooms the paired decode request, which would
|
// (non-2xx / transport error) dooms the paired decode request, which would
|
||||||
@@ -741,8 +816,7 @@ impl PDRouter {
|
|||||||
|
|
||||||
if prefill_failed {
|
if prefill_failed {
|
||||||
warn!(
|
warn!(
|
||||||
"Prefill failed, aborting paired decode request decode_url={} prefill_url={}",
|
"Prefill failed, aborting paired decode request prefill_url={}",
|
||||||
decode.url(),
|
|
||||||
prefill.url()
|
prefill.url()
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -767,17 +841,41 @@ impl PDRouter {
|
|||||||
),
|
),
|
||||||
};
|
};
|
||||||
response.extensions_mut().insert(BreakerOutcomesRecorded);
|
response.extensions_mut().insert(BreakerOutcomesRecorded);
|
||||||
return response;
|
return (response, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Prefill ok: take decode's result, awaiting it if still pending.
|
// Prefill ok: take decode's result, awaiting it if still pending.
|
||||||
let decode_result = match decode_early {
|
let leg = match decode_early {
|
||||||
Some(dr) => dr,
|
Some(leg) => leg,
|
||||||
None => (&mut decode_fut).await,
|
None => (&mut decode_leg).await,
|
||||||
|
};
|
||||||
|
let (decode, _decode_guard, decode_result) = match leg {
|
||||||
|
Ok(leg) => leg,
|
||||||
|
Err(response) => return (response, None),
|
||||||
};
|
};
|
||||||
|
|
||||||
events::RequestReceivedEvent {}.emit();
|
events::RequestReceivedEvent {}.emit();
|
||||||
|
|
||||||
|
let response = self
|
||||||
|
.process_pd_responses(
|
||||||
|
&context,
|
||||||
|
Arc::clone(&prefill),
|
||||||
|
Arc::clone(&decode),
|
||||||
|
prefill_result,
|
||||||
|
decode_result,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
(response, Some(decode))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process_pd_responses(
|
||||||
|
&self,
|
||||||
|
context: &PDRequestContext<'_>,
|
||||||
|
prefill: Arc<dyn Worker>,
|
||||||
|
decode: Arc<dyn Worker>,
|
||||||
|
prefill_result: Result<reqwest::Response, reqwest::Error>,
|
||||||
|
decode_result: Result<reqwest::Response, reqwest::Error>,
|
||||||
|
) -> Response {
|
||||||
// Process decode response
|
// Process decode response
|
||||||
match decode_result {
|
match decode_result {
|
||||||
Ok(res) => {
|
Ok(res) => {
|
||||||
@@ -821,7 +919,7 @@ impl PDRouter {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let mut response = self
|
let mut response = self
|
||||||
.handle_decode_error_response(res, &context, prefill, decode)
|
.handle_decode_error_response(res, context, prefill, decode)
|
||||||
.await;
|
.await;
|
||||||
response.extensions_mut().insert(BreakerOutcomesRecorded);
|
response.extensions_mut().insert(BreakerOutcomesRecorded);
|
||||||
return response;
|
return response;
|
||||||
@@ -979,79 +1077,83 @@ impl PDRouter {
|
|||||||
model_id: Option<&str>,
|
model_id: Option<&str>,
|
||||||
headers: Option<&HeaderMap>,
|
headers: Option<&HeaderMap>,
|
||||||
) -> Result<(Arc<dyn Worker>, Arc<dyn Worker>), String> {
|
) -> Result<(Arc<dyn Worker>, Arc<dyn Worker>), String> {
|
||||||
|
let prefill = self
|
||||||
|
.select_worker(request_text, model_id, headers, false)
|
||||||
|
.await?;
|
||||||
|
let decode = self
|
||||||
|
.select_worker(request_text, model_id, headers, true)
|
||||||
|
.await?;
|
||||||
|
Ok((prefill, decode))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn select_worker(
|
||||||
|
&self,
|
||||||
|
request_text: Option<&str>,
|
||||||
|
model_id: Option<&str>,
|
||||||
|
headers: Option<&HeaderMap>,
|
||||||
|
decode: bool,
|
||||||
|
) -> Result<Arc<dyn Worker>, String> {
|
||||||
let effective_model_id = if !self.enable_igw { None } else { model_id };
|
let effective_model_id = if !self.enable_igw { None } else { model_id };
|
||||||
|
|
||||||
debug!(
|
debug!(
|
||||||
"Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}",
|
"Selecting PD worker: decode={}, enable_igw={}, model_id={:?}, effective_model_id={:?}",
|
||||||
self.enable_igw, model_id, effective_model_id
|
decode, self.enable_igw, model_id, effective_model_id
|
||||||
);
|
);
|
||||||
|
|
||||||
let prefill_workers = if let Some(model) = effective_model_id {
|
let workers = match effective_model_id {
|
||||||
self.worker_registry
|
Some(model) => self
|
||||||
|
.worker_registry
|
||||||
.get_by_model(model)
|
.get_by_model(model)
|
||||||
.iter()
|
.iter()
|
||||||
.filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. }))
|
.filter(|w| match w.worker_type() {
|
||||||
|
WorkerType::Decode => decode,
|
||||||
|
WorkerType::Prefill { .. } => !decode,
|
||||||
|
_ => false,
|
||||||
|
})
|
||||||
.cloned()
|
.cloned()
|
||||||
.collect()
|
.collect(),
|
||||||
} else {
|
None if decode => self.worker_registry.get_decode_workers(),
|
||||||
self.worker_registry.get_prefill_workers()
|
None => self.worker_registry.get_prefill_workers(),
|
||||||
};
|
};
|
||||||
|
|
||||||
let decode_workers = if let Some(model) = effective_model_id {
|
let (policy, role, label) = if decode {
|
||||||
self.worker_registry
|
(
|
||||||
.get_by_model(model)
|
self.policy_registry.get_decode_policy(),
|
||||||
.iter()
|
"decode",
|
||||||
.filter(|w| matches!(w.worker_type(), WorkerType::Decode))
|
metrics_labels::WORKER_DECODE,
|
||||||
.cloned()
|
)
|
||||||
.collect()
|
|
||||||
} else {
|
} else {
|
||||||
self.worker_registry.get_decode_workers()
|
(
|
||||||
|
self.policy_registry.get_prefill_policy(),
|
||||||
|
"prefill",
|
||||||
|
metrics_labels::WORKER_PREFILL,
|
||||||
|
)
|
||||||
};
|
};
|
||||||
|
|
||||||
let prefill_policy = self.policy_registry.get_prefill_policy();
|
|
||||||
let decode_policy = self.policy_registry.get_decode_policy();
|
|
||||||
|
|
||||||
// Get cached hash ring for consistent hashing
|
// Get cached hash ring for consistent hashing
|
||||||
let hash_ring = self
|
let hash_ring = self
|
||||||
.worker_registry
|
.worker_registry
|
||||||
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
|
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
|
||||||
|
|
||||||
let prefill = Self::pick_worker_by_policy_arc(
|
let worker = Self::pick_worker_by_policy_arc(
|
||||||
&prefill_workers,
|
&workers,
|
||||||
&*prefill_policy,
|
&*policy,
|
||||||
request_text,
|
|
||||||
headers,
|
|
||||||
hash_ring.clone(),
|
|
||||||
"prefill",
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
|
|
||||||
let decode = Self::pick_worker_by_policy_arc(
|
|
||||||
&decode_workers,
|
|
||||||
&*decode_policy,
|
|
||||||
request_text,
|
request_text,
|
||||||
headers,
|
headers,
|
||||||
hash_ring,
|
hash_ring,
|
||||||
"decode",
|
role,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
// Record worker selection metrics (Layer 3)
|
// Record worker selection metrics (Layer 3)
|
||||||
let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
|
|
||||||
Metrics::record_worker_selection(
|
Metrics::record_worker_selection(
|
||||||
metrics_labels::WORKER_PREFILL,
|
label,
|
||||||
metrics_labels::CONNECTION_HTTP,
|
metrics_labels::CONNECTION_HTTP,
|
||||||
model,
|
model_id.unwrap_or(UNKNOWN_MODEL_ID),
|
||||||
prefill_policy.name(),
|
policy.name(),
|
||||||
);
|
|
||||||
Metrics::record_worker_selection(
|
|
||||||
metrics_labels::WORKER_DECODE,
|
|
||||||
metrics_labels::CONNECTION_HTTP,
|
|
||||||
model,
|
|
||||||
decode_policy.name(),
|
|
||||||
);
|
);
|
||||||
|
|
||||||
Ok((prefill, decode))
|
Ok(worker)
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn pick_worker_by_policy_arc(
|
async fn pick_worker_by_policy_arc(
|
||||||
@@ -1778,6 +1880,8 @@ mod tests {
|
|||||||
retry_config: RetryConfig::default(),
|
retry_config: RetryConfig::default(),
|
||||||
api_key: Some("test_api_key".to_string()),
|
api_key: Some("test_api_key".to_string()),
|
||||||
enable_igw: false,
|
enable_igw: false,
|
||||||
|
decode_gate: None,
|
||||||
|
request_timeout: Duration::from_secs(600),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -308,6 +308,7 @@ async fn generate_handler(
|
|||||||
Json(payload): Json<serde_json::Value>,
|
Json(payload): Json<serde_json::Value>,
|
||||||
) -> Response {
|
) -> Response {
|
||||||
let config = config.read().await;
|
let config = config.read().await;
|
||||||
|
record_request_arrival(config.port);
|
||||||
let worker_id = format!("worker-{}", config.port);
|
let worker_id = format!("worker-{}", config.port);
|
||||||
|
|
||||||
if should_fail(&config).await {
|
if should_fail(&config).await {
|
||||||
@@ -1470,6 +1471,27 @@ async fn responses_cancel_handler(
|
|||||||
// can enable slow streaming WITHOUT relying on the request payload
|
// can enable slow streaming WITHOUT relying on the request payload
|
||||||
// (the gateway deserializes/re-serializes the body, dropping unknown fields).
|
// (the gateway deserializes/re-serializes the body, dropping unknown fields).
|
||||||
|
|
||||||
|
static REQUEST_ARRIVALS: OnceLock<Mutex<HashMap<u16, Vec<std::time::Instant>>>> = OnceLock::new();
|
||||||
|
|
||||||
|
fn record_request_arrival(port: u16) {
|
||||||
|
REQUEST_ARRIVALS
|
||||||
|
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.entry(port)
|
||||||
|
.or_default()
|
||||||
|
.push(std::time::Instant::now());
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn take_request_arrivals(port: u16) -> Vec<std::time::Instant> {
|
||||||
|
REQUEST_ARRIVALS
|
||||||
|
.get_or_init(|| Mutex::new(HashMap::new()))
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.remove(&port)
|
||||||
|
.unwrap_or_default()
|
||||||
|
}
|
||||||
|
|
||||||
static SLOW_STREAM_CONFIG: OnceLock<Mutex<HashMap<u16, usize>>> = OnceLock::new();
|
static SLOW_STREAM_CONFIG: OnceLock<Mutex<HashMap<u16, usize>>> = OnceLock::new();
|
||||||
|
|
||||||
fn get_slow_stream_config() -> &'static Mutex<HashMap<u16, usize>> {
|
fn get_slow_stream_config() -> &'static Mutex<HashMap<u16, usize>> {
|
||||||
|
|||||||
@@ -219,6 +219,87 @@ mod pd_routing_tests {
|
|||||||
|
|
||||||
ctx.shutdown().await;
|
ctx.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Prefill gets both requests at once; the second decode waits for a slot.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_pd_decode_dispatch_limit_holds_decode_not_prefill() {
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use crate::common::mock_worker::take_request_arrivals;
|
||||||
|
|
||||||
|
let (prefill_port, decode_port) = (19858, 19859);
|
||||||
|
let config = RouterConfig::builder()
|
||||||
|
.prefill_decode_mode(
|
||||||
|
vec![(format!("http://127.0.0.1:{prefill_port}"), None)],
|
||||||
|
vec![format!("http://127.0.0.1:{decode_port}")],
|
||||||
|
)
|
||||||
|
.round_robin_policy()
|
||||||
|
.decode_dispatch_limit(1)
|
||||||
|
.host("127.0.0.1")
|
||||||
|
.port(3858)
|
||||||
|
.max_payload_size(256 * 1024 * 1024)
|
||||||
|
.request_timeout_secs(600)
|
||||||
|
.worker_startup_timeout_secs(5)
|
||||||
|
.worker_startup_check_interval_secs(1)
|
||||||
|
.max_concurrent_requests(64)
|
||||||
|
.queue_timeout_secs(60)
|
||||||
|
.build_unchecked();
|
||||||
|
let slow = |mut worker: MockWorkerConfig| {
|
||||||
|
worker.response_delay_ms = 600;
|
||||||
|
worker
|
||||||
|
};
|
||||||
|
let ctx = AppTestContext::new_with_config(
|
||||||
|
config,
|
||||||
|
vec![
|
||||||
|
slow(TestWorkerConfig::prefill(prefill_port)),
|
||||||
|
slow(TestWorkerConfig::decode(decode_port)),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let app = ctx.create_app().await;
|
||||||
|
take_request_arrivals(prefill_port);
|
||||||
|
take_request_arrivals(decode_port);
|
||||||
|
|
||||||
|
let send = || {
|
||||||
|
let app = app.clone();
|
||||||
|
async move {
|
||||||
|
let request = Request::builder()
|
||||||
|
.method("POST")
|
||||||
|
.uri("/generate")
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.body(Body::from(
|
||||||
|
json!({"text": "hi", "stream": false}).to_string(),
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
app.oneshot(request).await.unwrap().status()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let start = Instant::now();
|
||||||
|
let statuses = tokio::join!(send(), send());
|
||||||
|
assert_eq!(statuses, (StatusCode::OK, StatusCode::OK));
|
||||||
|
|
||||||
|
let since_start = |port| {
|
||||||
|
let mut arrivals: Vec<_> = take_request_arrivals(port)
|
||||||
|
.into_iter()
|
||||||
|
.map(|t| t - start)
|
||||||
|
.collect();
|
||||||
|
arrivals.sort();
|
||||||
|
arrivals
|
||||||
|
};
|
||||||
|
let prefill = since_start(prefill_port);
|
||||||
|
let decode = since_start(decode_port);
|
||||||
|
assert!(
|
||||||
|
prefill.len() == 2 && prefill[1] < Duration::from_millis(300),
|
||||||
|
"{prefill:?}"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
decode.len() == 2
|
||||||
|
&& decode[0] < Duration::from_millis(300)
|
||||||
|
&& decode[1] >= Duration::from_millis(500),
|
||||||
|
"{decode:?}"
|
||||||
|
);
|
||||||
|
ctx.shutdown().await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -321,6 +402,8 @@ mod pd_responses_routing_tests {
|
|||||||
retry_config: RetryConfig::default(),
|
retry_config: RetryConfig::default(),
|
||||||
api_key: None,
|
api_key: None,
|
||||||
enable_igw: false,
|
enable_igw: false,
|
||||||
|
decode_gate: None,
|
||||||
|
request_timeout: Duration::from_secs(600),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user