diff --git a/docs/docs/advanced_features/sgl_model_gateway.mdx b/docs/docs/advanced_features/sgl_model_gateway.mdx
index 049e6d4081..489ed8eb61 100644
--- a/docs/docs/advanced_features/sgl_model_gateway.mdx
+++ b/docs/docs/advanced_features/sgl_model_gateway.mdx
@@ -2670,6 +2670,12 @@ groups:
None |
Override policy for decode nodes |
+
+ | `--decode-dispatch-limit` |
+ int |
+ None |
+ Requests per prefill worker sent to decode before decode answers or prefill finishes; the rest pick their decode worker when sent. Needs prefill `--optimistic-prefill-attempts` above 0 |
+
| `--worker-startup-timeout-secs` |
int |
diff --git a/sgl-model-gateway/README.md b/sgl-model-gateway/README.md
index bffeae39f1..89a3c8e1d2 100644
--- a/sgl-model-gateway/README.md
+++ b/sgl-model-gateway/README.md
@@ -160,6 +160,8 @@ The `--version` (or `-V`) flag displays the version string. Use `--version-verbo
```
Prefill entries accept an optional bootstrap port. PD mode merges prefill metadata with decode outputs and streams results back to the client.
+A decode worker picked on arrival only starts decoding once prefill has queued, computed and sent the KV, so with long prefill queues, decode workers (with `--dp-aware`, DP ranks) that hold equal numbers of requests can run unequal batches. `--decode-dispatch-limit N` still sends every request to prefill on arrival, but sends at most N per prefill worker on to decode before decode answers or prefill finishes; the rest wait in arrival order and pick their decode worker when they are sent. Prefill must start a request before decode connects, so prefill engines need `--optimistic-prefill-attempts` above 0.
+
### Multi-Model Inference Gateway
Enable IGW mode to route multiple models through a single router while applying per-model policies:
```bash
diff --git a/sgl-model-gateway/bindings/python/src/lib.rs b/sgl-model-gateway/bindings/python/src/lib.rs
index f18bdf4ec6..447e4f87ca 100644
--- a/sgl-model-gateway/bindings/python/src/lib.rs
+++ b/sgl-model-gateway/bindings/python/src/lib.rs
@@ -412,6 +412,7 @@ struct Router {
queue_size: usize,
queue_timeout_secs: u64,
rate_limit_tokens_per_second: Option,
+ decode_dispatch_limit: Option,
connection_mode: core::ConnectionMode,
model_path: Option,
tokenizer_path: Option,
@@ -637,6 +638,7 @@ impl Router {
.maybe_log_level(self.log_level.as_ref())
.maybe_request_id_headers(self.request_id_headers.clone())
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
+ .maybe_decode_dispatch_limit(self.decode_dispatch_limit)
.maybe_model_path(self.model_path.as_ref())
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
.maybe_chat_template(self.chat_template.as_ref())
@@ -733,6 +735,7 @@ impl Router {
queue_size = 100,
queue_timeout_secs = 60,
rate_limit_tokens_per_second = None,
+ decode_dispatch_limit = None,
model_path = None,
tokenizer_path = None,
chat_template = None,
@@ -825,6 +828,7 @@ impl Router {
queue_size: usize,
queue_timeout_secs: u64,
rate_limit_tokens_per_second: Option,
+ decode_dispatch_limit: Option,
model_path: Option,
tokenizer_path: Option,
chat_template: Option,
@@ -930,6 +934,7 @@ impl Router {
queue_size,
queue_timeout_secs,
rate_limit_tokens_per_second,
+ decode_dispatch_limit,
connection_mode,
model_path,
tokenizer_path,
diff --git a/sgl-model-gateway/bindings/python/src/sglang_router/router.py b/sgl-model-gateway/bindings/python/src/sglang_router/router.py
index fa051b5765..0679bf192e 100644
--- a/sgl-model-gateway/bindings/python/src/sglang_router/router.py
+++ b/sgl-model-gateway/bindings/python/src/sglang_router/router.py
@@ -211,6 +211,7 @@ class Router:
queue_size: Queue size for pending requests when max concurrent limit reached (0 = no queue, return 429 immediately). Default: 100
queue_timeout_secs: Maximum time (in seconds) a request can wait in queue before timing out. Default: 60
rate_limit_tokens_per_second: Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests. Default: None
+ decode_dispatch_limit: Max requests per prefill worker in flight to decode (PD mode only). Default: None
cors_allowed_origins: List of allowed origins for CORS. Empty list allows all origins. Default: []
health_failure_threshold: Number of consecutive health check failures before marking worker unhealthy. Default: 3
health_success_threshold: Number of consecutive health check successes before marking worker healthy. Default: 2
diff --git a/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py b/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py
index a2ea4f2eab..5e7e0e7511 100644
--- a/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py
+++ b/sgl-model-gateway/bindings/python/src/sglang_router/router_args.py
@@ -98,6 +98,7 @@ class RouterArgs:
queue_timeout_secs: int = 60
# Token bucket refill rate (tokens per second). If not set, defaults to max_concurrent_requests
rate_limit_tokens_per_second: Optional[int] = None
+ decode_dispatch_limit: Optional[int] = None
# CORS allowed origins
cors_allowed_origins: List[str] = dataclasses.field(default_factory=list)
# Retry configuration
@@ -398,6 +399,13 @@ class RouterArgs:
metavar=("URL",),
help="Decode server URL. Can be specified multiple times.",
)
+ pd_group.add_argument(
+ f"--{prefix}decode-dispatch-limit",
+ type=int,
+ default=RouterArgs.decode_dispatch_limit,
+ help="Max requests per prefill worker in flight to decode; the rest pick their "
+ "decode worker when sent (PD mode only)",
+ )
pd_group.add_argument(
f"--{prefix}worker-startup-timeout-secs",
type=int,
diff --git a/sgl-model-gateway/src/config/builder.rs b/sgl-model-gateway/src/config/builder.rs
index 5d4891ca60..ad24d408d5 100644
--- a/sgl-model-gateway/src/config/builder.rs
+++ b/sgl-model-gateway/src/config/builder.rs
@@ -229,6 +229,11 @@ impl RouterConfigBuilder {
self
}
+ pub fn decode_dispatch_limit(mut self, limit: usize) -> Self {
+ self.config.decode_dispatch_limit = Some(limit);
+ self
+ }
+
pub fn rate_limit_tokens_per_second(mut self, tokens: i32) -> Self {
self.config.rate_limit_tokens_per_second = Some(tokens);
self
@@ -531,6 +536,11 @@ impl RouterConfigBuilder {
self
}
+ pub fn maybe_decode_dispatch_limit(mut self, limit: Option) -> Self {
+ self.config.decode_dispatch_limit = limit;
+ self
+ }
+
pub fn maybe_rate_limit_tokens_per_second(mut self, tokens: Option) -> Self {
self.config.rate_limit_tokens_per_second = tokens;
self
diff --git a/sgl-model-gateway/src/config/types.rs b/sgl-model-gateway/src/config/types.rs
index b19802642c..231047b6a2 100644
--- a/sgl-model-gateway/src/config/types.rs
+++ b/sgl-model-gateway/src/config/types.rs
@@ -26,6 +26,8 @@ pub struct RouterConfig {
pub worker_startup_timeout_secs: u64,
pub worker_startup_check_interval_secs: u64,
pub dp_aware: bool,
+ #[serde(default)]
+ pub decode_dispatch_limit: Option,
pub api_key: Option,
pub discovery: Option,
pub metrics: Option,
@@ -514,6 +516,7 @@ impl Default for RouterConfig {
worker_startup_timeout_secs: 1800, // 30 minutes for large model loading
worker_startup_check_interval_secs: 30,
dp_aware: false,
+ decode_dispatch_limit: None,
api_key: None,
discovery: None,
metrics: None,
diff --git a/sgl-model-gateway/src/config/validation.rs b/sgl-model-gateway/src/config/validation.rs
index bd3d38ab8f..47be1f2933 100644
--- a/sgl-model-gateway/src/config/validation.rs
+++ b/sgl-model-gateway/src/config/validation.rs
@@ -286,6 +286,14 @@ impl ConfigValidator {
});
}
+ if config.decode_dispatch_limit == Some(0) {
+ return Err(ConfigError::InvalidValue {
+ field: "decode_dispatch_limit".to_string(),
+ value: "0".to_string(),
+ reason: "Must be > 0 when specified".to_string(),
+ });
+ }
+
if let Some(tokens_per_second) = config.rate_limit_tokens_per_second {
// Allow 0 for pure concurrency limiting (semaphore behavior)
if tokens_per_second < 0 {
diff --git a/sgl-model-gateway/src/main.rs b/sgl-model-gateway/src/main.rs
index b389bdad19..15296826b7 100644
--- a/sgl-model-gateway/src/main.rs
+++ b/sgl-model-gateway/src/main.rs
@@ -211,6 +211,10 @@ struct CliArgs {
#[arg(long, value_parser = ["random", "round_robin", "cache_aware", "power_of_two", "prefix_hash", "manual"], help_heading = "PD Disaggregation")]
decode_policy: Option,
+ /// Max requests per prefill worker in flight to decode; the rest pick their decode worker when sent
+ #[arg(long, help_heading = "PD Disaggregation")]
+ decode_dispatch_limit: Option,
+
/// Timeout in seconds for worker startup and registration
#[arg(long, default_value_t = 1800, help_heading = "PD Disaggregation")]
worker_startup_timeout_secs: u64,
@@ -1056,6 +1060,7 @@ impl CliArgs {
(!self.request_id_headers.is_empty()).then(|| self.request_id_headers.clone()),
)
.maybe_rate_limit_tokens_per_second(self.rate_limit_tokens_per_second)
+ .maybe_decode_dispatch_limit(self.decode_dispatch_limit)
.maybe_model_path(self.model_path.as_ref())
.maybe_tokenizer_path(self.tokenizer_path.as_ref())
.maybe_chat_template(self.chat_template.as_ref())
diff --git a/sgl-model-gateway/src/routers/http/decode_gate.rs b/sgl-model-gateway/src/routers/http/decode_gate.rs
new file mode 100644
index 0000000000..27431e1f28
--- /dev/null
+++ b/sgl-model-gateway/src/routers/http/decode_gate.rs
@@ -0,0 +1,39 @@
+use std::sync::{Arc, Mutex};
+
+use dashmap::DashMap;
+use tokio::sync::{OwnedSemaphorePermit, Semaphore};
+
+#[derive(Debug)]
+pub struct DecodeGate {
+ limit: usize,
+ gates: DashMap>,
+}
+
+impl DecodeGate {
+ pub fn new(limit: usize) -> Self {
+ Self {
+ limit,
+ gates: DashMap::new(),
+ }
+ }
+
+ pub async fn admit(&self, prefill_url: &str, slot: &DecodeSlot) {
+ let gate = Arc::clone(
+ &self
+ .gates
+ .entry(prefill_url.to_string())
+ .or_insert_with(|| Arc::new(Semaphore::new(self.limit))),
+ );
+ let permit = gate.acquire_owned().await.expect("gates are never closed");
+ *slot.0.lock().unwrap() = Some(permit);
+ }
+}
+
+#[derive(Debug, Clone, Default)]
+pub struct DecodeSlot(Arc>>);
+
+impl DecodeSlot {
+ pub fn release(&self) {
+ self.0.lock().unwrap().take();
+ }
+}
diff --git a/sgl-model-gateway/src/routers/http/mod.rs b/sgl-model-gateway/src/routers/http/mod.rs
index 3f31b6f869..de8268eaeb 100644
--- a/sgl-model-gateway/src/routers/http/mod.rs
+++ b/sgl-model-gateway/src/routers/http/mod.rs
@@ -1,5 +1,6 @@
//! HTTP router implementations
+pub mod decode_gate;
pub mod pd_router;
pub mod pd_types;
pub mod router;
diff --git a/sgl-model-gateway/src/routers/http/pd_router.rs b/sgl-model-gateway/src/routers/http/pd_router.rs
index a0a196db64..ea60c3f299 100644
--- a/sgl-model-gateway/src/routers/http/pd_router.rs
+++ b/sgl-model-gateway/src/routers/http/pd_router.rs
@@ -1,4 +1,8 @@
-use std::{borrow::Cow, sync::Arc, time::Instant};
+use std::{
+ borrow::Cow,
+ sync::Arc,
+ time::{Duration, Instant},
+};
use async_trait::async_trait;
use axum::{
@@ -18,7 +22,10 @@ use serde_json::{json, Value};
use tokio_stream::wrappers::UnboundedReceiverStream;
use tracing::{debug, error, warn};
-use super::pd_types::api_path;
+use super::{
+ decode_gate::{DecodeGate, DecodeSlot},
+ pd_types::api_path,
+};
use crate::{
config::types::RetryConfig,
core::{
@@ -58,6 +65,8 @@ pub struct PDRouter {
pub retry_config: RetryConfig,
pub api_key: Option,
pub enable_igw: bool,
+ pub decode_gate: Option>,
+ pub request_timeout: Duration,
}
struct PreparedWorkerRequest<'a> {
@@ -189,6 +198,11 @@ impl PDRouter {
retry_config: ctx.router_config.effective_retry_config(),
api_key: ctx.router_config.api_key.clone(),
enable_igw: ctx.router_config.enable_igw,
+ decode_gate: ctx
+ .router_config
+ .decode_dispatch_limit
+ .map(|limit| Arc::new(DecodeGate::new(limit))),
+ request_timeout: Duration::from_secs(ctx.router_config.request_timeout_secs),
})
}
@@ -350,6 +364,7 @@ impl PDRouter {
})
}
+ #[cfg(test)]
async fn prepare_pd_worker_requests<'a>(
route: &'static str,
json_request: &'a Value,
@@ -398,14 +413,26 @@ impl PDRouter {
let shared_request = Arc::clone(&shared_request);
let context = context.clone();
async move {
- let (prefill, decode) = match self
- .select_pd_pair(
- context.request_text.as_deref(),
+ let request_text = context.request_text.as_deref();
+ let selected = if self.decode_gate.is_some() {
+ self.select_worker(
+ request_text,
+ context.model_id,
+ context.headers.as_ref(),
+ false,
+ )
+ .await
+ .map(|prefill| (prefill, None))
+ } else {
+ self.select_pd_pair(
+ request_text,
context.model_id,
context.headers.as_ref(),
)
.await
- {
+ .map(|(prefill, decode)| (prefill, Some(decode)))
+ };
+ let (prefill, decode) = match selected {
Ok(pair) => pair,
Err(e) => {
return Self::handle_server_selection_error(e);
@@ -413,10 +440,10 @@ impl PDRouter {
};
debug!(
- "PD retry attempt {} using prefill={} decode={}",
+ "PD retry attempt {} using prefill={} decode={:?}",
attempt,
prefill.url(),
- decode.url()
+ decode.as_ref().map(|d| d.url())
);
let mut json_request = match serde_json::to_value(shared_request.as_ref()) {
@@ -438,13 +465,13 @@ impl PDRouter {
};
let ctx_is_stream = context.is_stream;
- let response = self
+ let (response, decode) = self
.execute_dual_dispatch_internal(
headers,
json_request,
context,
Arc::clone(&prefill),
- Arc::clone(&decode),
+ decode,
start_time,
)
.await;
@@ -464,7 +491,7 @@ impl PDRouter {
// its byte stream records the outcome on drop. Skip the
// eager success record to avoid masking "200-then-broken"
// decode workers.
- if !ctx_is_stream {
+ if let (false, Some(decode)) = (ctx_is_stream, &decode) {
decode.record_outcome(not_error);
}
}
@@ -649,40 +676,40 @@ impl PDRouter {
}
// Internal method that performs the actual dual dispatch (without retry logic)
+ #[allow(clippy::too_many_arguments)]
async fn execute_dual_dispatch_internal(
&self,
headers: Option<&HeaderMap>,
json_request: Value,
context: PDRequestContext<'_>,
prefill: Arc,
- decode: Arc,
+ decode: Option>,
_start_time: Instant,
- ) -> Response {
+ ) -> (Response, Option>) {
// Count the request from dispatch so load-based policies see it before
// upstream headers arrive; streaming responses attach their own guards.
let _prefill_guard = WorkerLoadGuard::new(prefill.clone(), headers);
- let _decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
let mut headers_with_trace = headers.cloned().unwrap_or_default();
inject_trace_context_http(&mut headers_with_trace);
let headers = Some(&headers_with_trace);
- let (prepared_prefill, prepared_decode) = match Self::prepare_pd_worker_requests(
+ let prepared_prefill = match Self::prepare_worker_request(
context.route,
- &json_request,
prefill.as_ref(),
- decode.as_ref(),
+ Cow::Borrowed(&json_request),
)
.await
{
- Ok(requests) => requests,
+ Ok(request) => request,
Err(e) => {
error!("Failed to prepare PD worker requests: {}", e);
- return error::internal_error("pd_request_preparation_failed", e);
+ return (
+ error::internal_error("pd_request_preparation_failed", e),
+ None,
+ );
}
};
-
- // Build both requests
let prefill_request = self.build_post_with_headers(
&self.client,
&prepared_prefill.endpoint_url,
@@ -690,32 +717,77 @@ impl PDRouter {
headers,
false,
);
- let decode_request = self.build_post_with_headers(
- &self.client,
- &prepared_decode.endpoint_url,
- &prepared_decode.body,
- headers,
- false,
- );
+
+ let slot = decode.is_none().then(DecodeSlot::default);
+ let decode_leg = async {
+ if let (Some(gate), Some(slot)) = (&self.decode_gate, &slot) {
+ let admit = gate.admit(prefill.url(), slot);
+ if tokio::time::timeout(self.request_timeout, admit)
+ .await
+ .is_err()
+ {
+ return Err(error::create_error(
+ StatusCode::GATEWAY_TIMEOUT,
+ "decode_dispatch_timeout",
+ "Timed out waiting to dispatch the decode request",
+ ));
+ }
+ }
+ let decode = match decode {
+ Some(decode) => decode,
+ None => self
+ .select_worker(
+ context.request_text.as_deref(),
+ context.model_id,
+ context.headers.as_ref(),
+ true,
+ )
+ .await
+ .map_err(Self::handle_server_selection_error)?,
+ };
+ let decode_guard = WorkerLoadGuard::new(decode.clone(), headers);
+ let prepared_decode = async {
+ let body = Self::inject_prefill_dp_rank_for_decode(
+ Cow::Borrowed(&json_request),
+ prefill.as_ref(),
+ )?;
+ Self::prepare_worker_request(context.route, decode.as_ref(), body).await
+ }
+ .await
+ .map_err(|e| {
+ error!("Failed to prepare PD worker requests: {}", e);
+ error::internal_error("pd_request_preparation_failed", e)
+ })?;
+ let decode_request = self.build_post_with_headers(
+ &self.client,
+ &prepared_decode.endpoint_url,
+ &prepared_decode.body,
+ headers,
+ false,
+ );
+ events::RequestPDSentEvent {
+ prefill_url: prefill.url(),
+ decode_url: decode.url(),
+ }
+ .emit();
+ let result = decode_request.send().await;
+ if let Some(slot) = &slot {
+ slot.release();
+ }
+ Ok((decode, decode_guard, result))
+ };
// Run both in this handler task (not a detached tokio::spawn) so a client
// disconnect cancels the pending decode request too, keeping the
// upstream-cancel behavior from #19524.
- events::RequestPDSentEvent {
- prefill_url: prefill.url(),
- decode_url: decode.url(),
- }
- .emit();
-
let prefill_fut = prefill_request.send();
- let decode_fut = decode_request.send();
tokio::pin!(prefill_fut);
- tokio::pin!(decode_fut);
+ tokio::pin!(decode_leg);
// Poll both until prefill resolves; decode normally resolves later, but
// may resolve first if it rejects the request outright.
let prefill_result;
- let mut decode_early: Option> = None;
+ let mut decode_early = None;
loop {
tokio::select! {
biased;
@@ -723,11 +795,14 @@ impl PDRouter {
prefill_result = pr;
break;
}
- dr = &mut decode_fut, if decode_early.is_none() => {
+ dr = &mut decode_leg, if decode_early.is_none() => {
decode_early = Some(dr);
}
}
}
+ if let Some(slot) = &slot {
+ slot.release();
+ }
// Decode can't generate without prefill's KV, so any prefill failure
// (non-2xx / transport error) dooms the paired decode request, which would
@@ -741,8 +816,7 @@ impl PDRouter {
if prefill_failed {
warn!(
- "Prefill failed, aborting paired decode request decode_url={} prefill_url={}",
- decode.url(),
+ "Prefill failed, aborting paired decode request prefill_url={}",
prefill.url()
);
@@ -767,17 +841,41 @@ impl PDRouter {
),
};
response.extensions_mut().insert(BreakerOutcomesRecorded);
- return response;
+ return (response, None);
}
// Prefill ok: take decode's result, awaiting it if still pending.
- let decode_result = match decode_early {
- Some(dr) => dr,
- None => (&mut decode_fut).await,
+ let leg = match decode_early {
+ Some(leg) => leg,
+ None => (&mut decode_leg).await,
+ };
+ let (decode, _decode_guard, decode_result) = match leg {
+ Ok(leg) => leg,
+ Err(response) => return (response, None),
};
events::RequestReceivedEvent {}.emit();
+ let response = self
+ .process_pd_responses(
+ &context,
+ Arc::clone(&prefill),
+ Arc::clone(&decode),
+ prefill_result,
+ decode_result,
+ )
+ .await;
+ (response, Some(decode))
+ }
+
+ async fn process_pd_responses(
+ &self,
+ context: &PDRequestContext<'_>,
+ prefill: Arc,
+ decode: Arc,
+ prefill_result: Result,
+ decode_result: Result,
+ ) -> Response {
// Process decode response
match decode_result {
Ok(res) => {
@@ -821,7 +919,7 @@ impl PDRouter {
}
let mut response = self
- .handle_decode_error_response(res, &context, prefill, decode)
+ .handle_decode_error_response(res, context, prefill, decode)
.await;
response.extensions_mut().insert(BreakerOutcomesRecorded);
return response;
@@ -979,79 +1077,83 @@ impl PDRouter {
model_id: Option<&str>,
headers: Option<&HeaderMap>,
) -> Result<(Arc, Arc), String> {
+ let prefill = self
+ .select_worker(request_text, model_id, headers, false)
+ .await?;
+ let decode = self
+ .select_worker(request_text, model_id, headers, true)
+ .await?;
+ Ok((prefill, decode))
+ }
+
+ async fn select_worker(
+ &self,
+ request_text: Option<&str>,
+ model_id: Option<&str>,
+ headers: Option<&HeaderMap>,
+ decode: bool,
+ ) -> Result, String> {
let effective_model_id = if !self.enable_igw { None } else { model_id };
debug!(
- "Selecting PD pair: enable_igw={}, model_id={:?}, effective_model_id={:?}",
- self.enable_igw, model_id, effective_model_id
+ "Selecting PD worker: decode={}, enable_igw={}, model_id={:?}, effective_model_id={:?}",
+ decode, self.enable_igw, model_id, effective_model_id
);
- let prefill_workers = if let Some(model) = effective_model_id {
- self.worker_registry
+ let workers = match effective_model_id {
+ Some(model) => self
+ .worker_registry
.get_by_model(model)
.iter()
- .filter(|w| matches!(w.worker_type(), WorkerType::Prefill { .. }))
+ .filter(|w| match w.worker_type() {
+ WorkerType::Decode => decode,
+ WorkerType::Prefill { .. } => !decode,
+ _ => false,
+ })
.cloned()
- .collect()
- } else {
- self.worker_registry.get_prefill_workers()
+ .collect(),
+ None if decode => self.worker_registry.get_decode_workers(),
+ None => self.worker_registry.get_prefill_workers(),
};
- let decode_workers = if let Some(model) = effective_model_id {
- self.worker_registry
- .get_by_model(model)
- .iter()
- .filter(|w| matches!(w.worker_type(), WorkerType::Decode))
- .cloned()
- .collect()
+ let (policy, role, label) = if decode {
+ (
+ self.policy_registry.get_decode_policy(),
+ "decode",
+ metrics_labels::WORKER_DECODE,
+ )
} else {
- self.worker_registry.get_decode_workers()
+ (
+ self.policy_registry.get_prefill_policy(),
+ "prefill",
+ metrics_labels::WORKER_PREFILL,
+ )
};
- let prefill_policy = self.policy_registry.get_prefill_policy();
- let decode_policy = self.policy_registry.get_decode_policy();
-
// Get cached hash ring for consistent hashing
let hash_ring = self
.worker_registry
.get_hash_ring(effective_model_id.unwrap_or(UNKNOWN_MODEL_ID));
- let prefill = Self::pick_worker_by_policy_arc(
- &prefill_workers,
- &*prefill_policy,
- request_text,
- headers,
- hash_ring.clone(),
- "prefill",
- )
- .await?;
-
- let decode = Self::pick_worker_by_policy_arc(
- &decode_workers,
- &*decode_policy,
+ let worker = Self::pick_worker_by_policy_arc(
+ &workers,
+ &*policy,
request_text,
headers,
hash_ring,
- "decode",
+ role,
)
.await?;
// Record worker selection metrics (Layer 3)
- let model = model_id.unwrap_or(UNKNOWN_MODEL_ID);
Metrics::record_worker_selection(
- metrics_labels::WORKER_PREFILL,
+ label,
metrics_labels::CONNECTION_HTTP,
- model,
- prefill_policy.name(),
- );
- Metrics::record_worker_selection(
- metrics_labels::WORKER_DECODE,
- metrics_labels::CONNECTION_HTTP,
- model,
- decode_policy.name(),
+ model_id.unwrap_or(UNKNOWN_MODEL_ID),
+ policy.name(),
);
- Ok((prefill, decode))
+ Ok(worker)
}
async fn pick_worker_by_policy_arc(
@@ -1778,6 +1880,8 @@ mod tests {
retry_config: RetryConfig::default(),
api_key: Some("test_api_key".to_string()),
enable_igw: false,
+ decode_gate: None,
+ request_timeout: Duration::from_secs(600),
}
}
diff --git a/sgl-model-gateway/tests/common/mock_worker.rs b/sgl-model-gateway/tests/common/mock_worker.rs
index 131a5bbb63..148e623270 100755
--- a/sgl-model-gateway/tests/common/mock_worker.rs
+++ b/sgl-model-gateway/tests/common/mock_worker.rs
@@ -308,6 +308,7 @@ async fn generate_handler(
Json(payload): Json,
) -> Response {
let config = config.read().await;
+ record_request_arrival(config.port);
let worker_id = format!("worker-{}", config.port);
if should_fail(&config).await {
@@ -1470,6 +1471,27 @@ async fn responses_cancel_handler(
// can enable slow streaming WITHOUT relying on the request payload
// (the gateway deserializes/re-serializes the body, dropping unknown fields).
+static REQUEST_ARRIVALS: OnceLock>>> = OnceLock::new();
+
+fn record_request_arrival(port: u16) {
+ REQUEST_ARRIVALS
+ .get_or_init(|| Mutex::new(HashMap::new()))
+ .lock()
+ .unwrap()
+ .entry(port)
+ .or_default()
+ .push(std::time::Instant::now());
+}
+
+pub fn take_request_arrivals(port: u16) -> Vec {
+ REQUEST_ARRIVALS
+ .get_or_init(|| Mutex::new(HashMap::new()))
+ .lock()
+ .unwrap()
+ .remove(&port)
+ .unwrap_or_default()
+}
+
static SLOW_STREAM_CONFIG: OnceLock>> = OnceLock::new();
fn get_slow_stream_config() -> &'static Mutex> {
diff --git a/sgl-model-gateway/tests/routing/pd_routing_test.rs b/sgl-model-gateway/tests/routing/pd_routing_test.rs
index 6ad31453d7..423c0c1d4d 100644
--- a/sgl-model-gateway/tests/routing/pd_routing_test.rs
+++ b/sgl-model-gateway/tests/routing/pd_routing_test.rs
@@ -219,6 +219,87 @@ mod pd_routing_tests {
ctx.shutdown().await;
}
+
+ /// Prefill gets both requests at once; the second decode waits for a slot.
+ #[tokio::test]
+ async fn test_pd_decode_dispatch_limit_holds_decode_not_prefill() {
+ use std::time::{Duration, Instant};
+
+ use crate::common::mock_worker::take_request_arrivals;
+
+ let (prefill_port, decode_port) = (19858, 19859);
+ let config = RouterConfig::builder()
+ .prefill_decode_mode(
+ vec![(format!("http://127.0.0.1:{prefill_port}"), None)],
+ vec![format!("http://127.0.0.1:{decode_port}")],
+ )
+ .round_robin_policy()
+ .decode_dispatch_limit(1)
+ .host("127.0.0.1")
+ .port(3858)
+ .max_payload_size(256 * 1024 * 1024)
+ .request_timeout_secs(600)
+ .worker_startup_timeout_secs(5)
+ .worker_startup_check_interval_secs(1)
+ .max_concurrent_requests(64)
+ .queue_timeout_secs(60)
+ .build_unchecked();
+ let slow = |mut worker: MockWorkerConfig| {
+ worker.response_delay_ms = 600;
+ worker
+ };
+ let ctx = AppTestContext::new_with_config(
+ config,
+ vec![
+ slow(TestWorkerConfig::prefill(prefill_port)),
+ slow(TestWorkerConfig::decode(decode_port)),
+ ],
+ )
+ .await;
+ let app = ctx.create_app().await;
+ take_request_arrivals(prefill_port);
+ take_request_arrivals(decode_port);
+
+ let send = || {
+ let app = app.clone();
+ async move {
+ let request = Request::builder()
+ .method("POST")
+ .uri("/generate")
+ .header(CONTENT_TYPE, "application/json")
+ .body(Body::from(
+ json!({"text": "hi", "stream": false}).to_string(),
+ ))
+ .unwrap();
+ app.oneshot(request).await.unwrap().status()
+ }
+ };
+ let start = Instant::now();
+ let statuses = tokio::join!(send(), send());
+ assert_eq!(statuses, (StatusCode::OK, StatusCode::OK));
+
+ let since_start = |port| {
+ let mut arrivals: Vec<_> = take_request_arrivals(port)
+ .into_iter()
+ .map(|t| t - start)
+ .collect();
+ arrivals.sort();
+ arrivals
+ };
+ let prefill = since_start(prefill_port);
+ let decode = since_start(decode_port);
+ assert!(
+ prefill.len() == 2 && prefill[1] < Duration::from_millis(300),
+ "{prefill:?}"
+ );
+ assert!(
+ decode.len() == 2
+ && decode[0] < Duration::from_millis(300)
+ && decode[1] >= Duration::from_millis(500),
+ "{decode:?}"
+ );
+ ctx.shutdown().await;
+ }
}
#[cfg(test)]
@@ -321,6 +402,8 @@ mod pd_responses_routing_tests {
retry_config: RetryConfig::default(),
api_key: None,
enable_igw: false,
+ decode_gate: None,
+ request_timeout: Duration::from_secs(600),
}
}