From 4ef9765e76be4952d25f6b8ed160600aefc0757f Mon Sep 17 00:00:00 2001 From: lixiufei-leo Date: Tue, 20 Oct 2026 01:13:25 +0800 Subject: [PATCH] [Feat] sgl-router: Add --pd-stream-after-prefill (#45102) Co-authored-by: Xiufei Li <294029388+lixiufei-leo@users.noreply.github.com> --- experimental/sgl-router/src/config/cli.rs | 4 ++ experimental/sgl-router/src/config/types.rs | 3 + .../src/server/routes/chat/forward.rs | 66 +++++++++++++++---- 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/experimental/sgl-router/src/config/cli.rs b/experimental/sgl-router/src/config/cli.rs index f366d63cb7..492acf4a82 100644 --- a/experimental/sgl-router/src/config/cli.rs +++ b/experimental/sgl-router/src/config/cli.rs @@ -169,6 +169,9 @@ pub struct ServerArgs { /// Upper bound on the backoff between retries, in milliseconds. #[arg(long, default_value_t = ProxyConfig::default().max_backoff_ms)] pub retry_max_backoff_ms: u64, + /// PD: return decode's stream only once prefill completes, as sgl-model-gateway does. + #[arg(long)] + pub pd_stream_after_prefill: bool, /// PD: at most N requests per prefill DP rank go to decode before their /// first token; the rest wait and get their decode DP rank when sent. @@ -660,6 +663,7 @@ impl Cli { initial_backoff_ms: self.server.retry_initial_backoff_ms, max_backoff_ms: self.server.retry_max_backoff_ms, decode_dispatch_limit: self.server.decode_dispatch_limit, + stream_after_prefill: self.server.pd_stream_after_prefill, }, router_inflight_load: InflightLoadConfig { stale_request_timeout_secs: self.server.stale_request_timeout_secs, diff --git a/experimental/sgl-router/src/config/types.rs b/experimental/sgl-router/src/config/types.rs index 74808b75d2..62f3e4431c 100644 --- a/experimental/sgl-router/src/config/types.rs +++ b/experimental/sgl-router/src/config/types.rs @@ -32,6 +32,8 @@ pub struct ProxyConfig { pub max_backoff_ms: u64, /// PD: requests per prefill DP rank sent to decode before their first token. pub decode_dispatch_limit: Option, + /// PD: return decode's response only once prefill completes. + pub stream_after_prefill: bool, } impl ProxyConfig { @@ -64,6 +66,7 @@ impl Default for ProxyConfig { initial_backoff_ms: 50, max_backoff_ms: 2000, decode_dispatch_limit: None, + stream_after_prefill: false, } } } diff --git a/experimental/sgl-router/src/server/routes/chat/forward.rs b/experimental/sgl-router/src/server/routes/chat/forward.rs index ad286e7706..04f735dc81 100644 --- a/experimental/sgl-router/src/server/routes/chat/forward.rs +++ b/experimental/sgl-router/src/server/routes/chat/forward.rs @@ -344,7 +344,10 @@ async fn dispatch( ); let dispatch = async { match prefill_task { - Some((task, prefill)) => forward_pd(task, prefill, response).await, + Some((task, prefill)) => { + let wait = ctx.config.proxy.stream_after_prefill; + forward_pd(task, prefill, response, wait).await + } None => (response.await, None), } }; @@ -841,12 +844,14 @@ struct Blame { decode_dispatched: bool, } -/// Returns decode's response as soon as decode answers, unless prefill fails +/// Returns decode's response as soon as decode answers, or with +/// `stream_after_prefill` once prefill also completes, unless prefill fails /// first; then prefill's failure is returned along with the blamed prefill. async fn forward_pd( - task: tokio::task::JoinHandle>, + mut task: tokio::task::JoinHandle>, prefill: Arc, decode: impl Future>, + stream_after_prefill: bool, ) -> (Result, Option) { // Set on decode's first poll, which is where `forward_to_response_worker` // begins and the request reaches the worker. Read from the same task, so @@ -863,15 +868,26 @@ async fn forward_pd( decode_dispatched: dispatched.load(Ordering::Relaxed), }) }; - tokio::select! { - biased; - failure = task => match failure { - Ok(None) => (decode.await, None), - Ok(Some(failure)) => (failure, blame(prefill)), - Err(_) => (Err(ApiError::PrefillFailed { status: None }), blame(prefill)), - }, - // Dropping the handle leaves prefill running. - response = &mut decode => (response, None), + let mut held = None; + loop { + tokio::select! { + biased; + failure = &mut task => return match failure { + Ok(None) => match held { + Some(response) => (response, None), + None => (decode.await, None), + }, + Ok(Some(failure)) => (failure, blame(prefill)), + Err(_) => (Err(ApiError::PrefillFailed { status: None }), blame(prefill)), + }, + // Dropping the handle leaves prefill running. + response = &mut decode, if held.is_none() => { + if !stream_after_prefill { + return (response, None); + } + held = Some(response); + } + } } } @@ -1075,3 +1091,29 @@ impl Drop for RequestDurationGuard { .observe_request_duration(&self.model, self.request_started_at.elapsed().as_secs_f64()); } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn stream_after_prefill_waits_for_prefill() { + let prefill = Arc::new(Worker::new(crate::discovery::WorkerSpec { + mode: WorkerMode::Prefill, + ..Default::default() + })); + for wait in [false, true] { + let done = Arc::new(AtomicBool::new(false)); + let finished = Arc::clone(&done); + let task = tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + finished.store(true, Ordering::SeqCst); + None + }); + let decode = async { Ok(Response::new(Body::empty())) }; + let (response, blame) = forward_pd(task, Arc::clone(&prefill), decode, wait).await; + assert!(response.is_ok() && blame.is_none()); + assert_eq!(done.load(Ordering::SeqCst), wait); + } + } +}