[Feat] sgl-router: Add --pd-stream-after-prefill (#45102)

Co-authored-by: Xiufei Li <294029388+lixiufei-leo@users.noreply.github.com>
This commit is contained in:
lixiufei-leo
2026-10-20 01:13:25 +08:00
committed by GitHub
co-authored by Xiufei Li
parent 716c9151f2
commit 4ef9765e76
3 changed files with 61 additions and 12 deletions
@@ -169,6 +169,9 @@ pub struct ServerArgs {
/// Upper bound on the backoff between retries, in milliseconds.
#[arg(long, default_value_t = ProxyConfig::default().max_backoff_ms)]
pub retry_max_backoff_ms: u64,
/// PD: return decode's stream only once prefill completes, as sgl-model-gateway does.
#[arg(long)]
pub pd_stream_after_prefill: bool,
/// PD: at most N requests per prefill DP rank go to decode before their
/// first token; the rest wait and get their decode DP rank when sent.
@@ -660,6 +663,7 @@ impl Cli {
initial_backoff_ms: self.server.retry_initial_backoff_ms,
max_backoff_ms: self.server.retry_max_backoff_ms,
decode_dispatch_limit: self.server.decode_dispatch_limit,
stream_after_prefill: self.server.pd_stream_after_prefill,
},
router_inflight_load: InflightLoadConfig {
stale_request_timeout_secs: self.server.stale_request_timeout_secs,
@@ -32,6 +32,8 @@ pub struct ProxyConfig {
pub max_backoff_ms: u64,
/// PD: requests per prefill DP rank sent to decode before their first token.
pub decode_dispatch_limit: Option<NonZeroU32>,
/// PD: return decode's response only once prefill completes.
pub stream_after_prefill: bool,
}
impl ProxyConfig {
@@ -64,6 +66,7 @@ impl Default for ProxyConfig {
initial_backoff_ms: 50,
max_backoff_ms: 2000,
decode_dispatch_limit: None,
stream_after_prefill: false,
}
}
}
@@ -344,7 +344,10 @@ async fn dispatch<T: Transport>(
);
let dispatch = async {
match prefill_task {
Some((task, prefill)) => forward_pd(task, prefill, response).await,
Some((task, prefill)) => {
let wait = ctx.config.proxy.stream_after_prefill;
forward_pd(task, prefill, response, wait).await
}
None => (response.await, None),
}
};
@@ -841,12 +844,14 @@ struct Blame {
decode_dispatched: bool,
}
/// Returns decode's response as soon as decode answers, unless prefill fails
/// Returns decode's response as soon as decode answers, or with
/// `stream_after_prefill` once prefill also completes, unless prefill fails
/// first; then prefill's failure is returned along with the blamed prefill.
async fn forward_pd<R>(
task: tokio::task::JoinHandle<PrefillFailure<R>>,
mut task: tokio::task::JoinHandle<PrefillFailure<R>>,
prefill: Arc<Worker>,
decode: impl Future<Output = Result<R, ApiError>>,
stream_after_prefill: bool,
) -> (Result<R, ApiError>, Option<Blame>) {
// Set on decode's first poll, which is where `forward_to_response_worker`
// begins and the request reaches the worker. Read from the same task, so
@@ -863,15 +868,26 @@ async fn forward_pd<R>(
decode_dispatched: dispatched.load(Ordering::Relaxed),
})
};
tokio::select! {
biased;
failure = task => match failure {
Ok(None) => (decode.await, None),
Ok(Some(failure)) => (failure, blame(prefill)),
Err(_) => (Err(ApiError::PrefillFailed { status: None }), blame(prefill)),
},
// Dropping the handle leaves prefill running.
response = &mut decode => (response, None),
let mut held = None;
loop {
tokio::select! {
biased;
failure = &mut task => return match failure {
Ok(None) => match held {
Some(response) => (response, None),
None => (decode.await, None),
},
Ok(Some(failure)) => (failure, blame(prefill)),
Err(_) => (Err(ApiError::PrefillFailed { status: None }), blame(prefill)),
},
// Dropping the handle leaves prefill running.
response = &mut decode, if held.is_none() => {
if !stream_after_prefill {
return (response, None);
}
held = Some(response);
}
}
}
}
@@ -1075,3 +1091,29 @@ impl Drop for RequestDurationGuard {
.observe_request_duration(&self.model, self.request_started_at.elapsed().as_secs_f64());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn stream_after_prefill_waits_for_prefill() {
let prefill = Arc::new(Worker::new(crate::discovery::WorkerSpec {
mode: WorkerMode::Prefill,
..Default::default()
}));
for wait in [false, true] {
let done = Arc::new(AtomicBool::new(false));
let finished = Arc::clone(&done);
let task = tokio::spawn(async move {
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
finished.store(true, Ordering::SeqCst);
None
});
let decode = async { Ok(Response::new(Body::empty())) };
let (response, blame) = forward_pd(task, Arc::clone(&prefill), decode, wait).await;
assert!(response.is_ok() && blame.is_none());
assert_eq!(done.load(Ordering::SeqCst), wait);
}
}
}