[Feat] sgl-router: Add --pd-stream-after-prefill (#45102)
Co-authored-by: Xiufei Li <294029388+lixiufei-leo@users.noreply.github.com>
This commit is contained in:
co-authored by
Xiufei Li
parent
716c9151f2
commit
4ef9765e76
@@ -169,6 +169,9 @@ pub struct ServerArgs {
|
||||
/// Upper bound on the backoff between retries, in milliseconds.
|
||||
#[arg(long, default_value_t = ProxyConfig::default().max_backoff_ms)]
|
||||
pub retry_max_backoff_ms: u64,
|
||||
/// PD: return decode's stream only once prefill completes, as sgl-model-gateway does.
|
||||
#[arg(long)]
|
||||
pub pd_stream_after_prefill: bool,
|
||||
|
||||
/// PD: at most N requests per prefill DP rank go to decode before their
|
||||
/// first token; the rest wait and get their decode DP rank when sent.
|
||||
@@ -660,6 +663,7 @@ impl Cli {
|
||||
initial_backoff_ms: self.server.retry_initial_backoff_ms,
|
||||
max_backoff_ms: self.server.retry_max_backoff_ms,
|
||||
decode_dispatch_limit: self.server.decode_dispatch_limit,
|
||||
stream_after_prefill: self.server.pd_stream_after_prefill,
|
||||
},
|
||||
router_inflight_load: InflightLoadConfig {
|
||||
stale_request_timeout_secs: self.server.stale_request_timeout_secs,
|
||||
|
||||
@@ -32,6 +32,8 @@ pub struct ProxyConfig {
|
||||
pub max_backoff_ms: u64,
|
||||
/// PD: requests per prefill DP rank sent to decode before their first token.
|
||||
pub decode_dispatch_limit: Option<NonZeroU32>,
|
||||
/// PD: return decode's response only once prefill completes.
|
||||
pub stream_after_prefill: bool,
|
||||
}
|
||||
|
||||
impl ProxyConfig {
|
||||
@@ -64,6 +66,7 @@ impl Default for ProxyConfig {
|
||||
initial_backoff_ms: 50,
|
||||
max_backoff_ms: 2000,
|
||||
decode_dispatch_limit: None,
|
||||
stream_after_prefill: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -344,7 +344,10 @@ async fn dispatch<T: Transport>(
|
||||
);
|
||||
let dispatch = async {
|
||||
match prefill_task {
|
||||
Some((task, prefill)) => forward_pd(task, prefill, response).await,
|
||||
Some((task, prefill)) => {
|
||||
let wait = ctx.config.proxy.stream_after_prefill;
|
||||
forward_pd(task, prefill, response, wait).await
|
||||
}
|
||||
None => (response.await, None),
|
||||
}
|
||||
};
|
||||
@@ -841,12 +844,14 @@ struct Blame {
|
||||
decode_dispatched: bool,
|
||||
}
|
||||
|
||||
/// Returns decode's response as soon as decode answers, unless prefill fails
|
||||
/// Returns decode's response as soon as decode answers, or with
|
||||
/// `stream_after_prefill` once prefill also completes, unless prefill fails
|
||||
/// first; then prefill's failure is returned along with the blamed prefill.
|
||||
async fn forward_pd<R>(
|
||||
task: tokio::task::JoinHandle<PrefillFailure<R>>,
|
||||
mut task: tokio::task::JoinHandle<PrefillFailure<R>>,
|
||||
prefill: Arc<Worker>,
|
||||
decode: impl Future<Output = Result<R, ApiError>>,
|
||||
stream_after_prefill: bool,
|
||||
) -> (Result<R, ApiError>, Option<Blame>) {
|
||||
// Set on decode's first poll, which is where `forward_to_response_worker`
|
||||
// begins and the request reaches the worker. Read from the same task, so
|
||||
@@ -863,15 +868,26 @@ async fn forward_pd<R>(
|
||||
decode_dispatched: dispatched.load(Ordering::Relaxed),
|
||||
})
|
||||
};
|
||||
tokio::select! {
|
||||
biased;
|
||||
failure = task => match failure {
|
||||
Ok(None) => (decode.await, None),
|
||||
Ok(Some(failure)) => (failure, blame(prefill)),
|
||||
Err(_) => (Err(ApiError::PrefillFailed { status: None }), blame(prefill)),
|
||||
},
|
||||
// Dropping the handle leaves prefill running.
|
||||
response = &mut decode => (response, None),
|
||||
let mut held = None;
|
||||
loop {
|
||||
tokio::select! {
|
||||
biased;
|
||||
failure = &mut task => return match failure {
|
||||
Ok(None) => match held {
|
||||
Some(response) => (response, None),
|
||||
None => (decode.await, None),
|
||||
},
|
||||
Ok(Some(failure)) => (failure, blame(prefill)),
|
||||
Err(_) => (Err(ApiError::PrefillFailed { status: None }), blame(prefill)),
|
||||
},
|
||||
// Dropping the handle leaves prefill running.
|
||||
response = &mut decode, if held.is_none() => {
|
||||
if !stream_after_prefill {
|
||||
return (response, None);
|
||||
}
|
||||
held = Some(response);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1075,3 +1091,29 @@ impl Drop for RequestDurationGuard {
|
||||
.observe_request_duration(&self.model, self.request_started_at.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn stream_after_prefill_waits_for_prefill() {
|
||||
let prefill = Arc::new(Worker::new(crate::discovery::WorkerSpec {
|
||||
mode: WorkerMode::Prefill,
|
||||
..Default::default()
|
||||
}));
|
||||
for wait in [false, true] {
|
||||
let done = Arc::new(AtomicBool::new(false));
|
||||
let finished = Arc::clone(&done);
|
||||
let task = tokio::spawn(async move {
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
finished.store(true, Ordering::SeqCst);
|
||||
None
|
||||
});
|
||||
let decode = async { Ok(Response::new(Body::empty())) };
|
||||
let (response, blame) = forward_pd(task, Arc::clone(&prefill), decode, wait).await;
|
||||
assert!(response.is_ok() && blame.is_none());
|
||||
assert_eq!(done.load(Ordering::SeqCst), wait);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user