minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Super tiny extract route_typed_request_once (#14951)

Browse Source
This commit is contained in:
fzyzcjy
2025-12-12 11:10:10 -08:00
committed by GitHub
parent b243154614
commit ec242f516e
1 changed files with 78 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
sgl-model-gateway/src/routers/http/router.rs
+36 -23
View File
@@ -175,7 +175,42 @@ impl Router {
&self.retry_config,
// operation per attempt
|_: u32| async {
let worker = match self.select_worker_for_model(model_id, Some(&text)) {
self.route_typed_request_once(headers, typed_req, route, model_id, is_stream, &text)
.await
},
// should_retry predicate
|res, _attempt| is_retryable_status(res.status()),
// on_backoff hook
|delay, attempt| {
RouterMetrics::record_retry(route);
RouterMetrics::record_retry_backoff_duration(delay, attempt);
},
// on_exhausted hook
|| RouterMetrics::record_retries_exhausted(route),
)
.await;
if response.status().is_success() {
let duration = start.elapsed();
RouterMetrics::record_request(route);
RouterMetrics::record_generate_duration(duration);
} else if !is_retryable_status(response.status()) {
RouterMetrics::record_request_error(route, "non_retryable_error");
}
response
}
async fn route_typed_request_once<T: GenerationRequest + serde::Serialize + Clone>(
&self,
headers: Option<&HeaderMap>,
typed_req: &T,
route: &str,
model_id: Option<&str>,
is_stream: bool,
text: &str,
) -> Response {
let worker = match self.select_worker_for_model(model_id, Some(text)) {
Some(w) => w,
None => {
RouterMetrics::record_request_error(route, "no_available_workers");
@@ -239,28 +274,6 @@ impl Router {
}
}
response
},
// should_retry predicate
|res, _attempt| is_retryable_status(res.status()),
// on_backoff hook
|delay, attempt| {
RouterMetrics::record_retry(route);
RouterMetrics::record_retry_backoff_duration(delay, attempt);
},
// on_exhausted hook
|| RouterMetrics::record_retries_exhausted(route),
)
.await;
if response.status().is_success() {
let duration = start.elapsed();
RouterMetrics::record_request(route);
RouterMetrics::record_generate_duration(duration);
} else if !is_retryable_status(response.status()) {
RouterMetrics::record_request_error(route, "non_retryable_error");
}
response
}