From 996db4b60eecbbba7b1f3405eb2bf445bc69bb15 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Sat, 14 Nov 2026 13:03:58 +0800 Subject: [PATCH] gateway: forward /v1/responses raw bodies so unknown tool types pass through --- .../src/routers/http/pd_router.rs | 78 +++++++++++++++++++ sgl-model-gateway/src/routers/mod.rs | 21 +++++ .../src/routers/router_manager.rs | 24 ++++++ sgl-model-gateway/src/server.rs | 19 ++++- 4 files changed, 139 insertions(+), 3 deletions(-) diff --git a/sgl-model-gateway/src/routers/http/pd_router.rs b/sgl-model-gateway/src/routers/http/pd_router.rs index 4c0d649844..be76f30c3c 100644 --- a/sgl-model-gateway/src/routers/http/pd_router.rs +++ b/sgl-model-gateway/src/routers/http/pd_router.rs @@ -2027,6 +2027,84 @@ impl RouterTrait for PDRouter { self.execute_dual_dispatch(headers, body, context).await } + /// `/v1/responses` from its raw body: forward tool schemas the gateway does + /// not model (namespace, custom, tool_search, ...) without rejecting them. + async fn route_responses_raw( + &self, + headers: Option<&HeaderMap>, + body: &Value, + model_id: Option<&str>, + ) -> Response { + let is_stream = body + .get("stream") + .and_then(Value::as_bool) + .unwrap_or(false); + let background = body + .get("background") + .and_then(Value::as_bool) + .unwrap_or(false); + + if background && !is_stream { + warn!("PD mode does not support detached background responses; returning bad request"); + return error::bad_request( + "pd_unsupported_background_responses", + "PD mode does not support background responses without streaming", + ); + } + + let request_text = if self.policies_need_request_text() { + let text = Self::responses_request_text(body); + (!text.is_empty()).then_some(text) + } else { + None + }; + + let context = PDRequestContext { + route: "/v1/responses", + batch_size: None, + is_stream, + return_logprob: false, + request_text, + model_id, + headers: headers.cloned(), + }; + + self.execute_dual_dispatch(headers, body, context).await + } + + /// Plain text of a raw Responses body, mirroring the typed + /// `extract_text_for_routing` for cache-aware routing. + fn responses_request_text(body: &Value) -> String { + fn collect(value: &Value, out: &mut Vec) { + match value { + Value::String(text) => { + if !text.is_empty() { + out.push(text.clone()); + } + } + Value::Array(items) => { + for item in items { + collect(item, out); + } + } + Value::Object(map) => { + for key in ["text", "content", "arguments", "output"] { + if let Some(inner) = map.get(key) { + collect(inner, out); + } + } + } + _ => {} + } + } + + let mut parts = Vec::new(); + if let Some(input) = body.get("input") { + collect(input, &mut parts); + } + parts.join(" ") + } + async fn route_rerank( &self, headers: Option<&HeaderMap>, diff --git a/sgl-model-gateway/src/routers/mod.rs b/sgl-model-gateway/src/routers/mod.rs index 11cb05bbb8..ad3eec34bf 100644 --- a/sgl-model-gateway/src/routers/mod.rs +++ b/sgl-model-gateway/src/routers/mod.rs @@ -126,6 +126,27 @@ pub trait RouterTrait: Send + Sync + Debug { .into_response() } + /// Route a responses request from its raw JSON body. + /// + /// A gateway forwards tool schemas it does not model (namespace, custom, + /// tool_search, ...) unchanged; implementations that can forward the body + /// themselves should override this instead of rejecting unknown tool types. + async fn route_responses_raw( + &self, + headers: Option<&HeaderMap>, + body: &serde_json::Value, + model_id: Option<&str>, + ) -> Response { + match serde_json::from_value::(body.clone()) { + Ok(parsed) => self.route_responses(headers, &parsed, model_id).await, + Err(err) => ( + StatusCode::BAD_REQUEST, + format!("Invalid responses request: {err}"), + ) + .into_response(), + } + } + /// Retrieve a stored/background response by id async fn get_response( &self, diff --git a/sgl-model-gateway/src/routers/router_manager.rs b/sgl-model-gateway/src/routers/router_manager.rs index 01014e2dd4..48c0edf077 100644 --- a/sgl-model-gateway/src/routers/router_manager.rs +++ b/sgl-model-gateway/src/routers/router_manager.rs @@ -614,6 +614,30 @@ impl RouterTrait for RouterManager { } } + async fn route_responses_raw( + &self, + headers: Option<&HeaderMap>, + body: &serde_json::Value, + model_id: Option<&str>, + ) -> Response { + let selected_model = model_id + .map(str::to_string) + .or_else(|| body.get("model").and_then(|v| v.as_str()).map(str::to_string)); + let router = self.select_router_for_request(headers, selected_model.as_deref()); + + if let Some(router) = router { + router + .route_responses_raw(headers, body, selected_model.as_deref()) + .await + } else { + ( + StatusCode::NOT_FOUND, + "No router available to handle responses request", + ) + .into_response() + } + } + async fn get_response( &self, headers: Option<&HeaderMap>, diff --git a/sgl-model-gateway/src/server.rs b/sgl-model-gateway/src/server.rs index 99af905a37..85839d772d 100644 --- a/sgl-model-gateway/src/server.rs +++ b/sgl-model-gateway/src/server.rs @@ -47,7 +47,7 @@ use crate::{ generate::GenerateRequest, parser::{ParseFunctionCallRequest, SeparateReasoningRequest}, rerank::V1RerankReqInput, - responses::{ResponsesGetParams, ResponsesRequest}, + responses::ResponsesGetParams, tokenize::{AddTokenizerRequest, DetokenizeRequest, TokenizeRequest}, validated::ValidatedJson, worker_spec::{WorkerConfigRequest, WorkerUpdateRequest}, @@ -218,11 +218,24 @@ async fn v1_rerank( async fn v1_responses( State(state): State>, headers: http::HeaderMap, - ValidatedJson(body): ValidatedJson, + Json(body): Json, ) -> Response { + let model = match body.get("model").and_then(Value::as_str) { + Some(model) if !model.is_empty() => model.to_string(), + _ => { + return ( + StatusCode::BAD_REQUEST, + "model is required for /v1/responses", + ) + .into_response() + } + }; + + // Forward the raw body: the gateway must not reject tool types it does not + // model (namespace, custom, tool_search, ...); the engine owns the schema. state .router - .route_responses(Some(&headers), &body, Some(&body.model)) + .route_responses_raw(Some(&headers), &body, Some(&model)) .await }