gateway: forward /v1/responses raw bodies so unknown tool types pass through

This commit is contained in:
2026-11-14 05:18:54 +00:00
committed by ymk
parent 216880034a
commit 996db4b60e
4 changed files with 139 additions and 3 deletions
@@ -2027,6 +2027,84 @@ impl RouterTrait for PDRouter {
self.execute_dual_dispatch(headers, body, context).await
}
/// `/v1/responses` from its raw body: forward tool schemas the gateway does
/// not model (namespace, custom, tool_search, ...) without rejecting them.
async fn route_responses_raw(
&self,
headers: Option<&HeaderMap>,
body: &Value,
model_id: Option<&str>,
) -> Response {
let is_stream = body
.get("stream")
.and_then(Value::as_bool)
.unwrap_or(false);
let background = body
.get("background")
.and_then(Value::as_bool)
.unwrap_or(false);
if background && !is_stream {
warn!("PD mode does not support detached background responses; returning bad request");
return error::bad_request(
"pd_unsupported_background_responses",
"PD mode does not support background responses without streaming",
);
}
let request_text = if self.policies_need_request_text() {
let text = Self::responses_request_text(body);
(!text.is_empty()).then_some(text)
} else {
None
};
let context = PDRequestContext {
route: "/v1/responses",
batch_size: None,
is_stream,
return_logprob: false,
request_text,
model_id,
headers: headers.cloned(),
};
self.execute_dual_dispatch(headers, body, context).await
}
/// Plain text of a raw Responses body, mirroring the typed
/// `extract_text_for_routing` for cache-aware routing.
fn responses_request_text(body: &Value) -> String {
fn collect(value: &Value, out: &mut Vec<String>) {
match value {
Value::String(text) => {
if !text.is_empty() {
out.push(text.clone());
}
}
Value::Array(items) => {
for item in items {
collect(item, out);
}
}
Value::Object(map) => {
for key in ["text", "content", "arguments", "output"] {
if let Some(inner) = map.get(key) {
collect(inner, out);
}
}
}
_ => {}
}
}
let mut parts = Vec::new();
if let Some(input) = body.get("input") {
collect(input, &mut parts);
}
parts.join(" ")
}
async fn route_rerank(
&self,
headers: Option<&HeaderMap>,
+21
View File
@@ -126,6 +126,27 @@ pub trait RouterTrait: Send + Sync + Debug {
.into_response()
}
/// Route a responses request from its raw JSON body.
///
/// A gateway forwards tool schemas it does not model (namespace, custom,
/// tool_search, ...) unchanged; implementations that can forward the body
/// themselves should override this instead of rejecting unknown tool types.
async fn route_responses_raw(
&self,
headers: Option<&HeaderMap>,
body: &serde_json::Value,
model_id: Option<&str>,
) -> Response {
match serde_json::from_value::<ResponsesRequest>(body.clone()) {
Ok(parsed) => self.route_responses(headers, &parsed, model_id).await,
Err(err) => (
StatusCode::BAD_REQUEST,
format!("Invalid responses request: {err}"),
)
.into_response(),
}
}
/// Retrieve a stored/background response by id
async fn get_response(
&self,
@@ -614,6 +614,30 @@ impl RouterTrait for RouterManager {
}
}
async fn route_responses_raw(
&self,
headers: Option<&HeaderMap>,
body: &serde_json::Value,
model_id: Option<&str>,
) -> Response {
let selected_model = model_id
.map(str::to_string)
.or_else(|| body.get("model").and_then(|v| v.as_str()).map(str::to_string));
let router = self.select_router_for_request(headers, selected_model.as_deref());
if let Some(router) = router {
router
.route_responses_raw(headers, body, selected_model.as_deref())
.await
} else {
(
StatusCode::NOT_FOUND,
"No router available to handle responses request",
)
.into_response()
}
}
async fn get_response(
&self,
headers: Option<&HeaderMap>,
+16 -3
View File
@@ -47,7 +47,7 @@ use crate::{
generate::GenerateRequest,
parser::{ParseFunctionCallRequest, SeparateReasoningRequest},
rerank::V1RerankReqInput,
responses::{ResponsesGetParams, ResponsesRequest},
responses::ResponsesGetParams,
tokenize::{AddTokenizerRequest, DetokenizeRequest, TokenizeRequest},
validated::ValidatedJson,
worker_spec::{WorkerConfigRequest, WorkerUpdateRequest},
@@ -218,11 +218,24 @@ async fn v1_rerank(
async fn v1_responses(
State(state): State<Arc<AppState>>,
headers: http::HeaderMap,
ValidatedJson(body): ValidatedJson<ResponsesRequest>,
Json(body): Json<Value>,
) -> Response {
let model = match body.get("model").and_then(Value::as_str) {
Some(model) if !model.is_empty() => model.to_string(),
_ => {
return (
StatusCode::BAD_REQUEST,
"model is required for /v1/responses",
)
.into_response()
}
};
// Forward the raw body: the gateway must not reject tool types it does not
// model (namespace, custom, tool_search, ...); the engine owns the schema.
state
.router
.route_responses(Some(&headers), &body, Some(&body.model))
.route_responses_raw(Some(&headers), &body, Some(&model))
.await
}