gateway: forward /v1/responses raw bodies so unknown tool types pass through
This commit is contained in:
@@ -2027,6 +2027,84 @@ impl RouterTrait for PDRouter {
|
||||
self.execute_dual_dispatch(headers, body, context).await
|
||||
}
|
||||
|
||||
/// `/v1/responses` from its raw body: forward tool schemas the gateway does
|
||||
/// not model (namespace, custom, tool_search, ...) without rejecting them.
|
||||
async fn route_responses_raw(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
body: &Value,
|
||||
model_id: Option<&str>,
|
||||
) -> Response {
|
||||
let is_stream = body
|
||||
.get("stream")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
let background = body
|
||||
.get("background")
|
||||
.and_then(Value::as_bool)
|
||||
.unwrap_or(false);
|
||||
|
||||
if background && !is_stream {
|
||||
warn!("PD mode does not support detached background responses; returning bad request");
|
||||
return error::bad_request(
|
||||
"pd_unsupported_background_responses",
|
||||
"PD mode does not support background responses without streaming",
|
||||
);
|
||||
}
|
||||
|
||||
let request_text = if self.policies_need_request_text() {
|
||||
let text = Self::responses_request_text(body);
|
||||
(!text.is_empty()).then_some(text)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let context = PDRequestContext {
|
||||
route: "/v1/responses",
|
||||
batch_size: None,
|
||||
is_stream,
|
||||
return_logprob: false,
|
||||
request_text,
|
||||
model_id,
|
||||
headers: headers.cloned(),
|
||||
};
|
||||
|
||||
self.execute_dual_dispatch(headers, body, context).await
|
||||
}
|
||||
|
||||
/// Plain text of a raw Responses body, mirroring the typed
|
||||
/// `extract_text_for_routing` for cache-aware routing.
|
||||
fn responses_request_text(body: &Value) -> String {
|
||||
fn collect(value: &Value, out: &mut Vec<String>) {
|
||||
match value {
|
||||
Value::String(text) => {
|
||||
if !text.is_empty() {
|
||||
out.push(text.clone());
|
||||
}
|
||||
}
|
||||
Value::Array(items) => {
|
||||
for item in items {
|
||||
collect(item, out);
|
||||
}
|
||||
}
|
||||
Value::Object(map) => {
|
||||
for key in ["text", "content", "arguments", "output"] {
|
||||
if let Some(inner) = map.get(key) {
|
||||
collect(inner, out);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let mut parts = Vec::new();
|
||||
if let Some(input) = body.get("input") {
|
||||
collect(input, &mut parts);
|
||||
}
|
||||
parts.join(" ")
|
||||
}
|
||||
|
||||
async fn route_rerank(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
|
||||
@@ -126,6 +126,27 @@ pub trait RouterTrait: Send + Sync + Debug {
|
||||
.into_response()
|
||||
}
|
||||
|
||||
/// Route a responses request from its raw JSON body.
|
||||
///
|
||||
/// A gateway forwards tool schemas it does not model (namespace, custom,
|
||||
/// tool_search, ...) unchanged; implementations that can forward the body
|
||||
/// themselves should override this instead of rejecting unknown tool types.
|
||||
async fn route_responses_raw(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
body: &serde_json::Value,
|
||||
model_id: Option<&str>,
|
||||
) -> Response {
|
||||
match serde_json::from_value::<ResponsesRequest>(body.clone()) {
|
||||
Ok(parsed) => self.route_responses(headers, &parsed, model_id).await,
|
||||
Err(err) => (
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("Invalid responses request: {err}"),
|
||||
)
|
||||
.into_response(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Retrieve a stored/background response by id
|
||||
async fn get_response(
|
||||
&self,
|
||||
|
||||
@@ -614,6 +614,30 @@ impl RouterTrait for RouterManager {
|
||||
}
|
||||
}
|
||||
|
||||
async fn route_responses_raw(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
body: &serde_json::Value,
|
||||
model_id: Option<&str>,
|
||||
) -> Response {
|
||||
let selected_model = model_id
|
||||
.map(str::to_string)
|
||||
.or_else(|| body.get("model").and_then(|v| v.as_str()).map(str::to_string));
|
||||
let router = self.select_router_for_request(headers, selected_model.as_deref());
|
||||
|
||||
if let Some(router) = router {
|
||||
router
|
||||
.route_responses_raw(headers, body, selected_model.as_deref())
|
||||
.await
|
||||
} else {
|
||||
(
|
||||
StatusCode::NOT_FOUND,
|
||||
"No router available to handle responses request",
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_response(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
|
||||
@@ -47,7 +47,7 @@ use crate::{
|
||||
generate::GenerateRequest,
|
||||
parser::{ParseFunctionCallRequest, SeparateReasoningRequest},
|
||||
rerank::V1RerankReqInput,
|
||||
responses::{ResponsesGetParams, ResponsesRequest},
|
||||
responses::ResponsesGetParams,
|
||||
tokenize::{AddTokenizerRequest, DetokenizeRequest, TokenizeRequest},
|
||||
validated::ValidatedJson,
|
||||
worker_spec::{WorkerConfigRequest, WorkerUpdateRequest},
|
||||
@@ -218,11 +218,24 @@ async fn v1_rerank(
|
||||
async fn v1_responses(
|
||||
State(state): State<Arc<AppState>>,
|
||||
headers: http::HeaderMap,
|
||||
ValidatedJson(body): ValidatedJson<ResponsesRequest>,
|
||||
Json(body): Json<Value>,
|
||||
) -> Response {
|
||||
let model = match body.get("model").and_then(Value::as_str) {
|
||||
Some(model) if !model.is_empty() => model.to_string(),
|
||||
_ => {
|
||||
return (
|
||||
StatusCode::BAD_REQUEST,
|
||||
"model is required for /v1/responses",
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
};
|
||||
|
||||
// Forward the raw body: the gateway must not reject tool types it does not
|
||||
// model (namespace, custom, tool_search, ...); the engine owns the schema.
|
||||
state
|
||||
.router
|
||||
.route_responses(Some(&headers), &body, Some(&body.model))
|
||||
.route_responses_raw(Some(&headers), &body, Some(&model))
|
||||
.await
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user