diff --git a/sgl-model-gateway/src/server.rs b/sgl-model-gateway/src/server.rs index 85839d772d..e64dd8a9f8 100644 --- a/sgl-model-gateway/src/server.rs +++ b/sgl-model-gateway/src/server.rs @@ -184,14 +184,100 @@ async fn generate( async fn v1_chat_completions( State(state): State>, headers: http::HeaderMap, - ValidatedJson(body): ValidatedJson, + Json(mut body): Json, ) -> Response { + // Chat Completions has no namespace field; flatten a Responses-style + // namespace group to ``__`` function tools, matching + // the engine's convention, before the typed request is built. + flatten_chat_namespace_tools(&mut body); + let request: ChatCompletionRequest = match serde_json::from_value(body) { + Ok(request) => request, + Err(err) => { + return ( + StatusCode::BAD_REQUEST, + format!("Invalid chat completion request: {err}"), + ) + .into_response() + } + }; state .router - .route_chat(Some(&headers), &body, Some(&body.model)) + .route_chat(Some(&headers), &request, Some(&request.model)) .await } +fn flatten_chat_namespace_tools(body: &mut Value) { + let Some(tools) = body.get_mut("tools").and_then(Value::as_array_mut) else { + return; + }; + let mut flattened = Vec::with_capacity(tools.len()); + for tool in tools.drain(..) { + if tool.get("type").and_then(Value::as_str) != Some("namespace") { + flattened.push(tool); + continue; + } + let namespace = tool.get("name").and_then(Value::as_str).unwrap_or(""); + let namespace_description = tool + .get("description") + .and_then(Value::as_str) + .unwrap_or(""); + let Some(members) = tool.get("tools").and_then(Value::as_array) else { + continue; + }; + for member in members { + let mut function = if let Some(function) = member.get("function") { + function.clone() + } else if member.get("type").and_then(Value::as_str) == Some("function") { + let mut member = member.clone(); + if let Some(object) = member.as_object_mut() { + object.remove("type"); + } + member + } else { + continue; + }; + let Some(name) = function.get("name").and_then(Value::as_str) else { + continue; + }; + let flat = format!("{namespace}__{name}"); + if let Some(object) = function.as_object_mut() { + object.insert("name".into(), Value::String(flat)); + if !namespace_description.is_empty() { + let description = object + .get("description") + .and_then(Value::as_str) + .unwrap_or(""); + let merged = if description.is_empty() { + namespace_description.to_string() + } else { + format!( + "{namespace_description}{}{description}", + char::from(10u8) + ) + }; + object.insert("description".into(), Value::String(merged)); + } + } + flattened.push(json!({"type": "function", "function": function})); + } + } + body["tools"] = Value::Array(flattened); + + if let Some(choice) = body.get_mut("tool_choice") { + if let Some(object) = choice.as_object() { + if let (Some(namespace), Some(name)) = ( + object.get("namespace").and_then(Value::as_str), + object.get("name").and_then(Value::as_str), + ) { + *choice = json!({ + "type": "function", + "function": {"name": format!("{namespace}__{name}")} + }); + } + } + } +} + async fn v1_completions( State(state): State>, headers: http::HeaderMap,