diff --git a/sgl-model-gateway/Cargo.toml b/sgl-model-gateway/Cargo.toml index 12b06d9417..6498b108a9 100644 --- a/sgl-model-gateway/Cargo.toml +++ b/sgl-model-gateway/Cargo.toml @@ -205,3 +205,6 @@ codegen-units = 1 [profile.dev-opt] inherits = "dev" opt-level = 1 + +[patch.crates-io] +openai-protocol = { path = "vendor/openai-protocol" } diff --git a/sgl-model-gateway/src/routers/grpc/harmony/builder.rs b/sgl-model-gateway/src/routers/grpc/harmony/builder.rs index 62fb56fa95..4292d2ef22 100644 --- a/sgl-model-gateway/src/routers/grpc/harmony/builder.rs +++ b/sgl-model-gateway/src/routers/grpc/harmony/builder.rs @@ -301,11 +301,12 @@ impl HarmonyBuilder { .reasoning .as_ref() .and_then(|r| r.effort.as_ref()) - .map(|effort| match effort { - ResponsesReasoningEffort::High => ReasoningEffort::High, - ResponsesReasoningEffort::Medium => ReasoningEffort::Medium, - ResponsesReasoningEffort::Low => ReasoningEffort::Low, - ResponsesReasoningEffort::Minimal => ReasoningEffort::Low, + .and_then(|effort| match effort { + ResponsesReasoningEffort::None => None, + ResponsesReasoningEffort::High => Some(ReasoningEffort::High), + ResponsesReasoningEffort::Medium => Some(ReasoningEffort::Medium), + ResponsesReasoningEffort::Low => Some(ReasoningEffort::Low), + ResponsesReasoningEffort::Minimal => Some(ReasoningEffort::Low), }); self.build_system_message(reasoning_effort, with_custom_tools) diff --git a/sgl-model-gateway/vendor/openai-protocol/.cargo-ok b/sgl-model-gateway/vendor/openai-protocol/.cargo-ok new file mode 100644 index 0000000000..5f8b795830 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/sgl-model-gateway/vendor/openai-protocol/.cargo_vcs_info.json b/sgl-model-gateway/vendor/openai-protocol/.cargo_vcs_info.json new file mode 100644 index 0000000000..9130a68fa9 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "7455eaee1da5212ae5e0fd33d80a903029d2f981" + }, + "path_in_vcs": "protocols" +} \ No newline at end of file diff --git a/sgl-model-gateway/vendor/openai-protocol/Cargo.toml b/sgl-model-gateway/vendor/openai-protocol/Cargo.toml new file mode 100644 index 0000000000..bb97a44d9e --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/Cargo.toml @@ -0,0 +1,77 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "openai-protocol" +version = "1.0.0" +authors = [ + "Simo Lin ", + "Chang Su ", + "Keyang Ru ", +] +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "OpenAI-compatible API protocol definitions and types" +readme = false +keywords = [ + "openai", + "llm", + "api", + "protocol", + "types", +] +categories = [ + "api-bindings", + "data-structures", +] +license = "Apache-2.0" +repository = "https://github.com/lightseekorg/smg" + +[features] +axum = ["dep:axum"] +default = [] + +[lib] +name = "openai_protocol" +path = "src/lib.rs" + +[dependencies.axum] +version = "0.8.6" +optional = true + +[dependencies.chrono] +version = "0.4" + +[dependencies.rand] +version = "0.9.2" + +[dependencies.serde] +version = "1.0" +features = ["derive"] + +[dependencies.serde_json] +version = "1.0" +features = ["preserve_order"] + +[dependencies.tokio] +version = "1.42.0" + +[dependencies.tracing] +version = "0.1" + +[dependencies.validator] +version = "0.20.0" +features = ["derive"] diff --git a/sgl-model-gateway/vendor/openai-protocol/Cargo.toml.orig b/sgl-model-gateway/vendor/openai-protocol/Cargo.toml.orig new file mode 100644 index 0000000000..590898e0f9 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/Cargo.toml.orig @@ -0,0 +1,29 @@ +[package] +name = "openai-protocol" +version = "1.0.0" +edition = "2021" +description = "OpenAI-compatible API protocol definitions and types" +license = "Apache-2.0" +repository = "https://github.com/lightseekorg/smg" +authors = [ + "Simo Lin ", + "Chang Su ", + "Keyang Ru ", +] +keywords = ["openai", "llm", "api", "protocol", "types"] +categories = ["api-bindings", "data-structures"] + +[features] +default = [] +axum = ["dep:axum"] + +[dependencies] +axum = { workspace = true, optional = true } +chrono.workspace = true +rand.workspace = true +serde = { workspace = true, features = ["derive"] } +serde_json = { workspace = true, features = ["preserve_order"] } +tokio.workspace = true +tracing.workspace = true +# Only used by protocols +validator = { version = "0.20.0", features = ["derive"] } diff --git a/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/mod.rs b/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/mod.rs new file mode 100644 index 0000000000..61386b505c --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/mod.rs @@ -0,0 +1,7 @@ +//! Builders for Chat Completion API response types + +pub mod response; +pub mod stream_response; + +pub use response::ChatCompletionResponseBuilder; +pub use stream_response::ChatCompletionStreamResponseBuilder; diff --git a/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/response.rs b/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/response.rs new file mode 100644 index 0000000000..3362078c51 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/response.rs @@ -0,0 +1,218 @@ +//! Builder for ChatCompletionResponse +//! +//! Provides an ergonomic fluent API for constructing chat completion responses. + +use crate::{chat::*, common::Usage}; + +/// Builder for ChatCompletionResponse +/// +/// Provides a fluent interface for constructing chat completion responses with sensible defaults. +#[must_use = "Builder does nothing until .build() is called"] +#[derive(Clone, Debug)] +pub struct ChatCompletionResponseBuilder { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, + usage: Option, + system_fingerprint: Option, +} + +impl ChatCompletionResponseBuilder { + /// Create a new builder with required fields + /// + /// # Arguments + /// - `id`: Completion ID (e.g., "chatcmpl_abc123") + /// - `model`: Model name used for generation + pub fn new(id: impl Into, model: impl Into) -> Self { + Self { + id: id.into(), + object: "chat.completion".to_string(), + created: chrono::Utc::now().timestamp() as u64, + model: model.into(), + choices: Vec::new(), + usage: None, + system_fingerprint: None, + } + } + + /// Copy common fields from a ChatCompletionRequest + /// + /// This populates the model field from the request. + pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self { + self.model = request.model.clone(); + self + } + + /// Set the object type (default: "chat.completion") + pub fn object(mut self, object: impl Into) -> Self { + self.object = object.into(); + self + } + + /// Set the creation timestamp (default: current time) + pub fn created(mut self, timestamp: u64) -> Self { + self.created = timestamp; + self + } + + /// Set the choices + pub fn choices(mut self, choices: Vec) -> Self { + self.choices = choices; + self + } + + /// Add a single choice + pub fn add_choice(mut self, choice: ChatChoice) -> Self { + self.choices.push(choice); + self + } + + /// Set usage information + pub fn usage(mut self, usage: Usage) -> Self { + self.usage = Some(usage); + self + } + + /// Set usage if provided (handles Option) + pub fn maybe_usage(mut self, usage: Option) -> Self { + if let Some(u) = usage { + self.usage = Some(u); + } + self + } + + /// Set system fingerprint if provided (handles Option) + pub fn maybe_system_fingerprint(mut self, fingerprint: Option>) -> Self { + if let Some(fp) = fingerprint { + self.system_fingerprint = Some(fp.into()); + } + self + } + + /// Build the ChatCompletionResponse + pub fn build(self) -> ChatCompletionResponse { + ChatCompletionResponse { + id: self.id, + object: self.object, + created: self.created, + model: self.model, + choices: self.choices, + usage: self.usage, + system_fingerprint: self.system_fingerprint, + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_minimal() { + let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build(); + + assert_eq!(response.id, "chatcmpl_123"); + assert_eq!(response.model, "gpt-4"); + assert_eq!(response.object, "chat.completion"); + assert!(response.choices.is_empty()); + assert!(response.usage.is_none()); + assert!(response.system_fingerprint.is_none()); + } + + #[test] + fn test_build_complete() { + let choice = ChatChoice { + index: 0, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: Some("Hello!".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some("stop".to_string()), + matched_stop: None, + hidden_states: None, + }; + + let usage = Usage { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + completion_tokens_details: None, + }; + + let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4") + .choices(vec![choice.clone()]) + .maybe_usage(Some(usage)) + .maybe_system_fingerprint(Some("fp_123abc")) + .build(); + + assert_eq!(response.id, "chatcmpl_456"); + assert_eq!(response.choices.len(), 1); + assert_eq!(response.choices[0].index, 0); + assert!(response.usage.is_some()); + assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc"); + } + + #[test] + fn test_add_multiple_choices() { + let choice1 = ChatChoice { + index: 0, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: Some("Option 1".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some("stop".to_string()), + matched_stop: None, + hidden_states: None, + }; + + let choice2 = ChatChoice { + index: 1, + message: ChatCompletionMessage { + role: "assistant".to_string(), + content: Some("Option 2".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some("stop".to_string()), + matched_stop: None, + hidden_states: None, + }; + + let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4") + .add_choice(choice1) + .add_choice(choice2) + .build(); + + assert_eq!(response.choices.len(), 2); + assert_eq!(response.choices[0].index, 0); + assert_eq!(response.choices[1].index, 1); + } + + #[test] + fn test_copy_from_request() { + let request = ChatCompletionRequest { + messages: vec![], + model: "gpt-3.5-turbo".to_string(), + ..Default::default() + }; + + let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4") + .copy_from_request(&request) + .build(); + + assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/stream_response.rs b/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/stream_response.rs new file mode 100644 index 0000000000..85a67b1e1a --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/builders/chat/stream_response.rs @@ -0,0 +1,421 @@ +//! Builder for ChatCompletionStreamResponse +//! +//! Provides an ergonomic fluent API for constructing streaming chat completion responses. + +use std::borrow::Cow; + +use crate::{ + chat::*, + common::{FunctionCallDelta, ToolCallDelta, Usage}, +}; + +/// Builder for ChatCompletionStreamResponse +/// +/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults. +#[must_use = "Builder does nothing until .build() is called"] +#[derive(Clone, Debug)] +pub struct ChatCompletionStreamResponseBuilder { + id: String, + object: String, + created: u64, + model: String, + choices: Vec, + usage: Option, + system_fingerprint: Option, +} + +impl ChatCompletionStreamResponseBuilder { + /// Create a new builder with required fields + /// + /// # Arguments + /// - `id`: Completion ID (e.g., "chatcmpl_abc123") + /// - `model`: Model name used for generation + pub fn new(id: impl Into, model: impl Into) -> Self { + Self { + id: id.into(), + object: "chat.completion.chunk".to_string(), + created: chrono::Utc::now().timestamp() as u64, + model: model.into(), + choices: Vec::new(), + usage: None, + system_fingerprint: None, + } + } + + /// Copy common fields from a ChatCompletionRequest + /// + /// This populates the model field from the request. + pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self { + self.model = request.model.clone(); + self + } + + /// Set the object type (default: "chat.completion.chunk") + pub fn object(mut self, object: impl Into) -> Self { + self.object = object.into(); + self + } + + /// Set the creation timestamp (default: current time) + pub fn created(mut self, timestamp: u64) -> Self { + self.created = timestamp; + self + } + + /// Set the choices + pub fn choices(mut self, choices: Vec) -> Self { + self.choices = choices; + self + } + + /// Add a single choice (delta) + pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self { + self.choices.push(choice); + self + } + + /// Set usage information (typically sent in final chunk) + pub fn usage(mut self, usage: Usage) -> Self { + self.usage = Some(usage); + self + } + + /// Set system fingerprint if provided (handles Option) + pub fn maybe_system_fingerprint(mut self, fingerprint: Option>) -> Self { + if let Some(fp) = fingerprint { + self.system_fingerprint = Some(fp.into()); + } + self + } + + /// Set usage if provided (handles Option) + pub fn maybe_usage(mut self, usage: Option) -> Self { + if let Some(u) = usage { + self.usage = Some(u); + } + self + } + + /// Add a choice delta that sets `role` and `content` + pub fn add_choice_content( + mut self, + index: u32, + role: impl Into, + content: impl Into, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some(role.into()), + content: Some(content.into()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that sets `role`, `content`, and `logprobs` + pub fn add_choice_content_with_logprobs( + mut self, + index: u32, + role: impl Into, + content: impl Into, + logprobs: Option, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some(role.into()), + content: Some(content.into()), + tool_calls: None, + reasoning_content: None, + }, + logprobs, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that only sets `role` + pub fn add_choice_role(mut self, index: u32, role: impl Into) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some(role.into()), + content: None, + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that appends a tool-call *arguments delta* + /// Uses `Cow` so you can pass `&str` or `String` without extra clones + pub fn add_choice_tool_args( + mut self, + index: u32, + args_delta: impl Into>, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(vec![ToolCallDelta { + index: 0, + id: None, + tool_type: None, + function: Some(FunctionCallDelta { + name: None, + arguments: Some(args_delta.into().into_owned()), + }), + }]), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta that sets reasoning content (for models that stream reasoning) + pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: None, + reasoning_content: Some(reasoning.into()), + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta for tool call with function name and ID + pub fn add_choice_tool_name( + mut self, + index: u32, + tool_call_id: impl Into, + function_name: impl Into, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(vec![ToolCallDelta { + index: 0, + id: Some(tool_call_id.into()), + tool_type: Some("function".to_string()), + function: Some(FunctionCallDelta { + name: Some(function_name.into()), + arguments: None, + }), + }]), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice delta with a pre-constructed ToolCallDelta + /// Useful when you already have a ToolCallDelta object to emit + pub fn add_choice_tool_call_delta( + mut self, + index: u32, + tool_call_delta: ToolCallDelta, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: None, + tool_calls: Some(vec![tool_call_delta]), + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }); + self + } + + /// Add a choice with finish_reason (final chunk) + /// This is used for the last chunk in a stream to signal completion + pub fn add_choice_finish_reason( + mut self, + index: u32, + finish_reason: impl Into, + matched_stop: Option, + ) -> Self { + self.choices.push(ChatStreamChoice { + index, + delta: ChatMessageDelta { + role: None, + content: None, + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: Some(finish_reason.into()), + matched_stop, + }); + self + } + + /// Build the ChatCompletionStreamResponse + pub fn build(self) -> ChatCompletionStreamResponse { + ChatCompletionStreamResponse { + id: self.id, + object: self.object, + created: self.created, + model: self.model, + system_fingerprint: self.system_fingerprint, + choices: self.choices, + usage: self.usage, + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_minimal() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build(); + + assert_eq!(chunk.id, "chatcmpl_123"); + assert_eq!(chunk.model, "gpt-4"); + assert_eq!(chunk.object, "chat.completion.chunk"); + assert!(chunk.choices.is_empty()); + assert!(chunk.usage.is_none()); + } + + #[test] + fn test_with_content_delta() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4") + .add_choice_content(0, "assistant", "Hello") + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].index, 0); + assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello"); + assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); + assert!(chunk.choices[0].finish_reason.is_none()); + } + + #[test] + fn test_with_role_delta() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4") + .add_choice_role(0, "assistant") + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); + assert!(chunk.choices[0].delta.content.is_none()); + } + + #[test] + fn test_with_finish_reason() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4") + .add_choice_finish_reason(0, "stop", None) + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop"); + assert!(chunk.choices[0].delta.content.is_none()); + assert!(chunk.choices[0].delta.role.is_none()); + } + + #[test] + fn test_multiple_deltas() { + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4") + .add_choice_role(0, "assistant") + .add_choice_content(0, "assistant", "Hello") + .add_choice_content(0, "assistant", " world") + .add_choice_finish_reason(0, "stop", None) + .build(); + + assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish + } + + #[test] + fn test_with_usage() { + let usage = Usage { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + completion_tokens_details: None, + }; + + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4") + .add_choice_finish_reason(0, "stop", None) + .usage(usage) + .build(); + + assert!(chunk.usage.is_some()); + assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30); + } + + #[test] + fn test_copy_from_request() { + let request = ChatCompletionRequest { + messages: vec![], + model: "gpt-3.5-turbo".to_string(), + ..Default::default() + }; + + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4") + .copy_from_request(&request) + .add_choice_content(0, "assistant", "test") + .build(); + + assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request + } + + #[test] + fn test_add_choice_explicit() { + let choice = ChatStreamChoice { + index: 0, + delta: ChatMessageDelta { + role: Some("assistant".to_string()), + content: Some("Hello".to_string()), + tool_calls: None, + reasoning_content: None, + }, + logprobs: None, + finish_reason: None, + matched_stop: None, + }; + + let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4") + .add_choice(choice) + .build(); + + assert_eq!(chunk.choices.len(), 1); + assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant"); + assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello"); + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/builders/mod.rs b/sgl-model-gateway/vendor/openai-protocol/src/builders/mod.rs new file mode 100644 index 0000000000..9208441245 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/builders/mod.rs @@ -0,0 +1,27 @@ +//! Builder patterns for protocol response types +//! +//! This module provides ergonomic builders for response types with many optional fields. +//! Builders help avoid telescoping constructors and make construction intent clear. +//! +//! # Organization +//! +//! Builders are organized by API: +//! - `chat/` - Chat Completion API builders (response, stream_response) +//! - `responses/` - Responses API builder (response) +//! +//! # Optional Fields +//! +//! For optional fields, builders provide `maybe_*` methods that handle `Option` directly: +//! ```ignore +//! builder +//! .field(value) +//! .maybe_optional_field(optional_value) // Accepts Option +//! .build() +//! ``` + +pub mod chat; +pub mod responses; + +// Re-export all builders for convenient access +pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder}; +pub use responses::ResponsesResponseBuilder; diff --git a/sgl-model-gateway/vendor/openai-protocol/src/builders/responses/mod.rs b/sgl-model-gateway/vendor/openai-protocol/src/builders/responses/mod.rs new file mode 100644 index 0000000000..aca9d24923 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/builders/responses/mod.rs @@ -0,0 +1,5 @@ +//! Builders for Responses API response types + +pub mod response; + +pub use response::ResponsesResponseBuilder; diff --git a/sgl-model-gateway/vendor/openai-protocol/src/builders/responses/response.rs b/sgl-model-gateway/vendor/openai-protocol/src/builders/responses/response.rs new file mode 100644 index 0000000000..7db46c3f45 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/builders/responses/response.rs @@ -0,0 +1,411 @@ +//! Builder for ResponsesResponse +//! +//! Provides an ergonomic fluent API for constructing ResponsesResponse instances. + +use std::collections::HashMap; + +use serde_json::Value; + +use crate::responses::*; + +/// Builder for ResponsesResponse +/// +/// Provides a fluent interface for constructing responses with sensible defaults. +#[must_use = "Builder does nothing until .build() is called"] +#[derive(Clone, Debug)] +pub struct ResponsesResponseBuilder { + id: String, + object: String, + created_at: i64, + status: ResponseStatus, + error: Option, + incomplete_details: Option, + instructions: Option, + max_output_tokens: Option, + model: String, + output: Vec, + parallel_tool_calls: bool, + previous_response_id: Option, + reasoning: Option, + store: bool, + temperature: Option, + text: Option, + tool_choice: String, + tools: Vec, + top_p: Option, + truncation: Option, + usage: Option, + user: Option, + safety_identifier: Option, + metadata: HashMap, +} + +impl ResponsesResponseBuilder { + /// Create a new builder with required fields + /// + /// # Arguments + /// - `id`: Response ID (e.g., "resp_abc123") + /// - `model`: Model name used for generation + pub fn new(id: impl Into, model: impl Into) -> Self { + Self { + id: id.into(), + object: "response".to_string(), + created_at: chrono::Utc::now().timestamp(), + status: ResponseStatus::InProgress, + error: None, + incomplete_details: None, + instructions: None, + max_output_tokens: None, + model: model.into(), + output: Vec::new(), + parallel_tool_calls: true, + previous_response_id: None, + reasoning: None, + store: true, + temperature: None, + text: None, + tool_choice: "auto".to_string(), + tools: Vec::new(), + top_p: None, + truncation: None, + usage: None, + user: None, + safety_identifier: None, + metadata: HashMap::new(), + } + } + + /// Copy common fields from a ResponsesRequest + /// + /// This populates fields like instructions, max_output_tokens, temperature, etc. + /// from the original request, making it easy to construct a response that mirrors + /// the request parameters. + /// + /// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation + /// and should be set independently from the request's `user` field (which is for billing/tracking). + pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self { + self.instructions = request.instructions.clone(); + self.max_output_tokens = request.max_output_tokens; + self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true); + self.previous_response_id = request.previous_response_id.clone(); + self.store = request.store.unwrap_or(true); + self.temperature = request.temperature; + self.tool_choice = if let Some(ref tc) = request.tool_choice { + serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()) + } else { + "auto".to_string() + }; + self.tools = request.tools.clone().unwrap_or_default(); + self.top_p = request.top_p; + self.user = request.user.clone(); + self.metadata = request.metadata.clone().unwrap_or_default(); + self + } + + /// Set the object type (default: "response") + pub fn object(mut self, object: impl Into) -> Self { + self.object = object.into(); + self + } + + /// Set the creation timestamp (default: current time) + pub fn created_at(mut self, timestamp: i64) -> Self { + self.created_at = timestamp; + self + } + + /// Set the response status + pub fn status(mut self, status: ResponseStatus) -> Self { + self.status = status; + self + } + + /// Set error information (if status is failed) + pub fn error(mut self, error: Value) -> Self { + self.error = Some(error); + self + } + + /// Set incomplete details (if response was truncated) + pub fn incomplete_details(mut self, details: Value) -> Self { + self.incomplete_details = Some(details); + self + } + + /// Set system instructions + pub fn instructions(mut self, instructions: impl Into) -> Self { + self.instructions = Some(instructions.into()); + self + } + + /// Set max output tokens + pub fn max_output_tokens(mut self, tokens: u32) -> Self { + self.max_output_tokens = Some(tokens); + self + } + + /// Set output items + pub fn output(mut self, output: Vec) -> Self { + self.output = output; + self + } + + /// Add a single output item + pub fn add_output(mut self, item: ResponseOutputItem) -> Self { + self.output.push(item); + self + } + + /// Set whether parallel tool calls are enabled + pub fn parallel_tool_calls(mut self, enabled: bool) -> Self { + self.parallel_tool_calls = enabled; + self + } + + /// Set previous response ID (if continuation) + pub fn previous_response_id(mut self, id: impl Into) -> Self { + self.previous_response_id = Some(id.into()); + self + } + + /// Set reasoning information + pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self { + self.reasoning = Some(reasoning); + self + } + + /// Set whether the response is stored + pub fn store(mut self, store: bool) -> Self { + self.store = store; + self + } + + /// Set temperature setting + pub fn temperature(mut self, temperature: f32) -> Self { + self.temperature = Some(temperature); + self + } + + /// Set text format settings if provided (handles Option) + pub fn maybe_text(mut self, text: Option) -> Self { + if let Some(t) = text { + self.text = Some(t); + } + self + } + + /// Set tool choice setting + pub fn tool_choice(mut self, tool_choice: impl Into) -> Self { + self.tool_choice = tool_choice.into(); + self + } + + /// Set available tools + pub fn tools(mut self, tools: Vec) -> Self { + self.tools = tools; + self + } + + /// Set top-p setting + pub fn top_p(mut self, top_p: f32) -> Self { + self.top_p = Some(top_p); + self + } + + /// Set truncation strategy + pub fn truncation(mut self, truncation: impl Into) -> Self { + self.truncation = Some(truncation.into()); + self + } + + /// Set usage information + pub fn usage(mut self, usage: ResponsesUsage) -> Self { + self.usage = Some(usage); + self + } + + /// Set usage if provided (handles Option) + pub fn maybe_usage(mut self, usage: Option) -> Self { + if let Some(u) = usage { + self.usage = Some(u); + } + self + } + + /// Copy from request if provided (handles Option) + pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self { + if let Some(req) = request { + self = self.copy_from_request(req); + } + self + } + + /// Set user identifier + pub fn user(mut self, user: impl Into) -> Self { + self.user = Some(user.into()); + self + } + + /// Set safety identifier + pub fn safety_identifier(mut self, identifier: impl Into) -> Self { + self.safety_identifier = Some(identifier.into()); + self + } + + /// Set metadata + pub fn metadata(mut self, metadata: HashMap) -> Self { + self.metadata = metadata; + self + } + + /// Add a single metadata entry + pub fn add_metadata(mut self, key: impl Into, value: Value) -> Self { + self.metadata.insert(key.into(), value); + self + } + + /// Build the ResponsesResponse + pub fn build(self) -> ResponsesResponse { + ResponsesResponse { + id: self.id, + object: self.object, + created_at: self.created_at, + status: self.status, + error: self.error, + incomplete_details: self.incomplete_details, + instructions: self.instructions, + max_output_tokens: self.max_output_tokens, + model: self.model, + output: self.output, + parallel_tool_calls: self.parallel_tool_calls, + previous_response_id: self.previous_response_id, + reasoning: self.reasoning, + store: self.store, + temperature: self.temperature, + text: self.text, + tool_choice: self.tool_choice, + tools: self.tools, + top_p: self.top_p, + truncation: self.truncation, + usage: self.usage, + user: self.user, + safety_identifier: self.safety_identifier, + metadata: self.metadata, + } + } +} + +// ============================================================================ +// Tests +// ============================================================================ + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_minimal() { + let response = ResponsesResponse::builder("resp_123", "gpt-4").build(); + + assert_eq!(response.id, "resp_123"); + assert_eq!(response.model, "gpt-4"); + assert_eq!(response.object, "response"); + assert_eq!(response.status, ResponseStatus::InProgress); + assert!(response.output.is_empty()); + assert!(response.parallel_tool_calls); + assert!(response.store); + } + + #[test] + fn test_build_complete() { + let response = ResponsesResponse::builder("resp_123", "gpt-4") + .status(ResponseStatus::Completed) + .instructions("You are a helpful assistant") + .max_output_tokens(1000) + .temperature(0.7) + .top_p(0.9) + .parallel_tool_calls(false) + .store(false) + .build(); + + assert_eq!(response.status, ResponseStatus::Completed); + assert_eq!( + response.instructions.as_ref().unwrap(), + "You are a helpful assistant" + ); + assert_eq!(response.max_output_tokens, Some(1000)); + assert_eq!(response.temperature, Some(0.7)); + assert_eq!(response.top_p, Some(0.9)); + assert!(!response.parallel_tool_calls); + assert!(!response.store); + } + + #[test] + fn test_copy_from_request() { + let request = ResponsesRequest { + model: "gpt-4".to_string(), + input: ResponseInput::Text("test".to_string()), + instructions: Some("Be helpful".to_string()), + max_output_tokens: Some(500), + temperature: Some(0.8), + top_p: Some(0.95), + parallel_tool_calls: Some(false), + store: Some(false), + user: Some("user_123".to_string()), + metadata: Some(HashMap::from([( + "key".to_string(), + serde_json::json!("value"), + )])), + ..Default::default() + }; + + let response = ResponsesResponse::builder("resp_456", "gpt-4") + .copy_from_request(&request) + .status(ResponseStatus::Completed) + .build(); + + assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful"); + assert_eq!(response.max_output_tokens, Some(500)); + assert_eq!(response.temperature, Some(0.8)); + assert_eq!(response.top_p, Some(0.95)); + assert!(!response.parallel_tool_calls); + assert!(!response.store); + assert_eq!(response.user.as_ref().unwrap(), "user_123"); + assert_eq!( + response.metadata.get("key").unwrap(), + &serde_json::json!("value") + ); + } + + #[test] + fn test_add_output_items() { + let response = ResponsesResponse::builder("resp_789", "gpt-4") + .add_output(ResponseOutputItem::Message { + id: "msg_1".to_string(), + role: "assistant".to_string(), + content: vec![], + status: "completed".to_string(), + }) + .add_output(ResponseOutputItem::Message { + id: "msg_2".to_string(), + role: "assistant".to_string(), + content: vec![], + status: "completed".to_string(), + }) + .build(); + + assert_eq!(response.output.len(), 2); + } + + #[test] + fn test_add_metadata() { + let response = ResponsesResponse::builder("resp_101", "gpt-4") + .add_metadata("key1", serde_json::json!("value1")) + .add_metadata("key2", serde_json::json!(42)) + .build(); + + assert_eq!(response.metadata.len(), 2); + assert_eq!(response.metadata.get("key1").unwrap(), "value1"); + assert_eq!(response.metadata.get("key2").unwrap(), 42); + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/chat.rs b/sgl-model-gateway/vendor/openai-protocol/src/chat.rs new file mode 100644 index 0000000000..71150180ce --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/chat.rs @@ -0,0 +1,799 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::{ + common::{ + default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function, + FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions, + StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, + Usage, + }, + sampling_params::{validate_top_k_value, validate_top_p_value}, +}; +use crate::{ + builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder}, + validated::Normalizable, +}; + +// ============================================================================ +// Chat Messages +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "role")] +pub enum ChatMessage { + #[serde(rename = "system")] + System { + content: MessageContent, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + #[serde(rename = "user")] + User { + content: MessageContent, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + #[serde(rename = "assistant")] + Assistant { + #[serde(skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + tool_calls: Option>, + /// Reasoning content for O1-style models (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + reasoning_content: Option, + }, + #[serde(rename = "tool")] + Tool { + content: MessageContent, + tool_call_id: String, + }, + #[serde(rename = "function")] + Function { content: String, name: String }, + #[serde(rename = "developer")] + Developer { + content: MessageContent, + #[serde(skip_serializing_if = "Option::is_none")] + tools: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +pub enum MessageContent { + Text(String), + Parts(Vec), +} + +impl MessageContent { + /// Returns the text content, cloning only when necessary. + /// For simple text, returns a clone of the string. + /// For parts, concatenates text parts with spaces. + /// Optimized to avoid intermediate Vec allocation. + pub fn to_simple_string(&self) -> String { + match self { + MessageContent::Text(text) => text.clone(), + MessageContent::Parts(parts) => { + // Use fold to build string directly without intermediate Vec allocation + let mut result = String::new(); + let mut first = true; + for part in parts { + if let ContentPart::Text { text } = part { + if !first { + result.push(' '); + } + result.push_str(text); + first = false; + } + } + result + } + } + } + + /// Appends text content directly to a buffer, avoiding intermediate allocations. + /// Returns true if any content was appended. + #[inline] + pub fn append_text_to(&self, buffer: &mut String) -> bool { + match self { + MessageContent::Text(text) => { + if !text.is_empty() { + buffer.push_str(text); + true + } else { + false + } + } + MessageContent::Parts(parts) => { + let mut appended = false; + for part in parts { + if let ContentPart::Text { text } = part { + if !text.is_empty() { + if appended { + buffer.push(' '); + } + buffer.push_str(text); + appended = true; + } + } + } + appended + } + } + } + + /// Returns true if this content contains any non-empty text. + #[inline] + pub fn has_text(&self) -> bool { + match self { + MessageContent::Text(text) => !text.is_empty(), + MessageContent::Parts(parts) => parts + .iter() + .any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())), + } + } +} + +// ============================================================================ +// Chat Completion Request +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)] +#[validate(schema(function = "validate_chat_cross_parameters"))] +pub struct ChatCompletionRequest { + /// A list of messages comprising the conversation so far + #[validate(custom(function = "validate_messages"))] + pub messages: Vec, + + /// ID of the model to use + #[serde(default = "default_model")] + pub model: String, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, + + /// Deprecated: Replaced by tool_choice + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "Use tool_choice instead")] + pub function_call: Option, + + /// Deprecated: Replaced by tools + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "Use tools instead")] + pub functions: Option>, + + /// Modify the likelihood of specified tokens appearing in the completion + #[serde(skip_serializing_if = "Option::is_none")] + pub logit_bias: Option>, + + /// Whether to return log probabilities of the output tokens + #[serde(default)] + pub logprobs: bool, + + /// Deprecated: Replaced by max_completion_tokens + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "Use max_completion_tokens instead")] + #[validate(range(min = 1))] + pub max_tokens: Option, + + /// An upper bound for the number of tokens that can be generated for a completion + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub max_completion_tokens: Option, + + /// Developer-defined tags and values used for filtering completions in the dashboard + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + + /// Output types that you would like the model to generate for this request + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, + + /// How many chat completion choices to generate for each input message + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1, max = 10))] + pub n: Option, + + /// Whether to enable parallel function calling during tool use + #[serde(skip_serializing_if = "Option::is_none")] + pub parallel_tool_calls: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + + /// Cache key for prompts (beta feature) + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_cache_key: Option, + + /// Effort level for reasoning models (low, medium, high) + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + + /// An object specifying the format that the model must output + #[serde(skip_serializing_if = "Option::is_none")] + pub response_format: Option, + + /// Safety identifier for content moderation + #[serde(skip_serializing_if = "Option::is_none")] + pub safety_identifier: Option, + + /// Deprecated: This feature is in Legacy mode + #[serde(skip_serializing_if = "Option::is_none")] + #[deprecated(note = "This feature is in Legacy mode")] + pub seed: Option, + + /// The service tier to use for this request + #[serde(skip_serializing_if = "Option::is_none")] + pub service_tier: Option, + + /// Up to 4 sequences where the API will stop generating further tokens + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_stop"))] + pub stop: Option, + + /// If set, partial message deltas will be sent + #[serde(default)] + pub stream: bool, + + /// Options for streaming response + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + /// What sampling temperature to use, between 0 and 2 + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 2.0))] + pub temperature: Option, + + /// Controls which (if any) tool is called by the model + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + /// A list of tools the model may call + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + + /// An integer between 0 and 20 specifying the number of most likely tokens to return + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0, max = 20))] + pub top_logprobs: Option, + + /// An alternative to sampling with temperature + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, + + /// Verbosity level for debugging + #[serde(skip_serializing_if = "Option::is_none")] + pub verbosity: Option, + + // ============================================================================= + // Engine-Specific Sampling Parameters + // ============================================================================= + // These parameters are extensions beyond the OpenAI API specification and + // control model generation behavior in engine-specific ways. + // ============================================================================= + /// Top-k sampling parameter (-1 to disable) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_k_value"))] + pub top_k: Option, + + /// Min-p nucleus sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 1.0))] + pub min_p: Option, + + /// Minimum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub min_tokens: Option, + + /// Repetition penalty for reducing repetitive text + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 2.0))] + pub repetition_penalty: Option, + + /// Regex constraint for output generation + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + + /// EBNF grammar constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + + /// Specific token IDs to use as stop conditions + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + + /// Skip trimming stop tokens from output + #[serde(default)] + pub no_stop_trim: bool, + + /// Ignore end-of-sequence tokens during generation + #[serde(default)] + pub ignore_eos: bool, + + /// Continue generating from final assistant message + #[serde(default)] + pub continue_final_message: bool, + + /// Skip special tokens during detokenization + #[serde(default = "default_true")] + pub skip_special_tokens: bool, + + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Separate reasoning content from final answer (O1-style models) + #[serde(default = "default_true")] + pub separate_reasoning: bool, + + /// Stream reasoning tokens during generation + #[serde(default = "default_true")] + pub stream_reasoning: bool, + + /// Chat template kwargs + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template_kwargs: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// Random seed for sampling for deterministic outputs + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_seed: Option, +} + +// ============================================================================ +// Validation Functions +// ============================================================================ + +/// Validates messages array is not empty and has valid content +fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> { + if messages.is_empty() { + return Err(validator::ValidationError::new("messages cannot be empty")); + } + + for msg in messages.iter() { + if let ChatMessage::User { content, .. } = msg { + match content { + MessageContent::Text(text) if text.is_empty() => { + return Err(validator::ValidationError::new( + "message content cannot be empty", + )); + } + MessageContent::Parts(parts) if parts.is_empty() => { + return Err(validator::ValidationError::new( + "message content parts cannot be empty", + )); + } + _ => {} + } + } + } + Ok(()) +} + +/// Schema-level validation for cross-field dependencies +fn validate_chat_cross_parameters( + req: &ChatCompletionRequest, +) -> Result<(), validator::ValidationError> { + // 1. Validate logprobs dependency + if req.top_logprobs.is_some() && !req.logprobs { + let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs"); + e.message = Some("top_logprobs is only allowed when logprobs is enabled".into()); + return Err(e); + } + + // 2. Validate stream_options dependency + if req.stream_options.is_some() && !req.stream { + let mut e = validator::ValidationError::new("stream_options_requires_stream"); + e.message = + Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into()); + return Err(e); + } + + // 3. Validate token limits - min <= max + if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) { + if min > max { + let mut e = validator::ValidationError::new("min_tokens_exceeds_max"); + e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into()); + return Err(e); + } + } + + // 4. Validate structured output conflicts + let has_json_format = matches!( + req.response_format, + Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. }) + ); + + if has_json_format && req.regex.is_some() { + let mut e = validator::ValidationError::new("regex_conflicts_with_json"); + e.message = Some("cannot use regex constraint with JSON response format".into()); + return Err(e); + } + + if has_json_format && req.ebnf.is_some() { + let mut e = validator::ValidationError::new("ebnf_conflicts_with_json"); + e.message = Some("cannot use EBNF constraint with JSON response format".into()); + return Err(e); + } + + // 5. Validate mutually exclusive structured output constraints + let constraint_count = [ + req.regex.is_some(), + req.ebnf.is_some(), + matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })), + ] + .iter() + .filter(|&&x| x) + .count(); + + if constraint_count > 1 { + let mut e = validator::ValidationError::new("multiple_constraints"); + e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into()); + return Err(e); + } + + // 6. Validate response format JSON schema name + if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format { + if json_schema.name.is_empty() { + let mut e = validator::ValidationError::new("json_schema_name_empty"); + e.message = Some("JSON schema name cannot be empty".into()); + return Err(e); + } + } + + // 7. Validate tool_choice requires tools (except for "none") + if let Some(ref tool_choice) = req.tool_choice { + let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty()); + + // Check if tool_choice is anything other than "none" + let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None)); + + if is_some_choice && !has_tools { + let mut e = validator::ValidationError::new("tool_choice_requires_tools"); + e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into()); + return Err(e); + } + + // Additional validation when tools are present + if has_tools { + let tools = req.tools.as_ref().unwrap(); + + match tool_choice { + ToolChoice::Function { function, .. } => { + // Validate that the specified function name exists in tools + let function_exists = tools.iter().any(|tool| { + tool.tool_type == "function" && tool.function.name == function.name + }); + + if !function_exists { + let mut e = + validator::ValidationError::new("tool_choice_function_not_found"); + e.message = Some( + format!( + "Invalid value for 'tool_choice': function '{}' not found in 'tools'.", + function.name + ) + .into(), + ); + return Err(e); + } + } + ToolChoice::AllowedTools { + mode, + tools: allowed_tools, + .. + } => { + // Validate mode is "auto" or "required" + if mode != "auto" && mode != "required" { + let mut e = validator::ValidationError::new("tool_choice_invalid_mode"); + e.message = Some(format!( + "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.", + mode + ).into()); + return Err(e); + } + + // Validate that all ToolReferences are Function type (Chat API only supports function tools) + for tool_ref in allowed_tools { + match tool_ref { + ToolReference::Function { name } => { + // Validate that the function exists in tools array + let tool_exists = tools.iter().any(|tool| { + tool.tool_type == "function" && tool.function.name == *name + }); + + if !tool_exists { + let mut e = validator::ValidationError::new( + "tool_choice_tool_not_found", + ); + e.message = Some( + format!( + "Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.", + name + ) + .into(), + ); + return Err(e); + } + } + _ => { + // Chat Completion API only supports function tools in tool_choice + let mut e = validator::ValidationError::new( + "tool_choice_invalid_tool_type", + ); + e.message = Some( + format!( + "Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.", + tool_ref.identifier() + ) + .into(), + ); + return Err(e); + } + } + } + } + _ => {} + } + } + } + + Ok(()) +} + +// ============================================================================ +// Normalizable Implementation +// ============================================================================ + +impl Normalizable for ChatCompletionRequest { + /// Normalize the request by applying migrations and defaults: + /// 1. Migrate deprecated fields to their replacements + /// 2. Clear deprecated fields and log warnings + /// 3. Apply OpenAI defaults for tool_choice + fn normalize(&mut self) { + // Migrate deprecated max_tokens → max_completion_tokens + #[allow(deprecated)] + if self.max_completion_tokens.is_none() && self.max_tokens.is_some() { + self.max_completion_tokens = self.max_tokens; + self.max_tokens = None; // Clear deprecated field + } + + // Migrate deprecated functions → tools + #[allow(deprecated)] + if self.tools.is_none() && self.functions.is_some() { + tracing::warn!("functions is deprecated, use tools instead"); + self.tools = self.functions.as_ref().map(|functions| { + functions + .iter() + .map(|func| Tool { + tool_type: "function".to_string(), + function: func.clone(), + }) + .collect() + }); + self.functions = None; // Clear deprecated field + } + + // Migrate deprecated function_call → tool_choice + #[allow(deprecated)] + if self.tool_choice.is_none() && self.function_call.is_some() { + tracing::warn!("function_call is deprecated, use tool_choice instead"); + self.tool_choice = self.function_call.as_ref().map(|fc| match fc { + FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None), + FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto), + FunctionCall::Function { name } => ToolChoice::Function { + tool_type: "function".to_string(), + function: FunctionChoice { name: name.clone() }, + }, + }); + self.function_call = None; // Clear deprecated field + } + + // Apply tool_choice defaults + if self.tool_choice.is_none() { + if let Some(tools) = &self.tools { + let choice_value = if !tools.is_empty() { + ToolChoiceValue::Auto + } else { + ToolChoiceValue::None + }; + self.tool_choice = Some(ToolChoice::Value(choice_value)); + } + // If tools is None, leave tool_choice as None (don't set it) + } + } +} + +// ============================================================================ +// GenerationRequest Trait Implementation +// ============================================================================ + +impl GenerationRequest for ChatCompletionRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + // Extract text from messages for routing decisions + // Use a single buffer to avoid intermediate Vec allocations + let mut buffer = String::new(); + let mut has_content = false; + + for msg in &self.messages { + match msg { + ChatMessage::System { content, .. } + | ChatMessage::User { content, .. } + | ChatMessage::Tool { content, .. } + | ChatMessage::Developer { content, .. } => { + if has_content && content.has_text() { + buffer.push(' '); + } + if content.append_text_to(&mut buffer) { + has_content = true; + } + } + ChatMessage::Assistant { + content, + reasoning_content, + .. + } => { + // Append main content + if let Some(c) = content { + if has_content && c.has_text() { + buffer.push(' '); + } + if c.append_text_to(&mut buffer) { + has_content = true; + } + } + // Append reasoning content + if let Some(reasoning) = reasoning_content { + if !reasoning.is_empty() { + if has_content { + buffer.push(' '); + } + buffer.push_str(reasoning); + has_content = true; + } + } + } + ChatMessage::Function { content, .. } => { + if !content.is_empty() { + if has_content { + buffer.push(' '); + } + buffer.push_str(content); + has_content = true; + } + } + } + } + + buffer + } +} + +// ============================================================================ +// Response Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionResponse { + pub id: String, + pub object: String, // "chat.completion" + pub created: u64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +impl ChatCompletionResponse { + /// Create a new builder for ChatCompletionResponse + pub fn builder( + id: impl Into, + model: impl Into, + ) -> ChatCompletionResponseBuilder { + ChatCompletionResponseBuilder::new(id, model) + } +} + +/// Response message structure for ChatCompletionResponse (different from request ChatMessage) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionMessage { + pub role: String, // Always "assistant" for responses + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + pub reasoning_content: Option, + // Note: function_call is deprecated and not included + // Note: refusal, annotations, audio are not added yet +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatChoice { + pub index: u32, + pub message: ChatCompletionMessage, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, // "stop", "length", "tool_calls", "content_filter", "function_call" + /// Information about which stop condition was matched + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, // Can be string or integer + /// Hidden states from the model (SGLang extension) + #[serde(skip_serializing_if = "Option::is_none")] + pub hidden_states: Option>, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatCompletionStreamResponse { + pub id: String, + pub object: String, // "chat.completion.chunk" + pub created: u64, + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, +} + +impl ChatCompletionStreamResponse { + /// Create a new builder for ChatCompletionStreamResponse + pub fn builder( + id: impl Into, + model: impl Into, + ) -> ChatCompletionStreamResponseBuilder { + ChatCompletionStreamResponseBuilder::new(id, model) + } +} + +/// Delta structure for streaming chat completion responses +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatMessageDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + pub reasoning_content: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatStreamChoice { + pub index: u32, + pub delta: ChatMessageDelta, + pub logprobs: Option, + pub finish_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/classify.rs b/sgl-model-gateway/vendor/openai-protocol/src/classify.rs new file mode 100644 index 0000000000..492bf88b10 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/classify.rs @@ -0,0 +1,121 @@ +//! Classify API protocol definitions. +//! +//! This module defines the request and response types for the `/v1/classify` API, +//! which is compatible with vLLM's classification endpoint. +//! +//! Classification reuses the embedding backend - the scheduler returns logits as +//! "embeddings", and the classify layer applies softmax + label mapping. + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::common::{GenerationRequest, UsageInfo}; + +// ============================================================================ +// Classify API +// ============================================================================ + +/// Classification request - compatible with vLLM's /v1/classify API +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ClassifyRequest { + /// ID of the model to use + pub model: String, + + /// Input can be a string, array of strings, or token IDs + /// - Single string: "text to classify" + /// - Array of strings: ["text1", "text2"] + /// - Token IDs: [1, 2, 3] (advanced usage) + pub input: Value, + + /// Optional user identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// SGLang extension: request id for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, + + /// SGLang extension: request priority + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// SGLang extension: enable/disable logging of metrics + #[serde(skip_serializing_if = "Option::is_none")] + pub log_metrics: Option, +} + +impl GenerationRequest for ClassifyRequest { + fn is_stream(&self) -> bool { + false // Classification is always non-streaming + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + match &self.input { + Value::String(s) => s.clone(), + Value::Array(arr) => arr + .iter() + .filter_map(|v| v.as_str()) + .collect::>() + .join(" "), + _ => String::new(), + } + } +} + +// ============================================================================ +// Classify Response +// ============================================================================ + +/// Single classification result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClassifyData { + /// Index of this result (for batch requests) + pub index: u32, + /// Predicted class label (from id2label mapping) + pub label: String, + /// Probability distribution over all classes (softmax of logits) + pub probs: Vec, + /// Number of classes + pub num_classes: u32, +} + +/// Classification response +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ClassifyResponse { + /// Unique request ID (format: "classify-{uuid}") + pub id: String, + /// Always "list" + pub object: String, + /// Unix timestamp (seconds since epoch) + pub created: u64, + /// Model name + pub model: String, + /// Classification results (one per input in batch) + pub data: Vec, + /// Token usage info + pub usage: UsageInfo, +} + +impl ClassifyResponse { + /// Create a new ClassifyResponse with the given data + pub fn new( + id: String, + model: String, + created: u64, + data: Vec, + usage: UsageInfo, + ) -> Self { + Self { + id, + object: "list".to_string(), + created, + model, + data, + usage, + } + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/common.rs b/sgl-model-gateway/vendor/openai-protocol/src/common.rs new file mode 100644 index 0000000000..83e2d238af --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/common.rs @@ -0,0 +1,524 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator; + +use super::UNKNOWN_MODEL_ID; + +// ============================================================================ +// Default value helpers +// ============================================================================ + +/// Default model value when not specified +pub(crate) fn default_model() -> String { + UNKNOWN_MODEL_ID.to_string() +} + +/// Helper function for serde default value (returns true) +pub fn default_true() -> bool { + true +} + +// ============================================================================ +// GenerationRequest Trait +// ============================================================================ + +/// Trait for unified access to generation request properties +/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest, +/// EmbeddingRequest, RerankRequest, and ResponsesRequest +pub trait GenerationRequest: Send + Sync { + /// Check if the request is for streaming + fn is_stream(&self) -> bool; + + /// Get the model name if specified + fn get_model(&self) -> Option<&str>; + + /// Extract text content for routing decisions + fn extract_text_for_routing(&self) -> String; +} + +// ============================================================================ +// String/Array Utilities +// ============================================================================ + +/// A type that can be either a single string or an array of strings +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +#[serde(untagged)] +pub enum StringOrArray { + String(String), + Array(Vec), +} + +impl StringOrArray { + /// Get the number of items in the StringOrArray + pub fn len(&self) -> usize { + match self { + StringOrArray::String(_) => 1, + StringOrArray::Array(arr) => arr.len(), + } + } + + /// Check if the StringOrArray is empty + pub fn is_empty(&self) -> bool { + match self { + StringOrArray::String(s) => s.is_empty(), + StringOrArray::Array(arr) => arr.is_empty(), + } + } + + /// Convert to a vector of strings (clones the data) + pub fn to_vec(&self) -> Vec { + match self { + StringOrArray::String(s) => vec![s.clone()], + StringOrArray::Array(arr) => arr.clone(), + } + } + + /// Returns an iterator over string references without cloning. + /// Use this instead of `to_vec()` when you only need to iterate. + pub fn iter(&self) -> StringOrArrayIter<'_> { + StringOrArrayIter { + inner: self, + index: 0, + } + } + + /// Returns the first string, or None if empty + pub fn first(&self) -> Option<&str> { + match self { + StringOrArray::String(s) => { + if s.is_empty() { + None + } else { + Some(s) + } + } + StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()), + } + } +} + +/// Iterator over StringOrArray that yields string references without cloning +pub struct StringOrArrayIter<'a> { + inner: &'a StringOrArray, + index: usize, +} + +impl<'a> Iterator for StringOrArrayIter<'a> { + type Item = &'a str; + + fn next(&mut self) -> Option { + match self.inner { + StringOrArray::String(s) => { + if self.index == 0 { + self.index = 1; + Some(s.as_str()) + } else { + None + } + } + StringOrArray::Array(arr) => { + if self.index < arr.len() { + let item = &arr[self.index]; + self.index += 1; + Some(item.as_str()) + } else { + None + } + } + } + } + + fn size_hint(&self) -> (usize, Option) { + let remaining = match self.inner { + StringOrArray::String(_) => 1 - self.index, + StringOrArray::Array(arr) => arr.len() - self.index, + }; + (remaining, Some(remaining)) + } +} + +impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {} + +/// Validates stop sequences (max 4, non-empty strings) +/// Used by both ChatCompletionRequest and ResponsesRequest +pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> { + match stop { + StringOrArray::String(s) => { + if s.is_empty() { + return Err(validator::ValidationError::new( + "stop sequences cannot be empty", + )); + } + } + StringOrArray::Array(arr) => { + if arr.len() > 4 { + return Err(validator::ValidationError::new( + "maximum 4 stop sequences allowed", + )); + } + for s in arr { + if s.is_empty() { + return Err(validator::ValidationError::new( + "stop sequences cannot be empty", + )); + } + } + } + } + Ok(()) +} + +// ============================================================================ +// Content Parts (for multimodal messages) +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(tag = "type")] +pub enum ContentPart { + #[serde(rename = "text")] + Text { text: String }, + #[serde(rename = "image_url")] + ImageUrl { image_url: ImageUrl }, + #[serde(rename = "video_url")] + VideoUrl { video_url: VideoUrl }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct ImageUrl { + pub url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, // "auto", "low", or "high" +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +pub struct VideoUrl { + pub url: String, +} + +// ============================================================================ +// Response Format (for structured outputs) +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum ResponseFormat { + #[serde(rename = "text")] + Text, + #[serde(rename = "json_object")] + JsonObject, + #[serde(rename = "json_schema")] + JsonSchema { json_schema: JsonSchemaFormat }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct JsonSchemaFormat { + pub name: String, + pub schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +// ============================================================================ +// Streaming +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct StreamOptions { + #[serde(skip_serializing_if = "Option::is_none")] + pub include_usage: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCallDelta { + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "type")] + pub tool_type: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionCallDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub arguments: Option, +} + +// ============================================================================ +// Tools and Function Calling +// ============================================================================ + +/// Tool choice value for simple string options +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolChoiceValue { + Auto, + Required, + None, +} + +/// Tool choice for both Chat Completion and Responses APIs +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ToolChoice { + Value(ToolChoiceValue), + Function { + #[serde(rename = "type")] + tool_type: String, // "function" + function: FunctionChoice, + }, + AllowedTools { + #[serde(rename = "type")] + tool_type: String, // "allowed_tools" + mode: String, // "auto" | "required" TODO: need validation + tools: Vec, + }, +} + +impl Default for ToolChoice { + fn default() -> Self { + Self::Value(ToolChoiceValue::Auto) + } +} + +impl ToolChoice { + /// Serialize tool_choice to string for ResponsesResponse + /// + /// Returns the JSON-serialized tool_choice or "auto" as default + pub fn serialize_to_string(tool_choice: &Option) -> String { + tool_choice + .as_ref() + .map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())) + .unwrap_or_else(|| "auto".to_string()) + } +} + +/// Function choice specification for ToolChoice::Function +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionChoice { + pub name: String, +} + +/// Tool reference for ToolChoice::AllowedTools +/// +/// Represents a reference to a specific tool in the allowed_tools array. +/// Different tool types have different required fields. +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ToolReference { + /// Reference to a function tool + #[serde(rename = "function")] + Function { name: String }, + + /// Reference to an MCP tool + #[serde(rename = "mcp")] + Mcp { + server_label: String, + #[serde(skip_serializing_if = "Option::is_none")] + name: Option, + }, + + /// File search hosted tool + #[serde(rename = "file_search")] + FileSearch, + + /// Web search preview hosted tool + #[serde(rename = "web_search_preview")] + WebSearchPreview, + + /// Computer use preview hosted tool + #[serde(rename = "computer_use_preview")] + ComputerUsePreview, + + /// Code interpreter hosted tool + #[serde(rename = "code_interpreter")] + CodeInterpreter, + + /// Image generation hosted tool + #[serde(rename = "image_generation")] + ImageGeneration, +} + +impl ToolReference { + /// Get a unique identifier for this tool reference + pub fn identifier(&self) -> String { + match self { + ToolReference::Function { name } => format!("function:{}", name), + ToolReference::Mcp { server_label, name } => { + if let Some(n) = name { + format!("mcp:{}:{}", server_label, n) + } else { + format!("mcp:{}", server_label) + } + } + ToolReference::FileSearch => "file_search".to_string(), + ToolReference::WebSearchPreview => "web_search_preview".to_string(), + ToolReference::ComputerUsePreview => "computer_use_preview".to_string(), + ToolReference::CodeInterpreter => "code_interpreter".to_string(), + ToolReference::ImageGeneration => "image_generation".to_string(), + } + } + + /// Get the tool name if this is a function tool + pub fn function_name(&self) -> Option<&str> { + match self { + ToolReference::Function { name } => Some(name.as_str()), + _ => None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Tool { + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: Function, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Function { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub parameters: Value, // JSON Schema + /// Whether to enable strict schema adherence (OpenAI structured outputs) + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ToolCall { + pub id: String, + #[serde(rename = "type")] + pub tool_type: String, // "function" + pub function: FunctionCallResponse, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum FunctionCall { + None, + Auto, + Function { name: String }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FunctionCallResponse { + pub name: String, + #[serde(default)] + pub arguments: Option, // JSON string +} + +// ============================================================================ +// Usage and Logging +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct Usage { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub completion_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionTokensDetails { + pub reasoning_tokens: Option, +} + +/// Usage information (used by rerank and other endpoints) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct UsageInfo { + pub prompt_tokens: u32, + pub completion_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub prompt_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct PromptTokenUsageInfo { + pub cached_tokens: u32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct LogProbs { + pub tokens: Vec, + pub token_logprobs: Vec>, + pub top_logprobs: Vec>>, + pub text_offset: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ChatLogProbs { + Detailed { + #[serde(skip_serializing_if = "Option::is_none")] + content: Option>, + }, + Raw(Value), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ChatLogProbsContent { + pub token: String, + pub logprob: f32, + pub bytes: Option>, + pub top_logprobs: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TopLogProb { + pub token: String, + pub logprob: f32, + pub bytes: Option>, +} + +// ============================================================================ +// Error Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ErrorResponse { + pub error: ErrorDetail, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ErrorDetail { + pub message: String, + #[serde(rename = "type")] + pub error_type: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub param: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub code: Option, +} + +// ============================================================================ +// Input Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum InputIds { + Single(Vec), + Batch(Vec>), +} + +/// LoRA adapter path - can be single path or batch of paths (SGLang extension) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum LoRAPath { + Single(Option), + Batch(Vec>), +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/completion.rs b/sgl-model-gateway/vendor/openai-protocol/src/completion.rs new file mode 100644 index 0000000000..c6a4f638ae --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/completion.rs @@ -0,0 +1,214 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use super::common::*; + +// ============================================================================ +// Completions API (v1/completions) - DEPRECATED but still supported +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionRequest { + /// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang) + pub model: String, + + /// The prompt(s) to generate completions for + pub prompt: StringOrArray, + + /// The suffix that comes after a completion of inserted text + #[serde(skip_serializing_if = "Option::is_none")] + pub suffix: Option, + + /// The maximum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub max_tokens: Option, + + /// What sampling temperature to use, between 0 and 2 + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// An alternative to sampling with temperature (nucleus sampling) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// How many completions to generate for each prompt + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + + /// Whether to stream back partial progress + #[serde(default)] + pub stream: bool, + + /// Options for streaming response + #[serde(skip_serializing_if = "Option::is_none")] + pub stream_options: Option, + + /// Include the log probabilities on the logprobs most likely tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + + /// Echo back the prompt in addition to the completion + #[serde(default)] + pub echo: bool, + + /// Up to 4 sequences where the API will stop generating further tokens + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub presence_penalty: Option, + + /// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far + #[serde(skip_serializing_if = "Option::is_none")] + pub frequency_penalty: Option, + + /// Generates best_of completions server-side and returns the "best" + #[serde(skip_serializing_if = "Option::is_none")] + pub best_of: Option, + + /// Modify the likelihood of specified tokens appearing in the completion + #[serde(skip_serializing_if = "Option::is_none")] + pub logit_bias: Option>, + + /// A unique identifier representing your end-user + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// If specified, our system will make a best effort to sample deterministically + #[serde(skip_serializing_if = "Option::is_none")] + pub seed: Option, + + // -------- Engine Specific Sampling Parameters -------- + /// Top-k sampling parameter (-1 to disable) + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + /// Min-p nucleus sampling parameter + #[serde(skip_serializing_if = "Option::is_none")] + pub min_p: Option, + + /// Minimum number of tokens to generate + #[serde(skip_serializing_if = "Option::is_none")] + pub min_tokens: Option, + + /// Repetition penalty for reducing repetitive text + #[serde(skip_serializing_if = "Option::is_none")] + pub repetition_penalty: Option, + + /// Regex constraint for output generation + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + + /// EBNF grammar constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + + /// JSON schema constraint for structured output + #[serde(skip_serializing_if = "Option::is_none")] + pub json_schema: Option, + + /// Specific token IDs to use as stop conditions + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + + /// Skip trimming stop tokens from output + #[serde(default)] + pub no_stop_trim: bool, + + /// Ignore end-of-sequence tokens during generation + #[serde(default)] + pub ignore_eos: bool, + + /// Skip special tokens during detokenization + #[serde(default = "default_true")] + pub skip_special_tokens: bool, + + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// Sampling seed for deterministic outputs + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_seed: Option, + + /// Additional fields including bootstrap info for PD routing + #[serde(flatten)] + pub other: Map, +} + +impl GenerationRequest for CompletionRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + match &self.prompt { + StringOrArray::String(s) => s.clone(), + StringOrArray::Array(v) => v.join(" "), + } + } +} + +// ============================================================================ +// Response Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionResponse { + pub id: String, + pub object: String, // "text_completion" + pub created: u64, + pub model: String, + pub choices: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionChoice { + pub text: String, + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, // "stop", "length", "content_filter", etc. + /// Information about which stop condition was matched + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, // Can be string or integer +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionStreamResponse { + pub id: String, + pub object: String, // "text_completion" + pub created: u64, + pub choices: Vec, + pub model: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub system_fingerprint: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct CompletionStreamChoice { + pub text: String, + pub index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub logprobs: Option, + pub finish_reason: Option, +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/embedding.rs b/sgl-model-gateway/vendor/openai-protocol/src/embedding.rs new file mode 100644 index 0000000000..12e3daf19c --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/embedding.rs @@ -0,0 +1,76 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::common::{GenerationRequest, UsageInfo}; + +// ============================================================================ +// Embedding API +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct EmbeddingRequest { + /// ID of the model to use + pub model: String, + + /// Input can be a string, array of strings, tokens, or batch inputs + pub input: Value, + + /// Optional encoding format (e.g., "float", "base64") + #[serde(skip_serializing_if = "Option::is_none")] + pub encoding_format: Option, + + /// Optional user identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Optional number of dimensions for the embedding + #[serde(skip_serializing_if = "Option::is_none")] + pub dimensions: Option, + + /// SGLang extension: request id for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, + + /// SGLang extension: enable/disable logging of metrics for this request + #[serde(skip_serializing_if = "Option::is_none")] + pub log_metrics: Option, +} + +impl GenerationRequest for EmbeddingRequest { + fn is_stream(&self) -> bool { + // Embeddings are non-streaming + false + } + + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn extract_text_for_routing(&self) -> String { + // Best effort: extract text content for routing decisions + match &self.input { + Value::String(s) => s.clone(), + Value::Array(arr) => arr + .iter() + .filter_map(|v| v.as_str()) + .collect::>() + .join(" "), + _ => String::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmbeddingObject { + pub object: String, // "embedding" + pub embedding: Vec, + pub index: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EmbeddingResponse { + pub object: String, // "list" + pub data: Vec, + pub model: String, + pub usage: UsageInfo, +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/event_types.rs b/sgl-model-gateway/vendor/openai-protocol/src/event_types.rs new file mode 100644 index 0000000000..c36dc5d451 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/event_types.rs @@ -0,0 +1,228 @@ +use std::fmt; + +/// Response lifecycle events +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ResponseEvent { + Created, + InProgress, + Completed, +} + +impl ResponseEvent { + pub const CREATED: &'static str = "response.created"; + pub const IN_PROGRESS: &'static str = "response.in_progress"; + pub const COMPLETED: &'static str = "response.completed"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::Created => Self::CREATED, + Self::InProgress => Self::IN_PROGRESS, + Self::Completed => Self::COMPLETED, + } + } +} + +impl fmt::Display for ResponseEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Output item events for streaming +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OutputItemEvent { + Added, + Done, + Delta, +} + +impl OutputItemEvent { + pub const ADDED: &'static str = "response.output_item.added"; + pub const DONE: &'static str = "response.output_item.done"; + pub const DELTA: &'static str = "response.output_item.delta"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::Added => Self::ADDED, + Self::Done => Self::DONE, + Self::Delta => Self::DELTA, + } + } +} + +impl fmt::Display for OutputItemEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Function call argument streaming events +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FunctionCallEvent { + ArgumentsDelta, + ArgumentsDone, +} + +impl FunctionCallEvent { + pub const ARGUMENTS_DELTA: &'static str = "response.function_call_arguments.delta"; + pub const ARGUMENTS_DONE: &'static str = "response.function_call_arguments.done"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::ArgumentsDelta => Self::ARGUMENTS_DELTA, + Self::ArgumentsDone => Self::ARGUMENTS_DONE, + } + } +} + +impl fmt::Display for FunctionCallEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Content part streaming events +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ContentPartEvent { + Added, + Done, +} + +impl ContentPartEvent { + pub const ADDED: &'static str = "response.content_part.added"; + pub const DONE: &'static str = "response.content_part.done"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::Added => Self::ADDED, + Self::Done => Self::DONE, + } + } +} + +impl fmt::Display for ContentPartEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Output text streaming events +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum OutputTextEvent { + Delta, + Done, +} + +impl OutputTextEvent { + pub const DELTA: &'static str = "response.output_text.delta"; + pub const DONE: &'static str = "response.output_text.done"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::Delta => Self::DELTA, + Self::Done => Self::DONE, + } + } +} + +impl fmt::Display for OutputTextEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +// ============================================================================ +// MCP Events +// ============================================================================ + +/// MCP (Model Context Protocol) call events +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum McpEvent { + CallArgumentsDelta, + CallArgumentsDone, + CallInProgress, + CallCompleted, + CallFailed, + ListToolsInProgress, + ListToolsCompleted, +} + +impl McpEvent { + pub const CALL_ARGUMENTS_DELTA: &'static str = "response.mcp_call_arguments.delta"; + pub const CALL_ARGUMENTS_DONE: &'static str = "response.mcp_call_arguments.done"; + pub const CALL_IN_PROGRESS: &'static str = "response.mcp_call.in_progress"; + pub const CALL_COMPLETED: &'static str = "response.mcp_call.completed"; + pub const CALL_FAILED: &'static str = "response.mcp_call.failed"; + pub const LIST_TOOLS_IN_PROGRESS: &'static str = "response.mcp_list_tools.in_progress"; + pub const LIST_TOOLS_COMPLETED: &'static str = "response.mcp_list_tools.completed"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::CallArgumentsDelta => Self::CALL_ARGUMENTS_DELTA, + Self::CallArgumentsDone => Self::CALL_ARGUMENTS_DONE, + Self::CallInProgress => Self::CALL_IN_PROGRESS, + Self::CallCompleted => Self::CALL_COMPLETED, + Self::CallFailed => Self::CALL_FAILED, + Self::ListToolsInProgress => Self::LIST_TOOLS_IN_PROGRESS, + Self::ListToolsCompleted => Self::LIST_TOOLS_COMPLETED, + } + } +} + +impl fmt::Display for McpEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Item type discriminators used in output items +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ItemType { + FunctionCall, + FunctionToolCall, + McpCall, + Function, + McpListTools, +} + +impl ItemType { + pub const FUNCTION_CALL: &'static str = "function_call"; + pub const FUNCTION_TOOL_CALL: &'static str = "function_tool_call"; + pub const MCP_CALL: &'static str = "mcp_call"; + pub const FUNCTION: &'static str = "function"; + pub const MCP_LIST_TOOLS: &'static str = "mcp_list_tools"; + + pub const fn as_str(&self) -> &'static str { + match self { + Self::FunctionCall => Self::FUNCTION_CALL, + Self::FunctionToolCall => Self::FUNCTION_TOOL_CALL, + Self::McpCall => Self::MCP_CALL, + Self::Function => Self::FUNCTION, + Self::McpListTools => Self::MCP_LIST_TOOLS, + } + } + + /// Check if this is a function call variant (FunctionCall or FunctionToolCall) + pub const fn is_function_call(&self) -> bool { + matches!(self, Self::FunctionCall | Self::FunctionToolCall) + } +} + +impl fmt::Display for ItemType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Check if an event type string matches any response lifecycle event +pub fn is_response_event(event_type: &str) -> bool { + matches!( + event_type, + ResponseEvent::CREATED | ResponseEvent::IN_PROGRESS | ResponseEvent::COMPLETED + ) +} + +/// Check if an item type string is a function call variant +pub fn is_function_call_type(item_type: &str) -> bool { + item_type == ItemType::FUNCTION_CALL || item_type == ItemType::FUNCTION_TOOL_CALL +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/generate.rs b/sgl-model-gateway/vendor/openai-protocol/src/generate.rs new file mode 100644 index 0000000000..0c81184745 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/generate.rs @@ -0,0 +1,297 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::{ + common::{default_true, GenerationRequest, InputIds}, + sampling_params::SamplingParams, +}; +use crate::validated::Normalizable; + +// ============================================================================ +// SGLang Generate API (native format) +// ============================================================================ + +#[derive(Clone, Debug, Serialize, Deserialize, Validate)] +#[validate(schema(function = "validate_generate_request"))] +pub struct GenerateRequest { + /// Text input - SGLang native format + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + + pub model: Option, + + /// Input IDs for tokenized input + #[serde(skip_serializing_if = "Option::is_none")] + pub input_ids: Option, + + /// Input embeddings for direct embedding input + /// Can be a 2D array (single request) or 3D array (batch of requests) + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub input_embeds: Option, + + /// Image input data + /// Can be an image instance, file name, URL, or base64 encoded string + /// Supports single images, lists of images, or nested lists for batch processing + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub image_data: Option, + + /// Video input data + /// Can be a file name, URL, or base64 encoded string + /// Supports single videos, lists of videos, or nested lists for batch processing + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub video_data: Option, + + /// Audio input data + /// Can be a file name, URL, or base64 encoded string + /// Supports single audio files, lists of audio, or nested lists for batch processing + /// Placeholder for future use + #[serde(skip_serializing_if = "Option::is_none")] + pub audio_data: Option, + + /// Sampling parameters (sglang style) + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_params: Option, + + /// Whether to return logprobs + #[serde(skip_serializing_if = "Option::is_none")] + pub return_logprob: Option, + + /// If return logprobs, the start location in the prompt for returning logprobs. + #[serde(skip_serializing_if = "Option::is_none")] + pub logprob_start_len: Option, + + /// If return logprobs, the number of top logprobs to return at each position. + #[serde(skip_serializing_if = "Option::is_none")] + pub top_logprobs_num: Option, + + /// If return logprobs, the token ids to return logprob for. + #[serde(skip_serializing_if = "Option::is_none")] + pub token_ids_logprob: Option>, + + /// Whether to detokenize tokens in text in the returned logprobs. + #[serde(default)] + pub return_text_in_logprobs: bool, + + /// Whether to stream the response + #[serde(default)] + pub stream: bool, + + /// Whether to log metrics for this request (e.g. health_generate calls do not log metrics) + #[serde(default = "default_true")] + pub log_metrics: bool, + + /// Return model hidden states + #[serde(default)] + pub return_hidden_states: bool, + + /// The modalities of the image data [image, multi-images, video] + #[serde(skip_serializing_if = "Option::is_none")] + pub modalities: Option>, + + /// Session parameters for continual prompting + #[serde(skip_serializing_if = "Option::is_none")] + pub session_params: Option>, + + /// Path to LoRA adapter(s) for model customization + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_path: Option, + + /// LoRA adapter ID (if pre-loaded) + #[serde(skip_serializing_if = "Option::is_none")] + pub lora_id: Option, + + /// Custom logit processor for advanced sampling control. Must be a serialized instance + /// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py + /// Use the processor's `to_str()` method to generate the serialized string. + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_logit_processor: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_host: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_port: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_room: Option, + + /// For disaggregated inference + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_pair_key: Option, + + /// Data parallel rank routing + #[serde(skip_serializing_if = "Option::is_none")] + pub data_parallel_rank: Option, + + /// Background response + #[serde(default)] + pub background: bool, + + /// Conversation ID for tracking + #[serde(skip_serializing_if = "Option::is_none")] + pub conversation_id: Option, + + /// Priority for the request + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// Extra key for classifying the request (e.g. cache_salt) + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_key: Option, + + /// Whether to disallow logging for this request (e.g. due to ZDR) + #[serde(default)] + pub no_logs: bool, + + /// Custom metric labels + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_labels: Option>, + + /// Whether to return bytes for image generation + #[serde(default)] + pub return_bytes: bool, + + /// Whether to return entropy + #[serde(default)] + pub return_entropy: bool, + + /// Request ID for tracking (inherited from BaseReq in Python) + #[serde(skip_serializing_if = "Option::is_none")] + pub rid: Option, +} + +impl Normalizable for GenerateRequest { + // Use default no-op implementation - no normalization needed for GenerateRequest +} + +/// Validation function for GenerateRequest - ensure exactly one input type is provided +fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> { + // Exactly one of text or input_ids must be provided + // Note: input_embeds not yet supported in Rust implementation + let has_text = req.text.is_some(); + let has_input_ids = req.input_ids.is_some(); + + let count = [has_text, has_input_ids].iter().filter(|&&x| x).count(); + + if count == 0 { + return Err(validator::ValidationError::new( + "Either text or input_ids should be provided.", + )); + } + + if count > 1 { + return Err(validator::ValidationError::new( + "Either text or input_ids should be provided.", + )); + } + + Ok(()) +} + +impl GenerationRequest for GenerateRequest { + fn is_stream(&self) -> bool { + self.stream + } + + fn get_model(&self) -> Option<&str> { + // Generate requests have an optional model field + if let Some(s) = &self.model { + Some(s.as_str()) + } else { + None + } + } + + fn extract_text_for_routing(&self) -> String { + // Check fields in priority order: text, input_ids + if let Some(ref text) = self.text { + return text.clone(); + } + + if let Some(ref input_ids) = self.input_ids { + return match input_ids { + InputIds::Single(ids) => ids + .iter() + .map(|&id| id.to_string()) + .collect::>() + .join(" "), + InputIds::Batch(batches) => batches + .iter() + .flat_map(|batch| batch.iter().map(|&id| id.to_string())) + .collect::>() + .join(" "), + }; + } + + // No text input found + String::new() + } +} + +// ============================================================================ +// SGLang Generate Response Types +// ============================================================================ + +/// SGLang generate response (single completion or array for n>1) +/// +/// Format for n=1: +/// ```json +/// { +/// "text": "...", +/// "output_ids": [...], +/// "meta_info": { ... } +/// } +/// ``` +/// +/// Format for n>1: +/// ```json +/// [ +/// {"text": "...", "output_ids": [...], "meta_info": {...}}, +/// {"text": "...", "output_ids": [...], "meta_info": {...}} +/// ] +/// ``` +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateResponse { + pub text: String, + pub output_ids: Vec, + pub meta_info: GenerateMetaInfo, +} + +/// Metadata for a single generate completion +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateMetaInfo { + pub id: String, + pub finish_reason: GenerateFinishReason, + pub prompt_tokens: u32, + pub weight_version: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_token_logprobs: Option>>>, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_token_logprobs: Option>>>, + pub completion_tokens: u32, + pub cached_tokens: u32, + pub e2e_latency: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub matched_stop: Option, +} + +/// Finish reason for generate endpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +pub enum GenerateFinishReason { + Length { + length: u32, + }, + Stop, + #[serde(untagged)] + Other(Value), +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/lib.rs b/sgl-model-gateway/vendor/openai-protocol/src/lib.rs new file mode 100644 index 0000000000..6196611473 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/lib.rs @@ -0,0 +1,25 @@ +// Protocol definitions and validation for various LLM APIs +// This module provides a structured approach to handling different API protocols + +/// Default model identifier used when no model is specified. +/// +/// This constant should be used instead of hardcoded "unknown" strings +/// throughout the codebase for consistency. +pub const UNKNOWN_MODEL_ID: &str = "unknown"; + +pub mod builders; +pub mod chat; +pub mod classify; +pub mod common; +pub mod completion; +pub mod embedding; +pub mod event_types; +pub mod generate; +pub mod messages; +pub mod parser; +pub mod rerank; +pub mod responses; +pub mod sampling_params; +pub mod tokenize; +pub mod validated; +pub mod worker_spec; diff --git a/sgl-model-gateway/vendor/openai-protocol/src/messages.rs b/sgl-model-gateway/vendor/openai-protocol/src/messages.rs new file mode 100644 index 0000000000..b180e1224d --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/messages.rs @@ -0,0 +1,1726 @@ +//! Anthropic Messages API protocol definitions +//! +//! This module provides Rust types for the Anthropic Messages API. +//! See: https://docs.anthropic.com/en/api/messages + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +// ============================================================================ +// Request Types +// ============================================================================ + +/// Request to create a message using the Anthropic Messages API. +/// +/// This is the main request type for `/v1/messages` endpoint. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreateMessageRequest { + /// The model that will complete your prompt. + pub model: String, + + /// Input messages for the conversation. + pub messages: Vec, + + /// The maximum number of tokens to generate before stopping. + pub max_tokens: u32, + + /// An object describing metadata about the request. + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option, + + /// Service tier for the request (auto or standard_only). + #[serde(skip_serializing_if = "Option::is_none")] + pub service_tier: Option, + + /// Custom text sequences that will cause the model to stop generating. + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_sequences: Option>, + + /// Whether to incrementally stream the response using server-sent events. + #[serde(skip_serializing_if = "Option::is_none")] + pub stream: Option, + + /// System prompt for providing context and instructions. + #[serde(skip_serializing_if = "Option::is_none")] + pub system: Option, + + /// Amount of randomness injected into the response (0.0 to 1.0). + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// Configuration for extended thinking. + #[serde(skip_serializing_if = "Option::is_none")] + pub thinking: Option, + + /// How the model should use the provided tools. + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + /// Definitions of tools that the model may use. + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, + + /// Only sample from the top K options for each subsequent token. + #[serde(skip_serializing_if = "Option::is_none")] + pub top_k: Option, + + /// Use nucleus sampling. + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + // Beta features + /// Container configuration for code execution (beta). + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, + + /// MCP servers to be utilized in this request (beta). + #[serde(skip_serializing_if = "Option::is_none")] + pub mcp_servers: Option>, +} + +impl CreateMessageRequest { + /// Check if the request is for streaming + pub fn is_stream(&self) -> bool { + self.stream.unwrap_or(false) + } + + /// Get the model name + pub fn get_model(&self) -> &str { + &self.model + } +} + +/// Request metadata +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Metadata { + /// An external identifier for the user who is associated with the request. + #[serde(skip_serializing_if = "Option::is_none")] + pub user_id: Option, +} + +/// Service tier options +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ServiceTier { + Auto, + StandardOnly, +} + +/// System content can be a string or an array of text blocks +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum SystemContent { + String(String), + Blocks(Vec), +} + +/// A single input message in a conversation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InputMessage { + /// The role of the message sender (user or assistant) + pub role: Role, + + /// The content of the message + pub content: InputContent, +} + +/// Role of a message sender +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum Role { + User, + Assistant, +} + +/// Input content can be a string or an array of content blocks +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum InputContent { + String(String), + Blocks(Vec), +} + +// ============================================================================ +// Input Content Blocks +// ============================================================================ + +/// Input content block types +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum InputContentBlock { + /// Text content + Text(TextBlock), + /// Image content + Image(ImageBlock), + /// Document content + Document(DocumentBlock), + /// Tool use block (for assistant messages) + ToolUse(ToolUseBlock), + /// Tool result block (for user messages) + ToolResult(ToolResultBlock), + /// Thinking block + Thinking(ThinkingBlock), + /// Redacted thinking block + RedactedThinking(RedactedThinkingBlock), + /// Server tool use block + ServerToolUse(ServerToolUseBlock), + /// Search result block + SearchResult(SearchResultBlock), + /// Web search tool result block + WebSearchToolResult(WebSearchToolResultBlock), +} + +/// Text content block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextBlock { + /// The text content + pub text: String, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, + + /// Citations for this text block + #[serde(skip_serializing_if = "Option::is_none")] + pub citations: Option>, +} + +/// Image content block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageBlock { + /// The image source + pub source: ImageSource, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Image source (base64 or URL) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ImageSource { + Base64 { media_type: String, data: String }, + Url { url: String }, +} + +/// Document content block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DocumentBlock { + /// The document source + pub source: DocumentSource, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, + + /// Optional title for the document + #[serde(skip_serializing_if = "Option::is_none")] + pub title: Option, + + /// Optional context for the document + #[serde(skip_serializing_if = "Option::is_none")] + pub context: Option, + + /// Citations configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub citations: Option, +} + +/// Document source (base64, text, or URL) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum DocumentSource { + Base64 { media_type: String, data: String }, + Text { data: String }, + Url { url: String }, + Content { content: Vec }, +} + +/// Tool use block (in assistant messages) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolUseBlock { + /// Unique identifier for this tool use + pub id: String, + + /// Name of the tool being used + pub name: String, + + /// Input arguments for the tool + pub input: Value, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Tool result block (in user messages) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The result content (string or blocks) + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + + /// Whether this result indicates an error + #[serde(skip_serializing_if = "Option::is_none")] + pub is_error: Option, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Tool result content (string or blocks) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ToolResultContent { + String(String), + Blocks(Vec), +} + +/// Content blocks allowed in tool results +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolResultContentBlock { + Text(TextBlock), + Image(ImageBlock), + Document(DocumentBlock), + SearchResult(SearchResultBlock), +} + +/// Thinking block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ThinkingBlock { + /// The thinking content + pub thinking: String, + + /// Signature for the thinking block + pub signature: String, +} + +/// Redacted thinking block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RedactedThinkingBlock { + /// The encrypted/redacted data + pub data: String, +} + +/// Server tool use block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServerToolUseBlock { + /// Unique identifier for this tool use + pub id: String, + + /// Name of the server tool + pub name: String, + + /// Input arguments for the tool + pub input: Value, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Search result block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchResultBlock { + /// Source URL or identifier + pub source: String, + + /// Title of the search result + pub title: String, + + /// Content of the search result + pub content: Vec, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, + + /// Citations configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub citations: Option, +} + +/// Web search tool result block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchToolResultBlock { + /// The tool use ID this result is for + pub tool_use_id: String, + + /// The search results or error + pub content: WebSearchToolResultContent, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Web search tool result content +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WebSearchToolResultContent { + Results(Vec), + Error(WebSearchToolResultError), +} + +/// Web search result block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchResultBlock { + /// Title of the search result + pub title: String, + + /// URL of the search result + pub url: String, + + /// Encrypted content + pub encrypted_content: String, + + /// Page age (if available) + #[serde(skip_serializing_if = "Option::is_none")] + pub page_age: Option, +} + +/// Web search tool result error +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchToolResultError { + #[serde(rename = "type")] + pub error_type: String, + pub error_code: WebSearchToolResultErrorCode, +} + +/// Web search tool result error codes +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WebSearchToolResultErrorCode { + InvalidToolInput, + Unavailable, + MaxUsesExceeded, + TooManyRequests, + QueryTooLong, +} + +/// Cache control configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CacheControl { + Ephemeral, +} + +/// Citations configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CitationsConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, +} + +/// Citation types +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Citation { + CharLocation(CharLocationCitation), + PageLocation(PageLocationCitation), + ContentBlockLocation(ContentBlockLocationCitation), + WebSearchResultLocation(WebSearchResultLocationCitation), + SearchResultLocation(SearchResultLocationCitation), +} + +/// Character location citation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CharLocationCitation { + pub cited_text: String, + pub document_index: u32, + pub document_title: Option, + pub start_char_index: u32, + pub end_char_index: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub file_id: Option, +} + +/// Page location citation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PageLocationCitation { + pub cited_text: String, + pub document_index: u32, + pub document_title: Option, + pub start_page_number: u32, + pub end_page_number: u32, +} + +/// Content block location citation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContentBlockLocationCitation { + pub cited_text: String, + pub document_index: u32, + pub document_title: Option, + pub start_block_index: u32, + pub end_block_index: u32, +} + +/// Web search result location citation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchResultLocationCitation { + pub cited_text: String, + pub url: String, + pub title: Option, + pub encrypted_index: String, +} + +/// Search result location citation +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchResultLocationCitation { + pub cited_text: String, + pub search_result_index: u32, + pub source: String, + pub title: Option, + pub start_block_index: u32, + pub end_block_index: u32, +} + +// ============================================================================ +// Tool Definitions +// ============================================================================ + +/// Tool definition +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Tool { + /// Custom tool definition + Custom(CustomTool), + /// Bash tool (computer use) + Bash(BashTool), + /// Text editor tool (computer use) + TextEditor(TextEditorTool), + /// Web search tool + WebSearch(WebSearchTool), +} + +/// Custom tool definition +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CustomTool { + /// Name of the tool + pub name: String, + + /// Optional type (defaults to "custom") + #[serde(rename = "type", skip_serializing_if = "Option::is_none")] + pub tool_type: Option, + + /// Description of what this tool does + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + + /// JSON schema for the tool's input + pub input_schema: InputSchema, + + /// Cache control for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// JSON Schema for tool input +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct InputSchema { + #[serde(rename = "type")] + pub schema_type: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub properties: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub required: Option>, + + /// Additional properties can be stored here + #[serde(flatten)] + pub additional: HashMap, +} + +/// Bash tool for computer use +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BashTool { + #[serde(rename = "type")] + pub tool_type: String, // "bash_20250124" + + pub name: String, // "bash" + + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Text editor tool for computer use +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextEditorTool { + #[serde(rename = "type")] + pub tool_type: String, // "text_editor_20250124", etc. + + pub name: String, // "str_replace_editor" + + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Web search tool +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebSearchTool { + #[serde(rename = "type")] + pub tool_type: String, // "web_search_20250305" + + pub name: String, // "web_search" + + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_domains: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub blocked_domains: Option>, + + #[serde(skip_serializing_if = "Option::is_none")] + pub max_uses: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub user_location: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// User location for web search +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UserLocation { + #[serde(rename = "type")] + pub location_type: String, // "approximate" + + #[serde(skip_serializing_if = "Option::is_none")] + pub city: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub region: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub country: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub timezone: Option, +} + +// ============================================================================ +// Tool Choice +// ============================================================================ + +/// How the model should use the provided tools +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolChoice { + /// The model will automatically decide whether to use tools + Auto { + #[serde(skip_serializing_if = "Option::is_none")] + disable_parallel_tool_use: Option, + }, + /// The model will use any available tools + Any { + #[serde(skip_serializing_if = "Option::is_none")] + disable_parallel_tool_use: Option, + }, + /// The model will use the specified tool + Tool { + name: String, + #[serde(skip_serializing_if = "Option::is_none")] + disable_parallel_tool_use: Option, + }, + /// The model will not use tools + None, +} + +// ============================================================================ +// Thinking Configuration +// ============================================================================ + +/// Configuration for extended thinking +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ThinkingConfig { + /// Enable extended thinking + Enabled { + /// Budget in tokens for thinking (minimum 1024) + budget_tokens: u32, + }, + /// Disable extended thinking + Disabled, +} + +// ============================================================================ +// Response Types +// ============================================================================ + +/// Response message from the API +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Message { + /// Unique object identifier + pub id: String, + + /// Object type (always "message") + #[serde(rename = "type")] + pub message_type: String, + + /// Conversational role (always "assistant") + pub role: String, + + /// Content generated by the model + pub content: Vec, + + /// The model that generated the message + pub model: String, + + /// The reason the model stopped generating + pub stop_reason: Option, + + /// Which custom stop sequence was generated (if any) + pub stop_sequence: Option, + + /// Billing and rate-limit usage + pub usage: Usage, +} + +/// Output content block types +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentBlock { + /// Text content + Text { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + citations: Option>, + }, + /// Tool use by the model + ToolUse { + id: String, + name: String, + input: Value, + }, + /// Thinking content + Thinking { thinking: String, signature: String }, + /// Redacted thinking content + RedactedThinking { data: String }, + /// Server tool use + ServerToolUse { + id: String, + name: String, + input: Value, + }, + /// Web search tool result + WebSearchToolResult { + tool_use_id: String, + content: WebSearchToolResultContent, + }, +} + +/// Stop reasons +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum StopReason { + /// The model reached a natural stopping point + EndTurn, + /// We exceeded the requested max_tokens + MaxTokens, + /// One of the custom stop_sequences was generated + StopSequence, + /// The model invoked one or more tools + ToolUse, + /// We paused a long-running turn + PauseTurn, + /// Streaming classifiers intervened + Refusal, +} + +/// Billing and rate-limit usage +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Usage { + /// The number of input tokens used + pub input_tokens: u32, + + /// The number of output tokens used + pub output_tokens: u32, + + /// The number of input tokens used to create the cache entry + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_input_tokens: Option, + + /// The number of input tokens read from the cache + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read_input_tokens: Option, + + /// Breakdown of cached tokens by TTL + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation: Option, + + /// Server tool usage information + #[serde(skip_serializing_if = "Option::is_none")] + pub server_tool_use: Option, + + /// Service tier used for the request + #[serde(skip_serializing_if = "Option::is_none")] + pub service_tier: Option, +} + +/// Cache creation breakdown +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheCreation { + #[serde(flatten)] + pub tokens_by_ttl: HashMap, +} + +/// Server tool usage information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ServerToolUsage { + pub web_search_requests: u32, +} + +// ============================================================================ +// Streaming Event Types +// ============================================================================ + +/// Server-sent event wrapper +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MessageStreamEvent { + /// Start of a new message + MessageStart { message: Message }, + /// Update to a message + MessageDelta { + delta: MessageDelta, + usage: MessageDeltaUsage, + }, + /// End of a message + MessageStop, + /// Start of a content block + ContentBlockStart { + index: u32, + content_block: ContentBlock, + }, + /// Update to a content block + ContentBlockDelta { + index: u32, + delta: ContentBlockDelta, + }, + /// End of a content block + ContentBlockStop { index: u32 }, + /// Ping event (for keep-alive) + Ping, + /// Error event + Error { error: ErrorResponse }, +} + +/// Message delta for streaming updates +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MessageDelta { + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_reason: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_sequence: Option, +} + +/// Usage delta for streaming updates +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MessageDeltaUsage { + pub output_tokens: u32, + + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_input_tokens: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read_input_tokens: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub server_tool_use: Option, +} + +/// Content block delta for streaming updates +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentBlockDelta { + /// Text delta + TextDelta { text: String }, + /// JSON input delta (for tool use) + InputJsonDelta { partial_json: String }, + /// Thinking delta + ThinkingDelta { thinking: String }, + /// Signature delta + SignatureDelta { signature: String }, + /// Citations delta + CitationsDelta { citation: Citation }, +} + +// ============================================================================ +// Error Types +// ============================================================================ + +/// Error response +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ErrorResponse { + #[serde(rename = "type")] + pub error_type: String, + + pub message: String, +} + +/// API error types +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ApiError { + InvalidRequestError { message: String }, + AuthenticationError { message: String }, + BillingError { message: String }, + PermissionError { message: String }, + NotFoundError { message: String }, + RateLimitError { message: String }, + TimeoutError { message: String }, + ApiError { message: String }, + OverloadedError { message: String }, +} + +// ============================================================================ +// Count Tokens Types +// ============================================================================ + +/// Request to count tokens in a message +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CountMessageTokensRequest { + /// The model to use for token counting + pub model: String, + + /// Input messages + pub messages: Vec, + + /// System prompt + #[serde(skip_serializing_if = "Option::is_none")] + pub system: Option, + + /// Thinking configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub thinking: Option, + + /// Tool choice + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_choice: Option, + + /// Tool definitions + #[serde(skip_serializing_if = "Option::is_none")] + pub tools: Option>, +} + +/// Response from token counting +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CountMessageTokensResponse { + pub input_tokens: u32, +} + +// ============================================================================ +// Model Info Types +// ============================================================================ + +/// Model information +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelInfo { + /// Object type (always "model") + #[serde(rename = "type")] + pub model_type: String, + + /// Model ID + pub id: String, + + /// Display name + pub display_name: String, + + /// When the model was created + pub created_at: String, +} + +/// List of models response +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListModelsResponse { + pub data: Vec, + pub has_more: bool, + pub first_id: Option, + pub last_id: Option, +} + +// ============================================================================ +// Beta Features - Container & MCP Configuration +// ============================================================================ + +/// Container configuration for code execution (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContainerConfig { + /// Container ID for reuse across requests + #[serde(skip_serializing_if = "Option::is_none")] + pub id: Option, + + /// Skills to be loaded in the container + #[serde(skip_serializing_if = "Option::is_none")] + pub skills: Option>, +} + +/// MCP server configuration (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpServerConfig { + /// Name of the MCP server + pub name: String, + + /// MCP server URL + pub url: String, + + /// Authorization token (if required) + #[serde(skip_serializing_if = "Option::is_none")] + pub authorization_token: Option, + + /// Tool configuration for this server + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_configuration: Option, +} + +/// MCP tool configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolConfiguration { + /// Whether to allow all tools + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, + + /// Allowed tool names + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_tools: Option>, +} + +// ============================================================================ +// Beta Features - MCP Tool Types +// ============================================================================ + +/// MCP tool use block (beta) - for assistant messages +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolUseBlock { + /// Unique identifier for this tool use + pub id: String, + + /// Name of the tool being used + pub name: String, + + /// Name of the MCP server + pub server_name: String, + + /// Input arguments for the tool + pub input: Value, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// MCP tool result block (beta) - for user messages +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The result content + #[serde(skip_serializing_if = "Option::is_none")] + pub content: Option, + + /// Whether this result indicates an error + #[serde(skip_serializing_if = "Option::is_none")] + pub is_error: Option, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// MCP toolset definition (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolset { + #[serde(rename = "type")] + pub toolset_type: String, // "mcp_toolset" + + /// Name of the MCP server to configure tools for + pub mcp_server_name: String, + + /// Default configuration applied to all tools from this server + #[serde(skip_serializing_if = "Option::is_none")] + pub default_config: Option, + + /// Configuration overrides for specific tools + #[serde(skip_serializing_if = "Option::is_none")] + pub configs: Option>, + + /// Cache control for this toolset + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Default configuration for MCP tools +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolDefaultConfig { + /// Whether tools are enabled + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, + + /// Whether to defer loading + #[serde(skip_serializing_if = "Option::is_none")] + pub defer_loading: Option, +} + +/// Per-tool MCP configuration +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpToolConfig { + /// Whether this tool is enabled + #[serde(skip_serializing_if = "Option::is_none")] + pub enabled: Option, + + /// Whether to defer loading + #[serde(skip_serializing_if = "Option::is_none")] + pub defer_loading: Option, +} + +// ============================================================================ +// Beta Features - Code Execution Types +// ============================================================================ + +/// Code execution tool (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CodeExecutionTool { + #[serde(rename = "type")] + pub tool_type: String, // "code_execution_20250522" or "code_execution_20250825" + + pub name: String, // "code_execution" + + /// Allowed callers for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_callers: Option>, + + /// Whether to defer loading + #[serde(skip_serializing_if = "Option::is_none")] + pub defer_loading: Option, + + /// Whether to use strict mode + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, + + /// Cache control for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Code execution result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CodeExecutionResultBlock { + /// Stdout output + pub stdout: String, + + /// Stderr output + pub stderr: String, + + /// Return code + pub return_code: i32, + + /// Output files + pub content: Vec, +} + +/// Code execution output file reference +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CodeExecutionOutputBlock { + #[serde(rename = "type")] + pub block_type: String, // "code_execution_output" + + /// File ID + pub file_id: String, +} + +/// Code execution tool result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CodeExecutionToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The result content (success or error) + pub content: CodeExecutionToolResultContent, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Code execution tool result content +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum CodeExecutionToolResultContent { + Success(CodeExecutionResultBlock), + Error(CodeExecutionToolResultError), +} + +/// Code execution tool result error +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CodeExecutionToolResultError { + #[serde(rename = "type")] + pub error_type: String, // "code_execution_tool_result_error" + + pub error_code: CodeExecutionToolResultErrorCode, +} + +/// Code execution error codes +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CodeExecutionToolResultErrorCode { + Unavailable, + CodeExecutionExceededTimeout, + ContainerExpired, + InvalidToolInput, +} + +/// Bash code execution result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BashCodeExecutionResultBlock { + /// Stdout output + pub stdout: String, + + /// Stderr output + pub stderr: String, + + /// Return code + pub return_code: i32, + + /// Output files + pub content: Vec, +} + +/// Bash code execution output file reference +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BashCodeExecutionOutputBlock { + #[serde(rename = "type")] + pub block_type: String, // "bash_code_execution_output" + + /// File ID + pub file_id: String, +} + +/// Bash code execution tool result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BashCodeExecutionToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The result content (success or error) + pub content: BashCodeExecutionToolResultContent, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Bash code execution tool result content +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum BashCodeExecutionToolResultContent { + Success(BashCodeExecutionResultBlock), + Error(BashCodeExecutionToolResultError), +} + +/// Bash code execution tool result error +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct BashCodeExecutionToolResultError { + #[serde(rename = "type")] + pub error_type: String, // "bash_code_execution_tool_result_error" + + pub error_code: BashCodeExecutionToolResultErrorCode, +} + +/// Bash code execution error codes +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BashCodeExecutionToolResultErrorCode { + Unavailable, + CodeExecutionExceededTimeout, + ContainerExpired, + InvalidToolInput, +} + +/// Text editor code execution tool result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextEditorCodeExecutionToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The result content + pub content: TextEditorCodeExecutionToolResultContent, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Text editor code execution result content +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum TextEditorCodeExecutionToolResultContent { + CreateResult(TextEditorCodeExecutionCreateResultBlock), + StrReplaceResult(TextEditorCodeExecutionStrReplaceResultBlock), + ViewResult(TextEditorCodeExecutionViewResultBlock), + Error(TextEditorCodeExecutionToolResultError), +} + +/// Text editor create result block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextEditorCodeExecutionCreateResultBlock { + #[serde(rename = "type")] + pub block_type: String, // "text_editor_code_execution_create_result" +} + +/// Text editor str_replace result block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextEditorCodeExecutionStrReplaceResultBlock { + #[serde(rename = "type")] + pub block_type: String, // "text_editor_code_execution_str_replace_result" + + /// Snippet of content around the replacement + #[serde(skip_serializing_if = "Option::is_none")] + pub snippet: Option, +} + +/// Text editor view result block +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextEditorCodeExecutionViewResultBlock { + #[serde(rename = "type")] + pub block_type: String, // "text_editor_code_execution_view_result" + + /// Content of the viewed file + pub content: String, +} + +/// Text editor code execution tool result error +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TextEditorCodeExecutionToolResultError { + #[serde(rename = "type")] + pub error_type: String, + + pub error_code: TextEditorCodeExecutionToolResultErrorCode, +} + +/// Text editor code execution error codes +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum TextEditorCodeExecutionToolResultErrorCode { + Unavailable, + InvalidToolInput, + FileNotFound, + ContainerExpired, +} + +// ============================================================================ +// Beta Features - Web Fetch Types +// ============================================================================ + +/// Web fetch tool (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebFetchTool { + #[serde(rename = "type")] + pub tool_type: String, // "web_fetch_20250305" or similar + + pub name: String, // "web_fetch" + + /// Allowed callers for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_callers: Option>, + + /// Maximum number of uses + #[serde(skip_serializing_if = "Option::is_none")] + pub max_uses: Option, + + /// Cache control for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Web fetch result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebFetchResultBlock { + #[serde(rename = "type")] + pub block_type: String, // "web_fetch_result" + + /// The URL that was fetched + pub url: String, + + /// The document content + pub content: DocumentBlock, + + /// When the content was retrieved + #[serde(skip_serializing_if = "Option::is_none")] + pub retrieved_at: Option, +} + +/// Web fetch tool result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebFetchToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The result content (success or error) + pub content: WebFetchToolResultContent, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Web fetch tool result content +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum WebFetchToolResultContent { + Success(WebFetchResultBlock), + Error(WebFetchToolResultError), +} + +/// Web fetch tool result error +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WebFetchToolResultError { + #[serde(rename = "type")] + pub error_type: String, // "web_fetch_tool_result_error" + + pub error_code: WebFetchToolResultErrorCode, +} + +/// Web fetch error codes +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WebFetchToolResultErrorCode { + InvalidToolInput, + Unavailable, + MaxUsesExceeded, + TooManyRequests, + UrlNotAllowed, + FetchFailed, + ContentTooLarge, +} + +// ============================================================================ +// Beta Features - Tool Search Types +// ============================================================================ + +/// Tool search tool (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolSearchTool { + #[serde(rename = "type")] + pub tool_type: String, // "tool_search_tool_regex" or "tool_search_tool_bm25" + + pub name: String, + + /// Allowed callers for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_callers: Option>, + + /// Cache control for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +/// Tool reference block (beta) - returned by tool search +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolReferenceBlock { + #[serde(rename = "type")] + pub block_type: String, // "tool_reference" + + /// Tool name + pub tool_name: String, + + /// Tool description + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, +} + +/// Tool search result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolSearchResultBlock { + #[serde(rename = "type")] + pub block_type: String, // "tool_search_tool_search_result" + + /// Tool name + pub tool_name: String, + + /// Relevance score + #[serde(skip_serializing_if = "Option::is_none")] + pub score: Option, +} + +/// Tool search tool result block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolSearchToolResultBlock { + /// The ID of the tool use this is a result for + pub tool_use_id: String, + + /// The search results + pub content: Vec, + + /// Cache control for this block + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +// ============================================================================ +// Beta Features - Container Upload Types +// ============================================================================ + +/// Container upload block (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ContainerUploadBlock { + #[serde(rename = "type")] + pub block_type: String, // "container_upload" + + /// File ID + pub file_id: String, + + /// File name + pub file_name: String, + + /// File path in container + #[serde(skip_serializing_if = "Option::is_none")] + pub file_path: Option, +} + +// ============================================================================ +// Beta Features - Memory Tool Types +// ============================================================================ + +/// Memory tool (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct MemoryTool { + #[serde(rename = "type")] + pub tool_type: String, // "memory_20250818" + + pub name: String, // "memory" + + /// Allowed callers for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_callers: Option>, + + /// Whether to defer loading + #[serde(skip_serializing_if = "Option::is_none")] + pub defer_loading: Option, + + /// Whether to use strict mode + #[serde(skip_serializing_if = "Option::is_none")] + pub strict: Option, + + /// Input examples + #[serde(skip_serializing_if = "Option::is_none")] + pub input_examples: Option>, + + /// Cache control for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +// ============================================================================ +// Beta Features - Computer Use Tool Types +// ============================================================================ + +/// Computer use tool (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ComputerUseTool { + #[serde(rename = "type")] + pub tool_type: String, // "computer_20241022" or "computer_20250124" + + pub name: String, // "computer" + + /// Display width + pub display_width_px: u32, + + /// Display height + pub display_height_px: u32, + + /// Display number (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub display_number: Option, + + /// Allowed callers for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_callers: Option>, + + /// Cache control for this tool + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_control: Option, +} + +// ============================================================================ +// Beta Features - Extended Input Content Block Enum +// ============================================================================ + +/// Beta input content block types (extends InputContentBlock) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BetaInputContentBlock { + // Standard types + Text(TextBlock), + Image(ImageBlock), + Document(DocumentBlock), + ToolUse(ToolUseBlock), + ToolResult(ToolResultBlock), + Thinking(ThinkingBlock), + RedactedThinking(RedactedThinkingBlock), + ServerToolUse(ServerToolUseBlock), + SearchResult(SearchResultBlock), + WebSearchToolResult(WebSearchToolResultBlock), + + // Beta MCP types + McpToolUse(McpToolUseBlock), + McpToolResult(McpToolResultBlock), + + // Beta code execution types + CodeExecutionToolResult(CodeExecutionToolResultBlock), + BashCodeExecutionToolResult(BashCodeExecutionToolResultBlock), + TextEditorCodeExecutionToolResult(TextEditorCodeExecutionToolResultBlock), + + // Beta web fetch types + WebFetchToolResult(WebFetchToolResultBlock), + + // Beta tool search types + ToolSearchToolResult(ToolSearchToolResultBlock), + ToolReference(ToolReferenceBlock), + + // Beta container types + ContainerUpload(ContainerUploadBlock), +} + +/// Beta output content block types (extends ContentBlock) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BetaContentBlock { + // Standard types + Text { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + citations: Option>, + }, + ToolUse { + id: String, + name: String, + input: Value, + }, + Thinking { + thinking: String, + signature: String, + }, + RedactedThinking { + data: String, + }, + ServerToolUse { + id: String, + name: String, + input: Value, + }, + WebSearchToolResult { + tool_use_id: String, + content: WebSearchToolResultContent, + }, + + // Beta MCP types + McpToolUse { + id: String, + name: String, + server_name: String, + input: Value, + }, + McpToolResult { + tool_use_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + content: Option, + #[serde(skip_serializing_if = "Option::is_none")] + is_error: Option, + }, + + // Beta code execution types + CodeExecutionToolResult { + tool_use_id: String, + content: CodeExecutionToolResultContent, + }, + BashCodeExecutionToolResult { + tool_use_id: String, + content: BashCodeExecutionToolResultContent, + }, + TextEditorCodeExecutionToolResult { + tool_use_id: String, + content: TextEditorCodeExecutionToolResultContent, + }, + + // Beta web fetch types + WebFetchToolResult { + tool_use_id: String, + content: WebFetchToolResultContent, + }, + + // Beta tool search types + ToolSearchToolResult { + tool_use_id: String, + content: Vec, + }, + ToolReference { + tool_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + }, + + // Beta container types + ContainerUpload { + file_id: String, + file_name: String, + #[serde(skip_serializing_if = "Option::is_none")] + file_path: Option, + }, +} + +/// Beta tool definition (extends Tool) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum BetaTool { + // Standard tools + Custom(CustomTool), + Bash(BashTool), + TextEditor(TextEditorTool), + WebSearch(WebSearchTool), + + // Beta tools + CodeExecution(CodeExecutionTool), + McpToolset(McpToolset), + WebFetch(WebFetchTool), + ToolSearch(ToolSearchTool), + Memory(MemoryTool), + ComputerUse(ComputerUseTool), +} + +/// Server tool names for beta features +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum BetaServerToolName { + WebSearch, + WebFetch, + CodeExecution, + BashCodeExecution, + TextEditorCodeExecution, + ToolSearchToolRegex, + ToolSearchToolBm25, +} + +/// Server tool caller types (beta) +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ServerToolCaller { + /// Direct caller (the model itself) + Direct, + /// Code execution caller + #[serde(rename = "code_execution_20250825")] + CodeExecution20250825, +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/parser.rs b/sgl-model-gateway/vendor/openai-protocol/src/parser.rs new file mode 100644 index 0000000000..3d1f03f746 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/parser.rs @@ -0,0 +1,23 @@ +use serde::Deserialize; + +use crate::common::Tool; + +/// Request to parse function calls from model output text +#[derive(Deserialize)] +pub struct ParseFunctionCallRequest { + /// The text to parse for function calls + pub text: String, + /// The parser type/name to use for parsing (e.g., "json", "pythonic") + pub tool_call_parser: String, + /// The list of available tools that the model can call + pub tools: Vec, +} + +/// Request to separate reasoning from normal text in model output +#[derive(Deserialize)] +pub struct SeparateReasoningRequest { + /// The text to parse for reasoning content + pub text: String, + /// The parser type/name to use for reasoning detection (e.g., "step3", "deepseek_r1") + pub reasoning_parser: String, +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/rerank.rs b/sgl-model-gateway/vendor/openai-protocol/src/rerank.rs new file mode 100644 index 0000000000..6775f5d8a4 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/rerank.rs @@ -0,0 +1,212 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::common::{default_model, default_true, GenerationRequest, StringOrArray, UsageInfo}; + +fn default_rerank_object() -> String { + "rerank".to_string() +} + +/// TODO: Create timestamp should not be in protocol layer +fn current_timestamp() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_else(|_| std::time::Duration::from_secs(0)) + .as_secs() as i64 +} + +// ============================================================================ +// Rerank API +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[validate(schema(function = "validate_rerank_request"))] +pub struct RerankRequest { + /// The query text to rank documents against + #[validate(custom(function = "validate_query"))] + pub query: String, + + /// List of documents to be ranked + #[validate(custom(function = "validate_documents"))] + pub documents: Vec, + + /// Model to use for reranking + #[serde(default = "default_model")] + pub model: String, + + /// Maximum number of documents to return (optional) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub top_k: Option, + + /// Whether to return documents in addition to scores + #[serde(default = "default_true")] + pub return_documents: bool, + + // SGLang specific extensions + /// Request ID for tracking + pub rid: Option, + + /// User identifier + pub user: Option, +} + +impl GenerationRequest for RerankRequest { + fn get_model(&self) -> Option<&str> { + Some(&self.model) + } + + fn is_stream(&self) -> bool { + false // Reranking doesn't support streaming + } + + fn extract_text_for_routing(&self) -> String { + self.query.clone() + } +} + +impl super::validated::Normalizable for RerankRequest { + // Use default no-op normalization +} + +// ============================================================================ +// Validation Functions +// ============================================================================ + +/// Validates that the query is not empty +fn validate_query(query: &str) -> Result<(), validator::ValidationError> { + if query.trim().is_empty() { + return Err(validator::ValidationError::new("query cannot be empty")); + } + Ok(()) +} + +/// Validates that the documents list is not empty +fn validate_documents(documents: &[String]) -> Result<(), validator::ValidationError> { + if documents.is_empty() { + return Err(validator::ValidationError::new( + "documents list cannot be empty", + )); + } + Ok(()) +} + +/// Schema-level validation for cross-field dependencies +fn validate_rerank_request(req: &RerankRequest) -> Result<(), validator::ValidationError> { + // Validate top_k if specified + if let Some(k) = req.top_k { + if k > req.documents.len() { + // This is allowed but we log a warning + tracing::warn!( + "top_k ({}) is greater than number of documents ({})", + k, + req.documents.len() + ); + } + } + Ok(()) +} + +impl RerankRequest { + /// Get the effective top_k value + pub fn effective_top_k(&self) -> usize { + self.top_k.unwrap_or(self.documents.len()) + } +} + +/// Individual rerank result +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResult { + /// Relevance score for the document + pub score: f32, + + /// The document text (if return_documents was true) + #[serde(skip_serializing_if = "Option::is_none")] + pub document: Option, + + /// Original index of the document in the request + pub index: usize, + + /// Additional metadata about the ranking + #[serde(skip_serializing_if = "Option::is_none")] + pub meta_info: Option>, +} + +/// Rerank response containing sorted results +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RerankResponse { + /// Ranked results sorted by score (highest first) + pub results: Vec, + + /// Model used for reranking + pub model: String, + + /// Usage information + pub usage: Option, + + /// Response object type + #[serde(default = "default_rerank_object")] + pub object: String, + + /// Response ID + pub id: Option, + + /// Creation timestamp + pub created: i64, +} + +impl RerankResponse { + /// Create a new RerankResponse with the given results and model + pub fn new( + results: Vec, + model: String, + request_id: Option, + ) -> Self { + RerankResponse { + results, + model, + usage: None, + object: default_rerank_object(), + id: request_id, + created: current_timestamp(), + } + } + + /// Apply top_k limit to results + pub fn apply_top_k(&mut self, k: usize) { + self.results.truncate(k); + } + + /// Drop documents from results (when return_documents is false) + pub fn drop_documents(&mut self) { + for result in &mut self.results { + result.document = None; + } + } +} + +/// V1 API compatibility format for rerank requests +/// Matches Python's V1RerankReqInput +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct V1RerankReqInput { + pub query: String, + pub documents: Vec, +} + +/// Convert V1RerankReqInput to RerankRequest +impl From for RerankRequest { + fn from(v1: V1RerankReqInput) -> Self { + RerankRequest { + query: v1.query, + documents: v1.documents, + model: default_model(), + top_k: None, + return_documents: true, + rid: None, + user: None, + } + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/responses.rs b/sgl-model-gateway/vendor/openai-protocol/src/responses.rs new file mode 100644 index 0000000000..77221f158a --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/responses.rs @@ -0,0 +1,1329 @@ +// OpenAI Responses API types +// https://platform.openai.com/docs/api-reference/responses + +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use validator::Validate; + +use super::{ + common::{ + default_model, default_true, validate_stop, ChatLogProbs, Function, GenerationRequest, + PromptTokenUsageInfo, StringOrArray, ToolChoice, ToolChoiceValue, ToolReference, UsageInfo, + }, + sampling_params::{validate_top_k_value, validate_top_p_value}, +}; +use crate::{builders::ResponsesResponseBuilder, validated::Normalizable}; + +/// Responses API 允许扁平写法 {"type":"function","name":"foo"}, +/// 而 Chat 侧是嵌套 {"type":"function","function":{"name":"foo"}}。 +/// 反序列化时统一成嵌套形态,避免网关 400。 +fn deserialize_tool_choice_compat<'de, D>(deserializer: D) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + let Some(mut value) = value else { + return Ok(None); + }; + if let Value::Object(ref mut map) = value { + if map.get("type").and_then(Value::as_str) == Some("function") + && map.contains_key("name") + && !map.contains_key("function") + { + let name = map.remove("name").unwrap_or(Value::Null); + map.insert("function".to_string(), serde_json::json!({ "name": name })); + } + } + serde_json::from_value::(value) + .map(Some) + .map_err(|e| ::custom(e)) +} + + +// ============================================================================ +// Response Tools (MCP and others) +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseTool { + #[serde(rename = "type")] + pub r#type: ResponseToolType, + // Function tool fields (used when type == "function") + // In Responses API, function fields are flattened at the top level + #[serde(flatten)] + #[serde(skip_serializing_if = "Option::is_none")] + pub function: Option, + // MCP-specific fields (used when type == "mcp") + #[serde(skip_serializing_if = "Option::is_none")] + pub server_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub authorization: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_label: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub server_description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub require_approval: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub allowed_tools: Option>, +} + +impl Default for ResponseTool { + fn default() -> Self { + Self { + r#type: ResponseToolType::WebSearchPreview, + function: None, + server_url: None, + authorization: None, + server_label: None, + server_description: None, + require_approval: None, + allowed_tools: None, + } + } +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ResponseToolType { + Function, + WebSearchPreview, + CodeInterpreter, + Mcp, +} + +// ============================================================================ +// Reasoning Parameters +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseReasoningParam { + #[serde(default = "default_reasoning_effort")] + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +fn default_reasoning_effort() -> Option { + Some(ReasoningEffort::Medium) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningEffort { + None, + Minimal, + Low, + Medium, + High, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ReasoningSummary { + Auto, + Concise, + Detailed, +} + +// ============================================================================ +// Input/Output Items +// ============================================================================ + +/// Content can be either a simple string or array of content parts (for SimpleInputMessage) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum StringOrContentParts { + String(String), + Array(Vec), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseInputOutputItem { + #[serde(rename = "message")] + Message { + id: String, + role: String, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "reasoning")] + Reasoning { + id: String, + summary: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_call")] + FunctionToolCall { + id: String, + call_id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_call_output")] + FunctionCallOutput { + id: Option, + call_id: String, + output: String, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(untagged)] + SimpleInputMessage { + content: StringOrContentParts, + role: String, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "type")] + r#type: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseContentPart { + #[serde(rename = "output_text")] + OutputText { + text: String, + #[serde(default)] + #[serde(skip_serializing_if = "Vec::is_empty")] + annotations: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + logprobs: Option, + }, + #[serde(rename = "input_text")] + InputText { text: String }, + #[serde(other)] + Unknown, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseReasoningContent { + #[serde(rename = "reasoning_text")] + ReasoningText { text: String }, +} + +/// MCP Tool information for the mcp_list_tools output item +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct McpToolInfo { + pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub description: Option, + pub input_schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + pub annotations: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +#[serde(rename_all = "snake_case")] +pub enum ResponseOutputItem { + #[serde(rename = "message")] + Message { + id: String, + role: String, + content: Vec, + status: String, + }, + #[serde(rename = "reasoning")] + Reasoning { + id: String, + summary: Vec, + content: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + status: Option, + }, + #[serde(rename = "function_call")] + FunctionToolCall { + id: String, + call_id: String, + name: String, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + output: Option, + status: String, + }, + #[serde(rename = "mcp_list_tools")] + McpListTools { + id: String, + server_label: String, + tools: Vec, + }, + #[serde(rename = "mcp_call")] + McpCall { + id: String, + status: String, + #[serde(skip_serializing_if = "Option::is_none")] + approval_request_id: Option, + arguments: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + name: String, + output: String, + server_label: String, + }, +} + +// ============================================================================ +// Configuration Enums +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum ServiceTier { + #[default] + Auto, + Default, + Flex, + Scale, + Priority, +} + +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum Truncation { + Auto, + #[default] + Disabled, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResponseStatus { + Queued, + InProgress, + Completed, + Failed, + Cancelled, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ReasoningInfo { + #[serde(skip_serializing_if = "Option::is_none")] + pub effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub summary: Option, +} + +// ============================================================================ +// Text Format (structured outputs) +// ============================================================================ + +/// Text configuration for structured output requests +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TextConfig { + #[serde(skip_serializing_if = "Option::is_none")] + pub format: Option, +} + +/// Text format: text (default), json_object (legacy), or json_schema (recommended) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(tag = "type")] +pub enum TextFormat { + #[serde(rename = "text")] + Text, + + #[serde(rename = "json_object")] + JsonObject, + + #[serde(rename = "json_schema")] + JsonSchema { + name: String, + schema: Value, + #[serde(skip_serializing_if = "Option::is_none")] + description: Option, + #[serde(skip_serializing_if = "Option::is_none")] + strict: Option, + }, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum IncludeField { + #[serde(rename = "code_interpreter_call.outputs")] + CodeInterpreterCallOutputs, + #[serde(rename = "computer_call_output.output.image_url")] + ComputerCallOutputImageUrl, + #[serde(rename = "file_search_call.results")] + FileSearchCallResults, + #[serde(rename = "message.input_image.image_url")] + MessageInputImageUrl, + #[serde(rename = "message.output_text.logprobs")] + MessageOutputTextLogprobs, + #[serde(rename = "reasoning.encrypted_content")] + ReasoningEncryptedContent, +} + +// ============================================================================ +// Usage Types (Responses API format) +// ============================================================================ + +/// OpenAI Responses API usage format (different from standard UsageInfo) +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponseUsage { + pub input_tokens: u32, + pub output_tokens: u32, + pub total_tokens: u32, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens_details: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_tokens_details: Option, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ResponsesUsage { + Classic(UsageInfo), + Modern(ResponseUsage), +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct InputTokensDetails { + pub cached_tokens: u32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct OutputTokensDetails { + pub reasoning_tokens: u32, +} + +impl UsageInfo { + /// Convert to OpenAI Responses API format + pub fn to_response_usage(&self) -> ResponseUsage { + ResponseUsage { + input_tokens: self.prompt_tokens, + output_tokens: self.completion_tokens, + total_tokens: self.total_tokens, + input_tokens_details: self.prompt_tokens_details.as_ref().map(|details| { + InputTokensDetails { + cached_tokens: details.cached_tokens, + } + }), + output_tokens_details: self.reasoning_tokens.map(|tokens| OutputTokensDetails { + reasoning_tokens: tokens, + }), + } + } +} + +impl From for ResponseUsage { + fn from(usage: UsageInfo) -> Self { + usage.to_response_usage() + } +} + +impl ResponseUsage { + /// Convert back to standard UsageInfo format + pub fn to_usage_info(&self) -> UsageInfo { + UsageInfo { + prompt_tokens: self.input_tokens, + completion_tokens: self.output_tokens, + total_tokens: self.total_tokens, + reasoning_tokens: self + .output_tokens_details + .as_ref() + .map(|details| details.reasoning_tokens), + prompt_tokens_details: self.input_tokens_details.as_ref().map(|details| { + PromptTokenUsageInfo { + cached_tokens: details.cached_tokens, + } + }), + } + } +} + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub struct ResponsesGetParams { + #[serde(default)] + pub include: Vec, + #[serde(default)] + pub include_obfuscation: Option, + #[serde(default)] + pub starting_after: Option, + #[serde(default)] + pub stream: Option, +} + +impl ResponsesUsage { + pub fn to_response_usage(&self) -> ResponseUsage { + match self { + ResponsesUsage::Classic(usage) => usage.to_response_usage(), + ResponsesUsage::Modern(usage) => usage.clone(), + } + } + + pub fn to_usage_info(&self) -> UsageInfo { + match self { + ResponsesUsage::Classic(usage) => usage.clone(), + ResponsesUsage::Modern(usage) => usage.to_usage_info(), + } + } +} + +// ============================================================================ +// Helper Functions for Defaults +// ============================================================================ + +fn default_top_k() -> i32 { + -1 +} + +fn default_repetition_penalty() -> f32 { + 1.0 +} + +fn default_temperature() -> Option { + Some(1.0) +} + +fn default_top_p() -> Option { + Some(1.0) +} + +// ============================================================================ +// Request/Response Types +// ============================================================================ + +#[derive(Debug, Clone, Deserialize, Serialize, Validate)] +#[validate(schema(function = "validate_responses_cross_parameters"))] +pub struct ResponsesRequest { + /// Run the request in the background + #[serde(skip_serializing_if = "Option::is_none")] + pub background: Option, + + /// Fields to include in the response + #[serde(skip_serializing_if = "Option::is_none")] + pub include: Option>, + + /// Input content - can be string or structured items + #[validate(custom(function = "validate_response_input"))] + pub input: ResponseInput, + + /// System instructions for the model + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option, + + /// Maximum number of output tokens + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub max_output_tokens: Option, + + /// Maximum number of tool calls + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 1))] + pub max_tool_calls: Option, + + /// Additional metadata + #[serde(skip_serializing_if = "Option::is_none")] + pub metadata: Option>, + + /// Model to use + #[serde(default = "default_model")] + pub model: String, + + /// Optional conversation id to persist input/output as items + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_conversation_id"))] + pub conversation: Option, + + /// Whether to enable parallel tool calls + #[serde(skip_serializing_if = "Option::is_none")] + pub parallel_tool_calls: Option, + + /// ID of previous response to continue from + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, + + /// Reasoning configuration + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + + /// Service tier + #[serde(skip_serializing_if = "Option::is_none")] + pub service_tier: Option, + + /// Whether to store the response + #[serde(skip_serializing_if = "Option::is_none")] + pub store: Option, + + /// Whether to stream the response + #[serde(default)] + pub stream: Option, + + /// Temperature for sampling + #[serde( + default = "default_temperature", + skip_serializing_if = "Option::is_none" + )] + #[validate(range(min = 0.0, max = 2.0))] + pub temperature: Option, + + /// Tool choice behavior + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "deserialize_tool_choice_compat" + )] + pub tool_choice: Option, + + /// Available tools + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_response_tools"))] + pub tools: Option>, + + /// Number of top logprobs to return + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0, max = 20))] + pub top_logprobs: Option, + + /// Top-p sampling parameter + #[serde(default = "default_top_p", skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, + + /// Truncation behavior + #[serde(skip_serializing_if = "Option::is_none")] + pub truncation: Option, + + /// Text format for structured outputs (text, json_object, json_schema) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_text_format"))] + pub text: Option, + + /// User identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Request ID + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, + + /// Request priority + #[serde(default)] + pub priority: i32, + + /// Frequency penalty + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, + + /// Presence penalty + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + + /// Stop sequences + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_stop"))] + pub stop: Option, + + /// Top-k sampling parameter (SGLang extension) + #[serde(default = "default_top_k")] + #[validate(custom(function = "validate_top_k_value"))] + pub top_k: i32, + + /// Min-p sampling parameter (SGLang extension) + #[serde(default)] + #[validate(range(min = 0.0, max = 1.0))] + pub min_p: f32, + + /// Repetition penalty (SGLang extension) + #[serde(default = "default_repetition_penalty")] + #[validate(range(min = 0.0, max = 2.0))] + pub repetition_penalty: f32, +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum ResponseInput { + Items(Vec), + Text(String), +} + +impl Default for ResponsesRequest { + fn default() -> Self { + Self { + background: None, + include: None, + input: ResponseInput::Text(String::new()), + instructions: None, + max_output_tokens: None, + max_tool_calls: None, + metadata: None, + model: default_model(), + conversation: None, + parallel_tool_calls: None, + previous_response_id: None, + reasoning: None, + service_tier: None, + store: None, + stream: None, + temperature: None, + tool_choice: None, + tools: None, + top_logprobs: None, + top_p: None, + truncation: None, + text: None, + user: None, + request_id: None, + priority: 0, + frequency_penalty: None, + presence_penalty: None, + stop: None, + top_k: default_top_k(), + min_p: 0.0, + repetition_penalty: default_repetition_penalty(), + } + } +} + +impl Normalizable for ResponsesRequest { + /// Normalize the request by applying defaults: + /// 1. Apply tool_choice defaults based on tools presence + /// 2. Apply parallel_tool_calls defaults + /// 3. Apply store field defaults + fn normalize(&mut self) { + // 1. Apply tool_choice defaults + if self.tool_choice.is_none() { + if let Some(tools) = &self.tools { + let choice_value = if !tools.is_empty() { + ToolChoiceValue::Auto + } else { + ToolChoiceValue::None + }; + self.tool_choice = Some(ToolChoice::Value(choice_value)); + } + // If tools is None, leave tool_choice as None (don't set it) + } + + // 2. Apply default for parallel_tool_calls if tools are present + if self.parallel_tool_calls.is_none() && self.tools.is_some() { + self.parallel_tool_calls = Some(true); + } + + // 3. Ensure store defaults to true if not specified + if self.store.is_none() { + self.store = Some(true); + } + } +} + +impl GenerationRequest for ResponsesRequest { + fn is_stream(&self) -> bool { + self.stream.unwrap_or(false) + } + + fn get_model(&self) -> Option<&str> { + Some(self.model.as_str()) + } + + fn extract_text_for_routing(&self) -> String { + match &self.input { + ResponseInput::Text(text) => text.clone(), + ResponseInput::Items(items) => items + .iter() + .filter_map(|item| match item { + ResponseInputOutputItem::Message { content, .. } => { + let texts: Vec = content + .iter() + .filter_map(|part| match part { + ResponseContentPart::OutputText { text, .. } => Some(text.clone()), + ResponseContentPart::InputText { text } => Some(text.clone()), + ResponseContentPart::Unknown => None, + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + ResponseInputOutputItem::SimpleInputMessage { content, .. } => { + match content { + StringOrContentParts::String(s) => Some(s.clone()), + StringOrContentParts::Array(parts) => { + // SimpleInputMessage only supports InputText + let texts: Vec = parts + .iter() + .filter_map(|part| match part { + ResponseContentPart::InputText { text } => { + Some(text.clone()) + } + _ => None, + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + } + } + ResponseInputOutputItem::Reasoning { content, .. } => { + let texts: Vec = content + .iter() + .map(|part| match part { + ResponseReasoningContent::ReasoningText { text } => text.clone(), + }) + .collect(); + if texts.is_empty() { + None + } else { + Some(texts.join(" ")) + } + } + ResponseInputOutputItem::FunctionToolCall { arguments, .. } => { + Some(arguments.clone()) + } + ResponseInputOutputItem::FunctionCallOutput { output, .. } => { + Some(output.clone()) + } + }) + .collect::>() + .join(" "), + } + } +} + +/// Validate conversation ID format +pub fn validate_conversation_id(conv_id: &str) -> Result<(), validator::ValidationError> { + if !conv_id.starts_with("conv_") { + let mut error = validator::ValidationError::new("invalid_conversation_id"); + error.message = Some(std::borrow::Cow::Owned(format!( + "Invalid 'conversation': '{}'. Expected an ID that begins with 'conv_'.", + conv_id + ))); + return Err(error); + } + + // Check if the conversation ID contains only valid characters + let is_valid = conv_id + .chars() + .all(|c| c.is_alphanumeric() || c == '_' || c == '-'); + + if !is_valid { + let mut error = validator::ValidationError::new("invalid_conversation_id"); + error.message = Some(std::borrow::Cow::Owned(format!( + "Invalid 'conversation': '{}'. Expected an ID that contains letters, numbers, underscores, or dashes, but this value contained additional characters.", + conv_id + ))); + return Err(error); + } + Ok(()) +} + +/// Validates tool_choice requires tools and references exist +fn validate_tool_choice_with_tools( + request: &ResponsesRequest, +) -> Result<(), validator::ValidationError> { + let Some(tool_choice) = &request.tool_choice else { + return Ok(()); + }; + + let has_tools = request.tools.as_ref().is_some_and(|t| !t.is_empty()); + let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None)); + + // Check if tool_choice requires tools but none are provided + if is_some_choice && !has_tools { + let mut e = validator::ValidationError::new("tool_choice_requires_tools"); + e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into()); + return Err(e); + } + + // Validate tool references exist when tools are present + if !has_tools { + return Ok(()); + } + + // Extract function tool names from ResponseTools + let tools = request.tools.as_ref().unwrap(); + let function_tool_names: Vec<&str> = tools + .iter() + .filter_map(|t| match t.r#type { + ResponseToolType::Function => t.function.as_ref().map(|f| f.name.as_str()), + _ => None, + }) + .collect(); + + // Validate tool references exist + match tool_choice { + ToolChoice::Function { function, .. } => { + if !function_tool_names.contains(&function.name.as_str()) { + let mut e = validator::ValidationError::new("tool_choice_function_not_found"); + e.message = Some( + format!( + "Invalid value for 'tool_choice': function '{}' not found in 'tools'.", + function.name + ) + .into(), + ); + return Err(e); + } + } + ToolChoice::AllowedTools { + mode, + tools: allowed_tools, + .. + } => { + // Validate mode is "auto" or "required" + if mode != "auto" && mode != "required" { + let mut e = validator::ValidationError::new("tool_choice_invalid_mode"); + e.message = Some( + format!( + "Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.", + mode + ) + .into(), + ); + return Err(e); + } + + // Validate that all function tool references exist + for tool_ref in allowed_tools { + if let ToolReference::Function { name } = tool_ref { + if !function_tool_names.contains(&name.as_str()) { + let mut e = validator::ValidationError::new("tool_choice_tool_not_found"); + e.message = Some( + format!( + "Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.", + name + ) + .into(), + ); + return Err(e); + } + } + // Note: MCP and hosted tools don't need existence validation here + // as they are resolved dynamically at runtime + } + } + _ => {} + } + + Ok(()) +} + +/// Schema-level validation for cross-field dependencies +fn validate_responses_cross_parameters( + request: &ResponsesRequest, +) -> Result<(), validator::ValidationError> { + // 1. Validate tool_choice requires tools (enhanced) + validate_tool_choice_with_tools(request)?; + + // 2. Validate top_logprobs requires include field + if request.top_logprobs.is_some() { + let has_logprobs_include = request + .include + .as_ref() + .is_some_and(|inc| inc.contains(&IncludeField::MessageOutputTextLogprobs)); + + if !has_logprobs_include { + let mut e = validator::ValidationError::new("top_logprobs_requires_include"); + e.message = Some( + "top_logprobs requires include field with 'message.output_text.logprobs'".into(), + ); + return Err(e); + } + } + + // 3. Validate background/stream conflict + if request.background == Some(true) && request.stream == Some(true) { + let mut e = validator::ValidationError::new("background_conflicts_with_stream"); + e.message = Some("Cannot use background mode with streaming".into()); + return Err(e); + } + + // 4. Validate conversation and previous_response_id are mutually exclusive + if request.conversation.is_some() && request.previous_response_id.is_some() { + let mut e = validator::ValidationError::new("mutually_exclusive_parameters"); + e.message = Some("Mutually exclusive parameters. Ensure you are only providing one of: 'previous_response_id' or 'conversation'.".into()); + return Err(e); + } + + // 5. Validate input items structure + if let ResponseInput::Items(items) = &request.input { + // Check for at least one valid input message + let has_valid_input = items.iter().any(|item| { + matches!( + item, + ResponseInputOutputItem::Message { .. } + | ResponseInputOutputItem::SimpleInputMessage { .. } + ) + }); + + if !has_valid_input { + let mut e = validator::ValidationError::new("input_missing_user_message"); + e.message = Some("Input items must contain at least one message".into()); + return Err(e); + } + } + + // 6. Validate text format conflicts (for future structured output constraints) + // Currently, Responses API doesn't have regex/ebnf like Chat API, + // but this is here for completeness and future-proofing + + Ok(()) +} + +// ============================================================================ +// Field-Level Validation Functions +// ============================================================================ + +/// Validates response input is not empty and has valid content +fn validate_response_input(input: &ResponseInput) -> Result<(), validator::ValidationError> { + match input { + ResponseInput::Text(text) => { + if text.is_empty() { + let mut e = validator::ValidationError::new("input_text_empty"); + e.message = Some("Input text cannot be empty".into()); + return Err(e); + } + } + ResponseInput::Items(items) => { + if items.is_empty() { + let mut e = validator::ValidationError::new("input_items_empty"); + e.message = Some("Input items cannot be empty".into()); + return Err(e); + } + // Validate each item has valid content + for item in items { + validate_input_item(item)?; + } + } + } + Ok(()) +} + +/// Validates individual input items have valid content +fn validate_input_item(item: &ResponseInputOutputItem) -> Result<(), validator::ValidationError> { + match item { + ResponseInputOutputItem::Message { content, .. } => { + if content.is_empty() { + let mut e = validator::ValidationError::new("message_content_empty"); + e.message = Some("Message content cannot be empty".into()); + return Err(e); + } + } + ResponseInputOutputItem::SimpleInputMessage { content, .. } => match content { + StringOrContentParts::String(s) if s.is_empty() => { + let mut e = validator::ValidationError::new("message_content_empty"); + e.message = Some("Message content cannot be empty".into()); + return Err(e); + } + StringOrContentParts::Array(parts) if parts.is_empty() => { + let mut e = validator::ValidationError::new("message_content_empty"); + e.message = Some("Message content parts cannot be empty".into()); + return Err(e); + } + _ => {} + }, + ResponseInputOutputItem::Reasoning { .. } => { + // Reasoning content can be empty - no validation needed + } + ResponseInputOutputItem::FunctionCallOutput { output, .. } => { + if output.is_empty() { + let mut e = validator::ValidationError::new("function_output_empty"); + e.message = Some("Function call output cannot be empty".into()); + return Err(e); + } + } + _ => {} + } + Ok(()) +} + +/// Validates ResponseTool structure based on tool type +fn validate_response_tools(tools: &[ResponseTool]) -> Result<(), validator::ValidationError> { + for tool in tools { + match tool.r#type { + ResponseToolType::Function => { + if tool.function.is_none() { + let mut e = validator::ValidationError::new("function_tool_missing_function"); + e.message = Some("Function tool must have a function definition".into()); + return Err(e); + } + } + ResponseToolType::Mcp => { + if tool.server_url.is_none() { + let mut e = validator::ValidationError::new("mcp_tool_missing_server_url"); + e.message = Some("MCP tool must have a server_url".into()); + return Err(e); + } + } + _ => {} + } + } + Ok(()) +} + +/// Validates text format configuration (JSON schema name cannot be empty) +fn validate_text_format(text: &TextConfig) -> Result<(), validator::ValidationError> { + if let Some(TextFormat::JsonSchema { name, .. }) = &text.format { + if name.is_empty() { + let mut e = validator::ValidationError::new("json_schema_name_empty"); + e.message = Some("JSON schema name cannot be empty".into()); + return Err(e); + } + } + Ok(()) +} + +/// Normalize a SimpleInputMessage to a proper Message item +/// +/// This helper converts SimpleInputMessage (which can have flexible content) +/// into a fully-structured Message item with a generated ID, role, and content array. +/// +/// SimpleInputMessage items are converted to Message items with IDs generated using +/// the centralized ID generation pattern with "msg_" prefix for consistency. +/// +/// # Arguments +/// * `item` - The input item to normalize +/// +/// # Returns +/// A normalized ResponseInputOutputItem (either Message if converted, or original if not SimpleInputMessage) +pub fn normalize_input_item(item: &ResponseInputOutputItem) -> ResponseInputOutputItem { + match item { + ResponseInputOutputItem::SimpleInputMessage { content, role, .. } => { + let content_vec = match content { + StringOrContentParts::String(s) => { + vec![ResponseContentPart::InputText { text: s.clone() }] + } + StringOrContentParts::Array(parts) => parts.clone(), + }; + + ResponseInputOutputItem::Message { + id: generate_id("msg"), + role: role.clone(), + content: content_vec, + status: Some("completed".to_string()), + } + } + _ => item.clone(), + } +} + +pub fn generate_id(prefix: &str) -> String { + use rand::RngCore; + let mut rng = rand::rng(); + // Generate exactly 50 hex characters (25 bytes) for the part after the underscore + let mut bytes = [0u8; 25]; + rng.fill_bytes(&mut bytes); + let hex_string: String = bytes.iter().map(|b| format!("{:02x}", b)).collect(); + format!("{}_{}", prefix, hex_string) +} + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ResponsesResponse { + /// Response ID + pub id: String, + + /// Object type + #[serde(default = "default_object_type")] + pub object: String, + + /// Creation timestamp + pub created_at: i64, + + /// Response status + pub status: ResponseStatus, + + /// Error information if status is failed + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + + /// Incomplete details if response was truncated + #[serde(skip_serializing_if = "Option::is_none")] + pub incomplete_details: Option, + + /// System instructions used + #[serde(skip_serializing_if = "Option::is_none")] + pub instructions: Option, + + /// Max output tokens setting + #[serde(skip_serializing_if = "Option::is_none")] + pub max_output_tokens: Option, + + /// Model name + pub model: String, + + /// Output items + #[serde(default)] + pub output: Vec, + + /// Whether parallel tool calls are enabled + #[serde(default = "default_true")] + pub parallel_tool_calls: bool, + + /// Previous response ID if this is a continuation + #[serde(skip_serializing_if = "Option::is_none")] + pub previous_response_id: Option, + + /// Reasoning information + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning: Option, + + /// Whether the response is stored + #[serde(default = "default_true")] + pub store: bool, + + /// Temperature setting used + #[serde(skip_serializing_if = "Option::is_none")] + pub temperature: Option, + + /// Text format settings + #[serde(skip_serializing_if = "Option::is_none")] + pub text: Option, + + /// Tool choice setting + #[serde(default = "default_tool_choice")] + pub tool_choice: String, + + /// Available tools + #[serde(default)] + pub tools: Vec, + + /// Top-p setting used + #[serde(skip_serializing_if = "Option::is_none")] + pub top_p: Option, + + /// Truncation strategy used + #[serde(skip_serializing_if = "Option::is_none")] + pub truncation: Option, + + /// Usage information + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + + /// User identifier + #[serde(skip_serializing_if = "Option::is_none")] + pub user: Option, + + /// Safety identifier for content moderation + #[serde(skip_serializing_if = "Option::is_none")] + pub safety_identifier: Option, + + /// Additional metadata + #[serde(default)] + pub metadata: HashMap, +} + +fn default_object_type() -> String { + "response".to_string() +} + +fn default_tool_choice() -> String { + "auto".to_string() +} + +impl ResponsesResponse { + /// Create a builder for constructing a ResponsesResponse + pub fn builder(id: impl Into, model: impl Into) -> ResponsesResponseBuilder { + ResponsesResponseBuilder::new(id, model) + } + + /// Check if the response is complete + pub fn is_complete(&self) -> bool { + matches!(self.status, ResponseStatus::Completed) + } + + /// Check if the response is in progress + pub fn is_in_progress(&self) -> bool { + matches!(self.status, ResponseStatus::InProgress) + } + + /// Check if the response failed + pub fn is_failed(&self) -> bool { + matches!(self.status, ResponseStatus::Failed) + } +} + +impl ResponseOutputItem { + /// Create a new message output item + pub fn new_message( + id: String, + role: String, + content: Vec, + status: String, + ) -> Self { + Self::Message { + id, + role, + content, + status, + } + } + + /// Create a new reasoning output item + pub fn new_reasoning( + id: String, + summary: Vec, + content: Vec, + status: Option, + ) -> Self { + Self::Reasoning { + id, + summary, + content, + status, + } + } + + /// Create a new function tool call output item + pub fn new_function_tool_call( + id: String, + call_id: String, + name: String, + arguments: String, + output: Option, + status: String, + ) -> Self { + Self::FunctionToolCall { + id, + call_id, + name, + arguments, + output, + status, + } + } +} + +impl ResponseContentPart { + /// Create a new text content part + pub fn new_text( + text: String, + annotations: Vec, + logprobs: Option, + ) -> Self { + Self::OutputText { + text, + annotations, + logprobs, + } + } +} + +impl ResponseReasoningContent { + /// Create a new reasoning text content + pub fn new_reasoning_text(text: String) -> Self { + Self::ReasoningText { text } + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/sampling_params.rs b/sgl-model-gateway/vendor/openai-protocol/src/sampling_params.rs new file mode 100644 index 0000000000..9055a53dd3 --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/sampling_params.rs @@ -0,0 +1,119 @@ +use serde::{Deserialize, Serialize}; +use validator::Validate; + +use super::common::StringOrArray; + +/// Sampling parameters for text generation +#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)] +#[validate(schema(function = "validate_sampling_params"))] +pub struct SamplingParams { + /// Temperature for sampling (must be >= 0.0, no upper limit) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0))] + pub temperature: Option, + /// Maximum number of new tokens to generate (must be >= 0) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0))] + pub max_new_tokens: Option, + /// Top-p nucleus sampling (0.0 < top_p <= 1.0) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_p_value"))] + pub top_p: Option, + /// Top-k sampling (-1 to disable, or >= 1) + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(custom(function = "validate_top_k_value"))] + pub top_k: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub frequency_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = -2.0, max = 2.0))] + pub presence_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 2.0))] + pub repetition_penalty: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ignore_eos: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub skip_special_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub json_schema: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub regex: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ebnf: Option, + #[serde(skip_serializing_if = "Option::is_none")] + #[validate(range(min = 0.0, max = 1.0))] + pub min_p: Option, + /// Minimum number of new tokens (validated in schema function for cross-field check with max_new_tokens) + #[serde(skip_serializing_if = "Option::is_none")] + pub min_new_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub stop_token_ids: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub no_stop_trim: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub n: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub sampling_seed: Option, +} + +// ============================================================================ +// Shared Validation Functions +// ============================================================================ + +/// Validates top_p: 0.0 < top_p <= 1.0 (can't use range validator for open interval) +pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> { + if !(top_p > 0.0 && top_p <= 1.0) { + return Err(validator::ValidationError::new( + "top_p must be in (0, 1] - greater than 0.0 and at most 1.0", + )); + } + Ok(()) +} + +/// Validates top_k: -1 (disabled) or >= 1 (special -1 case - can't use range validator) +pub fn validate_top_k_value(top_k: i32) -> Result<(), validator::ValidationError> { + if top_k != -1 && top_k < 1 { + return Err(validator::ValidationError::new( + "top_k must be -1 (disabled) or at least 1", + )); + } + Ok(()) +} + +// ============================================================================ +// SamplingParams-Specific Validation +// ============================================================================ + +/// Validation function for SamplingParams - cross-field validation only +fn validate_sampling_params(params: &SamplingParams) -> Result<(), validator::ValidationError> { + // 1. Cross-field validation: min_new_tokens <= max_new_tokens + if let (Some(min), Some(max)) = (params.min_new_tokens, params.max_new_tokens) { + if min > max { + return Err(validator::ValidationError::new( + "min_new_tokens cannot exceed max_new_tokens", + )); + } + } + + // 2. Validate mutually exclusive structured output constraints + let constraint_count = [ + params.regex.is_some(), + params.ebnf.is_some(), + params.json_schema.is_some(), + ] + .iter() + .filter(|&&x| x) + .count(); + + if constraint_count > 1 { + return Err(validator::ValidationError::new( + "only one of regex, ebnf, or json_schema can be set", + )); + } + + Ok(()) +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/tokenize.rs b/sgl-model-gateway/vendor/openai-protocol/src/tokenize.rs new file mode 100644 index 0000000000..62a19c23bd --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/tokenize.rs @@ -0,0 +1,279 @@ +//! Tokenize and Detokenize API protocol types +//! +//! These types mirror the SGLang Python implementation for compatibility. +//! See: python/sglang/srt/entrypoints/openai/protocol.py + +use serde::{Deserialize, Serialize}; + +use super::UNKNOWN_MODEL_ID; + +// ============================================================================ +// Tokenize API +// ============================================================================ + +/// Request schema for the /v1/tokenize endpoint +/// +/// Supports both single string and batch tokenization. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct TokenizeRequest { + /// Model name for tokenizer selection + #[serde(default = "default_model_name")] + pub model: String, + + /// Text(s) to tokenize - can be a single string or array of strings + pub prompt: StringOrArray, +} + +/// Response schema for the /v1/tokenize endpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TokenizeResponse { + /// Token IDs - single list for single input, nested list for batch + pub tokens: TokensResult, + + /// Token count(s) - single int for single input, list for batch + pub count: CountResult, + + /// Character count(s) of input - single int for single input, list for batch + pub char_count: CountResult, +} + +/// Token IDs result - either single or batch +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum TokensResult { + Single(Vec), + Batch(Vec>), +} + +/// Count result - either single or batch +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum CountResult { + Single(i32), + Batch(Vec), +} + +// ============================================================================ +// Detokenize API +// ============================================================================ + +/// Request schema for the /v1/detokenize endpoint +/// +/// Supports both single sequence and batch detokenization. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct DetokenizeRequest { + /// Model name for tokenizer selection + #[serde(default = "default_model_name")] + pub model: String, + + /// Token IDs to detokenize - single list or batch (list of lists) + pub tokens: TokensInput, + + /// Whether to skip special tokens (e.g., padding or EOS) during decoding + #[serde(default = "default_true")] + pub skip_special_tokens: bool, +} + +/// Token input - either single sequence or batch +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum TokensInput { + /// Single sequence of token IDs + Single(Vec), + /// Batch of token sequences + Batch(Vec>), +} + +impl TokensInput { + /// Check if this is a batch input + pub fn is_batch(&self) -> bool { + matches!(self, TokensInput::Batch(_)) + } + + /// Get the sequences (always returns a vec of vecs for uniform processing) + pub fn sequences(&self) -> Vec<&[u32]> { + match self { + TokensInput::Single(seq) => vec![seq.as_slice()], + TokensInput::Batch(seqs) => seqs.iter().map(|s| s.as_slice()).collect(), + } + } +} + +/// Response schema for the /v1/detokenize endpoint +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DetokenizeResponse { + /// Decoded text - single string for single input, list for batch + pub text: TextResult, +} + +/// Text result - either single or batch +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(untagged)] +pub enum TextResult { + Single(String), + Batch(Vec), +} + +// ============================================================================ +// Tokenizer Management API +// ============================================================================ + +/// Request schema for adding a tokenizer +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct AddTokenizerRequest { + /// Name to register the tokenizer under + pub name: String, + + /// Source: either a local path or HuggingFace model ID + pub source: String, + + /// Optional path to chat template file + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template_path: Option, +} + +/// Response schema for adding a tokenizer (async) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AddTokenizerResponse { + /// Unique identifier for the tokenizer (UUID) + pub id: String, + /// Status of the request: "pending", "processing", "completed", "failed" + pub status: String, + pub message: String, + /// Vocabulary size of the loaded tokenizer (only set on completion) + #[serde(skip_serializing_if = "Option::is_none")] + pub vocab_size: Option, +} + +/// Response schema for listing tokenizers +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListTokenizersResponse { + pub tokenizers: Vec, +} + +/// Information about a registered tokenizer +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TokenizerInfo { + /// Unique identifier (UUID) + pub id: String, + /// User-provided name + pub name: String, + /// Source path or HuggingFace model ID + pub source: String, + pub vocab_size: usize, +} + +/// Request schema for removing a tokenizer +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct RemoveTokenizerRequest { + /// Name of the tokenizer to remove + pub name: String, +} + +/// Response schema for removing a tokenizer +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RemoveTokenizerResponse { + pub success: bool, + pub message: String, +} + +// ============================================================================ +// Helper Types +// ============================================================================ + +/// String or array of strings (for flexible input) +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(untagged)] +pub enum StringOrArray { + Single(String), + Array(Vec), +} + +impl StringOrArray { + /// Check if this is a batch (array) input + pub fn is_batch(&self) -> bool { + matches!(self, StringOrArray::Array(_)) + } + + /// Get all strings as a slice (converts single to vec) + pub fn as_strings(&self) -> Vec<&str> { + match self { + StringOrArray::Single(s) => vec![s.as_str()], + StringOrArray::Array(arr) => arr.iter().map(|s| s.as_str()).collect(), + } + } +} + +// ============================================================================ +// Default Functions +// ============================================================================ + +fn default_model_name() -> String { + UNKNOWN_MODEL_ID.to_string() +} + +fn default_true() -> bool { + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_tokenize_request_single() { + let json = r#"{"prompt": "Hello world"}"#; + let req: TokenizeRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.model, "unknown"); + assert!(matches!(req.prompt, StringOrArray::Single(_))); + } + + #[test] + fn test_tokenize_request_batch() { + let json = r#"{"model": "llama", "prompt": ["Hello", "World"]}"#; + let req: TokenizeRequest = serde_json::from_str(json).unwrap(); + assert_eq!(req.model, "llama"); + assert!(matches!(req.prompt, StringOrArray::Array(_))); + } + + #[test] + fn test_detokenize_request_single() { + let json = r#"{"tokens": [1, 2, 3]}"#; + let req: DetokenizeRequest = serde_json::from_str(json).unwrap(); + assert!(matches!(req.tokens, TokensInput::Single(_))); + assert!(req.skip_special_tokens); + } + + #[test] + fn test_detokenize_request_batch() { + let json = r#"{"tokens": [[1, 2], [3, 4, 5]], "skip_special_tokens": false}"#; + let req: DetokenizeRequest = serde_json::from_str(json).unwrap(); + assert!(matches!(req.tokens, TokensInput::Batch(_))); + assert!(!req.skip_special_tokens); + } + + #[test] + fn test_tokenize_response_single() { + let resp = TokenizeResponse { + tokens: TokensResult::Single(vec![1, 2, 3]), + count: CountResult::Single(3), + char_count: CountResult::Single(11), + }; + let json = serde_json::to_string(&resp).unwrap(); + assert!(json.contains("[1,2,3]")); + assert!(json.contains("\"count\":3")); + assert!(json.contains("\"char_count\":11")); + } + + #[test] + fn test_tokenize_response_batch() { + let resp = TokenizeResponse { + tokens: TokensResult::Batch(vec![vec![1, 2], vec![3, 4, 5]]), + count: CountResult::Batch(vec![2, 3]), + char_count: CountResult::Batch(vec![5, 5]), + }; + let json = serde_json::to_string(&resp).unwrap(); + assert!(json.contains("[[1,2],[3,4,5]]")); + assert!(json.contains("[2,3]")); + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/validated.rs b/sgl-model-gateway/vendor/openai-protocol/src/validated.rs new file mode 100644 index 0000000000..15c759f55e --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/validated.rs @@ -0,0 +1,172 @@ +// Validated JSON extractor for automatic request validation +// +// This module provides a ValidatedJson extractor that automatically validates +// requests using the validator crate's Validate trait. + +/// Trait for request types that need post-deserialization normalization +pub trait Normalizable { + /// Normalize the request by applying defaults and transformations + fn normalize(&mut self) { + // Default: no-op + } +} + +#[cfg(feature = "axum")] +use axum::{ + extract::{rejection::JsonRejection, FromRequest, Request}, + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +#[cfg(feature = "axum")] +use serde::de::DeserializeOwned; +#[cfg(feature = "axum")] +use serde_json::json; +#[cfg(feature = "axum")] +use validator::Validate; + +/// A JSON extractor that automatically validates and normalizes the request body +/// +/// This extractor deserializes the request body and automatically calls `.validate()` +/// on types that implement the `Validate` trait. If validation fails, it returns +/// a 400 Bad Request with detailed error information. +/// +/// # Example +/// +/// ```rust,ignore +/// async fn create_chat( +/// ValidatedJson(request): ValidatedJson, +/// ) -> Response { +/// // request is guaranteed to be valid here +/// process_request(request).await +/// } +/// ``` +#[cfg(feature = "axum")] +pub struct ValidatedJson(pub T); + +#[cfg(feature = "axum")] +impl FromRequest for ValidatedJson +where + T: DeserializeOwned + Validate + Normalizable + Send, + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request(req: Request, state: &S) -> Result { + // First, extract and deserialize the JSON + let Json(mut data) = + Json::::from_request(req, state) + .await + .map_err(|err: JsonRejection| { + let error_message = match err { + JsonRejection::JsonDataError(e) => { + format!("Invalid JSON data: {}", e) + } + JsonRejection::JsonSyntaxError(e) => { + format!("JSON syntax error: {}", e) + } + JsonRejection::MissingJsonContentType(_) => { + "Missing Content-Type: application/json header".to_string() + } + _ => format!("Failed to parse JSON: {}", err), + }; + + ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": error_message, + "type": "invalid_request_error", + "code": "json_parse_error" + } + })), + ) + .into_response() + })?; + + // Normalize the request (apply defaults based on other fields) + data.normalize(); + + // Then, automatically validate the data + data.validate().map_err(|validation_errors| { + ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": { + "message": validation_errors.to_string(), + "type": "invalid_request_error", + "code": 400 + } + })), + ) + .into_response() + })?; + + Ok(ValidatedJson(data)) + } +} + +// Implement Deref to allow transparent access to the inner value +#[cfg(feature = "axum")] +impl std::ops::Deref for ValidatedJson { + type Target = T; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[cfg(feature = "axum")] +impl std::ops::DerefMut for ValidatedJson { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +#[cfg(all(test, feature = "axum"))] +mod tests { + use serde::{Deserialize, Serialize}; + use validator::Validate; + + use super::*; + + #[derive(Debug, Deserialize, Serialize, Validate)] + struct TestRequest { + #[validate(range(min = 0.0, max = 1.0))] + value: f32, + #[validate(length(min = 1))] + name: String, + } + + impl Normalizable for TestRequest { + // Use default no-op implementation + } + + #[tokio::test] + async fn test_validated_json_valid() { + // This test is conceptual - actual testing would require Axum test harness + let request = TestRequest { + value: 0.5, + name: "test".to_string(), + }; + assert!(request.validate().is_ok()); + } + + #[tokio::test] + async fn test_validated_json_invalid_range() { + let request = TestRequest { + value: 1.5, // Out of range + name: "test".to_string(), + }; + assert!(request.validate().is_err()); + } + + #[tokio::test] + async fn test_validated_json_invalid_length() { + let request = TestRequest { + value: 0.5, + name: "".to_string(), // Empty name + }; + assert!(request.validate().is_err()); + } +} diff --git a/sgl-model-gateway/vendor/openai-protocol/src/worker_spec.rs b/sgl-model-gateway/vendor/openai-protocol/src/worker_spec.rs new file mode 100644 index 0000000000..230a0f1ffc --- /dev/null +++ b/sgl-model-gateway/vendor/openai-protocol/src/worker_spec.rs @@ -0,0 +1,469 @@ +//! Worker management API specifications +//! +//! Defines the request/response structures for worker management endpoints + +use std::collections::HashMap; + +#[cfg(feature = "axum")] +use axum::{ + http::StatusCode, + response::{IntoResponse, Response}, + Json, +}; +use serde::{Deserialize, Serialize}; +#[cfg(feature = "axum")] +use serde_json::{json, Value}; + +use super::UNKNOWN_MODEL_ID; + +/// Worker configuration for API requests +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WorkerConfigRequest { + /// Worker URL (required) + pub url: String, + + /// Worker API key (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub api_key: Option, + + /// Model ID (optional, will query from server if not provided) + #[serde(skip_serializing_if = "Option::is_none")] + pub model_id: Option, + + /// Worker priority (optional, default: 50, higher = preferred) + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// Worker cost factor (optional, default: 1.0) + #[serde(skip_serializing_if = "Option::is_none")] + pub cost: Option, + + /// Worker type (optional: "regular", "prefill", "decode") + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_type: Option, + + /// Bootstrap port for prefill workers (optional) + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_port: Option, + + /// Runtime type (optional: "sglang", "vllm", default: "sglang") + /// Only relevant for gRPC workers + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime: Option, + + // gRPC-specific configuration (optional, ignored in HTTP mode) + /// Tokenizer path for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + /// Reasoning parser type for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + /// Tool parser type for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + /// Chat template for gRPC mode + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template: Option, + + /// Additional labels (optional) + #[serde(default, skip_serializing_if = "HashMap::is_empty")] + pub labels: HashMap, + + /// Health check timeout in seconds (default: 30) + #[serde(default = "default_health_check_timeout")] + pub health_check_timeout_secs: u64, + + /// Health check interval in seconds (default: 60) + #[serde(default = "default_health_check_interval")] + pub health_check_interval_secs: u64, + + /// Number of successful health checks needed to mark worker as healthy (default: 2) + #[serde(default = "default_health_success_threshold")] + pub health_success_threshold: u32, + + /// Number of failed health checks before marking worker as unhealthy (default: 3) + #[serde(default = "default_health_failure_threshold")] + pub health_failure_threshold: u32, + + /// Disable periodic health checks for this worker (default: false) + #[serde(default)] + pub disable_health_check: bool, + + /// Maximum connection attempts during worker registration (default: 20) + #[serde(default = "default_max_connection_attempts")] + pub max_connection_attempts: u32, + + /// Enable data parallelism aware scheduling (default: false) + #[serde(default)] + pub dp_aware: bool, +} + +// Default value functions for serde +fn default_health_check_timeout() -> u64 { + 30 +} + +fn default_health_check_interval() -> u64 { + 60 +} + +fn default_health_success_threshold() -> u32 { + 2 +} + +fn default_health_failure_threshold() -> u32 { + 3 +} + +fn default_max_connection_attempts() -> u32 { + 20 +} + +/// Worker information for API responses +#[derive(Debug, Clone, Serialize)] +pub struct WorkerInfo { + /// Worker unique identifier + pub id: String, + + /// Worker URL + pub url: String, + + /// Model ID this worker serves + pub model_id: String, + + /// Worker priority + pub priority: u32, + + /// Worker cost factor + pub cost: f32, + + /// Worker type + pub worker_type: String, + + /// Whether the worker is healthy + pub is_healthy: bool, + + /// Current load on the worker + pub load: usize, + + /// Connection mode (http or grpc) + pub connection_mode: String, + + /// Runtime type (sglang or vllm, for gRPC workers) + #[serde(skip_serializing_if = "Option::is_none")] + pub runtime_type: Option, + + // gRPC-specific fields (None for HTTP workers) + #[serde(skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template: Option, + + /// Bootstrap port for prefill workers + #[serde(skip_serializing_if = "Option::is_none")] + pub bootstrap_port: Option, + + /// Additional metadata + #[serde(skip_serializing_if = "HashMap::is_empty")] + pub metadata: HashMap, + + /// Whether health checks are disabled for this worker + pub disable_health_check: bool, + + /// Job status for async operations (if available) + #[serde(skip_serializing_if = "Option::is_none")] + pub job_status: Option, +} + +impl WorkerInfo { + /// Create a partial WorkerInfo for pending workers (not yet registered). + /// Used when a worker ID maps to a URL but the worker is still being registered. + pub fn pending(worker_id: &str, url: String, job_status: Option) -> Self { + Self { + id: worker_id.to_string(), + url, + model_id: UNKNOWN_MODEL_ID.to_string(), + priority: 0, + cost: 1.0, + worker_type: UNKNOWN_MODEL_ID.to_string(), + is_healthy: false, + load: 0, + connection_mode: UNKNOWN_MODEL_ID.to_string(), + runtime_type: None, + tokenizer_path: None, + reasoning_parser: None, + tool_parser: None, + chat_template: None, + bootstrap_port: None, + metadata: HashMap::new(), + disable_health_check: false, + job_status, + } + } +} + +/// Job status for async control plane operations +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobStatus { + pub job_type: String, + pub worker_url: String, + pub status: String, + pub message: Option, + pub timestamp: u64, +} + +impl JobStatus { + /// Create a pending job status + pub fn pending(job_type: &str, worker_url: &str) -> Self { + Self { + job_type: job_type.to_string(), + worker_url: worker_url.to_string(), + status: "pending".to_string(), + message: None, + timestamp: std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } + + /// Create a processing job status + pub fn processing(job_type: &str, worker_url: &str) -> Self { + Self { + job_type: job_type.to_string(), + worker_url: worker_url.to_string(), + status: "processing".to_string(), + message: None, + timestamp: std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } + + /// Create a failed job status + pub fn failed(job_type: &str, worker_url: &str, error: String) -> Self { + Self { + job_type: job_type.to_string(), + worker_url: worker_url.to_string(), + status: "failed".to_string(), + message: Some(error), + timestamp: std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + } + } +} + +/// Worker list response +#[derive(Debug, Clone, Serialize)] +pub struct WorkerListResponse { + /// List of workers + pub workers: Vec, + + /// Total count + pub total: usize, + + /// Statistics + pub stats: WorkerStats, +} + +/// Worker statistics +#[derive(Debug, Clone, Serialize)] +pub struct WorkerStats { + pub total_workers: usize, + pub healthy_workers: usize, + pub total_models: usize, + pub total_load: usize, + pub by_type: WorkerTypeStats, +} + +/// Worker statistics by type +#[derive(Debug, Clone, Serialize)] +pub struct WorkerTypeStats { + pub regular: usize, + pub prefill: usize, + pub decode: usize, +} + +/// Worker update request +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkerUpdateRequest { + /// Update priority + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + /// Update cost + #[serde(skip_serializing_if = "Option::is_none")] + pub cost: Option, + + /// Update labels + #[serde(skip_serializing_if = "Option::is_none")] + pub labels: Option>, + + /// Update API key (for key rotation) + #[serde(skip_serializing_if = "Option::is_none")] + pub api_key: Option, + + /// Update health check timeout in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub health_check_timeout_secs: Option, + + /// Update health check interval in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub health_check_interval_secs: Option, + + /// Update health success threshold + #[serde(skip_serializing_if = "Option::is_none")] + pub health_success_threshold: Option, + + /// Update health failure threshold + #[serde(skip_serializing_if = "Option::is_none")] + pub health_failure_threshold: Option, + + /// Disable periodic health checks for this worker + #[serde(skip_serializing_if = "Option::is_none")] + pub disable_health_check: Option, +} + +/// Generic API response +#[derive(Debug, Clone, Serialize)] +pub struct WorkerApiResponse { + pub success: bool, + pub message: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub worker: Option, +} + +/// Error response +#[derive(Debug, Clone, Serialize)] +pub struct WorkerErrorResponse { + pub error: String, + pub code: String, +} + +/// Server info response from /get_server_info endpoint +#[derive(Debug, Clone, Deserialize)] +pub struct ServerInfo { + #[serde(skip_serializing_if = "Option::is_none")] + pub model_id: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub model_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub priority: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub cost: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_type: Option, + + // gRPC-specific + #[serde(skip_serializing_if = "Option::is_none")] + pub tokenizer_path: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub tool_parser: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub chat_template: Option, +} + +/// Result from flush cache operations across workers +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct FlushCacheResult { + /// URLs of workers where cache flush succeeded + pub successful: Vec, + /// URLs and error messages for workers where cache flush failed + pub failed: Vec<(String, String)>, + /// Total number of workers attempted + pub total_workers: usize, + /// Number of HTTP workers (gRPC workers don't support flush cache) + pub http_workers: usize, + /// Human-readable summary message + pub message: String, +} + +/// Result from getting worker loads +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WorkerLoadsResult { + /// Worker URL and load pairs + pub loads: Vec, + /// Total number of workers + pub total_workers: usize, + /// Number of workers with successful load fetches + pub successful: usize, + /// Number of workers with failed load fetches + pub failed: usize, +} + +/// Individual worker load information +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct WorkerLoadInfo { + /// Worker URL + pub worker: String, + /// Worker type (regular, prefill, decode) + #[serde(skip_serializing_if = "Option::is_none")] + pub worker_type: Option, + /// Current load (-1 indicates failure to fetch) + pub load: isize, +} + +#[cfg(feature = "axum")] +impl IntoResponse for FlushCacheResult { + fn into_response(self) -> Response { + let status = if self.failed.is_empty() { + StatusCode::OK + } else { + StatusCode::PARTIAL_CONTENT + }; + + let mut body = json!({ + "status": if self.failed.is_empty() { "success" } else { "partial_success" }, + "message": self.message, + "workers_flushed": self.successful.len(), + "total_http_workers": self.http_workers, + "total_workers": self.total_workers + }); + + if !self.failed.is_empty() { + body["successful"] = json!(self.successful); + body["failed"] = json!(self + .failed + .into_iter() + .map(|(url, err)| json!({"worker": url, "error": err})) + .collect::>()); + } + + (status, Json(body)).into_response() + } +} + +#[cfg(feature = "axum")] +impl IntoResponse for WorkerLoadsResult { + fn into_response(self) -> Response { + let loads: Vec = self + .loads + .iter() + .map(|info| json!({"worker": &info.worker, "load": info.load})) + .collect(); + Json(json!({"workers": loads})).into_response() + } +}