Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
216880034a | ||
|
|
6da0a40b02 | ||
|
|
caf3403086 | ||
|
|
617f19ba61 | ||
|
|
ba3318ffbd | ||
|
|
8279d3d6df | ||
|
|
ef4cccb6a0 | ||
|
|
98b950366d | ||
|
|
87ab6c0155 | ||
|
|
0308920bdb | ||
|
|
57db2a4ae9 | ||
|
|
7fd7fd58a5 | ||
|
|
71a72b8fea | ||
|
|
6c9518fcd8 | ||
|
|
3a4ab451ac | ||
|
|
5d38fc96a9 | ||
|
|
261ebbc55b | ||
|
|
53fb9be7ee | ||
|
|
e5be84f795 | ||
|
|
39ceba8a3d | ||
|
|
b9d259441c |
@@ -25,7 +25,7 @@
|
||||
# 端口: P1=30000 P2=30010 P3=30020 D=30001 router=30002
|
||||
# bootstrap: P1=8998 P2=8999 P3=9000 D=9001
|
||||
x-sglang-common: &sglang-common
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-6612af9e0-local-20261011-0740
|
||||
image: ymkymx/sglang:dsv41-pd-visioncp-e7dc175e9-local-20261011-0940
|
||||
shm_size: "32gb"
|
||||
ipc: host
|
||||
pid: host
|
||||
|
||||
@@ -2998,12 +2998,22 @@ class MooncakeKVManager(StagingManagerMixin, CommonKVManager):
|
||||
# add further chunks into the transfer queue.
|
||||
return
|
||||
|
||||
# NOTE(shangming): sharding according to the dst_infos to make sure
|
||||
# requests with the same dst_sessions will be added into the same
|
||||
# queue, which enables early abort with failed sessions.
|
||||
# Shard by dst session ports AND bootstrap_room. The port term alone
|
||||
# degenerates to a constant when all requests target a single decode
|
||||
# instance (its session ports are fixed), collapsing every transfer
|
||||
# onto 1/16 of the lanes and throttling P->D intake (measured at
|
||||
# client concurrency 300: #transfer-req piled at 100+ while decode
|
||||
# itself stayed healthy). Mixing in bootstrap_room keeps all chunks
|
||||
# of one request on the same lane (per-request FIFO order preserved)
|
||||
# while spreading distinct rooms across all lanes. Abort/cleanup is
|
||||
# unaffected: chunks are dropped per-room at dequeue via
|
||||
# request_status (see the worker loop) and per-request via
|
||||
# failed_sessions before send — no lane-level purge exists, and the
|
||||
# deferred abort ack is room-scoped (_staging_outstanding). Same
|
||||
# trade-off as mori/conn.py which shards by bootstrap_room alone.
|
||||
dst_infos = self.transfer_infos[bootstrap_room].keys()
|
||||
session_port_sum = sum(int(session.rsplit(":", 1)[1]) for session in dst_infos)
|
||||
shard_idx = session_port_sum % len(self.transfer_queues)
|
||||
shard_idx = (session_port_sum + bootstrap_room) % len(self.transfer_queues)
|
||||
|
||||
if trace_ctx is None:
|
||||
trace_ctx = TraceNullContext()
|
||||
|
||||
@@ -205,3 +205,6 @@ codegen-units = 1
|
||||
[profile.dev-opt]
|
||||
inherits = "dev"
|
||||
opt-level = 1
|
||||
|
||||
[patch.crates-io]
|
||||
openai-protocol = { path = "vendor/openai-protocol" }
|
||||
|
||||
@@ -378,6 +378,12 @@ impl RouterConfigBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// G9i pool-aware admission control + fast-P routing config (PD mode).
|
||||
pub fn g9i(mut self, g9i: Option<crate::routers::http::g9i::G9iConfig>) -> Self {
|
||||
self.config.g9i = g9i;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn model_path<S: Into<String>>(mut self, path: S) -> Self {
|
||||
self.config.model_path = Some(path.into());
|
||||
self
|
||||
|
||||
@@ -99,6 +99,9 @@ pub struct RouterConfig {
|
||||
/// Enable WASM support
|
||||
#[serde(default)]
|
||||
pub enable_wasm: bool,
|
||||
/// G9i pool-aware admission control + fast-P routing (PD mode, HTTP only).
|
||||
#[serde(default)]
|
||||
pub g9i: Option<crate::routers::http::g9i::G9iConfig>,
|
||||
}
|
||||
|
||||
/// Tokenizer cache configuration
|
||||
@@ -551,6 +554,7 @@ impl Default for RouterConfig {
|
||||
ca_certificates: vec![],
|
||||
mcp_config: None,
|
||||
enable_wasm: false,
|
||||
g9i: None,
|
||||
server_cert: None,
|
||||
server_key: None,
|
||||
}
|
||||
|
||||
@@ -219,6 +219,71 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 30, help_heading = "PD Disaggregation")]
|
||||
worker_startup_check_interval: u64,
|
||||
|
||||
// ==================== G9i: Pool Admission + Fast-P Routing ====================
|
||||
/// Enable G9i pool-aware admission control + fast-P routing (PD HTTP mode)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_enabled: bool,
|
||||
|
||||
/// Fast-P pool member base URLs (repeatable; router-side role, engine unaware)
|
||||
#[arg(long, action = ArgAction::Append, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_prefill_url: Vec<String>,
|
||||
|
||||
/// Small-prefill cutoff on NEW tokens (total - prefix hit). Tentative; tune by bench.
|
||||
#[arg(long, default_value_t = 4096, help_heading = "G9i Pool Admission")]
|
||||
g9i_max_new_tokens: u64,
|
||||
|
||||
/// 429 when the least-wait fast-P queue estimate exceeds this many seconds. Tentative.
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_fast_p_queue_reject_secs: f64,
|
||||
|
||||
/// D-pool admission ceiling: sum of in-processing requests across decode nodes. Tentative.
|
||||
#[arg(long, default_value_t = 256, help_heading = "G9i Pool Admission")]
|
||||
g9i_decode_max_running: u64,
|
||||
|
||||
/// 429 when the least-wait REGULAR pool queue estimate exceeds this many seconds. Tentative 10.0; 0=off.
|
||||
#[arg(long, default_value_t = 10.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_p_queue_reject_secs: f64,
|
||||
|
||||
/// Drain-rate floor (tok/s) for queue-wait estimates (div-by-zero / cold-start guard)
|
||||
#[arg(long, default_value_t = 1000.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_drain_rate_floor: f64,
|
||||
|
||||
/// Snapshot poll interval (seconds)
|
||||
#[arg(long, default_value_t = 1, help_heading = "G9i Pool Admission")]
|
||||
g9i_poll_interval_secs: u64,
|
||||
|
||||
/// Snapshot TTL — stale nodes leave pool decisions (seconds)
|
||||
#[arg(long, default_value_t = 5, help_heading = "G9i Pool Admission")]
|
||||
g9i_snapshot_ttl_secs: u64,
|
||||
|
||||
/// No fresh D-pool snapshot: "allow" (default, fail-open + warn) or "reject"
|
||||
#[arg(long, default_value = "allow", value_parser = ["allow", "reject"], help_heading = "G9i Pool Admission")]
|
||||
g9i_d_no_snapshot: String,
|
||||
|
||||
/// /query_cache_hit probe timeout per request (ms)
|
||||
#[arg(long, default_value_t = 2000, help_heading = "G9i Pool Admission")]
|
||||
g9i_cache_hit_query_timeout_ms: u64,
|
||||
|
||||
/// Also least-wait route the regular P pool (default: keep configured policy)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_regular_pool_least_wait: bool,
|
||||
|
||||
/// Sliding window for TTFT p95/p99 statistics publication (seconds; observability only, never gates)
|
||||
#[arg(long, default_value_t = 300, help_heading = "G9i Pool Admission")]
|
||||
g9i_ttft_window_s: u64,
|
||||
|
||||
/// G9i v2 shadow mode: prefix-chain bookkeeping + shadow classifier + queue model (observability only, zero routing effect; requires --g9i-enabled)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_enabled: bool,
|
||||
|
||||
/// Shadow prefix-table capacity in entries (LRU + TTL evicted)
|
||||
#[arg(long, default_value_t = 100_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_max_entries: usize,
|
||||
|
||||
/// Shadow prefix-table entry TTL (seconds)
|
||||
#[arg(long, default_value_t = 86400, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_ttl_secs: u64,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1070,11 +1135,39 @@ impl CliArgs {
|
||||
.circuit_breaker(!self.disable_circuit_breaker)
|
||||
.enable_wasm(self.enable_wasm)
|
||||
.igw(self.enable_igw)
|
||||
.g9i(self.build_g9i_config()?)
|
||||
.maybe_server_cert_and_key(self.tls_cert_path.as_ref(), self.tls_key_path.as_ref());
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
/// G9i pool admission + fast-P routing config; None when the feature is off.
|
||||
fn build_g9i_config(&self) -> ConfigResult<Option<smg::routers::http::g9i::G9iConfig>> {
|
||||
if !self.g9i_enabled && self.g9i_fast_prefill_url.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let config = smg::routers::http::g9i::G9iConfig {
|
||||
enabled: self.g9i_enabled,
|
||||
fast_prefill_urls: self.g9i_fast_prefill_url.clone(),
|
||||
max_new_tokens: self.g9i_max_new_tokens,
|
||||
fast_p_queue_reject_secs: self.g9i_fast_p_queue_reject_secs,
|
||||
regular_p_queue_reject_secs: self.g9i_regular_p_queue_reject_secs,
|
||||
decode_max_running: self.g9i_decode_max_running,
|
||||
drain_rate_floor: self.g9i_drain_rate_floor,
|
||||
poll_interval_secs: self.g9i_poll_interval_secs,
|
||||
snapshot_ttl_secs: self.g9i_snapshot_ttl_secs,
|
||||
d_no_snapshot: self.g9i_d_no_snapshot.clone(),
|
||||
cache_hit_query_timeout_ms: self.g9i_cache_hit_query_timeout_ms,
|
||||
regular_pool_least_wait: self.g9i_regular_pool_least_wait,
|
||||
ttft_window_s: self.g9i_ttft_window_s,
|
||||
shadow_enabled: self.g9i_shadow_enabled,
|
||||
shadow_table_max_entries: self.g9i_shadow_table_max_entries,
|
||||
shadow_table_ttl_secs: self.g9i_shadow_table_ttl_secs,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
}
|
||||
|
||||
fn to_server_config(&self, router_config: RouterConfig) -> ServerConfig {
|
||||
let service_discovery_config = if self.service_discovery {
|
||||
// Get router discovery config from router_config.discovery if available
|
||||
|
||||
@@ -301,11 +301,12 @@ impl HarmonyBuilder {
|
||||
.reasoning
|
||||
.as_ref()
|
||||
.and_then(|r| r.effort.as_ref())
|
||||
.map(|effort| match effort {
|
||||
ResponsesReasoningEffort::High => ReasoningEffort::High,
|
||||
ResponsesReasoningEffort::Medium => ReasoningEffort::Medium,
|
||||
ResponsesReasoningEffort::Low => ReasoningEffort::Low,
|
||||
ResponsesReasoningEffort::Minimal => ReasoningEffort::Low,
|
||||
.and_then(|effort| match effort {
|
||||
ResponsesReasoningEffort::None => None,
|
||||
ResponsesReasoningEffort::High => Some(ReasoningEffort::High),
|
||||
ResponsesReasoningEffort::Medium => Some(ReasoningEffort::Medium),
|
||||
ResponsesReasoningEffort::Low => Some(ReasoningEffort::Low),
|
||||
ResponsesReasoningEffort::Minimal => Some(ReasoningEffort::Low),
|
||||
});
|
||||
|
||||
self.build_system_message(reasoning_effort, with_custom_tools)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,675 @@
|
||||
//! G9i v2 P1 shadow mode — T1: prefix hash chain (pure function layer).
|
||||
//!
|
||||
//! Design source: `experiments/g9i/g9i-v2-design-20261013.md` §3.2/§4.1,
|
||||
//! content-handling constraints C4/C10. This module is **read-only with
|
||||
//! respect to routing**: it computes hashes, it never touches dispatch.
|
||||
//!
|
||||
//! Why a chain: a chat body with `m` messages has exactly `m` prefix
|
||||
//! boundaries, and the session bookkeeping table ("prefix → last serving
|
||||
//! node + exact token count") needs a stable key per boundary. Rolling the
|
||||
//! hash forward per message gives all `m` keys in one pass; lookup then
|
||||
//! probes from deep to shallow and takes the deepest hit. In practice a new
|
||||
//! turn is `previous body + assistant reply + new user message`, so hits
|
||||
//! usually land at `h[m-2]`.
|
||||
//!
|
||||
//! seed = xxh3(canonical_json({model, chat_template_kwargs, tools_hash, system}))
|
||||
//! h[0] = seed
|
||||
//! h[k] = xxh3(h[k-1] LE bytes ‖ len(role) ‖ role ‖ tagged content
|
||||
//! ‖ tagged canonical(tool_calls) ‖ tagged meta) k = 1..=m
|
||||
//!
|
||||
//! Encoding decisions (why, not what):
|
||||
//! * Streaming `Xxh3` instead of concatenating into a `Vec`: chat bodies
|
||||
//! with base64 images are routinely hundreds of KB; copying them per
|
||||
//! chain step would be pure waste.
|
||||
//! * Every variable-length field is length-prefixed (u64 LE) and every
|
||||
//! content piece carries a type tag. Without that, `["ab","c"]` and
|
||||
//! `["a","bc"]` (or role/content, part/part splits) would hash
|
||||
//! identically — silent false cache hits.
|
||||
//! * Missing role defaults to "user" (OpenAI clients routinely omit it);
|
||||
//! missing `model` / `chat_template_kwargs` / `tools` / system message
|
||||
//! serialize as JSON `null` in the seed document — a fixed placeholder
|
||||
//! that cannot collide with any real value because `null` is not a
|
||||
//! string/object the field could otherwise hold.
|
||||
//! * Byte-level semantics for multimodal (C4): `image_url` URLs are
|
||||
//! hashed as the raw string — `http(s):` URLs as-is, `data:` URLs
|
||||
//! including the base64 payload, **never fetched, never decoded**.
|
||||
//! Same image at different URLs hashes differently; that is correct —
|
||||
//! the rendered prompt bytes differ.
|
||||
//! * `is_unclassifiable` and `chain_hashes` share one traversal
|
||||
//! (`walk_messages`) so the "weird content" judgement can never drift
|
||||
//! between the two call sites (C10). Unknown part types still
|
||||
//! contribute a deterministic tag to the chain — the chain stays
|
||||
//! well-defined, the caller just must not trust it for table lookups
|
||||
//! when `is_unclassifiable` is true.
|
||||
//! * `content: null` / missing renders as empty content in every chat
|
||||
//! template, so it hashes exactly like `""` — NOT unclassifiable.
|
||||
//! Flagging it would disenfranchise tool-use conversations (assistant
|
||||
//! turns with `content: null` + `tool_calls`), our main traffic shape.
|
||||
//! Genuinely malformed content (number/object/bool) keeps the
|
||||
//! Empty + unclassifiable treatment.
|
||||
//! * `tool_calls` are rendered into the prompt by the chat template, so
|
||||
//! they MUST enter the chain (as canonical JSON, key-order
|
||||
//! insensitive) — otherwise two conversations with different tool
|
||||
//! calls could share a hash: a false hit in the dangerous direction.
|
||||
//! `reasoning_content` is deliberately excluded: templates never
|
||||
//! render it, so it cannot change prompt bytes.
|
||||
//! * `tool_call_id` (on `role:"tool"` results) and `name` (any role)
|
||||
//! are rendered by most templates, so they enter the chain as one
|
||||
//! canonical meta object. The META segment is unconditional — a
|
||||
//! message without these fields contributes `TAG_META ‖ len=0`,
|
||||
//! keeping the formula single-path (no presence/absence ambiguity).
|
||||
|
||||
use serde_json::Value;
|
||||
use xxhash_rust::xxh3::{xxh3_64, Xxh3};
|
||||
|
||||
/// Part-type tags mixed in before each content piece. Terminated with `\0`
|
||||
/// so tags are self-delimiting; content bytes are length-prefixed on top.
|
||||
const TAG_TEXT: &[u8] = b"text\0";
|
||||
const TAG_IMAGE_URL: &[u8] = b"image_url\0";
|
||||
const TAG_UNKNOWN: &[u8] = b"unknown\0";
|
||||
const TAG_TOOL_CALLS: &[u8] = b"tool_calls\0";
|
||||
const TAG_META: &[u8] = b"meta\0";
|
||||
|
||||
/// Chain seed: folds every body field that changes the rendered prompt but
|
||||
/// lives outside `messages` into one u64.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub struct ChainSeed {
|
||||
pub value: u64,
|
||||
}
|
||||
|
||||
impl ChainSeed {
|
||||
/// Build the seed from a chat request body.
|
||||
///
|
||||
/// Inputs: `model`, `chat_template_kwargs`, `tools` (folded to a single
|
||||
/// hash of its canonical form), and the first `role=="system"` message's
|
||||
/// string content. Missing fields become JSON `null` in the seed
|
||||
/// document (fixed placeholder, see module docs). Canonicalization
|
||||
/// sorts object keys recursively and emits no whitespace, so semantically
|
||||
/// equal bodies from different clients produce the same seed.
|
||||
///
|
||||
/// Note: a system message is *also* hashed as `messages[0]` inside the
|
||||
/// chain — deliberately. The chain must stay 1:1 with `messages`
|
||||
/// indices (the lookup table keys are `h[k]` by position), and the
|
||||
/// redundancy only costs one extra mix of already-known bytes.
|
||||
pub fn from_body(body: &Value) -> ChainSeed {
|
||||
let system_content: Option<&str> = body
|
||||
.get("messages")
|
||||
.and_then(Value::as_array)
|
||||
.and_then(|msgs| {
|
||||
msgs.iter()
|
||||
.find(|m| m.get("role").and_then(Value::as_str) == Some("system"))
|
||||
})
|
||||
.and_then(|m| m.get("content"))
|
||||
.and_then(Value::as_str);
|
||||
|
||||
let tools_hash: Option<u64> = body
|
||||
.get("tools")
|
||||
.map(|t| xxh3_64(canonical_json(t).as_bytes()));
|
||||
|
||||
let seed_doc = serde_json::json!({
|
||||
"model": body.get("model").cloned().unwrap_or(Value::Null),
|
||||
"chat_template_kwargs": body
|
||||
.get("chat_template_kwargs")
|
||||
.cloned()
|
||||
.unwrap_or(Value::Null),
|
||||
"tools_hash": tools_hash,
|
||||
"system": system_content,
|
||||
});
|
||||
ChainSeed {
|
||||
value: xxh3_64(canonical_json(&seed_doc).as_bytes()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the chain hashes `h[1..=m]`, one per message, in order.
|
||||
///
|
||||
/// `messages` missing / not an array / empty → empty `Vec`. The caller
|
||||
/// treats an empty chain as "cannot classify" (same fall-back path as
|
||||
/// `is_unclassifiable`); we never panic on malformed bodies because this
|
||||
/// sits on the hot request path.
|
||||
///
|
||||
/// The returned chain is deterministic even when `is_unclassifiable(body)`
|
||||
/// is true — unknown parts contribute their tag — but such chains are only
|
||||
/// suitable for logging/shadow comparison, not for bookkeeping lookups.
|
||||
pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for msg in &msgs {
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
let role = msg.role.as_bytes();
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
PartView::Text(t) => update_tagged(&mut h, TAG_TEXT, t.as_bytes()),
|
||||
PartView::ImageUrl(u) => {
|
||||
update_tagged(&mut h, TAG_IMAGE_URL, u.as_bytes())
|
||||
}
|
||||
PartView::Unknown => h.update(TAG_UNKNOWN),
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Empty => {}
|
||||
}
|
||||
|
||||
// Third segment: tool_calls are rendered into prompt bytes by the
|
||||
// chat template, so they must be in the chain.
|
||||
if let Some(tc) = &msg.tool_calls {
|
||||
update_tagged(&mut h, TAG_TOOL_CALLS, tc.as_bytes());
|
||||
}
|
||||
|
||||
// Fourth segment (unconditional): name / tool_call_id meta. Empty
|
||||
// meta hashes as TAG_META ‖ len=0 — one formula for all messages.
|
||||
update_tagged(&mut h, TAG_META, msg.extra.as_bytes());
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
}
|
||||
chain
|
||||
}
|
||||
|
||||
/// Whether the body contains content this module has no hashing semantics
|
||||
/// for: unknown part types (`audio_url`, `video_url`, `file`, ...),
|
||||
/// malformed parts, unsupported `image_url` schemes, or genuinely
|
||||
/// malformed message content (number/object/bool). Null/missing content
|
||||
/// and `tool_calls`-carrying assistant turns return false — both have
|
||||
/// well-defined rendering semantics. See `walk_messages` — this is the
|
||||
/// same traversal, so the answer cannot disagree with what `chain_hashes`
|
||||
/// actually hashed.
|
||||
pub fn is_unclassifiable(body: &Value) -> bool {
|
||||
walk_messages(body).1
|
||||
}
|
||||
|
||||
/// Mix `tag ‖ len(bytes) ‖ bytes` into the hasher. Length prefix kills
|
||||
/// concatenation ambiguity between adjacent pieces.
|
||||
fn update_tagged(h: &mut Xxh3, tag: &[u8], bytes: &[u8]) {
|
||||
h.update(tag);
|
||||
h.update(&(bytes.len() as u64).to_le_bytes());
|
||||
h.update(bytes);
|
||||
}
|
||||
|
||||
/// One message, flattened to just what the hash needs. Borrows from the
|
||||
/// body — zero copies on the hot path, except `tool_calls` / `extra`
|
||||
/// whose canonical forms must be owned.
|
||||
struct MessageView<'a> {
|
||||
role: &'a str,
|
||||
content: ContentView<'a>,
|
||||
/// Canonical JSON of `tool_calls` when the field exists and is an
|
||||
/// array (a non-array shape can't be rendered by the template either,
|
||||
/// so it's treated as absent).
|
||||
tool_calls: Option<String>,
|
||||
/// Canonical JSON of the meta object {"name"?, "tool_call_id"?}, or
|
||||
/// empty when neither field applies. See `walk_messages` for which
|
||||
/// fields qualify per role.
|
||||
extra: String,
|
||||
}
|
||||
|
||||
enum ContentView<'a> {
|
||||
Str(&'a str),
|
||||
Parts(Vec<PartView<'a>>),
|
||||
/// Genuinely malformed content (number/object/bool): contributes
|
||||
/// nothing, flagged unclassifiable. Null/missing content never lands
|
||||
/// here — it maps to `Str("")`.
|
||||
Empty,
|
||||
}
|
||||
|
||||
enum PartView<'a> {
|
||||
Text(&'a str),
|
||||
ImageUrl(&'a str),
|
||||
/// Unrecognized or malformed part: deterministic placeholder.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
/// Single traversal shared by `chain_hashes` and `is_unclassifiable`
|
||||
/// (C10: one source of truth for "what can we hash").
|
||||
fn walk_messages(body: &Value) -> (Vec<MessageView<'_>>, bool) {
|
||||
let Some(msgs) = body.get("messages").and_then(Value::as_array) else {
|
||||
return (Vec::new(), false);
|
||||
};
|
||||
let mut unclassifiable = false;
|
||||
let mut out = Vec::with_capacity(msgs.len());
|
||||
for m in msgs {
|
||||
let role = m.get("role").and_then(Value::as_str).unwrap_or("user");
|
||||
// `reasoning_content` is deliberately NOT hashed: chat templates do
|
||||
// not render it, so it cannot change prompt bytes — mixing it in
|
||||
// would only manufacture misses.
|
||||
let tool_calls = match m.get("tool_calls") {
|
||||
Some(v @ Value::Array(_)) => Some(canonical_json(v)),
|
||||
_ => None,
|
||||
};
|
||||
// Meta segment: `name` (any role) and `tool_call_id` (tool results
|
||||
// only) are rendered by most templates -> they must be in the
|
||||
// chain. String and non-string values alike go through the
|
||||
// canonical serializer, so wire-shape differences can't collide.
|
||||
// Explicit null counts as absent (templates render it as nothing).
|
||||
let mut meta = serde_json::Map::new();
|
||||
if let Some(v) = m.get("name").filter(|v| !v.is_null()) {
|
||||
meta.insert("name".to_string(), v.clone());
|
||||
}
|
||||
if role == "tool" {
|
||||
if let Some(v) = m.get("tool_call_id").filter(|v| !v.is_null()) {
|
||||
meta.insert("tool_call_id".to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
let extra = if meta.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
canonical_json(&Value::Object(meta))
|
||||
};
|
||||
let content = match m.get("content") {
|
||||
Some(Value::String(s)) => ContentView::Str(s.as_str()),
|
||||
Some(Value::Array(parts)) => {
|
||||
let mut views = Vec::with_capacity(parts.len());
|
||||
for p in parts {
|
||||
match classify_part(p) {
|
||||
Some(view) => views.push(view),
|
||||
None => {
|
||||
unclassifiable = true;
|
||||
views.push(PartView::Unknown);
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Parts(views)
|
||||
}
|
||||
// null / absent renders as empty content in every chat
|
||||
// template — hash exactly like "" and stay classifiable. This
|
||||
// is the tool-use conversation shape (assistant turn with
|
||||
// content:null + tool_calls), our main traffic.
|
||||
None | Some(Value::Null) => ContentView::Str(""),
|
||||
// number/object/bool content: genuinely malformed, no defined
|
||||
// rendering semantics — fail closed.
|
||||
_ => {
|
||||
unclassifiable = true;
|
||||
ContentView::Empty
|
||||
}
|
||||
};
|
||||
out.push(MessageView {
|
||||
role,
|
||||
content,
|
||||
tool_calls,
|
||||
extra,
|
||||
});
|
||||
}
|
||||
(out, unclassifiable)
|
||||
}
|
||||
|
||||
/// Classify one content part. `None` = no hashing semantics for this part
|
||||
/// (unknown type, malformed shape, or an `image_url` scheme outside
|
||||
/// http/https/data — e.g. `file://` would tempt a local read we must never
|
||||
/// do).
|
||||
fn classify_part(p: &Value) -> Option<PartView<'_>> {
|
||||
let ty = p.get("type").and_then(Value::as_str)?;
|
||||
match ty {
|
||||
"text" => p
|
||||
.get("text")
|
||||
.and_then(Value::as_str)
|
||||
.map(PartView::Text),
|
||||
"image_url" => {
|
||||
// Both wire shapes exist in the wild:
|
||||
// {"image_url":{"url":...}} and {"image_url":"..."}.
|
||||
let url = match p.get("image_url") {
|
||||
Some(Value::String(s)) => Some(s.as_str()),
|
||||
Some(v @ Value::Object(_)) => v.get("url").and_then(Value::as_str),
|
||||
_ => None,
|
||||
}?;
|
||||
if url.starts_with("http://") || url.starts_with("https://") || url.starts_with("data:")
|
||||
{
|
||||
Some(PartView::ImageUrl(url))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Canonical JSON: object keys sorted recursively, no whitespace. Hand-rolled
|
||||
/// (no new crate) because we only need it for seed inputs, which are small —
|
||||
/// `tools` is hashed from its canonical form, and the seed doc itself is
|
||||
/// four fields.
|
||||
fn canonical_json(v: &Value) -> String {
|
||||
let mut out = String::new();
|
||||
write_canonical(v, &mut out);
|
||||
out
|
||||
}
|
||||
|
||||
fn write_canonical(v: &Value, out: &mut String) {
|
||||
match v {
|
||||
Value::Object(map) => {
|
||||
out.push('{');
|
||||
let mut keys: Vec<&String> = map.keys().collect();
|
||||
keys.sort();
|
||||
for (i, k) in keys.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
// Serializing a &str Value is infallible; avoid unwrap anyway.
|
||||
if let Ok(k_json) = serde_json::to_string(k) {
|
||||
out.push_str(&k_json);
|
||||
}
|
||||
out.push(':');
|
||||
write_canonical(&map[*k], out);
|
||||
}
|
||||
out.push('}');
|
||||
}
|
||||
Value::Array(arr) => {
|
||||
out.push('[');
|
||||
for (i, item) in arr.iter().enumerate() {
|
||||
if i > 0 {
|
||||
out.push(',');
|
||||
}
|
||||
write_canonical(item, out);
|
||||
}
|
||||
out.push(']');
|
||||
}
|
||||
other => {
|
||||
if let Ok(s) = serde_json::to_string(other) {
|
||||
out.push_str(&s);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn text_body(msgs: Vec<Value>) -> Value {
|
||||
json!({"model": "dsv41-flash", "messages": msgs})
|
||||
}
|
||||
|
||||
fn user(t: &str) -> Value {
|
||||
json!({"role": "user", "content": t})
|
||||
}
|
||||
|
||||
fn assistant(t: &str) -> Value {
|
||||
json!({"role": "assistant", "content": t})
|
||||
}
|
||||
|
||||
fn chain(body: &Value) -> Vec<u64> {
|
||||
chain_hashes(&ChainSeed::from_body(body), body)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_prefix_different_tail() {
|
||||
let a = text_body(vec![user("hello"), assistant("hi"), user("tail A")]);
|
||||
let b = text_body(vec![user("hello"), assistant("hi"), user("tail B")]);
|
||||
let ca = chain(&a);
|
||||
let cb = chain(&b);
|
||||
assert_eq!(ca.len(), 3);
|
||||
assert_eq!(cb.len(), 3);
|
||||
assert_eq!(ca[..2], cb[..2], "shared prefix must hash identically");
|
||||
assert_ne!(ca[2], cb[2], "different tail must diverge");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn forked_conversation_shares_prefix() {
|
||||
let base = vec![user("q1"), assistant("a1"), user("q2"), assistant("a2")];
|
||||
let mut fork_x = base.clone();
|
||||
fork_x.push(user("branch X"));
|
||||
let mut fork_y = base;
|
||||
fork_y.push(user("branch Y"));
|
||||
let bx = text_body(fork_x);
|
||||
let by = text_body(fork_y);
|
||||
let cx = chain(&bx);
|
||||
let cy = chain(&by);
|
||||
assert_eq!(cx[..4], cy[..4]);
|
||||
assert_ne!(cx[4], cy[4]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn editing_history_rewrites_from_that_point_on() {
|
||||
let a = text_body(vec![user("q1"), assistant("old answer"), user("q2")]);
|
||||
let b = text_body(vec![user("q1"), assistant("edited answer"), user("q2")]);
|
||||
let ca = chain(&a);
|
||||
let cb = chain(&b);
|
||||
assert_eq!(ca[0], cb[0], "before the edit: unchanged");
|
||||
assert_ne!(ca[1], cb[1], "at the edit: changed");
|
||||
assert_ne!(ca[2], cb[2], "after the edit: still changed even with identical text");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multimodal_byte_level_semantics() {
|
||||
let mk = |url: &str| {
|
||||
text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": url}}
|
||||
]
|
||||
})])
|
||||
};
|
||||
// Same image content, different URLs -> different chain (the rendered
|
||||
// prompt bytes differ; this is the correct byte-level behaviour).
|
||||
let http_a = mk("https://example.com/a.png");
|
||||
let http_b = mk("https://example.com/b.png");
|
||||
assert_ne!(chain(&http_a), chain(&http_b));
|
||||
|
||||
// Identical base64 payload -> identical chain (hit).
|
||||
let data_1 = mk("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==");
|
||||
let data_2 = mk("data:image/png;base64,iVBORw0KGgoAAAANSUhEUg==");
|
||||
assert_eq!(chain(&data_1), chain(&data_2));
|
||||
|
||||
// base64 vs http form of the "same" image -> different (bytes differ).
|
||||
assert_ne!(chain(&data_1), chain(&http_a));
|
||||
|
||||
// Wire-shape variant {"image_url":"..."} hashes like the object form.
|
||||
let flat = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": "https://example.com/a.png"}
|
||||
]
|
||||
})]);
|
||||
assert_eq!(chain(&flat), chain(&http_a));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_fields_change_whole_chain() {
|
||||
let msgs = || vec![user("same"), assistant("same")];
|
||||
let base = text_body(msgs());
|
||||
let other_model = json!({"model": "other-model", "messages": msgs()});
|
||||
let with_kwargs = json!({
|
||||
"model": "dsv41-flash",
|
||||
"chat_template_kwargs": {"thinking": true},
|
||||
"messages": msgs()
|
||||
});
|
||||
let with_tools = json!({
|
||||
"model": "dsv41-flash",
|
||||
"tools": [{"type": "function", "function": {"name": "f"}}],
|
||||
"messages": msgs()
|
||||
});
|
||||
|
||||
let c_base = chain(&base);
|
||||
assert_ne!(ChainSeed::from_body(&base), ChainSeed::from_body(&other_model));
|
||||
assert_ne!(chain(&other_model), c_base);
|
||||
assert_ne!(chain(&with_kwargs), c_base);
|
||||
assert_ne!(chain(&with_tools), c_base);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seed_canonicalization_is_key_order_insensitive() {
|
||||
let a = json!({"model": "m", "chat_template_kwargs": {"x": 1, "y": 2}, "messages": [user("hi")]});
|
||||
let b = json!({"messages": [user("hi")], "chat_template_kwargs": {"y": 2, "x": 1}, "model": "m"});
|
||||
assert_eq!(ChainSeed::from_body(&a), ChainSeed::from_body(&b));
|
||||
assert_eq!(chain(&a), chain(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_or_malformed_messages_give_empty_chain() {
|
||||
let no_messages = json!({"model": "m"});
|
||||
let empty_messages = json!({"model": "m", "messages": []});
|
||||
let wrong_type = json!({"model": "m", "messages": "not-an-array"});
|
||||
for b in [&no_messages, &empty_messages, &wrong_type] {
|
||||
assert!(chain(b).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_part_types_flag_unclassifiable() {
|
||||
let audio = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [{"type": "audio_url", "audio_url": {"url": "https://x/a.mp3"}}]
|
||||
})]);
|
||||
assert!(is_unclassifiable(&audio));
|
||||
// Chain is still full-length and deterministic — just not to be trusted.
|
||||
assert_eq!(chain(&audio).len(), 1);
|
||||
assert_eq!(chain(&audio), chain(&audio));
|
||||
|
||||
let file_scheme = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [{"type": "image_url", "image_url": {"url": "file:///etc/passwd"}}]
|
||||
})]);
|
||||
assert!(is_unclassifiable(&file_scheme));
|
||||
|
||||
let plain = text_body(vec![user("hello"), assistant("hi")]);
|
||||
assert!(!is_unclassifiable(&plain));
|
||||
|
||||
let multimodal_ok = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}}
|
||||
]
|
||||
})]);
|
||||
assert!(!is_unclassifiable(&multimodal_ok));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_role_defaults_to_user() {
|
||||
let implicit = text_body(vec![json!({"content": "hi"})]);
|
||||
let explicit = text_body(vec![user("hi")]);
|
||||
assert_eq!(chain(&implicit), chain(&explicit));
|
||||
assert!(!is_unclassifiable(&implicit));
|
||||
}
|
||||
|
||||
fn tool_call(name: &str, args: &str) -> Value {
|
||||
json!([{
|
||||
"id": format!("call_{}", name),
|
||||
"type": "function",
|
||||
"function": {"name": name, "arguments": args}
|
||||
}])
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_calls_enter_the_chain() {
|
||||
let mk = |tc: Value| {
|
||||
text_body(vec![
|
||||
user("weather in Shanghai?"),
|
||||
json!({"role": "assistant", "content": null, "tool_calls": tc}),
|
||||
user("and tomorrow?"),
|
||||
])
|
||||
};
|
||||
let a = mk(tool_call("get_weather", "{\"city\":\"sh\"}"));
|
||||
let b = mk(tool_call("get_weather", "{\"city\":\"bj\"}"));
|
||||
let c = mk(tool_call("get_weather", "{\"city\":\"sh\"}"));
|
||||
let (ca, cb, cc) = (chain(&a), chain(&b), chain(&c));
|
||||
assert_eq!(ca[0], cb[0], "before the tool call: unchanged");
|
||||
assert_ne!(ca[1], cb[1], "different tool_calls must diverge");
|
||||
assert_ne!(ca[2], cb[2], "and stay diverged afterwards");
|
||||
assert_eq!(ca, cc, "identical tool_calls must hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_null_with_tool_calls_is_classifiable() {
|
||||
let tc = tool_call("get_weather", "{}");
|
||||
let null_content = text_body(vec![json!({
|
||||
"role": "assistant", "content": null, "tool_calls": tc.clone()
|
||||
})]);
|
||||
let empty_content = text_body(vec![json!({
|
||||
"role": "assistant", "content": "", "tool_calls": tc
|
||||
})]);
|
||||
assert!(!is_unclassifiable(&null_content));
|
||||
assert_eq!(
|
||||
chain(&null_content),
|
||||
chain(&empty_content),
|
||||
"content:null renders as empty -> must hash like content:\"\""
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn content_null_without_tool_calls_is_empty() {
|
||||
let null_content = text_body(vec![json!({"role": "assistant", "content": null})]);
|
||||
let missing_content = text_body(vec![json!({"role": "assistant"})]);
|
||||
let empty_content = text_body(vec![assistant("")]);
|
||||
assert!(!is_unclassifiable(&null_content));
|
||||
assert!(!is_unclassifiable(&missing_content));
|
||||
assert_eq!(chain(&null_content), chain(&empty_content));
|
||||
assert_eq!(chain(&missing_content), chain(&empty_content));
|
||||
|
||||
// Genuinely malformed content types keep the fail-closed treatment.
|
||||
let malformed = text_body(vec![json!({"role": "assistant", "content": 42})]);
|
||||
assert!(is_unclassifiable(&malformed));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_calls_canonicalization_key_order_insensitive() {
|
||||
let a = text_body(vec![json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "f", "arguments": "{}"}
|
||||
}]
|
||||
})]);
|
||||
let b = text_body(vec![json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": [{
|
||||
"function": {"arguments": "{}", "name": "f"},
|
||||
"type": "function",
|
||||
"id": "call_1"
|
||||
}]
|
||||
})]);
|
||||
assert_eq!(chain(&a), chain(&b));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_call_id_enters_chain() {
|
||||
let mk = |id: &str| {
|
||||
text_body(vec![
|
||||
user("weather in Shanghai?"),
|
||||
json!({
|
||||
"role": "assistant",
|
||||
"content": null,
|
||||
"tool_calls": tool_call("get_weather", "{}")
|
||||
}),
|
||||
json!({"role": "tool", "tool_call_id": id, "content": "sunny"}),
|
||||
user("thanks"),
|
||||
])
|
||||
};
|
||||
let a = mk("call_1");
|
||||
let b = mk("call_2");
|
||||
let c = mk("call_1");
|
||||
let (ca, cb, cc) = (chain(&a), chain(&b), chain(&c));
|
||||
assert_eq!(ca[..2], cb[..2], "before the tool result: unchanged");
|
||||
assert_ne!(ca[2], cb[2], "different tool_call_id must diverge");
|
||||
assert_ne!(ca[3], cb[3], "and stay diverged afterwards");
|
||||
assert_eq!(ca, cc, "identical tool_call_id must hit");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn name_field_enters_chain() {
|
||||
let a = text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]);
|
||||
let b = text_body(vec![json!({"role": "user", "name": "bob", "content": "hi"})]);
|
||||
let c = text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]);
|
||||
assert_ne!(chain(&a), chain(&b), "different name must diverge");
|
||||
assert_eq!(chain(&a), chain(&c), "identical name must hit");
|
||||
|
||||
// Absent name and empty meta stay equivalent to a plain message.
|
||||
let plain = text_body(vec![user("hi")]);
|
||||
let no_name = text_body(vec![json!({"role": "user", "content": "hi"})]);
|
||||
assert_eq!(chain(&plain), chain(&no_name));
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,8 @@
|
||||
//! HTTP router implementations
|
||||
|
||||
pub mod g9i;
|
||||
pub mod g9i_prefix_chain;
|
||||
pub mod g9i_shadow;
|
||||
pub mod pd_router;
|
||||
pub mod pd_types;
|
||||
pub mod router;
|
||||
|
||||
@@ -18,6 +18,7 @@ use serde_json::{json, Value};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::g9i_shadow::{ShadowRequestContext, ShadowResponseContext};
|
||||
use super::pd_types::api_path;
|
||||
use crate::{
|
||||
config::types::RetryConfig,
|
||||
@@ -58,6 +59,8 @@ pub struct PDRouter {
|
||||
pub retry_config: RetryConfig,
|
||||
pub api_key: Option<String>,
|
||||
pub enable_igw: bool,
|
||||
/// G9i pool admission + fast-P routing state; None = feature off.
|
||||
pub g9i: Option<Arc<super::g9i::G9iState>>,
|
||||
}
|
||||
|
||||
struct PreparedWorkerRequest<'a> {
|
||||
@@ -182,6 +185,14 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
pub async fn new(ctx: &Arc<crate::app_context::AppContext>) -> Result<Self, String> {
|
||||
let g9i = match &ctx.router_config.g9i {
|
||||
Some(config) if config.enabled => {
|
||||
let state = Arc::new(super::g9i::G9iState::new(config.clone()));
|
||||
state.start_monitor(Arc::clone(&ctx.worker_registry), ctx.client.clone());
|
||||
Some(state)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
Ok(PDRouter {
|
||||
worker_registry: Arc::clone(&ctx.worker_registry),
|
||||
policy_registry: Arc::clone(&ctx.policy_registry),
|
||||
@@ -189,6 +200,7 @@ impl PDRouter {
|
||||
retry_config: ctx.router_config.effective_retry_config(),
|
||||
api_key: ctx.router_config.api_key.clone(),
|
||||
enable_igw: ctx.router_config.enable_igw,
|
||||
g9i,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -390,6 +402,90 @@ impl PDRouter {
|
||||
// Clone request once outside the retry loop, then use Arc to share across attempts
|
||||
// This avoids O(retries) clones by sharing the same data
|
||||
let shared_request = Arc::new(original_request.clone());
|
||||
|
||||
// G9i admission (router-side, zero engine changes). Computed once per
|
||||
// request, outside the retry loop: our own 429s must never be
|
||||
// retried. `ttft_path` labels the TTFT histogram dimension for this
|
||||
// request's eventual first-token sample. `shadow_ctx` is the G9i v2
|
||||
// shadow bookkeeping carrier (T2): Some only for chat requests with
|
||||
// a classifiable prefix chain while shadow mode is on; it never
|
||||
// influences routing.
|
||||
let (g9i_prefill_override, g9i_ttft_path, shadow_ctx): (
|
||||
Option<String>,
|
||||
&'static str,
|
||||
Option<ShadowRequestContext>,
|
||||
) = if let Some(g9i) = &self.g9i {
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
if context.route == "/v1/chat/completions" {
|
||||
match serde_json::to_value(original_request) {
|
||||
Ok(body) => {
|
||||
// Shadow T2: compute the prefix chain once, here,
|
||||
// alongside the v1 classification. Skipped entirely
|
||||
// when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await
|
||||
{
|
||||
(plan, path, probe_new_work) => {
|
||||
// Shadow T3: shadow-classify alongside the v1
|
||||
// verdict and record agreement/divergence
|
||||
// counters. Routing still follows the v1 plan
|
||||
// alone; the decision only rides the context
|
||||
// into dispatch for queue bookkeeping.
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
if let Some(decision) = g9i.shadow_classify(&ctx) {
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
ctx.decision = Some(decision);
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
match plan.into_override() {
|
||||
Ok(override_url) => (override_url, path, shadow_ctx),
|
||||
Err(info) => {
|
||||
metrics::counter!(
|
||||
"smg_g9i_rejects_total",
|
||||
"reason" => info.reason
|
||||
)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(_) => (None, "regular", None),
|
||||
}
|
||||
} else {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
(None, "non_chat", None)
|
||||
}
|
||||
} else {
|
||||
(None, "untracked", None)
|
||||
};
|
||||
|
||||
let response = RetryExecutor::execute_response_with_retry(
|
||||
&self.retry_config,
|
||||
{
|
||||
@@ -397,6 +493,8 @@ impl PDRouter {
|
||||
// Clone Arc (cheap reference count increment) instead of cloning the entire request
|
||||
let shared_request = Arc::clone(&shared_request);
|
||||
let context = context.clone();
|
||||
let g9i_prefill_override = g9i_prefill_override.clone();
|
||||
let shadow_ctx = shadow_ctx.clone();
|
||||
async move {
|
||||
let (prefill, decode) = match self
|
||||
.select_pd_pair(
|
||||
@@ -412,6 +510,38 @@ impl PDRouter {
|
||||
}
|
||||
};
|
||||
|
||||
// G9i least-wait override: route onto the classified
|
||||
// fast-P node instead of the policy pick.
|
||||
let (prefill, decode) =
|
||||
if let Some(override_url) = g9i_prefill_override.as_ref() {
|
||||
let matched = self
|
||||
.worker_registry
|
||||
.get_prefill_workers()
|
||||
.into_iter()
|
||||
.find(|w| {
|
||||
w.url().trim_end_matches('/') == override_url.as_str()
|
||||
&& w.is_available()
|
||||
});
|
||||
match matched {
|
||||
Some(w) => {
|
||||
debug!(
|
||||
"G9i: overriding prefill selection with {}",
|
||||
override_url
|
||||
);
|
||||
(w, decode)
|
||||
}
|
||||
None => {
|
||||
warn!(
|
||||
"G9i: override target {} unavailable, keeping policy pick",
|
||||
override_url
|
||||
);
|
||||
(prefill, decode)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(prefill, decode)
|
||||
};
|
||||
|
||||
debug!(
|
||||
"PD retry attempt {} using prefill={} decode={}",
|
||||
attempt,
|
||||
@@ -428,6 +558,15 @@ impl PDRouter {
|
||||
json_request["stream"] = Value::Bool(context.is_stream);
|
||||
}
|
||||
|
||||
// G9i shadow T2: inject the usage / cache-details
|
||||
// flags the client didn't set (chat + shadow mode
|
||||
// only; `shadow_ctx` is None otherwise). Paired with
|
||||
// response-side stripping: we inject exactly what we
|
||||
// later strip, so clients see no change.
|
||||
if let Some(sctx) = &shadow_ctx {
|
||||
sctx.inject(&mut json_request);
|
||||
}
|
||||
|
||||
json_request = match Self::inject_bootstrap_into_value(
|
||||
json_request,
|
||||
prefill.as_ref(),
|
||||
@@ -446,6 +585,8 @@ impl PDRouter {
|
||||
Arc::clone(&prefill),
|
||||
Arc::clone(&decode),
|
||||
start_time,
|
||||
g9i_ttft_path,
|
||||
shadow_ctx,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -575,6 +716,9 @@ impl PDRouter {
|
||||
Some(response_headers),
|
||||
prefill,
|
||||
decode,
|
||||
None,
|
||||
// Error envelope: no usage will ever arrive — no shadow hook.
|
||||
None,
|
||||
)
|
||||
} else {
|
||||
// Handle non-streaming error response
|
||||
@@ -649,6 +793,7 @@ impl PDRouter {
|
||||
}
|
||||
|
||||
// Internal method that performs the actual dual dispatch (without retry logic)
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn execute_dual_dispatch_internal(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
@@ -656,7 +801,11 @@ impl PDRouter {
|
||||
context: PDRequestContext<'_>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
_start_time: Instant,
|
||||
start_time: Instant,
|
||||
g9i_path: &'static str,
|
||||
// G9i shadow T2: prefix chain + injection flags; None = shadow off
|
||||
// or unclassifiable request — zero behavior change.
|
||||
shadow_ctx: Option<ShadowRequestContext>,
|
||||
) -> Response {
|
||||
// For non-streaming: use guard for automatic load management
|
||||
// For streaming: load will be managed in create_streaming_response
|
||||
@@ -868,6 +1017,23 @@ impl PDRouter {
|
||||
|
||||
let response_headers = header_utils::preserve_response_headers(res.headers());
|
||||
|
||||
// Shadow T2: hand the chain + the ACTUAL prefill landing
|
||||
// node (post-override) to the stream wrapper, which
|
||||
// harvests the usage event and strips injected fields.
|
||||
let shadow = shadow_ctx.map(|ctx| {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
// ACTUAL landing node; the stream wrapper pairs this with
|
||||
// exactly one complete (usage recorded) or release
|
||||
// (abort/disconnect/error) before teardown.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
|
||||
self.create_streaming_response(
|
||||
res.bytes_stream(),
|
||||
status,
|
||||
@@ -876,15 +1042,31 @@ impl PDRouter {
|
||||
Some(response_headers),
|
||||
prefill,
|
||||
decode,
|
||||
Some((start_time, g9i_path)),
|
||||
shadow,
|
||||
)
|
||||
} else {
|
||||
// Non-streaming response
|
||||
let shadow = shadow_ctx.map(|ctx| {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
// ACTUAL landing node; paired with complete/release in
|
||||
// shadow_record_non_stream (or the read-failure paths
|
||||
// below / in process_non_streaming_response).
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
if context.return_logprob {
|
||||
self.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
@@ -894,6 +1076,8 @@ impl PDRouter {
|
||||
|
||||
match res.bytes().await {
|
||||
Ok(decode_body) => {
|
||||
let decode_body =
|
||||
self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
@@ -901,6 +1085,11 @@ impl PDRouter {
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to read decode response: {}", e);
|
||||
// Shadow T3: the admitted dispatch produced no
|
||||
// readable body — release the queue slot.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_release(s);
|
||||
}
|
||||
error::internal_error(
|
||||
"read_response_failed",
|
||||
"Failed to read response",
|
||||
@@ -1116,11 +1305,39 @@ impl PDRouter {
|
||||
headers: Option<HeaderMap>,
|
||||
prefill: Arc<dyn Worker>,
|
||||
decode: Arc<dyn Worker>,
|
||||
// TTFT sampling hook: Some((dispatch-entry instant, path label)) for
|
||||
// real upstream streams; None for synthetic error envelopes (their
|
||||
// "first chunk" is not a token — sampling them would poison the
|
||||
// window with near-zero error samples) and for tests.
|
||||
ttft: Option<(Instant, &'static str)>,
|
||||
// G9i shadow T2 hook: Some only for shadowable chat requests on a
|
||||
// success-status stream; None on error envelopes and tests.
|
||||
shadow: Option<ShadowResponseContext>,
|
||||
) -> Response {
|
||||
use crate::core::AttachedBody;
|
||||
|
||||
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
|
||||
|
||||
// G9i TTFT sample: record on the first upstream chunk — that moment is
|
||||
// the request's time-to-first-token as seen by the gateway (dispatch
|
||||
// entry -> first token). Non-streaming requests have no observable
|
||||
// first-token moment and are not sampled.
|
||||
let g9i_ttft = self.g9i.clone().zip(ttft);
|
||||
let mut ttft_recorded = false;
|
||||
|
||||
// G9i shadow T2: zip the state handle with the per-request context
|
||||
// (chain + node + strip flags) for the stream loop below.
|
||||
let g9i_shadow = self.g9i.clone().zip(shadow);
|
||||
// Shadow T3 queue bookkeeping state. `ttft` is Option<(Instant,
|
||||
// &'static str)> which is Copy, so zip() above left it usable:
|
||||
// mirror the dispatch-entry instant so the first chunk can also feed
|
||||
// the shadow TTFT calibration. `shadow_done` latches once usage is
|
||||
// recorded; teardown releases the admission only when no completion
|
||||
// happened (abort / client disconnect / upstream error).
|
||||
let ttft_start = ttft.map(|(t, _)| t);
|
||||
let mut shadow_ttft_secs: Option<f64> = None;
|
||||
let mut shadow_done = false;
|
||||
|
||||
// Uses select! to race stream.next() against tx.closed() so that
|
||||
// when the client disconnects the upstream HTTP connection is dropped
|
||||
// promptly, allowing the engine to abort the request.
|
||||
@@ -1149,8 +1366,54 @@ impl PDRouter {
|
||||
chunk_result = tracked.next() => {
|
||||
match chunk_result {
|
||||
Some(Ok(chunk)) => {
|
||||
// G9i shadow T2: harvest the usage event for
|
||||
// prefix-table bookkeeping and strip exactly
|
||||
// the fields we injected on the way in.
|
||||
// Chunks without a usage field pass through
|
||||
// byte-identically. A chunk reduced to nothing
|
||||
// (usage-only event we must hide from the
|
||||
// client) is dropped entirely.
|
||||
let chunk = if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
let (usage, forward) =
|
||||
super::g9i_shadow::process_sse_chunk(
|
||||
&chunk,
|
||||
sctx.strip_usage,
|
||||
sctx.strip_cached_details,
|
||||
);
|
||||
if let Some(json) = usage {
|
||||
// Shadow T3: exactly one completion
|
||||
// per admission — the latch keeps a
|
||||
// (malformed) repeat usage event from
|
||||
// double-completing the queue slot.
|
||||
if !shadow_done {
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs);
|
||||
shadow_done = true;
|
||||
}
|
||||
}
|
||||
match forward {
|
||||
Some(bytes) => bytes,
|
||||
None => continue,
|
||||
}
|
||||
} else {
|
||||
chunk
|
||||
};
|
||||
|
||||
let is_done = memmem::find(&chunk, b"data: [DONE]").is_some();
|
||||
|
||||
if !ttft_recorded {
|
||||
ttft_recorded = true;
|
||||
if let Some((g9i, (start_time, g9i_path))) = &g9i_ttft {
|
||||
g9i.record_ttft(
|
||||
g9i_path,
|
||||
start_time.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
// Shadow T3: the same first-token moment
|
||||
// calibrates est_TTFT (est − actual).
|
||||
shadow_ttft_secs =
|
||||
ttft_start.map(|t| t.elapsed().as_secs_f64());
|
||||
}
|
||||
|
||||
let result = if return_logprob && prefill_logprobs.is_some() {
|
||||
Self::merge_streaming_logprobs(prefill_logprobs.clone(), &chunk)
|
||||
.unwrap_or(chunk)
|
||||
@@ -1199,6 +1462,15 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shadow T3: the stream ended without a recorded usage event
|
||||
// (abort, client disconnect, or upstream error) — release the
|
||||
// queue admission so outstanding stays conserved. No-op when the
|
||||
// usage event already completed it.
|
||||
if !shadow_done {
|
||||
if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
g9i.shadow_release(sctx);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let stream = UnboundedReceiverStream::new(rx);
|
||||
@@ -1226,16 +1498,26 @@ impl PDRouter {
|
||||
status: StatusCode,
|
||||
return_logprob: bool,
|
||||
prefill_body: Option<bytes::Bytes>,
|
||||
shadow: Option<ShadowResponseContext>,
|
||||
) -> Response {
|
||||
let response = res.bytes().await;
|
||||
let decode_body = match response {
|
||||
Ok(decode_body) => decode_body,
|
||||
Err(e) => {
|
||||
error!("Failed to read decode response: {}", e);
|
||||
// Shadow T3: the admitted dispatch produced no readable
|
||||
// body — release the queue slot.
|
||||
if let (Some(g9i), Some(s)) = (&self.g9i, &shadow) {
|
||||
g9i.shadow_release(s);
|
||||
}
|
||||
return error::internal_error("read_response_failed", "Failed to read response");
|
||||
}
|
||||
};
|
||||
|
||||
// Shadow T2: record usage + strip injected fields before any
|
||||
// logprob merging (merge touches choices only, usage is untouched).
|
||||
let decode_body = self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
|
||||
if !return_logprob {
|
||||
return (status, decode_body).into_response();
|
||||
}
|
||||
@@ -1265,6 +1547,47 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// G9i shadow bookkeeping for non-streaming responses: parse the body,
|
||||
/// record usage into the prefix table (T2), pair the T3 queue admission
|
||||
/// with exactly one complete (parseable body) or release (non-success /
|
||||
/// unparseable), and strip `cached_tokens_details` when (and only when)
|
||||
/// we injected the request flag ourselves.
|
||||
///
|
||||
/// Passthrough guarantees: shadow off / non-success status / unparseable
|
||||
/// body / nothing stripped -> the ORIGINAL bytes are returned untouched,
|
||||
/// so clients never see a re-serialized response unless we actually
|
||||
/// removed an injected field.
|
||||
fn shadow_record_non_stream(
|
||||
&self,
|
||||
body: bytes::Bytes,
|
||||
status: StatusCode,
|
||||
shadow: &Option<ShadowResponseContext>,
|
||||
) -> bytes::Bytes {
|
||||
let (Some(g9i), Some(ctx)) = (&self.g9i, shadow) else {
|
||||
return body;
|
||||
};
|
||||
// Only 200 responses with usage may write the table (design §3.3-3).
|
||||
// A non-success response still releases the T3 queue admission.
|
||||
if !status.is_success() {
|
||||
g9i.shadow_release(ctx);
|
||||
return body;
|
||||
}
|
||||
let (parsed, rewritten) =
|
||||
super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details);
|
||||
match &parsed {
|
||||
// TTFT calibration is streaming-only (non-streaming has no
|
||||
// observable first-token moment) -> None.
|
||||
Some(json) => g9i.shadow_record(ctx, json, None),
|
||||
// Unparseable success body: nothing to record, but the dispatch
|
||||
// is over — release rather than leak the outstanding count.
|
||||
None => g9i.shadow_release(ctx),
|
||||
}
|
||||
match rewritten {
|
||||
Some(bytes) => bytes::Bytes::from(bytes),
|
||||
None => body,
|
||||
}
|
||||
}
|
||||
|
||||
// Helper to process prefill response and extract body if needed for logprobs
|
||||
async fn process_prefill_response(
|
||||
&self,
|
||||
@@ -1780,6 +2103,7 @@ mod tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: Some("test_api_key".to_string()),
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2048,6 +2372,8 @@ mod tests {
|
||||
None,
|
||||
prefill_ref.clone(),
|
||||
decode_ref.clone(),
|
||||
None,
|
||||
None,
|
||||
);
|
||||
|
||||
// Guards are now attached to response body, so load should be 1
|
||||
|
||||
@@ -321,6 +321,7 @@ mod pd_responses_routing_tests {
|
||||
retry_config: RetryConfig::default(),
|
||||
api_key: None,
|
||||
enable_igw: false,
|
||||
g9i: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"v":1}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "7455eaee1da5212ae5e0fd33d80a903029d2f981"
|
||||
},
|
||||
"path_in_vcs": "protocols"
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
name = "openai-protocol"
|
||||
version = "1.0.0"
|
||||
authors = [
|
||||
"Simo Lin <linsimo.mark@gmail.com>",
|
||||
"Chang Su <mckvtl@gmail.com>",
|
||||
"Keyang Ru <rukeyang@gmail.com>",
|
||||
]
|
||||
build = false
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = "OpenAI-compatible API protocol definitions and types"
|
||||
readme = false
|
||||
keywords = [
|
||||
"openai",
|
||||
"llm",
|
||||
"api",
|
||||
"protocol",
|
||||
"types",
|
||||
]
|
||||
categories = [
|
||||
"api-bindings",
|
||||
"data-structures",
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/lightseekorg/smg"
|
||||
|
||||
[features]
|
||||
axum = ["dep:axum"]
|
||||
default = []
|
||||
|
||||
[lib]
|
||||
name = "openai_protocol"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies.axum]
|
||||
version = "0.8.6"
|
||||
optional = true
|
||||
|
||||
[dependencies.chrono]
|
||||
version = "0.4"
|
||||
|
||||
[dependencies.rand]
|
||||
version = "0.9.2"
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1.0"
|
||||
features = ["derive"]
|
||||
|
||||
[dependencies.serde_json]
|
||||
version = "1.0"
|
||||
features = ["preserve_order"]
|
||||
|
||||
[dependencies.tokio]
|
||||
version = "1.42.0"
|
||||
|
||||
[dependencies.tracing]
|
||||
version = "0.1"
|
||||
|
||||
[dependencies.validator]
|
||||
version = "0.20.0"
|
||||
features = ["derive"]
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "openai-protocol"
|
||||
version = "1.0.0"
|
||||
edition = "2021"
|
||||
description = "OpenAI-compatible API protocol definitions and types"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/lightseekorg/smg"
|
||||
authors = [
|
||||
"Simo Lin <linsimo.mark@gmail.com>",
|
||||
"Chang Su <mckvtl@gmail.com>",
|
||||
"Keyang Ru <rukeyang@gmail.com>",
|
||||
]
|
||||
keywords = ["openai", "llm", "api", "protocol", "types"]
|
||||
categories = ["api-bindings", "data-structures"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
axum = ["dep:axum"]
|
||||
|
||||
[dependencies]
|
||||
axum = { workspace = true, optional = true }
|
||||
chrono.workspace = true
|
||||
rand.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
# Only used by protocols
|
||||
validator = { version = "0.20.0", features = ["derive"] }
|
||||
@@ -0,0 +1,7 @@
|
||||
//! Builders for Chat Completion API response types
|
||||
|
||||
pub mod response;
|
||||
pub mod stream_response;
|
||||
|
||||
pub use response::ChatCompletionResponseBuilder;
|
||||
pub use stream_response::ChatCompletionStreamResponseBuilder;
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Builder for ChatCompletionResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing chat completion responses.
|
||||
|
||||
use crate::{chat::*, common::Usage};
|
||||
|
||||
/// Builder for ChatCompletionResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing chat completion responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice
|
||||
pub fn add_choice(mut self, choice: ChatChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionResponse
|
||||
pub fn build(self) -> ChatCompletionResponse {
|
||||
ChatCompletionResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "chat.completion");
|
||||
assert!(response.choices.is_empty());
|
||||
assert!(response.usage.is_none());
|
||||
assert!(response.system_fingerprint.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let choice = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Hello!".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4")
|
||||
.choices(vec![choice.clone()])
|
||||
.maybe_usage(Some(usage))
|
||||
.maybe_system_fingerprint(Some("fp_123abc"))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_456");
|
||||
assert_eq!(response.choices.len(), 1);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert!(response.usage.is_some());
|
||||
assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_multiple_choices() {
|
||||
let choice1 = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 1".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let choice2 = ChatChoice {
|
||||
index: 1,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 2".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4")
|
||||
.add_choice(choice1)
|
||||
.add_choice(choice2)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.choices.len(), 2);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert_eq!(response.choices[1].index, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
}
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
//! Builder for ChatCompletionStreamResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing streaming chat completion responses.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::{
|
||||
chat::*,
|
||||
common::{FunctionCallDelta, ToolCallDelta, Usage},
|
||||
};
|
||||
|
||||
/// Builder for ChatCompletionStreamResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionStreamResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatStreamChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion.chunk")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatStreamChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice (delta)
|
||||
pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information (typically sent in final chunk)
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role` and `content`
|
||||
pub fn add_choice_content(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role`, `content`, and `logprobs`
|
||||
pub fn add_choice_content_with_logprobs(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
logprobs: Option<crate::common::ChatLogProbs>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that only sets `role`
|
||||
pub fn add_choice_role(mut self, index: u32, role: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that appends a tool-call *arguments delta*
|
||||
/// Uses `Cow` so you can pass `&str` or `String` without extra clones
|
||||
pub fn add_choice_tool_args(
|
||||
mut self,
|
||||
index: u32,
|
||||
args_delta: impl Into<Cow<'static, str>>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: None,
|
||||
tool_type: None,
|
||||
function: Some(FunctionCallDelta {
|
||||
name: None,
|
||||
arguments: Some(args_delta.into().into_owned()),
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets reasoning content (for models that stream reasoning)
|
||||
pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: Some(reasoning.into()),
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta for tool call with function name and ID
|
||||
pub fn add_choice_tool_name(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_id: impl Into<String>,
|
||||
function_name: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: Some(tool_call_id.into()),
|
||||
tool_type: Some("function".to_string()),
|
||||
function: Some(FunctionCallDelta {
|
||||
name: Some(function_name.into()),
|
||||
arguments: None,
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta with a pre-constructed ToolCallDelta
|
||||
/// Useful when you already have a ToolCallDelta object to emit
|
||||
pub fn add_choice_tool_call_delta(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_delta: ToolCallDelta,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![tool_call_delta]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice with finish_reason (final chunk)
|
||||
/// This is used for the last chunk in a stream to signal completion
|
||||
pub fn add_choice_finish_reason(
|
||||
mut self,
|
||||
index: u32,
|
||||
finish_reason: impl Into<String>,
|
||||
matched_stop: Option<serde_json::Value>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: None,
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some(finish_reason.into()),
|
||||
matched_stop,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionStreamResponse
|
||||
pub fn build(self) -> ChatCompletionStreamResponse {
|
||||
ChatCompletionStreamResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(chunk.id, "chatcmpl_123");
|
||||
assert_eq!(chunk.model, "gpt-4");
|
||||
assert_eq!(chunk.object, "chat.completion.chunk");
|
||||
assert!(chunk.choices.is_empty());
|
||||
assert!(chunk.usage.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_content_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].index, 0);
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].finish_reason.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_role_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_finish_reason() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
assert!(chunk.choices[0].delta.role.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_deltas() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.add_choice_content(0, "assistant", " world")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_usage() {
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.usage(usage)
|
||||
.build();
|
||||
|
||||
assert!(chunk.usage.is_some());
|
||||
assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.add_choice_content(0, "assistant", "test")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_choice_explicit() {
|
||||
let choice = ChatStreamChoice {
|
||||
index: 0,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: Some("Hello".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4")
|
||||
.add_choice(choice)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Builder patterns for protocol response types
|
||||
//!
|
||||
//! This module provides ergonomic builders for response types with many optional fields.
|
||||
//! Builders help avoid telescoping constructors and make construction intent clear.
|
||||
//!
|
||||
//! # Organization
|
||||
//!
|
||||
//! Builders are organized by API:
|
||||
//! - `chat/` - Chat Completion API builders (response, stream_response)
|
||||
//! - `responses/` - Responses API builder (response)
|
||||
//!
|
||||
//! # Optional Fields
|
||||
//!
|
||||
//! For optional fields, builders provide `maybe_*` methods that handle `Option<T>` directly:
|
||||
//! ```ignore
|
||||
//! builder
|
||||
//! .field(value)
|
||||
//! .maybe_optional_field(optional_value) // Accepts Option<T>
|
||||
//! .build()
|
||||
//! ```
|
||||
|
||||
pub mod chat;
|
||||
pub mod responses;
|
||||
|
||||
// Re-export all builders for convenient access
|
||||
pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder};
|
||||
pub use responses::ResponsesResponseBuilder;
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Builders for Responses API response types
|
||||
|
||||
pub mod response;
|
||||
|
||||
pub use response::ResponsesResponseBuilder;
|
||||
@@ -0,0 +1,411 @@
|
||||
//! Builder for ResponsesResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing ResponsesResponse instances.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::responses::*;
|
||||
|
||||
/// Builder for ResponsesResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ResponsesResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created_at: i64,
|
||||
status: ResponseStatus,
|
||||
error: Option<Value>,
|
||||
incomplete_details: Option<Value>,
|
||||
instructions: Option<String>,
|
||||
max_output_tokens: Option<u32>,
|
||||
model: String,
|
||||
output: Vec<ResponseOutputItem>,
|
||||
parallel_tool_calls: bool,
|
||||
previous_response_id: Option<String>,
|
||||
reasoning: Option<ReasoningInfo>,
|
||||
store: bool,
|
||||
temperature: Option<f32>,
|
||||
text: Option<TextConfig>,
|
||||
tool_choice: String,
|
||||
tools: Vec<ResponseTool>,
|
||||
top_p: Option<f32>,
|
||||
truncation: Option<String>,
|
||||
usage: Option<ResponsesUsage>,
|
||||
user: Option<String>,
|
||||
safety_identifier: Option<String>,
|
||||
metadata: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ResponsesResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Response ID (e.g., "resp_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "response".to_string(),
|
||||
created_at: chrono::Utc::now().timestamp(),
|
||||
status: ResponseStatus::InProgress,
|
||||
error: None,
|
||||
incomplete_details: None,
|
||||
instructions: None,
|
||||
max_output_tokens: None,
|
||||
model: model.into(),
|
||||
output: Vec::new(),
|
||||
parallel_tool_calls: true,
|
||||
previous_response_id: None,
|
||||
reasoning: None,
|
||||
store: true,
|
||||
temperature: None,
|
||||
text: None,
|
||||
tool_choice: "auto".to_string(),
|
||||
tools: Vec::new(),
|
||||
top_p: None,
|
||||
truncation: None,
|
||||
usage: None,
|
||||
user: None,
|
||||
safety_identifier: None,
|
||||
metadata: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ResponsesRequest
|
||||
///
|
||||
/// This populates fields like instructions, max_output_tokens, temperature, etc.
|
||||
/// from the original request, making it easy to construct a response that mirrors
|
||||
/// the request parameters.
|
||||
///
|
||||
/// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation
|
||||
/// and should be set independently from the request's `user` field (which is for billing/tracking).
|
||||
pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self {
|
||||
self.instructions = request.instructions.clone();
|
||||
self.max_output_tokens = request.max_output_tokens;
|
||||
self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true);
|
||||
self.previous_response_id = request.previous_response_id.clone();
|
||||
self.store = request.store.unwrap_or(true);
|
||||
self.temperature = request.temperature;
|
||||
self.tool_choice = if let Some(ref tc) = request.tool_choice {
|
||||
serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())
|
||||
} else {
|
||||
"auto".to_string()
|
||||
};
|
||||
self.tools = request.tools.clone().unwrap_or_default();
|
||||
self.top_p = request.top_p;
|
||||
self.user = request.user.clone();
|
||||
self.metadata = request.metadata.clone().unwrap_or_default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "response")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created_at(mut self, timestamp: i64) -> Self {
|
||||
self.created_at = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the response status
|
||||
pub fn status(mut self, status: ResponseStatus) -> Self {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set error information (if status is failed)
|
||||
pub fn error(mut self, error: Value) -> Self {
|
||||
self.error = Some(error);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set incomplete details (if response was truncated)
|
||||
pub fn incomplete_details(mut self, details: Value) -> Self {
|
||||
self.incomplete_details = Some(details);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system instructions
|
||||
pub fn instructions(mut self, instructions: impl Into<String>) -> Self {
|
||||
self.instructions = Some(instructions.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set max output tokens
|
||||
pub fn max_output_tokens(mut self, tokens: u32) -> Self {
|
||||
self.max_output_tokens = Some(tokens);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set output items
|
||||
pub fn output(mut self, output: Vec<ResponseOutputItem>) -> Self {
|
||||
self.output = output;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single output item
|
||||
pub fn add_output(mut self, item: ResponseOutputItem) -> Self {
|
||||
self.output.push(item);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether parallel tool calls are enabled
|
||||
pub fn parallel_tool_calls(mut self, enabled: bool) -> Self {
|
||||
self.parallel_tool_calls = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set previous response ID (if continuation)
|
||||
pub fn previous_response_id(mut self, id: impl Into<String>) -> Self {
|
||||
self.previous_response_id = Some(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set reasoning information
|
||||
pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self {
|
||||
self.reasoning = Some(reasoning);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether the response is stored
|
||||
pub fn store(mut self, store: bool) -> Self {
|
||||
self.store = store;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set temperature setting
|
||||
pub fn temperature(mut self, temperature: f32) -> Self {
|
||||
self.temperature = Some(temperature);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set text format settings if provided (handles Option)
|
||||
pub fn maybe_text(mut self, text: Option<TextConfig>) -> Self {
|
||||
if let Some(t) = text {
|
||||
self.text = Some(t);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set tool choice setting
|
||||
pub fn tool_choice(mut self, tool_choice: impl Into<String>) -> Self {
|
||||
self.tool_choice = tool_choice.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set available tools
|
||||
pub fn tools(mut self, tools: Vec<ResponseTool>) -> Self {
|
||||
self.tools = tools;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set top-p setting
|
||||
pub fn top_p(mut self, top_p: f32) -> Self {
|
||||
self.top_p = Some(top_p);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set truncation strategy
|
||||
pub fn truncation(mut self, truncation: impl Into<String>) -> Self {
|
||||
self.truncation = Some(truncation.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: ResponsesUsage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<ResponsesUsage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Copy from request if provided (handles Option)
|
||||
pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self {
|
||||
if let Some(req) = request {
|
||||
self = self.copy_from_request(req);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set user identifier
|
||||
pub fn user(mut self, user: impl Into<String>) -> Self {
|
||||
self.user = Some(user.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set safety identifier
|
||||
pub fn safety_identifier(mut self, identifier: impl Into<String>) -> Self {
|
||||
self.safety_identifier = Some(identifier.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set metadata
|
||||
pub fn metadata(mut self, metadata: HashMap<String, Value>) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single metadata entry
|
||||
pub fn add_metadata(mut self, key: impl Into<String>, value: Value) -> Self {
|
||||
self.metadata.insert(key.into(), value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ResponsesResponse
|
||||
pub fn build(self) -> ResponsesResponse {
|
||||
ResponsesResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created_at: self.created_at,
|
||||
status: self.status,
|
||||
error: self.error,
|
||||
incomplete_details: self.incomplete_details,
|
||||
instructions: self.instructions,
|
||||
max_output_tokens: self.max_output_tokens,
|
||||
model: self.model,
|
||||
output: self.output,
|
||||
parallel_tool_calls: self.parallel_tool_calls,
|
||||
previous_response_id: self.previous_response_id,
|
||||
reasoning: self.reasoning,
|
||||
store: self.store,
|
||||
temperature: self.temperature,
|
||||
text: self.text,
|
||||
tool_choice: self.tool_choice,
|
||||
tools: self.tools,
|
||||
top_p: self.top_p,
|
||||
truncation: self.truncation,
|
||||
usage: self.usage,
|
||||
user: self.user,
|
||||
safety_identifier: self.safety_identifier,
|
||||
metadata: self.metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "resp_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "response");
|
||||
assert_eq!(response.status, ResponseStatus::InProgress);
|
||||
assert!(response.output.is_empty());
|
||||
assert!(response.parallel_tool_calls);
|
||||
assert!(response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4")
|
||||
.status(ResponseStatus::Completed)
|
||||
.instructions("You are a helpful assistant")
|
||||
.max_output_tokens(1000)
|
||||
.temperature(0.7)
|
||||
.top_p(0.9)
|
||||
.parallel_tool_calls(false)
|
||||
.store(false)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.status, ResponseStatus::Completed);
|
||||
assert_eq!(
|
||||
response.instructions.as_ref().unwrap(),
|
||||
"You are a helpful assistant"
|
||||
);
|
||||
assert_eq!(response.max_output_tokens, Some(1000));
|
||||
assert_eq!(response.temperature, Some(0.7));
|
||||
assert_eq!(response.top_p, Some(0.9));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ResponsesRequest {
|
||||
model: "gpt-4".to_string(),
|
||||
input: ResponseInput::Text("test".to_string()),
|
||||
instructions: Some("Be helpful".to_string()),
|
||||
max_output_tokens: Some(500),
|
||||
temperature: Some(0.8),
|
||||
top_p: Some(0.95),
|
||||
parallel_tool_calls: Some(false),
|
||||
store: Some(false),
|
||||
user: Some("user_123".to_string()),
|
||||
metadata: Some(HashMap::from([(
|
||||
"key".to_string(),
|
||||
serde_json::json!("value"),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ResponsesResponse::builder("resp_456", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.status(ResponseStatus::Completed)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful");
|
||||
assert_eq!(response.max_output_tokens, Some(500));
|
||||
assert_eq!(response.temperature, Some(0.8));
|
||||
assert_eq!(response.top_p, Some(0.95));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
assert_eq!(response.user.as_ref().unwrap(), "user_123");
|
||||
assert_eq!(
|
||||
response.metadata.get("key").unwrap(),
|
||||
&serde_json::json!("value")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_output_items() {
|
||||
let response = ResponsesResponse::builder("resp_789", "gpt-4")
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_1".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_2".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.build();
|
||||
|
||||
assert_eq!(response.output.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_metadata() {
|
||||
let response = ResponsesResponse::builder("resp_101", "gpt-4")
|
||||
.add_metadata("key1", serde_json::json!("value1"))
|
||||
.add_metadata("key2", serde_json::json!(42))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.metadata.len(), 2);
|
||||
assert_eq!(response.metadata.get("key1").unwrap(), "value1");
|
||||
assert_eq!(response.metadata.get("key2").unwrap(), 42);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,799 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{
|
||||
default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function,
|
||||
FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions,
|
||||
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference,
|
||||
Usage,
|
||||
},
|
||||
sampling_params::{validate_top_k_value, validate_top_p_value},
|
||||
};
|
||||
use crate::{
|
||||
builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder},
|
||||
validated::Normalizable,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Chat Messages
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "role")]
|
||||
pub enum ChatMessage {
|
||||
#[serde(rename = "system")]
|
||||
System {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "user")]
|
||||
User {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "assistant")]
|
||||
Assistant {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<MessageContent>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tool_calls: Option<Vec<ToolCall>>,
|
||||
/// Reasoning content for O1-style models (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning_content: Option<String>,
|
||||
},
|
||||
#[serde(rename = "tool")]
|
||||
Tool {
|
||||
content: MessageContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
#[serde(rename = "function")]
|
||||
Function { content: String, name: String },
|
||||
#[serde(rename = "developer")]
|
||||
Developer {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tools: Option<Vec<Tool>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ContentPart>),
|
||||
}
|
||||
|
||||
impl MessageContent {
|
||||
/// Returns the text content, cloning only when necessary.
|
||||
/// For simple text, returns a clone of the string.
|
||||
/// For parts, concatenates text parts with spaces.
|
||||
/// Optimized to avoid intermediate Vec allocation.
|
||||
pub fn to_simple_string(&self) -> String {
|
||||
match self {
|
||||
MessageContent::Text(text) => text.clone(),
|
||||
MessageContent::Parts(parts) => {
|
||||
// Use fold to build string directly without intermediate Vec allocation
|
||||
let mut result = String::new();
|
||||
let mut first = true;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !first {
|
||||
result.push(' ');
|
||||
}
|
||||
result.push_str(text);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends text content directly to a buffer, avoiding intermediate allocations.
|
||||
/// Returns true if any content was appended.
|
||||
#[inline]
|
||||
pub fn append_text_to(&self, buffer: &mut String) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
buffer.push_str(text);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
MessageContent::Parts(parts) => {
|
||||
let mut appended = false;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !text.is_empty() {
|
||||
if appended {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(text);
|
||||
appended = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
appended
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this content contains any non-empty text.
|
||||
#[inline]
|
||||
pub fn has_text(&self) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => !text.is_empty(),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Chat Completion Request
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_chat_cross_parameters"))]
|
||||
pub struct ChatCompletionRequest {
|
||||
/// A list of messages comprising the conversation so far
|
||||
#[validate(custom(function = "validate_messages"))]
|
||||
pub messages: Vec<ChatMessage>,
|
||||
|
||||
/// ID of the model to use
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Deprecated: Replaced by tool_choice
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tool_choice instead")]
|
||||
pub function_call: Option<FunctionCall>,
|
||||
|
||||
/// Deprecated: Replaced by tools
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tools instead")]
|
||||
pub functions: Option<Vec<Function>>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Whether to return log probabilities of the output tokens
|
||||
#[serde(default)]
|
||||
pub logprobs: bool,
|
||||
|
||||
/// Deprecated: Replaced by max_completion_tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use max_completion_tokens instead")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// An upper bound for the number of tokens that can be generated for a completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_completion_tokens: Option<u32>,
|
||||
|
||||
/// Developer-defined tags and values used for filtering completions in the dashboard
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<HashMap<String, String>>,
|
||||
|
||||
/// Output types that you would like the model to generate for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// How many chat completion choices to generate for each input message
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1, max = 10))]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to enable parallel function calling during tool use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parallel_tool_calls: Option<bool>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Cache key for prompts (beta feature)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_cache_key: Option<String>,
|
||||
|
||||
/// Effort level for reasoning models (low, medium, high)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_effort: Option<String>,
|
||||
|
||||
/// An object specifying the format that the model must output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub response_format: Option<ResponseFormat>,
|
||||
|
||||
/// Safety identifier for content moderation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub safety_identifier: Option<String>,
|
||||
|
||||
/// Deprecated: This feature is in Legacy mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "This feature is in Legacy mode")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
/// The service tier to use for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub service_tier: Option<String>,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_stop"))]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// If set, partial message deltas will be sent
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// Controls which (if any) tool is called by the model
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_choice: Option<ToolChoice>,
|
||||
|
||||
/// A list of tools the model may call
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<Tool>>,
|
||||
|
||||
/// An integer between 0 and 20 specifying the number of most likely tokens to return
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0, max = 20))]
|
||||
pub top_logprobs: Option<u32>,
|
||||
|
||||
/// An alternative to sampling with temperature
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// Verbosity level for debugging
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verbosity: Option<i32>,
|
||||
|
||||
// =============================================================================
|
||||
// Engine-Specific Sampling Parameters
|
||||
// =============================================================================
|
||||
// These parameters are extensions beyond the OpenAI API specification and
|
||||
// control model generation behavior in engine-specific ways.
|
||||
// =============================================================================
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Continue generating from final assistant message
|
||||
#[serde(default)]
|
||||
pub continue_final_message: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Separate reasoning content from final answer (O1-style models)
|
||||
#[serde(default = "default_true")]
|
||||
pub separate_reasoning: bool,
|
||||
|
||||
/// Stream reasoning tokens during generation
|
||||
#[serde(default = "default_true")]
|
||||
pub stream_reasoning: bool,
|
||||
|
||||
/// Chat template kwargs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Random seed for sampling for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates messages array is not empty and has valid content
|
||||
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
|
||||
if messages.is_empty() {
|
||||
return Err(validator::ValidationError::new("messages cannot be empty"));
|
||||
}
|
||||
|
||||
for msg in messages.iter() {
|
||||
if let ChatMessage::User { content, .. } = msg {
|
||||
match content {
|
||||
MessageContent::Text(text) if text.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content cannot be empty",
|
||||
));
|
||||
}
|
||||
MessageContent::Parts(parts) if parts.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content parts cannot be empty",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_chat_cross_parameters(
|
||||
req: &ChatCompletionRequest,
|
||||
) -> Result<(), validator::ValidationError> {
|
||||
// 1. Validate logprobs dependency
|
||||
if req.top_logprobs.is_some() && !req.logprobs {
|
||||
let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs");
|
||||
e.message = Some("top_logprobs is only allowed when logprobs is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 2. Validate stream_options dependency
|
||||
if req.stream_options.is_some() && !req.stream {
|
||||
let mut e = validator::ValidationError::new("stream_options_requires_stream");
|
||||
e.message =
|
||||
Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 3. Validate token limits - min <= max
|
||||
if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) {
|
||||
if min > max {
|
||||
let mut e = validator::ValidationError::new("min_tokens_exceeds_max");
|
||||
e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Validate structured output conflicts
|
||||
let has_json_format = matches!(
|
||||
req.response_format,
|
||||
Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. })
|
||||
);
|
||||
|
||||
if has_json_format && req.regex.is_some() {
|
||||
let mut e = validator::ValidationError::new("regex_conflicts_with_json");
|
||||
e.message = Some("cannot use regex constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if has_json_format && req.ebnf.is_some() {
|
||||
let mut e = validator::ValidationError::new("ebnf_conflicts_with_json");
|
||||
e.message = Some("cannot use EBNF constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 5. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
req.regex.is_some(),
|
||||
req.ebnf.is_some(),
|
||||
matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
let mut e = validator::ValidationError::new("multiple_constraints");
|
||||
e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 6. Validate response format JSON schema name
|
||||
if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format {
|
||||
if json_schema.name.is_empty() {
|
||||
let mut e = validator::ValidationError::new("json_schema_name_empty");
|
||||
e.message = Some("JSON schema name cannot be empty".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Validate tool_choice requires tools (except for "none")
|
||||
if let Some(ref tool_choice) = req.tool_choice {
|
||||
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
|
||||
|
||||
// Check if tool_choice is anything other than "none"
|
||||
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
|
||||
if is_some_choice && !has_tools {
|
||||
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
|
||||
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Additional validation when tools are present
|
||||
if has_tools {
|
||||
let tools = req.tools.as_ref().unwrap();
|
||||
|
||||
match tool_choice {
|
||||
ToolChoice::Function { function, .. } => {
|
||||
// Validate that the specified function name exists in tools
|
||||
let function_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == function.name
|
||||
});
|
||||
|
||||
if !function_exists {
|
||||
let mut e =
|
||||
validator::ValidationError::new("tool_choice_function_not_found");
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
|
||||
function.name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
ToolChoice::AllowedTools {
|
||||
mode,
|
||||
tools: allowed_tools,
|
||||
..
|
||||
} => {
|
||||
// Validate mode is "auto" or "required"
|
||||
if mode != "auto" && mode != "required" {
|
||||
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
|
||||
e.message = Some(format!(
|
||||
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.",
|
||||
mode
|
||||
).into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Validate that all ToolReferences are Function type (Chat API only supports function tools)
|
||||
for tool_ref in allowed_tools {
|
||||
match tool_ref {
|
||||
ToolReference::Function { name } => {
|
||||
// Validate that the function exists in tools array
|
||||
let tool_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == *name
|
||||
});
|
||||
|
||||
if !tool_exists {
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_tool_not_found",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.",
|
||||
name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Chat Completion API only supports function tools in tool_choice
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_invalid_tool_type",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
|
||||
tool_ref.identifier()
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Normalizable Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl Normalizable for ChatCompletionRequest {
|
||||
/// Normalize the request by applying migrations and defaults:
|
||||
/// 1. Migrate deprecated fields to their replacements
|
||||
/// 2. Clear deprecated fields and log warnings
|
||||
/// 3. Apply OpenAI defaults for tool_choice
|
||||
fn normalize(&mut self) {
|
||||
// Migrate deprecated max_tokens → max_completion_tokens
|
||||
#[allow(deprecated)]
|
||||
if self.max_completion_tokens.is_none() && self.max_tokens.is_some() {
|
||||
self.max_completion_tokens = self.max_tokens;
|
||||
self.max_tokens = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated functions → tools
|
||||
#[allow(deprecated)]
|
||||
if self.tools.is_none() && self.functions.is_some() {
|
||||
tracing::warn!("functions is deprecated, use tools instead");
|
||||
self.tools = self.functions.as_ref().map(|functions| {
|
||||
functions
|
||||
.iter()
|
||||
.map(|func| Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: func.clone(),
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
self.functions = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated function_call → tool_choice
|
||||
#[allow(deprecated)]
|
||||
if self.tool_choice.is_none() && self.function_call.is_some() {
|
||||
tracing::warn!("function_call is deprecated, use tool_choice instead");
|
||||
self.tool_choice = self.function_call.as_ref().map(|fc| match fc {
|
||||
FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None),
|
||||
FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto),
|
||||
FunctionCall::Function { name } => ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionChoice { name: name.clone() },
|
||||
},
|
||||
});
|
||||
self.function_call = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Apply tool_choice defaults
|
||||
if self.tool_choice.is_none() {
|
||||
if let Some(tools) = &self.tools {
|
||||
let choice_value = if !tools.is_empty() {
|
||||
ToolChoiceValue::Auto
|
||||
} else {
|
||||
ToolChoiceValue::None
|
||||
};
|
||||
self.tool_choice = Some(ToolChoice::Value(choice_value));
|
||||
}
|
||||
// If tools is None, leave tool_choice as None (don't set it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl GenerationRequest for ChatCompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Extract text from messages for routing decisions
|
||||
// Use a single buffer to avoid intermediate Vec<String> allocations
|
||||
let mut buffer = String::new();
|
||||
let mut has_content = false;
|
||||
|
||||
for msg in &self.messages {
|
||||
match msg {
|
||||
ChatMessage::System { content, .. }
|
||||
| ChatMessage::User { content, .. }
|
||||
| ChatMessage::Tool { content, .. }
|
||||
| ChatMessage::Developer { content, .. } => {
|
||||
if has_content && content.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if content.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
ChatMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
..
|
||||
} => {
|
||||
// Append main content
|
||||
if let Some(c) = content {
|
||||
if has_content && c.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if c.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
// Append reasoning content
|
||||
if let Some(reasoning) = reasoning_content {
|
||||
if !reasoning.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(reasoning);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
ChatMessage::Function { content, .. } => {
|
||||
if !content.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(content);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<ChatChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponse {
|
||||
/// Create a new builder for ChatCompletionResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionResponseBuilder {
|
||||
ChatCompletionResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Response message structure for ChatCompletionResponse (different from request ChatMessage)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionMessage {
|
||||
pub role: String, // Always "assistant" for responses
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCall>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
// Note: function_call is deprecated and not included
|
||||
// Note: refusal, annotations, audio are not added yet
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatChoice {
|
||||
pub index: u32,
|
||||
pub message: ChatCompletionMessage,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "tool_calls", "content_filter", "function_call"
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
/// Hidden states from the model (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub hidden_states: Option<Vec<f32>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion.chunk"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub choices: Vec<ChatStreamChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponse {
|
||||
/// Create a new builder for ChatCompletionStreamResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionStreamResponseBuilder {
|
||||
ChatCompletionStreamResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Delta structure for streaming chat completion responses
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatMessageDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCallDelta>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatStreamChoice {
|
||||
pub index: u32,
|
||||
pub delta: ChatMessageDelta,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//! Classify API protocol definitions.
|
||||
//!
|
||||
//! This module defines the request and response types for the `/v1/classify` API,
|
||||
//! which is compatible with vLLM's classification endpoint.
|
||||
//!
|
||||
//! Classification reuses the embedding backend - the scheduler returns logits as
|
||||
//! "embeddings", and the classify layer applies softmax + label mapping.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Classify API
|
||||
// ============================================================================
|
||||
|
||||
/// Classification request - compatible with vLLM's /v1/classify API
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ClassifyRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, or token IDs
|
||||
/// - Single string: "text to classify"
|
||||
/// - Array of strings: ["text1", "text2"]
|
||||
/// - Token IDs: [1, 2, 3] (advanced usage)
|
||||
pub input: Value,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: request priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for ClassifyRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Classification is always non-streaming
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Classify Response
|
||||
// ============================================================================
|
||||
|
||||
/// Single classification result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyData {
|
||||
/// Index of this result (for batch requests)
|
||||
pub index: u32,
|
||||
/// Predicted class label (from id2label mapping)
|
||||
pub label: String,
|
||||
/// Probability distribution over all classes (softmax of logits)
|
||||
pub probs: Vec<f32>,
|
||||
/// Number of classes
|
||||
pub num_classes: u32,
|
||||
}
|
||||
|
||||
/// Classification response
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyResponse {
|
||||
/// Unique request ID (format: "classify-{uuid}")
|
||||
pub id: String,
|
||||
/// Always "list"
|
||||
pub object: String,
|
||||
/// Unix timestamp (seconds since epoch)
|
||||
pub created: u64,
|
||||
/// Model name
|
||||
pub model: String,
|
||||
/// Classification results (one per input in batch)
|
||||
pub data: Vec<ClassifyData>,
|
||||
/// Token usage info
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
|
||||
impl ClassifyResponse {
|
||||
/// Create a new ClassifyResponse with the given data
|
||||
pub fn new(
|
||||
id: String,
|
||||
model: String,
|
||||
created: u64,
|
||||
data: Vec<ClassifyData>,
|
||||
usage: UsageInfo,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
object: "list".to_string(),
|
||||
created,
|
||||
model,
|
||||
data,
|
||||
usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator;
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Default value helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Default model value when not specified
|
||||
pub(crate) fn default_model() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
/// Helper function for serde default value (returns true)
|
||||
pub fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait
|
||||
// ============================================================================
|
||||
|
||||
/// Trait for unified access to generation request properties
|
||||
/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest,
|
||||
/// EmbeddingRequest, RerankRequest, and ResponsesRequest
|
||||
pub trait GenerationRequest: Send + Sync {
|
||||
/// Check if the request is for streaming
|
||||
fn is_stream(&self) -> bool;
|
||||
|
||||
/// Get the model name if specified
|
||||
fn get_model(&self) -> Option<&str>;
|
||||
|
||||
/// Extract text content for routing decisions
|
||||
fn extract_text_for_routing(&self) -> String;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// String/Array Utilities
|
||||
// ============================================================================
|
||||
|
||||
/// A type that can be either a single string or an array of strings
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
String(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Get the number of items in the StringOrArray
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
StringOrArray::String(_) => 1,
|
||||
StringOrArray::Array(arr) => arr.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the StringOrArray is empty
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
StringOrArray::String(s) => s.is_empty(),
|
||||
StringOrArray::Array(arr) => arr.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to a vector of strings (clones the data)
|
||||
pub fn to_vec(&self) -> Vec<String> {
|
||||
match self {
|
||||
StringOrArray::String(s) => vec![s.clone()],
|
||||
StringOrArray::Array(arr) => arr.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over string references without cloning.
|
||||
/// Use this instead of `to_vec()` when you only need to iterate.
|
||||
pub fn iter(&self) -> StringOrArrayIter<'_> {
|
||||
StringOrArrayIter {
|
||||
inner: self,
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first string, or None if empty
|
||||
pub fn first(&self) -> Option<&str> {
|
||||
match self {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over StringOrArray that yields string references without cloning
|
||||
pub struct StringOrArrayIter<'a> {
|
||||
inner: &'a StringOrArray,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for StringOrArrayIter<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self.inner {
|
||||
StringOrArray::String(s) => {
|
||||
if self.index == 0 {
|
||||
self.index = 1;
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if self.index < arr.len() {
|
||||
let item = &arr[self.index];
|
||||
self.index += 1;
|
||||
Some(item.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let remaining = match self.inner {
|
||||
StringOrArray::String(_) => 1 - self.index,
|
||||
StringOrArray::Array(arr) => arr.len() - self.index,
|
||||
};
|
||||
(remaining, Some(remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {}
|
||||
|
||||
/// Validates stop sequences (max 4, non-empty strings)
|
||||
/// Used by both ChatCompletionRequest and ResponsesRequest
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
match stop {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if arr.len() > 4 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"maximum 4 stop sequences allowed",
|
||||
));
|
||||
}
|
||||
for s in arr {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Content Parts (for multimodal messages)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ContentPart {
|
||||
#[serde(rename = "text")]
|
||||
Text { text: String },
|
||||
#[serde(rename = "image_url")]
|
||||
ImageUrl { image_url: ImageUrl },
|
||||
#[serde(rename = "video_url")]
|
||||
VideoUrl { video_url: VideoUrl },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct ImageUrl {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>, // "auto", "low", or "high"
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Format (for structured outputs)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ResponseFormat {
|
||||
#[serde(rename = "text")]
|
||||
Text,
|
||||
#[serde(rename = "json_object")]
|
||||
JsonObject,
|
||||
#[serde(rename = "json_schema")]
|
||||
JsonSchema { json_schema: JsonSchemaFormat },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct JsonSchemaFormat {
|
||||
pub name: String,
|
||||
pub schema: Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct StreamOptions {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub include_usage: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCallDelta {
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub function: Option<FunctionCallDelta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub arguments: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tools and Function Calling
|
||||
// ============================================================================
|
||||
|
||||
/// Tool choice value for simple string options
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolChoiceValue {
|
||||
Auto,
|
||||
Required,
|
||||
None,
|
||||
}
|
||||
|
||||
/// Tool choice for both Chat Completion and Responses APIs
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ToolChoice {
|
||||
Value(ToolChoiceValue),
|
||||
Function {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "function"
|
||||
function: FunctionChoice,
|
||||
},
|
||||
AllowedTools {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "allowed_tools"
|
||||
mode: String, // "auto" | "required" TODO: need validation
|
||||
tools: Vec<ToolReference>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for ToolChoice {
|
||||
fn default() -> Self {
|
||||
Self::Value(ToolChoiceValue::Auto)
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolChoice {
|
||||
/// Serialize tool_choice to string for ResponsesResponse
|
||||
///
|
||||
/// Returns the JSON-serialized tool_choice or "auto" as default
|
||||
pub fn serialize_to_string(tool_choice: &Option<ToolChoice>) -> String {
|
||||
tool_choice
|
||||
.as_ref()
|
||||
.map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()))
|
||||
.unwrap_or_else(|| "auto".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function choice specification for ToolChoice::Function
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionChoice {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Tool reference for ToolChoice::AllowedTools
|
||||
///
|
||||
/// Represents a reference to a specific tool in the allowed_tools array.
|
||||
/// Different tool types have different required fields.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolReference {
|
||||
/// Reference to a function tool
|
||||
#[serde(rename = "function")]
|
||||
Function { name: String },
|
||||
|
||||
/// Reference to an MCP tool
|
||||
#[serde(rename = "mcp")]
|
||||
Mcp {
|
||||
server_label: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
|
||||
/// File search hosted tool
|
||||
#[serde(rename = "file_search")]
|
||||
FileSearch,
|
||||
|
||||
/// Web search preview hosted tool
|
||||
#[serde(rename = "web_search_preview")]
|
||||
WebSearchPreview,
|
||||
|
||||
/// Computer use preview hosted tool
|
||||
#[serde(rename = "computer_use_preview")]
|
||||
ComputerUsePreview,
|
||||
|
||||
/// Code interpreter hosted tool
|
||||
#[serde(rename = "code_interpreter")]
|
||||
CodeInterpreter,
|
||||
|
||||
/// Image generation hosted tool
|
||||
#[serde(rename = "image_generation")]
|
||||
ImageGeneration,
|
||||
}
|
||||
|
||||
impl ToolReference {
|
||||
/// Get a unique identifier for this tool reference
|
||||
pub fn identifier(&self) -> String {
|
||||
match self {
|
||||
ToolReference::Function { name } => format!("function:{}", name),
|
||||
ToolReference::Mcp { server_label, name } => {
|
||||
if let Some(n) = name {
|
||||
format!("mcp:{}:{}", server_label, n)
|
||||
} else {
|
||||
format!("mcp:{}", server_label)
|
||||
}
|
||||
}
|
||||
ToolReference::FileSearch => "file_search".to_string(),
|
||||
ToolReference::WebSearchPreview => "web_search_preview".to_string(),
|
||||
ToolReference::ComputerUsePreview => "computer_use_preview".to_string(),
|
||||
ToolReference::CodeInterpreter => "code_interpreter".to_string(),
|
||||
ToolReference::ImageGeneration => "image_generation".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the tool name if this is a function tool
|
||||
pub fn function_name(&self) -> Option<&str> {
|
||||
match self {
|
||||
ToolReference::Function { name } => Some(name.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Tool {
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: Function,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Function {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub parameters: Value, // JSON Schema
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCall {
|
||||
pub id: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: FunctionCallResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum FunctionCall {
|
||||
None,
|
||||
Auto,
|
||||
Function { name: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallResponse {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub arguments: Option<String>, // JSON string
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Usage and Logging
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionTokensDetails {
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Usage information (used by rerank and other endpoints)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct UsageInfo {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct LogProbs {
|
||||
pub tokens: Vec<String>,
|
||||
pub token_logprobs: Vec<Option<f32>>,
|
||||
pub top_logprobs: Vec<Option<HashMap<String, f32>>>,
|
||||
pub text_offset: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatLogProbs {
|
||||
Detailed {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<Vec<ChatLogProbsContent>>,
|
||||
},
|
||||
Raw(Value),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatLogProbsContent {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
pub top_logprobs: Vec<TopLogProb>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TopLogProb {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorResponse {
|
||||
pub error: ErrorDetail,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorDetail {
|
||||
pub message: String,
|
||||
#[serde(rename = "type")]
|
||||
pub error_type: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Input Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum InputIds {
|
||||
Single(Vec<i32>),
|
||||
Batch(Vec<Vec<i32>>),
|
||||
}
|
||||
|
||||
/// LoRA adapter path - can be single path or batch of paths (SGLang extension)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum LoRAPath {
|
||||
Single(Option<String>),
|
||||
Batch(Vec<Option<String>>),
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::common::*;
|
||||
|
||||
// ============================================================================
|
||||
// Completions API (v1/completions) - DEPRECATED but still supported
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionRequest {
|
||||
/// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang)
|
||||
pub model: String,
|
||||
|
||||
/// The prompt(s) to generate completions for
|
||||
pub prompt: StringOrArray,
|
||||
|
||||
/// The suffix that comes after a completion of inserted text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suffix: Option<String>,
|
||||
|
||||
/// The maximum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// An alternative to sampling with temperature (nucleus sampling)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// How many completions to generate for each prompt
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to stream back partial progress
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// Include the log probabilities on the logprobs most likely tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<u32>,
|
||||
|
||||
/// Echo back the prompt in addition to the completion
|
||||
#[serde(default)]
|
||||
pub echo: bool,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Generates best_of completions server-side and returns the "best"
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub best_of: Option<u32>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// A unique identifier representing your end-user
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// If specified, our system will make a best effort to sample deterministically
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
// -------- Engine Specific Sampling Parameters --------
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// JSON schema constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Sampling seed for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
|
||||
/// Additional fields including bootstrap info for PD routing
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for CompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.prompt {
|
||||
StringOrArray::String(s) => s.clone(),
|
||||
StringOrArray::Array(v) => v.join(" "),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<CompletionChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "content_filter", etc.
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub choices: Vec<CompletionStreamChoice>,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Embedding API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct EmbeddingRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, tokens, or batch inputs
|
||||
pub input: Value,
|
||||
|
||||
/// Optional encoding format (e.g., "float", "base64")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub encoding_format: Option<String>,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// Optional number of dimensions for the embedding
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dimensions: Option<u32>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for EmbeddingRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
// Embeddings are non-streaming
|
||||
false
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Best effort: extract text content for routing decisions
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingObject {
|
||||
pub object: String, // "embedding"
|
||||
pub embedding: Vec<f32>,
|
||||
pub index: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingResponse {
|
||||
pub object: String, // "list"
|
||||
pub data: Vec<EmbeddingObject>,
|
||||
pub model: String,
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
use std::fmt;
|
||||
|
||||
/// Response lifecycle events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ResponseEvent {
|
||||
Created,
|
||||
InProgress,
|
||||
Completed,
|
||||
}
|
||||
|
||||
impl ResponseEvent {
|
||||
pub const CREATED: &'static str = "response.created";
|
||||
pub const IN_PROGRESS: &'static str = "response.in_progress";
|
||||
pub const COMPLETED: &'static str = "response.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Created => Self::CREATED,
|
||||
Self::InProgress => Self::IN_PROGRESS,
|
||||
Self::Completed => Self::COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ResponseEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output item events for streaming
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputItemEvent {
|
||||
Added,
|
||||
Done,
|
||||
Delta,
|
||||
}
|
||||
|
||||
impl OutputItemEvent {
|
||||
pub const ADDED: &'static str = "response.output_item.added";
|
||||
pub const DONE: &'static str = "response.output_item.done";
|
||||
pub const DELTA: &'static str = "response.output_item.delta";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
Self::Delta => Self::DELTA,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputItemEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function call argument streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum FunctionCallEvent {
|
||||
ArgumentsDelta,
|
||||
ArgumentsDone,
|
||||
}
|
||||
|
||||
impl FunctionCallEvent {
|
||||
pub const ARGUMENTS_DELTA: &'static str = "response.function_call_arguments.delta";
|
||||
pub const ARGUMENTS_DONE: &'static str = "response.function_call_arguments.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ArgumentsDelta => Self::ARGUMENTS_DELTA,
|
||||
Self::ArgumentsDone => Self::ARGUMENTS_DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FunctionCallEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Content part streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ContentPartEvent {
|
||||
Added,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl ContentPartEvent {
|
||||
pub const ADDED: &'static str = "response.content_part.added";
|
||||
pub const DONE: &'static str = "response.content_part.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ContentPartEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output text streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputTextEvent {
|
||||
Delta,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl OutputTextEvent {
|
||||
pub const DELTA: &'static str = "response.output_text.delta";
|
||||
pub const DONE: &'static str = "response.output_text.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Delta => Self::DELTA,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputTextEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MCP Events
|
||||
// ============================================================================
|
||||
|
||||
/// MCP (Model Context Protocol) call events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum McpEvent {
|
||||
CallArgumentsDelta,
|
||||
CallArgumentsDone,
|
||||
CallInProgress,
|
||||
CallCompleted,
|
||||
CallFailed,
|
||||
ListToolsInProgress,
|
||||
ListToolsCompleted,
|
||||
}
|
||||
|
||||
impl McpEvent {
|
||||
pub const CALL_ARGUMENTS_DELTA: &'static str = "response.mcp_call_arguments.delta";
|
||||
pub const CALL_ARGUMENTS_DONE: &'static str = "response.mcp_call_arguments.done";
|
||||
pub const CALL_IN_PROGRESS: &'static str = "response.mcp_call.in_progress";
|
||||
pub const CALL_COMPLETED: &'static str = "response.mcp_call.completed";
|
||||
pub const CALL_FAILED: &'static str = "response.mcp_call.failed";
|
||||
pub const LIST_TOOLS_IN_PROGRESS: &'static str = "response.mcp_list_tools.in_progress";
|
||||
pub const LIST_TOOLS_COMPLETED: &'static str = "response.mcp_list_tools.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::CallArgumentsDelta => Self::CALL_ARGUMENTS_DELTA,
|
||||
Self::CallArgumentsDone => Self::CALL_ARGUMENTS_DONE,
|
||||
Self::CallInProgress => Self::CALL_IN_PROGRESS,
|
||||
Self::CallCompleted => Self::CALL_COMPLETED,
|
||||
Self::CallFailed => Self::CALL_FAILED,
|
||||
Self::ListToolsInProgress => Self::LIST_TOOLS_IN_PROGRESS,
|
||||
Self::ListToolsCompleted => Self::LIST_TOOLS_COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for McpEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Item type discriminators used in output items
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ItemType {
|
||||
FunctionCall,
|
||||
FunctionToolCall,
|
||||
McpCall,
|
||||
Function,
|
||||
McpListTools,
|
||||
}
|
||||
|
||||
impl ItemType {
|
||||
pub const FUNCTION_CALL: &'static str = "function_call";
|
||||
pub const FUNCTION_TOOL_CALL: &'static str = "function_tool_call";
|
||||
pub const MCP_CALL: &'static str = "mcp_call";
|
||||
pub const FUNCTION: &'static str = "function";
|
||||
pub const MCP_LIST_TOOLS: &'static str = "mcp_list_tools";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::FunctionCall => Self::FUNCTION_CALL,
|
||||
Self::FunctionToolCall => Self::FUNCTION_TOOL_CALL,
|
||||
Self::McpCall => Self::MCP_CALL,
|
||||
Self::Function => Self::FUNCTION,
|
||||
Self::McpListTools => Self::MCP_LIST_TOOLS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if this is a function call variant (FunctionCall or FunctionToolCall)
|
||||
pub const fn is_function_call(&self) -> bool {
|
||||
matches!(self, Self::FunctionCall | Self::FunctionToolCall)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ItemType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if an event type string matches any response lifecycle event
|
||||
pub fn is_response_event(event_type: &str) -> bool {
|
||||
matches!(
|
||||
event_type,
|
||||
ResponseEvent::CREATED | ResponseEvent::IN_PROGRESS | ResponseEvent::COMPLETED
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if an item type string is a function call variant
|
||||
pub fn is_function_call_type(item_type: &str) -> bool {
|
||||
item_type == ItemType::FUNCTION_CALL || item_type == ItemType::FUNCTION_TOOL_CALL
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{default_true, GenerationRequest, InputIds},
|
||||
sampling_params::SamplingParams,
|
||||
};
|
||||
use crate::validated::Normalizable;
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate API (native format)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Validate)]
|
||||
#[validate(schema(function = "validate_generate_request"))]
|
||||
pub struct GenerateRequest {
|
||||
/// Text input - SGLang native format
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
|
||||
pub model: Option<String>,
|
||||
|
||||
/// Input IDs for tokenized input
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_ids: Option<InputIds>,
|
||||
|
||||
/// Input embeddings for direct embedding input
|
||||
/// Can be a 2D array (single request) or 3D array (batch of requests)
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_embeds: Option<Value>,
|
||||
|
||||
/// Image input data
|
||||
/// Can be an image instance, file name, URL, or base64 encoded string
|
||||
/// Supports single images, lists of images, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub image_data: Option<Value>,
|
||||
|
||||
/// Video input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single videos, lists of videos, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub video_data: Option<Value>,
|
||||
|
||||
/// Audio input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single audio files, lists of audio, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub audio_data: Option<Value>,
|
||||
|
||||
/// Sampling parameters (sglang style)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_params: Option<SamplingParams>,
|
||||
|
||||
/// Whether to return logprobs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub return_logprob: Option<bool>,
|
||||
|
||||
/// If return logprobs, the start location in the prompt for returning logprobs.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprob_start_len: Option<i32>,
|
||||
|
||||
/// If return logprobs, the number of top logprobs to return at each position.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_logprobs_num: Option<i32>,
|
||||
|
||||
/// If return logprobs, the token ids to return logprob for.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub token_ids_logprob: Option<Vec<u32>>,
|
||||
|
||||
/// Whether to detokenize tokens in text in the returned logprobs.
|
||||
#[serde(default)]
|
||||
pub return_text_in_logprobs: bool,
|
||||
|
||||
/// Whether to stream the response
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Whether to log metrics for this request (e.g. health_generate calls do not log metrics)
|
||||
#[serde(default = "default_true")]
|
||||
pub log_metrics: bool,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// The modalities of the image data [image, multi-images, video]
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// LoRA adapter ID (if pre-loaded)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_id: Option<String>,
|
||||
|
||||
/// Custom logit processor for advanced sampling control. Must be a serialized instance
|
||||
/// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py
|
||||
/// Use the processor's `to_str()` method to generate the serialized string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_logit_processor: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_host: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_room: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_pair_key: Option<String>,
|
||||
|
||||
/// Data parallel rank routing
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data_parallel_rank: Option<i32>,
|
||||
|
||||
/// Background response
|
||||
#[serde(default)]
|
||||
pub background: bool,
|
||||
|
||||
/// Conversation ID for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub conversation_id: Option<String>,
|
||||
|
||||
/// Priority for the request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// Extra key for classifying the request (e.g. cache_salt)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub extra_key: Option<String>,
|
||||
|
||||
/// Whether to disallow logging for this request (e.g. due to ZDR)
|
||||
#[serde(default)]
|
||||
pub no_logs: bool,
|
||||
|
||||
/// Custom metric labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Whether to return bytes for image generation
|
||||
#[serde(default)]
|
||||
pub return_bytes: bool,
|
||||
|
||||
/// Whether to return entropy
|
||||
#[serde(default)]
|
||||
pub return_entropy: bool,
|
||||
|
||||
/// Request ID for tracking (inherited from BaseReq in Python)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
}
|
||||
|
||||
impl Normalizable for GenerateRequest {
|
||||
// Use default no-op implementation - no normalization needed for GenerateRequest
|
||||
}
|
||||
|
||||
/// Validation function for GenerateRequest - ensure exactly one input type is provided
|
||||
fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> {
|
||||
// Exactly one of text or input_ids must be provided
|
||||
// Note: input_embeds not yet supported in Rust implementation
|
||||
let has_text = req.text.is_some();
|
||||
let has_input_ids = req.input_ids.is_some();
|
||||
|
||||
let count = [has_text, has_input_ids].iter().filter(|&&x| x).count();
|
||||
|
||||
if count == 0 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
if count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl GenerationRequest for GenerateRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
// Generate requests have an optional model field
|
||||
if let Some(s) = &self.model {
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Check fields in priority order: text, input_ids
|
||||
if let Some(ref text) = self.text {
|
||||
return text.clone();
|
||||
}
|
||||
|
||||
if let Some(ref input_ids) = self.input_ids {
|
||||
return match input_ids {
|
||||
InputIds::Single(ids) => ids
|
||||
.iter()
|
||||
.map(|&id| id.to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
InputIds::Batch(batches) => batches
|
||||
.iter()
|
||||
.flat_map(|batch| batch.iter().map(|&id| id.to_string()))
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
};
|
||||
}
|
||||
|
||||
// No text input found
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate Response Types
|
||||
// ============================================================================
|
||||
|
||||
/// SGLang generate response (single completion or array for n>1)
|
||||
///
|
||||
/// Format for n=1:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "text": "...",
|
||||
/// "output_ids": [...],
|
||||
/// "meta_info": { ... }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Format for n>1:
|
||||
/// ```json
|
||||
/// [
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}},
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}}
|
||||
/// ]
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateResponse {
|
||||
pub text: String,
|
||||
pub output_ids: Vec<u32>,
|
||||
pub meta_info: GenerateMetaInfo,
|
||||
}
|
||||
|
||||
/// Metadata for a single generate completion
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateMetaInfo {
|
||||
pub id: String,
|
||||
pub finish_reason: GenerateFinishReason,
|
||||
pub prompt_tokens: u32,
|
||||
pub weight_version: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
pub completion_tokens: u32,
|
||||
pub cached_tokens: u32,
|
||||
pub e2e_latency: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
|
||||
/// Finish reason for generate endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "lowercase")]
|
||||
pub enum GenerateFinishReason {
|
||||
Length {
|
||||
length: u32,
|
||||
},
|
||||
Stop,
|
||||
#[serde(untagged)]
|
||||
Other(Value),
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// Protocol definitions and validation for various LLM APIs
|
||||
// This module provides a structured approach to handling different API protocols
|
||||
|
||||
/// Default model identifier used when no model is specified.
|
||||
///
|
||||
/// This constant should be used instead of hardcoded "unknown" strings
|
||||
/// throughout the codebase for consistency.
|
||||
pub const UNKNOWN_MODEL_ID: &str = "unknown";
|
||||
|
||||
pub mod builders;
|
||||
pub mod chat;
|
||||
pub mod classify;
|
||||
pub mod common;
|
||||
pub mod completion;
|
||||
pub mod embedding;
|
||||
pub mod event_types;
|
||||
pub mod generate;
|
||||
pub mod messages;
|
||||
pub mod parser;
|
||||
pub mod rerank;
|
||||
pub mod responses;
|
||||
pub mod sampling_params;
|
||||
pub mod tokenize;
|
||||
pub mod validated;
|
||||
pub mod worker_spec;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::common::Tool;
|
||||
|
||||
/// Request to parse function calls from model output text
|
||||
#[derive(Deserialize)]
|
||||
pub struct ParseFunctionCallRequest {
|
||||
/// The text to parse for function calls
|
||||
pub text: String,
|
||||
/// The parser type/name to use for parsing (e.g., "json", "pythonic")
|
||||
pub tool_call_parser: String,
|
||||
/// The list of available tools that the model can call
|
||||
pub tools: Vec<Tool>,
|
||||
}
|
||||
|
||||
/// Request to separate reasoning from normal text in model output
|
||||
#[derive(Deserialize)]
|
||||
pub struct SeparateReasoningRequest {
|
||||
/// The text to parse for reasoning content
|
||||
pub text: String,
|
||||
/// The parser type/name to use for reasoning detection (e.g., "step3", "deepseek_r1")
|
||||
pub reasoning_parser: String,
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::common::{default_model, default_true, GenerationRequest, StringOrArray, UsageInfo};
|
||||
|
||||
fn default_rerank_object() -> String {
|
||||
"rerank".to_string()
|
||||
}
|
||||
|
||||
/// TODO: Create timestamp should not be in protocol layer
|
||||
fn current_timestamp() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_else(|_| std::time::Duration::from_secs(0))
|
||||
.as_secs() as i64
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Rerank API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
#[validate(schema(function = "validate_rerank_request"))]
|
||||
pub struct RerankRequest {
|
||||
/// The query text to rank documents against
|
||||
#[validate(custom(function = "validate_query"))]
|
||||
pub query: String,
|
||||
|
||||
/// List of documents to be ranked
|
||||
#[validate(custom(function = "validate_documents"))]
|
||||
pub documents: Vec<String>,
|
||||
|
||||
/// Model to use for reranking
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Maximum number of documents to return (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub top_k: Option<usize>,
|
||||
|
||||
/// Whether to return documents in addition to scores
|
||||
#[serde(default = "default_true")]
|
||||
pub return_documents: bool,
|
||||
|
||||
// SGLang specific extensions
|
||||
/// Request ID for tracking
|
||||
pub rid: Option<StringOrArray>,
|
||||
|
||||
/// User identifier
|
||||
pub user: Option<String>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for RerankRequest {
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Reranking doesn't support streaming
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
self.query.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl super::validated::Normalizable for RerankRequest {
|
||||
// Use default no-op normalization
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates that the query is not empty
|
||||
fn validate_query(query: &str) -> Result<(), validator::ValidationError> {
|
||||
if query.trim().is_empty() {
|
||||
return Err(validator::ValidationError::new("query cannot be empty"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates that the documents list is not empty
|
||||
fn validate_documents(documents: &[String]) -> Result<(), validator::ValidationError> {
|
||||
if documents.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"documents list cannot be empty",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_rerank_request(req: &RerankRequest) -> Result<(), validator::ValidationError> {
|
||||
// Validate top_k if specified
|
||||
if let Some(k) = req.top_k {
|
||||
if k > req.documents.len() {
|
||||
// This is allowed but we log a warning
|
||||
tracing::warn!(
|
||||
"top_k ({}) is greater than number of documents ({})",
|
||||
k,
|
||||
req.documents.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RerankRequest {
|
||||
/// Get the effective top_k value
|
||||
pub fn effective_top_k(&self) -> usize {
|
||||
self.top_k.unwrap_or(self.documents.len())
|
||||
}
|
||||
}
|
||||
|
||||
/// Individual rerank result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RerankResult {
|
||||
/// Relevance score for the document
|
||||
pub score: f32,
|
||||
|
||||
/// The document text (if return_documents was true)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub document: Option<String>,
|
||||
|
||||
/// Original index of the document in the request
|
||||
pub index: usize,
|
||||
|
||||
/// Additional metadata about the ranking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub meta_info: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
/// Rerank response containing sorted results
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RerankResponse {
|
||||
/// Ranked results sorted by score (highest first)
|
||||
pub results: Vec<RerankResult>,
|
||||
|
||||
/// Model used for reranking
|
||||
pub model: String,
|
||||
|
||||
/// Usage information
|
||||
pub usage: Option<UsageInfo>,
|
||||
|
||||
/// Response object type
|
||||
#[serde(default = "default_rerank_object")]
|
||||
pub object: String,
|
||||
|
||||
/// Response ID
|
||||
pub id: Option<StringOrArray>,
|
||||
|
||||
/// Creation timestamp
|
||||
pub created: i64,
|
||||
}
|
||||
|
||||
impl RerankResponse {
|
||||
/// Create a new RerankResponse with the given results and model
|
||||
pub fn new(
|
||||
results: Vec<RerankResult>,
|
||||
model: String,
|
||||
request_id: Option<StringOrArray>,
|
||||
) -> Self {
|
||||
RerankResponse {
|
||||
results,
|
||||
model,
|
||||
usage: None,
|
||||
object: default_rerank_object(),
|
||||
id: request_id,
|
||||
created: current_timestamp(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply top_k limit to results
|
||||
pub fn apply_top_k(&mut self, k: usize) {
|
||||
self.results.truncate(k);
|
||||
}
|
||||
|
||||
/// Drop documents from results (when return_documents is false)
|
||||
pub fn drop_documents(&mut self) {
|
||||
for result in &mut self.results {
|
||||
result.document = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// V1 API compatibility format for rerank requests
|
||||
/// Matches Python's V1RerankReqInput
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct V1RerankReqInput {
|
||||
pub query: String,
|
||||
pub documents: Vec<String>,
|
||||
}
|
||||
|
||||
/// Convert V1RerankReqInput to RerankRequest
|
||||
impl From<V1RerankReqInput> for RerankRequest {
|
||||
fn from(v1: V1RerankReqInput) -> Self {
|
||||
RerankRequest {
|
||||
query: v1.query,
|
||||
documents: v1.documents,
|
||||
model: default_model(),
|
||||
top_k: None,
|
||||
return_documents: true,
|
||||
rid: None,
|
||||
user: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,119 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use validator::Validate;
|
||||
|
||||
use super::common::StringOrArray;
|
||||
|
||||
/// Sampling parameters for text generation
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_sampling_params"))]
|
||||
pub struct SamplingParams {
|
||||
/// Temperature for sampling (must be >= 0.0, no upper limit)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0))]
|
||||
pub temperature: Option<f32>,
|
||||
/// Maximum number of new tokens to generate (must be >= 0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0))]
|
||||
pub max_new_tokens: Option<u32>,
|
||||
/// Top-p nucleus sampling (0.0 < top_p <= 1.0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
/// Top-k sampling (-1 to disable, or >= 1)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ignore_eos: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub skip_special_tokens: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
/// Minimum number of new tokens (validated in schema function for cross-field check with max_new_tokens)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_new_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub no_stop_trim: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Shared Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates top_p: 0.0 < top_p <= 1.0 (can't use range validator for open interval)
|
||||
pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> {
|
||||
if !(top_p > 0.0 && top_p <= 1.0) {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_p must be in (0, 1] - greater than 0.0 and at most 1.0",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates top_k: -1 (disabled) or >= 1 (special -1 case - can't use range validator)
|
||||
pub fn validate_top_k_value(top_k: i32) -> Result<(), validator::ValidationError> {
|
||||
if top_k != -1 && top_k < 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_k must be -1 (disabled) or at least 1",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SamplingParams-Specific Validation
|
||||
// ============================================================================
|
||||
|
||||
/// Validation function for SamplingParams - cross-field validation only
|
||||
fn validate_sampling_params(params: &SamplingParams) -> Result<(), validator::ValidationError> {
|
||||
// 1. Cross-field validation: min_new_tokens <= max_new_tokens
|
||||
if let (Some(min), Some(max)) = (params.min_new_tokens, params.max_new_tokens) {
|
||||
if min > max {
|
||||
return Err(validator::ValidationError::new(
|
||||
"min_new_tokens cannot exceed max_new_tokens",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
params.regex.is_some(),
|
||||
params.ebnf.is_some(),
|
||||
params.json_schema.is_some(),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"only one of regex, ebnf, or json_schema can be set",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Tokenize and Detokenize API protocol types
|
||||
//!
|
||||
//! These types mirror the SGLang Python implementation for compatibility.
|
||||
//! See: python/sglang/srt/entrypoints/openai/protocol.py
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Tokenize API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for the /v1/tokenize endpoint
|
||||
///
|
||||
/// Supports both single string and batch tokenization.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TokenizeRequest {
|
||||
/// Model name for tokenizer selection
|
||||
#[serde(default = "default_model_name")]
|
||||
pub model: String,
|
||||
|
||||
/// Text(s) to tokenize - can be a single string or array of strings
|
||||
pub prompt: StringOrArray,
|
||||
}
|
||||
|
||||
/// Response schema for the /v1/tokenize endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TokenizeResponse {
|
||||
/// Token IDs - single list for single input, nested list for batch
|
||||
pub tokens: TokensResult,
|
||||
|
||||
/// Token count(s) - single int for single input, list for batch
|
||||
pub count: CountResult,
|
||||
|
||||
/// Character count(s) of input - single int for single input, list for batch
|
||||
pub char_count: CountResult,
|
||||
}
|
||||
|
||||
/// Token IDs result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TokensResult {
|
||||
Single(Vec<u32>),
|
||||
Batch(Vec<Vec<u32>>),
|
||||
}
|
||||
|
||||
/// Count result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum CountResult {
|
||||
Single(i32),
|
||||
Batch(Vec<i32>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Detokenize API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for the /v1/detokenize endpoint
|
||||
///
|
||||
/// Supports both single sequence and batch detokenization.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DetokenizeRequest {
|
||||
/// Model name for tokenizer selection
|
||||
#[serde(default = "default_model_name")]
|
||||
pub model: String,
|
||||
|
||||
/// Token IDs to detokenize - single list or batch (list of lists)
|
||||
pub tokens: TokensInput,
|
||||
|
||||
/// Whether to skip special tokens (e.g., padding or EOS) during decoding
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
}
|
||||
|
||||
/// Token input - either single sequence or batch
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TokensInput {
|
||||
/// Single sequence of token IDs
|
||||
Single(Vec<u32>),
|
||||
/// Batch of token sequences
|
||||
Batch(Vec<Vec<u32>>),
|
||||
}
|
||||
|
||||
impl TokensInput {
|
||||
/// Check if this is a batch input
|
||||
pub fn is_batch(&self) -> bool {
|
||||
matches!(self, TokensInput::Batch(_))
|
||||
}
|
||||
|
||||
/// Get the sequences (always returns a vec of vecs for uniform processing)
|
||||
pub fn sequences(&self) -> Vec<&[u32]> {
|
||||
match self {
|
||||
TokensInput::Single(seq) => vec![seq.as_slice()],
|
||||
TokensInput::Batch(seqs) => seqs.iter().map(|s| s.as_slice()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Response schema for the /v1/detokenize endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DetokenizeResponse {
|
||||
/// Decoded text - single string for single input, list for batch
|
||||
pub text: TextResult,
|
||||
}
|
||||
|
||||
/// Text result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TextResult {
|
||||
Single(String),
|
||||
Batch(Vec<String>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tokenizer Management API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for adding a tokenizer
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct AddTokenizerRequest {
|
||||
/// Name to register the tokenizer under
|
||||
pub name: String,
|
||||
|
||||
/// Source: either a local path or HuggingFace model ID
|
||||
pub source: String,
|
||||
|
||||
/// Optional path to chat template file
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Response schema for adding a tokenizer (async)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AddTokenizerResponse {
|
||||
/// Unique identifier for the tokenizer (UUID)
|
||||
pub id: String,
|
||||
/// Status of the request: "pending", "processing", "completed", "failed"
|
||||
pub status: String,
|
||||
pub message: String,
|
||||
/// Vocabulary size of the loaded tokenizer (only set on completion)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub vocab_size: Option<usize>,
|
||||
}
|
||||
|
||||
/// Response schema for listing tokenizers
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListTokenizersResponse {
|
||||
pub tokenizers: Vec<TokenizerInfo>,
|
||||
}
|
||||
|
||||
/// Information about a registered tokenizer
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TokenizerInfo {
|
||||
/// Unique identifier (UUID)
|
||||
pub id: String,
|
||||
/// User-provided name
|
||||
pub name: String,
|
||||
/// Source path or HuggingFace model ID
|
||||
pub source: String,
|
||||
pub vocab_size: usize,
|
||||
}
|
||||
|
||||
/// Request schema for removing a tokenizer
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct RemoveTokenizerRequest {
|
||||
/// Name of the tokenizer to remove
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Response schema for removing a tokenizer
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RemoveTokenizerResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Types
|
||||
// ============================================================================
|
||||
|
||||
/// String or array of strings (for flexible input)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
Single(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Check if this is a batch (array) input
|
||||
pub fn is_batch(&self) -> bool {
|
||||
matches!(self, StringOrArray::Array(_))
|
||||
}
|
||||
|
||||
/// Get all strings as a slice (converts single to vec)
|
||||
pub fn as_strings(&self) -> Vec<&str> {
|
||||
match self {
|
||||
StringOrArray::Single(s) => vec![s.as_str()],
|
||||
StringOrArray::Array(arr) => arr.iter().map(|s| s.as_str()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Default Functions
|
||||
// ============================================================================
|
||||
|
||||
fn default_model_name() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_request_single() {
|
||||
let json = r#"{"prompt": "Hello world"}"#;
|
||||
let req: TokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(req.model, "unknown");
|
||||
assert!(matches!(req.prompt, StringOrArray::Single(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_request_batch() {
|
||||
let json = r#"{"model": "llama", "prompt": ["Hello", "World"]}"#;
|
||||
let req: TokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(req.model, "llama");
|
||||
assert!(matches!(req.prompt, StringOrArray::Array(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detokenize_request_single() {
|
||||
let json = r#"{"tokens": [1, 2, 3]}"#;
|
||||
let req: DetokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(req.tokens, TokensInput::Single(_)));
|
||||
assert!(req.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detokenize_request_batch() {
|
||||
let json = r#"{"tokens": [[1, 2], [3, 4, 5]], "skip_special_tokens": false}"#;
|
||||
let req: DetokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(req.tokens, TokensInput::Batch(_)));
|
||||
assert!(!req.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_response_single() {
|
||||
let resp = TokenizeResponse {
|
||||
tokens: TokensResult::Single(vec![1, 2, 3]),
|
||||
count: CountResult::Single(3),
|
||||
char_count: CountResult::Single(11),
|
||||
};
|
||||
let json = serde_json::to_string(&resp).unwrap();
|
||||
assert!(json.contains("[1,2,3]"));
|
||||
assert!(json.contains("\"count\":3"));
|
||||
assert!(json.contains("\"char_count\":11"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_response_batch() {
|
||||
let resp = TokenizeResponse {
|
||||
tokens: TokensResult::Batch(vec![vec![1, 2], vec![3, 4, 5]]),
|
||||
count: CountResult::Batch(vec![2, 3]),
|
||||
char_count: CountResult::Batch(vec![5, 5]),
|
||||
};
|
||||
let json = serde_json::to_string(&resp).unwrap();
|
||||
assert!(json.contains("[[1,2],[3,4,5]]"));
|
||||
assert!(json.contains("[2,3]"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
// Validated JSON extractor for automatic request validation
|
||||
//
|
||||
// This module provides a ValidatedJson extractor that automatically validates
|
||||
// requests using the validator crate's Validate trait.
|
||||
|
||||
/// Trait for request types that need post-deserialization normalization
|
||||
pub trait Normalizable {
|
||||
/// Normalize the request by applying defaults and transformations
|
||||
fn normalize(&mut self) {
|
||||
// Default: no-op
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
use axum::{
|
||||
extract::{rejection::JsonRejection, FromRequest, Request},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
#[cfg(feature = "axum")]
|
||||
use serde::de::DeserializeOwned;
|
||||
#[cfg(feature = "axum")]
|
||||
use serde_json::json;
|
||||
#[cfg(feature = "axum")]
|
||||
use validator::Validate;
|
||||
|
||||
/// A JSON extractor that automatically validates and normalizes the request body
|
||||
///
|
||||
/// This extractor deserializes the request body and automatically calls `.validate()`
|
||||
/// on types that implement the `Validate` trait. If validation fails, it returns
|
||||
/// a 400 Bad Request with detailed error information.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// async fn create_chat(
|
||||
/// ValidatedJson(request): ValidatedJson<ChatCompletionRequest>,
|
||||
/// ) -> Response {
|
||||
/// // request is guaranteed to be valid here
|
||||
/// process_request(request).await
|
||||
/// }
|
||||
/// ```
|
||||
#[cfg(feature = "axum")]
|
||||
pub struct ValidatedJson<T>(pub T);
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl<S, T> FromRequest<S> for ValidatedJson<T>
|
||||
where
|
||||
T: DeserializeOwned + Validate + Normalizable + Send,
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = Response;
|
||||
|
||||
async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
|
||||
// First, extract and deserialize the JSON
|
||||
let Json(mut data) =
|
||||
Json::<T>::from_request(req, state)
|
||||
.await
|
||||
.map_err(|err: JsonRejection| {
|
||||
let error_message = match err {
|
||||
JsonRejection::JsonDataError(e) => {
|
||||
format!("Invalid JSON data: {}", e)
|
||||
}
|
||||
JsonRejection::JsonSyntaxError(e) => {
|
||||
format!("JSON syntax error: {}", e)
|
||||
}
|
||||
JsonRejection::MissingJsonContentType(_) => {
|
||||
"Missing Content-Type: application/json header".to_string()
|
||||
}
|
||||
_ => format!("Failed to parse JSON: {}", err),
|
||||
};
|
||||
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": error_message,
|
||||
"type": "invalid_request_error",
|
||||
"code": "json_parse_error"
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
// Normalize the request (apply defaults based on other fields)
|
||||
data.normalize();
|
||||
|
||||
// Then, automatically validate the data
|
||||
data.validate().map_err(|validation_errors| {
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": validation_errors.to_string(),
|
||||
"type": "invalid_request_error",
|
||||
"code": 400
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
Ok(ValidatedJson(data))
|
||||
}
|
||||
}
|
||||
|
||||
// Implement Deref to allow transparent access to the inner value
|
||||
#[cfg(feature = "axum")]
|
||||
impl<T> std::ops::Deref for ValidatedJson<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl<T> std::ops::DerefMut for ValidatedJson<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "axum"))]
|
||||
mod tests {
|
||||
use serde::{Deserialize, Serialize};
|
||||
use validator::Validate;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Validate)]
|
||||
struct TestRequest {
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
value: f32,
|
||||
#[validate(length(min = 1))]
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl Normalizable for TestRequest {
|
||||
// Use default no-op implementation
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_valid() {
|
||||
// This test is conceptual - actual testing would require Axum test harness
|
||||
let request = TestRequest {
|
||||
value: 0.5,
|
||||
name: "test".to_string(),
|
||||
};
|
||||
assert!(request.validate().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_invalid_range() {
|
||||
let request = TestRequest {
|
||||
value: 1.5, // Out of range
|
||||
name: "test".to_string(),
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_invalid_length() {
|
||||
let request = TestRequest {
|
||||
value: 0.5,
|
||||
name: "".to_string(), // Empty name
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
//! Worker management API specifications
|
||||
//!
|
||||
//! Defines the request/response structures for worker management endpoints
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
use axum::{
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[cfg(feature = "axum")]
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
/// Worker configuration for API requests
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerConfigRequest {
|
||||
/// Worker URL (required)
|
||||
pub url: String,
|
||||
|
||||
/// Worker API key (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// Model ID (optional, will query from server if not provided)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_id: Option<String>,
|
||||
|
||||
/// Worker priority (optional, default: 50, higher = preferred)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
/// Worker cost factor (optional, default: 1.0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
/// Worker type (optional: "regular", "prefill", "decode")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
|
||||
/// Bootstrap port for prefill workers (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<u16>,
|
||||
|
||||
/// Runtime type (optional: "sglang", "vllm", default: "sglang")
|
||||
/// Only relevant for gRPC workers
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime: Option<String>,
|
||||
|
||||
// gRPC-specific configuration (optional, ignored in HTTP mode)
|
||||
/// Tokenizer path for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
/// Reasoning parser type for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
/// Tool parser type for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
/// Chat template for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Additional labels (optional)
|
||||
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
|
||||
pub labels: HashMap<String, String>,
|
||||
|
||||
/// Health check timeout in seconds (default: 30)
|
||||
#[serde(default = "default_health_check_timeout")]
|
||||
pub health_check_timeout_secs: u64,
|
||||
|
||||
/// Health check interval in seconds (default: 60)
|
||||
#[serde(default = "default_health_check_interval")]
|
||||
pub health_check_interval_secs: u64,
|
||||
|
||||
/// Number of successful health checks needed to mark worker as healthy (default: 2)
|
||||
#[serde(default = "default_health_success_threshold")]
|
||||
pub health_success_threshold: u32,
|
||||
|
||||
/// Number of failed health checks before marking worker as unhealthy (default: 3)
|
||||
#[serde(default = "default_health_failure_threshold")]
|
||||
pub health_failure_threshold: u32,
|
||||
|
||||
/// Disable periodic health checks for this worker (default: false)
|
||||
#[serde(default)]
|
||||
pub disable_health_check: bool,
|
||||
|
||||
/// Maximum connection attempts during worker registration (default: 20)
|
||||
#[serde(default = "default_max_connection_attempts")]
|
||||
pub max_connection_attempts: u32,
|
||||
|
||||
/// Enable data parallelism aware scheduling (default: false)
|
||||
#[serde(default)]
|
||||
pub dp_aware: bool,
|
||||
}
|
||||
|
||||
// Default value functions for serde
|
||||
fn default_health_check_timeout() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_health_check_interval() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
fn default_health_success_threshold() -> u32 {
|
||||
2
|
||||
}
|
||||
|
||||
fn default_health_failure_threshold() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_max_connection_attempts() -> u32 {
|
||||
20
|
||||
}
|
||||
|
||||
/// Worker information for API responses
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerInfo {
|
||||
/// Worker unique identifier
|
||||
pub id: String,
|
||||
|
||||
/// Worker URL
|
||||
pub url: String,
|
||||
|
||||
/// Model ID this worker serves
|
||||
pub model_id: String,
|
||||
|
||||
/// Worker priority
|
||||
pub priority: u32,
|
||||
|
||||
/// Worker cost factor
|
||||
pub cost: f32,
|
||||
|
||||
/// Worker type
|
||||
pub worker_type: String,
|
||||
|
||||
/// Whether the worker is healthy
|
||||
pub is_healthy: bool,
|
||||
|
||||
/// Current load on the worker
|
||||
pub load: usize,
|
||||
|
||||
/// Connection mode (http or grpc)
|
||||
pub connection_mode: String,
|
||||
|
||||
/// Runtime type (sglang or vllm, for gRPC workers)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime_type: Option<String>,
|
||||
|
||||
// gRPC-specific fields (None for HTTP workers)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Bootstrap port for prefill workers
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<u16>,
|
||||
|
||||
/// Additional metadata
|
||||
#[serde(skip_serializing_if = "HashMap::is_empty")]
|
||||
pub metadata: HashMap<String, String>,
|
||||
|
||||
/// Whether health checks are disabled for this worker
|
||||
pub disable_health_check: bool,
|
||||
|
||||
/// Job status for async operations (if available)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub job_status: Option<JobStatus>,
|
||||
}
|
||||
|
||||
impl WorkerInfo {
|
||||
/// Create a partial WorkerInfo for pending workers (not yet registered).
|
||||
/// Used when a worker ID maps to a URL but the worker is still being registered.
|
||||
pub fn pending(worker_id: &str, url: String, job_status: Option<JobStatus>) -> Self {
|
||||
Self {
|
||||
id: worker_id.to_string(),
|
||||
url,
|
||||
model_id: UNKNOWN_MODEL_ID.to_string(),
|
||||
priority: 0,
|
||||
cost: 1.0,
|
||||
worker_type: UNKNOWN_MODEL_ID.to_string(),
|
||||
is_healthy: false,
|
||||
load: 0,
|
||||
connection_mode: UNKNOWN_MODEL_ID.to_string(),
|
||||
runtime_type: None,
|
||||
tokenizer_path: None,
|
||||
reasoning_parser: None,
|
||||
tool_parser: None,
|
||||
chat_template: None,
|
||||
bootstrap_port: None,
|
||||
metadata: HashMap::new(),
|
||||
disable_health_check: false,
|
||||
job_status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Job status for async control plane operations
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobStatus {
|
||||
pub job_type: String,
|
||||
pub worker_url: String,
|
||||
pub status: String,
|
||||
pub message: Option<String>,
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
impl JobStatus {
|
||||
/// Create a pending job status
|
||||
pub fn pending(job_type: &str, worker_url: &str) -> Self {
|
||||
Self {
|
||||
job_type: job_type.to_string(),
|
||||
worker_url: worker_url.to_string(),
|
||||
status: "pending".to_string(),
|
||||
message: None,
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a processing job status
|
||||
pub fn processing(job_type: &str, worker_url: &str) -> Self {
|
||||
Self {
|
||||
job_type: job_type.to_string(),
|
||||
worker_url: worker_url.to_string(),
|
||||
status: "processing".to_string(),
|
||||
message: None,
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a failed job status
|
||||
pub fn failed(job_type: &str, worker_url: &str, error: String) -> Self {
|
||||
Self {
|
||||
job_type: job_type.to_string(),
|
||||
worker_url: worker_url.to_string(),
|
||||
status: "failed".to_string(),
|
||||
message: Some(error),
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Worker list response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerListResponse {
|
||||
/// List of workers
|
||||
pub workers: Vec<WorkerInfo>,
|
||||
|
||||
/// Total count
|
||||
pub total: usize,
|
||||
|
||||
/// Statistics
|
||||
pub stats: WorkerStats,
|
||||
}
|
||||
|
||||
/// Worker statistics
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerStats {
|
||||
pub total_workers: usize,
|
||||
pub healthy_workers: usize,
|
||||
pub total_models: usize,
|
||||
pub total_load: usize,
|
||||
pub by_type: WorkerTypeStats,
|
||||
}
|
||||
|
||||
/// Worker statistics by type
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerTypeStats {
|
||||
pub regular: usize,
|
||||
pub prefill: usize,
|
||||
pub decode: usize,
|
||||
}
|
||||
|
||||
/// Worker update request
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkerUpdateRequest {
|
||||
/// Update priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
/// Update cost
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
/// Update labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Update API key (for key rotation)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// Update health check timeout in seconds
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_check_timeout_secs: Option<u64>,
|
||||
|
||||
/// Update health check interval in seconds
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_check_interval_secs: Option<u64>,
|
||||
|
||||
/// Update health success threshold
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_success_threshold: Option<u32>,
|
||||
|
||||
/// Update health failure threshold
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_failure_threshold: Option<u32>,
|
||||
|
||||
/// Disable periodic health checks for this worker
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disable_health_check: Option<bool>,
|
||||
}
|
||||
|
||||
/// Generic API response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerApiResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker: Option<WorkerInfo>,
|
||||
}
|
||||
|
||||
/// Error response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerErrorResponse {
|
||||
pub error: String,
|
||||
pub code: String,
|
||||
}
|
||||
|
||||
/// Server info response from /get_server_info endpoint
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ServerInfo {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_id: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
|
||||
// gRPC-specific
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
}
|
||||
|
||||
/// Result from flush cache operations across workers
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FlushCacheResult {
|
||||
/// URLs of workers where cache flush succeeded
|
||||
pub successful: Vec<String>,
|
||||
/// URLs and error messages for workers where cache flush failed
|
||||
pub failed: Vec<(String, String)>,
|
||||
/// Total number of workers attempted
|
||||
pub total_workers: usize,
|
||||
/// Number of HTTP workers (gRPC workers don't support flush cache)
|
||||
pub http_workers: usize,
|
||||
/// Human-readable summary message
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Result from getting worker loads
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerLoadsResult {
|
||||
/// Worker URL and load pairs
|
||||
pub loads: Vec<WorkerLoadInfo>,
|
||||
/// Total number of workers
|
||||
pub total_workers: usize,
|
||||
/// Number of workers with successful load fetches
|
||||
pub successful: usize,
|
||||
/// Number of workers with failed load fetches
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
/// Individual worker load information
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerLoadInfo {
|
||||
/// Worker URL
|
||||
pub worker: String,
|
||||
/// Worker type (regular, prefill, decode)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
/// Current load (-1 indicates failure to fetch)
|
||||
pub load: isize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl IntoResponse for FlushCacheResult {
|
||||
fn into_response(self) -> Response {
|
||||
let status = if self.failed.is_empty() {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::PARTIAL_CONTENT
|
||||
};
|
||||
|
||||
let mut body = json!({
|
||||
"status": if self.failed.is_empty() { "success" } else { "partial_success" },
|
||||
"message": self.message,
|
||||
"workers_flushed": self.successful.len(),
|
||||
"total_http_workers": self.http_workers,
|
||||
"total_workers": self.total_workers
|
||||
});
|
||||
|
||||
if !self.failed.is_empty() {
|
||||
body["successful"] = json!(self.successful);
|
||||
body["failed"] = json!(self
|
||||
.failed
|
||||
.into_iter()
|
||||
.map(|(url, err)| json!({"worker": url, "error": err}))
|
||||
.collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
(status, Json(body)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl IntoResponse for WorkerLoadsResult {
|
||||
fn into_response(self) -> Response {
|
||||
let loads: Vec<Value> = self
|
||||
.loads
|
||||
.iter()
|
||||
.map(|info| json!({"worker": &info.worker, "load": info.load}))
|
||||
.collect();
|
||||
Json(json!({"workers": loads})).into_response()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user