Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
216880034a |
@@ -21,8 +21,6 @@ from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import zlib
|
||||
from array import array
|
||||
from collections import deque
|
||||
from http import HTTPStatus
|
||||
@@ -107,64 +105,6 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_is_npu = is_npu()
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): per-step batch-decision fingerprint
|
||||
# and event-loop phase logging. Same env as custom_all_reduce_v2 ARDBG.
|
||||
# 1 = sampled/one-line, 2 = + stacks on ONE_SHOT_PUSH. Off by default.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_plan_seq = 0
|
||||
_plan_none_seq = 0
|
||||
|
||||
|
||||
def _dbg_plan(self, batch) -> None:
|
||||
global _plan_seq, _plan_none_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
_plan_seq += 1
|
||||
rank = torch.distributed.get_rank()
|
||||
if batch is None:
|
||||
# 空转步采样记录,避免 idle 自旋刷爆日志
|
||||
_plan_none_seq += 1
|
||||
if _plan_none_seq & 511:
|
||||
return
|
||||
logger.info("[ARDBG] plan rank=%s step=%d batch=None", rank, _plan_seq)
|
||||
return
|
||||
reqs = batch.reqs
|
||||
fp = 0
|
||||
ext_total = 0
|
||||
for r in reqs:
|
||||
# crc32 而非 hash():后者按进程随机化种子,跨 rank 不可比
|
||||
fp = (fp * 131 + zlib.crc32(r.rid.encode())) & 0xFFFFFFFFFFFFFFFF
|
||||
er = getattr(r, "extend_range", None)
|
||||
el = int(er.length) if er is not None else -1
|
||||
fp = (fp * 131 + el) & 0xFFFFFFFFFFFFFFFF
|
||||
fp = (fp * 131 + int(getattr(r, "prefix_len", -1))) & 0xFFFFFFFFFFFFFFFF
|
||||
if el > 0:
|
||||
ext_total += el
|
||||
logger.info(
|
||||
"[ARDBG] plan rank=%s step=%d nreq=%d ext=%d fp=%016x",
|
||||
rank,
|
||||
_plan_seq,
|
||||
len(reqs),
|
||||
ext_total,
|
||||
fp,
|
||||
)
|
||||
|
||||
|
||||
def _dbg_phase(label: str, batch=None) -> None:
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
if batch is None and label in ("inflight_enter", "inflight_exit"):
|
||||
# 空转迭代的 inflight 相位价值低,采样记录
|
||||
if _plan_seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] phase rank=%s step=%s %s nreq=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_plan_seq,
|
||||
label,
|
||||
len(batch.reqs) if batch is not None else "-",
|
||||
)
|
||||
|
||||
|
||||
def should_force_retry(req: Req) -> bool:
|
||||
"""Test hook to force a request into optimistic prefill retry."""
|
||||
@@ -688,7 +628,6 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
set_schedule_time_batch(batch)
|
||||
|
||||
_dbg_plan(self, batch)
|
||||
return NextBatchPlan(batch_to_run=batch, running_batch=running_batch)
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -719,17 +658,13 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self.process_batch_result(batch, result)
|
||||
else:
|
||||
self._sched_idled = True
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Update last_batch
|
||||
self.last_batch = batch
|
||||
@@ -763,9 +698,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
if batch:
|
||||
if self.enable_staging:
|
||||
self.maybe_prefetch_staging_for_batch(batch)
|
||||
_dbg_phase("run_batch_enter", batch)
|
||||
batch_result = self.run_batch(batch)
|
||||
_dbg_phase("run_batch_exit", batch)
|
||||
self._apply_war_barrier()
|
||||
self.result_queue.append((batch.copy(), batch_result))
|
||||
else:
|
||||
@@ -780,9 +713,7 @@ class SchedulerDisaggregationPrefillMixin:
|
||||
# When the server is idle, do self-check and re-init some states
|
||||
self.on_idle()
|
||||
|
||||
_dbg_phase("inflight_enter")
|
||||
self.process_disagg_prefill_inflight_queue()
|
||||
_dbg_phase("inflight_exit")
|
||||
|
||||
# Run sample of the current batch
|
||||
# It depends on the result of the last batch (e.g., grammar), so we run it after the last batch is processed.
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from collections import deque
|
||||
from contextlib import nullcontext
|
||||
@@ -45,32 +43,6 @@ if is_npu():
|
||||
DSV4NPUTokenToKVPool,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for host-side
|
||||
# gloo all-reduce sync points. Mirrors SGLANG_AR_SEQ_DEBUG in
|
||||
# custom_all_reduce_v2.py. 1 = one-line, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_host_ar_seq = 0
|
||||
|
||||
|
||||
def _dbg_log_host_ar(phase: str, polls) -> None:
|
||||
global _host_ar_seq
|
||||
if not _AR_SEQ_DEBUG:
|
||||
return
|
||||
seq = _host_ar_seq
|
||||
_host_ar_seq += 1
|
||||
# 空转迭代(无 poller)采样 1/256,避免 idle 自旋刷爆日志
|
||||
if (not polls) and seq & 255:
|
||||
return
|
||||
logger.info(
|
||||
"[ARDBG] host rank=%s seq=%d phase=%s polls=%s",
|
||||
dist.get_rank(),
|
||||
seq,
|
||||
phase,
|
||||
polls,
|
||||
)
|
||||
|
||||
#########################
|
||||
# Constants & Enums
|
||||
#########################
|
||||
@@ -241,20 +213,13 @@ def poll_and_all_reduce_attn_cp_tp_group(
|
||||
attn_cp_cpu_group: dist.ProcessGroup,
|
||||
attn_tp_cpu_group: dist.ProcessGroup,
|
||||
):
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("enter", len(pollers))
|
||||
# First sync across attn-tp ranks so all TP participants for a given (dp, cp)
|
||||
# shard observe the same status transitions.
|
||||
polls = poll_and_all_reduce(pollers, attn_tp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("tp-done", polls)
|
||||
|
||||
# Then sync across attn-cp ranks, so all TPxCP participants in one DP shard
|
||||
# converge to the same global status.
|
||||
result = _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
# DEBUG-ONLY
|
||||
_dbg_log_host_ar("cp-done", result)
|
||||
return result
|
||||
return _all_reduce_polls(polls, attn_cp_cpu_group)
|
||||
|
||||
|
||||
def poll_and_all_reduce_with_staging(
|
||||
|
||||
@@ -25,8 +25,6 @@ the kernel captured in the graph dereferences its row at replay time.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import traceback
|
||||
from contextlib import contextmanager
|
||||
from typing import List, NamedTuple, Optional, Tuple
|
||||
|
||||
@@ -70,9 +68,6 @@ MB = 1024 * 1024
|
||||
_ALIGN_BYTES = 1024
|
||||
_SEMAPHORE_BYTES = 128
|
||||
_MAX_GRAPH_INPUTS = 131072
|
||||
# DEBUG-ONLY (p2 wedge repro 2026-10-13): env-gated seq logging for TP device
|
||||
# all-reduce calls. 1 = one-line per call, 2 = + python stack. Not for prod.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
# resolved once at import time; explicit constructor sizes take precedence
|
||||
_DEFAULT_MAX_SIZE = envs.SGLANG_CUSTOM_ALL_REDUCE_V2_MAX_SIZE_KB.get() * 1024
|
||||
# forced per-direction workspace sizes; highest priority, override both the
|
||||
@@ -302,9 +297,6 @@ class CustomAllReduceV2:
|
||||
self.pull_enabled,
|
||||
)
|
||||
dist.barrier(group=self.group)
|
||||
# DEBUG-ONLY: per-instance seq counter for SGLANG_AR_SEQ_DEBUG logging
|
||||
self._dbg_seq = 0
|
||||
self._dbg_rank = self.rank
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Algo selection
|
||||
@@ -384,27 +376,6 @@ class CustomAllReduceV2:
|
||||
assert config is not None, f"No config for {nbytes = }"
|
||||
algo, use_graph, use_multicast = config
|
||||
graph_params = self._allocate_graph_row(input, nbytes) if use_graph else None
|
||||
if _AR_SEQ_DEBUG:
|
||||
seq = self._dbg_seq
|
||||
self._dbg_seq += 1
|
||||
# 采样 1/1024:生产高频路径下全量日志会拖垮 P 节点
|
||||
if seq & 1023 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] dev rank=%s seq=%d nbytes=%d algo=%s graph=%s capturing=%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
nbytes,
|
||||
getattr(algo, "name", algo),
|
||||
use_graph,
|
||||
torch.cuda.is_current_stream_capturing(),
|
||||
)
|
||||
if _AR_SEQ_DEBUG >= 2 and getattr(algo, "name", algo) == "ONE_SHOT_PUSH":
|
||||
logger.info(
|
||||
"[ARDBG] stack rank=%s seq=%d\n%s",
|
||||
self._dbg_rank,
|
||||
seq,
|
||||
"".join(traceback.format_stack(limit=15)),
|
||||
)
|
||||
return custom_all_reduce(
|
||||
self.obj,
|
||||
input,
|
||||
|
||||
@@ -2,7 +2,6 @@ from __future__ import annotations
|
||||
|
||||
import atexit
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import replace
|
||||
@@ -133,11 +132,6 @@ COMPONENT_REGISTRY: dict[ComponentType, type[TreeComponent]] = {
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# DEBUG-ONLY (p2 wedge trap 2026-10-13): env-gated logging for the per-step
|
||||
# hicache event sync. Same SGLANG_AR_SEQ_DEBUG switch as the ARDBG sites.
|
||||
_AR_SEQ_DEBUG = int(os.environ.get("SGLANG_AR_SEQ_DEBUG", "0"))
|
||||
_hc_step = 0
|
||||
|
||||
|
||||
class _OngoingWriteThrough(NamedTuple):
|
||||
"""Tracks an in-flight D→H write-through operation."""
|
||||
@@ -3539,16 +3533,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
|
||||
def check_hicache_events(self) -> None:
|
||||
"""Called per scheduler step to poll async HiCache events."""
|
||||
global _hc_step
|
||||
if _AR_SEQ_DEBUG:
|
||||
_hc_step += 1
|
||||
if _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcev rank=%s step=%d linker=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
self.linker is not None,
|
||||
)
|
||||
if self.linker is not None:
|
||||
finish_counts = torch.tensor(
|
||||
[
|
||||
@@ -3582,15 +3566,6 @@ class UnifiedRadixCache(BasePrefixCache):
|
||||
storage_queue_sizes,
|
||||
extra_pool_names,
|
||||
) = self._sync_hicache_ready_counts()
|
||||
if _AR_SEQ_DEBUG and _hc_step & 255 == 0:
|
||||
logger.info(
|
||||
"[ARDBG] hcsync rank=%s step=%d write=%s load=%s storage=%s",
|
||||
torch.distributed.get_rank(),
|
||||
_hc_step,
|
||||
write_finish_count,
|
||||
load_finish_count,
|
||||
storage_queue_sizes,
|
||||
)
|
||||
self.writing_check(finish_count=write_finish_count)
|
||||
self.loading_check(finish_count=load_finish_count)
|
||||
|
||||
|
||||
@@ -205,3 +205,6 @@ codegen-units = 1
|
||||
[profile.dev-opt]
|
||||
inherits = "dev"
|
||||
opt-level = 1
|
||||
|
||||
[patch.crates-io]
|
||||
openai-protocol = { path = "vendor/openai-protocol" }
|
||||
|
||||
@@ -284,90 +284,6 @@ struct CliArgs {
|
||||
#[arg(long, default_value_t = 86400, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_table_ttl_secs: u64,
|
||||
|
||||
/// est_T_D shadow estimate (D-side prealloc wait folded into shadow est_TTFT; observability only, default on; set false to roll back to the P-only model exactly)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_est_td_enabled: bool,
|
||||
|
||||
/// Shadow-consistent routing: live prefill pick follows the shadow decision's matched_node (requires --g9i-shadow-enabled; rejection gates unchanged; default off)
|
||||
#[arg(long, default_value_t = false, help_heading = "G9i Pool Admission")]
|
||||
g9i_shadow_route: bool,
|
||||
|
||||
/// T_D admission mode: off (default, ≡ current behavior) | shadow (log/count would-rejects, allow) | enforce (real 429s; requires --g9i-shadow-enabled)
|
||||
#[arg(long, default_value = "off", help_heading = "G9i Pool Admission")]
|
||||
g9i_td_admit: String,
|
||||
|
||||
/// T_D hard gate (seconds): when min over decode nodes of T_D exceeds this, every class gets 429 d_prealloc_saturated (0 = disabled)
|
||||
#[arg(long, default_value_t = 5.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_td_gate_secs: f64,
|
||||
|
||||
/// T_PREFER (seconds): prefix-holder affinity yields to a cheaper prefill candidate when the holder's est_TTFT exceeds this
|
||||
#[arg(long, default_value_t = 2.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_t_prefer_secs: f64,
|
||||
|
||||
/// Hard P-queue token cap (physical backstop; effective cap = min(this, drain_ewma x budget)) — 429 p_queue_cap only when ALL nodes over (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 16_000_000, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_token_cap: u64,
|
||||
|
||||
/// P-queue real-queue gate: skip prefill nodes whose engine-reported waiting queue (num_waiting_reqs) exceeds this (0 = disabled; follows --g9i-td-admit mode)
|
||||
#[arg(long, default_value_t = 10, help_heading = "G9i Pool Admission")]
|
||||
g9i_p_queue_reqs_cap: u64,
|
||||
|
||||
/// Admission hold queue: park holdable rejections (d_window_full / p_queue_cap) up to this many seconds instead of an immediate 429 (0 = disabled; enforce mode only; hold time counts into TTFT)
|
||||
#[arg(long, default_value_t = 1.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_secs: f64,
|
||||
|
||||
/// Admission hold queue bound: at most this many parked requests (full queue = immediate 429)
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_admit_hold_max: u64,
|
||||
|
||||
/// D-ledger pipeline booking TTL (seconds): force-release bookings that never saw a first token (wedge backstop; 0 = disabled)
|
||||
#[arg(long, default_value_t = 120, help_heading = "G9i Pool Admission")]
|
||||
g9i_pipeline_ttl_secs: u64,
|
||||
|
||||
/// D-ledger streaming booking TTL (seconds): force-release bookings stuck mid-decode (0 = disabled)
|
||||
#[arg(long, default_value_t = 600, help_heading = "G9i Pool Admission")]
|
||||
g9i_streaming_ttl_secs: u64,
|
||||
|
||||
/// Ledger reconcile: clamp the D ledger down to the engine-side in-flight when the excess is sustained >60s (subtract-only)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_ledger_reconcile: bool,
|
||||
|
||||
/// AIMD window initial pipeline depth per decode node (warmup start; clamped to [16, 256])
|
||||
#[arg(long, default_value_t = 128.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_init: f64,
|
||||
|
||||
/// AIMD additive-increase step per no-congestion tick (slo_ok / blind_recover; cut & holds unchanged)
|
||||
#[arg(long, default_value_t = 64.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_add: f64,
|
||||
|
||||
/// Pin the AIMD admission window at this value (clamped to [16, 256]); skips all window dynamics (0 = normal AIMD)
|
||||
#[arg(long, default_value_t = 0.0, help_heading = "G9i Pool Admission")]
|
||||
g9i_window_fixed: f64,
|
||||
|
||||
/// Prompt-length hard validation: 400 prompt_too_long at the front of admission when even the lower-bound estimate exceeds the context limit (default on)
|
||||
#[arg(long, default_value_t = true, help_heading = "G9i Pool Admission")]
|
||||
g9i_prompt_validation: bool,
|
||||
|
||||
/// Context-length limit (tokens) for the prompt_too_long lower-bound check (0 = disabled)
|
||||
#[arg(long, default_value_t = 1_048_576, help_heading = "G9i Pool Admission")]
|
||||
g9i_prompt_token_limit: u64,
|
||||
|
||||
/// T25 hot-channel admission A/B policy: off (default ≡ current behavior) | reservoir (Hot-class holdable rejections park in a priority reservoir up to --g9i-reservoir-max-ms instead of 429) | hardlane (cold classes may use only (1 - --g9i-hot-reserve-frac) of the d_window/p_queue budgets)
|
||||
#[arg(long, default_value = "off", help_heading = "G9i Pool Admission")]
|
||||
g9i_hot_policy: String,
|
||||
|
||||
/// T25-A reservoir wait budget in milliseconds (timeout → 429 reservoir_timeout; the wait counts into TTFT)
|
||||
#[arg(long, default_value_t = 1000, help_heading = "G9i Pool Admission")]
|
||||
g9i_reservoir_max_ms: u64,
|
||||
|
||||
/// T25-A reservoir capacity: at most this many parked requests (full → immediate 429 reservoir_full)
|
||||
#[arg(long, default_value_t = 512, help_heading = "G9i Pool Admission")]
|
||||
g9i_reservoir_cap: u64,
|
||||
|
||||
/// T25-B hardlane cold-budget reserve fraction: cold classes use at most (1 - frac) of the AIMD d_window and p_queue caps (fixed value, no adaptation)
|
||||
#[arg(long, default_value_t = 0.25, help_heading = "G9i Pool Admission")]
|
||||
g9i_hot_reserve_frac: f64,
|
||||
|
||||
// ==================== Service Discovery (Kubernetes) ====================
|
||||
/// Enable Kubernetes service discovery
|
||||
#[arg(
|
||||
@@ -1247,27 +1163,6 @@ impl CliArgs {
|
||||
shadow_enabled: self.g9i_shadow_enabled,
|
||||
shadow_table_max_entries: self.g9i_shadow_table_max_entries,
|
||||
shadow_table_ttl_secs: self.g9i_shadow_table_ttl_secs,
|
||||
est_td_enabled: self.g9i_est_td_enabled,
|
||||
shadow_route: self.g9i_shadow_route,
|
||||
td_admit: self.g9i_td_admit.clone(),
|
||||
td_gate_secs: self.g9i_td_gate_secs,
|
||||
t_prefer_secs: self.g9i_t_prefer_secs,
|
||||
p_queue_token_cap: self.g9i_p_queue_token_cap,
|
||||
p_queue_reqs_cap: self.g9i_p_queue_reqs_cap,
|
||||
admit_hold_secs: self.g9i_admit_hold_secs,
|
||||
admit_hold_max: self.g9i_admit_hold_max,
|
||||
pipeline_ttl_secs: self.g9i_pipeline_ttl_secs,
|
||||
streaming_ttl_secs: self.g9i_streaming_ttl_secs,
|
||||
ledger_reconcile: self.g9i_ledger_reconcile,
|
||||
window_init: self.g9i_window_init,
|
||||
window_add: self.g9i_window_add,
|
||||
window_fixed: self.g9i_window_fixed,
|
||||
prompt_validation: self.g9i_prompt_validation,
|
||||
prompt_token_limit: self.g9i_prompt_token_limit,
|
||||
hot_policy: self.g9i_hot_policy.clone(),
|
||||
reservoir_max_ms: self.g9i_reservoir_max_ms,
|
||||
reservoir_cap: self.g9i_reservoir_cap,
|
||||
hot_reserve_frac: self.g9i_hot_reserve_frac,
|
||||
};
|
||||
config.validate().map_err(|reason| ConfigError::ValidationFailed { reason })?;
|
||||
Ok(Some(config))
|
||||
|
||||
@@ -334,64 +334,6 @@ pub(crate) fn init_metrics() {
|
||||
smg_mesh::metrics::init_mesh_metrics();
|
||||
}
|
||||
|
||||
/// T25b: explicit Prometheus histogram buckets for every gateway histogram
|
||||
/// NOT covered by the `duration_seconds` suffix matcher (those keep the
|
||||
/// configurable duration_buckets). Root cause being fixed: without explicit
|
||||
/// buckets the exporter renders histograms as client-side SUMMARIES
|
||||
/// (DDSketch over a 3×20s rolling window); in the v16 deployment those
|
||||
/// summaries exported all-zero quantiles while _sum/_count stayed healthy
|
||||
/// (the rolling-window path silently drops samples when the quanta clock
|
||||
/// reading falls behind the newest bucket begin, poisoning the summary
|
||||
/// permanently). Explicit buckets render real Prometheus histograms —
|
||||
/// aggregable across time and replicas with histogram_quantile, and immune
|
||||
/// to the summary path entirely.
|
||||
///
|
||||
/// Groups by scale (keep in sync when adding a histogram!):
|
||||
/// * TTFT-scale seconds (0.5–120s): TTFT, e2e, est_T_P/T_D, residuals
|
||||
/// (signed values fall into the lowest bucket — acceptable),
|
||||
/// bootstrap actuals;
|
||||
/// * fine seconds (5ms–5s): TPOT, hold/reservoir waits, retry backoff;
|
||||
/// * prefix lookup depth (ring count, 1–400);
|
||||
/// * shadow-table evicted entry age (1s–1d).
|
||||
fn histogram_bucket_overrides() -> Vec<(Matcher, Vec<f64>)> {
|
||||
let ttft: Vec<f64> = vec![0.5, 1.0, 2.0, 5.0, 10.0, 20.0, 30.0, 60.0, 120.0];
|
||||
let fine: Vec<f64> = vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0];
|
||||
let depth: Vec<f64> = vec![1.0, 2.0, 3.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 400.0];
|
||||
let age: Vec<f64> = vec![1.0, 10.0, 60.0, 300.0, 900.0, 3600.0, 21600.0, 86400.0];
|
||||
let mut out: Vec<(Matcher, Vec<f64>)> = Vec::new();
|
||||
for name in [
|
||||
"smg_g9i_ttft_seconds",
|
||||
"smg_g9i_class_ttft_seconds",
|
||||
"smg_g9i_nonstream_e2e_secs",
|
||||
"smg_g9i_est_tp_secs",
|
||||
"smg_g9i_est_td_secs",
|
||||
"smg_g9i_est_td_slot_secs",
|
||||
"smg_g9i_shadow_ttft_est_error_seconds",
|
||||
"smg_g9i_est_vs_actual_residual",
|
||||
"smg_g9i_bootstrap_actual_secs",
|
||||
"smg_router_ttft_seconds",
|
||||
] {
|
||||
out.push((Matcher::Full(name.to_string()), ttft.clone()));
|
||||
}
|
||||
for name in [
|
||||
"smg_router_tpot_seconds",
|
||||
"smg_g9i_admit_hold_secs",
|
||||
"smg_g9i_reservoir_wait_seconds",
|
||||
"smg_worker_retry_backoff_seconds",
|
||||
] {
|
||||
out.push((Matcher::Full(name.to_string()), fine.clone()));
|
||||
}
|
||||
out.push((
|
||||
Matcher::Full("smg_g9i_shadow_prefix_lookup_depth".to_string()),
|
||||
depth,
|
||||
));
|
||||
out.push((
|
||||
Matcher::Full("smg_g9i_shadow_prefix_table_evicted_age_seconds".to_string()),
|
||||
age,
|
||||
));
|
||||
out
|
||||
}
|
||||
|
||||
pub fn start_prometheus(config: PrometheusConfig) {
|
||||
init_metrics();
|
||||
|
||||
@@ -409,20 +351,11 @@ pub fn start_prometheus(config: PrometheusConfig) {
|
||||
.unwrap_or(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)));
|
||||
let socket_addr = SocketAddr::new(ip_addr, config.port);
|
||||
|
||||
let mut builder = PrometheusBuilder::new()
|
||||
PrometheusBuilder::new()
|
||||
.with_http_listener(socket_addr)
|
||||
.upkeep_timeout(Duration::from_secs(5 * 60))
|
||||
.set_buckets_for_metric(duration_matcher, &duration_bucket)
|
||||
.expect("failed to set duration bucket");
|
||||
// T25b: every remaining gateway histogram gets explicit buckets (see
|
||||
// histogram_bucket_overrides) — no metric falls back to the broken
|
||||
// client-side summary path.
|
||||
for (matcher, buckets) in histogram_bucket_overrides() {
|
||||
builder = builder
|
||||
.set_buckets_for_metric(matcher, &buckets)
|
||||
.expect("failed to set g9i histogram buckets");
|
||||
}
|
||||
builder
|
||||
.expect("failed to set duration bucket")
|
||||
.install()
|
||||
.expect("failed to install Prometheus metrics exporter");
|
||||
}
|
||||
@@ -1580,61 +1513,4 @@ mod tests {
|
||||
assert_eq!(method_to_static_str("POST"), "POST");
|
||||
assert_eq!(method_to_static_str("UNKNOWN"), "OTHER");
|
||||
}
|
||||
|
||||
/// T25b smoke: with the production bucket overrides applied, g9i
|
||||
/// histograms render as real Prometheus histograms with non-zero le
|
||||
/// bucket counts (histogram_quantile-able) — never the client-side
|
||||
/// summary path that exported all-zero quantiles in v16.
|
||||
#[test]
|
||||
fn t25b_g9i_histograms_render_bucketed() {
|
||||
let mut builder = PrometheusBuilder::new();
|
||||
for (matcher, buckets) in histogram_bucket_overrides() {
|
||||
builder = builder
|
||||
.set_buckets_for_metric(matcher, &buckets)
|
||||
.expect("bucket override must apply");
|
||||
}
|
||||
let recorder = builder.build_recorder();
|
||||
let handle = recorder.handle();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
let ttft = histogram!("smg_g9i_class_ttft_seconds", "class" => "hot");
|
||||
for v in [0.8, 1.5, 3.0, 12.0, 45.0] {
|
||||
ttft.record(v);
|
||||
}
|
||||
histogram!("smg_g9i_shadow_prefix_lookup_depth").record(7.0);
|
||||
histogram!("smg_g9i_shadow_prefix_lookup_depth").record(120.0);
|
||||
});
|
||||
let out = handle.render();
|
||||
// Real histogram exposition, not summary.
|
||||
assert!(
|
||||
out.contains("# TYPE smg_g9i_class_ttft_seconds histogram"),
|
||||
"class ttft renders as histogram:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
!out.contains("smg_g9i_class_ttft_seconds{quantile="),
|
||||
"no summary quantile lines:\n{out}"
|
||||
);
|
||||
// 3 of the 5 samples (0.8, 1.5, 3.0) land at or below le=5.
|
||||
assert!(
|
||||
out.contains("smg_g9i_class_ttft_seconds_bucket{class=\"hot\",le=\"5\"} 3"),
|
||||
"le=5 bucket counts 3 samples:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_class_ttft_seconds_bucket{class=\"hot\",le=\"+Inf\"} 5"),
|
||||
"+Inf bucket counts all 5:\n{out}"
|
||||
);
|
||||
// lookup_depth: 7 -> le=10; both samples (7, 120) <= 200
|
||||
// (Prometheus buckets are cumulative).
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_bucket{le=\"10\"} 1"),
|
||||
"depth 7 lands in le=10:\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_bucket{le=\"200\"} 2"),
|
||||
"both depths land at or below le=200 (cumulative):\n{out}"
|
||||
);
|
||||
assert!(
|
||||
out.contains("smg_g9i_shadow_prefix_lookup_depth_count 2"),
|
||||
"count line present:\n{out}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -301,11 +301,12 @@ impl HarmonyBuilder {
|
||||
.reasoning
|
||||
.as_ref()
|
||||
.and_then(|r| r.effort.as_ref())
|
||||
.map(|effort| match effort {
|
||||
ResponsesReasoningEffort::High => ReasoningEffort::High,
|
||||
ResponsesReasoningEffort::Medium => ReasoningEffort::Medium,
|
||||
ResponsesReasoningEffort::Low => ReasoningEffort::Low,
|
||||
ResponsesReasoningEffort::Minimal => ReasoningEffort::Low,
|
||||
.and_then(|effort| match effort {
|
||||
ResponsesReasoningEffort::None => None,
|
||||
ResponsesReasoningEffort::High => Some(ReasoningEffort::High),
|
||||
ResponsesReasoningEffort::Medium => Some(ReasoningEffort::Medium),
|
||||
ResponsesReasoningEffort::Low => Some(ReasoningEffort::Low),
|
||||
ResponsesReasoningEffort::Minimal => Some(ReasoningEffort::Low),
|
||||
});
|
||||
|
||||
self.build_system_message(reasoning_effort, with_custom_tools)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -58,23 +58,6 @@
|
||||
//! canonical meta object. The META segment is unconditional — a
|
||||
//! message without these fields contributes `TAG_META ‖ len=0`,
|
||||
//! keeping the formula single-path (no presence/absence ambiguity).
|
||||
//!
|
||||
//! T23 (intra-message chunking): a text piece longer than `CHUNK_BYTES`
|
||||
//! (2048) is first rolled into bare chunk rings `xxh3(prev ‖ chunk)` —
|
||||
//! no tag, no length prefix — in content order, and the message-final
|
||||
//! ring then carries only the remaining ≤2048-byte tail in its content
|
||||
//! segment. Why: a per-message chain is structurally blind inside one
|
||||
//! long message (one message = one ring, so a shared 100KB prefix with
|
||||
//! a different tail hits nothing — v13 measured 9034/9034 depth-0
|
||||
//! misses on single-message long-prompt traffic). Chunk rings give the
|
||||
//! lookup table mid-message anchors at a fixed granularity. Texts of
|
||||
//! ≤2048 bytes emit no chunk rings, so short-message chains stay
|
||||
//! byte-identical to the pre-T23 formula (pinned by
|
||||
//! `short_messages_match_legacy_chain_byte_for_byte`). Because one
|
||||
//! message may now own several rings, `chain_hashes_indexed` also
|
||||
//! returns each ring's message index; `classify` needs it to map a
|
||||
//! hit's ring position back into the message index space of
|
||||
//! `msg_bytes` / `msg_image`.
|
||||
|
||||
use serde_json::Value;
|
||||
use xxhash_rust::xxh3::{xxh3_64, Xxh3};
|
||||
@@ -87,13 +70,6 @@ const TAG_UNKNOWN: &[u8] = b"unknown\0";
|
||||
const TAG_TOOL_CALLS: &[u8] = b"tool_calls\0";
|
||||
const TAG_META: &[u8] = b"meta\0";
|
||||
|
||||
/// T23: intra-message chunk size in bytes. A text piece longer than this
|
||||
/// contributes `ceil`-style bare chunk rings before its message-final
|
||||
/// ring (see module docs). 2048 bytes ≈ 500-600 tokens of English/JSON —
|
||||
/// fine enough to rescue shared-prefix hits, coarse enough that typical
|
||||
/// multi-turn chats (short messages) never pay the extra hash passes.
|
||||
const CHUNK_BYTES: usize = 2048;
|
||||
|
||||
/// Chain seed: folds every body field that changes the rendered prompt but
|
||||
/// lives outside `messages` into one u64.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
@@ -145,8 +121,7 @@ impl ChainSeed {
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the chain hashes `h[1..=m]`, one ring per message (plus T23
|
||||
/// intra-message chunk rings), in order.
|
||||
/// Compute the chain hashes `h[1..=m]`, one per message, in order.
|
||||
///
|
||||
/// `messages` missing / not an array / empty → empty `Vec`. The caller
|
||||
/// treats an empty chain as "cannot classify" (same fall-back path as
|
||||
@@ -156,53 +131,11 @@ impl ChainSeed {
|
||||
/// The returned chain is deterministic even when `is_unclassifiable(body)`
|
||||
/// is true — unknown parts contribute their tag — but such chains are only
|
||||
/// suitable for logging/shadow comparison, not for bookkeeping lookups.
|
||||
///
|
||||
/// This is the chain-only view of `chain_hashes_indexed`; callers that
|
||||
/// index into per-message arrays (`msg_bytes`, `msg_image`) need the
|
||||
/// indexed form to translate ring positions to message indices.
|
||||
pub fn chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
chain_hashes_indexed(seed, body).0
|
||||
}
|
||||
|
||||
/// `chain_hashes` plus, for each ring, the index of the message it
|
||||
/// belongs to (chunk rings and the message-final ring of message `k`
|
||||
/// all carry `k`). Both returned vecs always have the same length.
|
||||
pub fn chain_hashes_indexed(seed: &ChainSeed, body: &Value) -> (Vec<u64>, Vec<usize>) {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut ring_msg = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for (mi, msg) in msgs.iter().enumerate() {
|
||||
// T23 pass 1: roll oversized text pieces into bare chunk rings
|
||||
// (h = xxh3(prev ‖ chunk), no tag/length) and keep the ≤2048-byte
|
||||
// tails for the message-final ring. Short texts emit nothing here
|
||||
// and their tails ARE the originals — the pre-T23 byte stream.
|
||||
let content = match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
ContentView::Str(roll_chunks(&mut prev, &mut chain, &mut ring_msg, mi, s))
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
let mut viewed = Vec::with_capacity(parts.len());
|
||||
for part in parts {
|
||||
viewed.push(match part {
|
||||
PartView::Text(t) => PartView::Text(roll_chunks(
|
||||
&mut prev,
|
||||
&mut chain,
|
||||
&mut ring_msg,
|
||||
mi,
|
||||
t,
|
||||
)),
|
||||
PartView::ImageUrl(u) => PartView::ImageUrl(u),
|
||||
PartView::Unknown => PartView::Unknown,
|
||||
});
|
||||
}
|
||||
ContentView::Parts(viewed)
|
||||
}
|
||||
ContentView::Empty => ContentView::Empty,
|
||||
};
|
||||
|
||||
// Pass 2: the message-final ring — the pre-T23 formula unchanged,
|
||||
// except its content segment now hashes the pass-1 tails.
|
||||
for msg in &msgs {
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
@@ -210,7 +143,7 @@ pub fn chain_hashes_indexed(seed: &ChainSeed, body: &Value) -> (Vec<u64>, Vec<us
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &content {
|
||||
match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
@@ -240,47 +173,8 @@ pub fn chain_hashes_indexed(seed: &ChainSeed, body: &Value) -> (Vec<u64>, Vec<us
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
ring_msg.push(mi);
|
||||
}
|
||||
(chain, ring_msg)
|
||||
}
|
||||
|
||||
/// Roll `prev` forward over `text`'s full chunks, emitting one bare ring
|
||||
/// per chunk (`xxh3(prev ‖ chunk bytes)`), and return the remaining tail
|
||||
/// (≤ CHUNK_BYTES, possibly empty) for the message-final ring. Texts of
|
||||
/// ≤ CHUNK_BYTES emit nothing and are returned untouched.
|
||||
fn roll_chunks<'a>(
|
||||
prev: &mut u64,
|
||||
chain: &mut Vec<u64>,
|
||||
ring_msg: &mut Vec<usize>,
|
||||
mi: usize,
|
||||
text: &'a str,
|
||||
) -> &'a str {
|
||||
let mut rest = text;
|
||||
while rest.len() > CHUNK_BYTES {
|
||||
let (chunk, tail) = take_chunk(rest);
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
h.update(chunk.as_bytes());
|
||||
*prev = h.digest();
|
||||
chain.push(*prev);
|
||||
ring_msg.push(mi);
|
||||
rest = tail;
|
||||
}
|
||||
rest
|
||||
}
|
||||
|
||||
/// Split off the leading chunk of at most CHUNK_BYTES bytes, backing off
|
||||
/// to the previous UTF-8 char boundary when the cut lands mid-character.
|
||||
/// The caller guarantees `s.len() > CHUNK_BYTES`, so the boundary walk
|
||||
/// can never reach 0 (the first char is at most 4 bytes).
|
||||
fn take_chunk(s: &str) -> (&str, &str) {
|
||||
debug_assert!(s.len() > CHUNK_BYTES);
|
||||
let mut end = CHUNK_BYTES;
|
||||
while !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
s.split_at(end)
|
||||
chain
|
||||
}
|
||||
|
||||
/// Whether the body contains content this module has no hashing semantics
|
||||
@@ -778,202 +672,4 @@ mod tests {
|
||||
let no_name = text_body(vec![json!({"role": "user", "content": "hi"})]);
|
||||
assert_eq!(chain(&plain), chain(&no_name));
|
||||
}
|
||||
|
||||
/// `reasoning_content` must NOT change the chain: chat templates do not
|
||||
/// render historical reasoning into the prompt, so hashing it would
|
||||
/// manufacture misses for thinking-enabled clients that echo it back.
|
||||
/// Pins the walk_messages assumption (thinking:true is our default).
|
||||
#[test]
|
||||
fn reasoning_content_does_not_enter_chain() {
|
||||
let plain = text_body(vec![user("q"), assistant("a"), user("q2")]);
|
||||
let with_reasoning = text_body(vec![
|
||||
user("q"),
|
||||
json!({"role": "assistant", "content": "a", "reasoning_content": "let me think..."}),
|
||||
user("q2"),
|
||||
]);
|
||||
assert_eq!(
|
||||
chain(&plain),
|
||||
chain(&with_reasoning),
|
||||
"reasoning_content is not rendered -> must not change the chain"
|
||||
);
|
||||
assert!(
|
||||
!is_unclassifiable(&with_reasoning),
|
||||
"reasoning-carrying turns stay classifiable"
|
||||
);
|
||||
}
|
||||
|
||||
// ==================== T23: intra-message chunking ====================
|
||||
|
||||
/// The pre-T23 `chain_hashes`, verbatim copy. Short-message bodies must
|
||||
/// produce byte-identical chains under the chunked implementation.
|
||||
fn legacy_chain_hashes(seed: &ChainSeed, body: &Value) -> Vec<u64> {
|
||||
let (msgs, _) = walk_messages(body);
|
||||
let mut chain = Vec::with_capacity(msgs.len());
|
||||
let mut prev = seed.value;
|
||||
for msg in &msgs {
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&prev.to_le_bytes());
|
||||
|
||||
let role = msg.role.as_bytes();
|
||||
h.update(&(role.len() as u64).to_le_bytes());
|
||||
h.update(role);
|
||||
|
||||
match &msg.content {
|
||||
ContentView::Str(s) => {
|
||||
update_tagged(&mut h, TAG_TEXT, s.as_bytes());
|
||||
}
|
||||
ContentView::Parts(parts) => {
|
||||
for part in parts {
|
||||
match part {
|
||||
PartView::Text(t) => update_tagged(&mut h, TAG_TEXT, t.as_bytes()),
|
||||
PartView::ImageUrl(u) => {
|
||||
update_tagged(&mut h, TAG_IMAGE_URL, u.as_bytes())
|
||||
}
|
||||
PartView::Unknown => h.update(TAG_UNKNOWN),
|
||||
}
|
||||
}
|
||||
}
|
||||
ContentView::Empty => {}
|
||||
}
|
||||
|
||||
if let Some(tc) = &msg.tool_calls {
|
||||
update_tagged(&mut h, TAG_TOOL_CALLS, tc.as_bytes());
|
||||
}
|
||||
|
||||
update_tagged(&mut h, TAG_META, msg.extra.as_bytes());
|
||||
|
||||
prev = h.digest();
|
||||
chain.push(prev);
|
||||
}
|
||||
chain
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_messages_match_legacy_chain_byte_for_byte() {
|
||||
let exact_boundary = "x".repeat(CHUNK_BYTES);
|
||||
let bodies = vec![
|
||||
// pure-text multi-turn
|
||||
text_body(vec![user("q1"), assistant("a1"), user("q2")]),
|
||||
// multimodal parts (short text + image)
|
||||
text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "look"},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}}
|
||||
]
|
||||
})]),
|
||||
// tool_calls + content:null + tool result (meta on both)
|
||||
text_body(vec![
|
||||
user("weather?"),
|
||||
json!({"role": "assistant", "content": null,
|
||||
"tool_calls": tool_call("get_weather", "{\"city\":\"sh\"}")}),
|
||||
json!({"role": "tool", "tool_call_id": "call_1", "content": "sunny"}),
|
||||
]),
|
||||
// name meta
|
||||
text_body(vec![json!({"role": "user", "name": "alice", "content": "hi"})]),
|
||||
// null content without tool_calls
|
||||
text_body(vec![json!({"role": "assistant", "content": null})]),
|
||||
// exactly CHUNK_BYTES: never splits (only > CHUNK_BYTES does)
|
||||
text_body(vec![user(&exact_boundary)]),
|
||||
];
|
||||
for b in &bodies {
|
||||
let seed = ChainSeed::from_body(b);
|
||||
assert_eq!(
|
||||
chain_hashes(&seed, b),
|
||||
legacy_chain_hashes(&seed, b),
|
||||
"short-message chain must stay byte-identical to pre-T23"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_message_splits_into_chunk_rings() {
|
||||
// CHUNK_BYTES + 1 -> one chunk ring + the message-final ring.
|
||||
let text = "x".repeat(CHUNK_BYTES + 1);
|
||||
let body = text_body(vec![user(&text)]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(idx, vec![0, 0], "both rings belong to message 0");
|
||||
// The chunk ring is bare: xxh3(prev_le ‖ chunk), no tag, no length.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update("x".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
|
||||
// Exactly CHUNK_BYTES -> single ring (only > CHUNK_BYTES splits);
|
||||
// already pinned by the legacy-equivalence battery above.
|
||||
assert_eq!(chain(&text_body(vec![user(&"y".repeat(CHUNK_BYTES))])).len(), 1);
|
||||
}
|
||||
|
||||
/// The T23 acceptance shape: two single-message bodies sharing a 4KB
|
||||
/// prefix but differing in the tail now share their chunk rings — the
|
||||
/// pre-T23 chain (one ring per message) could never hit here.
|
||||
#[test]
|
||||
fn shared_long_prefix_different_tail_shares_chunk_rings() {
|
||||
let prefix = "a".repeat(2 * CHUNK_BYTES);
|
||||
let a = text_body(vec![user(&format!("{prefix}{}", "b".repeat(100)))]);
|
||||
let b = text_body(vec![user(&format!("{prefix}{}", "c".repeat(200)))]);
|
||||
let (ca, ia) = chain_hashes_indexed(&ChainSeed::from_body(&a), &a);
|
||||
let (cb, ib) = chain_hashes_indexed(&ChainSeed::from_body(&b), &b);
|
||||
assert_eq!(ca.len(), 3, "2 chunk rings + message-final ring");
|
||||
assert_eq!(cb.len(), 3);
|
||||
assert_eq!(ia, vec![0, 0, 0]);
|
||||
assert_eq!(ib, vec![0, 0, 0]);
|
||||
assert_eq!(
|
||||
ca[..2],
|
||||
cb[..2],
|
||||
"identical 4KB prefix -> identical chunk rings"
|
||||
);
|
||||
assert_ne!(ca[2], cb[2], "different tail -> final ring diverges");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utf8_boundary_chunk_never_splits_a_character() {
|
||||
// 700 x 3 bytes = 2100 > 2048; the 2048 cut lands mid-汉
|
||||
// (2048 % 3 = 2), so the chunk must back off to 2046 bytes.
|
||||
let text = "汉".repeat(700);
|
||||
let (chunk, tail) = take_chunk(&text);
|
||||
assert_eq!(chunk.len(), 2046);
|
||||
assert_eq!(tail.len(), 54);
|
||||
assert_eq!(format!("{chunk}{tail}"), text, "chunk + tail reconstruct");
|
||||
|
||||
let body = text_body(vec![user(&text)]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 2);
|
||||
assert_eq!(idx, vec![0, 0]);
|
||||
// The chunk ring carries exactly the boundary-aligned bytes.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update(chunk.as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multi_part_long_text_chunks_per_part_in_order() {
|
||||
let t1 = "p".repeat(CHUNK_BYTES + 10); // 1 chunk + 10-byte tail
|
||||
let t2 = "q".repeat(CHUNK_BYTES + 20); // 1 chunk + 20-byte tail
|
||||
let body = text_body(vec![json!({
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": t1},
|
||||
{"type": "image_url", "image_url": {"url": "https://x/a.png"}},
|
||||
{"type": "text", "text": t2}
|
||||
]
|
||||
})]);
|
||||
let seed = ChainSeed::from_body(&body);
|
||||
let (c, idx) = chain_hashes_indexed(&seed, &body);
|
||||
assert_eq!(c.len(), 3, "t1 chunk + t2 chunk + message-final ring");
|
||||
assert_eq!(idx, vec![0, 0, 0]);
|
||||
// Chunk rings follow part order: t1's chunk first, then t2's.
|
||||
let mut h = Xxh3::new();
|
||||
h.update(&seed.value.to_le_bytes());
|
||||
h.update("p".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[0], h.digest());
|
||||
let mut h2 = Xxh3::new();
|
||||
h2.update(&c[0].to_le_bytes());
|
||||
h2.update("q".repeat(CHUNK_BYTES).as_bytes());
|
||||
assert_eq!(c[1], h2.digest());
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -378,92 +378,6 @@ impl PDRouter {
|
||||
Ok((prefill_request, decode_request))
|
||||
}
|
||||
|
||||
/// G9i admission + shadow booking, ONE attempt (T15: the caller retries
|
||||
/// holdable rejections through the hold queue). None = G9i not
|
||||
/// configured ("untracked"). Err = rejected (the caller counts +
|
||||
/// answers 429); the rejections are structured so that a rejected
|
||||
/// attempt never entered the shadow bookkeeping (T11 拒绝路径零记账).
|
||||
/// T25-A: `prior_wait_secs` is the time the request already spent
|
||||
/// parked in the reservoir across re-runs — it counts into TTFT, so
|
||||
/// the re-judged est_TTFT carries it.
|
||||
async fn g9i_admission<T: Serialize + Clone>(
|
||||
&self,
|
||||
context: &PDRequestContext<'_>,
|
||||
original_request: &T,
|
||||
prior_wait_secs: f64,
|
||||
) -> Option<Result<(Option<String>, &'static str, Option<ShadowRequestContext>), super::g9i::RejectInfo>>
|
||||
{
|
||||
let g9i = self.g9i.as_ref()?;
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
return Some(Err(info));
|
||||
}
|
||||
if context.route != "/v1/chat/completions" {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
return Some(Err(info));
|
||||
}
|
||||
return Some(Ok((None, "non_chat", None)));
|
||||
}
|
||||
let body = match serde_json::to_value(original_request) {
|
||||
Ok(body) => body,
|
||||
Err(_) => return Some(Ok((None, "regular", None))),
|
||||
};
|
||||
// Shadow T2: compute the prefix chain once, here, alongside the v1
|
||||
// classification. Skipped entirely when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let (plan, path, probe_new_work) = g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await;
|
||||
// T11: v1 rejection FIRST — a request rejected by any gate must
|
||||
// never enter the shadow bookkeeping (拒绝路径零记账).
|
||||
let v1_override = match plan.into_override() {
|
||||
Ok(o) => o,
|
||||
Err(info) => return Some(Err(info)),
|
||||
};
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
// T11: classify + candidate pick + cap verdict + admission
|
||||
// in ONE queue-model critical section (atomic
|
||||
// check-and-act) — the check→act window no longer spans
|
||||
// the TTFT wait. Err = p_queue_cap (enforce), nothing
|
||||
// admitted.
|
||||
match g9i.shadow_classify_and_admit(&mut ctx) {
|
||||
Ok(Some(mut decision)) => {
|
||||
// T25-A: time already parked in the reservoir counts
|
||||
// into TTFT — the re-judged estimate carries it.
|
||||
if prior_wait_secs > 0.0 {
|
||||
decision.est_ttft_secs += prior_wait_secs;
|
||||
ctx.decision = Some(decision.clone());
|
||||
}
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(info) => return Some(Err(info)),
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
// T7 shadow-consistent routing (--g9i-shadow-route): the live
|
||||
// prefill override follows the shadow decision's matched_node (=
|
||||
// the booked node), so est vs actual is pure estimation error.
|
||||
let override_url = g9i.shadow_route_pick(
|
||||
v1_override,
|
||||
shadow_ctx.as_ref().and_then(|c| c.decision.as_ref()),
|
||||
);
|
||||
Some(Ok((override_url, path, shadow_ctx)))
|
||||
}
|
||||
|
||||
async fn execute_dual_dispatch<T: Serialize + Clone>(
|
||||
&self,
|
||||
headers: Option<&HeaderMap>,
|
||||
@@ -496,130 +410,82 @@ impl PDRouter {
|
||||
// shadow bookkeeping carrier (T2): Some only for chat requests with
|
||||
// a classifiable prefix chain while shadow mode is on; it never
|
||||
// influences routing.
|
||||
//
|
||||
// T15: holdable rejections (d_window_full / p_queue_cap) park the
|
||||
// request in the FIFO hold queue instead of an immediate 429
|
||||
// (429 太伤吞吐 — 测试端把 429 当失败). On pass the COMPLETE
|
||||
// admission re-runs (every gate re-checked; the T11 booking happens
|
||||
// only on a passing attempt — a parked request occupies no ledger).
|
||||
// T20 (B): prompt_too_long hard validation — the FRONT-MOST check,
|
||||
// before any admission gate / est_TTFT / ledger / queue bookkeeping
|
||||
// and before the hold loop (an invalid request must never park).
|
||||
// 400 (client bug), counted in smg_g9i_invalid_request_total, never
|
||||
// in smg_g9i_rejects_total. Chat-completions only.
|
||||
if context.route == "/v1/chat/completions" {
|
||||
if let Some(g9i) = &self.g9i {
|
||||
if let Ok(body) = serde_json::to_value(original_request) {
|
||||
if let Some(resp) = g9i.prompt_too_long_rejection(&body) {
|
||||
return resp;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut hold_deadline: Option<Instant> = None;
|
||||
// T25-A reservoir park state: (deadline, seq id) across re-runs +
|
||||
// the accumulated wait (counts into the est_TTFT re-judgment).
|
||||
let mut reservoir_state: Option<(Instant, u64)> = None;
|
||||
let mut reservoir_waited_secs: f64 = 0.0;
|
||||
let (g9i_prefill_override, g9i_ttft_path, shadow_ctx): (
|
||||
Option<String>,
|
||||
&'static str,
|
||||
Option<ShadowRequestContext>,
|
||||
) = loop {
|
||||
match self
|
||||
.g9i_admission(&context, original_request, reservoir_waited_secs)
|
||||
.await
|
||||
{
|
||||
None => {
|
||||
// T25: per-request admission outcome (untracked = no
|
||||
// G9i configured — class "none").
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => "none",
|
||||
"result" => "admitted",
|
||||
"reason" => "none",
|
||||
)
|
||||
) = if let Some(g9i) = &self.g9i {
|
||||
if let Err(info) = g9i.check_decode_pool(&self.worker_registry).await {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
break (None, "untracked", None);
|
||||
}
|
||||
Some(Ok(outcome)) => {
|
||||
// T25: per-request admission outcome — the shadow
|
||||
// decision's class (the experiment's primary counter).
|
||||
let class = outcome
|
||||
.2
|
||||
.as_ref()
|
||||
.and_then(|c| c.decision.as_ref())
|
||||
.map(|d| d.class.as_str())
|
||||
.unwrap_or("none");
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => class,
|
||||
"result" => "admitted",
|
||||
"reason" => "none",
|
||||
)
|
||||
.increment(1);
|
||||
break outcome;
|
||||
}
|
||||
Some(Err(mut info)) => {
|
||||
let mut retry = false;
|
||||
if let Some(g9i) = &self.g9i {
|
||||
// T25-A: under hot_policy=reservoir, HOT-class
|
||||
// holdable rejections park in the priority
|
||||
// reservoir INSTEAD of the T15 hold (the A/B stays
|
||||
// clean — every other class keeps the hold path).
|
||||
if g9i.reservoir_eligible(&info) {
|
||||
match g9i
|
||||
.reservoir_or_reject(&info, &mut reservoir_state)
|
||||
.await
|
||||
{
|
||||
Ok(waited) => {
|
||||
reservoir_waited_secs += waited;
|
||||
retry = true;
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
if context.route == "/v1/chat/completions" {
|
||||
match serde_json::to_value(original_request) {
|
||||
Ok(body) => {
|
||||
// Shadow T2: compute the prefix chain once, here,
|
||||
// alongside the v1 classification. Skipped entirely
|
||||
// when shadow mode is off.
|
||||
let shadow_ctx = if g9i.shadow_enabled() {
|
||||
ShadowRequestContext::from_body(&body, context.is_stream)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
match g9i
|
||||
.classify_prefill(&self.worker_registry, &self.client, &body)
|
||||
.await
|
||||
{
|
||||
(plan, path, probe_new_work) => {
|
||||
// Shadow T3: shadow-classify alongside the v1
|
||||
// verdict and record agreement/divergence
|
||||
// counters. Routing still follows the v1 plan
|
||||
// alone; the decision only rides the context
|
||||
// into dispatch for queue bookkeeping.
|
||||
let shadow_ctx = match shadow_ctx {
|
||||
Some(mut ctx) => {
|
||||
if let Some(decision) = g9i.shadow_classify(&ctx) {
|
||||
super::g9i_shadow::record_agreement(
|
||||
path,
|
||||
probe_new_work,
|
||||
&ctx,
|
||||
&decision,
|
||||
);
|
||||
ctx.decision = Some(decision);
|
||||
}
|
||||
Some(ctx)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
match plan.into_override() {
|
||||
Ok(override_url) => (override_url, path, shadow_ctx),
|
||||
Err(info) => {
|
||||
metrics::counter!(
|
||||
"smg_g9i_rejects_total",
|
||||
"reason" => info.reason
|
||||
)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
}
|
||||
Err(final_info) => info = final_info,
|
||||
}
|
||||
} else if info.holdable() {
|
||||
let body_bytes = context
|
||||
.request_text
|
||||
.as_deref()
|
||||
.map(|s| s.len() as u64)
|
||||
.unwrap_or(0);
|
||||
match g9i
|
||||
.hold_or_reject(&info, body_bytes, &mut hold_deadline)
|
||||
.await
|
||||
{
|
||||
Ok(()) => retry = true,
|
||||
Err(final_info) => info = final_info,
|
||||
}
|
||||
}
|
||||
}
|
||||
if retry {
|
||||
continue;
|
||||
}
|
||||
// T25: per-request rejection outcome (class from the
|
||||
// rejecting gate; "none" on non-chat/early gates).
|
||||
metrics::counter!(
|
||||
super::g9i::METRIC_ADMISSION_TOTAL,
|
||||
"class" => info.hold_class.unwrap_or("none"),
|
||||
"result" => "rejected",
|
||||
"reason" => info.reason,
|
||||
)
|
||||
.increment(1);
|
||||
Err(_) => (None, "regular", None),
|
||||
}
|
||||
} else {
|
||||
// Non-chat routes skip classification but still land on
|
||||
// the regular pool — the regular queue gate applies.
|
||||
if let Err(info) = g9i.check_regular_queue(&self.worker_registry) {
|
||||
metrics::counter!("smg_g9i_rejects_total", "reason" => info.reason)
|
||||
.increment(1);
|
||||
return super::g9i::reject_response(&info);
|
||||
}
|
||||
(None, "non_chat", None)
|
||||
}
|
||||
} else {
|
||||
(None, "untracked", None)
|
||||
};
|
||||
|
||||
// T11: handle of the decision-time booking, extracted before the
|
||||
// retry closure moves `shadow_ctx`. If the dispatch never reached
|
||||
// the response path (all attempts failed before headers), the
|
||||
// booking is released after the loop — exactly once, paired with
|
||||
// shadow_admit's consume mark.
|
||||
let admission_mark = shadow_ctx.as_ref().and_then(|c| c.admission.clone());
|
||||
|
||||
let response = RetryExecutor::execute_response_with_retry(
|
||||
&self.retry_config,
|
||||
{
|
||||
@@ -799,13 +665,6 @@ impl PDRouter {
|
||||
);
|
||||
}
|
||||
|
||||
// T11: release a decision-time booking that no response path ever
|
||||
// consumed (all attempts failed before headers). No-op when the
|
||||
// stream/non-stream path already took it over.
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.shadow_release_unconsumed(admission_mark);
|
||||
}
|
||||
|
||||
response
|
||||
}
|
||||
|
||||
@@ -1165,7 +1024,6 @@ impl PDRouter {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
super::g9i::normalize_url(decode.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
@@ -1193,7 +1051,6 @@ impl PDRouter {
|
||||
ShadowResponseContext::from_request(
|
||||
ctx,
|
||||
super::g9i::normalize_url(prefill.url()),
|
||||
super::g9i::normalize_url(decode.url()),
|
||||
)
|
||||
});
|
||||
// Shadow T3: admit into the queue model keyed by the
|
||||
@@ -1204,21 +1061,14 @@ impl PDRouter {
|
||||
g9i.shadow_admit(s);
|
||||
}
|
||||
if context.return_logprob {
|
||||
let resp = self
|
||||
.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await;
|
||||
// T17: non-streaming e2e (observability only — the
|
||||
// AIMD blind state reads local signals instead).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
resp
|
||||
self.process_non_streaming_response(
|
||||
res,
|
||||
status,
|
||||
context.return_logprob,
|
||||
prefill_body,
|
||||
shadow,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
// Direct passthrough when no logprobs needed
|
||||
let response_headers =
|
||||
@@ -1228,15 +1078,9 @@ impl PDRouter {
|
||||
Ok(decode_body) => {
|
||||
let decode_body =
|
||||
self.shadow_record_non_stream(decode_body, status, &shadow);
|
||||
let response = Self::non_stream_passthrough_response(
|
||||
status,
|
||||
decode_body,
|
||||
response_headers,
|
||||
);
|
||||
// T17: non-streaming e2e (observability only).
|
||||
if let Some(g9i) = &self.g9i {
|
||||
g9i.record_nonstream_e2e(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
let mut response = Response::new(Body::from(decode_body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = response_headers;
|
||||
response
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -1542,11 +1386,7 @@ impl PDRouter {
|
||||
// (malformed) repeat usage event from
|
||||
// double-completing the queue slot.
|
||||
if !shadow_done {
|
||||
// Streaming harvests the usage
|
||||
// event pre-strip, so the
|
||||
// breakdown extraction inside
|
||||
// works unaided (None).
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs, None);
|
||||
g9i.shadow_record(sctx, &json, shadow_ttft_secs);
|
||||
shadow_done = true;
|
||||
}
|
||||
}
|
||||
@@ -1567,34 +1407,11 @@ impl PDRouter {
|
||||
g9i_path,
|
||||
start_time.elapsed().as_secs_f64(),
|
||||
);
|
||||
// T25: TTFT by shadow class — the
|
||||
// same gateway-observed first-token
|
||||
// moment (start_time predates the
|
||||
// admission loop, so reservoir /
|
||||
// hold waits are included).
|
||||
if let Some((_, sctx)) = &g9i_shadow {
|
||||
if let Some(class) =
|
||||
sctx.class.map(|c| c.as_str())
|
||||
{
|
||||
metrics::histogram!(
|
||||
super::g9i::METRIC_CLASS_TTFT,
|
||||
"class" => class,
|
||||
)
|
||||
.record(start_time.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
}
|
||||
// Shadow T3: the same first-token moment
|
||||
// calibrates est_TTFT (est − actual).
|
||||
shadow_ttft_secs =
|
||||
ttft_start.map(|t| t.elapsed().as_secs_f64());
|
||||
// T16: first token -> the D booking moves
|
||||
// from the pipeline to the streaming
|
||||
// sub-ledger (the window gates pipeline
|
||||
// depth only).
|
||||
if let Some((g9i, sctx)) = &g9i_shadow {
|
||||
g9i.shadow_promote_streaming(sctx);
|
||||
}
|
||||
}
|
||||
|
||||
let result = if return_logprob && prefill_logprobs.is_some() {
|
||||
@@ -1755,15 +1572,12 @@ impl PDRouter {
|
||||
g9i.shadow_release(ctx);
|
||||
return body;
|
||||
}
|
||||
let (parsed, rewritten, breakdown) =
|
||||
let (parsed, rewritten) =
|
||||
super::g9i_shadow::strip_and_extract(&body, ctx.strip_cached_details);
|
||||
match &parsed {
|
||||
// TTFT calibration is streaming-only (non-streaming has no
|
||||
// observable first-token moment) -> None. The breakdown was
|
||||
// extracted pre-strip and is passed through — the parsed JSON
|
||||
// here is post-strip, so re-extracting from it would always
|
||||
// miss (the 2026-10-17 order bug).
|
||||
Some(json) => g9i.shadow_record(ctx, json, None, breakdown),
|
||||
// observable first-token moment) -> None.
|
||||
Some(json) => g9i.shadow_record(ctx, json, None),
|
||||
// Unparseable success body: nothing to record, but the dispatch
|
||||
// is over — release rather than leak the outstanding count.
|
||||
None => g9i.shadow_release(ctx),
|
||||
@@ -1774,27 +1588,6 @@ impl PDRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// Assemble the non-streaming passthrough response. The shadow strip
|
||||
/// may have REWRITTEN (shortened) the body, so the upstream
|
||||
/// Content-Length in `headers` is stale; forwarding it verbatim
|
||||
/// truncated cache-hit responses at exactly the stripped length (the
|
||||
/// 2026-10-17 IncompleteRead incident). Pin Content-Length to the
|
||||
/// actual body length — correct for rewritten and untouched bodies
|
||||
/// alike.
|
||||
fn non_stream_passthrough_response(
|
||||
status: StatusCode,
|
||||
body: bytes::Bytes,
|
||||
mut headers: HeaderMap,
|
||||
) -> Response {
|
||||
if let Ok(v) = HeaderValue::from_str(body.len().to_string().as_str()) {
|
||||
headers.insert(CONTENT_LENGTH, v);
|
||||
}
|
||||
let mut response = Response::new(Body::from(body));
|
||||
*response.status_mut() = status;
|
||||
*response.headers_mut() = headers;
|
||||
response
|
||||
}
|
||||
|
||||
// Helper to process prefill response and extract body if needed for logprobs
|
||||
async fn process_prefill_response(
|
||||
&self,
|
||||
@@ -2322,39 +2115,6 @@ mod tests {
|
||||
Box::new(worker)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn non_stream_passthrough_pins_content_length_to_body() {
|
||||
// 2026-10-17 IncompleteRead regression: the shadow strip shortens
|
||||
// the body on engine cache hits; the upstream Content-Length must
|
||||
// never survive onto the rewritten body.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("637"));
|
||||
let body = bytes::Bytes::from_static(b"{}");
|
||||
let resp =
|
||||
PDRouter::non_stream_passthrough_response(StatusCode::OK, body.clone(), headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2"),
|
||||
"stale upstream length is replaced by the actual body length"
|
||||
);
|
||||
|
||||
// Untouched body: the pinned length matches the original.
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(CONTENT_LENGTH, HeaderValue::from_static("2"));
|
||||
headers.insert(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||
let resp = PDRouter::non_stream_passthrough_response(StatusCode::OK, body, headers);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_LENGTH).unwrap(),
|
||||
&HeaderValue::from_static("2")
|
||||
);
|
||||
assert_eq!(
|
||||
resp.headers().get(CONTENT_TYPE).unwrap(),
|
||||
&HeaderValue::from_static("application/json"),
|
||||
"other preserved headers survive"
|
||||
);
|
||||
assert_eq!(resp.status(), StatusCode::OK);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_chat_request_text_uses_full_conversation() {
|
||||
// Regression test for https://github.com/sgl-project/sglang/issues/26263
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"v":1}
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"git": {
|
||||
"sha1": "7455eaee1da5212ae5e0fd33d80a903029d2f981"
|
||||
},
|
||||
"path_in_vcs": "protocols"
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
|
||||
#
|
||||
# When uploading crates to the registry Cargo will automatically
|
||||
# "normalize" Cargo.toml files for maximal compatibility
|
||||
# with all versions of Cargo and also rewrite `path` dependencies
|
||||
# to registry (e.g., crates.io) dependencies.
|
||||
#
|
||||
# If you are reading this file be aware that the original Cargo.toml
|
||||
# will likely look very different (and much more reasonable).
|
||||
# See Cargo.toml.orig for the original contents.
|
||||
|
||||
[package]
|
||||
edition = "2021"
|
||||
name = "openai-protocol"
|
||||
version = "1.0.0"
|
||||
authors = [
|
||||
"Simo Lin <linsimo.mark@gmail.com>",
|
||||
"Chang Su <mckvtl@gmail.com>",
|
||||
"Keyang Ru <rukeyang@gmail.com>",
|
||||
]
|
||||
build = false
|
||||
autolib = false
|
||||
autobins = false
|
||||
autoexamples = false
|
||||
autotests = false
|
||||
autobenches = false
|
||||
description = "OpenAI-compatible API protocol definitions and types"
|
||||
readme = false
|
||||
keywords = [
|
||||
"openai",
|
||||
"llm",
|
||||
"api",
|
||||
"protocol",
|
||||
"types",
|
||||
]
|
||||
categories = [
|
||||
"api-bindings",
|
||||
"data-structures",
|
||||
]
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/lightseekorg/smg"
|
||||
|
||||
[features]
|
||||
axum = ["dep:axum"]
|
||||
default = []
|
||||
|
||||
[lib]
|
||||
name = "openai_protocol"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[dependencies.axum]
|
||||
version = "0.8.6"
|
||||
optional = true
|
||||
|
||||
[dependencies.chrono]
|
||||
version = "0.4"
|
||||
|
||||
[dependencies.rand]
|
||||
version = "0.9.2"
|
||||
|
||||
[dependencies.serde]
|
||||
version = "1.0"
|
||||
features = ["derive"]
|
||||
|
||||
[dependencies.serde_json]
|
||||
version = "1.0"
|
||||
features = ["preserve_order"]
|
||||
|
||||
[dependencies.tokio]
|
||||
version = "1.42.0"
|
||||
|
||||
[dependencies.tracing]
|
||||
version = "0.1"
|
||||
|
||||
[dependencies.validator]
|
||||
version = "0.20.0"
|
||||
features = ["derive"]
|
||||
+29
@@ -0,0 +1,29 @@
|
||||
[package]
|
||||
name = "openai-protocol"
|
||||
version = "1.0.0"
|
||||
edition = "2021"
|
||||
description = "OpenAI-compatible API protocol definitions and types"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/lightseekorg/smg"
|
||||
authors = [
|
||||
"Simo Lin <linsimo.mark@gmail.com>",
|
||||
"Chang Su <mckvtl@gmail.com>",
|
||||
"Keyang Ru <rukeyang@gmail.com>",
|
||||
]
|
||||
keywords = ["openai", "llm", "api", "protocol", "types"]
|
||||
categories = ["api-bindings", "data-structures"]
|
||||
|
||||
[features]
|
||||
default = []
|
||||
axum = ["dep:axum"]
|
||||
|
||||
[dependencies]
|
||||
axum = { workspace = true, optional = true }
|
||||
chrono.workspace = true
|
||||
rand.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
tokio.workspace = true
|
||||
tracing.workspace = true
|
||||
# Only used by protocols
|
||||
validator = { version = "0.20.0", features = ["derive"] }
|
||||
@@ -0,0 +1,7 @@
|
||||
//! Builders for Chat Completion API response types
|
||||
|
||||
pub mod response;
|
||||
pub mod stream_response;
|
||||
|
||||
pub use response::ChatCompletionResponseBuilder;
|
||||
pub use stream_response::ChatCompletionStreamResponseBuilder;
|
||||
@@ -0,0 +1,218 @@
|
||||
//! Builder for ChatCompletionResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing chat completion responses.
|
||||
|
||||
use crate::{chat::*, common::Usage};
|
||||
|
||||
/// Builder for ChatCompletionResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing chat completion responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice
|
||||
pub fn add_choice(mut self, choice: ChatChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionResponse
|
||||
pub fn build(self) -> ChatCompletionResponse {
|
||||
ChatCompletionResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "chat.completion");
|
||||
assert!(response.choices.is_empty());
|
||||
assert!(response.usage.is_none());
|
||||
assert!(response.system_fingerprint.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let choice = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Hello!".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_456", "gpt-4")
|
||||
.choices(vec![choice.clone()])
|
||||
.maybe_usage(Some(usage))
|
||||
.maybe_system_fingerprint(Some("fp_123abc"))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.id, "chatcmpl_456");
|
||||
assert_eq!(response.choices.len(), 1);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert!(response.usage.is_some());
|
||||
assert_eq!(response.system_fingerprint.as_ref().unwrap(), "fp_123abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_multiple_choices() {
|
||||
let choice1 = ChatChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 1".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let choice2 = ChatChoice {
|
||||
index: 1,
|
||||
message: ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Option 2".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some("stop".to_string()),
|
||||
matched_stop: None,
|
||||
hidden_states: None,
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_789", "gpt-4")
|
||||
.add_choice(choice1)
|
||||
.add_choice(choice2)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.choices.len(), 2);
|
||||
assert_eq!(response.choices[0].index, 0);
|
||||
assert_eq!(response.choices[1].index, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ChatCompletionResponse::builder("chatcmpl_101", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
}
|
||||
+421
@@ -0,0 +1,421 @@
|
||||
//! Builder for ChatCompletionStreamResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing streaming chat completion responses.
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use crate::{
|
||||
chat::*,
|
||||
common::{FunctionCallDelta, ToolCallDelta, Usage},
|
||||
};
|
||||
|
||||
/// Builder for ChatCompletionStreamResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing streaming chat completion chunks with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ChatCompletionStreamResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created: u64,
|
||||
model: String,
|
||||
choices: Vec<ChatStreamChoice>,
|
||||
usage: Option<Usage>,
|
||||
system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Completion ID (e.g., "chatcmpl_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "chat.completion.chunk".to_string(),
|
||||
created: chrono::Utc::now().timestamp() as u64,
|
||||
model: model.into(),
|
||||
choices: Vec::new(),
|
||||
usage: None,
|
||||
system_fingerprint: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ChatCompletionRequest
|
||||
///
|
||||
/// This populates the model field from the request.
|
||||
pub fn copy_from_request(mut self, request: &ChatCompletionRequest) -> Self {
|
||||
self.model = request.model.clone();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "chat.completion.chunk")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created(mut self, timestamp: u64) -> Self {
|
||||
self.created = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the choices
|
||||
pub fn choices(mut self, choices: Vec<ChatStreamChoice>) -> Self {
|
||||
self.choices = choices;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single choice (delta)
|
||||
pub fn add_choice(mut self, choice: ChatStreamChoice) -> Self {
|
||||
self.choices.push(choice);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information (typically sent in final chunk)
|
||||
pub fn usage(mut self, usage: Usage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system fingerprint if provided (handles Option)
|
||||
pub fn maybe_system_fingerprint(mut self, fingerprint: Option<impl Into<String>>) -> Self {
|
||||
if let Some(fp) = fingerprint {
|
||||
self.system_fingerprint = Some(fp.into());
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<Usage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role` and `content`
|
||||
pub fn add_choice_content(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets `role`, `content`, and `logprobs`
|
||||
pub fn add_choice_content_with_logprobs(
|
||||
mut self,
|
||||
index: u32,
|
||||
role: impl Into<String>,
|
||||
content: impl Into<String>,
|
||||
logprobs: Option<crate::common::ChatLogProbs>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: Some(content.into()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that only sets `role`
|
||||
pub fn add_choice_role(mut self, index: u32, role: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some(role.into()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that appends a tool-call *arguments delta*
|
||||
/// Uses `Cow` so you can pass `&str` or `String` without extra clones
|
||||
pub fn add_choice_tool_args(
|
||||
mut self,
|
||||
index: u32,
|
||||
args_delta: impl Into<Cow<'static, str>>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: None,
|
||||
tool_type: None,
|
||||
function: Some(FunctionCallDelta {
|
||||
name: None,
|
||||
arguments: Some(args_delta.into().into_owned()),
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta that sets reasoning content (for models that stream reasoning)
|
||||
pub fn add_choice_reasoning(mut self, index: u32, reasoning: impl Into<String>) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: Some(reasoning.into()),
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta for tool call with function name and ID
|
||||
pub fn add_choice_tool_name(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_id: impl Into<String>,
|
||||
function_name: impl Into<String>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![ToolCallDelta {
|
||||
index: 0,
|
||||
id: Some(tool_call_id.into()),
|
||||
tool_type: Some("function".to_string()),
|
||||
function: Some(FunctionCallDelta {
|
||||
name: Some(function_name.into()),
|
||||
arguments: None,
|
||||
}),
|
||||
}]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice delta with a pre-constructed ToolCallDelta
|
||||
/// Useful when you already have a ToolCallDelta object to emit
|
||||
pub fn add_choice_tool_call_delta(
|
||||
mut self,
|
||||
index: u32,
|
||||
tool_call_delta: ToolCallDelta,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: None,
|
||||
tool_calls: Some(vec![tool_call_delta]),
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a choice with finish_reason (final chunk)
|
||||
/// This is used for the last chunk in a stream to signal completion
|
||||
pub fn add_choice_finish_reason(
|
||||
mut self,
|
||||
index: u32,
|
||||
finish_reason: impl Into<String>,
|
||||
matched_stop: Option<serde_json::Value>,
|
||||
) -> Self {
|
||||
self.choices.push(ChatStreamChoice {
|
||||
index,
|
||||
delta: ChatMessageDelta {
|
||||
role: None,
|
||||
content: None,
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: Some(finish_reason.into()),
|
||||
matched_stop,
|
||||
});
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ChatCompletionStreamResponse
|
||||
pub fn build(self) -> ChatCompletionStreamResponse {
|
||||
ChatCompletionStreamResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created: self.created,
|
||||
model: self.model,
|
||||
system_fingerprint: self.system_fingerprint,
|
||||
choices: self.choices,
|
||||
usage: self.usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(chunk.id, "chatcmpl_123");
|
||||
assert_eq!(chunk.model, "gpt-4");
|
||||
assert_eq!(chunk.object, "chat.completion.chunk");
|
||||
assert!(chunk.choices.is_empty());
|
||||
assert!(chunk.usage.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_content_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_456", "gpt-4")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].index, 0);
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].finish_reason.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_role_delta() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_789", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_finish_reason() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_101", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].finish_reason.as_ref().unwrap(), "stop");
|
||||
assert!(chunk.choices[0].delta.content.is_none());
|
||||
assert!(chunk.choices[0].delta.role.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_deltas() {
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_202", "gpt-4")
|
||||
.add_choice_role(0, "assistant")
|
||||
.add_choice_content(0, "assistant", "Hello")
|
||||
.add_choice_content(0, "assistant", " world")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 4); // role + 2 content + finish
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_usage() {
|
||||
let usage = Usage {
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
total_tokens: 30,
|
||||
completion_tokens_details: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_303", "gpt-4")
|
||||
.add_choice_finish_reason(0, "stop", None)
|
||||
.usage(usage)
|
||||
.build();
|
||||
|
||||
assert!(chunk.usage.is_some());
|
||||
assert_eq!(chunk.usage.as_ref().unwrap().total_tokens, 30);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ChatCompletionRequest {
|
||||
messages: vec![],
|
||||
model: "gpt-3.5-turbo".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_404", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.add_choice_content(0, "assistant", "test")
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.model, "gpt-3.5-turbo"); // Copied from request
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_choice_explicit() {
|
||||
let choice = ChatStreamChoice {
|
||||
index: 0,
|
||||
delta: ChatMessageDelta {
|
||||
role: Some("assistant".to_string()),
|
||||
content: Some("Hello".to_string()),
|
||||
tool_calls: None,
|
||||
reasoning_content: None,
|
||||
},
|
||||
logprobs: None,
|
||||
finish_reason: None,
|
||||
matched_stop: None,
|
||||
};
|
||||
|
||||
let chunk = ChatCompletionStreamResponseBuilder::new("chatcmpl_505", "gpt-4")
|
||||
.add_choice(choice)
|
||||
.build();
|
||||
|
||||
assert_eq!(chunk.choices.len(), 1);
|
||||
assert_eq!(chunk.choices[0].delta.role.as_ref().unwrap(), "assistant");
|
||||
assert_eq!(chunk.choices[0].delta.content.as_ref().unwrap(), "Hello");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
//! Builder patterns for protocol response types
|
||||
//!
|
||||
//! This module provides ergonomic builders for response types with many optional fields.
|
||||
//! Builders help avoid telescoping constructors and make construction intent clear.
|
||||
//!
|
||||
//! # Organization
|
||||
//!
|
||||
//! Builders are organized by API:
|
||||
//! - `chat/` - Chat Completion API builders (response, stream_response)
|
||||
//! - `responses/` - Responses API builder (response)
|
||||
//!
|
||||
//! # Optional Fields
|
||||
//!
|
||||
//! For optional fields, builders provide `maybe_*` methods that handle `Option<T>` directly:
|
||||
//! ```ignore
|
||||
//! builder
|
||||
//! .field(value)
|
||||
//! .maybe_optional_field(optional_value) // Accepts Option<T>
|
||||
//! .build()
|
||||
//! ```
|
||||
|
||||
pub mod chat;
|
||||
pub mod responses;
|
||||
|
||||
// Re-export all builders for convenient access
|
||||
pub use chat::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder};
|
||||
pub use responses::ResponsesResponseBuilder;
|
||||
@@ -0,0 +1,5 @@
|
||||
//! Builders for Responses API response types
|
||||
|
||||
pub mod response;
|
||||
|
||||
pub use response::ResponsesResponseBuilder;
|
||||
@@ -0,0 +1,411 @@
|
||||
//! Builder for ResponsesResponse
|
||||
//!
|
||||
//! Provides an ergonomic fluent API for constructing ResponsesResponse instances.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::responses::*;
|
||||
|
||||
/// Builder for ResponsesResponse
|
||||
///
|
||||
/// Provides a fluent interface for constructing responses with sensible defaults.
|
||||
#[must_use = "Builder does nothing until .build() is called"]
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ResponsesResponseBuilder {
|
||||
id: String,
|
||||
object: String,
|
||||
created_at: i64,
|
||||
status: ResponseStatus,
|
||||
error: Option<Value>,
|
||||
incomplete_details: Option<Value>,
|
||||
instructions: Option<String>,
|
||||
max_output_tokens: Option<u32>,
|
||||
model: String,
|
||||
output: Vec<ResponseOutputItem>,
|
||||
parallel_tool_calls: bool,
|
||||
previous_response_id: Option<String>,
|
||||
reasoning: Option<ReasoningInfo>,
|
||||
store: bool,
|
||||
temperature: Option<f32>,
|
||||
text: Option<TextConfig>,
|
||||
tool_choice: String,
|
||||
tools: Vec<ResponseTool>,
|
||||
top_p: Option<f32>,
|
||||
truncation: Option<String>,
|
||||
usage: Option<ResponsesUsage>,
|
||||
user: Option<String>,
|
||||
safety_identifier: Option<String>,
|
||||
metadata: HashMap<String, Value>,
|
||||
}
|
||||
|
||||
impl ResponsesResponseBuilder {
|
||||
/// Create a new builder with required fields
|
||||
///
|
||||
/// # Arguments
|
||||
/// - `id`: Response ID (e.g., "resp_abc123")
|
||||
/// - `model`: Model name used for generation
|
||||
pub fn new(id: impl Into<String>, model: impl Into<String>) -> Self {
|
||||
Self {
|
||||
id: id.into(),
|
||||
object: "response".to_string(),
|
||||
created_at: chrono::Utc::now().timestamp(),
|
||||
status: ResponseStatus::InProgress,
|
||||
error: None,
|
||||
incomplete_details: None,
|
||||
instructions: None,
|
||||
max_output_tokens: None,
|
||||
model: model.into(),
|
||||
output: Vec::new(),
|
||||
parallel_tool_calls: true,
|
||||
previous_response_id: None,
|
||||
reasoning: None,
|
||||
store: true,
|
||||
temperature: None,
|
||||
text: None,
|
||||
tool_choice: "auto".to_string(),
|
||||
tools: Vec::new(),
|
||||
top_p: None,
|
||||
truncation: None,
|
||||
usage: None,
|
||||
user: None,
|
||||
safety_identifier: None,
|
||||
metadata: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy common fields from a ResponsesRequest
|
||||
///
|
||||
/// This populates fields like instructions, max_output_tokens, temperature, etc.
|
||||
/// from the original request, making it easy to construct a response that mirrors
|
||||
/// the request parameters.
|
||||
///
|
||||
/// Note: `safety_identifier` is intentionally NOT copied as it is for content moderation
|
||||
/// and should be set independently from the request's `user` field (which is for billing/tracking).
|
||||
pub fn copy_from_request(mut self, request: &ResponsesRequest) -> Self {
|
||||
self.instructions = request.instructions.clone();
|
||||
self.max_output_tokens = request.max_output_tokens;
|
||||
self.parallel_tool_calls = request.parallel_tool_calls.unwrap_or(true);
|
||||
self.previous_response_id = request.previous_response_id.clone();
|
||||
self.store = request.store.unwrap_or(true);
|
||||
self.temperature = request.temperature;
|
||||
self.tool_choice = if let Some(ref tc) = request.tool_choice {
|
||||
serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string())
|
||||
} else {
|
||||
"auto".to_string()
|
||||
};
|
||||
self.tools = request.tools.clone().unwrap_or_default();
|
||||
self.top_p = request.top_p;
|
||||
self.user = request.user.clone();
|
||||
self.metadata = request.metadata.clone().unwrap_or_default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object type (default: "response")
|
||||
pub fn object(mut self, object: impl Into<String>) -> Self {
|
||||
self.object = object.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the creation timestamp (default: current time)
|
||||
pub fn created_at(mut self, timestamp: i64) -> Self {
|
||||
self.created_at = timestamp;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the response status
|
||||
pub fn status(mut self, status: ResponseStatus) -> Self {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set error information (if status is failed)
|
||||
pub fn error(mut self, error: Value) -> Self {
|
||||
self.error = Some(error);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set incomplete details (if response was truncated)
|
||||
pub fn incomplete_details(mut self, details: Value) -> Self {
|
||||
self.incomplete_details = Some(details);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set system instructions
|
||||
pub fn instructions(mut self, instructions: impl Into<String>) -> Self {
|
||||
self.instructions = Some(instructions.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set max output tokens
|
||||
pub fn max_output_tokens(mut self, tokens: u32) -> Self {
|
||||
self.max_output_tokens = Some(tokens);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set output items
|
||||
pub fn output(mut self, output: Vec<ResponseOutputItem>) -> Self {
|
||||
self.output = output;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single output item
|
||||
pub fn add_output(mut self, item: ResponseOutputItem) -> Self {
|
||||
self.output.push(item);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether parallel tool calls are enabled
|
||||
pub fn parallel_tool_calls(mut self, enabled: bool) -> Self {
|
||||
self.parallel_tool_calls = enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set previous response ID (if continuation)
|
||||
pub fn previous_response_id(mut self, id: impl Into<String>) -> Self {
|
||||
self.previous_response_id = Some(id.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set reasoning information
|
||||
pub fn reasoning(mut self, reasoning: ReasoningInfo) -> Self {
|
||||
self.reasoning = Some(reasoning);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether the response is stored
|
||||
pub fn store(mut self, store: bool) -> Self {
|
||||
self.store = store;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set temperature setting
|
||||
pub fn temperature(mut self, temperature: f32) -> Self {
|
||||
self.temperature = Some(temperature);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set text format settings if provided (handles Option)
|
||||
pub fn maybe_text(mut self, text: Option<TextConfig>) -> Self {
|
||||
if let Some(t) = text {
|
||||
self.text = Some(t);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set tool choice setting
|
||||
pub fn tool_choice(mut self, tool_choice: impl Into<String>) -> Self {
|
||||
self.tool_choice = tool_choice.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set available tools
|
||||
pub fn tools(mut self, tools: Vec<ResponseTool>) -> Self {
|
||||
self.tools = tools;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set top-p setting
|
||||
pub fn top_p(mut self, top_p: f32) -> Self {
|
||||
self.top_p = Some(top_p);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set truncation strategy
|
||||
pub fn truncation(mut self, truncation: impl Into<String>) -> Self {
|
||||
self.truncation = Some(truncation.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage information
|
||||
pub fn usage(mut self, usage: ResponsesUsage) -> Self {
|
||||
self.usage = Some(usage);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set usage if provided (handles Option)
|
||||
pub fn maybe_usage(mut self, usage: Option<ResponsesUsage>) -> Self {
|
||||
if let Some(u) = usage {
|
||||
self.usage = Some(u);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Copy from request if provided (handles Option)
|
||||
pub fn maybe_copy_from_request(mut self, request: Option<&ResponsesRequest>) -> Self {
|
||||
if let Some(req) = request {
|
||||
self = self.copy_from_request(req);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Set user identifier
|
||||
pub fn user(mut self, user: impl Into<String>) -> Self {
|
||||
self.user = Some(user.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set safety identifier
|
||||
pub fn safety_identifier(mut self, identifier: impl Into<String>) -> Self {
|
||||
self.safety_identifier = Some(identifier.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set metadata
|
||||
pub fn metadata(mut self, metadata: HashMap<String, Value>) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a single metadata entry
|
||||
pub fn add_metadata(mut self, key: impl Into<String>, value: Value) -> Self {
|
||||
self.metadata.insert(key.into(), value);
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the ResponsesResponse
|
||||
pub fn build(self) -> ResponsesResponse {
|
||||
ResponsesResponse {
|
||||
id: self.id,
|
||||
object: self.object,
|
||||
created_at: self.created_at,
|
||||
status: self.status,
|
||||
error: self.error,
|
||||
incomplete_details: self.incomplete_details,
|
||||
instructions: self.instructions,
|
||||
max_output_tokens: self.max_output_tokens,
|
||||
model: self.model,
|
||||
output: self.output,
|
||||
parallel_tool_calls: self.parallel_tool_calls,
|
||||
previous_response_id: self.previous_response_id,
|
||||
reasoning: self.reasoning,
|
||||
store: self.store,
|
||||
temperature: self.temperature,
|
||||
text: self.text,
|
||||
tool_choice: self.tool_choice,
|
||||
tools: self.tools,
|
||||
top_p: self.top_p,
|
||||
truncation: self.truncation,
|
||||
usage: self.usage,
|
||||
user: self.user,
|
||||
safety_identifier: self.safety_identifier,
|
||||
metadata: self.metadata,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tests
|
||||
// ============================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_build_minimal() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4").build();
|
||||
|
||||
assert_eq!(response.id, "resp_123");
|
||||
assert_eq!(response.model, "gpt-4");
|
||||
assert_eq!(response.object, "response");
|
||||
assert_eq!(response.status, ResponseStatus::InProgress);
|
||||
assert!(response.output.is_empty());
|
||||
assert!(response.parallel_tool_calls);
|
||||
assert!(response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_complete() {
|
||||
let response = ResponsesResponse::builder("resp_123", "gpt-4")
|
||||
.status(ResponseStatus::Completed)
|
||||
.instructions("You are a helpful assistant")
|
||||
.max_output_tokens(1000)
|
||||
.temperature(0.7)
|
||||
.top_p(0.9)
|
||||
.parallel_tool_calls(false)
|
||||
.store(false)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.status, ResponseStatus::Completed);
|
||||
assert_eq!(
|
||||
response.instructions.as_ref().unwrap(),
|
||||
"You are a helpful assistant"
|
||||
);
|
||||
assert_eq!(response.max_output_tokens, Some(1000));
|
||||
assert_eq!(response.temperature, Some(0.7));
|
||||
assert_eq!(response.top_p, Some(0.9));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_copy_from_request() {
|
||||
let request = ResponsesRequest {
|
||||
model: "gpt-4".to_string(),
|
||||
input: ResponseInput::Text("test".to_string()),
|
||||
instructions: Some("Be helpful".to_string()),
|
||||
max_output_tokens: Some(500),
|
||||
temperature: Some(0.8),
|
||||
top_p: Some(0.95),
|
||||
parallel_tool_calls: Some(false),
|
||||
store: Some(false),
|
||||
user: Some("user_123".to_string()),
|
||||
metadata: Some(HashMap::from([(
|
||||
"key".to_string(),
|
||||
serde_json::json!("value"),
|
||||
)])),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let response = ResponsesResponse::builder("resp_456", "gpt-4")
|
||||
.copy_from_request(&request)
|
||||
.status(ResponseStatus::Completed)
|
||||
.build();
|
||||
|
||||
assert_eq!(response.instructions.as_ref().unwrap(), "Be helpful");
|
||||
assert_eq!(response.max_output_tokens, Some(500));
|
||||
assert_eq!(response.temperature, Some(0.8));
|
||||
assert_eq!(response.top_p, Some(0.95));
|
||||
assert!(!response.parallel_tool_calls);
|
||||
assert!(!response.store);
|
||||
assert_eq!(response.user.as_ref().unwrap(), "user_123");
|
||||
assert_eq!(
|
||||
response.metadata.get("key").unwrap(),
|
||||
&serde_json::json!("value")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_output_items() {
|
||||
let response = ResponsesResponse::builder("resp_789", "gpt-4")
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_1".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.add_output(ResponseOutputItem::Message {
|
||||
id: "msg_2".to_string(),
|
||||
role: "assistant".to_string(),
|
||||
content: vec![],
|
||||
status: "completed".to_string(),
|
||||
})
|
||||
.build();
|
||||
|
||||
assert_eq!(response.output.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_metadata() {
|
||||
let response = ResponsesResponse::builder("resp_101", "gpt-4")
|
||||
.add_metadata("key1", serde_json::json!("value1"))
|
||||
.add_metadata("key2", serde_json::json!(42))
|
||||
.build();
|
||||
|
||||
assert_eq!(response.metadata.len(), 2);
|
||||
assert_eq!(response.metadata.get("key1").unwrap(), "value1");
|
||||
assert_eq!(response.metadata.get("key2").unwrap(), 42);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,799 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{
|
||||
default_model, default_true, validate_stop, ChatLogProbs, ContentPart, Function,
|
||||
FunctionCall, FunctionChoice, GenerationRequest, ResponseFormat, StreamOptions,
|
||||
StringOrArray, Tool, ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference,
|
||||
Usage,
|
||||
},
|
||||
sampling_params::{validate_top_k_value, validate_top_p_value},
|
||||
};
|
||||
use crate::{
|
||||
builders::{ChatCompletionResponseBuilder, ChatCompletionStreamResponseBuilder},
|
||||
validated::Normalizable,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// Chat Messages
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "role")]
|
||||
pub enum ChatMessage {
|
||||
#[serde(rename = "system")]
|
||||
System {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "user")]
|
||||
User {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
#[serde(rename = "assistant")]
|
||||
Assistant {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<MessageContent>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tool_calls: Option<Vec<ToolCall>>,
|
||||
/// Reasoning content for O1-style models (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
reasoning_content: Option<String>,
|
||||
},
|
||||
#[serde(rename = "tool")]
|
||||
Tool {
|
||||
content: MessageContent,
|
||||
tool_call_id: String,
|
||||
},
|
||||
#[serde(rename = "function")]
|
||||
Function { content: String, name: String },
|
||||
#[serde(rename = "developer")]
|
||||
Developer {
|
||||
content: MessageContent,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
tools: Option<Vec<Tool>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(untagged)]
|
||||
pub enum MessageContent {
|
||||
Text(String),
|
||||
Parts(Vec<ContentPart>),
|
||||
}
|
||||
|
||||
impl MessageContent {
|
||||
/// Returns the text content, cloning only when necessary.
|
||||
/// For simple text, returns a clone of the string.
|
||||
/// For parts, concatenates text parts with spaces.
|
||||
/// Optimized to avoid intermediate Vec allocation.
|
||||
pub fn to_simple_string(&self) -> String {
|
||||
match self {
|
||||
MessageContent::Text(text) => text.clone(),
|
||||
MessageContent::Parts(parts) => {
|
||||
// Use fold to build string directly without intermediate Vec allocation
|
||||
let mut result = String::new();
|
||||
let mut first = true;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !first {
|
||||
result.push(' ');
|
||||
}
|
||||
result.push_str(text);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Appends text content directly to a buffer, avoiding intermediate allocations.
|
||||
/// Returns true if any content was appended.
|
||||
#[inline]
|
||||
pub fn append_text_to(&self, buffer: &mut String) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => {
|
||||
if !text.is_empty() {
|
||||
buffer.push_str(text);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
MessageContent::Parts(parts) => {
|
||||
let mut appended = false;
|
||||
for part in parts {
|
||||
if let ContentPart::Text { text } = part {
|
||||
if !text.is_empty() {
|
||||
if appended {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(text);
|
||||
appended = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
appended
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this content contains any non-empty text.
|
||||
#[inline]
|
||||
pub fn has_text(&self) -> bool {
|
||||
match self {
|
||||
MessageContent::Text(text) => !text.is_empty(),
|
||||
MessageContent::Parts(parts) => parts
|
||||
.iter()
|
||||
.any(|part| matches!(part, ContentPart::Text { text } if !text.is_empty())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Chat Completion Request
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_chat_cross_parameters"))]
|
||||
pub struct ChatCompletionRequest {
|
||||
/// A list of messages comprising the conversation so far
|
||||
#[validate(custom(function = "validate_messages"))]
|
||||
pub messages: Vec<ChatMessage>,
|
||||
|
||||
/// ID of the model to use
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Deprecated: Replaced by tool_choice
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tool_choice instead")]
|
||||
pub function_call: Option<FunctionCall>,
|
||||
|
||||
/// Deprecated: Replaced by tools
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use tools instead")]
|
||||
pub functions: Option<Vec<Function>>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// Whether to return log probabilities of the output tokens
|
||||
#[serde(default)]
|
||||
pub logprobs: bool,
|
||||
|
||||
/// Deprecated: Replaced by max_completion_tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "Use max_completion_tokens instead")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// An upper bound for the number of tokens that can be generated for a completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub max_completion_tokens: Option<u32>,
|
||||
|
||||
/// Developer-defined tags and values used for filtering completions in the dashboard
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<HashMap<String, String>>,
|
||||
|
||||
/// Output types that you would like the model to generate for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// How many chat completion choices to generate for each input message
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1, max = 10))]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to enable parallel function calling during tool use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub parallel_tool_calls: Option<bool>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Cache key for prompts (beta feature)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_cache_key: Option<String>,
|
||||
|
||||
/// Effort level for reasoning models (low, medium, high)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_effort: Option<String>,
|
||||
|
||||
/// An object specifying the format that the model must output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub response_format: Option<ResponseFormat>,
|
||||
|
||||
/// Safety identifier for content moderation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub safety_identifier: Option<String>,
|
||||
|
||||
/// Deprecated: This feature is in Legacy mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[deprecated(note = "This feature is in Legacy mode")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
/// The service tier to use for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub service_tier: Option<String>,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_stop"))]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// If set, partial message deltas will be sent
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// Controls which (if any) tool is called by the model
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_choice: Option<ToolChoice>,
|
||||
|
||||
/// A list of tools the model may call
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tools: Option<Vec<Tool>>,
|
||||
|
||||
/// An integer between 0 and 20 specifying the number of most likely tokens to return
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0, max = 20))]
|
||||
pub top_logprobs: Option<u32>,
|
||||
|
||||
/// An alternative to sampling with temperature
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// Verbosity level for debugging
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub verbosity: Option<i32>,
|
||||
|
||||
// =============================================================================
|
||||
// Engine-Specific Sampling Parameters
|
||||
// =============================================================================
|
||||
// These parameters are extensions beyond the OpenAI API specification and
|
||||
// control model generation behavior in engine-specific ways.
|
||||
// =============================================================================
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Continue generating from final assistant message
|
||||
#[serde(default)]
|
||||
pub continue_final_message: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Separate reasoning content from final answer (O1-style models)
|
||||
#[serde(default = "default_true")]
|
||||
pub separate_reasoning: bool,
|
||||
|
||||
/// Stream reasoning tokens during generation
|
||||
#[serde(default = "default_true")]
|
||||
pub stream_reasoning: bool,
|
||||
|
||||
/// Chat template kwargs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_kwargs: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Random seed for sampling for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates messages array is not empty and has valid content
|
||||
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
|
||||
if messages.is_empty() {
|
||||
return Err(validator::ValidationError::new("messages cannot be empty"));
|
||||
}
|
||||
|
||||
for msg in messages.iter() {
|
||||
if let ChatMessage::User { content, .. } = msg {
|
||||
match content {
|
||||
MessageContent::Text(text) if text.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content cannot be empty",
|
||||
));
|
||||
}
|
||||
MessageContent::Parts(parts) if parts.is_empty() => {
|
||||
return Err(validator::ValidationError::new(
|
||||
"message content parts cannot be empty",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_chat_cross_parameters(
|
||||
req: &ChatCompletionRequest,
|
||||
) -> Result<(), validator::ValidationError> {
|
||||
// 1. Validate logprobs dependency
|
||||
if req.top_logprobs.is_some() && !req.logprobs {
|
||||
let mut e = validator::ValidationError::new("top_logprobs_requires_logprobs");
|
||||
e.message = Some("top_logprobs is only allowed when logprobs is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 2. Validate stream_options dependency
|
||||
if req.stream_options.is_some() && !req.stream {
|
||||
let mut e = validator::ValidationError::new("stream_options_requires_stream");
|
||||
e.message =
|
||||
Some("The 'stream_options' parameter is only allowed when 'stream' is enabled".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 3. Validate token limits - min <= max
|
||||
if let (Some(min), Some(max)) = (req.min_tokens, req.max_completion_tokens) {
|
||||
if min > max {
|
||||
let mut e = validator::ValidationError::new("min_tokens_exceeds_max");
|
||||
e.message = Some("min_tokens cannot exceed max_tokens/max_completion_tokens".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Validate structured output conflicts
|
||||
let has_json_format = matches!(
|
||||
req.response_format,
|
||||
Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. })
|
||||
);
|
||||
|
||||
if has_json_format && req.regex.is_some() {
|
||||
let mut e = validator::ValidationError::new("regex_conflicts_with_json");
|
||||
e.message = Some("cannot use regex constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if has_json_format && req.ebnf.is_some() {
|
||||
let mut e = validator::ValidationError::new("ebnf_conflicts_with_json");
|
||||
e.message = Some("cannot use EBNF constraint with JSON response format".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 5. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
req.regex.is_some(),
|
||||
req.ebnf.is_some(),
|
||||
matches!(req.response_format, Some(ResponseFormat::JsonSchema { .. })),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
let mut e = validator::ValidationError::new("multiple_constraints");
|
||||
e.message = Some("only one structured output constraint (regex, ebnf, or json_schema) can be active at a time".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// 6. Validate response format JSON schema name
|
||||
if let Some(ResponseFormat::JsonSchema { json_schema }) = &req.response_format {
|
||||
if json_schema.name.is_empty() {
|
||||
let mut e = validator::ValidationError::new("json_schema_name_empty");
|
||||
e.message = Some("JSON schema name cannot be empty".into());
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Validate tool_choice requires tools (except for "none")
|
||||
if let Some(ref tool_choice) = req.tool_choice {
|
||||
let has_tools = req.tools.as_ref().is_some_and(|t| !t.is_empty());
|
||||
|
||||
// Check if tool_choice is anything other than "none"
|
||||
let is_some_choice = !matches!(tool_choice, ToolChoice::Value(ToolChoiceValue::None));
|
||||
|
||||
if is_some_choice && !has_tools {
|
||||
let mut e = validator::ValidationError::new("tool_choice_requires_tools");
|
||||
e.message = Some("Invalid value for 'tool_choice': 'tool_choice' is only allowed when 'tools' are specified.".into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Additional validation when tools are present
|
||||
if has_tools {
|
||||
let tools = req.tools.as_ref().unwrap();
|
||||
|
||||
match tool_choice {
|
||||
ToolChoice::Function { function, .. } => {
|
||||
// Validate that the specified function name exists in tools
|
||||
let function_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == function.name
|
||||
});
|
||||
|
||||
if !function_exists {
|
||||
let mut e =
|
||||
validator::ValidationError::new("tool_choice_function_not_found");
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice': function '{}' not found in 'tools'.",
|
||||
function.name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
ToolChoice::AllowedTools {
|
||||
mode,
|
||||
tools: allowed_tools,
|
||||
..
|
||||
} => {
|
||||
// Validate mode is "auto" or "required"
|
||||
if mode != "auto" && mode != "required" {
|
||||
let mut e = validator::ValidationError::new("tool_choice_invalid_mode");
|
||||
e.message = Some(format!(
|
||||
"Invalid value for 'tool_choice.mode': must be 'auto' or 'required', got '{}'.",
|
||||
mode
|
||||
).into());
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
// Validate that all ToolReferences are Function type (Chat API only supports function tools)
|
||||
for tool_ref in allowed_tools {
|
||||
match tool_ref {
|
||||
ToolReference::Function { name } => {
|
||||
// Validate that the function exists in tools array
|
||||
let tool_exists = tools.iter().any(|tool| {
|
||||
tool.tool_type == "function" && tool.function.name == *name
|
||||
});
|
||||
|
||||
if !tool_exists {
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_tool_not_found",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': tool '{}' not found in 'tools'.",
|
||||
name
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
// Chat Completion API only supports function tools in tool_choice
|
||||
let mut e = validator::ValidationError::new(
|
||||
"tool_choice_invalid_tool_type",
|
||||
);
|
||||
e.message = Some(
|
||||
format!(
|
||||
"Invalid value for 'tool_choice.tools': Chat Completion API only supports function tools, got '{}'.",
|
||||
tool_ref.identifier()
|
||||
)
|
||||
.into(),
|
||||
);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Normalizable Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl Normalizable for ChatCompletionRequest {
|
||||
/// Normalize the request by applying migrations and defaults:
|
||||
/// 1. Migrate deprecated fields to their replacements
|
||||
/// 2. Clear deprecated fields and log warnings
|
||||
/// 3. Apply OpenAI defaults for tool_choice
|
||||
fn normalize(&mut self) {
|
||||
// Migrate deprecated max_tokens → max_completion_tokens
|
||||
#[allow(deprecated)]
|
||||
if self.max_completion_tokens.is_none() && self.max_tokens.is_some() {
|
||||
self.max_completion_tokens = self.max_tokens;
|
||||
self.max_tokens = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated functions → tools
|
||||
#[allow(deprecated)]
|
||||
if self.tools.is_none() && self.functions.is_some() {
|
||||
tracing::warn!("functions is deprecated, use tools instead");
|
||||
self.tools = self.functions.as_ref().map(|functions| {
|
||||
functions
|
||||
.iter()
|
||||
.map(|func| Tool {
|
||||
tool_type: "function".to_string(),
|
||||
function: func.clone(),
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
self.functions = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Migrate deprecated function_call → tool_choice
|
||||
#[allow(deprecated)]
|
||||
if self.tool_choice.is_none() && self.function_call.is_some() {
|
||||
tracing::warn!("function_call is deprecated, use tool_choice instead");
|
||||
self.tool_choice = self.function_call.as_ref().map(|fc| match fc {
|
||||
FunctionCall::None => ToolChoice::Value(ToolChoiceValue::None),
|
||||
FunctionCall::Auto => ToolChoice::Value(ToolChoiceValue::Auto),
|
||||
FunctionCall::Function { name } => ToolChoice::Function {
|
||||
tool_type: "function".to_string(),
|
||||
function: FunctionChoice { name: name.clone() },
|
||||
},
|
||||
});
|
||||
self.function_call = None; // Clear deprecated field
|
||||
}
|
||||
|
||||
// Apply tool_choice defaults
|
||||
if self.tool_choice.is_none() {
|
||||
if let Some(tools) = &self.tools {
|
||||
let choice_value = if !tools.is_empty() {
|
||||
ToolChoiceValue::Auto
|
||||
} else {
|
||||
ToolChoiceValue::None
|
||||
};
|
||||
self.tool_choice = Some(ToolChoice::Value(choice_value));
|
||||
}
|
||||
// If tools is None, leave tool_choice as None (don't set it)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait Implementation
|
||||
// ============================================================================
|
||||
|
||||
impl GenerationRequest for ChatCompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Extract text from messages for routing decisions
|
||||
// Use a single buffer to avoid intermediate Vec<String> allocations
|
||||
let mut buffer = String::new();
|
||||
let mut has_content = false;
|
||||
|
||||
for msg in &self.messages {
|
||||
match msg {
|
||||
ChatMessage::System { content, .. }
|
||||
| ChatMessage::User { content, .. }
|
||||
| ChatMessage::Tool { content, .. }
|
||||
| ChatMessage::Developer { content, .. } => {
|
||||
if has_content && content.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if content.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
ChatMessage::Assistant {
|
||||
content,
|
||||
reasoning_content,
|
||||
..
|
||||
} => {
|
||||
// Append main content
|
||||
if let Some(c) = content {
|
||||
if has_content && c.has_text() {
|
||||
buffer.push(' ');
|
||||
}
|
||||
if c.append_text_to(&mut buffer) {
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
// Append reasoning content
|
||||
if let Some(reasoning) = reasoning_content {
|
||||
if !reasoning.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(reasoning);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
ChatMessage::Function { content, .. } => {
|
||||
if !content.is_empty() {
|
||||
if has_content {
|
||||
buffer.push(' ');
|
||||
}
|
||||
buffer.push_str(content);
|
||||
has_content = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
buffer
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<ChatChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
impl ChatCompletionResponse {
|
||||
/// Create a new builder for ChatCompletionResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionResponseBuilder {
|
||||
ChatCompletionResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Response message structure for ChatCompletionResponse (different from request ChatMessage)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionMessage {
|
||||
pub role: String, // Always "assistant" for responses
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCall>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
// Note: function_call is deprecated and not included
|
||||
// Note: refusal, annotations, audio are not added yet
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatChoice {
|
||||
pub index: u32,
|
||||
pub message: ChatCompletionMessage,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "tool_calls", "content_filter", "function_call"
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
/// Hidden states from the model (SGLang extension)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub hidden_states: Option<Vec<f32>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatCompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "chat.completion.chunk"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
pub choices: Vec<ChatStreamChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
}
|
||||
|
||||
impl ChatCompletionStreamResponse {
|
||||
/// Create a new builder for ChatCompletionStreamResponse
|
||||
pub fn builder(
|
||||
id: impl Into<String>,
|
||||
model: impl Into<String>,
|
||||
) -> ChatCompletionStreamResponseBuilder {
|
||||
ChatCompletionStreamResponseBuilder::new(id, model)
|
||||
}
|
||||
}
|
||||
|
||||
/// Delta structure for streaming chat completion responses
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatMessageDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub content: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_calls: Option<Vec<ToolCallDelta>>,
|
||||
pub reasoning_content: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatStreamChoice {
|
||||
pub index: u32,
|
||||
pub delta: ChatMessageDelta,
|
||||
pub logprobs: Option<ChatLogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
//! Classify API protocol definitions.
|
||||
//!
|
||||
//! This module defines the request and response types for the `/v1/classify` API,
|
||||
//! which is compatible with vLLM's classification endpoint.
|
||||
//!
|
||||
//! Classification reuses the embedding backend - the scheduler returns logits as
|
||||
//! "embeddings", and the classify layer applies softmax + label mapping.
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Classify API
|
||||
// ============================================================================
|
||||
|
||||
/// Classification request - compatible with vLLM's /v1/classify API
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ClassifyRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, or token IDs
|
||||
/// - Single string: "text to classify"
|
||||
/// - Array of strings: ["text1", "text2"]
|
||||
/// - Token IDs: [1, 2, 3] (advanced usage)
|
||||
pub input: Value,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: request priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for ClassifyRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Classification is always non-streaming
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Classify Response
|
||||
// ============================================================================
|
||||
|
||||
/// Single classification result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyData {
|
||||
/// Index of this result (for batch requests)
|
||||
pub index: u32,
|
||||
/// Predicted class label (from id2label mapping)
|
||||
pub label: String,
|
||||
/// Probability distribution over all classes (softmax of logits)
|
||||
pub probs: Vec<f32>,
|
||||
/// Number of classes
|
||||
pub num_classes: u32,
|
||||
}
|
||||
|
||||
/// Classification response
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ClassifyResponse {
|
||||
/// Unique request ID (format: "classify-{uuid}")
|
||||
pub id: String,
|
||||
/// Always "list"
|
||||
pub object: String,
|
||||
/// Unix timestamp (seconds since epoch)
|
||||
pub created: u64,
|
||||
/// Model name
|
||||
pub model: String,
|
||||
/// Classification results (one per input in batch)
|
||||
pub data: Vec<ClassifyData>,
|
||||
/// Token usage info
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
|
||||
impl ClassifyResponse {
|
||||
/// Create a new ClassifyResponse with the given data
|
||||
pub fn new(
|
||||
id: String,
|
||||
model: String,
|
||||
created: u64,
|
||||
data: Vec<ClassifyData>,
|
||||
usage: UsageInfo,
|
||||
) -> Self {
|
||||
Self {
|
||||
id,
|
||||
object: "list".to_string(),
|
||||
created,
|
||||
model,
|
||||
data,
|
||||
usage,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,524 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator;
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Default value helpers
|
||||
// ============================================================================
|
||||
|
||||
/// Default model value when not specified
|
||||
pub(crate) fn default_model() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
/// Helper function for serde default value (returns true)
|
||||
pub fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// GenerationRequest Trait
|
||||
// ============================================================================
|
||||
|
||||
/// Trait for unified access to generation request properties
|
||||
/// Implemented by ChatCompletionRequest, CompletionRequest, GenerateRequest,
|
||||
/// EmbeddingRequest, RerankRequest, and ResponsesRequest
|
||||
pub trait GenerationRequest: Send + Sync {
|
||||
/// Check if the request is for streaming
|
||||
fn is_stream(&self) -> bool;
|
||||
|
||||
/// Get the model name if specified
|
||||
fn get_model(&self) -> Option<&str>;
|
||||
|
||||
/// Extract text content for routing decisions
|
||||
fn extract_text_for_routing(&self) -> String;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// String/Array Utilities
|
||||
// ============================================================================
|
||||
|
||||
/// A type that can be either a single string or an array of strings
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
String(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Get the number of items in the StringOrArray
|
||||
pub fn len(&self) -> usize {
|
||||
match self {
|
||||
StringOrArray::String(_) => 1,
|
||||
StringOrArray::Array(arr) => arr.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the StringOrArray is empty
|
||||
pub fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
StringOrArray::String(s) => s.is_empty(),
|
||||
StringOrArray::Array(arr) => arr.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert to a vector of strings (clones the data)
|
||||
pub fn to_vec(&self) -> Vec<String> {
|
||||
match self {
|
||||
StringOrArray::String(s) => vec![s.clone()],
|
||||
StringOrArray::Array(arr) => arr.clone(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over string references without cloning.
|
||||
/// Use this instead of `to_vec()` when you only need to iterate.
|
||||
pub fn iter(&self) -> StringOrArrayIter<'_> {
|
||||
StringOrArrayIter {
|
||||
inner: self,
|
||||
index: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the first string, or None if empty
|
||||
pub fn first(&self) -> Option<&str> {
|
||||
match self {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(s)
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => arr.first().map(|s| s.as_str()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Iterator over StringOrArray that yields string references without cloning
|
||||
pub struct StringOrArrayIter<'a> {
|
||||
inner: &'a StringOrArray,
|
||||
index: usize,
|
||||
}
|
||||
|
||||
impl<'a> Iterator for StringOrArrayIter<'a> {
|
||||
type Item = &'a str;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
match self.inner {
|
||||
StringOrArray::String(s) => {
|
||||
if self.index == 0 {
|
||||
self.index = 1;
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if self.index < arr.len() {
|
||||
let item = &arr[self.index];
|
||||
self.index += 1;
|
||||
Some(item.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
let remaining = match self.inner {
|
||||
StringOrArray::String(_) => 1 - self.index,
|
||||
StringOrArray::Array(arr) => arr.len() - self.index,
|
||||
};
|
||||
(remaining, Some(remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> ExactSizeIterator for StringOrArrayIter<'a> {}
|
||||
|
||||
/// Validates stop sequences (max 4, non-empty strings)
|
||||
/// Used by both ChatCompletionRequest and ResponsesRequest
|
||||
pub fn validate_stop(stop: &StringOrArray) -> Result<(), validator::ValidationError> {
|
||||
match stop {
|
||||
StringOrArray::String(s) => {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
StringOrArray::Array(arr) => {
|
||||
if arr.len() > 4 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"maximum 4 stop sequences allowed",
|
||||
));
|
||||
}
|
||||
for s in arr {
|
||||
if s.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"stop sequences cannot be empty",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Content Parts (for multimodal messages)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ContentPart {
|
||||
#[serde(rename = "text")]
|
||||
Text { text: String },
|
||||
#[serde(rename = "image_url")]
|
||||
ImageUrl { image_url: ImageUrl },
|
||||
#[serde(rename = "video_url")]
|
||||
VideoUrl { video_url: VideoUrl },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct ImageUrl {
|
||||
pub url: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>, // "auto", "low", or "high"
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)]
|
||||
pub struct VideoUrl {
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Format (for structured outputs)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum ResponseFormat {
|
||||
#[serde(rename = "text")]
|
||||
Text,
|
||||
#[serde(rename = "json_object")]
|
||||
JsonObject,
|
||||
#[serde(rename = "json_schema")]
|
||||
JsonSchema { json_schema: JsonSchemaFormat },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct JsonSchemaFormat {
|
||||
pub name: String,
|
||||
pub schema: Value,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Streaming
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct StreamOptions {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub include_usage: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCallDelta {
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub id: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub function: Option<FunctionCallDelta>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallDelta {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub arguments: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tools and Function Calling
|
||||
// ============================================================================
|
||||
|
||||
/// Tool choice value for simple string options
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolChoiceValue {
|
||||
Auto,
|
||||
Required,
|
||||
None,
|
||||
}
|
||||
|
||||
/// Tool choice for both Chat Completion and Responses APIs
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ToolChoice {
|
||||
Value(ToolChoiceValue),
|
||||
Function {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "function"
|
||||
function: FunctionChoice,
|
||||
},
|
||||
AllowedTools {
|
||||
#[serde(rename = "type")]
|
||||
tool_type: String, // "allowed_tools"
|
||||
mode: String, // "auto" | "required" TODO: need validation
|
||||
tools: Vec<ToolReference>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Default for ToolChoice {
|
||||
fn default() -> Self {
|
||||
Self::Value(ToolChoiceValue::Auto)
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolChoice {
|
||||
/// Serialize tool_choice to string for ResponsesResponse
|
||||
///
|
||||
/// Returns the JSON-serialized tool_choice or "auto" as default
|
||||
pub fn serialize_to_string(tool_choice: &Option<ToolChoice>) -> String {
|
||||
tool_choice
|
||||
.as_ref()
|
||||
.map(|tc| serde_json::to_string(tc).unwrap_or_else(|_| "auto".to_string()))
|
||||
.unwrap_or_else(|| "auto".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function choice specification for ToolChoice::Function
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionChoice {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Tool reference for ToolChoice::AllowedTools
|
||||
///
|
||||
/// Represents a reference to a specific tool in the allowed_tools array.
|
||||
/// Different tool types have different required fields.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ToolReference {
|
||||
/// Reference to a function tool
|
||||
#[serde(rename = "function")]
|
||||
Function { name: String },
|
||||
|
||||
/// Reference to an MCP tool
|
||||
#[serde(rename = "mcp")]
|
||||
Mcp {
|
||||
server_label: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
name: Option<String>,
|
||||
},
|
||||
|
||||
/// File search hosted tool
|
||||
#[serde(rename = "file_search")]
|
||||
FileSearch,
|
||||
|
||||
/// Web search preview hosted tool
|
||||
#[serde(rename = "web_search_preview")]
|
||||
WebSearchPreview,
|
||||
|
||||
/// Computer use preview hosted tool
|
||||
#[serde(rename = "computer_use_preview")]
|
||||
ComputerUsePreview,
|
||||
|
||||
/// Code interpreter hosted tool
|
||||
#[serde(rename = "code_interpreter")]
|
||||
CodeInterpreter,
|
||||
|
||||
/// Image generation hosted tool
|
||||
#[serde(rename = "image_generation")]
|
||||
ImageGeneration,
|
||||
}
|
||||
|
||||
impl ToolReference {
|
||||
/// Get a unique identifier for this tool reference
|
||||
pub fn identifier(&self) -> String {
|
||||
match self {
|
||||
ToolReference::Function { name } => format!("function:{}", name),
|
||||
ToolReference::Mcp { server_label, name } => {
|
||||
if let Some(n) = name {
|
||||
format!("mcp:{}:{}", server_label, n)
|
||||
} else {
|
||||
format!("mcp:{}", server_label)
|
||||
}
|
||||
}
|
||||
ToolReference::FileSearch => "file_search".to_string(),
|
||||
ToolReference::WebSearchPreview => "web_search_preview".to_string(),
|
||||
ToolReference::ComputerUsePreview => "computer_use_preview".to_string(),
|
||||
ToolReference::CodeInterpreter => "code_interpreter".to_string(),
|
||||
ToolReference::ImageGeneration => "image_generation".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the tool name if this is a function tool
|
||||
pub fn function_name(&self) -> Option<&str> {
|
||||
match self {
|
||||
ToolReference::Function { name } => Some(name.as_str()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Tool {
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: Function,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Function {
|
||||
pub name: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub description: Option<String>,
|
||||
pub parameters: Value, // JSON Schema
|
||||
/// Whether to enable strict schema adherence (OpenAI structured outputs)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub strict: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ToolCall {
|
||||
pub id: String,
|
||||
#[serde(rename = "type")]
|
||||
pub tool_type: String, // "function"
|
||||
pub function: FunctionCallResponse,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum FunctionCall {
|
||||
None,
|
||||
Auto,
|
||||
Function { name: String },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FunctionCallResponse {
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub arguments: Option<String>, // JSON string
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Usage and Logging
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct Usage {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub completion_tokens_details: Option<CompletionTokensDetails>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionTokensDetails {
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
/// Usage information (used by rerank and other endpoints)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct UsageInfo {
|
||||
pub prompt_tokens: u32,
|
||||
pub completion_tokens: u32,
|
||||
pub total_tokens: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub prompt_tokens_details: Option<PromptTokenUsageInfo>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct PromptTokenUsageInfo {
|
||||
pub cached_tokens: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct LogProbs {
|
||||
pub tokens: Vec<String>,
|
||||
pub token_logprobs: Vec<Option<f32>>,
|
||||
pub top_logprobs: Vec<Option<HashMap<String, f32>>>,
|
||||
pub text_offset: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatLogProbs {
|
||||
Detailed {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
content: Option<Vec<ChatLogProbsContent>>,
|
||||
},
|
||||
Raw(Value),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ChatLogProbsContent {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
pub top_logprobs: Vec<TopLogProb>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TopLogProb {
|
||||
pub token: String,
|
||||
pub logprob: f32,
|
||||
pub bytes: Option<Vec<u8>>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Error Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorResponse {
|
||||
pub error: ErrorDetail,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct ErrorDetail {
|
||||
pub message: String,
|
||||
#[serde(rename = "type")]
|
||||
pub error_type: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub param: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub code: Option<String>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Input Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum InputIds {
|
||||
Single(Vec<i32>),
|
||||
Batch(Vec<Vec<i32>>),
|
||||
}
|
||||
|
||||
/// LoRA adapter path - can be single path or batch of paths (SGLang extension)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum LoRAPath {
|
||||
Single(Option<String>),
|
||||
Batch(Vec<Option<String>>),
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::common::*;
|
||||
|
||||
// ============================================================================
|
||||
// Completions API (v1/completions) - DEPRECATED but still supported
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionRequest {
|
||||
/// ID of the model to use (required for OpenAI, optional for some implementations, such as SGLang)
|
||||
pub model: String,
|
||||
|
||||
/// The prompt(s) to generate completions for
|
||||
pub prompt: StringOrArray,
|
||||
|
||||
/// The suffix that comes after a completion of inserted text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub suffix: Option<String>,
|
||||
|
||||
/// The maximum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub max_tokens: Option<u32>,
|
||||
|
||||
/// What sampling temperature to use, between 0 and 2
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub temperature: Option<f32>,
|
||||
|
||||
/// An alternative to sampling with temperature (nucleus sampling)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_p: Option<f32>,
|
||||
|
||||
/// How many completions to generate for each prompt
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
|
||||
/// Whether to stream back partial progress
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Options for streaming response
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stream_options: Option<StreamOptions>,
|
||||
|
||||
/// Include the log probabilities on the logprobs most likely tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<u32>,
|
||||
|
||||
/// Echo back the prompt in addition to the completion
|
||||
#[serde(default)]
|
||||
pub echo: bool,
|
||||
|
||||
/// Up to 4 sequences where the API will stop generating further tokens
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on whether they appear in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub presence_penalty: Option<f32>,
|
||||
|
||||
/// Number between -2.0 and 2.0. Positive values penalize new tokens based on their existing frequency in the text so far
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
|
||||
/// Generates best_of completions server-side and returns the "best"
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub best_of: Option<u32>,
|
||||
|
||||
/// Modify the likelihood of specified tokens appearing in the completion
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logit_bias: Option<HashMap<String, f32>>,
|
||||
|
||||
/// A unique identifier representing your end-user
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// If specified, our system will make a best effort to sample deterministically
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub seed: Option<i64>,
|
||||
|
||||
// -------- Engine Specific Sampling Parameters --------
|
||||
/// Top-k sampling parameter (-1 to disable)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_k: Option<i32>,
|
||||
|
||||
/// Min-p nucleus sampling parameter
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_p: Option<f32>,
|
||||
|
||||
/// Minimum number of tokens to generate
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_tokens: Option<u32>,
|
||||
|
||||
/// Repetition penalty for reducing repetitive text
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
|
||||
/// Regex constraint for output generation
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
|
||||
/// EBNF grammar constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
|
||||
/// JSON schema constraint for structured output
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
|
||||
/// Specific token IDs to use as stop conditions
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
|
||||
/// Skip trimming stop tokens from output
|
||||
#[serde(default)]
|
||||
pub no_stop_trim: bool,
|
||||
|
||||
/// Ignore end-of-sequence tokens during generation
|
||||
#[serde(default)]
|
||||
pub ignore_eos: bool,
|
||||
|
||||
/// Skip special tokens during detokenization
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// Sampling seed for deterministic outputs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
|
||||
/// Additional fields including bootstrap info for PD routing
|
||||
#[serde(flatten)]
|
||||
pub other: Map<String, Value>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for CompletionRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
match &self.prompt {
|
||||
StringOrArray::String(s) => s.clone(),
|
||||
StringOrArray::Array(v) => v.join(" "),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Response Types
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub model: String,
|
||||
pub choices: Vec<CompletionChoice>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub usage: Option<Usage>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>, // "stop", "length", "content_filter", etc.
|
||||
/// Information about which stop condition was matched
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>, // Can be string or integer
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamResponse {
|
||||
pub id: String,
|
||||
pub object: String, // "text_completion"
|
||||
pub created: u64,
|
||||
pub choices: Vec<CompletionStreamChoice>,
|
||||
pub model: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub system_fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct CompletionStreamChoice {
|
||||
pub text: String,
|
||||
pub index: u32,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprobs: Option<LogProbs>,
|
||||
pub finish_reason: Option<String>,
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
|
||||
use super::common::{GenerationRequest, UsageInfo};
|
||||
|
||||
// ============================================================================
|
||||
// Embedding API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct EmbeddingRequest {
|
||||
/// ID of the model to use
|
||||
pub model: String,
|
||||
|
||||
/// Input can be a string, array of strings, tokens, or batch inputs
|
||||
pub input: Value,
|
||||
|
||||
/// Optional encoding format (e.g., "float", "base64")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub encoding_format: Option<String>,
|
||||
|
||||
/// Optional user identifier
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub user: Option<String>,
|
||||
|
||||
/// Optional number of dimensions for the embedding
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub dimensions: Option<u32>,
|
||||
|
||||
/// SGLang extension: request id for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
|
||||
/// SGLang extension: enable/disable logging of metrics for this request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub log_metrics: Option<bool>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for EmbeddingRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
// Embeddings are non-streaming
|
||||
false
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Best effort: extract text content for routing decisions
|
||||
match &self.input {
|
||||
Value::String(s) => s.clone(),
|
||||
Value::Array(arr) => arr
|
||||
.iter()
|
||||
.filter_map(|v| v.as_str())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" "),
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingObject {
|
||||
pub object: String, // "embedding"
|
||||
pub embedding: Vec<f32>,
|
||||
pub index: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct EmbeddingResponse {
|
||||
pub object: String, // "list"
|
||||
pub data: Vec<EmbeddingObject>,
|
||||
pub model: String,
|
||||
pub usage: UsageInfo,
|
||||
}
|
||||
@@ -0,0 +1,228 @@
|
||||
use std::fmt;
|
||||
|
||||
/// Response lifecycle events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ResponseEvent {
|
||||
Created,
|
||||
InProgress,
|
||||
Completed,
|
||||
}
|
||||
|
||||
impl ResponseEvent {
|
||||
pub const CREATED: &'static str = "response.created";
|
||||
pub const IN_PROGRESS: &'static str = "response.in_progress";
|
||||
pub const COMPLETED: &'static str = "response.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Created => Self::CREATED,
|
||||
Self::InProgress => Self::IN_PROGRESS,
|
||||
Self::Completed => Self::COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ResponseEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output item events for streaming
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputItemEvent {
|
||||
Added,
|
||||
Done,
|
||||
Delta,
|
||||
}
|
||||
|
||||
impl OutputItemEvent {
|
||||
pub const ADDED: &'static str = "response.output_item.added";
|
||||
pub const DONE: &'static str = "response.output_item.done";
|
||||
pub const DELTA: &'static str = "response.output_item.delta";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
Self::Delta => Self::DELTA,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputItemEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Function call argument streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum FunctionCallEvent {
|
||||
ArgumentsDelta,
|
||||
ArgumentsDone,
|
||||
}
|
||||
|
||||
impl FunctionCallEvent {
|
||||
pub const ARGUMENTS_DELTA: &'static str = "response.function_call_arguments.delta";
|
||||
pub const ARGUMENTS_DONE: &'static str = "response.function_call_arguments.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::ArgumentsDelta => Self::ARGUMENTS_DELTA,
|
||||
Self::ArgumentsDone => Self::ARGUMENTS_DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for FunctionCallEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Content part streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ContentPartEvent {
|
||||
Added,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl ContentPartEvent {
|
||||
pub const ADDED: &'static str = "response.content_part.added";
|
||||
pub const DONE: &'static str = "response.content_part.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Added => Self::ADDED,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ContentPartEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Output text streaming events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum OutputTextEvent {
|
||||
Delta,
|
||||
Done,
|
||||
}
|
||||
|
||||
impl OutputTextEvent {
|
||||
pub const DELTA: &'static str = "response.output_text.delta";
|
||||
pub const DONE: &'static str = "response.output_text.done";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Delta => Self::DELTA,
|
||||
Self::Done => Self::DONE,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for OutputTextEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MCP Events
|
||||
// ============================================================================
|
||||
|
||||
/// MCP (Model Context Protocol) call events
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum McpEvent {
|
||||
CallArgumentsDelta,
|
||||
CallArgumentsDone,
|
||||
CallInProgress,
|
||||
CallCompleted,
|
||||
CallFailed,
|
||||
ListToolsInProgress,
|
||||
ListToolsCompleted,
|
||||
}
|
||||
|
||||
impl McpEvent {
|
||||
pub const CALL_ARGUMENTS_DELTA: &'static str = "response.mcp_call_arguments.delta";
|
||||
pub const CALL_ARGUMENTS_DONE: &'static str = "response.mcp_call_arguments.done";
|
||||
pub const CALL_IN_PROGRESS: &'static str = "response.mcp_call.in_progress";
|
||||
pub const CALL_COMPLETED: &'static str = "response.mcp_call.completed";
|
||||
pub const CALL_FAILED: &'static str = "response.mcp_call.failed";
|
||||
pub const LIST_TOOLS_IN_PROGRESS: &'static str = "response.mcp_list_tools.in_progress";
|
||||
pub const LIST_TOOLS_COMPLETED: &'static str = "response.mcp_list_tools.completed";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::CallArgumentsDelta => Self::CALL_ARGUMENTS_DELTA,
|
||||
Self::CallArgumentsDone => Self::CALL_ARGUMENTS_DONE,
|
||||
Self::CallInProgress => Self::CALL_IN_PROGRESS,
|
||||
Self::CallCompleted => Self::CALL_COMPLETED,
|
||||
Self::CallFailed => Self::CALL_FAILED,
|
||||
Self::ListToolsInProgress => Self::LIST_TOOLS_IN_PROGRESS,
|
||||
Self::ListToolsCompleted => Self::LIST_TOOLS_COMPLETED,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for McpEvent {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Item type discriminators used in output items
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
|
||||
pub enum ItemType {
|
||||
FunctionCall,
|
||||
FunctionToolCall,
|
||||
McpCall,
|
||||
Function,
|
||||
McpListTools,
|
||||
}
|
||||
|
||||
impl ItemType {
|
||||
pub const FUNCTION_CALL: &'static str = "function_call";
|
||||
pub const FUNCTION_TOOL_CALL: &'static str = "function_tool_call";
|
||||
pub const MCP_CALL: &'static str = "mcp_call";
|
||||
pub const FUNCTION: &'static str = "function";
|
||||
pub const MCP_LIST_TOOLS: &'static str = "mcp_list_tools";
|
||||
|
||||
pub const fn as_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::FunctionCall => Self::FUNCTION_CALL,
|
||||
Self::FunctionToolCall => Self::FUNCTION_TOOL_CALL,
|
||||
Self::McpCall => Self::MCP_CALL,
|
||||
Self::Function => Self::FUNCTION,
|
||||
Self::McpListTools => Self::MCP_LIST_TOOLS,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if this is a function call variant (FunctionCall or FunctionToolCall)
|
||||
pub const fn is_function_call(&self) -> bool {
|
||||
matches!(self, Self::FunctionCall | Self::FunctionToolCall)
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ItemType {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(self.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if an event type string matches any response lifecycle event
|
||||
pub fn is_response_event(event_type: &str) -> bool {
|
||||
matches!(
|
||||
event_type,
|
||||
ResponseEvent::CREATED | ResponseEvent::IN_PROGRESS | ResponseEvent::COMPLETED
|
||||
)
|
||||
}
|
||||
|
||||
/// Check if an item type string is a function call variant
|
||||
pub fn is_function_call_type(item_type: &str) -> bool {
|
||||
item_type == ItemType::FUNCTION_CALL || item_type == ItemType::FUNCTION_TOOL_CALL
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::{
|
||||
common::{default_true, GenerationRequest, InputIds},
|
||||
sampling_params::SamplingParams,
|
||||
};
|
||||
use crate::validated::Normalizable;
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate API (native format)
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, Validate)]
|
||||
#[validate(schema(function = "validate_generate_request"))]
|
||||
pub struct GenerateRequest {
|
||||
/// Text input - SGLang native format
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub text: Option<String>,
|
||||
|
||||
pub model: Option<String>,
|
||||
|
||||
/// Input IDs for tokenized input
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_ids: Option<InputIds>,
|
||||
|
||||
/// Input embeddings for direct embedding input
|
||||
/// Can be a 2D array (single request) or 3D array (batch of requests)
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_embeds: Option<Value>,
|
||||
|
||||
/// Image input data
|
||||
/// Can be an image instance, file name, URL, or base64 encoded string
|
||||
/// Supports single images, lists of images, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub image_data: Option<Value>,
|
||||
|
||||
/// Video input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single videos, lists of videos, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub video_data: Option<Value>,
|
||||
|
||||
/// Audio input data
|
||||
/// Can be a file name, URL, or base64 encoded string
|
||||
/// Supports single audio files, lists of audio, or nested lists for batch processing
|
||||
/// Placeholder for future use
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub audio_data: Option<Value>,
|
||||
|
||||
/// Sampling parameters (sglang style)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_params: Option<SamplingParams>,
|
||||
|
||||
/// Whether to return logprobs
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub return_logprob: Option<bool>,
|
||||
|
||||
/// If return logprobs, the start location in the prompt for returning logprobs.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub logprob_start_len: Option<i32>,
|
||||
|
||||
/// If return logprobs, the number of top logprobs to return at each position.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub top_logprobs_num: Option<i32>,
|
||||
|
||||
/// If return logprobs, the token ids to return logprob for.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub token_ids_logprob: Option<Vec<u32>>,
|
||||
|
||||
/// Whether to detokenize tokens in text in the returned logprobs.
|
||||
#[serde(default)]
|
||||
pub return_text_in_logprobs: bool,
|
||||
|
||||
/// Whether to stream the response
|
||||
#[serde(default)]
|
||||
pub stream: bool,
|
||||
|
||||
/// Whether to log metrics for this request (e.g. health_generate calls do not log metrics)
|
||||
#[serde(default = "default_true")]
|
||||
pub log_metrics: bool,
|
||||
|
||||
/// Return model hidden states
|
||||
#[serde(default)]
|
||||
pub return_hidden_states: bool,
|
||||
|
||||
/// The modalities of the image data [image, multi-images, video]
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub modalities: Option<Vec<String>>,
|
||||
|
||||
/// Session parameters for continual prompting
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub session_params: Option<HashMap<String, Value>>,
|
||||
|
||||
/// Path to LoRA adapter(s) for model customization
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_path: Option<String>,
|
||||
|
||||
/// LoRA adapter ID (if pre-loaded)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub lora_id: Option<String>,
|
||||
|
||||
/// Custom logit processor for advanced sampling control. Must be a serialized instance
|
||||
/// of `CustomLogitProcessor` in python/sglang/srt/sampling/custom_logit_processor.py
|
||||
/// Use the processor's `to_str()` method to generate the serialized string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_logit_processor: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_host: Option<String>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_room: Option<i32>,
|
||||
|
||||
/// For disaggregated inference
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_pair_key: Option<String>,
|
||||
|
||||
/// Data parallel rank routing
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub data_parallel_rank: Option<i32>,
|
||||
|
||||
/// Background response
|
||||
#[serde(default)]
|
||||
pub background: bool,
|
||||
|
||||
/// Conversation ID for tracking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub conversation_id: Option<String>,
|
||||
|
||||
/// Priority for the request
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<i32>,
|
||||
|
||||
/// Extra key for classifying the request (e.g. cache_salt)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub extra_key: Option<String>,
|
||||
|
||||
/// Whether to disallow logging for this request (e.g. due to ZDR)
|
||||
#[serde(default)]
|
||||
pub no_logs: bool,
|
||||
|
||||
/// Custom metric labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub custom_labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Whether to return bytes for image generation
|
||||
#[serde(default)]
|
||||
pub return_bytes: bool,
|
||||
|
||||
/// Whether to return entropy
|
||||
#[serde(default)]
|
||||
pub return_entropy: bool,
|
||||
|
||||
/// Request ID for tracking (inherited from BaseReq in Python)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub rid: Option<String>,
|
||||
}
|
||||
|
||||
impl Normalizable for GenerateRequest {
|
||||
// Use default no-op implementation - no normalization needed for GenerateRequest
|
||||
}
|
||||
|
||||
/// Validation function for GenerateRequest - ensure exactly one input type is provided
|
||||
fn validate_generate_request(req: &GenerateRequest) -> Result<(), validator::ValidationError> {
|
||||
// Exactly one of text or input_ids must be provided
|
||||
// Note: input_embeds not yet supported in Rust implementation
|
||||
let has_text = req.text.is_some();
|
||||
let has_input_ids = req.input_ids.is_some();
|
||||
|
||||
let count = [has_text, has_input_ids].iter().filter(|&&x| x).count();
|
||||
|
||||
if count == 0 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
if count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"Either text or input_ids should be provided.",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl GenerationRequest for GenerateRequest {
|
||||
fn is_stream(&self) -> bool {
|
||||
self.stream
|
||||
}
|
||||
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
// Generate requests have an optional model field
|
||||
if let Some(s) = &self.model {
|
||||
Some(s.as_str())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
// Check fields in priority order: text, input_ids
|
||||
if let Some(ref text) = self.text {
|
||||
return text.clone();
|
||||
}
|
||||
|
||||
if let Some(ref input_ids) = self.input_ids {
|
||||
return match input_ids {
|
||||
InputIds::Single(ids) => ids
|
||||
.iter()
|
||||
.map(|&id| id.to_string())
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
InputIds::Batch(batches) => batches
|
||||
.iter()
|
||||
.flat_map(|batch| batch.iter().map(|&id| id.to_string()))
|
||||
.collect::<Vec<String>>()
|
||||
.join(" "),
|
||||
};
|
||||
}
|
||||
|
||||
// No text input found
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SGLang Generate Response Types
|
||||
// ============================================================================
|
||||
|
||||
/// SGLang generate response (single completion or array for n>1)
|
||||
///
|
||||
/// Format for n=1:
|
||||
/// ```json
|
||||
/// {
|
||||
/// "text": "...",
|
||||
/// "output_ids": [...],
|
||||
/// "meta_info": { ... }
|
||||
/// }
|
||||
/// ```
|
||||
///
|
||||
/// Format for n>1:
|
||||
/// ```json
|
||||
/// [
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}},
|
||||
/// {"text": "...", "output_ids": [...], "meta_info": {...}}
|
||||
/// ]
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateResponse {
|
||||
pub text: String,
|
||||
pub output_ids: Vec<u32>,
|
||||
pub meta_info: GenerateMetaInfo,
|
||||
}
|
||||
|
||||
/// Metadata for a single generate completion
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct GenerateMetaInfo {
|
||||
pub id: String,
|
||||
pub finish_reason: GenerateFinishReason,
|
||||
pub prompt_tokens: u32,
|
||||
pub weight_version: String,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub input_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub output_token_logprobs: Option<Vec<Vec<Option<f64>>>>,
|
||||
pub completion_tokens: u32,
|
||||
pub cached_tokens: u32,
|
||||
pub e2e_latency: f64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub matched_stop: Option<Value>,
|
||||
}
|
||||
|
||||
/// Finish reason for generate endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "lowercase")]
|
||||
pub enum GenerateFinishReason {
|
||||
Length {
|
||||
length: u32,
|
||||
},
|
||||
Stop,
|
||||
#[serde(untagged)]
|
||||
Other(Value),
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
// Protocol definitions and validation for various LLM APIs
|
||||
// This module provides a structured approach to handling different API protocols
|
||||
|
||||
/// Default model identifier used when no model is specified.
|
||||
///
|
||||
/// This constant should be used instead of hardcoded "unknown" strings
|
||||
/// throughout the codebase for consistency.
|
||||
pub const UNKNOWN_MODEL_ID: &str = "unknown";
|
||||
|
||||
pub mod builders;
|
||||
pub mod chat;
|
||||
pub mod classify;
|
||||
pub mod common;
|
||||
pub mod completion;
|
||||
pub mod embedding;
|
||||
pub mod event_types;
|
||||
pub mod generate;
|
||||
pub mod messages;
|
||||
pub mod parser;
|
||||
pub mod rerank;
|
||||
pub mod responses;
|
||||
pub mod sampling_params;
|
||||
pub mod tokenize;
|
||||
pub mod validated;
|
||||
pub mod worker_spec;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,23 @@
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::common::Tool;
|
||||
|
||||
/// Request to parse function calls from model output text
|
||||
#[derive(Deserialize)]
|
||||
pub struct ParseFunctionCallRequest {
|
||||
/// The text to parse for function calls
|
||||
pub text: String,
|
||||
/// The parser type/name to use for parsing (e.g., "json", "pythonic")
|
||||
pub tool_call_parser: String,
|
||||
/// The list of available tools that the model can call
|
||||
pub tools: Vec<Tool>,
|
||||
}
|
||||
|
||||
/// Request to separate reasoning from normal text in model output
|
||||
#[derive(Deserialize)]
|
||||
pub struct SeparateReasoningRequest {
|
||||
/// The text to parse for reasoning content
|
||||
pub text: String,
|
||||
/// The parser type/name to use for reasoning detection (e.g., "step3", "deepseek_r1")
|
||||
pub reasoning_parser: String,
|
||||
}
|
||||
@@ -0,0 +1,212 @@
|
||||
use std::collections::HashMap;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use validator::Validate;
|
||||
|
||||
use super::common::{default_model, default_true, GenerationRequest, StringOrArray, UsageInfo};
|
||||
|
||||
fn default_rerank_object() -> String {
|
||||
"rerank".to_string()
|
||||
}
|
||||
|
||||
/// TODO: Create timestamp should not be in protocol layer
|
||||
fn current_timestamp() -> i64 {
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_else(|_| std::time::Duration::from_secs(0))
|
||||
.as_secs() as i64
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Rerank API
|
||||
// ============================================================================
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Validate)]
|
||||
#[validate(schema(function = "validate_rerank_request"))]
|
||||
pub struct RerankRequest {
|
||||
/// The query text to rank documents against
|
||||
#[validate(custom(function = "validate_query"))]
|
||||
pub query: String,
|
||||
|
||||
/// List of documents to be ranked
|
||||
#[validate(custom(function = "validate_documents"))]
|
||||
pub documents: Vec<String>,
|
||||
|
||||
/// Model to use for reranking
|
||||
#[serde(default = "default_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Maximum number of documents to return (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 1))]
|
||||
pub top_k: Option<usize>,
|
||||
|
||||
/// Whether to return documents in addition to scores
|
||||
#[serde(default = "default_true")]
|
||||
pub return_documents: bool,
|
||||
|
||||
// SGLang specific extensions
|
||||
/// Request ID for tracking
|
||||
pub rid: Option<StringOrArray>,
|
||||
|
||||
/// User identifier
|
||||
pub user: Option<String>,
|
||||
}
|
||||
|
||||
impl GenerationRequest for RerankRequest {
|
||||
fn get_model(&self) -> Option<&str> {
|
||||
Some(&self.model)
|
||||
}
|
||||
|
||||
fn is_stream(&self) -> bool {
|
||||
false // Reranking doesn't support streaming
|
||||
}
|
||||
|
||||
fn extract_text_for_routing(&self) -> String {
|
||||
self.query.clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl super::validated::Normalizable for RerankRequest {
|
||||
// Use default no-op normalization
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates that the query is not empty
|
||||
fn validate_query(query: &str) -> Result<(), validator::ValidationError> {
|
||||
if query.trim().is_empty() {
|
||||
return Err(validator::ValidationError::new("query cannot be empty"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates that the documents list is not empty
|
||||
fn validate_documents(documents: &[String]) -> Result<(), validator::ValidationError> {
|
||||
if documents.is_empty() {
|
||||
return Err(validator::ValidationError::new(
|
||||
"documents list cannot be empty",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Schema-level validation for cross-field dependencies
|
||||
fn validate_rerank_request(req: &RerankRequest) -> Result<(), validator::ValidationError> {
|
||||
// Validate top_k if specified
|
||||
if let Some(k) = req.top_k {
|
||||
if k > req.documents.len() {
|
||||
// This is allowed but we log a warning
|
||||
tracing::warn!(
|
||||
"top_k ({}) is greater than number of documents ({})",
|
||||
k,
|
||||
req.documents.len()
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
impl RerankRequest {
|
||||
/// Get the effective top_k value
|
||||
pub fn effective_top_k(&self) -> usize {
|
||||
self.top_k.unwrap_or(self.documents.len())
|
||||
}
|
||||
}
|
||||
|
||||
/// Individual rerank result
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RerankResult {
|
||||
/// Relevance score for the document
|
||||
pub score: f32,
|
||||
|
||||
/// The document text (if return_documents was true)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub document: Option<String>,
|
||||
|
||||
/// Original index of the document in the request
|
||||
pub index: usize,
|
||||
|
||||
/// Additional metadata about the ranking
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub meta_info: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
/// Rerank response containing sorted results
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RerankResponse {
|
||||
/// Ranked results sorted by score (highest first)
|
||||
pub results: Vec<RerankResult>,
|
||||
|
||||
/// Model used for reranking
|
||||
pub model: String,
|
||||
|
||||
/// Usage information
|
||||
pub usage: Option<UsageInfo>,
|
||||
|
||||
/// Response object type
|
||||
#[serde(default = "default_rerank_object")]
|
||||
pub object: String,
|
||||
|
||||
/// Response ID
|
||||
pub id: Option<StringOrArray>,
|
||||
|
||||
/// Creation timestamp
|
||||
pub created: i64,
|
||||
}
|
||||
|
||||
impl RerankResponse {
|
||||
/// Create a new RerankResponse with the given results and model
|
||||
pub fn new(
|
||||
results: Vec<RerankResult>,
|
||||
model: String,
|
||||
request_id: Option<StringOrArray>,
|
||||
) -> Self {
|
||||
RerankResponse {
|
||||
results,
|
||||
model,
|
||||
usage: None,
|
||||
object: default_rerank_object(),
|
||||
id: request_id,
|
||||
created: current_timestamp(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply top_k limit to results
|
||||
pub fn apply_top_k(&mut self, k: usize) {
|
||||
self.results.truncate(k);
|
||||
}
|
||||
|
||||
/// Drop documents from results (when return_documents is false)
|
||||
pub fn drop_documents(&mut self) {
|
||||
for result in &mut self.results {
|
||||
result.document = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// V1 API compatibility format for rerank requests
|
||||
/// Matches Python's V1RerankReqInput
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct V1RerankReqInput {
|
||||
pub query: String,
|
||||
pub documents: Vec<String>,
|
||||
}
|
||||
|
||||
/// Convert V1RerankReqInput to RerankRequest
|
||||
impl From<V1RerankReqInput> for RerankRequest {
|
||||
fn from(v1: V1RerankReqInput) -> Self {
|
||||
RerankRequest {
|
||||
query: v1.query,
|
||||
documents: v1.documents,
|
||||
model: default_model(),
|
||||
top_k: None,
|
||||
return_documents: true,
|
||||
rid: None,
|
||||
user: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,119 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use validator::Validate;
|
||||
|
||||
use super::common::StringOrArray;
|
||||
|
||||
/// Sampling parameters for text generation
|
||||
#[derive(Debug, Clone, Deserialize, Serialize, Default, Validate)]
|
||||
#[validate(schema(function = "validate_sampling_params"))]
|
||||
pub struct SamplingParams {
|
||||
/// Temperature for sampling (must be >= 0.0, no upper limit)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0))]
|
||||
pub temperature: Option<f32>,
|
||||
/// Maximum number of new tokens to generate (must be >= 0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0))]
|
||||
pub max_new_tokens: Option<u32>,
|
||||
/// Top-p nucleus sampling (0.0 < top_p <= 1.0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_p_value"))]
|
||||
pub top_p: Option<f32>,
|
||||
/// Top-k sampling (-1 to disable, or >= 1)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(custom(function = "validate_top_k_value"))]
|
||||
pub top_k: Option<i32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub frequency_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = -2.0, max = 2.0))]
|
||||
pub presence_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 2.0))]
|
||||
pub repetition_penalty: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop: Option<StringOrArray>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ignore_eos: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub skip_special_tokens: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub json_schema: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub regex: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub ebnf: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
pub min_p: Option<f32>,
|
||||
/// Minimum number of new tokens (validated in schema function for cross-field check with max_new_tokens)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub min_new_tokens: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub stop_token_ids: Option<Vec<u32>>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub no_stop_trim: Option<bool>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub n: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sampling_seed: Option<u64>,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Shared Validation Functions
|
||||
// ============================================================================
|
||||
|
||||
/// Validates top_p: 0.0 < top_p <= 1.0 (can't use range validator for open interval)
|
||||
pub fn validate_top_p_value(top_p: f32) -> Result<(), validator::ValidationError> {
|
||||
if !(top_p > 0.0 && top_p <= 1.0) {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_p must be in (0, 1] - greater than 0.0 and at most 1.0",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validates top_k: -1 (disabled) or >= 1 (special -1 case - can't use range validator)
|
||||
pub fn validate_top_k_value(top_k: i32) -> Result<(), validator::ValidationError> {
|
||||
if top_k != -1 && top_k < 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"top_k must be -1 (disabled) or at least 1",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// SamplingParams-Specific Validation
|
||||
// ============================================================================
|
||||
|
||||
/// Validation function for SamplingParams - cross-field validation only
|
||||
fn validate_sampling_params(params: &SamplingParams) -> Result<(), validator::ValidationError> {
|
||||
// 1. Cross-field validation: min_new_tokens <= max_new_tokens
|
||||
if let (Some(min), Some(max)) = (params.min_new_tokens, params.max_new_tokens) {
|
||||
if min > max {
|
||||
return Err(validator::ValidationError::new(
|
||||
"min_new_tokens cannot exceed max_new_tokens",
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Validate mutually exclusive structured output constraints
|
||||
let constraint_count = [
|
||||
params.regex.is_some(),
|
||||
params.ebnf.is_some(),
|
||||
params.json_schema.is_some(),
|
||||
]
|
||||
.iter()
|
||||
.filter(|&&x| x)
|
||||
.count();
|
||||
|
||||
if constraint_count > 1 {
|
||||
return Err(validator::ValidationError::new(
|
||||
"only one of regex, ebnf, or json_schema can be set",
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
//! Tokenize and Detokenize API protocol types
|
||||
//!
|
||||
//! These types mirror the SGLang Python implementation for compatibility.
|
||||
//! See: python/sglang/srt/entrypoints/openai/protocol.py
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
// ============================================================================
|
||||
// Tokenize API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for the /v1/tokenize endpoint
|
||||
///
|
||||
/// Supports both single string and batch tokenization.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct TokenizeRequest {
|
||||
/// Model name for tokenizer selection
|
||||
#[serde(default = "default_model_name")]
|
||||
pub model: String,
|
||||
|
||||
/// Text(s) to tokenize - can be a single string or array of strings
|
||||
pub prompt: StringOrArray,
|
||||
}
|
||||
|
||||
/// Response schema for the /v1/tokenize endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TokenizeResponse {
|
||||
/// Token IDs - single list for single input, nested list for batch
|
||||
pub tokens: TokensResult,
|
||||
|
||||
/// Token count(s) - single int for single input, list for batch
|
||||
pub count: CountResult,
|
||||
|
||||
/// Character count(s) of input - single int for single input, list for batch
|
||||
pub char_count: CountResult,
|
||||
}
|
||||
|
||||
/// Token IDs result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TokensResult {
|
||||
Single(Vec<u32>),
|
||||
Batch(Vec<Vec<u32>>),
|
||||
}
|
||||
|
||||
/// Count result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum CountResult {
|
||||
Single(i32),
|
||||
Batch(Vec<i32>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Detokenize API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for the /v1/detokenize endpoint
|
||||
///
|
||||
/// Supports both single sequence and batch detokenization.
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct DetokenizeRequest {
|
||||
/// Model name for tokenizer selection
|
||||
#[serde(default = "default_model_name")]
|
||||
pub model: String,
|
||||
|
||||
/// Token IDs to detokenize - single list or batch (list of lists)
|
||||
pub tokens: TokensInput,
|
||||
|
||||
/// Whether to skip special tokens (e.g., padding or EOS) during decoding
|
||||
#[serde(default = "default_true")]
|
||||
pub skip_special_tokens: bool,
|
||||
}
|
||||
|
||||
/// Token input - either single sequence or batch
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TokensInput {
|
||||
/// Single sequence of token IDs
|
||||
Single(Vec<u32>),
|
||||
/// Batch of token sequences
|
||||
Batch(Vec<Vec<u32>>),
|
||||
}
|
||||
|
||||
impl TokensInput {
|
||||
/// Check if this is a batch input
|
||||
pub fn is_batch(&self) -> bool {
|
||||
matches!(self, TokensInput::Batch(_))
|
||||
}
|
||||
|
||||
/// Get the sequences (always returns a vec of vecs for uniform processing)
|
||||
pub fn sequences(&self) -> Vec<&[u32]> {
|
||||
match self {
|
||||
TokensInput::Single(seq) => vec![seq.as_slice()],
|
||||
TokensInput::Batch(seqs) => seqs.iter().map(|s| s.as_slice()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Response schema for the /v1/detokenize endpoint
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct DetokenizeResponse {
|
||||
/// Decoded text - single string for single input, list for batch
|
||||
pub text: TextResult,
|
||||
}
|
||||
|
||||
/// Text result - either single or batch
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum TextResult {
|
||||
Single(String),
|
||||
Batch(Vec<String>),
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Tokenizer Management API
|
||||
// ============================================================================
|
||||
|
||||
/// Request schema for adding a tokenizer
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct AddTokenizerRequest {
|
||||
/// Name to register the tokenizer under
|
||||
pub name: String,
|
||||
|
||||
/// Source: either a local path or HuggingFace model ID
|
||||
pub source: String,
|
||||
|
||||
/// Optional path to chat template file
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Response schema for adding a tokenizer (async)
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct AddTokenizerResponse {
|
||||
/// Unique identifier for the tokenizer (UUID)
|
||||
pub id: String,
|
||||
/// Status of the request: "pending", "processing", "completed", "failed"
|
||||
pub status: String,
|
||||
pub message: String,
|
||||
/// Vocabulary size of the loaded tokenizer (only set on completion)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub vocab_size: Option<usize>,
|
||||
}
|
||||
|
||||
/// Response schema for listing tokenizers
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ListTokenizersResponse {
|
||||
pub tokenizers: Vec<TokenizerInfo>,
|
||||
}
|
||||
|
||||
/// Information about a registered tokenizer
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TokenizerInfo {
|
||||
/// Unique identifier (UUID)
|
||||
pub id: String,
|
||||
/// User-provided name
|
||||
pub name: String,
|
||||
/// Source path or HuggingFace model ID
|
||||
pub source: String,
|
||||
pub vocab_size: usize,
|
||||
}
|
||||
|
||||
/// Request schema for removing a tokenizer
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct RemoveTokenizerRequest {
|
||||
/// Name of the tokenizer to remove
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
/// Response schema for removing a tokenizer
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct RemoveTokenizerResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Helper Types
|
||||
// ============================================================================
|
||||
|
||||
/// String or array of strings (for flexible input)
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum StringOrArray {
|
||||
Single(String),
|
||||
Array(Vec<String>),
|
||||
}
|
||||
|
||||
impl StringOrArray {
|
||||
/// Check if this is a batch (array) input
|
||||
pub fn is_batch(&self) -> bool {
|
||||
matches!(self, StringOrArray::Array(_))
|
||||
}
|
||||
|
||||
/// Get all strings as a slice (converts single to vec)
|
||||
pub fn as_strings(&self) -> Vec<&str> {
|
||||
match self {
|
||||
StringOrArray::Single(s) => vec![s.as_str()],
|
||||
StringOrArray::Array(arr) => arr.iter().map(|s| s.as_str()).collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Default Functions
|
||||
// ============================================================================
|
||||
|
||||
fn default_model_name() -> String {
|
||||
UNKNOWN_MODEL_ID.to_string()
|
||||
}
|
||||
|
||||
fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_request_single() {
|
||||
let json = r#"{"prompt": "Hello world"}"#;
|
||||
let req: TokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(req.model, "unknown");
|
||||
assert!(matches!(req.prompt, StringOrArray::Single(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_request_batch() {
|
||||
let json = r#"{"model": "llama", "prompt": ["Hello", "World"]}"#;
|
||||
let req: TokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert_eq!(req.model, "llama");
|
||||
assert!(matches!(req.prompt, StringOrArray::Array(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detokenize_request_single() {
|
||||
let json = r#"{"tokens": [1, 2, 3]}"#;
|
||||
let req: DetokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(req.tokens, TokensInput::Single(_)));
|
||||
assert!(req.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_detokenize_request_batch() {
|
||||
let json = r#"{"tokens": [[1, 2], [3, 4, 5]], "skip_special_tokens": false}"#;
|
||||
let req: DetokenizeRequest = serde_json::from_str(json).unwrap();
|
||||
assert!(matches!(req.tokens, TokensInput::Batch(_)));
|
||||
assert!(!req.skip_special_tokens);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_response_single() {
|
||||
let resp = TokenizeResponse {
|
||||
tokens: TokensResult::Single(vec![1, 2, 3]),
|
||||
count: CountResult::Single(3),
|
||||
char_count: CountResult::Single(11),
|
||||
};
|
||||
let json = serde_json::to_string(&resp).unwrap();
|
||||
assert!(json.contains("[1,2,3]"));
|
||||
assert!(json.contains("\"count\":3"));
|
||||
assert!(json.contains("\"char_count\":11"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_tokenize_response_batch() {
|
||||
let resp = TokenizeResponse {
|
||||
tokens: TokensResult::Batch(vec![vec![1, 2], vec![3, 4, 5]]),
|
||||
count: CountResult::Batch(vec![2, 3]),
|
||||
char_count: CountResult::Batch(vec![5, 5]),
|
||||
};
|
||||
let json = serde_json::to_string(&resp).unwrap();
|
||||
assert!(json.contains("[[1,2],[3,4,5]]"));
|
||||
assert!(json.contains("[2,3]"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,172 @@
|
||||
// Validated JSON extractor for automatic request validation
|
||||
//
|
||||
// This module provides a ValidatedJson extractor that automatically validates
|
||||
// requests using the validator crate's Validate trait.
|
||||
|
||||
/// Trait for request types that need post-deserialization normalization
|
||||
pub trait Normalizable {
|
||||
/// Normalize the request by applying defaults and transformations
|
||||
fn normalize(&mut self) {
|
||||
// Default: no-op
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
use axum::{
|
||||
extract::{rejection::JsonRejection, FromRequest, Request},
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
#[cfg(feature = "axum")]
|
||||
use serde::de::DeserializeOwned;
|
||||
#[cfg(feature = "axum")]
|
||||
use serde_json::json;
|
||||
#[cfg(feature = "axum")]
|
||||
use validator::Validate;
|
||||
|
||||
/// A JSON extractor that automatically validates and normalizes the request body
|
||||
///
|
||||
/// This extractor deserializes the request body and automatically calls `.validate()`
|
||||
/// on types that implement the `Validate` trait. If validation fails, it returns
|
||||
/// a 400 Bad Request with detailed error information.
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```rust,ignore
|
||||
/// async fn create_chat(
|
||||
/// ValidatedJson(request): ValidatedJson<ChatCompletionRequest>,
|
||||
/// ) -> Response {
|
||||
/// // request is guaranteed to be valid here
|
||||
/// process_request(request).await
|
||||
/// }
|
||||
/// ```
|
||||
#[cfg(feature = "axum")]
|
||||
pub struct ValidatedJson<T>(pub T);
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl<S, T> FromRequest<S> for ValidatedJson<T>
|
||||
where
|
||||
T: DeserializeOwned + Validate + Normalizable + Send,
|
||||
S: Send + Sync,
|
||||
{
|
||||
type Rejection = Response;
|
||||
|
||||
async fn from_request(req: Request, state: &S) -> Result<Self, Self::Rejection> {
|
||||
// First, extract and deserialize the JSON
|
||||
let Json(mut data) =
|
||||
Json::<T>::from_request(req, state)
|
||||
.await
|
||||
.map_err(|err: JsonRejection| {
|
||||
let error_message = match err {
|
||||
JsonRejection::JsonDataError(e) => {
|
||||
format!("Invalid JSON data: {}", e)
|
||||
}
|
||||
JsonRejection::JsonSyntaxError(e) => {
|
||||
format!("JSON syntax error: {}", e)
|
||||
}
|
||||
JsonRejection::MissingJsonContentType(_) => {
|
||||
"Missing Content-Type: application/json header".to_string()
|
||||
}
|
||||
_ => format!("Failed to parse JSON: {}", err),
|
||||
};
|
||||
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": error_message,
|
||||
"type": "invalid_request_error",
|
||||
"code": "json_parse_error"
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
// Normalize the request (apply defaults based on other fields)
|
||||
data.normalize();
|
||||
|
||||
// Then, automatically validate the data
|
||||
data.validate().map_err(|validation_errors| {
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
Json(json!({
|
||||
"error": {
|
||||
"message": validation_errors.to_string(),
|
||||
"type": "invalid_request_error",
|
||||
"code": 400
|
||||
}
|
||||
})),
|
||||
)
|
||||
.into_response()
|
||||
})?;
|
||||
|
||||
Ok(ValidatedJson(data))
|
||||
}
|
||||
}
|
||||
|
||||
// Implement Deref to allow transparent access to the inner value
|
||||
#[cfg(feature = "axum")]
|
||||
impl<T> std::ops::Deref for ValidatedJson<T> {
|
||||
type Target = T;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
&self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl<T> std::ops::DerefMut for ValidatedJson<T> {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
&mut self.0
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "axum"))]
|
||||
mod tests {
|
||||
use serde::{Deserialize, Serialize};
|
||||
use validator::Validate;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Validate)]
|
||||
struct TestRequest {
|
||||
#[validate(range(min = 0.0, max = 1.0))]
|
||||
value: f32,
|
||||
#[validate(length(min = 1))]
|
||||
name: String,
|
||||
}
|
||||
|
||||
impl Normalizable for TestRequest {
|
||||
// Use default no-op implementation
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_valid() {
|
||||
// This test is conceptual - actual testing would require Axum test harness
|
||||
let request = TestRequest {
|
||||
value: 0.5,
|
||||
name: "test".to_string(),
|
||||
};
|
||||
assert!(request.validate().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_invalid_range() {
|
||||
let request = TestRequest {
|
||||
value: 1.5, // Out of range
|
||||
name: "test".to_string(),
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_validated_json_invalid_length() {
|
||||
let request = TestRequest {
|
||||
value: 0.5,
|
||||
name: "".to_string(), // Empty name
|
||||
};
|
||||
assert!(request.validate().is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,469 @@
|
||||
//! Worker management API specifications
|
||||
//!
|
||||
//! Defines the request/response structures for worker management endpoints
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
use axum::{
|
||||
http::StatusCode,
|
||||
response::{IntoResponse, Response},
|
||||
Json,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[cfg(feature = "axum")]
|
||||
use serde_json::{json, Value};
|
||||
|
||||
use super::UNKNOWN_MODEL_ID;
|
||||
|
||||
/// Worker configuration for API requests
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerConfigRequest {
|
||||
/// Worker URL (required)
|
||||
pub url: String,
|
||||
|
||||
/// Worker API key (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// Model ID (optional, will query from server if not provided)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_id: Option<String>,
|
||||
|
||||
/// Worker priority (optional, default: 50, higher = preferred)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
/// Worker cost factor (optional, default: 1.0)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
/// Worker type (optional: "regular", "prefill", "decode")
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
|
||||
/// Bootstrap port for prefill workers (optional)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<u16>,
|
||||
|
||||
/// Runtime type (optional: "sglang", "vllm", default: "sglang")
|
||||
/// Only relevant for gRPC workers
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime: Option<String>,
|
||||
|
||||
// gRPC-specific configuration (optional, ignored in HTTP mode)
|
||||
/// Tokenizer path for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
/// Reasoning parser type for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
/// Tool parser type for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
/// Chat template for gRPC mode
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Additional labels (optional)
|
||||
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
|
||||
pub labels: HashMap<String, String>,
|
||||
|
||||
/// Health check timeout in seconds (default: 30)
|
||||
#[serde(default = "default_health_check_timeout")]
|
||||
pub health_check_timeout_secs: u64,
|
||||
|
||||
/// Health check interval in seconds (default: 60)
|
||||
#[serde(default = "default_health_check_interval")]
|
||||
pub health_check_interval_secs: u64,
|
||||
|
||||
/// Number of successful health checks needed to mark worker as healthy (default: 2)
|
||||
#[serde(default = "default_health_success_threshold")]
|
||||
pub health_success_threshold: u32,
|
||||
|
||||
/// Number of failed health checks before marking worker as unhealthy (default: 3)
|
||||
#[serde(default = "default_health_failure_threshold")]
|
||||
pub health_failure_threshold: u32,
|
||||
|
||||
/// Disable periodic health checks for this worker (default: false)
|
||||
#[serde(default)]
|
||||
pub disable_health_check: bool,
|
||||
|
||||
/// Maximum connection attempts during worker registration (default: 20)
|
||||
#[serde(default = "default_max_connection_attempts")]
|
||||
pub max_connection_attempts: u32,
|
||||
|
||||
/// Enable data parallelism aware scheduling (default: false)
|
||||
#[serde(default)]
|
||||
pub dp_aware: bool,
|
||||
}
|
||||
|
||||
// Default value functions for serde
|
||||
fn default_health_check_timeout() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
fn default_health_check_interval() -> u64 {
|
||||
60
|
||||
}
|
||||
|
||||
fn default_health_success_threshold() -> u32 {
|
||||
2
|
||||
}
|
||||
|
||||
fn default_health_failure_threshold() -> u32 {
|
||||
3
|
||||
}
|
||||
|
||||
fn default_max_connection_attempts() -> u32 {
|
||||
20
|
||||
}
|
||||
|
||||
/// Worker information for API responses
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerInfo {
|
||||
/// Worker unique identifier
|
||||
pub id: String,
|
||||
|
||||
/// Worker URL
|
||||
pub url: String,
|
||||
|
||||
/// Model ID this worker serves
|
||||
pub model_id: String,
|
||||
|
||||
/// Worker priority
|
||||
pub priority: u32,
|
||||
|
||||
/// Worker cost factor
|
||||
pub cost: f32,
|
||||
|
||||
/// Worker type
|
||||
pub worker_type: String,
|
||||
|
||||
/// Whether the worker is healthy
|
||||
pub is_healthy: bool,
|
||||
|
||||
/// Current load on the worker
|
||||
pub load: usize,
|
||||
|
||||
/// Connection mode (http or grpc)
|
||||
pub connection_mode: String,
|
||||
|
||||
/// Runtime type (sglang or vllm, for gRPC workers)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub runtime_type: Option<String>,
|
||||
|
||||
// gRPC-specific fields (None for HTTP workers)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
|
||||
/// Bootstrap port for prefill workers
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub bootstrap_port: Option<u16>,
|
||||
|
||||
/// Additional metadata
|
||||
#[serde(skip_serializing_if = "HashMap::is_empty")]
|
||||
pub metadata: HashMap<String, String>,
|
||||
|
||||
/// Whether health checks are disabled for this worker
|
||||
pub disable_health_check: bool,
|
||||
|
||||
/// Job status for async operations (if available)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub job_status: Option<JobStatus>,
|
||||
}
|
||||
|
||||
impl WorkerInfo {
|
||||
/// Create a partial WorkerInfo for pending workers (not yet registered).
|
||||
/// Used when a worker ID maps to a URL but the worker is still being registered.
|
||||
pub fn pending(worker_id: &str, url: String, job_status: Option<JobStatus>) -> Self {
|
||||
Self {
|
||||
id: worker_id.to_string(),
|
||||
url,
|
||||
model_id: UNKNOWN_MODEL_ID.to_string(),
|
||||
priority: 0,
|
||||
cost: 1.0,
|
||||
worker_type: UNKNOWN_MODEL_ID.to_string(),
|
||||
is_healthy: false,
|
||||
load: 0,
|
||||
connection_mode: UNKNOWN_MODEL_ID.to_string(),
|
||||
runtime_type: None,
|
||||
tokenizer_path: None,
|
||||
reasoning_parser: None,
|
||||
tool_parser: None,
|
||||
chat_template: None,
|
||||
bootstrap_port: None,
|
||||
metadata: HashMap::new(),
|
||||
disable_health_check: false,
|
||||
job_status,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Job status for async control plane operations
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct JobStatus {
|
||||
pub job_type: String,
|
||||
pub worker_url: String,
|
||||
pub status: String,
|
||||
pub message: Option<String>,
|
||||
pub timestamp: u64,
|
||||
}
|
||||
|
||||
impl JobStatus {
|
||||
/// Create a pending job status
|
||||
pub fn pending(job_type: &str, worker_url: &str) -> Self {
|
||||
Self {
|
||||
job_type: job_type.to_string(),
|
||||
worker_url: worker_url.to_string(),
|
||||
status: "pending".to_string(),
|
||||
message: None,
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a processing job status
|
||||
pub fn processing(job_type: &str, worker_url: &str) -> Self {
|
||||
Self {
|
||||
job_type: job_type.to_string(),
|
||||
worker_url: worker_url.to_string(),
|
||||
status: "processing".to_string(),
|
||||
message: None,
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a failed job status
|
||||
pub fn failed(job_type: &str, worker_url: &str, error: String) -> Self {
|
||||
Self {
|
||||
job_type: job_type.to_string(),
|
||||
worker_url: worker_url.to_string(),
|
||||
status: "failed".to_string(),
|
||||
message: Some(error),
|
||||
timestamp: std::time::SystemTime::now()
|
||||
.duration_since(std::time::SystemTime::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Worker list response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerListResponse {
|
||||
/// List of workers
|
||||
pub workers: Vec<WorkerInfo>,
|
||||
|
||||
/// Total count
|
||||
pub total: usize,
|
||||
|
||||
/// Statistics
|
||||
pub stats: WorkerStats,
|
||||
}
|
||||
|
||||
/// Worker statistics
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerStats {
|
||||
pub total_workers: usize,
|
||||
pub healthy_workers: usize,
|
||||
pub total_models: usize,
|
||||
pub total_load: usize,
|
||||
pub by_type: WorkerTypeStats,
|
||||
}
|
||||
|
||||
/// Worker statistics by type
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerTypeStats {
|
||||
pub regular: usize,
|
||||
pub prefill: usize,
|
||||
pub decode: usize,
|
||||
}
|
||||
|
||||
/// Worker update request
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct WorkerUpdateRequest {
|
||||
/// Update priority
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
/// Update cost
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
/// Update labels
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub labels: Option<HashMap<String, String>>,
|
||||
|
||||
/// Update API key (for key rotation)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub api_key: Option<String>,
|
||||
|
||||
/// Update health check timeout in seconds
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_check_timeout_secs: Option<u64>,
|
||||
|
||||
/// Update health check interval in seconds
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_check_interval_secs: Option<u64>,
|
||||
|
||||
/// Update health success threshold
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_success_threshold: Option<u32>,
|
||||
|
||||
/// Update health failure threshold
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub health_failure_threshold: Option<u32>,
|
||||
|
||||
/// Disable periodic health checks for this worker
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub disable_health_check: Option<bool>,
|
||||
}
|
||||
|
||||
/// Generic API response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerApiResponse {
|
||||
pub success: bool,
|
||||
pub message: String,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker: Option<WorkerInfo>,
|
||||
}
|
||||
|
||||
/// Error response
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct WorkerErrorResponse {
|
||||
pub error: String,
|
||||
pub code: String,
|
||||
}
|
||||
|
||||
/// Server info response from /get_server_info endpoint
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
pub struct ServerInfo {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_id: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub model_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub priority: Option<u32>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub cost: Option<f32>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
|
||||
// gRPC-specific
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tokenizer_path: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reasoning_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub tool_parser: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub chat_template: Option<String>,
|
||||
}
|
||||
|
||||
/// Result from flush cache operations across workers
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct FlushCacheResult {
|
||||
/// URLs of workers where cache flush succeeded
|
||||
pub successful: Vec<String>,
|
||||
/// URLs and error messages for workers where cache flush failed
|
||||
pub failed: Vec<(String, String)>,
|
||||
/// Total number of workers attempted
|
||||
pub total_workers: usize,
|
||||
/// Number of HTTP workers (gRPC workers don't support flush cache)
|
||||
pub http_workers: usize,
|
||||
/// Human-readable summary message
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
/// Result from getting worker loads
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerLoadsResult {
|
||||
/// Worker URL and load pairs
|
||||
pub loads: Vec<WorkerLoadInfo>,
|
||||
/// Total number of workers
|
||||
pub total_workers: usize,
|
||||
/// Number of workers with successful load fetches
|
||||
pub successful: usize,
|
||||
/// Number of workers with failed load fetches
|
||||
pub failed: usize,
|
||||
}
|
||||
|
||||
/// Individual worker load information
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub struct WorkerLoadInfo {
|
||||
/// Worker URL
|
||||
pub worker: String,
|
||||
/// Worker type (regular, prefill, decode)
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub worker_type: Option<String>,
|
||||
/// Current load (-1 indicates failure to fetch)
|
||||
pub load: isize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl IntoResponse for FlushCacheResult {
|
||||
fn into_response(self) -> Response {
|
||||
let status = if self.failed.is_empty() {
|
||||
StatusCode::OK
|
||||
} else {
|
||||
StatusCode::PARTIAL_CONTENT
|
||||
};
|
||||
|
||||
let mut body = json!({
|
||||
"status": if self.failed.is_empty() { "success" } else { "partial_success" },
|
||||
"message": self.message,
|
||||
"workers_flushed": self.successful.len(),
|
||||
"total_http_workers": self.http_workers,
|
||||
"total_workers": self.total_workers
|
||||
});
|
||||
|
||||
if !self.failed.is_empty() {
|
||||
body["successful"] = json!(self.successful);
|
||||
body["failed"] = json!(self
|
||||
.failed
|
||||
.into_iter()
|
||||
.map(|(url, err)| json!({"worker": url, "error": err}))
|
||||
.collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
(status, Json(body)).into_response()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "axum")]
|
||||
impl IntoResponse for WorkerLoadsResult {
|
||||
fn into_response(self) -> Response {
|
||||
let loads: Vec<Value> = self
|
||||
.loads
|
||||
.iter()
|
||||
.map(|info| json!({"worker": &info.worker, "load": info.load}))
|
||||
.collect();
|
||||
Json(json!({"workers": loads})).into_response()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user