Files

1424 lines
41 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// responses-shim: DeepSeek Responses API (/v1/responses) → OpenAI Chat Completions 协议转换 shim
// 后置部署:new-api 渠道上游指向本服务,本服务再转发到 sglang router。
// - POST /v1/responses → 翻译成 /v1/chat/completions 请求,响应/流式事件翻译回 Responses 协议
// - POST /v1/chat/completions → 透传(仅做 usage 归一化:sglang 顶层 reasoning_tokens
// 收编进 completion_tokens_details 标准位置;解析失败的报文回退字节级透传)
// - GET /healthz → 200
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。
// 超时:流式请求 300s 响应头超时(防上游假死);非流式不设上限——sglang 要等整段生成完才回
// 响应头,可能远超 300s(2026-10-09 用户指令),仅靠客户端断连回收。
package main
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"os"
"strings"
"sync/atomic"
"time"
)
var (
upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002")
listenAddr = envOr("LISTEN", ":8080")
// 流式:上游首字节即回响应头,300s ResponseHeaderTimeout 只防上游假死。
httpClientStream = newUpstreamClient(300 * time.Second)
// 非流式:sglang 要等整个生成结束才回响应头,生成可能远超 300s——不设上限(2026-10-09 用户指令),
// 仅靠客户端断连(r.Context())回收。
httpClientNonStream = newUpstreamClient(0)
idCounter int64
stopScope = envOr("STOP_SCOPE", "all")
)
func newUpstreamClient(respHeaderTimeout time.Duration) *http.Client {
return &http.Client{
Timeout: 0, // 生成可能很长,不设总超时
Transport: &http.Transport{
MaxIdleConns: 512,
MaxIdleConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: respHeaderTimeout,
},
}
}
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
const shimVersion = "v0.2.5"
func envOr(k, def string) string {
if v := os.Getenv(k); v != "" {
return v
}
return def
}
func nextID(prefix string) string {
return fmt.Sprintf("%s_%x%x", prefix, time.Now().UnixNano(), atomic.AddInt64(&idCounter, 1))
}
func main() {
mux := http.NewServeMux()
mux.HandleFunc("/v1/responses", handleResponses)
mux.HandleFunc("/v1/chat/completions", handlePassthrough)
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte("ok"))
})
log.Printf("responses-shim %s listening on %s, upstream=%s", shimVersion, listenAddr, upstreamBase)
log.Fatal(http.ListenAndServe(listenAddr, mux))
}
// ---------------- 公共:转发 ----------------
// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。
// streaming=true 走带 300s 响应头超时的 client(流式首包即回头,超时=上游假死);
// false 走无上限 client(非流式要等整段生成完才回头,不能设上限)。
func forwardToUpstream(r *http.Request, path string, body []byte, streaming bool) (*http.Response, error) {
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body))
if err != nil {
return nil, err
}
req.Header.Set("Content-Type", "application/json")
if auth := r.Header.Get("Authorization"); auth != "" {
req.Header.Set("Authorization", auth)
}
req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层
client := httpClientNonStream
if streaming {
client = httpClientStream
}
return client.Do(req)
}
// handlePassthrough 透传 /v1/chat/completions。
// STOP_SCOPE=content 且请求带 stop 时:stop 不下发上游,shim 只在 content 上匹配截断
// (思维链不受影响),命中后 finish_reason=stop 并回填 matched_stop。
func handlePassthrough(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
return
}
body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20))
if err != nil {
http.Error(w, `{"error":"read request failed"}`, http.StatusBadRequest)
return
}
stops, upstreamBody, stream := maybeStripStop(body)
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody, stream)
if err != nil {
http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway)
return
}
defer resp.Body.Close()
copyHeader(w.Header(), resp.Header)
if len(stops) == 0 || resp.StatusCode != http.StatusOK {
w.WriteHeader(resp.StatusCode)
if resp.StatusCode == http.StatusOK {
ct := resp.Header.Get("Content-Type")
switch {
case strings.Contains(ct, "text/event-stream"):
streamNormalizeUsage(w, resp.Body)
return
case strings.Contains(ct, "json"):
normalizeUsageJSON(w, resp.Body)
return
}
}
streamCopy(w, resp.Body)
return
}
if !stream {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
applyStopNonStream(w, resp.Body, stops)
return
}
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.WriteHeader(http.StatusOK)
applyStopStream(w, resp.Body, stops)
}
// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。
// stream 字段始终探测(非流式要选无超时上限的上游 client),与 stop 是否摘除无关。
func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) {
out = body
var req map[string]any
if err := json.Unmarshal(body, &req); err != nil {
return nil, out, false
}
stream, _ = req["stream"].(bool)
if stopScope != "content" {
return nil, out, stream
}
switch v := req["stop"].(type) {
case string:
if v != "" {
stops = []string{v}
}
case []any:
for _, s := range v {
if str, ok := s.(string); ok && str != "" {
stops = append(stops, str)
}
}
}
if len(stops) == 0 {
return nil, out, stream
}
delete(req, "stop")
if b, err := json.Marshal(req); err == nil {
out = b
}
return stops, out, stream
}
// earliestStop 返回 text 中最早命中的 stop 位置与串;未命中返回 -1。
func earliestStop(text string, stops []string) (int, string) {
best, hit := -1, ""
for _, s := range stops {
if i := strings.Index(text, s); i >= 0 && (best < 0 || i < best) {
best, hit = i, s
}
}
return best, hit
}
func maxStopLen(stops []string) int {
m := 0
for _, s := range stops {
if len(s) > m {
m = len(s)
}
}
return m
}
// truncateChoicesContent 截断每个 choice 的 content,命中则 finish_reason=stop + matched_stop。
func truncateChoicesContent(resp map[string]any, stops []string) {
choices, ok := resp["choices"].([]any)
if !ok {
return
}
for _, c := range choices {
cm, ok := c.(map[string]any)
if !ok {
continue
}
msg, ok := cm["message"].(map[string]any)
if !ok {
continue
}
content, _ := msg["content"].(string)
if i, hit := earliestStop(content, stops); i >= 0 {
msg["content"] = content[:i]
cm["finish_reason"] = "stop"
cm["matched_stop"] = hit
}
}
}
// applyStopNonStream 非流式:截断每个 choice 的 content,命中则 finish_reason=stop。
func applyStopNonStream(w http.ResponseWriter, src io.Reader, stops []string) {
var resp map[string]any
if err := json.NewDecoder(src).Decode(&resp); err != nil {
http.Error(w, `{"error":"decode upstream failed"}`, http.StatusBadGateway)
return
}
truncateChoicesContent(resp, stops)
normalizeUsage(resp)
_ = json.NewEncoder(w).Encode(resp)
}
// stopScanner 流式内容侧 stop 扫描器(每请求一个实例,无共享状态)。
type stopScanner struct {
w http.ResponseWriter
fl http.Flusher
stops []string
tailKeep int
pending strings.Builder
chunkID string
model string
finished bool
}
func (s *stopScanner) emitChunk(chunk map[string]any) {
b, err := json.Marshal(chunk)
if err != nil {
return
}
fmt.Fprintf(s.w, "data: %s\n\n", b)
if s.fl != nil {
s.fl.Flush()
}
}
func (s *stopScanner) emitRaw(line string) {
fmt.Fprintf(s.w, "%s\n\n", line)
if s.fl != nil {
s.fl.Flush()
}
}
// flushPending 把缓冲尾巴作为内容 chunk 发出(在 finish chunk 之前调用,保证顺序)。
func (s *stopScanner) flushPending() {
if s.pending.Len() == 0 {
return
}
content := s.pending.String()
s.pending.Reset()
s.emitChunk(map[string]any{
"id": s.chunkID, "object": "chat.completion.chunk", "model": s.model,
"created": time.Now().Unix(),
"choices": []any{map[string]any{
"index": 0, "delta": map[string]any{"content": content}, "finish_reason": nil,
}},
})
}
// feed 处理一个 chunk:content 入缓冲并吐出安全前缀;finish chunk 前先冲刷缓冲。
// 命中 stop 时返回命中串,chunk 的 content 已被截到命中点之前。
func (s *stopScanner) feed(chunk map[string]any) (matched string, hit bool) {
if id, ok := chunk["id"].(string); ok && id != "" {
s.chunkID = id
}
if m, ok := chunk["model"].(string); ok && m != "" {
s.model = m
}
choices, ok := chunk["choices"].([]any)
if !ok || len(choices) == 0 {
return "", false // usage-only chunk 等,原样透传
}
c, ok := choices[0].(map[string]any)
if !ok {
return "", false
}
// finish chunk:先把缓冲内容吐干净再放行
if fr, _ := c["finish_reason"].(string); fr != "" {
s.flushPending()
return "", false
}
delta, ok := c["delta"].(map[string]any)
if !ok {
return "", false
}
content, _ := delta["content"].(string)
if content != "" {
s.pending.WriteString(content)
}
full := s.pending.String()
if i, mstr := earliestStop(full, s.stops); i >= 0 {
safe := full[:i]
if safe != "" {
delta["content"] = safe
} else {
delete(delta, "content")
}
s.pending.Reset()
return mstr, true
}
// 未命中:吐出除尾巴外的安全前缀
safeLen := len(full) - s.tailKeep
if safeLen < 0 {
safeLen = 0
}
if safe := full[:safeLen]; safe != "" {
delta["content"] = safe
} else {
delete(delta, "content")
}
s.pending.Reset()
s.pending.WriteString(full[safeLen:])
return "", false
}
// emitFinish 命中 stop 后补发 finish chunk(带 matched_stop)+ [DONE]。
func (s *stopScanner) emitFinish(matched string) {
s.emitChunk(map[string]any{
"id": s.chunkID, "object": "chat.completion.chunk", "model": s.model,
"created": time.Now().Unix(),
"choices": []any{map[string]any{
"index": 0, "delta": map[string]any{}, "finish_reason": "stop", "matched_stop": matched,
}},
})
s.emitRaw("data: [DONE]")
}
// applyStopStream 流式:逐 chunk 扫描 content delta,跨 chunk 缓冲防边界漏配;
// 命中后补发 finish chunk + [DONE] 并断开上游(触发 sglang abort,省后续 token)。
// reasoning_content 不参与匹配,思维链完整生成。
func applyStopStream(w http.ResponseWriter, src io.Reader, stops []string) {
fl, _ := w.(http.Flusher)
sc := &stopScanner{w: w, fl: fl, stops: stops, tailKeep: maxStopLen(stops) - 1}
reader := bufio.NewReader(src)
matched := false
var matchStr string
for {
line, err := reader.ReadString('\n')
trimmed := strings.TrimRight(line, "\r\n")
if trimmed != "" && strings.HasPrefix(trimmed, "data:") {
data := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))
if data == "[DONE]" {
break
}
var chunk map[string]any
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
mstr, hit := sc.feed(chunk)
normalizeUsage(chunk)
sc.emitChunk(chunk)
if hit {
matched, matchStr = true, mstr
break
}
} else {
sc.emitRaw(trimmed) // 解析失败的行原样透传
}
} else if trimmed != "" {
sc.emitRaw(trimmed)
}
if err != nil {
break
}
}
if matched {
sc.emitFinish(matchStr)
return // 断开上游:defer resp.Body.Close() 触发 abort
}
// 自然结束(上游 [DONE] 或 EOF):flush 残余缓冲 + [DONE]
sc.flushPending()
sc.emitRaw("data: [DONE]")
}
func copyHeader(dst, src http.Header) {
for _, k := range []string{"Content-Type", "Cache-Control"} {
if v := src.Get(k); v != "" {
dst.Set(k, v)
}
}
}
// streamCopy 边读边写边 flush,SSE 友好。
func streamCopy(w http.ResponseWriter, src io.Reader) {
fl, _ := w.(http.Flusher)
buf := make([]byte, 32<<10)
for {
n, err := src.Read(buf)
if n > 0 {
_, _ = w.Write(buf[:n])
if fl != nil {
fl.Flush()
}
}
if err != nil {
return
}
}
}
// ---------------- 公共:usage 归一化 ----------------
// normalizeUsage 把 sglang 放在 usage 顶层的 reasoning_tokens 收编进
// completion_tokens_details(OpenAI 标准位置;new-api 等下游只认标准位置,
// 否则 reasoning_tokens 恒显示 0)。只增不改:顶层原字段保留。
func normalizeUsage(m map[string]any) {
usage, ok := m["usage"].(map[string]any)
if !ok {
return
}
rt, ok := usage["reasoning_tokens"].(float64)
if !ok || rt <= 0 {
return
}
ctd, _ := usage["completion_tokens_details"].(map[string]any)
if ctd == nil {
ctd = map[string]any{}
usage["completion_tokens_details"] = ctd
}
if existing, _ := ctd["reasoning_tokens"].(float64); existing <= 0 {
ctd["reasoning_tokens"] = rt
}
}
// normalizeUsageJSON 非流式透传:仅归一化 usage 后写回;解析失败回退字节透传。
func normalizeUsageJSON(w http.ResponseWriter, src io.Reader) {
body, err := io.ReadAll(src)
if err != nil {
return
}
var m map[string]any
if err := json.Unmarshal(body, &m); err != nil {
_, _ = w.Write(body)
return
}
normalizeUsage(m)
out, err := json.Marshal(m)
if err != nil {
_, _ = w.Write(body)
return
}
_, _ = w.Write(out)
}
// streamNormalizeUsage 流式透传:逐行扫描 SSE,含 usage 对象的 data 帧归一化后
// 重写,其余行原样透传(保序保边界);解析失败的帧同样原样透传。
func streamNormalizeUsage(w http.ResponseWriter, src io.Reader) {
fl, _ := w.(http.Flusher)
reader := bufio.NewReader(src)
for {
line, err := reader.ReadString('\n')
trimmed := strings.TrimRight(line, "\r\n")
out := line
if strings.HasPrefix(trimmed, "data:") {
data := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))
if data != "[DONE]" {
var chunk map[string]any
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
if _, has := chunk["usage"].(map[string]any); has {
normalizeUsage(chunk)
if b, merr := json.Marshal(chunk); merr == nil {
out = "data: " + string(b) + "\n"
}
}
}
}
}
_, _ = w.Write([]byte(out))
if fl != nil {
fl.Flush()
}
if err != nil {
return
}
}
}
// ---------------- Responses → Chat 请求翻译 ----------------
type responsesRequest struct {
Model string `json:"model"`
Input json.RawMessage `json:"input"`
Instructions string `json:"instructions"`
Reasoning *struct {
Effort string `json:"effort"`
} `json:"reasoning"`
MaxOutputTokens *int `json:"max_output_tokens"`
Temperature *float64 `json:"temperature"`
TopP *float64 `json:"top_p"`
Stream bool `json:"stream"`
Text *struct {
Format *struct {
Type string `json:"type"`
Name string `json:"name"`
Schema json.RawMessage `json:"schema"`
Strict *bool `json:"strict"`
} `json:"format"`
} `json:"text"`
Tools []map[string]any `json:"tools"`
ToolChoice json.RawMessage `json:"tool_choice"`
TopLogprobs *int `json:"top_logprobs"`
User string `json:"user"`
Stop json.RawMessage `json:"stop"`
}
// parseStops 解析 stop 的 string | string[] 两种形态。
func parseStops(raw json.RawMessage) []string {
if len(raw) == 0 || string(raw) == "null" {
return nil
}
var s string
if err := json.Unmarshal(raw, &s); err == nil {
if s != "" {
return []string{s}
}
return nil
}
var arr []string
if err := json.Unmarshal(raw, &arr); err == nil {
var out []string
for _, v := range arr {
if v != "" {
out = append(out, v)
}
}
return out
}
return nil
}
// effort 别名归一(DeepSeek 官方映射表):minimal→low,medium/xhigh→high,ultra→max
func normalizeEffort(e string) string {
switch e {
case "minimal":
return "low"
case "medium", "xhigh":
return "high"
case "ultra":
return "max"
default:
return e // none/low/high/max 及其他原样
}
}
func translateRequest(req *responsesRequest) (map[string]any, []string, error) {
out := map[string]any{"model": req.Model}
// messages
var messages []map[string]any
if req.Instructions != "" {
messages = append(messages, map[string]any{"role": "system", "content": req.Instructions})
}
if len(req.Input) > 0 && string(req.Input) != "null" {
ms, err := convertInput(req.Input)
if err != nil {
return nil, nil, err
}
messages = append(messages, ms...)
}
if len(messages) == 0 {
return nil, nil, fmt.Errorf("input 与 instructions 至少传一个")
}
out["messages"] = messages
// reasoning.effort → chat_template_kwargs(接 new-api/sglang 的思考开关语义)
if req.Reasoning != nil && req.Reasoning.Effort != "" {
effort := normalizeEffort(req.Reasoning.Effort)
ctk := map[string]any{}
if effort == "none" {
ctk["thinking"] = false
} else {
ctk["thinking"] = true
ctk["reasoning_effort"] = effort
}
out["chat_template_kwargs"] = ctk
}
if req.MaxOutputTokens != nil {
out["max_tokens"] = *req.MaxOutputTokens
}
if req.Temperature != nil {
out["temperature"] = *req.Temperature
}
if req.TopP != nil {
out["top_p"] = *req.TopP
}
if req.Stream {
out["stream"] = true
// 必须让上游回 usage,否则 response.completed 里没有计数
out["stream_options"] = map[string]any{"include_usage": true}
}
if req.Text != nil && req.Text.Format != nil {
switch req.Text.Format.Type {
case "json_object":
out["response_format"] = map[string]any{"type": "json_object"}
case "json_schema":
js := map[string]any{"name": req.Text.Format.Name}
if len(req.Text.Format.Schema) > 0 {
var schema any
if err := json.Unmarshal(req.Text.Format.Schema, &schema); err == nil {
js["schema"] = schema
}
}
if req.Text.Format.Strict != nil {
js["strict"] = *req.Text.Format.Strict
}
out["response_format"] = map[string]any{"type": "json_schema", "json_schema": js}
}
}
if len(req.Tools) > 0 {
var tools []map[string]any
for _, t := range req.Tools {
if t["type"] != "function" {
continue // 内置工具类型忽略
}
name, _ := t["name"].(string)
if strings.TrimSpace(name) == "" {
continue // name 为空的 function 工具跳过(上游会 400)
}
fn := map[string]any{"name": name}
if d, ok := t["description"]; ok {
fn["description"] = d
}
if p, ok := t["parameters"]; ok {
fn["parameters"] = p
}
tools = append(tools, map[string]any{"type": "function", "function": fn})
}
if len(tools) > 0 {
out["tools"] = tools
}
}
if len(req.ToolChoice) > 0 && string(req.ToolChoice) != "null" {
var s string
if err := json.Unmarshal(req.ToolChoice, &s); err == nil {
out["tool_choice"] = s // none/auto/required
} else {
var obj map[string]any
if err := json.Unmarshal(req.ToolChoice, &obj); err == nil {
if obj["type"] == "function" {
out["tool_choice"] = map[string]any{
"type": "function",
"function": map[string]any{"name": obj["name"]},
}
}
}
}
}
if req.TopLogprobs != nil {
out["logprobs"] = true
out["top_logprobs"] = *req.TopLogprobs
}
if req.User != "" {
out["user"] = req.User
}
// stop:STOP_SCOPE=content 时不下发上游(由 shim 在 content 上截断,保思维链);
// 否则原生透传给上游。
stops := parseStops(req.Stop)
if len(stops) > 0 && stopScope != "content" {
out["stop"] = stops
}
return out, stops, nil
}
// convertInput 处理 input 的两种形态:纯字符串 或 item 列表。
func convertInput(raw json.RawMessage) ([]map[string]any, error) {
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return []map[string]any{{"role": "user", "content": s}}, nil
}
var items []map[string]json.RawMessage
if err := json.Unmarshal(raw, &items); err != nil {
return nil, fmt.Errorf("input 必须是字符串或 item 列表: %w", err)
}
var messages []map[string]any
for _, item := range items {
var typ string
_ = json.Unmarshal(item["type"], &typ)
var role string
_ = json.Unmarshal(item["role"], &role)
switch {
case typ == "message" || (typ == "" && role != ""):
if role == "developer" {
role = "user" // DeepSeek: developer 视同 user
}
if role == "" {
role = "user"
}
msg := map[string]any{"role": role}
if len(item["content"]) > 0 {
c, err := convertContent(item["content"], role)
if err != nil {
return nil, err
}
msg["content"] = c
}
messages = append(messages, msg)
case typ == "function_call" || typ == "custom_tool_call":
var callID, name, args string
_ = json.Unmarshal(item["call_id"], &callID)
_ = json.Unmarshal(item["name"], &name)
if typ == "custom_tool_call" {
_ = json.Unmarshal(item["input"], &args)
} else {
_ = json.Unmarshal(item["arguments"], &args)
}
if callID == "" {
callID = nextID("call")
}
messages = append(messages, map[string]any{
"role": "assistant",
"tool_calls": []map[string]any{{
"id": callID, "type": "function",
"function": map[string]any{"name": name, "arguments": args},
}},
})
case typ == "function_call_output" || typ == "custom_tool_call_output":
var callID string
_ = json.Unmarshal(item["call_id"], &callID)
var outputStr string
if len(item["output"]) > 0 {
if err := json.Unmarshal(item["output"], &outputStr); err != nil {
// output 可能是结构化内容(含 input_image 块),转成 text/image parts
parts, err := convertContent(item["output"], "tool")
if err == nil {
messages = append(messages, map[string]any{
"role": "tool", "tool_call_id": callID, "content": parts,
})
continue
}
outputStr = string(item["output"])
}
}
messages = append(messages, map[string]any{
"role": "tool", "tool_call_id": callID, "content": outputStr,
})
case typ == "reasoning":
// 历史思维链不回放进上下文(与 DeepSeek 无 tools 多轮语义一致)
continue
default:
continue // 其他类型忽略
}
}
return messages, nil
}
// convertContent 处理 content 的字符串/blocks 两种形态。
func convertContent(raw json.RawMessage, role string) (any, error) {
var s string
if err := json.Unmarshal(raw, &s); err == nil {
return s, nil
}
var blocks []map[string]json.RawMessage
if err := json.Unmarshal(raw, &blocks); err != nil {
return nil, fmt.Errorf("content 格式不支持: %w", err)
}
var parts []map[string]any
var textBuf strings.Builder
for _, b := range blocks {
var typ string
_ = json.Unmarshal(b["type"], &typ)
switch typ {
case "input_text", "output_text", "text":
var t string
_ = json.Unmarshal(b["text"], &t)
textBuf.WriteString(t)
case "input_image":
var url string
_ = json.Unmarshal(b["image_url"], &url)
parts = append(parts, map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}})
default:
continue
}
}
// 纯文本时退回字符串形态(兼容性最好)
if len(parts) == 0 {
return textBuf.String(), nil
}
if textBuf.Len() > 0 {
parts = append([]map[string]any{{"type": "text", "text": textBuf.String()}}, parts...)
}
return parts, nil
}
// ---------------- Chat → Responses 响应翻译 ----------------
func translateResponse(chatResp map[string]any, respID string) map[string]any {
output := []any{}
outputIdx := 0
finishReason := ""
model, _ := chatResp["model"].(string)
if choices, ok := chatResp["choices"].([]any); ok && len(choices) > 0 {
if c, ok := choices[0].(map[string]any); ok {
finishReason, _ = c["finish_reason"].(string)
if msg, ok := c["message"].(map[string]any); ok {
if rc, ok := msg["reasoning_content"].(string); ok && rc != "" {
output = append(output, map[string]any{
"id": nextID("rs"),
"type": "reasoning",
"summary": []any{map[string]any{"type": "summary_text", "text": rc}},
})
outputIdx++
}
if content, ok := msg["content"].(string); ok {
output = append(output, map[string]any{
"id": nextID("msg"),
"type": "message",
"role": "assistant",
"status": "completed",
"content": []any{map[string]any{
"type": "output_text", "text": content, "annotations": []any{},
}},
})
outputIdx++
}
if tcs, ok := msg["tool_calls"].([]any); ok {
for _, tc := range tcs {
if tcm, ok := tc.(map[string]any); ok {
fn, _ := tcm["function"].(map[string]any)
output = append(output, map[string]any{
"id": nextID("fc"),
"type": "function_call",
"call_id": tcm["id"],
"name": fn["name"],
"arguments": fn["arguments"],
"status": "completed",
})
outputIdx++
}
}
}
}
}
}
status := "completed"
var incompleteDetails any
if finishReason == "length" {
status = "incomplete"
incompleteDetails = map[string]any{"reason": "max_output_tokens"}
}
resp := map[string]any{
"id": respID,
"object": "response",
"created_at": time.Now().Unix(),
"status": status,
"model": model,
"output": output,
"incomplete_details": incompleteDetails,
}
if usage, ok := chatResp["usage"].(map[string]any); ok {
resp["usage"] = translateUsage(usage)
}
return resp
}
func translateUsage(u map[string]any) map[string]any {
out := map[string]any{}
if v, ok := u["prompt_tokens"]; ok {
out["input_tokens"] = v
}
if v, ok := u["completion_tokens"]; ok {
out["output_tokens"] = v
}
if v, ok := u["total_tokens"]; ok {
out["total_tokens"] = v
}
// details 透明透传(字段名按 Responses 协议改名,内部字段原样保留):
// prompt_tokens_details → input_tokens_details(含 cached_tokens 缓存命中数)
// completion_tokens_details → output_tokens_details(含 reasoning_tokens)
if d, ok := u["prompt_tokens_details"].(map[string]any); ok && len(d) > 0 {
out["input_tokens_details"] = d
}
outDetails := map[string]any{}
if d, ok := u["completion_tokens_details"].(map[string]any); ok {
for k, v := range d {
outDetails[k] = v
}
}
// sglang 把 reasoning_tokens 放在顶层而非 completion_tokens_details 里,收编进来
if v, ok := u["reasoning_tokens"]; ok {
outDetails["reasoning_tokens"] = v
}
if len(outDetails) > 0 {
out["output_tokens_details"] = outDetails
}
return out
}
// ---------------- 流式翻译状态机 ----------------
type toolState struct {
opened bool
itemID string
callID string
name string
outIdx int
args strings.Builder
}
type streamState struct {
w http.ResponseWriter
fl http.Flusher
seq int
respID string
model string
createdAt int64
started bool
nextOutIdx int
reasoningOpen bool
reasoningID string
reasoningIdx int
reasoningText strings.Builder
msgOpen bool
msgID string
msgIdx int
partOpen bool
text strings.Builder
tools map[int]*toolState
toolOrder []int
finishReason string
usage map[string]any
failed bool
}
func newStreamState(w http.ResponseWriter) *streamState {
fl, _ := w.(http.Flusher)
return &streamState{
w: w, fl: fl, respID: nextID("resp"),
createdAt: time.Now().Unix(),
tools: map[int]*toolState{},
}
}
func (s *streamState) emit(eventType string, payload map[string]any) {
s.seq++
payload["type"] = eventType
payload["sequence_number"] = s.seq
data, err := json.Marshal(payload)
if err != nil {
return
}
fmt.Fprintf(s.w, "event: %s\ndata: %s\n\n", eventType, data)
if s.fl != nil {
s.fl.Flush()
}
}
func (s *streamState) responseSkeleton(status string) map[string]any {
return map[string]any{
"id": s.respID, "object": "response", "created_at": s.createdAt,
"status": status, "model": s.model, "output": []any{},
}
}
func (s *streamState) ensureStarted() {
if s.started {
return
}
s.started = true
s.emit("response.created", map[string]any{"response": s.responseSkeleton("in_progress")})
s.emit("response.in_progress", map[string]any{"response": s.responseSkeleton("in_progress")})
}
func (s *streamState) openReasoning() {
if s.reasoningOpen {
return
}
s.reasoningOpen = true
s.reasoningID = nextID("rs")
s.reasoningIdx = s.nextOutIdx
s.nextOutIdx++
s.emit("response.output_item.added", map[string]any{
"output_index": s.reasoningIdx,
"item": map[string]any{"id": s.reasoningID, "type": "reasoning", "summary": []any{}},
})
s.emit("response.reasoning_summary_part.added", map[string]any{
"item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0,
"part": map[string]any{"type": "summary_text", "text": ""},
})
}
func (s *streamState) closeReasoning() {
if !s.reasoningOpen {
return
}
s.reasoningOpen = false
full := s.reasoningText.String()
s.emit("response.reasoning_summary_text.done", map[string]any{
"item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0, "text": full,
})
s.emit("response.reasoning_summary_part.done", map[string]any{
"item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0,
"part": map[string]any{"type": "summary_text", "text": full},
})
s.emit("response.output_item.done", map[string]any{
"output_index": s.reasoningIdx,
"item": map[string]any{"id": s.reasoningID, "type": "reasoning",
"summary": []any{map[string]any{"type": "summary_text", "text": full}}},
})
}
func (s *streamState) openMessage() {
if s.msgOpen {
return
}
s.closeReasoning()
s.msgOpen = true
s.msgID = nextID("msg")
s.msgIdx = s.nextOutIdx
s.nextOutIdx++
s.emit("response.output_item.added", map[string]any{
"output_index": s.msgIdx,
"item": map[string]any{"id": s.msgID, "type": "message", "role": "assistant",
"status": "in_progress", "content": []any{}},
})
s.partOpen = true
s.emit("response.content_part.added", map[string]any{
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0,
"part": map[string]any{"type": "output_text", "text": "", "annotations": []any{}},
})
}
func (s *streamState) closeMessage() {
if !s.msgOpen {
return
}
s.msgOpen = false
full := s.text.String()
s.emit("response.output_text.done", map[string]any{
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, "text": full,
})
s.emit("response.content_part.done", map[string]any{
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0,
"part": map[string]any{"type": "output_text", "text": full, "annotations": []any{}},
})
s.emit("response.output_item.done", map[string]any{
"output_index": s.msgIdx,
"item": map[string]any{"id": s.msgID, "type": "message", "role": "assistant",
"status": "completed",
"content": []any{map[string]any{"type": "output_text", "text": full, "annotations": []any{}}}},
})
}
func (s *streamState) openTool(idx int, callID, name string) *toolState {
ts, ok := s.tools[idx]
if !ok {
ts = &toolState{}
s.tools[idx] = ts
s.toolOrder = append(s.toolOrder, idx)
}
if ts.opened {
return ts
}
s.closeReasoning()
s.closeMessage()
ts.opened = true
ts.itemID = nextID("fc")
ts.callID = callID
ts.name = name
ts.outIdx = s.nextOutIdx
s.nextOutIdx++
s.emit("response.output_item.added", map[string]any{
"output_index": ts.outIdx,
"item": map[string]any{"id": ts.itemID, "type": "function_call",
"call_id": ts.callID, "name": ts.name, "arguments": "", "status": "in_progress"},
})
return ts
}
func (s *streamState) closeAll() {
s.closeReasoning()
s.closeMessage()
for _, idx := range s.toolOrder {
ts := s.tools[idx]
if !ts.opened {
continue
}
ts.opened = false
s.emit("response.function_call_arguments.done", map[string]any{
"item_id": ts.itemID, "output_index": ts.outIdx, "arguments": ts.args.String(),
})
s.emit("response.output_item.done", map[string]any{
"output_index": ts.outIdx,
"item": map[string]any{"id": ts.itemID, "type": "function_call",
"call_id": ts.callID, "name": ts.name, "arguments": ts.args.String(), "status": "completed"},
})
}
}
// buildFinalOutput 重建完整 output 数组(用于 response.completed/incomplete)。
func (s *streamState) buildFinalOutput() []any {
output := make([]any, s.nextOutIdx)
if s.reasoningID != "" {
output[s.reasoningIdx] = map[string]any{"id": s.reasoningID, "type": "reasoning",
"summary": []any{map[string]any{"type": "summary_text", "text": s.reasoningText.String()}}}
}
if s.msgID != "" {
output[s.msgIdx] = map[string]any{"id": s.msgID, "type": "message", "role": "assistant",
"status": "completed",
"content": []any{map[string]any{"type": "output_text", "text": s.text.String(), "annotations": []any{}}}}
}
for _, idx := range s.toolOrder {
ts := s.tools[idx]
output[ts.outIdx] = map[string]any{"id": ts.itemID, "type": "function_call",
"call_id": ts.callID, "name": ts.name, "arguments": ts.args.String(), "status": "completed"}
}
// 防御:去掉未填充槽位
compact := output[:0]
for _, o := range output {
if o != nil {
compact = append(compact, o)
}
}
return compact
}
func (s *streamState) finish() {
s.closeAll()
status := "completed"
event := "response.completed"
var incompleteDetails any
if s.finishReason == "length" {
status = "incomplete"
event = "response.incomplete"
incompleteDetails = map[string]any{"reason": "max_output_tokens"}
}
resp := s.responseSkeleton(status)
resp["output"] = s.buildFinalOutput()
resp["incomplete_details"] = incompleteDetails
if s.usage != nil {
resp["usage"] = translateUsage(s.usage)
}
s.emit(event, map[string]any{"response": resp})
}
func (s *streamState) fail(msg string) {
if s.failed {
return
}
s.failed = true
resp := s.responseSkeleton("failed")
resp["error"] = map[string]any{"message": msg}
s.emit("response.failed", map[string]any{"response": resp})
}
// processChunk 处理一条 chat.completion.chunk。
func (s *streamState) processChunk(chunk map[string]any) {
if m, ok := chunk["model"].(string); ok && m != "" {
s.model = m
}
if usage, ok := chunk["usage"].(map[string]any); ok && usage != nil {
s.usage = usage
}
choices, ok := chunk["choices"].([]any)
if !ok || len(choices) == 0 {
return
}
c, ok := choices[0].(map[string]any)
if !ok {
return
}
s.ensureStarted()
if delta, ok := c["delta"].(map[string]any); ok {
if rc, ok := delta["reasoning_content"].(string); ok && rc != "" {
s.openReasoning()
s.reasoningText.WriteString(rc)
s.emit("response.reasoning_summary_text.delta", map[string]any{
"item_id": s.reasoningID, "output_index": s.reasoningIdx,
"summary_index": 0, "delta": rc,
})
}
if content, ok := delta["content"].(string); ok && content != "" {
s.openMessage()
s.text.WriteString(content)
s.emit("response.output_text.delta", map[string]any{
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, "delta": content,
})
}
if tcs, ok := delta["tool_calls"].([]any); ok {
for _, tc := range tcs {
tcm, ok := tc.(map[string]any)
if !ok {
continue
}
idxF, _ := tcm["index"].(float64)
idx := int(idxF)
callID, _ := tcm["id"].(string)
fn, _ := tcm["function"].(map[string]any)
name, _ := fn["name"].(string)
ts := s.openTool(idx, callID, name)
if ts.callID == "" {
ts.callID = callID
}
if ts.name == "" {
ts.name = name
}
if args, ok := fn["arguments"].(string); ok && args != "" {
ts.args.WriteString(args)
s.emit("response.function_call_arguments.delta", map[string]any{
"item_id": ts.itemID, "output_index": ts.outIdx, "delta": args,
})
}
}
}
}
if fr, ok := c["finish_reason"].(string); ok && fr != "" {
s.finishReason = fr
}
}
// ---------------- /v1/responses 入口 ----------------
func handleResponses(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
return
}
body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20))
if err != nil {
writeResponsesError(w, http.StatusBadRequest, "read request failed")
return
}
var req responsesRequest
if err := json.Unmarshal(body, &req); err != nil {
writeResponsesError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
return
}
chatReq, stops, err := translateRequest(&req)
if err != nil {
writeResponsesError(w, http.StatusBadRequest, err.Error())
return
}
chatBody, err := json.Marshal(chatReq)
if err != nil {
writeResponsesError(w, http.StatusInternalServerError, "marshal failed")
return
}
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody, req.Stream)
if err != nil {
writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error())
return
}
defer upstream.Body.Close()
if upstream.StatusCode != http.StatusOK {
// 上游错误原样透传(sglang 的错误体本身就是 OpenAI 格式)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(upstream.StatusCode)
_, _ = io.Copy(w, upstream.Body)
return
}
if !req.Stream {
var chatResp map[string]any
if err := json.NewDecoder(upstream.Body).Decode(&chatResp); err != nil {
writeResponsesError(w, http.StatusBadGateway, "decode upstream response failed")
return
}
if len(stops) > 0 {
truncateChoicesContent(chatResp, stops)
}
respID := nextID("resp")
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(translateResponse(chatResp, respID))
return
}
// 流式:chat SSE → responses SSE
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.WriteHeader(http.StatusOK)
state := newStreamState(w)
scanner := bufio.NewReader(upstream.Body)
stopped := false
// 内容侧 stop 扫描(STOP_SCOPE=content 且带 stop 时):跨 chunk 缓冲,
// reasoning_content 不参与匹配;命中即截断并结束响应。
var pending strings.Builder
tailKeep := 0
if len(stops) > 0 {
tailKeep = maxStopLen(stops) - 1
}
flushTail := func() {
if pending.Len() == 0 {
return
}
tail := pending.String()
pending.Reset()
state.processChunk(map[string]any{
"choices": []any{map[string]any{"delta": map[string]any{"content": tail}}},
})
}
feed := func(chunk map[string]any) (hit bool) {
if len(stops) == 0 {
state.processChunk(chunk)
return false
}
choices, _ := chunk["choices"].([]any)
if len(choices) == 0 {
state.processChunk(chunk)
return false
}
c, _ := choices[0].(map[string]any)
if c == nil {
state.processChunk(chunk)
return false
}
// finish chunk:先把缓冲尾巴吐进状态机再放行
if fr, _ := c["finish_reason"].(string); fr != "" {
flushTail()
state.processChunk(chunk)
return false
}
delta, _ := c["delta"].(map[string]any)
if delta == nil {
state.processChunk(chunk)
return false
}
content, _ := delta["content"].(string)
if content != "" {
pending.WriteString(content)
}
full := pending.String()
if i, _ := earliestStop(full, stops); i >= 0 {
if safe := full[:i]; safe != "" {
delta["content"] = safe
} else {
delete(delta, "content")
}
pending.Reset()
state.processChunk(chunk)
return true
}
safeLen := len(full) - tailKeep
if safeLen < 0 {
safeLen = 0
}
if safe := full[:safeLen]; safe != "" {
delta["content"] = safe
} else {
delete(delta, "content")
}
pending.Reset()
pending.WriteString(full[safeLen:])
state.processChunk(chunk)
return false
}
for {
line, err := scanner.ReadString('\n')
if len(line) > 0 {
line = strings.TrimRight(line, "\r\n")
if strings.HasPrefix(line, "data:") {
data := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
if data == "[DONE]" {
break
}
var chunk map[string]any
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
if feed(chunk) {
stopped = true
break
}
}
}
}
if err != nil {
break
}
}
if !stopped {
flushTail() // 自然结束:残余缓冲进状态机
}
if state.started && !state.failed {
state.finish()
} else if !state.started {
state.fail("upstream closed stream without data")
}
}
func writeResponsesError(w http.ResponseWriter, code int, msg string) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
_ = json.NewEncoder(w).Encode(map[string]any{
"error": map[string]any{"message": msg, "type": "invalid_request_error"},
})
}