1424 lines
41 KiB
Go
1424 lines
41 KiB
Go
// responses-shim: DeepSeek Responses API (/v1/responses) → OpenAI Chat Completions 协议转换 shim
|
||
// 后置部署:new-api 渠道上游指向本服务,本服务再转发到 sglang router。
|
||
// - POST /v1/responses → 翻译成 /v1/chat/completions 请求,响应/流式事件翻译回 Responses 协议
|
||
// - POST /v1/chat/completions → 透传(仅做 usage 归一化:sglang 顶层 reasoning_tokens
|
||
// 收编进 completion_tokens_details 标准位置;解析失败的报文回退字节级透传)
|
||
// - GET /healthz → 200
|
||
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
|
||
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
|
||
// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。
|
||
// 超时:流式请求 300s 响应头超时(防上游假死);非流式不设上限——sglang 要等整段生成完才回
|
||
// 响应头,可能远超 300s(2026-10-09 用户指令),仅靠客户端断连回收。
|
||
package main
|
||
|
||
import (
|
||
"bufio"
|
||
"bytes"
|
||
"encoding/json"
|
||
"fmt"
|
||
"io"
|
||
"log"
|
||
"net/http"
|
||
"os"
|
||
"strings"
|
||
"sync/atomic"
|
||
"time"
|
||
)
|
||
|
||
var (
|
||
upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002")
|
||
listenAddr = envOr("LISTEN", ":8080")
|
||
// 流式:上游首字节即回响应头,300s ResponseHeaderTimeout 只防上游假死。
|
||
httpClientStream = newUpstreamClient(300 * time.Second)
|
||
// 非流式:sglang 要等整个生成结束才回响应头,生成可能远超 300s——不设上限(2026-10-09 用户指令),
|
||
// 仅靠客户端断连(r.Context())回收。
|
||
httpClientNonStream = newUpstreamClient(0)
|
||
idCounter int64
|
||
stopScope = envOr("STOP_SCOPE", "all")
|
||
)
|
||
|
||
func newUpstreamClient(respHeaderTimeout time.Duration) *http.Client {
|
||
return &http.Client{
|
||
Timeout: 0, // 生成可能很长,不设总超时
|
||
Transport: &http.Transport{
|
||
MaxIdleConns: 512,
|
||
MaxIdleConnsPerHost: 256,
|
||
IdleConnTimeout: 90 * time.Second,
|
||
ResponseHeaderTimeout: respHeaderTimeout,
|
||
},
|
||
}
|
||
}
|
||
|
||
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
|
||
const shimVersion = "v0.2.5"
|
||
|
||
func envOr(k, def string) string {
|
||
if v := os.Getenv(k); v != "" {
|
||
return v
|
||
}
|
||
return def
|
||
}
|
||
|
||
func nextID(prefix string) string {
|
||
return fmt.Sprintf("%s_%x%x", prefix, time.Now().UnixNano(), atomic.AddInt64(&idCounter, 1))
|
||
}
|
||
|
||
func main() {
|
||
mux := http.NewServeMux()
|
||
mux.HandleFunc("/v1/responses", handleResponses)
|
||
mux.HandleFunc("/v1/chat/completions", handlePassthrough)
|
||
mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
|
||
w.WriteHeader(http.StatusOK)
|
||
_, _ = w.Write([]byte("ok"))
|
||
})
|
||
log.Printf("responses-shim %s listening on %s, upstream=%s", shimVersion, listenAddr, upstreamBase)
|
||
log.Fatal(http.ListenAndServe(listenAddr, mux))
|
||
}
|
||
|
||
// ---------------- 公共:转发 ----------------
|
||
|
||
// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。
|
||
// streaming=true 走带 300s 响应头超时的 client(流式首包即回头,超时=上游假死);
|
||
// false 走无上限 client(非流式要等整段生成完才回头,不能设上限)。
|
||
func forwardToUpstream(r *http.Request, path string, body []byte, streaming bool) (*http.Response, error) {
|
||
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body))
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("Content-Type", "application/json")
|
||
if auth := r.Header.Get("Authorization"); auth != "" {
|
||
req.Header.Set("Authorization", auth)
|
||
}
|
||
req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层
|
||
client := httpClientNonStream
|
||
if streaming {
|
||
client = httpClientStream
|
||
}
|
||
return client.Do(req)
|
||
}
|
||
|
||
// handlePassthrough 透传 /v1/chat/completions。
|
||
// STOP_SCOPE=content 且请求带 stop 时:stop 不下发上游,shim 只在 content 上匹配截断
|
||
// (思维链不受影响),命中后 finish_reason=stop 并回填 matched_stop。
|
||
func handlePassthrough(w http.ResponseWriter, r *http.Request) {
|
||
if r.Method != http.MethodPost {
|
||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20))
|
||
if err != nil {
|
||
http.Error(w, `{"error":"read request failed"}`, http.StatusBadRequest)
|
||
return
|
||
}
|
||
|
||
stops, upstreamBody, stream := maybeStripStop(body)
|
||
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody, stream)
|
||
if err != nil {
|
||
http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway)
|
||
return
|
||
}
|
||
defer resp.Body.Close()
|
||
copyHeader(w.Header(), resp.Header)
|
||
|
||
if len(stops) == 0 || resp.StatusCode != http.StatusOK {
|
||
w.WriteHeader(resp.StatusCode)
|
||
if resp.StatusCode == http.StatusOK {
|
||
ct := resp.Header.Get("Content-Type")
|
||
switch {
|
||
case strings.Contains(ct, "text/event-stream"):
|
||
streamNormalizeUsage(w, resp.Body)
|
||
return
|
||
case strings.Contains(ct, "json"):
|
||
normalizeUsageJSON(w, resp.Body)
|
||
return
|
||
}
|
||
}
|
||
streamCopy(w, resp.Body)
|
||
return
|
||
}
|
||
if !stream {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(http.StatusOK)
|
||
applyStopNonStream(w, resp.Body, stops)
|
||
return
|
||
}
|
||
w.Header().Set("Content-Type", "text/event-stream")
|
||
w.Header().Set("Cache-Control", "no-cache")
|
||
w.WriteHeader(http.StatusOK)
|
||
applyStopStream(w, resp.Body, stops)
|
||
}
|
||
|
||
// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。
|
||
// stream 字段始终探测(非流式要选无超时上限的上游 client),与 stop 是否摘除无关。
|
||
func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) {
|
||
out = body
|
||
var req map[string]any
|
||
if err := json.Unmarshal(body, &req); err != nil {
|
||
return nil, out, false
|
||
}
|
||
stream, _ = req["stream"].(bool)
|
||
if stopScope != "content" {
|
||
return nil, out, stream
|
||
}
|
||
switch v := req["stop"].(type) {
|
||
case string:
|
||
if v != "" {
|
||
stops = []string{v}
|
||
}
|
||
case []any:
|
||
for _, s := range v {
|
||
if str, ok := s.(string); ok && str != "" {
|
||
stops = append(stops, str)
|
||
}
|
||
}
|
||
}
|
||
if len(stops) == 0 {
|
||
return nil, out, stream
|
||
}
|
||
delete(req, "stop")
|
||
if b, err := json.Marshal(req); err == nil {
|
||
out = b
|
||
}
|
||
return stops, out, stream
|
||
}
|
||
|
||
// earliestStop 返回 text 中最早命中的 stop 位置与串;未命中返回 -1。
|
||
func earliestStop(text string, stops []string) (int, string) {
|
||
best, hit := -1, ""
|
||
for _, s := range stops {
|
||
if i := strings.Index(text, s); i >= 0 && (best < 0 || i < best) {
|
||
best, hit = i, s
|
||
}
|
||
}
|
||
return best, hit
|
||
}
|
||
|
||
func maxStopLen(stops []string) int {
|
||
m := 0
|
||
for _, s := range stops {
|
||
if len(s) > m {
|
||
m = len(s)
|
||
}
|
||
}
|
||
return m
|
||
}
|
||
|
||
// truncateChoicesContent 截断每个 choice 的 content,命中则 finish_reason=stop + matched_stop。
|
||
func truncateChoicesContent(resp map[string]any, stops []string) {
|
||
choices, ok := resp["choices"].([]any)
|
||
if !ok {
|
||
return
|
||
}
|
||
for _, c := range choices {
|
||
cm, ok := c.(map[string]any)
|
||
if !ok {
|
||
continue
|
||
}
|
||
msg, ok := cm["message"].(map[string]any)
|
||
if !ok {
|
||
continue
|
||
}
|
||
content, _ := msg["content"].(string)
|
||
if i, hit := earliestStop(content, stops); i >= 0 {
|
||
msg["content"] = content[:i]
|
||
cm["finish_reason"] = "stop"
|
||
cm["matched_stop"] = hit
|
||
}
|
||
}
|
||
}
|
||
|
||
// applyStopNonStream 非流式:截断每个 choice 的 content,命中则 finish_reason=stop。
|
||
func applyStopNonStream(w http.ResponseWriter, src io.Reader, stops []string) {
|
||
var resp map[string]any
|
||
if err := json.NewDecoder(src).Decode(&resp); err != nil {
|
||
http.Error(w, `{"error":"decode upstream failed"}`, http.StatusBadGateway)
|
||
return
|
||
}
|
||
truncateChoicesContent(resp, stops)
|
||
normalizeUsage(resp)
|
||
_ = json.NewEncoder(w).Encode(resp)
|
||
}
|
||
|
||
// stopScanner 流式内容侧 stop 扫描器(每请求一个实例,无共享状态)。
|
||
type stopScanner struct {
|
||
w http.ResponseWriter
|
||
fl http.Flusher
|
||
stops []string
|
||
tailKeep int
|
||
pending strings.Builder
|
||
chunkID string
|
||
model string
|
||
finished bool
|
||
}
|
||
|
||
func (s *stopScanner) emitChunk(chunk map[string]any) {
|
||
b, err := json.Marshal(chunk)
|
||
if err != nil {
|
||
return
|
||
}
|
||
fmt.Fprintf(s.w, "data: %s\n\n", b)
|
||
if s.fl != nil {
|
||
s.fl.Flush()
|
||
}
|
||
}
|
||
|
||
func (s *stopScanner) emitRaw(line string) {
|
||
fmt.Fprintf(s.w, "%s\n\n", line)
|
||
if s.fl != nil {
|
||
s.fl.Flush()
|
||
}
|
||
}
|
||
|
||
// flushPending 把缓冲尾巴作为内容 chunk 发出(在 finish chunk 之前调用,保证顺序)。
|
||
func (s *stopScanner) flushPending() {
|
||
if s.pending.Len() == 0 {
|
||
return
|
||
}
|
||
content := s.pending.String()
|
||
s.pending.Reset()
|
||
s.emitChunk(map[string]any{
|
||
"id": s.chunkID, "object": "chat.completion.chunk", "model": s.model,
|
||
"created": time.Now().Unix(),
|
||
"choices": []any{map[string]any{
|
||
"index": 0, "delta": map[string]any{"content": content}, "finish_reason": nil,
|
||
}},
|
||
})
|
||
}
|
||
|
||
// feed 处理一个 chunk:content 入缓冲并吐出安全前缀;finish chunk 前先冲刷缓冲。
|
||
// 命中 stop 时返回命中串,chunk 的 content 已被截到命中点之前。
|
||
func (s *stopScanner) feed(chunk map[string]any) (matched string, hit bool) {
|
||
if id, ok := chunk["id"].(string); ok && id != "" {
|
||
s.chunkID = id
|
||
}
|
||
if m, ok := chunk["model"].(string); ok && m != "" {
|
||
s.model = m
|
||
}
|
||
choices, ok := chunk["choices"].([]any)
|
||
if !ok || len(choices) == 0 {
|
||
return "", false // usage-only chunk 等,原样透传
|
||
}
|
||
c, ok := choices[0].(map[string]any)
|
||
if !ok {
|
||
return "", false
|
||
}
|
||
// finish chunk:先把缓冲内容吐干净再放行
|
||
if fr, _ := c["finish_reason"].(string); fr != "" {
|
||
s.flushPending()
|
||
return "", false
|
||
}
|
||
delta, ok := c["delta"].(map[string]any)
|
||
if !ok {
|
||
return "", false
|
||
}
|
||
content, _ := delta["content"].(string)
|
||
if content != "" {
|
||
s.pending.WriteString(content)
|
||
}
|
||
full := s.pending.String()
|
||
if i, mstr := earliestStop(full, s.stops); i >= 0 {
|
||
safe := full[:i]
|
||
if safe != "" {
|
||
delta["content"] = safe
|
||
} else {
|
||
delete(delta, "content")
|
||
}
|
||
s.pending.Reset()
|
||
return mstr, true
|
||
}
|
||
// 未命中:吐出除尾巴外的安全前缀
|
||
safeLen := len(full) - s.tailKeep
|
||
if safeLen < 0 {
|
||
safeLen = 0
|
||
}
|
||
if safe := full[:safeLen]; safe != "" {
|
||
delta["content"] = safe
|
||
} else {
|
||
delete(delta, "content")
|
||
}
|
||
s.pending.Reset()
|
||
s.pending.WriteString(full[safeLen:])
|
||
return "", false
|
||
}
|
||
|
||
// emitFinish 命中 stop 后补发 finish chunk(带 matched_stop)+ [DONE]。
|
||
func (s *stopScanner) emitFinish(matched string) {
|
||
s.emitChunk(map[string]any{
|
||
"id": s.chunkID, "object": "chat.completion.chunk", "model": s.model,
|
||
"created": time.Now().Unix(),
|
||
"choices": []any{map[string]any{
|
||
"index": 0, "delta": map[string]any{}, "finish_reason": "stop", "matched_stop": matched,
|
||
}},
|
||
})
|
||
s.emitRaw("data: [DONE]")
|
||
}
|
||
|
||
// applyStopStream 流式:逐 chunk 扫描 content delta,跨 chunk 缓冲防边界漏配;
|
||
// 命中后补发 finish chunk + [DONE] 并断开上游(触发 sglang abort,省后续 token)。
|
||
// reasoning_content 不参与匹配,思维链完整生成。
|
||
func applyStopStream(w http.ResponseWriter, src io.Reader, stops []string) {
|
||
fl, _ := w.(http.Flusher)
|
||
sc := &stopScanner{w: w, fl: fl, stops: stops, tailKeep: maxStopLen(stops) - 1}
|
||
|
||
reader := bufio.NewReader(src)
|
||
matched := false
|
||
var matchStr string
|
||
for {
|
||
line, err := reader.ReadString('\n')
|
||
trimmed := strings.TrimRight(line, "\r\n")
|
||
if trimmed != "" && strings.HasPrefix(trimmed, "data:") {
|
||
data := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))
|
||
if data == "[DONE]" {
|
||
break
|
||
}
|
||
var chunk map[string]any
|
||
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
|
||
mstr, hit := sc.feed(chunk)
|
||
normalizeUsage(chunk)
|
||
sc.emitChunk(chunk)
|
||
if hit {
|
||
matched, matchStr = true, mstr
|
||
break
|
||
}
|
||
} else {
|
||
sc.emitRaw(trimmed) // 解析失败的行原样透传
|
||
}
|
||
} else if trimmed != "" {
|
||
sc.emitRaw(trimmed)
|
||
}
|
||
if err != nil {
|
||
break
|
||
}
|
||
}
|
||
|
||
if matched {
|
||
sc.emitFinish(matchStr)
|
||
return // 断开上游:defer resp.Body.Close() 触发 abort
|
||
}
|
||
// 自然结束(上游 [DONE] 或 EOF):flush 残余缓冲 + [DONE]
|
||
sc.flushPending()
|
||
sc.emitRaw("data: [DONE]")
|
||
}
|
||
|
||
func copyHeader(dst, src http.Header) {
|
||
for _, k := range []string{"Content-Type", "Cache-Control"} {
|
||
if v := src.Get(k); v != "" {
|
||
dst.Set(k, v)
|
||
}
|
||
}
|
||
}
|
||
|
||
// streamCopy 边读边写边 flush,SSE 友好。
|
||
func streamCopy(w http.ResponseWriter, src io.Reader) {
|
||
fl, _ := w.(http.Flusher)
|
||
buf := make([]byte, 32<<10)
|
||
for {
|
||
n, err := src.Read(buf)
|
||
if n > 0 {
|
||
_, _ = w.Write(buf[:n])
|
||
if fl != nil {
|
||
fl.Flush()
|
||
}
|
||
}
|
||
if err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------- 公共:usage 归一化 ----------------
|
||
|
||
// normalizeUsage 把 sglang 放在 usage 顶层的 reasoning_tokens 收编进
|
||
// completion_tokens_details(OpenAI 标准位置;new-api 等下游只认标准位置,
|
||
// 否则 reasoning_tokens 恒显示 0)。只增不改:顶层原字段保留。
|
||
func normalizeUsage(m map[string]any) {
|
||
usage, ok := m["usage"].(map[string]any)
|
||
if !ok {
|
||
return
|
||
}
|
||
rt, ok := usage["reasoning_tokens"].(float64)
|
||
if !ok || rt <= 0 {
|
||
return
|
||
}
|
||
ctd, _ := usage["completion_tokens_details"].(map[string]any)
|
||
if ctd == nil {
|
||
ctd = map[string]any{}
|
||
usage["completion_tokens_details"] = ctd
|
||
}
|
||
if existing, _ := ctd["reasoning_tokens"].(float64); existing <= 0 {
|
||
ctd["reasoning_tokens"] = rt
|
||
}
|
||
}
|
||
|
||
// normalizeUsageJSON 非流式透传:仅归一化 usage 后写回;解析失败回退字节透传。
|
||
func normalizeUsageJSON(w http.ResponseWriter, src io.Reader) {
|
||
body, err := io.ReadAll(src)
|
||
if err != nil {
|
||
return
|
||
}
|
||
var m map[string]any
|
||
if err := json.Unmarshal(body, &m); err != nil {
|
||
_, _ = w.Write(body)
|
||
return
|
||
}
|
||
normalizeUsage(m)
|
||
out, err := json.Marshal(m)
|
||
if err != nil {
|
||
_, _ = w.Write(body)
|
||
return
|
||
}
|
||
_, _ = w.Write(out)
|
||
}
|
||
|
||
// streamNormalizeUsage 流式透传:逐行扫描 SSE,含 usage 对象的 data 帧归一化后
|
||
// 重写,其余行原样透传(保序保边界);解析失败的帧同样原样透传。
|
||
func streamNormalizeUsage(w http.ResponseWriter, src io.Reader) {
|
||
fl, _ := w.(http.Flusher)
|
||
reader := bufio.NewReader(src)
|
||
for {
|
||
line, err := reader.ReadString('\n')
|
||
trimmed := strings.TrimRight(line, "\r\n")
|
||
out := line
|
||
if strings.HasPrefix(trimmed, "data:") {
|
||
data := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))
|
||
if data != "[DONE]" {
|
||
var chunk map[string]any
|
||
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
|
||
if _, has := chunk["usage"].(map[string]any); has {
|
||
normalizeUsage(chunk)
|
||
if b, merr := json.Marshal(chunk); merr == nil {
|
||
out = "data: " + string(b) + "\n"
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
_, _ = w.Write([]byte(out))
|
||
if fl != nil {
|
||
fl.Flush()
|
||
}
|
||
if err != nil {
|
||
return
|
||
}
|
||
}
|
||
}
|
||
|
||
// ---------------- Responses → Chat 请求翻译 ----------------
|
||
|
||
type responsesRequest struct {
|
||
Model string `json:"model"`
|
||
Input json.RawMessage `json:"input"`
|
||
Instructions string `json:"instructions"`
|
||
Reasoning *struct {
|
||
Effort string `json:"effort"`
|
||
} `json:"reasoning"`
|
||
MaxOutputTokens *int `json:"max_output_tokens"`
|
||
Temperature *float64 `json:"temperature"`
|
||
TopP *float64 `json:"top_p"`
|
||
Stream bool `json:"stream"`
|
||
Text *struct {
|
||
Format *struct {
|
||
Type string `json:"type"`
|
||
Name string `json:"name"`
|
||
Schema json.RawMessage `json:"schema"`
|
||
Strict *bool `json:"strict"`
|
||
} `json:"format"`
|
||
} `json:"text"`
|
||
Tools []map[string]any `json:"tools"`
|
||
ToolChoice json.RawMessage `json:"tool_choice"`
|
||
TopLogprobs *int `json:"top_logprobs"`
|
||
User string `json:"user"`
|
||
Stop json.RawMessage `json:"stop"`
|
||
}
|
||
|
||
// parseStops 解析 stop 的 string | string[] 两种形态。
|
||
func parseStops(raw json.RawMessage) []string {
|
||
if len(raw) == 0 || string(raw) == "null" {
|
||
return nil
|
||
}
|
||
var s string
|
||
if err := json.Unmarshal(raw, &s); err == nil {
|
||
if s != "" {
|
||
return []string{s}
|
||
}
|
||
return nil
|
||
}
|
||
var arr []string
|
||
if err := json.Unmarshal(raw, &arr); err == nil {
|
||
var out []string
|
||
for _, v := range arr {
|
||
if v != "" {
|
||
out = append(out, v)
|
||
}
|
||
}
|
||
return out
|
||
}
|
||
return nil
|
||
}
|
||
|
||
// effort 别名归一(DeepSeek 官方映射表):minimal→low,medium/xhigh→high,ultra→max
|
||
func normalizeEffort(e string) string {
|
||
switch e {
|
||
case "minimal":
|
||
return "low"
|
||
case "medium", "xhigh":
|
||
return "high"
|
||
case "ultra":
|
||
return "max"
|
||
default:
|
||
return e // none/low/high/max 及其他原样
|
||
}
|
||
}
|
||
|
||
func translateRequest(req *responsesRequest) (map[string]any, []string, error) {
|
||
out := map[string]any{"model": req.Model}
|
||
|
||
// messages
|
||
var messages []map[string]any
|
||
if req.Instructions != "" {
|
||
messages = append(messages, map[string]any{"role": "system", "content": req.Instructions})
|
||
}
|
||
if len(req.Input) > 0 && string(req.Input) != "null" {
|
||
ms, err := convertInput(req.Input)
|
||
if err != nil {
|
||
return nil, nil, err
|
||
}
|
||
messages = append(messages, ms...)
|
||
}
|
||
if len(messages) == 0 {
|
||
return nil, nil, fmt.Errorf("input 与 instructions 至少传一个")
|
||
}
|
||
out["messages"] = messages
|
||
|
||
// reasoning.effort → chat_template_kwargs(接 new-api/sglang 的思考开关语义)
|
||
if req.Reasoning != nil && req.Reasoning.Effort != "" {
|
||
effort := normalizeEffort(req.Reasoning.Effort)
|
||
ctk := map[string]any{}
|
||
if effort == "none" {
|
||
ctk["thinking"] = false
|
||
} else {
|
||
ctk["thinking"] = true
|
||
ctk["reasoning_effort"] = effort
|
||
}
|
||
out["chat_template_kwargs"] = ctk
|
||
}
|
||
|
||
if req.MaxOutputTokens != nil {
|
||
out["max_tokens"] = *req.MaxOutputTokens
|
||
}
|
||
if req.Temperature != nil {
|
||
out["temperature"] = *req.Temperature
|
||
}
|
||
if req.TopP != nil {
|
||
out["top_p"] = *req.TopP
|
||
}
|
||
if req.Stream {
|
||
out["stream"] = true
|
||
// 必须让上游回 usage,否则 response.completed 里没有计数
|
||
out["stream_options"] = map[string]any{"include_usage": true}
|
||
}
|
||
if req.Text != nil && req.Text.Format != nil {
|
||
switch req.Text.Format.Type {
|
||
case "json_object":
|
||
out["response_format"] = map[string]any{"type": "json_object"}
|
||
case "json_schema":
|
||
js := map[string]any{"name": req.Text.Format.Name}
|
||
if len(req.Text.Format.Schema) > 0 {
|
||
var schema any
|
||
if err := json.Unmarshal(req.Text.Format.Schema, &schema); err == nil {
|
||
js["schema"] = schema
|
||
}
|
||
}
|
||
if req.Text.Format.Strict != nil {
|
||
js["strict"] = *req.Text.Format.Strict
|
||
}
|
||
out["response_format"] = map[string]any{"type": "json_schema", "json_schema": js}
|
||
}
|
||
}
|
||
if len(req.Tools) > 0 {
|
||
var tools []map[string]any
|
||
for _, t := range req.Tools {
|
||
if t["type"] != "function" {
|
||
continue // 内置工具类型忽略
|
||
}
|
||
name, _ := t["name"].(string)
|
||
if strings.TrimSpace(name) == "" {
|
||
continue // name 为空的 function 工具跳过(上游会 400)
|
||
}
|
||
fn := map[string]any{"name": name}
|
||
if d, ok := t["description"]; ok {
|
||
fn["description"] = d
|
||
}
|
||
if p, ok := t["parameters"]; ok {
|
||
fn["parameters"] = p
|
||
}
|
||
tools = append(tools, map[string]any{"type": "function", "function": fn})
|
||
}
|
||
if len(tools) > 0 {
|
||
out["tools"] = tools
|
||
}
|
||
}
|
||
if len(req.ToolChoice) > 0 && string(req.ToolChoice) != "null" {
|
||
var s string
|
||
if err := json.Unmarshal(req.ToolChoice, &s); err == nil {
|
||
out["tool_choice"] = s // none/auto/required
|
||
} else {
|
||
var obj map[string]any
|
||
if err := json.Unmarshal(req.ToolChoice, &obj); err == nil {
|
||
if obj["type"] == "function" {
|
||
out["tool_choice"] = map[string]any{
|
||
"type": "function",
|
||
"function": map[string]any{"name": obj["name"]},
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if req.TopLogprobs != nil {
|
||
out["logprobs"] = true
|
||
out["top_logprobs"] = *req.TopLogprobs
|
||
}
|
||
if req.User != "" {
|
||
out["user"] = req.User
|
||
}
|
||
// stop:STOP_SCOPE=content 时不下发上游(由 shim 在 content 上截断,保思维链);
|
||
// 否则原生透传给上游。
|
||
stops := parseStops(req.Stop)
|
||
if len(stops) > 0 && stopScope != "content" {
|
||
out["stop"] = stops
|
||
}
|
||
return out, stops, nil
|
||
}
|
||
|
||
// convertInput 处理 input 的两种形态:纯字符串 或 item 列表。
|
||
func convertInput(raw json.RawMessage) ([]map[string]any, error) {
|
||
var s string
|
||
if err := json.Unmarshal(raw, &s); err == nil {
|
||
return []map[string]any{{"role": "user", "content": s}}, nil
|
||
}
|
||
var items []map[string]json.RawMessage
|
||
if err := json.Unmarshal(raw, &items); err != nil {
|
||
return nil, fmt.Errorf("input 必须是字符串或 item 列表: %w", err)
|
||
}
|
||
var messages []map[string]any
|
||
for _, item := range items {
|
||
var typ string
|
||
_ = json.Unmarshal(item["type"], &typ)
|
||
var role string
|
||
_ = json.Unmarshal(item["role"], &role)
|
||
|
||
switch {
|
||
case typ == "message" || (typ == "" && role != ""):
|
||
if role == "developer" {
|
||
role = "user" // DeepSeek: developer 视同 user
|
||
}
|
||
if role == "" {
|
||
role = "user"
|
||
}
|
||
msg := map[string]any{"role": role}
|
||
if len(item["content"]) > 0 {
|
||
c, err := convertContent(item["content"], role)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
msg["content"] = c
|
||
}
|
||
messages = append(messages, msg)
|
||
|
||
case typ == "function_call" || typ == "custom_tool_call":
|
||
var callID, name, args string
|
||
_ = json.Unmarshal(item["call_id"], &callID)
|
||
_ = json.Unmarshal(item["name"], &name)
|
||
if typ == "custom_tool_call" {
|
||
_ = json.Unmarshal(item["input"], &args)
|
||
} else {
|
||
_ = json.Unmarshal(item["arguments"], &args)
|
||
}
|
||
if callID == "" {
|
||
callID = nextID("call")
|
||
}
|
||
messages = append(messages, map[string]any{
|
||
"role": "assistant",
|
||
"tool_calls": []map[string]any{{
|
||
"id": callID, "type": "function",
|
||
"function": map[string]any{"name": name, "arguments": args},
|
||
}},
|
||
})
|
||
|
||
case typ == "function_call_output" || typ == "custom_tool_call_output":
|
||
var callID string
|
||
_ = json.Unmarshal(item["call_id"], &callID)
|
||
var outputStr string
|
||
if len(item["output"]) > 0 {
|
||
if err := json.Unmarshal(item["output"], &outputStr); err != nil {
|
||
// output 可能是结构化内容(含 input_image 块),转成 text/image parts
|
||
parts, err := convertContent(item["output"], "tool")
|
||
if err == nil {
|
||
messages = append(messages, map[string]any{
|
||
"role": "tool", "tool_call_id": callID, "content": parts,
|
||
})
|
||
continue
|
||
}
|
||
outputStr = string(item["output"])
|
||
}
|
||
}
|
||
messages = append(messages, map[string]any{
|
||
"role": "tool", "tool_call_id": callID, "content": outputStr,
|
||
})
|
||
|
||
case typ == "reasoning":
|
||
// 历史思维链不回放进上下文(与 DeepSeek 无 tools 多轮语义一致)
|
||
continue
|
||
default:
|
||
continue // 其他类型忽略
|
||
}
|
||
}
|
||
return messages, nil
|
||
}
|
||
|
||
// convertContent 处理 content 的字符串/blocks 两种形态。
|
||
func convertContent(raw json.RawMessage, role string) (any, error) {
|
||
var s string
|
||
if err := json.Unmarshal(raw, &s); err == nil {
|
||
return s, nil
|
||
}
|
||
var blocks []map[string]json.RawMessage
|
||
if err := json.Unmarshal(raw, &blocks); err != nil {
|
||
return nil, fmt.Errorf("content 格式不支持: %w", err)
|
||
}
|
||
var parts []map[string]any
|
||
var textBuf strings.Builder
|
||
for _, b := range blocks {
|
||
var typ string
|
||
_ = json.Unmarshal(b["type"], &typ)
|
||
switch typ {
|
||
case "input_text", "output_text", "text":
|
||
var t string
|
||
_ = json.Unmarshal(b["text"], &t)
|
||
textBuf.WriteString(t)
|
||
case "input_image":
|
||
var url string
|
||
_ = json.Unmarshal(b["image_url"], &url)
|
||
parts = append(parts, map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}})
|
||
default:
|
||
continue
|
||
}
|
||
}
|
||
// 纯文本时退回字符串形态(兼容性最好)
|
||
if len(parts) == 0 {
|
||
return textBuf.String(), nil
|
||
}
|
||
if textBuf.Len() > 0 {
|
||
parts = append([]map[string]any{{"type": "text", "text": textBuf.String()}}, parts...)
|
||
}
|
||
return parts, nil
|
||
}
|
||
|
||
// ---------------- Chat → Responses 响应翻译 ----------------
|
||
|
||
func translateResponse(chatResp map[string]any, respID string) map[string]any {
|
||
output := []any{}
|
||
outputIdx := 0
|
||
finishReason := ""
|
||
model, _ := chatResp["model"].(string)
|
||
|
||
if choices, ok := chatResp["choices"].([]any); ok && len(choices) > 0 {
|
||
if c, ok := choices[0].(map[string]any); ok {
|
||
finishReason, _ = c["finish_reason"].(string)
|
||
if msg, ok := c["message"].(map[string]any); ok {
|
||
if rc, ok := msg["reasoning_content"].(string); ok && rc != "" {
|
||
output = append(output, map[string]any{
|
||
"id": nextID("rs"),
|
||
"type": "reasoning",
|
||
"summary": []any{map[string]any{"type": "summary_text", "text": rc}},
|
||
})
|
||
outputIdx++
|
||
}
|
||
if content, ok := msg["content"].(string); ok {
|
||
output = append(output, map[string]any{
|
||
"id": nextID("msg"),
|
||
"type": "message",
|
||
"role": "assistant",
|
||
"status": "completed",
|
||
"content": []any{map[string]any{
|
||
"type": "output_text", "text": content, "annotations": []any{},
|
||
}},
|
||
})
|
||
outputIdx++
|
||
}
|
||
if tcs, ok := msg["tool_calls"].([]any); ok {
|
||
for _, tc := range tcs {
|
||
if tcm, ok := tc.(map[string]any); ok {
|
||
fn, _ := tcm["function"].(map[string]any)
|
||
output = append(output, map[string]any{
|
||
"id": nextID("fc"),
|
||
"type": "function_call",
|
||
"call_id": tcm["id"],
|
||
"name": fn["name"],
|
||
"arguments": fn["arguments"],
|
||
"status": "completed",
|
||
})
|
||
outputIdx++
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
status := "completed"
|
||
var incompleteDetails any
|
||
if finishReason == "length" {
|
||
status = "incomplete"
|
||
incompleteDetails = map[string]any{"reason": "max_output_tokens"}
|
||
}
|
||
|
||
resp := map[string]any{
|
||
"id": respID,
|
||
"object": "response",
|
||
"created_at": time.Now().Unix(),
|
||
"status": status,
|
||
"model": model,
|
||
"output": output,
|
||
"incomplete_details": incompleteDetails,
|
||
}
|
||
if usage, ok := chatResp["usage"].(map[string]any); ok {
|
||
resp["usage"] = translateUsage(usage)
|
||
}
|
||
return resp
|
||
}
|
||
|
||
func translateUsage(u map[string]any) map[string]any {
|
||
out := map[string]any{}
|
||
if v, ok := u["prompt_tokens"]; ok {
|
||
out["input_tokens"] = v
|
||
}
|
||
if v, ok := u["completion_tokens"]; ok {
|
||
out["output_tokens"] = v
|
||
}
|
||
if v, ok := u["total_tokens"]; ok {
|
||
out["total_tokens"] = v
|
||
}
|
||
// details 透明透传(字段名按 Responses 协议改名,内部字段原样保留):
|
||
// prompt_tokens_details → input_tokens_details(含 cached_tokens 缓存命中数)
|
||
// completion_tokens_details → output_tokens_details(含 reasoning_tokens)
|
||
if d, ok := u["prompt_tokens_details"].(map[string]any); ok && len(d) > 0 {
|
||
out["input_tokens_details"] = d
|
||
}
|
||
outDetails := map[string]any{}
|
||
if d, ok := u["completion_tokens_details"].(map[string]any); ok {
|
||
for k, v := range d {
|
||
outDetails[k] = v
|
||
}
|
||
}
|
||
// sglang 把 reasoning_tokens 放在顶层而非 completion_tokens_details 里,收编进来
|
||
if v, ok := u["reasoning_tokens"]; ok {
|
||
outDetails["reasoning_tokens"] = v
|
||
}
|
||
if len(outDetails) > 0 {
|
||
out["output_tokens_details"] = outDetails
|
||
}
|
||
return out
|
||
}
|
||
|
||
// ---------------- 流式翻译状态机 ----------------
|
||
|
||
type toolState struct {
|
||
opened bool
|
||
itemID string
|
||
callID string
|
||
name string
|
||
outIdx int
|
||
args strings.Builder
|
||
}
|
||
|
||
type streamState struct {
|
||
w http.ResponseWriter
|
||
fl http.Flusher
|
||
seq int
|
||
respID string
|
||
model string
|
||
createdAt int64
|
||
started bool
|
||
nextOutIdx int
|
||
reasoningOpen bool
|
||
reasoningID string
|
||
reasoningIdx int
|
||
reasoningText strings.Builder
|
||
msgOpen bool
|
||
msgID string
|
||
msgIdx int
|
||
partOpen bool
|
||
text strings.Builder
|
||
tools map[int]*toolState
|
||
toolOrder []int
|
||
finishReason string
|
||
usage map[string]any
|
||
failed bool
|
||
}
|
||
|
||
func newStreamState(w http.ResponseWriter) *streamState {
|
||
fl, _ := w.(http.Flusher)
|
||
return &streamState{
|
||
w: w, fl: fl, respID: nextID("resp"),
|
||
createdAt: time.Now().Unix(),
|
||
tools: map[int]*toolState{},
|
||
}
|
||
}
|
||
|
||
func (s *streamState) emit(eventType string, payload map[string]any) {
|
||
s.seq++
|
||
payload["type"] = eventType
|
||
payload["sequence_number"] = s.seq
|
||
data, err := json.Marshal(payload)
|
||
if err != nil {
|
||
return
|
||
}
|
||
fmt.Fprintf(s.w, "event: %s\ndata: %s\n\n", eventType, data)
|
||
if s.fl != nil {
|
||
s.fl.Flush()
|
||
}
|
||
}
|
||
|
||
func (s *streamState) responseSkeleton(status string) map[string]any {
|
||
return map[string]any{
|
||
"id": s.respID, "object": "response", "created_at": s.createdAt,
|
||
"status": status, "model": s.model, "output": []any{},
|
||
}
|
||
}
|
||
|
||
func (s *streamState) ensureStarted() {
|
||
if s.started {
|
||
return
|
||
}
|
||
s.started = true
|
||
s.emit("response.created", map[string]any{"response": s.responseSkeleton("in_progress")})
|
||
s.emit("response.in_progress", map[string]any{"response": s.responseSkeleton("in_progress")})
|
||
}
|
||
|
||
func (s *streamState) openReasoning() {
|
||
if s.reasoningOpen {
|
||
return
|
||
}
|
||
s.reasoningOpen = true
|
||
s.reasoningID = nextID("rs")
|
||
s.reasoningIdx = s.nextOutIdx
|
||
s.nextOutIdx++
|
||
s.emit("response.output_item.added", map[string]any{
|
||
"output_index": s.reasoningIdx,
|
||
"item": map[string]any{"id": s.reasoningID, "type": "reasoning", "summary": []any{}},
|
||
})
|
||
s.emit("response.reasoning_summary_part.added", map[string]any{
|
||
"item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0,
|
||
"part": map[string]any{"type": "summary_text", "text": ""},
|
||
})
|
||
}
|
||
|
||
func (s *streamState) closeReasoning() {
|
||
if !s.reasoningOpen {
|
||
return
|
||
}
|
||
s.reasoningOpen = false
|
||
full := s.reasoningText.String()
|
||
s.emit("response.reasoning_summary_text.done", map[string]any{
|
||
"item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0, "text": full,
|
||
})
|
||
s.emit("response.reasoning_summary_part.done", map[string]any{
|
||
"item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0,
|
||
"part": map[string]any{"type": "summary_text", "text": full},
|
||
})
|
||
s.emit("response.output_item.done", map[string]any{
|
||
"output_index": s.reasoningIdx,
|
||
"item": map[string]any{"id": s.reasoningID, "type": "reasoning",
|
||
"summary": []any{map[string]any{"type": "summary_text", "text": full}}},
|
||
})
|
||
}
|
||
|
||
func (s *streamState) openMessage() {
|
||
if s.msgOpen {
|
||
return
|
||
}
|
||
s.closeReasoning()
|
||
s.msgOpen = true
|
||
s.msgID = nextID("msg")
|
||
s.msgIdx = s.nextOutIdx
|
||
s.nextOutIdx++
|
||
s.emit("response.output_item.added", map[string]any{
|
||
"output_index": s.msgIdx,
|
||
"item": map[string]any{"id": s.msgID, "type": "message", "role": "assistant",
|
||
"status": "in_progress", "content": []any{}},
|
||
})
|
||
s.partOpen = true
|
||
s.emit("response.content_part.added", map[string]any{
|
||
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0,
|
||
"part": map[string]any{"type": "output_text", "text": "", "annotations": []any{}},
|
||
})
|
||
}
|
||
|
||
func (s *streamState) closeMessage() {
|
||
if !s.msgOpen {
|
||
return
|
||
}
|
||
s.msgOpen = false
|
||
full := s.text.String()
|
||
s.emit("response.output_text.done", map[string]any{
|
||
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, "text": full,
|
||
})
|
||
s.emit("response.content_part.done", map[string]any{
|
||
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0,
|
||
"part": map[string]any{"type": "output_text", "text": full, "annotations": []any{}},
|
||
})
|
||
s.emit("response.output_item.done", map[string]any{
|
||
"output_index": s.msgIdx,
|
||
"item": map[string]any{"id": s.msgID, "type": "message", "role": "assistant",
|
||
"status": "completed",
|
||
"content": []any{map[string]any{"type": "output_text", "text": full, "annotations": []any{}}}},
|
||
})
|
||
}
|
||
|
||
func (s *streamState) openTool(idx int, callID, name string) *toolState {
|
||
ts, ok := s.tools[idx]
|
||
if !ok {
|
||
ts = &toolState{}
|
||
s.tools[idx] = ts
|
||
s.toolOrder = append(s.toolOrder, idx)
|
||
}
|
||
if ts.opened {
|
||
return ts
|
||
}
|
||
s.closeReasoning()
|
||
s.closeMessage()
|
||
ts.opened = true
|
||
ts.itemID = nextID("fc")
|
||
ts.callID = callID
|
||
ts.name = name
|
||
ts.outIdx = s.nextOutIdx
|
||
s.nextOutIdx++
|
||
s.emit("response.output_item.added", map[string]any{
|
||
"output_index": ts.outIdx,
|
||
"item": map[string]any{"id": ts.itemID, "type": "function_call",
|
||
"call_id": ts.callID, "name": ts.name, "arguments": "", "status": "in_progress"},
|
||
})
|
||
return ts
|
||
}
|
||
|
||
func (s *streamState) closeAll() {
|
||
s.closeReasoning()
|
||
s.closeMessage()
|
||
for _, idx := range s.toolOrder {
|
||
ts := s.tools[idx]
|
||
if !ts.opened {
|
||
continue
|
||
}
|
||
ts.opened = false
|
||
s.emit("response.function_call_arguments.done", map[string]any{
|
||
"item_id": ts.itemID, "output_index": ts.outIdx, "arguments": ts.args.String(),
|
||
})
|
||
s.emit("response.output_item.done", map[string]any{
|
||
"output_index": ts.outIdx,
|
||
"item": map[string]any{"id": ts.itemID, "type": "function_call",
|
||
"call_id": ts.callID, "name": ts.name, "arguments": ts.args.String(), "status": "completed"},
|
||
})
|
||
}
|
||
}
|
||
|
||
// buildFinalOutput 重建完整 output 数组(用于 response.completed/incomplete)。
|
||
func (s *streamState) buildFinalOutput() []any {
|
||
output := make([]any, s.nextOutIdx)
|
||
if s.reasoningID != "" {
|
||
output[s.reasoningIdx] = map[string]any{"id": s.reasoningID, "type": "reasoning",
|
||
"summary": []any{map[string]any{"type": "summary_text", "text": s.reasoningText.String()}}}
|
||
}
|
||
if s.msgID != "" {
|
||
output[s.msgIdx] = map[string]any{"id": s.msgID, "type": "message", "role": "assistant",
|
||
"status": "completed",
|
||
"content": []any{map[string]any{"type": "output_text", "text": s.text.String(), "annotations": []any{}}}}
|
||
}
|
||
for _, idx := range s.toolOrder {
|
||
ts := s.tools[idx]
|
||
output[ts.outIdx] = map[string]any{"id": ts.itemID, "type": "function_call",
|
||
"call_id": ts.callID, "name": ts.name, "arguments": ts.args.String(), "status": "completed"}
|
||
}
|
||
// 防御:去掉未填充槽位
|
||
compact := output[:0]
|
||
for _, o := range output {
|
||
if o != nil {
|
||
compact = append(compact, o)
|
||
}
|
||
}
|
||
return compact
|
||
}
|
||
|
||
func (s *streamState) finish() {
|
||
s.closeAll()
|
||
status := "completed"
|
||
event := "response.completed"
|
||
var incompleteDetails any
|
||
if s.finishReason == "length" {
|
||
status = "incomplete"
|
||
event = "response.incomplete"
|
||
incompleteDetails = map[string]any{"reason": "max_output_tokens"}
|
||
}
|
||
resp := s.responseSkeleton(status)
|
||
resp["output"] = s.buildFinalOutput()
|
||
resp["incomplete_details"] = incompleteDetails
|
||
if s.usage != nil {
|
||
resp["usage"] = translateUsage(s.usage)
|
||
}
|
||
s.emit(event, map[string]any{"response": resp})
|
||
}
|
||
|
||
func (s *streamState) fail(msg string) {
|
||
if s.failed {
|
||
return
|
||
}
|
||
s.failed = true
|
||
resp := s.responseSkeleton("failed")
|
||
resp["error"] = map[string]any{"message": msg}
|
||
s.emit("response.failed", map[string]any{"response": resp})
|
||
}
|
||
|
||
// processChunk 处理一条 chat.completion.chunk。
|
||
func (s *streamState) processChunk(chunk map[string]any) {
|
||
if m, ok := chunk["model"].(string); ok && m != "" {
|
||
s.model = m
|
||
}
|
||
if usage, ok := chunk["usage"].(map[string]any); ok && usage != nil {
|
||
s.usage = usage
|
||
}
|
||
choices, ok := chunk["choices"].([]any)
|
||
if !ok || len(choices) == 0 {
|
||
return
|
||
}
|
||
c, ok := choices[0].(map[string]any)
|
||
if !ok {
|
||
return
|
||
}
|
||
s.ensureStarted()
|
||
|
||
if delta, ok := c["delta"].(map[string]any); ok {
|
||
if rc, ok := delta["reasoning_content"].(string); ok && rc != "" {
|
||
s.openReasoning()
|
||
s.reasoningText.WriteString(rc)
|
||
s.emit("response.reasoning_summary_text.delta", map[string]any{
|
||
"item_id": s.reasoningID, "output_index": s.reasoningIdx,
|
||
"summary_index": 0, "delta": rc,
|
||
})
|
||
}
|
||
if content, ok := delta["content"].(string); ok && content != "" {
|
||
s.openMessage()
|
||
s.text.WriteString(content)
|
||
s.emit("response.output_text.delta", map[string]any{
|
||
"item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, "delta": content,
|
||
})
|
||
}
|
||
if tcs, ok := delta["tool_calls"].([]any); ok {
|
||
for _, tc := range tcs {
|
||
tcm, ok := tc.(map[string]any)
|
||
if !ok {
|
||
continue
|
||
}
|
||
idxF, _ := tcm["index"].(float64)
|
||
idx := int(idxF)
|
||
callID, _ := tcm["id"].(string)
|
||
fn, _ := tcm["function"].(map[string]any)
|
||
name, _ := fn["name"].(string)
|
||
ts := s.openTool(idx, callID, name)
|
||
if ts.callID == "" {
|
||
ts.callID = callID
|
||
}
|
||
if ts.name == "" {
|
||
ts.name = name
|
||
}
|
||
if args, ok := fn["arguments"].(string); ok && args != "" {
|
||
ts.args.WriteString(args)
|
||
s.emit("response.function_call_arguments.delta", map[string]any{
|
||
"item_id": ts.itemID, "output_index": ts.outIdx, "delta": args,
|
||
})
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if fr, ok := c["finish_reason"].(string); ok && fr != "" {
|
||
s.finishReason = fr
|
||
}
|
||
}
|
||
|
||
// ---------------- /v1/responses 入口 ----------------
|
||
|
||
func handleResponses(w http.ResponseWriter, r *http.Request) {
|
||
if r.Method != http.MethodPost {
|
||
http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed)
|
||
return
|
||
}
|
||
body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20))
|
||
if err != nil {
|
||
writeResponsesError(w, http.StatusBadRequest, "read request failed")
|
||
return
|
||
}
|
||
var req responsesRequest
|
||
if err := json.Unmarshal(body, &req); err != nil {
|
||
writeResponsesError(w, http.StatusBadRequest, "invalid JSON: "+err.Error())
|
||
return
|
||
}
|
||
chatReq, stops, err := translateRequest(&req)
|
||
if err != nil {
|
||
writeResponsesError(w, http.StatusBadRequest, err.Error())
|
||
return
|
||
}
|
||
chatBody, err := json.Marshal(chatReq)
|
||
if err != nil {
|
||
writeResponsesError(w, http.StatusInternalServerError, "marshal failed")
|
||
return
|
||
}
|
||
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody, req.Stream)
|
||
if err != nil {
|
||
writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error())
|
||
return
|
||
}
|
||
defer upstream.Body.Close()
|
||
|
||
if upstream.StatusCode != http.StatusOK {
|
||
// 上游错误原样透传(sglang 的错误体本身就是 OpenAI 格式)
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(upstream.StatusCode)
|
||
_, _ = io.Copy(w, upstream.Body)
|
||
return
|
||
}
|
||
|
||
if !req.Stream {
|
||
var chatResp map[string]any
|
||
if err := json.NewDecoder(upstream.Body).Decode(&chatResp); err != nil {
|
||
writeResponsesError(w, http.StatusBadGateway, "decode upstream response failed")
|
||
return
|
||
}
|
||
if len(stops) > 0 {
|
||
truncateChoicesContent(chatResp, stops)
|
||
}
|
||
respID := nextID("resp")
|
||
w.Header().Set("Content-Type", "application/json")
|
||
_ = json.NewEncoder(w).Encode(translateResponse(chatResp, respID))
|
||
return
|
||
}
|
||
|
||
// 流式:chat SSE → responses SSE
|
||
w.Header().Set("Content-Type", "text/event-stream")
|
||
w.Header().Set("Cache-Control", "no-cache")
|
||
w.Header().Set("Connection", "keep-alive")
|
||
w.WriteHeader(http.StatusOK)
|
||
state := newStreamState(w)
|
||
|
||
scanner := bufio.NewReader(upstream.Body)
|
||
stopped := false
|
||
// 内容侧 stop 扫描(STOP_SCOPE=content 且带 stop 时):跨 chunk 缓冲,
|
||
// reasoning_content 不参与匹配;命中即截断并结束响应。
|
||
var pending strings.Builder
|
||
tailKeep := 0
|
||
if len(stops) > 0 {
|
||
tailKeep = maxStopLen(stops) - 1
|
||
}
|
||
flushTail := func() {
|
||
if pending.Len() == 0 {
|
||
return
|
||
}
|
||
tail := pending.String()
|
||
pending.Reset()
|
||
state.processChunk(map[string]any{
|
||
"choices": []any{map[string]any{"delta": map[string]any{"content": tail}}},
|
||
})
|
||
}
|
||
feed := func(chunk map[string]any) (hit bool) {
|
||
if len(stops) == 0 {
|
||
state.processChunk(chunk)
|
||
return false
|
||
}
|
||
choices, _ := chunk["choices"].([]any)
|
||
if len(choices) == 0 {
|
||
state.processChunk(chunk)
|
||
return false
|
||
}
|
||
c, _ := choices[0].(map[string]any)
|
||
if c == nil {
|
||
state.processChunk(chunk)
|
||
return false
|
||
}
|
||
// finish chunk:先把缓冲尾巴吐进状态机再放行
|
||
if fr, _ := c["finish_reason"].(string); fr != "" {
|
||
flushTail()
|
||
state.processChunk(chunk)
|
||
return false
|
||
}
|
||
delta, _ := c["delta"].(map[string]any)
|
||
if delta == nil {
|
||
state.processChunk(chunk)
|
||
return false
|
||
}
|
||
content, _ := delta["content"].(string)
|
||
if content != "" {
|
||
pending.WriteString(content)
|
||
}
|
||
full := pending.String()
|
||
if i, _ := earliestStop(full, stops); i >= 0 {
|
||
if safe := full[:i]; safe != "" {
|
||
delta["content"] = safe
|
||
} else {
|
||
delete(delta, "content")
|
||
}
|
||
pending.Reset()
|
||
state.processChunk(chunk)
|
||
return true
|
||
}
|
||
safeLen := len(full) - tailKeep
|
||
if safeLen < 0 {
|
||
safeLen = 0
|
||
}
|
||
if safe := full[:safeLen]; safe != "" {
|
||
delta["content"] = safe
|
||
} else {
|
||
delete(delta, "content")
|
||
}
|
||
pending.Reset()
|
||
pending.WriteString(full[safeLen:])
|
||
state.processChunk(chunk)
|
||
return false
|
||
}
|
||
for {
|
||
line, err := scanner.ReadString('\n')
|
||
if len(line) > 0 {
|
||
line = strings.TrimRight(line, "\r\n")
|
||
if strings.HasPrefix(line, "data:") {
|
||
data := strings.TrimSpace(strings.TrimPrefix(line, "data:"))
|
||
if data == "[DONE]" {
|
||
break
|
||
}
|
||
var chunk map[string]any
|
||
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
|
||
if feed(chunk) {
|
||
stopped = true
|
||
break
|
||
}
|
||
}
|
||
}
|
||
}
|
||
if err != nil {
|
||
break
|
||
}
|
||
}
|
||
if !stopped {
|
||
flushTail() // 自然结束:残余缓冲进状态机
|
||
}
|
||
if state.started && !state.failed {
|
||
state.finish()
|
||
} else if !state.started {
|
||
state.fail("upstream closed stream without data")
|
||
}
|
||
}
|
||
|
||
func writeResponsesError(w http.ResponseWriter, code int, msg string) {
|
||
w.Header().Set("Content-Type", "application/json")
|
||
w.WriteHeader(code)
|
||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||
"error": map[string]any{"message": msg, "type": "invalid_request_error"},
|
||
})
|
||
}
|