From ccdd9df1d3f2dc6aef023f0b697e044f88bd9cc4 Mon Sep 17 00:00:00 2001 From: "minke.yu" Date: Wed, 30 Sep 2026 17:44:41 +0800 Subject: [PATCH] responses-shim v0.2.3: DeepSeek Responses -> chat completions shim MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - /v1/responses 双向翻译(含流式 response.* 事件、tools、usage 全字段) - /v1/chat/completions 字节级透传 - STOP_SCOPE=content 内容侧 stop(保思维链 + matched_stop 回填) - b300 docker 构建 + 孔明 kaniko 构建 + 两版 cw manifest --- .gitignore | 2 + Dockerfile | 6 + Dockerfile.kaniko | 15 + README.md | 43 + deploy/responses-shim-b300-cw.yaml | 71 ++ feishu-changelog.xml | 4 + feishu-chapter.xml | 52 ++ feishu-diagram.xml | 11 + go.mod | 3 + main.go | 1312 ++++++++++++++++++++++++++++ 10 files changed, 1519 insertions(+) create mode 100644 .gitignore create mode 100644 Dockerfile create mode 100644 Dockerfile.kaniko create mode 100644 README.md create mode 100644 deploy/responses-shim-b300-cw.yaml create mode 100644 feishu-changelog.xml create mode 100644 feishu-chapter.xml create mode 100644 feishu-diagram.xml create mode 100644 go.mod create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..40af49e --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +shim.exe +shim-linux diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1490815 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +# responses-shim: 静态二进制,scratch 镜像(离线构建无需拉取任何镜像) +# b300 构建: docker build -t ymkymx/responses-shim:v0.1.0 . +FROM scratch +COPY shim-linux /shim +EXPOSE 8080 +ENTRYPOINT ["/shim"] diff --git a/Dockerfile.kaniko b/Dockerfile.kaniko new file mode 100644 index 0000000..d15c52f --- /dev/null +++ b/Dockerfile.kaniko @@ -0,0 +1,15 @@ +# 孔明 kmctl kaniko 构建(dockerfile 模式,无构建上下文) +# 不 COPY 本地文件,从 Gitee 匿名 clone 源码编译(与 sglang 的 build-kmctl/Dockerfile 同套路)。 +# 用法: kmctl --cluster <集群> create image responses-shim --build-mode dockerfile \ +# --dockerfile Dockerfile.kaniko --node-group <节点组> --wait +FROM uhub.service.ucloud.cn/umirror/golang:1.23 AS build + +RUN git clone --depth 1 https://gitee.com/ymkymkymkymx/responses-shim.git /src \ + && cd /src \ + && CGO_ENABLED=0 go build -ldflags="-s -w" -o /shim . \ + && git rev-parse HEAD > /etc/shim-commit || true + +FROM scratch +COPY --from=build /shim /shim +EXPOSE 8080 +ENTRYPOINT ["/shim"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..1f04b52 --- /dev/null +++ b/README.md @@ -0,0 +1,43 @@ +# responses-shim + +DeepSeek Responses API(`/v1/responses`)→ OpenAI Chat Completions 的协议转换 shim。 +后置部署在 new-api 与 sglang 之间:客户端打 new-api 的 `/v1/responses`,new-api 按原路径透传给 +本服务(渠道上游),本服务翻译后直连 sglang router。 + +## 功能 + +- `POST /v1/responses` → 翻译为 `/v1/chat/completions`: + - `input`(纯字符串 / item 列表:message、function_call、function_call_output、custom_tool_call*、reasoning)→ `messages` + - `instructions` → 首条 system 消息 + - `reasoning.effort`(none/low/high/max,别名 minimal→low、medium/xhigh→high、ultra→max)→ `chat_template_kwargs` + - `max_output_tokens` → `max_tokens`;`text.format` → `response_format`;`tools`/`tool_choice` 拍平; + `top_logprobs` / `user` / `stop` 透传 + - 响应:`choices` → `output[]`(reasoning / message / function_call item);usage 全字段 + (含 `cached_tokens` 缓存命中、`reasoning_tokens`) + - 流式:chat SSE → `response.*` 事件序列(created/in_progress/output_item/delta/completed/incomplete/failed) +- `POST /v1/chat/completions` → 字节级透传 +- `GET /healthz` + +## STOP_SCOPE:内容侧 stop + +`STOP_SCOPE=content`(生产配置)时,chat 与 responses 两条路径的 `stop` 都**不下发上游**, +由 shim 在 content 流上跨 chunk 匹配截断:思维链(reasoning_content)完整生成不被误杀, +命中后 `finish_reason=stop` + `matched_stop` 回填,流式命中即断开上游触发 sglang abort。 +`STOP_SCOPE=all`(默认)为原生语义:stop 透传上游,token 级截断(含思维链)。 + +## 环境变量 + +| 变量 | 默认 | 说明 | +|---|---|---| +| `UPSTREAM_BASE` | `http://127.0.0.1:50002` | sglang router 地址 | +| `LISTEN` | `:8080` | 监听地址 | +| `STOP_SCOPE` | `all` | `all` / `content` | + +## 构建与部署 + +- 本地/b300 docker 构建(scratch,需先交叉编译):`GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags="-s -w" -o shim-linux . && docker build -t ymkymx/responses-shim:vX .` +- 孔明 kaniko 构建(无上下文,从 Gitee clone 源码): + `kmctl create image responses-shim --build-mode dockerfile --dockerfile Dockerfile.kaniko --node-group --wait` +- 部署清单:`deploy/`(b300 版 + h20 版),`kmctl create cw -f --node-group ` + +仅标准库,零依赖。源码 source of truth 在本仓库;b300 构建目录 `/data/ymk/build/responses-shim/` 是镜像构建缓存。 diff --git a/deploy/responses-shim-b300-cw.yaml b/deploy/responses-shim-b300-cw.yaml new file mode 100644 index 0000000..09952cb --- /dev/null +++ b/deploy/responses-shim-b300-cw.yaml @@ -0,0 +1,71 @@ +# responses-shim — DeepSeek Responses API (/v1/responses) → chat completions 协议转换 +# 架构: 客户端 → new-api(:31000, /v1/responses 原路径透传) → 本服务(渠道上游) → ds41-3p1d:50002 +# 部署: wsl kmctl create cw ds41-resp-shim -f /mnt/d/B300/build-kmctl/responses-shim-cw.yaml --node-group B300 +# 镜像: ymkymx/responses-shim:v0.1.0(scratch+静态 Go 二进制,5.6MB, +# 已 docker save | ctr -n k8s.io images import 侧载到 b300-01;Kyverno 对未映射镜像保持原名) +# 网络: 普通 Pod 网络 + ClusterIP Service(同 jaeger 之外的常规姿势);new-api 渠道 base URL 填 +# http://ds41-resp-shim:8080 (同 namespace 短名可达,与 ds41-3p1d 同理) +# 透传: /v1/chat/completions 字节级透传(渠道测试/健康检查/误路由的 chat 请求都正常) +# 源码: D:\B300\responses-shim\main.go(另存 b300 /data/ymk/build/responses-shim/) +apiVersion: apps/v1 +kind: Deployment +metadata: + name: responses-shim + labels: + app: responses-shim +spec: + replicas: 1 + strategy: + type: Recreate # 无状态但无需滚动,重建越快越好 + selector: + matchLabels: + app: responses-shim + template: + metadata: + labels: + app: responses-shim + spec: + nodeSelector: + kubernetes.io/hostname: b300-01 + containers: + - name: shim + image: ymkymx/responses-shim:v0.2.3 + imagePullPolicy: IfNotPresent + env: + - name: UPSTREAM_BASE + value: "http://ds41-3p1d:50002" + - name: LISTEN + value: ":8080" + - name: STOP_SCOPE + value: "content" # stop 只在 content 上截断,思维链不被误杀(Claude stop_sequences 语义) + ports: + - {containerPort: 8080} + resources: + requests: + cpu: 200m + memory: 128Mi + limits: + cpu: "1" + memory: 512Mi + readinessProbe: + httpGet: {path: /healthz, port: 8080} + initialDelaySeconds: 2 + periodSeconds: 10 + livenessProbe: + httpGet: {path: /healthz, port: 8080} + initialDelaySeconds: 5 + periodSeconds: 20 +--- +apiVersion: v1 +kind: Service +metadata: + name: ds41-resp-shim + labels: + app: responses-shim +spec: + selector: + app: responses-shim + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/feishu-changelog.xml b/feishu-changelog.xml new file mode 100644 index 0000000..9bc8834 --- /dev/null +++ b/feishu-changelog.xml @@ -0,0 +1,4 @@ +

2026-09-30 | new-api 参数覆盖链 + responses-shim 上线(v0.1.0→v0.2.3)

+

背景:sglang 不支持官方 reasoning_effort / thinking 开关语义与 /v1/responses 协议,new-api 只能透传。

+

改动:①new-api 渠道参数覆盖链——thinking⇄chat_template_kwargs 映射、reasoning_effort 归一(含 minimal/medium/xhigh 别名)、temperature 思考模式钉 1(关思考跟随用户)、top_p 钳 0.95、max_tokens 缺省 65536 keep_origin;preflight 验收 13/13。②新建 responses-shim(Go stdlib 单文件,scratch 镜像 5.6MB),孔明 cw ds41-resp-shim(ClusterIP ds41-resp-shim:8080,钉 b300-01,上游 ds41-3p1d:50002);new-api 渠道上游改指 shim。当天迭代至 v0.2.3:内容侧 stop 截断(STOP_SCOPE=content)、usage details 透传、reasoning_tokens 收编、responses 路径 stop 修复。

+

边界:chat 路径响应 stop_sequence=null 属 new-api Claude 转换器限制救不了;previous_response_id 不支持;上游 sglang PR #5209 合入后 shim 可退役。详见六、new-api 网关层。

diff --git a/feishu-chapter.xml b/feishu-chapter.xml new file mode 100644 index 0000000..72261bf --- /dev/null +++ b/feishu-chapter.xml @@ -0,0 +1,52 @@ +

六、new-api 网关层:参数覆盖链 + responses-shim(2026-09-30 上线)

+

对外入口统一走 new-api 网关(http://125.93.200.207:31000/,模型名 deepseek-v4.1-flash)。sglang 原生不支持的官方 API 参数(reasoning_effort / thinking 开关 / responses 协议)在网关层补齐:new-api 渠道"参数覆盖"负责采样参数归一,responses-shim(孔明 cw ds41-resp-shim)负责 /v1/responses ⇄ /v1/chat/completions 双向翻译。new-api 渠道上游已改为 shim(http://ds41-resp-shim:8080),shim 再转发 ds41-3p1d router(ClusterIP ds41-3p1d:50002)。

+flowchart LR + C[客户端] --> N[new-api :31000 参数覆盖链] + N --> S[responses-shim :8080 cw ds41-resp-shim] + S --> R[sgl-router ds41-3p1d:50002] + R --> P[P x3 prefill cp4+dspark] + R --> D[D decode dp2] + +

6.1 new-api 参数覆盖链(渠道高级设置 → 参数覆盖)

+

目标:对上暴露 DeepSeek 官方 API 语义,对下翻译成 sglang 认识的参数。覆盖 ops 按顺序执行,要点:

+
    +
  • thinking 映射:Claude 风格 thinking.type=enabled/disabled → chat_template_kwargs.thinking=true/false(sglang 用 chat template kwargs 控制思维链)。
  • +
  • reasoning_effort 归一:官方取值 none/low/high/max,别名 minimal→low、medium/xhigh→high;落到 chat_template_kwargs 的 thinking 开关与 reasoning_effort。responses 请求另有一条豁免:reasoning.effort 由 shim 翻译,不在覆盖链重复处理。
  • +
  • temperature 条件钉 1:思考模式下官方语义=不生效,强制 set 1;关闭思考(none / thinking disabled)时跟随用户设置——条件带 invert + pass_missing_key,缺省不补。
  • +
  • top_p 钳制:思考模式有效范围 0.95–1.0,低于 0.95 按 0.95;非思考模式恒 1.0(传入忽略)。
  • +
  • max_tokens 缺省 65536:keep_origin=true,用户传了不覆盖;responses 侧的 max_output_tokens 同理 keep_origin。
  • +
+

验收:llm-api-preflight 13/13 通过(采样参数生效性、思考开关、默认值、/v1/messages 兼容路径),基线存档 D:\B300\preflight\newapi-thinking-baseline-20260930.md。

+

6.2 responses-shim(/v1/responses 协议适配)

+

sglang 不支持 官方 /v1/responses 协议,new-api 只能透传不能翻译,所以在中间加了一层 shim:

+
    +
  • 实现:Go 单文件、纯 stdlib 零依赖;源码 D:\B300\responses-shim\main.go,b300 构建目录 /data/ymk/build/responses-shim/。
  • +
  • 镜像:ymkymx/responses-shim:v0.2.3(scratch 基础,~5.6MB,本机交叉编译 GOOS=linux GOARCH=amd64 CGO_ENABLED=0 → scp → b300 docker build → ctr -n k8s.io images import 侧载)。
  • +
  • 部署:孔明 cw ds41-resp-shim(manifest D:\B300\build-kmctl\responses-shim-cw.yaml),ClusterIP ds41-resp-shim:8080,钉 b300-01,资源 1 核 / 512Mi;env:UPSTREAM_BASE=http://ds41-3p1d:50002、STOP_SCOPE=content。
  • +
  • 功能:/v1/responses 双向翻译——入参 input / instructions / reasoning.effort / max_output_tokens / text.format / tools / tool_choice / top_logprobs / user / stop;流式输出完整 response.* 事件序列;usage 全字段透传(含 cached_tokens、reasoning_tokens——sglang 把 reasoning_tokens 放在顶层,v0.2.2 起收编进 output_tokens_details)。/v1/chat/completions 字节级透传不解析。
  • +
  • 版本履历:v0.2.0 内容侧 stop 截断(STOP_SCOPE=content);v0.2.1 usage details 透传;v0.2.2 reasoning_tokens 收编;v0.2.3 responses 路径 stop 参数修复。
  • +
+

6.3 已知坑与边界

+ +
    +
  • stop_sequence=null 救不了:chat 路径 new-api 的 Claude 转换器不读上游 matched_stop 字段,响应里 stop_sequence 恒 null——shim 无法补齐,属网关转换器限制。
  • +
  • STOP_SCOPE=content 语义偏离:stop 只截正文不截思维链(官方行为是连 reasoning 一起截)。如需严格对齐官方,改 STOP_SCOPE=all 重启。
  • +
  • previous_response_id 不支持:responses 链式会话 ID 不生效(多轮需客户端自带历史)。
  • +
  • shim 单副本承接全量流量:chat 透传也过 shim,1 核 / 512Mi 实测 1000 RPM 余量充足;若日后流量翻倍再扩副本(无状态可直接加)。
  • +
  • Windows scp 丢执行位:二进制传到 b300 后必须 chmod +x 再 docker build,否则容器启动报 permission denied。
  • +
  • 上游 PR #5209 若合入(sglang 原生 responses 支持)shim 可退役,届时回滚 new-api 渠道上游直指 ds41-3p1d:50002 即可。
  • +
+
+

6.4 运维速查

+
# 查状态 / 日志 / 启停
+wsl kmctl logs cw ds41-resp-shim
+wsl kmctl {events,stop,start} cw ds41-resp-shim
+
+# 改代码重建(本机绿色 Go:D:\B300\tools\go\bin\go.exe)
+cd D:\B300\responses-shim
+GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o responses-shim .
+scp responses-shim 24.125.93.200.207:/data/ymk/build/responses-shim/
+# b300 上:chmod +x → docker build -t ymkymx/responses-shim:vX.Y.Z .
+# → docker save | ctr -n k8s.io images import -
+# 改 manifest tag 后:wsl kmctl update cw ds41-resp-shim
+
diff --git a/feishu-diagram.xml b/feishu-diagram.xml new file mode 100644 index 0000000..9faea2e --- /dev/null +++ b/feishu-diagram.xml @@ -0,0 +1,11 @@ +
客户端(/v1/chat/completions · /v1/messages · /v1/responses)
+  │
+  ▼
+new-api 网关 :31000 ── 参数覆盖链(thinking / reasoning_effort / temperature / top_p / max_tokens)
+  │ 渠道上游
+  ▼
+responses-shim :8080(孔明 cw ds41-resp-shim;responses ⇄ chat 翻译,chat 字节透传)
+  │ UPSTREAM_BASE=http://ds41-3p1d:50002
+  ▼
+sgl-router ds41-3p1d:50002 ──┬── P ×3(prefill,cp4 + dspark)
+                             └── D(decode,dp2)
diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..f116628 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module responses-shim + +go 1.23.12 diff --git a/main.go b/main.go new file mode 100644 index 0000000..cbc9eb8 --- /dev/null +++ b/main.go @@ -0,0 +1,1312 @@ +// responses-shim: DeepSeek Responses API (/v1/responses) → OpenAI Chat Completions 协议转换 shim +// 后置部署:new-api 渠道上游指向本服务,本服务再转发到 sglang router。 +// - POST /v1/responses → 翻译成 /v1/chat/completions 请求,响应/流式事件翻译回 Responses 协议 +// - POST /v1/chat/completions → 纯透传(字节级转发,含 SSE) +// - GET /healthz → 200 +// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、 +// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content +// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。 +package main + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "strings" + "sync/atomic" + "time" +) + +var ( + upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002") + listenAddr = envOr("LISTEN", ":8080") + httpClient = &http.Client{ + Timeout: 0, // 生成可能很长,不设总超时;靠 ResponseHeaderTimeout 防上游假死 + Transport: &http.Transport{ + MaxIdleConns: 512, + MaxIdleConnsPerHost: 256, + IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: 300 * time.Second, + }, + } + idCounter int64 + stopScope = envOr("STOP_SCOPE", "all") +) + +func envOr(k, def string) string { + if v := os.Getenv(k); v != "" { + return v + } + return def +} + +func nextID(prefix string) string { + return fmt.Sprintf("%s_%x%x", prefix, time.Now().UnixNano(), atomic.AddInt64(&idCounter, 1)) +} + +func main() { + mux := http.NewServeMux() + mux.HandleFunc("/v1/responses", handleResponses) + mux.HandleFunc("/v1/chat/completions", handlePassthrough) + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("ok")) + }) + log.Printf("responses-shim listening on %s, upstream=%s", listenAddr, upstreamBase) + log.Fatal(http.ListenAndServe(listenAddr, mux)) +} + +// ---------------- 公共:转发 ---------------- + +// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。 +func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Response, error) { + req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + if auth := r.Header.Get("Authorization"); auth != "" { + req.Header.Set("Authorization", auth) + } + req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层 + return httpClient.Do(req) +} + +// handlePassthrough 透传 /v1/chat/completions。 +// STOP_SCOPE=content 且请求带 stop 时:stop 不下发上游,shim 只在 content 上匹配截断 +// (思维链不受影响),命中后 finish_reason=stop 并回填 matched_stop。 +func handlePassthrough(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20)) + if err != nil { + http.Error(w, `{"error":"read request failed"}`, http.StatusBadRequest) + return + } + + stops, upstreamBody, stream := maybeStripStop(body) + resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody) + if err != nil { + http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway) + return + } + defer resp.Body.Close() + copyHeader(w.Header(), resp.Header) + + if len(stops) == 0 || resp.StatusCode != http.StatusOK { + w.WriteHeader(resp.StatusCode) + streamCopy(w, resp.Body) + return + } + if !stream { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + applyStopNonStream(w, resp.Body, stops) + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.WriteHeader(http.StatusOK) + applyStopStream(w, resp.Body, stops) +} + +// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。 +func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) { + out = body + if stopScope != "content" { + return nil, out, false + } + var req map[string]any + if err := json.Unmarshal(body, &req); err != nil { + return nil, out, false + } + stream, _ = req["stream"].(bool) + switch v := req["stop"].(type) { + case string: + if v != "" { + stops = []string{v} + } + case []any: + for _, s := range v { + if str, ok := s.(string); ok && str != "" { + stops = append(stops, str) + } + } + } + if len(stops) == 0 { + return nil, out, stream + } + delete(req, "stop") + if b, err := json.Marshal(req); err == nil { + out = b + } + return stops, out, stream +} + +// earliestStop 返回 text 中最早命中的 stop 位置与串;未命中返回 -1。 +func earliestStop(text string, stops []string) (int, string) { + best, hit := -1, "" + for _, s := range stops { + if i := strings.Index(text, s); i >= 0 && (best < 0 || i < best) { + best, hit = i, s + } + } + return best, hit +} + +func maxStopLen(stops []string) int { + m := 0 + for _, s := range stops { + if len(s) > m { + m = len(s) + } + } + return m +} + +// truncateChoicesContent 截断每个 choice 的 content,命中则 finish_reason=stop + matched_stop。 +func truncateChoicesContent(resp map[string]any, stops []string) { + choices, ok := resp["choices"].([]any) + if !ok { + return + } + for _, c := range choices { + cm, ok := c.(map[string]any) + if !ok { + continue + } + msg, ok := cm["message"].(map[string]any) + if !ok { + continue + } + content, _ := msg["content"].(string) + if i, hit := earliestStop(content, stops); i >= 0 { + msg["content"] = content[:i] + cm["finish_reason"] = "stop" + cm["matched_stop"] = hit + } + } +} + +// applyStopNonStream 非流式:截断每个 choice 的 content,命中则 finish_reason=stop。 +func applyStopNonStream(w http.ResponseWriter, src io.Reader, stops []string) { + var resp map[string]any + if err := json.NewDecoder(src).Decode(&resp); err != nil { + http.Error(w, `{"error":"decode upstream failed"}`, http.StatusBadGateway) + return + } + truncateChoicesContent(resp, stops) + _ = json.NewEncoder(w).Encode(resp) +} + +// stopScanner 流式内容侧 stop 扫描器(每请求一个实例,无共享状态)。 +type stopScanner struct { + w http.ResponseWriter + fl http.Flusher + stops []string + tailKeep int + pending strings.Builder + chunkID string + model string + finished bool +} + +func (s *stopScanner) emitChunk(chunk map[string]any) { + b, err := json.Marshal(chunk) + if err != nil { + return + } + fmt.Fprintf(s.w, "data: %s\n\n", b) + if s.fl != nil { + s.fl.Flush() + } +} + +func (s *stopScanner) emitRaw(line string) { + fmt.Fprintf(s.w, "%s\n\n", line) + if s.fl != nil { + s.fl.Flush() + } +} + +// flushPending 把缓冲尾巴作为内容 chunk 发出(在 finish chunk 之前调用,保证顺序)。 +func (s *stopScanner) flushPending() { + if s.pending.Len() == 0 { + return + } + content := s.pending.String() + s.pending.Reset() + s.emitChunk(map[string]any{ + "id": s.chunkID, "object": "chat.completion.chunk", "model": s.model, + "created": time.Now().Unix(), + "choices": []any{map[string]any{ + "index": 0, "delta": map[string]any{"content": content}, "finish_reason": nil, + }}, + }) +} + +// feed 处理一个 chunk:content 入缓冲并吐出安全前缀;finish chunk 前先冲刷缓冲。 +// 命中 stop 时返回命中串,chunk 的 content 已被截到命中点之前。 +func (s *stopScanner) feed(chunk map[string]any) (matched string, hit bool) { + if id, ok := chunk["id"].(string); ok && id != "" { + s.chunkID = id + } + if m, ok := chunk["model"].(string); ok && m != "" { + s.model = m + } + choices, ok := chunk["choices"].([]any) + if !ok || len(choices) == 0 { + return "", false // usage-only chunk 等,原样透传 + } + c, ok := choices[0].(map[string]any) + if !ok { + return "", false + } + // finish chunk:先把缓冲内容吐干净再放行 + if fr, _ := c["finish_reason"].(string); fr != "" { + s.flushPending() + return "", false + } + delta, ok := c["delta"].(map[string]any) + if !ok { + return "", false + } + content, _ := delta["content"].(string) + if content != "" { + s.pending.WriteString(content) + } + full := s.pending.String() + if i, mstr := earliestStop(full, s.stops); i >= 0 { + safe := full[:i] + if safe != "" { + delta["content"] = safe + } else { + delete(delta, "content") + } + s.pending.Reset() + return mstr, true + } + // 未命中:吐出除尾巴外的安全前缀 + safeLen := len(full) - s.tailKeep + if safeLen < 0 { + safeLen = 0 + } + if safe := full[:safeLen]; safe != "" { + delta["content"] = safe + } else { + delete(delta, "content") + } + s.pending.Reset() + s.pending.WriteString(full[safeLen:]) + return "", false +} + +// emitFinish 命中 stop 后补发 finish chunk(带 matched_stop)+ [DONE]。 +func (s *stopScanner) emitFinish(matched string) { + s.emitChunk(map[string]any{ + "id": s.chunkID, "object": "chat.completion.chunk", "model": s.model, + "created": time.Now().Unix(), + "choices": []any{map[string]any{ + "index": 0, "delta": map[string]any{}, "finish_reason": "stop", "matched_stop": matched, + }}, + }) + s.emitRaw("data: [DONE]") +} + +// applyStopStream 流式:逐 chunk 扫描 content delta,跨 chunk 缓冲防边界漏配; +// 命中后补发 finish chunk + [DONE] 并断开上游(触发 sglang abort,省后续 token)。 +// reasoning_content 不参与匹配,思维链完整生成。 +func applyStopStream(w http.ResponseWriter, src io.Reader, stops []string) { + fl, _ := w.(http.Flusher) + sc := &stopScanner{w: w, fl: fl, stops: stops, tailKeep: maxStopLen(stops) - 1} + + reader := bufio.NewReader(src) + matched := false + var matchStr string + for { + line, err := reader.ReadString('\n') + trimmed := strings.TrimRight(line, "\r\n") + if trimmed != "" && strings.HasPrefix(trimmed, "data:") { + data := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:")) + if data == "[DONE]" { + break + } + var chunk map[string]any + if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil { + mstr, hit := sc.feed(chunk) + sc.emitChunk(chunk) + if hit { + matched, matchStr = true, mstr + break + } + } else { + sc.emitRaw(trimmed) // 解析失败的行原样透传 + } + } else if trimmed != "" { + sc.emitRaw(trimmed) + } + if err != nil { + break + } + } + + if matched { + sc.emitFinish(matchStr) + return // 断开上游:defer resp.Body.Close() 触发 abort + } + // 自然结束(上游 [DONE] 或 EOF):flush 残余缓冲 + [DONE] + sc.flushPending() + sc.emitRaw("data: [DONE]") +} + +func copyHeader(dst, src http.Header) { + for _, k := range []string{"Content-Type", "Cache-Control"} { + if v := src.Get(k); v != "" { + dst.Set(k, v) + } + } +} + +// streamCopy 边读边写边 flush,SSE 友好。 +func streamCopy(w http.ResponseWriter, src io.Reader) { + fl, _ := w.(http.Flusher) + buf := make([]byte, 32<<10) + for { + n, err := src.Read(buf) + if n > 0 { + _, _ = w.Write(buf[:n]) + if fl != nil { + fl.Flush() + } + } + if err != nil { + return + } + } +} + +// ---------------- Responses → Chat 请求翻译 ---------------- + +type responsesRequest struct { + Model string `json:"model"` + Input json.RawMessage `json:"input"` + Instructions string `json:"instructions"` + Reasoning *struct { + Effort string `json:"effort"` + } `json:"reasoning"` + MaxOutputTokens *int `json:"max_output_tokens"` + Temperature *float64 `json:"temperature"` + TopP *float64 `json:"top_p"` + Stream bool `json:"stream"` + Text *struct { + Format *struct { + Type string `json:"type"` + Name string `json:"name"` + Schema json.RawMessage `json:"schema"` + Strict *bool `json:"strict"` + } `json:"format"` + } `json:"text"` + Tools []map[string]any `json:"tools"` + ToolChoice json.RawMessage `json:"tool_choice"` + TopLogprobs *int `json:"top_logprobs"` + User string `json:"user"` + Stop json.RawMessage `json:"stop"` +} + +// parseStops 解析 stop 的 string | string[] 两种形态。 +func parseStops(raw json.RawMessage) []string { + if len(raw) == 0 || string(raw) == "null" { + return nil + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + if s != "" { + return []string{s} + } + return nil + } + var arr []string + if err := json.Unmarshal(raw, &arr); err == nil { + var out []string + for _, v := range arr { + if v != "" { + out = append(out, v) + } + } + return out + } + return nil +} + +// effort 别名归一(DeepSeek 官方映射表):minimal→low,medium/xhigh→high,ultra→max +func normalizeEffort(e string) string { + switch e { + case "minimal": + return "low" + case "medium", "xhigh": + return "high" + case "ultra": + return "max" + default: + return e // none/low/high/max 及其他原样 + } +} + +func translateRequest(req *responsesRequest) (map[string]any, []string, error) { + out := map[string]any{"model": req.Model} + + // messages + var messages []map[string]any + if req.Instructions != "" { + messages = append(messages, map[string]any{"role": "system", "content": req.Instructions}) + } + if len(req.Input) > 0 && string(req.Input) != "null" { + ms, err := convertInput(req.Input) + if err != nil { + return nil, nil, err + } + messages = append(messages, ms...) + } + if len(messages) == 0 { + return nil, nil, fmt.Errorf("input 与 instructions 至少传一个") + } + out["messages"] = messages + + // reasoning.effort → chat_template_kwargs(接 new-api/sglang 的思考开关语义) + if req.Reasoning != nil && req.Reasoning.Effort != "" { + effort := normalizeEffort(req.Reasoning.Effort) + ctk := map[string]any{} + if effort == "none" { + ctk["thinking"] = false + } else { + ctk["thinking"] = true + ctk["reasoning_effort"] = effort + } + out["chat_template_kwargs"] = ctk + } + + if req.MaxOutputTokens != nil { + out["max_tokens"] = *req.MaxOutputTokens + } + if req.Temperature != nil { + out["temperature"] = *req.Temperature + } + if req.TopP != nil { + out["top_p"] = *req.TopP + } + if req.Stream { + out["stream"] = true + // 必须让上游回 usage,否则 response.completed 里没有计数 + out["stream_options"] = map[string]any{"include_usage": true} + } + if req.Text != nil && req.Text.Format != nil { + switch req.Text.Format.Type { + case "json_object": + out["response_format"] = map[string]any{"type": "json_object"} + case "json_schema": + js := map[string]any{"name": req.Text.Format.Name} + if len(req.Text.Format.Schema) > 0 { + var schema any + if err := json.Unmarshal(req.Text.Format.Schema, &schema); err == nil { + js["schema"] = schema + } + } + if req.Text.Format.Strict != nil { + js["strict"] = *req.Text.Format.Strict + } + out["response_format"] = map[string]any{"type": "json_schema", "json_schema": js} + } + } + if len(req.Tools) > 0 { + var tools []map[string]any + for _, t := range req.Tools { + if t["type"] != "function" { + continue // 内置工具类型忽略 + } + name, _ := t["name"].(string) + if strings.TrimSpace(name) == "" { + continue // name 为空的 function 工具跳过(上游会 400) + } + fn := map[string]any{"name": name} + if d, ok := t["description"]; ok { + fn["description"] = d + } + if p, ok := t["parameters"]; ok { + fn["parameters"] = p + } + tools = append(tools, map[string]any{"type": "function", "function": fn}) + } + if len(tools) > 0 { + out["tools"] = tools + } + } + if len(req.ToolChoice) > 0 && string(req.ToolChoice) != "null" { + var s string + if err := json.Unmarshal(req.ToolChoice, &s); err == nil { + out["tool_choice"] = s // none/auto/required + } else { + var obj map[string]any + if err := json.Unmarshal(req.ToolChoice, &obj); err == nil { + if obj["type"] == "function" { + out["tool_choice"] = map[string]any{ + "type": "function", + "function": map[string]any{"name": obj["name"]}, + } + } + } + } + } + if req.TopLogprobs != nil { + out["logprobs"] = true + out["top_logprobs"] = *req.TopLogprobs + } + if req.User != "" { + out["user"] = req.User + } + // stop:STOP_SCOPE=content 时不下发上游(由 shim 在 content 上截断,保思维链); + // 否则原生透传给上游。 + stops := parseStops(req.Stop) + if len(stops) > 0 && stopScope != "content" { + out["stop"] = stops + } + return out, stops, nil +} + +// convertInput 处理 input 的两种形态:纯字符串 或 item 列表。 +func convertInput(raw json.RawMessage) ([]map[string]any, error) { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return []map[string]any{{"role": "user", "content": s}}, nil + } + var items []map[string]json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil { + return nil, fmt.Errorf("input 必须是字符串或 item 列表: %w", err) + } + var messages []map[string]any + for _, item := range items { + var typ string + _ = json.Unmarshal(item["type"], &typ) + var role string + _ = json.Unmarshal(item["role"], &role) + + switch { + case typ == "message" || (typ == "" && role != ""): + if role == "developer" { + role = "user" // DeepSeek: developer 视同 user + } + if role == "" { + role = "user" + } + msg := map[string]any{"role": role} + if len(item["content"]) > 0 { + c, err := convertContent(item["content"], role) + if err != nil { + return nil, err + } + msg["content"] = c + } + messages = append(messages, msg) + + case typ == "function_call" || typ == "custom_tool_call": + var callID, name, args string + _ = json.Unmarshal(item["call_id"], &callID) + _ = json.Unmarshal(item["name"], &name) + if typ == "custom_tool_call" { + _ = json.Unmarshal(item["input"], &args) + } else { + _ = json.Unmarshal(item["arguments"], &args) + } + if callID == "" { + callID = nextID("call") + } + messages = append(messages, map[string]any{ + "role": "assistant", + "tool_calls": []map[string]any{{ + "id": callID, "type": "function", + "function": map[string]any{"name": name, "arguments": args}, + }}, + }) + + case typ == "function_call_output" || typ == "custom_tool_call_output": + var callID string + _ = json.Unmarshal(item["call_id"], &callID) + var outputStr string + if len(item["output"]) > 0 { + if err := json.Unmarshal(item["output"], &outputStr); err != nil { + // output 可能是结构化内容(含 input_image 块),转成 text/image parts + parts, err := convertContent(item["output"], "tool") + if err == nil { + messages = append(messages, map[string]any{ + "role": "tool", "tool_call_id": callID, "content": parts, + }) + continue + } + outputStr = string(item["output"]) + } + } + messages = append(messages, map[string]any{ + "role": "tool", "tool_call_id": callID, "content": outputStr, + }) + + case typ == "reasoning": + // 历史思维链不回放进上下文(与 DeepSeek 无 tools 多轮语义一致) + continue + default: + continue // 其他类型忽略 + } + } + return messages, nil +} + +// convertContent 处理 content 的字符串/blocks 两种形态。 +func convertContent(raw json.RawMessage, role string) (any, error) { + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s, nil + } + var blocks []map[string]json.RawMessage + if err := json.Unmarshal(raw, &blocks); err != nil { + return nil, fmt.Errorf("content 格式不支持: %w", err) + } + var parts []map[string]any + var textBuf strings.Builder + for _, b := range blocks { + var typ string + _ = json.Unmarshal(b["type"], &typ) + switch typ { + case "input_text", "output_text", "text": + var t string + _ = json.Unmarshal(b["text"], &t) + textBuf.WriteString(t) + case "input_image": + var url string + _ = json.Unmarshal(b["image_url"], &url) + parts = append(parts, map[string]any{"type": "image_url", "image_url": map[string]any{"url": url}}) + default: + continue + } + } + // 纯文本时退回字符串形态(兼容性最好) + if len(parts) == 0 { + return textBuf.String(), nil + } + if textBuf.Len() > 0 { + parts = append([]map[string]any{{"type": "text", "text": textBuf.String()}}, parts...) + } + return parts, nil +} + +// ---------------- Chat → Responses 响应翻译 ---------------- + +func translateResponse(chatResp map[string]any, respID string) map[string]any { + output := []any{} + outputIdx := 0 + finishReason := "" + model, _ := chatResp["model"].(string) + + if choices, ok := chatResp["choices"].([]any); ok && len(choices) > 0 { + if c, ok := choices[0].(map[string]any); ok { + finishReason, _ = c["finish_reason"].(string) + if msg, ok := c["message"].(map[string]any); ok { + if rc, ok := msg["reasoning_content"].(string); ok && rc != "" { + output = append(output, map[string]any{ + "id": nextID("rs"), + "type": "reasoning", + "summary": []any{map[string]any{"type": "summary_text", "text": rc}}, + }) + outputIdx++ + } + if content, ok := msg["content"].(string); ok { + output = append(output, map[string]any{ + "id": nextID("msg"), + "type": "message", + "role": "assistant", + "status": "completed", + "content": []any{map[string]any{ + "type": "output_text", "text": content, "annotations": []any{}, + }}, + }) + outputIdx++ + } + if tcs, ok := msg["tool_calls"].([]any); ok { + for _, tc := range tcs { + if tcm, ok := tc.(map[string]any); ok { + fn, _ := tcm["function"].(map[string]any) + output = append(output, map[string]any{ + "id": nextID("fc"), + "type": "function_call", + "call_id": tcm["id"], + "name": fn["name"], + "arguments": fn["arguments"], + "status": "completed", + }) + outputIdx++ + } + } + } + } + } + } + + status := "completed" + var incompleteDetails any + if finishReason == "length" { + status = "incomplete" + incompleteDetails = map[string]any{"reason": "max_output_tokens"} + } + + resp := map[string]any{ + "id": respID, + "object": "response", + "created_at": time.Now().Unix(), + "status": status, + "model": model, + "output": output, + "incomplete_details": incompleteDetails, + } + if usage, ok := chatResp["usage"].(map[string]any); ok { + resp["usage"] = translateUsage(usage) + } + return resp +} + +func translateUsage(u map[string]any) map[string]any { + out := map[string]any{} + if v, ok := u["prompt_tokens"]; ok { + out["input_tokens"] = v + } + if v, ok := u["completion_tokens"]; ok { + out["output_tokens"] = v + } + if v, ok := u["total_tokens"]; ok { + out["total_tokens"] = v + } + // details 透明透传(字段名按 Responses 协议改名,内部字段原样保留): + // prompt_tokens_details → input_tokens_details(含 cached_tokens 缓存命中数) + // completion_tokens_details → output_tokens_details(含 reasoning_tokens) + if d, ok := u["prompt_tokens_details"].(map[string]any); ok && len(d) > 0 { + out["input_tokens_details"] = d + } + outDetails := map[string]any{} + if d, ok := u["completion_tokens_details"].(map[string]any); ok { + for k, v := range d { + outDetails[k] = v + } + } + // sglang 把 reasoning_tokens 放在顶层而非 completion_tokens_details 里,收编进来 + if v, ok := u["reasoning_tokens"]; ok { + outDetails["reasoning_tokens"] = v + } + if len(outDetails) > 0 { + out["output_tokens_details"] = outDetails + } + return out +} + +// ---------------- 流式翻译状态机 ---------------- + +type toolState struct { + opened bool + itemID string + callID string + name string + outIdx int + args strings.Builder +} + +type streamState struct { + w http.ResponseWriter + fl http.Flusher + seq int + respID string + model string + createdAt int64 + started bool + nextOutIdx int + reasoningOpen bool + reasoningID string + reasoningIdx int + reasoningText strings.Builder + msgOpen bool + msgID string + msgIdx int + partOpen bool + text strings.Builder + tools map[int]*toolState + toolOrder []int + finishReason string + usage map[string]any + failed bool +} + +func newStreamState(w http.ResponseWriter) *streamState { + fl, _ := w.(http.Flusher) + return &streamState{ + w: w, fl: fl, respID: nextID("resp"), + createdAt: time.Now().Unix(), + tools: map[int]*toolState{}, + } +} + +func (s *streamState) emit(eventType string, payload map[string]any) { + s.seq++ + payload["type"] = eventType + payload["sequence_number"] = s.seq + data, err := json.Marshal(payload) + if err != nil { + return + } + fmt.Fprintf(s.w, "event: %s\ndata: %s\n\n", eventType, data) + if s.fl != nil { + s.fl.Flush() + } +} + +func (s *streamState) responseSkeleton(status string) map[string]any { + return map[string]any{ + "id": s.respID, "object": "response", "created_at": s.createdAt, + "status": status, "model": s.model, "output": []any{}, + } +} + +func (s *streamState) ensureStarted() { + if s.started { + return + } + s.started = true + s.emit("response.created", map[string]any{"response": s.responseSkeleton("in_progress")}) + s.emit("response.in_progress", map[string]any{"response": s.responseSkeleton("in_progress")}) +} + +func (s *streamState) openReasoning() { + if s.reasoningOpen { + return + } + s.reasoningOpen = true + s.reasoningID = nextID("rs") + s.reasoningIdx = s.nextOutIdx + s.nextOutIdx++ + s.emit("response.output_item.added", map[string]any{ + "output_index": s.reasoningIdx, + "item": map[string]any{"id": s.reasoningID, "type": "reasoning", "summary": []any{}}, + }) + s.emit("response.reasoning_summary_part.added", map[string]any{ + "item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0, + "part": map[string]any{"type": "summary_text", "text": ""}, + }) +} + +func (s *streamState) closeReasoning() { + if !s.reasoningOpen { + return + } + s.reasoningOpen = false + full := s.reasoningText.String() + s.emit("response.reasoning_summary_text.done", map[string]any{ + "item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0, "text": full, + }) + s.emit("response.reasoning_summary_part.done", map[string]any{ + "item_id": s.reasoningID, "output_index": s.reasoningIdx, "summary_index": 0, + "part": map[string]any{"type": "summary_text", "text": full}, + }) + s.emit("response.output_item.done", map[string]any{ + "output_index": s.reasoningIdx, + "item": map[string]any{"id": s.reasoningID, "type": "reasoning", + "summary": []any{map[string]any{"type": "summary_text", "text": full}}}, + }) +} + +func (s *streamState) openMessage() { + if s.msgOpen { + return + } + s.closeReasoning() + s.msgOpen = true + s.msgID = nextID("msg") + s.msgIdx = s.nextOutIdx + s.nextOutIdx++ + s.emit("response.output_item.added", map[string]any{ + "output_index": s.msgIdx, + "item": map[string]any{"id": s.msgID, "type": "message", "role": "assistant", + "status": "in_progress", "content": []any{}}, + }) + s.partOpen = true + s.emit("response.content_part.added", map[string]any{ + "item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, + "part": map[string]any{"type": "output_text", "text": "", "annotations": []any{}}, + }) +} + +func (s *streamState) closeMessage() { + if !s.msgOpen { + return + } + s.msgOpen = false + full := s.text.String() + s.emit("response.output_text.done", map[string]any{ + "item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, "text": full, + }) + s.emit("response.content_part.done", map[string]any{ + "item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, + "part": map[string]any{"type": "output_text", "text": full, "annotations": []any{}}, + }) + s.emit("response.output_item.done", map[string]any{ + "output_index": s.msgIdx, + "item": map[string]any{"id": s.msgID, "type": "message", "role": "assistant", + "status": "completed", + "content": []any{map[string]any{"type": "output_text", "text": full, "annotations": []any{}}}}, + }) +} + +func (s *streamState) openTool(idx int, callID, name string) *toolState { + ts, ok := s.tools[idx] + if !ok { + ts = &toolState{} + s.tools[idx] = ts + s.toolOrder = append(s.toolOrder, idx) + } + if ts.opened { + return ts + } + s.closeReasoning() + s.closeMessage() + ts.opened = true + ts.itemID = nextID("fc") + ts.callID = callID + ts.name = name + ts.outIdx = s.nextOutIdx + s.nextOutIdx++ + s.emit("response.output_item.added", map[string]any{ + "output_index": ts.outIdx, + "item": map[string]any{"id": ts.itemID, "type": "function_call", + "call_id": ts.callID, "name": ts.name, "arguments": "", "status": "in_progress"}, + }) + return ts +} + +func (s *streamState) closeAll() { + s.closeReasoning() + s.closeMessage() + for _, idx := range s.toolOrder { + ts := s.tools[idx] + if !ts.opened { + continue + } + ts.opened = false + s.emit("response.function_call_arguments.done", map[string]any{ + "item_id": ts.itemID, "output_index": ts.outIdx, "arguments": ts.args.String(), + }) + s.emit("response.output_item.done", map[string]any{ + "output_index": ts.outIdx, + "item": map[string]any{"id": ts.itemID, "type": "function_call", + "call_id": ts.callID, "name": ts.name, "arguments": ts.args.String(), "status": "completed"}, + }) + } +} + +// buildFinalOutput 重建完整 output 数组(用于 response.completed/incomplete)。 +func (s *streamState) buildFinalOutput() []any { + output := make([]any, s.nextOutIdx) + if s.reasoningID != "" { + output[s.reasoningIdx] = map[string]any{"id": s.reasoningID, "type": "reasoning", + "summary": []any{map[string]any{"type": "summary_text", "text": s.reasoningText.String()}}} + } + if s.msgID != "" { + output[s.msgIdx] = map[string]any{"id": s.msgID, "type": "message", "role": "assistant", + "status": "completed", + "content": []any{map[string]any{"type": "output_text", "text": s.text.String(), "annotations": []any{}}}} + } + for _, idx := range s.toolOrder { + ts := s.tools[idx] + output[ts.outIdx] = map[string]any{"id": ts.itemID, "type": "function_call", + "call_id": ts.callID, "name": ts.name, "arguments": ts.args.String(), "status": "completed"} + } + // 防御:去掉未填充槽位 + compact := output[:0] + for _, o := range output { + if o != nil { + compact = append(compact, o) + } + } + return compact +} + +func (s *streamState) finish() { + s.closeAll() + status := "completed" + event := "response.completed" + var incompleteDetails any + if s.finishReason == "length" { + status = "incomplete" + event = "response.incomplete" + incompleteDetails = map[string]any{"reason": "max_output_tokens"} + } + resp := s.responseSkeleton(status) + resp["output"] = s.buildFinalOutput() + resp["incomplete_details"] = incompleteDetails + if s.usage != nil { + resp["usage"] = translateUsage(s.usage) + } + s.emit(event, map[string]any{"response": resp}) +} + +func (s *streamState) fail(msg string) { + if s.failed { + return + } + s.failed = true + resp := s.responseSkeleton("failed") + resp["error"] = map[string]any{"message": msg} + s.emit("response.failed", map[string]any{"response": resp}) +} + +// processChunk 处理一条 chat.completion.chunk。 +func (s *streamState) processChunk(chunk map[string]any) { + if m, ok := chunk["model"].(string); ok && m != "" { + s.model = m + } + if usage, ok := chunk["usage"].(map[string]any); ok && usage != nil { + s.usage = usage + } + choices, ok := chunk["choices"].([]any) + if !ok || len(choices) == 0 { + return + } + c, ok := choices[0].(map[string]any) + if !ok { + return + } + s.ensureStarted() + + if delta, ok := c["delta"].(map[string]any); ok { + if rc, ok := delta["reasoning_content"].(string); ok && rc != "" { + s.openReasoning() + s.reasoningText.WriteString(rc) + s.emit("response.reasoning_summary_text.delta", map[string]any{ + "item_id": s.reasoningID, "output_index": s.reasoningIdx, + "summary_index": 0, "delta": rc, + }) + } + if content, ok := delta["content"].(string); ok && content != "" { + s.openMessage() + s.text.WriteString(content) + s.emit("response.output_text.delta", map[string]any{ + "item_id": s.msgID, "output_index": s.msgIdx, "content_index": 0, "delta": content, + }) + } + if tcs, ok := delta["tool_calls"].([]any); ok { + for _, tc := range tcs { + tcm, ok := tc.(map[string]any) + if !ok { + continue + } + idxF, _ := tcm["index"].(float64) + idx := int(idxF) + callID, _ := tcm["id"].(string) + fn, _ := tcm["function"].(map[string]any) + name, _ := fn["name"].(string) + ts := s.openTool(idx, callID, name) + if ts.callID == "" { + ts.callID = callID + } + if ts.name == "" { + ts.name = name + } + if args, ok := fn["arguments"].(string); ok && args != "" { + ts.args.WriteString(args) + s.emit("response.function_call_arguments.delta", map[string]any{ + "item_id": ts.itemID, "output_index": ts.outIdx, "delta": args, + }) + } + } + } + } + if fr, ok := c["finish_reason"].(string); ok && fr != "" { + s.finishReason = fr + } +} + +// ---------------- /v1/responses 入口 ---------------- + +func handleResponses(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, `{"error":"method not allowed"}`, http.StatusMethodNotAllowed) + return + } + body, err := io.ReadAll(io.LimitReader(r.Body, 64<<20)) + if err != nil { + writeResponsesError(w, http.StatusBadRequest, "read request failed") + return + } + var req responsesRequest + if err := json.Unmarshal(body, &req); err != nil { + writeResponsesError(w, http.StatusBadRequest, "invalid JSON: "+err.Error()) + return + } + chatReq, stops, err := translateRequest(&req) + if err != nil { + writeResponsesError(w, http.StatusBadRequest, err.Error()) + return + } + chatBody, err := json.Marshal(chatReq) + if err != nil { + writeResponsesError(w, http.StatusInternalServerError, "marshal failed") + return + } + upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody) + if err != nil { + writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error()) + return + } + defer upstream.Body.Close() + + if upstream.StatusCode != http.StatusOK { + // 上游错误原样透传(sglang 的错误体本身就是 OpenAI 格式) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(upstream.StatusCode) + _, _ = io.Copy(w, upstream.Body) + return + } + + if !req.Stream { + var chatResp map[string]any + if err := json.NewDecoder(upstream.Body).Decode(&chatResp); err != nil { + writeResponsesError(w, http.StatusBadGateway, "decode upstream response failed") + return + } + if len(stops) > 0 { + truncateChoicesContent(chatResp, stops) + } + respID := nextID("resp") + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(translateResponse(chatResp, respID)) + return + } + + // 流式:chat SSE → responses SSE + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + state := newStreamState(w) + + scanner := bufio.NewReader(upstream.Body) + stopped := false + // 内容侧 stop 扫描(STOP_SCOPE=content 且带 stop 时):跨 chunk 缓冲, + // reasoning_content 不参与匹配;命中即截断并结束响应。 + var pending strings.Builder + tailKeep := 0 + if len(stops) > 0 { + tailKeep = maxStopLen(stops) - 1 + } + flushTail := func() { + if pending.Len() == 0 { + return + } + tail := pending.String() + pending.Reset() + state.processChunk(map[string]any{ + "choices": []any{map[string]any{"delta": map[string]any{"content": tail}}}, + }) + } + feed := func(chunk map[string]any) (hit bool) { + if len(stops) == 0 { + state.processChunk(chunk) + return false + } + choices, _ := chunk["choices"].([]any) + if len(choices) == 0 { + state.processChunk(chunk) + return false + } + c, _ := choices[0].(map[string]any) + if c == nil { + state.processChunk(chunk) + return false + } + // finish chunk:先把缓冲尾巴吐进状态机再放行 + if fr, _ := c["finish_reason"].(string); fr != "" { + flushTail() + state.processChunk(chunk) + return false + } + delta, _ := c["delta"].(map[string]any) + if delta == nil { + state.processChunk(chunk) + return false + } + content, _ := delta["content"].(string) + if content != "" { + pending.WriteString(content) + } + full := pending.String() + if i, _ := earliestStop(full, stops); i >= 0 { + if safe := full[:i]; safe != "" { + delta["content"] = safe + } else { + delete(delta, "content") + } + pending.Reset() + state.processChunk(chunk) + return true + } + safeLen := len(full) - tailKeep + if safeLen < 0 { + safeLen = 0 + } + if safe := full[:safeLen]; safe != "" { + delta["content"] = safe + } else { + delete(delta, "content") + } + pending.Reset() + pending.WriteString(full[safeLen:]) + state.processChunk(chunk) + return false + } + for { + line, err := scanner.ReadString('\n') + if len(line) > 0 { + line = strings.TrimRight(line, "\r\n") + if strings.HasPrefix(line, "data:") { + data := strings.TrimSpace(strings.TrimPrefix(line, "data:")) + if data == "[DONE]" { + break + } + var chunk map[string]any + if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil { + if feed(chunk) { + stopped = true + break + } + } + } + } + if err != nil { + break + } + } + if !stopped { + flushTail() // 自然结束:残余缓冲进状态机 + } + if state.started && !state.failed { + state.finish() + } else if !state.started { + state.fail("upstream closed stream without data") + } +} + +func writeResponsesError(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]any{ + "error": map[string]any{"message": msg, "type": "invalid_request_error"}, + }) +}