diff --git a/README.md b/README.md index b6567eb..4bcc4e5 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,13 @@ DeepSeek Responses API(`/v1/responses`)→ OpenAI Chat Completions 的协议 命中后 `finish_reason=stop` + `matched_stop` 回填,流式命中即断开上游触发 sglang abort。 `STOP_SCOPE=all`(默认)为原生语义:stop 透传上游,token 级截断(含思维链)。 +## 超时设计(v0.2.5 起) + +- 流式请求:300s **响应头**超时——流式首包即回响应头,300s 没回头 = 上游假死,断开。 +- 非流式请求:**不设上限**(2026-10-09 用户指令)——sglang 要等整段生成完才回响应头, + 长生成(如 max_tokens 65536 的思考链)可能远超 300s,设上限会误杀正常请求; + 仅靠客户端断连(request context)回收连接。 + ## 环境变量 | 变量 | 默认 | 说明 | diff --git a/deploy/responses-shim-b300-cw.yaml b/deploy/responses-shim-b300-cw.yaml index 2f58aa2..fc8ae16 100644 --- a/deploy/responses-shim-b300-cw.yaml +++ b/deploy/responses-shim-b300-cw.yaml @@ -29,7 +29,7 @@ spec: kubernetes.io/hostname: b300-01 containers: - name: shim - image: ymkymx/responses-shim:v0.2.3 + image: ymkymx/responses-shim:v0.2.5 imagePullPolicy: IfNotPresent env: - name: UPSTREAM_BASE diff --git a/main.go b/main.go index 2a2789b..2995177 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,8 @@ // 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、 // STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content // 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。 +// 超时:流式请求 300s 响应头超时(防上游假死);非流式不设上限——sglang 要等整段生成完才回 +// 响应头,可能远超 300s(2026-10-09 用户指令),仅靠客户端断连回收。 package main import ( @@ -26,21 +28,29 @@ import ( var ( upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002") listenAddr = envOr("LISTEN", ":8080") - httpClient = &http.Client{ - Timeout: 0, // 生成可能很长,不设总超时;靠 ResponseHeaderTimeout 防上游假死 - Transport: &http.Transport{ - MaxIdleConns: 512, - MaxIdleConnsPerHost: 256, - IdleConnTimeout: 90 * time.Second, - ResponseHeaderTimeout: 300 * time.Second, - }, - } - idCounter int64 - stopScope = envOr("STOP_SCOPE", "all") + // 流式:上游首字节即回响应头,300s ResponseHeaderTimeout 只防上游假死。 + httpClientStream = newUpstreamClient(300 * time.Second) + // 非流式:sglang 要等整个生成结束才回响应头,生成可能远超 300s——不设上限(2026-10-09 用户指令), + // 仅靠客户端断连(r.Context())回收。 + httpClientNonStream = newUpstreamClient(0) + idCounter int64 + stopScope = envOr("STOP_SCOPE", "all") ) +func newUpstreamClient(respHeaderTimeout time.Duration) *http.Client { + return &http.Client{ + Timeout: 0, // 生成可能很长,不设总超时 + Transport: &http.Transport{ + MaxIdleConns: 512, + MaxIdleConnsPerHost: 256, + IdleConnTimeout: 90 * time.Second, + ResponseHeaderTimeout: respHeaderTimeout, + }, + } +} + // shimVersion 随每次行为变更递增,启动日志可核对线上版本。 -const shimVersion = "v0.2.4" +const shimVersion = "v0.2.5" func envOr(k, def string) string { if v := os.Getenv(k); v != "" { @@ -68,7 +78,9 @@ func main() { // ---------------- 公共:转发 ---------------- // forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。 -func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Response, error) { +// streaming=true 走带 300s 响应头超时的 client(流式首包即回头,超时=上游假死); +// false 走无上限 client(非流式要等整段生成完才回头,不能设上限)。 +func forwardToUpstream(r *http.Request, path string, body []byte, streaming bool) (*http.Response, error) { req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body)) if err != nil { return nil, err @@ -78,7 +90,11 @@ func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Respons req.Header.Set("Authorization", auth) } req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层 - return httpClient.Do(req) + client := httpClientNonStream + if streaming { + client = httpClientStream + } + return client.Do(req) } // handlePassthrough 透传 /v1/chat/completions。 @@ -96,7 +112,7 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) { } stops, upstreamBody, stream := maybeStripStop(body) - resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody) + resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody, stream) if err != nil { http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway) return @@ -133,16 +149,17 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) { } // maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。 +// stream 字段始终探测(非流式要选无超时上限的上游 client),与 stop 是否摘除无关。 func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) { out = body - if stopScope != "content" { - return nil, out, false - } var req map[string]any if err := json.Unmarshal(body, &req); err != nil { return nil, out, false } stream, _ = req["stream"].(bool) + if stopScope != "content" { + return nil, out, stream + } switch v := req["stop"].(type) { case string: if v != "" { @@ -1254,7 +1271,7 @@ func handleResponses(w http.ResponseWriter, r *http.Request) { writeResponsesError(w, http.StatusInternalServerError, "marshal failed") return } - upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody) + upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody, req.Stream) if err != nil { writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error()) return