v0.2.5: 非流式请求去掉 300s 响应头超时上限(双 client:流式 300s 防假死 / 非流式无上限,2026-10-09 用户指令);b300 manifest 升 v0.2.5

This commit is contained in:
2026-10-09 11:35:50 +08:00
parent 3af935acd8
commit 39af1071b8
3 changed files with 44 additions and 20 deletions
+7
View File
@@ -25,6 +25,13 @@ DeepSeek Responses API(`/v1/responses`)→ OpenAI Chat Completions 的协议
命中后 `finish_reason=stop` + `matched_stop` 回填,流式命中即断开上游触发 sglang abort。
`STOP_SCOPE=all`(默认)为原生语义:stop 透传上游,token 级截断(含思维链)。
## 超时设计(v0.2.5 起)
- 流式请求:300s **响应头**超时——流式首包即回响应头,300s 没回头 = 上游假死,断开。
- 非流式请求:**不设上限**(2026-10-09 用户指令)——sglang 要等整段生成完才回响应头,
长生成(如 max_tokens 65536 的思考链)可能远超 300s,设上限会误杀正常请求;
仅靠客户端断连(request context)回收连接。
## 环境变量
| 变量 | 默认 | 说明 |
+1 -1
View File
@@ -29,7 +29,7 @@ spec:
kubernetes.io/hostname: b300-01
containers:
- name: shim
image: ymkymx/responses-shim:v0.2.3
image: ymkymx/responses-shim:v0.2.5
imagePullPolicy: IfNotPresent
env:
- name: UPSTREAM_BASE
+36 -19
View File
@@ -7,6 +7,8 @@
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。
// 超时:流式请求 300s 响应头超时(防上游假死);非流式不设上限——sglang 要等整段生成完才回
// 响应头,可能远超 300s(2026-10-09 用户指令),仅靠客户端断连回收。
package main
import (
@@ -26,21 +28,29 @@ import (
var (
upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002")
listenAddr = envOr("LISTEN", ":8080")
httpClient = &http.Client{
Timeout: 0, // 生成可能很长,不设总超时;靠 ResponseHeaderTimeout 防上游假死
Transport: &http.Transport{
MaxIdleConns: 512,
MaxIdleConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: 300 * time.Second,
},
}
idCounter int64
stopScope = envOr("STOP_SCOPE", "all")
// 流式:上游首字节即回响应头,300s ResponseHeaderTimeout 只防上游假死。
httpClientStream = newUpstreamClient(300 * time.Second)
// 非流式:sglang 要等整个生成结束才回响应头,生成可能远超 300s——不设上限(2026-10-09 用户指令),
// 仅靠客户端断连(r.Context())回收。
httpClientNonStream = newUpstreamClient(0)
idCounter int64
stopScope = envOr("STOP_SCOPE", "all")
)
func newUpstreamClient(respHeaderTimeout time.Duration) *http.Client {
return &http.Client{
Timeout: 0, // 生成可能很长,不设总超时
Transport: &http.Transport{
MaxIdleConns: 512,
MaxIdleConnsPerHost: 256,
IdleConnTimeout: 90 * time.Second,
ResponseHeaderTimeout: respHeaderTimeout,
},
}
}
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
const shimVersion = "v0.2.4"
const shimVersion = "v0.2.5"
func envOr(k, def string) string {
if v := os.Getenv(k); v != "" {
@@ -68,7 +78,9 @@ func main() {
// ---------------- 公共:转发 ----------------
// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。
func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Response, error) {
// streaming=true 走带 300s 响应头超时的 client(流式首包即回头,超时=上游假死);
// false 走无上限 client(非流式要等整段生成完才回头,不能设上限)。
func forwardToUpstream(r *http.Request, path string, body []byte, streaming bool) (*http.Response, error) {
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body))
if err != nil {
return nil, err
@@ -78,7 +90,11 @@ func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Respons
req.Header.Set("Authorization", auth)
}
req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层
return httpClient.Do(req)
client := httpClientNonStream
if streaming {
client = httpClientStream
}
return client.Do(req)
}
// handlePassthrough 透传 /v1/chat/completions。
@@ -96,7 +112,7 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
}
stops, upstreamBody, stream := maybeStripStop(body)
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody)
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody, stream)
if err != nil {
http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway)
return
@@ -133,16 +149,17 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
}
// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。
// stream 字段始终探测(非流式要选无超时上限的上游 client),与 stop 是否摘除无关。
func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) {
out = body
if stopScope != "content" {
return nil, out, false
}
var req map[string]any
if err := json.Unmarshal(body, &req); err != nil {
return nil, out, false
}
stream, _ = req["stream"].(bool)
if stopScope != "content" {
return nil, out, stream
}
switch v := req["stop"].(type) {
case string:
if v != "" {
@@ -1254,7 +1271,7 @@ func handleResponses(w http.ResponseWriter, r *http.Request) {
writeResponsesError(w, http.StatusInternalServerError, "marshal failed")
return
}
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody)
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody, req.Stream)
if err != nil {
writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error())
return