v0.2.5: 非流式请求去掉 300s 响应头超时上限(双 client:流式 300s 防假死 / 非流式无上限,2026-10-09 用户指令);b300 manifest 升 v0.2.5
This commit is contained in:
@@ -25,6 +25,13 @@ DeepSeek Responses API(`/v1/responses`)→ OpenAI Chat Completions 的协议
|
||||
命中后 `finish_reason=stop` + `matched_stop` 回填,流式命中即断开上游触发 sglang abort。
|
||||
`STOP_SCOPE=all`(默认)为原生语义:stop 透传上游,token 级截断(含思维链)。
|
||||
|
||||
## 超时设计(v0.2.5 起)
|
||||
|
||||
- 流式请求:300s **响应头**超时——流式首包即回响应头,300s 没回头 = 上游假死,断开。
|
||||
- 非流式请求:**不设上限**(2026-10-09 用户指令)——sglang 要等整段生成完才回响应头,
|
||||
长生成(如 max_tokens 65536 的思考链)可能远超 300s,设上限会误杀正常请求;
|
||||
仅靠客户端断连(request context)回收连接。
|
||||
|
||||
## 环境变量
|
||||
|
||||
| 变量 | 默认 | 说明 |
|
||||
|
||||
@@ -29,7 +29,7 @@ spec:
|
||||
kubernetes.io/hostname: b300-01
|
||||
containers:
|
||||
- name: shim
|
||||
image: ymkymx/responses-shim:v0.2.3
|
||||
image: ymkymx/responses-shim:v0.2.5
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: UPSTREAM_BASE
|
||||
|
||||
@@ -7,6 +7,8 @@
|
||||
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
|
||||
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
|
||||
// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。
|
||||
// 超时:流式请求 300s 响应头超时(防上游假死);非流式不设上限——sglang 要等整段生成完才回
|
||||
// 响应头,可能远超 300s(2026-10-09 用户指令),仅靠客户端断连回收。
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -26,21 +28,29 @@ import (
|
||||
var (
|
||||
upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002")
|
||||
listenAddr = envOr("LISTEN", ":8080")
|
||||
httpClient = &http.Client{
|
||||
Timeout: 0, // 生成可能很长,不设总超时;靠 ResponseHeaderTimeout 防上游假死
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 512,
|
||||
MaxIdleConnsPerHost: 256,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ResponseHeaderTimeout: 300 * time.Second,
|
||||
},
|
||||
}
|
||||
idCounter int64
|
||||
stopScope = envOr("STOP_SCOPE", "all")
|
||||
// 流式:上游首字节即回响应头,300s ResponseHeaderTimeout 只防上游假死。
|
||||
httpClientStream = newUpstreamClient(300 * time.Second)
|
||||
// 非流式:sglang 要等整个生成结束才回响应头,生成可能远超 300s——不设上限(2026-10-09 用户指令),
|
||||
// 仅靠客户端断连(r.Context())回收。
|
||||
httpClientNonStream = newUpstreamClient(0)
|
||||
idCounter int64
|
||||
stopScope = envOr("STOP_SCOPE", "all")
|
||||
)
|
||||
|
||||
func newUpstreamClient(respHeaderTimeout time.Duration) *http.Client {
|
||||
return &http.Client{
|
||||
Timeout: 0, // 生成可能很长,不设总超时
|
||||
Transport: &http.Transport{
|
||||
MaxIdleConns: 512,
|
||||
MaxIdleConnsPerHost: 256,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
ResponseHeaderTimeout: respHeaderTimeout,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
|
||||
const shimVersion = "v0.2.4"
|
||||
const shimVersion = "v0.2.5"
|
||||
|
||||
func envOr(k, def string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
@@ -68,7 +78,9 @@ func main() {
|
||||
// ---------------- 公共:转发 ----------------
|
||||
|
||||
// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。
|
||||
func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Response, error) {
|
||||
// streaming=true 走带 300s 响应头超时的 client(流式首包即回头,超时=上游假死);
|
||||
// false 走无上限 client(非流式要等整段生成完才回头,不能设上限)。
|
||||
func forwardToUpstream(r *http.Request, path string, body []byte, streaming bool) (*http.Response, error) {
|
||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -78,7 +90,11 @@ func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Respons
|
||||
req.Header.Set("Authorization", auth)
|
||||
}
|
||||
req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层
|
||||
return httpClient.Do(req)
|
||||
client := httpClientNonStream
|
||||
if streaming {
|
||||
client = httpClientStream
|
||||
}
|
||||
return client.Do(req)
|
||||
}
|
||||
|
||||
// handlePassthrough 透传 /v1/chat/completions。
|
||||
@@ -96,7 +112,7 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
stops, upstreamBody, stream := maybeStripStop(body)
|
||||
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody)
|
||||
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody, stream)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway)
|
||||
return
|
||||
@@ -133,16 +149,17 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。
|
||||
// stream 字段始终探测(非流式要选无超时上限的上游 client),与 stop 是否摘除无关。
|
||||
func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) {
|
||||
out = body
|
||||
if stopScope != "content" {
|
||||
return nil, out, false
|
||||
}
|
||||
var req map[string]any
|
||||
if err := json.Unmarshal(body, &req); err != nil {
|
||||
return nil, out, false
|
||||
}
|
||||
stream, _ = req["stream"].(bool)
|
||||
if stopScope != "content" {
|
||||
return nil, out, stream
|
||||
}
|
||||
switch v := req["stop"].(type) {
|
||||
case string:
|
||||
if v != "" {
|
||||
@@ -1254,7 +1271,7 @@ func handleResponses(w http.ResponseWriter, r *http.Request) {
|
||||
writeResponsesError(w, http.StatusInternalServerError, "marshal failed")
|
||||
return
|
||||
}
|
||||
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody)
|
||||
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody, req.Stream)
|
||||
if err != nil {
|
||||
writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error())
|
||||
return
|
||||
|
||||
Reference in New Issue
Block a user