v0.2.5: 非流式请求去掉 300s 响应头超时上限(双 client:流式 300s 防假死 / 非流式无上限,2026-10-09 用户指令);b300 manifest 升 v0.2.5
This commit is contained in:
@@ -25,6 +25,13 @@ DeepSeek Responses API(`/v1/responses`)→ OpenAI Chat Completions 的协议
|
|||||||
命中后 `finish_reason=stop` + `matched_stop` 回填,流式命中即断开上游触发 sglang abort。
|
命中后 `finish_reason=stop` + `matched_stop` 回填,流式命中即断开上游触发 sglang abort。
|
||||||
`STOP_SCOPE=all`(默认)为原生语义:stop 透传上游,token 级截断(含思维链)。
|
`STOP_SCOPE=all`(默认)为原生语义:stop 透传上游,token 级截断(含思维链)。
|
||||||
|
|
||||||
|
## 超时设计(v0.2.5 起)
|
||||||
|
|
||||||
|
- 流式请求:300s **响应头**超时——流式首包即回响应头,300s 没回头 = 上游假死,断开。
|
||||||
|
- 非流式请求:**不设上限**(2026-10-09 用户指令)——sglang 要等整段生成完才回响应头,
|
||||||
|
长生成(如 max_tokens 65536 的思考链)可能远超 300s,设上限会误杀正常请求;
|
||||||
|
仅靠客户端断连(request context)回收连接。
|
||||||
|
|
||||||
## 环境变量
|
## 环境变量
|
||||||
|
|
||||||
| 变量 | 默认 | 说明 |
|
| 变量 | 默认 | 说明 |
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ spec:
|
|||||||
kubernetes.io/hostname: b300-01
|
kubernetes.io/hostname: b300-01
|
||||||
containers:
|
containers:
|
||||||
- name: shim
|
- name: shim
|
||||||
image: ymkymx/responses-shim:v0.2.3
|
image: ymkymx/responses-shim:v0.2.5
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
env:
|
env:
|
||||||
- name: UPSTREAM_BASE
|
- name: UPSTREAM_BASE
|
||||||
|
|||||||
@@ -7,6 +7,8 @@
|
|||||||
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
|
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
|
||||||
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
|
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
|
||||||
// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。
|
// 上匹配截断,避免思维链里命中 stop 串导致正文零输出——Claude stop_sequences 语义修复)。
|
||||||
|
// 超时:流式请求 300s 响应头超时(防上游假死);非流式不设上限——sglang 要等整段生成完才回
|
||||||
|
// 响应头,可能远超 300s(2026-10-09 用户指令),仅靠客户端断连回收。
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -26,21 +28,29 @@ import (
|
|||||||
var (
|
var (
|
||||||
upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002")
|
upstreamBase = envOr("UPSTREAM_BASE", "http://127.0.0.1:50002")
|
||||||
listenAddr = envOr("LISTEN", ":8080")
|
listenAddr = envOr("LISTEN", ":8080")
|
||||||
httpClient = &http.Client{
|
// 流式:上游首字节即回响应头,300s ResponseHeaderTimeout 只防上游假死。
|
||||||
Timeout: 0, // 生成可能很长,不设总超时;靠 ResponseHeaderTimeout 防上游假死
|
httpClientStream = newUpstreamClient(300 * time.Second)
|
||||||
Transport: &http.Transport{
|
// 非流式:sglang 要等整个生成结束才回响应头,生成可能远超 300s——不设上限(2026-10-09 用户指令),
|
||||||
MaxIdleConns: 512,
|
// 仅靠客户端断连(r.Context())回收。
|
||||||
MaxIdleConnsPerHost: 256,
|
httpClientNonStream = newUpstreamClient(0)
|
||||||
IdleConnTimeout: 90 * time.Second,
|
idCounter int64
|
||||||
ResponseHeaderTimeout: 300 * time.Second,
|
stopScope = envOr("STOP_SCOPE", "all")
|
||||||
},
|
|
||||||
}
|
|
||||||
idCounter int64
|
|
||||||
stopScope = envOr("STOP_SCOPE", "all")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func newUpstreamClient(respHeaderTimeout time.Duration) *http.Client {
|
||||||
|
return &http.Client{
|
||||||
|
Timeout: 0, // 生成可能很长,不设总超时
|
||||||
|
Transport: &http.Transport{
|
||||||
|
MaxIdleConns: 512,
|
||||||
|
MaxIdleConnsPerHost: 256,
|
||||||
|
IdleConnTimeout: 90 * time.Second,
|
||||||
|
ResponseHeaderTimeout: respHeaderTimeout,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
|
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
|
||||||
const shimVersion = "v0.2.4"
|
const shimVersion = "v0.2.5"
|
||||||
|
|
||||||
func envOr(k, def string) string {
|
func envOr(k, def string) string {
|
||||||
if v := os.Getenv(k); v != "" {
|
if v := os.Getenv(k); v != "" {
|
||||||
@@ -68,7 +78,9 @@ func main() {
|
|||||||
// ---------------- 公共:转发 ----------------
|
// ---------------- 公共:转发 ----------------
|
||||||
|
|
||||||
// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。
|
// forwardToUpstream 把 body POST 到上游 path,透传 Authorization 头。
|
||||||
func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Response, error) {
|
// streaming=true 走带 300s 响应头超时的 client(流式首包即回头,超时=上游假死);
|
||||||
|
// false 走无上限 client(非流式要等整段生成完才回头,不能设上限)。
|
||||||
|
func forwardToUpstream(r *http.Request, path string, body []byte, streaming bool) (*http.Response, error) {
|
||||||
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body))
|
req, err := http.NewRequestWithContext(r.Context(), http.MethodPost, upstreamBase+path, bytes.NewReader(body))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
@@ -78,7 +90,11 @@ func forwardToUpstream(r *http.Request, path string, body []byte) (*http.Respons
|
|||||||
req.Header.Set("Authorization", auth)
|
req.Header.Set("Authorization", auth)
|
||||||
}
|
}
|
||||||
req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层
|
req.Header.Set("Accept-Encoding", "identity") // SSE 不能过压缩层
|
||||||
return httpClient.Do(req)
|
client := httpClientNonStream
|
||||||
|
if streaming {
|
||||||
|
client = httpClientStream
|
||||||
|
}
|
||||||
|
return client.Do(req)
|
||||||
}
|
}
|
||||||
|
|
||||||
// handlePassthrough 透传 /v1/chat/completions。
|
// handlePassthrough 透传 /v1/chat/completions。
|
||||||
@@ -96,7 +112,7 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
stops, upstreamBody, stream := maybeStripStop(body)
|
stops, upstreamBody, stream := maybeStripStop(body)
|
||||||
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody)
|
resp, err := forwardToUpstream(r, "/v1/chat/completions", upstreamBody, stream)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway)
|
http.Error(w, fmt.Sprintf(`{"error":"upstream unreachable: %s"}`, err), http.StatusBadGateway)
|
||||||
return
|
return
|
||||||
@@ -133,16 +149,17 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。
|
// maybeStripStop 在 STOP_SCOPE=content 时从请求体摘除 stop 并返回 stop 列表;否则原样返回。
|
||||||
|
// stream 字段始终探测(非流式要选无超时上限的上游 client),与 stop 是否摘除无关。
|
||||||
func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) {
|
func maybeStripStop(body []byte) (stops []string, out []byte, stream bool) {
|
||||||
out = body
|
out = body
|
||||||
if stopScope != "content" {
|
|
||||||
return nil, out, false
|
|
||||||
}
|
|
||||||
var req map[string]any
|
var req map[string]any
|
||||||
if err := json.Unmarshal(body, &req); err != nil {
|
if err := json.Unmarshal(body, &req); err != nil {
|
||||||
return nil, out, false
|
return nil, out, false
|
||||||
}
|
}
|
||||||
stream, _ = req["stream"].(bool)
|
stream, _ = req["stream"].(bool)
|
||||||
|
if stopScope != "content" {
|
||||||
|
return nil, out, stream
|
||||||
|
}
|
||||||
switch v := req["stop"].(type) {
|
switch v := req["stop"].(type) {
|
||||||
case string:
|
case string:
|
||||||
if v != "" {
|
if v != "" {
|
||||||
@@ -1254,7 +1271,7 @@ func handleResponses(w http.ResponseWriter, r *http.Request) {
|
|||||||
writeResponsesError(w, http.StatusInternalServerError, "marshal failed")
|
writeResponsesError(w, http.StatusInternalServerError, "marshal failed")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody)
|
upstream, err := forwardToUpstream(r, "/v1/chat/completions", chatBody, req.Stream)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error())
|
writeResponsesError(w, http.StatusBadGateway, "upstream unreachable: "+err.Error())
|
||||||
return
|
return
|
||||||
|
|||||||
Reference in New Issue
Block a user