v0.2.4: chat 透传 usage 归一化——sglang 顶层 reasoning_tokens 收编进 completion_tokens_details(OpenAI 标准位置),非流式/流式/stop 路径全覆盖;启动日志带版本号
This commit is contained in:
@@ -1,7 +1,8 @@
|
||||
// responses-shim: DeepSeek Responses API (/v1/responses) → OpenAI Chat Completions 协议转换 shim
|
||||
// 后置部署:new-api 渠道上游指向本服务,本服务再转发到 sglang router。
|
||||
// - POST /v1/responses → 翻译成 /v1/chat/completions 请求,响应/流式事件翻译回 Responses 协议
|
||||
// - POST /v1/chat/completions → 纯透传(字节级转发,含 SSE)
|
||||
// - POST /v1/chat/completions → 透传(仅做 usage 归一化:sglang 顶层 reasoning_tokens
|
||||
// 收编进 completion_tokens_details 标准位置;解析失败的报文回退字节级透传)
|
||||
// - GET /healthz → 200
|
||||
// 仅使用标准库。配置通过环境变量:UPSTREAM_BASE(默认 http://127.0.0.1:50002)、LISTEN(默认 :8080)、
|
||||
// STOP_SCOPE=all|content(默认 all=原生透传 stop;content=stop 不下发上游,由 shim 只在 content
|
||||
@@ -38,6 +39,9 @@ var (
|
||||
stopScope = envOr("STOP_SCOPE", "all")
|
||||
)
|
||||
|
||||
// shimVersion 随每次行为变更递增,启动日志可核对线上版本。
|
||||
const shimVersion = "v0.2.4"
|
||||
|
||||
func envOr(k, def string) string {
|
||||
if v := os.Getenv(k); v != "" {
|
||||
return v
|
||||
@@ -57,7 +61,7 @@ func main() {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
})
|
||||
log.Printf("responses-shim listening on %s, upstream=%s", listenAddr, upstreamBase)
|
||||
log.Printf("responses-shim %s listening on %s, upstream=%s", shimVersion, listenAddr, upstreamBase)
|
||||
log.Fatal(http.ListenAndServe(listenAddr, mux))
|
||||
}
|
||||
|
||||
@@ -102,6 +106,17 @@ func handlePassthrough(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
if len(stops) == 0 || resp.StatusCode != http.StatusOK {
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
if resp.StatusCode == http.StatusOK {
|
||||
ct := resp.Header.Get("Content-Type")
|
||||
switch {
|
||||
case strings.Contains(ct, "text/event-stream"):
|
||||
streamNormalizeUsage(w, resp.Body)
|
||||
return
|
||||
case strings.Contains(ct, "json"):
|
||||
normalizeUsageJSON(w, resp.Body)
|
||||
return
|
||||
}
|
||||
}
|
||||
streamCopy(w, resp.Body)
|
||||
return
|
||||
}
|
||||
@@ -203,6 +218,7 @@ func applyStopNonStream(w http.ResponseWriter, src io.Reader, stops []string) {
|
||||
return
|
||||
}
|
||||
truncateChoicesContent(resp, stops)
|
||||
normalizeUsage(resp)
|
||||
_ = json.NewEncoder(w).Encode(resp)
|
||||
}
|
||||
|
||||
@@ -341,6 +357,7 @@ func applyStopStream(w http.ResponseWriter, src io.Reader, stops []string) {
|
||||
var chunk map[string]any
|
||||
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
|
||||
mstr, hit := sc.feed(chunk)
|
||||
normalizeUsage(chunk)
|
||||
sc.emitChunk(chunk)
|
||||
if hit {
|
||||
matched, matchStr = true, mstr
|
||||
@@ -392,6 +409,83 @@ func streamCopy(w http.ResponseWriter, src io.Reader) {
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- 公共:usage 归一化 ----------------
|
||||
|
||||
// normalizeUsage 把 sglang 放在 usage 顶层的 reasoning_tokens 收编进
|
||||
// completion_tokens_details(OpenAI 标准位置;new-api 等下游只认标准位置,
|
||||
// 否则 reasoning_tokens 恒显示 0)。只增不改:顶层原字段保留。
|
||||
func normalizeUsage(m map[string]any) {
|
||||
usage, ok := m["usage"].(map[string]any)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
rt, ok := usage["reasoning_tokens"].(float64)
|
||||
if !ok || rt <= 0 {
|
||||
return
|
||||
}
|
||||
ctd, _ := usage["completion_tokens_details"].(map[string]any)
|
||||
if ctd == nil {
|
||||
ctd = map[string]any{}
|
||||
usage["completion_tokens_details"] = ctd
|
||||
}
|
||||
if existing, _ := ctd["reasoning_tokens"].(float64); existing <= 0 {
|
||||
ctd["reasoning_tokens"] = rt
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeUsageJSON 非流式透传:仅归一化 usage 后写回;解析失败回退字节透传。
|
||||
func normalizeUsageJSON(w http.ResponseWriter, src io.Reader) {
|
||||
body, err := io.ReadAll(src)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(body, &m); err != nil {
|
||||
_, _ = w.Write(body)
|
||||
return
|
||||
}
|
||||
normalizeUsage(m)
|
||||
out, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
_, _ = w.Write(body)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(out)
|
||||
}
|
||||
|
||||
// streamNormalizeUsage 流式透传:逐行扫描 SSE,含 usage 对象的 data 帧归一化后
|
||||
// 重写,其余行原样透传(保序保边界);解析失败的帧同样原样透传。
|
||||
func streamNormalizeUsage(w http.ResponseWriter, src io.Reader) {
|
||||
fl, _ := w.(http.Flusher)
|
||||
reader := bufio.NewReader(src)
|
||||
for {
|
||||
line, err := reader.ReadString('\n')
|
||||
trimmed := strings.TrimRight(line, "\r\n")
|
||||
out := line
|
||||
if strings.HasPrefix(trimmed, "data:") {
|
||||
data := strings.TrimSpace(strings.TrimPrefix(trimmed, "data:"))
|
||||
if data != "[DONE]" {
|
||||
var chunk map[string]any
|
||||
if jerr := json.Unmarshal([]byte(data), &chunk); jerr == nil {
|
||||
if _, has := chunk["usage"].(map[string]any); has {
|
||||
normalizeUsage(chunk)
|
||||
if b, merr := json.Marshal(chunk); merr == nil {
|
||||
out = "data: " + string(b) + "\n"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_, _ = w.Write([]byte(out))
|
||||
if fl != nil {
|
||||
fl.Flush()
|
||||
}
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------- Responses → Chat 请求翻译 ----------------
|
||||
|
||||
type responsesRequest struct {
|
||||
|
||||
Reference in New Issue
Block a user