minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[Bench] Fix bench_serving missing reasoning_content stream chunks (#23954)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-04-30 15:00:27 -07:00
committed by GitHub
parent 340efca244
commit 989a16187d
2 changed files with 225 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/bench_serving.py
+15 -8
View File
@@ -462,10 +462,22 @@ async def async_request_openai_chat_completions(
pass
else:
data = json.loads(chunk)
# Check for usage info in final chunks. OpenAI-compatible
# servers may emit usage-only chunks with choices=[].
output_len = (data.get("usage") or {}).get(
"completion_tokens", output_len
)
# Check if this chunk contains content
delta = data.get("choices", [{}])[0].get("delta", {})
content = delta.get("content", "")
choices = data.get("choices") or []
if not choices:
continue
# Reasoning models stream thoughts via
# `reasoning_content`; count them like content.
delta = choices[0].get("delta") or {}
content = (delta.get("reasoning_content") or "") + (
delta.get("content") or ""
)
if content:
timestamp = time.perf_counter()
@@ -484,11 +496,6 @@ async def async_request_openai_chat_completions(
most_recent_timestamp = timestamp
generated_text += content
# Check for usage info in final chunk
output_len = (data.get("usage") or {}).get(
"completion_tokens", output_len
)
output.generated_text = generated_text
output.success = True
output.latency = latency