Fix bench_serving non-stream reasoning content (#25298)

This commit is contained in:
Ratish P
2026-05-20 18:41:46 +00:00
committed by GitHub
parent 1f209b4433
commit 5e7bf73757
2 changed files with 131 additions and 6 deletions
+7 -6
View File
@@ -131,6 +131,10 @@ def get_request_headers() -> Dict[str, str]:
return headers
def _combine_openai_chat_content(message: Dict[str, Any]) -> str:
return (message.get("reasoning_content") or "") + (message.get("content") or "")
def wait_for_endpoint(url: str, timeout_sec: int = 60) -> bool:
"""Wait for the server to become ready by polling the given URL."""
print(f"Waiting up to {timeout_sec}s for {url} to become ready...")
@@ -440,9 +444,8 @@ async def async_request_openai_chat_completions(
if args.disable_stream:
# Non-streaming response
response_json = await response.json()
output.generated_text = response_json["choices"][0]["message"][
"content"
]
message = response_json["choices"][0]["message"]
output.generated_text = _combine_openai_chat_content(message)
output.success = True
output.latency = time.perf_counter() - st
output.ttft = (
@@ -477,9 +480,7 @@ async def async_request_openai_chat_completions(
# Reasoning models stream thoughts via
# `reasoning_content`; count them like content.
delta = choices[0].get("delta") or {}
content = (delta.get("reasoning_content") or "") + (
delta.get("content") or ""
)
content = _combine_openai_chat_content(delta)
if content:
timestamp = time.perf_counter()