minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

Responses support (#32689)

Browse Source 
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Harmya Bhatt <harmyacs@gmail.com>
Co-authored-by: harmya <harmya@modal.com>
Co-authored-by: Xinyuan <xinyuan@radixark.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Liangsheng Yin
2026-08-07 13:21:46 -07:00
committed by GitHub
co-authored by github-actions[bot] Harmya Bhatt harmya Xinyuan Xinyuan Tong Xinyuan Tong
parent 6c7498113f
commit 3c51e29deb
9 changed files with 1135 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/entrypoints/context.py
+11
View File
@@ -79,6 +79,7 @@ class HarmonyContext(ConversationContext):
self.num_cached_tokens = 0
self.num_output_tokens = 0
self.num_reasoning_tokens = 0
self.finish_reason = None
def append_output(self, output) -> None:
if isinstance(output, dict) and "output_ids" in output:
@@ -97,12 +98,20 @@ class HarmonyContext(ConversationContext):
self.num_cached_tokens = meta_info["cached_tokens"]
if "completion_tokens" in meta_info:
self.num_output_tokens += meta_info["completion_tokens"]
self._record_finish_reason(meta_info)
else:
output_msgs = output
self._messages.extend(output_msgs)
def _record_finish_reason(self, meta_info: dict) -> None:
# Last non-null wins: a builtin-tool continuation turn supersedes the
# reason recorded for the turn before it.
reason = meta_info.get("finish_reason")
if reason is not None:
self.finish_reason = reason
@property
def messages(self) -> list:
return self._messages
@@ -210,6 +219,8 @@ class StreamingHarmonyContext(HarmonyContext):
new_token_ids = output_token_ids
self.num_processed_tokens += len(output_token_ids)
self._record_finish_reason(meta_info)
for token_id in new_token_ids:
self.parser.process(token_id)
python/sglang/srt/entrypoints/harmony_utils.py
+14 -6
View File
@@ -5,7 +5,7 @@
import datetime
import logging
from collections.abc import Iterable
from typing import Literal, Optional, Union
from typing import Optional, Union
import orjson
from openai.types.responses import (
@@ -40,7 +40,10 @@ from openai_harmony import (
load_harmony_encoding,
)
from sglang.srt.entrypoints.openai.protocol import ResponseInputOutputItem
from sglang.srt.entrypoints.openai.protocol import (
ReasoningEffortTier,
ResponseInputOutputItem,
)
from sglang.srt.utils import random_uuid
logger = logging.getLogger(__name__)
@@ -49,6 +52,11 @@ REASONING_EFFORT = {
"high": ReasoningEffort.HIGH,
"medium": ReasoningEffort.MEDIUM,
"low": ReasoningEffort.LOW,
# harmony has only these three, so outer tiers clamp inward. "none" is left
# out so it falls through to the harmony default.
"minimal": ReasoningEffort.LOW,
"xhigh": ReasoningEffort.HIGH,
"max": ReasoningEffort.HIGH,
}
_harmony_encoding = None
@@ -63,7 +71,7 @@ def get_encoding():
def get_system_message(
model_identity: Optional[str] = None,
reasoning_effort: Optional[Literal["high", "medium", "low"]] = None,
reasoning_effort: Optional[ReasoningEffortTier] = None,
start_date: Optional[str] = None,
browser_description: Optional[str] = None,
python_description: Optional[str] = None,
@@ -72,9 +80,9 @@ def get_system_message(
if model_identity is not None:
sys_msg_content = sys_msg_content.with_model_identity(model_identity)
if reasoning_effort is not None:
sys_msg_content = sys_msg_content.with_reasoning_effort(
REASONING_EFFORT[reasoning_effort]
)
effort = REASONING_EFFORT.get(reasoning_effort)
if effort is not None:
sys_msg_content = sys_msg_content.with_reasoning_effort(effort)
if start_date is None:
start_date = datetime.datetime.now().strftime("%Y-%m-%d")
sys_msg_content = sys_msg_content.with_conversation_start_date(start_date)
python/sglang/srt/entrypoints/openai/protocol.py
+131 -9
View File
@@ -42,12 +42,18 @@ from openai.types.responses import (
ResponseOutputMessage,
ResponseOutputText,
ResponseReasoningItem,
ResponseTextConfig,
)
from openai.types.responses.response import ToolChoice
from openai.types.responses.response_format_text_json_schema_config import (
ResponseFormatTextJSONSchemaConfig,
)
from openai.types.shared.response_format_json_object import ResponseFormatJSONObject
from pydantic import (
BaseModel,
ConfigDict,
Field,
field_serializer,
field_validator,
model_serializer,
model_validator,
@@ -710,6 +716,9 @@ class ToolChoice(BaseModel):
ReasoningEffortTier = Literal[
"none", "minimal", "low", "medium", "high", "xhigh", "max"
]
# The typed /v1/responses stream events validate against OpenAI's narrower
# ``Reasoning.effort``; echoing a tier outside this set kills the stream.
ECHOABLE_REASONING_EFFORTS = frozenset({"minimal", "low", "medium", "high"})
# Chat Completions and /v1/tokenize additionally accept a fine-grained float in
# [0.0, 0.99] as an sglang extension (not part of the OpenAI schema, so the
# /v1/responses surface deliberately keeps the string tiers only). Single-sourced
@@ -1509,7 +1518,8 @@ class ResponsesRequest(BaseModel):
store: Optional[bool] = True
stream: Optional[bool] = False
temperature: Optional[float] = None
tool_choice: Literal["auto", "required", "none"] = "auto"
text: Optional[ResponseTextConfig] = None
tool_choice: Union[Literal["auto", "required", "none"], Dict[str, Any]] = "auto"
tools: List[ResponseTool] = Field(default_factory=list)
top_logprobs: Optional[int] = 0
top_p: Optional[float] = None
@@ -1517,6 +1527,7 @@ class ResponsesRequest(BaseModel):
user: Optional[str] = None
# Extra SGLang parameters
chat_template_kwargs: Optional[Dict[str, Any]] = None
request_id: str = Field(
default_factory=lambda: f"resp_{uuid.uuid4().hex}",
description="The request_id related to this request. If the caller does not set it, a random uuid will be generated.",
@@ -1548,6 +1559,34 @@ class ResponsesRequest(BaseModel):
"repetition_penalty": 1.0,
}
@model_validator(mode="before")
@classmethod
def normalize_reasoning_to_thinking(cls, values):
"""Turn reasoning.effort == "none" into a thinking toggle, as
ChatCompletionRequest does. Both keys are set: families differ
("thinking" for deepseek/kimi, "enable_thinking" for qwen3/glm)."""
if not isinstance(values, dict):
return values
# mode="before", so reasoning is still raw: a dict from JSON, or a
# built param object when constructed in Python.
r = values.get("reasoning")
effort = None
if isinstance(r, dict):
effort = r.get("effort") or r.get("reasoning_effort")
elif isinstance(r, ResponseReasoningParam):
effort = r.effort
if effort == "none":
existing = values.get("chat_template_kwargs")
existing = existing if isinstance(existing, dict) else {}
# existing last: an explicit caller value wins.
values["chat_template_kwargs"] = {
"thinking": False,
"enable_thinking": False,
**existing,
}
return values
@model_validator(mode="before")
@classmethod
def normalize_responses_input(cls, values):
@@ -1569,11 +1608,17 @@ class ResponsesRequest(BaseModel):
if not isinstance(item, dict):
return item
# an output item replayed into input carries a string id; without this it'd
# be read as an item-reference, drop its content, and fail as an empty {}.
# input-item ids aren't resolved server-side, so just drop it.
if isinstance(item.get("id"), str) and item.get("content") is not None:
item = {k: v for k, v in item.items() if k != "id"}
content = item.get("content")
if not isinstance(content, list):
return item
item = item.copy()
item = dict(item)
item["content"] = [
ResponsesRequest._normalize_content_part_for_validation(part)
for part in content
@@ -1593,6 +1638,40 @@ class ResponsesRequest(BaseModel):
part["detail"] = "auto"
return part
@staticmethod
def _json_schema_from_text_format(
text: Optional[ResponseTextConfig],
) -> Optional[str]:
"""Map a Responses ``text.format`` to a json_schema string, or None when
no JSON constraint applies (``text`` and anything unrecognized)."""
response_format = text.format if text is not None else None
if isinstance(response_format, ResponseFormatJSONObject):
return '{"type": "object"}'
if not isinstance(response_format, ResponseFormatTextJSONSchemaConfig):
return None
schema = response_format.schema_
return convert_json_schema_to_str(schema) if schema is not None else None
def is_include_output_logprobs(self) -> bool:
return bool(self.include and "message.output_text.logprobs" in self.include)
def has_json_schema_constraint(self) -> bool:
return self._json_schema_from_text_format(self.text) is not None
def effective_tool_choice(self) -> Union[str, Dict[str, Any]]:
"""``tool_choice`` reduced to what the server can actually honor: of the
object forms only a named ``function`` survives, the rest (web_search,
mcp, ...) can't be forced through the tool-call parser."""
tool_choice = self.tool_choice
if not isinstance(tool_choice, dict):
return tool_choice
name = tool_choice.get("name") or (tool_choice.get("function") or {}).get(
"name"
)
if tool_choice.get("type") == "function" and name:
return {"type": "function", "name": name}
return "auto"
def to_sampling_params(
self,
default_max_tokens: int,
@@ -1645,6 +1724,10 @@ class ResponsesRequest(BaseModel):
if key not in params or params[key] is None:
params[key] = value
json_schema = self._json_schema_from_text_format(self.text)
if json_schema is not None:
params["json_schema"] = json_schema
has_existing_constraints = (
params.get("regex")
or params.get("ebnf")
@@ -1655,7 +1738,8 @@ class ResponsesRequest(BaseModel):
# Refuse rather than silently drop the tool-call grammar.
raise ValueError(
"Cannot combine tool calls with constrained decoding "
"(regex / ebnf / structural_tag / json_schema). Remove one."
"(text.format / regex / ebnf / structural_tag / json_schema). "
"Remove one."
)
if tool_call_constraint:
constraint_type, constraint_value = tool_call_constraint
@@ -1688,10 +1772,12 @@ class ResponsesResponse(BaseModel):
output: List[
Union[ResponseOutputItem, ResponseReasoningItem, ResponseFunctionToolCall]
] = Field(default_factory=list)
status: Literal["queued", "in_progress", "completed", "failed", "cancelled"]
status: Literal[
"queued", "in_progress", "completed", "incomplete", "failed", "cancelled"
]
usage: Optional[UsageInfo] = None
parallel_tool_calls: bool = True
tool_choice: str = "auto"
tool_choice: Union[str, Dict[str, Any]] = "auto"
tools: List[ResponseTool] = Field(default_factory=list)
# OpenAI compatibility fields. not all are used at the moment.
@@ -1714,6 +1800,28 @@ class ResponsesResponse(BaseModel):
user: Optional[str] = None
metadata: Optional[Dict[str, Any]] = None
@field_serializer("usage")
def _serialize_usage(self, usage: Optional[UsageInfo], _info):
"""Emit the Responses usage shape, not the chat one UsageInfo carries."""
if usage is None:
return None
cached = (
usage.prompt_tokens_details.cached_tokens
if usage.prompt_tokens_details
else 0
)
return {
"input_tokens": usage.prompt_tokens,
"input_tokens_details": {
"cached_tokens": cached,
# required (no default) in the SDK's InputTokensDetails model
"cache_write_tokens": 0,
},
"output_tokens": usage.completion_tokens or 0,
"output_tokens_details": {"reasoning_tokens": usage.reasoning_tokens or 0},
"total_tokens": usage.total_tokens,
}
@classmethod
def from_request(
cls,
@@ -1761,7 +1869,13 @@ class ResponsesResponse(BaseModel):
return False
return True
text_format = {"format": {"type": "text"}} if _is_text_only(output) else None
if request.text is not None:
# by_alias keeps the wire key "schema" rather than the attr schema_.
text_format = request.text.model_dump(by_alias=True, exclude_none=True)
else:
text_format = (
{"format": {"type": "text"}} if _is_text_only(output) else None
)
return cls(
id=request.request_id,
@@ -1775,16 +1889,23 @@ class ResponsesResponse(BaseModel):
if request.parallel_tool_calls is not None
else True
),
tool_choice=request.tool_choice,
tool_choice=request.effective_tool_choice(),
tools=request.tools,
# fields for parity with v1/responses
error=None,
incomplete_details=None,
incomplete_details=(
{"reason": "max_output_tokens"} if status == "incomplete" else None
),
instructions=request.instructions,
max_output_tokens=request.max_output_tokens,
previous_response_id=request.previous_response_id, # TODO(v): ensure this is propagated if retrieved from store
reasoning={
"effort": request.reasoning.effort if request.reasoning else None,
"effort": (
request.reasoning.effort
if request.reasoning
and request.reasoning.effort in ECHOABLE_REASONING_EFFORTS
else None
),
"summary": None, # unused
},
store=request.store,
@@ -1814,6 +1935,7 @@ class MessageProcessingResult:
modalities: List[str]
stop: List[str]
tool_call_constraint: Optional[ToolCallConstraint] = None
skip_special_tokens: bool = True
require_reasoning: bool = False
python/sglang/srt/entrypoints/openai/serving_responses.py
+287 -153
View File
@@ -23,6 +23,7 @@ from openai.types.responses import (
ResponseReasoningItem,
)
from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall
from openai.types.responses.response_output_text import Logprob, LogprobTopLogprob
from openai.types.responses.response_reasoning_item import (
Content as ResponseReasoningTextContent,
)
@@ -67,6 +68,7 @@ from sglang.srt.entrypoints.openai.protocol import (
)
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer
from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs
from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.function_call.json_array_parser import JsonArrayParser
from sglang.srt.managers.io_struct import GenerateReqInput
@@ -84,6 +86,54 @@ class _MediaInputValidationError(ValueError):
pass
def _build_output_text_logprobs(meta_info: dict) -> list[Logprob]:
"""Reshape decoded ``meta_info`` logprobs into the Responses logprob type,
covering every generated token."""
decoded = to_openai_style_logprobs(
output_token_logprobs=meta_info.get("output_token_logprobs"),
output_top_logprobs=meta_info.get("output_top_logprobs"),
)
top_lists = decoded.top_logprobs or []
logprobs: list[Logprob] = []
for index, (token, logprob) in enumerate(
zip(decoded.tokens, decoded.token_logprobs)
):
top_entry = top_lists[index] if index < len(top_lists) else None
top_logprobs = [
LogprobTopLogprob(
token=top_token,
logprob=top_logprob,
bytes=list(top_token.encode("utf-8")),
)
for top_token, top_logprob in (top_entry or {}).items()
]
logprobs.append(
Logprob(
token=token,
logprob=logprob,
bytes=list(token.encode("utf-8")),
top_logprobs=top_logprobs,
)
)
return logprobs
def _should_emit_normal_text_as_message(
text: str, *, any_tool_call_in_progress: bool
) -> bool:
"""Whether ``text`` should open / extend a user-visible message item.
qwen3-coder separates adjacent tool-call blocks with ``\\n``, which the
streaming detector cannot tell from real content -- so whitespace arriving
while a call is open is treated as a separator.
"""
if not text:
return False
if any_tool_call_in_progress and not text.strip():
return False
return True
class OpenAIServingResponses(OpenAIServingChat):
"""Handler for /v1/responses requests"""
@@ -202,6 +252,23 @@ class OpenAIServingResponses(OpenAIServingChat):
'type="function"; other built-in tool types cannot be forced.'
)
# harmony emits raw tokens; per-token logprobs aren't wired there.
if self.use_harmony and request.is_include_output_logprobs():
return self.create_error_response(
"logprobs are not supported with gpt-oss models", param="logprobs"
)
# streaming skips the logprobs build path; reject so the include doesn't silently no-op.
if request.stream and request.is_include_output_logprobs():
return self.create_error_response(
"logprobs are not supported in streaming mode", param="logprobs"
)
# harmony output opens with <|channel|>analysis<|message|>, so a whole-output
# json_schema forces "{" at the first token and the harmony parse then fails.
if self.use_harmony and request.has_json_schema_constraint():
return self.create_error_response(
"structured output (text.format) is not supported with gpt-oss models",
param="text",
)
if (
self.use_harmony
and self._has_response_tool(request, "web_search", "web_search_preview")
@@ -328,6 +395,13 @@ class OpenAIServingResponses(OpenAIServingChat):
),
)
# _process_messages set skip_special_tokens on a chat_request
# we then discard, so re-apply it to the engine sampling dict.
if processed_messages is not None and (
not processed_messages.skip_special_tokens
):
sampling_params["skip_special_tokens"] = False
context: ConversationContext
if self.use_harmony:
if request.stream:
@@ -343,8 +417,20 @@ class OpenAIServingResponses(OpenAIServingChat):
else:
prompt_kwargs = {"input_ids": engine_prompt}
logprob_kwargs = (
{
"return_logprob": True,
"logprob_start_len": -1,
"top_logprobs_num": request.top_logprobs or 0,
"return_text_in_logprobs": True,
}
if request.is_include_output_logprobs()
else {}
)
adapted_request = GenerateReqInput(
**prompt_kwargs,
**logprob_kwargs,
image_data=(
processed_messages.image_data
if processed_messages
@@ -370,7 +456,8 @@ class OpenAIServingResponses(OpenAIServingChat):
rid=request.request_id,
session_id=request.session_id,
extra_key=self._compute_extra_key(request),
background=request.background,
# background+stream streams on this connection, so don't detach.
background=request.background and not request.stream,
require_reasoning=require_reasoning,
)
@@ -394,7 +481,7 @@ class OpenAIServingResponses(OpenAIServingChat):
if request.store:
self.msg_store[request.request_id] = messages
if request.background:
if request.background and not request.stream:
created_time = int(time.time())
response = ResponsesResponse.from_request(
request,
@@ -484,7 +571,11 @@ class OpenAIServingResponses(OpenAIServingChat):
messages=messages,
stream=request.stream,
tools=chat_tools or None,
tool_choice=request.tool_choice if chat_tools else "none",
tool_choice=(
self._chat_tool_choice(request.effective_tool_choice())
if chat_tools
else "none"
),
parallel_tool_calls=(
request.parallel_tool_calls
if request.parallel_tool_calls is not None
@@ -492,6 +583,7 @@ class OpenAIServingResponses(OpenAIServingChat):
),
stop=request.stop,
reasoning_effort=(request.reasoning.effort if request.reasoning else None),
chat_template_kwargs=request.chat_template_kwargs,
)
media_error = self._validate_media_content(chat_request)
@@ -500,6 +592,7 @@ class OpenAIServingResponses(OpenAIServingChat):
is_multimodal = self.tokenizer_manager.model_config.is_multimodal
processed_messages = self._process_messages(chat_request, is_multimodal)
processed_messages.skip_special_tokens = chat_request.skip_special_tokens
if is_multimodal:
request_prompts = [processed_messages.prompt]
@@ -548,6 +641,7 @@ class OpenAIServingResponses(OpenAIServingChat):
except ValueError as e:
return self.create_error_response(str(e))
status = "completed"
if self.use_harmony:
assert isinstance(context, HarmonyContext)
output = self._make_response_output_items_with_harmony(context)
@@ -556,19 +650,12 @@ class OpenAIServingResponses(OpenAIServingChat):
num_generated_tokens = context.num_output_tokens
num_cached_tokens = context.num_cached_tokens
num_reasoning_tokens = context.num_reasoning_tokens
status = self._status_from_finish_reason(context.finish_reason)
else:
assert isinstance(context, SimpleContext)
final_res = context.last_output
assert final_res is not None
output = self._make_response_output_items(
request,
final_res["text"],
tokenizer,
require_reasoning=require_reasoning,
)
# Calculate usage from actual output
num_reasoning_tokens = 0
meta_info = None
if isinstance(final_res, dict) and isinstance(
@@ -578,11 +665,25 @@ class OpenAIServingResponses(OpenAIServingChat):
elif hasattr(final_res, "meta_info"):
meta_info = final_res.meta_info
output_logprobs = (
_build_output_text_logprobs(meta_info)
if request.is_include_output_logprobs() and isinstance(meta_info, dict)
else None
)
output = self._make_response_output_items(
request,
final_res["text"],
tokenizer,
output_logprobs=output_logprobs,
require_reasoning=require_reasoning,
)
if meta_info is not None:
num_prompt_tokens = meta_info.get("prompt_tokens", 0)
num_generated_tokens = meta_info.get("completion_tokens", 0)
num_cached_tokens = meta_info.get("cached_tokens", 0)
num_reasoning_tokens = meta_info.get("reasoning_tokens", 0)
status = self._status_from_finish_reason(meta_info.get("finish_reason"))
elif isinstance(final_res, dict) and (
final_res.get("prompt_token_ids") is not None
or final_res.get("output_ids") is not None
@@ -630,7 +731,7 @@ class OpenAIServingResponses(OpenAIServingChat):
model_name=model_name,
created_time=created_time,
output=output,
status="completed",
status=status,
usage=usage,
)
@@ -647,9 +748,27 @@ class OpenAIServingResponses(OpenAIServingChat):
def _wants_reasoning_summary(request: ResponsesRequest) -> bool:
return request.reasoning is not None and request.reasoning.summary is not None
@staticmethod
def _status_from_finish_reason(finish_reason: Any) -> str:
"""Only a length-capped generation is ``incomplete``; anything that got
here otherwise finished normally."""
reason = None
if isinstance(finish_reason, dict):
reason = finish_reason.get("type")
elif isinstance(finish_reason, str):
reason = finish_reason
return "incomplete" if reason == "length" else "completed"
def _is_thinking_enabled_for_request(self, request: ResponsesRequest) -> bool:
if not self.reasoning_parser:
return False
# an explicit toggle wins; the key differs by family (enable_thinking vs thinking).
ctk = request.chat_template_kwargs or {}
thinking_toggles = (ctk.get("enable_thinking"), ctk.get("thinking"))
if any(toggle is False for toggle in thinking_toggles):
return False
if any(toggle is True for toggle in thinking_toggles):
return True
effort = request.reasoning.effort if request.reasoning is not None else None
if self.reasoning_parser == "hunyuan":
return effort not in (None, "none", "no_think")
@@ -685,6 +804,7 @@ class OpenAIServingResponses(OpenAIServingChat):
request: ResponsesRequest,
final_output: Any,
tokenizer: Any,
output_logprobs: Optional[list] = None,
*,
require_reasoning: bool,
):
@@ -692,7 +812,11 @@ class OpenAIServingResponses(OpenAIServingChat):
reasoning_parser = ReasoningParser(
model_type=self.reasoning_parser,
stream_reasoning=False,
force_reasoning=require_reasoning,
# A template that prefills <think> forces the parser open even
# when the request itself did not ask for reasoning, same as chat.
force_reasoning=(
self.template_manager.force_reasoning or require_reasoning
),
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
)
@@ -794,7 +918,8 @@ class OpenAIServingResponses(OpenAIServingChat):
text=content,
annotations=[], # TODO
type="output_text",
logprobs=None, # TODO
# logprobs cover all generated tokens, not just the stripped content.
logprobs=output_logprobs,
)
message = ResponseOutputMessage(
id=f"msg_{random_uuid()}",
@@ -821,6 +946,14 @@ class OpenAIServingResponses(OpenAIServingChat):
output_items.extend(last_items)
return output_items
@staticmethod
def _chat_tool_choice(tool_choice: Any) -> Any:
"""Nest an ``effective_tool_choice()`` result the way chat expects:
``{"type":"function","name":X}`` -> ``{...,"function":{"name":X}}``."""
if not isinstance(tool_choice, dict):
return tool_choice
return {"type": "function", "function": {"name": tool_choice["name"]}}
@staticmethod
def _response_tools_to_chat_tools(request: ResponsesRequest) -> list[Tool]:
# Only ``function`` tools flow to chat; built-ins go through harmony.
@@ -929,10 +1062,15 @@ class OpenAIServingResponses(OpenAIServingChat):
],
}
if msg_type == "function_call_output":
# ``output`` may be a string or an array of content parts (OpenAI
# allows both); the chat tool message needs a string, so flatten.
out = message.get("output", "")
if isinstance(out, list):
out = "".join(p.get("text", "") for p in out if isinstance(p, dict))
return {
"role": "tool",
"tool_call_id": message.get("call_id"),
"content": message.get("output", ""),
"content": out,
}
# Reasoning items render as {role: assistant, reasoning_content};
# empty ones drop instead of injecting an empty assistant block.
@@ -1274,10 +1412,8 @@ class OpenAIServingResponses(OpenAIServingChat):
prev_status = response.status
if prev_status not in ("queued", "in_progress"):
return self.create_error_response(
err_type="invalid_request_error",
message="Cannot cancel a synchronous response.",
)
# already terminal; a second cancel is a no-op, return as-is.
return response
# Update the status to "cancelled"
response.status = "cancelled"
@@ -1374,7 +1510,6 @@ class OpenAIServingResponses(OpenAIServingChat):
)
async for ctx in result_generator:
# Only process context objects that implement the `is_expecting_start()` method,
# which indicates they support per-turn streaming (e.g., StreamingHarmonyContext).
# Contexts without this method are skipped, as they do not represent a new turn
@@ -1731,26 +1866,10 @@ class OpenAIServingResponses(OpenAIServingChat):
created_time=created_time,
require_reasoning=require_reasoning,
)
# Convert final_response to the format expected by ResponseCompletedEvent
response_dict = final_response.model_dump()
# OpenAI SDK's Tool union may not know extended types; drop echo.
response_dict["tools"] = []
# Convert UsageInfo to ResponseUsage format
if response_dict.get("usage"):
usage_info = response_dict["usage"]
response_dict["usage"] = {
"input_tokens": usage_info.get("prompt_tokens", 0),
"input_tokens_details": {
"cached_tokens": usage_info.get("cached_tokens", 0)
},
"output_tokens": usage_info.get("completion_tokens", 0),
"output_tokens_details": {
"reasoning_tokens": usage_info.get("reasoning_tokens", 0)
},
"total_tokens": usage_info.get("total_tokens", 0),
}
yield _send_event(
openai_responses_types.ResponseCompletedEvent(
type="response.completed",
@@ -1851,7 +1970,11 @@ class OpenAIServingResponses(OpenAIServingChat):
reasoning_parser_obj = ReasoningParser(
model_type=self.reasoning_parser,
stream_reasoning=True,
force_reasoning=require_reasoning,
# A template that prefills <think> forces the parser open even
# when the request itself did not ask for reasoning, same as chat.
force_reasoning=(
self.template_manager.force_reasoning or require_reasoning
),
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
)
@@ -2166,118 +2289,142 @@ class OpenAIServingResponses(OpenAIServingChat):
else:
normal_text, tool_calls = delta, []
# Close any open tool-call item before opening a message so
# ``output_item.done`` lands before the next ``added``.
if normal_text:
if reasoning_state["open"]:
for ev in _close_reasoning_item():
yield ev
for tool_index in list(tool_call_states):
for ev in _close_tool_call_state(tool_index):
yield ev
if not message_state["open"]:
item_id = _open_message_item()
yield _send_event(
openai_responses_types.ResponseOutputItemAddedEvent(
type="response.output_item.added",
sequence_number=-1,
output_index=message_state["output_index"],
item=ResponseOutputMessage(
id=item_id,
type="message",
role="assistant",
content=[],
status="in_progress",
),
def _emit_tool_calls(calls):
nonlocal current_output_index
if calls:
if reasoning_state["open"]:
for ev in _close_reasoning_item():
yield ev
if message_state["open"]:
for ev in _close_message_item():
yield ev
for call in calls:
tool_index = call.tool_index
state = tool_call_states.get(tool_index)
if state is None or state.get("done"):
# Close other open calls first, so their
# output_item.done precedes the next added.
for other_index in list(tool_call_states):
if other_index != tool_index:
for ev in _close_tool_call_state(other_index):
yield ev
current_output_index += 1
item_id = f"fc_{random_uuid()[:8]}"
call_id = f"call_{random_uuid()[:24]}"
state = {
"item_id": item_id,
"call_id": call_id,
"output_index": current_output_index,
"name": call.name or "",
"arguments": "",
"added": False,
"done": False,
}
tool_call_states[tool_index] = state
if not state["added"]:
state["added"] = True
yield _send_event(
openai_responses_types.ResponseOutputItemAddedEvent(
type="response.output_item.added",
sequence_number=-1,
output_index=state["output_index"],
item=ResponseFunctionToolCall(
arguments="",
call_id=state["call_id"],
name=state["name"],
type="function_call",
id=state["item_id"],
status="in_progress",
),
)
)
)
if call.parameters:
state["arguments"] += call.parameters
yield _send_event(
openai_responses_types.ResponseFunctionCallArgumentsDeltaEvent(
type="response.function_call_arguments.delta",
sequence_number=-1,
item_id=state["item_id"],
output_index=state["output_index"],
delta=call.parameters,
)
)
def _emit_normal_text():
if normal_text and _should_emit_normal_text_as_message(
normal_text,
any_tool_call_in_progress=any(
not s.get("done") for s in tool_call_states.values()
),
):
if reasoning_state["open"]:
for ev in _close_reasoning_item():
yield ev
for tool_index in list(tool_call_states):
for ev in _close_tool_call_state(tool_index):
yield ev
if not message_state["open"]:
item_id = _open_message_item()
yield _send_event(
openai_responses_types.ResponseOutputItemAddedEvent(
type="response.output_item.added",
sequence_number=-1,
output_index=message_state["output_index"],
item=ResponseOutputMessage(
id=item_id,
type="message",
role="assistant",
content=[],
status="in_progress",
),
)
)
yield _send_event(
openai_responses_types.ResponseContentPartAddedEvent(
type="response.content_part.added",
sequence_number=-1,
output_index=message_state["output_index"],
item_id=message_state["item_id"],
content_index=0,
part=openai_responses_types.ResponseOutputText(
type="output_text",
text="",
annotations=[],
logprobs=None,
),
)
)
message_state["text"] += normal_text
yield _send_event(
openai_responses_types.ResponseContentPartAddedEvent(
type="response.content_part.added",
openai_responses_types.ResponseTextDeltaEvent(
type="response.output_text.delta",
sequence_number=-1,
content_index=0,
output_index=message_state["output_index"],
item_id=message_state["item_id"],
content_index=0,
part=openai_responses_types.ResponseOutputText(
type="output_text",
text="",
annotations=[],
logprobs=None,
),
delta=normal_text,
logprobs=[],
)
)
message_state["text"] += normal_text
yield _send_event(
openai_responses_types.ResponseTextDeltaEvent(
type="response.output_text.delta",
sequence_number=-1,
content_index=0,
output_index=message_state["output_index"],
item_id=message_state["item_id"],
delta=normal_text,
logprobs=[],
)
)
if not tool_calls:
continue
# The parser's (text, calls) tuple is unordered, but positions
# are recoverable: continuing arguments precede this delta's
# text, a newly opened call follows it. Classify first --
# emitting mutates tool_call_states.
def _is_continuing(call):
state = tool_call_states.get(call.tool_index)
return state is not None and not state.get("done")
if reasoning_state["open"]:
for ev in _close_reasoning_item():
yield ev
if message_state["open"]:
for ev in _close_message_item():
yield ev
continuing = [c for c in tool_calls if _is_continuing(c)]
opening = [c for c in tool_calls if not _is_continuing(c)]
for call in tool_calls:
tool_index = call.tool_index
state = tool_call_states.get(tool_index)
if state is None or state.get("done"):
current_output_index += 1
item_id = f"fc_{random_uuid()[:8]}"
call_id = f"call_{random_uuid()[:24]}"
state = {
"item_id": item_id,
"call_id": call_id,
"output_index": current_output_index,
"name": call.name or "",
"arguments": "",
"added": False,
"done": False,
}
tool_call_states[tool_index] = state
if not state["added"]:
state["added"] = True
# Capture ``call.name`` before the ``added`` event so
# the name is set on the first emitted item.
if call.name and not state["name"]:
state["name"] = call.name
yield _send_event(
openai_responses_types.ResponseOutputItemAddedEvent(
type="response.output_item.added",
sequence_number=-1,
output_index=state["output_index"],
item=ResponseFunctionToolCall(
arguments="",
call_id=state["call_id"],
name=state["name"],
type="function_call",
id=state["item_id"],
status="in_progress",
),
)
)
if call.parameters:
state["arguments"] += call.parameters
yield _send_event(
openai_responses_types.ResponseFunctionCallArgumentsDeltaEvent(
type="response.function_call_arguments.delta",
sequence_number=-1,
item_id=state["item_id"],
output_index=state["output_index"],
delta=call.parameters,
)
)
for ev in _emit_tool_calls(continuing):
yield ev
for ev in _emit_normal_text():
yield ev
for ev in _emit_tool_calls(opening):
yield ev
except Exception:
logger.exception("Error while streaming /v1/responses")
failed = _sanitize_response_dict(
@@ -2328,7 +2475,7 @@ class OpenAIServingResponses(OpenAIServingChat):
model_name=model_name,
created_time=created_time,
output=final_output_items,
status="completed",
status=self._status_from_finish_reason(finish_reason),
usage=usage,
)
if request.store:
@@ -2338,19 +2485,6 @@ class OpenAIServingResponses(OpenAIServingChat):
self.response_store[final_response.id] = final_response
response_dict = _sanitize_response_dict(final_response.model_dump())
if response_dict.get("usage"):
usage_info = response_dict["usage"]
response_dict["usage"] = {
"input_tokens": usage_info.get("prompt_tokens", 0),
"input_tokens_details": {
"cached_tokens": cached_tokens,
},
"output_tokens": usage_info.get("completion_tokens", 0),
"output_tokens_details": {
"reasoning_tokens": reasoning_tokens_meta,
},
"total_tokens": usage_info.get("total_tokens", 0),
}
yield _send_event(
openai_responses_types.ResponseCompletedEvent(