[Feat] entrypoints: add a chat encoding adapter registry (#45300)

This commit is contained in:
Xinyuan Tong
2026-10-20 14:10:01 +08:00
committed by GitHub
parent 4a651146de
commit 4b83f215f4
5 changed files with 494 additions and 149 deletions
@@ -0,0 +1,149 @@
"""Unregistered chat encoding specs retain the existing serving branches;
concrete adapters load only when selected."""
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol
import msgspec
if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest, ToolCall
from sglang.srt.function_call.core_types import ToolCallItem
EncodingBackend = Literal["builtin", "recipe"]
class ChatOutputHooks(msgspec.Struct, frozen=True, kw_only=True):
"""Chat-handler policies an output adapter needs, bound to one handler."""
tool_call_parsing_active: Callable[[ChatCompletionRequest], bool]
history_tool_calls_count: Callable[[ChatCompletionRequest], int]
tool_call_id: Callable[[ToolCallItem, int], str]
class ResponsesOutputStream(Protocol):
def step(
self, chunk: dict[str, Any], finish_reason: Optional[dict[str, Any]]
) -> tuple[str, str, list[ToolCallItem]]: ...
class ChatEncodingOutputAdapter(Protocol):
def parse_chat_response(
self,
*,
request: ChatCompletionRequest,
output_ids: list[int],
finish_reason: dict[str, Any],
) -> tuple[Optional[str], str, Optional[list[ToolCall]], dict[str, Any]]: ...
def stream_chat_chunks(
self,
*,
content: dict[str, Any],
index: int,
request: ChatCompletionRequest,
created: int,
parser_dict: dict[int, Any],
has_tool_calls: dict[int, bool],
choice_logprobs: Optional[dict],
finish_reason_type: Optional[str],
usage: Optional[dict[str, Any]],
) -> list[str]: ...
def parse_responses_output(
self,
*,
output_ids: list[int],
finish_reason: Optional[dict[str, Any]],
separate_reasoning: bool,
parse_tool_calls: bool,
) -> tuple[str, str, list[ToolCallItem]]: ...
def new_responses_stream(
self, *, separate_reasoning: bool, parse_tool_calls: bool
) -> ResponsesOutputStream: ...
class ChatEncodingAdapter(Protocol):
collects_openai_media: bool
"""Whether media still has to be collected from the OpenAI messages after
``encode_messages`` returns prompt ids."""
@property
def output(self) -> Optional[ChatEncodingOutputAdapter]:
"""None when the configured parsers leave output to the text parsers."""
...
def encode_messages(
self,
*,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
tools: Optional[list[dict]],
) -> list[int]: ...
class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
"""Everything a builder may read; a builder takes only what its encoding needs.
Consumers:
tokenizer: prompt encoding (Inkling).
reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing.
hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id).
model_path, revision: checkpoint files (DeepSeek tokenizer.json and the
encoder file for effort-profile detection).
backend: selects the builder; "recipe" is an explicit opt-in.
"""
chat_encoding_spec: str
tokenizer: Any
reasoning_parser: Optional[str]
tool_call_parser: Optional[str]
chat_hooks: ChatOutputHooks
hf_config: Any
model_path: str
revision: Optional[str]
backend: EncodingBackend = "builtin"
def require(self, field: str) -> Any:
value = getattr(self, field)
if value is None:
raise ValueError(
f"chat encoding spec {self.chat_encoding_spec!r} ({self.backend} backend) "
f"requires {field}, which this server does not provide"
)
return value
def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter:
from sglang.srt.entrypoints.openai.serving_inkling import InklingChatEncodingAdapter
return InklingChatEncodingAdapter(tokenizer=ctx.tokenizer)
_BUILDERS: dict[
str,
dict[EncodingBackend, Callable[[ChatEncodingAdapterContext], ChatEncodingAdapter]],
] = {
"inkling": {"builtin": _build_inkling},
}
def has_chat_encoding_adapter(chat_encoding_spec: Optional[str]) -> bool:
return chat_encoding_spec in _BUILDERS
def build_chat_encoding_adapter(
ctx: ChatEncodingAdapterContext,
) -> Optional[ChatEncodingAdapter]:
builders = _BUILDERS.get(ctx.chat_encoding_spec, {})
builder = builders.get(ctx.backend)
if builder is not None:
return builder(ctx)
if ctx.backend != "builtin":
raise ValueError(
f"chat encoding spec {ctx.chat_encoding_spec!r} has no {ctx.backend} backend; "
f"available: {sorted(builders) or ['builtin']}"
)
return None
@@ -42,6 +42,7 @@ from jsonschema import Draft202012Validator, SchemaError
from sglang.srt.entrypoints.openai import (
chat_encoding,
chat_encoding_adapters,
encoding_dsv4,
encoding_dsv32,
encoding_dsv41,
@@ -345,14 +346,7 @@ class OpenAIServingChat(OpenAIServingBase):
else None
)
# Resolve the env-configured Inkling effort default once: the env var is
# frozen for the server's lifetime, and a misconfigured value should
# fail at boot, not 400 every request.
self._inkling_default_reasoning_effort: float | None = (
self._get_inkling_default_reasoning_effort()
if self.chat_encoding_spec == "inkling"
else None
)
self._encoding_adapter = self._build_chat_encoding_adapter()
self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = (
chat_encoding.default_dsv41_reasoning_effort_from_env(
envs.SGLANG_DSV41_REASONING_EFFORT.get()
@@ -489,6 +483,37 @@ class OpenAIServingChat(OpenAIServingBase):
tool_call_parser=self.tool_call_parser,
)
def _build_chat_encoding_adapter(
self,
) -> Optional[chat_encoding_adapters.ChatEncodingAdapter]:
if not chat_encoding_adapters.has_chat_encoding_adapter(
self.chat_encoding_spec
):
return None
return chat_encoding_adapters.build_chat_encoding_adapter(
chat_encoding_adapters.ChatEncodingAdapterContext(
chat_encoding_spec=self.chat_encoding_spec,
tokenizer=self.tokenizer_manager.tokenizer,
reasoning_parser=self.reasoning_parser,
tool_call_parser=self.tool_call_parser,
chat_hooks=chat_encoding_adapters.ChatOutputHooks(
tool_call_parsing_active=self._tool_call_parsing_active,
history_tool_calls_count=self._get_history_tool_calls_cnt,
tool_call_id=self._process_tool_call_id,
),
hf_config=self.tokenizer_manager.model_config.hf_config,
model_path=self.tokenizer_manager.model_path,
revision=get_model().revision,
)
)
def _encoding_output(
self,
) -> Optional[chat_encoding_adapters.ChatEncodingOutputAdapter]:
if self._encoding_adapter is None:
return None
return self._encoding_adapter.output
def _request_id_prefix(self) -> str:
return "chatcmpl-"
@@ -588,44 +613,10 @@ class OpenAIServingChat(OpenAIServingBase):
Returns prompt_ids if handled, None to use default encoding.
"""
if self.chat_encoding_spec == "inkling":
# Inkling: render messages -> input_ids with framing tokens + ONE placeholder per
# media (encoding/expansion happens later in InklingMultimodalProcessor). The
# server's tokenizer is the base tiktoken backend; wrap it so encode_special
# supplies the framing-token overlay.
from sglang.srt.parser.inkling_renderer import render_inkling_messages
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_TEXT,
MESSAGE_MODEL,
InklingTokenizer,
if self._encoding_adapter is not None:
return self._encoding_adapter.encode_messages(
messages=messages, request=request, tools=tools
)
inkling_tokenizer = InklingTokenizer(
tokenizer=self.tokenizer_manager.tokenizer
)
reasoning_effort = self._parse_inkling_reasoning_effort(
request.reasoning_effort
)
if reasoning_effort is None:
reasoning_effort = self._inkling_default_reasoning_effort
assistant_prefix = self._pop_inkling_assistant_prefix(messages, request)
prompt_ids = render_inkling_messages(
messages,
inkling_tokenizer,
add_generation_prompt=False,
tools=tools,
reasoning_effort=reasoning_effort,
)
if assistant_prefix is not None:
# Continue the final assistant message inside an OPEN model text
# block: header + payload, no <|end_message|> and no
# <|content_model_end_sampling|>, so the model resumes the turn.
prompt_ids += [
inkling_tokenizer.encode_special(MESSAGE_MODEL),
inkling_tokenizer.encode_special(CONTENT_TEXT),
*inkling_tokenizer.encode_text(assistant_prefix),
]
return prompt_ids
if self.chat_encoding_spec == "kimi_k3":
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
messages, request
@@ -900,6 +891,30 @@ class OpenAIServingChat(OpenAIServingBase):
completion_tokens: dict[int, int],
) -> AsyncGenerator[str, None]:
"""Generate SSE chunks for streaming content."""
encoding_output = self._encoding_output()
if encoding_output is not None:
usage = None
if continuous_usage_stats:
usage = UsageProcessor.calculate_token_usage(
prompt_tokens=self._reported_prompt_tokens(content["meta_info"]),
reasoning_tokens=content["meta_info"].get("reasoning_tokens", 0),
completion_tokens=content["meta_info"].get("completion_tokens", 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
for chunk in encoding_output.stream_chat_chunks(
content=content,
index=index,
request=request,
created=created,
parser_dict=parser_dict,
has_tool_calls=has_tool_calls,
choice_logprobs=choice_logprobs,
finish_reason_type=finish_reason_type,
usage=usage,
):
yield chunk
return
offset = stream_offsets.get(index, 0)
if get_serving().incremental_streaming_output:
delta = content["text"]
@@ -1556,7 +1571,12 @@ class OpenAIServingChat(OpenAIServingBase):
)
if prompt_ids is not None:
if self.chat_encoding_spec in ("inkling", "kimi_k3"):
collects_openai_media = (
self._encoding_adapter.collects_openai_media
if self._encoding_adapter is not None
else self.chat_encoding_spec == "kimi_k3"
)
if collects_openai_media:
for message in request.messages:
msg_dict = message.model_dump()
if msg_dict.get("content") is None:
@@ -2397,6 +2417,7 @@ class OpenAIServingChat(OpenAIServingBase):
output_ids=output_ids,
)
encoding_output = self._encoding_output()
for idx, ret_item in enumerate(ret):
# Process logprobs
choice_logprobs = None
@@ -2414,7 +2435,16 @@ class OpenAIServingChat(OpenAIServingBase):
# Handle reasoning content
reasoning_text = None
if self.reasoning_parser and request.separate_reasoning:
tool_calls = None
if encoding_output is not None:
reasoning_text, text, tool_calls, finish_reason = (
encoding_output.parse_chat_response(
request=request,
output_ids=ret_item["output_ids"],
finish_reason=finish_reason,
)
)
elif self.reasoning_parser and request.separate_reasoning:
force_reasoning = (
self.template_manager.force_reasoning
or self._get_reasoning_from_request(request)
@@ -2438,9 +2468,8 @@ class OpenAIServingChat(OpenAIServingBase):
)
# Handle tool calls
tool_calls = None
effective_tools = self._effective_tools(request)
if self._tool_call_parsing_active(request):
if encoding_output is None and self._tool_call_parsing_active(request):
history_tool_calls_cnt = self._get_history_tool_calls_cnt(request)
tool_calls, text, finish_reason = self._process_tool_calls(
text,
@@ -0,0 +1,67 @@
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Optional
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
if TYPE_CHECKING:
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
class InklingChatEncodingAdapter:
"""Inkling prompt encoding through ``inkling_renderer``; output stays with
the registered ``inkling`` reasoning and tool-call parsers."""
collects_openai_media = True
output = None
def __init__(self, *, tokenizer: Any):
self._tokenizer = tokenizer
# Resolved at boot: a misconfigured env default fails startup, not requests.
self._default_reasoning_effort = (
OpenAIServingChat._get_inkling_default_reasoning_effort()
)
def encode_messages(
self,
*,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
tools: Optional[list[dict]],
) -> list[int]:
from sglang.srt.parser.inkling_renderer import render_inkling_messages
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_TEXT,
MESSAGE_MODEL,
InklingTokenizer,
)
# Framing tokens + ONE placeholder per media (encoding/expansion happens later
# in InklingMultimodalProcessor). The server's tokenizer is the base tiktoken
# backend; wrap it so encode_special supplies the framing-token overlay.
inkling_tokenizer = InklingTokenizer(tokenizer=self._tokenizer)
reasoning_effort = OpenAIServingChat._parse_inkling_reasoning_effort(
request.reasoning_effort
)
if reasoning_effort is None:
reasoning_effort = self._default_reasoning_effort
assistant_prefix = OpenAIServingChat._pop_inkling_assistant_prefix(
messages, request
)
prompt_ids = render_inkling_messages(
messages,
inkling_tokenizer,
add_generation_prompt=False,
tools=tools,
reasoning_effort=reasoning_effort,
)
if assistant_prefix is not None:
# Continue the final assistant message inside an OPEN model text
# block: header + payload, no <|end_message|> and no
# <|content_model_end_sampling|>, so the model resumes the turn.
prompt_ids += [
inkling_tokenizer.encode_special(MESSAGE_MODEL),
inkling_tokenizer.encode_special(CONTENT_TEXT),
*inkling_tokenizer.encode_text(assistant_prefix),
]
return prompt_ids
@@ -87,6 +87,7 @@ from sglang.srt.entrypoints.openai.responses_adapters import (
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer
from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs
from sglang.srt.function_call.core_types import ToolCallItem
from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.function_call.json_array_parser import JsonArrayParser
from sglang.srt.managers.io_struct import GenerateReqInput
@@ -799,6 +800,8 @@ class OpenAIServingResponses(OpenAIServingChat):
tokenizer,
output_logprobs=output_logprobs,
require_reasoning=require_reasoning,
output_ids=final_res.get("output_ids"),
finish_reason=meta_info.get("finish_reason") if meta_info else None,
)
if meta_info is not None:
@@ -1022,8 +1025,84 @@ class OpenAIServingResponses(OpenAIServingChat):
output_logprobs: Optional[list] = None,
*,
require_reasoning: bool,
output_ids: Optional[list[int]] = None,
finish_reason: Optional[dict[str, Any]] = None,
):
chat_tools = self._response_tools_to_chat_tools(request)
custom_names = custom_tool_names(request.tools)
encoding_output = self._encoding_output()
if encoding_output is not None:
required_json = self._uses_required_tool_json(request, chat_tools)
reasoning_content, content, calls = encoding_output.parse_responses_output(
output_ids=output_ids or [],
finish_reason=finish_reason,
separate_reasoning=bool(self.reasoning_parser),
parse_tool_calls=(
self._tool_call_parsing_enabled(request, chat_tools)
and not required_json
),
)
if required_json and content:
content, tool_call_items = self._parse_required_tool_json(
content, custom_names
)
else:
tool_call_items = [
self._make_tool_call_item(call.name, call.parameters, custom_names)
for call in calls
]
else:
reasoning_content, content, tool_call_items = self._parse_text_output(
request,
final_output,
chat_tools,
custom_names,
require_reasoning=require_reasoning,
)
output_items = []
if reasoning_content:
output_items.append(
self._make_reasoning_item(
request,
reasoning_content,
item_id=f"rs_{random_uuid()}",
status=None,
)
)
if content:
output_text = ResponseOutputText(
text=content,
annotations=[], # TODO
type="output_text",
# logprobs cover all generated tokens, not just the stripped content.
logprobs=output_logprobs,
)
message = ResponseOutputMessage(
phase="commentary" if tool_call_items else "final_answer",
id=f"msg_{random_uuid()}",
content=[output_text],
role="assistant",
status="completed",
type="message",
)
output_items.append(message)
output_items.extend(tool_call_items)
return output_items
def _parse_text_output(
self,
request: ResponsesRequest,
final_output: Any,
chat_tools: list[Tool],
custom_names: set[str],
*,
require_reasoning: bool,
) -> tuple[
Optional[str],
Any,
list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]],
]:
if self.reasoning_parser:
reasoning_parser = ReasoningParser(
model_type=self.reasoning_parser,
@@ -1046,20 +1125,8 @@ class OpenAIServingResponses(OpenAIServingChat):
reasoning_content = None
content = final_output
output_items = []
if reasoning_content:
output_items.append(
self._make_reasoning_item(
request,
reasoning_content,
item_id=f"rs_{random_uuid()}",
status=None,
)
)
tool_choice = request.effective_tool_choice()
is_required = tool_choice == "required" or isinstance(tool_choice, dict)
custom_names = custom_tool_names(request.tools)
tool_call_items: list[
Union[ResponseFunctionToolCall, ResponseCustomToolCall]
] = []
@@ -1101,45 +1168,32 @@ class OpenAIServingResponses(OpenAIServingChat):
and not parsed_via_native
and not detector_owns_format
):
try:
tool_call_data = orjson.loads(content)
if isinstance(tool_call_data, dict):
tool_call_data = [tool_call_data]
if isinstance(tool_call_data, list):
for tool in tool_call_data:
if not isinstance(tool, dict) or "name" not in tool:
continue
arguments = json.dumps(
tool.get("parameters", {}), ensure_ascii=False
)
tool_call_items.append(
self._make_tool_call_item(
tool["name"], arguments, custom_names
)
)
content = ""
except Exception as e:
logger.error("Required tool JSON parse error: %s", e)
content, json_items = self._parse_required_tool_json(content, custom_names)
tool_call_items.extend(json_items)
return reasoning_content, content, tool_call_items
if content:
output_text = ResponseOutputText(
text=content,
annotations=[], # TODO
type="output_text",
# logprobs cover all generated tokens, not just the stripped content.
logprobs=output_logprobs,
)
message = ResponseOutputMessage(
phase="commentary" if tool_call_items else "final_answer",
id=f"msg_{random_uuid()}",
content=[output_text],
role="assistant",
status="completed",
type="message",
)
output_items.append(message)
output_items.extend(tool_call_items)
return output_items
def _parse_required_tool_json(
self, content: str, custom_names: set[str]
) -> tuple[str, list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]]]:
tool_call_items = []
try:
tool_call_data = orjson.loads(content)
if isinstance(tool_call_data, dict):
tool_call_data = [tool_call_data]
if isinstance(tool_call_data, list):
for tool in tool_call_data:
if not isinstance(tool, dict) or "name" not in tool:
continue
arguments = json.dumps(
tool.get("parameters", {}), ensure_ascii=False
)
tool_call_items.append(
self._make_tool_call_item(tool["name"], arguments, custom_names)
)
content = ""
except Exception as e:
logger.error("Required tool JSON parse error: %s", e)
return content, tool_call_items
def _make_response_output_items_with_harmony(
self,
@@ -1159,10 +1213,37 @@ class OpenAIServingResponses(OpenAIServingChat):
output_items.extend(last_items)
return output_items
def _tool_call_parsing_enabled(
self, request: ResponsesRequest, chat_tools: list[Tool]
) -> bool:
return bool(
chat_tools and self.tool_call_parser and request.tool_choice != "none"
)
@staticmethod
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
return parser.owns_tool_format()
def _uses_required_tool_json(
self, request: ResponsesRequest, chat_tools: list[Tool]
) -> bool:
"""Whether required/named tool output is constrained to the generic JSON
array, rather than to the configured detector's native framing."""
tool_choice = request.effective_tool_choice()
if not chat_tools or not (
tool_choice == "required" or isinstance(tool_choice, dict)
):
return False
if not self.tool_call_parser:
return True
probe = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
return not self._tool_parser_owns_format(probe)
@staticmethod
def _chat_tool_choice(tool_choice: Any) -> Any:
"""Nest an ``effective_tool_choice()`` result the way chat expects:
@@ -1998,29 +2079,33 @@ class OpenAIServingResponses(OpenAIServingChat):
chat_tools = self._response_tools_to_chat_tools(request)
custom_names = custom_tool_names(request.tools)
tool_choice = request.effective_tool_choice()
is_required = tool_choice == "required" or isinstance(tool_choice, dict)
required_json = self._uses_required_tool_json(request, chat_tools)
encoding_output = self._encoding_output()
token_stream = (
encoding_output.new_responses_stream(
separate_reasoning=bool(self.reasoning_parser),
parse_tool_calls=(
self._tool_call_parsing_enabled(request, chat_tools)
and not required_json
),
)
if encoding_output is not None
else None
)
tool_parser: Optional[Union[FunctionCallParser, JsonArrayParser]] = None
if chat_tools and request.tool_choice != "none":
detector_owns_format = False
if self.tool_call_parser:
probe = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
detector_owns_format = self._tool_parser_owns_format(probe)
if is_required and not detector_owns_format:
tool_parser = JsonArrayParser()
elif self.tool_call_parser:
tool_parser = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
if required_json:
tool_parser = JsonArrayParser()
elif token_stream is None and self._tool_call_parsing_enabled(
request, chat_tools
):
tool_parser = FunctionCallParser(
chat_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
reasoning_parser_obj: Optional[ReasoningParser] = None
if self.reasoning_parser:
if token_stream is None and self.reasoning_parser:
reasoning_parser_obj = ReasoningParser(
model_type=self.reasoning_parser,
stream_reasoning=True,
@@ -2308,36 +2393,45 @@ class OpenAIServingResponses(OpenAIServingChat):
)
finish_reason = meta.get("finish_reason") or finish_reason
text = chunk.get("text", "") or ""
if incremental:
delta = text
else:
delta = text[stream_offset:]
stream_offset = len(text)
if not delta and finish_reason is None:
continue
# finish_reason is sticky, so it would otherwise re-flush.
flush = (
not flushed
and finish_reason is not None
and finish_reason.get("type") != "abort"
)
flushed = flushed or flush
if reasoning_parser_obj is not None:
reasoning_chunk, delta = reasoning_parser_obj.parse_stream_chunk(
delta
token_calls: list[ToolCallItem] = []
if token_stream is not None:
reasoning_chunk, delta, token_calls = token_stream.step(
chunk, finish_reason
)
if flush:
end_reasoning, end_normal = (
reasoning_parser_obj.parse_stream_end()
)
if end_reasoning:
reasoning_chunk = (reasoning_chunk or "") + end_reasoning
if end_normal:
delta = (delta or "") + end_normal
flush = False
else:
reasoning_chunk = None
text = chunk.get("text", "") or ""
if incremental:
delta = text
else:
delta = text[stream_offset:]
stream_offset = len(text)
if not delta and finish_reason is None:
continue
# finish_reason is sticky, so it would otherwise re-flush.
flush = (
not flushed
and finish_reason is not None
and finish_reason.get("type") != "abort"
)
flushed = flushed or flush
if reasoning_parser_obj is not None:
reasoning_chunk, delta = (
reasoning_parser_obj.parse_stream_chunk(delta)
)
if flush:
end_reasoning, end_normal = (
reasoning_parser_obj.parse_stream_end()
)
if end_reasoning:
reasoning_chunk = (
reasoning_chunk or ""
) + end_reasoning
if end_normal:
delta = (delta or "") + end_normal
else:
reasoning_chunk = None
if reasoning_chunk:
if message_state["open"]:
@@ -2412,12 +2506,14 @@ class OpenAIServingResponses(OpenAIServingChat):
)
)
if not delta and not flush:
if not delta and not flush and not token_calls:
continue
if isinstance(tool_parser, JsonArrayParser):
sp = tool_parser.parse_streaming_increment(delta, chat_tools)
normal_text, tool_calls = sp.normal_text or "", sp.calls
elif token_stream is not None:
normal_text, tool_calls = delta, token_calls
elif tool_parser is not None:
normal_text, tool_calls = tool_parser.parse_stream_chunk(delta)
if flush: