[Feat] entrypoints: add a chat encoding adapter registry (#45300)
This commit is contained in:
@@ -0,0 +1,149 @@
|
||||
"""Unregistered chat encoding specs retain the existing serving branches;
|
||||
concrete adapters load only when selected."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol
|
||||
|
||||
import msgspec
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest, ToolCall
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
|
||||
EncodingBackend = Literal["builtin", "recipe"]
|
||||
|
||||
|
||||
class ChatOutputHooks(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Chat-handler policies an output adapter needs, bound to one handler."""
|
||||
|
||||
tool_call_parsing_active: Callable[[ChatCompletionRequest], bool]
|
||||
history_tool_calls_count: Callable[[ChatCompletionRequest], int]
|
||||
tool_call_id: Callable[[ToolCallItem, int], str]
|
||||
|
||||
|
||||
class ResponsesOutputStream(Protocol):
|
||||
def step(
|
||||
self, chunk: dict[str, Any], finish_reason: Optional[dict[str, Any]]
|
||||
) -> tuple[str, str, list[ToolCallItem]]: ...
|
||||
|
||||
|
||||
class ChatEncodingOutputAdapter(Protocol):
|
||||
def parse_chat_response(
|
||||
self,
|
||||
*,
|
||||
request: ChatCompletionRequest,
|
||||
output_ids: list[int],
|
||||
finish_reason: dict[str, Any],
|
||||
) -> tuple[Optional[str], str, Optional[list[ToolCall]], dict[str, Any]]: ...
|
||||
|
||||
def stream_chat_chunks(
|
||||
self,
|
||||
*,
|
||||
content: dict[str, Any],
|
||||
index: int,
|
||||
request: ChatCompletionRequest,
|
||||
created: int,
|
||||
parser_dict: dict[int, Any],
|
||||
has_tool_calls: dict[int, bool],
|
||||
choice_logprobs: Optional[dict],
|
||||
finish_reason_type: Optional[str],
|
||||
usage: Optional[dict[str, Any]],
|
||||
) -> list[str]: ...
|
||||
|
||||
def parse_responses_output(
|
||||
self,
|
||||
*,
|
||||
output_ids: list[int],
|
||||
finish_reason: Optional[dict[str, Any]],
|
||||
separate_reasoning: bool,
|
||||
parse_tool_calls: bool,
|
||||
) -> tuple[str, str, list[ToolCallItem]]: ...
|
||||
|
||||
def new_responses_stream(
|
||||
self, *, separate_reasoning: bool, parse_tool_calls: bool
|
||||
) -> ResponsesOutputStream: ...
|
||||
|
||||
|
||||
class ChatEncodingAdapter(Protocol):
|
||||
collects_openai_media: bool
|
||||
"""Whether media still has to be collected from the OpenAI messages after
|
||||
``encode_messages`` returns prompt ids."""
|
||||
|
||||
@property
|
||||
def output(self) -> Optional[ChatEncodingOutputAdapter]:
|
||||
"""None when the configured parsers leave output to the text parsers."""
|
||||
...
|
||||
|
||||
def encode_messages(
|
||||
self,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
request: ChatCompletionRequest,
|
||||
tools: Optional[list[dict]],
|
||||
) -> list[int]: ...
|
||||
|
||||
|
||||
class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
|
||||
"""Everything a builder may read; a builder takes only what its encoding needs.
|
||||
|
||||
Consumers:
|
||||
tokenizer: prompt encoding (Inkling).
|
||||
reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing.
|
||||
hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id).
|
||||
model_path, revision: checkpoint files (DeepSeek tokenizer.json and the
|
||||
encoder file for effort-profile detection).
|
||||
backend: selects the builder; "recipe" is an explicit opt-in.
|
||||
"""
|
||||
|
||||
chat_encoding_spec: str
|
||||
tokenizer: Any
|
||||
reasoning_parser: Optional[str]
|
||||
tool_call_parser: Optional[str]
|
||||
chat_hooks: ChatOutputHooks
|
||||
hf_config: Any
|
||||
model_path: str
|
||||
revision: Optional[str]
|
||||
backend: EncodingBackend = "builtin"
|
||||
|
||||
def require(self, field: str) -> Any:
|
||||
value = getattr(self, field)
|
||||
if value is None:
|
||||
raise ValueError(
|
||||
f"chat encoding spec {self.chat_encoding_spec!r} ({self.backend} backend) "
|
||||
f"requires {field}, which this server does not provide"
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter:
|
||||
from sglang.srt.entrypoints.openai.serving_inkling import InklingChatEncodingAdapter
|
||||
|
||||
return InklingChatEncodingAdapter(tokenizer=ctx.tokenizer)
|
||||
|
||||
|
||||
_BUILDERS: dict[
|
||||
str,
|
||||
dict[EncodingBackend, Callable[[ChatEncodingAdapterContext], ChatEncodingAdapter]],
|
||||
] = {
|
||||
"inkling": {"builtin": _build_inkling},
|
||||
}
|
||||
|
||||
|
||||
def has_chat_encoding_adapter(chat_encoding_spec: Optional[str]) -> bool:
|
||||
return chat_encoding_spec in _BUILDERS
|
||||
|
||||
|
||||
def build_chat_encoding_adapter(
|
||||
ctx: ChatEncodingAdapterContext,
|
||||
) -> Optional[ChatEncodingAdapter]:
|
||||
builders = _BUILDERS.get(ctx.chat_encoding_spec, {})
|
||||
builder = builders.get(ctx.backend)
|
||||
if builder is not None:
|
||||
return builder(ctx)
|
||||
if ctx.backend != "builtin":
|
||||
raise ValueError(
|
||||
f"chat encoding spec {ctx.chat_encoding_spec!r} has no {ctx.backend} backend; "
|
||||
f"available: {sorted(builders) or ['builtin']}"
|
||||
)
|
||||
return None
|
||||
@@ -42,6 +42,7 @@ from jsonschema import Draft202012Validator, SchemaError
|
||||
|
||||
from sglang.srt.entrypoints.openai import (
|
||||
chat_encoding,
|
||||
chat_encoding_adapters,
|
||||
encoding_dsv4,
|
||||
encoding_dsv32,
|
||||
encoding_dsv41,
|
||||
@@ -345,14 +346,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
else None
|
||||
)
|
||||
|
||||
# Resolve the env-configured Inkling effort default once: the env var is
|
||||
# frozen for the server's lifetime, and a misconfigured value should
|
||||
# fail at boot, not 400 every request.
|
||||
self._inkling_default_reasoning_effort: float | None = (
|
||||
self._get_inkling_default_reasoning_effort()
|
||||
if self.chat_encoding_spec == "inkling"
|
||||
else None
|
||||
)
|
||||
self._encoding_adapter = self._build_chat_encoding_adapter()
|
||||
self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = (
|
||||
chat_encoding.default_dsv41_reasoning_effort_from_env(
|
||||
envs.SGLANG_DSV41_REASONING_EFFORT.get()
|
||||
@@ -489,6 +483,37 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
)
|
||||
|
||||
def _build_chat_encoding_adapter(
|
||||
self,
|
||||
) -> Optional[chat_encoding_adapters.ChatEncodingAdapter]:
|
||||
if not chat_encoding_adapters.has_chat_encoding_adapter(
|
||||
self.chat_encoding_spec
|
||||
):
|
||||
return None
|
||||
return chat_encoding_adapters.build_chat_encoding_adapter(
|
||||
chat_encoding_adapters.ChatEncodingAdapterContext(
|
||||
chat_encoding_spec=self.chat_encoding_spec,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
reasoning_parser=self.reasoning_parser,
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
chat_hooks=chat_encoding_adapters.ChatOutputHooks(
|
||||
tool_call_parsing_active=self._tool_call_parsing_active,
|
||||
history_tool_calls_count=self._get_history_tool_calls_cnt,
|
||||
tool_call_id=self._process_tool_call_id,
|
||||
),
|
||||
hf_config=self.tokenizer_manager.model_config.hf_config,
|
||||
model_path=self.tokenizer_manager.model_path,
|
||||
revision=get_model().revision,
|
||||
)
|
||||
)
|
||||
|
||||
def _encoding_output(
|
||||
self,
|
||||
) -> Optional[chat_encoding_adapters.ChatEncodingOutputAdapter]:
|
||||
if self._encoding_adapter is None:
|
||||
return None
|
||||
return self._encoding_adapter.output
|
||||
|
||||
def _request_id_prefix(self) -> str:
|
||||
return "chatcmpl-"
|
||||
|
||||
@@ -588,44 +613,10 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
Returns prompt_ids if handled, None to use default encoding.
|
||||
"""
|
||||
if self.chat_encoding_spec == "inkling":
|
||||
# Inkling: render messages -> input_ids with framing tokens + ONE placeholder per
|
||||
# media (encoding/expansion happens later in InklingMultimodalProcessor). The
|
||||
# server's tokenizer is the base tiktoken backend; wrap it so encode_special
|
||||
# supplies the framing-token overlay.
|
||||
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_TEXT,
|
||||
MESSAGE_MODEL,
|
||||
InklingTokenizer,
|
||||
if self._encoding_adapter is not None:
|
||||
return self._encoding_adapter.encode_messages(
|
||||
messages=messages, request=request, tools=tools
|
||||
)
|
||||
|
||||
inkling_tokenizer = InklingTokenizer(
|
||||
tokenizer=self.tokenizer_manager.tokenizer
|
||||
)
|
||||
reasoning_effort = self._parse_inkling_reasoning_effort(
|
||||
request.reasoning_effort
|
||||
)
|
||||
if reasoning_effort is None:
|
||||
reasoning_effort = self._inkling_default_reasoning_effort
|
||||
assistant_prefix = self._pop_inkling_assistant_prefix(messages, request)
|
||||
prompt_ids = render_inkling_messages(
|
||||
messages,
|
||||
inkling_tokenizer,
|
||||
add_generation_prompt=False,
|
||||
tools=tools,
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
if assistant_prefix is not None:
|
||||
# Continue the final assistant message inside an OPEN model text
|
||||
# block: header + payload, no <|end_message|> and no
|
||||
# <|content_model_end_sampling|>, so the model resumes the turn.
|
||||
prompt_ids += [
|
||||
inkling_tokenizer.encode_special(MESSAGE_MODEL),
|
||||
inkling_tokenizer.encode_special(CONTENT_TEXT),
|
||||
*inkling_tokenizer.encode_text(assistant_prefix),
|
||||
]
|
||||
return prompt_ids
|
||||
if self.chat_encoding_spec == "kimi_k3":
|
||||
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
|
||||
messages, request
|
||||
@@ -900,6 +891,30 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
completion_tokens: dict[int, int],
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Generate SSE chunks for streaming content."""
|
||||
encoding_output = self._encoding_output()
|
||||
if encoding_output is not None:
|
||||
usage = None
|
||||
if continuous_usage_stats:
|
||||
usage = UsageProcessor.calculate_token_usage(
|
||||
prompt_tokens=self._reported_prompt_tokens(content["meta_info"]),
|
||||
reasoning_tokens=content["meta_info"].get("reasoning_tokens", 0),
|
||||
completion_tokens=content["meta_info"].get("completion_tokens", 0),
|
||||
cached_tokens=self._continuous_usage_cached_details(content),
|
||||
).model_dump()
|
||||
for chunk in encoding_output.stream_chat_chunks(
|
||||
content=content,
|
||||
index=index,
|
||||
request=request,
|
||||
created=created,
|
||||
parser_dict=parser_dict,
|
||||
has_tool_calls=has_tool_calls,
|
||||
choice_logprobs=choice_logprobs,
|
||||
finish_reason_type=finish_reason_type,
|
||||
usage=usage,
|
||||
):
|
||||
yield chunk
|
||||
return
|
||||
|
||||
offset = stream_offsets.get(index, 0)
|
||||
if get_serving().incremental_streaming_output:
|
||||
delta = content["text"]
|
||||
@@ -1556,7 +1571,12 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
|
||||
if prompt_ids is not None:
|
||||
if self.chat_encoding_spec in ("inkling", "kimi_k3"):
|
||||
collects_openai_media = (
|
||||
self._encoding_adapter.collects_openai_media
|
||||
if self._encoding_adapter is not None
|
||||
else self.chat_encoding_spec == "kimi_k3"
|
||||
)
|
||||
if collects_openai_media:
|
||||
for message in request.messages:
|
||||
msg_dict = message.model_dump()
|
||||
if msg_dict.get("content") is None:
|
||||
@@ -2397,6 +2417,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
output_ids=output_ids,
|
||||
)
|
||||
|
||||
encoding_output = self._encoding_output()
|
||||
for idx, ret_item in enumerate(ret):
|
||||
# Process logprobs
|
||||
choice_logprobs = None
|
||||
@@ -2414,7 +2435,16 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
|
||||
# Handle reasoning content
|
||||
reasoning_text = None
|
||||
if self.reasoning_parser and request.separate_reasoning:
|
||||
tool_calls = None
|
||||
if encoding_output is not None:
|
||||
reasoning_text, text, tool_calls, finish_reason = (
|
||||
encoding_output.parse_chat_response(
|
||||
request=request,
|
||||
output_ids=ret_item["output_ids"],
|
||||
finish_reason=finish_reason,
|
||||
)
|
||||
)
|
||||
elif self.reasoning_parser and request.separate_reasoning:
|
||||
force_reasoning = (
|
||||
self.template_manager.force_reasoning
|
||||
or self._get_reasoning_from_request(request)
|
||||
@@ -2438,9 +2468,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
)
|
||||
|
||||
# Handle tool calls
|
||||
tool_calls = None
|
||||
effective_tools = self._effective_tools(request)
|
||||
if self._tool_call_parsing_active(request):
|
||||
if encoding_output is None and self._tool_call_parsing_active(request):
|
||||
history_tool_calls_cnt = self._get_history_tool_calls_cnt(request)
|
||||
tool_calls, text, finish_reason = self._process_tool_calls(
|
||||
text,
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Optional
|
||||
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
|
||||
|
||||
class InklingChatEncodingAdapter:
|
||||
"""Inkling prompt encoding through ``inkling_renderer``; output stays with
|
||||
the registered ``inkling`` reasoning and tool-call parsers."""
|
||||
|
||||
collects_openai_media = True
|
||||
output = None
|
||||
|
||||
def __init__(self, *, tokenizer: Any):
|
||||
self._tokenizer = tokenizer
|
||||
# Resolved at boot: a misconfigured env default fails startup, not requests.
|
||||
self._default_reasoning_effort = (
|
||||
OpenAIServingChat._get_inkling_default_reasoning_effort()
|
||||
)
|
||||
|
||||
def encode_messages(
|
||||
self,
|
||||
*,
|
||||
messages: list[dict[str, Any]],
|
||||
request: ChatCompletionRequest,
|
||||
tools: Optional[list[dict]],
|
||||
) -> list[int]:
|
||||
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_TEXT,
|
||||
MESSAGE_MODEL,
|
||||
InklingTokenizer,
|
||||
)
|
||||
|
||||
# Framing tokens + ONE placeholder per media (encoding/expansion happens later
|
||||
# in InklingMultimodalProcessor). The server's tokenizer is the base tiktoken
|
||||
# backend; wrap it so encode_special supplies the framing-token overlay.
|
||||
inkling_tokenizer = InklingTokenizer(tokenizer=self._tokenizer)
|
||||
reasoning_effort = OpenAIServingChat._parse_inkling_reasoning_effort(
|
||||
request.reasoning_effort
|
||||
)
|
||||
if reasoning_effort is None:
|
||||
reasoning_effort = self._default_reasoning_effort
|
||||
assistant_prefix = OpenAIServingChat._pop_inkling_assistant_prefix(
|
||||
messages, request
|
||||
)
|
||||
prompt_ids = render_inkling_messages(
|
||||
messages,
|
||||
inkling_tokenizer,
|
||||
add_generation_prompt=False,
|
||||
tools=tools,
|
||||
reasoning_effort=reasoning_effort,
|
||||
)
|
||||
if assistant_prefix is not None:
|
||||
# Continue the final assistant message inside an OPEN model text
|
||||
# block: header + payload, no <|end_message|> and no
|
||||
# <|content_model_end_sampling|>, so the model resumes the turn.
|
||||
prompt_ids += [
|
||||
inkling_tokenizer.encode_special(MESSAGE_MODEL),
|
||||
inkling_tokenizer.encode_special(CONTENT_TEXT),
|
||||
*inkling_tokenizer.encode_text(assistant_prefix),
|
||||
]
|
||||
return prompt_ids
|
||||
@@ -87,6 +87,7 @@ from sglang.srt.entrypoints.openai.responses_adapters import (
|
||||
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||
from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer
|
||||
from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
from sglang.srt.function_call.json_array_parser import JsonArrayParser
|
||||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||||
@@ -799,6 +800,8 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
tokenizer,
|
||||
output_logprobs=output_logprobs,
|
||||
require_reasoning=require_reasoning,
|
||||
output_ids=final_res.get("output_ids"),
|
||||
finish_reason=meta_info.get("finish_reason") if meta_info else None,
|
||||
)
|
||||
|
||||
if meta_info is not None:
|
||||
@@ -1022,8 +1025,84 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
output_logprobs: Optional[list] = None,
|
||||
*,
|
||||
require_reasoning: bool,
|
||||
output_ids: Optional[list[int]] = None,
|
||||
finish_reason: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
encoding_output = self._encoding_output()
|
||||
if encoding_output is not None:
|
||||
required_json = self._uses_required_tool_json(request, chat_tools)
|
||||
reasoning_content, content, calls = encoding_output.parse_responses_output(
|
||||
output_ids=output_ids or [],
|
||||
finish_reason=finish_reason,
|
||||
separate_reasoning=bool(self.reasoning_parser),
|
||||
parse_tool_calls=(
|
||||
self._tool_call_parsing_enabled(request, chat_tools)
|
||||
and not required_json
|
||||
),
|
||||
)
|
||||
if required_json and content:
|
||||
content, tool_call_items = self._parse_required_tool_json(
|
||||
content, custom_names
|
||||
)
|
||||
else:
|
||||
tool_call_items = [
|
||||
self._make_tool_call_item(call.name, call.parameters, custom_names)
|
||||
for call in calls
|
||||
]
|
||||
else:
|
||||
reasoning_content, content, tool_call_items = self._parse_text_output(
|
||||
request,
|
||||
final_output,
|
||||
chat_tools,
|
||||
custom_names,
|
||||
require_reasoning=require_reasoning,
|
||||
)
|
||||
|
||||
output_items = []
|
||||
if reasoning_content:
|
||||
output_items.append(
|
||||
self._make_reasoning_item(
|
||||
request,
|
||||
reasoning_content,
|
||||
item_id=f"rs_{random_uuid()}",
|
||||
status=None,
|
||||
)
|
||||
)
|
||||
if content:
|
||||
output_text = ResponseOutputText(
|
||||
text=content,
|
||||
annotations=[], # TODO
|
||||
type="output_text",
|
||||
# logprobs cover all generated tokens, not just the stripped content.
|
||||
logprobs=output_logprobs,
|
||||
)
|
||||
message = ResponseOutputMessage(
|
||||
phase="commentary" if tool_call_items else "final_answer",
|
||||
id=f"msg_{random_uuid()}",
|
||||
content=[output_text],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
)
|
||||
output_items.append(message)
|
||||
output_items.extend(tool_call_items)
|
||||
return output_items
|
||||
|
||||
def _parse_text_output(
|
||||
self,
|
||||
request: ResponsesRequest,
|
||||
final_output: Any,
|
||||
chat_tools: list[Tool],
|
||||
custom_names: set[str],
|
||||
*,
|
||||
require_reasoning: bool,
|
||||
) -> tuple[
|
||||
Optional[str],
|
||||
Any,
|
||||
list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]],
|
||||
]:
|
||||
if self.reasoning_parser:
|
||||
reasoning_parser = ReasoningParser(
|
||||
model_type=self.reasoning_parser,
|
||||
@@ -1046,20 +1125,8 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
reasoning_content = None
|
||||
content = final_output
|
||||
|
||||
output_items = []
|
||||
if reasoning_content:
|
||||
output_items.append(
|
||||
self._make_reasoning_item(
|
||||
request,
|
||||
reasoning_content,
|
||||
item_id=f"rs_{random_uuid()}",
|
||||
status=None,
|
||||
)
|
||||
)
|
||||
|
||||
tool_choice = request.effective_tool_choice()
|
||||
is_required = tool_choice == "required" or isinstance(tool_choice, dict)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
tool_call_items: list[
|
||||
Union[ResponseFunctionToolCall, ResponseCustomToolCall]
|
||||
] = []
|
||||
@@ -1101,45 +1168,32 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
and not parsed_via_native
|
||||
and not detector_owns_format
|
||||
):
|
||||
try:
|
||||
tool_call_data = orjson.loads(content)
|
||||
if isinstance(tool_call_data, dict):
|
||||
tool_call_data = [tool_call_data]
|
||||
if isinstance(tool_call_data, list):
|
||||
for tool in tool_call_data:
|
||||
if not isinstance(tool, dict) or "name" not in tool:
|
||||
continue
|
||||
arguments = json.dumps(
|
||||
tool.get("parameters", {}), ensure_ascii=False
|
||||
)
|
||||
tool_call_items.append(
|
||||
self._make_tool_call_item(
|
||||
tool["name"], arguments, custom_names
|
||||
)
|
||||
)
|
||||
content = ""
|
||||
except Exception as e:
|
||||
logger.error("Required tool JSON parse error: %s", e)
|
||||
content, json_items = self._parse_required_tool_json(content, custom_names)
|
||||
tool_call_items.extend(json_items)
|
||||
return reasoning_content, content, tool_call_items
|
||||
|
||||
if content:
|
||||
output_text = ResponseOutputText(
|
||||
text=content,
|
||||
annotations=[], # TODO
|
||||
type="output_text",
|
||||
# logprobs cover all generated tokens, not just the stripped content.
|
||||
logprobs=output_logprobs,
|
||||
)
|
||||
message = ResponseOutputMessage(
|
||||
phase="commentary" if tool_call_items else "final_answer",
|
||||
id=f"msg_{random_uuid()}",
|
||||
content=[output_text],
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
)
|
||||
output_items.append(message)
|
||||
output_items.extend(tool_call_items)
|
||||
return output_items
|
||||
def _parse_required_tool_json(
|
||||
self, content: str, custom_names: set[str]
|
||||
) -> tuple[str, list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]]]:
|
||||
tool_call_items = []
|
||||
try:
|
||||
tool_call_data = orjson.loads(content)
|
||||
if isinstance(tool_call_data, dict):
|
||||
tool_call_data = [tool_call_data]
|
||||
if isinstance(tool_call_data, list):
|
||||
for tool in tool_call_data:
|
||||
if not isinstance(tool, dict) or "name" not in tool:
|
||||
continue
|
||||
arguments = json.dumps(
|
||||
tool.get("parameters", {}), ensure_ascii=False
|
||||
)
|
||||
tool_call_items.append(
|
||||
self._make_tool_call_item(tool["name"], arguments, custom_names)
|
||||
)
|
||||
content = ""
|
||||
except Exception as e:
|
||||
logger.error("Required tool JSON parse error: %s", e)
|
||||
return content, tool_call_items
|
||||
|
||||
def _make_response_output_items_with_harmony(
|
||||
self,
|
||||
@@ -1159,10 +1213,37 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
output_items.extend(last_items)
|
||||
return output_items
|
||||
|
||||
def _tool_call_parsing_enabled(
|
||||
self, request: ResponsesRequest, chat_tools: list[Tool]
|
||||
) -> bool:
|
||||
return bool(
|
||||
chat_tools and self.tool_call_parser and request.tool_choice != "none"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _tool_parser_owns_format(parser: FunctionCallParser) -> bool:
|
||||
return parser.owns_tool_format()
|
||||
|
||||
def _uses_required_tool_json(
|
||||
self, request: ResponsesRequest, chat_tools: list[Tool]
|
||||
) -> bool:
|
||||
"""Whether required/named tool output is constrained to the generic JSON
|
||||
array, rather than to the configured detector's native framing."""
|
||||
tool_choice = request.effective_tool_choice()
|
||||
if not chat_tools or not (
|
||||
tool_choice == "required" or isinstance(tool_choice, dict)
|
||||
):
|
||||
return False
|
||||
if not self.tool_call_parser:
|
||||
return True
|
||||
probe = FunctionCallParser(
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
return not self._tool_parser_owns_format(probe)
|
||||
|
||||
@staticmethod
|
||||
def _chat_tool_choice(tool_choice: Any) -> Any:
|
||||
"""Nest an ``effective_tool_choice()`` result the way chat expects:
|
||||
@@ -1998,29 +2079,33 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
chat_tools = self._response_tools_to_chat_tools(request)
|
||||
custom_names = custom_tool_names(request.tools)
|
||||
tool_choice = request.effective_tool_choice()
|
||||
is_required = tool_choice == "required" or isinstance(tool_choice, dict)
|
||||
required_json = self._uses_required_tool_json(request, chat_tools)
|
||||
encoding_output = self._encoding_output()
|
||||
token_stream = (
|
||||
encoding_output.new_responses_stream(
|
||||
separate_reasoning=bool(self.reasoning_parser),
|
||||
parse_tool_calls=(
|
||||
self._tool_call_parsing_enabled(request, chat_tools)
|
||||
and not required_json
|
||||
),
|
||||
)
|
||||
if encoding_output is not None
|
||||
else None
|
||||
)
|
||||
tool_parser: Optional[Union[FunctionCallParser, JsonArrayParser]] = None
|
||||
if chat_tools and request.tool_choice != "none":
|
||||
detector_owns_format = False
|
||||
if self.tool_call_parser:
|
||||
probe = FunctionCallParser(
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
detector_owns_format = self._tool_parser_owns_format(probe)
|
||||
if is_required and not detector_owns_format:
|
||||
tool_parser = JsonArrayParser()
|
||||
elif self.tool_call_parser:
|
||||
tool_parser = FunctionCallParser(
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
if required_json:
|
||||
tool_parser = JsonArrayParser()
|
||||
elif token_stream is None and self._tool_call_parsing_enabled(
|
||||
request, chat_tools
|
||||
):
|
||||
tool_parser = FunctionCallParser(
|
||||
chat_tools,
|
||||
self.tool_call_parser,
|
||||
tokenizer=self.tokenizer_manager.tokenizer,
|
||||
tool_choice=tool_choice,
|
||||
)
|
||||
reasoning_parser_obj: Optional[ReasoningParser] = None
|
||||
if self.reasoning_parser:
|
||||
if token_stream is None and self.reasoning_parser:
|
||||
reasoning_parser_obj = ReasoningParser(
|
||||
model_type=self.reasoning_parser,
|
||||
stream_reasoning=True,
|
||||
@@ -2308,36 +2393,45 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
finish_reason = meta.get("finish_reason") or finish_reason
|
||||
|
||||
text = chunk.get("text", "") or ""
|
||||
if incremental:
|
||||
delta = text
|
||||
else:
|
||||
delta = text[stream_offset:]
|
||||
stream_offset = len(text)
|
||||
if not delta and finish_reason is None:
|
||||
continue
|
||||
# finish_reason is sticky, so it would otherwise re-flush.
|
||||
flush = (
|
||||
not flushed
|
||||
and finish_reason is not None
|
||||
and finish_reason.get("type") != "abort"
|
||||
)
|
||||
flushed = flushed or flush
|
||||
|
||||
if reasoning_parser_obj is not None:
|
||||
reasoning_chunk, delta = reasoning_parser_obj.parse_stream_chunk(
|
||||
delta
|
||||
token_calls: list[ToolCallItem] = []
|
||||
if token_stream is not None:
|
||||
reasoning_chunk, delta, token_calls = token_stream.step(
|
||||
chunk, finish_reason
|
||||
)
|
||||
if flush:
|
||||
end_reasoning, end_normal = (
|
||||
reasoning_parser_obj.parse_stream_end()
|
||||
)
|
||||
if end_reasoning:
|
||||
reasoning_chunk = (reasoning_chunk or "") + end_reasoning
|
||||
if end_normal:
|
||||
delta = (delta or "") + end_normal
|
||||
flush = False
|
||||
else:
|
||||
reasoning_chunk = None
|
||||
text = chunk.get("text", "") or ""
|
||||
if incremental:
|
||||
delta = text
|
||||
else:
|
||||
delta = text[stream_offset:]
|
||||
stream_offset = len(text)
|
||||
if not delta and finish_reason is None:
|
||||
continue
|
||||
# finish_reason is sticky, so it would otherwise re-flush.
|
||||
flush = (
|
||||
not flushed
|
||||
and finish_reason is not None
|
||||
and finish_reason.get("type") != "abort"
|
||||
)
|
||||
flushed = flushed or flush
|
||||
|
||||
if reasoning_parser_obj is not None:
|
||||
reasoning_chunk, delta = (
|
||||
reasoning_parser_obj.parse_stream_chunk(delta)
|
||||
)
|
||||
if flush:
|
||||
end_reasoning, end_normal = (
|
||||
reasoning_parser_obj.parse_stream_end()
|
||||
)
|
||||
if end_reasoning:
|
||||
reasoning_chunk = (
|
||||
reasoning_chunk or ""
|
||||
) + end_reasoning
|
||||
if end_normal:
|
||||
delta = (delta or "") + end_normal
|
||||
else:
|
||||
reasoning_chunk = None
|
||||
|
||||
if reasoning_chunk:
|
||||
if message_state["open"]:
|
||||
@@ -2412,12 +2506,14 @@ class OpenAIServingResponses(OpenAIServingChat):
|
||||
)
|
||||
)
|
||||
|
||||
if not delta and not flush:
|
||||
if not delta and not flush and not token_calls:
|
||||
continue
|
||||
|
||||
if isinstance(tool_parser, JsonArrayParser):
|
||||
sp = tool_parser.parse_streaming_increment(delta, chat_tools)
|
||||
normal_text, tool_calls = sp.normal_text or "", sp.calls
|
||||
elif token_stream is not None:
|
||||
normal_text, tool_calls = delta, token_calls
|
||||
elif tool_parser is not None:
|
||||
normal_text, tool_calls = tool_parser.parse_stream_chunk(delta)
|
||||
if flush:
|
||||
|
||||
Reference in New Issue
Block a user