From 4b83f215f498bc0371587bfae4a9470f24e77737 Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Tue, 20 Oct 2026 14:10:01 +0800 Subject: [PATCH] [Feat] entrypoints: add a chat encoding adapter registry (#45300) --- .../openai/chat_encoding_adapters.py | 149 +++++++++ .../srt/entrypoints/openai/serving_chat.py | 127 +++++--- .../srt/entrypoints/openai/serving_inkling.py | 67 ++++ .../entrypoints/openai/serving_responses.py | 296 ++++++++++++------ .../entrypoints/openai/test_serving_chat.py | 4 + 5 files changed, 494 insertions(+), 149 deletions(-) create mode 100644 python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py create mode 100644 python/sglang/srt/entrypoints/openai/serving_inkling.py diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py b/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py new file mode 100644 index 0000000000..52fa6a1df9 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py @@ -0,0 +1,149 @@ +"""Unregistered chat encoding specs retain the existing serving branches; +concrete adapters load only when selected.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Callable, Literal, Optional, Protocol + +import msgspec + +if TYPE_CHECKING: + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest, ToolCall + from sglang.srt.function_call.core_types import ToolCallItem + +EncodingBackend = Literal["builtin", "recipe"] + + +class ChatOutputHooks(msgspec.Struct, frozen=True, kw_only=True): + """Chat-handler policies an output adapter needs, bound to one handler.""" + + tool_call_parsing_active: Callable[[ChatCompletionRequest], bool] + history_tool_calls_count: Callable[[ChatCompletionRequest], int] + tool_call_id: Callable[[ToolCallItem, int], str] + + +class ResponsesOutputStream(Protocol): + def step( + self, chunk: dict[str, Any], finish_reason: Optional[dict[str, Any]] + ) -> tuple[str, str, list[ToolCallItem]]: ... + + +class ChatEncodingOutputAdapter(Protocol): + def parse_chat_response( + self, + *, + request: ChatCompletionRequest, + output_ids: list[int], + finish_reason: dict[str, Any], + ) -> tuple[Optional[str], str, Optional[list[ToolCall]], dict[str, Any]]: ... + + def stream_chat_chunks( + self, + *, + content: dict[str, Any], + index: int, + request: ChatCompletionRequest, + created: int, + parser_dict: dict[int, Any], + has_tool_calls: dict[int, bool], + choice_logprobs: Optional[dict], + finish_reason_type: Optional[str], + usage: Optional[dict[str, Any]], + ) -> list[str]: ... + + def parse_responses_output( + self, + *, + output_ids: list[int], + finish_reason: Optional[dict[str, Any]], + separate_reasoning: bool, + parse_tool_calls: bool, + ) -> tuple[str, str, list[ToolCallItem]]: ... + + def new_responses_stream( + self, *, separate_reasoning: bool, parse_tool_calls: bool + ) -> ResponsesOutputStream: ... + + +class ChatEncodingAdapter(Protocol): + collects_openai_media: bool + """Whether media still has to be collected from the OpenAI messages after + ``encode_messages`` returns prompt ids.""" + + @property + def output(self) -> Optional[ChatEncodingOutputAdapter]: + """None when the configured parsers leave output to the text parsers.""" + ... + + def encode_messages( + self, + *, + messages: list[dict[str, Any]], + request: ChatCompletionRequest, + tools: Optional[list[dict]], + ) -> list[int]: ... + + +class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True): + """Everything a builder may read; a builder takes only what its encoding needs. + + Consumers: + tokenizer: prompt encoding (Inkling). + reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing. + hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id). + model_path, revision: checkpoint files (DeepSeek tokenizer.json and the + encoder file for effort-profile detection). + backend: selects the builder; "recipe" is an explicit opt-in. + """ + + chat_encoding_spec: str + tokenizer: Any + reasoning_parser: Optional[str] + tool_call_parser: Optional[str] + chat_hooks: ChatOutputHooks + hf_config: Any + model_path: str + revision: Optional[str] + backend: EncodingBackend = "builtin" + + def require(self, field: str) -> Any: + value = getattr(self, field) + if value is None: + raise ValueError( + f"chat encoding spec {self.chat_encoding_spec!r} ({self.backend} backend) " + f"requires {field}, which this server does not provide" + ) + return value + + +def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter: + from sglang.srt.entrypoints.openai.serving_inkling import InklingChatEncodingAdapter + + return InklingChatEncodingAdapter(tokenizer=ctx.tokenizer) + + +_BUILDERS: dict[ + str, + dict[EncodingBackend, Callable[[ChatEncodingAdapterContext], ChatEncodingAdapter]], +] = { + "inkling": {"builtin": _build_inkling}, +} + + +def has_chat_encoding_adapter(chat_encoding_spec: Optional[str]) -> bool: + return chat_encoding_spec in _BUILDERS + + +def build_chat_encoding_adapter( + ctx: ChatEncodingAdapterContext, +) -> Optional[ChatEncodingAdapter]: + builders = _BUILDERS.get(ctx.chat_encoding_spec, {}) + builder = builders.get(ctx.backend) + if builder is not None: + return builder(ctx) + if ctx.backend != "builtin": + raise ValueError( + f"chat encoding spec {ctx.chat_encoding_spec!r} has no {ctx.backend} backend; " + f"available: {sorted(builders) or ['builtin']}" + ) + return None diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 9ecca0c71e..f195b837dd 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -42,6 +42,7 @@ from jsonschema import Draft202012Validator, SchemaError from sglang.srt.entrypoints.openai import ( chat_encoding, + chat_encoding_adapters, encoding_dsv4, encoding_dsv32, encoding_dsv41, @@ -345,14 +346,7 @@ class OpenAIServingChat(OpenAIServingBase): else None ) - # Resolve the env-configured Inkling effort default once: the env var is - # frozen for the server's lifetime, and a misconfigured value should - # fail at boot, not 400 every request. - self._inkling_default_reasoning_effort: float | None = ( - self._get_inkling_default_reasoning_effort() - if self.chat_encoding_spec == "inkling" - else None - ) + self._encoding_adapter = self._build_chat_encoding_adapter() self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = ( chat_encoding.default_dsv41_reasoning_effort_from_env( envs.SGLANG_DSV41_REASONING_EFFORT.get() @@ -489,6 +483,37 @@ class OpenAIServingChat(OpenAIServingBase): tool_call_parser=self.tool_call_parser, ) + def _build_chat_encoding_adapter( + self, + ) -> Optional[chat_encoding_adapters.ChatEncodingAdapter]: + if not chat_encoding_adapters.has_chat_encoding_adapter( + self.chat_encoding_spec + ): + return None + return chat_encoding_adapters.build_chat_encoding_adapter( + chat_encoding_adapters.ChatEncodingAdapterContext( + chat_encoding_spec=self.chat_encoding_spec, + tokenizer=self.tokenizer_manager.tokenizer, + reasoning_parser=self.reasoning_parser, + tool_call_parser=self.tool_call_parser, + chat_hooks=chat_encoding_adapters.ChatOutputHooks( + tool_call_parsing_active=self._tool_call_parsing_active, + history_tool_calls_count=self._get_history_tool_calls_cnt, + tool_call_id=self._process_tool_call_id, + ), + hf_config=self.tokenizer_manager.model_config.hf_config, + model_path=self.tokenizer_manager.model_path, + revision=get_model().revision, + ) + ) + + def _encoding_output( + self, + ) -> Optional[chat_encoding_adapters.ChatEncodingOutputAdapter]: + if self._encoding_adapter is None: + return None + return self._encoding_adapter.output + def _request_id_prefix(self) -> str: return "chatcmpl-" @@ -588,44 +613,10 @@ class OpenAIServingChat(OpenAIServingBase): Returns prompt_ids if handled, None to use default encoding. """ - if self.chat_encoding_spec == "inkling": - # Inkling: render messages -> input_ids with framing tokens + ONE placeholder per - # media (encoding/expansion happens later in InklingMultimodalProcessor). The - # server's tokenizer is the base tiktoken backend; wrap it so encode_special - # supplies the framing-token overlay. - from sglang.srt.parser.inkling_renderer import render_inkling_messages - from sglang.srt.parser.inkling_tokenizer import ( - CONTENT_TEXT, - MESSAGE_MODEL, - InklingTokenizer, + if self._encoding_adapter is not None: + return self._encoding_adapter.encode_messages( + messages=messages, request=request, tools=tools ) - - inkling_tokenizer = InklingTokenizer( - tokenizer=self.tokenizer_manager.tokenizer - ) - reasoning_effort = self._parse_inkling_reasoning_effort( - request.reasoning_effort - ) - if reasoning_effort is None: - reasoning_effort = self._inkling_default_reasoning_effort - assistant_prefix = self._pop_inkling_assistant_prefix(messages, request) - prompt_ids = render_inkling_messages( - messages, - inkling_tokenizer, - add_generation_prompt=False, - tools=tools, - reasoning_effort=reasoning_effort, - ) - if assistant_prefix is not None: - # Continue the final assistant message inside an OPEN model text - # block: header + payload, no <|end_message|> and no - # <|content_model_end_sampling|>, so the model resumes the turn. - prompt_ids += [ - inkling_tokenizer.encode_special(MESSAGE_MODEL), - inkling_tokenizer.encode_special(CONTENT_TEXT), - *inkling_tokenizer.encode_text(assistant_prefix), - ] - return prompt_ids if self.chat_encoding_spec == "kimi_k3": messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages( messages, request @@ -900,6 +891,30 @@ class OpenAIServingChat(OpenAIServingBase): completion_tokens: dict[int, int], ) -> AsyncGenerator[str, None]: """Generate SSE chunks for streaming content.""" + encoding_output = self._encoding_output() + if encoding_output is not None: + usage = None + if continuous_usage_stats: + usage = UsageProcessor.calculate_token_usage( + prompt_tokens=self._reported_prompt_tokens(content["meta_info"]), + reasoning_tokens=content["meta_info"].get("reasoning_tokens", 0), + completion_tokens=content["meta_info"].get("completion_tokens", 0), + cached_tokens=self._continuous_usage_cached_details(content), + ).model_dump() + for chunk in encoding_output.stream_chat_chunks( + content=content, + index=index, + request=request, + created=created, + parser_dict=parser_dict, + has_tool_calls=has_tool_calls, + choice_logprobs=choice_logprobs, + finish_reason_type=finish_reason_type, + usage=usage, + ): + yield chunk + return + offset = stream_offsets.get(index, 0) if get_serving().incremental_streaming_output: delta = content["text"] @@ -1556,7 +1571,12 @@ class OpenAIServingChat(OpenAIServingBase): ) if prompt_ids is not None: - if self.chat_encoding_spec in ("inkling", "kimi_k3"): + collects_openai_media = ( + self._encoding_adapter.collects_openai_media + if self._encoding_adapter is not None + else self.chat_encoding_spec == "kimi_k3" + ) + if collects_openai_media: for message in request.messages: msg_dict = message.model_dump() if msg_dict.get("content") is None: @@ -2397,6 +2417,7 @@ class OpenAIServingChat(OpenAIServingBase): output_ids=output_ids, ) + encoding_output = self._encoding_output() for idx, ret_item in enumerate(ret): # Process logprobs choice_logprobs = None @@ -2414,7 +2435,16 @@ class OpenAIServingChat(OpenAIServingBase): # Handle reasoning content reasoning_text = None - if self.reasoning_parser and request.separate_reasoning: + tool_calls = None + if encoding_output is not None: + reasoning_text, text, tool_calls, finish_reason = ( + encoding_output.parse_chat_response( + request=request, + output_ids=ret_item["output_ids"], + finish_reason=finish_reason, + ) + ) + elif self.reasoning_parser and request.separate_reasoning: force_reasoning = ( self.template_manager.force_reasoning or self._get_reasoning_from_request(request) @@ -2438,9 +2468,8 @@ class OpenAIServingChat(OpenAIServingBase): ) # Handle tool calls - tool_calls = None effective_tools = self._effective_tools(request) - if self._tool_call_parsing_active(request): + if encoding_output is None and self._tool_call_parsing_active(request): history_tool_calls_cnt = self._get_history_tool_calls_cnt(request) tool_calls, text, finish_reason = self._process_tool_calls( text, diff --git a/python/sglang/srt/entrypoints/openai/serving_inkling.py b/python/sglang/srt/entrypoints/openai/serving_inkling.py new file mode 100644 index 0000000000..c65d610909 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_inkling.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional + +from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + +if TYPE_CHECKING: + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + +class InklingChatEncodingAdapter: + """Inkling prompt encoding through ``inkling_renderer``; output stays with + the registered ``inkling`` reasoning and tool-call parsers.""" + + collects_openai_media = True + output = None + + def __init__(self, *, tokenizer: Any): + self._tokenizer = tokenizer + # Resolved at boot: a misconfigured env default fails startup, not requests. + self._default_reasoning_effort = ( + OpenAIServingChat._get_inkling_default_reasoning_effort() + ) + + def encode_messages( + self, + *, + messages: list[dict[str, Any]], + request: ChatCompletionRequest, + tools: Optional[list[dict]], + ) -> list[int]: + from sglang.srt.parser.inkling_renderer import render_inkling_messages + from sglang.srt.parser.inkling_tokenizer import ( + CONTENT_TEXT, + MESSAGE_MODEL, + InklingTokenizer, + ) + + # Framing tokens + ONE placeholder per media (encoding/expansion happens later + # in InklingMultimodalProcessor). The server's tokenizer is the base tiktoken + # backend; wrap it so encode_special supplies the framing-token overlay. + inkling_tokenizer = InklingTokenizer(tokenizer=self._tokenizer) + reasoning_effort = OpenAIServingChat._parse_inkling_reasoning_effort( + request.reasoning_effort + ) + if reasoning_effort is None: + reasoning_effort = self._default_reasoning_effort + assistant_prefix = OpenAIServingChat._pop_inkling_assistant_prefix( + messages, request + ) + prompt_ids = render_inkling_messages( + messages, + inkling_tokenizer, + add_generation_prompt=False, + tools=tools, + reasoning_effort=reasoning_effort, + ) + if assistant_prefix is not None: + # Continue the final assistant message inside an OPEN model text + # block: header + payload, no <|end_message|> and no + # <|content_model_end_sampling|>, so the model resumes the turn. + prompt_ids += [ + inkling_tokenizer.encode_special(MESSAGE_MODEL), + inkling_tokenizer.encode_special(CONTENT_TEXT), + *inkling_tokenizer.encode_text(assistant_prefix), + ] + return prompt_ids diff --git a/python/sglang/srt/entrypoints/openai/serving_responses.py b/python/sglang/srt/entrypoints/openai/serving_responses.py index d4e4c2d300..6ce269e0fe 100644 --- a/python/sglang/srt/entrypoints/openai/serving_responses.py +++ b/python/sglang/srt/entrypoints/openai/serving_responses.py @@ -87,6 +87,7 @@ from sglang.srt.entrypoints.openai.responses_adapters import ( from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat from sglang.srt.entrypoints.openai.tool_server import MCPToolServer, ToolServer from sglang.srt.entrypoints.openai.utils import to_openai_style_logprobs +from sglang.srt.function_call.core_types import ToolCallItem from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.managers.io_struct import GenerateReqInput @@ -799,6 +800,8 @@ class OpenAIServingResponses(OpenAIServingChat): tokenizer, output_logprobs=output_logprobs, require_reasoning=require_reasoning, + output_ids=final_res.get("output_ids"), + finish_reason=meta_info.get("finish_reason") if meta_info else None, ) if meta_info is not None: @@ -1022,8 +1025,84 @@ class OpenAIServingResponses(OpenAIServingChat): output_logprobs: Optional[list] = None, *, require_reasoning: bool, + output_ids: Optional[list[int]] = None, + finish_reason: Optional[dict[str, Any]] = None, ): chat_tools = self._response_tools_to_chat_tools(request) + custom_names = custom_tool_names(request.tools) + encoding_output = self._encoding_output() + if encoding_output is not None: + required_json = self._uses_required_tool_json(request, chat_tools) + reasoning_content, content, calls = encoding_output.parse_responses_output( + output_ids=output_ids or [], + finish_reason=finish_reason, + separate_reasoning=bool(self.reasoning_parser), + parse_tool_calls=( + self._tool_call_parsing_enabled(request, chat_tools) + and not required_json + ), + ) + if required_json and content: + content, tool_call_items = self._parse_required_tool_json( + content, custom_names + ) + else: + tool_call_items = [ + self._make_tool_call_item(call.name, call.parameters, custom_names) + for call in calls + ] + else: + reasoning_content, content, tool_call_items = self._parse_text_output( + request, + final_output, + chat_tools, + custom_names, + require_reasoning=require_reasoning, + ) + + output_items = [] + if reasoning_content: + output_items.append( + self._make_reasoning_item( + request, + reasoning_content, + item_id=f"rs_{random_uuid()}", + status=None, + ) + ) + if content: + output_text = ResponseOutputText( + text=content, + annotations=[], # TODO + type="output_text", + # logprobs cover all generated tokens, not just the stripped content. + logprobs=output_logprobs, + ) + message = ResponseOutputMessage( + phase="commentary" if tool_call_items else "final_answer", + id=f"msg_{random_uuid()}", + content=[output_text], + role="assistant", + status="completed", + type="message", + ) + output_items.append(message) + output_items.extend(tool_call_items) + return output_items + + def _parse_text_output( + self, + request: ResponsesRequest, + final_output: Any, + chat_tools: list[Tool], + custom_names: set[str], + *, + require_reasoning: bool, + ) -> tuple[ + Optional[str], + Any, + list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]], + ]: if self.reasoning_parser: reasoning_parser = ReasoningParser( model_type=self.reasoning_parser, @@ -1046,20 +1125,8 @@ class OpenAIServingResponses(OpenAIServingChat): reasoning_content = None content = final_output - output_items = [] - if reasoning_content: - output_items.append( - self._make_reasoning_item( - request, - reasoning_content, - item_id=f"rs_{random_uuid()}", - status=None, - ) - ) - tool_choice = request.effective_tool_choice() is_required = tool_choice == "required" or isinstance(tool_choice, dict) - custom_names = custom_tool_names(request.tools) tool_call_items: list[ Union[ResponseFunctionToolCall, ResponseCustomToolCall] ] = [] @@ -1101,45 +1168,32 @@ class OpenAIServingResponses(OpenAIServingChat): and not parsed_via_native and not detector_owns_format ): - try: - tool_call_data = orjson.loads(content) - if isinstance(tool_call_data, dict): - tool_call_data = [tool_call_data] - if isinstance(tool_call_data, list): - for tool in tool_call_data: - if not isinstance(tool, dict) or "name" not in tool: - continue - arguments = json.dumps( - tool.get("parameters", {}), ensure_ascii=False - ) - tool_call_items.append( - self._make_tool_call_item( - tool["name"], arguments, custom_names - ) - ) - content = "" - except Exception as e: - logger.error("Required tool JSON parse error: %s", e) + content, json_items = self._parse_required_tool_json(content, custom_names) + tool_call_items.extend(json_items) + return reasoning_content, content, tool_call_items - if content: - output_text = ResponseOutputText( - text=content, - annotations=[], # TODO - type="output_text", - # logprobs cover all generated tokens, not just the stripped content. - logprobs=output_logprobs, - ) - message = ResponseOutputMessage( - phase="commentary" if tool_call_items else "final_answer", - id=f"msg_{random_uuid()}", - content=[output_text], - role="assistant", - status="completed", - type="message", - ) - output_items.append(message) - output_items.extend(tool_call_items) - return output_items + def _parse_required_tool_json( + self, content: str, custom_names: set[str] + ) -> tuple[str, list[Union[ResponseFunctionToolCall, ResponseCustomToolCall]]]: + tool_call_items = [] + try: + tool_call_data = orjson.loads(content) + if isinstance(tool_call_data, dict): + tool_call_data = [tool_call_data] + if isinstance(tool_call_data, list): + for tool in tool_call_data: + if not isinstance(tool, dict) or "name" not in tool: + continue + arguments = json.dumps( + tool.get("parameters", {}), ensure_ascii=False + ) + tool_call_items.append( + self._make_tool_call_item(tool["name"], arguments, custom_names) + ) + content = "" + except Exception as e: + logger.error("Required tool JSON parse error: %s", e) + return content, tool_call_items def _make_response_output_items_with_harmony( self, @@ -1159,10 +1213,37 @@ class OpenAIServingResponses(OpenAIServingChat): output_items.extend(last_items) return output_items + def _tool_call_parsing_enabled( + self, request: ResponsesRequest, chat_tools: list[Tool] + ) -> bool: + return bool( + chat_tools and self.tool_call_parser and request.tool_choice != "none" + ) + @staticmethod def _tool_parser_owns_format(parser: FunctionCallParser) -> bool: return parser.owns_tool_format() + def _uses_required_tool_json( + self, request: ResponsesRequest, chat_tools: list[Tool] + ) -> bool: + """Whether required/named tool output is constrained to the generic JSON + array, rather than to the configured detector's native framing.""" + tool_choice = request.effective_tool_choice() + if not chat_tools or not ( + tool_choice == "required" or isinstance(tool_choice, dict) + ): + return False + if not self.tool_call_parser: + return True + probe = FunctionCallParser( + chat_tools, + self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, + tool_choice=tool_choice, + ) + return not self._tool_parser_owns_format(probe) + @staticmethod def _chat_tool_choice(tool_choice: Any) -> Any: """Nest an ``effective_tool_choice()`` result the way chat expects: @@ -1998,29 +2079,33 @@ class OpenAIServingResponses(OpenAIServingChat): chat_tools = self._response_tools_to_chat_tools(request) custom_names = custom_tool_names(request.tools) tool_choice = request.effective_tool_choice() - is_required = tool_choice == "required" or isinstance(tool_choice, dict) + required_json = self._uses_required_tool_json(request, chat_tools) + encoding_output = self._encoding_output() + token_stream = ( + encoding_output.new_responses_stream( + separate_reasoning=bool(self.reasoning_parser), + parse_tool_calls=( + self._tool_call_parsing_enabled(request, chat_tools) + and not required_json + ), + ) + if encoding_output is not None + else None + ) tool_parser: Optional[Union[FunctionCallParser, JsonArrayParser]] = None - if chat_tools and request.tool_choice != "none": - detector_owns_format = False - if self.tool_call_parser: - probe = FunctionCallParser( - chat_tools, - self.tool_call_parser, - tokenizer=self.tokenizer_manager.tokenizer, - tool_choice=tool_choice, - ) - detector_owns_format = self._tool_parser_owns_format(probe) - if is_required and not detector_owns_format: - tool_parser = JsonArrayParser() - elif self.tool_call_parser: - tool_parser = FunctionCallParser( - chat_tools, - self.tool_call_parser, - tokenizer=self.tokenizer_manager.tokenizer, - tool_choice=tool_choice, - ) + if required_json: + tool_parser = JsonArrayParser() + elif token_stream is None and self._tool_call_parsing_enabled( + request, chat_tools + ): + tool_parser = FunctionCallParser( + chat_tools, + self.tool_call_parser, + tokenizer=self.tokenizer_manager.tokenizer, + tool_choice=tool_choice, + ) reasoning_parser_obj: Optional[ReasoningParser] = None - if self.reasoning_parser: + if token_stream is None and self.reasoning_parser: reasoning_parser_obj = ReasoningParser( model_type=self.reasoning_parser, stream_reasoning=True, @@ -2308,36 +2393,45 @@ class OpenAIServingResponses(OpenAIServingChat): ) finish_reason = meta.get("finish_reason") or finish_reason - text = chunk.get("text", "") or "" - if incremental: - delta = text - else: - delta = text[stream_offset:] - stream_offset = len(text) - if not delta and finish_reason is None: - continue - # finish_reason is sticky, so it would otherwise re-flush. - flush = ( - not flushed - and finish_reason is not None - and finish_reason.get("type") != "abort" - ) - flushed = flushed or flush - - if reasoning_parser_obj is not None: - reasoning_chunk, delta = reasoning_parser_obj.parse_stream_chunk( - delta + token_calls: list[ToolCallItem] = [] + if token_stream is not None: + reasoning_chunk, delta, token_calls = token_stream.step( + chunk, finish_reason ) - if flush: - end_reasoning, end_normal = ( - reasoning_parser_obj.parse_stream_end() - ) - if end_reasoning: - reasoning_chunk = (reasoning_chunk or "") + end_reasoning - if end_normal: - delta = (delta or "") + end_normal + flush = False else: - reasoning_chunk = None + text = chunk.get("text", "") or "" + if incremental: + delta = text + else: + delta = text[stream_offset:] + stream_offset = len(text) + if not delta and finish_reason is None: + continue + # finish_reason is sticky, so it would otherwise re-flush. + flush = ( + not flushed + and finish_reason is not None + and finish_reason.get("type") != "abort" + ) + flushed = flushed or flush + + if reasoning_parser_obj is not None: + reasoning_chunk, delta = ( + reasoning_parser_obj.parse_stream_chunk(delta) + ) + if flush: + end_reasoning, end_normal = ( + reasoning_parser_obj.parse_stream_end() + ) + if end_reasoning: + reasoning_chunk = ( + reasoning_chunk or "" + ) + end_reasoning + if end_normal: + delta = (delta or "") + end_normal + else: + reasoning_chunk = None if reasoning_chunk: if message_state["open"]: @@ -2412,12 +2506,14 @@ class OpenAIServingResponses(OpenAIServingChat): ) ) - if not delta and not flush: + if not delta and not flush and not token_calls: continue if isinstance(tool_parser, JsonArrayParser): sp = tool_parser.parse_streaming_increment(delta, chat_tools) normal_text, tool_calls = sp.normal_text or "", sp.calls + elif token_stream is not None: + normal_text, tool_calls = delta, token_calls elif tool_parser is not None: normal_text, tool_calls = tool_parser.parse_stream_chunk(delta) if flush: diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index ccbf0f3cfb..8679166e42 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -38,6 +38,7 @@ from sglang.srt.entrypoints.openai.serving_chat import ( OpenAIServingChat, normalize_tool_content, ) +from sglang.srt.entrypoints.openai.serving_inkling import InklingChatEncodingAdapter from sglang.srt.environ import envs from sglang.srt.function_call.kimik3_format import TOOLS_CLOSE, TOOLS_OPEN from sglang.srt.managers.io_struct import GenerateReqInput @@ -5236,6 +5237,7 @@ class InklingReasoningEffortTest(unittest.TestCase): serving = object.__new__(OpenAIServingChat) serving.chat_encoding_spec = "inkling" serving.tokenizer_manager = Mock(tokenizer=Tokenizer()) + serving._encoding_adapter = InklingChatEncodingAdapter(tokenizer=Tokenizer()) request = ChatCompletionRequest( model="test-model", messages=[{"role": "user", "content": "hello"}], @@ -5263,6 +5265,7 @@ class InklingReasoningEffortTest(unittest.TestCase): serving = object.__new__(OpenAIServingChat) serving.chat_encoding_spec = "inkling" serving.tokenizer_manager = Mock(tokenizer=Tokenizer()) + serving._encoding_adapter = InklingChatEncodingAdapter(tokenizer=Tokenizer()) request = ChatCompletionRequest( model="test-model", messages=[ @@ -5299,6 +5302,7 @@ class InklingReasoningEffortTest(unittest.TestCase): serving = object.__new__(OpenAIServingChat) serving.chat_encoding_spec = "inkling" serving.tokenizer_manager = Mock(tokenizer=Tokenizer()) + serving._encoding_adapter = InklingChatEncodingAdapter(tokenizer=Tokenizer()) request = ChatCompletionRequest( model="test-model", messages=[