Files
sglang/python/sglang/srt/entrypoints/openai/serving_chat.py
T
minke.yu 77e7555ae3 G9 P0 fix: /query_cache_hit 绕过 _tokenize_one_request 的 ReqState 依赖
_tokenize_one_request._create_tokenized_object 会读 rid_to_state[obj.rid]
(正常生成路径在 tokenize 前注册 ReqState),纯查询无 rid → KeyError: None。
改为:dsv41 chat encoding 直接渲染的 prompt_ids 直接用;否则对渲染后
text 调 _tokenize_texts(与真实路径同一分词入口)。tok8 环境实测 200。

(cherry picked from commit 71c6d8bc55)
2026-10-19 10:46:23 +08:00

3277 lines
135 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import copy
import json
import logging
import math
import time
import uuid
from collections import OrderedDict
from collections.abc import AsyncGenerator
from enum import Enum
from http import HTTPStatus
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
from sglang.srt.runtime_context import get_model, get_serving
class ThinkingMode(str, Enum):
"""Mode for message encoding - chat vs thinking/reasoning."""
CHAT = "chat"
THINKING = "thinking"
import jinja2
import orjson
from fastapi import Request
try:
from mistral_common.exceptions import MistralCommonException
_MISTRAL_COMMON_ERRORS: tuple[type[BaseException], ...] = (MistralCommonException,)
except ImportError:
_MISTRAL_COMMON_ERRORS = ()
_CHAT_TEMPLATE_CLIENT_ERRORS: tuple[type[BaseException], ...] = (
jinja2.TemplateError,
TypeError,
) + _MISTRAL_COMMON_ERRORS
from fastapi.responses import ORJSONResponse, StreamingResponse
from jsonschema import Draft202012Validator, SchemaError
from sglang.srt.entrypoints.openai import (
chat_encoding,
encoding_dsv4,
encoding_dsv32,
encoding_dsv41,
)
from sglang.srt.entrypoints.openai.protocol import (
ChatCompletionMessageContentTextPart,
ChatCompletionMessageContentVideoPart,
ChatCompletionMessageGenericParam,
ChatCompletionMessageUserParam,
ChatCompletionRequest,
ChatCompletionResponse,
ChatCompletionResponseChoice,
ChatCompletionResponseStreamChoice,
ChatCompletionStreamResponse,
ChatCompletionTokenLogprob,
ChatMessage,
ChoiceLogprobs,
DeltaMessage,
ErrorResponse,
FunctionResponse,
MessageProcessingResult,
PromptTokensDetails,
ResponseParserProtocol,
SglExt,
Tool,
ToolCall,
ToolCallProcessingResult,
ToolChoice,
TopLogprob,
)
from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
from sglang.srt.entrypoints.openai.sse_utils import build_sse_content
from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor
from sglang.srt.entrypoints.openai.utils import (
cached_tokens_details_from_dict,
process_cached_tokens_details_from_ret,
process_hidden_states_for_response,
process_hidden_states_from_ret,
process_routed_experts_from_ret,
process_spec_tokens_details_from_ret,
should_include_usage,
spec_tokens_details_from_meta_info,
token_id_to_bytes,
)
from sglang.srt.entrypoints.request_headers import apply_header_overrides
from sglang.srt.environ import envs
from sglang.srt.function_call.core_types import ToolCallItem
from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.function_call.json_array_parser import JsonArrayParser
from sglang.srt.function_call.utils import (
get_json_schema_constraint,
normalize_json_schema_types,
strip_structural_tag_excludes,
)
from sglang.srt.managers.io_struct import GenerateReqInput
from sglang.srt.parser.conversation import generate_chat_conv
from sglang.srt.parser.hunyuan_reasoning import (
normalize_hunyuan_reasoning_effort,
uses_hunyuan_reasoning_effort,
)
from sglang.srt.parser.jinja_template_utils import (
MEDIA_URL_PART_TYPES,
process_content_for_template_format,
)
from sglang.srt.parser.reasoning_parser import (
IQuestQ1ReasoningDetector,
ReasoningParser,
)
from sglang.srt.sampling.sampling_params import (
set_request_reasoning_end_token_ids,
)
from sglang.srt.utils import ImageData
from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata
if TYPE_CHECKING:
from sglang.srt.managers.tokenizer_manager import TokenizerManager
from sglang.srt.parser.template_manager import TemplateManager
logger = logging.getLogger(__name__)
_MEDIA_CONTENT_PART_TYPES = frozenset({"image_url", "video_url", "audio_url"})
_CHAT_TEMPLATE_CACHE_MAX_SIZE = 128
def normalize_tool_content(role: str, content):
"""Normalize tool message content from OpenAI array format to plain string.
OpenAI clients may send tool content as a list of content parts
(e.g. [{"type":"text","text":"..."}]) but most chat templates expect
a plain string for tool messages. Only flatten when ALL items are
pure OpenAI text parts; preserve lists containing non-text-type items
that some templates intentionally iterate over.
"""
if role != "tool" or not isinstance(content, list):
return content
parts = content
is_openai_text_parts = all(
(isinstance(p, dict) and p.get("type") == "text") or isinstance(p, str)
for p in parts
)
if is_openai_text_parts:
text_parts = [p.get("text", "") if isinstance(p, dict) else p for p in parts]
return " ".join(text_parts)
return content
def parse_tool_call_arguments(arguments: str) -> dict[str, Any]:
"""Parse OpenAI tool call arguments for chat templates."""
try:
parsed_arguments = orjson.loads(arguments)
except orjson.JSONDecodeError as exc:
raise ValueError(
"Assistant tool call function.arguments must be valid JSON."
) from exc
if not isinstance(parsed_arguments, dict):
raise ValueError(
"Assistant tool call function.arguments must be a JSON object."
)
return parsed_arguments
def normalize_assistant_tool_call_arguments(
message: dict[str, Any], *, strict: bool = True
) -> None:
"""Normalize assistant history tool call arguments in-place."""
if message.get("role") != "assistant" or not isinstance(
message.get("tool_calls"), list
):
return
for item in message["tool_calls"]:
function = item.get("function") if isinstance(item, dict) else None
if not isinstance(function, dict):
continue
if "arguments" in function and isinstance(function["arguments"], str):
try:
function["arguments"] = parse_tool_call_arguments(function["arguments"])
except ValueError:
if strict:
raise
def _extract_max_dynamic_patch(request: ChatCompletionRequest):
img_vals = []
vid_vals = []
for msg in request.messages or []:
content = getattr(msg, "content", None)
if not isinstance(content, list):
continue
for part in content:
# pydantic object or dict type
if getattr(part, "type", None) == "image_url":
iu = getattr(part, "image_url", None)
mdp = getattr(iu, "max_dynamic_patch", None) if iu else None
if mdp is not None:
img_vals.append(int(mdp))
elif getattr(part, "type", None) == "video_url":
vu = getattr(part, "video_url", None)
mdp = getattr(vu, "max_dynamic_patch", None) if vu else None
if mdp is not None:
vid_vals.append(int(mdp))
# TODO(yuan-luo): per-item max_dynamic_patch for both image and video
img_max_dynamic_patch = min(img_vals) if img_vals else None
vid_max_dynamic_patch = min(vid_vals) if vid_vals else None
return img_max_dynamic_patch, vid_max_dynamic_patch
KIMI_K3_IMAGE_PLACEHOLDER = "<|kimi_image_placeholder|>"
KIMI_K3_IMAGE_PLACEHOLDER_ESCAPED = "<| kimi_image_placeholder |>"
def neutralize_kimi_k3_image_placeholder(text: str) -> str:
return text.replace(KIMI_K3_IMAGE_PLACEHOLDER, KIMI_K3_IMAGE_PLACEHOLDER_ESCAPED)
def neutralize_kimi_k3_image_placeholder_value(value: Any) -> Any:
if isinstance(value, str):
return neutralize_kimi_k3_image_placeholder(value)
if isinstance(value, list):
return [neutralize_kimi_k3_image_placeholder_value(item) for item in value]
if isinstance(value, dict):
return {
key: neutralize_kimi_k3_image_placeholder_value(item)
for key, item in value.items()
}
return value
def _extract_video_question(request: ChatCompletionRequest) -> str | None:
"""Return text paired with a video in the last user turn."""
for message in reversed(request.messages or []):
if not isinstance(message, ChatCompletionMessageUserParam):
continue
content = message.content
if not isinstance(content, list):
continue
has_video = any(
isinstance(part, ChatCompletionMessageContentVideoPart) for part in content
)
if not has_video:
continue
return "".join(
part.text
for part in content
if isinstance(part, ChatCompletionMessageContentTextPart)
)
return None
def _build_video_config(request: ChatCompletionRequest) -> dict[str, Any] | None:
"""Build request-scoped video processor config without model-specific fields."""
config = dict(request.video_config or {})
question = _extract_video_question(request)
if question is not None:
# Internal metadata derived from the message must not be overridden by
# a model-specific public processor option.
config["_question"] = question
return config or None
class OpenAIServingChat(OpenAIServingBase):
"""Handler for /v1/chat/completions requests"""
_default_sampling_params_logged = False
_KIMI_K3_GENERATION_STUB_TOKENS = 3
def __init__(
self,
tokenizer_manager: TokenizerManager,
template_manager: TemplateManager,
):
super().__init__(tokenizer_manager)
self.template_manager = template_manager
self.tool_call_parser = self.tokenizer_manager.config_value("tool_call_parser")
self.reasoning_parser = self.tokenizer_manager.config_value("reasoning_parser")
self.default_chat_template_kwargs = (
get_serving().default_chat_template_kwargs or {}
)
self._reasoning_detector = None
if self.reasoning_parser:
try:
rp = ReasoningParser(
model_type=self.reasoning_parser,
stream_reasoning=True,
tokenizer=self.tokenizer_manager.tokenizer,
)
self._reasoning_detector = rp.detector
except ValueError as e:
logger.warning(
"Failed to initialize reasoning detector for parser '%s': %s",
self.reasoning_parser,
e,
)
# Get default sampling parameters from model's generation config
self.default_sampling_params = (
self.tokenizer_manager.model_config.get_default_sampling_params()
)
if (
self.default_sampling_params
and not OpenAIServingChat._default_sampling_params_logged
):
logger.info(
f"Using default chat sampling params from model generation config: {self.default_sampling_params}",
)
OpenAIServingChat._default_sampling_params_logged = True
# Check if the model is a GPT-OSS model
self.is_gpt_oss = (
hasattr(self.tokenizer_manager.model_config, "hf_config")
and hasattr(self.tokenizer_manager.model_config.hf_config, "model_type")
and self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss"
)
self.is_gemma4 = (
hasattr(self.tokenizer_manager.model_config, "hf_config")
and hasattr(self.tokenizer_manager.model_config.hf_config, "model_type")
and self.tokenizer_manager.model_config.hf_config.model_type
in ("gemma4", "gemma4_unified")
)
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
self._dsv4_reasoning_effort_profile = (
chat_encoding.resolve_dsv4_reasoning_effort_profile(
model_path=self.tokenizer_manager.model_path,
revision=get_model().revision,
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
),
)
if self.chat_encoding_spec == "dsv4"
else None
)
# Resolve the env-configured Inkling effort default once: the env var is
# frozen for the server's lifetime, and a misconfigured value should
# fail at boot, not 400 every request.
self._inkling_default_reasoning_effort: float | None = (
self._get_inkling_default_reasoning_effort()
if self.chat_encoding_spec == "inkling"
else None
)
self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = (
chat_encoding.default_dsv41_reasoning_effort_from_env(
envs.SGLANG_DSV41_REASONING_EFFORT.get()
)
if self.chat_encoding_spec == "dsv41"
else None
)
# Per-request response parser for custom decoding (set by _encode_messages)
self._response_parser: ResponseParserProtocol | None = None
# Probe whether ``encode("")`` returns specials. If it does, we must
# keep ``add_special_tokens=False`` at the chat-template encode site
# to avoid double BOS; otherwise the kwarg is a no-op and dropping it
# lets slow tokenizers (e.g. Kimi's TikTokenTokenizer) stay on the
# fast internal path.
try:
self._tokenizer_auto_adds_specials = (
len(self.tokenizer_manager.tokenizer.encode("")) > 0
)
except Exception:
self._tokenizer_auto_adds_specials = True
self._prompt_text_round_trip_is_lossy = self._probe_prompt_text_round_trip()
self._chat_template_cache: OrderedDict[bytes, tuple[tuple[int, ...], str]] = (
OrderedDict()
)
def _probe_prompt_text_round_trip(self) -> bool:
"""Does rendering the chat template to text and re-encoding lose anything?
mistral_common tokenizers emit control tokens ([INST],
[AVAILABLE_TOOLS], ...) that have no text form. Rendering to a string
turns them into literal characters and re-encoding also prepends a
second BOS, so the model sees the letters "AVAILABLE_TOOLS" instead of
the control token that frames the tool block. Encoding straight to ids
is the only faithful route on such tokenizers, so compare the two here
once and remember which to trust.
"""
probe = [{"role": "user", "content": "x"}]
try:
tokenizer = self.tokenizer_manager.tokenizer
rendered = tokenizer.apply_chat_template(
probe, tokenize=False, add_generation_prompt=True, return_dict=False
)
encode_kwargs = (
{"add_special_tokens": False}
if self._tokenizer_auto_adds_specials
else {}
)
via_text = tokenizer.encode(rendered, **encode_kwargs)
via_ids = tokenizer.apply_chat_template(
probe, tokenize=True, add_generation_prompt=True, return_dict=False
)
return list(via_text) != list(via_ids)
except Exception:
# A template that needs kwargs this probe does not supply tells us
# nothing; keep the long-standing text path.
return False
def _handle_last_assistant_message(
self,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
) -> tuple[list[dict[str, Any]], str | None]:
"""
Handle continue_final_message feature: separate final assistant message.
If continue_final_message is enabled and the last message is from assistant,
extract its content and remove it from the message list.
If continue_final_message is False and the last message is from assistant,
convert it to a user message to ensure the last message is always from user.
Only processes text-based content (strings), ignoring multimodal content (lists).
Args:
messages: List of message dictionaries
request: ChatCompletionRequest with continue_final_message flag
Returns:
Tuple of (processed_messages, assistant_prefix)
- processed_messages: Messages with last assistant message handled appropriately
- assistant_prefix: Content of the last assistant message (string only), or None
"""
assistant_prefix = None
if messages and messages[-1].get("role") == "assistant":
last_content = messages[-1].get("content")
# Only process string content, ignore multimodal content (lists)
if isinstance(last_content, str):
if request.continue_final_message:
# Extract content and remove the assistant message
assistant_prefix = last_content
messages = messages[:-1]
else:
# Convert the last assistant message to user message
messages[-1] = {"role": "user", "content": last_content}
return messages, assistant_prefix
def _append_assistant_prefix_to_prompt_ids(
self, prompt_ids: list[int], assistant_prefix: str
) -> list[int]:
"""
Append assistant prefix to prompt_ids.
Args:
prompt_ids: Current prompt token IDs
assistant_prefix: Assistant message content to append
Returns:
Updated prompt_ids with assistant prefix appended
"""
encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix)
if encoded and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id:
encoded = encoded[1:]
return prompt_ids + encoded
def _resolve_chat_encoding_spec(self) -> str | None:
"""Determine which chat encoding spec to use.
Override in subclass to add custom encoding specs.
"""
return chat_encoding.resolve_chat_encoding_spec(
hf_config=self.tokenizer_manager.model_config.hf_config,
tokenizer=self.tokenizer_manager.tokenizer,
tool_call_parser=self.tool_call_parser,
)
def _request_id_prefix(self) -> str:
return "chatcmpl-"
def _effective_tools(self, request: ChatCompletionRequest) -> list[Tool]:
tools = list(request.tools or [])
for message in request.messages:
if (
isinstance(message, ChatCompletionMessageGenericParam)
and message.role in ("system", "developer")
and message.tools
):
tools.extend(message.tools)
return tools
def _prepare_kimi_k3_messages(
self,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
) -> tuple[list[dict[str, Any]], int, str | None]:
image_count = 0
for index, message in enumerate(messages):
content = message.get("content")
if isinstance(content, list):
parts = []
for part in content:
if not isinstance(part, dict):
continue
part_type = part.get("type")
if part_type in ("text", "input_text"):
parts.append(
{
"type": "text",
"text": neutralize_kimi_k3_image_placeholder(
part["text"]
),
}
)
elif part_type in ("image_url", "input_image"):
image = part.get("image_url") or {}
if isinstance(image, str):
image = {"url": image, "detail": part.get("detail")}
parts.append({"type": "image_url", "image_url": image})
image_count += 1
message["content"] = parts
elif isinstance(content, str):
message["content"] = neutralize_kimi_k3_image_placeholder(content)
elif content is None:
message["content"] = ""
if message.get("role") == "assistant":
for key in ("reasoning_content", "reasoning"):
if key in message:
message[key] = neutralize_kimi_k3_image_placeholder_value(
message[key]
)
for tool_call in message.get("tool_calls") or []:
function = (
tool_call.get("function")
if isinstance(tool_call, dict)
else None
)
if isinstance(function, dict) and "arguments" in function:
function["arguments"] = (
neutralize_kimi_k3_image_placeholder_value(
function["arguments"]
)
)
source = request.messages[index]
if (
isinstance(source, ChatCompletionMessageGenericParam)
and source.role in ("system", "developer")
and source.tools
):
message["tools"] = [
tool.model_dump(exclude_unset=True, by_alias=True)
for tool in source.tools
]
if message.get("role") == "developer":
message["role"] = "system"
assistant_prefix = None
if request.continue_final_message:
messages, assistant_prefix = self._handle_last_assistant_message(
messages, request
)
return messages, image_count, assistant_prefix
def _encode_messages(
self,
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
thinking_mode: ThinkingMode,
tools: list[dict] | None = None,
) -> list[int] | None:
"""Encode messages for custom chat_encoding_spec values.
Returns prompt_ids if handled, None to use default encoding.
"""
if self.chat_encoding_spec == "inkling":
# Inkling: render messages -> input_ids with framing tokens + ONE placeholder per
# media (encoding/expansion happens later in InklingMultimodalProcessor). The
# server's tokenizer is the base tiktoken backend; wrap it so encode_special
# supplies the framing-token overlay.
from sglang.srt.parser.inkling_renderer import render_inkling_messages
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_TEXT,
MESSAGE_MODEL,
InklingTokenizer,
)
inkling_tokenizer = InklingTokenizer(
tokenizer=self.tokenizer_manager.tokenizer
)
reasoning_effort = self._parse_inkling_reasoning_effort(
request.reasoning_effort
)
if reasoning_effort is None:
reasoning_effort = self._inkling_default_reasoning_effort
assistant_prefix = self._pop_inkling_assistant_prefix(messages, request)
prompt_ids = render_inkling_messages(
messages,
inkling_tokenizer,
add_generation_prompt=False,
tools=tools,
reasoning_effort=reasoning_effort,
)
if assistant_prefix is not None:
# Continue the final assistant message inside an OPEN model text
# block: header + payload, no <|end_message|> and no
# <|content_model_end_sampling|>, so the model resumes the turn.
prompt_ids += [
inkling_tokenizer.encode_special(MESSAGE_MODEL),
inkling_tokenizer.encode_special(CONTENT_TEXT),
*inkling_tokenizer.encode_text(assistant_prefix),
]
return prompt_ids
if self.chat_encoding_spec == "kimi_k3":
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
messages, request
)
template_kwargs = dict(request.chat_template_kwargs or {})
template_kwargs.pop("tokenize", None)
template_kwargs.pop("return_dict", None)
template_kwargs.pop("image_prompts", None)
if image_count:
template_kwargs["image_prompts"] = ["<|media_pad|>"] * image_count
if (
request.reasoning_effort in ("low", "high", "max")
and "thinking_effort" not in template_kwargs
):
template_kwargs["thinking_effort"] = request.reasoning_effort
elif request.reasoning_effort not in (
None,
"none",
"low",
"high",
"max",
):
logger.warning(
"Kimi K3 does not support reasoning_effort=%r; using the "
"encoder default.",
request.reasoning_effort,
)
effective_tools = self._effective_tools(request)
if (
effective_tools
and isinstance(request.tool_choice, str)
and request.tool_choice in ("required", "none")
):
template_kwargs.setdefault("tool_choice", request.tool_choice)
if request.response_format is not None:
template_kwargs.setdefault(
"response_format",
request.response_format.model_dump(
exclude_unset=True, by_alias=True
),
)
request_tools = (
[
tool.model_dump(exclude_unset=True, by_alias=True)
for tool in request.tools
]
if request.tools
else None
)
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
tools=request_tools,
return_dict=False,
**template_kwargs,
)
if assistant_prefix:
prompt_ids = self._append_assistant_prefix_to_prompt_ids(
prompt_ids, assistant_prefix
)
return prompt_ids
return None
@staticmethod
def _pop_inkling_assistant_prefix(
messages: list[dict[str, Any]],
request: ChatCompletionRequest,
) -> str | None:
"""Extract the trailing assistant text for ``continue_final_message``.
Only a plain-string assistant message with no tool calls and no
reasoning content can be continued; anything else renders as a closed
historical turn. Mutates ``messages`` in place (callers pass a copy).
"""
if not request.continue_final_message or not messages:
return None
last = messages[-1]
if (
last.get("role") != "assistant"
or not isinstance(last.get("content"), str)
or last.get("tool_calls")
or last.get("reasoning_content")
):
return None
messages.pop()
return last["content"]
@staticmethod
def _parse_inkling_reasoning_effort(
value: str | float | None,
) -> float | None:
"""Convert an OpenAI-style reasoning_effort to an Inkling float."""
if value is None:
return None
if isinstance(value, bool):
raise ValueError("Inkling reasoning_effort must not be a boolean")
if isinstance(value, (int, float)):
parsed = float(value)
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
return parsed
_EFFORT_MAP = {
"none": 0.0,
"minimal": 0.1,
"low": 0.2,
"medium": 0.7,
"high": 0.9,
"xhigh": 0.99,
"max": 0.99,
}
if value in _EFFORT_MAP:
return _EFFORT_MAP[value]
try:
parsed = float(value)
except (ValueError, TypeError) as exc:
raise ValueError(f"invalid Inkling reasoning_effort: {value!r}") from exc
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
return parsed
def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]:
"""Request effort for the V4.1 encoder; unsupported values warn and fall back."""
effort = chat_encoding.parse_dsv41_reasoning_effort(value)
if effort is not None:
return effort
if value is not None and value != "none":
logger.warning(
"DeepSeek-V4.1 does not support reasoning_effort=%r; using the "
"default %r (low/high/xhigh/max, a float in [0, 0.99], or an "
"integer budget in [1, 100] via chat_template_kwargs are accepted).",
value,
self._dsv41_default_reasoning_effort,
)
return self._dsv41_default_reasoning_effort
@staticmethod
def _get_inkling_default_reasoning_effort() -> float:
"""Read the default Inkling reasoning effort from the environment."""
from sglang.srt.environ import envs
val = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT.get()
if not val:
return 0.9
try:
parsed = float(val)
except (ValueError, TypeError) as exc:
raise ValueError(
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be numeric"
) from exc
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
raise ValueError(
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be in [0.0, 0.99]"
)
return parsed
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
"""Extract text from response."""
return ret_item["text"]
def _get_parsed_response_fields(
self,
reasoning_text: str | None,
tool_calls: list[dict] | None,
) -> tuple[str | None, list[dict] | None]:
"""Post-process reasoning and tool_calls before building response."""
return reasoning_text, tool_calls
def _should_return_input_ids(self, request: ChatCompletionRequest) -> bool:
"""Whether prompt (input) token ids should be returned via sglext."""
return request.return_input_ids_in_sglext or get_serving().return_input_ids
def _should_return_output_ids(self, request: ChatCompletionRequest) -> bool:
"""Whether sampled output token ids should be returned via sglext."""
return request.return_output_ids_in_sglext or get_serving().return_output_ids
def _continuous_usage_cached_details(
self, content: dict[str, Any]
) -> PromptTokensDetails | None:
if not get_serving().enable_cache_report:
return None
return UsageProcessor._details_if_cached(
content["meta_info"].get("cached_tokens", 0)
)
def _reported_prompt_tokens(self, meta_info: dict[str, Any]) -> int:
prompt_tokens = meta_info.get("prompt_tokens", 0)
if self.chat_encoding_spec == "kimi_k3":
# K3's three-token assistant generation stub is model input, but the
# reference API excludes it from billed/reported prompt tokens.
prompt_tokens = max(0, prompt_tokens - self._KIMI_K3_GENERATION_STUB_TOKENS)
return prompt_tokens
@staticmethod
def _sort_tool_message_run(
run: list[dict[str, Any]], tool_calls: list[dict[str, Any]]
) -> list[dict[str, Any]]:
"""Order a tool-message run by tool_call position.
Templates that associate results by tool_call_id render the run in
tool_calls order; sorting the run upfront keeps extraction order and
placeholder order the same. Runs the template itself would refuse to
associate (missing/duplicate/unknown ids) are left untouched, as are
text-only runs, whose order text-only templates may rely on.
"""
if len(run) < 2:
return run
call_ids = [tc.get("id") for tc in tool_calls]
if any(call_id is None for call_id in call_ids) or len(set(call_ids)) != len(
call_ids
):
return run
result_ids = [message.get("tool_call_id") for message in run]
if any(result_id not in call_ids for result_id in result_ids) or len(
set(result_ids)
) != len(result_ids):
return run
has_media = any(
isinstance(message.get("content"), list)
and any(
isinstance(part, dict) and part.get("type") in MEDIA_URL_PART_TYPES
for part in message["content"]
)
for message in run
)
if not has_media:
return run
position = {call_id: index for index, call_id in enumerate(call_ids)}
return sorted(run, key=lambda message: position[message["tool_call_id"]])
@classmethod
def _canonicalize_tool_message_order(
cls, messages: list[dict[str, Any]]
) -> list[dict[str, Any]]:
canonical = []
index = 0
while index < len(messages):
message = messages[index]
canonical.append(message)
index += 1
tool_calls = message.get("tool_calls") or []
if message.get("role") != "assistant" or not tool_calls:
continue
run = []
while index < len(messages) and messages[index].get("role") in (
"tool",
"function",
):
run.append(messages[index])
index += 1
canonical.extend(cls._sort_tool_message_run(run, tool_calls))
return canonical
async def _generate_stream_content(
self,
content: dict[str, Any],
index: int,
request: ChatCompletionRequest,
stream_offsets: dict[int, int],
reasoning_parser_dict: dict,
parser_dict: dict,
has_tool_calls: dict[int, bool],
choice_logprobs: dict | None,
finish_reason_type: str | None,
continuous_usage_stats: bool,
prompt_tokens: dict[int, int],
reasoning_tokens: dict[int, int],
completion_tokens: dict[int, int],
) -> AsyncGenerator[str, None]:
"""Generate SSE chunks for streaming content."""
offset = stream_offsets.get(index, 0)
if get_serving().incremental_streaming_output:
delta = content["text"]
else:
delta = content["text"][offset:]
stream_offsets[index] = len(content["text"])
# Attach logprobs to the first chunk emitted this step (reasoning,
# tool-call, or content) so they aren't dropped when a parser is active
# nor duplicated across chunks; flush any leftover at the end.
remaining_logprobs = choice_logprobs
# Handle reasoning content
if self.reasoning_parser and request.separate_reasoning:
reasoning_text, delta = self._process_reasoning_stream(
index,
delta,
reasoning_parser_dict,
content,
request,
finish_reason_type,
)
if reasoning_text:
usage = None
if continuous_usage_stats:
usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
yield build_sse_content(
chunk_id=content["meta_info"]["id"],
created=int(time.time()),
model=request.model,
index=index,
reasoning_content=reasoning_text,
logprobs=remaining_logprobs,
usage=usage,
)
remaining_logprobs = None
# Handle tool calls
if self._tool_call_parsing_active(request):
async for chunk in self._process_tool_call_stream(
index,
delta,
parser_dict,
content,
request,
has_tool_calls,
continuous_usage_stats,
flush=finish_reason_type is not None and finish_reason_type != "abort",
):
if chunk:
yield chunk
# Send any remaining tool call arguments when generation finishes
if finish_reason_type is not None and index in parser_dict:
parser = parser_dict[index]
remaining_chunk = self._check_for_unstreamed_tool_args(
parser, content, request, index
)
if remaining_chunk:
yield remaining_chunk
else:
# Regular content
if delta:
usage = None
if continuous_usage_stats:
usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
yield build_sse_content(
chunk_id=content["meta_info"]["id"],
created=int(time.time()),
model=request.model,
index=index,
content=delta,
logprobs=remaining_logprobs,
usage=usage,
)
remaining_logprobs = None
# Flush logprobs still unattached this step — only when a parser is
# active, since _process_tool_call_stream may consume the delta and emit
# no content chunk. On the plain path an empty-delta step has no chunk
# to attach to either way, and a standalone empty-delta logprobs chunk
# is not a shape clients expect.
if remaining_logprobs is not None and (
self.reasoning_parser or self.tool_call_parser
):
usage = None
if continuous_usage_stats:
usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens.get(index, 0),
reasoning_tokens=reasoning_tokens.get(index, 0),
completion_tokens=completion_tokens.get(index, 0),
cached_tokens=self._continuous_usage_cached_details(content),
).model_dump()
yield build_sse_content(
chunk_id=content["meta_info"]["id"],
created=int(time.time()),
model=request.model,
index=index,
logprobs=remaining_logprobs,
usage=usage,
)
def _tool_call_parsing_active(self, request: ChatCompletionRequest) -> bool:
"""Whether this request's output runs through the tool-call parser.
The reasoning parser is told the same thing, so channel-framed formats
keep their framing intact exactly when a tool-call parser consumes it.
"""
return bool(
request.tool_choice != "none"
and self._effective_tools(request)
and self.tool_call_parser
)
def _validate_request(self, request: ChatCompletionRequest) -> str | None:
"""Validate that the input is valid."""
if not request.messages:
return "Messages cannot be empty."
if (
request.chat_template_kwargs
and "chat_template" in request.chat_template_kwargs
and not get_serving().trust_request_chat_template
):
return (
"Request-supplied chat_template is not allowed. Start the server "
"with --trust-request-chat-template to permit it."
)
if request.return_sampling_mask and not request.return_meta_info:
return "return_sampling_mask requires return_meta_info=true."
media_error = self._validate_media_content(request)
if media_error:
return media_error
effective_tools = self._effective_tools(request)
has_message_tools = any(
isinstance(message, ChatCompletionMessageGenericParam)
and message.role in ("system", "developer")
and message.tools
for message in request.messages
)
if (
isinstance(request.tool_choice, str)
and request.tool_choice.lower() == "required"
and not effective_tools
):
return "Tools cannot be empty if tool choice is set to required."
if request.tool_choice is not None and not isinstance(request.tool_choice, str):
if not effective_tools:
return "Tools cannot be empty if tool choice is set to a specific tool."
tool_name = request.tool_choice.function.name
tool_exists = any(
tool.function.name == tool_name for tool in effective_tools
)
if not tool_exists:
return f"Tool '{tool_name}' not found in tools list."
if has_message_tools:
names = [tool.function.name for tool in effective_tools]
if len(names) != len(set(names)):
return "Tool names must be unique across request and message tools."
# Validate tool definitions
for i, tool in enumerate(effective_tools):
if tool.function.parameters is None:
continue
try:
# Rewrite DB/ORM-style aliases (e.g. "varchar", "enum", "int")
# to standard JSON Schema types before validation. RecursionError
# guards against hand-crafted cyclic schemas so the request gets
# a 400 instead of crashing into a 500.
normalize_json_schema_types(tool.function.parameters)
Draft202012Validator.check_schema(tool.function.parameters)
except SchemaError as e:
return f"Tool {i} function has invalid 'parameters' schema: {str(e)}"
except RecursionError:
return (
f"Tool {i} function 'parameters' schema is too deeply nested "
"or contains a cycle."
)
max_output_tokens = request.max_completion_tokens or request.max_tokens
server_context_length = get_model().context_length
if (
max_output_tokens
and server_context_length
and max_output_tokens > server_context_length
) and not get_serving().allow_auto_truncate:
return (
f"max_completion_tokens is too large: {max_output_tokens}."
f"This model supports at most {server_context_length} completion tokens."
)
if request.response_format and request.response_format.type == "json_schema":
schema = getattr(request.response_format.json_schema, "schema_", None)
if schema is None:
return "schema_ is required for json_schema response format request."
return None
def _validate_media_content(self, request: ChatCompletionRequest) -> str | None:
if self.tokenizer_manager.model_config.is_multimodal:
return None
media_type = next(
(
part.type
for message in request.messages
if isinstance(message.content, list)
for part in message.content
if part.type in _MEDIA_CONTENT_PART_TYPES
),
None,
)
if media_type is None:
return None
return (
"Model only supports text input; "
f"received unsupported content type '{media_type}'."
)
def _engine_prompt(
self, processed_messages: MessageProcessingResult, is_multimodal: bool
) -> tuple[str, Any]:
"""Standard VLMs render a text prompt (with placeholder strings) for
the MM processor to tokenize. Token-first encoders instead produce
pre-rendered input_ids with single placeholder ids and leave the text
empty; pass those through rather than re-tokenizing an empty prompt.
"""
# A lossy text round-trip makes the rendered prompt unusable, so send the
# ids instead. Only when nothing needs placeholder expansion: with media
# attached the MM processor still has to tokenize the text itself.
prefers_prompt_ids = (
self._prompt_text_round_trip_is_lossy
and isinstance(processed_messages.prompt_ids, list)
and processed_messages.prompt_ids
and not (
processed_messages.image_data
or processed_messages.video_data
or processed_messages.audio_data
)
)
if (
is_multimodal
and not chat_encoding.spec_renders_prompt_ids(self.chat_encoding_spec)
and not prefers_prompt_ids
):
return "text", processed_messages.prompt
if isinstance(processed_messages.prompt_ids, str):
return "text", processed_messages.prompt_ids
return "input_ids", processed_messages.prompt_ids
async def query_cache_hit(
self, request: ChatCompletionRequest, raw_request: Request = None
) -> ORJSONResponse:
"""G9 probe: report how much of this chat request's prompt is already
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
any prefill. The router uses it for conditional PD-bypass decisions.
Multimodal requests are rejected (v1): their sentinel-token expansion
costs ~10-100ms per image for only ~2% of traffic; the router skips
them for bypass anyway.
"""
try:
internal_req, _ = self._convert_to_internal_request(request, raw_request)
except Exception as e:
return ORJSONResponse(
status_code=400,
content={"error": f"failed to convert chat request: {e}"},
)
if internal_req.contains_mm_input():
return ORJSONResponse(
status_code=400,
content={"error": "multimodal requests are not supported by /query_cache_hit"},
)
try:
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
if internal_req.input_ids is not None:
input_ids = list(internal_req.input_ids)
elif internal_req.text:
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
internal_req.text, False
)
else:
return ORJSONResponse(
status_code=400,
content={"error": "request produced neither input_ids nor text"},
)
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
result = await self.tokenizer_manager.query_cache_hit(
QueryCacheHitReqInput(
input_ids=list(input_ids),
extra_key=getattr(internal_req, "extra_key", None),
cache_salt=getattr(internal_req, "cache_salt", None),
)
)
except Exception as e:
return ORJSONResponse(
status_code=500, content={"error": f"cache-hit query failed: {e}"}
)
if result.error is not None:
return ORJSONResponse(status_code=500, content={"error": result.error})
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
return ORJSONResponse(
content={
"total_tokens": result.total_tokens,
"l1_hit_tokens": result.l1_hit_tokens,
"l2_hit_tokens": result.l2_hit_tokens,
"l3_hit_tokens": result.l3_hit_tokens,
"hit_tokens": hit,
"new_tokens": max(0, result.total_tokens - hit),
"dp_rank": result.dp_rank,
}
)
def _convert_to_internal_request(
self,
request: ChatCompletionRequest,
raw_request: Request = None,
) -> tuple[GenerateReqInput, ChatCompletionRequest]:
# Header-based opt-in (same rationale as request_headers.py).
if raw_request is not None and not request.return_input_ids_in_sglext:
if raw_request.headers.get("x-sglext-return-input-ids") == "1":
request.return_input_ids_in_sglext = True
if raw_request is not None and not request.return_output_ids_in_sglext:
if raw_request.headers.get("x-sglext-return-output-ids") == "1":
request.return_output_ids_in_sglext = True
reasoning_effort = None
if not uses_hunyuan_reasoning_effort(
self.reasoning_parser, self.template_manager.reasoning_config
):
reasoning_effort = (
request.chat_template_kwargs.pop("reasoning_effort", None)
if request.chat_template_kwargs
else None
)
if self.is_gpt_oss and reasoning_effort == "none":
raise ValueError(
f"Harmony does not support reasoning effort {reasoning_effort}"
)
if reasoning_effort is not None:
request.reasoning_effort = reasoning_effort
if request.stream:
if request.return_prompt_token_ids:
raise ValueError(
"return_prompt_token_ids is not supported with streaming. "
"Please set stream=false when using return_prompt_token_ids=true."
)
if request.return_token_ids:
raise ValueError(
"return_token_ids is not supported with streaming on "
"/v1/chat/completions. Please set stream=false when using "
"return_token_ids=true."
)
if request.return_meta_info:
raise ValueError(
"return_meta_info is not supported with streaming. "
"Please set stream=false when using return_meta_info=true."
)
is_multimodal = self.tokenizer_manager.model_config.is_multimodal
# Process messages and apply chat template
processed_messages = self._process_messages(request, is_multimodal)
# Build sampling parameters
sampling_params = request.to_sampling_params(
stop=processed_messages.stop,
model_generation_config=self.default_sampling_params,
tool_call_constraint=processed_messages.tool_call_constraint,
renderer_handles_response_format=self.chat_encoding_spec == "kimi_k3",
)
set_request_reasoning_end_token_ids(
sampling_params, processed_messages.reasoning_end_token_ids
)
# Handle single vs multiple requests
if request.input_ids is not None:
prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
else:
prompt_key, prompt_value = self._engine_prompt(
processed_messages, is_multimodal
)
prompt_kwargs = {prompt_key: prompt_value}
# Extract custom labels from raw request headers
custom_labels = self.extract_custom_labels(raw_request)
# Extract routed_dp_rank from header (has higher priority than body)
effective_routed_dp_rank = self.extract_routed_dp_rank_from_header(
raw_request, request.routed_dp_rank
)
# Resolve LoRA adapter from model parameter or explicit lora_path
lora_path = self._resolve_lora_path(request.model, request.lora_path)
img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch(
request
)
adapted_request = GenerateReqInput(
**prompt_kwargs,
image_data=processed_messages.image_data,
video_data=processed_messages.video_data,
audio_data=processed_messages.audio_data,
sampling_params=sampling_params,
return_logprob=request.logprobs,
logprob_start_len=-1,
top_logprobs_num=request.top_logprobs or 0,
return_sampling_mask=request.return_sampling_mask,
sampling_logprobs_mode=request.sampling_logprobs_mode,
stream=request.stream,
return_text_in_logprobs=True,
modalities=processed_messages.modalities,
lora_path=lora_path,
bootstrap_host=request.bootstrap_host,
bootstrap_port=request.bootstrap_port,
bootstrap_room=request.bootstrap_room,
routed_dp_rank=effective_routed_dp_rank,
disagg_prefill_dp_rank=request.disagg_prefill_dp_rank,
return_hidden_states=request.return_hidden_states,
return_routed_experts=request.return_routed_experts,
routed_experts_start_len=request.routed_experts_start_len,
rid=request.rid,
session_id=request.session_id,
extra_key=request.extra_key,
cache_salt=request.cache_salt,
require_reasoning=processed_messages.require_reasoning,
priority=request.priority,
routing_key=self.extract_routing_key(raw_request),
custom_labels=custom_labels,
custom_logit_processor=request.custom_logit_processor,
images_config=getattr(request, "images_config", None),
video_config=_build_video_config(request),
image_max_dynamic_patch=img_max_dynamic_patch,
video_max_dynamic_patch=vid_max_dynamic_patch,
max_dynamic_patch=getattr(request, "max_dynamic_patch", None),
use_audio_in_video=getattr(request, "use_audio_in_video", False),
return_prompt_token_ids=(
request.return_prompt_token_ids
or request.return_token_ids
or self._should_return_input_ids(request)
),
)
if (
raw_request is not None
and envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get()
):
apply_header_overrides(adapted_request, raw_request.headers)
return adapted_request, request
def _process_messages(
self, request: ChatCompletionRequest, is_multimodal: bool
) -> MessageProcessingResult:
"""Process chat messages and apply chat template"""
if self.default_chat_template_kwargs:
ctk = dict(request.chat_template_kwargs or {})
for k, v in self.default_chat_template_kwargs.items():
ctk.setdefault(k, v)
request.chat_template_kwargs = ctk
effort = ctk.get("reasoning_effort")
if effort is not None and request.reasoning_effort is None:
request.reasoning_effort = effort
normalize_hunyuan_reasoning_effort(
request, self.reasoning_parser, self.template_manager.reasoning_config
)
# GptOss model needs to keep special tokens for harmony parsing
if self.is_gpt_oss or self.is_gemma4:
request.skip_special_tokens = False
self._patch_reasoning_skip_special_tokens(request)
thinking_mode = self._get_reasoning_from_request(request)
# SGLang's ReasonerGrammarBackend owns the reasoning prefix
# when --reasoning-parser is configured, so builtin xgrammar
# tags must describe only the post-reasoning tool-call suffix.
xgrammar_reasoning = thinking_mode and (self.reasoning_parser is None)
tool_call_constraint = None
effective_tools = self._effective_tools(request)
glm_constraint = self.tool_call_parser == "glm47" and not any(
tool.function.strict for tool in effective_tools
)
if glm_constraint:
enable_thinking = (request.chat_template_kwargs or {}).get(
"enable_thinking"
)
parser = FunctionCallParser(request.tools or [], self.tool_call_parser)
tool_call_constraint = parser.get_structure_constraint(
request.tool_choice,
parallel_tool_calls=request.parallel_tool_calls,
thinking_mode=True
if enable_thinking is None
else bool(enable_thinking),
)
# Apply chat template and its stop strings
tools = None
tool_call_stop = None
required_parsed_natively = glm_constraint
if effective_tools and request.tool_choice != "none":
request.skip_special_tokens = False
if not isinstance(request.tool_choice, str):
tools = [
item.model_dump()
for item in request.tools or []
if item.function.name == request.tool_choice.function.name
] or None
elif request.tools:
tools = [item.model_dump() for item in request.tools]
if self.tool_call_parser and not glm_constraint:
parser = FunctionCallParser(
effective_tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
tool_call_constraint = parser.get_structure_constraint(
request.tool_choice,
parallel_tool_calls=request.parallel_tool_calls,
thinking_mode=xgrammar_reasoning,
)
if (
tool_call_constraint is not None
and tool_call_constraint[0] == "structural_tag"
and self._reasoning_detector is not None
):
# Same ownership rule: the reasoning parser handles think
# tokens, so the grammar must not forbid them in free text.
strip_structural_tag_excludes(
tool_call_constraint[1],
(
self._reasoning_detector.think_start_token,
self._reasoning_detector.think_end_token,
),
)
required_parsed_natively = parser.detector.parses_required_natively()
if self.chat_encoding_spec == "kimi_k3":
tool_call_stop = parser.detector.eot_token
if (
tool_call_constraint is None
and not required_parsed_natively
and not (
self.chat_encoding_spec == "kimi_k3"
and self.tool_call_parser == "kimi_k3"
)
and (
request.tool_choice == "required"
or isinstance(request.tool_choice, ToolChoice)
)
):
json_schema = get_json_schema_constraint(
effective_tools,
request.tool_choice,
parallel_tool_calls=request.parallel_tool_calls,
)
tool_call_constraint = ("json_schema", json_schema)
# When input_ids are provided, skip template tokenization entirely;
# only stop tokens and tool_call_constraint are needed.
if request.input_ids is not None:
result = MessageProcessingResult(
prompt="",
prompt_ids=request.input_ids,
image_data=None,
audio_data=None,
video_data=None,
modalities=[],
stop=request.stop or [],
)
elif self.template_manager.chat_template_name is None:
result = self._apply_jinja_template(request, tools, is_multimodal)
else:
result = self._apply_conversation_template(request, is_multimodal)
if tool_call_stop is not None:
if isinstance(result.stop, str):
result.stop = [result.stop]
elif result.stop is None:
result.stop = []
else:
result.stop = list(result.stop)
if tool_call_stop not in result.stop:
result.stop.append(tool_call_stop)
result.tool_call_constraint = tool_call_constraint
result.require_reasoning = thinking_mode
result.skip_special_tokens = request.skip_special_tokens
if self.reasoning_parser == "k2_horizon" and thinking_mode:
parser = ReasoningParser(
model_type=self.reasoning_parser,
stream_reasoning=False,
force_reasoning=True,
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
)
token_ids = self.tokenizer_manager.tokenizer.encode(
parser.detector.think_end_token,
add_special_tokens=False,
)
if hasattr(token_ids, "tolist"):
token_ids = token_ids.tolist()
if (
not isinstance(token_ids, list)
or not token_ids
or any(
type(token_id) is not int or token_id < 0 for token_id in token_ids
)
):
raise ValueError(
"The selected K2 reasoning terminator could not be encoded"
)
result.reasoning_end_token_ids = list(token_ids)
return result
def _apply_jinja_template(
self,
request: ChatCompletionRequest,
tools: list[dict] | None,
is_multimodal: bool,
) -> MessageProcessingResult:
"""Apply Jinja chat template"""
prompt = ""
prompt_ids = []
decoded_prompt = None
openai_compatible_messages = []
image_data = []
video_data = []
audio_data = []
modalities = []
template_content_format = self.template_manager.jinja_template_content_format
# Try custom encoding first (override in subclass for custom renderers)
thinking_requested = (request.chat_template_kwargs or {}).get(
"thinking", envs.SGLANG_DEFAULT_THINKING.get()
)
thinking_mode = (
ThinkingMode.THINKING if thinking_requested else ThinkingMode.CHAT
)
messages = [msg.model_dump() for msg in request.messages]
for message in messages:
normalize_assistant_tool_call_arguments(
message, strict=self.chat_encoding_spec != "kimi_k3"
)
prompt_ids = self._encode_messages(
copy.deepcopy(messages),
request,
thinking_mode,
tools=tools,
)
if prompt_ids is not None:
if self.chat_encoding_spec in ("inkling", "kimi_k3"):
for message in request.messages:
msg_dict = message.model_dump()
if msg_dict.get("content") is None:
msg_dict["content"] = ""
process_content_for_template_format(
msg_dict,
"openai",
image_data,
video_data,
audio_data,
modalities,
)
elif self.chat_encoding_spec is not None:
# dsv4/dsv41/dsv32 encoding path
messages = copy.deepcopy(messages)
is_dsv41 = self.chat_encoding_spec == "dsv41"
for msg in messages:
if msg.get("content") is None:
msg["content"] = ""
# The V4.1 encoder consumes OpenAI parts lists itself; dsv4/dsv32
# are text-only, so their parts-list content is flattened first.
if not is_dsv41:
for i, msg in enumerate(messages):
if isinstance(msg.get("content"), list):
messages[i] = process_content_for_template_format(
msg, "string", [], [], [], []
)
for msg in messages:
processed_msg = process_content_for_template_format(
msg,
template_content_format,
image_data,
video_data,
audio_data,
modalities,
use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32",
)
msg.update(processed_msg)
# Handle continue_final_message: separate final assistant message
messages, assistant_prefix = self._handle_last_assistant_message(
messages, request
)
# An empty system message hosts the request tools; dsv41 renders a
# system token for it, so it only gets one when tools need the host.
if messages[0]["role"] != "system" and (request.tools or not is_dsv41):
messages.insert(0, {"role": "system", "content": ""})
if request.tools:
messages[0]["tools"] = [
(
chat_encoding.dsv41_tool_payload(tool)
if is_dsv41
else tool.model_dump()
)
for tool in request.tools
]
# Default encoding (dsv4/dsv41/dsv32)
if self.chat_encoding_spec == "dsv4":
effort_source = request.reasoning_effort
if effort_source is None:
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
if env_val:
effort_source = env_val
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
assert reasoning_effort_profile is not None
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
reasoning_effort_profile
]
v4_reasoning_effort = (
effort_source if effort_source in accepted_efforts else None
)
if request.task is not None:
encoding_dsv4.attach_task_to_last_user_message(
messages, request.task
)
real_input = encoding_dsv4.encode_messages(
messages,
thinking_mode=thinking_mode,
reasoning_effort=v4_reasoning_effort,
reasoning_effort_profile=reasoning_effort_profile,
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
elif is_dsv41:
if request.task is not None:
encoding_dsv41.attach_task_to_last_user_message(
messages, request.task
)
real_input, media = encoding_dsv41.encode_messages(
messages,
thinking_mode=thinking_mode,
reasoning_effort=self._resolve_dsv41_reasoning_effort(
request.reasoning_effort
),
return_multi_modal_data=True,
)
if media["images"]:
if not is_multimodal:
raise ValueError("image input is not supported for this model")
image_data.extend(
ImageData(url=image["url"]) for image in media["images"]
)
tokenizer = self.tokenizer_manager.tokenizer
real_input = real_input.replace(
encoding_dsv41.IMAGE_PLACEHOLDER,
tokenizer.convert_ids_to_tokens(
self.tokenizer_manager.image_token_id
),
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
else:
real_input = encoding_dsv32.encode_messages(
messages, thinking_mode=thinking_mode
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
# Append assistant prefix if continue_final_message is enabled
if assistant_prefix:
prompt_ids = self._append_assistant_prefix_to_prompt_ids(
prompt_ids, assistant_prefix
)
else:
if self.template_manager.jinja_template_may_reorder_tool_results:
messages = self._canonicalize_tool_message_order(messages)
for msg_dict in copy.deepcopy(messages):
if msg_dict.get("content") is None:
msg_dict["content"] = ""
# Process content based on detected template format
processed_msg = process_content_for_template_format(
msg_dict,
template_content_format,
image_data,
video_data,
audio_data,
modalities,
)
processed_msg["content"] = normalize_tool_content(
processed_msg["role"], processed_msg.get("content")
)
openai_compatible_messages.append(processed_msg)
# Handle continue_final_message: separate final assistant message
openai_compatible_messages, assistant_prefix = (
self._handle_last_assistant_message(openai_compatible_messages, request)
)
extra_template_kwargs = {}
if request.reasoning_effort is not None:
extra_template_kwargs["reasoning_effort"] = request.reasoning_effort
if request.chat_template_kwargs:
extra_template_kwargs.update(request.chat_template_kwargs)
rc = self.template_manager.reasoning_config
if rc is not None and rc.effort_kwarg is not None:
if request.reasoning_effort == "low":
extra_template_kwargs.setdefault(rc.effort_kwarg, True)
elif request.reasoning_effort in ("medium", "high", "max"):
logger.warning(
"Model '%s' supports only 'low' reasoning effort; "
"requested '%s' treated as default thinking",
self.tokenizer_manager.served_model_name,
request.reasoning_effort,
)
# Split apply_chat_template(tokenize=True) into render + encode so we
# can skip add_special_tokens=False on tokenizers that don't auto-add
# specials (Kimi-like, OpenAI-chat analogue of #25265). Chat
# templates already include role/special tokens, so the encode must
# avoid double BOS on tokenizers that would add it.
encode_kwargs = (
{"add_special_tokens": False}
if self._tokenizer_auto_adds_specials
else {}
)
try:
prompt_ids, decoded_prompt = self._render_and_encode_chat_template(
openai_compatible_messages,
tools=tools,
template_kwargs=extra_template_kwargs,
encode_kwargs=encode_kwargs,
use_cache=is_multimodal,
)
except Exception:
# If the first attempt fails, try with flat function-only format.
# Some templates (e.g. Mistral) expect tools without the OpenAI wrapper.
tools = (
[t["function"] if "function" in t else t for t in tools]
if tools
else None
)
try:
prompt_ids, decoded_prompt = self._render_and_encode_chat_template(
openai_compatible_messages,
tools=tools,
template_kwargs=extra_template_kwargs,
encode_kwargs=encode_kwargs,
use_cache=is_multimodal,
)
except _CHAT_TEMPLATE_CLIENT_ERRORS as template_error:
# Template errors (e.g., from raise_exception in Jinja templates)
# and TypeError (e.g., tojson filter on Jinja2 Undefined variables)
# should be treated as client errors (400 BadRequest)
raise ValueError(str(template_error)) from template_error
# Append assistant prefix if continue_final_message is enabled
if assistant_prefix:
prompt_ids = self._append_assistant_prefix_to_prompt_ids(
prompt_ids, assistant_prefix
)
# The cached decode corresponds to prompt_ids before the prefix.
decoded_prompt = None
if is_multimodal:
prompt = (
decoded_prompt
if decoded_prompt is not None
else self.tokenizer_manager.tokenizer.decode(prompt_ids)
)
stop = request.stop
image_data = image_data if image_data else None
audio_data = audio_data if audio_data else None
video_data = video_data if video_data else None
modalities = modalities if modalities else []
return MessageProcessingResult(
prompt=prompt,
prompt_ids=prompt_ids,
image_data=image_data,
video_data=video_data,
audio_data=audio_data,
modalities=modalities,
stop=stop,
)
def _render_and_encode_chat_template(
self,
messages: List[Dict[str, Any]],
*,
tools: Optional[List[Dict]],
template_kwargs: Dict[str, Any],
encode_kwargs: Dict[str, Any],
use_cache: bool,
) -> tuple[List[int], Optional[str]]:
cache_key = None
if use_cache:
try:
# Key order is part of the key: templates render dicts in their given order.
cache_key = orjson.dumps(
(
getattr(
self.tokenizer_manager.tokenizer,
"chat_template",
None,
),
messages,
tools,
template_kwargs,
encode_kwargs,
),
)
except TypeError:
pass
if cache_key is not None:
cached = self._chat_template_cache.get(cache_key)
if cached is not None:
self._chat_template_cache.move_to_end(cache_key)
prompt_ids, decoded_prompt = cached
return list(prompt_ids), decoded_prompt
if self._prompt_text_round_trip_is_lossy:
# Re-encoding rendered text would drop the template's control tokens.
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
messages,
tokenize=True,
add_generation_prompt=True,
tools=tools,
return_dict=False,
**template_kwargs,
)
else:
rendered_prompt = self.tokenizer_manager.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
tools=tools,
return_dict=False,
**template_kwargs,
)
prompt_ids = self.tokenizer_manager.tokenizer.encode(
rendered_prompt, **encode_kwargs
)
decoded_prompt = (
self.tokenizer_manager.tokenizer.decode(prompt_ids)
if cache_key is not None
else None
)
if cache_key is not None:
self._chat_template_cache[cache_key] = (tuple(prompt_ids), decoded_prompt)
if len(self._chat_template_cache) > _CHAT_TEMPLATE_CACHE_MAX_SIZE:
self._chat_template_cache.popitem(last=False)
return prompt_ids, decoded_prompt
def _apply_conversation_template(
self,
request: ChatCompletionRequest,
is_multimodal: bool,
) -> MessageProcessingResult:
"""Apply conversation template"""
prompt = ""
prompt_ids = []
conv = generate_chat_conv(request, self.template_manager.chat_template_name)
# If we should continue the final assistant message, adjust the conversation.
if (
request.continue_final_message
and request.messages
and request.messages[-1].role == "assistant"
):
# Remove the auto-added blank assistant turn, if present.
if conv.messages and conv.messages[-1][1] is None:
conv.messages.pop()
# Rebuild the prompt from the conversation.
prompt = conv.get_prompt()
# Strip trailing stop tokens or separators that indicate end-of-assistant.
if isinstance(conv.stop_str, list):
for stop_token in conv.stop_str:
if prompt.endswith(stop_token):
prompt = prompt[: -len(stop_token)]
elif isinstance(conv.stop_str, str) and prompt.endswith(conv.stop_str):
prompt = prompt[: -len(conv.stop_str)]
if conv.sep and prompt.endswith(conv.sep):
prompt = prompt[: -len(conv.sep)]
if getattr(conv, "sep2", None) and prompt.endswith(conv.sep2):
prompt = prompt[: -len(conv.sep2)]
else:
prompt = conv.get_prompt()
if self._get_reasoning_from_request(request) and (
self._reasoning_detector is None
or not self._reasoning_detector.thinks_internally
):
# Models with thinks_internally=True think without a leading <think> token
prompt += "<think>" # Note(Xinyuan): hard code thinking token
image_data = conv.image_data if conv.image_data else None
video_data = conv.video_data if conv.video_data else None
audio_data = conv.audio_data if conv.audio_data else None
modalities = conv.modalities if conv.modalities else []
stop = copy.copy(conv.stop_str or [] if not request.ignore_eos else [])
if request.stop:
if isinstance(request.stop, str):
stop.append(request.stop)
else:
stop.extend(request.stop)
if not is_multimodal:
prompt_ids = self.tokenizer_manager.tokenizer.encode(prompt)
return MessageProcessingResult(
prompt=prompt,
prompt_ids=prompt_ids,
image_data=image_data,
video_data=video_data,
audio_data=audio_data,
modalities=modalities,
stop=stop,
)
async def _handle_streaming_request(
self,
adapted_request: GenerateReqInput,
request: ChatCompletionRequest,
raw_request: Request,
) -> StreamingResponse | ErrorResponse:
"""Handle streaming chat completion request"""
generator = self._generate_chat_stream(adapted_request, request, raw_request)
# Kick-start the generator to trigger validation before HTTP 200 is sent.
# If validation fails (e.g., context length exceeded), we can still return
# a proper HTTP 400 error response instead of streaming it as SSE payload.
try:
first_chunk = await generator.__anext__()
except ValueError as e:
return self.create_error_response(str(e))
async def prepend_first_chunk():
yield first_chunk
async for chunk in generator:
yield chunk
return StreamingResponse(
prepend_first_chunk(),
media_type="text/event-stream",
background=self.tokenizer_manager.create_abort_task(adapted_request),
)
async def _generate_chat_stream(
self,
adapted_request: GenerateReqInput,
request: ChatCompletionRequest,
raw_request: Request,
) -> AsyncGenerator[str, None]:
"""Generate streaming chat completion response"""
# Parsers for tool calls and reasoning
parser_dict = {}
reasoning_parser_dict = {}
# State tracking for streaming
is_firsts = {}
stream_offsets = {}
n_prev_tokens = {}
has_tool_calls = {}
finish_reasons = {}
# Usage tracking
prompt_tokens = {}
reasoning_tokens = {}
completion_tokens = {}
cached_tokens = {}
hidden_states = {}
routed_experts = {}
cached_tokens_details = {}
spec_tokens_details = {}
image_tokens = {}
audio_tokens = {}
video_tokens = {}
input_ids: list[int] | None = None
output_ids: dict[int, list[int]] = {}
stream_started = False
error_aborted = False
try:
include_usage, continuous_usage_stats = should_include_usage(
request.stream_options,
get_serving().stream_response_default_include_usage,
)
return_input_ids = self._should_return_input_ids(request)
return_output_ids = self._should_return_output_ids(request)
ids_framed = (
raw_request is not None
and raw_request.headers.get("x-sglext-ids-framed") == "1"
)
async for content in self.tokenizer_manager.generate_request(
adapted_request, raw_request
):
index = content.get("index", 0)
prompt_tokens[index] = self._reported_prompt_tokens(
content["meta_info"]
)
completion_tokens[index] = content["meta_info"].get(
"completion_tokens", 0
)
reasoning_tokens[index] = content["meta_info"].get(
"reasoning_tokens", 0
)
cached_tokens[index] = content["meta_info"].get("cached_tokens", 0)
hidden_states[index] = content["meta_info"].get("hidden_states", None)
routed_experts[index] = content["meta_info"].get("routed_experts", None)
cached_tokens_details[index] = content["meta_info"].get(
"cached_tokens_details", None
)
if request.return_spec_tokens_details:
spec_tokens_details[index] = spec_tokens_details_from_meta_info(
content["meta_info"]
)
image_tokens[index] = content["meta_info"].get("image_tokens", 0)
audio_tokens[index] = content["meta_info"].get("audio_tokens", 0)
video_tokens[index] = content["meta_info"].get("video_tokens", 0)
finish_reason = content["meta_info"].get("finish_reason", None)
finish_reason_type = finish_reason["type"] if finish_reason else None
if return_input_ids and input_ids is None:
# The prompt is the full, shared prompt (same across choices
# and constant across chunks), so capture it once.
chunk_input_ids = content.get("prompt_token_ids")
if chunk_input_ids is not None:
input_ids = list(chunk_input_ids)
if return_output_ids:
chunk_output_ids = content.get("output_ids")
if chunk_output_ids is not None:
if get_serving().incremental_streaming_output:
accumulated = output_ids.setdefault(index, [])
if finish_reason_type == "abort":
# The abort chunk re-sends the last token plus any coalesced deltas;
# keep only what brings the total up to completion_tokens.
keep = completion_tokens[index] - len(accumulated)
chunk_output_ids = chunk_output_ids[: max(keep, 0)]
accumulated.extend(chunk_output_ids)
else:
# Intermediate chunks share the live state.output_ids
# list; the final chunk is a stable copy.
output_ids[index] = chunk_output_ids
# Handle logprobs
choice_logprobs = None
if request.logprobs:
n_prev_token = n_prev_tokens.get(index, 0)
total_output_logprobs = content["meta_info"][
"output_token_logprobs_length"
]
if n_prev_token < total_output_logprobs:
choice_logprobs = self._process_streaming_logprobs(
content, n_prev_token, total_output_logprobs
).model_dump()
n_prev_tokens[index] = total_output_logprobs
# Track finish_reason for each index
if finish_reason_type:
# Abort with an explicit error status_code is a system error
# (timeout, OOM, validation): emit a streaming error chunk.
# A graceful abort (no status_code, e.g. user-initiated via
# /abort_request or session lifecycle cleanup) falls through
# to the normal chunk path, matching the non-stream behavior
# in tokenizer_manager._handle_abort_finish_reason.
if finish_reason_type == "abort" and isinstance(
finish_reason.get("status_code"), HTTPStatus
):
code = finish_reason["status_code"]
error = self.create_streaming_error_response(
finish_reason.get("message", "Generation aborted."),
code.name,
code.value,
)
yield f"data: {error}\n\n"
error_aborted = True
break
finish_reasons[index] = finish_reason
# First chunk with role
if is_firsts.get(index, True):
is_firsts[index] = False
yield build_sse_content(
chunk_id=content["meta_info"]["id"],
created=int(time.time()),
model=request.model,
index=index,
role="assistant",
content="",
)
stream_started = True
# Generate streaming content (override in subclass for custom behavior)
async for chunk in self._generate_stream_content(
content=content,
index=index,
request=request,
stream_offsets=stream_offsets,
reasoning_parser_dict=reasoning_parser_dict,
parser_dict=parser_dict,
has_tool_calls=has_tool_calls,
choice_logprobs=choice_logprobs,
finish_reason_type=finish_reason_type,
continuous_usage_stats=continuous_usage_stats,
prompt_tokens=prompt_tokens,
reasoning_tokens=reasoning_tokens,
completion_tokens=completion_tokens,
):
yield chunk
# Send finish_reason chunks for each index that completed
for idx, finish_reason_data in finish_reasons.items():
finish_reason_type = finish_reason_data["type"]
# Change finish_reason to "tool_calls" if we had tool calls and stopped naturally
final_finish_reason = finish_reason_type
if has_tool_calls.get(idx, False) and finish_reason_type == "stop":
final_finish_reason = "tool_calls"
matched_stop = finish_reason_data.get("matched")
yield build_sse_content(
chunk_id=content["meta_info"]["id"],
created=int(time.time()),
model=request.model,
index=idx,
finish_reason=final_finish_reason,
matched_stop=matched_stop,
)
# Send hidden states if requested
if request.return_hidden_states and hidden_states:
for index, choice_hidden_states in hidden_states.items():
if choice_hidden_states:
response_hidden_states = process_hidden_states_for_response(
choice_hidden_states, request.return_hidden_states
)
hidden_states_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[
ChatCompletionResponseStreamChoice(
index=index,
delta=DeltaMessage(
hidden_states=response_hidden_states
),
finish_reason=None, # Hidden states don't need finish_reason
)
],
model=request.model,
)
yield f"data: {hidden_states_chunk.model_dump_json()}\n\n"
sglext_routed = None
if request.return_routed_experts and routed_experts:
sglext_routed = next(
(v for v in routed_experts.values() if v is not None), None
)
sglext_cached_tokens_details = None
if request.return_cached_tokens_details and cached_tokens_details:
first_details = next(
(v for v in cached_tokens_details.values() if v is not None), None
)
if first_details is not None:
sglext_cached_tokens_details = cached_tokens_details_from_dict(
first_details
)
sglext_spec_tokens_details = None
if request.return_spec_tokens_details and spec_tokens_details:
spec_details = [
spec_tokens_details[index]
for index in sorted(spec_tokens_details)
if spec_tokens_details[index] is not None
]
if spec_details:
sglext_spec_tokens_details = (
spec_details if request.n > 1 else spec_details[0]
)
# Omit token ids after an error abort.
sglext_input_ids = None
if return_input_ids and input_ids and not error_aborted:
sglext_input_ids = list(input_ids)
sglext_output_ids = None
if return_output_ids and output_ids and not error_aborted:
sglext_output_ids = [
list(output_ids.get(i, [])) for i in range(request.n)
]
sglext_full = SglExt(
routed_experts=sglext_routed,
cached_tokens_details=sglext_cached_tokens_details,
spec_tokens_details=sglext_spec_tokens_details,
input_ids=sglext_input_ids,
output_ids=sglext_output_ids,
)
sglext_non_ids, sglext_ids = sglext_full.split_ids()
if ids_framed:
# A named SSE event lets transit hops pick the ids out without parsing JSON;
# the other sglext fields keep the plain data-chunk shape.
if sglext_non_ids is not None:
sglext_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[],
model=request.model,
sglext=sglext_non_ids,
)
yield f"data: {sglext_chunk.model_dump_json()}\n\n"
if sglext_ids is not None:
sglext_ids_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[],
model=request.model,
sglext=sglext_ids,
)
yield f"event: sglext_ids\ndata: {sglext_ids_chunk.model_dump_json()}\n\n"
elif sglext_non_ids is not None or sglext_ids is not None:
sglext_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[], # sglext is at response level
model=request.model,
sglext=sglext_full,
)
yield f"data: {sglext_chunk.model_dump_json()}\n\n"
# Additional usage chunk
if include_usage:
# Multimodal tokens are per-prompt (input side), so aggregate
# once per prompt (first choice), matching prompt/cached semantics.
total_image_tokens = sum(
tok for idx, tok in image_tokens.items() if idx % request.n == 0
)
total_audio_tokens = sum(
tok for idx, tok in audio_tokens.items() if idx % request.n == 0
)
total_video_tokens = sum(
tok for idx, tok in video_tokens.items() if idx % request.n == 0
)
usage = UsageProcessor.calculate_streaming_usage(
prompt_tokens,
reasoning_tokens,
completion_tokens,
cached_tokens=cached_tokens,
n_choices=request.n,
enable_cache_report=get_serving().enable_cache_report,
image_tokens=total_image_tokens,
audio_tokens=total_audio_tokens,
video_tokens=total_video_tokens,
)
usage_chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[], # Empty choices array as per OpenAI spec
model=request.model,
usage=usage,
)
yield f"data: {usage_chunk.model_dump_json()}\n\n"
except ValueError as e:
if not stream_started:
raise
error = self.create_streaming_error_response(str(e))
yield f"data: {error}\n\n"
yield "data: [DONE]\n\n"
async def _handle_non_streaming_request(
self,
adapted_request: GenerateReqInput,
request: ChatCompletionRequest,
raw_request: Request,
) -> ChatCompletionResponse | ErrorResponse | ORJSONResponse:
"""Handle non-streaming chat completion request"""
try:
ret = await self.tokenizer_manager.generate_request(
adapted_request, raw_request
).__anext__()
except ValueError as e:
return self.create_error_response(str(e))
if not isinstance(ret, list):
ret = [ret]
response = self._build_chat_response(
request,
ret,
int(time.time()),
)
return response
def _build_chat_response(
self,
request: ChatCompletionRequest,
ret: list[dict[str, Any]],
created: int,
) -> ChatCompletionResponse | ORJSONResponse:
"""Build chat completion response from generation results"""
if self.chat_encoding_spec == "kimi_k3":
ret = [
{
**item,
"meta_info": {
**item["meta_info"],
"prompt_tokens": self._reported_prompt_tokens(
item["meta_info"]
),
},
}
for item in ret
]
choices = []
# Build sglext at response level (from first ret_item, as these are per-request)
first_ret = ret[0]
routed_experts = (
None
if request.return_meta_info
else process_routed_experts_from_ret(first_ret, request)
)
cached_tokens_details = process_cached_tokens_details_from_ret(
first_ret, request
)
spec_details = [
detail
for detail in (
process_spec_tokens_details_from_ret(item, request) for item in ret
)
if detail is not None
]
spec_tokens_details = (
spec_details
if request.n > 1
else (spec_details[0] if spec_details else None)
)
input_ids = None
if self._should_return_input_ids(request) and "prompt_token_ids" in ret[0]:
input_ids = list(ret[0]["prompt_token_ids"])
output_ids = None
if self._should_return_output_ids(request):
output_ids = [list(ret_item["output_ids"]) for ret_item in ret]
response_sglext = None
if (
routed_experts
or cached_tokens_details
or spec_tokens_details
or input_ids is not None
or output_ids is not None
):
response_sglext = SglExt(
routed_experts=routed_experts,
cached_tokens_details=cached_tokens_details,
spec_tokens_details=spec_tokens_details,
input_ids=input_ids,
output_ids=output_ids,
)
for idx, ret_item in enumerate(ret):
# Process logprobs
choice_logprobs = None
if request.logprobs:
choice_logprobs = self._process_response_logprobs(ret_item)
# Handle hidden states
hidden_states = process_hidden_states_from_ret(ret_item, request)
finish_reason = ret_item["meta_info"]["finish_reason"]
text = self._decode_response(ret_item)
if isinstance(text, ErrorResponse):
return ORJSONResponse(content=text.model_dump(), status_code=text.code)
# Handle reasoning content
reasoning_text = None
if self.reasoning_parser and request.separate_reasoning:
force_reasoning = (
self.template_manager.force_reasoning
or self._get_reasoning_from_request(request)
)
try:
parser = ReasoningParser(
model_type=self.reasoning_parser,
stream_reasoning=False,
force_reasoning=force_reasoning,
request=request,
tokenizer=self.tokenizer_manager.tokenizer,
tool_call_parser_active=self._tool_call_parsing_active(request),
)
reasoning_text, text = parser.parse_non_stream(text)
except Exception as e:
logger.error(f"Reasoning parsing error: {e}")
return self.create_error_response(
"Failed to parse reasoning content",
err_type="InternalServerError",
status_code=500,
)
# Handle tool calls
tool_calls = None
effective_tools = self._effective_tools(request)
if self._tool_call_parsing_active(request):
history_tool_calls_cnt = self._get_history_tool_calls_cnt(request)
tool_calls, text, finish_reason = self._process_tool_calls(
text,
effective_tools,
finish_reason,
request.tool_choice,
history_tool_calls_cnt,
)
# Extract prompt_token_ids if requested
choice_prompt_token_ids = (
ret_item.get("prompt_token_ids")
if request.return_prompt_token_ids or request.return_token_ids
else None
)
choice_token_ids = (
ret_item["output_ids"] if request.return_token_ids else None
)
choice_meta_info = (
ret_item["meta_info"] if request.return_meta_info else None
)
# NOTE: content should not be None but empty string to make sure retokenize consistency.
reasoning_text, tool_calls = self._get_parsed_response_fields(
reasoning_text, tool_calls
)
choice_data = ChatCompletionResponseChoice(
index=idx,
message=ChatMessage(
role="assistant",
content=text if text else "",
tool_calls=tool_calls,
reasoning_content=reasoning_text if reasoning_text else None,
),
logprobs=choice_logprobs,
finish_reason=finish_reason["type"] if finish_reason else None,
matched_stop=(
finish_reason["matched"]
if finish_reason and "matched" in finish_reason
else None
),
hidden_states=hidden_states,
prompt_token_ids=choice_prompt_token_ids,
response_token_ids=choice_token_ids,
meta_info=choice_meta_info,
)
choices.append(choice_data)
# Calculate usage. Multimodal tokens are per-prompt (input side), so
# aggregate once per prompt (stride by n), matching prompt/cached semantics.
image_tokens = sum(
ret[i]["meta_info"].get("image_tokens", 0)
for i in range(0, len(ret), request.n)
)
audio_tokens = sum(
ret[i]["meta_info"].get("audio_tokens", 0)
for i in range(0, len(ret), request.n)
)
video_tokens = sum(
ret[i]["meta_info"].get("video_tokens", 0)
for i in range(0, len(ret), request.n)
)
usage = UsageProcessor.calculate_response_usage(
ret,
n_choices=request.n,
enable_cache_report=get_serving().enable_cache_report,
image_tokens=image_tokens,
audio_tokens=audio_tokens,
video_tokens=video_tokens,
)
return ChatCompletionResponse(
id=ret[0]["meta_info"]["id"],
created=created,
model=request.model,
choices=choices,
usage=usage,
metadata=build_endpoint_weight_version_metadata(ret[0]["meta_info"]),
sglext=response_sglext,
)
def _process_response_logprobs(self, ret_item: dict[str, Any]) -> ChoiceLogprobs:
"""Process logprobs for non-streaming response"""
output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"]
output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None)
token_logprobs = self._build_token_logprobs_from_raw(
output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
def _build_token_logprobs_from_raw(
self,
output_token_logprobs: list[Any],
output_top_logprobs: list[Any] | None,
) -> list[ChatCompletionTokenLogprob]:
"""Build OpenAI ChatCompletionTokenLogprob from the engine's raw
``(logprob, token_id, token_text)`` triples.
The engine always keeps the real token id in the triple; the token text
is a *detokenized display* string that loses fragmentary byte-level
tokens (e.g. the four single-byte BPE pieces of U+20BB7 decode to
U+FFFD). Recovering the OpenAI ``bytes`` field from the display text
therefore corrupts fragments; we recover the true bytes from the token
id through the GPT-2 byte decoder instead.
Only byte-level BPE tokenizers (GPT-2 family) use the byte decoder;
SentencePiece tokenizers (Mistral, Gemma) fall back to the display
text, so multi-byte characters like é are NOT corrupted to [233].
"""
tokenizer = self.tokenizer_manager.tokenizer
from sglang.srt.entrypoints.openai.utils import _is_byte_level_tokenizer
is_byte_level = _is_byte_level_tokenizer(tokenizer)
token_logprobs: list[ChatCompletionTokenLogprob] = []
for token_idx, item in enumerate(output_token_logprobs):
# item = (logprob, token_id, token_text)
logprob, token_id, token_text = item
if is_byte_level:
token_bytes = token_id_to_bytes(tokenizer, token_id)
else:
token_bytes = None
if token_bytes is None:
token_bytes = list((token_text or "").encode("utf-8"))
top_logprobs: list[TopLogprob] = []
if output_top_logprobs:
# Both callers pass rows aligned with output_token_logprobs.
if token_idx < len(output_top_logprobs):
top_row = output_top_logprobs[token_idx]
if top_row is not None:
for top_logprob, top_id, top_text in top_row:
if is_byte_level:
top_bytes = token_id_to_bytes(tokenizer, top_id)
else:
top_bytes = None
if top_bytes is None:
top_bytes = list((top_text or "").encode("utf-8"))
top_logprobs.append(
TopLogprob(
token=top_text or "",
bytes=top_bytes,
logprob=top_logprob,
)
)
token_logprobs.append(
ChatCompletionTokenLogprob(
token=token_text or "",
bytes=token_bytes,
logprob=logprob,
top_logprobs=top_logprobs,
)
)
return token_logprobs
def _process_tool_call_id(
self,
call_item: ToolCallItem,
history_tool_calls_cnt: int,
) -> str:
"""Process for generating a new and unique `tool_call_id`"""
if self.tool_call_parser == "kimi_k3":
return f"{call_item.name}:{history_tool_calls_cnt + call_item.tool_index}"
if self.tool_call_parser != "kimi_k2":
# A simple uuid is sufficient for all models except for Kimi-K2.
tool_call_id = f"call_{uuid.uuid4().hex[:24]}"
return tool_call_id
tool_call_id = (
f"functions.{call_item.name}:"
f"{history_tool_calls_cnt + call_item.tool_index}"
)
logger.debug(
f"Process tool call idx, parser: {self.tool_call_parser}, tool_call_id: {tool_call_id}, history_cnt: {history_tool_calls_cnt}"
)
return tool_call_id
def _process_tool_calls(
self,
text: str,
tools: list[Any],
finish_reason: dict[str, Any],
tool_choice: str | ToolChoice | None = None,
history_tool_calls_cnt: int = 0,
) -> ToolCallProcessingResult:
"""Process tool calls in the response"""
is_required = tool_choice == "required" or isinstance(tool_choice, ToolChoice)
# Try model-specific parser when output is in native format.
# For required/named: only use parser when structural_tag was used
# as constraint (mirrors the streaming path). For auto: always try.
if self.tool_call_parser:
parser = FunctionCallParser(
tools,
self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=tool_choice,
)
detector_owns_format = parser.owns_tool_format()
should_try_parser = not is_required or detector_owns_format
if should_try_parser and parser.has_tool_call(text):
try:
text, call_info_list = parser.parse_non_stream(text)
if not call_info_list:
logger.warning(
"Tool call marker present but no complete call parsed "
"from %s output; dropping the incomplete call",
self.tool_call_parser,
)
logger.debug(
"Unparsed tool call output (%d chars): %r",
len(text),
text[:2000],
)
return ToolCallProcessingResult(None, text, finish_reason)
tool_calls = []
for index, call_info in enumerate(call_info_list):
tool_id = self._process_tool_call_id(
call_info, history_tool_calls_cnt
)
# Call ordinal, as in the streaming deltas;
# tool_index is the tool's position in the request.
tool_calls.append(
ToolCall(
id=tool_id,
index=index,
function=FunctionResponse(
name=call_info.name,
arguments=call_info.parameters,
),
)
)
if finish_reason["type"] == "stop":
finish_reason["type"] = "tool_calls"
finish_reason["matched"] = None
return ToolCallProcessingResult(tool_calls, text, finish_reason)
except Exception as e:
logger.error(f"Tool call parsing error: {e}")
return ToolCallProcessingResult(None, text, finish_reason)
if is_required and detector_owns_format:
logger.warning(
"Required tool call missing from %s output (%d chars)",
self.tool_call_parser,
len(text),
)
logger.debug("Unparsed required tool call output: %r", text[:2000])
return ToolCallProcessingResult(None, text, finish_reason)
# json_schema constraint → JSON array output for required/named
if is_required:
original_finish_type = finish_reason["type"]
if finish_reason["type"] == "stop":
finish_reason["type"] = "tool_calls"
finish_reason["matched"] = None
try:
tool_call_data = orjson.loads(text)
if isinstance(tool_call_data, dict):
tool_call_data = [tool_call_data]
if not isinstance(tool_call_data, list):
raise ValueError(
"expected a JSON array of tool calls, got "
f"{type(tool_call_data).__name__}"
)
if not all(
isinstance(tool, dict) and "name" in tool for tool in tool_call_data
):
raise ValueError(
"every tool call must be a JSON object with a 'name'"
)
tool_calls = []
for i, tool in enumerate(tool_call_data):
parameters = json.dumps(
tool.get("parameters", {}), ensure_ascii=False
)
call_info = ToolCallItem(
tool_index=i,
name=tool["name"],
parameters=parameters,
)
tool_id = self._process_tool_call_id(
call_info, history_tool_calls_cnt
)
tool_calls.append(
ToolCall(
id=tool_id,
index=i,
function=FunctionResponse(
name=tool["name"],
arguments=parameters,
),
)
)
return ToolCallProcessingResult(tool_calls, "", finish_reason)
except Exception as e:
logger.error(f"Tool call parsing error: {e}")
logger.debug("Unparsed required tool call output: %r", text[:2000])
finish_reason["type"] = original_finish_type
return ToolCallProcessingResult(None, text, finish_reason)
return ToolCallProcessingResult(None, text, finish_reason)
def _process_streaming_logprobs(
self,
content: dict[str, Any],
n_prev_token: int,
total_output_logprobs: int,
) -> ChoiceLogprobs:
"""Process logprobs for streaming response"""
output_token_logprobs = content["meta_info"]["output_token_logprobs"]
output_top_logprobs = content["meta_info"].get("output_top_logprobs", [])
if not get_serving().incremental_streaming_output:
output_token_logprobs = output_token_logprobs[
n_prev_token:total_output_logprobs
]
output_top_logprobs = output_top_logprobs[
n_prev_token:total_output_logprobs
]
token_logprobs = self._build_token_logprobs_from_raw(
output_token_logprobs, output_top_logprobs
)
return ChoiceLogprobs(content=token_logprobs)
def _process_reasoning_stream(
self,
index: int,
delta: str,
reasoning_parser_dict: dict[int, ReasoningParser],
content: dict[str, Any],
request: ChatCompletionRequest,
finish_reason_type: str | None = None,
) -> tuple[str | None, str]:
"""Process reasoning content in streaming response"""
if index not in reasoning_parser_dict:
is_force_reasoning = (
self.template_manager.force_reasoning
or self._get_reasoning_from_request(request)
)
reasoning_parser_dict[index] = ReasoningParser(
self.reasoning_parser,
request.stream_reasoning,
is_force_reasoning,
request,
tokenizer=self.tokenizer_manager.tokenizer,
tool_call_parser_active=self._tool_call_parsing_active(request),
)
reasoning_parser = reasoning_parser_dict[index]
reasoning_text, normal_text = reasoning_parser.parse_stream_chunk(delta)
if finish_reason_type is not None and finish_reason_type != "abort":
end_reasoning_text, end_normal_text = reasoning_parser.parse_stream_end()
if end_reasoning_text:
reasoning_text = (reasoning_text or "") + end_reasoning_text
if end_normal_text:
normal_text = (normal_text or "") + end_normal_text
return reasoning_text, normal_text
def _get_history_tool_calls_cnt(self, request: ChatCompletionRequest) -> int:
"""Counts the number of tool calls in the request's message history.
NOTE: This method is only useful for models that include self-increasing
history tool call idx in tool calls id, such as kimi-k2
Args:
request: The chat completion request object.
Returns:
The total number of tool calls in the history, or 0 if not applicable.
"""
messages = getattr(request, "messages", [])
idx = 0
for msg in messages:
if msg.role == "assistant":
tool_calls = getattr(msg, "tool_calls", None)
idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa
return idx
def _patch_reasoning_skip_special_tokens(
self, request: ChatCompletionRequest
) -> None:
"""Keep parser-specific reasoning markers in the decoded text.
Some reasoning parsers rely on special-token delimiters that would be
removed during detokenization when ``skip_special_tokens=True``.
"""
if self.reasoning_parser == "apertus2509":
request.skip_special_tokens = False
if self.reasoning_parser == "kimi_k3" or self.chat_encoding_spec == "kimi_k3":
request.skip_special_tokens = False
if (
self.reasoning_parser in ["mistral"]
and request.reasoning_effort is not None
and request.reasoning_effort != "none"
):
request.skip_special_tokens = False
elif self.reasoning_parser == "inkling":
request.skip_special_tokens = False
elif self.reasoning_parser == "muse":
request.skip_special_tokens = False
def supports_native_reasoning_history(self) -> bool:
"""Whether the chat encoder takes history as ``reasoning_content`` rather
than via :meth:`wrap_reasoning_history`; see
:func:`chat_encoding.spec_owns_reasoning_history` for why.
"""
return chat_encoding.spec_owns_reasoning_history(self.chat_encoding_spec)
def wrap_reasoning_history(self, reasoning_text: str) -> str:
"""Wrap prior-turn reasoning in the detector's own start/end tokens.
Pulling the delimiters from the detector keeps adapters in lockstep
with any future parser that ships non-``<think>`` markers — Mistral's
``[THINK]``, Gemma4's ``think_start_self_label = "thought\\n"``, etc.
Falling back to a plain string is unsafe: it would let prior
thinking text reach a non-reasoning model as ordinary assistant
content, so the caller must surface this state, not paper over it.
"""
if self._reasoning_detector is None:
raise ValueError(
"Cannot rewrap thinking history: no reasoning detector is "
"configured for this model"
)
d = self._reasoning_detector
return (
f"{d.think_start_token}{d.think_start_self_label}"
f"{reasoning_text}\n{d.think_end_token}"
)
def _reasoning_default_mode(self) -> str | None:
if self._reasoning_detector is None:
return None
return self._reasoning_detector.reasoning_default
def _get_reasoning_toggle_param(self) -> str | None:
"""Resolve the chat-template kwarg that toggles reasoning, if any."""
config = self.template_manager.reasoning_config
if config is not None:
return config.toggle_param
mode = self._reasoning_default_mode()
if mode in ("thinking", "enable_thinking"):
return mode
if mode in ("explicit_thinking", "explicit_enable_thinking"):
return mode.replace("explicit_", "")
return None
def apply_reasoning_enabled(
self, request: ChatCompletionRequest, enabled: bool
) -> None:
"""Force the request into the requested reasoning-on/off mode.
Mirrors the read-side logic in ``_get_reasoning_from_request``;
the two must stay in sync. Always-on models cannot be disabled,
so explicit ``enabled=False`` raises rather than silently leaving
reasoning on.
"""
if not self.reasoning_parser:
if enabled:
raise ValueError(
"Anthropic thinking is not supported for models without "
"a reasoning parser"
)
return
if self.reasoning_parser == "hunyuan":
config = self.template_manager.reasoning_config
if config is not None and config.special_case == "hunyuan_effort":
request.reasoning_effort = "high" if enabled else "no_think"
else:
request.reasoning_effort = "medium" if enabled else "no_think"
return
if self.reasoning_parser == "iquest_q1":
request.chat_template_kwargs = {
**(request.chat_template_kwargs or {}),
"thinking": enabled,
"enable_thinking": enabled,
}
return
if self.reasoning_parser == "inkling":
# Effort-conditioned, not toggled: "none" (0.0) is the off switch.
if not enabled:
request.reasoning_effort = "none"
return
config = self.template_manager.reasoning_config
is_mistral = (config is not None and config.special_case == "mistral") or (
config is None and self._reasoning_default_mode() == "mistral"
)
if is_mistral:
request.reasoning_effort = "medium" if enabled else "none"
return
is_always_on = (config is not None and config.special_case == "always") or (
config is None and self._reasoning_default_mode() == "always"
)
if is_always_on:
if not enabled:
raise ValueError(
f"Reasoning parser '{self.reasoning_parser}' is always-on "
f"and cannot be disabled via Anthropic thinking"
)
return
toggle_param = self._get_reasoning_toggle_param()
# The read side (``_get_reasoning_from_request``) returns False
# whenever ``config.toggle_param is None`` OR
# ``config.default_enabled is None``. The write side must mirror
# both conditions: if ``default_enabled`` is unset we cannot
# actually honor an ``enabled=True`` request even when the toggle
# name itself is resolvable, so writing the kwarg would set up the
# template to emit reasoning tokens while the parser ignores them
# (literal ``<think>`` markers leak into the assistant text).
config = self.template_manager.reasoning_config
read_side_supported = toggle_param is not None and (
config is None or config.default_enabled is not None
)
if not read_side_supported:
if not enabled:
return
raise ValueError(
f"Anthropic thinking is not supported for reasoning parser "
f"'{self.reasoning_parser}'"
)
chat_template_kwargs = dict(request.chat_template_kwargs or {})
chat_template_kwargs[toggle_param] = enabled
request.chat_template_kwargs = chat_template_kwargs
def _get_reasoning_from_request(self, request: ChatCompletionRequest) -> bool:
"""Determine whether reasoning mode should be enabled for this request.
NOTE: This is predefined based on model's chat template
"""
if not self.reasoning_parser:
return False
if self.reasoning_parser == "iquest_q1":
return IQuestQ1ReasoningDetector.thinking_enabled(
request.chat_template_kwargs or {}
)
if self.reasoning_parser == "minimax-m3":
# M3 template prefills <mm:think> for thinking_mode=enabled, so it never
# appears in output and reasoning must be forced. Mirrors reasoning_parser.py.
return (request.chat_template_kwargs or {}).get(
"thinking_mode"
) == "enabled"
if self.reasoning_parser == "hunyuan":
config = self.template_manager.reasoning_config
if config is not None and config.special_case == "hunyuan_effort":
return request.reasoning_effort not in ("none", "no_think")
# Hy3-preview template emits no <think> when reasoning_effort is
# "no_think" / "none" / unset; forcing reasoning would route all
# output into reasoning_content.
return request.reasoning_effort not in (None, "none", "no_think")
config = self.template_manager.reasoning_config
if config is None:
# Fallback to parser-level defaults when template toggle config
# cannot be inferred (e.g., parser-only <think> templates).
mode = (
self._reasoning_detector.reasoning_default
if self._reasoning_detector is not None
else None
)
if mode is None:
return False
if mode == "always":
return True
if mode == "mistral":
return (
request.reasoning_effort is not None
and request.reasoning_effort != "none"
)
if mode in ("thinking", "enable_thinking"):
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get(mode) is not False
)
if mode in ("explicit_thinking", "explicit_enable_thinking"):
toggle = mode.replace("explicit_", "")
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get(toggle) is True
)
logger.warning(
"Unknown reasoning_default mode '%s', defaulting to reasoning disabled",
mode,
)
return False
if config.special_case == "always":
return True
if config.special_case == "mistral":
return (
request.reasoning_effort is not None
and request.reasoning_effort != "none"
)
if config.toggle_param is None or config.default_enabled is None:
return False
if config.default_enabled:
return (
not request.chat_template_kwargs
or request.chat_template_kwargs.get(config.toggle_param) is not False
)
return (
request.chat_template_kwargs is not None
and request.chat_template_kwargs.get(config.toggle_param) is True
)
async def _process_tool_call_stream(
self,
index: int,
delta: str,
parser_dict: dict[int, FunctionCallParser],
content: dict[str, Any],
request: ChatCompletionRequest,
has_tool_calls: dict[int, bool],
continuous_usage_stats: bool = False,
flush: bool = False,
):
"""Process tool calls in streaming response.
With flush=True (the terminal delta), the parser also drains text it
held back waiting for a marker that can no longer arrive.
"""
effective_tools = self._effective_tools(request)
if index not in parser_dict:
is_required = request.tool_choice == "required" or isinstance(
request.tool_choice, ToolChoice
)
# For required/named tool choice: use JsonArrayParser when the
# constrained output is plain JSON (detector doesn't support
# structural_tag or no parser configured). Use FunctionCallParser
# only when the detector supports structural_tag and will produce
# native format output.
if is_required:
use_native_parser = False
if self.tool_call_parser:
probe = FunctionCallParser(
tools=effective_tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
tool_choice=request.tool_choice,
)
use_native_parser = probe.owns_tool_format()
if use_native_parser:
parser_dict[index] = probe
else:
parser_dict[index] = JsonArrayParser()
else:
parser_dict[index] = FunctionCallParser(
tools=effective_tools,
tool_call_parser=self.tool_call_parser,
tokenizer=self.tokenizer_manager.tokenizer,
)
parser = parser_dict[index]
# Handle both FunctionCallParser and JsonArrayParser
if isinstance(parser, JsonArrayParser):
result = parser.parse_streaming_increment(delta, effective_tools)
normal_text, calls = result.normal_text, result.calls
else:
normal_text, calls = parser.parse_stream_chunk(delta)
if flush:
end_text, end_calls = parser.parse_stream_end()
normal_text = (normal_text or "") + end_text
calls = list(calls) + end_calls
# Yield normal text
if normal_text:
choice_data = ChatCompletionResponseStreamChoice(
index=index,
delta=DeltaMessage(content=normal_text),
finish_reason=None,
)
chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[choice_data],
model=request.model,
)
# Add usage stats if continuous_usage_stats is enabled
if continuous_usage_stats:
prompt_tokens = self._reported_prompt_tokens(content["meta_info"])
completion_tokens = content["meta_info"].get("completion_tokens", 0)
reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0)
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
cached_tokens=self._continuous_usage_cached_details(content),
)
yield f"data: {chunk.model_dump_json()}\n\n"
# Yield tool calls
history_tool_calls_cnt = self._get_history_tool_calls_cnt(request)
for call_item in calls:
# Mark that this choice has tool calls
has_tool_calls[index] = True
# Tool call ID should be generated only once per tool call
if call_item.name:
# First chunk: include ID and function name
tool_call_id = self._process_tool_call_id(
call_item, history_tool_calls_cnt
)
function_name = call_item.name
else:
# Subsequent chunks: null ID and name for argument deltas
tool_call_id = None
function_name = None
tool_call = ToolCall(
id=tool_call_id,
index=call_item.tool_index,
function=FunctionResponse(
name=function_name,
arguments=call_item.parameters,
),
)
choice_data = ChatCompletionResponseStreamChoice(
index=index,
delta=DeltaMessage(tool_calls=[tool_call]),
finish_reason=None,
)
chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[choice_data],
model=request.model,
)
# Add usage stats if continuous_usage_stats is enabled
if continuous_usage_stats:
prompt_tokens = self._reported_prompt_tokens(content["meta_info"])
completion_tokens = content["meta_info"].get("completion_tokens", 0)
reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0)
chunk.usage = UsageProcessor.calculate_token_usage(
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
reasoning_tokens=reasoning_tokens,
cached_tokens=self._continuous_usage_cached_details(content),
)
yield f"data: {chunk.model_dump_json()}\n\n"
def _check_for_unstreamed_tool_args(
self,
parser: FunctionCallParser | JsonArrayParser,
content: dict[str, Any],
request: ChatCompletionRequest,
index: int,
) -> str | None:
"""
Check for any remaining tool call arguments that need to be streamed
when generation finishes. This ensures tool calls are properly completed
even if the model generates the final arguments in the last chunk.
"""
# Get the detector - either from FunctionCallParser or directly if json detector
detector = parser.detector if hasattr(parser, "detector") else parser
# Only check if we have tool calls and the detector has tracked data
if (
not hasattr(detector, "prev_tool_call_arr")
or not detector.prev_tool_call_arr
):
return None
if (
not hasattr(detector, "streamed_args_for_tool")
or not detector.streamed_args_for_tool
):
return None
# Get the last tool call that was being processed
tool_index = len(detector.prev_tool_call_arr) - 1
if tool_index < 0 or tool_index >= len(detector.streamed_args_for_tool):
return None
# Get expected vs actual arguments
expected_args = detector.prev_tool_call_arr[tool_index].get("arguments", {})
if isinstance(expected_args, str):
expected_call = expected_args
else:
expected_call = json.dumps(expected_args, ensure_ascii=False)
actual_call = detector.streamed_args_for_tool[tool_index]
# Check if there are remaining arguments to send
remaining_call = (
expected_call[len(actual_call) :]
if expected_call.startswith(actual_call)
else ""
)
if remaining_call:
# Create tool call chunk with remaining arguments
tool_call = ToolCall(
id=None, # No ID for argument deltas
index=tool_index,
function=FunctionResponse(
name=None, # No name for argument deltas
arguments=remaining_call,
),
)
choice_data = ChatCompletionResponseStreamChoice(
index=index,
delta=DeltaMessage(tool_calls=[tool_call]),
finish_reason=None, # Don't send finish_reason with this chunk
)
chunk = ChatCompletionStreamResponse(
id=content["meta_info"]["id"],
created=int(time.time()),
choices=[choice_data],
model=request.model,
)
return f"data: {chunk.model_dump_json()}\n\n"
return None