_tokenize_one_request._create_tokenized_object 会读 rid_to_state[obj.rid]
(正常生成路径在 tokenize 前注册 ReqState),纯查询无 rid → KeyError: None。
改为:dsv41 chat encoding 直接渲染的 prompt_ids 直接用;否则对渲染后
text 调 _tokenize_texts(与真实路径同一分词入口)。tok8 环境实测 200。
(cherry picked from commit 71c6d8bc55)
3277 lines
135 KiB
Python
3277 lines
135 KiB
Python
from __future__ import annotations
|
||
|
||
import copy
|
||
import json
|
||
import logging
|
||
import math
|
||
import time
|
||
import uuid
|
||
from collections import OrderedDict
|
||
from collections.abc import AsyncGenerator
|
||
from enum import Enum
|
||
from http import HTTPStatus
|
||
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
|
||
|
||
from sglang.srt.runtime_context import get_model, get_serving
|
||
|
||
|
||
class ThinkingMode(str, Enum):
|
||
"""Mode for message encoding - chat vs thinking/reasoning."""
|
||
|
||
CHAT = "chat"
|
||
THINKING = "thinking"
|
||
|
||
|
||
import jinja2
|
||
import orjson
|
||
from fastapi import Request
|
||
|
||
try:
|
||
from mistral_common.exceptions import MistralCommonException
|
||
|
||
_MISTRAL_COMMON_ERRORS: tuple[type[BaseException], ...] = (MistralCommonException,)
|
||
except ImportError:
|
||
_MISTRAL_COMMON_ERRORS = ()
|
||
|
||
_CHAT_TEMPLATE_CLIENT_ERRORS: tuple[type[BaseException], ...] = (
|
||
jinja2.TemplateError,
|
||
TypeError,
|
||
) + _MISTRAL_COMMON_ERRORS
|
||
from fastapi.responses import ORJSONResponse, StreamingResponse
|
||
from jsonschema import Draft202012Validator, SchemaError
|
||
|
||
from sglang.srt.entrypoints.openai import (
|
||
chat_encoding,
|
||
encoding_dsv4,
|
||
encoding_dsv32,
|
||
encoding_dsv41,
|
||
)
|
||
from sglang.srt.entrypoints.openai.protocol import (
|
||
ChatCompletionMessageContentTextPart,
|
||
ChatCompletionMessageContentVideoPart,
|
||
ChatCompletionMessageGenericParam,
|
||
ChatCompletionMessageUserParam,
|
||
ChatCompletionRequest,
|
||
ChatCompletionResponse,
|
||
ChatCompletionResponseChoice,
|
||
ChatCompletionResponseStreamChoice,
|
||
ChatCompletionStreamResponse,
|
||
ChatCompletionTokenLogprob,
|
||
ChatMessage,
|
||
ChoiceLogprobs,
|
||
DeltaMessage,
|
||
ErrorResponse,
|
||
FunctionResponse,
|
||
MessageProcessingResult,
|
||
PromptTokensDetails,
|
||
ResponseParserProtocol,
|
||
SglExt,
|
||
Tool,
|
||
ToolCall,
|
||
ToolCallProcessingResult,
|
||
ToolChoice,
|
||
TopLogprob,
|
||
)
|
||
from sglang.srt.entrypoints.openai.serving_base import OpenAIServingBase
|
||
from sglang.srt.entrypoints.openai.sse_utils import build_sse_content
|
||
from sglang.srt.entrypoints.openai.usage_processor import UsageProcessor
|
||
from sglang.srt.entrypoints.openai.utils import (
|
||
cached_tokens_details_from_dict,
|
||
process_cached_tokens_details_from_ret,
|
||
process_hidden_states_for_response,
|
||
process_hidden_states_from_ret,
|
||
process_routed_experts_from_ret,
|
||
process_spec_tokens_details_from_ret,
|
||
should_include_usage,
|
||
spec_tokens_details_from_meta_info,
|
||
token_id_to_bytes,
|
||
)
|
||
from sglang.srt.entrypoints.request_headers import apply_header_overrides
|
||
from sglang.srt.environ import envs
|
||
from sglang.srt.function_call.core_types import ToolCallItem
|
||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||
from sglang.srt.function_call.json_array_parser import JsonArrayParser
|
||
from sglang.srt.function_call.utils import (
|
||
get_json_schema_constraint,
|
||
normalize_json_schema_types,
|
||
strip_structural_tag_excludes,
|
||
)
|
||
from sglang.srt.managers.io_struct import GenerateReqInput
|
||
from sglang.srt.parser.conversation import generate_chat_conv
|
||
from sglang.srt.parser.hunyuan_reasoning import (
|
||
normalize_hunyuan_reasoning_effort,
|
||
uses_hunyuan_reasoning_effort,
|
||
)
|
||
from sglang.srt.parser.jinja_template_utils import (
|
||
MEDIA_URL_PART_TYPES,
|
||
process_content_for_template_format,
|
||
)
|
||
from sglang.srt.parser.reasoning_parser import (
|
||
IQuestQ1ReasoningDetector,
|
||
ReasoningParser,
|
||
)
|
||
from sglang.srt.sampling.sampling_params import (
|
||
set_request_reasoning_end_token_ids,
|
||
)
|
||
from sglang.srt.utils import ImageData
|
||
from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata
|
||
|
||
if TYPE_CHECKING:
|
||
from sglang.srt.managers.tokenizer_manager import TokenizerManager
|
||
from sglang.srt.parser.template_manager import TemplateManager
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_MEDIA_CONTENT_PART_TYPES = frozenset({"image_url", "video_url", "audio_url"})
|
||
_CHAT_TEMPLATE_CACHE_MAX_SIZE = 128
|
||
|
||
|
||
def normalize_tool_content(role: str, content):
|
||
"""Normalize tool message content from OpenAI array format to plain string.
|
||
|
||
OpenAI clients may send tool content as a list of content parts
|
||
(e.g. [{"type":"text","text":"..."}]) but most chat templates expect
|
||
a plain string for tool messages. Only flatten when ALL items are
|
||
pure OpenAI text parts; preserve lists containing non-text-type items
|
||
that some templates intentionally iterate over.
|
||
"""
|
||
if role != "tool" or not isinstance(content, list):
|
||
return content
|
||
parts = content
|
||
is_openai_text_parts = all(
|
||
(isinstance(p, dict) and p.get("type") == "text") or isinstance(p, str)
|
||
for p in parts
|
||
)
|
||
if is_openai_text_parts:
|
||
text_parts = [p.get("text", "") if isinstance(p, dict) else p for p in parts]
|
||
return " ".join(text_parts)
|
||
return content
|
||
|
||
|
||
def parse_tool_call_arguments(arguments: str) -> dict[str, Any]:
|
||
"""Parse OpenAI tool call arguments for chat templates."""
|
||
try:
|
||
parsed_arguments = orjson.loads(arguments)
|
||
except orjson.JSONDecodeError as exc:
|
||
raise ValueError(
|
||
"Assistant tool call function.arguments must be valid JSON."
|
||
) from exc
|
||
|
||
if not isinstance(parsed_arguments, dict):
|
||
raise ValueError(
|
||
"Assistant tool call function.arguments must be a JSON object."
|
||
)
|
||
|
||
return parsed_arguments
|
||
|
||
|
||
def normalize_assistant_tool_call_arguments(
|
||
message: dict[str, Any], *, strict: bool = True
|
||
) -> None:
|
||
"""Normalize assistant history tool call arguments in-place."""
|
||
if message.get("role") != "assistant" or not isinstance(
|
||
message.get("tool_calls"), list
|
||
):
|
||
return
|
||
|
||
for item in message["tool_calls"]:
|
||
function = item.get("function") if isinstance(item, dict) else None
|
||
if not isinstance(function, dict):
|
||
continue
|
||
if "arguments" in function and isinstance(function["arguments"], str):
|
||
try:
|
||
function["arguments"] = parse_tool_call_arguments(function["arguments"])
|
||
except ValueError:
|
||
if strict:
|
||
raise
|
||
|
||
|
||
def _extract_max_dynamic_patch(request: ChatCompletionRequest):
|
||
img_vals = []
|
||
vid_vals = []
|
||
for msg in request.messages or []:
|
||
content = getattr(msg, "content", None)
|
||
if not isinstance(content, list):
|
||
continue
|
||
for part in content:
|
||
# pydantic object or dict type
|
||
if getattr(part, "type", None) == "image_url":
|
||
iu = getattr(part, "image_url", None)
|
||
mdp = getattr(iu, "max_dynamic_patch", None) if iu else None
|
||
if mdp is not None:
|
||
img_vals.append(int(mdp))
|
||
elif getattr(part, "type", None) == "video_url":
|
||
vu = getattr(part, "video_url", None)
|
||
mdp = getattr(vu, "max_dynamic_patch", None) if vu else None
|
||
if mdp is not None:
|
||
vid_vals.append(int(mdp))
|
||
|
||
# TODO(yuan-luo): per-item max_dynamic_patch for both image and video
|
||
img_max_dynamic_patch = min(img_vals) if img_vals else None
|
||
vid_max_dynamic_patch = min(vid_vals) if vid_vals else None
|
||
return img_max_dynamic_patch, vid_max_dynamic_patch
|
||
|
||
|
||
KIMI_K3_IMAGE_PLACEHOLDER = "<|kimi_image_placeholder|>"
|
||
KIMI_K3_IMAGE_PLACEHOLDER_ESCAPED = "<| kimi_image_placeholder |>"
|
||
|
||
|
||
def neutralize_kimi_k3_image_placeholder(text: str) -> str:
|
||
return text.replace(KIMI_K3_IMAGE_PLACEHOLDER, KIMI_K3_IMAGE_PLACEHOLDER_ESCAPED)
|
||
|
||
|
||
def neutralize_kimi_k3_image_placeholder_value(value: Any) -> Any:
|
||
if isinstance(value, str):
|
||
return neutralize_kimi_k3_image_placeholder(value)
|
||
if isinstance(value, list):
|
||
return [neutralize_kimi_k3_image_placeholder_value(item) for item in value]
|
||
if isinstance(value, dict):
|
||
return {
|
||
key: neutralize_kimi_k3_image_placeholder_value(item)
|
||
for key, item in value.items()
|
||
}
|
||
return value
|
||
|
||
|
||
def _extract_video_question(request: ChatCompletionRequest) -> str | None:
|
||
"""Return text paired with a video in the last user turn."""
|
||
for message in reversed(request.messages or []):
|
||
if not isinstance(message, ChatCompletionMessageUserParam):
|
||
continue
|
||
content = message.content
|
||
if not isinstance(content, list):
|
||
continue
|
||
has_video = any(
|
||
isinstance(part, ChatCompletionMessageContentVideoPart) for part in content
|
||
)
|
||
if not has_video:
|
||
continue
|
||
return "".join(
|
||
part.text
|
||
for part in content
|
||
if isinstance(part, ChatCompletionMessageContentTextPart)
|
||
)
|
||
return None
|
||
|
||
|
||
def _build_video_config(request: ChatCompletionRequest) -> dict[str, Any] | None:
|
||
"""Build request-scoped video processor config without model-specific fields."""
|
||
config = dict(request.video_config or {})
|
||
question = _extract_video_question(request)
|
||
if question is not None:
|
||
# Internal metadata derived from the message must not be overridden by
|
||
# a model-specific public processor option.
|
||
config["_question"] = question
|
||
return config or None
|
||
|
||
|
||
class OpenAIServingChat(OpenAIServingBase):
|
||
"""Handler for /v1/chat/completions requests"""
|
||
|
||
_default_sampling_params_logged = False
|
||
_KIMI_K3_GENERATION_STUB_TOKENS = 3
|
||
|
||
def __init__(
|
||
self,
|
||
tokenizer_manager: TokenizerManager,
|
||
template_manager: TemplateManager,
|
||
):
|
||
super().__init__(tokenizer_manager)
|
||
self.template_manager = template_manager
|
||
self.tool_call_parser = self.tokenizer_manager.config_value("tool_call_parser")
|
||
self.reasoning_parser = self.tokenizer_manager.config_value("reasoning_parser")
|
||
self.default_chat_template_kwargs = (
|
||
get_serving().default_chat_template_kwargs or {}
|
||
)
|
||
self._reasoning_detector = None
|
||
if self.reasoning_parser:
|
||
try:
|
||
rp = ReasoningParser(
|
||
model_type=self.reasoning_parser,
|
||
stream_reasoning=True,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
)
|
||
self._reasoning_detector = rp.detector
|
||
except ValueError as e:
|
||
logger.warning(
|
||
"Failed to initialize reasoning detector for parser '%s': %s",
|
||
self.reasoning_parser,
|
||
e,
|
||
)
|
||
|
||
# Get default sampling parameters from model's generation config
|
||
self.default_sampling_params = (
|
||
self.tokenizer_manager.model_config.get_default_sampling_params()
|
||
)
|
||
if (
|
||
self.default_sampling_params
|
||
and not OpenAIServingChat._default_sampling_params_logged
|
||
):
|
||
logger.info(
|
||
f"Using default chat sampling params from model generation config: {self.default_sampling_params}",
|
||
)
|
||
OpenAIServingChat._default_sampling_params_logged = True
|
||
|
||
# Check if the model is a GPT-OSS model
|
||
self.is_gpt_oss = (
|
||
hasattr(self.tokenizer_manager.model_config, "hf_config")
|
||
and hasattr(self.tokenizer_manager.model_config.hf_config, "model_type")
|
||
and self.tokenizer_manager.model_config.hf_config.model_type == "gpt_oss"
|
||
)
|
||
self.is_gemma4 = (
|
||
hasattr(self.tokenizer_manager.model_config, "hf_config")
|
||
and hasattr(self.tokenizer_manager.model_config.hf_config, "model_type")
|
||
and self.tokenizer_manager.model_config.hf_config.model_type
|
||
in ("gemma4", "gemma4_unified")
|
||
)
|
||
|
||
# Which Python-based chat encoder (if any) bypasses apply_chat_template.
|
||
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
||
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
||
self._dsv4_reasoning_effort_profile = (
|
||
chat_encoding.resolve_dsv4_reasoning_effort_profile(
|
||
model_path=self.tokenizer_manager.model_path,
|
||
revision=get_model().revision,
|
||
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
|
||
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
|
||
),
|
||
)
|
||
if self.chat_encoding_spec == "dsv4"
|
||
else None
|
||
)
|
||
|
||
# Resolve the env-configured Inkling effort default once: the env var is
|
||
# frozen for the server's lifetime, and a misconfigured value should
|
||
# fail at boot, not 400 every request.
|
||
self._inkling_default_reasoning_effort: float | None = (
|
||
self._get_inkling_default_reasoning_effort()
|
||
if self.chat_encoding_spec == "inkling"
|
||
else None
|
||
)
|
||
self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = (
|
||
chat_encoding.default_dsv41_reasoning_effort_from_env(
|
||
envs.SGLANG_DSV41_REASONING_EFFORT.get()
|
||
)
|
||
if self.chat_encoding_spec == "dsv41"
|
||
else None
|
||
)
|
||
|
||
# Per-request response parser for custom decoding (set by _encode_messages)
|
||
self._response_parser: ResponseParserProtocol | None = None
|
||
|
||
# Probe whether ``encode("")`` returns specials. If it does, we must
|
||
# keep ``add_special_tokens=False`` at the chat-template encode site
|
||
# to avoid double BOS; otherwise the kwarg is a no-op and dropping it
|
||
# lets slow tokenizers (e.g. Kimi's TikTokenTokenizer) stay on the
|
||
# fast internal path.
|
||
try:
|
||
self._tokenizer_auto_adds_specials = (
|
||
len(self.tokenizer_manager.tokenizer.encode("")) > 0
|
||
)
|
||
except Exception:
|
||
self._tokenizer_auto_adds_specials = True
|
||
self._prompt_text_round_trip_is_lossy = self._probe_prompt_text_round_trip()
|
||
self._chat_template_cache: OrderedDict[bytes, tuple[tuple[int, ...], str]] = (
|
||
OrderedDict()
|
||
)
|
||
|
||
def _probe_prompt_text_round_trip(self) -> bool:
|
||
"""Does rendering the chat template to text and re-encoding lose anything?
|
||
|
||
mistral_common tokenizers emit control tokens ([INST],
|
||
[AVAILABLE_TOOLS], ...) that have no text form. Rendering to a string
|
||
turns them into literal characters and re-encoding also prepends a
|
||
second BOS, so the model sees the letters "AVAILABLE_TOOLS" instead of
|
||
the control token that frames the tool block. Encoding straight to ids
|
||
is the only faithful route on such tokenizers, so compare the two here
|
||
once and remember which to trust.
|
||
"""
|
||
probe = [{"role": "user", "content": "x"}]
|
||
try:
|
||
tokenizer = self.tokenizer_manager.tokenizer
|
||
rendered = tokenizer.apply_chat_template(
|
||
probe, tokenize=False, add_generation_prompt=True, return_dict=False
|
||
)
|
||
encode_kwargs = (
|
||
{"add_special_tokens": False}
|
||
if self._tokenizer_auto_adds_specials
|
||
else {}
|
||
)
|
||
via_text = tokenizer.encode(rendered, **encode_kwargs)
|
||
via_ids = tokenizer.apply_chat_template(
|
||
probe, tokenize=True, add_generation_prompt=True, return_dict=False
|
||
)
|
||
return list(via_text) != list(via_ids)
|
||
except Exception:
|
||
# A template that needs kwargs this probe does not supply tells us
|
||
# nothing; keep the long-standing text path.
|
||
return False
|
||
|
||
def _handle_last_assistant_message(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
request: ChatCompletionRequest,
|
||
) -> tuple[list[dict[str, Any]], str | None]:
|
||
"""
|
||
Handle continue_final_message feature: separate final assistant message.
|
||
|
||
If continue_final_message is enabled and the last message is from assistant,
|
||
extract its content and remove it from the message list.
|
||
If continue_final_message is False and the last message is from assistant,
|
||
convert it to a user message to ensure the last message is always from user.
|
||
|
||
Only processes text-based content (strings), ignoring multimodal content (lists).
|
||
|
||
Args:
|
||
messages: List of message dictionaries
|
||
request: ChatCompletionRequest with continue_final_message flag
|
||
|
||
Returns:
|
||
Tuple of (processed_messages, assistant_prefix)
|
||
- processed_messages: Messages with last assistant message handled appropriately
|
||
- assistant_prefix: Content of the last assistant message (string only), or None
|
||
"""
|
||
assistant_prefix = None
|
||
if messages and messages[-1].get("role") == "assistant":
|
||
last_content = messages[-1].get("content")
|
||
# Only process string content, ignore multimodal content (lists)
|
||
if isinstance(last_content, str):
|
||
if request.continue_final_message:
|
||
# Extract content and remove the assistant message
|
||
assistant_prefix = last_content
|
||
messages = messages[:-1]
|
||
else:
|
||
# Convert the last assistant message to user message
|
||
messages[-1] = {"role": "user", "content": last_content}
|
||
return messages, assistant_prefix
|
||
|
||
def _append_assistant_prefix_to_prompt_ids(
|
||
self, prompt_ids: list[int], assistant_prefix: str
|
||
) -> list[int]:
|
||
"""
|
||
Append assistant prefix to prompt_ids.
|
||
|
||
Args:
|
||
prompt_ids: Current prompt token IDs
|
||
assistant_prefix: Assistant message content to append
|
||
|
||
Returns:
|
||
Updated prompt_ids with assistant prefix appended
|
||
"""
|
||
encoded = self.tokenizer_manager.tokenizer.encode(assistant_prefix)
|
||
if encoded and encoded[0] == self.tokenizer_manager.tokenizer.bos_token_id:
|
||
encoded = encoded[1:]
|
||
return prompt_ids + encoded
|
||
|
||
def _resolve_chat_encoding_spec(self) -> str | None:
|
||
"""Determine which chat encoding spec to use.
|
||
|
||
Override in subclass to add custom encoding specs.
|
||
"""
|
||
return chat_encoding.resolve_chat_encoding_spec(
|
||
hf_config=self.tokenizer_manager.model_config.hf_config,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
tool_call_parser=self.tool_call_parser,
|
||
)
|
||
|
||
def _request_id_prefix(self) -> str:
|
||
return "chatcmpl-"
|
||
|
||
def _effective_tools(self, request: ChatCompletionRequest) -> list[Tool]:
|
||
tools = list(request.tools or [])
|
||
for message in request.messages:
|
||
if (
|
||
isinstance(message, ChatCompletionMessageGenericParam)
|
||
and message.role in ("system", "developer")
|
||
and message.tools
|
||
):
|
||
tools.extend(message.tools)
|
||
return tools
|
||
|
||
def _prepare_kimi_k3_messages(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
request: ChatCompletionRequest,
|
||
) -> tuple[list[dict[str, Any]], int, str | None]:
|
||
image_count = 0
|
||
for index, message in enumerate(messages):
|
||
content = message.get("content")
|
||
if isinstance(content, list):
|
||
parts = []
|
||
for part in content:
|
||
if not isinstance(part, dict):
|
||
continue
|
||
part_type = part.get("type")
|
||
if part_type in ("text", "input_text"):
|
||
parts.append(
|
||
{
|
||
"type": "text",
|
||
"text": neutralize_kimi_k3_image_placeholder(
|
||
part["text"]
|
||
),
|
||
}
|
||
)
|
||
elif part_type in ("image_url", "input_image"):
|
||
image = part.get("image_url") or {}
|
||
if isinstance(image, str):
|
||
image = {"url": image, "detail": part.get("detail")}
|
||
parts.append({"type": "image_url", "image_url": image})
|
||
image_count += 1
|
||
message["content"] = parts
|
||
elif isinstance(content, str):
|
||
message["content"] = neutralize_kimi_k3_image_placeholder(content)
|
||
elif content is None:
|
||
message["content"] = ""
|
||
|
||
if message.get("role") == "assistant":
|
||
for key in ("reasoning_content", "reasoning"):
|
||
if key in message:
|
||
message[key] = neutralize_kimi_k3_image_placeholder_value(
|
||
message[key]
|
||
)
|
||
for tool_call in message.get("tool_calls") or []:
|
||
function = (
|
||
tool_call.get("function")
|
||
if isinstance(tool_call, dict)
|
||
else None
|
||
)
|
||
if isinstance(function, dict) and "arguments" in function:
|
||
function["arguments"] = (
|
||
neutralize_kimi_k3_image_placeholder_value(
|
||
function["arguments"]
|
||
)
|
||
)
|
||
|
||
source = request.messages[index]
|
||
if (
|
||
isinstance(source, ChatCompletionMessageGenericParam)
|
||
and source.role in ("system", "developer")
|
||
and source.tools
|
||
):
|
||
message["tools"] = [
|
||
tool.model_dump(exclude_unset=True, by_alias=True)
|
||
for tool in source.tools
|
||
]
|
||
if message.get("role") == "developer":
|
||
message["role"] = "system"
|
||
|
||
assistant_prefix = None
|
||
if request.continue_final_message:
|
||
messages, assistant_prefix = self._handle_last_assistant_message(
|
||
messages, request
|
||
)
|
||
return messages, image_count, assistant_prefix
|
||
|
||
def _encode_messages(
|
||
self,
|
||
messages: list[dict[str, Any]],
|
||
request: ChatCompletionRequest,
|
||
thinking_mode: ThinkingMode,
|
||
tools: list[dict] | None = None,
|
||
) -> list[int] | None:
|
||
"""Encode messages for custom chat_encoding_spec values.
|
||
|
||
Returns prompt_ids if handled, None to use default encoding.
|
||
"""
|
||
if self.chat_encoding_spec == "inkling":
|
||
# Inkling: render messages -> input_ids with framing tokens + ONE placeholder per
|
||
# media (encoding/expansion happens later in InklingMultimodalProcessor). The
|
||
# server's tokenizer is the base tiktoken backend; wrap it so encode_special
|
||
# supplies the framing-token overlay.
|
||
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
||
from sglang.srt.parser.inkling_tokenizer import (
|
||
CONTENT_TEXT,
|
||
MESSAGE_MODEL,
|
||
InklingTokenizer,
|
||
)
|
||
|
||
inkling_tokenizer = InklingTokenizer(
|
||
tokenizer=self.tokenizer_manager.tokenizer
|
||
)
|
||
reasoning_effort = self._parse_inkling_reasoning_effort(
|
||
request.reasoning_effort
|
||
)
|
||
if reasoning_effort is None:
|
||
reasoning_effort = self._inkling_default_reasoning_effort
|
||
assistant_prefix = self._pop_inkling_assistant_prefix(messages, request)
|
||
prompt_ids = render_inkling_messages(
|
||
messages,
|
||
inkling_tokenizer,
|
||
add_generation_prompt=False,
|
||
tools=tools,
|
||
reasoning_effort=reasoning_effort,
|
||
)
|
||
if assistant_prefix is not None:
|
||
# Continue the final assistant message inside an OPEN model text
|
||
# block: header + payload, no <|end_message|> and no
|
||
# <|content_model_end_sampling|>, so the model resumes the turn.
|
||
prompt_ids += [
|
||
inkling_tokenizer.encode_special(MESSAGE_MODEL),
|
||
inkling_tokenizer.encode_special(CONTENT_TEXT),
|
||
*inkling_tokenizer.encode_text(assistant_prefix),
|
||
]
|
||
return prompt_ids
|
||
if self.chat_encoding_spec == "kimi_k3":
|
||
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
|
||
messages, request
|
||
)
|
||
template_kwargs = dict(request.chat_template_kwargs or {})
|
||
template_kwargs.pop("tokenize", None)
|
||
template_kwargs.pop("return_dict", None)
|
||
template_kwargs.pop("image_prompts", None)
|
||
if image_count:
|
||
template_kwargs["image_prompts"] = ["<|media_pad|>"] * image_count
|
||
|
||
if (
|
||
request.reasoning_effort in ("low", "high", "max")
|
||
and "thinking_effort" not in template_kwargs
|
||
):
|
||
template_kwargs["thinking_effort"] = request.reasoning_effort
|
||
elif request.reasoning_effort not in (
|
||
None,
|
||
"none",
|
||
"low",
|
||
"high",
|
||
"max",
|
||
):
|
||
logger.warning(
|
||
"Kimi K3 does not support reasoning_effort=%r; using the "
|
||
"encoder default.",
|
||
request.reasoning_effort,
|
||
)
|
||
|
||
effective_tools = self._effective_tools(request)
|
||
if (
|
||
effective_tools
|
||
and isinstance(request.tool_choice, str)
|
||
and request.tool_choice in ("required", "none")
|
||
):
|
||
template_kwargs.setdefault("tool_choice", request.tool_choice)
|
||
if request.response_format is not None:
|
||
template_kwargs.setdefault(
|
||
"response_format",
|
||
request.response_format.model_dump(
|
||
exclude_unset=True, by_alias=True
|
||
),
|
||
)
|
||
|
||
request_tools = (
|
||
[
|
||
tool.model_dump(exclude_unset=True, by_alias=True)
|
||
for tool in request.tools
|
||
]
|
||
if request.tools
|
||
else None
|
||
)
|
||
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
|
||
messages,
|
||
tokenize=True,
|
||
add_generation_prompt=True,
|
||
tools=request_tools,
|
||
return_dict=False,
|
||
**template_kwargs,
|
||
)
|
||
if assistant_prefix:
|
||
prompt_ids = self._append_assistant_prefix_to_prompt_ids(
|
||
prompt_ids, assistant_prefix
|
||
)
|
||
return prompt_ids
|
||
return None
|
||
|
||
@staticmethod
|
||
def _pop_inkling_assistant_prefix(
|
||
messages: list[dict[str, Any]],
|
||
request: ChatCompletionRequest,
|
||
) -> str | None:
|
||
"""Extract the trailing assistant text for ``continue_final_message``.
|
||
|
||
Only a plain-string assistant message with no tool calls and no
|
||
reasoning content can be continued; anything else renders as a closed
|
||
historical turn. Mutates ``messages`` in place (callers pass a copy).
|
||
"""
|
||
if not request.continue_final_message or not messages:
|
||
return None
|
||
last = messages[-1]
|
||
if (
|
||
last.get("role") != "assistant"
|
||
or not isinstance(last.get("content"), str)
|
||
or last.get("tool_calls")
|
||
or last.get("reasoning_content")
|
||
):
|
||
return None
|
||
messages.pop()
|
||
return last["content"]
|
||
|
||
@staticmethod
|
||
def _parse_inkling_reasoning_effort(
|
||
value: str | float | None,
|
||
) -> float | None:
|
||
"""Convert an OpenAI-style reasoning_effort to an Inkling float."""
|
||
if value is None:
|
||
return None
|
||
if isinstance(value, bool):
|
||
raise ValueError("Inkling reasoning_effort must not be a boolean")
|
||
if isinstance(value, (int, float)):
|
||
parsed = float(value)
|
||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
|
||
return parsed
|
||
_EFFORT_MAP = {
|
||
"none": 0.0,
|
||
"minimal": 0.1,
|
||
"low": 0.2,
|
||
"medium": 0.7,
|
||
"high": 0.9,
|
||
"xhigh": 0.99,
|
||
"max": 0.99,
|
||
}
|
||
if value in _EFFORT_MAP:
|
||
return _EFFORT_MAP[value]
|
||
try:
|
||
parsed = float(value)
|
||
except (ValueError, TypeError) as exc:
|
||
raise ValueError(f"invalid Inkling reasoning_effort: {value!r}") from exc
|
||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||
raise ValueError("Inkling reasoning_effort must be in [0.0, 0.99]")
|
||
return parsed
|
||
|
||
def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]:
|
||
"""Request effort for the V4.1 encoder; unsupported values warn and fall back."""
|
||
effort = chat_encoding.parse_dsv41_reasoning_effort(value)
|
||
if effort is not None:
|
||
return effort
|
||
if value is not None and value != "none":
|
||
logger.warning(
|
||
"DeepSeek-V4.1 does not support reasoning_effort=%r; using the "
|
||
"default %r (low/high/xhigh/max, a float in [0, 0.99], or an "
|
||
"integer budget in [1, 100] via chat_template_kwargs are accepted).",
|
||
value,
|
||
self._dsv41_default_reasoning_effort,
|
||
)
|
||
return self._dsv41_default_reasoning_effort
|
||
|
||
@staticmethod
|
||
def _get_inkling_default_reasoning_effort() -> float:
|
||
"""Read the default Inkling reasoning effort from the environment."""
|
||
from sglang.srt.environ import envs
|
||
|
||
val = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT.get()
|
||
if not val:
|
||
return 0.9
|
||
try:
|
||
parsed = float(val)
|
||
except (ValueError, TypeError) as exc:
|
||
raise ValueError(
|
||
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be numeric"
|
||
) from exc
|
||
if not math.isfinite(parsed) or not 0.0 <= parsed <= 0.99:
|
||
raise ValueError(
|
||
"SGLANG_INKLING_DEFAULT_REASONING_EFFORT must be in [0.0, 0.99]"
|
||
)
|
||
return parsed
|
||
|
||
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
|
||
"""Extract text from response."""
|
||
return ret_item["text"]
|
||
|
||
def _get_parsed_response_fields(
|
||
self,
|
||
reasoning_text: str | None,
|
||
tool_calls: list[dict] | None,
|
||
) -> tuple[str | None, list[dict] | None]:
|
||
"""Post-process reasoning and tool_calls before building response."""
|
||
return reasoning_text, tool_calls
|
||
|
||
def _should_return_input_ids(self, request: ChatCompletionRequest) -> bool:
|
||
"""Whether prompt (input) token ids should be returned via sglext."""
|
||
return request.return_input_ids_in_sglext or get_serving().return_input_ids
|
||
|
||
def _should_return_output_ids(self, request: ChatCompletionRequest) -> bool:
|
||
"""Whether sampled output token ids should be returned via sglext."""
|
||
return request.return_output_ids_in_sglext or get_serving().return_output_ids
|
||
|
||
def _continuous_usage_cached_details(
|
||
self, content: dict[str, Any]
|
||
) -> PromptTokensDetails | None:
|
||
if not get_serving().enable_cache_report:
|
||
return None
|
||
return UsageProcessor._details_if_cached(
|
||
content["meta_info"].get("cached_tokens", 0)
|
||
)
|
||
|
||
def _reported_prompt_tokens(self, meta_info: dict[str, Any]) -> int:
|
||
prompt_tokens = meta_info.get("prompt_tokens", 0)
|
||
if self.chat_encoding_spec == "kimi_k3":
|
||
# K3's three-token assistant generation stub is model input, but the
|
||
# reference API excludes it from billed/reported prompt tokens.
|
||
prompt_tokens = max(0, prompt_tokens - self._KIMI_K3_GENERATION_STUB_TOKENS)
|
||
return prompt_tokens
|
||
|
||
@staticmethod
|
||
def _sort_tool_message_run(
|
||
run: list[dict[str, Any]], tool_calls: list[dict[str, Any]]
|
||
) -> list[dict[str, Any]]:
|
||
"""Order a tool-message run by tool_call position.
|
||
|
||
Templates that associate results by tool_call_id render the run in
|
||
tool_calls order; sorting the run upfront keeps extraction order and
|
||
placeholder order the same. Runs the template itself would refuse to
|
||
associate (missing/duplicate/unknown ids) are left untouched, as are
|
||
text-only runs, whose order text-only templates may rely on.
|
||
"""
|
||
if len(run) < 2:
|
||
return run
|
||
call_ids = [tc.get("id") for tc in tool_calls]
|
||
if any(call_id is None for call_id in call_ids) or len(set(call_ids)) != len(
|
||
call_ids
|
||
):
|
||
return run
|
||
result_ids = [message.get("tool_call_id") for message in run]
|
||
if any(result_id not in call_ids for result_id in result_ids) or len(
|
||
set(result_ids)
|
||
) != len(result_ids):
|
||
return run
|
||
has_media = any(
|
||
isinstance(message.get("content"), list)
|
||
and any(
|
||
isinstance(part, dict) and part.get("type") in MEDIA_URL_PART_TYPES
|
||
for part in message["content"]
|
||
)
|
||
for message in run
|
||
)
|
||
if not has_media:
|
||
return run
|
||
position = {call_id: index for index, call_id in enumerate(call_ids)}
|
||
return sorted(run, key=lambda message: position[message["tool_call_id"]])
|
||
|
||
@classmethod
|
||
def _canonicalize_tool_message_order(
|
||
cls, messages: list[dict[str, Any]]
|
||
) -> list[dict[str, Any]]:
|
||
canonical = []
|
||
index = 0
|
||
while index < len(messages):
|
||
message = messages[index]
|
||
canonical.append(message)
|
||
index += 1
|
||
tool_calls = message.get("tool_calls") or []
|
||
if message.get("role") != "assistant" or not tool_calls:
|
||
continue
|
||
run = []
|
||
while index < len(messages) and messages[index].get("role") in (
|
||
"tool",
|
||
"function",
|
||
):
|
||
run.append(messages[index])
|
||
index += 1
|
||
canonical.extend(cls._sort_tool_message_run(run, tool_calls))
|
||
return canonical
|
||
|
||
async def _generate_stream_content(
|
||
self,
|
||
content: dict[str, Any],
|
||
index: int,
|
||
request: ChatCompletionRequest,
|
||
stream_offsets: dict[int, int],
|
||
reasoning_parser_dict: dict,
|
||
parser_dict: dict,
|
||
has_tool_calls: dict[int, bool],
|
||
choice_logprobs: dict | None,
|
||
finish_reason_type: str | None,
|
||
continuous_usage_stats: bool,
|
||
prompt_tokens: dict[int, int],
|
||
reasoning_tokens: dict[int, int],
|
||
completion_tokens: dict[int, int],
|
||
) -> AsyncGenerator[str, None]:
|
||
"""Generate SSE chunks for streaming content."""
|
||
offset = stream_offsets.get(index, 0)
|
||
if get_serving().incremental_streaming_output:
|
||
delta = content["text"]
|
||
else:
|
||
delta = content["text"][offset:]
|
||
stream_offsets[index] = len(content["text"])
|
||
|
||
# Attach logprobs to the first chunk emitted this step (reasoning,
|
||
# tool-call, or content) so they aren't dropped when a parser is active
|
||
# nor duplicated across chunks; flush any leftover at the end.
|
||
remaining_logprobs = choice_logprobs
|
||
|
||
# Handle reasoning content
|
||
if self.reasoning_parser and request.separate_reasoning:
|
||
reasoning_text, delta = self._process_reasoning_stream(
|
||
index,
|
||
delta,
|
||
reasoning_parser_dict,
|
||
content,
|
||
request,
|
||
finish_reason_type,
|
||
)
|
||
if reasoning_text:
|
||
usage = None
|
||
if continuous_usage_stats:
|
||
usage = UsageProcessor.calculate_token_usage(
|
||
prompt_tokens=prompt_tokens.get(index, 0),
|
||
reasoning_tokens=reasoning_tokens.get(index, 0),
|
||
completion_tokens=completion_tokens.get(index, 0),
|
||
cached_tokens=self._continuous_usage_cached_details(content),
|
||
).model_dump()
|
||
|
||
yield build_sse_content(
|
||
chunk_id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
model=request.model,
|
||
index=index,
|
||
reasoning_content=reasoning_text,
|
||
logprobs=remaining_logprobs,
|
||
usage=usage,
|
||
)
|
||
remaining_logprobs = None
|
||
|
||
# Handle tool calls
|
||
if self._tool_call_parsing_active(request):
|
||
async for chunk in self._process_tool_call_stream(
|
||
index,
|
||
delta,
|
||
parser_dict,
|
||
content,
|
||
request,
|
||
has_tool_calls,
|
||
continuous_usage_stats,
|
||
flush=finish_reason_type is not None and finish_reason_type != "abort",
|
||
):
|
||
if chunk:
|
||
yield chunk
|
||
|
||
# Send any remaining tool call arguments when generation finishes
|
||
if finish_reason_type is not None and index in parser_dict:
|
||
parser = parser_dict[index]
|
||
remaining_chunk = self._check_for_unstreamed_tool_args(
|
||
parser, content, request, index
|
||
)
|
||
if remaining_chunk:
|
||
yield remaining_chunk
|
||
|
||
else:
|
||
# Regular content
|
||
if delta:
|
||
usage = None
|
||
if continuous_usage_stats:
|
||
usage = UsageProcessor.calculate_token_usage(
|
||
prompt_tokens=prompt_tokens.get(index, 0),
|
||
reasoning_tokens=reasoning_tokens.get(index, 0),
|
||
completion_tokens=completion_tokens.get(index, 0),
|
||
cached_tokens=self._continuous_usage_cached_details(content),
|
||
).model_dump()
|
||
|
||
yield build_sse_content(
|
||
chunk_id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
model=request.model,
|
||
index=index,
|
||
content=delta,
|
||
logprobs=remaining_logprobs,
|
||
usage=usage,
|
||
)
|
||
remaining_logprobs = None
|
||
|
||
# Flush logprobs still unattached this step — only when a parser is
|
||
# active, since _process_tool_call_stream may consume the delta and emit
|
||
# no content chunk. On the plain path an empty-delta step has no chunk
|
||
# to attach to either way, and a standalone empty-delta logprobs chunk
|
||
# is not a shape clients expect.
|
||
if remaining_logprobs is not None and (
|
||
self.reasoning_parser or self.tool_call_parser
|
||
):
|
||
usage = None
|
||
if continuous_usage_stats:
|
||
usage = UsageProcessor.calculate_token_usage(
|
||
prompt_tokens=prompt_tokens.get(index, 0),
|
||
reasoning_tokens=reasoning_tokens.get(index, 0),
|
||
completion_tokens=completion_tokens.get(index, 0),
|
||
cached_tokens=self._continuous_usage_cached_details(content),
|
||
).model_dump()
|
||
|
||
yield build_sse_content(
|
||
chunk_id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
model=request.model,
|
||
index=index,
|
||
logprobs=remaining_logprobs,
|
||
usage=usage,
|
||
)
|
||
|
||
def _tool_call_parsing_active(self, request: ChatCompletionRequest) -> bool:
|
||
"""Whether this request's output runs through the tool-call parser.
|
||
|
||
The reasoning parser is told the same thing, so channel-framed formats
|
||
keep their framing intact exactly when a tool-call parser consumes it.
|
||
"""
|
||
return bool(
|
||
request.tool_choice != "none"
|
||
and self._effective_tools(request)
|
||
and self.tool_call_parser
|
||
)
|
||
|
||
def _validate_request(self, request: ChatCompletionRequest) -> str | None:
|
||
"""Validate that the input is valid."""
|
||
if not request.messages:
|
||
return "Messages cannot be empty."
|
||
|
||
if (
|
||
request.chat_template_kwargs
|
||
and "chat_template" in request.chat_template_kwargs
|
||
and not get_serving().trust_request_chat_template
|
||
):
|
||
return (
|
||
"Request-supplied chat_template is not allowed. Start the server "
|
||
"with --trust-request-chat-template to permit it."
|
||
)
|
||
|
||
if request.return_sampling_mask and not request.return_meta_info:
|
||
return "return_sampling_mask requires return_meta_info=true."
|
||
|
||
media_error = self._validate_media_content(request)
|
||
if media_error:
|
||
return media_error
|
||
|
||
effective_tools = self._effective_tools(request)
|
||
has_message_tools = any(
|
||
isinstance(message, ChatCompletionMessageGenericParam)
|
||
and message.role in ("system", "developer")
|
||
and message.tools
|
||
for message in request.messages
|
||
)
|
||
if (
|
||
isinstance(request.tool_choice, str)
|
||
and request.tool_choice.lower() == "required"
|
||
and not effective_tools
|
||
):
|
||
return "Tools cannot be empty if tool choice is set to required."
|
||
|
||
if request.tool_choice is not None and not isinstance(request.tool_choice, str):
|
||
if not effective_tools:
|
||
return "Tools cannot be empty if tool choice is set to a specific tool."
|
||
tool_name = request.tool_choice.function.name
|
||
tool_exists = any(
|
||
tool.function.name == tool_name for tool in effective_tools
|
||
)
|
||
if not tool_exists:
|
||
return f"Tool '{tool_name}' not found in tools list."
|
||
|
||
if has_message_tools:
|
||
names = [tool.function.name for tool in effective_tools]
|
||
if len(names) != len(set(names)):
|
||
return "Tool names must be unique across request and message tools."
|
||
|
||
# Validate tool definitions
|
||
for i, tool in enumerate(effective_tools):
|
||
if tool.function.parameters is None:
|
||
continue
|
||
try:
|
||
# Rewrite DB/ORM-style aliases (e.g. "varchar", "enum", "int")
|
||
# to standard JSON Schema types before validation. RecursionError
|
||
# guards against hand-crafted cyclic schemas so the request gets
|
||
# a 400 instead of crashing into a 500.
|
||
normalize_json_schema_types(tool.function.parameters)
|
||
Draft202012Validator.check_schema(tool.function.parameters)
|
||
except SchemaError as e:
|
||
return f"Tool {i} function has invalid 'parameters' schema: {str(e)}"
|
||
except RecursionError:
|
||
return (
|
||
f"Tool {i} function 'parameters' schema is too deeply nested "
|
||
"or contains a cycle."
|
||
)
|
||
|
||
max_output_tokens = request.max_completion_tokens or request.max_tokens
|
||
server_context_length = get_model().context_length
|
||
if (
|
||
max_output_tokens
|
||
and server_context_length
|
||
and max_output_tokens > server_context_length
|
||
) and not get_serving().allow_auto_truncate:
|
||
return (
|
||
f"max_completion_tokens is too large: {max_output_tokens}."
|
||
f"This model supports at most {server_context_length} completion tokens."
|
||
)
|
||
|
||
if request.response_format and request.response_format.type == "json_schema":
|
||
schema = getattr(request.response_format.json_schema, "schema_", None)
|
||
if schema is None:
|
||
return "schema_ is required for json_schema response format request."
|
||
|
||
return None
|
||
|
||
def _validate_media_content(self, request: ChatCompletionRequest) -> str | None:
|
||
if self.tokenizer_manager.model_config.is_multimodal:
|
||
return None
|
||
|
||
media_type = next(
|
||
(
|
||
part.type
|
||
for message in request.messages
|
||
if isinstance(message.content, list)
|
||
for part in message.content
|
||
if part.type in _MEDIA_CONTENT_PART_TYPES
|
||
),
|
||
None,
|
||
)
|
||
if media_type is None:
|
||
return None
|
||
|
||
return (
|
||
"Model only supports text input; "
|
||
f"received unsupported content type '{media_type}'."
|
||
)
|
||
|
||
def _engine_prompt(
|
||
self, processed_messages: MessageProcessingResult, is_multimodal: bool
|
||
) -> tuple[str, Any]:
|
||
"""Standard VLMs render a text prompt (with placeholder strings) for
|
||
the MM processor to tokenize. Token-first encoders instead produce
|
||
pre-rendered input_ids with single placeholder ids and leave the text
|
||
empty; pass those through rather than re-tokenizing an empty prompt.
|
||
"""
|
||
# A lossy text round-trip makes the rendered prompt unusable, so send the
|
||
# ids instead. Only when nothing needs placeholder expansion: with media
|
||
# attached the MM processor still has to tokenize the text itself.
|
||
prefers_prompt_ids = (
|
||
self._prompt_text_round_trip_is_lossy
|
||
and isinstance(processed_messages.prompt_ids, list)
|
||
and processed_messages.prompt_ids
|
||
and not (
|
||
processed_messages.image_data
|
||
or processed_messages.video_data
|
||
or processed_messages.audio_data
|
||
)
|
||
)
|
||
if (
|
||
is_multimodal
|
||
and not chat_encoding.spec_renders_prompt_ids(self.chat_encoding_spec)
|
||
and not prefers_prompt_ids
|
||
):
|
||
return "text", processed_messages.prompt
|
||
if isinstance(processed_messages.prompt_ids, str):
|
||
return "text", processed_messages.prompt_ids
|
||
return "input_ids", processed_messages.prompt_ids
|
||
|
||
async def query_cache_hit(
|
||
self, request: ChatCompletionRequest, raw_request: Request = None
|
||
) -> ORJSONResponse:
|
||
"""G9 probe: report how much of this chat request's prompt is already
|
||
cached (per-rank-best L1/L2 radix + shared L3 storage) without running
|
||
any prefill. The router uses it for conditional PD-bypass decisions.
|
||
|
||
Multimodal requests are rejected (v1): their sentinel-token expansion
|
||
costs ~10-100ms per image for only ~2% of traffic; the router skips
|
||
them for bypass anyway.
|
||
"""
|
||
try:
|
||
internal_req, _ = self._convert_to_internal_request(request, raw_request)
|
||
except Exception as e:
|
||
return ORJSONResponse(
|
||
status_code=400,
|
||
content={"error": f"failed to convert chat request: {e}"},
|
||
)
|
||
if internal_req.contains_mm_input():
|
||
return ORJSONResponse(
|
||
status_code=400,
|
||
content={"error": "multimodal requests are not supported by /query_cache_hit"},
|
||
)
|
||
try:
|
||
# 只取 input_ids,不走 _tokenize_one_request——它要求 rid_to_state
|
||
# 里已有 ReqState(正常生成路径在 tokenize 前注册),纯查询没有。
|
||
# dsv41 系 chat encoding 直接渲染 prompt_ids;否则对渲染后的
|
||
# text 走 tokenizer(_tokenize_texts 与真实路径同一入口)。
|
||
if internal_req.input_ids is not None:
|
||
input_ids = list(internal_req.input_ids)
|
||
elif internal_req.text:
|
||
input_ids, _ = await self.tokenizer_manager._tokenize_texts(
|
||
internal_req.text, False
|
||
)
|
||
else:
|
||
return ORJSONResponse(
|
||
status_code=400,
|
||
content={"error": "request produced neither input_ids nor text"},
|
||
)
|
||
from sglang.srt.managers.io_struct import QueryCacheHitReqInput
|
||
|
||
result = await self.tokenizer_manager.query_cache_hit(
|
||
QueryCacheHitReqInput(
|
||
input_ids=list(input_ids),
|
||
extra_key=getattr(internal_req, "extra_key", None),
|
||
cache_salt=getattr(internal_req, "cache_salt", None),
|
||
)
|
||
)
|
||
except Exception as e:
|
||
return ORJSONResponse(
|
||
status_code=500, content={"error": f"cache-hit query failed: {e}"}
|
||
)
|
||
if result.error is not None:
|
||
return ORJSONResponse(status_code=500, content={"error": result.error})
|
||
hit = result.l1_hit_tokens + result.l2_hit_tokens + result.l3_hit_tokens
|
||
return ORJSONResponse(
|
||
content={
|
||
"total_tokens": result.total_tokens,
|
||
"l1_hit_tokens": result.l1_hit_tokens,
|
||
"l2_hit_tokens": result.l2_hit_tokens,
|
||
"l3_hit_tokens": result.l3_hit_tokens,
|
||
"hit_tokens": hit,
|
||
"new_tokens": max(0, result.total_tokens - hit),
|
||
"dp_rank": result.dp_rank,
|
||
}
|
||
)
|
||
|
||
def _convert_to_internal_request(
|
||
self,
|
||
request: ChatCompletionRequest,
|
||
raw_request: Request = None,
|
||
) -> tuple[GenerateReqInput, ChatCompletionRequest]:
|
||
|
||
# Header-based opt-in (same rationale as request_headers.py).
|
||
if raw_request is not None and not request.return_input_ids_in_sglext:
|
||
if raw_request.headers.get("x-sglext-return-input-ids") == "1":
|
||
request.return_input_ids_in_sglext = True
|
||
|
||
if raw_request is not None and not request.return_output_ids_in_sglext:
|
||
if raw_request.headers.get("x-sglext-return-output-ids") == "1":
|
||
request.return_output_ids_in_sglext = True
|
||
|
||
reasoning_effort = None
|
||
if not uses_hunyuan_reasoning_effort(
|
||
self.reasoning_parser, self.template_manager.reasoning_config
|
||
):
|
||
reasoning_effort = (
|
||
request.chat_template_kwargs.pop("reasoning_effort", None)
|
||
if request.chat_template_kwargs
|
||
else None
|
||
)
|
||
|
||
if self.is_gpt_oss and reasoning_effort == "none":
|
||
raise ValueError(
|
||
f"Harmony does not support reasoning effort {reasoning_effort}"
|
||
)
|
||
|
||
if reasoning_effort is not None:
|
||
request.reasoning_effort = reasoning_effort
|
||
|
||
if request.stream:
|
||
if request.return_prompt_token_ids:
|
||
raise ValueError(
|
||
"return_prompt_token_ids is not supported with streaming. "
|
||
"Please set stream=false when using return_prompt_token_ids=true."
|
||
)
|
||
if request.return_token_ids:
|
||
raise ValueError(
|
||
"return_token_ids is not supported with streaming on "
|
||
"/v1/chat/completions. Please set stream=false when using "
|
||
"return_token_ids=true."
|
||
)
|
||
if request.return_meta_info:
|
||
raise ValueError(
|
||
"return_meta_info is not supported with streaming. "
|
||
"Please set stream=false when using return_meta_info=true."
|
||
)
|
||
|
||
is_multimodal = self.tokenizer_manager.model_config.is_multimodal
|
||
|
||
# Process messages and apply chat template
|
||
processed_messages = self._process_messages(request, is_multimodal)
|
||
# Build sampling parameters
|
||
sampling_params = request.to_sampling_params(
|
||
stop=processed_messages.stop,
|
||
model_generation_config=self.default_sampling_params,
|
||
tool_call_constraint=processed_messages.tool_call_constraint,
|
||
renderer_handles_response_format=self.chat_encoding_spec == "kimi_k3",
|
||
)
|
||
set_request_reasoning_end_token_ids(
|
||
sampling_params, processed_messages.reasoning_end_token_ids
|
||
)
|
||
|
||
# Handle single vs multiple requests
|
||
if request.input_ids is not None:
|
||
prompt_kwargs = {"input_ids": processed_messages.prompt_ids}
|
||
else:
|
||
prompt_key, prompt_value = self._engine_prompt(
|
||
processed_messages, is_multimodal
|
||
)
|
||
prompt_kwargs = {prompt_key: prompt_value}
|
||
|
||
# Extract custom labels from raw request headers
|
||
custom_labels = self.extract_custom_labels(raw_request)
|
||
|
||
# Extract routed_dp_rank from header (has higher priority than body)
|
||
effective_routed_dp_rank = self.extract_routed_dp_rank_from_header(
|
||
raw_request, request.routed_dp_rank
|
||
)
|
||
|
||
# Resolve LoRA adapter from model parameter or explicit lora_path
|
||
lora_path = self._resolve_lora_path(request.model, request.lora_path)
|
||
img_max_dynamic_patch, vid_max_dynamic_patch = _extract_max_dynamic_patch(
|
||
request
|
||
)
|
||
adapted_request = GenerateReqInput(
|
||
**prompt_kwargs,
|
||
image_data=processed_messages.image_data,
|
||
video_data=processed_messages.video_data,
|
||
audio_data=processed_messages.audio_data,
|
||
sampling_params=sampling_params,
|
||
return_logprob=request.logprobs,
|
||
logprob_start_len=-1,
|
||
top_logprobs_num=request.top_logprobs or 0,
|
||
return_sampling_mask=request.return_sampling_mask,
|
||
sampling_logprobs_mode=request.sampling_logprobs_mode,
|
||
stream=request.stream,
|
||
return_text_in_logprobs=True,
|
||
modalities=processed_messages.modalities,
|
||
lora_path=lora_path,
|
||
bootstrap_host=request.bootstrap_host,
|
||
bootstrap_port=request.bootstrap_port,
|
||
bootstrap_room=request.bootstrap_room,
|
||
routed_dp_rank=effective_routed_dp_rank,
|
||
disagg_prefill_dp_rank=request.disagg_prefill_dp_rank,
|
||
return_hidden_states=request.return_hidden_states,
|
||
return_routed_experts=request.return_routed_experts,
|
||
routed_experts_start_len=request.routed_experts_start_len,
|
||
rid=request.rid,
|
||
session_id=request.session_id,
|
||
extra_key=request.extra_key,
|
||
cache_salt=request.cache_salt,
|
||
require_reasoning=processed_messages.require_reasoning,
|
||
priority=request.priority,
|
||
routing_key=self.extract_routing_key(raw_request),
|
||
custom_labels=custom_labels,
|
||
custom_logit_processor=request.custom_logit_processor,
|
||
images_config=getattr(request, "images_config", None),
|
||
video_config=_build_video_config(request),
|
||
image_max_dynamic_patch=img_max_dynamic_patch,
|
||
video_max_dynamic_patch=vid_max_dynamic_patch,
|
||
max_dynamic_patch=getattr(request, "max_dynamic_patch", None),
|
||
use_audio_in_video=getattr(request, "use_audio_in_video", False),
|
||
return_prompt_token_ids=(
|
||
request.return_prompt_token_ids
|
||
or request.return_token_ids
|
||
or self._should_return_input_ids(request)
|
||
),
|
||
)
|
||
if (
|
||
raw_request is not None
|
||
and envs.SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES.get()
|
||
):
|
||
apply_header_overrides(adapted_request, raw_request.headers)
|
||
|
||
return adapted_request, request
|
||
|
||
def _process_messages(
|
||
self, request: ChatCompletionRequest, is_multimodal: bool
|
||
) -> MessageProcessingResult:
|
||
"""Process chat messages and apply chat template"""
|
||
if self.default_chat_template_kwargs:
|
||
ctk = dict(request.chat_template_kwargs or {})
|
||
for k, v in self.default_chat_template_kwargs.items():
|
||
ctk.setdefault(k, v)
|
||
request.chat_template_kwargs = ctk
|
||
effort = ctk.get("reasoning_effort")
|
||
if effort is not None and request.reasoning_effort is None:
|
||
request.reasoning_effort = effort
|
||
|
||
normalize_hunyuan_reasoning_effort(
|
||
request, self.reasoning_parser, self.template_manager.reasoning_config
|
||
)
|
||
|
||
# GptOss model needs to keep special tokens for harmony parsing
|
||
if self.is_gpt_oss or self.is_gemma4:
|
||
request.skip_special_tokens = False
|
||
|
||
self._patch_reasoning_skip_special_tokens(request)
|
||
|
||
thinking_mode = self._get_reasoning_from_request(request)
|
||
# SGLang's ReasonerGrammarBackend owns the reasoning prefix
|
||
# when --reasoning-parser is configured, so builtin xgrammar
|
||
# tags must describe only the post-reasoning tool-call suffix.
|
||
xgrammar_reasoning = thinking_mode and (self.reasoning_parser is None)
|
||
tool_call_constraint = None
|
||
|
||
effective_tools = self._effective_tools(request)
|
||
glm_constraint = self.tool_call_parser == "glm47" and not any(
|
||
tool.function.strict for tool in effective_tools
|
||
)
|
||
if glm_constraint:
|
||
enable_thinking = (request.chat_template_kwargs or {}).get(
|
||
"enable_thinking"
|
||
)
|
||
parser = FunctionCallParser(request.tools or [], self.tool_call_parser)
|
||
tool_call_constraint = parser.get_structure_constraint(
|
||
request.tool_choice,
|
||
parallel_tool_calls=request.parallel_tool_calls,
|
||
thinking_mode=True
|
||
if enable_thinking is None
|
||
else bool(enable_thinking),
|
||
)
|
||
|
||
# Apply chat template and its stop strings
|
||
tools = None
|
||
tool_call_stop = None
|
||
required_parsed_natively = glm_constraint
|
||
if effective_tools and request.tool_choice != "none":
|
||
request.skip_special_tokens = False
|
||
if not isinstance(request.tool_choice, str):
|
||
tools = [
|
||
item.model_dump()
|
||
for item in request.tools or []
|
||
if item.function.name == request.tool_choice.function.name
|
||
] or None
|
||
elif request.tools:
|
||
tools = [item.model_dump() for item in request.tools]
|
||
if self.tool_call_parser and not glm_constraint:
|
||
parser = FunctionCallParser(
|
||
effective_tools,
|
||
self.tool_call_parser,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
)
|
||
tool_call_constraint = parser.get_structure_constraint(
|
||
request.tool_choice,
|
||
parallel_tool_calls=request.parallel_tool_calls,
|
||
thinking_mode=xgrammar_reasoning,
|
||
)
|
||
if (
|
||
tool_call_constraint is not None
|
||
and tool_call_constraint[0] == "structural_tag"
|
||
and self._reasoning_detector is not None
|
||
):
|
||
# Same ownership rule: the reasoning parser handles think
|
||
# tokens, so the grammar must not forbid them in free text.
|
||
strip_structural_tag_excludes(
|
||
tool_call_constraint[1],
|
||
(
|
||
self._reasoning_detector.think_start_token,
|
||
self._reasoning_detector.think_end_token,
|
||
),
|
||
)
|
||
required_parsed_natively = parser.detector.parses_required_natively()
|
||
if self.chat_encoding_spec == "kimi_k3":
|
||
tool_call_stop = parser.detector.eot_token
|
||
if (
|
||
tool_call_constraint is None
|
||
and not required_parsed_natively
|
||
and not (
|
||
self.chat_encoding_spec == "kimi_k3"
|
||
and self.tool_call_parser == "kimi_k3"
|
||
)
|
||
and (
|
||
request.tool_choice == "required"
|
||
or isinstance(request.tool_choice, ToolChoice)
|
||
)
|
||
):
|
||
json_schema = get_json_schema_constraint(
|
||
effective_tools,
|
||
request.tool_choice,
|
||
parallel_tool_calls=request.parallel_tool_calls,
|
||
)
|
||
tool_call_constraint = ("json_schema", json_schema)
|
||
|
||
# When input_ids are provided, skip template tokenization entirely;
|
||
# only stop tokens and tool_call_constraint are needed.
|
||
if request.input_ids is not None:
|
||
result = MessageProcessingResult(
|
||
prompt="",
|
||
prompt_ids=request.input_ids,
|
||
image_data=None,
|
||
audio_data=None,
|
||
video_data=None,
|
||
modalities=[],
|
||
stop=request.stop or [],
|
||
)
|
||
elif self.template_manager.chat_template_name is None:
|
||
result = self._apply_jinja_template(request, tools, is_multimodal)
|
||
else:
|
||
result = self._apply_conversation_template(request, is_multimodal)
|
||
|
||
if tool_call_stop is not None:
|
||
if isinstance(result.stop, str):
|
||
result.stop = [result.stop]
|
||
elif result.stop is None:
|
||
result.stop = []
|
||
else:
|
||
result.stop = list(result.stop)
|
||
if tool_call_stop not in result.stop:
|
||
result.stop.append(tool_call_stop)
|
||
|
||
result.tool_call_constraint = tool_call_constraint
|
||
result.require_reasoning = thinking_mode
|
||
result.skip_special_tokens = request.skip_special_tokens
|
||
if self.reasoning_parser == "k2_horizon" and thinking_mode:
|
||
parser = ReasoningParser(
|
||
model_type=self.reasoning_parser,
|
||
stream_reasoning=False,
|
||
force_reasoning=True,
|
||
request=request,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
)
|
||
token_ids = self.tokenizer_manager.tokenizer.encode(
|
||
parser.detector.think_end_token,
|
||
add_special_tokens=False,
|
||
)
|
||
if hasattr(token_ids, "tolist"):
|
||
token_ids = token_ids.tolist()
|
||
if (
|
||
not isinstance(token_ids, list)
|
||
or not token_ids
|
||
or any(
|
||
type(token_id) is not int or token_id < 0 for token_id in token_ids
|
||
)
|
||
):
|
||
raise ValueError(
|
||
"The selected K2 reasoning terminator could not be encoded"
|
||
)
|
||
result.reasoning_end_token_ids = list(token_ids)
|
||
return result
|
||
|
||
def _apply_jinja_template(
|
||
self,
|
||
request: ChatCompletionRequest,
|
||
tools: list[dict] | None,
|
||
is_multimodal: bool,
|
||
) -> MessageProcessingResult:
|
||
"""Apply Jinja chat template"""
|
||
prompt = ""
|
||
prompt_ids = []
|
||
decoded_prompt = None
|
||
openai_compatible_messages = []
|
||
image_data = []
|
||
video_data = []
|
||
audio_data = []
|
||
modalities = []
|
||
|
||
template_content_format = self.template_manager.jinja_template_content_format
|
||
|
||
# Try custom encoding first (override in subclass for custom renderers)
|
||
thinking_requested = (request.chat_template_kwargs or {}).get(
|
||
"thinking", envs.SGLANG_DEFAULT_THINKING.get()
|
||
)
|
||
thinking_mode = (
|
||
ThinkingMode.THINKING if thinking_requested else ThinkingMode.CHAT
|
||
)
|
||
messages = [msg.model_dump() for msg in request.messages]
|
||
for message in messages:
|
||
normalize_assistant_tool_call_arguments(
|
||
message, strict=self.chat_encoding_spec != "kimi_k3"
|
||
)
|
||
|
||
prompt_ids = self._encode_messages(
|
||
copy.deepcopy(messages),
|
||
request,
|
||
thinking_mode,
|
||
tools=tools,
|
||
)
|
||
|
||
if prompt_ids is not None:
|
||
if self.chat_encoding_spec in ("inkling", "kimi_k3"):
|
||
for message in request.messages:
|
||
msg_dict = message.model_dump()
|
||
if msg_dict.get("content") is None:
|
||
msg_dict["content"] = ""
|
||
process_content_for_template_format(
|
||
msg_dict,
|
||
"openai",
|
||
image_data,
|
||
video_data,
|
||
audio_data,
|
||
modalities,
|
||
)
|
||
elif self.chat_encoding_spec is not None:
|
||
# dsv4/dsv41/dsv32 encoding path
|
||
messages = copy.deepcopy(messages)
|
||
is_dsv41 = self.chat_encoding_spec == "dsv41"
|
||
for msg in messages:
|
||
if msg.get("content") is None:
|
||
msg["content"] = ""
|
||
|
||
# The V4.1 encoder consumes OpenAI parts lists itself; dsv4/dsv32
|
||
# are text-only, so their parts-list content is flattened first.
|
||
if not is_dsv41:
|
||
for i, msg in enumerate(messages):
|
||
if isinstance(msg.get("content"), list):
|
||
messages[i] = process_content_for_template_format(
|
||
msg, "string", [], [], [], []
|
||
)
|
||
|
||
for msg in messages:
|
||
processed_msg = process_content_for_template_format(
|
||
msg,
|
||
template_content_format,
|
||
image_data,
|
||
video_data,
|
||
audio_data,
|
||
modalities,
|
||
use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32",
|
||
)
|
||
msg.update(processed_msg)
|
||
|
||
# Handle continue_final_message: separate final assistant message
|
||
messages, assistant_prefix = self._handle_last_assistant_message(
|
||
messages, request
|
||
)
|
||
|
||
# An empty system message hosts the request tools; dsv41 renders a
|
||
# system token for it, so it only gets one when tools need the host.
|
||
if messages[0]["role"] != "system" and (request.tools or not is_dsv41):
|
||
messages.insert(0, {"role": "system", "content": ""})
|
||
if request.tools:
|
||
messages[0]["tools"] = [
|
||
(
|
||
chat_encoding.dsv41_tool_payload(tool)
|
||
if is_dsv41
|
||
else tool.model_dump()
|
||
)
|
||
for tool in request.tools
|
||
]
|
||
|
||
# Default encoding (dsv4/dsv41/dsv32)
|
||
if self.chat_encoding_spec == "dsv4":
|
||
effort_source = request.reasoning_effort
|
||
if effort_source is None:
|
||
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
|
||
if env_val:
|
||
effort_source = env_val
|
||
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
|
||
assert reasoning_effort_profile is not None
|
||
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
|
||
reasoning_effort_profile
|
||
]
|
||
v4_reasoning_effort = (
|
||
effort_source if effort_source in accepted_efforts else None
|
||
)
|
||
if request.task is not None:
|
||
encoding_dsv4.attach_task_to_last_user_message(
|
||
messages, request.task
|
||
)
|
||
real_input = encoding_dsv4.encode_messages(
|
||
messages,
|
||
thinking_mode=thinking_mode,
|
||
reasoning_effort=v4_reasoning_effort,
|
||
reasoning_effort_profile=reasoning_effort_profile,
|
||
)
|
||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
||
elif is_dsv41:
|
||
if request.task is not None:
|
||
encoding_dsv41.attach_task_to_last_user_message(
|
||
messages, request.task
|
||
)
|
||
real_input, media = encoding_dsv41.encode_messages(
|
||
messages,
|
||
thinking_mode=thinking_mode,
|
||
reasoning_effort=self._resolve_dsv41_reasoning_effort(
|
||
request.reasoning_effort
|
||
),
|
||
return_multi_modal_data=True,
|
||
)
|
||
if media["images"]:
|
||
if not is_multimodal:
|
||
raise ValueError("image input is not supported for this model")
|
||
image_data.extend(
|
||
ImageData(url=image["url"]) for image in media["images"]
|
||
)
|
||
tokenizer = self.tokenizer_manager.tokenizer
|
||
real_input = real_input.replace(
|
||
encoding_dsv41.IMAGE_PLACEHOLDER,
|
||
tokenizer.convert_ids_to_tokens(
|
||
self.tokenizer_manager.image_token_id
|
||
),
|
||
)
|
||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
||
else:
|
||
real_input = encoding_dsv32.encode_messages(
|
||
messages, thinking_mode=thinking_mode
|
||
)
|
||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
||
|
||
# Append assistant prefix if continue_final_message is enabled
|
||
if assistant_prefix:
|
||
prompt_ids = self._append_assistant_prefix_to_prompt_ids(
|
||
prompt_ids, assistant_prefix
|
||
)
|
||
else:
|
||
if self.template_manager.jinja_template_may_reorder_tool_results:
|
||
messages = self._canonicalize_tool_message_order(messages)
|
||
for msg_dict in copy.deepcopy(messages):
|
||
if msg_dict.get("content") is None:
|
||
msg_dict["content"] = ""
|
||
|
||
# Process content based on detected template format
|
||
processed_msg = process_content_for_template_format(
|
||
msg_dict,
|
||
template_content_format,
|
||
image_data,
|
||
video_data,
|
||
audio_data,
|
||
modalities,
|
||
)
|
||
|
||
processed_msg["content"] = normalize_tool_content(
|
||
processed_msg["role"], processed_msg.get("content")
|
||
)
|
||
|
||
openai_compatible_messages.append(processed_msg)
|
||
|
||
# Handle continue_final_message: separate final assistant message
|
||
openai_compatible_messages, assistant_prefix = (
|
||
self._handle_last_assistant_message(openai_compatible_messages, request)
|
||
)
|
||
|
||
extra_template_kwargs = {}
|
||
if request.reasoning_effort is not None:
|
||
extra_template_kwargs["reasoning_effort"] = request.reasoning_effort
|
||
if request.chat_template_kwargs:
|
||
extra_template_kwargs.update(request.chat_template_kwargs)
|
||
|
||
rc = self.template_manager.reasoning_config
|
||
if rc is not None and rc.effort_kwarg is not None:
|
||
if request.reasoning_effort == "low":
|
||
extra_template_kwargs.setdefault(rc.effort_kwarg, True)
|
||
elif request.reasoning_effort in ("medium", "high", "max"):
|
||
logger.warning(
|
||
"Model '%s' supports only 'low' reasoning effort; "
|
||
"requested '%s' treated as default thinking",
|
||
self.tokenizer_manager.served_model_name,
|
||
request.reasoning_effort,
|
||
)
|
||
|
||
# Split apply_chat_template(tokenize=True) into render + encode so we
|
||
# can skip add_special_tokens=False on tokenizers that don't auto-add
|
||
# specials (Kimi-like, OpenAI-chat analogue of #25265). Chat
|
||
# templates already include role/special tokens, so the encode must
|
||
# avoid double BOS on tokenizers that would add it.
|
||
encode_kwargs = (
|
||
{"add_special_tokens": False}
|
||
if self._tokenizer_auto_adds_specials
|
||
else {}
|
||
)
|
||
try:
|
||
prompt_ids, decoded_prompt = self._render_and_encode_chat_template(
|
||
openai_compatible_messages,
|
||
tools=tools,
|
||
template_kwargs=extra_template_kwargs,
|
||
encode_kwargs=encode_kwargs,
|
||
use_cache=is_multimodal,
|
||
)
|
||
except Exception:
|
||
# If the first attempt fails, try with flat function-only format.
|
||
# Some templates (e.g. Mistral) expect tools without the OpenAI wrapper.
|
||
tools = (
|
||
[t["function"] if "function" in t else t for t in tools]
|
||
if tools
|
||
else None
|
||
)
|
||
try:
|
||
prompt_ids, decoded_prompt = self._render_and_encode_chat_template(
|
||
openai_compatible_messages,
|
||
tools=tools,
|
||
template_kwargs=extra_template_kwargs,
|
||
encode_kwargs=encode_kwargs,
|
||
use_cache=is_multimodal,
|
||
)
|
||
except _CHAT_TEMPLATE_CLIENT_ERRORS as template_error:
|
||
# Template errors (e.g., from raise_exception in Jinja templates)
|
||
# and TypeError (e.g., tojson filter on Jinja2 Undefined variables)
|
||
# should be treated as client errors (400 BadRequest)
|
||
raise ValueError(str(template_error)) from template_error
|
||
|
||
# Append assistant prefix if continue_final_message is enabled
|
||
if assistant_prefix:
|
||
prompt_ids = self._append_assistant_prefix_to_prompt_ids(
|
||
prompt_ids, assistant_prefix
|
||
)
|
||
# The cached decode corresponds to prompt_ids before the prefix.
|
||
decoded_prompt = None
|
||
|
||
if is_multimodal:
|
||
prompt = (
|
||
decoded_prompt
|
||
if decoded_prompt is not None
|
||
else self.tokenizer_manager.tokenizer.decode(prompt_ids)
|
||
)
|
||
|
||
stop = request.stop
|
||
image_data = image_data if image_data else None
|
||
audio_data = audio_data if audio_data else None
|
||
video_data = video_data if video_data else None
|
||
modalities = modalities if modalities else []
|
||
return MessageProcessingResult(
|
||
prompt=prompt,
|
||
prompt_ids=prompt_ids,
|
||
image_data=image_data,
|
||
video_data=video_data,
|
||
audio_data=audio_data,
|
||
modalities=modalities,
|
||
stop=stop,
|
||
)
|
||
|
||
def _render_and_encode_chat_template(
|
||
self,
|
||
messages: List[Dict[str, Any]],
|
||
*,
|
||
tools: Optional[List[Dict]],
|
||
template_kwargs: Dict[str, Any],
|
||
encode_kwargs: Dict[str, Any],
|
||
use_cache: bool,
|
||
) -> tuple[List[int], Optional[str]]:
|
||
cache_key = None
|
||
if use_cache:
|
||
try:
|
||
# Key order is part of the key: templates render dicts in their given order.
|
||
cache_key = orjson.dumps(
|
||
(
|
||
getattr(
|
||
self.tokenizer_manager.tokenizer,
|
||
"chat_template",
|
||
None,
|
||
),
|
||
messages,
|
||
tools,
|
||
template_kwargs,
|
||
encode_kwargs,
|
||
),
|
||
)
|
||
except TypeError:
|
||
pass
|
||
|
||
if cache_key is not None:
|
||
cached = self._chat_template_cache.get(cache_key)
|
||
if cached is not None:
|
||
self._chat_template_cache.move_to_end(cache_key)
|
||
prompt_ids, decoded_prompt = cached
|
||
return list(prompt_ids), decoded_prompt
|
||
|
||
if self._prompt_text_round_trip_is_lossy:
|
||
# Re-encoding rendered text would drop the template's control tokens.
|
||
prompt_ids = self.tokenizer_manager.tokenizer.apply_chat_template(
|
||
messages,
|
||
tokenize=True,
|
||
add_generation_prompt=True,
|
||
tools=tools,
|
||
return_dict=False,
|
||
**template_kwargs,
|
||
)
|
||
else:
|
||
rendered_prompt = self.tokenizer_manager.tokenizer.apply_chat_template(
|
||
messages,
|
||
tokenize=False,
|
||
add_generation_prompt=True,
|
||
tools=tools,
|
||
return_dict=False,
|
||
**template_kwargs,
|
||
)
|
||
prompt_ids = self.tokenizer_manager.tokenizer.encode(
|
||
rendered_prompt, **encode_kwargs
|
||
)
|
||
decoded_prompt = (
|
||
self.tokenizer_manager.tokenizer.decode(prompt_ids)
|
||
if cache_key is not None
|
||
else None
|
||
)
|
||
|
||
if cache_key is not None:
|
||
self._chat_template_cache[cache_key] = (tuple(prompt_ids), decoded_prompt)
|
||
if len(self._chat_template_cache) > _CHAT_TEMPLATE_CACHE_MAX_SIZE:
|
||
self._chat_template_cache.popitem(last=False)
|
||
|
||
return prompt_ids, decoded_prompt
|
||
|
||
def _apply_conversation_template(
|
||
self,
|
||
request: ChatCompletionRequest,
|
||
is_multimodal: bool,
|
||
) -> MessageProcessingResult:
|
||
"""Apply conversation template"""
|
||
prompt = ""
|
||
prompt_ids = []
|
||
conv = generate_chat_conv(request, self.template_manager.chat_template_name)
|
||
|
||
# If we should continue the final assistant message, adjust the conversation.
|
||
if (
|
||
request.continue_final_message
|
||
and request.messages
|
||
and request.messages[-1].role == "assistant"
|
||
):
|
||
# Remove the auto-added blank assistant turn, if present.
|
||
if conv.messages and conv.messages[-1][1] is None:
|
||
conv.messages.pop()
|
||
# Rebuild the prompt from the conversation.
|
||
prompt = conv.get_prompt()
|
||
# Strip trailing stop tokens or separators that indicate end-of-assistant.
|
||
if isinstance(conv.stop_str, list):
|
||
for stop_token in conv.stop_str:
|
||
if prompt.endswith(stop_token):
|
||
prompt = prompt[: -len(stop_token)]
|
||
elif isinstance(conv.stop_str, str) and prompt.endswith(conv.stop_str):
|
||
prompt = prompt[: -len(conv.stop_str)]
|
||
if conv.sep and prompt.endswith(conv.sep):
|
||
prompt = prompt[: -len(conv.sep)]
|
||
if getattr(conv, "sep2", None) and prompt.endswith(conv.sep2):
|
||
prompt = prompt[: -len(conv.sep2)]
|
||
else:
|
||
prompt = conv.get_prompt()
|
||
if self._get_reasoning_from_request(request) and (
|
||
self._reasoning_detector is None
|
||
or not self._reasoning_detector.thinks_internally
|
||
):
|
||
# Models with thinks_internally=True think without a leading <think> token
|
||
prompt += "<think>" # Note(Xinyuan): hard code thinking token
|
||
|
||
image_data = conv.image_data if conv.image_data else None
|
||
video_data = conv.video_data if conv.video_data else None
|
||
audio_data = conv.audio_data if conv.audio_data else None
|
||
modalities = conv.modalities if conv.modalities else []
|
||
stop = copy.copy(conv.stop_str or [] if not request.ignore_eos else [])
|
||
|
||
if request.stop:
|
||
if isinstance(request.stop, str):
|
||
stop.append(request.stop)
|
||
else:
|
||
stop.extend(request.stop)
|
||
|
||
if not is_multimodal:
|
||
prompt_ids = self.tokenizer_manager.tokenizer.encode(prompt)
|
||
|
||
return MessageProcessingResult(
|
||
prompt=prompt,
|
||
prompt_ids=prompt_ids,
|
||
image_data=image_data,
|
||
video_data=video_data,
|
||
audio_data=audio_data,
|
||
modalities=modalities,
|
||
stop=stop,
|
||
)
|
||
|
||
async def _handle_streaming_request(
|
||
self,
|
||
adapted_request: GenerateReqInput,
|
||
request: ChatCompletionRequest,
|
||
raw_request: Request,
|
||
) -> StreamingResponse | ErrorResponse:
|
||
"""Handle streaming chat completion request"""
|
||
generator = self._generate_chat_stream(adapted_request, request, raw_request)
|
||
|
||
# Kick-start the generator to trigger validation before HTTP 200 is sent.
|
||
# If validation fails (e.g., context length exceeded), we can still return
|
||
# a proper HTTP 400 error response instead of streaming it as SSE payload.
|
||
try:
|
||
first_chunk = await generator.__anext__()
|
||
except ValueError as e:
|
||
return self.create_error_response(str(e))
|
||
|
||
async def prepend_first_chunk():
|
||
yield first_chunk
|
||
async for chunk in generator:
|
||
yield chunk
|
||
|
||
return StreamingResponse(
|
||
prepend_first_chunk(),
|
||
media_type="text/event-stream",
|
||
background=self.tokenizer_manager.create_abort_task(adapted_request),
|
||
)
|
||
|
||
async def _generate_chat_stream(
|
||
self,
|
||
adapted_request: GenerateReqInput,
|
||
request: ChatCompletionRequest,
|
||
raw_request: Request,
|
||
) -> AsyncGenerator[str, None]:
|
||
"""Generate streaming chat completion response"""
|
||
# Parsers for tool calls and reasoning
|
||
parser_dict = {}
|
||
reasoning_parser_dict = {}
|
||
|
||
# State tracking for streaming
|
||
is_firsts = {}
|
||
stream_offsets = {}
|
||
n_prev_tokens = {}
|
||
has_tool_calls = {}
|
||
finish_reasons = {}
|
||
|
||
# Usage tracking
|
||
prompt_tokens = {}
|
||
reasoning_tokens = {}
|
||
completion_tokens = {}
|
||
cached_tokens = {}
|
||
hidden_states = {}
|
||
routed_experts = {}
|
||
cached_tokens_details = {}
|
||
spec_tokens_details = {}
|
||
image_tokens = {}
|
||
audio_tokens = {}
|
||
video_tokens = {}
|
||
input_ids: list[int] | None = None
|
||
output_ids: dict[int, list[int]] = {}
|
||
|
||
stream_started = False
|
||
error_aborted = False
|
||
try:
|
||
include_usage, continuous_usage_stats = should_include_usage(
|
||
request.stream_options,
|
||
get_serving().stream_response_default_include_usage,
|
||
)
|
||
|
||
return_input_ids = self._should_return_input_ids(request)
|
||
return_output_ids = self._should_return_output_ids(request)
|
||
|
||
ids_framed = (
|
||
raw_request is not None
|
||
and raw_request.headers.get("x-sglext-ids-framed") == "1"
|
||
)
|
||
|
||
async for content in self.tokenizer_manager.generate_request(
|
||
adapted_request, raw_request
|
||
):
|
||
index = content.get("index", 0)
|
||
|
||
prompt_tokens[index] = self._reported_prompt_tokens(
|
||
content["meta_info"]
|
||
)
|
||
completion_tokens[index] = content["meta_info"].get(
|
||
"completion_tokens", 0
|
||
)
|
||
reasoning_tokens[index] = content["meta_info"].get(
|
||
"reasoning_tokens", 0
|
||
)
|
||
cached_tokens[index] = content["meta_info"].get("cached_tokens", 0)
|
||
hidden_states[index] = content["meta_info"].get("hidden_states", None)
|
||
routed_experts[index] = content["meta_info"].get("routed_experts", None)
|
||
cached_tokens_details[index] = content["meta_info"].get(
|
||
"cached_tokens_details", None
|
||
)
|
||
if request.return_spec_tokens_details:
|
||
spec_tokens_details[index] = spec_tokens_details_from_meta_info(
|
||
content["meta_info"]
|
||
)
|
||
image_tokens[index] = content["meta_info"].get("image_tokens", 0)
|
||
audio_tokens[index] = content["meta_info"].get("audio_tokens", 0)
|
||
video_tokens[index] = content["meta_info"].get("video_tokens", 0)
|
||
|
||
finish_reason = content["meta_info"].get("finish_reason", None)
|
||
finish_reason_type = finish_reason["type"] if finish_reason else None
|
||
|
||
if return_input_ids and input_ids is None:
|
||
# The prompt is the full, shared prompt (same across choices
|
||
# and constant across chunks), so capture it once.
|
||
chunk_input_ids = content.get("prompt_token_ids")
|
||
if chunk_input_ids is not None:
|
||
input_ids = list(chunk_input_ids)
|
||
|
||
if return_output_ids:
|
||
chunk_output_ids = content.get("output_ids")
|
||
if chunk_output_ids is not None:
|
||
if get_serving().incremental_streaming_output:
|
||
accumulated = output_ids.setdefault(index, [])
|
||
if finish_reason_type == "abort":
|
||
# The abort chunk re-sends the last token plus any coalesced deltas;
|
||
# keep only what brings the total up to completion_tokens.
|
||
keep = completion_tokens[index] - len(accumulated)
|
||
chunk_output_ids = chunk_output_ids[: max(keep, 0)]
|
||
accumulated.extend(chunk_output_ids)
|
||
else:
|
||
# Intermediate chunks share the live state.output_ids
|
||
# list; the final chunk is a stable copy.
|
||
output_ids[index] = chunk_output_ids
|
||
|
||
# Handle logprobs
|
||
choice_logprobs = None
|
||
if request.logprobs:
|
||
n_prev_token = n_prev_tokens.get(index, 0)
|
||
total_output_logprobs = content["meta_info"][
|
||
"output_token_logprobs_length"
|
||
]
|
||
if n_prev_token < total_output_logprobs:
|
||
choice_logprobs = self._process_streaming_logprobs(
|
||
content, n_prev_token, total_output_logprobs
|
||
).model_dump()
|
||
n_prev_tokens[index] = total_output_logprobs
|
||
|
||
# Track finish_reason for each index
|
||
if finish_reason_type:
|
||
# Abort with an explicit error status_code is a system error
|
||
# (timeout, OOM, validation): emit a streaming error chunk.
|
||
# A graceful abort (no status_code, e.g. user-initiated via
|
||
# /abort_request or session lifecycle cleanup) falls through
|
||
# to the normal chunk path, matching the non-stream behavior
|
||
# in tokenizer_manager._handle_abort_finish_reason.
|
||
if finish_reason_type == "abort" and isinstance(
|
||
finish_reason.get("status_code"), HTTPStatus
|
||
):
|
||
code = finish_reason["status_code"]
|
||
error = self.create_streaming_error_response(
|
||
finish_reason.get("message", "Generation aborted."),
|
||
code.name,
|
||
code.value,
|
||
)
|
||
yield f"data: {error}\n\n"
|
||
error_aborted = True
|
||
break
|
||
finish_reasons[index] = finish_reason
|
||
|
||
# First chunk with role
|
||
if is_firsts.get(index, True):
|
||
is_firsts[index] = False
|
||
yield build_sse_content(
|
||
chunk_id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
model=request.model,
|
||
index=index,
|
||
role="assistant",
|
||
content="",
|
||
)
|
||
stream_started = True
|
||
|
||
# Generate streaming content (override in subclass for custom behavior)
|
||
async for chunk in self._generate_stream_content(
|
||
content=content,
|
||
index=index,
|
||
request=request,
|
||
stream_offsets=stream_offsets,
|
||
reasoning_parser_dict=reasoning_parser_dict,
|
||
parser_dict=parser_dict,
|
||
has_tool_calls=has_tool_calls,
|
||
choice_logprobs=choice_logprobs,
|
||
finish_reason_type=finish_reason_type,
|
||
continuous_usage_stats=continuous_usage_stats,
|
||
prompt_tokens=prompt_tokens,
|
||
reasoning_tokens=reasoning_tokens,
|
||
completion_tokens=completion_tokens,
|
||
):
|
||
yield chunk
|
||
|
||
# Send finish_reason chunks for each index that completed
|
||
for idx, finish_reason_data in finish_reasons.items():
|
||
finish_reason_type = finish_reason_data["type"]
|
||
|
||
# Change finish_reason to "tool_calls" if we had tool calls and stopped naturally
|
||
final_finish_reason = finish_reason_type
|
||
if has_tool_calls.get(idx, False) and finish_reason_type == "stop":
|
||
final_finish_reason = "tool_calls"
|
||
|
||
matched_stop = finish_reason_data.get("matched")
|
||
yield build_sse_content(
|
||
chunk_id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
model=request.model,
|
||
index=idx,
|
||
finish_reason=final_finish_reason,
|
||
matched_stop=matched_stop,
|
||
)
|
||
|
||
# Send hidden states if requested
|
||
if request.return_hidden_states and hidden_states:
|
||
for index, choice_hidden_states in hidden_states.items():
|
||
if choice_hidden_states:
|
||
response_hidden_states = process_hidden_states_for_response(
|
||
choice_hidden_states, request.return_hidden_states
|
||
)
|
||
hidden_states_chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[
|
||
ChatCompletionResponseStreamChoice(
|
||
index=index,
|
||
delta=DeltaMessage(
|
||
hidden_states=response_hidden_states
|
||
),
|
||
finish_reason=None, # Hidden states don't need finish_reason
|
||
)
|
||
],
|
||
model=request.model,
|
||
)
|
||
yield f"data: {hidden_states_chunk.model_dump_json()}\n\n"
|
||
|
||
sglext_routed = None
|
||
if request.return_routed_experts and routed_experts:
|
||
sglext_routed = next(
|
||
(v for v in routed_experts.values() if v is not None), None
|
||
)
|
||
|
||
sglext_cached_tokens_details = None
|
||
if request.return_cached_tokens_details and cached_tokens_details:
|
||
first_details = next(
|
||
(v for v in cached_tokens_details.values() if v is not None), None
|
||
)
|
||
if first_details is not None:
|
||
sglext_cached_tokens_details = cached_tokens_details_from_dict(
|
||
first_details
|
||
)
|
||
|
||
sglext_spec_tokens_details = None
|
||
if request.return_spec_tokens_details and spec_tokens_details:
|
||
spec_details = [
|
||
spec_tokens_details[index]
|
||
for index in sorted(spec_tokens_details)
|
||
if spec_tokens_details[index] is not None
|
||
]
|
||
if spec_details:
|
||
sglext_spec_tokens_details = (
|
||
spec_details if request.n > 1 else spec_details[0]
|
||
)
|
||
|
||
# Omit token ids after an error abort.
|
||
sglext_input_ids = None
|
||
if return_input_ids and input_ids and not error_aborted:
|
||
sglext_input_ids = list(input_ids)
|
||
|
||
sglext_output_ids = None
|
||
if return_output_ids and output_ids and not error_aborted:
|
||
sglext_output_ids = [
|
||
list(output_ids.get(i, [])) for i in range(request.n)
|
||
]
|
||
|
||
sglext_full = SglExt(
|
||
routed_experts=sglext_routed,
|
||
cached_tokens_details=sglext_cached_tokens_details,
|
||
spec_tokens_details=sglext_spec_tokens_details,
|
||
input_ids=sglext_input_ids,
|
||
output_ids=sglext_output_ids,
|
||
)
|
||
sglext_non_ids, sglext_ids = sglext_full.split_ids()
|
||
|
||
if ids_framed:
|
||
# A named SSE event lets transit hops pick the ids out without parsing JSON;
|
||
# the other sglext fields keep the plain data-chunk shape.
|
||
if sglext_non_ids is not None:
|
||
sglext_chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[],
|
||
model=request.model,
|
||
sglext=sglext_non_ids,
|
||
)
|
||
yield f"data: {sglext_chunk.model_dump_json()}\n\n"
|
||
if sglext_ids is not None:
|
||
sglext_ids_chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[],
|
||
model=request.model,
|
||
sglext=sglext_ids,
|
||
)
|
||
yield f"event: sglext_ids\ndata: {sglext_ids_chunk.model_dump_json()}\n\n"
|
||
elif sglext_non_ids is not None or sglext_ids is not None:
|
||
sglext_chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[], # sglext is at response level
|
||
model=request.model,
|
||
sglext=sglext_full,
|
||
)
|
||
yield f"data: {sglext_chunk.model_dump_json()}\n\n"
|
||
|
||
# Additional usage chunk
|
||
if include_usage:
|
||
# Multimodal tokens are per-prompt (input side), so aggregate
|
||
# once per prompt (first choice), matching prompt/cached semantics.
|
||
total_image_tokens = sum(
|
||
tok for idx, tok in image_tokens.items() if idx % request.n == 0
|
||
)
|
||
total_audio_tokens = sum(
|
||
tok for idx, tok in audio_tokens.items() if idx % request.n == 0
|
||
)
|
||
total_video_tokens = sum(
|
||
tok for idx, tok in video_tokens.items() if idx % request.n == 0
|
||
)
|
||
usage = UsageProcessor.calculate_streaming_usage(
|
||
prompt_tokens,
|
||
reasoning_tokens,
|
||
completion_tokens,
|
||
cached_tokens=cached_tokens,
|
||
n_choices=request.n,
|
||
enable_cache_report=get_serving().enable_cache_report,
|
||
image_tokens=total_image_tokens,
|
||
audio_tokens=total_audio_tokens,
|
||
video_tokens=total_video_tokens,
|
||
)
|
||
usage_chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[], # Empty choices array as per OpenAI spec
|
||
model=request.model,
|
||
usage=usage,
|
||
)
|
||
yield f"data: {usage_chunk.model_dump_json()}\n\n"
|
||
|
||
except ValueError as e:
|
||
if not stream_started:
|
||
raise
|
||
error = self.create_streaming_error_response(str(e))
|
||
yield f"data: {error}\n\n"
|
||
|
||
yield "data: [DONE]\n\n"
|
||
|
||
async def _handle_non_streaming_request(
|
||
self,
|
||
adapted_request: GenerateReqInput,
|
||
request: ChatCompletionRequest,
|
||
raw_request: Request,
|
||
) -> ChatCompletionResponse | ErrorResponse | ORJSONResponse:
|
||
"""Handle non-streaming chat completion request"""
|
||
try:
|
||
ret = await self.tokenizer_manager.generate_request(
|
||
adapted_request, raw_request
|
||
).__anext__()
|
||
except ValueError as e:
|
||
return self.create_error_response(str(e))
|
||
|
||
if not isinstance(ret, list):
|
||
ret = [ret]
|
||
|
||
response = self._build_chat_response(
|
||
request,
|
||
ret,
|
||
int(time.time()),
|
||
)
|
||
|
||
return response
|
||
|
||
def _build_chat_response(
|
||
self,
|
||
request: ChatCompletionRequest,
|
||
ret: list[dict[str, Any]],
|
||
created: int,
|
||
) -> ChatCompletionResponse | ORJSONResponse:
|
||
"""Build chat completion response from generation results"""
|
||
if self.chat_encoding_spec == "kimi_k3":
|
||
ret = [
|
||
{
|
||
**item,
|
||
"meta_info": {
|
||
**item["meta_info"],
|
||
"prompt_tokens": self._reported_prompt_tokens(
|
||
item["meta_info"]
|
||
),
|
||
},
|
||
}
|
||
for item in ret
|
||
]
|
||
|
||
choices = []
|
||
|
||
# Build sglext at response level (from first ret_item, as these are per-request)
|
||
first_ret = ret[0]
|
||
routed_experts = (
|
||
None
|
||
if request.return_meta_info
|
||
else process_routed_experts_from_ret(first_ret, request)
|
||
)
|
||
cached_tokens_details = process_cached_tokens_details_from_ret(
|
||
first_ret, request
|
||
)
|
||
spec_details = [
|
||
detail
|
||
for detail in (
|
||
process_spec_tokens_details_from_ret(item, request) for item in ret
|
||
)
|
||
if detail is not None
|
||
]
|
||
spec_tokens_details = (
|
||
spec_details
|
||
if request.n > 1
|
||
else (spec_details[0] if spec_details else None)
|
||
)
|
||
input_ids = None
|
||
if self._should_return_input_ids(request) and "prompt_token_ids" in ret[0]:
|
||
input_ids = list(ret[0]["prompt_token_ids"])
|
||
output_ids = None
|
||
if self._should_return_output_ids(request):
|
||
output_ids = [list(ret_item["output_ids"]) for ret_item in ret]
|
||
response_sglext = None
|
||
if (
|
||
routed_experts
|
||
or cached_tokens_details
|
||
or spec_tokens_details
|
||
or input_ids is not None
|
||
or output_ids is not None
|
||
):
|
||
response_sglext = SglExt(
|
||
routed_experts=routed_experts,
|
||
cached_tokens_details=cached_tokens_details,
|
||
spec_tokens_details=spec_tokens_details,
|
||
input_ids=input_ids,
|
||
output_ids=output_ids,
|
||
)
|
||
|
||
for idx, ret_item in enumerate(ret):
|
||
# Process logprobs
|
||
choice_logprobs = None
|
||
if request.logprobs:
|
||
choice_logprobs = self._process_response_logprobs(ret_item)
|
||
|
||
# Handle hidden states
|
||
hidden_states = process_hidden_states_from_ret(ret_item, request)
|
||
|
||
finish_reason = ret_item["meta_info"]["finish_reason"]
|
||
|
||
text = self._decode_response(ret_item)
|
||
if isinstance(text, ErrorResponse):
|
||
return ORJSONResponse(content=text.model_dump(), status_code=text.code)
|
||
|
||
# Handle reasoning content
|
||
reasoning_text = None
|
||
if self.reasoning_parser and request.separate_reasoning:
|
||
force_reasoning = (
|
||
self.template_manager.force_reasoning
|
||
or self._get_reasoning_from_request(request)
|
||
)
|
||
try:
|
||
parser = ReasoningParser(
|
||
model_type=self.reasoning_parser,
|
||
stream_reasoning=False,
|
||
force_reasoning=force_reasoning,
|
||
request=request,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
tool_call_parser_active=self._tool_call_parsing_active(request),
|
||
)
|
||
reasoning_text, text = parser.parse_non_stream(text)
|
||
except Exception as e:
|
||
logger.error(f"Reasoning parsing error: {e}")
|
||
return self.create_error_response(
|
||
"Failed to parse reasoning content",
|
||
err_type="InternalServerError",
|
||
status_code=500,
|
||
)
|
||
|
||
# Handle tool calls
|
||
tool_calls = None
|
||
effective_tools = self._effective_tools(request)
|
||
if self._tool_call_parsing_active(request):
|
||
history_tool_calls_cnt = self._get_history_tool_calls_cnt(request)
|
||
tool_calls, text, finish_reason = self._process_tool_calls(
|
||
text,
|
||
effective_tools,
|
||
finish_reason,
|
||
request.tool_choice,
|
||
history_tool_calls_cnt,
|
||
)
|
||
|
||
# Extract prompt_token_ids if requested
|
||
choice_prompt_token_ids = (
|
||
ret_item.get("prompt_token_ids")
|
||
if request.return_prompt_token_ids or request.return_token_ids
|
||
else None
|
||
)
|
||
choice_token_ids = (
|
||
ret_item["output_ids"] if request.return_token_ids else None
|
||
)
|
||
|
||
choice_meta_info = (
|
||
ret_item["meta_info"] if request.return_meta_info else None
|
||
)
|
||
# NOTE: content should not be None but empty string to make sure retokenize consistency.
|
||
reasoning_text, tool_calls = self._get_parsed_response_fields(
|
||
reasoning_text, tool_calls
|
||
)
|
||
|
||
choice_data = ChatCompletionResponseChoice(
|
||
index=idx,
|
||
message=ChatMessage(
|
||
role="assistant",
|
||
content=text if text else "",
|
||
tool_calls=tool_calls,
|
||
reasoning_content=reasoning_text if reasoning_text else None,
|
||
),
|
||
logprobs=choice_logprobs,
|
||
finish_reason=finish_reason["type"] if finish_reason else None,
|
||
matched_stop=(
|
||
finish_reason["matched"]
|
||
if finish_reason and "matched" in finish_reason
|
||
else None
|
||
),
|
||
hidden_states=hidden_states,
|
||
prompt_token_ids=choice_prompt_token_ids,
|
||
response_token_ids=choice_token_ids,
|
||
meta_info=choice_meta_info,
|
||
)
|
||
choices.append(choice_data)
|
||
|
||
# Calculate usage. Multimodal tokens are per-prompt (input side), so
|
||
# aggregate once per prompt (stride by n), matching prompt/cached semantics.
|
||
image_tokens = sum(
|
||
ret[i]["meta_info"].get("image_tokens", 0)
|
||
for i in range(0, len(ret), request.n)
|
||
)
|
||
audio_tokens = sum(
|
||
ret[i]["meta_info"].get("audio_tokens", 0)
|
||
for i in range(0, len(ret), request.n)
|
||
)
|
||
video_tokens = sum(
|
||
ret[i]["meta_info"].get("video_tokens", 0)
|
||
for i in range(0, len(ret), request.n)
|
||
)
|
||
usage = UsageProcessor.calculate_response_usage(
|
||
ret,
|
||
n_choices=request.n,
|
||
enable_cache_report=get_serving().enable_cache_report,
|
||
image_tokens=image_tokens,
|
||
audio_tokens=audio_tokens,
|
||
video_tokens=video_tokens,
|
||
)
|
||
|
||
return ChatCompletionResponse(
|
||
id=ret[0]["meta_info"]["id"],
|
||
created=created,
|
||
model=request.model,
|
||
choices=choices,
|
||
usage=usage,
|
||
metadata=build_endpoint_weight_version_metadata(ret[0]["meta_info"]),
|
||
sglext=response_sglext,
|
||
)
|
||
|
||
def _process_response_logprobs(self, ret_item: dict[str, Any]) -> ChoiceLogprobs:
|
||
"""Process logprobs for non-streaming response"""
|
||
output_token_logprobs = ret_item["meta_info"]["output_token_logprobs"]
|
||
output_top_logprobs = ret_item["meta_info"].get("output_top_logprobs", None)
|
||
token_logprobs = self._build_token_logprobs_from_raw(
|
||
output_token_logprobs, output_top_logprobs
|
||
)
|
||
return ChoiceLogprobs(content=token_logprobs)
|
||
|
||
def _build_token_logprobs_from_raw(
|
||
self,
|
||
output_token_logprobs: list[Any],
|
||
output_top_logprobs: list[Any] | None,
|
||
) -> list[ChatCompletionTokenLogprob]:
|
||
"""Build OpenAI ChatCompletionTokenLogprob from the engine's raw
|
||
``(logprob, token_id, token_text)`` triples.
|
||
|
||
The engine always keeps the real token id in the triple; the token text
|
||
is a *detokenized display* string that loses fragmentary byte-level
|
||
tokens (e.g. the four single-byte BPE pieces of U+20BB7 decode to
|
||
U+FFFD). Recovering the OpenAI ``bytes`` field from the display text
|
||
therefore corrupts fragments; we recover the true bytes from the token
|
||
id through the GPT-2 byte decoder instead.
|
||
|
||
Only byte-level BPE tokenizers (GPT-2 family) use the byte decoder;
|
||
SentencePiece tokenizers (Mistral, Gemma) fall back to the display
|
||
text, so multi-byte characters like é are NOT corrupted to [233].
|
||
"""
|
||
tokenizer = self.tokenizer_manager.tokenizer
|
||
from sglang.srt.entrypoints.openai.utils import _is_byte_level_tokenizer
|
||
|
||
is_byte_level = _is_byte_level_tokenizer(tokenizer)
|
||
token_logprobs: list[ChatCompletionTokenLogprob] = []
|
||
|
||
for token_idx, item in enumerate(output_token_logprobs):
|
||
# item = (logprob, token_id, token_text)
|
||
logprob, token_id, token_text = item
|
||
if is_byte_level:
|
||
token_bytes = token_id_to_bytes(tokenizer, token_id)
|
||
else:
|
||
token_bytes = None
|
||
if token_bytes is None:
|
||
token_bytes = list((token_text or "").encode("utf-8"))
|
||
|
||
top_logprobs: list[TopLogprob] = []
|
||
if output_top_logprobs:
|
||
# Both callers pass rows aligned with output_token_logprobs.
|
||
if token_idx < len(output_top_logprobs):
|
||
top_row = output_top_logprobs[token_idx]
|
||
if top_row is not None:
|
||
for top_logprob, top_id, top_text in top_row:
|
||
if is_byte_level:
|
||
top_bytes = token_id_to_bytes(tokenizer, top_id)
|
||
else:
|
||
top_bytes = None
|
||
if top_bytes is None:
|
||
top_bytes = list((top_text or "").encode("utf-8"))
|
||
top_logprobs.append(
|
||
TopLogprob(
|
||
token=top_text or "",
|
||
bytes=top_bytes,
|
||
logprob=top_logprob,
|
||
)
|
||
)
|
||
|
||
token_logprobs.append(
|
||
ChatCompletionTokenLogprob(
|
||
token=token_text or "",
|
||
bytes=token_bytes,
|
||
logprob=logprob,
|
||
top_logprobs=top_logprobs,
|
||
)
|
||
)
|
||
|
||
return token_logprobs
|
||
|
||
def _process_tool_call_id(
|
||
self,
|
||
call_item: ToolCallItem,
|
||
history_tool_calls_cnt: int,
|
||
) -> str:
|
||
"""Process for generating a new and unique `tool_call_id`"""
|
||
if self.tool_call_parser == "kimi_k3":
|
||
return f"{call_item.name}:{history_tool_calls_cnt + call_item.tool_index}"
|
||
if self.tool_call_parser != "kimi_k2":
|
||
# A simple uuid is sufficient for all models except for Kimi-K2.
|
||
tool_call_id = f"call_{uuid.uuid4().hex[:24]}"
|
||
return tool_call_id
|
||
tool_call_id = (
|
||
f"functions.{call_item.name}:"
|
||
f"{history_tool_calls_cnt + call_item.tool_index}"
|
||
)
|
||
logger.debug(
|
||
f"Process tool call idx, parser: {self.tool_call_parser}, tool_call_id: {tool_call_id}, history_cnt: {history_tool_calls_cnt}"
|
||
)
|
||
return tool_call_id
|
||
|
||
def _process_tool_calls(
|
||
self,
|
||
text: str,
|
||
tools: list[Any],
|
||
finish_reason: dict[str, Any],
|
||
tool_choice: str | ToolChoice | None = None,
|
||
history_tool_calls_cnt: int = 0,
|
||
) -> ToolCallProcessingResult:
|
||
"""Process tool calls in the response"""
|
||
|
||
is_required = tool_choice == "required" or isinstance(tool_choice, ToolChoice)
|
||
|
||
# Try model-specific parser when output is in native format.
|
||
# For required/named: only use parser when structural_tag was used
|
||
# as constraint (mirrors the streaming path). For auto: always try.
|
||
if self.tool_call_parser:
|
||
parser = FunctionCallParser(
|
||
tools,
|
||
self.tool_call_parser,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
tool_choice=tool_choice,
|
||
)
|
||
detector_owns_format = parser.owns_tool_format()
|
||
should_try_parser = not is_required or detector_owns_format
|
||
if should_try_parser and parser.has_tool_call(text):
|
||
try:
|
||
text, call_info_list = parser.parse_non_stream(text)
|
||
if not call_info_list:
|
||
logger.warning(
|
||
"Tool call marker present but no complete call parsed "
|
||
"from %s output; dropping the incomplete call",
|
||
self.tool_call_parser,
|
||
)
|
||
logger.debug(
|
||
"Unparsed tool call output (%d chars): %r",
|
||
len(text),
|
||
text[:2000],
|
||
)
|
||
return ToolCallProcessingResult(None, text, finish_reason)
|
||
|
||
tool_calls = []
|
||
for index, call_info in enumerate(call_info_list):
|
||
tool_id = self._process_tool_call_id(
|
||
call_info, history_tool_calls_cnt
|
||
)
|
||
# Call ordinal, as in the streaming deltas;
|
||
# tool_index is the tool's position in the request.
|
||
tool_calls.append(
|
||
ToolCall(
|
||
id=tool_id,
|
||
index=index,
|
||
function=FunctionResponse(
|
||
name=call_info.name,
|
||
arguments=call_info.parameters,
|
||
),
|
||
)
|
||
)
|
||
if finish_reason["type"] == "stop":
|
||
finish_reason["type"] = "tool_calls"
|
||
finish_reason["matched"] = None
|
||
return ToolCallProcessingResult(tool_calls, text, finish_reason)
|
||
except Exception as e:
|
||
logger.error(f"Tool call parsing error: {e}")
|
||
return ToolCallProcessingResult(None, text, finish_reason)
|
||
|
||
if is_required and detector_owns_format:
|
||
logger.warning(
|
||
"Required tool call missing from %s output (%d chars)",
|
||
self.tool_call_parser,
|
||
len(text),
|
||
)
|
||
logger.debug("Unparsed required tool call output: %r", text[:2000])
|
||
return ToolCallProcessingResult(None, text, finish_reason)
|
||
|
||
# json_schema constraint → JSON array output for required/named
|
||
if is_required:
|
||
original_finish_type = finish_reason["type"]
|
||
if finish_reason["type"] == "stop":
|
||
finish_reason["type"] = "tool_calls"
|
||
finish_reason["matched"] = None
|
||
try:
|
||
tool_call_data = orjson.loads(text)
|
||
if isinstance(tool_call_data, dict):
|
||
tool_call_data = [tool_call_data]
|
||
if not isinstance(tool_call_data, list):
|
||
raise ValueError(
|
||
"expected a JSON array of tool calls, got "
|
||
f"{type(tool_call_data).__name__}"
|
||
)
|
||
if not all(
|
||
isinstance(tool, dict) and "name" in tool for tool in tool_call_data
|
||
):
|
||
raise ValueError(
|
||
"every tool call must be a JSON object with a 'name'"
|
||
)
|
||
tool_calls = []
|
||
for i, tool in enumerate(tool_call_data):
|
||
parameters = json.dumps(
|
||
tool.get("parameters", {}), ensure_ascii=False
|
||
)
|
||
call_info = ToolCallItem(
|
||
tool_index=i,
|
||
name=tool["name"],
|
||
parameters=parameters,
|
||
)
|
||
tool_id = self._process_tool_call_id(
|
||
call_info, history_tool_calls_cnt
|
||
)
|
||
tool_calls.append(
|
||
ToolCall(
|
||
id=tool_id,
|
||
index=i,
|
||
function=FunctionResponse(
|
||
name=tool["name"],
|
||
arguments=parameters,
|
||
),
|
||
)
|
||
)
|
||
return ToolCallProcessingResult(tool_calls, "", finish_reason)
|
||
except Exception as e:
|
||
logger.error(f"Tool call parsing error: {e}")
|
||
logger.debug("Unparsed required tool call output: %r", text[:2000])
|
||
finish_reason["type"] = original_finish_type
|
||
return ToolCallProcessingResult(None, text, finish_reason)
|
||
|
||
return ToolCallProcessingResult(None, text, finish_reason)
|
||
|
||
def _process_streaming_logprobs(
|
||
self,
|
||
content: dict[str, Any],
|
||
n_prev_token: int,
|
||
total_output_logprobs: int,
|
||
) -> ChoiceLogprobs:
|
||
"""Process logprobs for streaming response"""
|
||
output_token_logprobs = content["meta_info"]["output_token_logprobs"]
|
||
output_top_logprobs = content["meta_info"].get("output_top_logprobs", [])
|
||
if not get_serving().incremental_streaming_output:
|
||
output_token_logprobs = output_token_logprobs[
|
||
n_prev_token:total_output_logprobs
|
||
]
|
||
output_top_logprobs = output_top_logprobs[
|
||
n_prev_token:total_output_logprobs
|
||
]
|
||
token_logprobs = self._build_token_logprobs_from_raw(
|
||
output_token_logprobs, output_top_logprobs
|
||
)
|
||
return ChoiceLogprobs(content=token_logprobs)
|
||
|
||
def _process_reasoning_stream(
|
||
self,
|
||
index: int,
|
||
delta: str,
|
||
reasoning_parser_dict: dict[int, ReasoningParser],
|
||
content: dict[str, Any],
|
||
request: ChatCompletionRequest,
|
||
finish_reason_type: str | None = None,
|
||
) -> tuple[str | None, str]:
|
||
"""Process reasoning content in streaming response"""
|
||
if index not in reasoning_parser_dict:
|
||
is_force_reasoning = (
|
||
self.template_manager.force_reasoning
|
||
or self._get_reasoning_from_request(request)
|
||
)
|
||
reasoning_parser_dict[index] = ReasoningParser(
|
||
self.reasoning_parser,
|
||
request.stream_reasoning,
|
||
is_force_reasoning,
|
||
request,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
tool_call_parser_active=self._tool_call_parsing_active(request),
|
||
)
|
||
reasoning_parser = reasoning_parser_dict[index]
|
||
reasoning_text, normal_text = reasoning_parser.parse_stream_chunk(delta)
|
||
if finish_reason_type is not None and finish_reason_type != "abort":
|
||
end_reasoning_text, end_normal_text = reasoning_parser.parse_stream_end()
|
||
if end_reasoning_text:
|
||
reasoning_text = (reasoning_text or "") + end_reasoning_text
|
||
if end_normal_text:
|
||
normal_text = (normal_text or "") + end_normal_text
|
||
return reasoning_text, normal_text
|
||
|
||
def _get_history_tool_calls_cnt(self, request: ChatCompletionRequest) -> int:
|
||
"""Counts the number of tool calls in the request's message history.
|
||
|
||
NOTE: This method is only useful for models that include self-increasing
|
||
history tool call idx in tool calls id, such as kimi-k2
|
||
|
||
Args:
|
||
request: The chat completion request object.
|
||
|
||
Returns:
|
||
The total number of tool calls in the history, or 0 if not applicable.
|
||
"""
|
||
messages = getattr(request, "messages", [])
|
||
idx = 0
|
||
for msg in messages:
|
||
if msg.role == "assistant":
|
||
tool_calls = getattr(msg, "tool_calls", None)
|
||
idx += len(list(tool_calls)) if tool_calls is not None else 0 # noqa
|
||
return idx
|
||
|
||
def _patch_reasoning_skip_special_tokens(
|
||
self, request: ChatCompletionRequest
|
||
) -> None:
|
||
"""Keep parser-specific reasoning markers in the decoded text.
|
||
|
||
Some reasoning parsers rely on special-token delimiters that would be
|
||
removed during detokenization when ``skip_special_tokens=True``.
|
||
"""
|
||
if self.reasoning_parser == "apertus2509":
|
||
request.skip_special_tokens = False
|
||
if self.reasoning_parser == "kimi_k3" or self.chat_encoding_spec == "kimi_k3":
|
||
request.skip_special_tokens = False
|
||
|
||
if (
|
||
self.reasoning_parser in ["mistral"]
|
||
and request.reasoning_effort is not None
|
||
and request.reasoning_effort != "none"
|
||
):
|
||
request.skip_special_tokens = False
|
||
elif self.reasoning_parser == "inkling":
|
||
request.skip_special_tokens = False
|
||
elif self.reasoning_parser == "muse":
|
||
request.skip_special_tokens = False
|
||
|
||
def supports_native_reasoning_history(self) -> bool:
|
||
"""Whether the chat encoder takes history as ``reasoning_content`` rather
|
||
than via :meth:`wrap_reasoning_history`; see
|
||
:func:`chat_encoding.spec_owns_reasoning_history` for why.
|
||
"""
|
||
return chat_encoding.spec_owns_reasoning_history(self.chat_encoding_spec)
|
||
|
||
def wrap_reasoning_history(self, reasoning_text: str) -> str:
|
||
"""Wrap prior-turn reasoning in the detector's own start/end tokens.
|
||
|
||
Pulling the delimiters from the detector keeps adapters in lockstep
|
||
with any future parser that ships non-``<think>`` markers — Mistral's
|
||
``[THINK]``, Gemma4's ``think_start_self_label = "thought\\n"``, etc.
|
||
Falling back to a plain string is unsafe: it would let prior
|
||
thinking text reach a non-reasoning model as ordinary assistant
|
||
content, so the caller must surface this state, not paper over it.
|
||
"""
|
||
if self._reasoning_detector is None:
|
||
raise ValueError(
|
||
"Cannot rewrap thinking history: no reasoning detector is "
|
||
"configured for this model"
|
||
)
|
||
d = self._reasoning_detector
|
||
return (
|
||
f"{d.think_start_token}{d.think_start_self_label}"
|
||
f"{reasoning_text}\n{d.think_end_token}"
|
||
)
|
||
|
||
def _reasoning_default_mode(self) -> str | None:
|
||
if self._reasoning_detector is None:
|
||
return None
|
||
return self._reasoning_detector.reasoning_default
|
||
|
||
def _get_reasoning_toggle_param(self) -> str | None:
|
||
"""Resolve the chat-template kwarg that toggles reasoning, if any."""
|
||
config = self.template_manager.reasoning_config
|
||
if config is not None:
|
||
return config.toggle_param
|
||
|
||
mode = self._reasoning_default_mode()
|
||
if mode in ("thinking", "enable_thinking"):
|
||
return mode
|
||
if mode in ("explicit_thinking", "explicit_enable_thinking"):
|
||
return mode.replace("explicit_", "")
|
||
return None
|
||
|
||
def apply_reasoning_enabled(
|
||
self, request: ChatCompletionRequest, enabled: bool
|
||
) -> None:
|
||
"""Force the request into the requested reasoning-on/off mode.
|
||
|
||
Mirrors the read-side logic in ``_get_reasoning_from_request``;
|
||
the two must stay in sync. Always-on models cannot be disabled,
|
||
so explicit ``enabled=False`` raises rather than silently leaving
|
||
reasoning on.
|
||
"""
|
||
if not self.reasoning_parser:
|
||
if enabled:
|
||
raise ValueError(
|
||
"Anthropic thinking is not supported for models without "
|
||
"a reasoning parser"
|
||
)
|
||
return
|
||
|
||
if self.reasoning_parser == "hunyuan":
|
||
config = self.template_manager.reasoning_config
|
||
if config is not None and config.special_case == "hunyuan_effort":
|
||
request.reasoning_effort = "high" if enabled else "no_think"
|
||
else:
|
||
request.reasoning_effort = "medium" if enabled else "no_think"
|
||
return
|
||
|
||
if self.reasoning_parser == "iquest_q1":
|
||
request.chat_template_kwargs = {
|
||
**(request.chat_template_kwargs or {}),
|
||
"thinking": enabled,
|
||
"enable_thinking": enabled,
|
||
}
|
||
return
|
||
|
||
if self.reasoning_parser == "inkling":
|
||
# Effort-conditioned, not toggled: "none" (0.0) is the off switch.
|
||
if not enabled:
|
||
request.reasoning_effort = "none"
|
||
return
|
||
|
||
config = self.template_manager.reasoning_config
|
||
is_mistral = (config is not None and config.special_case == "mistral") or (
|
||
config is None and self._reasoning_default_mode() == "mistral"
|
||
)
|
||
if is_mistral:
|
||
request.reasoning_effort = "medium" if enabled else "none"
|
||
return
|
||
|
||
is_always_on = (config is not None and config.special_case == "always") or (
|
||
config is None and self._reasoning_default_mode() == "always"
|
||
)
|
||
if is_always_on:
|
||
if not enabled:
|
||
raise ValueError(
|
||
f"Reasoning parser '{self.reasoning_parser}' is always-on "
|
||
f"and cannot be disabled via Anthropic thinking"
|
||
)
|
||
return
|
||
|
||
toggle_param = self._get_reasoning_toggle_param()
|
||
# The read side (``_get_reasoning_from_request``) returns False
|
||
# whenever ``config.toggle_param is None`` OR
|
||
# ``config.default_enabled is None``. The write side must mirror
|
||
# both conditions: if ``default_enabled`` is unset we cannot
|
||
# actually honor an ``enabled=True`` request even when the toggle
|
||
# name itself is resolvable, so writing the kwarg would set up the
|
||
# template to emit reasoning tokens while the parser ignores them
|
||
# (literal ``<think>`` markers leak into the assistant text).
|
||
config = self.template_manager.reasoning_config
|
||
read_side_supported = toggle_param is not None and (
|
||
config is None or config.default_enabled is not None
|
||
)
|
||
if not read_side_supported:
|
||
if not enabled:
|
||
return
|
||
raise ValueError(
|
||
f"Anthropic thinking is not supported for reasoning parser "
|
||
f"'{self.reasoning_parser}'"
|
||
)
|
||
|
||
chat_template_kwargs = dict(request.chat_template_kwargs or {})
|
||
chat_template_kwargs[toggle_param] = enabled
|
||
request.chat_template_kwargs = chat_template_kwargs
|
||
|
||
def _get_reasoning_from_request(self, request: ChatCompletionRequest) -> bool:
|
||
"""Determine whether reasoning mode should be enabled for this request.
|
||
|
||
NOTE: This is predefined based on model's chat template
|
||
"""
|
||
if not self.reasoning_parser:
|
||
return False
|
||
|
||
if self.reasoning_parser == "iquest_q1":
|
||
return IQuestQ1ReasoningDetector.thinking_enabled(
|
||
request.chat_template_kwargs or {}
|
||
)
|
||
|
||
if self.reasoning_parser == "minimax-m3":
|
||
# M3 template prefills <mm:think> for thinking_mode=enabled, so it never
|
||
# appears in output and reasoning must be forced. Mirrors reasoning_parser.py.
|
||
return (request.chat_template_kwargs or {}).get(
|
||
"thinking_mode"
|
||
) == "enabled"
|
||
|
||
if self.reasoning_parser == "hunyuan":
|
||
config = self.template_manager.reasoning_config
|
||
if config is not None and config.special_case == "hunyuan_effort":
|
||
return request.reasoning_effort not in ("none", "no_think")
|
||
# Hy3-preview template emits no <think> when reasoning_effort is
|
||
# "no_think" / "none" / unset; forcing reasoning would route all
|
||
# output into reasoning_content.
|
||
return request.reasoning_effort not in (None, "none", "no_think")
|
||
|
||
config = self.template_manager.reasoning_config
|
||
if config is None:
|
||
# Fallback to parser-level defaults when template toggle config
|
||
# cannot be inferred (e.g., parser-only <think> templates).
|
||
mode = (
|
||
self._reasoning_detector.reasoning_default
|
||
if self._reasoning_detector is not None
|
||
else None
|
||
)
|
||
if mode is None:
|
||
return False
|
||
if mode == "always":
|
||
return True
|
||
if mode == "mistral":
|
||
return (
|
||
request.reasoning_effort is not None
|
||
and request.reasoning_effort != "none"
|
||
)
|
||
if mode in ("thinking", "enable_thinking"):
|
||
return (
|
||
not request.chat_template_kwargs
|
||
or request.chat_template_kwargs.get(mode) is not False
|
||
)
|
||
if mode in ("explicit_thinking", "explicit_enable_thinking"):
|
||
toggle = mode.replace("explicit_", "")
|
||
return (
|
||
request.chat_template_kwargs is not None
|
||
and request.chat_template_kwargs.get(toggle) is True
|
||
)
|
||
logger.warning(
|
||
"Unknown reasoning_default mode '%s', defaulting to reasoning disabled",
|
||
mode,
|
||
)
|
||
return False
|
||
|
||
if config.special_case == "always":
|
||
return True
|
||
|
||
if config.special_case == "mistral":
|
||
return (
|
||
request.reasoning_effort is not None
|
||
and request.reasoning_effort != "none"
|
||
)
|
||
|
||
if config.toggle_param is None or config.default_enabled is None:
|
||
return False
|
||
|
||
if config.default_enabled:
|
||
return (
|
||
not request.chat_template_kwargs
|
||
or request.chat_template_kwargs.get(config.toggle_param) is not False
|
||
)
|
||
return (
|
||
request.chat_template_kwargs is not None
|
||
and request.chat_template_kwargs.get(config.toggle_param) is True
|
||
)
|
||
|
||
async def _process_tool_call_stream(
|
||
self,
|
||
index: int,
|
||
delta: str,
|
||
parser_dict: dict[int, FunctionCallParser],
|
||
content: dict[str, Any],
|
||
request: ChatCompletionRequest,
|
||
has_tool_calls: dict[int, bool],
|
||
continuous_usage_stats: bool = False,
|
||
flush: bool = False,
|
||
):
|
||
"""Process tool calls in streaming response.
|
||
|
||
With flush=True (the terminal delta), the parser also drains text it
|
||
held back waiting for a marker that can no longer arrive.
|
||
"""
|
||
effective_tools = self._effective_tools(request)
|
||
if index not in parser_dict:
|
||
is_required = request.tool_choice == "required" or isinstance(
|
||
request.tool_choice, ToolChoice
|
||
)
|
||
# For required/named tool choice: use JsonArrayParser when the
|
||
# constrained output is plain JSON (detector doesn't support
|
||
# structural_tag or no parser configured). Use FunctionCallParser
|
||
# only when the detector supports structural_tag and will produce
|
||
# native format output.
|
||
if is_required:
|
||
use_native_parser = False
|
||
if self.tool_call_parser:
|
||
probe = FunctionCallParser(
|
||
tools=effective_tools,
|
||
tool_call_parser=self.tool_call_parser,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
tool_choice=request.tool_choice,
|
||
)
|
||
use_native_parser = probe.owns_tool_format()
|
||
if use_native_parser:
|
||
parser_dict[index] = probe
|
||
else:
|
||
parser_dict[index] = JsonArrayParser()
|
||
else:
|
||
parser_dict[index] = FunctionCallParser(
|
||
tools=effective_tools,
|
||
tool_call_parser=self.tool_call_parser,
|
||
tokenizer=self.tokenizer_manager.tokenizer,
|
||
)
|
||
|
||
parser = parser_dict[index]
|
||
|
||
# Handle both FunctionCallParser and JsonArrayParser
|
||
if isinstance(parser, JsonArrayParser):
|
||
result = parser.parse_streaming_increment(delta, effective_tools)
|
||
normal_text, calls = result.normal_text, result.calls
|
||
else:
|
||
normal_text, calls = parser.parse_stream_chunk(delta)
|
||
if flush:
|
||
end_text, end_calls = parser.parse_stream_end()
|
||
normal_text = (normal_text or "") + end_text
|
||
calls = list(calls) + end_calls
|
||
|
||
# Yield normal text
|
||
if normal_text:
|
||
choice_data = ChatCompletionResponseStreamChoice(
|
||
index=index,
|
||
delta=DeltaMessage(content=normal_text),
|
||
finish_reason=None,
|
||
)
|
||
chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[choice_data],
|
||
model=request.model,
|
||
)
|
||
|
||
# Add usage stats if continuous_usage_stats is enabled
|
||
if continuous_usage_stats:
|
||
prompt_tokens = self._reported_prompt_tokens(content["meta_info"])
|
||
completion_tokens = content["meta_info"].get("completion_tokens", 0)
|
||
reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0)
|
||
chunk.usage = UsageProcessor.calculate_token_usage(
|
||
prompt_tokens=prompt_tokens,
|
||
completion_tokens=completion_tokens,
|
||
reasoning_tokens=reasoning_tokens,
|
||
cached_tokens=self._continuous_usage_cached_details(content),
|
||
)
|
||
|
||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||
|
||
# Yield tool calls
|
||
history_tool_calls_cnt = self._get_history_tool_calls_cnt(request)
|
||
for call_item in calls:
|
||
# Mark that this choice has tool calls
|
||
has_tool_calls[index] = True
|
||
|
||
# Tool call ID should be generated only once per tool call
|
||
if call_item.name:
|
||
# First chunk: include ID and function name
|
||
tool_call_id = self._process_tool_call_id(
|
||
call_item, history_tool_calls_cnt
|
||
)
|
||
function_name = call_item.name
|
||
else:
|
||
# Subsequent chunks: null ID and name for argument deltas
|
||
tool_call_id = None
|
||
function_name = None
|
||
|
||
tool_call = ToolCall(
|
||
id=tool_call_id,
|
||
index=call_item.tool_index,
|
||
function=FunctionResponse(
|
||
name=function_name,
|
||
arguments=call_item.parameters,
|
||
),
|
||
)
|
||
|
||
choice_data = ChatCompletionResponseStreamChoice(
|
||
index=index,
|
||
delta=DeltaMessage(tool_calls=[tool_call]),
|
||
finish_reason=None,
|
||
)
|
||
chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[choice_data],
|
||
model=request.model,
|
||
)
|
||
|
||
# Add usage stats if continuous_usage_stats is enabled
|
||
if continuous_usage_stats:
|
||
prompt_tokens = self._reported_prompt_tokens(content["meta_info"])
|
||
completion_tokens = content["meta_info"].get("completion_tokens", 0)
|
||
reasoning_tokens = content["meta_info"].get("reasoning_tokens", 0)
|
||
chunk.usage = UsageProcessor.calculate_token_usage(
|
||
prompt_tokens=prompt_tokens,
|
||
completion_tokens=completion_tokens,
|
||
reasoning_tokens=reasoning_tokens,
|
||
cached_tokens=self._continuous_usage_cached_details(content),
|
||
)
|
||
|
||
yield f"data: {chunk.model_dump_json()}\n\n"
|
||
|
||
def _check_for_unstreamed_tool_args(
|
||
self,
|
||
parser: FunctionCallParser | JsonArrayParser,
|
||
content: dict[str, Any],
|
||
request: ChatCompletionRequest,
|
||
index: int,
|
||
) -> str | None:
|
||
"""
|
||
Check for any remaining tool call arguments that need to be streamed
|
||
when generation finishes. This ensures tool calls are properly completed
|
||
even if the model generates the final arguments in the last chunk.
|
||
"""
|
||
# Get the detector - either from FunctionCallParser or directly if json detector
|
||
detector = parser.detector if hasattr(parser, "detector") else parser
|
||
|
||
# Only check if we have tool calls and the detector has tracked data
|
||
if (
|
||
not hasattr(detector, "prev_tool_call_arr")
|
||
or not detector.prev_tool_call_arr
|
||
):
|
||
return None
|
||
|
||
if (
|
||
not hasattr(detector, "streamed_args_for_tool")
|
||
or not detector.streamed_args_for_tool
|
||
):
|
||
return None
|
||
|
||
# Get the last tool call that was being processed
|
||
tool_index = len(detector.prev_tool_call_arr) - 1
|
||
if tool_index < 0 or tool_index >= len(detector.streamed_args_for_tool):
|
||
return None
|
||
|
||
# Get expected vs actual arguments
|
||
expected_args = detector.prev_tool_call_arr[tool_index].get("arguments", {})
|
||
if isinstance(expected_args, str):
|
||
expected_call = expected_args
|
||
else:
|
||
expected_call = json.dumps(expected_args, ensure_ascii=False)
|
||
actual_call = detector.streamed_args_for_tool[tool_index]
|
||
|
||
# Check if there are remaining arguments to send
|
||
remaining_call = (
|
||
expected_call[len(actual_call) :]
|
||
if expected_call.startswith(actual_call)
|
||
else ""
|
||
)
|
||
|
||
if remaining_call:
|
||
# Create tool call chunk with remaining arguments
|
||
tool_call = ToolCall(
|
||
id=None, # No ID for argument deltas
|
||
index=tool_index,
|
||
function=FunctionResponse(
|
||
name=None, # No name for argument deltas
|
||
arguments=remaining_call,
|
||
),
|
||
)
|
||
|
||
choice_data = ChatCompletionResponseStreamChoice(
|
||
index=index,
|
||
delta=DeltaMessage(tool_calls=[tool_call]),
|
||
finish_reason=None, # Don't send finish_reason with this chunk
|
||
)
|
||
|
||
chunk = ChatCompletionStreamResponse(
|
||
id=content["meta_info"]["id"],
|
||
created=int(time.time()),
|
||
choices=[choice_data],
|
||
model=request.model,
|
||
)
|
||
|
||
return f"data: {chunk.model_dump_json()}\n\n"
|
||
|
||
return None
|