diff --git a/docs/docs/references/environment_variables.mdx b/docs/docs/references/environment_variables.mdx index c6fde0398b..686f20f150 100644 --- a/docs/docs/references/environment_variables.mdx +++ b/docs/docs/references/environment_variables.mdx @@ -757,12 +757,12 @@ SGLang supports various environment variables that can be used to configure its SGLANG_DSV4_REASONING_EFFORT - Default reasoning_effort for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts high and max; the official profile accepts low, high, and max. The profile is detected from the bundled encoder. Override it with --json-model-override-args '{"dsv4_reasoning_effort_profile":"official"}'. + Default reasoning_effort for DeepSeek-V4 when a request does not set it: a deepseek-recipe tier name. Unset means deepseek-recipe's default. "" SGLANG_DSV41_REASONING_EFFORT - Default reasoning_effort for the DeepSeek-V4.1 chat encoder when a request does not set it: one of low, high, xhigh, max, or an integer budget in [1, 100]. Unset means the encoder default high. + Default reasoning_effort for DeepSeek-V4.1 when a request does not set it: a deepseek-recipe tier name or an integer budget in [1, 100]. Unset means deepseek-recipe's default. "" diff --git a/python/pyproject.toml b/python/pyproject.toml index b974ad8655..ded56076c0 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -32,6 +32,7 @@ dependencies = [ "cuda-python>=13.0", "datasets", "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", + "deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'", "distro", "easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports "einops", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 964c9159a1..f2115fcbec 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -23,6 +23,7 @@ dependencies = [ "build", "compressed-tensors", "datasets", + "deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'", "easydict", "einops", "fastapi", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 42a4d83acf..c2ea03edab 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -24,6 +24,7 @@ dependencies = [ "compressed-tensors", "datasets", "decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", + "deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'", "easydict", "einops", "fastapi", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 849a49d8c4..31bfbbd822 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -28,6 +28,7 @@ runtime_base = [ "blobfile==3.0.0", "build", "datasets", + "deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'", "easydict", "einops", "fastapi", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index 2256a8417e..02e82aa496 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -24,6 +24,7 @@ dependencies = [ "build", "compressed-tensors", "datasets", + "deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'", "easydict", "einops", "fastapi", diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding.py b/python/sglang/srt/entrypoints/openai/chat_encoding.py index 63996a3143..a153eb8290 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding.py @@ -7,102 +7,7 @@ it here instead of re-deriving it from model architectures themselves. from __future__ import annotations -import ast -import logging -from pathlib import Path -from typing import Any, Dict, List, Optional, Union - -from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv41 - -logger = logging.getLogger(__name__) - -DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile" -_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py" -_MAX_DSV4_ENCODER_BYTES = 1 << 20 - - -def _detect_dsv4_reasoning_effort_profile( - model_path: str, revision: Optional[str] = None -) -> Optional[str]: - encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER - try: - if not encoder_path.is_file(): - from huggingface_hub import hf_hub_download - - encoder_path = Path( - hf_hub_download( - model_path, - _DSV4_REASONING_EFFORT_ENCODER, - revision=revision, - ) - ) - if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES: - return None - tree = ast.parse(encoder_path.read_text(encoding="utf-8")) - except Exception as error: - logger.debug( - "Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s", - encoder_path, - error, - ) - return None - - assignments = {} - for node in tree.body: - if isinstance(node, ast.Assign): - targets = node.targets - value = node.value - elif isinstance(node, ast.AnnAssign): - targets = [node.target] - value = node.value - else: - continue - - for target in targets: - if not isinstance(target, ast.Name): - continue - try: - assignments[target.id] = ast.literal_eval(value) - except (TypeError, ValueError): - continue - - prompts = assignments.get("REASONING_EFFORT_PROMPTS") - if ( - assignments.get("DEFAULT_REASONING_EFFORT") == "low" - and isinstance(prompts, dict) - and {"low", "high", "max"} <= prompts.keys() - ): - return "official" - if "REASONING_EFFORT_MAX" in assignments: - return "preview" - return None - - -def _validate_dsv4_reasoning_effort_profile(profile: str) -> str: - if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES: - raise ValueError( - f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; " - f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}" - ) - return profile - - -def resolve_dsv4_reasoning_effort_profile( - *, - model_path: str, - revision: Optional[str] = None, - override: Optional[str] = None, -) -> str: - if override is not None: - return _validate_dsv4_reasoning_effort_profile(override) - - return ( - _detect_dsv4_reasoning_effort_profile( - model_path=model_path, - revision=revision, - ) - or "preview" - ) +from typing import Any, Dict, List, Optional def is_deepseek_v41_arch(*, arch: str, model_type: str) -> bool: @@ -154,56 +59,6 @@ def resolve_chat_encoding_spec( return None -def parse_dsv41_reasoning_effort(value: Any) -> Union[str, int, None]: - """Map an API ``reasoning_effort`` onto what the V4.1 encoder accepts. - - An int budget only reaches here through ``chat_template_kwargs``; None - means unsupported, and the caller applies its default. - """ - if isinstance(value, bool): - return None - if isinstance(value, int): - return value if 1 <= value <= 100 else None - if isinstance(value, float): - return max(1, round(value * 100)) if 0.0 <= value <= 0.99 else None - if value in encoding_dsv41.REASONING_EFFORT_MAPPINGS: - return value - return None - - -_OPENAI_FUNCTION_FIELD_ORDER = ("name", "description", "parameters") - - -def dsv41_tool_payload(tool: Any) -> Dict[str, Any]: - """The tool dict the V4.1 encoder renders verbatim into the prompt. - - Only fields the client sent, in the OpenAI field order; pydantic would - otherwise add defaults (strict=false) and reorder keys by declaration. - """ - payload = tool.model_dump(exclude_unset=True, exclude_none=True) - function = dict(payload.get("function") or {}) - ordered = { - k: function.pop(k) for k in _OPENAI_FUNCTION_FIELD_ORDER if k in function - } - ordered.update(function) - payload["function"] = ordered - return payload - - -def default_dsv41_reasoning_effort_from_env(raw: Optional[str]) -> Union[str, int]: - """Parse ``SGLANG_DSV41_REASONING_EFFORT``; raises so a bad value fails at boot.""" - if raw is None or not raw.strip(): - return encoding_dsv41.DEFAULT_REASONING_EFFORT - value: Any = int(raw) if raw.strip().isdigit() else raw.strip() - effort = parse_dsv41_reasoning_effort(value) - if effort is None: - raise ValueError( - f"Invalid SGLANG_DSV41_REASONING_EFFORT={raw!r}; expected one of " - f"{list(encoding_dsv41.REASONING_EFFORT_MAPPINGS)} or an integer in [1, 100]" - ) - return effort - - def spec_supports_inline_system(spec: Optional[str]) -> bool: """Native encoders verified to render mid-conversation system messages in place.""" # K3's tokenizer renders in Python, so there is no Jinja source to probe. @@ -244,34 +99,30 @@ def encode_simple_chat( Minimal encode for offline tools: no tools, no multimodal content, no continue_final_message; the serving path keeps its full request-level - pipeline in ``serving_chat``. System-message handling matches - ``serving_chat``: dsv4/dsv32 get an empty one prepended, dsv41 does not - (it renders a system token even for empty content). + pipeline in ``serving_chat``. dsv32 gets an empty system message prepended, + as ``serving_chat`` does; DeepSeek-V4 / V4.1 render through deepseek-recipe. """ if spec == "inkling": from sglang.srt.parser.inkling_renderer import render_inkling_messages return render_inkling_messages(messages) - if spec in ("dsv4", "dsv32", "dsv41"): - if spec != "dsv41" and messages and messages[0]["role"] != "system": + if spec in ("dsv4", "dsv41"): + from sglang.srt.parser.deepseek_recipe_renderer import RecipeEncoder + + encoder = RecipeEncoder.from_hf_tokenizer(spec, tokenizer) + return encoder.encode( + {"messages": list(messages), "thinking": thinking_mode == "thinking"} + ) + + if spec == "dsv32": + from sglang.srt.entrypoints.openai import encoding_dsv32 + + if messages and messages[0]["role"] != "system": messages = [{"role": "system", "content": ""}] + list(messages) - if spec == "dsv4": - from sglang.srt.entrypoints.openai import encoding_dsv4 - - real_input = encoding_dsv4.encode_messages( - messages, thinking_mode=thinking_mode - ) - elif spec == "dsv41": - real_input = encoding_dsv41.encode_messages( - messages, thinking_mode=thinking_mode - ) - else: - from sglang.srt.entrypoints.openai import encoding_dsv32 - - real_input = encoding_dsv32.encode_messages( - messages, thinking_mode=thinking_mode - ) + real_input = encoding_dsv32.encode_messages( + messages, thinking_mode=thinking_mode + ) return tokenizer.encode(real_input) if getattr(tokenizer, "chat_template", None) is None: diff --git a/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py b/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py index fef98fb98c..99801968b5 100644 --- a/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py +++ b/python/sglang/srt/entrypoints/openai/chat_encoding_adapters.py @@ -70,6 +70,10 @@ class ChatEncodingAdapter(Protocol): """Whether media still has to be collected from the OpenAI messages after ``encode_messages`` returns prompt ids.""" + strict_tool_call_arguments: bool + """Whether history tool-call ``arguments`` must parse as JSON before + ``encode_messages`` sees them; False when the encoding serializes them itself.""" + @property def output(self) -> Optional[ChatEncodingOutputAdapter]: """None when the configured parsers leave output to the text parsers.""" @@ -81,19 +85,26 @@ class ChatEncodingAdapter(Protocol): messages: list[dict[str, Any]], request: ChatCompletionRequest, tools: Optional[list[dict]], - ) -> list[int]: ... + thinking: bool, + image_data: list, + ) -> list[int]: + """``thinking`` is the handler's thinking decision for ``request``. An + encoding that collects media itself appends the prompt's images to + ``image_data`` in placeholder order.""" + ... class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True): """Everything a builder may read; a builder takes only what its encoding needs. Consumers: - tokenizer: prompt encoding (Inkling). + tokenizer: prompt encoding (Inkling, DeepSeek-V4 / V4.1). reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing. hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id). - model_path, revision: checkpoint files (DeepSeek tokenizer.json and the - encoder file for effort-profile detection). - backend: selects the builder; "recipe" is an explicit opt-in. + model_path, revision: checkpoint files. + default_chat_template_kwargs: the workers' --default-chat-template-kwargs + (deepseek-recipe validates the default effort at startup). + backend: selects the builder; every spec has a "builtin" one. """ chat_encoding_spec: str @@ -104,6 +115,7 @@ class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True): hf_config: Any model_path: str revision: Optional[str] + default_chat_template_kwargs: Optional[dict[str, Any]] = None backend: EncodingBackend = "builtin" def require(self, field: str) -> Any: @@ -127,11 +139,32 @@ def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter: ) +def _build_deepseek_recipe( + ctx: ChatEncodingAdapterContext, +) -> Optional[ChatEncodingAdapter]: + if ctx.tokenizer is None: + return None # --skip-tokenizer-init: the server encodes no chat prompts. + from sglang.srt.entrypoints.openai.serving_deepseek_recipe import ( + DeepSeekRecipeChatEncodingAdapter, + ) + + return DeepSeekRecipeChatEncodingAdapter( + chat_encoding_spec=ctx.chat_encoding_spec, + tokenizer=ctx.tokenizer, + default_chat_template_kwargs=ctx.default_chat_template_kwargs, + ) + + _BUILDERS: dict[ str, - dict[EncodingBackend, Callable[[ChatEncodingAdapterContext], ChatEncodingAdapter]], + dict[ + EncodingBackend, + Callable[[ChatEncodingAdapterContext], Optional[ChatEncodingAdapter]], + ], ] = { "inkling": {"builtin": _build_inkling}, + "dsv4": {"builtin": _build_deepseek_recipe}, + "dsv41": {"builtin": _build_deepseek_recipe}, } diff --git a/python/sglang/srt/entrypoints/openai/encoding_dsv4.py b/python/sglang/srt/entrypoints/openai/encoding_dsv4.py deleted file mode 100644 index 52c8bebd77..0000000000 --- a/python/sglang/srt/entrypoints/openai/encoding_dsv4.py +++ /dev/null @@ -1,884 +0,0 @@ -# Adapted from the DeepSeek-V4 release reference implementation. -""" -DeepSeek-V4 Encoding - -A self-contained implementation for encoding/decoding DeepSeek-V4 chat messages -with tool calling, thinking mode, and quick instruction task support. -""" - -import copy -import json -import re -from typing import Any, Dict, List, Optional, Tuple, Union - -# ============================================================ -# Special Tokens -# ============================================================ - -bos_token: str = "<|begin▁of▁sentence|>" -eos_token: str = "<|end▁of▁sentence|>" -thinking_start_token: str = "" -thinking_end_token: str = "" -dsml_token: str = "|DSML|" - -USER_SP_TOKEN = "<|User|>" -ASSISTANT_SP_TOKEN = "<|Assistant|>" -LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" - -# Task special tokens for internal classification tasks -DS_TASK_SP_TOKENS = { - "action": "<|action|>", - "query": "<|query|>", - "authority": "<|authority|>", - "domain": "<|domain|>", - "title": "<|title|>", - "read_url": "<|read_url|>", -} -VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) - -# ============================================================ -# Templates -# ============================================================ - -system_msg_template: str = "{content}" -user_msg_template: str = "{content}" -latest_reminder_msg_template: str = "{content}" -assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token -assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}" -thinking_template: str = "{reasoning_content}" - -response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" -tool_call_template: str = ( - '<{dsml_token}invoke name="{name}">\n{arguments}\n' -) -tool_calls_template = ( - "<{dsml_token}{tc_block_name}>\n{tool_calls}\n" -) -tool_calls_block_name: str = "tool_calls" - -tool_output_template: str = "{content}" - -REASONING_EFFORT_PREVIEW_MAX = ( - "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" - "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" - "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" -) - -REASONING_EFFORT_OFFICIAL_MAX = ( - "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" - "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" - "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" -) - -REASONING_EFFORT_PROFILES = { - "preview": { - "high": "", - "max": REASONING_EFFORT_PREVIEW_MAX, - }, - "official": { - "low": "", - "high": REASONING_EFFORT_PREVIEW_MAX, - "max": REASONING_EFFORT_OFFICIAL_MAX, - }, -} - -TOOLS_TEMPLATE = """## Tools - -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: - -<{dsml_token}tool_calls> -<{dsml_token}invoke name="$TOOL_NAME"> -<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE -... - -<{dsml_token}invoke name="$TOOL_NAME2"> -... - - - -String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. - -If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. - -Otherwise, output directly after {thinking_end_token} with tool calls or final response. - -### Available Tool Schemas - -{tool_schemas} - -You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -""" - -# ============================================================ -# Utility Functions -# ============================================================ - - -def to_json(value: Any) -> str: - """Serialize a value to JSON string.""" - try: - return json.dumps(value, ensure_ascii=False) - except: - return json.dumps(value, ensure_ascii=True) - - -def tools_from_openai_format(tools): - """Extract function definitions from OpenAI-format tool list.""" - return [tool["function"] for tool in tools] - - -def tool_calls_from_openai_format(tool_calls): - """Convert OpenAI-format tool calls to internal format.""" - return [ - { - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - } - for tool_call in tool_calls - ] - - -def tool_calls_to_openai_format(tool_calls): - """Convert internal tool calls to OpenAI format.""" - return [ - { - "type": "function", - "function": { - "name": tool_call["name"], - "arguments": tool_call["arguments"], - }, - } - for tool_call in tool_calls - ] - - -def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str: - """ - Encode tool call arguments into DSML parameter format. - - Args: - tool_call: Dict with "name" and "arguments" keys. - - Returns: - DSML-formatted parameter string. - """ - p_dsml_template = '<{dsml_token}parameter name="{key}" string="{is_str}">{value}' - P_dsml_strs = [] - - raw_arguments = tool_call["arguments"] - arguments = ( - json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments - ) - if not isinstance(arguments, dict): - raise ValueError( - "Assistant tool call function.arguments must be a JSON object." - ) - - for k, v in arguments.items(): - p_dsml_str = p_dsml_template.format( - dsml_token=dsml_token, - key=k, - is_str="true" if isinstance(v, str) else "false", - value=v if isinstance(v, str) else to_json(v), - ) - P_dsml_strs.append(p_dsml_str) - - return "\n".join(P_dsml_strs) - - -def decode_dsml_to_arguments( - tool_name: str, tool_args: Dict[str, Tuple[str, str]] -) -> Dict[str, str]: - """ - Decode DSML parameters back to a tool call dict. - - Args: - tool_name: Name of the tool. - tool_args: Dict mapping param_name -> (value, is_string_flag). - - Returns: - Dict with "name" and "arguments" (JSON string) keys. - """ - - def _decode_value(key: str, value: str, string: str): - if string == "true": - value = to_json(value) - return f"{to_json(key)}: {value}" - - tool_args_json = ( - "{" - + ", ".join( - [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()] - ) - + "}" - ) - return dict(name=tool_name, arguments=tool_args_json) - - -def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: - """ - Render tool schemas into the system prompt format. - - Args: - tools: List of tool schema dicts (each with name, description, parameters). - - Returns: - Formatted tools section string. - """ - tools_json = [to_json(t) for t in tools] - - return TOOLS_TEMPLATE.format( - tool_schemas="\n".join(tools_json), - dsml_token=dsml_token, - thinking_start_token=thinking_start_token, - thinking_end_token=thinking_end_token, - ) - - -def find_last_user_index(messages: List[Dict[str, Any]]) -> int: - """Find the index of the last user/developer message.""" - last_user_index = -1 - for idx in range(len(messages) - 1, -1, -1): - if messages[idx].get("role") in ["user", "developer"]: - last_user_index = idx - break - return last_user_index - - -def attach_task_to_last_user_message(messages: List[Dict[str, Any]], task: str) -> None: - """Set `task` on the most recent user/developer message; raise if none exists.""" - idx = find_last_user_index(messages) - if idx == -1: - raise ValueError( - "`task` requires at least one message with role='user' or 'developer'." - ) - messages[idx]["task"] = task - - -# ============================================================ -# Message Rendering -# ============================================================ - - -def render_message( - index: int, - messages: List[Dict[str, Any]], - thinking_mode: str, - drop_thinking: bool = True, - reasoning_effort: Optional[str] = None, - reasoning_effort_profile: str = "preview", -) -> str: - """ - Render a single message at the given index into its encoded string form. - - This is the core function that converts each message in the conversation - into the DeepSeek-V4 format. - - Args: - index: Index of the message to render. - messages: Full list of messages in the conversation. - thinking_mode: Either "chat" or "thinking". - drop_thinking: Whether to drop reasoning content from earlier turns. - reasoning_effort: Optional reasoning effort level. The preview profile accepts - "high" and "max"; the official profile accepts "low", "high", and "max". - reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official"). - - Returns: - Encoded string for this message. - """ - assert 0 <= index < len(messages) - assert thinking_mode in [ - "chat", - "thinking", - ], f"Invalid thinking_mode `{thinking_mode}`" - - prompt = "" - msg = messages[index] - last_user_idx = find_last_user_index(messages) - - role = msg.get("role") - content = msg.get("content") - tools = msg.get("tools") - response_format = msg.get("response_format") - tool_calls = msg.get("tool_calls") - reasoning_content = msg.get("reasoning_content") - wo_eos = msg.get("wo_eos", False) - - if tools: - tools = tools_from_openai_format(tools) - if tool_calls: - tool_calls = tool_calls_from_openai_format(tool_calls) - - if reasoning_effort_profile not in REASONING_EFFORT_PROFILES: - raise ValueError( - f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; " - f"expected one of {list(REASONING_EFFORT_PROFILES)}" - ) - effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile] - if reasoning_effort is None: - reasoning_effort = "low" if reasoning_effort_profile == "official" else "high" - if reasoning_effort not in effort_prompts: - raise ValueError( - f"Invalid reasoning effort {reasoning_effort!r} for profile " - f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}" - ) - if index == 0 and thinking_mode == "thinking": - prompt += effort_prompts[reasoning_effort] - - if role == "system": - prompt += system_msg_template.format(content=content or "") - if tools: - prompt += "\n\n" + render_tools(tools) - if response_format: - prompt += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - elif role == "developer": - assert content, f"Invalid message for role `{role}`: {msg}" - - content_developer = USER_SP_TOKEN - content_developer += content - - if tools: - content_developer += "\n\n" + render_tools(tools) - if response_format: - content_developer += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - prompt += user_msg_template.format(content=content_developer) - - elif role == "user": - prompt += USER_SP_TOKEN - - # Handle content blocks (tool results mixed with text) - content_blocks = msg.get("content_blocks") - if content_blocks: - parts = [] - for block in content_blocks: - block_type = block.get("type") - if block_type == "text": - parts.append(block.get("text", "")) - elif block_type == "tool_result": - tool_content = block.get("content", "") - if isinstance(tool_content, list): - text_parts = [] - for b in tool_content: - if b.get("type") == "text": - text_parts.append(b.get("text", "")) - else: - text_parts.append(f"[Unsupported {b.get('type')}]") - tool_content = "\n\n".join(text_parts) - parts.append(tool_output_template.format(content=tool_content)) - else: - parts.append(f"[Unsupported {block_type}]") - prompt += "\n\n".join(parts) - else: - prompt += content or "" - - elif role == "latest_reminder": - prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format( - content=content - ) - - elif role == "tool": - raise NotImplementedError( - "deepseek_v4 merges tool messages into user; please preprocess with merge_tool_messages()" - ) - - elif role == "assistant": - thinking_part = "" - tc_content = "" - - if tool_calls: - tc_list = [ - tool_call_template.format( - dsml_token=dsml_token, - name=tc.get("name"), - arguments=encode_arguments_to_dsml(tc), - ) - for tc in tool_calls - ] - tc_content += "\n\n" + tool_calls_template.format( - dsml_token=dsml_token, - tool_calls="\n".join(tc_list), - tc_block_name=tool_calls_block_name, - ) - - summary_content = content or "" - rc = reasoning_content or "" - - # Check if previous message has a task - if so, this is a task output (no thinking) - prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None - - if thinking_mode == "thinking" and not prev_has_task: - if not drop_thinking or index > last_user_idx: - thinking_part = ( - thinking_template.format(reasoning_content=rc) + thinking_end_token - ) - else: - thinking_part = "" - - if wo_eos: - prompt += assistant_msg_wo_eos_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - prompt += assistant_msg_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - raise NotImplementedError(f"Unknown role: {role}") - - # Append transition tokens based on what follows - if index + 1 < len(messages) and messages[index + 1].get("role") not in [ - "assistant", - "latest_reminder", - ]: - return prompt - - task = messages[index].get("task") - if task is not None: - # Task special token for internal classification tasks - assert task in VALID_TASKS, ( - f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}" - ) - task_sp_token = DS_TASK_SP_TOKENS[task] - - if task != "action": - # Non-action tasks: append task sp token directly after the message - prompt += task_sp_token - else: - # Action task: append Assistant + thinking token + action sp token - prompt += ASSISTANT_SP_TOKEN - prompt += ( - thinking_end_token - if thinking_mode != "thinking" - else thinking_start_token - ) - prompt += task_sp_token - - elif messages[index].get("role") in ["user", "developer"]: - # Normal generation: append Assistant + thinking token - prompt += ASSISTANT_SP_TOKEN - if not drop_thinking and thinking_mode == "thinking": - prompt += thinking_start_token - elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx: - prompt += thinking_start_token - else: - prompt += thinking_end_token - - return prompt - - -# ============================================================ -# Preprocessing -# ============================================================ - - -def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Merge tool messages into the preceding user message using content_blocks format. - - DeepSeek-V4 does not have a standalone "tool" role; instead, tool results - are encoded as blocks within user messages. - - This function converts a standard OpenAI-format conversation (with separate - "tool" role messages) into V4 format where tool results are merged into - user messages. - - Args: - messages: List of message dicts in OpenAI format. - - Returns: - Processed message list with tool messages merged into user messages. - """ - merged: List[Dict[str, Any]] = [] - - for msg in messages: - msg = copy.deepcopy(msg) - role = msg.get("role") - - if role == "tool": - # Convert tool message to a user message with tool_result block - tool_block = { - "type": "tool_result", - "tool_use_id": msg.get("tool_call_id", ""), - "content": msg.get("content", ""), - } - # Merge into previous message if it's already a user (merged tool) - if ( - merged - and merged[-1].get("role") == "user" - and "content_blocks" in merged[-1] - ): - merged[-1]["content_blocks"].append(tool_block) - else: - merged.append( - { - "role": "user", - "content_blocks": [tool_block], - } - ) - elif role == "user": - text_block = {"type": "text", "text": msg.get("content", "")} - if ( - merged - and merged[-1].get("role") == "user" - and "content_blocks" in merged[-1] - and merged[-1].get("task") is None - ): - merged[-1]["content_blocks"].append(text_block) - else: - new_msg = { - "role": "user", - "content": msg.get("content", ""), - "content_blocks": [text_block], - } - # Preserve extra fields (task, wo_eos, mask, etc.) - for key in ("task", "wo_eos", "mask"): - if key in msg: - new_msg[key] = msg[key] - merged.append(new_msg) - else: - merged.append(msg) - - return merged - - -def sort_tool_results_by_call_order( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - """ - Sort tool_result blocks within user messages by the order of tool_calls - in the preceding assistant message. - - Args: - messages: Preprocessed message list (after merge_tool_messages). - - Returns: - Message list with sorted tool result blocks. - """ - last_tool_call_order: Dict[str, int] = {} - - for msg in messages: - role = msg.get("role") - if role == "assistant" and msg.get("tool_calls"): - last_tool_call_order = {} - for idx, tc in enumerate(msg["tool_calls"]): - tc_id = tc.get("id") or tc.get("function", {}).get("id", "") - if tc_id: - last_tool_call_order[tc_id] = idx - - elif role == "user" and msg.get("content_blocks"): - tool_blocks = [ - b for b in msg["content_blocks"] if b.get("type") == "tool_result" - ] - if len(tool_blocks) > 1 and last_tool_call_order: - sorted_blocks = sorted( - tool_blocks, - key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0), - ) - sorted_idx = 0 - new_blocks = [] - for block in msg["content_blocks"]: - if block.get("type") == "tool_result": - new_blocks.append(sorted_blocks[sorted_idx]) - sorted_idx += 1 - else: - new_blocks.append(block) - msg["content_blocks"] = new_blocks - - return messages - - -# ============================================================ -# Main Encoding Function -# ============================================================ - - -def encode_messages( - messages: List[Dict[str, Any]], - thinking_mode: str, - context: Optional[List[Dict[str, Any]]] = None, - drop_thinking: bool = True, - add_default_bos_token: bool = True, - reasoning_effort: Optional[str] = None, - reasoning_effort_profile: str = "preview", -) -> str: - """ - Encode a list of messages into the DeepSeek-V4 prompt format. - - This is the main entry point for encoding conversations. It handles: - - BOS token insertion - - Thinking mode with optional reasoning content dropping - - Tool message merging into user messages - - Multi-turn conversation context - - Args: - messages: List of message dicts to encode. - thinking_mode: Either "chat" or "thinking". - context: Optional preceding context messages (already encoded prefix). - drop_thinking: If True, drop reasoning_content from earlier assistant turns - (only keep reasoning for messages after the last user message). - add_default_bos_token: Whether to prepend BOS token at conversation start. - reasoning_effort: Optional reasoning effort level. The preview profile accepts - "high" and "max"; the official profile accepts "low", "high", and "max". - reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official"). - - Returns: - The encoded prompt string. - """ - context = context if context else [] - - # Preprocess: merge tool messages and sort tool results - messages = merge_tool_messages(messages) - messages = sort_tool_results_by_call_order(context + messages)[len(context) :] - if context: - context = merge_tool_messages(context) - context = sort_tool_results_by_call_order(context) - - full_messages = context + messages - - prompt = bos_token if add_default_bos_token and len(context) == 0 else "" - - # Resolve drop_thinking: if any message has tools defined, don't drop thinking - effective_drop_thinking = drop_thinking - if any(m.get("tools") for m in full_messages): - effective_drop_thinking = False - - if thinking_mode == "thinking" and effective_drop_thinking: - full_messages = _drop_thinking_messages(full_messages) - # After dropping, recalculate how many messages to render - # (context may have shrunk too) - num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) - context_len = len(full_messages) - num_to_render - else: - num_to_render = len(messages) - context_len = len(context) - - for idx in range(num_to_render): - prompt += render_message( - idx + context_len, - full_messages, - thinking_mode=thinking_mode, - drop_thinking=effective_drop_thinking, - reasoning_effort=reasoning_effort, - reasoning_effort_profile=reasoning_effort_profile, - ) - - return prompt - - -def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """ - Drop reasoning_content and non-essential messages before the last user message. - - Behavior: - - Messages with role in ["user", "system", "tool", "latest_reminder"] are always kept. - - Messages at or after the last user index are always kept. - - Assistant messages before the last user get reasoning_content removed. - - Developer messages before the last user are dropped entirely. - """ - last_user_idx = find_last_user_index(messages) - result = [] - keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} - - for idx, msg in enumerate(messages): - role = msg.get("role") - if role in keep_roles or idx >= last_user_idx: - result.append(msg) - elif role == "assistant": - msg = copy.copy(msg) - msg.pop("reasoning_content", None) - result.append(msg) - # developer and other roles before last_user_idx are dropped - - return result - - -# ============================================================ -# Parsing (Decoding model output) -# ============================================================ - - -def _read_until_stop( - index: int, text: str, stop: List[str] -) -> Tuple[int, str, Optional[str]]: - """ - Read text from index until one of the stop strings is found. - - Returns: - Tuple of (new_index, content_before_stop, matched_stop_string_or_None). - """ - min_pos = len(text) - matched_stop = None - - for s in stop: - pos = text.find(s, index) - if pos != -1 and pos < min_pos: - min_pos = pos - matched_stop = s - - if matched_stop: - content = text[index:min_pos] - return min_pos + len(matched_stop), content, matched_stop - else: - content = text[index:] - return len(text), content, None - - -def parse_tool_calls( - index: int, text: str -) -> Tuple[int, Optional[str], List[Dict[str, str]]]: - """ - Parse DSML tool calls from text starting at the given index. - - Args: - index: Starting position in text. - text: The full text to parse. - - Returns: - Tuple of (new_index, last_stop_token, list_of_tool_call_dicts). - Each tool call dict has "name" and "arguments" keys. - """ - tool_calls: List[Dict[str, Any]] = [] - stop_token = None - tool_calls_end_token = f"" - - while index < len(text): - index, _, stop_token = _read_until_stop( - index, text, [f"<{dsml_token}invoke", tool_calls_end_token] - ) - if _ != ">\n": - raise ValueError(f"Tool call format error: expected '>\\n' but got '{_}'") - - if stop_token == tool_calls_end_token: - break - - if stop_token is None: - raise ValueError("Missing special token in tool calls") - - index, tool_name_content, stop_token = _read_until_stop( - index, text, [f"<{dsml_token}parameter", f"\n$', tool_name_content, flags=re.DOTALL - ) - if len(p_tool_name) != 1: - raise ValueError(f"Tool name format error: '{tool_name_content}'") - tool_name = p_tool_name[0] - - tool_args: Dict[str, Tuple[str, str]] = {} - while stop_token == f"<{dsml_token}parameter": - index, param_content, stop_token = _read_until_stop( - index, text, [f"/{dsml_token}parameter"] - ) - - param_kv = re.findall( - r'^ name="(.*?)" string="(true|false)">(.*?)<$', - param_content, - flags=re.DOTALL, - ) - if len(param_kv) != 1: - raise ValueError(f"Parameter format error: '{param_content}'") - param_name, string, param_value = param_kv[0] - - if param_name in tool_args: - raise ValueError(f"Duplicate parameter name: '{param_name}'") - tool_args[param_name] = (param_value, string) - - index, content, stop_token = _read_until_stop( - index, text, [f"<{dsml_token}parameter", f"\n": - raise ValueError( - f"Parameter format error: expected '>\\n' but got '{content}'" - ) - - tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args) - tool_calls.append(tool_call) - - return index, stop_token, tool_calls - - -def parse_message_from_completion_text(text: str, thinking_mode: str) -> Dict[str, Any]: - """ - Parse a model completion text into a structured assistant message. - - This function takes the raw text output from the model (a single assistant turn) - and extracts: - - reasoning_content (thinking block) - - content (summary/response) - - tool_calls (if any) - - NOTE: This function is designed to parse only correctly formatted strings and - will raise ValueError for malformed output. - - Args: - text: The raw completion text (including EOS token). - thinking_mode: Either "chat" or "thinking". - - Returns: - Dict with keys: "role", "content", "reasoning_content", "tool_calls". - tool_calls are in OpenAI format. - """ - summary_content, reasoning_content, tool_calls = "", "", [] - index, stop_token = 0, None - tool_calls_start_token = f"\n\n<{dsml_token}{tool_calls_block_name}" - - is_thinking = thinking_mode == "thinking" - is_tool_calling = False - - if is_thinking: - index, content_delta, stop_token = _read_until_stop( - index, text, [thinking_end_token, tool_calls_start_token] - ) - reasoning_content = content_delta - assert stop_token == thinking_end_token, ( - "Invalid thinking format: missing " - ) - - index, content_delta, stop_token = _read_until_stop( - index, text, [eos_token, tool_calls_start_token] - ) - summary_content = content_delta - if stop_token == tool_calls_start_token: - is_tool_calling = True - else: - assert stop_token == eos_token, "Invalid format: missing EOS token" - - if is_tool_calling: - index, stop_token, tool_calls = parse_tool_calls(index, text) - - index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token]) - assert not tool_ends_text, "Unexpected content after tool calls" - - assert len(text) == index and stop_token in [ - eos_token, - None, - ], "Unexpected content at end" - - for sp_token in [ - bos_token, - eos_token, - thinking_start_token, - thinking_end_token, - dsml_token, - ]: - assert sp_token not in summary_content and sp_token not in reasoning_content, ( - f"Unexpected special token '{sp_token}' in content" - ) - - return { - "role": "assistant", - "content": summary_content, - "reasoning_content": reasoning_content, - "tool_calls": tool_calls_to_openai_format(tool_calls), - } diff --git a/python/sglang/srt/entrypoints/openai/encoding_dsv41.py b/python/sglang/srt/entrypoints/openai/encoding_dsv41.py deleted file mode 100644 index c8eb19344e..0000000000 --- a/python/sglang/srt/entrypoints/openai/encoding_dsv41.py +++ /dev/null @@ -1,719 +0,0 @@ -# Adapted from the DeepSeek-V4.1 release reference implementation. -"""Encode DeepSeek-V4.1 chat messages. - -Mid-conversation system messages trigger the assistant generation header. -""" - -import copy -import json -from typing import Any, Dict, List, Optional, Tuple, Union - -# ============================================================ -# Special Tokens -# ============================================================ - -bos_token: str = "<|begin▁of▁sentence|>" -eos_token: str = "<|end▁of▁sentence|>" -thinking_start_token: str = "" -thinking_end_token: str = "" -dsml_token: str = "|DSML|" - -USER_SP_TOKEN = "<|User|>" -ASSISTANT_SP_TOKEN = "<|Assistant|>" -SYSTEM_SP_TOKEN = "<|System|>" -LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>" - -IMAGE_PLACEHOLDER = "<|deepseek_image|>" - -# Task special tokens for internal classification tasks -DS_TASK_SP_TOKENS = { - "action": "<|action|>", - "query": "<|query|>", - "authority": "<|authority|>", - "domain": "<|domain|>", - "title": "<|title|>", - "read_url": "<|read_url|>", -} -VALID_TASKS = set(DS_TASK_SP_TOKENS.keys()) - -# ============================================================ -# Templates -# ============================================================ - -system_msg_template: str = "{content}" -user_msg_template: str = "{content}" -latest_reminder_msg_template: str = "{content}" -assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token -assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}" -thinking_template: str = "{reasoning_content}" - -response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" - -tool_calls_block_name: str = " calls" -tool_call_tag_name: str = " invoke" -tool_parameter_tag_name: str = " parameter" - -tool_call_template: str = '<{dsml_token}{tool_call_tag_name} name="{name}">\n{arguments}\n' -tool_calls_template = ( - "<{dsml_token}{tc_block_name}>\n{tool_calls}\n" -) - -tool_output_template: str = "{content}" - -REASONING_EFFORT_TEMPLATE = ( - "Reasoning Effort: {budget} " - "(range 1-100, the higher the value, the more thorough the reasoning)\n\n" -) - -REASONING_EFFORT_MAPPINGS: Dict[str, int] = { - "low": 50, - "high": 75, - "xhigh": 75, - "max": 100, -} -DEFAULT_REASONING_EFFORT = "high" - -TOOLS_TEMPLATE = """## Tools - -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following: - -<{dsml_token}{tc_block_name}> -<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME"> -<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE -... - -<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2"> -... - - - -String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. - -If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. - -Otherwise, output directly after {thinking_end_token} with tool calls or final response. - -### Available Tool Schemas - -{tool_schemas} - -You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -""" - -# ============================================================ -# Utility Functions -# ============================================================ - - -def to_json(value: Any) -> str: - try: - return json.dumps(value, ensure_ascii=False) - except: - return json.dumps(value, ensure_ascii=True) - - -def tools_from_openai_format(tools): - return [tool["function"] for tool in tools] - - -def tool_calls_from_openai_format(tool_calls): - return [ - { - "name": tool_call["function"]["name"], - "arguments": tool_call["function"]["arguments"], - } - for tool_call in tool_calls - ] - - -def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str: - p_dsml_template = ( - '<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">' - "{value}" - ) - P_dsml_strs = [] - - raw_arguments = tool_call["arguments"] - arguments = ( - json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments - ) - if not isinstance(arguments, dict): - raise ValueError( - "Assistant tool call function.arguments must be a JSON object." - ) - - for k, v in arguments.items(): - P_dsml_strs.append( - p_dsml_template.format( - dsml_token=dsml_token, - tool_parameter_tag_name=tool_parameter_tag_name, - key=k, - is_str="true" if isinstance(v, str) else "false", - value=v if isinstance(v, str) else to_json(v), - ) - ) - - return "\n".join(P_dsml_strs) - - -def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: - tools_json = [to_json(t) for t in tools] - - return TOOLS_TEMPLATE.format( - tool_schemas="\n".join(tools_json), - dsml_token=dsml_token, - tc_block_name=tool_calls_block_name, - tool_call_tag_name=tool_call_tag_name, - tool_parameter_tag_name=tool_parameter_tag_name, - thinking_start_token=thinking_start_token, - thinking_end_token=thinking_end_token, - ) - - -def render_reasoning_effort( - index: int, - thinking_mode: str, - effort: Union[str, int, None], -) -> str: - """Render the numeric reasoning effort prefix (thinking mode, index 0 only).""" - if effort is None: - effort = DEFAULT_REASONING_EFFORT - if not ( - (type(effort) is int and 1 <= effort <= 100) - or effort in REASONING_EFFORT_MAPPINGS - ): - raise ValueError( - f"Invalid reasoning effort for deepseek_v41: {effort!r}, should be " - f"int within [1,100] or {list(REASONING_EFFORT_MAPPINGS)}" - ) - if type(effort) is str: - effort = REASONING_EFFORT_MAPPINGS[effort] - if index == 0 and thinking_mode == "thinking": - return REASONING_EFFORT_TEMPLATE.format(budget=effort) - return "" - - -def find_last_user_index(messages: List[Dict[str, Any]]) -> int: - """Mid-conversation system messages also count as user messages here; - they trigger the assistant generation header. - User messages containing only tool results do not start a new turn. - """ - last_user_index = -1 - for idx in range(len(messages) - 1, -1, -1): - role = messages[idx].get("role") - if role in ["user", "developer"] or (role == "system" and idx > 0): - content_blocks = messages[idx].get("content_blocks") - if content_blocks and not any( - block.get("type") in ("text", "image", "image_url") - for block in content_blocks - ): - continue - last_user_index = idx - break - return last_user_index - - -def attach_task_to_last_user_message(messages: List[Dict[str, Any]], task: str) -> None: - """Set `task` on the most recent user/developer message; raise if none exists.""" - idx = find_last_user_index(messages) - if idx == -1: - raise ValueError( - "`task` requires at least one message with role='user' or 'developer'." - ) - messages[idx]["task"] = task - - -# ============================================================ -# Message Rendering -# ============================================================ - - -def render_message( - index: int, - messages: List[Dict[str, Any]], - thinking_mode: str, - drop_thinking: bool = True, - reasoning_effort: Union[str, int, None] = None, -) -> str: - assert 0 <= index < len(messages) - assert thinking_mode in [ - "chat", - "thinking", - ], f"Invalid thinking_mode `{thinking_mode}`" - - msg = messages[index] - last_user_idx = find_last_user_index(messages) - - role = msg.get("role") - content = msg.get("content") - tools = msg.get("tools") - response_format = msg.get("response_format") - tool_calls = msg.get("tool_calls") - reasoning_content = msg.get("reasoning_content") - wo_eos = msg.get("wo_eos", False) - - if tools: - tools = tools_from_openai_format(tools) - if tool_calls: - tool_calls = tool_calls_from_openai_format(tool_calls) - - reasoning_effort_prompt = render_reasoning_effort( - index, thinking_mode, reasoning_effort - ) - # Index 0 also emits the system token for the effort prompt, even before a user. - prompt = ( - SYSTEM_SP_TOKEN - if index == 0 and (reasoning_effort_prompt or role == "system") - else "" - ) - prompt += reasoning_effort_prompt - - if role == "system": - if index > 0: - prompt += SYSTEM_SP_TOKEN - prompt += system_msg_template.format(content=content or "") - if tools: - prompt += "\n\n" + render_tools(tools) - if response_format: - prompt += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - elif role == "developer": - assert content, f"Invalid message for role `{role}`: {msg}" - - content_developer = USER_SP_TOKEN - content_developer += content - - if tools: - content_developer += "\n\n" + render_tools(tools) - if response_format: - content_developer += "\n\n" + response_format_template.format( - schema=to_json(response_format) - ) - - prompt += user_msg_template.format(content=content_developer) - - elif role == "user": - prompt += USER_SP_TOKEN - - # Handle content blocks (tool results mixed with text) - content_blocks = msg.get("content_blocks") - if content_blocks: - parts = [] - for block in content_blocks: - block_type = block.get("type") - if block_type == "text": - parts.append(block.get("text", "")) - elif block_type == "tool_result": - tool_content = block.get("content", "") - if isinstance(tool_content, list): - text_parts = [] - for b in tool_content: - if b.get("type") == "text": - text_parts.append(b.get("text", "")) - else: - text_parts.append(f"[Unsupported {b.get('type')}]") - tool_content = "\n\n".join(text_parts) - parts.append(tool_output_template.format(content=tool_content)) - else: - parts.append(f"[Unsupported {block_type}]") - prompt += "\n\n".join(parts) - else: - prompt += content or "" - - elif role == "latest_reminder": - prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format( - content=content - ) - - elif role == "tool": - raise NotImplementedError( - "deepseek_v41 merges tool messages into user; please preprocess with merge_tool_messages()" - ) - - elif role == "assistant": - thinking_part = "" - tc_content = "" - - if tool_calls: - tc_list = [ - tool_call_template.format( - dsml_token=dsml_token, - tool_call_tag_name=tool_call_tag_name, - name=tc.get("name"), - arguments=encode_arguments_to_dsml(tc), - ) - for tc in tool_calls - ] - tc_content += "\n\n" + tool_calls_template.format( - dsml_token=dsml_token, - tool_calls="\n".join(tc_list), - tc_block_name=tool_calls_block_name, - ) - - summary_content = content or "" - rc = reasoning_content or "" - - # Check if previous message has a task - if so, this is a task output (no thinking) - prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None - - if thinking_mode == "thinking" and not prev_has_task: - if not drop_thinking or index > last_user_idx: - thinking_part = ( - thinking_template.format(reasoning_content=rc) + thinking_end_token - ) - else: - thinking_part = "" - - if wo_eos: - prompt += assistant_msg_wo_eos_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - prompt += assistant_msg_template.format( - reasoning=thinking_part, - content=summary_content, - tool_calls=tc_content, - ) - else: - raise NotImplementedError(f"Unknown role: {role}") - - # Append transition tokens based on what follows - if index + 1 < len(messages) and messages[index + 1].get("role") not in [ - "assistant", - "latest_reminder", - ]: - return prompt - - task = messages[index].get("task") - if task is not None: - # Task special token for internal classification tasks - assert task in VALID_TASKS, ( - f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}" - ) - task_sp_token = DS_TASK_SP_TOKENS[task] - - if task != "action": - # Non-action tasks: append task sp token directly after the message - prompt += task_sp_token - else: - # Action task: append Assistant + thinking token + action sp token - prompt += ASSISTANT_SP_TOKEN - prompt += ( - thinking_end_token - if thinking_mode != "thinking" - else thinking_start_token - ) - prompt += task_sp_token - - elif messages[index].get("role") in ["user", "developer"] or ( - messages[index].get("role") == "system" and index > 0 - ): - # Normal generation: append Assistant + thinking token - prompt += ASSISTANT_SP_TOKEN - if not drop_thinking and thinking_mode == "thinking": - prompt += thinking_start_token - elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx: - prompt += thinking_start_token - else: - prompt += thinking_end_token - - return prompt - - -# ============================================================ -# Preprocessing -# ============================================================ - - -def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Tool results are encoded within user messages; - DeepSeek-V4.1 has no standalone tool role. - """ - merged: List[Dict[str, Any]] = [] - - for msg in messages: - msg = copy.deepcopy(msg) - role = msg.get("role") - - if role == "tool": - # Convert tool message to a user message with tool_result block - content = msg.get("content", "") - if msg.get("content_blocks") and not isinstance(msg.get("content"), str): - content = msg["content_blocks"] - tool_block = { - "type": "tool_result", - "tool_use_id": msg.get("tool_call_id", ""), - "content": content, - } - # Merge into previous message if it's already a user (merged tool) - if ( - merged - and merged[-1].get("role") == "user" - and "content_blocks" in merged[-1] - ): - merged[-1]["content_blocks"].append(tool_block) - else: - merged.append( - { - "role": "user", - "content_blocks": [tool_block], - } - ) - elif role == "user": - content_blocks = msg.get("content_blocks") - if content_blocks is None: - content = msg.get("content", "") - content_blocks = ( - content - if isinstance(content, list) - else [{"type": "text", "text": content}] - ) - if ( - merged - and merged[-1].get("role") == "user" - and "content_blocks" in merged[-1] - and merged[-1].get("task") is None - ): - merged[-1]["content_blocks"].extend(content_blocks) - else: - # Keeps structured content and every message-level field. - new_msg = msg - new_msg["content_blocks"] = content_blocks - merged.append(new_msg) - else: - merged.append(msg) - - return merged - - -def sort_tool_results_by_call_order( - messages: List[Dict[str, Any]], -) -> List[Dict[str, Any]]: - last_tool_call_order: Dict[str, int] = {} - - for msg in messages: - role = msg.get("role") - if role == "assistant" and msg.get("tool_calls"): - last_tool_call_order = {} - for idx, tc in enumerate(msg["tool_calls"]): - tc_id = tc.get("id") or tc.get("function", {}).get("id", "") - if tc_id: - last_tool_call_order[tc_id] = idx - - elif role == "user" and msg.get("content_blocks"): - tool_blocks = [ - b for b in msg["content_blocks"] if b.get("type") == "tool_result" - ] - if len(tool_blocks) > 1 and last_tool_call_order: - sorted_blocks = sorted( - tool_blocks, - key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0), - ) - sorted_idx = 0 - new_blocks = [] - for block in msg["content_blocks"]: - if block.get("type") == "tool_result": - new_blocks.append(sorted_blocks[sorted_idx]) - sorted_idx += 1 - else: - new_blocks.append(block) - msg["content_blocks"] = new_blocks - - return messages - - -# ============================================================ -# Vision Message Preprocessing -# ============================================================ - - -def _is_image_block(block: Dict[str, Any]) -> bool: - return isinstance(block, dict) and block.get("type") == "image_url" - - -def _extract_image(block: Dict[str, Any]) -> Dict[str, Any]: - image_url = block.get("image_url") - url = image_url if isinstance(image_url, str) else (image_url or {}).get("url", "") - if not url: - raise ValueError("Image block does not contain a valid source") - return {"url": url} - - -def _process_image_blocks( - blocks: List[Any], image_placeholder: str = IMAGE_PLACEHOLDER -) -> Tuple[List[Any], List[Dict[str, Any]]]: - new_blocks: List[Any] = [] - images: List[Dict[str, Any]] = [] - for block in blocks: - if not isinstance(block, dict): - new_blocks.append(block) - continue - if _is_image_block(block): - new_blocks.append({"type": "text", "text": image_placeholder}) - images.append(_extract_image(block)) - elif block.get("type") == "tool_result" and isinstance( - block.get("content"), list - ): - block = copy.copy(block) - block["content"], nested_images = _process_image_blocks( - block["content"], image_placeholder - ) - new_blocks.append(block) - images.extend(nested_images) - elif block.get("type") == "text": - text = block.get("text") or "" - if IMAGE_PLACEHOLDER in text: - raise ValueError( - f"Text block contains image placeholder '{IMAGE_PLACEHOLDER}': " - f"'{text[:100]}'. Images should be separate content blocks." - ) - new_blocks.append(block) - else: - new_blocks.append(block) - return new_blocks, images - - -def _validate_no_image_sp_tokens(msg: Dict[str, Any]) -> None: - content = msg.get("content") - if isinstance(content, str) and IMAGE_PLACEHOLDER in content: - raise ValueError( - f"Message content contains image special token '{IMAGE_PLACEHOLDER}'. " - "Images should be provided as image content blocks." - ) - reasoning_content = msg.get("reasoning_content") - if isinstance(reasoning_content, str) and IMAGE_PLACEHOLDER in reasoning_content: - raise ValueError( - f"reasoning_content contains image special token '{IMAGE_PLACEHOLDER}'" - ) - - -def process_image_messages( - messages: List[Dict[str, Any]], -) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: - processed: List[Dict[str, Any]] = [] - images: List[Dict[str, Any]] = [] - for msg in messages: - msg = copy.deepcopy(msg) - _validate_no_image_sp_tokens(msg) - - if isinstance(msg.get("content"), list) and "content_blocks" not in msg: - msg["content_blocks"] = msg.pop("content") - - if msg.get("content_blocks"): - msg["content_blocks"], message_images = _process_image_blocks( - msg["content_blocks"] - ) - images.extend(message_images) - if not isinstance(msg.get("content"), str): - texts = [ - block.get("text", "") - for block in msg["content_blocks"] - if isinstance(block, dict) and block.get("type") == "text" - ] - msg["content"] = "\n\n".join(texts) - - processed.append(msg) - return processed, images - - -# ============================================================ -# Main Encoding Function -# ============================================================ - - -def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - last_user_idx = find_last_user_index(messages) - result = [] - keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"} - - for idx, msg in enumerate(messages): - role = msg.get("role") - if role in keep_roles or idx >= last_user_idx: - result.append(msg) - elif role == "assistant": - msg = copy.copy(msg) - msg.pop("reasoning_content", None) - result.append(msg) - # developer and other roles before last_user_idx are dropped - - return result - - -def _encode_messages_text( - messages: List[Dict[str, Any]], - thinking_mode: str, - context: Optional[List[Dict[str, Any]]] = None, - drop_thinking: bool = True, - add_default_bos_token: bool = True, - reasoning_effort: Union[str, int, None] = None, -) -> str: - """Encode preprocessed (text-only) messages into the V4.1 prompt format.""" - context = context if context else [] - full_messages = context + messages - - prompt = bos_token if add_default_bos_token and len(context) == 0 else "" - - # Resolve drop_thinking: if any message has tools defined, don't drop thinking - effective_drop_thinking = drop_thinking - if any(m.get("tools") for m in full_messages): - effective_drop_thinking = False - - if thinking_mode == "thinking" and effective_drop_thinking: - full_messages = _drop_thinking_messages(full_messages) - num_to_render = len(full_messages) - len(_drop_thinking_messages(context)) - context_len = len(full_messages) - num_to_render - else: - num_to_render = len(messages) - context_len = len(context) - - for idx in range(num_to_render): - prompt += render_message( - idx + context_len, - full_messages, - thinking_mode=thinking_mode, - drop_thinking=effective_drop_thinking, - reasoning_effort=reasoning_effort, - ) - - return prompt - - -def encode_messages( - messages: List[Dict[str, Any]], - thinking_mode: str, - context: Optional[List[Dict[str, Any]]] = None, - drop_thinking: bool = True, - add_default_bos_token: bool = True, - reasoning_effort: Union[str, int, None] = None, - return_multi_modal_data: bool = False, -) -> Any: - """Encode a list of messages into the DeepSeek-V4.1 prompt format. - - Returns the prompt string, or ``(prompt, {"images": [...]})`` when - ``return_multi_modal_data`` is set; the image records are in prompt order. - """ - # Validate before merging can discard message-level textual fields. - for msg in (context or []) + messages: - _validate_no_image_sp_tokens(msg) - - # Collect images only after tool results are in their final prompt order. - context = merge_tool_messages(context or []) - messages = merge_tool_messages(messages) - messages = sort_tool_results_by_call_order(context + messages)[len(context) :] - processed_context, _ = process_image_messages(context) if context else ([], []) - processed_messages, images = process_image_messages(messages) - prompt = _encode_messages_text( - processed_messages, - thinking_mode=thinking_mode, - context=processed_context if processed_context else None, - drop_thinking=drop_thinking, - add_default_bos_token=add_default_bos_token, - reasoning_effort=reasoning_effort, - ) - if return_multi_modal_data: - return prompt, {"images": images} - return prompt diff --git a/python/sglang/srt/entrypoints/openai/protocol.py b/python/sglang/srt/entrypoints/openai/protocol.py index 1734c22b99..c854d7e560 100644 --- a/python/sglang/srt/entrypoints/openai/protocol.py +++ b/python/sglang/srt/entrypoints/openai/protocol.py @@ -745,6 +745,8 @@ class ChatCompletionMessageGenericParam(BaseModel): reasoning_content: Optional[str] = None tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None]) tools: Optional[List[Tool]] = Field(default=None, examples=[None]) + # DeepSeek API prefix completion: continue this final assistant message. + prefix: Optional[bool] = None @field_validator("role", mode="before") @classmethod @@ -838,9 +840,10 @@ class ToolChoice(BaseModel): # OpenAI-spec string tiers for reasoning effort (current Responses/Chat API): -# none/minimal/low/medium/high/xhigh/max. Used as-is by /v1/responses. +# none/minimal/low/medium/high/xhigh/max, plus DeepSeek's ultra. Used as-is by +# /v1/responses. ReasoningEffortTier = Literal[ - "none", "minimal", "low", "medium", "high", "xhigh", "max" + "none", "minimal", "low", "medium", "high", "xhigh", "ultra", "max" ] # The typed /v1/responses stream events validate against OpenAI's narrower # ``Reasoning.effort``; echoing a tier outside this set kills the stream. @@ -934,7 +937,7 @@ class ChatCompletionRequest(PDRoutingFields): reasoning_effort: ReasoningEffortType = Field( default=None, description="Constrains effort on reasoning for reasoning models. " - "Accepts string levels ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max') or a " + "Accepts string levels ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'ultra', 'max') or a " "float in [0.0, 0.99] for fine-grained control. " "'none' disables reasoning entirely, 'low' is the least effort, 'high' is the most effort. " "Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning " @@ -1063,6 +1066,7 @@ class ChatCompletionRequest(PDRoutingFields): "medium", "high", "xhigh", + "ultra", "max", }: values["reasoning_effort"] = effort @@ -1778,7 +1782,7 @@ class ResponseReasoningParam(BaseModel): default="medium", description="Constrains effort on reasoning for reasoning models. " "Accepts the OpenAI string tiers " - "('none','minimal','low','medium','high','xhigh','max').", + "('none','minimal','low','medium','high','xhigh','ultra','max').", ) summary: Optional[Literal["auto", "concise", "detailed"]] = Field( default=None, diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index 32e30503db..341a12f8b4 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -9,7 +9,7 @@ from collections import OrderedDict from collections.abc import AsyncGenerator from enum import Enum from http import HTTPStatus -from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional from sglang.srt.runtime_context import get_model, get_serving @@ -42,9 +42,7 @@ from jsonschema import Draft202012Validator, SchemaError from sglang.srt.entrypoints.openai import ( chat_encoding, chat_encoding_adapters, - encoding_dsv4, encoding_dsv32, - encoding_dsv41, ) from sglang.srt.entrypoints.openai.parallel_prompt_encode import ( parallel_prompt_encode, @@ -127,7 +125,6 @@ from sglang.srt.parser.template_detection import detect_inline_system_support from sglang.srt.sampling.sampling_params import ( set_request_reasoning_end_token_ids, ) -from sglang.srt.utils import ImageData from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata if TYPE_CHECKING: @@ -348,26 +345,7 @@ class OpenAIServingChat(OpenAIServingBase): # Values: "dsv32", "dsv4", or custom values set by subclass. None for default. self.chat_encoding_spec = self._resolve_chat_encoding_spec() self.supports_inline_system = self._resolve_inline_system_support() - self._dsv4_reasoning_effort_profile = ( - chat_encoding.resolve_dsv4_reasoning_effort_profile( - model_path=self.tokenizer_manager.model_path, - revision=get_model().revision, - override=self.tokenizer_manager.model_config.hf_config.to_dict().get( - chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE - ), - ) - if self.chat_encoding_spec == "dsv4" - else None - ) - self._encoding_adapter = self._build_chat_encoding_adapter() - self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = ( - chat_encoding.default_dsv41_reasoning_effort_from_env( - envs.SGLANG_DSV41_REASONING_EFFORT.get() - ) - if self.chat_encoding_spec == "dsv41" - else None - ) # Per-request response parser for custom decoding (set by _encode_messages) self._response_parser: ResponseParserProtocol | None = None @@ -521,6 +499,7 @@ class OpenAIServingChat(OpenAIServingBase): hf_config=self.tokenizer_manager.model_config.hf_config, model_path=self.tokenizer_manager.model_path, revision=get_model().revision, + default_chat_template_kwargs=get_serving().default_chat_template_kwargs, ) ) @@ -625,6 +604,8 @@ class OpenAIServingChat(OpenAIServingBase): request: ChatCompletionRequest, thinking_mode: ThinkingMode, tools: list[dict] | None = None, + *, + image_data: list, ) -> list[int] | None: """Encode messages for custom chat_encoding_spec values. @@ -632,7 +613,11 @@ class OpenAIServingChat(OpenAIServingBase): """ if self._encoding_adapter is not None: return self._encoding_adapter.encode_messages( - messages=messages, request=request, tools=tools + messages=messages, + request=request, + tools=tools, + thinking=thinking_mode == ThinkingMode.THINKING, + image_data=image_data, ) if self.chat_encoding_spec == "kimi_k3": messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages( @@ -701,21 +686,6 @@ class OpenAIServingChat(OpenAIServingBase): return prompt_ids return None - def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]: - """Request effort for the V4.1 encoder; unsupported values warn and fall back.""" - effort = chat_encoding.parse_dsv41_reasoning_effort(value) - if effort is not None: - return effort - if value is not None and value != "none": - logger.warning( - "DeepSeek-V4.1 does not support reasoning_effort=%r; using the " - "default %r (low/high/xhigh/max, a float in [0, 0.99], or an " - "integer budget in [1, 100] via chat_template_kwargs are accepted).", - value, - self._dsv41_default_reasoning_effort, - ) - return self._dsv41_default_reasoning_effort - def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse: """Extract text from response.""" return ret_item["text"] @@ -1591,9 +1561,14 @@ class OpenAIServingChat(OpenAIServingBase): ThinkingMode.THINKING if thinking_requested else ThinkingMode.CHAT ) messages = [msg.model_dump() for msg in request.messages] + strict_tool_call_arguments = ( + self._encoding_adapter.strict_tool_call_arguments + if self._encoding_adapter is not None + else self.chat_encoding_spec != "kimi_k3" + ) for message in messages: normalize_assistant_tool_call_arguments( - message, strict=self.chat_encoding_spec != "kimi_k3" + message, strict=strict_tool_call_arguments ) prompt_ids = self._encode_messages( @@ -1601,9 +1576,12 @@ class OpenAIServingChat(OpenAIServingBase): request, thinking_mode, tools=tools, + image_data=image_data, ) if prompt_ids is not None: + if image_data and not is_multimodal: + raise ValueError("image input is not supported for this model") collects_openai_media = ( self._encoding_adapter.collects_openai_media if self._encoding_adapter is not None @@ -1623,33 +1601,30 @@ class OpenAIServingChat(OpenAIServingBase): modalities, ) elif self.chat_encoding_spec is not None: - # dsv4/dsv41/dsv32 encoding path + # dsv32 encoding path messages = copy.deepcopy(messages) - is_dsv41 = self.chat_encoding_spec == "dsv41" for msg in messages: if msg.get("content") is None: msg["content"] = "" - # The V4.1 encoder consumes OpenAI parts lists itself; dsv4/dsv32 - # are text-only, so their parts-list content is flattened first. - if not is_dsv41: - for i, msg in enumerate(messages): - if isinstance(msg.get("content"), list): - messages[i] = process_content_for_template_format( - msg, "string", [], [], [], [] - ) - - for msg in messages: - processed_msg = process_content_for_template_format( - msg, - template_content_format, - image_data, - video_data, - audio_data, - modalities, - use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32", + # The dsv32 encoder is text-only, so parts-list content is flattened first. + for i, msg in enumerate(messages): + if isinstance(msg.get("content"), list): + messages[i] = process_content_for_template_format( + msg, "string", [], [], [], [] ) - msg.update(processed_msg) + + for msg in messages: + processed_msg = process_content_for_template_format( + msg, + template_content_format, + image_data, + video_data, + audio_data, + modalities, + use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32", + ) + msg.update(processed_msg) # Handle continue_final_message: separate final assistant message messages, assistant_prefix = self._handle_last_assistant_message( @@ -1658,78 +1633,16 @@ class OpenAIServingChat(OpenAIServingBase): request_tools = request.tools if tools else None - # An empty system message hosts the request tools; dsv41 renders a - # system token for it, so it only gets one when tools need the host. - if messages[0]["role"] != "system" and (request_tools or not is_dsv41): + # An empty system message hosts the request tools. + if messages[0]["role"] != "system": messages.insert(0, {"role": "system", "content": ""}) if request_tools: - messages[0]["tools"] = [ - ( - chat_encoding.dsv41_tool_payload(tool) - if is_dsv41 - else tool.model_dump() - ) - for tool in request_tools - ] + messages[0]["tools"] = [tool.model_dump() for tool in request_tools] - # Default encoding (dsv4/dsv41/dsv32) - if self.chat_encoding_spec == "dsv4": - effort_source = request.reasoning_effort - if effort_source is None: - env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get() - if env_val: - effort_source = env_val - reasoning_effort_profile = self._dsv4_reasoning_effort_profile - assert reasoning_effort_profile is not None - accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[ - reasoning_effort_profile - ] - v4_reasoning_effort = ( - effort_source if effort_source in accepted_efforts else None - ) - if request.task is not None: - encoding_dsv4.attach_task_to_last_user_message( - messages, request.task - ) - real_input = encoding_dsv4.encode_messages( - messages, - thinking_mode=thinking_mode, - reasoning_effort=v4_reasoning_effort, - reasoning_effort_profile=reasoning_effort_profile, - ) - prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) - elif is_dsv41: - if request.task is not None: - encoding_dsv41.attach_task_to_last_user_message( - messages, request.task - ) - real_input, media = encoding_dsv41.encode_messages( - messages, - thinking_mode=thinking_mode, - reasoning_effort=self._resolve_dsv41_reasoning_effort( - request.reasoning_effort - ), - return_multi_modal_data=True, - ) - if media["images"]: - if not is_multimodal: - raise ValueError("image input is not supported for this model") - image_data.extend( - ImageData(url=image["url"]) for image in media["images"] - ) - tokenizer = self.tokenizer_manager.tokenizer - real_input = real_input.replace( - encoding_dsv41.IMAGE_PLACEHOLDER, - tokenizer.convert_ids_to_tokens( - self.tokenizer_manager.image_token_id - ), - ) - prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) - else: - real_input = encoding_dsv32.encode_messages( - messages, thinking_mode=thinking_mode - ) - prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) + real_input = encoding_dsv32.encode_messages( + messages, thinking_mode=thinking_mode + ) + prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) # Append assistant prefix if continue_final_message is enabled if assistant_prefix: diff --git a/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py b/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py new file mode 100644 index 0000000000..fbe9b910e0 --- /dev/null +++ b/python/sglang/srt/entrypoints/openai/serving_deepseek_recipe.py @@ -0,0 +1,127 @@ +"""DeepSeek-V4 / V4.1 serving glue for the ``dsv4`` / ``dsv41`` chat encoding +specs: prompt encoding through deepseek-recipe under SGLang's request semantics.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Optional, Union + +from sglang.srt.environ import envs +from sglang.srt.parser.deepseek_recipe_renderer import ( + RecipeEncoder, + parse_effort_budget, +) +from sglang.srt.utils import ImageData + +if TYPE_CHECKING: + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + +DEFAULT_EFFORT_ENVS = { + "dsv4": envs.SGLANG_DSV4_REASONING_EFFORT, + "dsv41": envs.SGLANG_DSV41_REASONING_EFFORT, +} + + +class DeepSeekRecipeChatEncodingAdapter: + # recipe sorts tool results by call order, so images come from its render. + collects_openai_media = False + output = None + strict_tool_call_arguments = False # recipe serializes object arguments itself. + + def __init__( + self, + *, + chat_encoding_spec: str, + tokenizer: Any, + default_chat_template_kwargs: Optional[dict[str, Any]], + ): + self._spec = chat_encoding_spec + self.encoder = RecipeEncoder.from_hf_tokenizer(chat_encoding_spec, tokenizer) + raw = (DEFAULT_EFFORT_ENVS[chat_encoding_spec].get() or "").strip() + self._default_effort = int(raw) if raw.isdigit() else raw or None + # Encoded at boot, so a bad server default fails startup, not requests. + defaults = default_chat_template_kwargs or {} + self._encode( + {"messages": [{"role": "user", "content": ""}], "thinking": True}, + effort=defaults.get("reasoning_effort"), + ) + + def encode_messages( + self, + *, + messages: list[dict[str, Any]], + request: ChatCompletionRequest, + tools: Optional[list[dict]], + thinking: bool, + image_data: list, + ) -> list[int]: + # The wire request: the handler's message dumps carry None for unset fields. + wire = request.model_dump(exclude_unset=True, by_alias=True, mode="json") + wire_messages = wire["messages"] + continue_final = request.continue_final_message + task_prefix = None + # SGLang's trailing-assistant rule, as `_handle_last_assistant_message`. + last = wire_messages[-1] if wire_messages else None + if ( + last + and last["role"] == "assistant" + and isinstance(last.get("content"), str) + ): + if not continue_final: + wire_messages[-1] = {"role": "user", "content": last["content"]} + elif request.task is not None: + # A task renders its own tail, so the prefix follows it. + task_prefix = wire_messages.pop()["content"] + continue_final = False + body = { + "messages": wire_messages, + "thinking": thinking, + "continue_final_message": continue_final, + "response_format": wire.get("response_format"), + } + if tools is not None: + # `model_dump` adds None for unset fields, which recipe renders as values. + body["tools"] = [ + {k: v for k, v in tool.items() if v is not None} + | { + "function": { + k: v for k, v in tool["function"].items() if v is not None + } + } + for tool in tools + ] + body["tool_choice"] = wire.get("tool_choice") + body = {key: value for key, value in body.items() if value is not None} + image_urls = [] + prompt_ids = self._encode( + body, + effort=request.reasoning_effort, + task=request.task, + image_urls=image_urls, + ) + image_data.extend(ImageData(url=url) for url in image_urls) + if task_prefix: + prompt_ids += self.encoder.encode_text(task_prefix) + return prompt_ids + + def _encode( + self, + body: dict[str, Any], + *, + effort: Union[str, int, float, None], + task: Optional[str] = None, + image_urls: Optional[list[str]] = None, + ) -> list[int]: + if effort is None or effort == "none": + effort = self._default_effort + budget = None + if effort is not None and not isinstance(effort, str): + if self._spec != "dsv41": + raise ValueError( + f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}" + ) + budget = parse_effort_budget(effort) + elif effort is not None: + body["reasoning_effort"] = effort + return self.encoder.encode( + body, task=task, effort_budget=budget, image_urls=image_urls + ) diff --git a/python/sglang/srt/entrypoints/openai/serving_inkling.py b/python/sglang/srt/entrypoints/openai/serving_inkling.py index 214bbe23f9..acfb8ba297 100644 --- a/python/sglang/srt/entrypoints/openai/serving_inkling.py +++ b/python/sglang/srt/entrypoints/openai/serving_inkling.py @@ -35,6 +35,7 @@ if TYPE_CHECKING: class InklingChatEncodingAdapter: collects_openai_media = True + strict_tool_call_arguments = True def __init__( self, @@ -73,7 +74,11 @@ class InklingChatEncodingAdapter: messages: list[dict[str, Any]], request: ChatCompletionRequest, tools: list[dict] | None, + thinking: bool, + image_data: list, ) -> list[int]: + # Inkling's thinking is effort-conditioned and the handler collects its + # media, so `thinking` and `image_data` are not read. reasoning_effort = parse_reasoning_effort(request.reasoning_effort) if reasoning_effort is None: reasoning_effort = self._default_reasoning_effort diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index be871c7251..d5d71bda98 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1699,10 +1699,12 @@ class Envs: SGLANG_DSV4_FP4_EXPERTS = EnvBool(True) # Set True to dequantize the FP4 experts to FP8 at runtime SGLANG_DSV4_FP4_DEQUANT = EnvBool(False) - # Flash-0731 also accepts "low"; the active profile is checkpoint-resolved. + # DeepSeek-V4 default when a request carries no reasoning_effort: a + # deepseek-recipe tier name; unset -> deepseek-recipe's default. SGLANG_DSV4_REASONING_EFFORT = EnvStr("") # DeepSeek-V4.1 default when a request carries no reasoning_effort: one of - # low/high/xhigh/max or an integer budget in [1, 100]; unset -> the encoder default. + # a deepseek-recipe tier name or an integer budget in [1, 100]; unset -> + # deepseek-recipe's default. SGLANG_DSV41_REASONING_EFFORT = EnvStr(None) # Quantize the SWA fp8 KV cache from bf16-rounded values (matches # trainer-side QAT and the DSA-CP path) instead of fp32 registers. diff --git a/python/sglang/srt/parser/deepseek_recipe_renderer.py b/python/sglang/srt/parser/deepseek_recipe_renderer.py new file mode 100644 index 0000000000..0a44dabf4e --- /dev/null +++ b/python/sglang/srt/parser/deepseek_recipe_renderer.py @@ -0,0 +1,199 @@ +"""DeepSeek-V4 / V4.1 prompts through deepseek-recipe, plus the SGLang extensions +recipe does not model (quick instruction tasks, V4.1 integer effort).""" + +from __future__ import annotations + +import copy +import functools +import json +from types import ModuleType +from typing import Any, Optional, Union + +DEEPSEEK_RECIPE_INSTALL_HINT = ( + "Serving DeepSeek-V4 / V4.1 requires the `deepseek-recipe` package (DeepSeek's " + "reference encoder and parser; wheels for Linux and macOS). Install it with " + "`pip install deepseek-recipe`." +) + +MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"} +# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>. +IMAGE_TOKEN = "<|deepseek_image|>" +ASSISTANT_TOKEN = "<|Assistant|>" +THINK_START_TOKEN = "" +THINK_END_TOKEN = "" +TASK_TOKENS = { + "action": "<|action|>", + "query": "<|query|>", + "authority": "<|authority|>", + "domain": "<|domain|>", + "title": "<|title|>", + "read_url": "<|read_url|>", +} +# V4.1 renders an integer budget as this line; recipe's "high" tier is budget 75. +BUDGET_TIER, BUDGET_TIER_VALUE = "high", 75 +BUDGET_LINE = "Reasoning Effort: {budget}" + + +@functools.cache +def load_deepseek_recipe() -> ModuleType: + try: + import deepseek_recipe + except ImportError as exc: + raise ImportError(DEEPSEEK_RECIPE_INSTALL_HINT) from exc + return deepseek_recipe + + +def parse_effort_budget(value: Union[int, float]) -> int: + if isinstance(value, bool): + raise ValueError("reasoning_effort must not be a boolean") + if isinstance(value, int) and 1 <= value <= 100: + return value + if isinstance(value, float) and 0.0 <= value <= 0.99: + return max(1, round(value * 100)) + raise ValueError( + f"reasoning_effort={value!r} is outside the integer budget [1, 100] " + "and the float range [0, 0.99]" + ) + + +def pre_normalize(body: dict[str, Any]) -> dict[str, Any]: + """Rewrite an SGLang chat request body for recipe's Chat Completions adapter. + + `body` carries only what the prompt depends on: messages, thinking and, when + set, tools, tool_choice, response_format, continue_final_message and a named + reasoning_effort. + """ + request = copy.deepcopy(body) + request["thinking"] = {"type": "enabled" if request["thinking"] else "disabled"} + # The host enforces structural_tag with its grammar. + response_format = request.get("response_format") + if response_format and response_format.get("type") == "structural_tag": + del request["response_format"] + choice = request.get("tool_choice") + if not request.get("tools") or choice == "none": + request.pop("tools", None) + request.pop("tool_choice", None) + elif choice == "required" or isinstance(choice, dict): + # SGLang enforces the choice with its grammar, which expects the model + # to open the calls block itself. + request["tool_choice"] = "auto" + messages = request["messages"] + for message in messages: + if message["role"] == "developer": + message["role"] = "system" + for call in message.get("tool_calls") or []: + function = call["function"] + if "arguments" in function and not isinstance(function["arguments"], str): + function["arguments"] = json.dumps( + function["arguments"], ensure_ascii=False, separators=(",", ":") + ) + continue_final = request.pop("continue_final_message", False) + if continue_final and messages and messages[-1]["role"] == "assistant": + messages[-1]["prefix"] = True + return request + + +def _error_message(body: Any) -> str: + try: + return json.loads(body)["error"]["message"] + except (TypeError, ValueError, KeyError): + return str(body) + + +class RecipeEncoder: + """`tokenizer` is the checkpoint's `tokenizers.Tokenizer`; recipe encodes prompts + with its own copy of it, and the SGLang extensions splice ids from the original.""" + + def __init__(self, chat_encoding_spec: str, tokenizer: Any): + recipe = load_deepseek_recipe() + self.model = MODEL_NAMES[chat_encoding_spec] + self.hf_tokenizer = tokenizer + special = recipe.SpecialTokens(image=IMAGE_TOKEN) + self._options = recipe.ConversionOptions().with_special_tokens(special) + encoding = ( + recipe.DeepseekV4Encoding() + if chat_encoding_spec == "dsv4" + else recipe.DeepseekV41Encoding() + ) + self._encoding = encoding.with_special_tokens(special).with_tokenizer( + recipe.Tokenizer.from_str(tokenizer.to_str()) + ) + + @classmethod + def from_hf_tokenizer( + cls, chat_encoding_spec: str, tokenizer: Any + ) -> RecipeEncoder: + backend = getattr(tokenizer, "backend_tokenizer", None) + if backend is None: + raise ValueError( + "deepseek-recipe needs the checkpoint's fast tokenizer (tokenizer.json)" + ) + return cls(chat_encoding_spec, backend) + + def encode( + self, + body: dict[str, Any], + *, + task: Optional[str] = None, + effort_budget: Optional[int] = None, + image_urls: Optional[list[str]] = None, + ) -> list[int]: + """Prompt ids for `body` (see `pre_normalize`); a request recipe rejects + raises ValueError. `image_urls` receives the images in placeholder order.""" + recipe = load_deepseek_recipe() + normalized = pre_normalize(body) + normalized["model"] = self.model + if effort_budget is not None: + normalized["reasoning_effort"] = BUDGET_TIER + try: + converted = recipe.ChatCompletionRequest(normalized).convert(self._options) + except recipe.ConversionError as error: + raise ValueError(_error_message(error.body)) from error + prompt_ids = self._encoding.encode(converted.conversation) + if image_urls is not None: + rendered = self._encoding.render_conversation(converted.conversation) + image_urls.extend( + source.url or source.data_url for source in rendered.image_sources + ) + if effort_budget is not None and body["thinking"]: + prompt_ids = self._replace_effort_budget(prompt_ids, effort_budget) + if task is not None: + prompt_ids = self._attach_task(prompt_ids, task, body["thinking"]) + return prompt_ids + + def encode_text(self, text: str) -> list[int]: + return self.hf_tokenizer.encode(text, add_special_tokens=False).ids + + def _replace_effort_budget(self, prompt_ids: list[int], budget: int) -> list[int]: + # The effort line opens the prompt, so its first match is the line itself. + rendered = self.encode_text(BUDGET_LINE.format(budget=BUDGET_TIER_VALUE)) + for start in range(len(prompt_ids) - len(rendered) + 1): + if prompt_ids[start : start + len(rendered)] == rendered: + end = start + len(rendered) + return ( + prompt_ids[:start] + + self.encode_text(BUDGET_LINE.format(budget=budget)) + + prompt_ids[end:] + ) + raise ValueError("deepseek-recipe rendered no reasoning effort line") + + def _attach_task( + self, prompt_ids: list[int], task: str, thinking: bool + ) -> list[int]: + """A quick instruction task on the last user turn, as DeepSeek's checkpoint + encoder renders it: `action` follows the generation header, other tasks + replace it.""" + header = [ + self.hf_tokenizer.token_to_id(ASSISTANT_TOKEN), + self.hf_tokenizer.token_to_id( + THINK_START_TOKEN if thinking else THINK_END_TOKEN + ), + ] + if prompt_ids[-len(header) :] != header: + raise ValueError( + "task requires the conversation to end with a user message" + ) + task_id = self.hf_tokenizer.token_to_id(TASK_TOKENS[task]) + if task == "action": + return prompt_ids + [task_id] + return prompt_ids[: -len(header)] + [task_id] diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index c149eb8faf..c09090eb83 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -4,14 +4,6 @@ from typing import Dict, List, Optional, Tuple, Type from transformers import PreTrainedTokenizerBase -from sglang.srt.entrypoints.openai.encoding_dsv4 import dsml_token as dsv4_dsml_token -from sglang.srt.entrypoints.openai.encoding_dsv4 import eos_token as dsv4_eos_token -from sglang.srt.entrypoints.openai.encoding_dsv4 import ( - thinking_end_token as dsv4_thinking_end_token, -) -from sglang.srt.entrypoints.openai.encoding_dsv4 import ( - thinking_start_token as dsv4_thinking_start_token, -) from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, ResponsesRequest, @@ -1525,6 +1517,12 @@ class _DeepSeekV3Detector(Qwen3Detector): self.reasoning_default = "explicit_thinking" +DSV4_EOS_TOKEN = "<|end▁of▁sentence|>" +DSV4_THINK_START_TOKEN = "" +DSV4_THINK_END_TOKEN = "" +DSV4_DSML_TOKEN = "|DSML|" + + class DeepSeekV4Detector(BaseReasoningFormatDetector): def __init__( self, @@ -1535,13 +1533,13 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector): force_nonempty_content: bool = False, ): super().__init__( - dsv4_thinking_start_token, - dsv4_thinking_end_token, - think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token], + DSV4_THINK_START_TOKEN, + DSV4_THINK_END_TOKEN, + think_excluded_tokens=[DSV4_EOS_TOKEN, DSV4_DSML_TOKEN], # The reference parser splits reasoning at `\n\n<|DSML|tool_calls`; # anchoring on the full tag at a line start keeps a mid-sentence # mention of the tag inside reasoning from ending the block. - tool_start_token=f"<{dsv4_dsml_token}tool_calls>", + tool_start_token=f"<{DSV4_DSML_TOKEN}tool_calls>", tool_start_at_line_start=True, force_reasoning=force_reasoning, stream_reasoning=stream_reasoning, @@ -1559,7 +1557,7 @@ class DeepSeekV41ReasoningDetector(DeepSeekV4Detector): def __init__(self, **kwargs): super().__init__(**kwargs) - self.tool_start_token = f"<{dsv4_dsml_token} calls>" + self.tool_start_token = f"<{DSV4_DSML_TOKEN} calls>" class _MimoDetector(Qwen3Detector): diff --git a/rust/sglang-processor/tests/scripts/generate_parity.py b/rust/sglang-processor/tests/scripts/generate_parity.py index b6b63bded5..3fed976ba9 100644 --- a/rust/sglang-processor/tests/scripts/generate_parity.py +++ b/rust/sglang-processor/tests/scripts/generate_parity.py @@ -37,7 +37,6 @@ from sglang.srt.runtime_context import publish, reset_context from sglang.srt.server_args import ServerArgs FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/parity" -DSV4_PROFILE = chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE class RecordingTokenizer: @@ -87,18 +86,6 @@ def serving_chat(model, path, revision, config, tokenizer, tool_call_parser): tokenizer=tokenizer, tool_call_parser=tool_call_parser, ) - server._dsv4_reasoning_effort_profile = ( - chat_encoding.resolve_dsv4_reasoning_effort_profile( - model_path=path, revision=revision, override=config.get(DSV4_PROFILE) - ) - if server.chat_encoding_spec == "dsv4" - else None - ) - server._dsv41_default_reasoning_effort = ( - chat_encoding.default_dsv41_reasoning_effort_from_env(None) - if server.chat_encoding_spec == "dsv41" - else None - ) server._tokenizer_auto_adds_specials = len(tokenizer.encode("")) > 0 server._prompt_text_round_trip_is_lossy = server._probe_prompt_text_round_trip() server._chat_template_cache = OrderedDict() @@ -130,13 +117,10 @@ def generate(fixture_path): tok, fixture.get("tool_call_parser"), ) - # What the processor reads from config.json; the resolved profile stands in - # for the checkpoint's encoder file so the Rust test runs offline. + # What the processor reads from config.json. fixture["config"] = { key: config[key] for key in ("model_type", "architectures") if key in config } - if server._dsv4_reasoning_effort_profile: - fixture["config"][DSV4_PROFILE] = server._dsv4_reasoning_effort_profile fixture["bos_token_id"] = tok.bos_token_id # Jinja checkpoints render with their template; keep it so the test runs offline. if server.chat_encoding_spec is None: diff --git a/test/registered/unit/entrypoints/openai/test_protocol.py b/test/registered/unit/entrypoints/openai/test_protocol.py index 1cae9d3b22..cf9de9ad03 100644 --- a/test/registered/unit/entrypoints/openai/test_protocol.py +++ b/test/registered/unit/entrypoints/openai/test_protocol.py @@ -261,7 +261,7 @@ class TestChatCompletionRequest(unittest.TestCase): from pydantic import ValidationError messages = [{"role": "user", "content": "Hello"}] - for effort in ("xhigh", "max"): + for effort in ("xhigh", "ultra", "max"): with self.subTest(effort=effort, request_form="top-level"): request = ChatCompletionRequest( model="test-model", @@ -283,7 +283,7 @@ class TestChatCompletionRequest(unittest.TestCase): ChatCompletionRequest( model="test-model", messages=messages, - reasoning_effort="ultra", + reasoning_effort="extreme", ) def test_chat_completion_reasoning_effort_is_strictly_validated(self): diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index a8bb48b9c1..873b0a3bc4 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -11,23 +11,18 @@ from sglang.test.test_utils import CustomTestCase, enter_override, maybe_stub_sg maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel import asyncio -import copy import gc import json -import tempfile import unittest import uuid from http import HTTPStatus -from pathlib import Path from typing import Optional from unittest.mock import Mock, patch from fastapi import Request from transformers.utils.chat_template_utils import _compile_jinja_template -from sglang.srt.entrypoints.openai.chat_encoding import ( - resolve_dsv4_reasoning_effort_profile, -) +from sglang.srt.entrypoints.openai.chat_encoding import resolve_chat_encoding_spec from sglang.srt.entrypoints.openai.protocol import ( ChatCompletionRequest, Function, @@ -157,15 +152,6 @@ _TOOL_RESULT_REORDER_TEMPLATE = """ """ -def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str: - model_dir = tempfile.TemporaryDirectory() - test_case.addCleanup(model_dir.cleanup) - encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py" - encoder_path.parent.mkdir(parents=True) - encoder_path.write_text(source, encoding="utf-8") - return model_dir.name - - class _MockTokenizerManager: """Minimal mock that satisfies OpenAIServingChat.""" @@ -1915,7 +1901,6 @@ class ServingChatTestCase(CustomTestCase): def test_dsv_tool_choice_none_omits_tools_from_prompt(self): self.template_manager.chat_template_name = None self.template_manager.jinja_template_content_format = "string" - self.chat._dsv4_reasoning_effort_profile = "preview" tool = { "type": "function", "function": { @@ -1924,25 +1909,21 @@ class ServingChatTestCase(CustomTestCase): }, } - for chat_encoding_spec in ("dsv4", "dsv32", "dsv41"): - for tool_choice, expects_tools in (("none", False), ("auto", True)): - with self.subTest( - chat_encoding_spec=chat_encoding_spec, + self.chat.chat_encoding_spec = "dsv32" + for tool_choice, expects_tools in (("none", False), ("auto", True)): + with self.subTest(tool_choice=tool_choice): + self.tm.tokenizer.encode.reset_mock() + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Solve 100 / 7."}], + tools=[tool], tool_choice=tool_choice, - ): - self.chat.chat_encoding_spec = chat_encoding_spec - self.tm.tokenizer.encode.reset_mock() - request = ChatCompletionRequest( - model="x", - messages=[{"role": "user", "content": "Solve 100 / 7."}], - tools=[tool], - tool_choice=tool_choice, - ) + ) - self.chat._process_messages(request, is_multimodal=False) + self.chat._process_messages(request, is_multimodal=False) - prompt = self.tm.tokenizer.encode.call_args.args[0] - self.assertEqual("calculator" in prompt, expects_tools) + prompt = self.tm.tokenizer.encode.call_args.args[0] + self.assertEqual("calculator" in prompt, expects_tools) def test_kimi_k3_encoder_receives_wire_request_fields(self): self.template_manager.chat_template_name = None @@ -2200,76 +2181,70 @@ class ServingChatTestCase(CustomTestCase): """DeepSeek encoders should reject history tool call scalars as BadRequest.""" self.template_manager.chat_template_name = None self.template_manager.jinja_template_content_format = "string" - self.chat._dsv4_reasoning_effort_profile = "preview" - for chat_encoding_spec in ("dsv4", "dsv32"): - with self.subTest(chat_encoding_spec=chat_encoding_spec): - self.chat.chat_encoding_spec = chat_encoding_spec - req = ChatCompletionRequest( - model="x", - messages=[ - {"role": "user", "content": "Where is it raining?"}, + self.chat.chat_encoding_spec = "dsv32" + req = ChatCompletionRequest( + model="x", + messages=[ + {"role": "user", "content": "Where is it raining?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '"Beijing"', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": "sunny", - }, + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '"Beijing"', + }, + } ], - ) + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "sunny", + }, + ], + ) - with self.assertRaisesRegex(ValueError, "must be a JSON object"): - self.chat._process_messages(req, is_multimodal=False) + with self.assertRaisesRegex(ValueError, "must be a JSON object"): + self.chat._process_messages(req, is_multimodal=False) def test_dsv_encoders_accept_object_tool_call_arguments_string(self): """DeepSeek encoders accept object-shaped OpenAI JSON string arguments.""" self.template_manager.chat_template_name = None self.template_manager.jinja_template_content_format = "string" - self.chat._dsv4_reasoning_effort_profile = "preview" - for chat_encoding_spec in ("dsv4", "dsv32"): - with self.subTest(chat_encoding_spec=chat_encoding_spec): - self.chat.chat_encoding_spec = chat_encoding_spec - req = ChatCompletionRequest( - model="x", - messages=[ - {"role": "user", "content": "Where is it raining?"}, + self.chat.chat_encoding_spec = "dsv32" + req = ChatCompletionRequest( + model="x", + messages=[ + {"role": "user", "content": "Where is it raining?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": '{"city": "Beijing"}', - }, - } - ], - }, - { - "role": "tool", - "tool_call_id": "call_1", - "content": "sunny", - }, + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": '{"city": "Beijing"}', + }, + } ], - ) + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": "sunny", + }, + ], + ) - self.chat._process_messages(req, is_multimodal=False) + self.chat._process_messages(req, is_multimodal=False) def test_stop_str_isolation_between_requests(self): """Test that stop strings from one request don't affect subsequent requests. @@ -3110,17 +3085,14 @@ class ServingChatTestCase(CustomTestCase): # Case 4: DeepseekV4 arch -> always dsv4, even with chat_template # (release ships a stale V3 jinja we deliberately override). mock_hf_config.architectures = ["DeepseekV4ForCausalLM"] - mock_hf_config.to_dict.return_value = { - "dsv4_reasoning_effort_profile": "preview" - } - tm.model_path = "deepseek-ai/DeepSeek-V4-Flash" - tm.tokenizer.chat_template = "stale v3 jinja" - serving_chat = OpenAIServingChat(tm, TemplateManager()) - self.assertEqual(serving_chat.chat_encoding_spec, "dsv4") - - tm.tokenizer.chat_template = None - serving_chat = OpenAIServingChat(tm, TemplateManager()) - self.assertEqual(serving_chat.chat_encoding_spec, "dsv4") + for chat_template in ("stale v3 jinja", None): + tm.tokenizer.chat_template = chat_template + self.assertEqual( + resolve_chat_encoding_spec( + hf_config=mock_hf_config, tokenizer=tm.tokenizer + ), + "dsv4", + ) def test_kimi_k3_encoding_detection(self): from sglang.srt.parser.template_manager import TemplateManager @@ -3151,7 +3123,7 @@ class ServingChatTestCase(CustomTestCase): self.chat.chat_encoding_spec = None self.assertFalse(self.chat.supports_native_reasoning_history()) - # ------------- dsv4 task + latest_reminder ------------- + # ------------- dsv4 task ------------- def test_dsv4_task_field_schema(self): """Top-level `task` accepts the 6 DS task tokens and rejects others.""" for valid in ("action", "query", "authority", "domain", "title", "read_url"): @@ -3175,407 +3147,6 @@ class ServingChatTestCase(CustomTestCase): task="bogus", ) - def test_attach_task_to_last_user_message(self): - """Helper attaches task to the nearest user/developer message.""" - from sglang.srt.entrypoints.openai import encoding_dsv4 - - messages = [{"role": "user", "content": "Hi"}] - encoding_dsv4.attach_task_to_last_user_message(messages, "domain") - self.assertEqual(messages[0]["task"], "domain") - - # Prefers the LAST user message across a multi-turn conversation. - messages = [ - {"role": "user", "content": "first"}, - {"role": "assistant", "content": "ok"}, - {"role": "user", "content": "second"}, - ] - encoding_dsv4.attach_task_to_last_user_message(messages, "query") - self.assertNotIn("task", messages[0]) - self.assertEqual(messages[2]["task"], "query") - - # `developer` role is treated like `user` (matches encoder semantics). - messages = [{"role": "developer", "content": "dev"}] - encoding_dsv4.attach_task_to_last_user_message(messages, "authority") - self.assertEqual(messages[0]["task"], "authority") - - # No user/developer present -> raises. - with self.assertRaises(ValueError): - encoding_dsv4.attach_task_to_last_user_message( - [{"role": "system", "content": "s"}], "domain" - ) - - def test_dsv4_content_parts_list_normalized(self): - """OpenAI list-of-parts content flattens to text before reaching the encoder.""" - from sglang.srt.entrypoints.openai import encoding_dsv4 - from sglang.srt.parser.jinja_template_utils import ( - process_content_for_template_format, - ) - - req = ChatCompletionRequest( - model="x", - messages=[ - { - "role": "user", - "content": [{"type": "text", "text": "say hi"}], - } - ], - ) - messages = [m.model_dump() for m in req.messages] - # Mirror the boundary normalization _process_messages does for any - # non-None chat_encoding_spec. - for i, msg in enumerate(messages): - if isinstance(msg.get("content"), list): - messages[i] = process_content_for_template_format( - msg, "string", [], [], [], [] - ) - out = encoding_dsv4.encode_messages(messages, thinking_mode="chat") - self.assertIn("<|User|>say hi", out) - - # Multiple text parts concat with single space; non-text parts dropped. - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": "describe"}, - {"type": "image_url", "image_url": {"url": "x"}}, - ], - } - ] - for i, msg in enumerate(messages): - if isinstance(msg.get("content"), list): - messages[i] = process_content_for_template_format( - msg, "string", [], [], [], [] - ) - out = encoding_dsv4.encode_messages(messages, thinking_mode="chat") - self.assertIn("<|User|>describe", out) - self.assertNotIn("image_url", out) - - def test_dsv4_task_and_reminder_encode_end_to_end(self): - """Task + latest_reminder plumb through to the dsv4 encoder correctly.""" - from sglang.srt.entrypoints.openai import encoding_dsv4 - - # 1) task='domain' in chat mode -> `<|domain|>` appended, no Assistant - # prefix (this is a single-shot classification, not a chat turn). - req = ChatCompletionRequest( - model="x", - messages=[{"role": "user", "content": "What is SGLang?"}], - task="domain", - ) - messages = [m.model_dump() for m in req.messages] - encoding_dsv4.attach_task_to_last_user_message(messages, req.task) - out = encoding_dsv4.encode_messages(messages, thinking_mode="chat") - self.assertIn("<|domain|>", out) - self.assertTrue(out.rstrip().endswith("<|domain|>")) - self.assertNotIn("<|Assistant|>", out) - - # 2) task='action' in thinking mode -> Assistant + + <|action|> - # (action is the one task that still runs a reasoning pass). - req = ChatCompletionRequest( - model="x", - messages=[{"role": "user", "content": "Hi"}], - task="action", - ) - messages = [m.model_dump() for m in req.messages] - encoding_dsv4.attach_task_to_last_user_message(messages, req.task) - out = encoding_dsv4.encode_messages(messages, thinking_mode="thinking") - self.assertIn("<|Assistant|>", out) - self.assertIn("", out) - self.assertTrue(out.rstrip().endswith("<|action|>")) - - # 3) latest_reminder preceding user -> reminder renders before user, - # Assistant prefix still comes after user. - req = ChatCompletionRequest( - model="x", - messages=[ - {"role": "latest_reminder", "content": "Be terse."}, - {"role": "user", "content": "Hello"}, - ], - ) - messages = [m.model_dump() for m in req.messages] - out = encoding_dsv4.encode_messages(messages, thinking_mode="chat") - self.assertIn("<|latest_reminder|>Be terse.", out) - self.assertIn("<|User|>Hello", out) - self.assertLess( - out.index("<|latest_reminder|>"), - out.index("<|User|>"), - ) - self.assertIn("<|Assistant|>", out) - - def test_dsv4_reasoning_effort_profiles(self): - from sglang.srt.entrypoints.openai import encoding_dsv4 - - messages = [ - {"role": "system", "content": ""}, - {"role": "user", "content": "Solve this."}, - ] - absolute_maximum = ( - "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" - "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" - "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" - ) - beyond_maximum = ( - "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" - "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" - "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" - ) - - def encode(profile, effort): - return encoding_dsv4.encode_messages( - messages, - thinking_mode="thinking", - reasoning_effort=effort, - reasoning_effort_profile=profile, - ) - - preview_high = encode("preview", "high") - preview_max = encode("preview", "max") - official_low = encode("official", "low") - official_high = encode("official", "high") - official_max = encode("official", "max") - - self.assertNotIn("Reasoning Effort:", preview_high) - self.assertTrue( - preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum) - ) - self.assertNotIn("Reasoning Effort:", official_low) - self.assertEqual( - official_high, - encoding_dsv4.bos_token - + absolute_maximum - + official_low.removeprefix(encoding_dsv4.bos_token), - ) - self.assertEqual( - official_max, - encoding_dsv4.bos_token - + beyond_maximum - + official_low.removeprefix(encoding_dsv4.bos_token), - ) - self.assertEqual(encode("preview", None), preview_high) - self.assertEqual(encode("official", None), official_low) - self.assertEqual(len({official_low, official_high, official_max}), 3) - - with self.assertRaises(ValueError): - encode("preview", "low") - - def test_dsv4_reasoning_effort_profile_resolution(self): - resolve = resolve_dsv4_reasoning_effort_profile - preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER) - official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER) - inconclusive_model_path = _create_dsv4_checkpoint( - self, 'UNRELATED_METADATA = "value"\n' - ) - self.assertEqual(resolve(model_path=preview_model_path), "preview") - self.assertEqual(resolve(model_path=official_model_path), "official") - self.assertEqual(resolve(model_path=inconclusive_model_path), "preview") - - self.assertEqual( - resolve(model_path="renamed/model", override="official"), "official" - ) - self.assertEqual( - resolve(model_path="renamed/model", override="preview"), "preview" - ) - with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"): - resolve(model_path="renamed/model", override="auto") - - def test_dsv4_reasoning_effort_profile_from_checkpoint(self): - from sglang.srt.parser.template_manager import TemplateManager - - official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER) - tm = _MockTokenizerManager() - tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"] - tm.model_config.hf_config.to_dict.return_value = {} - tm.model_config.hf_config.dspark_block_size = 5 - tm.model_config.hf_config.dspark_markov_rank = 256 - tm.model_path = official_model_path - tm.server_args.model_path = tm.model_path - serving_chat = OpenAIServingChat(tm, TemplateManager()) - - request = ChatCompletionRequest( - model="x", - messages=[{"role": "user", "content": "Hello"}], - reasoning_effort="max", - ) - serving_chat._process_messages(request, is_multimodal=False) - prompt = tm.tokenizer.encode.call_args.args[0] - self.assertIn("Reasoning Effort: Beyond maximum", prompt) - - def test_dsv4_reasoning_effort_profile_override_from_model_config(self): - from sglang.srt.parser.template_manager import TemplateManager - - tm = _MockTokenizerManager() - tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"] - tm.model_config.hf_config.to_dict.return_value = { - "dsv4_reasoning_effort_profile": "official" - } - serving_chat = OpenAIServingChat(tm, TemplateManager()) - request = ChatCompletionRequest( - model="x", - messages=[{"role": "user", "content": "Hello"}], - reasoning_effort="max", - ) - - serving_chat._process_messages(request, is_multimodal=False) - - prompt = tm.tokenizer.encode.call_args.args[0] - self.assertIn("Reasoning Effort: Beyond maximum", prompt) - - def test_dsv4_invalid_profile_override_fails_at_construction(self): - from sglang.srt.parser.template_manager import TemplateManager - - tm = _MockTokenizerManager() - tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"] - tm.model_config.hf_config.to_dict.return_value = { - "dsv4_reasoning_effort_profile": "invalid" - } - with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"): - OpenAIServingChat(tm, TemplateManager()) - - def test_dsv41_rejects_image_placeholder_in_text(self): - from sglang.srt.entrypoints.openai import encoding_dsv41 - - for role in ("user", "tool"): - with self.subTest(role=role): - with self.assertRaises(ValueError): - encoding_dsv41.encode_messages( - [ - { - "role": role, - "content": f"hi {encoding_dsv41.IMAGE_PLACEHOLDER}", - } - ], - thinking_mode="chat", - ) - - def test_dsv41_tool_result_images_follow_prompt_order(self): - """Out-of-order tool results are sorted by call order; images must follow.""" - from sglang.srt.entrypoints.openai import encoding_dsv41 - - image = encoding_dsv41.IMAGE_PLACEHOLDER - history = [ - { - "role": "user", - "content": [ - {"type": "image_url", "image_url": {"url": "history.png"}}, - ], - }, - { - "role": "assistant", - "tool_calls": [ - { - "id": name, - "type": "function", - "function": {"name": name, "arguments": "{}"}, - } - for name in ("a", "b") - ], - }, - ] - results = [ - { - "type": "tool_result", - "tool_use_id": name, - "content": [ - {"type": "text", "text": f"result {name}"}, - {"type": "image_url", "image_url": {"url": f"{name}.png"}}, - ], - } - for name in ("b", "a") - ] - - for input_format in ( - "content_blocks", - "tool_messages", - "tool_content_blocks", - "context", - ): - with self.subTest(input_format=input_format): - if input_format in ("tool_messages", "tool_content_blocks"): - content_field = ( - "content_blocks" - if input_format == "tool_content_blocks" - else "content" - ) - messages = history + [ - { - "role": "tool", - "tool_call_id": block["tool_use_id"], - content_field: block["content"], - } - for block in results - ] - context = [] - elif input_format == "context": - messages = [{"role": "user", "content_blocks": results}] - context = history - else: - messages = history + [{"role": "user", "content_blocks": results}] - context = [] - original = copy.deepcopy((messages, context)) - - prompt, media = encoding_dsv41.encode_messages( - messages, - thinking_mode="chat", - context=context, - return_multi_modal_data=True, - ) - - self.assertTrue( - prompt.endswith( - f"<|User|>result a\n\n{image}\n\n" - f"result b\n\n{image}" - "<|Assistant|>" - ), - prompt, - ) - expected_urls = ( - ["a.png", "b.png"] if context else ["history.png", "a.png", "b.png"] - ) - self.assertEqual( - media, - {"images": [{"url": url} for url in expected_urls]}, - ) - self.assertEqual((messages, context), original) - - def test_dsv41_drop_thinking_without_tools(self): - """Tool results do not start a new turn, so current-turn reasoning is kept.""" - from sglang.srt.entrypoints.openai import encoding_dsv41 - - messages = [ - {"role": "user", "content": "q1"}, - {"role": "assistant", "content": "a1", "reasoning_content": "r1"}, - {"role": "user", "content": "q2"}, - { - "role": "assistant", - "reasoning_content": "r2", - "tool_calls": [ - { - "id": "c1", - "type": "function", - "function": {"name": "lookup", "arguments": "{}"}, - } - ], - }, - {"role": "tool", "tool_call_id": "c1", "content": "value"}, - ] - prompt = encoding_dsv41.encode_messages( - messages, thinking_mode="thinking", drop_thinking=True - ) - self.assertIn( - "<|User|>q1<|Assistant|>a1<|end▁of▁sentence|>", prompt - ) - self.assertNotIn("r1", prompt) - self.assertIn("r2", prompt) - self.assertTrue( - prompt.endswith("value<|Assistant|>") - ) - - next_turn = encoding_dsv41.encode_messages( - messages + [{"role": "user", "content": "q3"}], - thinking_mode="thinking", - ) - self.assertNotIn("r2", next_turn) - self.assertTrue(next_turn.endswith("q3<|Assistant|>")) - def test_streaming_abort_yields_error(self): """Test that an abort finish reason during streaming correctly yields an error and stops.""" err_msg = "Aborted by scheduler" @@ -5499,7 +5070,7 @@ class ServingChatTestCase(CustomTestCase): # ------------- hook method tests ------------- def test_encode_messages_returns_none_by_default(self): """Default _encode_messages returns None (use standard encoding).""" - result = self.chat._encode_messages([], Mock(), False) + result = self.chat._encode_messages([], Mock(), False, image_data=[]) self.assertIsNone(result) diff --git a/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py b/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py new file mode 100644 index 0000000000..97e83a7935 --- /dev/null +++ b/test/registered/unit/entrypoints/openai/test_serving_deepseek_recipe.py @@ -0,0 +1,298 @@ +import unittest +from types import SimpleNamespace + +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, is_in_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +MODEL = "deepseek-ai/DeepSeek-V4.1-Flash-1017" +REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170" + + +def _tokenizer_file(): + """The V4.1 tokenizer; only outside CI does an unreachable Hub skip the test.""" + from huggingface_hub import hf_hub_download + + try: + return hf_hub_download(MODEL, "tokenizer.json", revision=REVISION) + except Exception as error: # noqa: BLE001 + if is_in_ci(): + raise + raise unittest.SkipTest(f"the tokenizer is not available: {error}") + + +def _serving(test, tokenizer, *, is_multimodal=False, **server_args): + from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat + from sglang.srt.runtime_context import publish, reset_context + from sglang.srt.server_args import ServerArgs + + reset_context() + test.addCleanup(reset_context) + args = ServerArgs( + model_path="dummy", + reasoning_parser="deepseek-v41", + tool_call_parser="deepseekv41", + **server_args, + ) + publish(args, role="tokenizer") + manager = SimpleNamespace( + model_path="dummy", + served_model_name="dummy", + server_args=args, + tokenizer=tokenizer, + model_config=SimpleNamespace( + is_multimodal=is_multimodal, + context_len=131072, + hf_config=SimpleNamespace( + model_type="deepseek_v41", + architectures=["DeepseekV41ForCausalLM"], + to_dict=lambda: {}, + ), + get_default_sampling_params=lambda: {}, + ), + config_value=lambda name: getattr(args, name), + ) + template_manager = SimpleNamespace( + chat_template_name=None, + jinja_template_content_format=None, + reasoning_config=None, + force_reasoning=False, + python_chat_template=None, + jinja_template_may_reorder_tool_results=False, + ) + return OpenAIServingChat(manager, template_manager) + + +class TestAdapterRegistry(CustomTestCase): + def _context(self, spec, tokenizer=None, default_kwargs=None): + from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry + + return registry.ChatEncodingAdapterContext( + chat_encoding_spec=spec, + tokenizer=tokenizer, + reasoning_parser=None, + tool_call_parser=None, + chat_hooks=registry.ChatOutputHooks( + tool_call_parsing_active=lambda request: False, + history_tool_calls_count=lambda request: 0, + tool_call_id=lambda item, index: "", + ), + hf_config=SimpleNamespace( + model_type="deepseek_v41", architectures=["DeepseekV41ForCausalLM"] + ), + model_path="", + revision=None, + default_chat_template_kwargs=default_kwargs, + ) + + def _fast_tokenizer(self): + from tokenizers import Tokenizer + + return SimpleNamespace(backend_tokenizer=Tokenizer.from_file(_tokenizer_file())) + + def test_bad_default_effort_fails_startup(self): + from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry + + tokenizer = self._fast_tokenizer() + for spec, effort in (("dsv41", "extreme"), ("dsv4", 60)): + with self.subTest(spec=spec, effort=effort), self.assertRaises(ValueError): + registry.build_chat_encoding_adapter( + self._context( + spec, + tokenizer=tokenizer, + default_kwargs={"reasoning_effort": effort}, + ) + ) + + def test_no_tokenizer_builds_no_adapter(self): + """--skip-tokenizer-init servers start, with no chat prompts to encode.""" + from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry + + for spec in ("dsv4", "dsv41"): + with self.subTest(spec=spec): + self.assertIsNone( + registry.build_chat_encoding_adapter(self._context(spec)) + ) + + def test_tool_dumps_render_as_the_client_sent_them(self): + from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + adapter = registry.build_chat_encoding_adapter( + self._context("dsv41", tokenizer=self._fast_tokenizer()) + ) + # The handler's tool dumps carry None for unset fields. + tool = { + "type": "function", + "function": { + "name": "f", + "description": "d", + "parameters": {"type": "object", "properties": {}}, + }, + } + dumped = { + "type": "function", + "function": {**tool["function"], "strict": None}, + "defer_loading": None, + } + request = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}]) + self.assertEqual( + adapter.encode_messages( + messages=[], + request=request, + tools=[dumped], + thinking=False, + image_data=[], + ), + adapter.encode_messages( + messages=[], + request=request, + tools=[tool], + thinking=False, + image_data=[], + ), + ) + + +class TestHandlerRequestSemantics(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + from transformers import PreTrainedTokenizerFast + + cls.tokenizer = PreTrainedTokenizerFast(tokenizer_file=_tokenizer_file()) + + def _token(self, token): + return self.tokenizer.convert_tokens_to_ids(token) + + def _prompt(self, chat, **fields): + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + request = ChatCompletionRequest( + messages=[{"role": "user", "content": "hi"}], **fields + ) + return chat._process_messages(request, False) + + def test_thinking_defaults_off(self): + chat = _serving(self, self.tokenizer) + default = self._prompt(chat) + self.assertEqual(default.prompt_ids[-1], self._token("")) + self.assertFalse(default.require_reasoning) + thinking = self._prompt(chat, chat_template_kwargs={"thinking": True}) + self.assertEqual(thinking.prompt_ids[-1], self._token("")) + self.assertTrue(thinking.require_reasoning) + + def test_request_effort_wins_over_the_server_default(self): + thinking = {"chat_template_kwargs": {"thinking": True}} + plain = _serving(self, self.tokenizer) + low = self._prompt(plain, reasoning_effort="low", **thinking).prompt_ids + self.assertNotEqual( + low, self._prompt(plain, reasoning_effort="max", **thinking).prompt_ids + ) + chat = _serving( + self, + self.tokenizer, + default_chat_template_kwargs={"reasoning_effort": "max"}, + ) + self.assertEqual( + self._prompt(chat, reasoning_effort="low", **thinking).prompt_ids, low + ) + + def test_trailing_assistant_follows_sglang_rules(self): + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + chat = _serving(self, self.tokenizer) + + def prompt(messages, **fields): + request = ChatCompletionRequest(messages=messages, **fields) + return chat._process_messages(request, False).prompt_ids + + answer = {"role": "assistant", "content": "Hel"} + # Without continue_final_message the trailing turn becomes a user turn. + self.assertEqual( + prompt([{"role": "user", "content": "hi"}, answer]), + prompt( + [{"role": "user", "content": "hi"}, {"role": "user", "content": "Hel"}] + ), + ) + # A task renders its own tail; the continued prefix follows it. + self.assertEqual( + prompt( + [{"role": "user", "content": "hi"}, answer], + task="title", + continue_final_message=True, + ), + prompt([{"role": "user", "content": "hi"}], task="title") + + self.tokenizer.encode("Hel", add_special_tokens=False), + ) + + def test_tool_result_images_follow_prompt_order(self): + """Out-of-order tool results render in call order; their images must follow.""" + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + chat = _serving(self, self.tokenizer, is_multimodal=True) + names = ("a", "b") + urls = {name: f"https://example.com/{name}.png" for name in names} + calls = [ + { + "id": name, + "type": "function", + "function": {"name": name, "arguments": "{}"}, + } + for name in names + ] + results = [ + { + "role": "tool", + "tool_call_id": name, + "content": [ + {"type": "text", "text": f"result {name}"}, + {"type": "image_url", "image_url": {"url": urls[name]}}, + ], + } + for name in reversed(names) + ] + request = ChatCompletionRequest( + messages=[ + {"role": "user", "content": "both?"}, + {"role": "assistant", "content": "", "tool_calls": calls}, + *results, + ], + tools=[ + { + "type": "function", + "function": {"name": name, "parameters": {"type": "object"}}, + } + for name in names + ], + ) + processed = chat._process_messages(request, True) + prompt = self.tokenizer.decode(processed.prompt_ids) + self.assertLess(prompt.index("result a"), prompt.index("result b")) + self.assertEqual( + [image.url for image in processed.image_data], [urls["a"], urls["b"]] + ) + + def test_request_reaches_the_engine_with_sglang_semantics(self): + from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest + + chat = _serving(self, self.tokenizer) + request = ChatCompletionRequest( + messages=[{"role": "user", "content": "hi"}], + temperature=0.0, + chat_template_kwargs={"thinking": True, "reasoning_effort": 30}, + ) + adapted, _ = chat._convert_to_internal_request(request) + self.assertEqual(adapted.sampling_params["temperature"], 0.0) + self.assertIsNone(adapted.sampling_params["max_new_tokens"]) + self.assertIn( + "Reasoning Effort: 30 (range 1-100", + self.tokenizer.decode(adapted.input_ids), + ) + titled = self._prompt(chat, task="title").prompt_ids + self.assertEqual(titled[-1], self._token("<|title|>")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/unit/entrypoints/openai/test_serving_inkling.py b/test/registered/unit/entrypoints/openai/test_serving_inkling.py index 5fcacad624..e821420334 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_inkling.py +++ b/test/registered/unit/entrypoints/openai/test_serving_inkling.py @@ -111,6 +111,7 @@ class InklingPromptTest(unittest.TestCase): [message.model_dump() for message in request.messages], request, thinking_mode=None, + image_data=[], ) def test_prompt_tail(self): @@ -626,6 +627,8 @@ class InklingTokenOutputTest(CustomTestCase): messages=[message.model_dump() for message in request.messages], request=request, tools=None, + thinking=False, + image_data=[], ) key = reasoner.get_request_grammar_key( ("regex", "yes"), input_ids=prompt_ids, thinking_budget=None diff --git a/test/registered/unit/entrypoints/openai/test_serving_responses.py b/test/registered/unit/entrypoints/openai/test_serving_responses.py index 93d5fefda5..13ac0bd13d 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_responses.py +++ b/test/registered/unit/entrypoints/openai/test_serving_responses.py @@ -1448,7 +1448,7 @@ class ChatEncodingAdapterDispatchTestCase(CustomTestCase): ) with patch.dict(sys.modules, {"tml_renderers": None}): - for encoding in ("kimi_k3", "dsv4", "dsv41", "dsv32"): + for encoding in ("kimi_k3", "dsv32"): with self.subTest(encoding=encoding): self.assertIsNone( chat_encoding_adapters.build_chat_encoding_adapter( diff --git a/test/registered/unit/function_call/test_deepseekv41_detector.py b/test/registered/unit/function_call/test_deepseekv41_detector.py index a7a960d953..53b15d4e04 100644 --- a/test/registered/unit/function_call/test_deepseekv41_detector.py +++ b/test/registered/unit/function_call/test_deepseekv41_detector.py @@ -8,7 +8,6 @@ import xgrammar as xgr from xgrammar.structural_tag import JSONSchemaFormat from xgrammar.testing import _is_grammar_accept_string -from sglang.srt.entrypoints.openai import encoding_dsv41 from sglang.srt.entrypoints.openai.protocol import ( Function, Tool, @@ -17,6 +16,7 @@ from sglang.srt.entrypoints.openai.protocol import ( ) from sglang.srt.function_call.deepseekv41_detector import DeepSeekV41Detector from sglang.srt.function_call.function_call_parser import FunctionCallParser +from sglang.srt.parser.deepseek_recipe_renderer import load_deepseek_recipe from sglang.srt.parser.reasoning_parser import ReasoningParser from sglang.test.ci.ci_register import register_cpu_ci from sglang.test.test_utils import CustomTestCase @@ -25,6 +25,7 @@ register_cpu_ci(est_time=2, suite="base-a-test-cpu") CHUNK_SIZES = [1, 2, 3, 5, 7, 11, 23, 1000] DSML = "|DSML|" +EOS = "<|end▁of▁sentence|>" def _tools(): @@ -59,6 +60,34 @@ def _tools(): ] +def _render_assistant(message, *, thinking): + """The assistant turn as deepseek-recipe renders it, minus the generation + header (`` / ``) that ends the prompt before it.""" + recipe = load_deepseek_recipe() + calls = [ + {**call, "id": f"call_{i}"} for i, call in enumerate(message["tool_calls"]) + ] + results = [ + {"role": "tool", "tool_call_id": call["id"], "content": "ok"} for call in calls + ] + body = { + "model": "DeepSeek-V4.1-Flash", + "messages": [ + {"role": "user", "content": "question"}, + {**message, "tool_calls": calls}, + *results, + ], + "thinking": {"type": "enabled" if thinking else "disabled"}, + # Declared tools keep the turn's reasoning in the rendered history. + "tools": [tool.model_dump(exclude_none=True) for tool in _tools()], + } + converted = recipe.ChatCompletionRequest(body).convert(recipe.ConversionOptions()) + prompt = recipe.DeepseekV41Encoding().render_conversation(converted.conversation) + turn = prompt.prompt.split("<|User|>question<|Assistant|>", 1)[1] + turn = turn[: turn.index(EOS) + len(EOS)] + return turn.removeprefix("" if thinking else "") + + def _assemble(calls): """Streamed ToolCallItems -> [(name, parsed arguments)] per tool_index.""" by_index = {} @@ -81,26 +110,22 @@ class TestDeepSeekV41RoundTrip(CustomTestCase): def setUp(self): self.tools = _tools() - self.completion = encoding_dsv41.render_message( - 1, - [ - {"role": "user", "content": "question"}, - { - "role": "assistant", - "reasoning_content": "reason", - "content": "summary", - "tool_calls": [ - { - "type": "function", - "function": { - "name": "lookup", - "arguments": json.dumps(self.ARGUMENTS), - }, - } - ], - }, - ], - thinking_mode="thinking", + self.completion = _render_assistant( + { + "role": "assistant", + "reasoning_content": "reason", + "content": "summary", + "tool_calls": [ + { + "type": "function", + "function": { + "name": "lookup", + "arguments": json.dumps(self.ARGUMENTS), + }, + } + ], + }, + thinking=True, ) self.completion = "" + self.completion self.expected = [("lookup", self.ARGUMENTS)] @@ -221,29 +246,21 @@ class TestDeepSeekV41ParameterGrammar(CustomTestCase): @staticmethod def _render(arguments, *, thinking=False, count=1, name="get_weather"): - return encoding_dsv41.render_message( - 1, - [ - {"role": "user", "content": "question"}, - { - "role": "assistant", - "content": "", - "reasoning_content": "reason", - "wo_eos": True, - "tool_calls": [ - { - "type": "function", - "function": { - "name": name, - "arguments": json.dumps(arguments), - }, - } - ] - * count, - }, - ], - thinking_mode="thinking" if thinking else "chat", - ) + return _render_assistant( + { + "role": "assistant", + "content": "", + "reasoning_content": "reason", + "tool_calls": [ + { + "type": "function", + "function": {"name": name, "arguments": json.dumps(arguments)}, + } + ] + * count, + }, + thinking=thinking, + ).removesuffix(EOS) def _grammar(self, choice="required", thinking=False, parallel=True): constraint = FunctionCallParser( diff --git a/test/registered/unit/parser/test_deepseek_recipe_renderer.py b/test/registered/unit/parser/test_deepseek_recipe_renderer.py new file mode 100644 index 0000000000..6d59d1e3ad --- /dev/null +++ b/test/registered/unit/parser/test_deepseek_recipe_renderer.py @@ -0,0 +1,206 @@ +import unittest + +from sglang.srt.parser import deepseek_recipe_renderer as renderer +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase, is_in_ci + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + +MODEL = "deepseek-ai/DeepSeek-V4.1-Flash-1017" +REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170" + + +def _tokenizer_file(): + """The V4.1 tokenizer; only outside CI does an unreachable Hub skip the test.""" + from huggingface_hub import hf_hub_download + + try: + return hf_hub_download(MODEL, "tokenizer.json", revision=REVISION) + except Exception as error: # noqa: BLE001 + if is_in_ci(): + raise + raise unittest.SkipTest(f"the tokenizer is not available: {error}") + + +class TestPreNormalize(CustomTestCase): + def test_developer_becomes_system(self): + body = renderer.pre_normalize( + { + "messages": [ + {"role": "developer", "content": "Be terse."}, + {"role": "user", "content": "x"}, + ], + "thinking": False, + } + ) + self.assertEqual(body["messages"][0]["role"], "system") + + def test_tool_choice_renders_as_auto(self): + tools = [ + {"type": "function", "function": {"name": "a"}}, + {"type": "function", "function": {"name": "b"}}, + ] + named = renderer.pre_normalize( + { + "messages": [{"role": "user", "content": "x"}], + "thinking": False, + "tools": tools, + "tool_choice": {"type": "function", "function": {"name": "b"}}, + } + ) + self.assertEqual(named["tool_choice"], "auto") + self.assertEqual(len(named["tools"]), 2) + required = renderer.pre_normalize( + { + "messages": [{"role": "user", "content": "x"}], + "thinking": False, + "tools": tools, + "tool_choice": "required", + } + ) + self.assertEqual(required["tool_choice"], "auto") + for tools_field, choice in ((tools, "none"), ([], "auto")): + off = renderer.pre_normalize( + { + "messages": [{"role": "user", "content": "x"}], + "thinking": False, + "tools": tools_field, + "tool_choice": choice, + } + ) + self.assertNotIn("tools", off) + self.assertNotIn("tool_choice", off) + + def test_history_translation(self): + body = renderer.pre_normalize( + { + "messages": [ + {"role": "user", "content": "weather?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_0", + "type": "function", + "function": { + "name": "get_weather", + "arguments": {"location": "Paris", "days": 1.5}, + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call_0", "content": "sunny"}, + {"role": "assistant", "content": "It is"}, + ], + "thinking": False, + "continue_final_message": True, + "response_format": {"type": "structural_tag", "structures": []}, + } + ) + call = body["messages"][1]["tool_calls"][0] + self.assertEqual( + call["function"]["arguments"], '{"location":"Paris","days":1.5}' + ) + self.assertTrue(body["messages"][3]["prefix"]) + self.assertNotIn("continue_final_message", body) + self.assertNotIn("response_format", body) + + +class TestEffortBudget(CustomTestCase): + def test_integer_and_float_budgets(self): + self.assertEqual(renderer.parse_effort_budget(30), 30) + self.assertEqual(renderer.parse_effort_budget(0.5), 50) + self.assertEqual(renderer.parse_effort_budget(0.0), 1) + for value in (0, 101, 1.0, True): + with self.subTest(value=value), self.assertRaises(ValueError): + renderer.parse_effort_budget(value) + + +class TestRecipeEncoder(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + from tokenizers import Tokenizer + + cls.tokenizer = Tokenizer.from_file(_tokenizer_file()) + cls.encoder = renderer.RecipeEncoder("dsv41", cls.tokenizer) + + def _ids(self, *tokens): + return [self.tokenizer.token_to_id(token) for token in tokens] + + def test_encodes_with_the_served_vocabulary(self): + self.assertEqual( + self.encoder.encode( + {"messages": [{"role": "user", "content": ""}], "thinking": False} + ), + self._ids( + "<|begin▁of▁sentence|>", "<|User|>", "<|Assistant|>", "" + ), + ) + thinking = self.encoder.encode( + {"messages": [{"role": "user", "content": "hi"}], "thinking": True} + ) + self.assertEqual(thinking[-2:], self._ids("<|Assistant|>", "")) + + def test_tasks_replace_or_follow_the_generation_header(self): + for thinking in (False, True): + body = { + "messages": [{"role": "user", "content": "hello"}], + "thinking": thinking, + } + plain = self.encoder.encode(body) + for task, token in renderer.TASK_TOKENS.items(): + with self.subTest(thinking=thinking, task=task): + expected = ( + plain + self._ids(token) + if task == "action" + else plain[:-2] + self._ids(token) + ) + self.assertEqual(self.encoder.encode(body, task=task), expected) + continued = { + "messages": [ + {"role": "user", "content": "count"}, + {"role": "assistant", "content": "1, 2,"}, + ], + "thinking": False, + "continue_final_message": True, + } + with self.assertRaisesRegex(ValueError, "end with a user message"): + self.encoder.encode(continued, task="query") + + def test_effort_budget_rewrites_the_effort_line(self): + body = {"messages": [{"role": "user", "content": "hi"}], "thinking": True} + high = self.tokenizer.decode( + self.encoder.encode({**body, "reasoning_effort": "high"}), + skip_special_tokens=False, + ) + self.assertIn("Reasoning Effort: 75 (range 1-100", high) + for budget in (1, 30, 100): + with self.subTest(budget=budget): + expected = high.replace( + "Reasoning Effort: 75 ", f"Reasoning Effort: {budget} ", 1 + ) + self.assertEqual( + self.encoder.encode(body, effort_budget=budget), + self.tokenizer.encode(expected, add_special_tokens=False).ids, + ) + # Chat mode renders no effort line, so the budget changes nothing. + chat = {"messages": [{"role": "user", "content": "hi"}], "thinking": False} + self.assertEqual( + self.encoder.encode(chat, effort_budget=30), self.encoder.encode(chat) + ) + + def test_api_validation_surfaces_as_value_error(self): + with self.assertRaises(ValueError): + self.encoder.encode( + { + "messages": [{"role": "user", "content": "hi"}], + "thinking": True, + "reasoning_effort": "x", + } + ) + + +if __name__ == "__main__": + unittest.main()