[Feat] openai: Render DeepSeek-V4/V4.1 chat prompts with deepseek-recipe (#45315)
Co-authored-by: hnyls2002 <lsyincs@gmail.com>
This commit is contained in:
@@ -757,12 +757,12 @@ SGLang supports various environment variables that can be used to configure its
|
|||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV4_REASONING_EFFORT</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek V4 chat encoder when a request does not set it. The preview profile accepts <code>high</code> and <code>max</code>; the official profile accepts <code>low</code>, <code>high</code>, and <code>max</code>. The profile is detected from the bundled encoder. Override it with <code>--json-model-override-args '{"dsv4_reasoning_effort_profile":"official"}'</code>.</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for DeepSeek-V4 when a request does not set it: a deepseek-recipe tier name. Unset means deepseek-recipe's default.</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV41_REASONING_EFFORT</code></td>
|
<td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>SGLANG_DSV41_REASONING_EFFORT</code></td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for the DeepSeek-V4.1 chat encoder when a request does not set it: one of <code>low</code>, <code>high</code>, <code>xhigh</code>, <code>max</code>, or an integer budget in [1, 100]. Unset means the encoder default <code>high</code>.</td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Default <code>reasoning_effort</code> for DeepSeek-V4.1 when a request does not set it: a deepseek-recipe tier name or an integer budget in [1, 100]. Unset means deepseek-recipe's default.</td>
|
||||||
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
<td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>""</code></td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ dependencies = [
|
|||||||
"cuda-python>=13.0",
|
"cuda-python>=13.0",
|
||||||
"datasets",
|
"datasets",
|
||||||
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
|
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
|
||||||
|
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
|
||||||
"distro",
|
"distro",
|
||||||
"easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports
|
"easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports
|
||||||
"einops",
|
"einops",
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ dependencies = [
|
|||||||
"build",
|
"build",
|
||||||
"compressed-tensors",
|
"compressed-tensors",
|
||||||
"datasets",
|
"datasets",
|
||||||
|
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
|
||||||
"easydict",
|
"easydict",
|
||||||
"einops",
|
"einops",
|
||||||
"fastapi",
|
"fastapi",
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ dependencies = [
|
|||||||
"compressed-tensors",
|
"compressed-tensors",
|
||||||
"datasets",
|
"datasets",
|
||||||
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
|
"decord2 ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
|
||||||
|
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
|
||||||
"easydict",
|
"easydict",
|
||||||
"einops",
|
"einops",
|
||||||
"fastapi",
|
"fastapi",
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ runtime_base = [
|
|||||||
"blobfile==3.0.0",
|
"blobfile==3.0.0",
|
||||||
"build",
|
"build",
|
||||||
"datasets",
|
"datasets",
|
||||||
|
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
|
||||||
"easydict",
|
"easydict",
|
||||||
"einops",
|
"einops",
|
||||||
"fastapi",
|
"fastapi",
|
||||||
|
|||||||
@@ -24,6 +24,7 @@ dependencies = [
|
|||||||
"build",
|
"build",
|
||||||
"compressed-tensors",
|
"compressed-tensors",
|
||||||
"datasets",
|
"datasets",
|
||||||
|
"deepseek-recipe==0.2.0 ; (sys_platform == 'linux' and (platform_machine == 'x86_64' or platform_machine == 'aarch64')) or sys_platform == 'darwin'",
|
||||||
"easydict",
|
"easydict",
|
||||||
"einops",
|
"einops",
|
||||||
"fastapi",
|
"fastapi",
|
||||||
|
|||||||
@@ -7,102 +7,7 @@ it here instead of re-deriving it from model architectures themselves.
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import ast
|
from typing import Any, Dict, List, Optional
|
||||||
import logging
|
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Dict, List, Optional, Union
|
|
||||||
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv41
|
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
DSV4_REASONING_EFFORT_PROFILE_OVERRIDE = "dsv4_reasoning_effort_profile"
|
|
||||||
_DSV4_REASONING_EFFORT_ENCODER = "encoding/encoding_dsv4.py"
|
|
||||||
_MAX_DSV4_ENCODER_BYTES = 1 << 20
|
|
||||||
|
|
||||||
|
|
||||||
def _detect_dsv4_reasoning_effort_profile(
|
|
||||||
model_path: str, revision: Optional[str] = None
|
|
||||||
) -> Optional[str]:
|
|
||||||
encoder_path = Path(model_path) / _DSV4_REASONING_EFFORT_ENCODER
|
|
||||||
try:
|
|
||||||
if not encoder_path.is_file():
|
|
||||||
from huggingface_hub import hf_hub_download
|
|
||||||
|
|
||||||
encoder_path = Path(
|
|
||||||
hf_hub_download(
|
|
||||||
model_path,
|
|
||||||
_DSV4_REASONING_EFFORT_ENCODER,
|
|
||||||
revision=revision,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
if encoder_path.stat().st_size > _MAX_DSV4_ENCODER_BYTES:
|
|
||||||
return None
|
|
||||||
tree = ast.parse(encoder_path.read_text(encoding="utf-8"))
|
|
||||||
except Exception as error:
|
|
||||||
logger.debug(
|
|
||||||
"Could not inspect DeepSeek-V4 checkpoint encoder at %s: %s",
|
|
||||||
encoder_path,
|
|
||||||
error,
|
|
||||||
)
|
|
||||||
return None
|
|
||||||
|
|
||||||
assignments = {}
|
|
||||||
for node in tree.body:
|
|
||||||
if isinstance(node, ast.Assign):
|
|
||||||
targets = node.targets
|
|
||||||
value = node.value
|
|
||||||
elif isinstance(node, ast.AnnAssign):
|
|
||||||
targets = [node.target]
|
|
||||||
value = node.value
|
|
||||||
else:
|
|
||||||
continue
|
|
||||||
|
|
||||||
for target in targets:
|
|
||||||
if not isinstance(target, ast.Name):
|
|
||||||
continue
|
|
||||||
try:
|
|
||||||
assignments[target.id] = ast.literal_eval(value)
|
|
||||||
except (TypeError, ValueError):
|
|
||||||
continue
|
|
||||||
|
|
||||||
prompts = assignments.get("REASONING_EFFORT_PROMPTS")
|
|
||||||
if (
|
|
||||||
assignments.get("DEFAULT_REASONING_EFFORT") == "low"
|
|
||||||
and isinstance(prompts, dict)
|
|
||||||
and {"low", "high", "max"} <= prompts.keys()
|
|
||||||
):
|
|
||||||
return "official"
|
|
||||||
if "REASONING_EFFORT_MAX" in assignments:
|
|
||||||
return "preview"
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_dsv4_reasoning_effort_profile(profile: str) -> str:
|
|
||||||
if profile not in encoding_dsv4.REASONING_EFFORT_PROFILES:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid {DSV4_REASONING_EFFORT_PROFILE_OVERRIDE}: {profile!r}; "
|
|
||||||
f"expected one of {list(encoding_dsv4.REASONING_EFFORT_PROFILES)}"
|
|
||||||
)
|
|
||||||
return profile
|
|
||||||
|
|
||||||
|
|
||||||
def resolve_dsv4_reasoning_effort_profile(
|
|
||||||
*,
|
|
||||||
model_path: str,
|
|
||||||
revision: Optional[str] = None,
|
|
||||||
override: Optional[str] = None,
|
|
||||||
) -> str:
|
|
||||||
if override is not None:
|
|
||||||
return _validate_dsv4_reasoning_effort_profile(override)
|
|
||||||
|
|
||||||
return (
|
|
||||||
_detect_dsv4_reasoning_effort_profile(
|
|
||||||
model_path=model_path,
|
|
||||||
revision=revision,
|
|
||||||
)
|
|
||||||
or "preview"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def is_deepseek_v41_arch(*, arch: str, model_type: str) -> bool:
|
def is_deepseek_v41_arch(*, arch: str, model_type: str) -> bool:
|
||||||
@@ -154,56 +59,6 @@ def resolve_chat_encoding_spec(
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def parse_dsv41_reasoning_effort(value: Any) -> Union[str, int, None]:
|
|
||||||
"""Map an API ``reasoning_effort`` onto what the V4.1 encoder accepts.
|
|
||||||
|
|
||||||
An int budget only reaches here through ``chat_template_kwargs``; None
|
|
||||||
means unsupported, and the caller applies its default.
|
|
||||||
"""
|
|
||||||
if isinstance(value, bool):
|
|
||||||
return None
|
|
||||||
if isinstance(value, int):
|
|
||||||
return value if 1 <= value <= 100 else None
|
|
||||||
if isinstance(value, float):
|
|
||||||
return max(1, round(value * 100)) if 0.0 <= value <= 0.99 else None
|
|
||||||
if value in encoding_dsv41.REASONING_EFFORT_MAPPINGS:
|
|
||||||
return value
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
_OPENAI_FUNCTION_FIELD_ORDER = ("name", "description", "parameters")
|
|
||||||
|
|
||||||
|
|
||||||
def dsv41_tool_payload(tool: Any) -> Dict[str, Any]:
|
|
||||||
"""The tool dict the V4.1 encoder renders verbatim into the prompt.
|
|
||||||
|
|
||||||
Only fields the client sent, in the OpenAI field order; pydantic would
|
|
||||||
otherwise add defaults (strict=false) and reorder keys by declaration.
|
|
||||||
"""
|
|
||||||
payload = tool.model_dump(exclude_unset=True, exclude_none=True)
|
|
||||||
function = dict(payload.get("function") or {})
|
|
||||||
ordered = {
|
|
||||||
k: function.pop(k) for k in _OPENAI_FUNCTION_FIELD_ORDER if k in function
|
|
||||||
}
|
|
||||||
ordered.update(function)
|
|
||||||
payload["function"] = ordered
|
|
||||||
return payload
|
|
||||||
|
|
||||||
|
|
||||||
def default_dsv41_reasoning_effort_from_env(raw: Optional[str]) -> Union[str, int]:
|
|
||||||
"""Parse ``SGLANG_DSV41_REASONING_EFFORT``; raises so a bad value fails at boot."""
|
|
||||||
if raw is None or not raw.strip():
|
|
||||||
return encoding_dsv41.DEFAULT_REASONING_EFFORT
|
|
||||||
value: Any = int(raw) if raw.strip().isdigit() else raw.strip()
|
|
||||||
effort = parse_dsv41_reasoning_effort(value)
|
|
||||||
if effort is None:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid SGLANG_DSV41_REASONING_EFFORT={raw!r}; expected one of "
|
|
||||||
f"{list(encoding_dsv41.REASONING_EFFORT_MAPPINGS)} or an integer in [1, 100]"
|
|
||||||
)
|
|
||||||
return effort
|
|
||||||
|
|
||||||
|
|
||||||
def spec_supports_inline_system(spec: Optional[str]) -> bool:
|
def spec_supports_inline_system(spec: Optional[str]) -> bool:
|
||||||
"""Native encoders verified to render mid-conversation system messages in place."""
|
"""Native encoders verified to render mid-conversation system messages in place."""
|
||||||
# K3's tokenizer renders in Python, so there is no Jinja source to probe.
|
# K3's tokenizer renders in Python, so there is no Jinja source to probe.
|
||||||
@@ -244,34 +99,30 @@ def encode_simple_chat(
|
|||||||
|
|
||||||
Minimal encode for offline tools: no tools, no multimodal content, no
|
Minimal encode for offline tools: no tools, no multimodal content, no
|
||||||
continue_final_message; the serving path keeps its full request-level
|
continue_final_message; the serving path keeps its full request-level
|
||||||
pipeline in ``serving_chat``. System-message handling matches
|
pipeline in ``serving_chat``. dsv32 gets an empty system message prepended,
|
||||||
``serving_chat``: dsv4/dsv32 get an empty one prepended, dsv41 does not
|
as ``serving_chat`` does; DeepSeek-V4 / V4.1 render through deepseek-recipe.
|
||||||
(it renders a system token even for empty content).
|
|
||||||
"""
|
"""
|
||||||
if spec == "inkling":
|
if spec == "inkling":
|
||||||
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
from sglang.srt.parser.inkling_renderer import render_inkling_messages
|
||||||
|
|
||||||
return render_inkling_messages(messages)
|
return render_inkling_messages(messages)
|
||||||
|
|
||||||
if spec in ("dsv4", "dsv32", "dsv41"):
|
if spec in ("dsv4", "dsv41"):
|
||||||
if spec != "dsv41" and messages and messages[0]["role"] != "system":
|
from sglang.srt.parser.deepseek_recipe_renderer import RecipeEncoder
|
||||||
|
|
||||||
|
encoder = RecipeEncoder.from_hf_tokenizer(spec, tokenizer)
|
||||||
|
return encoder.encode(
|
||||||
|
{"messages": list(messages), "thinking": thinking_mode == "thinking"}
|
||||||
|
)
|
||||||
|
|
||||||
|
if spec == "dsv32":
|
||||||
|
from sglang.srt.entrypoints.openai import encoding_dsv32
|
||||||
|
|
||||||
|
if messages and messages[0]["role"] != "system":
|
||||||
messages = [{"role": "system", "content": ""}] + list(messages)
|
messages = [{"role": "system", "content": ""}] + list(messages)
|
||||||
if spec == "dsv4":
|
real_input = encoding_dsv32.encode_messages(
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
messages, thinking_mode=thinking_mode
|
||||||
|
)
|
||||||
real_input = encoding_dsv4.encode_messages(
|
|
||||||
messages, thinking_mode=thinking_mode
|
|
||||||
)
|
|
||||||
elif spec == "dsv41":
|
|
||||||
real_input = encoding_dsv41.encode_messages(
|
|
||||||
messages, thinking_mode=thinking_mode
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv32
|
|
||||||
|
|
||||||
real_input = encoding_dsv32.encode_messages(
|
|
||||||
messages, thinking_mode=thinking_mode
|
|
||||||
)
|
|
||||||
return tokenizer.encode(real_input)
|
return tokenizer.encode(real_input)
|
||||||
|
|
||||||
if getattr(tokenizer, "chat_template", None) is None:
|
if getattr(tokenizer, "chat_template", None) is None:
|
||||||
|
|||||||
@@ -70,6 +70,10 @@ class ChatEncodingAdapter(Protocol):
|
|||||||
"""Whether media still has to be collected from the OpenAI messages after
|
"""Whether media still has to be collected from the OpenAI messages after
|
||||||
``encode_messages`` returns prompt ids."""
|
``encode_messages`` returns prompt ids."""
|
||||||
|
|
||||||
|
strict_tool_call_arguments: bool
|
||||||
|
"""Whether history tool-call ``arguments`` must parse as JSON before
|
||||||
|
``encode_messages`` sees them; False when the encoding serializes them itself."""
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def output(self) -> Optional[ChatEncodingOutputAdapter]:
|
def output(self) -> Optional[ChatEncodingOutputAdapter]:
|
||||||
"""None when the configured parsers leave output to the text parsers."""
|
"""None when the configured parsers leave output to the text parsers."""
|
||||||
@@ -81,19 +85,26 @@ class ChatEncodingAdapter(Protocol):
|
|||||||
messages: list[dict[str, Any]],
|
messages: list[dict[str, Any]],
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
tools: Optional[list[dict]],
|
tools: Optional[list[dict]],
|
||||||
) -> list[int]: ...
|
thinking: bool,
|
||||||
|
image_data: list,
|
||||||
|
) -> list[int]:
|
||||||
|
"""``thinking`` is the handler's thinking decision for ``request``. An
|
||||||
|
encoding that collects media itself appends the prompt's images to
|
||||||
|
``image_data`` in placeholder order."""
|
||||||
|
...
|
||||||
|
|
||||||
|
|
||||||
class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
|
class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
|
||||||
"""Everything a builder may read; a builder takes only what its encoding needs.
|
"""Everything a builder may read; a builder takes only what its encoding needs.
|
||||||
|
|
||||||
Consumers:
|
Consumers:
|
||||||
tokenizer: prompt encoding (Inkling).
|
tokenizer: prompt encoding (Inkling, DeepSeek-V4 / V4.1).
|
||||||
reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing.
|
reasoning_parser, tool_call_parser, chat_hooks: adapter-owned output parsing.
|
||||||
hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id).
|
hf_config: config-dependent encoding (DeepSeek V4.1 image_token_id).
|
||||||
model_path, revision: checkpoint files (DeepSeek tokenizer.json and the
|
model_path, revision: checkpoint files.
|
||||||
encoder file for effort-profile detection).
|
default_chat_template_kwargs: the workers' --default-chat-template-kwargs
|
||||||
backend: selects the builder; "recipe" is an explicit opt-in.
|
(deepseek-recipe validates the default effort at startup).
|
||||||
|
backend: selects the builder; every spec has a "builtin" one.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
chat_encoding_spec: str
|
chat_encoding_spec: str
|
||||||
@@ -104,6 +115,7 @@ class ChatEncodingAdapterContext(msgspec.Struct, frozen=True, kw_only=True):
|
|||||||
hf_config: Any
|
hf_config: Any
|
||||||
model_path: str
|
model_path: str
|
||||||
revision: Optional[str]
|
revision: Optional[str]
|
||||||
|
default_chat_template_kwargs: Optional[dict[str, Any]] = None
|
||||||
backend: EncodingBackend = "builtin"
|
backend: EncodingBackend = "builtin"
|
||||||
|
|
||||||
def require(self, field: str) -> Any:
|
def require(self, field: str) -> Any:
|
||||||
@@ -127,11 +139,32 @@ def _build_inkling(ctx: ChatEncodingAdapterContext) -> ChatEncodingAdapter:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_deepseek_recipe(
|
||||||
|
ctx: ChatEncodingAdapterContext,
|
||||||
|
) -> Optional[ChatEncodingAdapter]:
|
||||||
|
if ctx.tokenizer is None:
|
||||||
|
return None # --skip-tokenizer-init: the server encodes no chat prompts.
|
||||||
|
from sglang.srt.entrypoints.openai.serving_deepseek_recipe import (
|
||||||
|
DeepSeekRecipeChatEncodingAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
return DeepSeekRecipeChatEncodingAdapter(
|
||||||
|
chat_encoding_spec=ctx.chat_encoding_spec,
|
||||||
|
tokenizer=ctx.tokenizer,
|
||||||
|
default_chat_template_kwargs=ctx.default_chat_template_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
_BUILDERS: dict[
|
_BUILDERS: dict[
|
||||||
str,
|
str,
|
||||||
dict[EncodingBackend, Callable[[ChatEncodingAdapterContext], ChatEncodingAdapter]],
|
dict[
|
||||||
|
EncodingBackend,
|
||||||
|
Callable[[ChatEncodingAdapterContext], Optional[ChatEncodingAdapter]],
|
||||||
|
],
|
||||||
] = {
|
] = {
|
||||||
"inkling": {"builtin": _build_inkling},
|
"inkling": {"builtin": _build_inkling},
|
||||||
|
"dsv4": {"builtin": _build_deepseek_recipe},
|
||||||
|
"dsv41": {"builtin": _build_deepseek_recipe},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,884 +0,0 @@
|
|||||||
# Adapted from the DeepSeek-V4 release reference implementation.
|
|
||||||
"""
|
|
||||||
DeepSeek-V4 Encoding
|
|
||||||
|
|
||||||
A self-contained implementation for encoding/decoding DeepSeek-V4 chat messages
|
|
||||||
with tool calling, thinking mode, and quick instruction task support.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import copy
|
|
||||||
import json
|
|
||||||
import re
|
|
||||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Special Tokens
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
bos_token: str = "<|begin▁of▁sentence|>"
|
|
||||||
eos_token: str = "<|end▁of▁sentence|>"
|
|
||||||
thinking_start_token: str = "<think>"
|
|
||||||
thinking_end_token: str = "</think>"
|
|
||||||
dsml_token: str = "|DSML|"
|
|
||||||
|
|
||||||
USER_SP_TOKEN = "<|User|>"
|
|
||||||
ASSISTANT_SP_TOKEN = "<|Assistant|>"
|
|
||||||
LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>"
|
|
||||||
|
|
||||||
# Task special tokens for internal classification tasks
|
|
||||||
DS_TASK_SP_TOKENS = {
|
|
||||||
"action": "<|action|>",
|
|
||||||
"query": "<|query|>",
|
|
||||||
"authority": "<|authority|>",
|
|
||||||
"domain": "<|domain|>",
|
|
||||||
"title": "<|title|>",
|
|
||||||
"read_url": "<|read_url|>",
|
|
||||||
}
|
|
||||||
VALID_TASKS = set(DS_TASK_SP_TOKENS.keys())
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Templates
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
system_msg_template: str = "{content}"
|
|
||||||
user_msg_template: str = "{content}"
|
|
||||||
latest_reminder_msg_template: str = "{content}"
|
|
||||||
assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token
|
|
||||||
assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}"
|
|
||||||
thinking_template: str = "{reasoning_content}"
|
|
||||||
|
|
||||||
response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"
|
|
||||||
tool_call_template: str = (
|
|
||||||
'<{dsml_token}invoke name="{name}">\n{arguments}\n</{dsml_token}invoke>'
|
|
||||||
)
|
|
||||||
tool_calls_template = (
|
|
||||||
"<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
|
|
||||||
)
|
|
||||||
tool_calls_block_name: str = "tool_calls"
|
|
||||||
|
|
||||||
tool_output_template: str = "<tool_result>{content}</tool_result>"
|
|
||||||
|
|
||||||
REASONING_EFFORT_PREVIEW_MAX = (
|
|
||||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
|
||||||
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
|
||||||
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
REASONING_EFFORT_OFFICIAL_MAX = (
|
|
||||||
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
|
|
||||||
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
|
|
||||||
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
REASONING_EFFORT_PROFILES = {
|
|
||||||
"preview": {
|
|
||||||
"high": "",
|
|
||||||
"max": REASONING_EFFORT_PREVIEW_MAX,
|
|
||||||
},
|
|
||||||
"official": {
|
|
||||||
"low": "",
|
|
||||||
"high": REASONING_EFFORT_PREVIEW_MAX,
|
|
||||||
"max": REASONING_EFFORT_OFFICIAL_MAX,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
TOOLS_TEMPLATE = """## Tools
|
|
||||||
|
|
||||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following:
|
|
||||||
|
|
||||||
<{dsml_token}tool_calls>
|
|
||||||
<{dsml_token}invoke name="$TOOL_NAME">
|
|
||||||
<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}parameter>
|
|
||||||
...
|
|
||||||
</{dsml_token}invoke>
|
|
||||||
<{dsml_token}invoke name="$TOOL_NAME2">
|
|
||||||
...
|
|
||||||
</{dsml_token}invoke>
|
|
||||||
</{dsml_token}tool_calls>
|
|
||||||
|
|
||||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
|
||||||
|
|
||||||
If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
|
|
||||||
|
|
||||||
Otherwise, output directly after {thinking_end_token} with tool calls or final response.
|
|
||||||
|
|
||||||
### Available Tool Schemas
|
|
||||||
|
|
||||||
{tool_schemas}
|
|
||||||
|
|
||||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Utility Functions
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def to_json(value: Any) -> str:
|
|
||||||
"""Serialize a value to JSON string."""
|
|
||||||
try:
|
|
||||||
return json.dumps(value, ensure_ascii=False)
|
|
||||||
except:
|
|
||||||
return json.dumps(value, ensure_ascii=True)
|
|
||||||
|
|
||||||
|
|
||||||
def tools_from_openai_format(tools):
|
|
||||||
"""Extract function definitions from OpenAI-format tool list."""
|
|
||||||
return [tool["function"] for tool in tools]
|
|
||||||
|
|
||||||
|
|
||||||
def tool_calls_from_openai_format(tool_calls):
|
|
||||||
"""Convert OpenAI-format tool calls to internal format."""
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"name": tool_call["function"]["name"],
|
|
||||||
"arguments": tool_call["function"]["arguments"],
|
|
||||||
}
|
|
||||||
for tool_call in tool_calls
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def tool_calls_to_openai_format(tool_calls):
|
|
||||||
"""Convert internal tool calls to OpenAI format."""
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"type": "function",
|
|
||||||
"function": {
|
|
||||||
"name": tool_call["name"],
|
|
||||||
"arguments": tool_call["arguments"],
|
|
||||||
},
|
|
||||||
}
|
|
||||||
for tool_call in tool_calls
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str:
|
|
||||||
"""
|
|
||||||
Encode tool call arguments into DSML parameter format.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tool_call: Dict with "name" and "arguments" keys.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
DSML-formatted parameter string.
|
|
||||||
"""
|
|
||||||
p_dsml_template = '<{dsml_token}parameter name="{key}" string="{is_str}">{value}</{dsml_token}parameter>'
|
|
||||||
P_dsml_strs = []
|
|
||||||
|
|
||||||
raw_arguments = tool_call["arguments"]
|
|
||||||
arguments = (
|
|
||||||
json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
|
|
||||||
)
|
|
||||||
if not isinstance(arguments, dict):
|
|
||||||
raise ValueError(
|
|
||||||
"Assistant tool call function.arguments must be a JSON object."
|
|
||||||
)
|
|
||||||
|
|
||||||
for k, v in arguments.items():
|
|
||||||
p_dsml_str = p_dsml_template.format(
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
key=k,
|
|
||||||
is_str="true" if isinstance(v, str) else "false",
|
|
||||||
value=v if isinstance(v, str) else to_json(v),
|
|
||||||
)
|
|
||||||
P_dsml_strs.append(p_dsml_str)
|
|
||||||
|
|
||||||
return "\n".join(P_dsml_strs)
|
|
||||||
|
|
||||||
|
|
||||||
def decode_dsml_to_arguments(
|
|
||||||
tool_name: str, tool_args: Dict[str, Tuple[str, str]]
|
|
||||||
) -> Dict[str, str]:
|
|
||||||
"""
|
|
||||||
Decode DSML parameters back to a tool call dict.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tool_name: Name of the tool.
|
|
||||||
tool_args: Dict mapping param_name -> (value, is_string_flag).
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with "name" and "arguments" (JSON string) keys.
|
|
||||||
"""
|
|
||||||
|
|
||||||
def _decode_value(key: str, value: str, string: str):
|
|
||||||
if string == "true":
|
|
||||||
value = to_json(value)
|
|
||||||
return f"{to_json(key)}: {value}"
|
|
||||||
|
|
||||||
tool_args_json = (
|
|
||||||
"{"
|
|
||||||
+ ", ".join(
|
|
||||||
[_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()]
|
|
||||||
)
|
|
||||||
+ "}"
|
|
||||||
)
|
|
||||||
return dict(name=tool_name, arguments=tool_args_json)
|
|
||||||
|
|
||||||
|
|
||||||
def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str:
|
|
||||||
"""
|
|
||||||
Render tool schemas into the system prompt format.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tools: List of tool schema dicts (each with name, description, parameters).
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Formatted tools section string.
|
|
||||||
"""
|
|
||||||
tools_json = [to_json(t) for t in tools]
|
|
||||||
|
|
||||||
return TOOLS_TEMPLATE.format(
|
|
||||||
tool_schemas="\n".join(tools_json),
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
thinking_start_token=thinking_start_token,
|
|
||||||
thinking_end_token=thinking_end_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
|
|
||||||
"""Find the index of the last user/developer message."""
|
|
||||||
last_user_index = -1
|
|
||||||
for idx in range(len(messages) - 1, -1, -1):
|
|
||||||
if messages[idx].get("role") in ["user", "developer"]:
|
|
||||||
last_user_index = idx
|
|
||||||
break
|
|
||||||
return last_user_index
|
|
||||||
|
|
||||||
|
|
||||||
def attach_task_to_last_user_message(messages: List[Dict[str, Any]], task: str) -> None:
|
|
||||||
"""Set `task` on the most recent user/developer message; raise if none exists."""
|
|
||||||
idx = find_last_user_index(messages)
|
|
||||||
if idx == -1:
|
|
||||||
raise ValueError(
|
|
||||||
"`task` requires at least one message with role='user' or 'developer'."
|
|
||||||
)
|
|
||||||
messages[idx]["task"] = task
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Message Rendering
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def render_message(
|
|
||||||
index: int,
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
thinking_mode: str,
|
|
||||||
drop_thinking: bool = True,
|
|
||||||
reasoning_effort: Optional[str] = None,
|
|
||||||
reasoning_effort_profile: str = "preview",
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Render a single message at the given index into its encoded string form.
|
|
||||||
|
|
||||||
This is the core function that converts each message in the conversation
|
|
||||||
into the DeepSeek-V4 format.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
index: Index of the message to render.
|
|
||||||
messages: Full list of messages in the conversation.
|
|
||||||
thinking_mode: Either "chat" or "thinking".
|
|
||||||
drop_thinking: Whether to drop reasoning content from earlier turns.
|
|
||||||
reasoning_effort: Optional reasoning effort level. The preview profile accepts
|
|
||||||
"high" and "max"; the official profile accepts "low", "high", and "max".
|
|
||||||
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Encoded string for this message.
|
|
||||||
"""
|
|
||||||
assert 0 <= index < len(messages)
|
|
||||||
assert thinking_mode in [
|
|
||||||
"chat",
|
|
||||||
"thinking",
|
|
||||||
], f"Invalid thinking_mode `{thinking_mode}`"
|
|
||||||
|
|
||||||
prompt = ""
|
|
||||||
msg = messages[index]
|
|
||||||
last_user_idx = find_last_user_index(messages)
|
|
||||||
|
|
||||||
role = msg.get("role")
|
|
||||||
content = msg.get("content")
|
|
||||||
tools = msg.get("tools")
|
|
||||||
response_format = msg.get("response_format")
|
|
||||||
tool_calls = msg.get("tool_calls")
|
|
||||||
reasoning_content = msg.get("reasoning_content")
|
|
||||||
wo_eos = msg.get("wo_eos", False)
|
|
||||||
|
|
||||||
if tools:
|
|
||||||
tools = tools_from_openai_format(tools)
|
|
||||||
if tool_calls:
|
|
||||||
tool_calls = tool_calls_from_openai_format(tool_calls)
|
|
||||||
|
|
||||||
if reasoning_effort_profile not in REASONING_EFFORT_PROFILES:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid reasoning effort profile: {reasoning_effort_profile!r}; "
|
|
||||||
f"expected one of {list(REASONING_EFFORT_PROFILES)}"
|
|
||||||
)
|
|
||||||
effort_prompts = REASONING_EFFORT_PROFILES[reasoning_effort_profile]
|
|
||||||
if reasoning_effort is None:
|
|
||||||
reasoning_effort = "low" if reasoning_effort_profile == "official" else "high"
|
|
||||||
if reasoning_effort not in effort_prompts:
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid reasoning effort {reasoning_effort!r} for profile "
|
|
||||||
f"{reasoning_effort_profile!r}; expected one of {list(effort_prompts)}"
|
|
||||||
)
|
|
||||||
if index == 0 and thinking_mode == "thinking":
|
|
||||||
prompt += effort_prompts[reasoning_effort]
|
|
||||||
|
|
||||||
if role == "system":
|
|
||||||
prompt += system_msg_template.format(content=content or "")
|
|
||||||
if tools:
|
|
||||||
prompt += "\n\n" + render_tools(tools)
|
|
||||||
if response_format:
|
|
||||||
prompt += "\n\n" + response_format_template.format(
|
|
||||||
schema=to_json(response_format)
|
|
||||||
)
|
|
||||||
|
|
||||||
elif role == "developer":
|
|
||||||
assert content, f"Invalid message for role `{role}`: {msg}"
|
|
||||||
|
|
||||||
content_developer = USER_SP_TOKEN
|
|
||||||
content_developer += content
|
|
||||||
|
|
||||||
if tools:
|
|
||||||
content_developer += "\n\n" + render_tools(tools)
|
|
||||||
if response_format:
|
|
||||||
content_developer += "\n\n" + response_format_template.format(
|
|
||||||
schema=to_json(response_format)
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt += user_msg_template.format(content=content_developer)
|
|
||||||
|
|
||||||
elif role == "user":
|
|
||||||
prompt += USER_SP_TOKEN
|
|
||||||
|
|
||||||
# Handle content blocks (tool results mixed with text)
|
|
||||||
content_blocks = msg.get("content_blocks")
|
|
||||||
if content_blocks:
|
|
||||||
parts = []
|
|
||||||
for block in content_blocks:
|
|
||||||
block_type = block.get("type")
|
|
||||||
if block_type == "text":
|
|
||||||
parts.append(block.get("text", ""))
|
|
||||||
elif block_type == "tool_result":
|
|
||||||
tool_content = block.get("content", "")
|
|
||||||
if isinstance(tool_content, list):
|
|
||||||
text_parts = []
|
|
||||||
for b in tool_content:
|
|
||||||
if b.get("type") == "text":
|
|
||||||
text_parts.append(b.get("text", ""))
|
|
||||||
else:
|
|
||||||
text_parts.append(f"[Unsupported {b.get('type')}]")
|
|
||||||
tool_content = "\n\n".join(text_parts)
|
|
||||||
parts.append(tool_output_template.format(content=tool_content))
|
|
||||||
else:
|
|
||||||
parts.append(f"[Unsupported {block_type}]")
|
|
||||||
prompt += "\n\n".join(parts)
|
|
||||||
else:
|
|
||||||
prompt += content or ""
|
|
||||||
|
|
||||||
elif role == "latest_reminder":
|
|
||||||
prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(
|
|
||||||
content=content
|
|
||||||
)
|
|
||||||
|
|
||||||
elif role == "tool":
|
|
||||||
raise NotImplementedError(
|
|
||||||
"deepseek_v4 merges tool messages into user; please preprocess with merge_tool_messages()"
|
|
||||||
)
|
|
||||||
|
|
||||||
elif role == "assistant":
|
|
||||||
thinking_part = ""
|
|
||||||
tc_content = ""
|
|
||||||
|
|
||||||
if tool_calls:
|
|
||||||
tc_list = [
|
|
||||||
tool_call_template.format(
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
name=tc.get("name"),
|
|
||||||
arguments=encode_arguments_to_dsml(tc),
|
|
||||||
)
|
|
||||||
for tc in tool_calls
|
|
||||||
]
|
|
||||||
tc_content += "\n\n" + tool_calls_template.format(
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
tool_calls="\n".join(tc_list),
|
|
||||||
tc_block_name=tool_calls_block_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
summary_content = content or ""
|
|
||||||
rc = reasoning_content or ""
|
|
||||||
|
|
||||||
# Check if previous message has a task - if so, this is a task output (no thinking)
|
|
||||||
prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None
|
|
||||||
|
|
||||||
if thinking_mode == "thinking" and not prev_has_task:
|
|
||||||
if not drop_thinking or index > last_user_idx:
|
|
||||||
thinking_part = (
|
|
||||||
thinking_template.format(reasoning_content=rc) + thinking_end_token
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
thinking_part = ""
|
|
||||||
|
|
||||||
if wo_eos:
|
|
||||||
prompt += assistant_msg_wo_eos_template.format(
|
|
||||||
reasoning=thinking_part,
|
|
||||||
content=summary_content,
|
|
||||||
tool_calls=tc_content,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
prompt += assistant_msg_template.format(
|
|
||||||
reasoning=thinking_part,
|
|
||||||
content=summary_content,
|
|
||||||
tool_calls=tc_content,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise NotImplementedError(f"Unknown role: {role}")
|
|
||||||
|
|
||||||
# Append transition tokens based on what follows
|
|
||||||
if index + 1 < len(messages) and messages[index + 1].get("role") not in [
|
|
||||||
"assistant",
|
|
||||||
"latest_reminder",
|
|
||||||
]:
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
task = messages[index].get("task")
|
|
||||||
if task is not None:
|
|
||||||
# Task special token for internal classification tasks
|
|
||||||
assert task in VALID_TASKS, (
|
|
||||||
f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
|
|
||||||
)
|
|
||||||
task_sp_token = DS_TASK_SP_TOKENS[task]
|
|
||||||
|
|
||||||
if task != "action":
|
|
||||||
# Non-action tasks: append task sp token directly after the message
|
|
||||||
prompt += task_sp_token
|
|
||||||
else:
|
|
||||||
# Action task: append Assistant + thinking token + action sp token
|
|
||||||
prompt += ASSISTANT_SP_TOKEN
|
|
||||||
prompt += (
|
|
||||||
thinking_end_token
|
|
||||||
if thinking_mode != "thinking"
|
|
||||||
else thinking_start_token
|
|
||||||
)
|
|
||||||
prompt += task_sp_token
|
|
||||||
|
|
||||||
elif messages[index].get("role") in ["user", "developer"]:
|
|
||||||
# Normal generation: append Assistant + thinking token
|
|
||||||
prompt += ASSISTANT_SP_TOKEN
|
|
||||||
if not drop_thinking and thinking_mode == "thinking":
|
|
||||||
prompt += thinking_start_token
|
|
||||||
elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx:
|
|
||||||
prompt += thinking_start_token
|
|
||||||
else:
|
|
||||||
prompt += thinking_end_token
|
|
||||||
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Preprocessing
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Merge tool messages into the preceding user message using content_blocks format.
|
|
||||||
|
|
||||||
DeepSeek-V4 does not have a standalone "tool" role; instead, tool results
|
|
||||||
are encoded as <tool_result> blocks within user messages.
|
|
||||||
|
|
||||||
This function converts a standard OpenAI-format conversation (with separate
|
|
||||||
"tool" role messages) into V4 format where tool results are merged into
|
|
||||||
user messages.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
messages: List of message dicts in OpenAI format.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Processed message list with tool messages merged into user messages.
|
|
||||||
"""
|
|
||||||
merged: List[Dict[str, Any]] = []
|
|
||||||
|
|
||||||
for msg in messages:
|
|
||||||
msg = copy.deepcopy(msg)
|
|
||||||
role = msg.get("role")
|
|
||||||
|
|
||||||
if role == "tool":
|
|
||||||
# Convert tool message to a user message with tool_result block
|
|
||||||
tool_block = {
|
|
||||||
"type": "tool_result",
|
|
||||||
"tool_use_id": msg.get("tool_call_id", ""),
|
|
||||||
"content": msg.get("content", ""),
|
|
||||||
}
|
|
||||||
# Merge into previous message if it's already a user (merged tool)
|
|
||||||
if (
|
|
||||||
merged
|
|
||||||
and merged[-1].get("role") == "user"
|
|
||||||
and "content_blocks" in merged[-1]
|
|
||||||
):
|
|
||||||
merged[-1]["content_blocks"].append(tool_block)
|
|
||||||
else:
|
|
||||||
merged.append(
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content_blocks": [tool_block],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
elif role == "user":
|
|
||||||
text_block = {"type": "text", "text": msg.get("content", "")}
|
|
||||||
if (
|
|
||||||
merged
|
|
||||||
and merged[-1].get("role") == "user"
|
|
||||||
and "content_blocks" in merged[-1]
|
|
||||||
and merged[-1].get("task") is None
|
|
||||||
):
|
|
||||||
merged[-1]["content_blocks"].append(text_block)
|
|
||||||
else:
|
|
||||||
new_msg = {
|
|
||||||
"role": "user",
|
|
||||||
"content": msg.get("content", ""),
|
|
||||||
"content_blocks": [text_block],
|
|
||||||
}
|
|
||||||
# Preserve extra fields (task, wo_eos, mask, etc.)
|
|
||||||
for key in ("task", "wo_eos", "mask"):
|
|
||||||
if key in msg:
|
|
||||||
new_msg[key] = msg[key]
|
|
||||||
merged.append(new_msg)
|
|
||||||
else:
|
|
||||||
merged.append(msg)
|
|
||||||
|
|
||||||
return merged
|
|
||||||
|
|
||||||
|
|
||||||
def sort_tool_results_by_call_order(
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Sort tool_result blocks within user messages by the order of tool_calls
|
|
||||||
in the preceding assistant message.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
messages: Preprocessed message list (after merge_tool_messages).
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Message list with sorted tool result blocks.
|
|
||||||
"""
|
|
||||||
last_tool_call_order: Dict[str, int] = {}
|
|
||||||
|
|
||||||
for msg in messages:
|
|
||||||
role = msg.get("role")
|
|
||||||
if role == "assistant" and msg.get("tool_calls"):
|
|
||||||
last_tool_call_order = {}
|
|
||||||
for idx, tc in enumerate(msg["tool_calls"]):
|
|
||||||
tc_id = tc.get("id") or tc.get("function", {}).get("id", "")
|
|
||||||
if tc_id:
|
|
||||||
last_tool_call_order[tc_id] = idx
|
|
||||||
|
|
||||||
elif role == "user" and msg.get("content_blocks"):
|
|
||||||
tool_blocks = [
|
|
||||||
b for b in msg["content_blocks"] if b.get("type") == "tool_result"
|
|
||||||
]
|
|
||||||
if len(tool_blocks) > 1 and last_tool_call_order:
|
|
||||||
sorted_blocks = sorted(
|
|
||||||
tool_blocks,
|
|
||||||
key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0),
|
|
||||||
)
|
|
||||||
sorted_idx = 0
|
|
||||||
new_blocks = []
|
|
||||||
for block in msg["content_blocks"]:
|
|
||||||
if block.get("type") == "tool_result":
|
|
||||||
new_blocks.append(sorted_blocks[sorted_idx])
|
|
||||||
sorted_idx += 1
|
|
||||||
else:
|
|
||||||
new_blocks.append(block)
|
|
||||||
msg["content_blocks"] = new_blocks
|
|
||||||
|
|
||||||
return messages
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Main Encoding Function
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def encode_messages(
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
thinking_mode: str,
|
|
||||||
context: Optional[List[Dict[str, Any]]] = None,
|
|
||||||
drop_thinking: bool = True,
|
|
||||||
add_default_bos_token: bool = True,
|
|
||||||
reasoning_effort: Optional[str] = None,
|
|
||||||
reasoning_effort_profile: str = "preview",
|
|
||||||
) -> str:
|
|
||||||
"""
|
|
||||||
Encode a list of messages into the DeepSeek-V4 prompt format.
|
|
||||||
|
|
||||||
This is the main entry point for encoding conversations. It handles:
|
|
||||||
- BOS token insertion
|
|
||||||
- Thinking mode with optional reasoning content dropping
|
|
||||||
- Tool message merging into user messages
|
|
||||||
- Multi-turn conversation context
|
|
||||||
|
|
||||||
Args:
|
|
||||||
messages: List of message dicts to encode.
|
|
||||||
thinking_mode: Either "chat" or "thinking".
|
|
||||||
context: Optional preceding context messages (already encoded prefix).
|
|
||||||
drop_thinking: If True, drop reasoning_content from earlier assistant turns
|
|
||||||
(only keep reasoning for messages after the last user message).
|
|
||||||
add_default_bos_token: Whether to prepend BOS token at conversation start.
|
|
||||||
reasoning_effort: Optional reasoning effort level. The preview profile accepts
|
|
||||||
"high" and "max"; the official profile accepts "low", "high", and "max".
|
|
||||||
reasoning_effort_profile: DeepSeek-V4 effort mapping ("preview" or "official").
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
The encoded prompt string.
|
|
||||||
"""
|
|
||||||
context = context if context else []
|
|
||||||
|
|
||||||
# Preprocess: merge tool messages and sort tool results
|
|
||||||
messages = merge_tool_messages(messages)
|
|
||||||
messages = sort_tool_results_by_call_order(context + messages)[len(context) :]
|
|
||||||
if context:
|
|
||||||
context = merge_tool_messages(context)
|
|
||||||
context = sort_tool_results_by_call_order(context)
|
|
||||||
|
|
||||||
full_messages = context + messages
|
|
||||||
|
|
||||||
prompt = bos_token if add_default_bos_token and len(context) == 0 else ""
|
|
||||||
|
|
||||||
# Resolve drop_thinking: if any message has tools defined, don't drop thinking
|
|
||||||
effective_drop_thinking = drop_thinking
|
|
||||||
if any(m.get("tools") for m in full_messages):
|
|
||||||
effective_drop_thinking = False
|
|
||||||
|
|
||||||
if thinking_mode == "thinking" and effective_drop_thinking:
|
|
||||||
full_messages = _drop_thinking_messages(full_messages)
|
|
||||||
# After dropping, recalculate how many messages to render
|
|
||||||
# (context may have shrunk too)
|
|
||||||
num_to_render = len(full_messages) - len(_drop_thinking_messages(context))
|
|
||||||
context_len = len(full_messages) - num_to_render
|
|
||||||
else:
|
|
||||||
num_to_render = len(messages)
|
|
||||||
context_len = len(context)
|
|
||||||
|
|
||||||
for idx in range(num_to_render):
|
|
||||||
prompt += render_message(
|
|
||||||
idx + context_len,
|
|
||||||
full_messages,
|
|
||||||
thinking_mode=thinking_mode,
|
|
||||||
drop_thinking=effective_drop_thinking,
|
|
||||||
reasoning_effort=reasoning_effort,
|
|
||||||
reasoning_effort_profile=reasoning_effort_profile,
|
|
||||||
)
|
|
||||||
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
|
|
||||||
def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
||||||
"""
|
|
||||||
Drop reasoning_content and non-essential messages before the last user message.
|
|
||||||
|
|
||||||
Behavior:
|
|
||||||
- Messages with role in ["user", "system", "tool", "latest_reminder"] are always kept.
|
|
||||||
- Messages at or after the last user index are always kept.
|
|
||||||
- Assistant messages before the last user get reasoning_content removed.
|
|
||||||
- Developer messages before the last user are dropped entirely.
|
|
||||||
"""
|
|
||||||
last_user_idx = find_last_user_index(messages)
|
|
||||||
result = []
|
|
||||||
keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"}
|
|
||||||
|
|
||||||
for idx, msg in enumerate(messages):
|
|
||||||
role = msg.get("role")
|
|
||||||
if role in keep_roles or idx >= last_user_idx:
|
|
||||||
result.append(msg)
|
|
||||||
elif role == "assistant":
|
|
||||||
msg = copy.copy(msg)
|
|
||||||
msg.pop("reasoning_content", None)
|
|
||||||
result.append(msg)
|
|
||||||
# developer and other roles before last_user_idx are dropped
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Parsing (Decoding model output)
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def _read_until_stop(
|
|
||||||
index: int, text: str, stop: List[str]
|
|
||||||
) -> Tuple[int, str, Optional[str]]:
|
|
||||||
"""
|
|
||||||
Read text from index until one of the stop strings is found.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (new_index, content_before_stop, matched_stop_string_or_None).
|
|
||||||
"""
|
|
||||||
min_pos = len(text)
|
|
||||||
matched_stop = None
|
|
||||||
|
|
||||||
for s in stop:
|
|
||||||
pos = text.find(s, index)
|
|
||||||
if pos != -1 and pos < min_pos:
|
|
||||||
min_pos = pos
|
|
||||||
matched_stop = s
|
|
||||||
|
|
||||||
if matched_stop:
|
|
||||||
content = text[index:min_pos]
|
|
||||||
return min_pos + len(matched_stop), content, matched_stop
|
|
||||||
else:
|
|
||||||
content = text[index:]
|
|
||||||
return len(text), content, None
|
|
||||||
|
|
||||||
|
|
||||||
def parse_tool_calls(
|
|
||||||
index: int, text: str
|
|
||||||
) -> Tuple[int, Optional[str], List[Dict[str, str]]]:
|
|
||||||
"""
|
|
||||||
Parse DSML tool calls from text starting at the given index.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
index: Starting position in text.
|
|
||||||
text: The full text to parse.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple of (new_index, last_stop_token, list_of_tool_call_dicts).
|
|
||||||
Each tool call dict has "name" and "arguments" keys.
|
|
||||||
"""
|
|
||||||
tool_calls: List[Dict[str, Any]] = []
|
|
||||||
stop_token = None
|
|
||||||
tool_calls_end_token = f"</{dsml_token}{tool_calls_block_name}>"
|
|
||||||
|
|
||||||
while index < len(text):
|
|
||||||
index, _, stop_token = _read_until_stop(
|
|
||||||
index, text, [f"<{dsml_token}invoke", tool_calls_end_token]
|
|
||||||
)
|
|
||||||
if _ != ">\n":
|
|
||||||
raise ValueError(f"Tool call format error: expected '>\\n' but got '{_}'")
|
|
||||||
|
|
||||||
if stop_token == tool_calls_end_token:
|
|
||||||
break
|
|
||||||
|
|
||||||
if stop_token is None:
|
|
||||||
raise ValueError("Missing special token in tool calls")
|
|
||||||
|
|
||||||
index, tool_name_content, stop_token = _read_until_stop(
|
|
||||||
index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"]
|
|
||||||
)
|
|
||||||
|
|
||||||
p_tool_name = re.findall(
|
|
||||||
r'^\s*name="(.*?)">\n$', tool_name_content, flags=re.DOTALL
|
|
||||||
)
|
|
||||||
if len(p_tool_name) != 1:
|
|
||||||
raise ValueError(f"Tool name format error: '{tool_name_content}'")
|
|
||||||
tool_name = p_tool_name[0]
|
|
||||||
|
|
||||||
tool_args: Dict[str, Tuple[str, str]] = {}
|
|
||||||
while stop_token == f"<{dsml_token}parameter":
|
|
||||||
index, param_content, stop_token = _read_until_stop(
|
|
||||||
index, text, [f"/{dsml_token}parameter"]
|
|
||||||
)
|
|
||||||
|
|
||||||
param_kv = re.findall(
|
|
||||||
r'^ name="(.*?)" string="(true|false)">(.*?)<$',
|
|
||||||
param_content,
|
|
||||||
flags=re.DOTALL,
|
|
||||||
)
|
|
||||||
if len(param_kv) != 1:
|
|
||||||
raise ValueError(f"Parameter format error: '{param_content}'")
|
|
||||||
param_name, string, param_value = param_kv[0]
|
|
||||||
|
|
||||||
if param_name in tool_args:
|
|
||||||
raise ValueError(f"Duplicate parameter name: '{param_name}'")
|
|
||||||
tool_args[param_name] = (param_value, string)
|
|
||||||
|
|
||||||
index, content, stop_token = _read_until_stop(
|
|
||||||
index, text, [f"<{dsml_token}parameter", f"</{dsml_token}invoke"]
|
|
||||||
)
|
|
||||||
if content != ">\n":
|
|
||||||
raise ValueError(
|
|
||||||
f"Parameter format error: expected '>\\n' but got '{content}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args)
|
|
||||||
tool_calls.append(tool_call)
|
|
||||||
|
|
||||||
return index, stop_token, tool_calls
|
|
||||||
|
|
||||||
|
|
||||||
def parse_message_from_completion_text(text: str, thinking_mode: str) -> Dict[str, Any]:
|
|
||||||
"""
|
|
||||||
Parse a model completion text into a structured assistant message.
|
|
||||||
|
|
||||||
This function takes the raw text output from the model (a single assistant turn)
|
|
||||||
and extracts:
|
|
||||||
- reasoning_content (thinking block)
|
|
||||||
- content (summary/response)
|
|
||||||
- tool_calls (if any)
|
|
||||||
|
|
||||||
NOTE: This function is designed to parse only correctly formatted strings and
|
|
||||||
will raise ValueError for malformed output.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
text: The raw completion text (including EOS token).
|
|
||||||
thinking_mode: Either "chat" or "thinking".
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Dict with keys: "role", "content", "reasoning_content", "tool_calls".
|
|
||||||
tool_calls are in OpenAI format.
|
|
||||||
"""
|
|
||||||
summary_content, reasoning_content, tool_calls = "", "", []
|
|
||||||
index, stop_token = 0, None
|
|
||||||
tool_calls_start_token = f"\n\n<{dsml_token}{tool_calls_block_name}"
|
|
||||||
|
|
||||||
is_thinking = thinking_mode == "thinking"
|
|
||||||
is_tool_calling = False
|
|
||||||
|
|
||||||
if is_thinking:
|
|
||||||
index, content_delta, stop_token = _read_until_stop(
|
|
||||||
index, text, [thinking_end_token, tool_calls_start_token]
|
|
||||||
)
|
|
||||||
reasoning_content = content_delta
|
|
||||||
assert stop_token == thinking_end_token, (
|
|
||||||
"Invalid thinking format: missing </think>"
|
|
||||||
)
|
|
||||||
|
|
||||||
index, content_delta, stop_token = _read_until_stop(
|
|
||||||
index, text, [eos_token, tool_calls_start_token]
|
|
||||||
)
|
|
||||||
summary_content = content_delta
|
|
||||||
if stop_token == tool_calls_start_token:
|
|
||||||
is_tool_calling = True
|
|
||||||
else:
|
|
||||||
assert stop_token == eos_token, "Invalid format: missing EOS token"
|
|
||||||
|
|
||||||
if is_tool_calling:
|
|
||||||
index, stop_token, tool_calls = parse_tool_calls(index, text)
|
|
||||||
|
|
||||||
index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token])
|
|
||||||
assert not tool_ends_text, "Unexpected content after tool calls"
|
|
||||||
|
|
||||||
assert len(text) == index and stop_token in [
|
|
||||||
eos_token,
|
|
||||||
None,
|
|
||||||
], "Unexpected content at end"
|
|
||||||
|
|
||||||
for sp_token in [
|
|
||||||
bos_token,
|
|
||||||
eos_token,
|
|
||||||
thinking_start_token,
|
|
||||||
thinking_end_token,
|
|
||||||
dsml_token,
|
|
||||||
]:
|
|
||||||
assert sp_token not in summary_content and sp_token not in reasoning_content, (
|
|
||||||
f"Unexpected special token '{sp_token}' in content"
|
|
||||||
)
|
|
||||||
|
|
||||||
return {
|
|
||||||
"role": "assistant",
|
|
||||||
"content": summary_content,
|
|
||||||
"reasoning_content": reasoning_content,
|
|
||||||
"tool_calls": tool_calls_to_openai_format(tool_calls),
|
|
||||||
}
|
|
||||||
@@ -1,719 +0,0 @@
|
|||||||
# Adapted from the DeepSeek-V4.1 release reference implementation.
|
|
||||||
"""Encode DeepSeek-V4.1 chat messages.
|
|
||||||
|
|
||||||
Mid-conversation system messages trigger the assistant generation header.
|
|
||||||
"""
|
|
||||||
|
|
||||||
import copy
|
|
||||||
import json
|
|
||||||
from typing import Any, Dict, List, Optional, Tuple, Union
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Special Tokens
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
bos_token: str = "<|begin▁of▁sentence|>"
|
|
||||||
eos_token: str = "<|end▁of▁sentence|>"
|
|
||||||
thinking_start_token: str = "<think>"
|
|
||||||
thinking_end_token: str = "</think>"
|
|
||||||
dsml_token: str = "|DSML|"
|
|
||||||
|
|
||||||
USER_SP_TOKEN = "<|User|>"
|
|
||||||
ASSISTANT_SP_TOKEN = "<|Assistant|>"
|
|
||||||
SYSTEM_SP_TOKEN = "<|System|>"
|
|
||||||
LATEST_REMINDER_SP_TOKEN = "<|latest_reminder|>"
|
|
||||||
|
|
||||||
IMAGE_PLACEHOLDER = "<|deepseek_image|>"
|
|
||||||
|
|
||||||
# Task special tokens for internal classification tasks
|
|
||||||
DS_TASK_SP_TOKENS = {
|
|
||||||
"action": "<|action|>",
|
|
||||||
"query": "<|query|>",
|
|
||||||
"authority": "<|authority|>",
|
|
||||||
"domain": "<|domain|>",
|
|
||||||
"title": "<|title|>",
|
|
||||||
"read_url": "<|read_url|>",
|
|
||||||
}
|
|
||||||
VALID_TASKS = set(DS_TASK_SP_TOKENS.keys())
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Templates
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
system_msg_template: str = "{content}"
|
|
||||||
user_msg_template: str = "{content}"
|
|
||||||
latest_reminder_msg_template: str = "{content}"
|
|
||||||
assistant_msg_template: str = "{reasoning}{content}{tool_calls}" + eos_token
|
|
||||||
assistant_msg_wo_eos_template: str = "{reasoning}{content}{tool_calls}"
|
|
||||||
thinking_template: str = "{reasoning_content}"
|
|
||||||
|
|
||||||
response_format_template: str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"
|
|
||||||
|
|
||||||
tool_calls_block_name: str = " calls"
|
|
||||||
tool_call_tag_name: str = " invoke"
|
|
||||||
tool_parameter_tag_name: str = " parameter"
|
|
||||||
|
|
||||||
tool_call_template: str = '<{dsml_token}{tool_call_tag_name} name="{name}">\n{arguments}\n</{dsml_token}{tool_call_tag_name}>'
|
|
||||||
tool_calls_template = (
|
|
||||||
"<{dsml_token}{tc_block_name}>\n{tool_calls}\n</{dsml_token}{tc_block_name}>"
|
|
||||||
)
|
|
||||||
|
|
||||||
tool_output_template: str = "<tool_result>{content}</tool_result>"
|
|
||||||
|
|
||||||
REASONING_EFFORT_TEMPLATE = (
|
|
||||||
"Reasoning Effort: {budget} "
|
|
||||||
"(range 1-100, the higher the value, the more thorough the reasoning)\n\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
REASONING_EFFORT_MAPPINGS: Dict[str, int] = {
|
|
||||||
"low": 50,
|
|
||||||
"high": 75,
|
|
||||||
"xhigh": 75,
|
|
||||||
"max": 100,
|
|
||||||
}
|
|
||||||
DEFAULT_REASONING_EFFORT = "high"
|
|
||||||
|
|
||||||
TOOLS_TEMPLATE = """## Tools
|
|
||||||
|
|
||||||
You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}{tc_block_name}>" block like the following:
|
|
||||||
|
|
||||||
<{dsml_token}{tc_block_name}>
|
|
||||||
<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME">
|
|
||||||
<{dsml_token}{tool_parameter_tag_name} name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE</{dsml_token}{tool_parameter_tag_name}>
|
|
||||||
...
|
|
||||||
</{dsml_token}{tool_call_tag_name}>
|
|
||||||
<{dsml_token}{tool_call_tag_name} name="$TOOL_NAME2">
|
|
||||||
...
|
|
||||||
</{dsml_token}{tool_call_tag_name}>
|
|
||||||
</{dsml_token}{tc_block_name}>
|
|
||||||
|
|
||||||
String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`.
|
|
||||||
|
|
||||||
If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response.
|
|
||||||
|
|
||||||
Otherwise, output directly after {thinking_end_token} with tool calls or final response.
|
|
||||||
|
|
||||||
### Available Tool Schemas
|
|
||||||
|
|
||||||
{tool_schemas}
|
|
||||||
|
|
||||||
You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.
|
|
||||||
"""
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Utility Functions
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def to_json(value: Any) -> str:
|
|
||||||
try:
|
|
||||||
return json.dumps(value, ensure_ascii=False)
|
|
||||||
except:
|
|
||||||
return json.dumps(value, ensure_ascii=True)
|
|
||||||
|
|
||||||
|
|
||||||
def tools_from_openai_format(tools):
|
|
||||||
return [tool["function"] for tool in tools]
|
|
||||||
|
|
||||||
|
|
||||||
def tool_calls_from_openai_format(tool_calls):
|
|
||||||
return [
|
|
||||||
{
|
|
||||||
"name": tool_call["function"]["name"],
|
|
||||||
"arguments": tool_call["function"]["arguments"],
|
|
||||||
}
|
|
||||||
for tool_call in tool_calls
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
def encode_arguments_to_dsml(tool_call: Dict[str, Any]) -> str:
|
|
||||||
p_dsml_template = (
|
|
||||||
'<{dsml_token}{tool_parameter_tag_name} name="{key}" string="{is_str}">'
|
|
||||||
"{value}</{dsml_token}{tool_parameter_tag_name}>"
|
|
||||||
)
|
|
||||||
P_dsml_strs = []
|
|
||||||
|
|
||||||
raw_arguments = tool_call["arguments"]
|
|
||||||
arguments = (
|
|
||||||
json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments
|
|
||||||
)
|
|
||||||
if not isinstance(arguments, dict):
|
|
||||||
raise ValueError(
|
|
||||||
"Assistant tool call function.arguments must be a JSON object."
|
|
||||||
)
|
|
||||||
|
|
||||||
for k, v in arguments.items():
|
|
||||||
P_dsml_strs.append(
|
|
||||||
p_dsml_template.format(
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
tool_parameter_tag_name=tool_parameter_tag_name,
|
|
||||||
key=k,
|
|
||||||
is_str="true" if isinstance(v, str) else "false",
|
|
||||||
value=v if isinstance(v, str) else to_json(v),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
return "\n".join(P_dsml_strs)
|
|
||||||
|
|
||||||
|
|
||||||
def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str:
|
|
||||||
tools_json = [to_json(t) for t in tools]
|
|
||||||
|
|
||||||
return TOOLS_TEMPLATE.format(
|
|
||||||
tool_schemas="\n".join(tools_json),
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
tc_block_name=tool_calls_block_name,
|
|
||||||
tool_call_tag_name=tool_call_tag_name,
|
|
||||||
tool_parameter_tag_name=tool_parameter_tag_name,
|
|
||||||
thinking_start_token=thinking_start_token,
|
|
||||||
thinking_end_token=thinking_end_token,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def render_reasoning_effort(
|
|
||||||
index: int,
|
|
||||||
thinking_mode: str,
|
|
||||||
effort: Union[str, int, None],
|
|
||||||
) -> str:
|
|
||||||
"""Render the numeric reasoning effort prefix (thinking mode, index 0 only)."""
|
|
||||||
if effort is None:
|
|
||||||
effort = DEFAULT_REASONING_EFFORT
|
|
||||||
if not (
|
|
||||||
(type(effort) is int and 1 <= effort <= 100)
|
|
||||||
or effort in REASONING_EFFORT_MAPPINGS
|
|
||||||
):
|
|
||||||
raise ValueError(
|
|
||||||
f"Invalid reasoning effort for deepseek_v41: {effort!r}, should be "
|
|
||||||
f"int within [1,100] or {list(REASONING_EFFORT_MAPPINGS)}"
|
|
||||||
)
|
|
||||||
if type(effort) is str:
|
|
||||||
effort = REASONING_EFFORT_MAPPINGS[effort]
|
|
||||||
if index == 0 and thinking_mode == "thinking":
|
|
||||||
return REASONING_EFFORT_TEMPLATE.format(budget=effort)
|
|
||||||
return ""
|
|
||||||
|
|
||||||
|
|
||||||
def find_last_user_index(messages: List[Dict[str, Any]]) -> int:
|
|
||||||
"""Mid-conversation system messages also count as user messages here;
|
|
||||||
they trigger the assistant generation header.
|
|
||||||
User messages containing only tool results do not start a new turn.
|
|
||||||
"""
|
|
||||||
last_user_index = -1
|
|
||||||
for idx in range(len(messages) - 1, -1, -1):
|
|
||||||
role = messages[idx].get("role")
|
|
||||||
if role in ["user", "developer"] or (role == "system" and idx > 0):
|
|
||||||
content_blocks = messages[idx].get("content_blocks")
|
|
||||||
if content_blocks and not any(
|
|
||||||
block.get("type") in ("text", "image", "image_url")
|
|
||||||
for block in content_blocks
|
|
||||||
):
|
|
||||||
continue
|
|
||||||
last_user_index = idx
|
|
||||||
break
|
|
||||||
return last_user_index
|
|
||||||
|
|
||||||
|
|
||||||
def attach_task_to_last_user_message(messages: List[Dict[str, Any]], task: str) -> None:
|
|
||||||
"""Set `task` on the most recent user/developer message; raise if none exists."""
|
|
||||||
idx = find_last_user_index(messages)
|
|
||||||
if idx == -1:
|
|
||||||
raise ValueError(
|
|
||||||
"`task` requires at least one message with role='user' or 'developer'."
|
|
||||||
)
|
|
||||||
messages[idx]["task"] = task
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Message Rendering
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def render_message(
|
|
||||||
index: int,
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
thinking_mode: str,
|
|
||||||
drop_thinking: bool = True,
|
|
||||||
reasoning_effort: Union[str, int, None] = None,
|
|
||||||
) -> str:
|
|
||||||
assert 0 <= index < len(messages)
|
|
||||||
assert thinking_mode in [
|
|
||||||
"chat",
|
|
||||||
"thinking",
|
|
||||||
], f"Invalid thinking_mode `{thinking_mode}`"
|
|
||||||
|
|
||||||
msg = messages[index]
|
|
||||||
last_user_idx = find_last_user_index(messages)
|
|
||||||
|
|
||||||
role = msg.get("role")
|
|
||||||
content = msg.get("content")
|
|
||||||
tools = msg.get("tools")
|
|
||||||
response_format = msg.get("response_format")
|
|
||||||
tool_calls = msg.get("tool_calls")
|
|
||||||
reasoning_content = msg.get("reasoning_content")
|
|
||||||
wo_eos = msg.get("wo_eos", False)
|
|
||||||
|
|
||||||
if tools:
|
|
||||||
tools = tools_from_openai_format(tools)
|
|
||||||
if tool_calls:
|
|
||||||
tool_calls = tool_calls_from_openai_format(tool_calls)
|
|
||||||
|
|
||||||
reasoning_effort_prompt = render_reasoning_effort(
|
|
||||||
index, thinking_mode, reasoning_effort
|
|
||||||
)
|
|
||||||
# Index 0 also emits the system token for the effort prompt, even before a user.
|
|
||||||
prompt = (
|
|
||||||
SYSTEM_SP_TOKEN
|
|
||||||
if index == 0 and (reasoning_effort_prompt or role == "system")
|
|
||||||
else ""
|
|
||||||
)
|
|
||||||
prompt += reasoning_effort_prompt
|
|
||||||
|
|
||||||
if role == "system":
|
|
||||||
if index > 0:
|
|
||||||
prompt += SYSTEM_SP_TOKEN
|
|
||||||
prompt += system_msg_template.format(content=content or "")
|
|
||||||
if tools:
|
|
||||||
prompt += "\n\n" + render_tools(tools)
|
|
||||||
if response_format:
|
|
||||||
prompt += "\n\n" + response_format_template.format(
|
|
||||||
schema=to_json(response_format)
|
|
||||||
)
|
|
||||||
|
|
||||||
elif role == "developer":
|
|
||||||
assert content, f"Invalid message for role `{role}`: {msg}"
|
|
||||||
|
|
||||||
content_developer = USER_SP_TOKEN
|
|
||||||
content_developer += content
|
|
||||||
|
|
||||||
if tools:
|
|
||||||
content_developer += "\n\n" + render_tools(tools)
|
|
||||||
if response_format:
|
|
||||||
content_developer += "\n\n" + response_format_template.format(
|
|
||||||
schema=to_json(response_format)
|
|
||||||
)
|
|
||||||
|
|
||||||
prompt += user_msg_template.format(content=content_developer)
|
|
||||||
|
|
||||||
elif role == "user":
|
|
||||||
prompt += USER_SP_TOKEN
|
|
||||||
|
|
||||||
# Handle content blocks (tool results mixed with text)
|
|
||||||
content_blocks = msg.get("content_blocks")
|
|
||||||
if content_blocks:
|
|
||||||
parts = []
|
|
||||||
for block in content_blocks:
|
|
||||||
block_type = block.get("type")
|
|
||||||
if block_type == "text":
|
|
||||||
parts.append(block.get("text", ""))
|
|
||||||
elif block_type == "tool_result":
|
|
||||||
tool_content = block.get("content", "")
|
|
||||||
if isinstance(tool_content, list):
|
|
||||||
text_parts = []
|
|
||||||
for b in tool_content:
|
|
||||||
if b.get("type") == "text":
|
|
||||||
text_parts.append(b.get("text", ""))
|
|
||||||
else:
|
|
||||||
text_parts.append(f"[Unsupported {b.get('type')}]")
|
|
||||||
tool_content = "\n\n".join(text_parts)
|
|
||||||
parts.append(tool_output_template.format(content=tool_content))
|
|
||||||
else:
|
|
||||||
parts.append(f"[Unsupported {block_type}]")
|
|
||||||
prompt += "\n\n".join(parts)
|
|
||||||
else:
|
|
||||||
prompt += content or ""
|
|
||||||
|
|
||||||
elif role == "latest_reminder":
|
|
||||||
prompt += LATEST_REMINDER_SP_TOKEN + latest_reminder_msg_template.format(
|
|
||||||
content=content
|
|
||||||
)
|
|
||||||
|
|
||||||
elif role == "tool":
|
|
||||||
raise NotImplementedError(
|
|
||||||
"deepseek_v41 merges tool messages into user; please preprocess with merge_tool_messages()"
|
|
||||||
)
|
|
||||||
|
|
||||||
elif role == "assistant":
|
|
||||||
thinking_part = ""
|
|
||||||
tc_content = ""
|
|
||||||
|
|
||||||
if tool_calls:
|
|
||||||
tc_list = [
|
|
||||||
tool_call_template.format(
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
tool_call_tag_name=tool_call_tag_name,
|
|
||||||
name=tc.get("name"),
|
|
||||||
arguments=encode_arguments_to_dsml(tc),
|
|
||||||
)
|
|
||||||
for tc in tool_calls
|
|
||||||
]
|
|
||||||
tc_content += "\n\n" + tool_calls_template.format(
|
|
||||||
dsml_token=dsml_token,
|
|
||||||
tool_calls="\n".join(tc_list),
|
|
||||||
tc_block_name=tool_calls_block_name,
|
|
||||||
)
|
|
||||||
|
|
||||||
summary_content = content or ""
|
|
||||||
rc = reasoning_content or ""
|
|
||||||
|
|
||||||
# Check if previous message has a task - if so, this is a task output (no thinking)
|
|
||||||
prev_has_task = index - 1 >= 0 and messages[index - 1].get("task") is not None
|
|
||||||
|
|
||||||
if thinking_mode == "thinking" and not prev_has_task:
|
|
||||||
if not drop_thinking or index > last_user_idx:
|
|
||||||
thinking_part = (
|
|
||||||
thinking_template.format(reasoning_content=rc) + thinking_end_token
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
thinking_part = ""
|
|
||||||
|
|
||||||
if wo_eos:
|
|
||||||
prompt += assistant_msg_wo_eos_template.format(
|
|
||||||
reasoning=thinking_part,
|
|
||||||
content=summary_content,
|
|
||||||
tool_calls=tc_content,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
prompt += assistant_msg_template.format(
|
|
||||||
reasoning=thinking_part,
|
|
||||||
content=summary_content,
|
|
||||||
tool_calls=tc_content,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
raise NotImplementedError(f"Unknown role: {role}")
|
|
||||||
|
|
||||||
# Append transition tokens based on what follows
|
|
||||||
if index + 1 < len(messages) and messages[index + 1].get("role") not in [
|
|
||||||
"assistant",
|
|
||||||
"latest_reminder",
|
|
||||||
]:
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
task = messages[index].get("task")
|
|
||||||
if task is not None:
|
|
||||||
# Task special token for internal classification tasks
|
|
||||||
assert task in VALID_TASKS, (
|
|
||||||
f"Invalid task: '{task}'. Valid tasks are: {list(VALID_TASKS)}"
|
|
||||||
)
|
|
||||||
task_sp_token = DS_TASK_SP_TOKENS[task]
|
|
||||||
|
|
||||||
if task != "action":
|
|
||||||
# Non-action tasks: append task sp token directly after the message
|
|
||||||
prompt += task_sp_token
|
|
||||||
else:
|
|
||||||
# Action task: append Assistant + thinking token + action sp token
|
|
||||||
prompt += ASSISTANT_SP_TOKEN
|
|
||||||
prompt += (
|
|
||||||
thinking_end_token
|
|
||||||
if thinking_mode != "thinking"
|
|
||||||
else thinking_start_token
|
|
||||||
)
|
|
||||||
prompt += task_sp_token
|
|
||||||
|
|
||||||
elif messages[index].get("role") in ["user", "developer"] or (
|
|
||||||
messages[index].get("role") == "system" and index > 0
|
|
||||||
):
|
|
||||||
# Normal generation: append Assistant + thinking token
|
|
||||||
prompt += ASSISTANT_SP_TOKEN
|
|
||||||
if not drop_thinking and thinking_mode == "thinking":
|
|
||||||
prompt += thinking_start_token
|
|
||||||
elif drop_thinking and thinking_mode == "thinking" and index >= last_user_idx:
|
|
||||||
prompt += thinking_start_token
|
|
||||||
else:
|
|
||||||
prompt += thinking_end_token
|
|
||||||
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Preprocessing
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def merge_tool_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
||||||
"""Tool results are encoded within user messages;
|
|
||||||
DeepSeek-V4.1 has no standalone tool role.
|
|
||||||
"""
|
|
||||||
merged: List[Dict[str, Any]] = []
|
|
||||||
|
|
||||||
for msg in messages:
|
|
||||||
msg = copy.deepcopy(msg)
|
|
||||||
role = msg.get("role")
|
|
||||||
|
|
||||||
if role == "tool":
|
|
||||||
# Convert tool message to a user message with tool_result block
|
|
||||||
content = msg.get("content", "")
|
|
||||||
if msg.get("content_blocks") and not isinstance(msg.get("content"), str):
|
|
||||||
content = msg["content_blocks"]
|
|
||||||
tool_block = {
|
|
||||||
"type": "tool_result",
|
|
||||||
"tool_use_id": msg.get("tool_call_id", ""),
|
|
||||||
"content": content,
|
|
||||||
}
|
|
||||||
# Merge into previous message if it's already a user (merged tool)
|
|
||||||
if (
|
|
||||||
merged
|
|
||||||
and merged[-1].get("role") == "user"
|
|
||||||
and "content_blocks" in merged[-1]
|
|
||||||
):
|
|
||||||
merged[-1]["content_blocks"].append(tool_block)
|
|
||||||
else:
|
|
||||||
merged.append(
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content_blocks": [tool_block],
|
|
||||||
}
|
|
||||||
)
|
|
||||||
elif role == "user":
|
|
||||||
content_blocks = msg.get("content_blocks")
|
|
||||||
if content_blocks is None:
|
|
||||||
content = msg.get("content", "")
|
|
||||||
content_blocks = (
|
|
||||||
content
|
|
||||||
if isinstance(content, list)
|
|
||||||
else [{"type": "text", "text": content}]
|
|
||||||
)
|
|
||||||
if (
|
|
||||||
merged
|
|
||||||
and merged[-1].get("role") == "user"
|
|
||||||
and "content_blocks" in merged[-1]
|
|
||||||
and merged[-1].get("task") is None
|
|
||||||
):
|
|
||||||
merged[-1]["content_blocks"].extend(content_blocks)
|
|
||||||
else:
|
|
||||||
# Keeps structured content and every message-level field.
|
|
||||||
new_msg = msg
|
|
||||||
new_msg["content_blocks"] = content_blocks
|
|
||||||
merged.append(new_msg)
|
|
||||||
else:
|
|
||||||
merged.append(msg)
|
|
||||||
|
|
||||||
return merged
|
|
||||||
|
|
||||||
|
|
||||||
def sort_tool_results_by_call_order(
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
) -> List[Dict[str, Any]]:
|
|
||||||
last_tool_call_order: Dict[str, int] = {}
|
|
||||||
|
|
||||||
for msg in messages:
|
|
||||||
role = msg.get("role")
|
|
||||||
if role == "assistant" and msg.get("tool_calls"):
|
|
||||||
last_tool_call_order = {}
|
|
||||||
for idx, tc in enumerate(msg["tool_calls"]):
|
|
||||||
tc_id = tc.get("id") or tc.get("function", {}).get("id", "")
|
|
||||||
if tc_id:
|
|
||||||
last_tool_call_order[tc_id] = idx
|
|
||||||
|
|
||||||
elif role == "user" and msg.get("content_blocks"):
|
|
||||||
tool_blocks = [
|
|
||||||
b for b in msg["content_blocks"] if b.get("type") == "tool_result"
|
|
||||||
]
|
|
||||||
if len(tool_blocks) > 1 and last_tool_call_order:
|
|
||||||
sorted_blocks = sorted(
|
|
||||||
tool_blocks,
|
|
||||||
key=lambda b: last_tool_call_order.get(b.get("tool_use_id", ""), 0),
|
|
||||||
)
|
|
||||||
sorted_idx = 0
|
|
||||||
new_blocks = []
|
|
||||||
for block in msg["content_blocks"]:
|
|
||||||
if block.get("type") == "tool_result":
|
|
||||||
new_blocks.append(sorted_blocks[sorted_idx])
|
|
||||||
sorted_idx += 1
|
|
||||||
else:
|
|
||||||
new_blocks.append(block)
|
|
||||||
msg["content_blocks"] = new_blocks
|
|
||||||
|
|
||||||
return messages
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Vision Message Preprocessing
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def _is_image_block(block: Dict[str, Any]) -> bool:
|
|
||||||
return isinstance(block, dict) and block.get("type") == "image_url"
|
|
||||||
|
|
||||||
|
|
||||||
def _extract_image(block: Dict[str, Any]) -> Dict[str, Any]:
|
|
||||||
image_url = block.get("image_url")
|
|
||||||
url = image_url if isinstance(image_url, str) else (image_url or {}).get("url", "")
|
|
||||||
if not url:
|
|
||||||
raise ValueError("Image block does not contain a valid source")
|
|
||||||
return {"url": url}
|
|
||||||
|
|
||||||
|
|
||||||
def _process_image_blocks(
|
|
||||||
blocks: List[Any], image_placeholder: str = IMAGE_PLACEHOLDER
|
|
||||||
) -> Tuple[List[Any], List[Dict[str, Any]]]:
|
|
||||||
new_blocks: List[Any] = []
|
|
||||||
images: List[Dict[str, Any]] = []
|
|
||||||
for block in blocks:
|
|
||||||
if not isinstance(block, dict):
|
|
||||||
new_blocks.append(block)
|
|
||||||
continue
|
|
||||||
if _is_image_block(block):
|
|
||||||
new_blocks.append({"type": "text", "text": image_placeholder})
|
|
||||||
images.append(_extract_image(block))
|
|
||||||
elif block.get("type") == "tool_result" and isinstance(
|
|
||||||
block.get("content"), list
|
|
||||||
):
|
|
||||||
block = copy.copy(block)
|
|
||||||
block["content"], nested_images = _process_image_blocks(
|
|
||||||
block["content"], image_placeholder
|
|
||||||
)
|
|
||||||
new_blocks.append(block)
|
|
||||||
images.extend(nested_images)
|
|
||||||
elif block.get("type") == "text":
|
|
||||||
text = block.get("text") or ""
|
|
||||||
if IMAGE_PLACEHOLDER in text:
|
|
||||||
raise ValueError(
|
|
||||||
f"Text block contains image placeholder '{IMAGE_PLACEHOLDER}': "
|
|
||||||
f"'{text[:100]}'. Images should be separate content blocks."
|
|
||||||
)
|
|
||||||
new_blocks.append(block)
|
|
||||||
else:
|
|
||||||
new_blocks.append(block)
|
|
||||||
return new_blocks, images
|
|
||||||
|
|
||||||
|
|
||||||
def _validate_no_image_sp_tokens(msg: Dict[str, Any]) -> None:
|
|
||||||
content = msg.get("content")
|
|
||||||
if isinstance(content, str) and IMAGE_PLACEHOLDER in content:
|
|
||||||
raise ValueError(
|
|
||||||
f"Message content contains image special token '{IMAGE_PLACEHOLDER}'. "
|
|
||||||
"Images should be provided as image content blocks."
|
|
||||||
)
|
|
||||||
reasoning_content = msg.get("reasoning_content")
|
|
||||||
if isinstance(reasoning_content, str) and IMAGE_PLACEHOLDER in reasoning_content:
|
|
||||||
raise ValueError(
|
|
||||||
f"reasoning_content contains image special token '{IMAGE_PLACEHOLDER}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def process_image_messages(
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
|
|
||||||
processed: List[Dict[str, Any]] = []
|
|
||||||
images: List[Dict[str, Any]] = []
|
|
||||||
for msg in messages:
|
|
||||||
msg = copy.deepcopy(msg)
|
|
||||||
_validate_no_image_sp_tokens(msg)
|
|
||||||
|
|
||||||
if isinstance(msg.get("content"), list) and "content_blocks" not in msg:
|
|
||||||
msg["content_blocks"] = msg.pop("content")
|
|
||||||
|
|
||||||
if msg.get("content_blocks"):
|
|
||||||
msg["content_blocks"], message_images = _process_image_blocks(
|
|
||||||
msg["content_blocks"]
|
|
||||||
)
|
|
||||||
images.extend(message_images)
|
|
||||||
if not isinstance(msg.get("content"), str):
|
|
||||||
texts = [
|
|
||||||
block.get("text", "")
|
|
||||||
for block in msg["content_blocks"]
|
|
||||||
if isinstance(block, dict) and block.get("type") == "text"
|
|
||||||
]
|
|
||||||
msg["content"] = "\n\n".join(texts)
|
|
||||||
|
|
||||||
processed.append(msg)
|
|
||||||
return processed, images
|
|
||||||
|
|
||||||
|
|
||||||
# ============================================================
|
|
||||||
# Main Encoding Function
|
|
||||||
# ============================================================
|
|
||||||
|
|
||||||
|
|
||||||
def _drop_thinking_messages(messages: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
|
|
||||||
last_user_idx = find_last_user_index(messages)
|
|
||||||
result = []
|
|
||||||
keep_roles = {"user", "system", "tool", "latest_reminder", "direct_search_results"}
|
|
||||||
|
|
||||||
for idx, msg in enumerate(messages):
|
|
||||||
role = msg.get("role")
|
|
||||||
if role in keep_roles or idx >= last_user_idx:
|
|
||||||
result.append(msg)
|
|
||||||
elif role == "assistant":
|
|
||||||
msg = copy.copy(msg)
|
|
||||||
msg.pop("reasoning_content", None)
|
|
||||||
result.append(msg)
|
|
||||||
# developer and other roles before last_user_idx are dropped
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def _encode_messages_text(
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
thinking_mode: str,
|
|
||||||
context: Optional[List[Dict[str, Any]]] = None,
|
|
||||||
drop_thinking: bool = True,
|
|
||||||
add_default_bos_token: bool = True,
|
|
||||||
reasoning_effort: Union[str, int, None] = None,
|
|
||||||
) -> str:
|
|
||||||
"""Encode preprocessed (text-only) messages into the V4.1 prompt format."""
|
|
||||||
context = context if context else []
|
|
||||||
full_messages = context + messages
|
|
||||||
|
|
||||||
prompt = bos_token if add_default_bos_token and len(context) == 0 else ""
|
|
||||||
|
|
||||||
# Resolve drop_thinking: if any message has tools defined, don't drop thinking
|
|
||||||
effective_drop_thinking = drop_thinking
|
|
||||||
if any(m.get("tools") for m in full_messages):
|
|
||||||
effective_drop_thinking = False
|
|
||||||
|
|
||||||
if thinking_mode == "thinking" and effective_drop_thinking:
|
|
||||||
full_messages = _drop_thinking_messages(full_messages)
|
|
||||||
num_to_render = len(full_messages) - len(_drop_thinking_messages(context))
|
|
||||||
context_len = len(full_messages) - num_to_render
|
|
||||||
else:
|
|
||||||
num_to_render = len(messages)
|
|
||||||
context_len = len(context)
|
|
||||||
|
|
||||||
for idx in range(num_to_render):
|
|
||||||
prompt += render_message(
|
|
||||||
idx + context_len,
|
|
||||||
full_messages,
|
|
||||||
thinking_mode=thinking_mode,
|
|
||||||
drop_thinking=effective_drop_thinking,
|
|
||||||
reasoning_effort=reasoning_effort,
|
|
||||||
)
|
|
||||||
|
|
||||||
return prompt
|
|
||||||
|
|
||||||
|
|
||||||
def encode_messages(
|
|
||||||
messages: List[Dict[str, Any]],
|
|
||||||
thinking_mode: str,
|
|
||||||
context: Optional[List[Dict[str, Any]]] = None,
|
|
||||||
drop_thinking: bool = True,
|
|
||||||
add_default_bos_token: bool = True,
|
|
||||||
reasoning_effort: Union[str, int, None] = None,
|
|
||||||
return_multi_modal_data: bool = False,
|
|
||||||
) -> Any:
|
|
||||||
"""Encode a list of messages into the DeepSeek-V4.1 prompt format.
|
|
||||||
|
|
||||||
Returns the prompt string, or ``(prompt, {"images": [...]})`` when
|
|
||||||
``return_multi_modal_data`` is set; the image records are in prompt order.
|
|
||||||
"""
|
|
||||||
# Validate before merging can discard message-level textual fields.
|
|
||||||
for msg in (context or []) + messages:
|
|
||||||
_validate_no_image_sp_tokens(msg)
|
|
||||||
|
|
||||||
# Collect images only after tool results are in their final prompt order.
|
|
||||||
context = merge_tool_messages(context or [])
|
|
||||||
messages = merge_tool_messages(messages)
|
|
||||||
messages = sort_tool_results_by_call_order(context + messages)[len(context) :]
|
|
||||||
processed_context, _ = process_image_messages(context) if context else ([], [])
|
|
||||||
processed_messages, images = process_image_messages(messages)
|
|
||||||
prompt = _encode_messages_text(
|
|
||||||
processed_messages,
|
|
||||||
thinking_mode=thinking_mode,
|
|
||||||
context=processed_context if processed_context else None,
|
|
||||||
drop_thinking=drop_thinking,
|
|
||||||
add_default_bos_token=add_default_bos_token,
|
|
||||||
reasoning_effort=reasoning_effort,
|
|
||||||
)
|
|
||||||
if return_multi_modal_data:
|
|
||||||
return prompt, {"images": images}
|
|
||||||
return prompt
|
|
||||||
@@ -745,6 +745,8 @@ class ChatCompletionMessageGenericParam(BaseModel):
|
|||||||
reasoning_content: Optional[str] = None
|
reasoning_content: Optional[str] = None
|
||||||
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
|
tool_calls: Optional[List[ToolCall]] = Field(default=None, examples=[None])
|
||||||
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
|
tools: Optional[List[Tool]] = Field(default=None, examples=[None])
|
||||||
|
# DeepSeek API prefix completion: continue this final assistant message.
|
||||||
|
prefix: Optional[bool] = None
|
||||||
|
|
||||||
@field_validator("role", mode="before")
|
@field_validator("role", mode="before")
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -838,9 +840,10 @@ class ToolChoice(BaseModel):
|
|||||||
|
|
||||||
|
|
||||||
# OpenAI-spec string tiers for reasoning effort (current Responses/Chat API):
|
# OpenAI-spec string tiers for reasoning effort (current Responses/Chat API):
|
||||||
# none/minimal/low/medium/high/xhigh/max. Used as-is by /v1/responses.
|
# none/minimal/low/medium/high/xhigh/max, plus DeepSeek's ultra. Used as-is by
|
||||||
|
# /v1/responses.
|
||||||
ReasoningEffortTier = Literal[
|
ReasoningEffortTier = Literal[
|
||||||
"none", "minimal", "low", "medium", "high", "xhigh", "max"
|
"none", "minimal", "low", "medium", "high", "xhigh", "ultra", "max"
|
||||||
]
|
]
|
||||||
# The typed /v1/responses stream events validate against OpenAI's narrower
|
# The typed /v1/responses stream events validate against OpenAI's narrower
|
||||||
# ``Reasoning.effort``; echoing a tier outside this set kills the stream.
|
# ``Reasoning.effort``; echoing a tier outside this set kills the stream.
|
||||||
@@ -934,7 +937,7 @@ class ChatCompletionRequest(PDRoutingFields):
|
|||||||
reasoning_effort: ReasoningEffortType = Field(
|
reasoning_effort: ReasoningEffortType = Field(
|
||||||
default=None,
|
default=None,
|
||||||
description="Constrains effort on reasoning for reasoning models. "
|
description="Constrains effort on reasoning for reasoning models. "
|
||||||
"Accepts string levels ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'max') or a "
|
"Accepts string levels ('none', 'minimal', 'low', 'medium', 'high', 'xhigh', 'ultra', 'max') or a "
|
||||||
"float in [0.0, 0.99] for fine-grained control. "
|
"float in [0.0, 0.99] for fine-grained control. "
|
||||||
"'none' disables reasoning entirely, 'low' is the least effort, 'high' is the most effort. "
|
"'none' disables reasoning entirely, 'low' is the least effort, 'high' is the most effort. "
|
||||||
"Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning "
|
"Reducing reasoning effort can result in faster responses and fewer tokens used on reasoning "
|
||||||
@@ -1063,6 +1066,7 @@ class ChatCompletionRequest(PDRoutingFields):
|
|||||||
"medium",
|
"medium",
|
||||||
"high",
|
"high",
|
||||||
"xhigh",
|
"xhigh",
|
||||||
|
"ultra",
|
||||||
"max",
|
"max",
|
||||||
}:
|
}:
|
||||||
values["reasoning_effort"] = effort
|
values["reasoning_effort"] = effort
|
||||||
@@ -1778,7 +1782,7 @@ class ResponseReasoningParam(BaseModel):
|
|||||||
default="medium",
|
default="medium",
|
||||||
description="Constrains effort on reasoning for reasoning models. "
|
description="Constrains effort on reasoning for reasoning models. "
|
||||||
"Accepts the OpenAI string tiers "
|
"Accepts the OpenAI string tiers "
|
||||||
"('none','minimal','low','medium','high','xhigh','max').",
|
"('none','minimal','low','medium','high','xhigh','ultra','max').",
|
||||||
)
|
)
|
||||||
summary: Optional[Literal["auto", "concise", "detailed"]] = Field(
|
summary: Optional[Literal["auto", "concise", "detailed"]] = Field(
|
||||||
default=None,
|
default=None,
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from collections import OrderedDict
|
|||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional, Union
|
from typing import TYPE_CHECKING, Any, AsyncGenerator, Dict, List, Optional
|
||||||
|
|
||||||
from sglang.srt.runtime_context import get_model, get_serving
|
from sglang.srt.runtime_context import get_model, get_serving
|
||||||
|
|
||||||
@@ -42,9 +42,7 @@ from jsonschema import Draft202012Validator, SchemaError
|
|||||||
from sglang.srt.entrypoints.openai import (
|
from sglang.srt.entrypoints.openai import (
|
||||||
chat_encoding,
|
chat_encoding,
|
||||||
chat_encoding_adapters,
|
chat_encoding_adapters,
|
||||||
encoding_dsv4,
|
|
||||||
encoding_dsv32,
|
encoding_dsv32,
|
||||||
encoding_dsv41,
|
|
||||||
)
|
)
|
||||||
from sglang.srt.entrypoints.openai.parallel_prompt_encode import (
|
from sglang.srt.entrypoints.openai.parallel_prompt_encode import (
|
||||||
parallel_prompt_encode,
|
parallel_prompt_encode,
|
||||||
@@ -127,7 +125,6 @@ from sglang.srt.parser.template_detection import detect_inline_system_support
|
|||||||
from sglang.srt.sampling.sampling_params import (
|
from sglang.srt.sampling.sampling_params import (
|
||||||
set_request_reasoning_end_token_ids,
|
set_request_reasoning_end_token_ids,
|
||||||
)
|
)
|
||||||
from sglang.srt.utils import ImageData
|
|
||||||
from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata
|
from sglang.srt.utils.weight_versions import build_endpoint_weight_version_metadata
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
@@ -348,26 +345,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
# Values: "dsv32", "dsv4", or custom values set by subclass. None for default.
|
||||||
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
self.chat_encoding_spec = self._resolve_chat_encoding_spec()
|
||||||
self.supports_inline_system = self._resolve_inline_system_support()
|
self.supports_inline_system = self._resolve_inline_system_support()
|
||||||
self._dsv4_reasoning_effort_profile = (
|
|
||||||
chat_encoding.resolve_dsv4_reasoning_effort_profile(
|
|
||||||
model_path=self.tokenizer_manager.model_path,
|
|
||||||
revision=get_model().revision,
|
|
||||||
override=self.tokenizer_manager.model_config.hf_config.to_dict().get(
|
|
||||||
chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
|
|
||||||
),
|
|
||||||
)
|
|
||||||
if self.chat_encoding_spec == "dsv4"
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
|
|
||||||
self._encoding_adapter = self._build_chat_encoding_adapter()
|
self._encoding_adapter = self._build_chat_encoding_adapter()
|
||||||
self._dsv41_default_reasoning_effort: Optional[Union[str, int]] = (
|
|
||||||
chat_encoding.default_dsv41_reasoning_effort_from_env(
|
|
||||||
envs.SGLANG_DSV41_REASONING_EFFORT.get()
|
|
||||||
)
|
|
||||||
if self.chat_encoding_spec == "dsv41"
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
|
|
||||||
# Per-request response parser for custom decoding (set by _encode_messages)
|
# Per-request response parser for custom decoding (set by _encode_messages)
|
||||||
self._response_parser: ResponseParserProtocol | None = None
|
self._response_parser: ResponseParserProtocol | None = None
|
||||||
@@ -521,6 +499,7 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
hf_config=self.tokenizer_manager.model_config.hf_config,
|
hf_config=self.tokenizer_manager.model_config.hf_config,
|
||||||
model_path=self.tokenizer_manager.model_path,
|
model_path=self.tokenizer_manager.model_path,
|
||||||
revision=get_model().revision,
|
revision=get_model().revision,
|
||||||
|
default_chat_template_kwargs=get_serving().default_chat_template_kwargs,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -625,6 +604,8 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
thinking_mode: ThinkingMode,
|
thinking_mode: ThinkingMode,
|
||||||
tools: list[dict] | None = None,
|
tools: list[dict] | None = None,
|
||||||
|
*,
|
||||||
|
image_data: list,
|
||||||
) -> list[int] | None:
|
) -> list[int] | None:
|
||||||
"""Encode messages for custom chat_encoding_spec values.
|
"""Encode messages for custom chat_encoding_spec values.
|
||||||
|
|
||||||
@@ -632,7 +613,11 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
"""
|
"""
|
||||||
if self._encoding_adapter is not None:
|
if self._encoding_adapter is not None:
|
||||||
return self._encoding_adapter.encode_messages(
|
return self._encoding_adapter.encode_messages(
|
||||||
messages=messages, request=request, tools=tools
|
messages=messages,
|
||||||
|
request=request,
|
||||||
|
tools=tools,
|
||||||
|
thinking=thinking_mode == ThinkingMode.THINKING,
|
||||||
|
image_data=image_data,
|
||||||
)
|
)
|
||||||
if self.chat_encoding_spec == "kimi_k3":
|
if self.chat_encoding_spec == "kimi_k3":
|
||||||
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
|
messages, image_count, assistant_prefix = self._prepare_kimi_k3_messages(
|
||||||
@@ -701,21 +686,6 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
return prompt_ids
|
return prompt_ids
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _resolve_dsv41_reasoning_effort(self, value: Any) -> Union[str, int]:
|
|
||||||
"""Request effort for the V4.1 encoder; unsupported values warn and fall back."""
|
|
||||||
effort = chat_encoding.parse_dsv41_reasoning_effort(value)
|
|
||||||
if effort is not None:
|
|
||||||
return effort
|
|
||||||
if value is not None and value != "none":
|
|
||||||
logger.warning(
|
|
||||||
"DeepSeek-V4.1 does not support reasoning_effort=%r; using the "
|
|
||||||
"default %r (low/high/xhigh/max, a float in [0, 0.99], or an "
|
|
||||||
"integer budget in [1, 100] via chat_template_kwargs are accepted).",
|
|
||||||
value,
|
|
||||||
self._dsv41_default_reasoning_effort,
|
|
||||||
)
|
|
||||||
return self._dsv41_default_reasoning_effort
|
|
||||||
|
|
||||||
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
|
def _decode_response(self, ret_item: dict[str, Any]) -> str | ErrorResponse:
|
||||||
"""Extract text from response."""
|
"""Extract text from response."""
|
||||||
return ret_item["text"]
|
return ret_item["text"]
|
||||||
@@ -1591,9 +1561,14 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
ThinkingMode.THINKING if thinking_requested else ThinkingMode.CHAT
|
ThinkingMode.THINKING if thinking_requested else ThinkingMode.CHAT
|
||||||
)
|
)
|
||||||
messages = [msg.model_dump() for msg in request.messages]
|
messages = [msg.model_dump() for msg in request.messages]
|
||||||
|
strict_tool_call_arguments = (
|
||||||
|
self._encoding_adapter.strict_tool_call_arguments
|
||||||
|
if self._encoding_adapter is not None
|
||||||
|
else self.chat_encoding_spec != "kimi_k3"
|
||||||
|
)
|
||||||
for message in messages:
|
for message in messages:
|
||||||
normalize_assistant_tool_call_arguments(
|
normalize_assistant_tool_call_arguments(
|
||||||
message, strict=self.chat_encoding_spec != "kimi_k3"
|
message, strict=strict_tool_call_arguments
|
||||||
)
|
)
|
||||||
|
|
||||||
prompt_ids = self._encode_messages(
|
prompt_ids = self._encode_messages(
|
||||||
@@ -1601,9 +1576,12 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
request,
|
request,
|
||||||
thinking_mode,
|
thinking_mode,
|
||||||
tools=tools,
|
tools=tools,
|
||||||
|
image_data=image_data,
|
||||||
)
|
)
|
||||||
|
|
||||||
if prompt_ids is not None:
|
if prompt_ids is not None:
|
||||||
|
if image_data and not is_multimodal:
|
||||||
|
raise ValueError("image input is not supported for this model")
|
||||||
collects_openai_media = (
|
collects_openai_media = (
|
||||||
self._encoding_adapter.collects_openai_media
|
self._encoding_adapter.collects_openai_media
|
||||||
if self._encoding_adapter is not None
|
if self._encoding_adapter is not None
|
||||||
@@ -1623,33 +1601,30 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
modalities,
|
modalities,
|
||||||
)
|
)
|
||||||
elif self.chat_encoding_spec is not None:
|
elif self.chat_encoding_spec is not None:
|
||||||
# dsv4/dsv41/dsv32 encoding path
|
# dsv32 encoding path
|
||||||
messages = copy.deepcopy(messages)
|
messages = copy.deepcopy(messages)
|
||||||
is_dsv41 = self.chat_encoding_spec == "dsv41"
|
|
||||||
for msg in messages:
|
for msg in messages:
|
||||||
if msg.get("content") is None:
|
if msg.get("content") is None:
|
||||||
msg["content"] = ""
|
msg["content"] = ""
|
||||||
|
|
||||||
# The V4.1 encoder consumes OpenAI parts lists itself; dsv4/dsv32
|
# The dsv32 encoder is text-only, so parts-list content is flattened first.
|
||||||
# are text-only, so their parts-list content is flattened first.
|
for i, msg in enumerate(messages):
|
||||||
if not is_dsv41:
|
if isinstance(msg.get("content"), list):
|
||||||
for i, msg in enumerate(messages):
|
messages[i] = process_content_for_template_format(
|
||||||
if isinstance(msg.get("content"), list):
|
msg, "string", [], [], [], []
|
||||||
messages[i] = process_content_for_template_format(
|
|
||||||
msg, "string", [], [], [], []
|
|
||||||
)
|
|
||||||
|
|
||||||
for msg in messages:
|
|
||||||
processed_msg = process_content_for_template_format(
|
|
||||||
msg,
|
|
||||||
template_content_format,
|
|
||||||
image_data,
|
|
||||||
video_data,
|
|
||||||
audio_data,
|
|
||||||
modalities,
|
|
||||||
use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32",
|
|
||||||
)
|
)
|
||||||
msg.update(processed_msg)
|
|
||||||
|
for msg in messages:
|
||||||
|
processed_msg = process_content_for_template_format(
|
||||||
|
msg,
|
||||||
|
template_content_format,
|
||||||
|
image_data,
|
||||||
|
video_data,
|
||||||
|
audio_data,
|
||||||
|
modalities,
|
||||||
|
use_dpsk_v32_encoding=self.chat_encoding_spec == "dsv32",
|
||||||
|
)
|
||||||
|
msg.update(processed_msg)
|
||||||
|
|
||||||
# Handle continue_final_message: separate final assistant message
|
# Handle continue_final_message: separate final assistant message
|
||||||
messages, assistant_prefix = self._handle_last_assistant_message(
|
messages, assistant_prefix = self._handle_last_assistant_message(
|
||||||
@@ -1658,78 +1633,16 @@ class OpenAIServingChat(OpenAIServingBase):
|
|||||||
|
|
||||||
request_tools = request.tools if tools else None
|
request_tools = request.tools if tools else None
|
||||||
|
|
||||||
# An empty system message hosts the request tools; dsv41 renders a
|
# An empty system message hosts the request tools.
|
||||||
# system token for it, so it only gets one when tools need the host.
|
if messages[0]["role"] != "system":
|
||||||
if messages[0]["role"] != "system" and (request_tools or not is_dsv41):
|
|
||||||
messages.insert(0, {"role": "system", "content": ""})
|
messages.insert(0, {"role": "system", "content": ""})
|
||||||
if request_tools:
|
if request_tools:
|
||||||
messages[0]["tools"] = [
|
messages[0]["tools"] = [tool.model_dump() for tool in request_tools]
|
||||||
(
|
|
||||||
chat_encoding.dsv41_tool_payload(tool)
|
|
||||||
if is_dsv41
|
|
||||||
else tool.model_dump()
|
|
||||||
)
|
|
||||||
for tool in request_tools
|
|
||||||
]
|
|
||||||
|
|
||||||
# Default encoding (dsv4/dsv41/dsv32)
|
real_input = encoding_dsv32.encode_messages(
|
||||||
if self.chat_encoding_spec == "dsv4":
|
messages, thinking_mode=thinking_mode
|
||||||
effort_source = request.reasoning_effort
|
)
|
||||||
if effort_source is None:
|
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
||||||
env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get()
|
|
||||||
if env_val:
|
|
||||||
effort_source = env_val
|
|
||||||
reasoning_effort_profile = self._dsv4_reasoning_effort_profile
|
|
||||||
assert reasoning_effort_profile is not None
|
|
||||||
accepted_efforts = encoding_dsv4.REASONING_EFFORT_PROFILES[
|
|
||||||
reasoning_effort_profile
|
|
||||||
]
|
|
||||||
v4_reasoning_effort = (
|
|
||||||
effort_source if effort_source in accepted_efforts else None
|
|
||||||
)
|
|
||||||
if request.task is not None:
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(
|
|
||||||
messages, request.task
|
|
||||||
)
|
|
||||||
real_input = encoding_dsv4.encode_messages(
|
|
||||||
messages,
|
|
||||||
thinking_mode=thinking_mode,
|
|
||||||
reasoning_effort=v4_reasoning_effort,
|
|
||||||
reasoning_effort_profile=reasoning_effort_profile,
|
|
||||||
)
|
|
||||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
|
||||||
elif is_dsv41:
|
|
||||||
if request.task is not None:
|
|
||||||
encoding_dsv41.attach_task_to_last_user_message(
|
|
||||||
messages, request.task
|
|
||||||
)
|
|
||||||
real_input, media = encoding_dsv41.encode_messages(
|
|
||||||
messages,
|
|
||||||
thinking_mode=thinking_mode,
|
|
||||||
reasoning_effort=self._resolve_dsv41_reasoning_effort(
|
|
||||||
request.reasoning_effort
|
|
||||||
),
|
|
||||||
return_multi_modal_data=True,
|
|
||||||
)
|
|
||||||
if media["images"]:
|
|
||||||
if not is_multimodal:
|
|
||||||
raise ValueError("image input is not supported for this model")
|
|
||||||
image_data.extend(
|
|
||||||
ImageData(url=image["url"]) for image in media["images"]
|
|
||||||
)
|
|
||||||
tokenizer = self.tokenizer_manager.tokenizer
|
|
||||||
real_input = real_input.replace(
|
|
||||||
encoding_dsv41.IMAGE_PLACEHOLDER,
|
|
||||||
tokenizer.convert_ids_to_tokens(
|
|
||||||
self.tokenizer_manager.image_token_id
|
|
||||||
),
|
|
||||||
)
|
|
||||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
|
||||||
else:
|
|
||||||
real_input = encoding_dsv32.encode_messages(
|
|
||||||
messages, thinking_mode=thinking_mode
|
|
||||||
)
|
|
||||||
prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input)
|
|
||||||
|
|
||||||
# Append assistant prefix if continue_final_message is enabled
|
# Append assistant prefix if continue_final_message is enabled
|
||||||
if assistant_prefix:
|
if assistant_prefix:
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
"""DeepSeek-V4 / V4.1 serving glue for the ``dsv4`` / ``dsv41`` chat encoding
|
||||||
|
specs: prompt encoding through deepseek-recipe under SGLang's request semantics."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||||
|
|
||||||
|
from sglang.srt.environ import envs
|
||||||
|
from sglang.srt.parser.deepseek_recipe_renderer import (
|
||||||
|
RecipeEncoder,
|
||||||
|
parse_effort_budget,
|
||||||
|
)
|
||||||
|
from sglang.srt.utils import ImageData
|
||||||
|
|
||||||
|
if TYPE_CHECKING:
|
||||||
|
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
DEFAULT_EFFORT_ENVS = {
|
||||||
|
"dsv4": envs.SGLANG_DSV4_REASONING_EFFORT,
|
||||||
|
"dsv41": envs.SGLANG_DSV41_REASONING_EFFORT,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class DeepSeekRecipeChatEncodingAdapter:
|
||||||
|
# recipe sorts tool results by call order, so images come from its render.
|
||||||
|
collects_openai_media = False
|
||||||
|
output = None
|
||||||
|
strict_tool_call_arguments = False # recipe serializes object arguments itself.
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
chat_encoding_spec: str,
|
||||||
|
tokenizer: Any,
|
||||||
|
default_chat_template_kwargs: Optional[dict[str, Any]],
|
||||||
|
):
|
||||||
|
self._spec = chat_encoding_spec
|
||||||
|
self.encoder = RecipeEncoder.from_hf_tokenizer(chat_encoding_spec, tokenizer)
|
||||||
|
raw = (DEFAULT_EFFORT_ENVS[chat_encoding_spec].get() or "").strip()
|
||||||
|
self._default_effort = int(raw) if raw.isdigit() else raw or None
|
||||||
|
# Encoded at boot, so a bad server default fails startup, not requests.
|
||||||
|
defaults = default_chat_template_kwargs or {}
|
||||||
|
self._encode(
|
||||||
|
{"messages": [{"role": "user", "content": ""}], "thinking": True},
|
||||||
|
effort=defaults.get("reasoning_effort"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def encode_messages(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
messages: list[dict[str, Any]],
|
||||||
|
request: ChatCompletionRequest,
|
||||||
|
tools: Optional[list[dict]],
|
||||||
|
thinking: bool,
|
||||||
|
image_data: list,
|
||||||
|
) -> list[int]:
|
||||||
|
# The wire request: the handler's message dumps carry None for unset fields.
|
||||||
|
wire = request.model_dump(exclude_unset=True, by_alias=True, mode="json")
|
||||||
|
wire_messages = wire["messages"]
|
||||||
|
continue_final = request.continue_final_message
|
||||||
|
task_prefix = None
|
||||||
|
# SGLang's trailing-assistant rule, as `_handle_last_assistant_message`.
|
||||||
|
last = wire_messages[-1] if wire_messages else None
|
||||||
|
if (
|
||||||
|
last
|
||||||
|
and last["role"] == "assistant"
|
||||||
|
and isinstance(last.get("content"), str)
|
||||||
|
):
|
||||||
|
if not continue_final:
|
||||||
|
wire_messages[-1] = {"role": "user", "content": last["content"]}
|
||||||
|
elif request.task is not None:
|
||||||
|
# A task renders its own tail, so the prefix follows it.
|
||||||
|
task_prefix = wire_messages.pop()["content"]
|
||||||
|
continue_final = False
|
||||||
|
body = {
|
||||||
|
"messages": wire_messages,
|
||||||
|
"thinking": thinking,
|
||||||
|
"continue_final_message": continue_final,
|
||||||
|
"response_format": wire.get("response_format"),
|
||||||
|
}
|
||||||
|
if tools is not None:
|
||||||
|
# `model_dump` adds None for unset fields, which recipe renders as values.
|
||||||
|
body["tools"] = [
|
||||||
|
{k: v for k, v in tool.items() if v is not None}
|
||||||
|
| {
|
||||||
|
"function": {
|
||||||
|
k: v for k, v in tool["function"].items() if v is not None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for tool in tools
|
||||||
|
]
|
||||||
|
body["tool_choice"] = wire.get("tool_choice")
|
||||||
|
body = {key: value for key, value in body.items() if value is not None}
|
||||||
|
image_urls = []
|
||||||
|
prompt_ids = self._encode(
|
||||||
|
body,
|
||||||
|
effort=request.reasoning_effort,
|
||||||
|
task=request.task,
|
||||||
|
image_urls=image_urls,
|
||||||
|
)
|
||||||
|
image_data.extend(ImageData(url=url) for url in image_urls)
|
||||||
|
if task_prefix:
|
||||||
|
prompt_ids += self.encoder.encode_text(task_prefix)
|
||||||
|
return prompt_ids
|
||||||
|
|
||||||
|
def _encode(
|
||||||
|
self,
|
||||||
|
body: dict[str, Any],
|
||||||
|
*,
|
||||||
|
effort: Union[str, int, float, None],
|
||||||
|
task: Optional[str] = None,
|
||||||
|
image_urls: Optional[list[str]] = None,
|
||||||
|
) -> list[int]:
|
||||||
|
if effort is None or effort == "none":
|
||||||
|
effort = self._default_effort
|
||||||
|
budget = None
|
||||||
|
if effort is not None and not isinstance(effort, str):
|
||||||
|
if self._spec != "dsv41":
|
||||||
|
raise ValueError(
|
||||||
|
f"DeepSeek-V4 reasoning_effort must be a named tier, got {effort!r}"
|
||||||
|
)
|
||||||
|
budget = parse_effort_budget(effort)
|
||||||
|
elif effort is not None:
|
||||||
|
body["reasoning_effort"] = effort
|
||||||
|
return self.encoder.encode(
|
||||||
|
body, task=task, effort_budget=budget, image_urls=image_urls
|
||||||
|
)
|
||||||
@@ -35,6 +35,7 @@ if TYPE_CHECKING:
|
|||||||
|
|
||||||
class InklingChatEncodingAdapter:
|
class InklingChatEncodingAdapter:
|
||||||
collects_openai_media = True
|
collects_openai_media = True
|
||||||
|
strict_tool_call_arguments = True
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -73,7 +74,11 @@ class InklingChatEncodingAdapter:
|
|||||||
messages: list[dict[str, Any]],
|
messages: list[dict[str, Any]],
|
||||||
request: ChatCompletionRequest,
|
request: ChatCompletionRequest,
|
||||||
tools: list[dict] | None,
|
tools: list[dict] | None,
|
||||||
|
thinking: bool,
|
||||||
|
image_data: list,
|
||||||
) -> list[int]:
|
) -> list[int]:
|
||||||
|
# Inkling's thinking is effort-conditioned and the handler collects its
|
||||||
|
# media, so `thinking` and `image_data` are not read.
|
||||||
reasoning_effort = parse_reasoning_effort(request.reasoning_effort)
|
reasoning_effort = parse_reasoning_effort(request.reasoning_effort)
|
||||||
if reasoning_effort is None:
|
if reasoning_effort is None:
|
||||||
reasoning_effort = self._default_reasoning_effort
|
reasoning_effort = self._default_reasoning_effort
|
||||||
|
|||||||
@@ -1699,10 +1699,12 @@ class Envs:
|
|||||||
SGLANG_DSV4_FP4_EXPERTS = EnvBool(True)
|
SGLANG_DSV4_FP4_EXPERTS = EnvBool(True)
|
||||||
# Set True to dequantize the FP4 experts to FP8 at runtime
|
# Set True to dequantize the FP4 experts to FP8 at runtime
|
||||||
SGLANG_DSV4_FP4_DEQUANT = EnvBool(False)
|
SGLANG_DSV4_FP4_DEQUANT = EnvBool(False)
|
||||||
# Flash-0731 also accepts "low"; the active profile is checkpoint-resolved.
|
# DeepSeek-V4 default when a request carries no reasoning_effort: a
|
||||||
|
# deepseek-recipe tier name; unset -> deepseek-recipe's default.
|
||||||
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
|
SGLANG_DSV4_REASONING_EFFORT = EnvStr("")
|
||||||
# DeepSeek-V4.1 default when a request carries no reasoning_effort: one of
|
# DeepSeek-V4.1 default when a request carries no reasoning_effort: one of
|
||||||
# low/high/xhigh/max or an integer budget in [1, 100]; unset -> the encoder default.
|
# a deepseek-recipe tier name or an integer budget in [1, 100]; unset ->
|
||||||
|
# deepseek-recipe's default.
|
||||||
SGLANG_DSV41_REASONING_EFFORT = EnvStr(None)
|
SGLANG_DSV41_REASONING_EFFORT = EnvStr(None)
|
||||||
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
|
# Quantize the SWA fp8 KV cache from bf16-rounded values (matches
|
||||||
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
|
# trainer-side QAT and the DSA-CP path) instead of fp32 registers.
|
||||||
|
|||||||
@@ -0,0 +1,199 @@
|
|||||||
|
"""DeepSeek-V4 / V4.1 prompts through deepseek-recipe, plus the SGLang extensions
|
||||||
|
recipe does not model (quick instruction tasks, V4.1 integer effort)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import functools
|
||||||
|
import json
|
||||||
|
from types import ModuleType
|
||||||
|
from typing import Any, Optional, Union
|
||||||
|
|
||||||
|
DEEPSEEK_RECIPE_INSTALL_HINT = (
|
||||||
|
"Serving DeepSeek-V4 / V4.1 requires the `deepseek-recipe` package (DeepSeek's "
|
||||||
|
"reference encoder and parser; wheels for Linux and macOS). Install it with "
|
||||||
|
"`pip install deepseek-recipe`."
|
||||||
|
)
|
||||||
|
|
||||||
|
MODEL_NAMES = {"dsv4": "DeepSeek-V4-Pro", "dsv41": "DeepSeek-V4.1-Flash"}
|
||||||
|
# The checkpoint tokenizer's image token; recipe's bundled tokenizer spells it <|image|>.
|
||||||
|
IMAGE_TOKEN = "<|deepseek_image|>"
|
||||||
|
ASSISTANT_TOKEN = "<|Assistant|>"
|
||||||
|
THINK_START_TOKEN = "<think>"
|
||||||
|
THINK_END_TOKEN = "</think>"
|
||||||
|
TASK_TOKENS = {
|
||||||
|
"action": "<|action|>",
|
||||||
|
"query": "<|query|>",
|
||||||
|
"authority": "<|authority|>",
|
||||||
|
"domain": "<|domain|>",
|
||||||
|
"title": "<|title|>",
|
||||||
|
"read_url": "<|read_url|>",
|
||||||
|
}
|
||||||
|
# V4.1 renders an integer budget as this line; recipe's "high" tier is budget 75.
|
||||||
|
BUDGET_TIER, BUDGET_TIER_VALUE = "high", 75
|
||||||
|
BUDGET_LINE = "Reasoning Effort: {budget}"
|
||||||
|
|
||||||
|
|
||||||
|
@functools.cache
|
||||||
|
def load_deepseek_recipe() -> ModuleType:
|
||||||
|
try:
|
||||||
|
import deepseek_recipe
|
||||||
|
except ImportError as exc:
|
||||||
|
raise ImportError(DEEPSEEK_RECIPE_INSTALL_HINT) from exc
|
||||||
|
return deepseek_recipe
|
||||||
|
|
||||||
|
|
||||||
|
def parse_effort_budget(value: Union[int, float]) -> int:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
raise ValueError("reasoning_effort must not be a boolean")
|
||||||
|
if isinstance(value, int) and 1 <= value <= 100:
|
||||||
|
return value
|
||||||
|
if isinstance(value, float) and 0.0 <= value <= 0.99:
|
||||||
|
return max(1, round(value * 100))
|
||||||
|
raise ValueError(
|
||||||
|
f"reasoning_effort={value!r} is outside the integer budget [1, 100] "
|
||||||
|
"and the float range [0, 0.99]"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def pre_normalize(body: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""Rewrite an SGLang chat request body for recipe's Chat Completions adapter.
|
||||||
|
|
||||||
|
`body` carries only what the prompt depends on: messages, thinking and, when
|
||||||
|
set, tools, tool_choice, response_format, continue_final_message and a named
|
||||||
|
reasoning_effort.
|
||||||
|
"""
|
||||||
|
request = copy.deepcopy(body)
|
||||||
|
request["thinking"] = {"type": "enabled" if request["thinking"] else "disabled"}
|
||||||
|
# The host enforces structural_tag with its grammar.
|
||||||
|
response_format = request.get("response_format")
|
||||||
|
if response_format and response_format.get("type") == "structural_tag":
|
||||||
|
del request["response_format"]
|
||||||
|
choice = request.get("tool_choice")
|
||||||
|
if not request.get("tools") or choice == "none":
|
||||||
|
request.pop("tools", None)
|
||||||
|
request.pop("tool_choice", None)
|
||||||
|
elif choice == "required" or isinstance(choice, dict):
|
||||||
|
# SGLang enforces the choice with its grammar, which expects the model
|
||||||
|
# to open the calls block itself.
|
||||||
|
request["tool_choice"] = "auto"
|
||||||
|
messages = request["messages"]
|
||||||
|
for message in messages:
|
||||||
|
if message["role"] == "developer":
|
||||||
|
message["role"] = "system"
|
||||||
|
for call in message.get("tool_calls") or []:
|
||||||
|
function = call["function"]
|
||||||
|
if "arguments" in function and not isinstance(function["arguments"], str):
|
||||||
|
function["arguments"] = json.dumps(
|
||||||
|
function["arguments"], ensure_ascii=False, separators=(",", ":")
|
||||||
|
)
|
||||||
|
continue_final = request.pop("continue_final_message", False)
|
||||||
|
if continue_final and messages and messages[-1]["role"] == "assistant":
|
||||||
|
messages[-1]["prefix"] = True
|
||||||
|
return request
|
||||||
|
|
||||||
|
|
||||||
|
def _error_message(body: Any) -> str:
|
||||||
|
try:
|
||||||
|
return json.loads(body)["error"]["message"]
|
||||||
|
except (TypeError, ValueError, KeyError):
|
||||||
|
return str(body)
|
||||||
|
|
||||||
|
|
||||||
|
class RecipeEncoder:
|
||||||
|
"""`tokenizer` is the checkpoint's `tokenizers.Tokenizer`; recipe encodes prompts
|
||||||
|
with its own copy of it, and the SGLang extensions splice ids from the original."""
|
||||||
|
|
||||||
|
def __init__(self, chat_encoding_spec: str, tokenizer: Any):
|
||||||
|
recipe = load_deepseek_recipe()
|
||||||
|
self.model = MODEL_NAMES[chat_encoding_spec]
|
||||||
|
self.hf_tokenizer = tokenizer
|
||||||
|
special = recipe.SpecialTokens(image=IMAGE_TOKEN)
|
||||||
|
self._options = recipe.ConversionOptions().with_special_tokens(special)
|
||||||
|
encoding = (
|
||||||
|
recipe.DeepseekV4Encoding()
|
||||||
|
if chat_encoding_spec == "dsv4"
|
||||||
|
else recipe.DeepseekV41Encoding()
|
||||||
|
)
|
||||||
|
self._encoding = encoding.with_special_tokens(special).with_tokenizer(
|
||||||
|
recipe.Tokenizer.from_str(tokenizer.to_str())
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_hf_tokenizer(
|
||||||
|
cls, chat_encoding_spec: str, tokenizer: Any
|
||||||
|
) -> RecipeEncoder:
|
||||||
|
backend = getattr(tokenizer, "backend_tokenizer", None)
|
||||||
|
if backend is None:
|
||||||
|
raise ValueError(
|
||||||
|
"deepseek-recipe needs the checkpoint's fast tokenizer (tokenizer.json)"
|
||||||
|
)
|
||||||
|
return cls(chat_encoding_spec, backend)
|
||||||
|
|
||||||
|
def encode(
|
||||||
|
self,
|
||||||
|
body: dict[str, Any],
|
||||||
|
*,
|
||||||
|
task: Optional[str] = None,
|
||||||
|
effort_budget: Optional[int] = None,
|
||||||
|
image_urls: Optional[list[str]] = None,
|
||||||
|
) -> list[int]:
|
||||||
|
"""Prompt ids for `body` (see `pre_normalize`); a request recipe rejects
|
||||||
|
raises ValueError. `image_urls` receives the images in placeholder order."""
|
||||||
|
recipe = load_deepseek_recipe()
|
||||||
|
normalized = pre_normalize(body)
|
||||||
|
normalized["model"] = self.model
|
||||||
|
if effort_budget is not None:
|
||||||
|
normalized["reasoning_effort"] = BUDGET_TIER
|
||||||
|
try:
|
||||||
|
converted = recipe.ChatCompletionRequest(normalized).convert(self._options)
|
||||||
|
except recipe.ConversionError as error:
|
||||||
|
raise ValueError(_error_message(error.body)) from error
|
||||||
|
prompt_ids = self._encoding.encode(converted.conversation)
|
||||||
|
if image_urls is not None:
|
||||||
|
rendered = self._encoding.render_conversation(converted.conversation)
|
||||||
|
image_urls.extend(
|
||||||
|
source.url or source.data_url for source in rendered.image_sources
|
||||||
|
)
|
||||||
|
if effort_budget is not None and body["thinking"]:
|
||||||
|
prompt_ids = self._replace_effort_budget(prompt_ids, effort_budget)
|
||||||
|
if task is not None:
|
||||||
|
prompt_ids = self._attach_task(prompt_ids, task, body["thinking"])
|
||||||
|
return prompt_ids
|
||||||
|
|
||||||
|
def encode_text(self, text: str) -> list[int]:
|
||||||
|
return self.hf_tokenizer.encode(text, add_special_tokens=False).ids
|
||||||
|
|
||||||
|
def _replace_effort_budget(self, prompt_ids: list[int], budget: int) -> list[int]:
|
||||||
|
# The effort line opens the prompt, so its first match is the line itself.
|
||||||
|
rendered = self.encode_text(BUDGET_LINE.format(budget=BUDGET_TIER_VALUE))
|
||||||
|
for start in range(len(prompt_ids) - len(rendered) + 1):
|
||||||
|
if prompt_ids[start : start + len(rendered)] == rendered:
|
||||||
|
end = start + len(rendered)
|
||||||
|
return (
|
||||||
|
prompt_ids[:start]
|
||||||
|
+ self.encode_text(BUDGET_LINE.format(budget=budget))
|
||||||
|
+ prompt_ids[end:]
|
||||||
|
)
|
||||||
|
raise ValueError("deepseek-recipe rendered no reasoning effort line")
|
||||||
|
|
||||||
|
def _attach_task(
|
||||||
|
self, prompt_ids: list[int], task: str, thinking: bool
|
||||||
|
) -> list[int]:
|
||||||
|
"""A quick instruction task on the last user turn, as DeepSeek's checkpoint
|
||||||
|
encoder renders it: `action` follows the generation header, other tasks
|
||||||
|
replace it."""
|
||||||
|
header = [
|
||||||
|
self.hf_tokenizer.token_to_id(ASSISTANT_TOKEN),
|
||||||
|
self.hf_tokenizer.token_to_id(
|
||||||
|
THINK_START_TOKEN if thinking else THINK_END_TOKEN
|
||||||
|
),
|
||||||
|
]
|
||||||
|
if prompt_ids[-len(header) :] != header:
|
||||||
|
raise ValueError(
|
||||||
|
"task requires the conversation to end with a user message"
|
||||||
|
)
|
||||||
|
task_id = self.hf_tokenizer.token_to_id(TASK_TOKENS[task])
|
||||||
|
if task == "action":
|
||||||
|
return prompt_ids + [task_id]
|
||||||
|
return prompt_ids[: -len(header)] + [task_id]
|
||||||
@@ -4,14 +4,6 @@ from typing import Dict, List, Optional, Tuple, Type
|
|||||||
|
|
||||||
from transformers import PreTrainedTokenizerBase
|
from transformers import PreTrainedTokenizerBase
|
||||||
|
|
||||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import dsml_token as dsv4_dsml_token
|
|
||||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import eos_token as dsv4_eos_token
|
|
||||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
|
|
||||||
thinking_end_token as dsv4_thinking_end_token,
|
|
||||||
)
|
|
||||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
|
|
||||||
thinking_start_token as dsv4_thinking_start_token,
|
|
||||||
)
|
|
||||||
from sglang.srt.entrypoints.openai.protocol import (
|
from sglang.srt.entrypoints.openai.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
ResponsesRequest,
|
ResponsesRequest,
|
||||||
@@ -1525,6 +1517,12 @@ class _DeepSeekV3Detector(Qwen3Detector):
|
|||||||
self.reasoning_default = "explicit_thinking"
|
self.reasoning_default = "explicit_thinking"
|
||||||
|
|
||||||
|
|
||||||
|
DSV4_EOS_TOKEN = "<|end▁of▁sentence|>"
|
||||||
|
DSV4_THINK_START_TOKEN = "<think>"
|
||||||
|
DSV4_THINK_END_TOKEN = "</think>"
|
||||||
|
DSV4_DSML_TOKEN = "|DSML|"
|
||||||
|
|
||||||
|
|
||||||
class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@@ -1535,13 +1533,13 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
|||||||
force_nonempty_content: bool = False,
|
force_nonempty_content: bool = False,
|
||||||
):
|
):
|
||||||
super().__init__(
|
super().__init__(
|
||||||
dsv4_thinking_start_token,
|
DSV4_THINK_START_TOKEN,
|
||||||
dsv4_thinking_end_token,
|
DSV4_THINK_END_TOKEN,
|
||||||
think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token],
|
think_excluded_tokens=[DSV4_EOS_TOKEN, DSV4_DSML_TOKEN],
|
||||||
# The reference parser splits reasoning at `\n\n<|DSML|tool_calls`;
|
# The reference parser splits reasoning at `\n\n<|DSML|tool_calls`;
|
||||||
# anchoring on the full tag at a line start keeps a mid-sentence
|
# anchoring on the full tag at a line start keeps a mid-sentence
|
||||||
# mention of the tag inside reasoning from ending the block.
|
# mention of the tag inside reasoning from ending the block.
|
||||||
tool_start_token=f"<{dsv4_dsml_token}tool_calls>",
|
tool_start_token=f"<{DSV4_DSML_TOKEN}tool_calls>",
|
||||||
tool_start_at_line_start=True,
|
tool_start_at_line_start=True,
|
||||||
force_reasoning=force_reasoning,
|
force_reasoning=force_reasoning,
|
||||||
stream_reasoning=stream_reasoning,
|
stream_reasoning=stream_reasoning,
|
||||||
@@ -1559,7 +1557,7 @@ class DeepSeekV41ReasoningDetector(DeepSeekV4Detector):
|
|||||||
|
|
||||||
def __init__(self, **kwargs):
|
def __init__(self, **kwargs):
|
||||||
super().__init__(**kwargs)
|
super().__init__(**kwargs)
|
||||||
self.tool_start_token = f"<{dsv4_dsml_token} calls>"
|
self.tool_start_token = f"<{DSV4_DSML_TOKEN} calls>"
|
||||||
|
|
||||||
|
|
||||||
class _MimoDetector(Qwen3Detector):
|
class _MimoDetector(Qwen3Detector):
|
||||||
|
|||||||
@@ -37,7 +37,6 @@ from sglang.srt.runtime_context import publish, reset_context
|
|||||||
from sglang.srt.server_args import ServerArgs
|
from sglang.srt.server_args import ServerArgs
|
||||||
|
|
||||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/parity"
|
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures/parity"
|
||||||
DSV4_PROFILE = chat_encoding.DSV4_REASONING_EFFORT_PROFILE_OVERRIDE
|
|
||||||
|
|
||||||
|
|
||||||
class RecordingTokenizer:
|
class RecordingTokenizer:
|
||||||
@@ -87,18 +86,6 @@ def serving_chat(model, path, revision, config, tokenizer, tool_call_parser):
|
|||||||
tokenizer=tokenizer,
|
tokenizer=tokenizer,
|
||||||
tool_call_parser=tool_call_parser,
|
tool_call_parser=tool_call_parser,
|
||||||
)
|
)
|
||||||
server._dsv4_reasoning_effort_profile = (
|
|
||||||
chat_encoding.resolve_dsv4_reasoning_effort_profile(
|
|
||||||
model_path=path, revision=revision, override=config.get(DSV4_PROFILE)
|
|
||||||
)
|
|
||||||
if server.chat_encoding_spec == "dsv4"
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
server._dsv41_default_reasoning_effort = (
|
|
||||||
chat_encoding.default_dsv41_reasoning_effort_from_env(None)
|
|
||||||
if server.chat_encoding_spec == "dsv41"
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
server._tokenizer_auto_adds_specials = len(tokenizer.encode("")) > 0
|
server._tokenizer_auto_adds_specials = len(tokenizer.encode("")) > 0
|
||||||
server._prompt_text_round_trip_is_lossy = server._probe_prompt_text_round_trip()
|
server._prompt_text_round_trip_is_lossy = server._probe_prompt_text_round_trip()
|
||||||
server._chat_template_cache = OrderedDict()
|
server._chat_template_cache = OrderedDict()
|
||||||
@@ -130,13 +117,10 @@ def generate(fixture_path):
|
|||||||
tok,
|
tok,
|
||||||
fixture.get("tool_call_parser"),
|
fixture.get("tool_call_parser"),
|
||||||
)
|
)
|
||||||
# What the processor reads from config.json; the resolved profile stands in
|
# What the processor reads from config.json.
|
||||||
# for the checkpoint's encoder file so the Rust test runs offline.
|
|
||||||
fixture["config"] = {
|
fixture["config"] = {
|
||||||
key: config[key] for key in ("model_type", "architectures") if key in config
|
key: config[key] for key in ("model_type", "architectures") if key in config
|
||||||
}
|
}
|
||||||
if server._dsv4_reasoning_effort_profile:
|
|
||||||
fixture["config"][DSV4_PROFILE] = server._dsv4_reasoning_effort_profile
|
|
||||||
fixture["bos_token_id"] = tok.bos_token_id
|
fixture["bos_token_id"] = tok.bos_token_id
|
||||||
# Jinja checkpoints render with their template; keep it so the test runs offline.
|
# Jinja checkpoints render with their template; keep it so the test runs offline.
|
||||||
if server.chat_encoding_spec is None:
|
if server.chat_encoding_spec is None:
|
||||||
|
|||||||
@@ -261,7 +261,7 @@ class TestChatCompletionRequest(unittest.TestCase):
|
|||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
messages = [{"role": "user", "content": "Hello"}]
|
messages = [{"role": "user", "content": "Hello"}]
|
||||||
for effort in ("xhigh", "max"):
|
for effort in ("xhigh", "ultra", "max"):
|
||||||
with self.subTest(effort=effort, request_form="top-level"):
|
with self.subTest(effort=effort, request_form="top-level"):
|
||||||
request = ChatCompletionRequest(
|
request = ChatCompletionRequest(
|
||||||
model="test-model",
|
model="test-model",
|
||||||
@@ -283,7 +283,7 @@ class TestChatCompletionRequest(unittest.TestCase):
|
|||||||
ChatCompletionRequest(
|
ChatCompletionRequest(
|
||||||
model="test-model",
|
model="test-model",
|
||||||
messages=messages,
|
messages=messages,
|
||||||
reasoning_effort="ultra",
|
reasoning_effort="extreme",
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_chat_completion_reasoning_effort_is_strictly_validated(self):
|
def test_chat_completion_reasoning_effort_is_strictly_validated(self):
|
||||||
|
|||||||
@@ -11,23 +11,18 @@ from sglang.test.test_utils import CustomTestCase, enter_override, maybe_stub_sg
|
|||||||
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
import copy
|
|
||||||
import gc
|
import gc
|
||||||
import json
|
import json
|
||||||
import tempfile
|
|
||||||
import unittest
|
import unittest
|
||||||
import uuid
|
import uuid
|
||||||
from http import HTTPStatus
|
from http import HTTPStatus
|
||||||
from pathlib import Path
|
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock, patch
|
||||||
|
|
||||||
from fastapi import Request
|
from fastapi import Request
|
||||||
from transformers.utils.chat_template_utils import _compile_jinja_template
|
from transformers.utils.chat_template_utils import _compile_jinja_template
|
||||||
|
|
||||||
from sglang.srt.entrypoints.openai.chat_encoding import (
|
from sglang.srt.entrypoints.openai.chat_encoding import resolve_chat_encoding_spec
|
||||||
resolve_dsv4_reasoning_effort_profile,
|
|
||||||
)
|
|
||||||
from sglang.srt.entrypoints.openai.protocol import (
|
from sglang.srt.entrypoints.openai.protocol import (
|
||||||
ChatCompletionRequest,
|
ChatCompletionRequest,
|
||||||
Function,
|
Function,
|
||||||
@@ -157,15 +152,6 @@ _TOOL_RESULT_REORDER_TEMPLATE = """
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def _create_dsv4_checkpoint(test_case: unittest.TestCase, source: str) -> str:
|
|
||||||
model_dir = tempfile.TemporaryDirectory()
|
|
||||||
test_case.addCleanup(model_dir.cleanup)
|
|
||||||
encoder_path = Path(model_dir.name) / "encoding" / "encoding_dsv4.py"
|
|
||||||
encoder_path.parent.mkdir(parents=True)
|
|
||||||
encoder_path.write_text(source, encoding="utf-8")
|
|
||||||
return model_dir.name
|
|
||||||
|
|
||||||
|
|
||||||
class _MockTokenizerManager:
|
class _MockTokenizerManager:
|
||||||
"""Minimal mock that satisfies OpenAIServingChat."""
|
"""Minimal mock that satisfies OpenAIServingChat."""
|
||||||
|
|
||||||
@@ -1915,7 +1901,6 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
def test_dsv_tool_choice_none_omits_tools_from_prompt(self):
|
def test_dsv_tool_choice_none_omits_tools_from_prompt(self):
|
||||||
self.template_manager.chat_template_name = None
|
self.template_manager.chat_template_name = None
|
||||||
self.template_manager.jinja_template_content_format = "string"
|
self.template_manager.jinja_template_content_format = "string"
|
||||||
self.chat._dsv4_reasoning_effort_profile = "preview"
|
|
||||||
tool = {
|
tool = {
|
||||||
"type": "function",
|
"type": "function",
|
||||||
"function": {
|
"function": {
|
||||||
@@ -1924,25 +1909,21 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for chat_encoding_spec in ("dsv4", "dsv32", "dsv41"):
|
self.chat.chat_encoding_spec = "dsv32"
|
||||||
for tool_choice, expects_tools in (("none", False), ("auto", True)):
|
for tool_choice, expects_tools in (("none", False), ("auto", True)):
|
||||||
with self.subTest(
|
with self.subTest(tool_choice=tool_choice):
|
||||||
chat_encoding_spec=chat_encoding_spec,
|
self.tm.tokenizer.encode.reset_mock()
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
model="x",
|
||||||
|
messages=[{"role": "user", "content": "Solve 100 / 7."}],
|
||||||
|
tools=[tool],
|
||||||
tool_choice=tool_choice,
|
tool_choice=tool_choice,
|
||||||
):
|
)
|
||||||
self.chat.chat_encoding_spec = chat_encoding_spec
|
|
||||||
self.tm.tokenizer.encode.reset_mock()
|
|
||||||
request = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[{"role": "user", "content": "Solve 100 / 7."}],
|
|
||||||
tools=[tool],
|
|
||||||
tool_choice=tool_choice,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.chat._process_messages(request, is_multimodal=False)
|
self.chat._process_messages(request, is_multimodal=False)
|
||||||
|
|
||||||
prompt = self.tm.tokenizer.encode.call_args.args[0]
|
prompt = self.tm.tokenizer.encode.call_args.args[0]
|
||||||
self.assertEqual("calculator" in prompt, expects_tools)
|
self.assertEqual("calculator" in prompt, expects_tools)
|
||||||
|
|
||||||
def test_kimi_k3_encoder_receives_wire_request_fields(self):
|
def test_kimi_k3_encoder_receives_wire_request_fields(self):
|
||||||
self.template_manager.chat_template_name = None
|
self.template_manager.chat_template_name = None
|
||||||
@@ -2200,76 +2181,70 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
|
"""DeepSeek encoders should reject history tool call scalars as BadRequest."""
|
||||||
self.template_manager.chat_template_name = None
|
self.template_manager.chat_template_name = None
|
||||||
self.template_manager.jinja_template_content_format = "string"
|
self.template_manager.jinja_template_content_format = "string"
|
||||||
self.chat._dsv4_reasoning_effort_profile = "preview"
|
|
||||||
|
|
||||||
for chat_encoding_spec in ("dsv4", "dsv32"):
|
self.chat.chat_encoding_spec = "dsv32"
|
||||||
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
req = ChatCompletionRequest(
|
||||||
self.chat.chat_encoding_spec = chat_encoding_spec
|
model="x",
|
||||||
req = ChatCompletionRequest(
|
messages=[
|
||||||
model="x",
|
{"role": "user", "content": "Where is it raining?"},
|
||||||
messages=[
|
{
|
||||||
{"role": "user", "content": "Where is it raining?"},
|
"role": "assistant",
|
||||||
|
"content": None,
|
||||||
|
"tool_calls": [
|
||||||
{
|
{
|
||||||
"role": "assistant",
|
"id": "call_1",
|
||||||
"content": None,
|
"type": "function",
|
||||||
"tool_calls": [
|
"function": {
|
||||||
{
|
"name": "get_weather",
|
||||||
"id": "call_1",
|
"arguments": '"Beijing"',
|
||||||
"type": "function",
|
},
|
||||||
"function": {
|
}
|
||||||
"name": "get_weather",
|
|
||||||
"arguments": '"Beijing"',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"role": "tool",
|
|
||||||
"tool_call_id": "call_1",
|
|
||||||
"content": "sunny",
|
|
||||||
},
|
|
||||||
],
|
],
|
||||||
)
|
},
|
||||||
|
{
|
||||||
|
"role": "tool",
|
||||||
|
"tool_call_id": "call_1",
|
||||||
|
"content": "sunny",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
with self.assertRaisesRegex(ValueError, "must be a JSON object"):
|
with self.assertRaisesRegex(ValueError, "must be a JSON object"):
|
||||||
self.chat._process_messages(req, is_multimodal=False)
|
self.chat._process_messages(req, is_multimodal=False)
|
||||||
|
|
||||||
def test_dsv_encoders_accept_object_tool_call_arguments_string(self):
|
def test_dsv_encoders_accept_object_tool_call_arguments_string(self):
|
||||||
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
|
"""DeepSeek encoders accept object-shaped OpenAI JSON string arguments."""
|
||||||
self.template_manager.chat_template_name = None
|
self.template_manager.chat_template_name = None
|
||||||
self.template_manager.jinja_template_content_format = "string"
|
self.template_manager.jinja_template_content_format = "string"
|
||||||
self.chat._dsv4_reasoning_effort_profile = "preview"
|
|
||||||
|
|
||||||
for chat_encoding_spec in ("dsv4", "dsv32"):
|
self.chat.chat_encoding_spec = "dsv32"
|
||||||
with self.subTest(chat_encoding_spec=chat_encoding_spec):
|
req = ChatCompletionRequest(
|
||||||
self.chat.chat_encoding_spec = chat_encoding_spec
|
model="x",
|
||||||
req = ChatCompletionRequest(
|
messages=[
|
||||||
model="x",
|
{"role": "user", "content": "Where is it raining?"},
|
||||||
messages=[
|
{
|
||||||
{"role": "user", "content": "Where is it raining?"},
|
"role": "assistant",
|
||||||
|
"content": None,
|
||||||
|
"tool_calls": [
|
||||||
{
|
{
|
||||||
"role": "assistant",
|
"id": "call_1",
|
||||||
"content": None,
|
"type": "function",
|
||||||
"tool_calls": [
|
"function": {
|
||||||
{
|
"name": "get_weather",
|
||||||
"id": "call_1",
|
"arguments": '{"city": "Beijing"}',
|
||||||
"type": "function",
|
},
|
||||||
"function": {
|
}
|
||||||
"name": "get_weather",
|
|
||||||
"arguments": '{"city": "Beijing"}',
|
|
||||||
},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"role": "tool",
|
|
||||||
"tool_call_id": "call_1",
|
|
||||||
"content": "sunny",
|
|
||||||
},
|
|
||||||
],
|
],
|
||||||
)
|
},
|
||||||
|
{
|
||||||
|
"role": "tool",
|
||||||
|
"tool_call_id": "call_1",
|
||||||
|
"content": "sunny",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
self.chat._process_messages(req, is_multimodal=False)
|
self.chat._process_messages(req, is_multimodal=False)
|
||||||
|
|
||||||
def test_stop_str_isolation_between_requests(self):
|
def test_stop_str_isolation_between_requests(self):
|
||||||
"""Test that stop strings from one request don't affect subsequent requests.
|
"""Test that stop strings from one request don't affect subsequent requests.
|
||||||
@@ -3110,17 +3085,14 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
|
# Case 4: DeepseekV4 arch -> always dsv4, even with chat_template
|
||||||
# (release ships a stale V3 jinja we deliberately override).
|
# (release ships a stale V3 jinja we deliberately override).
|
||||||
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
mock_hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
||||||
mock_hf_config.to_dict.return_value = {
|
for chat_template in ("stale v3 jinja", None):
|
||||||
"dsv4_reasoning_effort_profile": "preview"
|
tm.tokenizer.chat_template = chat_template
|
||||||
}
|
self.assertEqual(
|
||||||
tm.model_path = "deepseek-ai/DeepSeek-V4-Flash"
|
resolve_chat_encoding_spec(
|
||||||
tm.tokenizer.chat_template = "stale v3 jinja"
|
hf_config=mock_hf_config, tokenizer=tm.tokenizer
|
||||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
),
|
||||||
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
|
"dsv4",
|
||||||
|
)
|
||||||
tm.tokenizer.chat_template = None
|
|
||||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
|
||||||
self.assertEqual(serving_chat.chat_encoding_spec, "dsv4")
|
|
||||||
|
|
||||||
def test_kimi_k3_encoding_detection(self):
|
def test_kimi_k3_encoding_detection(self):
|
||||||
from sglang.srt.parser.template_manager import TemplateManager
|
from sglang.srt.parser.template_manager import TemplateManager
|
||||||
@@ -3151,7 +3123,7 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
self.chat.chat_encoding_spec = None
|
self.chat.chat_encoding_spec = None
|
||||||
self.assertFalse(self.chat.supports_native_reasoning_history())
|
self.assertFalse(self.chat.supports_native_reasoning_history())
|
||||||
|
|
||||||
# ------------- dsv4 task + latest_reminder -------------
|
# ------------- dsv4 task -------------
|
||||||
def test_dsv4_task_field_schema(self):
|
def test_dsv4_task_field_schema(self):
|
||||||
"""Top-level `task` accepts the 6 DS task tokens and rejects others."""
|
"""Top-level `task` accepts the 6 DS task tokens and rejects others."""
|
||||||
for valid in ("action", "query", "authority", "domain", "title", "read_url"):
|
for valid in ("action", "query", "authority", "domain", "title", "read_url"):
|
||||||
@@ -3175,407 +3147,6 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
task="bogus",
|
task="bogus",
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_attach_task_to_last_user_message(self):
|
|
||||||
"""Helper attaches task to the nearest user/developer message."""
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
|
||||||
|
|
||||||
messages = [{"role": "user", "content": "Hi"}]
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(messages, "domain")
|
|
||||||
self.assertEqual(messages[0]["task"], "domain")
|
|
||||||
|
|
||||||
# Prefers the LAST user message across a multi-turn conversation.
|
|
||||||
messages = [
|
|
||||||
{"role": "user", "content": "first"},
|
|
||||||
{"role": "assistant", "content": "ok"},
|
|
||||||
{"role": "user", "content": "second"},
|
|
||||||
]
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(messages, "query")
|
|
||||||
self.assertNotIn("task", messages[0])
|
|
||||||
self.assertEqual(messages[2]["task"], "query")
|
|
||||||
|
|
||||||
# `developer` role is treated like `user` (matches encoder semantics).
|
|
||||||
messages = [{"role": "developer", "content": "dev"}]
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(messages, "authority")
|
|
||||||
self.assertEqual(messages[0]["task"], "authority")
|
|
||||||
|
|
||||||
# No user/developer present -> raises.
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(
|
|
||||||
[{"role": "system", "content": "s"}], "domain"
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_dsv4_content_parts_list_normalized(self):
|
|
||||||
"""OpenAI list-of-parts content flattens to text before reaching the encoder."""
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
|
||||||
from sglang.srt.parser.jinja_template_utils import (
|
|
||||||
process_content_for_template_format,
|
|
||||||
)
|
|
||||||
|
|
||||||
req = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content": [{"type": "text", "text": "say hi"}],
|
|
||||||
}
|
|
||||||
],
|
|
||||||
)
|
|
||||||
messages = [m.model_dump() for m in req.messages]
|
|
||||||
# Mirror the boundary normalization _process_messages does for any
|
|
||||||
# non-None chat_encoding_spec.
|
|
||||||
for i, msg in enumerate(messages):
|
|
||||||
if isinstance(msg.get("content"), list):
|
|
||||||
messages[i] = process_content_for_template_format(
|
|
||||||
msg, "string", [], [], [], []
|
|
||||||
)
|
|
||||||
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
|
|
||||||
self.assertIn("<|User|>say hi", out)
|
|
||||||
|
|
||||||
# Multiple text parts concat with single space; non-text parts dropped.
|
|
||||||
messages = [
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content": [
|
|
||||||
{"type": "text", "text": "describe"},
|
|
||||||
{"type": "image_url", "image_url": {"url": "x"}},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
]
|
|
||||||
for i, msg in enumerate(messages):
|
|
||||||
if isinstance(msg.get("content"), list):
|
|
||||||
messages[i] = process_content_for_template_format(
|
|
||||||
msg, "string", [], [], [], []
|
|
||||||
)
|
|
||||||
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
|
|
||||||
self.assertIn("<|User|>describe", out)
|
|
||||||
self.assertNotIn("image_url", out)
|
|
||||||
|
|
||||||
def test_dsv4_task_and_reminder_encode_end_to_end(self):
|
|
||||||
"""Task + latest_reminder plumb through to the dsv4 encoder correctly."""
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
|
||||||
|
|
||||||
# 1) task='domain' in chat mode -> `<|domain|>` appended, no Assistant
|
|
||||||
# prefix (this is a single-shot classification, not a chat turn).
|
|
||||||
req = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[{"role": "user", "content": "What is SGLang?"}],
|
|
||||||
task="domain",
|
|
||||||
)
|
|
||||||
messages = [m.model_dump() for m in req.messages]
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(messages, req.task)
|
|
||||||
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
|
|
||||||
self.assertIn("<|domain|>", out)
|
|
||||||
self.assertTrue(out.rstrip().endswith("<|domain|>"))
|
|
||||||
self.assertNotIn("<|Assistant|>", out)
|
|
||||||
|
|
||||||
# 2) task='action' in thinking mode -> Assistant + <think> + <|action|>
|
|
||||||
# (action is the one task that still runs a reasoning pass).
|
|
||||||
req = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[{"role": "user", "content": "Hi"}],
|
|
||||||
task="action",
|
|
||||||
)
|
|
||||||
messages = [m.model_dump() for m in req.messages]
|
|
||||||
encoding_dsv4.attach_task_to_last_user_message(messages, req.task)
|
|
||||||
out = encoding_dsv4.encode_messages(messages, thinking_mode="thinking")
|
|
||||||
self.assertIn("<|Assistant|>", out)
|
|
||||||
self.assertIn("<think>", out)
|
|
||||||
self.assertTrue(out.rstrip().endswith("<|action|>"))
|
|
||||||
|
|
||||||
# 3) latest_reminder preceding user -> reminder renders before user,
|
|
||||||
# Assistant prefix still comes after user.
|
|
||||||
req = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[
|
|
||||||
{"role": "latest_reminder", "content": "Be terse."},
|
|
||||||
{"role": "user", "content": "Hello"},
|
|
||||||
],
|
|
||||||
)
|
|
||||||
messages = [m.model_dump() for m in req.messages]
|
|
||||||
out = encoding_dsv4.encode_messages(messages, thinking_mode="chat")
|
|
||||||
self.assertIn("<|latest_reminder|>Be terse.", out)
|
|
||||||
self.assertIn("<|User|>Hello", out)
|
|
||||||
self.assertLess(
|
|
||||||
out.index("<|latest_reminder|>"),
|
|
||||||
out.index("<|User|>"),
|
|
||||||
)
|
|
||||||
self.assertIn("<|Assistant|>", out)
|
|
||||||
|
|
||||||
def test_dsv4_reasoning_effort_profiles(self):
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv4
|
|
||||||
|
|
||||||
messages = [
|
|
||||||
{"role": "system", "content": ""},
|
|
||||||
{"role": "user", "content": "Solve this."},
|
|
||||||
]
|
|
||||||
absolute_maximum = (
|
|
||||||
"Reasoning Effort: Absolute maximum with no shortcuts permitted.\n"
|
|
||||||
"You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n"
|
|
||||||
"Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"
|
|
||||||
)
|
|
||||||
beyond_maximum = (
|
|
||||||
"Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n"
|
|
||||||
"You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n"
|
|
||||||
"Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n"
|
|
||||||
)
|
|
||||||
|
|
||||||
def encode(profile, effort):
|
|
||||||
return encoding_dsv4.encode_messages(
|
|
||||||
messages,
|
|
||||||
thinking_mode="thinking",
|
|
||||||
reasoning_effort=effort,
|
|
||||||
reasoning_effort_profile=profile,
|
|
||||||
)
|
|
||||||
|
|
||||||
preview_high = encode("preview", "high")
|
|
||||||
preview_max = encode("preview", "max")
|
|
||||||
official_low = encode("official", "low")
|
|
||||||
official_high = encode("official", "high")
|
|
||||||
official_max = encode("official", "max")
|
|
||||||
|
|
||||||
self.assertNotIn("Reasoning Effort:", preview_high)
|
|
||||||
self.assertTrue(
|
|
||||||
preview_max.startswith(encoding_dsv4.bos_token + absolute_maximum)
|
|
||||||
)
|
|
||||||
self.assertNotIn("Reasoning Effort:", official_low)
|
|
||||||
self.assertEqual(
|
|
||||||
official_high,
|
|
||||||
encoding_dsv4.bos_token
|
|
||||||
+ absolute_maximum
|
|
||||||
+ official_low.removeprefix(encoding_dsv4.bos_token),
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
official_max,
|
|
||||||
encoding_dsv4.bos_token
|
|
||||||
+ beyond_maximum
|
|
||||||
+ official_low.removeprefix(encoding_dsv4.bos_token),
|
|
||||||
)
|
|
||||||
self.assertEqual(encode("preview", None), preview_high)
|
|
||||||
self.assertEqual(encode("official", None), official_low)
|
|
||||||
self.assertEqual(len({official_low, official_high, official_max}), 3)
|
|
||||||
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
encode("preview", "low")
|
|
||||||
|
|
||||||
def test_dsv4_reasoning_effort_profile_resolution(self):
|
|
||||||
resolve = resolve_dsv4_reasoning_effort_profile
|
|
||||||
preview_model_path = _create_dsv4_checkpoint(self, _DSV4_PREVIEW_ENCODER)
|
|
||||||
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
|
|
||||||
inconclusive_model_path = _create_dsv4_checkpoint(
|
|
||||||
self, 'UNRELATED_METADATA = "value"\n'
|
|
||||||
)
|
|
||||||
self.assertEqual(resolve(model_path=preview_model_path), "preview")
|
|
||||||
self.assertEqual(resolve(model_path=official_model_path), "official")
|
|
||||||
self.assertEqual(resolve(model_path=inconclusive_model_path), "preview")
|
|
||||||
|
|
||||||
self.assertEqual(
|
|
||||||
resolve(model_path="renamed/model", override="official"), "official"
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
resolve(model_path="renamed/model", override="preview"), "preview"
|
|
||||||
)
|
|
||||||
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
|
|
||||||
resolve(model_path="renamed/model", override="auto")
|
|
||||||
|
|
||||||
def test_dsv4_reasoning_effort_profile_from_checkpoint(self):
|
|
||||||
from sglang.srt.parser.template_manager import TemplateManager
|
|
||||||
|
|
||||||
official_model_path = _create_dsv4_checkpoint(self, _DSV4_OFFICIAL_ENCODER)
|
|
||||||
tm = _MockTokenizerManager()
|
|
||||||
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
|
||||||
tm.model_config.hf_config.to_dict.return_value = {}
|
|
||||||
tm.model_config.hf_config.dspark_block_size = 5
|
|
||||||
tm.model_config.hf_config.dspark_markov_rank = 256
|
|
||||||
tm.model_path = official_model_path
|
|
||||||
tm.server_args.model_path = tm.model_path
|
|
||||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
|
||||||
|
|
||||||
request = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[{"role": "user", "content": "Hello"}],
|
|
||||||
reasoning_effort="max",
|
|
||||||
)
|
|
||||||
serving_chat._process_messages(request, is_multimodal=False)
|
|
||||||
prompt = tm.tokenizer.encode.call_args.args[0]
|
|
||||||
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
|
|
||||||
|
|
||||||
def test_dsv4_reasoning_effort_profile_override_from_model_config(self):
|
|
||||||
from sglang.srt.parser.template_manager import TemplateManager
|
|
||||||
|
|
||||||
tm = _MockTokenizerManager()
|
|
||||||
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
|
||||||
tm.model_config.hf_config.to_dict.return_value = {
|
|
||||||
"dsv4_reasoning_effort_profile": "official"
|
|
||||||
}
|
|
||||||
serving_chat = OpenAIServingChat(tm, TemplateManager())
|
|
||||||
request = ChatCompletionRequest(
|
|
||||||
model="x",
|
|
||||||
messages=[{"role": "user", "content": "Hello"}],
|
|
||||||
reasoning_effort="max",
|
|
||||||
)
|
|
||||||
|
|
||||||
serving_chat._process_messages(request, is_multimodal=False)
|
|
||||||
|
|
||||||
prompt = tm.tokenizer.encode.call_args.args[0]
|
|
||||||
self.assertIn("Reasoning Effort: Beyond maximum", prompt)
|
|
||||||
|
|
||||||
def test_dsv4_invalid_profile_override_fails_at_construction(self):
|
|
||||||
from sglang.srt.parser.template_manager import TemplateManager
|
|
||||||
|
|
||||||
tm = _MockTokenizerManager()
|
|
||||||
tm.model_config.hf_config.architectures = ["DeepseekV4ForCausalLM"]
|
|
||||||
tm.model_config.hf_config.to_dict.return_value = {
|
|
||||||
"dsv4_reasoning_effort_profile": "invalid"
|
|
||||||
}
|
|
||||||
with self.assertRaisesRegex(ValueError, "dsv4_reasoning_effort_profile"):
|
|
||||||
OpenAIServingChat(tm, TemplateManager())
|
|
||||||
|
|
||||||
def test_dsv41_rejects_image_placeholder_in_text(self):
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv41
|
|
||||||
|
|
||||||
for role in ("user", "tool"):
|
|
||||||
with self.subTest(role=role):
|
|
||||||
with self.assertRaises(ValueError):
|
|
||||||
encoding_dsv41.encode_messages(
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"role": role,
|
|
||||||
"content": f"hi {encoding_dsv41.IMAGE_PLACEHOLDER}",
|
|
||||||
}
|
|
||||||
],
|
|
||||||
thinking_mode="chat",
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_dsv41_tool_result_images_follow_prompt_order(self):
|
|
||||||
"""Out-of-order tool results are sorted by call order; images must follow."""
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv41
|
|
||||||
|
|
||||||
image = encoding_dsv41.IMAGE_PLACEHOLDER
|
|
||||||
history = [
|
|
||||||
{
|
|
||||||
"role": "user",
|
|
||||||
"content": [
|
|
||||||
{"type": "image_url", "image_url": {"url": "history.png"}},
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"role": "assistant",
|
|
||||||
"tool_calls": [
|
|
||||||
{
|
|
||||||
"id": name,
|
|
||||||
"type": "function",
|
|
||||||
"function": {"name": name, "arguments": "{}"},
|
|
||||||
}
|
|
||||||
for name in ("a", "b")
|
|
||||||
],
|
|
||||||
},
|
|
||||||
]
|
|
||||||
results = [
|
|
||||||
{
|
|
||||||
"type": "tool_result",
|
|
||||||
"tool_use_id": name,
|
|
||||||
"content": [
|
|
||||||
{"type": "text", "text": f"result {name}"},
|
|
||||||
{"type": "image_url", "image_url": {"url": f"{name}.png"}},
|
|
||||||
],
|
|
||||||
}
|
|
||||||
for name in ("b", "a")
|
|
||||||
]
|
|
||||||
|
|
||||||
for input_format in (
|
|
||||||
"content_blocks",
|
|
||||||
"tool_messages",
|
|
||||||
"tool_content_blocks",
|
|
||||||
"context",
|
|
||||||
):
|
|
||||||
with self.subTest(input_format=input_format):
|
|
||||||
if input_format in ("tool_messages", "tool_content_blocks"):
|
|
||||||
content_field = (
|
|
||||||
"content_blocks"
|
|
||||||
if input_format == "tool_content_blocks"
|
|
||||||
else "content"
|
|
||||||
)
|
|
||||||
messages = history + [
|
|
||||||
{
|
|
||||||
"role": "tool",
|
|
||||||
"tool_call_id": block["tool_use_id"],
|
|
||||||
content_field: block["content"],
|
|
||||||
}
|
|
||||||
for block in results
|
|
||||||
]
|
|
||||||
context = []
|
|
||||||
elif input_format == "context":
|
|
||||||
messages = [{"role": "user", "content_blocks": results}]
|
|
||||||
context = history
|
|
||||||
else:
|
|
||||||
messages = history + [{"role": "user", "content_blocks": results}]
|
|
||||||
context = []
|
|
||||||
original = copy.deepcopy((messages, context))
|
|
||||||
|
|
||||||
prompt, media = encoding_dsv41.encode_messages(
|
|
||||||
messages,
|
|
||||||
thinking_mode="chat",
|
|
||||||
context=context,
|
|
||||||
return_multi_modal_data=True,
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertTrue(
|
|
||||||
prompt.endswith(
|
|
||||||
f"<|User|><tool_result>result a\n\n{image}</tool_result>\n\n"
|
|
||||||
f"<tool_result>result b\n\n{image}</tool_result>"
|
|
||||||
"<|Assistant|></think>"
|
|
||||||
),
|
|
||||||
prompt,
|
|
||||||
)
|
|
||||||
expected_urls = (
|
|
||||||
["a.png", "b.png"] if context else ["history.png", "a.png", "b.png"]
|
|
||||||
)
|
|
||||||
self.assertEqual(
|
|
||||||
media,
|
|
||||||
{"images": [{"url": url} for url in expected_urls]},
|
|
||||||
)
|
|
||||||
self.assertEqual((messages, context), original)
|
|
||||||
|
|
||||||
def test_dsv41_drop_thinking_without_tools(self):
|
|
||||||
"""Tool results do not start a new turn, so current-turn reasoning is kept."""
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv41
|
|
||||||
|
|
||||||
messages = [
|
|
||||||
{"role": "user", "content": "q1"},
|
|
||||||
{"role": "assistant", "content": "a1", "reasoning_content": "r1"},
|
|
||||||
{"role": "user", "content": "q2"},
|
|
||||||
{
|
|
||||||
"role": "assistant",
|
|
||||||
"reasoning_content": "r2",
|
|
||||||
"tool_calls": [
|
|
||||||
{
|
|
||||||
"id": "c1",
|
|
||||||
"type": "function",
|
|
||||||
"function": {"name": "lookup", "arguments": "{}"},
|
|
||||||
}
|
|
||||||
],
|
|
||||||
},
|
|
||||||
{"role": "tool", "tool_call_id": "c1", "content": "value"},
|
|
||||||
]
|
|
||||||
prompt = encoding_dsv41.encode_messages(
|
|
||||||
messages, thinking_mode="thinking", drop_thinking=True
|
|
||||||
)
|
|
||||||
self.assertIn(
|
|
||||||
"<|User|>q1<|Assistant|></think>a1<|end▁of▁sentence|>", prompt
|
|
||||||
)
|
|
||||||
self.assertNotIn("r1", prompt)
|
|
||||||
self.assertIn("<think>r2</think>", prompt)
|
|
||||||
self.assertTrue(
|
|
||||||
prompt.endswith("<tool_result>value</tool_result><|Assistant|><think>")
|
|
||||||
)
|
|
||||||
|
|
||||||
next_turn = encoding_dsv41.encode_messages(
|
|
||||||
messages + [{"role": "user", "content": "q3"}],
|
|
||||||
thinking_mode="thinking",
|
|
||||||
)
|
|
||||||
self.assertNotIn("r2", next_turn)
|
|
||||||
self.assertTrue(next_turn.endswith("q3<|Assistant|><think>"))
|
|
||||||
|
|
||||||
def test_streaming_abort_yields_error(self):
|
def test_streaming_abort_yields_error(self):
|
||||||
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
|
"""Test that an abort finish reason during streaming correctly yields an error and stops."""
|
||||||
err_msg = "Aborted by scheduler"
|
err_msg = "Aborted by scheduler"
|
||||||
@@ -5499,7 +5070,7 @@ class ServingChatTestCase(CustomTestCase):
|
|||||||
# ------------- hook method tests -------------
|
# ------------- hook method tests -------------
|
||||||
def test_encode_messages_returns_none_by_default(self):
|
def test_encode_messages_returns_none_by_default(self):
|
||||||
"""Default _encode_messages returns None (use standard encoding)."""
|
"""Default _encode_messages returns None (use standard encoding)."""
|
||||||
result = self.chat._encode_messages([], Mock(), False)
|
result = self.chat._encode_messages([], Mock(), False, image_data=[])
|
||||||
self.assertIsNone(result)
|
self.assertIsNone(result)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,298 @@
|
|||||||
|
import unittest
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from sglang.test.ci.ci_register import register_cpu_ci
|
||||||
|
from sglang.test.test_utils import CustomTestCase, is_in_ci
|
||||||
|
|
||||||
|
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||||
|
|
||||||
|
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash-1017"
|
||||||
|
REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170"
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenizer_file():
|
||||||
|
"""The V4.1 tokenizer; only outside CI does an unreachable Hub skip the test."""
|
||||||
|
from huggingface_hub import hf_hub_download
|
||||||
|
|
||||||
|
try:
|
||||||
|
return hf_hub_download(MODEL, "tokenizer.json", revision=REVISION)
|
||||||
|
except Exception as error: # noqa: BLE001
|
||||||
|
if is_in_ci():
|
||||||
|
raise
|
||||||
|
raise unittest.SkipTest(f"the tokenizer is not available: {error}")
|
||||||
|
|
||||||
|
|
||||||
|
def _serving(test, tokenizer, *, is_multimodal=False, **server_args):
|
||||||
|
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
|
||||||
|
from sglang.srt.runtime_context import publish, reset_context
|
||||||
|
from sglang.srt.server_args import ServerArgs
|
||||||
|
|
||||||
|
reset_context()
|
||||||
|
test.addCleanup(reset_context)
|
||||||
|
args = ServerArgs(
|
||||||
|
model_path="dummy",
|
||||||
|
reasoning_parser="deepseek-v41",
|
||||||
|
tool_call_parser="deepseekv41",
|
||||||
|
**server_args,
|
||||||
|
)
|
||||||
|
publish(args, role="tokenizer")
|
||||||
|
manager = SimpleNamespace(
|
||||||
|
model_path="dummy",
|
||||||
|
served_model_name="dummy",
|
||||||
|
server_args=args,
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
model_config=SimpleNamespace(
|
||||||
|
is_multimodal=is_multimodal,
|
||||||
|
context_len=131072,
|
||||||
|
hf_config=SimpleNamespace(
|
||||||
|
model_type="deepseek_v41",
|
||||||
|
architectures=["DeepseekV41ForCausalLM"],
|
||||||
|
to_dict=lambda: {},
|
||||||
|
),
|
||||||
|
get_default_sampling_params=lambda: {},
|
||||||
|
),
|
||||||
|
config_value=lambda name: getattr(args, name),
|
||||||
|
)
|
||||||
|
template_manager = SimpleNamespace(
|
||||||
|
chat_template_name=None,
|
||||||
|
jinja_template_content_format=None,
|
||||||
|
reasoning_config=None,
|
||||||
|
force_reasoning=False,
|
||||||
|
python_chat_template=None,
|
||||||
|
jinja_template_may_reorder_tool_results=False,
|
||||||
|
)
|
||||||
|
return OpenAIServingChat(manager, template_manager)
|
||||||
|
|
||||||
|
|
||||||
|
class TestAdapterRegistry(CustomTestCase):
|
||||||
|
def _context(self, spec, tokenizer=None, default_kwargs=None):
|
||||||
|
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
|
||||||
|
|
||||||
|
return registry.ChatEncodingAdapterContext(
|
||||||
|
chat_encoding_spec=spec,
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
reasoning_parser=None,
|
||||||
|
tool_call_parser=None,
|
||||||
|
chat_hooks=registry.ChatOutputHooks(
|
||||||
|
tool_call_parsing_active=lambda request: False,
|
||||||
|
history_tool_calls_count=lambda request: 0,
|
||||||
|
tool_call_id=lambda item, index: "",
|
||||||
|
),
|
||||||
|
hf_config=SimpleNamespace(
|
||||||
|
model_type="deepseek_v41", architectures=["DeepseekV41ForCausalLM"]
|
||||||
|
),
|
||||||
|
model_path="",
|
||||||
|
revision=None,
|
||||||
|
default_chat_template_kwargs=default_kwargs,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _fast_tokenizer(self):
|
||||||
|
from tokenizers import Tokenizer
|
||||||
|
|
||||||
|
return SimpleNamespace(backend_tokenizer=Tokenizer.from_file(_tokenizer_file()))
|
||||||
|
|
||||||
|
def test_bad_default_effort_fails_startup(self):
|
||||||
|
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
|
||||||
|
|
||||||
|
tokenizer = self._fast_tokenizer()
|
||||||
|
for spec, effort in (("dsv41", "extreme"), ("dsv4", 60)):
|
||||||
|
with self.subTest(spec=spec, effort=effort), self.assertRaises(ValueError):
|
||||||
|
registry.build_chat_encoding_adapter(
|
||||||
|
self._context(
|
||||||
|
spec,
|
||||||
|
tokenizer=tokenizer,
|
||||||
|
default_kwargs={"reasoning_effort": effort},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_no_tokenizer_builds_no_adapter(self):
|
||||||
|
"""--skip-tokenizer-init servers start, with no chat prompts to encode."""
|
||||||
|
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
|
||||||
|
|
||||||
|
for spec in ("dsv4", "dsv41"):
|
||||||
|
with self.subTest(spec=spec):
|
||||||
|
self.assertIsNone(
|
||||||
|
registry.build_chat_encoding_adapter(self._context(spec))
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_tool_dumps_render_as_the_client_sent_them(self):
|
||||||
|
from sglang.srt.entrypoints.openai import chat_encoding_adapters as registry
|
||||||
|
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
adapter = registry.build_chat_encoding_adapter(
|
||||||
|
self._context("dsv41", tokenizer=self._fast_tokenizer())
|
||||||
|
)
|
||||||
|
# The handler's tool dumps carry None for unset fields.
|
||||||
|
tool = {
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "f",
|
||||||
|
"description": "d",
|
||||||
|
"parameters": {"type": "object", "properties": {}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
dumped = {
|
||||||
|
"type": "function",
|
||||||
|
"function": {**tool["function"], "strict": None},
|
||||||
|
"defer_loading": None,
|
||||||
|
}
|
||||||
|
request = ChatCompletionRequest(messages=[{"role": "user", "content": "hi"}])
|
||||||
|
self.assertEqual(
|
||||||
|
adapter.encode_messages(
|
||||||
|
messages=[],
|
||||||
|
request=request,
|
||||||
|
tools=[dumped],
|
||||||
|
thinking=False,
|
||||||
|
image_data=[],
|
||||||
|
),
|
||||||
|
adapter.encode_messages(
|
||||||
|
messages=[],
|
||||||
|
request=request,
|
||||||
|
tools=[tool],
|
||||||
|
thinking=False,
|
||||||
|
image_data=[],
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestHandlerRequestSemantics(CustomTestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
super().setUpClass()
|
||||||
|
from transformers import PreTrainedTokenizerFast
|
||||||
|
|
||||||
|
cls.tokenizer = PreTrainedTokenizerFast(tokenizer_file=_tokenizer_file())
|
||||||
|
|
||||||
|
def _token(self, token):
|
||||||
|
return self.tokenizer.convert_tokens_to_ids(token)
|
||||||
|
|
||||||
|
def _prompt(self, chat, **fields):
|
||||||
|
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
messages=[{"role": "user", "content": "hi"}], **fields
|
||||||
|
)
|
||||||
|
return chat._process_messages(request, False)
|
||||||
|
|
||||||
|
def test_thinking_defaults_off(self):
|
||||||
|
chat = _serving(self, self.tokenizer)
|
||||||
|
default = self._prompt(chat)
|
||||||
|
self.assertEqual(default.prompt_ids[-1], self._token("</think>"))
|
||||||
|
self.assertFalse(default.require_reasoning)
|
||||||
|
thinking = self._prompt(chat, chat_template_kwargs={"thinking": True})
|
||||||
|
self.assertEqual(thinking.prompt_ids[-1], self._token("<think>"))
|
||||||
|
self.assertTrue(thinking.require_reasoning)
|
||||||
|
|
||||||
|
def test_request_effort_wins_over_the_server_default(self):
|
||||||
|
thinking = {"chat_template_kwargs": {"thinking": True}}
|
||||||
|
plain = _serving(self, self.tokenizer)
|
||||||
|
low = self._prompt(plain, reasoning_effort="low", **thinking).prompt_ids
|
||||||
|
self.assertNotEqual(
|
||||||
|
low, self._prompt(plain, reasoning_effort="max", **thinking).prompt_ids
|
||||||
|
)
|
||||||
|
chat = _serving(
|
||||||
|
self,
|
||||||
|
self.tokenizer,
|
||||||
|
default_chat_template_kwargs={"reasoning_effort": "max"},
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self._prompt(chat, reasoning_effort="low", **thinking).prompt_ids, low
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_trailing_assistant_follows_sglang_rules(self):
|
||||||
|
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
chat = _serving(self, self.tokenizer)
|
||||||
|
|
||||||
|
def prompt(messages, **fields):
|
||||||
|
request = ChatCompletionRequest(messages=messages, **fields)
|
||||||
|
return chat._process_messages(request, False).prompt_ids
|
||||||
|
|
||||||
|
answer = {"role": "assistant", "content": "Hel"}
|
||||||
|
# Without continue_final_message the trailing turn becomes a user turn.
|
||||||
|
self.assertEqual(
|
||||||
|
prompt([{"role": "user", "content": "hi"}, answer]),
|
||||||
|
prompt(
|
||||||
|
[{"role": "user", "content": "hi"}, {"role": "user", "content": "Hel"}]
|
||||||
|
),
|
||||||
|
)
|
||||||
|
# A task renders its own tail; the continued prefix follows it.
|
||||||
|
self.assertEqual(
|
||||||
|
prompt(
|
||||||
|
[{"role": "user", "content": "hi"}, answer],
|
||||||
|
task="title",
|
||||||
|
continue_final_message=True,
|
||||||
|
),
|
||||||
|
prompt([{"role": "user", "content": "hi"}], task="title")
|
||||||
|
+ self.tokenizer.encode("Hel", add_special_tokens=False),
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_tool_result_images_follow_prompt_order(self):
|
||||||
|
"""Out-of-order tool results render in call order; their images must follow."""
|
||||||
|
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
chat = _serving(self, self.tokenizer, is_multimodal=True)
|
||||||
|
names = ("a", "b")
|
||||||
|
urls = {name: f"https://example.com/{name}.png" for name in names}
|
||||||
|
calls = [
|
||||||
|
{
|
||||||
|
"id": name,
|
||||||
|
"type": "function",
|
||||||
|
"function": {"name": name, "arguments": "{}"},
|
||||||
|
}
|
||||||
|
for name in names
|
||||||
|
]
|
||||||
|
results = [
|
||||||
|
{
|
||||||
|
"role": "tool",
|
||||||
|
"tool_call_id": name,
|
||||||
|
"content": [
|
||||||
|
{"type": "text", "text": f"result {name}"},
|
||||||
|
{"type": "image_url", "image_url": {"url": urls[name]}},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
for name in reversed(names)
|
||||||
|
]
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
messages=[
|
||||||
|
{"role": "user", "content": "both?"},
|
||||||
|
{"role": "assistant", "content": "", "tool_calls": calls},
|
||||||
|
*results,
|
||||||
|
],
|
||||||
|
tools=[
|
||||||
|
{
|
||||||
|
"type": "function",
|
||||||
|
"function": {"name": name, "parameters": {"type": "object"}},
|
||||||
|
}
|
||||||
|
for name in names
|
||||||
|
],
|
||||||
|
)
|
||||||
|
processed = chat._process_messages(request, True)
|
||||||
|
prompt = self.tokenizer.decode(processed.prompt_ids)
|
||||||
|
self.assertLess(prompt.index("result a"), prompt.index("result b"))
|
||||||
|
self.assertEqual(
|
||||||
|
[image.url for image in processed.image_data], [urls["a"], urls["b"]]
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_request_reaches_the_engine_with_sglang_semantics(self):
|
||||||
|
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||||
|
|
||||||
|
chat = _serving(self, self.tokenizer)
|
||||||
|
request = ChatCompletionRequest(
|
||||||
|
messages=[{"role": "user", "content": "hi"}],
|
||||||
|
temperature=0.0,
|
||||||
|
chat_template_kwargs={"thinking": True, "reasoning_effort": 30},
|
||||||
|
)
|
||||||
|
adapted, _ = chat._convert_to_internal_request(request)
|
||||||
|
self.assertEqual(adapted.sampling_params["temperature"], 0.0)
|
||||||
|
self.assertIsNone(adapted.sampling_params["max_new_tokens"])
|
||||||
|
self.assertIn(
|
||||||
|
"Reasoning Effort: 30 (range 1-100",
|
||||||
|
self.tokenizer.decode(adapted.input_ids),
|
||||||
|
)
|
||||||
|
titled = self._prompt(chat, task="title").prompt_ids
|
||||||
|
self.assertEqual(titled[-1], self._token("<|title|>"))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -111,6 +111,7 @@ class InklingPromptTest(unittest.TestCase):
|
|||||||
[message.model_dump() for message in request.messages],
|
[message.model_dump() for message in request.messages],
|
||||||
request,
|
request,
|
||||||
thinking_mode=None,
|
thinking_mode=None,
|
||||||
|
image_data=[],
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_prompt_tail(self):
|
def test_prompt_tail(self):
|
||||||
@@ -626,6 +627,8 @@ class InklingTokenOutputTest(CustomTestCase):
|
|||||||
messages=[message.model_dump() for message in request.messages],
|
messages=[message.model_dump() for message in request.messages],
|
||||||
request=request,
|
request=request,
|
||||||
tools=None,
|
tools=None,
|
||||||
|
thinking=False,
|
||||||
|
image_data=[],
|
||||||
)
|
)
|
||||||
key = reasoner.get_request_grammar_key(
|
key = reasoner.get_request_grammar_key(
|
||||||
("regex", "yes"), input_ids=prompt_ids, thinking_budget=None
|
("regex", "yes"), input_ids=prompt_ids, thinking_budget=None
|
||||||
|
|||||||
@@ -1448,7 +1448,7 @@ class ChatEncodingAdapterDispatchTestCase(CustomTestCase):
|
|||||||
)
|
)
|
||||||
|
|
||||||
with patch.dict(sys.modules, {"tml_renderers": None}):
|
with patch.dict(sys.modules, {"tml_renderers": None}):
|
||||||
for encoding in ("kimi_k3", "dsv4", "dsv41", "dsv32"):
|
for encoding in ("kimi_k3", "dsv32"):
|
||||||
with self.subTest(encoding=encoding):
|
with self.subTest(encoding=encoding):
|
||||||
self.assertIsNone(
|
self.assertIsNone(
|
||||||
chat_encoding_adapters.build_chat_encoding_adapter(
|
chat_encoding_adapters.build_chat_encoding_adapter(
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ import xgrammar as xgr
|
|||||||
from xgrammar.structural_tag import JSONSchemaFormat
|
from xgrammar.structural_tag import JSONSchemaFormat
|
||||||
from xgrammar.testing import _is_grammar_accept_string
|
from xgrammar.testing import _is_grammar_accept_string
|
||||||
|
|
||||||
from sglang.srt.entrypoints.openai import encoding_dsv41
|
|
||||||
from sglang.srt.entrypoints.openai.protocol import (
|
from sglang.srt.entrypoints.openai.protocol import (
|
||||||
Function,
|
Function,
|
||||||
Tool,
|
Tool,
|
||||||
@@ -17,6 +16,7 @@ from sglang.srt.entrypoints.openai.protocol import (
|
|||||||
)
|
)
|
||||||
from sglang.srt.function_call.deepseekv41_detector import DeepSeekV41Detector
|
from sglang.srt.function_call.deepseekv41_detector import DeepSeekV41Detector
|
||||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||||
|
from sglang.srt.parser.deepseek_recipe_renderer import load_deepseek_recipe
|
||||||
from sglang.srt.parser.reasoning_parser import ReasoningParser
|
from sglang.srt.parser.reasoning_parser import ReasoningParser
|
||||||
from sglang.test.ci.ci_register import register_cpu_ci
|
from sglang.test.ci.ci_register import register_cpu_ci
|
||||||
from sglang.test.test_utils import CustomTestCase
|
from sglang.test.test_utils import CustomTestCase
|
||||||
@@ -25,6 +25,7 @@ register_cpu_ci(est_time=2, suite="base-a-test-cpu")
|
|||||||
|
|
||||||
CHUNK_SIZES = [1, 2, 3, 5, 7, 11, 23, 1000]
|
CHUNK_SIZES = [1, 2, 3, 5, 7, 11, 23, 1000]
|
||||||
DSML = "|DSML|"
|
DSML = "|DSML|"
|
||||||
|
EOS = "<|end▁of▁sentence|>"
|
||||||
|
|
||||||
|
|
||||||
def _tools():
|
def _tools():
|
||||||
@@ -59,6 +60,34 @@ def _tools():
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _render_assistant(message, *, thinking):
|
||||||
|
"""The assistant turn as deepseek-recipe renders it, minus the generation
|
||||||
|
header (`<think>` / `</think>`) that ends the prompt before it."""
|
||||||
|
recipe = load_deepseek_recipe()
|
||||||
|
calls = [
|
||||||
|
{**call, "id": f"call_{i}"} for i, call in enumerate(message["tool_calls"])
|
||||||
|
]
|
||||||
|
results = [
|
||||||
|
{"role": "tool", "tool_call_id": call["id"], "content": "ok"} for call in calls
|
||||||
|
]
|
||||||
|
body = {
|
||||||
|
"model": "DeepSeek-V4.1-Flash",
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "question"},
|
||||||
|
{**message, "tool_calls": calls},
|
||||||
|
*results,
|
||||||
|
],
|
||||||
|
"thinking": {"type": "enabled" if thinking else "disabled"},
|
||||||
|
# Declared tools keep the turn's reasoning in the rendered history.
|
||||||
|
"tools": [tool.model_dump(exclude_none=True) for tool in _tools()],
|
||||||
|
}
|
||||||
|
converted = recipe.ChatCompletionRequest(body).convert(recipe.ConversionOptions())
|
||||||
|
prompt = recipe.DeepseekV41Encoding().render_conversation(converted.conversation)
|
||||||
|
turn = prompt.prompt.split("<|User|>question<|Assistant|>", 1)[1]
|
||||||
|
turn = turn[: turn.index(EOS) + len(EOS)]
|
||||||
|
return turn.removeprefix("<think>" if thinking else "</think>")
|
||||||
|
|
||||||
|
|
||||||
def _assemble(calls):
|
def _assemble(calls):
|
||||||
"""Streamed ToolCallItems -> [(name, parsed arguments)] per tool_index."""
|
"""Streamed ToolCallItems -> [(name, parsed arguments)] per tool_index."""
|
||||||
by_index = {}
|
by_index = {}
|
||||||
@@ -81,26 +110,22 @@ class TestDeepSeekV41RoundTrip(CustomTestCase):
|
|||||||
|
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
self.tools = _tools()
|
self.tools = _tools()
|
||||||
self.completion = encoding_dsv41.render_message(
|
self.completion = _render_assistant(
|
||||||
1,
|
{
|
||||||
[
|
"role": "assistant",
|
||||||
{"role": "user", "content": "question"},
|
"reasoning_content": "reason",
|
||||||
{
|
"content": "summary",
|
||||||
"role": "assistant",
|
"tool_calls": [
|
||||||
"reasoning_content": "reason",
|
{
|
||||||
"content": "summary",
|
"type": "function",
|
||||||
"tool_calls": [
|
"function": {
|
||||||
{
|
"name": "lookup",
|
||||||
"type": "function",
|
"arguments": json.dumps(self.ARGUMENTS),
|
||||||
"function": {
|
},
|
||||||
"name": "lookup",
|
}
|
||||||
"arguments": json.dumps(self.ARGUMENTS),
|
],
|
||||||
},
|
},
|
||||||
}
|
thinking=True,
|
||||||
],
|
|
||||||
},
|
|
||||||
],
|
|
||||||
thinking_mode="thinking",
|
|
||||||
)
|
)
|
||||||
self.completion = "<think>" + self.completion
|
self.completion = "<think>" + self.completion
|
||||||
self.expected = [("lookup", self.ARGUMENTS)]
|
self.expected = [("lookup", self.ARGUMENTS)]
|
||||||
@@ -221,29 +246,21 @@ class TestDeepSeekV41ParameterGrammar(CustomTestCase):
|
|||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _render(arguments, *, thinking=False, count=1, name="get_weather"):
|
def _render(arguments, *, thinking=False, count=1, name="get_weather"):
|
||||||
return encoding_dsv41.render_message(
|
return _render_assistant(
|
||||||
1,
|
{
|
||||||
[
|
"role": "assistant",
|
||||||
{"role": "user", "content": "question"},
|
"content": "",
|
||||||
{
|
"reasoning_content": "reason",
|
||||||
"role": "assistant",
|
"tool_calls": [
|
||||||
"content": "",
|
{
|
||||||
"reasoning_content": "reason",
|
"type": "function",
|
||||||
"wo_eos": True,
|
"function": {"name": name, "arguments": json.dumps(arguments)},
|
||||||
"tool_calls": [
|
}
|
||||||
{
|
]
|
||||||
"type": "function",
|
* count,
|
||||||
"function": {
|
},
|
||||||
"name": name,
|
thinking=thinking,
|
||||||
"arguments": json.dumps(arguments),
|
).removesuffix(EOS)
|
||||||
},
|
|
||||||
}
|
|
||||||
]
|
|
||||||
* count,
|
|
||||||
},
|
|
||||||
],
|
|
||||||
thinking_mode="thinking" if thinking else "chat",
|
|
||||||
)
|
|
||||||
|
|
||||||
def _grammar(self, choice="required", thinking=False, parallel=True):
|
def _grammar(self, choice="required", thinking=False, parallel=True):
|
||||||
constraint = FunctionCallParser(
|
constraint = FunctionCallParser(
|
||||||
|
|||||||
@@ -0,0 +1,206 @@
|
|||||||
|
import unittest
|
||||||
|
|
||||||
|
from sglang.srt.parser import deepseek_recipe_renderer as renderer
|
||||||
|
from sglang.test.ci.ci_register import register_cpu_ci
|
||||||
|
from sglang.test.test_utils import CustomTestCase, is_in_ci
|
||||||
|
|
||||||
|
register_cpu_ci(est_time=10, suite="base-a-test-cpu")
|
||||||
|
|
||||||
|
MODEL = "deepseek-ai/DeepSeek-V4.1-Flash-1017"
|
||||||
|
REVISION = "1bae0e1088993b9b82de08f59de5be5a34e42170"
|
||||||
|
|
||||||
|
|
||||||
|
def _tokenizer_file():
|
||||||
|
"""The V4.1 tokenizer; only outside CI does an unreachable Hub skip the test."""
|
||||||
|
from huggingface_hub import hf_hub_download
|
||||||
|
|
||||||
|
try:
|
||||||
|
return hf_hub_download(MODEL, "tokenizer.json", revision=REVISION)
|
||||||
|
except Exception as error: # noqa: BLE001
|
||||||
|
if is_in_ci():
|
||||||
|
raise
|
||||||
|
raise unittest.SkipTest(f"the tokenizer is not available: {error}")
|
||||||
|
|
||||||
|
|
||||||
|
class TestPreNormalize(CustomTestCase):
|
||||||
|
def test_developer_becomes_system(self):
|
||||||
|
body = renderer.pre_normalize(
|
||||||
|
{
|
||||||
|
"messages": [
|
||||||
|
{"role": "developer", "content": "Be terse."},
|
||||||
|
{"role": "user", "content": "x"},
|
||||||
|
],
|
||||||
|
"thinking": False,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(body["messages"][0]["role"], "system")
|
||||||
|
|
||||||
|
def test_tool_choice_renders_as_auto(self):
|
||||||
|
tools = [
|
||||||
|
{"type": "function", "function": {"name": "a"}},
|
||||||
|
{"type": "function", "function": {"name": "b"}},
|
||||||
|
]
|
||||||
|
named = renderer.pre_normalize(
|
||||||
|
{
|
||||||
|
"messages": [{"role": "user", "content": "x"}],
|
||||||
|
"thinking": False,
|
||||||
|
"tools": tools,
|
||||||
|
"tool_choice": {"type": "function", "function": {"name": "b"}},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(named["tool_choice"], "auto")
|
||||||
|
self.assertEqual(len(named["tools"]), 2)
|
||||||
|
required = renderer.pre_normalize(
|
||||||
|
{
|
||||||
|
"messages": [{"role": "user", "content": "x"}],
|
||||||
|
"thinking": False,
|
||||||
|
"tools": tools,
|
||||||
|
"tool_choice": "required",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertEqual(required["tool_choice"], "auto")
|
||||||
|
for tools_field, choice in ((tools, "none"), ([], "auto")):
|
||||||
|
off = renderer.pre_normalize(
|
||||||
|
{
|
||||||
|
"messages": [{"role": "user", "content": "x"}],
|
||||||
|
"thinking": False,
|
||||||
|
"tools": tools_field,
|
||||||
|
"tool_choice": choice,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self.assertNotIn("tools", off)
|
||||||
|
self.assertNotIn("tool_choice", off)
|
||||||
|
|
||||||
|
def test_history_translation(self):
|
||||||
|
body = renderer.pre_normalize(
|
||||||
|
{
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "weather?"},
|
||||||
|
{
|
||||||
|
"role": "assistant",
|
||||||
|
"content": None,
|
||||||
|
"tool_calls": [
|
||||||
|
{
|
||||||
|
"id": "call_0",
|
||||||
|
"type": "function",
|
||||||
|
"function": {
|
||||||
|
"name": "get_weather",
|
||||||
|
"arguments": {"location": "Paris", "days": 1.5},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{"role": "tool", "tool_call_id": "call_0", "content": "sunny"},
|
||||||
|
{"role": "assistant", "content": "It is"},
|
||||||
|
],
|
||||||
|
"thinking": False,
|
||||||
|
"continue_final_message": True,
|
||||||
|
"response_format": {"type": "structural_tag", "structures": []},
|
||||||
|
}
|
||||||
|
)
|
||||||
|
call = body["messages"][1]["tool_calls"][0]
|
||||||
|
self.assertEqual(
|
||||||
|
call["function"]["arguments"], '{"location":"Paris","days":1.5}'
|
||||||
|
)
|
||||||
|
self.assertTrue(body["messages"][3]["prefix"])
|
||||||
|
self.assertNotIn("continue_final_message", body)
|
||||||
|
self.assertNotIn("response_format", body)
|
||||||
|
|
||||||
|
|
||||||
|
class TestEffortBudget(CustomTestCase):
|
||||||
|
def test_integer_and_float_budgets(self):
|
||||||
|
self.assertEqual(renderer.parse_effort_budget(30), 30)
|
||||||
|
self.assertEqual(renderer.parse_effort_budget(0.5), 50)
|
||||||
|
self.assertEqual(renderer.parse_effort_budget(0.0), 1)
|
||||||
|
for value in (0, 101, 1.0, True):
|
||||||
|
with self.subTest(value=value), self.assertRaises(ValueError):
|
||||||
|
renderer.parse_effort_budget(value)
|
||||||
|
|
||||||
|
|
||||||
|
class TestRecipeEncoder(CustomTestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
super().setUpClass()
|
||||||
|
from tokenizers import Tokenizer
|
||||||
|
|
||||||
|
cls.tokenizer = Tokenizer.from_file(_tokenizer_file())
|
||||||
|
cls.encoder = renderer.RecipeEncoder("dsv41", cls.tokenizer)
|
||||||
|
|
||||||
|
def _ids(self, *tokens):
|
||||||
|
return [self.tokenizer.token_to_id(token) for token in tokens]
|
||||||
|
|
||||||
|
def test_encodes_with_the_served_vocabulary(self):
|
||||||
|
self.assertEqual(
|
||||||
|
self.encoder.encode(
|
||||||
|
{"messages": [{"role": "user", "content": ""}], "thinking": False}
|
||||||
|
),
|
||||||
|
self._ids(
|
||||||
|
"<|begin▁of▁sentence|>", "<|User|>", "<|Assistant|>", "</think>"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
thinking = self.encoder.encode(
|
||||||
|
{"messages": [{"role": "user", "content": "hi"}], "thinking": True}
|
||||||
|
)
|
||||||
|
self.assertEqual(thinking[-2:], self._ids("<|Assistant|>", "<think>"))
|
||||||
|
|
||||||
|
def test_tasks_replace_or_follow_the_generation_header(self):
|
||||||
|
for thinking in (False, True):
|
||||||
|
body = {
|
||||||
|
"messages": [{"role": "user", "content": "hello"}],
|
||||||
|
"thinking": thinking,
|
||||||
|
}
|
||||||
|
plain = self.encoder.encode(body)
|
||||||
|
for task, token in renderer.TASK_TOKENS.items():
|
||||||
|
with self.subTest(thinking=thinking, task=task):
|
||||||
|
expected = (
|
||||||
|
plain + self._ids(token)
|
||||||
|
if task == "action"
|
||||||
|
else plain[:-2] + self._ids(token)
|
||||||
|
)
|
||||||
|
self.assertEqual(self.encoder.encode(body, task=task), expected)
|
||||||
|
continued = {
|
||||||
|
"messages": [
|
||||||
|
{"role": "user", "content": "count"},
|
||||||
|
{"role": "assistant", "content": "1, 2,"},
|
||||||
|
],
|
||||||
|
"thinking": False,
|
||||||
|
"continue_final_message": True,
|
||||||
|
}
|
||||||
|
with self.assertRaisesRegex(ValueError, "end with a user message"):
|
||||||
|
self.encoder.encode(continued, task="query")
|
||||||
|
|
||||||
|
def test_effort_budget_rewrites_the_effort_line(self):
|
||||||
|
body = {"messages": [{"role": "user", "content": "hi"}], "thinking": True}
|
||||||
|
high = self.tokenizer.decode(
|
||||||
|
self.encoder.encode({**body, "reasoning_effort": "high"}),
|
||||||
|
skip_special_tokens=False,
|
||||||
|
)
|
||||||
|
self.assertIn("Reasoning Effort: 75 (range 1-100", high)
|
||||||
|
for budget in (1, 30, 100):
|
||||||
|
with self.subTest(budget=budget):
|
||||||
|
expected = high.replace(
|
||||||
|
"Reasoning Effort: 75 ", f"Reasoning Effort: {budget} ", 1
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
self.encoder.encode(body, effort_budget=budget),
|
||||||
|
self.tokenizer.encode(expected, add_special_tokens=False).ids,
|
||||||
|
)
|
||||||
|
# Chat mode renders no effort line, so the budget changes nothing.
|
||||||
|
chat = {"messages": [{"role": "user", "content": "hi"}], "thinking": False}
|
||||||
|
self.assertEqual(
|
||||||
|
self.encoder.encode(chat, effort_budget=30), self.encoder.encode(chat)
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_api_validation_surfaces_as_value_error(self):
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.encoder.encode(
|
||||||
|
{
|
||||||
|
"messages": [{"role": "user", "content": "hi"}],
|
||||||
|
"thinking": True,
|
||||||
|
"reasoning_effort": "x",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user