[Feat] openai: Parse DeepSeek text for the native endpoints through deepseek-recipe (#46219)
This commit is contained in:
@@ -91,6 +91,7 @@ from sglang.srt.entrypoints.ollama.protocol import (
|
||||
OllamaShowRequest,
|
||||
)
|
||||
from sglang.srt.entrypoints.ollama.serving import OllamaServing
|
||||
from sglang.srt.entrypoints.openai.chat_encoding_adapters import native_text_parser
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ClassifyRequest,
|
||||
@@ -1716,15 +1717,27 @@ async def parse_function_call_request(
|
||||
"""
|
||||
A native API endpoint to parse function calls from a text.
|
||||
"""
|
||||
# 1) Initialize the parser based on the request body
|
||||
parser = FunctionCallParser(
|
||||
tools=obj.tools,
|
||||
tool_call_parser=obj.tool_call_parser,
|
||||
tokenizer=get_global_state().tokenizer_manager.tokenizer,
|
||||
)
|
||||
chat = request.app.state.openai_serving_chat
|
||||
try:
|
||||
text_parser = native_text_parser(
|
||||
obj.tool_call_parser,
|
||||
chat_encoding_spec=chat.chat_encoding_spec,
|
||||
adapter=chat.encoding_adapter,
|
||||
)
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(error))
|
||||
if text_parser is not None:
|
||||
normal_text, calls = text_parser.parse_function_call_text(obj.text, obj.tools)
|
||||
else:
|
||||
# 1) Initialize the parser based on the request body
|
||||
parser = FunctionCallParser(
|
||||
tools=obj.tools,
|
||||
tool_call_parser=obj.tool_call_parser,
|
||||
tokenizer=get_global_state().tokenizer_manager.tokenizer,
|
||||
)
|
||||
|
||||
# 2) Call the non-stream parsing method (non-stream)
|
||||
normal_text, calls = parser.parse_non_stream(obj.text)
|
||||
# 2) Call the non-stream parsing method (non-stream)
|
||||
normal_text, calls = parser.parse_non_stream(obj.text)
|
||||
|
||||
# 3) Organize the response content
|
||||
response_data = {
|
||||
@@ -1744,22 +1757,38 @@ async def separate_reasoning_request(
|
||||
"""
|
||||
A native API endpoint to separate reasoning from a text.
|
||||
"""
|
||||
# 1) Initialize the parser based on the request body
|
||||
parser = ReasoningParser(
|
||||
model_type=obj.reasoning_parser,
|
||||
request=request,
|
||||
tokenizer=get_global_state().tokenizer_manager.tokenizer,
|
||||
)
|
||||
|
||||
# 2) Call the non-stream parsing method (non-stream)
|
||||
if obj.return_blocks:
|
||||
blocks = parser.parse_non_stream_blocks(obj.text)
|
||||
reasoning_blocks = [b["text"] for b in blocks if b["type"] == "reasoning"]
|
||||
text_blocks = [b["text"] for b in blocks if b["type"] == "text"]
|
||||
reasoning_text = "".join(reasoning_blocks)
|
||||
normal_text = "".join(text_blocks)
|
||||
chat = request.app.state.openai_serving_chat
|
||||
try:
|
||||
text_parser = native_text_parser(
|
||||
obj.reasoning_parser,
|
||||
chat_encoding_spec=chat.chat_encoding_spec,
|
||||
adapter=chat.encoding_adapter,
|
||||
)
|
||||
except ValueError as error:
|
||||
raise HTTPException(status_code=HTTPStatus.BAD_REQUEST, detail=str(error))
|
||||
if text_parser is not None:
|
||||
reasoning_text, normal_text = text_parser.separate_reasoning_text(obj.text)
|
||||
blocks = (
|
||||
[{"type": "reasoning", "text": reasoning_text}] if reasoning_text else []
|
||||
)
|
||||
blocks.append({"type": "text", "text": normal_text})
|
||||
else:
|
||||
reasoning_text, normal_text = parser.parse_non_stream(obj.text)
|
||||
# 1) Initialize the parser based on the request body
|
||||
parser = ReasoningParser(
|
||||
model_type=obj.reasoning_parser,
|
||||
request=request,
|
||||
tokenizer=get_global_state().tokenizer_manager.tokenizer,
|
||||
)
|
||||
|
||||
# 2) Call the non-stream parsing method (non-stream)
|
||||
if obj.return_blocks:
|
||||
blocks = parser.parse_non_stream_blocks(obj.text)
|
||||
reasoning_text = "".join(
|
||||
b["text"] for b in blocks if b["type"] == "reasoning"
|
||||
)
|
||||
normal_text = "".join(b["text"] for b in blocks if b["type"] == "text")
|
||||
else:
|
||||
reasoning_text, normal_text = parser.parse_non_stream(obj.text)
|
||||
|
||||
# 3) Organize the response content
|
||||
response_data = {
|
||||
@@ -1767,8 +1796,12 @@ async def separate_reasoning_request(
|
||||
"text": normal_text,
|
||||
}
|
||||
if obj.return_blocks:
|
||||
response_data["reasoning_blocks"] = reasoning_blocks
|
||||
response_data["text_blocks"] = text_blocks
|
||||
response_data["reasoning_blocks"] = [
|
||||
b["text"] for b in blocks if b["type"] == "reasoning"
|
||||
]
|
||||
response_data["text_blocks"] = [
|
||||
b["text"] for b in blocks if b["type"] == "text"
|
||||
]
|
||||
response_data["blocks"] = blocks
|
||||
|
||||
return ORJSONResponse(content=response_data, status_code=200)
|
||||
|
||||
@@ -11,6 +11,7 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ResponsesRequest,
|
||||
Tool,
|
||||
ToolCall,
|
||||
)
|
||||
from sglang.srt.function_call.core_types import ToolCallItem
|
||||
@@ -106,6 +107,46 @@ class ChatEncodingAdapter(Protocol):
|
||||
...
|
||||
|
||||
|
||||
class NativeTextParser(Protocol):
|
||||
"""The native /separate_reasoning and /parse_function_call endpoints, for a
|
||||
parser whose output the served adapter parses."""
|
||||
|
||||
def separate_reasoning_text(self, text: str) -> tuple[str, str]: ...
|
||||
|
||||
def parse_function_call_text(
|
||||
self, text: str, tools: list[Tool]
|
||||
) -> tuple[str, list[ToolCallItem]]: ...
|
||||
|
||||
|
||||
# The reasoning and tool-call parser names whose output the spec's adapter parses.
|
||||
ADAPTER_OUTPUT_PARSERS: dict[str, tuple[str, str]] = {
|
||||
"dsv4": ("deepseek-v4", "deepseekv4"),
|
||||
"dsv41": ("deepseek-v41", "deepseekv41"),
|
||||
}
|
||||
|
||||
|
||||
def native_text_parser(
|
||||
parser_name: Optional[str],
|
||||
*,
|
||||
chat_encoding_spec: Optional[str],
|
||||
adapter: Optional[ChatEncodingAdapter],
|
||||
) -> Optional[NativeTextParser]:
|
||||
"""The served adapter when it parses `parser_name` output; None for the text
|
||||
parsers. A parser whose adapter is not served raises ValueError: the adapter
|
||||
parses with the served checkpoint's tokenizer."""
|
||||
owners = [
|
||||
spec for spec, names in ADAPTER_OUTPUT_PARSERS.items() if parser_name in names
|
||||
]
|
||||
if not owners:
|
||||
return None
|
||||
if adapter is None or chat_encoding_spec not in owners:
|
||||
raise ValueError(
|
||||
f"{parser_name!r} output is parsed with its model's tokenizer; this "
|
||||
"server does not serve that model"
|
||||
)
|
||||
return adapter
|
||||
|
||||
|
||||
def select_new_output_ids(
|
||||
*,
|
||||
output_ids: list[int],
|
||||
|
||||
@@ -478,6 +478,10 @@ class OpenAIServingChat(OpenAIServingBase):
|
||||
tool_call_parser=self.tool_call_parser,
|
||||
)
|
||||
|
||||
@property
|
||||
def encoding_adapter(self) -> Optional[chat_encoding_adapters.ChatEncodingAdapter]:
|
||||
return self._encoding_adapter
|
||||
|
||||
def _build_chat_encoding_adapter(
|
||||
self,
|
||||
) -> Optional[chat_encoding_adapters.ChatEncodingAdapter]:
|
||||
|
||||
@@ -8,6 +8,7 @@ from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, Any, Optional, Union
|
||||
|
||||
from sglang.srt.entrypoints.openai.chat_encoding_adapters import (
|
||||
ADAPTER_OUTPUT_PARSERS,
|
||||
select_new_output_ids,
|
||||
token_output_sse_chunks,
|
||||
)
|
||||
@@ -21,6 +22,7 @@ from sglang.srt.parser.deepseek_recipe_output import (
|
||||
DeepSeekRecipeOutputParser,
|
||||
RecipeToolCall,
|
||||
RecipeToolCallDelta,
|
||||
parse_output_text,
|
||||
whole_tool_calls,
|
||||
)
|
||||
from sglang.srt.parser.deepseek_recipe_renderer import (
|
||||
@@ -35,16 +37,13 @@ if TYPE_CHECKING:
|
||||
from sglang.srt.entrypoints.openai.protocol import (
|
||||
ChatCompletionRequest,
|
||||
ResponsesRequest,
|
||||
Tool,
|
||||
)
|
||||
|
||||
DEFAULT_EFFORT_ENVS = {
|
||||
"dsv4": envs.SGLANG_DSV4_REASONING_EFFORT,
|
||||
"dsv41": envs.SGLANG_DSV41_REASONING_EFFORT,
|
||||
}
|
||||
PARSERS = {
|
||||
"dsv4": ("deepseek-v4", "deepseekv4"),
|
||||
"dsv41": ("deepseek-v41", "deepseekv41"),
|
||||
}
|
||||
JSON_RESPONSE_FORMATS = ("json_object", "json_schema")
|
||||
|
||||
|
||||
@@ -77,7 +76,7 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
)
|
||||
# recipe parses the output once a DeepSeek parser is configured; other
|
||||
# parsers keep the text path.
|
||||
reasoning_name, tool_name = PARSERS[chat_encoding_spec]
|
||||
reasoning_name, tool_name = ADAPTER_OUTPUT_PARSERS[chat_encoding_spec]
|
||||
self._token_output = (
|
||||
reasoning_parser == reasoning_name or tool_call_parser == tool_name
|
||||
)
|
||||
@@ -293,6 +292,30 @@ class DeepSeekRecipeChatEncodingAdapter:
|
||||
)
|
||||
)
|
||||
|
||||
def separate_reasoning_text(self, text: str) -> tuple[str, str]:
|
||||
delta = parse_output_text(self.deepseek_recipe, text, parse_tool_calls=False)
|
||||
return delta.reasoning, delta.content
|
||||
|
||||
def parse_function_call_text(
|
||||
self, text: str, tools: list[Tool]
|
||||
) -> tuple[str, list[ToolCallItem]]:
|
||||
if not tools:
|
||||
return text, [] # as the text parsers: nothing to call
|
||||
delta = parse_output_text(
|
||||
self.deepseek_recipe, text, parse_tool_calls=True, thinking=False
|
||||
)
|
||||
# As in chat, a call to a tool outside `tools` is returned, at index -1.
|
||||
indices = {tool.function.name: i for i, tool in enumerate(tools)}
|
||||
calls = [
|
||||
ToolCallItem(
|
||||
tool_index=indices.get(call.name, -1),
|
||||
name=call.name,
|
||||
parameters=call.arguments,
|
||||
)
|
||||
for call in whole_tool_calls(delta)
|
||||
]
|
||||
return delta.content, calls
|
||||
|
||||
def _new_output_parser(
|
||||
self, request: ChatCompletionRequest
|
||||
) -> DeepSeekRecipeOutputParser:
|
||||
|
||||
@@ -12,12 +12,6 @@ class DeepSeekV41Detector(DeepSeekV32Detector):
|
||||
tool_calls_block_name = " calls"
|
||||
invoke_tag_name = " invoke"
|
||||
parameter_tag_name = " parameter"
|
||||
strip_string_param_value: bool = False
|
||||
|
||||
# The encoder joins an assistant turn's content and its calls block with a
|
||||
# blank line, and renders it even when there is no content.
|
||||
tool_calls_prefix = "\n\n"
|
||||
think_end_token = "</think>"
|
||||
|
||||
def get_structural_tag_name(self) -> Optional[str]:
|
||||
return "deepseek_v4_1"
|
||||
|
||||
@@ -11,6 +11,8 @@ from typing import Any, Optional
|
||||
import msgspec
|
||||
|
||||
from sglang.srt.parser.deepseek_recipe_renderer import (
|
||||
THINK_END_TOKEN,
|
||||
THINK_START_TOKEN,
|
||||
DeepSeekRecipe,
|
||||
load_deepseek_recipe,
|
||||
)
|
||||
@@ -220,3 +222,28 @@ class DeepSeekRecipeOutputParser:
|
||||
)
|
||||
)
|
||||
return delta
|
||||
|
||||
|
||||
def parse_output_text(
|
||||
deepseek_recipe: DeepSeekRecipe,
|
||||
text: str,
|
||||
*,
|
||||
parse_tool_calls: bool,
|
||||
thinking: Optional[bool] = None,
|
||||
) -> RecipeOutputDelta:
|
||||
"""`text` parsed as a whole output; the markers it spells count as the tokens
|
||||
the model would have sampled. With `thinking` None, text that opens with
|
||||
`<think>` or closes a thinking block is reasoning up to `</think>`."""
|
||||
tokenizer = deepseek_recipe.hf_tokenizer
|
||||
output_ids = tokenizer.encode(text, add_special_tokens=False).ids
|
||||
if thinking is None:
|
||||
thinking = output_ids[:1] == [tokenizer.token_to_id(THINK_START_TOKEN)]
|
||||
if thinking:
|
||||
output_ids = output_ids[1:]
|
||||
thinking = thinking or tokenizer.token_to_id(THINK_END_TOKEN) in output_ids
|
||||
parser = DeepSeekRecipeOutputParser(
|
||||
deepseek_recipe,
|
||||
thinking_stage=deepseek_recipe.parse_start(thinking=thinking),
|
||||
parse_tool_calls=parse_tool_calls,
|
||||
)
|
||||
return parser.finish(output_ids, {"type": "stop"})
|
||||
|
||||
@@ -1536,11 +1536,6 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
||||
DSV4_THINK_START_TOKEN,
|
||||
DSV4_THINK_END_TOKEN,
|
||||
think_excluded_tokens=[DSV4_EOS_TOKEN, DSV4_DSML_TOKEN],
|
||||
# The reference parser splits reasoning at `\n\n<|DSML|tool_calls`;
|
||||
# anchoring on the full tag at a line start keeps a mid-sentence
|
||||
# mention of the tag inside reasoning from ending the block.
|
||||
tool_start_token=f"<{DSV4_DSML_TOKEN}tool_calls>",
|
||||
tool_start_at_line_start=True,
|
||||
force_reasoning=force_reasoning,
|
||||
stream_reasoning=stream_reasoning,
|
||||
continue_final_message=continue_final_message,
|
||||
@@ -1551,15 +1546,6 @@ class DeepSeekV4Detector(BaseReasoningFormatDetector):
|
||||
)
|
||||
|
||||
|
||||
class DeepSeekV41ReasoningDetector(DeepSeekV4Detector):
|
||||
"""V4.1 emits the calls block with a spaced tag name (`<|DSML| calls>`),
|
||||
so the unspaced V4 tool-start anchor never interrupts its reasoning."""
|
||||
|
||||
def __init__(self, **kwargs):
|
||||
super().__init__(**kwargs)
|
||||
self.tool_start_token = f"<{DSV4_DSML_TOKEN} calls>"
|
||||
|
||||
|
||||
class _MimoDetector(Qwen3Detector):
|
||||
"""MIMO reuses Qwen3 tokens but requires explicit enable_thinking=True to enable."""
|
||||
|
||||
@@ -2271,7 +2257,7 @@ class ReasoningParser:
|
||||
"deepseek-r1": DeepSeekR1Detector,
|
||||
"deepseek-v3": _DeepSeekV3Detector,
|
||||
"deepseek-v4": DeepSeekV4Detector,
|
||||
"deepseek-v41": DeepSeekV41ReasoningDetector,
|
||||
"deepseek-v41": DeepSeekV4Detector,
|
||||
"dots": Qwen3Detector,
|
||||
"glm45": Glm45Detector,
|
||||
"ling3": Ling3Detector,
|
||||
|
||||
Reference in New Issue
Block a user