minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity
Files
b5c9b68f03c0552c94b717e200e7c2e6166c6741
sglang/python/sglang/srt/parser/reasoning_parser.py
T

2238 lines
89 KiB
Python
Raw Blame History

import inspect
import re
from typing import Dict, List, Optional, Tuple, Type
from sglang.srt.entrypoints.openai.encoding_dsv4 import dsml_token as dsv4_dsml_token
from sglang.srt.entrypoints.openai.encoding_dsv4 import eos_token as dsv4_eos_token
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
thinking_end_token as dsv4_thinking_end_token,
)
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
thinking_start_token as dsv4_thinking_start_token,
)
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens
from sglang.srt.function_call.kimik3_format import (
MESSAGE_CLOSE,
RESPONSE_CLOSE,
RESPONSE_OPEN,
THINK_CLOSE,
THINK_OPEN,
TOOLS_OPEN,
partial_suffix_len,
strip_partial_marker_suffix,
strip_response_wrappers,
)
from sglang.srt.function_call.muse_glimmer_format import (
EOM,
EOT,
MAX_CHANNEL_MARKER,
MESSAGE,
RECIPIENT_RE,
START,
could_start_header,
has_atem_markers,
partial_marker_len,
)
from sglang.srt.parser.harmony_parser import HarmonyParser
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_INVOKE_TOOL_JSON,
CONTENT_INVOKE_TOOL_TEXT,
CONTENT_MODEL_END_SAMPLING,
CONTENT_TEXT,
CONTENT_THINKING,
END_MESSAGE,
INKLING_CONTROL_TOKENS,
MESSAGE_MODEL,
)
class StreamingParseResult:
"""Result of streaming incremental parsing."""
def __init__(
self,
normal_text: Optional[str] = None,
reasoning_text: Optional[str] = None,
):
self.normal_text = normal_text or ""
self.reasoning_text = reasoning_text or ""
class BaseReasoningFormatDetector:
"""Base class providing two sets of interfaces: one-time and streaming incremental."""
def __init__(
self,
think_start_token: str,
think_end_token: str,
think_excluded_tokens: Optional[List[str]] = None,
force_reasoning: bool = False,
stream_reasoning: bool = True,
tool_start_token: Optional[str] = None,
continue_final_message: bool = False,
previous_content: str = "",
thinks_internally: bool = False,
reasoning_default: str = "always",
force_nonempty_content: bool = False,
):
self.think_start_token = think_start_token
self.think_end_token = think_end_token
self.think_excluded_tokens = think_excluded_tokens
self.tool_start_token = tool_start_token
self.force_reasoning = force_reasoning
self._in_reasoning = force_reasoning
self.stream_reasoning = stream_reasoning
self.thinks_internally = thinks_internally
self.reasoning_default = reasoning_default
self._buffer = ""
self.stripped_think_start = False
self.think_start_self_label = ""
self._force_nonempty_content = force_nonempty_content
self._accumulated_reasoning = ""
self.continue_final_message = continue_final_message
if self.continue_final_message:
self.previous_content = previous_content
self.previous_count = len(previous_content)
else:
self.previous_content = ""
self.previous_count = 0
if self.think_start_token in self.previous_content:
self._in_reasoning = True
if self.think_end_token in self.previous_content:
self._in_reasoning = False
def _maybe_apply_force_nonempty_content(
self, ret: StreamingParseResult
) -> StreamingParseResult:
if self._force_nonempty_content and not ret.normal_text:
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
return ret
def detect_and_parse(self, text: str) -> StreamingParseResult:
"""
One-time parsing: Detects and parses reasoning sections in the provided text.
Returns both reasoning content and normal text separately.
"""
return self._maybe_apply_force_nonempty_content(
self._detect_and_parse_impl(text)
)
def _detect_and_parse_impl(self, text: str) -> StreamingParseResult:
in_reasoning = self._in_reasoning or self.think_start_token in text
if not in_reasoning:
return StreamingParseResult(normal_text=text)
# The text is considered to be in a reasoning block.
think_start_text = self.think_start_token + self.think_start_self_label
processed_text = text
while processed_text.startswith(think_start_text):
processed_text = processed_text[len(think_start_text) :]
if (
self.think_end_token not in processed_text
and self.think_end_token not in self.previous_content
):
# Check for tool_start_token interruption
if (
in_reasoning
and self.tool_start_token is not None
and self.tool_start_token in processed_text
):
# Find the first occurrence of tool_start_token and split there
tool_idx = processed_text.find(self.tool_start_token)
reasoning_text = processed_text[:tool_idx]
# Preserve tool_start_token in normal text
normal_text = processed_text[tool_idx:]
return StreamingParseResult(
normal_text=normal_text, reasoning_text=reasoning_text
)
# Assume reasoning was truncated before end token
return StreamingParseResult(reasoning_text=processed_text)
# Extract reasoning content
if self.think_end_token in processed_text:
splits = processed_text.split(self.think_end_token, maxsplit=1)
reasoning_text = splits[0]
normal_text = splits[1]
return StreamingParseResult(
normal_text=normal_text, reasoning_text=reasoning_text
)
else:
# think_end_token is in self.previous_content for continue_final_message=True case
return StreamingParseResult(normal_text=processed_text)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
"""
Streaming incremental parsing for reasoning content.
Handles partial reasoning tags and content.
If stream_reasoning is False:
Accumulates reasoning content until the end tag is found
If stream_reasoning is True:
Streams reasoning content as it arrives
"""
ret = self._parse_streaming_increment_impl(new_text)
if self._force_nonempty_content:
if self._in_reasoning:
self._accumulated_reasoning += ret.reasoning_text
else:
self._accumulated_reasoning = ""
return ret
def _parse_streaming_increment_impl(self, new_text: str) -> StreamingParseResult:
self._buffer += new_text
current_text = self._buffer
think_start_text = self.think_start_token + self.think_start_self_label
# If the current text is a prefix of the think token, keep buffering
tokens_to_check = [think_start_text, self.think_end_token]
if self.tool_start_token:
tokens_to_check.append(self.tool_start_token)
if any(
token.startswith(current_text) and token != current_text
for token in tokens_to_check
):
return StreamingParseResult()
# Strip `<think>` token if present
if not self.stripped_think_start and think_start_text in current_text:
current_text = current_text.replace(think_start_text, "", 1)
# Write back, or stream_reasoning=False carries the token into finish().
self._buffer = current_text
self.stripped_think_start = True
self._in_reasoning = True
# Handle end of reasoning block
if self._in_reasoning and self.think_end_token in current_text:
end_idx = current_text.find(self.think_end_token)
reasoning_text = current_text[:end_idx]
self._buffer = ""
self._in_reasoning = False
normal_text = current_text[end_idx + len(self.think_end_token) :]
return StreamingParseResult(
normal_text=normal_text, reasoning_text=reasoning_text
)
# Continue with reasoning content
if self._in_reasoning:
# Check for tool_start_token interruption. Streaming cannot see a
# think_end_token that has not arrived yet; see the chunk_dependent test.
if self.tool_start_token and self.tool_start_token in current_text:
tool_idx = current_text.find(self.tool_start_token)
reasoning_text = current_text[:tool_idx]
# Preserve tool_start_token in normal text
normal_text = current_text[tool_idx:]
self._buffer = ""
self._in_reasoning = False
return StreamingParseResult(
normal_text=normal_text, reasoning_text=reasoning_text
)
if self.stream_reasoning:
# Minus any trailing slice that could be a token split across chunks.
holdback_tokens = [self.think_end_token]
if self.tool_start_token:
holdback_tokens.append(self.tool_start_token)
if not self.stripped_think_start:
# force_reasoning never saw the opening token; it can still split.
holdback_tokens.append(think_start_text)
holdback = max(
self._ends_with_partial_token(current_text, token)
for token in holdback_tokens
)
self._buffer = current_text[len(current_text) - holdback :]
return StreamingParseResult(
reasoning_text=current_text[: len(current_text) - holdback]
)
else:
return StreamingParseResult()
# If we're not in a reasoning block return as normal text
if not self._in_reasoning:
self._buffer = ""
return StreamingParseResult(normal_text=current_text)
return StreamingParseResult()
def _strip_leading_think_start(self, text: str) -> str:
think_start_text = self.think_start_token + self.think_start_self_label
if text.startswith(think_start_text):
return text[len(think_start_text) :]
return text
@staticmethod
def _ends_with_partial_token(buffer: str, token: str) -> int:
"""Length of the longest trailing slice of `buffer` that is a strict prefix
of `token`. Longest, so a token whose prefix repeats inside itself does not
get cut short and leak the rest of the marker."""
for i in range(min(len(buffer), len(token) - 1), 0, -1):
if token.startswith(buffer[-i:]):
return i
return 0
def finish(self) -> StreamingParseResult:
"""Flush reasoning still buffered when the stream ends before the end token
(e.g. max_tokens cut it short), instead of dropping it: the whole block under
stream_reasoning=False, the held-back token suffix under stream_reasoning=True.
force_nonempty_content emits it as normal_text, else as reasoning_text."""
if not self._in_reasoning:
# Same as the reasoning-side flush below: a held-back slice that never
# became a token is content.
leftover = self._buffer
self._buffer = ""
return StreamingParseResult(normal_text=leftover)
# Defensive: subclasses that fill _buffer themselves may not have stripped
# the opening think token that _parse_streaming_increment_impl removes.
buffer = self._strip_leading_think_start(self._buffer)
self._buffer = ""
if self._force_nonempty_content:
normal_text = self._accumulated_reasoning + buffer
self._accumulated_reasoning = ""
if normal_text:
return StreamingParseResult(normal_text=normal_text)
return StreamingParseResult()
if buffer:
return StreamingParseResult(reasoning_text=buffer)
return StreamingParseResult()
class DeepSeekR1Detector(BaseReasoningFormatDetector):
"""
Detector for DeepSeek-R1 model.
Assumes reasoning format:
(<think>)*(.*)</think>
Returns all the text before the </think> tag as `reasoning_text`
and the rest of the text as `normal_text`.
Supported models:
- DeepSeek-R1: Always generates thinking content without <think> start tag
- DeepSeek-R1-0528: Generates thinking content with <think> start tag
Format patterns:
- DeepSeek-R1: "I need to think about this...</think>The answer is 42."
- DeepSeek-R1-0528: "<think>I need to think about this...</think>The answer is 42."
Args:
stream_reasoning (bool): If False, accumulates reasoning content until the end tag.
If True, streams reasoning content as it arrives.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# DeepSeek-R1 is assumed to be reasoning until `</think>` token
super().__init__(
"<think>",
"</think>",
force_reasoning=True,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
# https://github.com/sgl-project/sglang/pull/3202#discussion_r1950153599
class Qwen3Detector(BaseReasoningFormatDetector):
"""
Detector for Qwen3 models (e.g., Qwen/Qwen3-235B-A22B).
Assumes reasoning format:
(<think>)*(.*)</think>
Qwen3 models released before 07/2025 supports switching between thinking mode and normal
mode using `enable_thinking` parameter in the request parameter.
- enable_thinking=True: "<think>reasoning content</think>The answer is 42."
- enable_thinking=False: "The answer is 42." (no thinking tokens)
Args:
stream_reasoning (bool): If False, accumulates reasoning content until the end tag.
If True, streams reasoning content as it arrives.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
think_excluded_tokens = [
"<tool_call>",
"</tool_call>",
"<|im_end|>",
"<|endoftext|>",
]
super().__init__(
"<think>",
"</think>",
think_excluded_tokens=think_excluded_tokens,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
# Qwen3.5 sometimes opens ``<tool_call>`` without closing
# ``</think>``; treat it as an implicit reasoning close.
tool_start_token="<tool_call>",
continue_final_message=continue_final_message,
previous_content=previous_content,
thinks_internally=True,
reasoning_default="enable_thinking",
force_nonempty_content=force_nonempty_content,
)
class KimiDetector(BaseReasoningFormatDetector):
"""
Detector for Kimi Thinking model.
Assumes reasoning format:
◁think▷*(.*)◁/think▷
Returns all the text before the ◁/think▷ tag as `reasoning_text`
and the rest of the text as `normal_text`.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"◁think▷",
"◁/think▷",
force_reasoning=False,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
class KimiK2Detector(BaseReasoningFormatDetector):
"""
Detector for Kimi K2 models.
Assumes reasoning format:
(<think>)*(.*)</think>
Kimi K2 can switch from reasoning to tool-call section with
`<|tool_calls_section_begin|>` before emitting `</think>`.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
think_excluded_tokens = [
"<think>",
"<|tool_calls_section_begin|>",
"<|tool_call_begin|>",
"<|tool_call_argument_begin|>",
"<|tool_call_section_end|>",
"<|tool_call_end|>",
"[EOS]",
"<|im_end|>",
"<|end_header_id|>",
"[EOT]",
]
super().__init__(
"<think>",
"</think>",
think_excluded_tokens=think_excluded_tokens,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token="<|tool_calls_section_begin|>",
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="thinking",
force_nonempty_content=force_nonempty_content,
)
class K2V3Detector(BaseReasoningFormatDetector):
"""Reasoning detector for canonical K2 Horizon IFM tokens.
K2 Horizon's template prefills the opening token, so generated text starts
inside reasoning and this parser is always forced on. ``reasoning_effort``
selects the matching IFM token pair.
"""
_EFFORT_TOKENS = {
"high": ("<ifm|think>", "</ifm|think>"),
"medium": ("<ifm|think_fast>", "</ifm|think_fast>"),
"low": ("<ifm|think_faster>", "</ifm|think_faster>"),
}
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
reasoning_effort: object = "high",
):
if not force_reasoning:
raise ValueError("K2-v3 reasoning parser requires force_reasoning=True")
# Five release templates reject unsupported levels. The 0.9B template's
# fallback emits the medium token, so medium is the only possible wire
# format for an unsupported value that reaches generation.
effort = (
reasoning_effort
if isinstance(reasoning_effort, str)
and reasoning_effort in self._EFFORT_TOKENS
else "medium"
)
start_token, end_token = self._EFFORT_TOKENS[effort]
super().__init__(
start_token,
end_token,
force_reasoning=True,
stream_reasoning=stream_reasoning,
# Common prefix of the singular and plural tool-call open tags.
# This also closes reasoning for a malformed turn that omits its
# explicit </ifm|think...> token.
tool_start_token="<ifm|tool_call",
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="always",
force_nonempty_content=force_nonempty_content,
)
# Catalog only: scheduler-side request validation encodes these, but
# the active matcher must use just the delimiter selected above.
self.request_selectable_think_end_tokens = tuple(
tokens[1] for tokens in self._EFFORT_TOKENS.values()
)
class KimiK3Detector(BaseReasoningFormatDetector):
"""Detector for the Kimi K3 XTML think channel.
K3 wraps reasoning as ``<|open|>think<|sep|>...<|close|>think<|sep|>``
where each marker is a multi-token special sequence, so partial markers
can straddle streaming chunks and must be held back. In thinking mode
the serving layer may feed the open marker as the generation prefix, so
output can begin inside the think channel with no open marker
(``force_reasoning=True`` covers this).
Post-reasoning content is unwrapped from the XTML ``response`` /
``message`` markers; a ``tools`` channel is passed through raw for the
kimi_k3 tool-call detector.
The model does not always honour the pre-filled think channel: on very long
prompts (~1M tokens) it sometimes emits a zero-length think section and
writes the reply directly, closing with
``<|close|>response<|sep|><|close|>message<|sep|>`` and never producing
``<|close|>think<|sep|>`` or ``<|open|>response<|sep|>``. A bare
``<|close|>response<|sep|>`` therefore proves the preceding text was the
response channel, and it is reported as content rather than reasoning
(see :meth:`_skipped_think_channel`).
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# strict-thinking flattens these to single token ids, so the full marker
# "<|open|>response<|sep|>" is inexpressible. The bare name works: it
# follows <|open|> unspaced, so it tokenizes to the no-space variant, not
# the " response"/" message" tokens prose uses -- at the cost of not being
# able to start those words unspaced mid-reasoning. tools is left out on
# purpose: the model may jump from think straight into that channel.
think_excluded_tokens = [
"response",
"message",
"<|end_of_msg|>",
"[EOS]",
"[EOT]",
]
super().__init__(
THINK_OPEN,
THINK_CLOSE,
think_excluded_tokens=think_excluded_tokens,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token=TOOLS_OPEN,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="thinking",
)
# Unlike the base class, K3 cannot use `normal_text == ""` alone:
# skipped-think and truncated marker-free reasoning end up identical.
self._force_nonempty_content = force_nonempty_content
self._reasoning_done = False
self._tools_passthrough = False
self._stream_text = ""
self._streamed_reasoning: list[str] = []
self._discard_delayed_think_close = False
def _clean_content(self, text: str) -> str:
tools_idx = text.find(TOOLS_OPEN)
if tools_idx != -1:
return strip_response_wrappers(text[:tools_idx]) + text[tools_idx:]
return strip_response_wrappers(text)
def _next_channel_idx(self, text: str, start: int = 0) -> int:
found = [
idx
for token in (RESPONSE_OPEN, self.tool_start_token)
if (idx := text.find(token, start)) != -1
]
return min(found) if found else -1
@staticmethod
def _skipped_think_channel(
text: str,
start: int = 0,
think_close_idx: int = -1,
next_channel_idx: int = -1,
) -> bool:
response_close_idx = text.find(RESPONSE_CLOSE, start)
return response_close_idx != -1 and all(
boundary_idx == -1 or response_close_idx < boundary_idx
for boundary_idx in (think_close_idx, next_channel_idx)
)
def _clean_skipped_think_content(self, text: str) -> str:
return self._clean_content(text.replace(self.think_end_token, ""))
def detect_and_parse(self, text: str) -> StreamingParseResult:
in_reasoning = self._in_reasoning or self.think_start_token in text
if not in_reasoning and self.think_end_token not in text:
return StreamingParseResult(normal_text=self._clean_content(text))
if self._force_nonempty_content and self._is_skipped_think_answer(text):
return StreamingParseResult(normal_text=self._clean_content(text))
open_idx = text.find(self.think_start_token)
start = open_idx + len(self.think_start_token) if open_idx != -1 else 0
close_idx = text.find(self.think_end_token, start)
tools_idx = text.find(self.tool_start_token, start)
channel_idx = self._next_channel_idx(text, start)
if self._skipped_think_channel(text, start, close_idx, channel_idx):
return StreamingParseResult(
normal_text=self._clean_skipped_think_content(text[start:])
)
if close_idx != -1 and tools_idx != -1 and tools_idx < close_idx:
return StreamingParseResult(
reasoning_text=strip_partial_marker_suffix(text[start:tools_idx]),
normal_text=self._clean_content(text[tools_idx:]),
)
if close_idx == -1:
if channel_idx != -1:
return StreamingParseResult(
reasoning_text=strip_partial_marker_suffix(text[start:channel_idx]),
normal_text=self._clean_content(text[channel_idx:]),
)
return StreamingParseResult(
reasoning_text=strip_partial_marker_suffix(text[start:])
)
reasoning_text = text[start:close_idx]
rest = text[close_idx + len(self.think_end_token) :]
return StreamingParseResult(
reasoning_text=reasoning_text, normal_text=self._clean_content(rest)
)
def _is_skipped_think_answer(self, text: str) -> bool:
return (
self.think_start_token not in text
and self.think_end_token not in text
and self.think_start_token.removesuffix("<|sep|>") not in text
and self.think_end_token.removesuffix("<|sep|>") not in text
and (RESPONSE_CLOSE in text or MESSAGE_CLOSE in text)
)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
self._buffer += new_text
if self._force_nonempty_content:
self._stream_text += new_text
if not self._in_reasoning and not self._reasoning_done:
open_idx = self._buffer.find(self.think_start_token)
if open_idx != -1:
self._buffer = self._buffer[open_idx + len(self.think_start_token) :]
self._in_reasoning = True
self.stripped_think_start = True
elif self.think_start_token.startswith(self._buffer):
return StreamingParseResult()
else:
self._reasoning_done = True
if self._in_reasoning:
buf = self._buffer
if not self.stripped_think_start:
open_idx = buf.find(self.think_start_token)
if open_idx != -1:
buf = buf[open_idx + len(self.think_start_token) :]
self._buffer = buf
self.stripped_think_start = True
close_idx = buf.find(self.think_end_token)
tools_idx = buf.find(self.tool_start_token)
channel_idx = self._next_channel_idx(buf)
if self._skipped_think_channel(
buf,
think_close_idx=close_idx,
next_channel_idx=channel_idx,
):
replay = "".join(self._streamed_reasoning)
self._streamed_reasoning.clear()
self._in_reasoning = False
self._reasoning_done = True
self._discard_delayed_think_close = True
return StreamingParseResult(
normal_text=(replay + self._drain_content()) or None
)
if close_idx != -1 and not (tools_idx != -1 and tools_idx < close_idx):
reasoning_text = buf[:close_idx]
self._buffer = buf[close_idx + len(self.think_end_token) :]
self._in_reasoning = False
self._reasoning_done = True
self._streamed_reasoning.clear()
return StreamingParseResult(
reasoning_text=reasoning_text or None,
normal_text=self._drain_content() or None,
)
if channel_idx != -1:
reasoning_text = strip_partial_marker_suffix(buf[:channel_idx])
self._buffer = buf[channel_idx:]
self._in_reasoning = False
self._reasoning_done = True
self._streamed_reasoning.clear()
self._tools_passthrough = buf.startswith(
self.tool_start_token, channel_idx
)
return StreamingParseResult(
reasoning_text=reasoning_text or None,
normal_text=self._drain_content() or None,
)
if not self.stream_reasoning:
return StreamingParseResult()
markers = [
self.think_end_token,
self.tool_start_token,
RESPONSE_OPEN,
RESPONSE_CLOSE,
MESSAGE_CLOSE,
]
if not self.stripped_think_start:
markers.append(self.think_start_token)
holdback = partial_suffix_len(buf, markers)
emit = buf[: len(buf) - holdback] if holdback else buf
emit = strip_partial_marker_suffix(emit)
self._buffer = buf[len(emit) :]
self._streamed_reasoning.append(emit)
return StreamingParseResult(reasoning_text=emit)
return StreamingParseResult(normal_text=self._drain_content())
def finish(self) -> StreamingParseResult:
self._streamed_reasoning.clear()
if not self._force_nonempty_content:
return super().finish()
text, self._stream_text = self._stream_text, ""
if self._in_reasoning and self._is_skipped_think_answer(text):
# _in_reasoning means no channel decision happened mid-stream, so the
# answer went out as reasoning; without this gate the re-emit duplicates
# answers already streamed as content (RESPONSE_OPEN / force_reasoning=False).
self._buffer = ""
return StreamingParseResult(normal_text=self._clean_content(text))
if self._in_reasoning and not self.stream_reasoning and self._buffer:
# super().finish() would emit this buffer as content under
# force_nonempty_content — the leak the flag exists to prevent.
buffer, self._buffer = self._buffer, ""
return StreamingParseResult(reasoning_text=buffer)
return StreamingParseResult()
def _drain_content(self) -> str:
buf = self._buffer
if not buf:
return ""
if self._tools_passthrough:
holdback = (
partial_suffix_len(buf, [self.think_end_token])
if self._discard_delayed_think_close
else 0
)
emit = buf[: len(buf) - holdback] if holdback else buf
self._buffer = buf[len(emit) :]
if self._discard_delayed_think_close:
emit = emit.replace(self.think_end_token, "")
return emit
tools_idx = buf.find(TOOLS_OPEN)
if tools_idx != -1:
holdback = (
partial_suffix_len(buf, [self.think_end_token])
if self._discard_delayed_think_close
else 0
)
emit = buf[: len(buf) - holdback] if holdback else buf
self._buffer = buf[len(emit) :]
head = emit[:tools_idx]
tail = emit[tools_idx:]
for marker in (RESPONSE_OPEN, RESPONSE_CLOSE, MESSAGE_CLOSE):
head = head.replace(marker, "")
if self._discard_delayed_think_close:
head = head.replace(self.think_end_token, "")
tail = tail.replace(self.think_end_token, "")
self._tools_passthrough = True
return head + tail
markers = [RESPONSE_OPEN, RESPONSE_CLOSE, MESSAGE_CLOSE, TOOLS_OPEN]
if self._discard_delayed_think_close:
markers.append(self.think_end_token)
holdback = partial_suffix_len(buf, markers)
emit = buf[: len(buf) - holdback] if holdback else buf
self._buffer = buf[len(emit) :]
for marker in (RESPONSE_OPEN, RESPONSE_CLOSE, MESSAGE_CLOSE):
emit = emit.replace(marker, "")
if self._discard_delayed_think_close:
emit = emit.replace(self.think_end_token, "")
return emit
class Glm45Detector(BaseReasoningFormatDetector):
"""
Detector for GLM-4.5 models.
Assumes reasoning format:
(<think>)*(.*)</think>
GLM-4.5 uses `<tool_call>` as the tool start token to switch from reasoning mode to normal mode.
Args:
stream_reasoning (bool): If False, accumulates reasoning content until the end tag.
If True, streams reasoning content as it arrives.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
force_nonempty_content: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
reasoning_default: str = "enable_thinking",
):
think_excluded_tokens = [
"<tool_call>",
"</tool_call>",
"<eop>",
"<|user|>",
"<|endoftext|>",
]
super().__init__(
"<think>",
"</think>",
think_excluded_tokens=think_excluded_tokens,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token="<tool_call>",
thinks_internally=True,
reasoning_default=reasoning_default,
force_nonempty_content=force_nonempty_content,
continue_final_message=continue_final_message,
previous_content=previous_content,
)
class Ling3Detector(Glm45Detector):
"""
Detector for Ling3 models.
Ling3 is a hybrid-thinking model whose chat template defaults to thinking
on (the template sets `thinking_option='on'` when `enable_thinking` is
omitted, which the generic template detector cannot infer). Tool calls also
terminate reasoning when the model omits </think>.
If non-streaming output only contains reasoning text and no tool call, Ling3
moves that text into normal content as a client-experience fallback. Streaming
parsing still emits reasoning increments as they arrive because this parser
does not receive a final end-of-generation signal.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = True,
):
super().__init__(
stream_reasoning=stream_reasoning,
force_reasoning=force_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="enable_thinking",
)
self._force_nonempty_content = force_nonempty_content
def detect_and_parse(self, text: str) -> StreamingParseResult:
ret = super().detect_and_parse(text)
if (
self._force_nonempty_content
and ret.reasoning_text
and not ret.normal_text
and self.tool_start_token not in text
):
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
return ret
class GptOssDetector(BaseReasoningFormatDetector):
"""
Detector for T4-style reasoning format (GPT-OSS), using the HarmonyParser.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<|channel|>analysis<|message|>",
"<|end|>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self.parser = HarmonyParser()
def detect_and_parse(self, text: str) -> StreamingParseResult:
events = self.parser.parse(text)
# Flush the buffer for one-shot parsing
events += self.parser.parse("")
reasoning_text = "".join(
[e.content for e in events if e.event_type == "reasoning"]
)
normal_parts = []
for e in events:
if e.event_type == "normal":
normal_parts.append(e.content)
elif e.event_type == "tool_call":
# Use raw_text to preserve structural markers for function call detector
normal_parts.append(e.raw_text if e.raw_text else e.content)
normal_text = "".join(normal_parts)
# Tool call events preserve raw text with structural markers
return self._maybe_apply_force_nonempty_content(
StreamingParseResult(
normal_text=normal_text,
reasoning_text=reasoning_text,
)
)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
events = self.parser.parse(new_text)
reasoning_text = "".join(
[e.content for e in events if e.event_type == "reasoning"]
)
normal_parts = []
for e in events:
if e.event_type == "normal":
normal_parts.append(e.content)
elif e.event_type == "tool_call":
# Use raw_text to preserve structural markers for function call detector
normal_parts.append(e.raw_text if e.raw_text else e.content)
normal_text = "".join(normal_parts)
return StreamingParseResult(
normal_text=normal_text,
reasoning_text=reasoning_text,
)
class MiniMaxAppendThinkDetector(BaseReasoningFormatDetector):
"""
Append `<think>` token to the beginning of the text.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# scheduler.py need `reasoning_parser.detector.think_end_token`
super().__init__(
"<think>",
"</think>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self.is_first_chunk = False
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
if not self.is_first_chunk:
self.is_first_chunk = True
new_text = self.think_start_token + new_text
return StreamingParseResult(normal_text=new_text)
def detect_and_parse(self, text: str) -> StreamingParseResult:
return StreamingParseResult(normal_text=self.think_start_token + text)
class Nemotron3Detector(BaseReasoningFormatDetector):
"""
Detector for Nemotron3 model.
Uses the same reasoning format as DeepSeek-R1: (<think>)*(.*)</think>
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<think>",
"</think>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token="<tool_call>",
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="enable_thinking",
force_nonempty_content=force_nonempty_content,
)
class MiniMaxM3Detector(BaseReasoningFormatDetector):
"""MiniMax-M3 detector. Format: (<mm:think>)*(.*)</mm:think>.
In multi-turn chats M3 prefixes earlier non-thinking turns with a bare
``</mm:think>``, so a non-thinking reply may open with one stray closer; drop it unless thinking.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<mm:think>",
"</mm:think>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
)
self._lead_buffer = ""
self._checked_leading_close = False
self._force_nonempty_content = force_nonempty_content
def detect_and_parse(self, text: str) -> StreamingParseResult:
if not self._in_reasoning and text.lstrip().startswith(self.think_end_token):
text = text.lstrip()[len(self.think_end_token) :]
ret = super().detect_and_parse(text)
if self._force_nonempty_content and not ret.normal_text:
ret.normal_text, ret.reasoning_text = ret.reasoning_text, ret.normal_text
return ret
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
# ``</mm:think>`` is a single token, so a stray leading closer arrives whole.
if not self._checked_leading_close and not self._in_reasoning:
self._lead_buffer += new_text
stripped = self._lead_buffer.lstrip()
if not stripped:
return StreamingParseResult()
self._checked_leading_close = True
if stripped.startswith(self.think_end_token):
new_text = stripped[len(self.think_end_token) :]
else:
new_text = self._lead_buffer
self._lead_buffer = ""
if not new_text:
return StreamingParseResult()
return super().parse_streaming_increment(new_text)
class MistralDetector(BaseReasoningFormatDetector):
"""
Detector for Mistral models with reasoning (e.g., Mistral-Small-4-119B-2603).
Assumes reasoning format:
[THINK]reasoning content[/THINK]answer
Reasoning is optional — it only appears when reasoning_effort="high" is set.
When reasoning_effort="none", the model outputs directly without thinking tokens.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"[THINK]",
"[/THINK]",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="mistral",
force_nonempty_content=force_nonempty_content,
)
class HunyuanDetector(BaseReasoningFormatDetector):
"""
Detector for Hunyuan models (e.g., tencent/Hunyuan-A13B-Instruct).
Like Glm45Detector but uses ``<tool_calls>`` (plural) as the tool start token.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
tokenizer=None,
force_nonempty_content: bool = False,
):
t = resolve_hunyuan_tokens(tokenizer)
think_open = t["think"]
think_close = (
"</" + think_open[1:] if think_open.startswith("<") else think_open
)
super().__init__(
think_open,
think_close,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
tool_start_token=t["tool_calls"],
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
class Gemma4Detector(BaseReasoningFormatDetector):
"""Gemma4 reasoning detector."""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<|channel>",
"<channel|>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
reasoning_default="explicit_enable_thinking",
force_nonempty_content=force_nonempty_content,
)
self.think_start_self_label = "thought\n"
_INKLING_CONTENT_KINDS = {
CONTENT_THINKING: "reasoning",
CONTENT_TEXT: "content",
}
_INKLING_END_TOKENS = {
CONTENT_MODEL_END_SAMPLING,
END_MESSAGE,
}
_INKLING_CONTROL_TOKENS = INKLING_CONTROL_TOKENS
_INKLING_CONTROL_RE = re.compile(
"|".join(re.escape(t) for t in sorted(_INKLING_CONTROL_TOKENS))
)
class InklingDetector(BaseReasoningFormatDetector):
"""Detector for Inkling typed content blocks."""
# Parse the model's sequence of typed content blocks, for example:
# <|message_model|><|content_thinking|>reasoning<|end_message|>
# <|message_model|><|content_text|>visible answer<|end_message|>
# <|content_model_end_sampling|>
# Special tokens must decode literally so thinking and visible text can be
# routed to their respective response fields.
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
del force_nonempty_content
super().__init__(
CONTENT_THINKING,
END_MESSAGE,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
thinks_internally=False,
reasoning_default="always",
)
self._kind: str | None = None
self._pending_header = ""
self._pending_reasoning = ""
def detect_and_parse(self, text: str) -> StreamingParseResult:
self._buffer = ""
self._kind = None
self._pending_header = ""
self._pending_reasoning = ""
ret = self._parse_blocks(text)
if self._kind == "reasoning" and not self.stream_reasoning:
ret.reasoning_text += self._pending_reasoning
self._kind = None
self._pending_header = ""
self._pending_reasoning = ""
return ret
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
text = self._buffer + new_text
partial_len = self._partial_control_length(text)
if partial_len:
self._buffer = text[-partial_len:]
text = text[:-partial_len]
else:
self._buffer = ""
return self._parse_blocks(text)
def finish(self) -> StreamingParseResult:
# Flush reasoning buffered under stream_reasoning=False when the stream
# ends before a control/end token closes the block (e.g. max_tokens cut
# a thinking block short). Mirrors the non-streaming flush in
# detect_and_parse; without it the trailing reasoning trace is dropped.
reasoning_text = ""
if self._kind == "reasoning" and not self.stream_reasoning:
reasoning_text = self._pending_reasoning
self._buffer = ""
self._pending_reasoning = ""
self._pending_header = ""
self._kind = None
return StreamingParseResult(reasoning_text=reasoning_text)
@staticmethod
def _partial_control_length(text: str) -> int:
max_token_len = max(map(len, _INKLING_CONTROL_TOKENS))
for length in range(min(len(text), max_token_len - 1), 0, -1):
suffix = text[-length:]
if any(
len(suffix) < len(token) and token.startswith(suffix)
for token in _INKLING_CONTROL_TOKENS
):
return length
return 0
def _parse_blocks(self, text: str) -> StreamingParseResult:
reasoning: list[str] = []
content: list[str] = []
saw_control = False
pos = 0
def emit(text: str) -> None:
if self._kind == "reasoning":
if self.stream_reasoning:
reasoning.append(text)
else:
self._pending_reasoning += text
elif self._kind == "content":
content.append(text)
elif self._kind == "tool":
content.append(text)
elif self._kind == "header":
self._pending_header += text
elif text:
# No open block — e.g. a continue_final_message stream resuming
# mid text block. Route to visible content, matching the
# no-control-token path below.
content.append(text)
def flush_reasoning() -> None:
if self._kind == "reasoning" and not self.stream_reasoning:
reasoning.append(self._pending_reasoning)
self._pending_reasoning = ""
for match in _INKLING_CONTROL_RE.finditer(text):
saw_control = True
emit(text[pos : match.start()])
token = match.group(0)
pos = match.end()
if token == MESSAGE_MODEL:
if self._kind in (None, "header"):
flush_reasoning()
self._pending_header = ""
self._kind = "header"
else:
# Inside an open block a decoded <|message_model|> string
# is payload the model wrote (e.g. quoting the protocol) —
# a real header can only follow an end token. Preserve it
# instead of rerouting the rest of the block into a header.
emit(token)
elif token in (CONTENT_INVOKE_TOOL_JSON, CONTENT_INVOKE_TOOL_TEXT):
# Preserve the tool-invocation framing (json and headerless raw
# text) in content so the tool-call detector receives it.
flush_reasoning()
if self._kind == "header":
content.extend((MESSAGE_MODEL, self._pending_header, token))
self._pending_header = ""
else:
content.append(token)
self._kind = "tool"
elif self._kind == "tool":
content.append(token)
if token in _INKLING_END_TOKENS:
self._kind = None
elif token in _INKLING_CONTENT_KINDS:
flush_reasoning()
self._pending_header = ""
self._kind = _INKLING_CONTENT_KINDS[token]
elif token in _INKLING_END_TOKENS:
flush_reasoning()
self._pending_header = ""
self._kind = None
tail = text[pos:]
if saw_control or self._kind is not None:
emit(tail)
else:
content.append(text)
return StreamingParseResult(
normal_text="".join(content),
reasoning_text="".join(reasoning),
)
class _DeepSeekV3Detector(Qwen3Detector):
"""DeepSeek-V3 reuses Qwen3 tokens but requires explicit thinking=True to enable."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reasoning_default = "explicit_thinking"
class DeepSeekV4Detector(BaseReasoningFormatDetector):
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
dsv4_thinking_start_token,
dsv4_thinking_end_token,
think_excluded_tokens=[dsv4_eos_token, dsv4_dsml_token],
# Leading "<" included: has_tool_call() matches on it.
tool_start_token=f"<{dsv4_dsml_token}",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
thinks_internally=True,
reasoning_default="explicit_thinking",
force_nonempty_content=force_nonempty_content,
)
class _MimoDetector(Qwen3Detector):
"""MIMO reuses Qwen3 tokens but requires explicit enable_thinking=True to enable."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reasoning_default = "explicit_enable_thinking"
class _PoolsideV1Detector(Qwen3Detector):
"""Poolside v1 (Laguna-XS.2) reuses Qwen3 <think> tokens but the HF chat template
defaults `enable_thinking=False`; reasoning is opt-in via `enable_thinking=True`."""
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.reasoning_default = "explicit_enable_thinking"
class Apertus2509Detector(BaseReasoningFormatDetector):
"""
Detector for Apertus 2509 models
Reasoning blocks are delimited by:
<|inner_prefix|> ... <|inner_suffix|>
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
super().__init__(
"<|inner_prefix|>",
"<|inner_suffix|>",
force_reasoning=False,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self._force_reasoning = force_reasoning
self._tool_start_token = "<|tools_prefix|>["
self._tool_end_token = "<|tools_suffix|>"
self._reasoning_acc: str = ""
self._in_inner_tool: bool = False
def detect_and_parse(self, text: str) -> StreamingParseResult:
blocks = self.detect_and_parse_block_sequence(text)
reasoning_parts = [t for k, t in blocks if k == "reasoning"]
text_parts = [t for k, t in blocks if k == "text"]
ret = StreamingParseResult(
normal_text="".join(text_parts),
reasoning_text="".join(reasoning_parts),
)
return self._maybe_apply_force_nonempty_content(ret)
def detect_and_parse_block_sequence(self, text: str) -> list[tuple[str, str]]:
"""Return an ordered sequence of blocks: [("reasoning"|"text", content), ...]"""
start_tok = self.think_start_token
end_tok = self.think_end_token
blocks: list[tuple[str, str]] = []
cursor = 0
# continue_final_message can resume inside an existing inner
if self._in_reasoning:
if (e := text.find(end_tok, cursor)) == -1:
blocks.extend(self._split_inner_reasoning(text[cursor:]))
blocks.append(("text", ""))
return blocks
blocks.extend(self._split_inner_reasoning(text[cursor:e]))
cursor = e + len(end_tok)
while True:
if (s := text.find(start_tok, cursor)) == -1:
# Always include the trailing text block (may be empty)
blocks.append(("text", text[cursor:]))
break
if s > cursor:
blocks.append(("text", text[cursor:s]))
cursor = s + len(start_tok)
if (e := text.find(end_tok, cursor)) == -1:
blocks.extend(self._split_inner_reasoning(text[cursor:]))
blocks.append(("text", ""))
break
blocks.extend(self._split_inner_reasoning(text[cursor:e]))
cursor = e + len(end_tok)
last_idx = len(blocks) - 1
blocks = [
(k, t)
for i, (k, t) in enumerate(blocks)
if not (k == "text" and t == "" and i != last_idx)
]
return blocks
def _split_inner_reasoning(self, inner_text: str) -> list[tuple[str, str]]:
"""
Split content inside <|inner_prefix|>...<|inner_suffix|> into:
- ("reasoning", <thoughts text>)
- ("text", <|tools_prefix|>[...]<|tools_suffix|>) for any tool calls inside reasoning
"""
tool_start = self._tool_start_token
tool_end = self._tool_end_token
out: list[tuple[str, str]] = []
cursor = 0
while True:
if (s := inner_text.find(tool_start, cursor)) == -1:
if (tail := inner_text[cursor:]) != "":
out.append(("reasoning", tail))
break
if s > cursor:
out.append(("reasoning", inner_text[cursor:s]))
if (e := inner_text.find(tool_end, s)) == -1:
out.append(("text", inner_text[s:]))
break
out.append(("text", inner_text[s : e + len(tool_end)]))
cursor = e + len(tool_end)
return out
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
self._buffer += new_text
out_reasoning = ""
out_normal = ""
start_tok = self.think_start_token
end_tok = self.think_end_token
tool_start = self._tool_start_token
tool_end = self._tool_end_token
while True:
if not self._in_reasoning:
if (s := self._buffer.find(start_tok)) == -1:
if partial := self._ends_with_partial_token(
self._buffer, start_tok
):
out_normal += self._buffer[:-partial]
self._buffer = self._buffer[-partial:]
else:
out_normal += self._buffer
self._buffer = ""
return StreamingParseResult(
normal_text=out_normal, reasoning_text=out_reasoning
)
out_normal += self._buffer[:s]
self._buffer = self._buffer[s + len(start_tok) :]
self._in_reasoning = True
self._reasoning_acc = ""
self._in_inner_tool = False
continue
if self._in_inner_tool:
if (end_pos := self._buffer.find(tool_end)) == -1:
if (
hold := self._ends_with_partial_token(self._buffer, tool_end)
) != 0:
out_normal += self._buffer[:-hold]
self._buffer = self._buffer[-hold:]
else:
out_normal += self._buffer
self._buffer = ""
return StreamingParseResult(
normal_text=out_normal, reasoning_text=out_reasoning
)
out_normal += self._buffer[: end_pos + len(tool_end)]
self._buffer = self._buffer[end_pos + len(tool_end) :]
self._in_inner_tool = False
continue
pos_tool = self._buffer.find(tool_start)
pos_end = self._buffer.find(end_tok)
if pos_tool == -1 and pos_end == -1:
if self.stream_reasoning:
if (
hold := max(
self._ends_with_partial_token(self._buffer, end_tok),
self._ends_with_partial_token(self._buffer, tool_start),
)
) != 0:
out_reasoning += self._buffer[:-hold]
self._buffer = self._buffer[-hold:]
else:
out_reasoning += self._buffer
self._buffer = ""
return StreamingParseResult(
normal_text=out_normal, reasoning_text=out_reasoning
)
next_pos = min(p for p in [pos_tool, pos_end] if p != -1)
if pos_end != -1 and pos_end == next_pos:
reasoning_chunk = self._buffer[:pos_end]
if self.stream_reasoning:
out_reasoning += reasoning_chunk
else:
self._reasoning_acc += reasoning_chunk
out_reasoning += self._reasoning_acc
self._reasoning_acc = ""
self._buffer = self._buffer[pos_end + len(end_tok) :]
self._in_reasoning = False
continue
reasoning_chunk = self._buffer[:pos_tool]
if self.stream_reasoning:
out_reasoning += reasoning_chunk
else:
self._reasoning_acc += reasoning_chunk
self._buffer = self._buffer[pos_tool:]
self._in_inner_tool = True
continue
class CohereCommand4Detector(BaseReasoningFormatDetector):
"""Detector for Cohere Command4 / Command-A family (incl. cohere2_moe and
cohere2_vision Command-A-Plus).
Generated format (the assistant prefix in the chat template already emits
``<|START_THINKING|>`` when ``reasoning=True``, so the *generated* text
typically begins inside the thinking block):
thinking_content<|END_THINKING|><|START_TEXT|>final_answer<|END_TEXT|>
When ``reasoning=False`` the chat template emits both START/END_THINKING
in the prefix and the generated text is just::
<|START_TEXT|>final_answer<|END_TEXT|>
This detector returns:
- ``reasoning_text`` = the thinking block (between START_THINKING and
END_THINKING, with the START tag stripped if the model echoed it).
- ``normal_text`` = the content between ``<|START_TEXT|>`` and
``<|END_TEXT|>``, with both markers stripped. If no ``<|START_TEXT|>``
appears (the model exhausted max_new_tokens still inside thinking),
``normal_text`` is the empty string.
Matches the public token names from the model's
``special_tokens_map.json`` (``<|START_THINKING|>`` etc.).
"""
TEXT_START_TOKEN = "<|START_TEXT|>"
TEXT_END_TOKEN = "<|END_TEXT|>"
# When the model decides to call tools instead of producing a final text
# block, it emits an action block instead of a text block. The reasoning
# parser must leave that block intact so the downstream tool-call parser
# can pick it up.
ACTION_START_TOKEN = "<|START_ACTION|>"
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = True,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
):
# The chat template puts <|START_THINKING|> in the assistant prefix
# when reasoning is enabled, so the *generated* text usually starts
# already inside thinking. ``force_reasoning=True`` makes the base
# detector treat the leading bytes as reasoning even though the
# generated stream typically does not echo <|START_THINKING|>.
super().__init__(
think_start_token="<|START_THINKING|>",
think_end_token="<|END_THINKING|>",
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
# Streaming state machine. The model emits, in order:
# 1. reasoning (between START_THINKING [in prefix] and END_THINKING)
# 2. either ``<|START_TEXT|>...<|END_TEXT|>`` (final answer) or
# ``<|START_ACTION|>...<|END_ACTION|>`` (tool calls) -- never both.
# When ``reasoning=False`` the chat template emits both START/END
# thinking in the prefix and step 1 is empty; the generated stream
# then starts directly with the text or action block.
self._reasoning_done = False
self._saw_text_start = False
self._saw_text_end = False
self._in_action_mode = False
@classmethod
def _strip_text_markers(cls, raw: str) -> str:
"""Extract the substring between ``<|START_TEXT|>`` and
``<|END_TEXT|>``. If ``<|START_TEXT|>`` is absent but a
``<|START_ACTION|>`` block is present, the model produced a tool
call instead of a text answer -- return the raw text untouched so
the downstream tool-call parser can pick up the action block. If
neither marker is present (ran out of tokens still inside
thinking) return ``""``. If ``<|END_TEXT|>`` is absent (stop token
or max_new_tokens cut the stream off inside the text block) return
everything after ``<|START_TEXT|>``.
"""
if not raw:
return ""
s = raw.find(cls.TEXT_START_TOKEN)
if s == -1:
if cls.ACTION_START_TOKEN in raw:
return raw
return ""
s += len(cls.TEXT_START_TOKEN)
tail = raw[s:]
e = tail.find(cls.TEXT_END_TOKEN)
if e == -1:
return tail
return tail[:e]
def detect_and_parse(self, text: str) -> StreamingParseResult:
# Direct parse: split on the (single) ``<|END_THINKING|>`` token if
# present. Anything before is reasoning, anything after is the
# final-text block. If no END_THINKING but a START_TEXT exists,
# we're in the reasoning=False case (chat template emitted both
# START/END thinking in the prefix; the model only generated the
# text block). Otherwise the model exhausted tokens still thinking
# and ``normal_text`` ends up empty -- matching the convention of
# the other detectors in this module (DeepSeekR1, Qwen3, ...). The
# empty content is propagated as ``message.content = None`` by
# serving_chat, and downstream code is expected to treat that as
# "no answer" rather than falling back to ``reasoning_content``.
end_think_idx = text.find(self.think_end_token)
text_start_idx = text.find(self.TEXT_START_TOKEN)
action_start_idx = text.find(self.ACTION_START_TOKEN)
if end_think_idx != -1:
reasoning = text[:end_think_idx]
rest = text[end_think_idx + len(self.think_end_token) :]
elif text_start_idx != -1:
reasoning = text[:text_start_idx]
rest = text[text_start_idx:]
elif action_start_idx != -1:
# reasoning=False + tool call: chat template emitted both
# START/END thinking in the prefix, the model only generated
# an action block. Treat the prefix before the action block as
# (probably empty) reasoning so the action block reaches the
# tool-call parser intact.
reasoning = text[:action_start_idx]
rest = text[action_start_idx:]
else:
reasoning = text
rest = ""
# Some checkpoints echo the START_THINKING token even though the
# chat template put it in the prefix; drop it if so.
think_start_text = self.think_start_token + self.think_start_self_label
if reasoning.startswith(think_start_text):
reasoning = reasoning[len(think_start_text) :]
return self._maybe_apply_force_nonempty_content(
StreamingParseResult(
normal_text=self._strip_text_markers(rest),
reasoning_text=reasoning,
)
)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
"""Streaming parse. Custom state machine -- we don't reuse the base
class because Cohere's "reasoning=False" path (the model emits no
``<|END_THINKING|>``, just goes straight to a text or action block)
is fundamentally incompatible with the base detector's
``force_reasoning`` semantics."""
self._buffer += new_text
buf = self._buffer
if not self._reasoning_done:
# Look for any marker that ends reasoning: an explicit
# END_THINKING, or an implicit transition via the start of the
# final-text or action block (reasoning=False case).
markers = (
(self.think_end_token, "think_end"),
(self.TEXT_START_TOKEN, "text"),
(self.ACTION_START_TOKEN, "action"),
)
first_pos = None
first_marker = None
first_kind = None
for marker_text, kind in markers:
p = buf.find(marker_text)
if p != -1 and (first_pos is None or p < first_pos):
first_pos, first_marker, first_kind = p, marker_text, kind
if first_pos is None:
# No marker seen yet. Stream the reasoning prefix, but keep
# enough tail in the buffer to recognise a marker split
# across chunk boundaries.
if not self.stream_reasoning:
return StreamingParseResult()
max_keep = max(len(m) for m, _ in markers) - 1
if len(buf) > max_keep:
head = buf[:-max_keep]
self._buffer = buf[-max_keep:]
return StreamingParseResult(reasoning_text=head)
return StreamingParseResult()
reasoning_chunk = buf[:first_pos]
if first_kind == "think_end":
self._buffer = buf[first_pos + len(first_marker) :]
else:
# Implicit reasoning-end: leave the start-of-block marker in
# the buffer for the post-thinking branch below to consume.
self._buffer = buf[first_pos:]
self._reasoning_done = True
if reasoning_chunk:
return StreamingParseResult(reasoning_text=reasoning_chunk)
buf = self._buffer
# Reasoning is closed. Decide between text-stripping and
# action-passthrough on first sight of a marker.
if self._in_action_mode:
if not buf:
return StreamingParseResult()
self._buffer = ""
return StreamingParseResult(normal_text=buf)
if not self._saw_text_start:
s_text = buf.find(self.TEXT_START_TOKEN)
s_action = buf.find(self.ACTION_START_TOKEN)
picks = [
(p, k) for p, k in ((s_text, "text"), (s_action, "action")) if p != -1
]
if not picks:
max_keep = (
max(len(self.TEXT_START_TOKEN), len(self.ACTION_START_TOKEN)) - 1
)
if len(buf) > max_keep:
self._buffer = buf[-max_keep:]
return StreamingParseResult()
picks.sort()
first_pos, first_kind = picks[0]
if first_kind == "action":
self._in_action_mode = True
out_normal = buf[first_pos:]
self._buffer = ""
return StreamingParseResult(normal_text=out_normal)
# Found <|START_TEXT|>. Drop everything up to and including the
# marker -- text content streams next.
self._buffer = buf[first_pos + len(self.TEXT_START_TOKEN) :]
self._saw_text_start = True
buf = self._buffer
if self._saw_text_start and not self._saw_text_end:
e = buf.find(self.TEXT_END_TOKEN)
if e == -1:
# Emit everything except a possible partial END_TEXT tail.
keep = len(self.TEXT_END_TOKEN) - 1
if len(buf) > keep:
out_normal = buf[:-keep]
self._buffer = buf[-keep:]
return StreamingParseResult(normal_text=out_normal)
return StreamingParseResult()
out_normal = buf[:e]
self._buffer = buf[e + len(self.TEXT_END_TOKEN) :]
self._saw_text_end = True
return StreamingParseResult(normal_text=out_normal)
return StreamingParseResult()
def finish(self) -> StreamingParseResult:
# _in_reasoning stays pinned True here (phase tracked via _reasoning_done), so
# the base finish() would misfile a truncated answer tail as reasoning.
buffer = self._buffer
self._buffer = ""
if not self._reasoning_done:
ret = StreamingParseResult(
reasoning_text=self._strip_leading_think_start(buffer)
)
elif self._saw_text_start and not self._saw_text_end:
ret = StreamingParseResult(normal_text=buffer)
else:
return StreamingParseResult()
return self._maybe_apply_force_nonempty_content(ret)
class MuseGlimmerDetector(BaseReasoningFormatDetector):
"""Detector for Muse Glimmer's recipient-channel format.
The chat template ends the generation prompt at ``<|start|>assistant`` with no
recipient and no ``<|message|>``, so the model itself emits the channel header as
ordinary text. A full turn looks like::
" to=self<|message|>" <reasoning> "<|eom|>"
"<|start|>assistant to=user<|message|>" <answer> "<|eot|>"
Reasoning is the ``to=self`` channel; the answer is ``to=user``. Any other recipient
is a tool call (``to=functions.get_weather``), whose body is an ATEM block that must
reach the function-call detector with its markers intact — so those channels are
emitted as normal text including their header, following GptOssDetector's precedent
of preserving raw structural text for tool calls.
When a tool-call parser consumes this detector's normal text
(``tool_call_parser_active=True``), the ``to=user`` channel keeps its framing too,
so the downstream detector sees every channel boundary and can tell a real tool
channel from one merely *quoted* inside the answer — unwrapping here would make a
quoted ``<|start|>assistant to=<tool><|message|>`` indistinguishable from a real
header and turn quoted markup into a live call. The tool detector unwraps
``to=user`` itself, so nothing framed leaks to the client. Non-streaming
additionally requires that a turn *without* any ATEM block come out unwrapped,
because serving bypasses the tool detector entirely when ``has_tool_call()`` is
false — hence the ATEM-presence branch in ``detect_and_parse``, mirroring the
vendor's reference reasoning parser.
Keying on ``<|message|>`` rather than the literal " to=self" mirrors the vendor's own
reference implementation (which slices past the last ``<|message|>`` token),
and is robust to the header varying with
the recipient. It does require the delimiters to survive detokenization, which is why
``muse`` is registered in ``_patch_reasoning_skip_special_tokens``.
A single channel may also be cut short by the token cap, in which case there is no
terminator and the partial body is still attributed to whichever channel was open.
"""
def __init__(
self,
stream_reasoning: bool = True,
force_reasoning: bool = False,
continue_final_message: bool = False,
previous_content: str = "",
force_nonempty_content: bool = False,
tool_call_parser_active: bool = False,
):
super().__init__(
" to=self" + MESSAGE,
EOM,
force_reasoning=force_reasoning,
stream_reasoning=stream_reasoning,
continue_final_message=continue_final_message,
previous_content=previous_content,
force_nonempty_content=force_nonempty_content,
)
self._recipient: Optional[str] = None
self._in_body = False
self._at_stream_start = True
self._pending_reasoning = ""
self._tool_call_parser_active = tool_call_parser_active
self._saw_reasoning_block = False
def _sink(self, recipient: Optional[str]) -> str:
return "reasoning" if recipient == "self" else "normal"
def _consume(self, flush: bool, preserve_channels: bool = False) -> Tuple[str, str]:
"""Drain self._buffer into (reasoning, normal).
With flush=False, holds back a short tail that could be the prefix of a marker
split across chunk boundaries; with flush=True, emits everything.
With preserve_channels=True, the ``to=user`` channel keeps its header and
terminator like tool channels do (see the class docstring for why the
function-call detector needs the framing intact); reasoning is always
extracted and never framed.
"""
reasoning_parts: List[str] = []
normal_parts: List[str] = []
while self._buffer:
if not self._in_body:
# Without this, unframed prose never streams: it buffers
# forever waiting for a <|message|> that never arrives.
if not (self._at_stream_start and could_start_header(self._buffer)):
ws = len(self._buffer) - len(self._buffer.lstrip())
head = self._buffer[ws : ws + len(START)]
if not START.startswith(head):
self._in_body = True
self._recipient = None
self._at_stream_start = False
continue
if ws:
normal_parts.append(self._buffer[:ws])
self._buffer = self._buffer[ws:]
if len(head) < len(START):
break
idx = self._buffer.find(MESSAGE)
if idx == -1:
if flush:
normal_parts.append(self._buffer)
self._buffer = ""
break
self._at_stream_start = False
header = self._buffer[:idx]
m = RECIPIENT_RE.search(header)
self._recipient = m.group(1) if m else "user"
self._buffer = self._buffer[idx + len(MESSAGE) :]
self._in_body = True
if self._sink(self._recipient) == "reasoning":
if self._saw_reasoning_block:
reasoning_parts.append("\n")
self._saw_reasoning_block = True
elif self._recipient != "user" or preserve_channels:
# Keep the header so the function-call detector sees it.
normal_parts.append(header + MESSAGE)
continue
end_idx, end_tok = -1, ""
for tok in (EOM, EOT):
i = self._buffer.find(tok)
if i != -1 and (end_idx == -1 or i < end_idx):
end_idx, end_tok = i, tok
if end_idx != -1:
body = self._buffer[:end_idx]
self._buffer = self._buffer[end_idx + len(end_tok) :]
self._in_body = False
if self._sink(self._recipient) == "reasoning":
reasoning_parts.append(body)
else:
normal_parts.append(body)
if self._recipient != "user" or preserve_channels:
normal_parts.append(end_tok)
self._recipient = None
continue
# Hold back only a genuine marker prefix.
if flush:
body, self._buffer = self._buffer, ""
else:
keep = partial_marker_len(
self._buffer, (EOM, EOT, START), MAX_CHANNEL_MARKER
)
if keep == len(self._buffer):
break
body = self._buffer[: len(self._buffer) - keep]
self._buffer = self._buffer[len(self._buffer) - keep :]
if not body:
break
if self._sink(self._recipient) == "reasoning":
reasoning_parts.append(body)
else:
normal_parts.append(body)
return "".join(reasoning_parts), "".join(normal_parts)
def detect_and_parse(self, text: str) -> StreamingParseResult:
self._buffer += text
raw = self._buffer
reasoning, normal = self._consume(flush=True)
if self._tool_call_parser_active and has_atem_markers(normal):
self._buffer = raw
self._recipient = None
self._in_body = False
self._at_stream_start = True
self._saw_reasoning_block = False
reasoning, normal = self._consume(flush=True, preserve_channels=True)
return self._maybe_apply_force_nonempty_content(
StreamingParseResult(normal_text=normal, reasoning_text=reasoning)
)
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
self._buffer += new_text
reasoning, normal = self._consume(
flush=False, preserve_channels=self._tool_call_parser_active
)
if not self.stream_reasoning:
self._pending_reasoning += reasoning
reasoning = ""
if not self._in_body and self._pending_reasoning:
reasoning, self._pending_reasoning = self._pending_reasoning, ""
if self._force_nonempty_content:
# Kept so finish() can promote it to content if the turn produces
# none. Dropped on real content, NOT when the channel closes --
# <|eom|> lands in the same chunk as the last reasoning text.
self._accumulated_reasoning += reasoning
if normal:
self._accumulated_reasoning = ""
return StreamingParseResult(normal_text=normal, reasoning_text=reasoning)
def finish(self) -> StreamingParseResult:
reasoning, normal = self._consume(
flush=True, preserve_channels=self._tool_call_parser_active
)
if self._pending_reasoning:
reasoning = self._pending_reasoning + reasoning
self._pending_reasoning = ""
if self._force_nonempty_content:
promoted = self._accumulated_reasoning + reasoning
self._accumulated_reasoning = ""
if not normal and promoted:
return StreamingParseResult(normal_text=promoted)
return StreamingParseResult(normal_text=normal, reasoning_text=reasoning)
class ReasoningParser:
"""
Parser that handles both streaming and non-streaming scenarios for extracting
reasoning content from model outputs.
Args:
model_type (str): Type of model to parse reasoning from
stream_reasoning (bool): If False, accumulates reasoning content until complete.
If True, streams reasoning content as it arrives.
tool_call_parser_active (bool): True when this parser's normal text feeds a
function-call parser rather than going straight to the client. Passed on
to detectors that accept it (channel-framed formats keep tool framing
intact for the downstream detector).
"""
DetectorMap: Dict[str, Type[BaseReasoningFormatDetector]] = {
"apertus2509": Apertus2509Detector,
"deepseek-r1": DeepSeekR1Detector,
"deepseek-v3": _DeepSeekV3Detector,
"deepseek-v4": DeepSeekV4Detector,
"dots": Qwen3Detector,
"glm45": Glm45Detector,
"ling3": Ling3Detector,
"hunyuan": HunyuanDetector,
"gpt-oss": GptOssDetector,
"k2_horizon": K2V3Detector,
"kimi": KimiDetector,
"kimi_k2": KimiK2Detector,
"kimi_k3": KimiK3Detector,
"mimo": _MimoDetector,
"muse": MuseGlimmerDetector,
"poolside_v1": _PoolsideV1Detector,
"qwen3": Qwen3Detector,
"qwen3-thinking": Qwen3Detector,
"minimax": Qwen3Detector,
"minimax-append-think": MiniMaxAppendThinkDetector,
"minimax-m3": MiniMaxM3Detector,
"nanbeige": Qwen3Detector,
"step3": DeepSeekR1Detector,
"step3p5": DeepSeekR1Detector,
"mistral": MistralDetector,
"nemotron_3": Nemotron3Detector,
"interns1": Qwen3Detector,
"gemma4": Gemma4Detector,
"inkling": InklingDetector,
"cohere_command4": CohereCommand4Detector,
}
def __init__(
self,
model_type: Optional[str] = None,
stream_reasoning: bool = True,
force_reasoning: Optional[bool] = None,
request: ChatCompletionRequest = None,
tokenizer=None,
tool_call_parser_active: bool = False,
):
if not model_type:
raise ValueError("Model type must be specified")
detector_class = self.DetectorMap.get(model_type.lower())
if not detector_class:
raise ValueError(f"Unsupported model type: {model_type}")
chat_template_kwargs = getattr(request, "chat_template_kwargs", None) or {}
# Special cases where we override force_reasoning
if model_type.lower() in {
"qwen3-thinking",
"gpt-oss",
"minimax",
}:
force_reasoning = True
# M3 consumes the <mm:think> start tag only for thinking_mode=enabled
# (absent from output → must force); mirror serving_chat's M3 branch.
if model_type.lower() == "minimax-m3" and force_reasoning is None:
force_reasoning = chat_template_kwargs.get("thinking_mode") == "enabled"
# Only pass force_reasoning if explicitly set, let detectors use their defaults
kwargs = {"stream_reasoning": stream_reasoning}
if force_reasoning is not None:
kwargs["force_reasoning"] = force_reasoning
if (
request is not None
and isinstance(request, ChatCompletionRequest)
and request.continue_final_message
and request.messages[-1].role == "assistant"
):
kwargs["continue_final_message"] = True
kwargs["previous_content"] = request.messages[-1].content
if chat_template_kwargs.get("force_nonempty_content") is True:
kwargs["force_nonempty_content"] = True
if model_type.lower() == "k2_horizon":
# Template kwargs are the final values passed to Jinja and therefore
# take precedence over the convenience fields on API requests.
effort = chat_template_kwargs.get("reasoning_effort")
if effort is None:
effort = getattr(request, "reasoning_effort", None)
if effort is None:
# The Responses API carries the same value in its standard
# nested shape (``reasoning.effort``). Prompt rendering already
# mirrors it into a ChatCompletionRequest; parsing must select
# the matching IFM delimiter as well.
reasoning = getattr(request, "reasoning", None)
effort = getattr(reasoning, "effort", None)
if effort is not None:
kwargs["reasoning_effort"] = effort
if tokenizer is not None:
sig = inspect.signature(detector_class)
if "tokenizer" in sig.parameters:
kwargs["tokenizer"] = tokenizer
if tool_call_parser_active:
sig = inspect.signature(detector_class)
if "tool_call_parser_active" in sig.parameters:
kwargs["tool_call_parser_active"] = True
self.detector = detector_class(**kwargs)
def parse_non_stream(self, full_text: str) -> Tuple[Optional[str], Optional[str]]:
"""Non-streaming call: one-time parsing"""
ret = self.detector.detect_and_parse(full_text)
return ret.reasoning_text, ret.normal_text
def parse_non_stream_blocks(self, full_text: str) -> list[dict]:
"""Non-streaming call: return an ordered sequence of reasoning/text blocks"""
if hasattr(self.detector, "detect_and_parse_block_sequence"):
seq = self.detector.detect_and_parse_block_sequence(full_text)
return [{"type": k, "text": t} for k, t in seq]
ret = self.detector.detect_and_parse(full_text)
blocks: list[dict] = []
if ret.reasoning_text:
blocks.append({"type": "reasoning", "text": ret.reasoning_text})
blocks.append({"type": "text", "text": ret.normal_text or ""})
return blocks
def parse_stream_chunk(
self, chunk_text: str
) -> Tuple[Optional[str], Optional[str]]:
"""Streaming call: incremental parsing"""
ret = self.detector.parse_streaming_increment(chunk_text)
return ret.reasoning_text, ret.normal_text
def parse_stream_end(self) -> Tuple[Optional[str], Optional[str]]:
"""Streaming call: flush any detector-specific buffered state once
the stream ends."""
ret = self.detector.finish()
return ret.reasoning_text, ret.normal_text
Reference in New Issue View Git Blame Copy Permalink