Add Inkling model support (#31681)
Co-authored-by: Chunan Zeng <zcnrex@gmail.com> Co-authored-by: Ke Bao <ispobaoke@gmail.com> Co-authored-by: Yanbin Jiang <jybsuper@gmail.com> Co-authored-by: Yuhao Yang <47235274+yhyang201@users.noreply.github.com> Co-authored-by: Qiaolin Yu <qiaolin.yu@radixark.ai> Co-authored-by: Zhichen Zeng <zczeng@uw.edu> Co-authored-by: Aurick Qiao <aurick@thinkingmachines.ai> Co-authored-by: Joseph <jk@thinkingmachines.ai>
This commit is contained in:
Cheng Wan
co-authored by
Chunan Zeng
Ke Bao
Yanbin Jiang
Yuhao Yang
Qiaolin Yu
Zhichen Zeng
Aurick Qiao
Joseph
Chunan Zeng
Ke Bao
Yanbin Jiang
Yuhao Yang
Qiaolin Yu
Zhichen Zeng
Aurick Qiao
Joseph
parent
829e9ce9d5
commit
02236fa38c
@@ -0,0 +1,333 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any, Protocol
|
||||
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
AUDIO_END,
|
||||
AUDIO_TOKEN_ID,
|
||||
CONTENT_AUDIO_INPUT,
|
||||
CONTENT_IMAGE,
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
CONTENT_XML,
|
||||
END_MESSAGE,
|
||||
IMAGE_TOKEN_ID,
|
||||
MESSAGE_MODEL,
|
||||
ROLE_MESSAGE_TOKENS,
|
||||
)
|
||||
|
||||
|
||||
class InklingTextTokenizer(Protocol):
|
||||
def encode_text(self, text: str) -> list[int]: ...
|
||||
|
||||
def encode_special(self, token: str) -> int: ...
|
||||
|
||||
|
||||
# OpenAI content-part type spellings that mean image / audio (render only needs the kind,
|
||||
# not the bytes — the bytes are encoded later in the MM processor).
|
||||
_IMAGE_PART_TYPES = frozenset({"image", "input_image", "image_url"})
|
||||
_AUDIO_PART_TYPES = frozenset({"audio", "input_audio", "audio_url"})
|
||||
INKLING_DEFAULT_REASONING_EFFORT = 0.9
|
||||
|
||||
|
||||
def render_inkling_messages(
|
||||
messages: Sequence[Mapping[str, Any]],
|
||||
tokenizer: InklingTextTokenizer,
|
||||
*,
|
||||
add_generation_prompt: bool = False,
|
||||
tools: Sequence[Mapping[str, Any]] | None = None,
|
||||
reasoning_effort: float | None = None,
|
||||
) -> list[int]:
|
||||
"""Render chat messages to Inkling input_ids with ONE placeholder per media item.
|
||||
|
||||
PURE renderer: emits Inkling framing + a single IMAGE_TOKEN_ID / AUDIO_TOKEN_ID
|
||||
per image / audio part. Media encoding and 1->N placeholder expansion happen
|
||||
later in the MM processor. Inkling normally emits its own assistant turn
|
||||
opener; ``add_generation_prompt`` is retained only for legacy callers. The
|
||||
conversation-level effort directive is emitted once in the initial prefix
|
||||
and defaults to 0.9.
|
||||
"""
|
||||
input_ids: list[int] = []
|
||||
tool_call_id_to_name: dict[str, str] = {}
|
||||
|
||||
if tools:
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"system",
|
||||
"xml",
|
||||
_tool_declare_json(tools),
|
||||
author_name="tool_declare",
|
||||
)
|
||||
|
||||
# Normalize the OpenAI "developer" role to "system" (as the Responses API
|
||||
# does in _normalize_response_message_for_chat) so developer-instruction
|
||||
# messages render instead of tripping _expect_role; the leading-system
|
||||
# grouping below then also sees them. Shallow-copy only affected messages so
|
||||
# the caller's list is left untouched.
|
||||
message_list = [
|
||||
{**message, "role": "system"} if message.get("role") == "developer" else message
|
||||
for message in messages
|
||||
]
|
||||
leading_system_count = 0
|
||||
for message in message_list:
|
||||
if message.get("role") != "system":
|
||||
break
|
||||
leading_system_count += 1
|
||||
|
||||
def append_effort() -> None:
|
||||
effort = (
|
||||
INKLING_DEFAULT_REASONING_EFFORT
|
||||
if reasoning_effort is None
|
||||
else reasoning_effort
|
||||
)
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"system",
|
||||
"text",
|
||||
f"Thinking effort level: {_format_reasoning_effort(effort)}",
|
||||
)
|
||||
|
||||
for message_index, message in enumerate(message_list):
|
||||
if message_index == leading_system_count:
|
||||
append_effort()
|
||||
role = _expect_role(message)
|
||||
if role == "tool":
|
||||
tool_name = message.get("name") or tool_call_id_to_name.get(
|
||||
message.get("tool_call_id") or "", ""
|
||||
)
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"tool",
|
||||
"text",
|
||||
_expect_string_content(message.get("content", "")),
|
||||
author_name=str(tool_name),
|
||||
)
|
||||
continue
|
||||
|
||||
parts = list(_iter_render_parts(message.get("content", "")))
|
||||
turn_start = len(input_ids)
|
||||
if role == "assistant":
|
||||
reasoning_content = message.get("reasoning_content")
|
||||
if reasoning_content:
|
||||
if not isinstance(reasoning_content, str):
|
||||
raise TypeError(
|
||||
"assistant reasoning_content must be a string for Inkling rendering"
|
||||
)
|
||||
if any(kind == "thinking" for kind, _ in parts):
|
||||
raise ValueError(
|
||||
"assistant message cannot mix reasoning_content with ordered thinking parts"
|
||||
)
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"assistant",
|
||||
"thinking",
|
||||
reasoning_content,
|
||||
)
|
||||
|
||||
for kind, text in parts:
|
||||
if kind == "thinking" and role != "assistant":
|
||||
raise ValueError("Inkling thinking parts require role='assistant'")
|
||||
_append_message(input_ids, tokenizer, role, kind, text)
|
||||
|
||||
if role == "assistant":
|
||||
for tool_call in message.get("tool_calls") or []:
|
||||
name, args = _tool_call_name_and_args(tool_call)
|
||||
tool_call_id = _as_mapping(tool_call).get("id")
|
||||
if tool_call_id:
|
||||
tool_call_id_to_name[str(tool_call_id)] = name
|
||||
_append_message(
|
||||
input_ids,
|
||||
tokenizer,
|
||||
"assistant",
|
||||
"invoke_tool_json",
|
||||
_tool_call_json(name, args),
|
||||
author_name=name,
|
||||
)
|
||||
if len(input_ids) > turn_start:
|
||||
# Close the historical model turn — but never emit a bare
|
||||
# terminator for an assistant message that rendered no blocks.
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_MODEL_END_SAMPLING))
|
||||
|
||||
if leading_system_count == len(message_list):
|
||||
append_effort()
|
||||
|
||||
if add_generation_prompt:
|
||||
input_ids.append(tokenizer.encode_special(MESSAGE_MODEL))
|
||||
return input_ids
|
||||
|
||||
|
||||
def _append_message(
|
||||
input_ids: list[int],
|
||||
tokenizer: InklingTextTokenizer,
|
||||
role: str,
|
||||
kind: str,
|
||||
text: str,
|
||||
*,
|
||||
author_name: str | None = None,
|
||||
) -> None:
|
||||
input_ids.append(tokenizer.encode_special(ROLE_MESSAGE_TOKENS[role]))
|
||||
if author_name:
|
||||
input_ids.extend(tokenizer.encode_text(author_name))
|
||||
|
||||
if kind == "text":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_TEXT))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
elif kind == "image":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_IMAGE))
|
||||
input_ids.append(IMAGE_TOKEN_ID)
|
||||
elif kind == "audio":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_AUDIO_INPUT))
|
||||
input_ids.append(AUDIO_TOKEN_ID)
|
||||
input_ids.append(tokenizer.encode_special(AUDIO_END))
|
||||
elif kind == "thinking":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_THINKING))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
elif kind == "xml":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_XML))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
elif kind == "invoke_tool_json":
|
||||
input_ids.append(tokenizer.encode_special(CONTENT_INVOKE_TOOL_JSON))
|
||||
input_ids.extend(tokenizer.encode_text(text))
|
||||
else:
|
||||
raise ValueError(f"unsupported Inkling render part kind: {kind!r}")
|
||||
|
||||
input_ids.append(tokenizer.encode_special(END_MESSAGE))
|
||||
|
||||
|
||||
def _iter_render_parts(content: Any):
|
||||
"""Yield ordered ``(kind, text)`` pairs from message content."""
|
||||
if content is None:
|
||||
return
|
||||
if isinstance(content, str):
|
||||
if content:
|
||||
yield ("text", content)
|
||||
return
|
||||
if not isinstance(content, Sequence) or isinstance(content, (bytes, bytearray)):
|
||||
raise TypeError("message content must be a string or a sequence of parts")
|
||||
for part in content:
|
||||
if isinstance(part, str):
|
||||
yield ("text", part)
|
||||
continue
|
||||
if not isinstance(part, Mapping):
|
||||
raise TypeError(f"content part must be mapping, got {type(part).__name__}")
|
||||
ptype = part.get("type")
|
||||
if ptype in (None, "text", "input_text"):
|
||||
text = part.get("text", "")
|
||||
yield ("text", text if isinstance(text, str) else "")
|
||||
elif ptype in ("thinking", "reasoning"):
|
||||
text = part.get("thinking")
|
||||
if text is None:
|
||||
text = part.get("text", "")
|
||||
if not isinstance(text, str):
|
||||
raise TypeError("Inkling thinking part payload must be a string")
|
||||
yield ("thinking", text)
|
||||
elif ptype in _IMAGE_PART_TYPES:
|
||||
yield ("image", "")
|
||||
elif ptype in _AUDIO_PART_TYPES:
|
||||
yield ("audio", "")
|
||||
else:
|
||||
raise ValueError(f"unsupported content part type: {ptype!r}")
|
||||
|
||||
|
||||
def _format_reasoning_effort(reasoning_effort: float) -> str:
|
||||
if isinstance(reasoning_effort, bool) or not isinstance(
|
||||
reasoning_effort, (int, float)
|
||||
):
|
||||
raise TypeError("Inkling reasoning_effort must be a number")
|
||||
value = float(reasoning_effort)
|
||||
if not math.isfinite(value) or not 0.0 <= value <= 0.99:
|
||||
raise ValueError("Inkling reasoning_effort must be finite and in [0.0, 0.99]")
|
||||
return f"{round(value, 2):g}"
|
||||
|
||||
|
||||
def _expect_string_content(content: Any) -> str:
|
||||
if content is None:
|
||||
return ""
|
||||
if not isinstance(content, str):
|
||||
raise TypeError(
|
||||
f"message content must be a string for this Inkling role, got {type(content).__name__}"
|
||||
)
|
||||
return content
|
||||
|
||||
|
||||
def _expect_role(message: Mapping[str, Any]) -> str:
|
||||
role = message.get("role")
|
||||
if role not in ROLE_MESSAGE_TOKENS:
|
||||
raise ValueError(
|
||||
f"unsupported Inkling message role {role!r}; expected one of {sorted(ROLE_MESSAGE_TOKENS)}"
|
||||
)
|
||||
return str(role)
|
||||
|
||||
|
||||
def _as_mapping(value: Any) -> Mapping[str, Any]:
|
||||
if isinstance(value, Mapping):
|
||||
return value
|
||||
if hasattr(value, "model_dump"):
|
||||
dumped = value.model_dump()
|
||||
if isinstance(dumped, Mapping):
|
||||
return dumped
|
||||
raise TypeError(f"expected mapping, got {type(value).__name__}")
|
||||
|
||||
|
||||
def _canonical_json(value: Any) -> str:
|
||||
return json.dumps(
|
||||
_sort_json(value),
|
||||
ensure_ascii=False,
|
||||
allow_nan=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _sort_json(value: Any) -> Any:
|
||||
if isinstance(value, Mapping):
|
||||
return {str(key): _sort_json(value[key]) for key in sorted(value)}
|
||||
if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)):
|
||||
return [_sort_json(item) for item in value]
|
||||
return value
|
||||
|
||||
|
||||
def _tool_declare_json(tools: Sequence[Mapping[str, Any]]) -> str:
|
||||
tool_specs = []
|
||||
for tool_value in tools:
|
||||
tool = _as_mapping(tool_value)
|
||||
function = _as_mapping(tool.get("function", {}))
|
||||
tool_specs.append(
|
||||
{
|
||||
"description": function.get("description") or "",
|
||||
"name": function["name"],
|
||||
"parameters": function.get("parameters") or {},
|
||||
"type": tool.get("type", "function"),
|
||||
}
|
||||
)
|
||||
return _canonical_json(tool_specs)
|
||||
|
||||
|
||||
def _tool_call_name_and_args(tool_call_value: Any) -> tuple[str, Mapping[str, Any]]:
|
||||
tool_call = _as_mapping(tool_call_value)
|
||||
function = _as_mapping(tool_call.get("function", {}))
|
||||
name = function.get("name")
|
||||
if not isinstance(name, str):
|
||||
raise TypeError("tool call function name must be a string")
|
||||
|
||||
raw_args = function.get("arguments") or {}
|
||||
if isinstance(raw_args, str):
|
||||
args = json.loads(raw_args) if raw_args else {}
|
||||
else:
|
||||
args = raw_args
|
||||
if not isinstance(args, Mapping):
|
||||
raise TypeError("tool call function arguments must decode to an object")
|
||||
return name, args
|
||||
|
||||
|
||||
def _tool_call_json(name: str, args: Mapping[str, Any]) -> str:
|
||||
name_json = json.dumps(name, ensure_ascii=False, allow_nan=False)
|
||||
return f'{{"name":{name_json},"args":{_canonical_json(args)}}}'
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
END_OF_TEXT = "<|endoftext|>"
|
||||
MESSAGE_USER = "<|message_user|>"
|
||||
MESSAGE_MODEL = "<|message_model|>"
|
||||
MESSAGE_SYSTEM = "<|message_system|>"
|
||||
MESSAGE_TOOL = "<|message_tool|>"
|
||||
CONTENT_TEXT = "<|content_text|>"
|
||||
CONTENT_IMAGE = "<|content_image|>"
|
||||
CONTENT_MODEL_END_SAMPLING = "<|content_model_end_sampling|>"
|
||||
CONTENT_THINKING = "<|content_thinking|>"
|
||||
CONTENT_AUDIO_INPUT = "<|content_audio_input|>"
|
||||
CONTENT_TOOL_ERROR = "<|content_tool_error|>"
|
||||
CONTENT_XML = "<|content_xml|>"
|
||||
CONTENT_INVOKE_TOOL_JSON = "<|content_invoke_tool_json|>"
|
||||
CONTENT_INVOKE_TOOL_TEXT = "<|content_invoke_tool_text|>"
|
||||
END_MESSAGE = "<|end_message|>"
|
||||
AUDIO_END = "<|audio_end|>"
|
||||
|
||||
IMAGE_TOKEN_ID = -101
|
||||
AUDIO_TOKEN_ID = -102
|
||||
|
||||
INKLING_SPECIAL_TOKEN_IDS: dict[str, int] = {
|
||||
END_OF_TEXT: 199999,
|
||||
MESSAGE_USER: 200000,
|
||||
MESSAGE_MODEL: 200001,
|
||||
MESSAGE_SYSTEM: 200002,
|
||||
MESSAGE_TOOL: 200003,
|
||||
CONTENT_TEXT: 200004,
|
||||
CONTENT_IMAGE: 200005,
|
||||
CONTENT_MODEL_END_SAMPLING: 200006,
|
||||
CONTENT_THINKING: 200008,
|
||||
END_MESSAGE: 200010,
|
||||
CONTENT_AUDIO_INPUT: 200020,
|
||||
CONTENT_TOOL_ERROR: 200022,
|
||||
CONTENT_XML: 200024,
|
||||
AUDIO_END: 200043,
|
||||
CONTENT_INVOKE_TOOL_JSON: 200049,
|
||||
CONTENT_INVOKE_TOOL_TEXT: 200057,
|
||||
}
|
||||
|
||||
INKLING_SPECIAL_TOKENS: frozenset[str] = frozenset(INKLING_SPECIAL_TOKEN_IDS)
|
||||
|
||||
# The full control alphabet the streaming parsers key on: every framing token
|
||||
# plus control tokens the model can emit that have no framing-ID mapping.
|
||||
# The reasoning parser and the tool-call detector MUST share this alphabet —
|
||||
# a token visible to one but not the other lets malformed headers slip through.
|
||||
INKLING_CONTROL_TOKENS: frozenset[str] = frozenset(
|
||||
{
|
||||
*INKLING_SPECIAL_TOKENS,
|
||||
"<|content_invoke_tool|>",
|
||||
"<|model_trigger_generation|>",
|
||||
}
|
||||
)
|
||||
|
||||
INKLING_SPECIAL_TOKEN_NAMES: dict[str, str] = {
|
||||
token.removeprefix("<|").removesuffix("|>"): token
|
||||
for token in INKLING_SPECIAL_TOKENS
|
||||
}
|
||||
|
||||
ROLE_MESSAGE_TOKENS: dict[str, str] = {
|
||||
"user": MESSAGE_USER,
|
||||
"assistant": MESSAGE_MODEL,
|
||||
"system": MESSAGE_SYSTEM,
|
||||
"tool": MESSAGE_TOOL,
|
||||
}
|
||||
|
||||
|
||||
def normalize_special_token(token: str) -> str:
|
||||
"""Accept either message_user or <|message_user|> spellings."""
|
||||
if token in INKLING_SPECIAL_TOKENS:
|
||||
return token
|
||||
try:
|
||||
return INKLING_SPECIAL_TOKEN_NAMES[token]
|
||||
except KeyError as exc:
|
||||
raise KeyError(f"unknown Inkling special token: {token!r}") from exc
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class InklingTokenizer:
|
||||
"""Small wrapper around a base text tokenizer plus Inkling framing IDs.
|
||||
|
||||
Plain text is encoded by the base tokenizer, while the minimal chat
|
||||
framing tokens are inserted from the fixed overlay map.
|
||||
"""
|
||||
|
||||
tokenizer: Any
|
||||
special_token_ids: Mapping[str, int] | None = None
|
||||
|
||||
def encode_text(self, text: str) -> list[int]:
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(f"text must be str, got {type(text).__name__}")
|
||||
return list(self.tokenizer.encode(text, add_special_tokens=False))
|
||||
|
||||
def encode_special(self, token: str) -> int:
|
||||
special = normalize_special_token(token)
|
||||
token_ids = self.special_token_ids or INKLING_SPECIAL_TOKEN_IDS
|
||||
return int(token_ids[special])
|
||||
|
||||
def decode(self, token_ids: list[int]) -> str:
|
||||
return self.tokenizer.decode(token_ids)
|
||||
@@ -1,9 +1,19 @@
|
||||
import inspect
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
|
||||
from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens
|
||||
from sglang.srt.parser.harmony_parser import HarmonyParser
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
END_MESSAGE,
|
||||
INKLING_CONTROL_TOKENS,
|
||||
MESSAGE_MODEL,
|
||||
)
|
||||
|
||||
|
||||
class StreamingParseResult:
|
||||
@@ -702,6 +712,169 @@ class Gemma4Detector(BaseReasoningFormatDetector):
|
||||
self.think_start_self_label = "thought\n"
|
||||
|
||||
|
||||
_INKLING_CONTENT_KINDS = {
|
||||
CONTENT_THINKING: "reasoning",
|
||||
CONTENT_TEXT: "content",
|
||||
}
|
||||
_INKLING_END_TOKENS = {
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
END_MESSAGE,
|
||||
}
|
||||
_INKLING_CONTROL_TOKENS = INKLING_CONTROL_TOKENS
|
||||
_INKLING_CONTROL_RE = re.compile(
|
||||
"|".join(re.escape(t) for t in sorted(_INKLING_CONTROL_TOKENS))
|
||||
)
|
||||
|
||||
|
||||
class InklingDetector(BaseReasoningFormatDetector):
|
||||
"""Detector for Inkling typed content blocks."""
|
||||
|
||||
# Parse the model's sequence of typed content blocks, for example:
|
||||
# <|message_model|><|content_thinking|>reasoning<|end_message|>
|
||||
# <|message_model|><|content_text|>visible answer<|end_message|>
|
||||
# <|content_model_end_sampling|>
|
||||
# Special tokens must decode literally so thinking and visible text can be
|
||||
# routed to their respective response fields.
|
||||
def __init__(
|
||||
self,
|
||||
stream_reasoning: bool = True,
|
||||
force_reasoning: bool = False,
|
||||
continue_final_message: bool = False,
|
||||
previous_content: str = "",
|
||||
force_nonempty_content: bool = False,
|
||||
):
|
||||
del force_nonempty_content
|
||||
super().__init__(
|
||||
CONTENT_THINKING,
|
||||
END_MESSAGE,
|
||||
force_reasoning=force_reasoning,
|
||||
stream_reasoning=stream_reasoning,
|
||||
continue_final_message=continue_final_message,
|
||||
previous_content=previous_content,
|
||||
thinks_internally=False,
|
||||
reasoning_default="always",
|
||||
)
|
||||
|
||||
self._kind: str | None = None
|
||||
self._pending_header = ""
|
||||
self._pending_reasoning = ""
|
||||
|
||||
def detect_and_parse(self, text: str) -> StreamingParseResult:
|
||||
self._buffer = ""
|
||||
self._kind = None
|
||||
self._pending_header = ""
|
||||
self._pending_reasoning = ""
|
||||
ret = self._parse_blocks(text)
|
||||
if self._kind == "reasoning" and not self.stream_reasoning:
|
||||
ret.reasoning_text += self._pending_reasoning
|
||||
self._kind = None
|
||||
self._pending_header = ""
|
||||
self._pending_reasoning = ""
|
||||
return ret
|
||||
|
||||
def parse_streaming_increment(self, new_text: str) -> StreamingParseResult:
|
||||
text = self._buffer + new_text
|
||||
partial_len = self._partial_control_length(text)
|
||||
if partial_len:
|
||||
self._buffer = text[-partial_len:]
|
||||
text = text[:-partial_len]
|
||||
else:
|
||||
self._buffer = ""
|
||||
return self._parse_blocks(text)
|
||||
|
||||
@staticmethod
|
||||
def _partial_control_length(text: str) -> int:
|
||||
max_token_len = max(map(len, _INKLING_CONTROL_TOKENS))
|
||||
for length in range(min(len(text), max_token_len - 1), 0, -1):
|
||||
suffix = text[-length:]
|
||||
if any(
|
||||
len(suffix) < len(token) and token.startswith(suffix)
|
||||
for token in _INKLING_CONTROL_TOKENS
|
||||
):
|
||||
return length
|
||||
return 0
|
||||
|
||||
def _parse_blocks(self, text: str) -> StreamingParseResult:
|
||||
reasoning: list[str] = []
|
||||
content: list[str] = []
|
||||
saw_control = False
|
||||
pos = 0
|
||||
|
||||
def emit(text: str) -> None:
|
||||
if self._kind == "reasoning":
|
||||
if self.stream_reasoning:
|
||||
reasoning.append(text)
|
||||
else:
|
||||
self._pending_reasoning += text
|
||||
elif self._kind == "content":
|
||||
content.append(text)
|
||||
elif self._kind == "tool":
|
||||
content.append(text)
|
||||
elif self._kind == "header":
|
||||
self._pending_header += text
|
||||
elif text:
|
||||
# No open block — e.g. a continue_final_message stream resuming
|
||||
# mid text block. Route to visible content, matching the
|
||||
# no-control-token path below.
|
||||
content.append(text)
|
||||
|
||||
def flush_reasoning() -> None:
|
||||
if self._kind == "reasoning" and not self.stream_reasoning:
|
||||
reasoning.append(self._pending_reasoning)
|
||||
self._pending_reasoning = ""
|
||||
|
||||
for match in _INKLING_CONTROL_RE.finditer(text):
|
||||
saw_control = True
|
||||
emit(text[pos : match.start()])
|
||||
|
||||
token = match.group(0)
|
||||
pos = match.end()
|
||||
if token == MESSAGE_MODEL:
|
||||
if self._kind in (None, "header"):
|
||||
flush_reasoning()
|
||||
self._pending_header = ""
|
||||
self._kind = "header"
|
||||
else:
|
||||
# Inside an open block a decoded <|message_model|> string
|
||||
# is payload the model wrote (e.g. quoting the protocol) —
|
||||
# a real header can only follow an end token. Preserve it
|
||||
# instead of rerouting the rest of the block into a header.
|
||||
emit(token)
|
||||
elif token == CONTENT_INVOKE_TOOL_JSON:
|
||||
flush_reasoning()
|
||||
if self._kind == "header":
|
||||
content.extend(
|
||||
(MESSAGE_MODEL, self._pending_header, CONTENT_INVOKE_TOOL_JSON)
|
||||
)
|
||||
self._pending_header = ""
|
||||
else:
|
||||
content.append(token)
|
||||
self._kind = "tool"
|
||||
elif self._kind == "tool":
|
||||
content.append(token)
|
||||
if token in _INKLING_END_TOKENS:
|
||||
self._kind = None
|
||||
elif token in _INKLING_CONTENT_KINDS:
|
||||
flush_reasoning()
|
||||
self._pending_header = ""
|
||||
self._kind = _INKLING_CONTENT_KINDS[token]
|
||||
elif token in _INKLING_END_TOKENS:
|
||||
flush_reasoning()
|
||||
self._pending_header = ""
|
||||
self._kind = None
|
||||
|
||||
tail = text[pos:]
|
||||
if saw_control or self._kind is not None:
|
||||
emit(tail)
|
||||
else:
|
||||
content.append(text)
|
||||
|
||||
return StreamingParseResult(
|
||||
normal_text="".join(content),
|
||||
reasoning_text="".join(reasoning),
|
||||
)
|
||||
|
||||
|
||||
class _DeepSeekV3Detector(Qwen3Detector):
|
||||
"""DeepSeek-V3 reuses Qwen3 tokens but requires explicit thinking=True to enable."""
|
||||
|
||||
@@ -1217,6 +1390,7 @@ class ReasoningParser:
|
||||
"nemotron_3": Nemotron3Detector,
|
||||
"interns1": Qwen3Detector,
|
||||
"gemma4": Gemma4Detector,
|
||||
"inkling": InklingDetector,
|
||||
"cohere_command4": CohereCommand4Detector,
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user