Fix Inkling tool-call parsing recovery, content handling, and streaming (#32861)
This commit is contained in:
Ke Bao
@@ -1,11 +1,8 @@
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
from typing import List, Optional
|
||||
|
||||
from partial_json_parser.core.exceptions import MalformedJSON
|
||||
from partial_json_parser.core.options import Allow
|
||||
from xgrammar import StructuralTag
|
||||
|
||||
from sglang.srt.entrypoints.openai.protocol import Tool
|
||||
@@ -16,9 +13,9 @@ from sglang.srt.function_call.core_types import (
|
||||
ToolCallItem,
|
||||
_GetInfoFunc,
|
||||
)
|
||||
from sglang.srt.function_call.utils import _is_complete_json, _partial_json_loads
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_INVOKE_TOOL_TEXT,
|
||||
END_MESSAGE,
|
||||
INKLING_CONTROL_TOKENS,
|
||||
INKLING_SPECIAL_TOKEN_IDS,
|
||||
@@ -28,6 +25,11 @@ from sglang.srt.parser.inkling_tokenizer import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _reject_nonfinite_number(value: str) -> float:
|
||||
# Strict tool-payload parsing rejects NaN/Infinity; only recovery accepts them.
|
||||
raise ValueError(f"{value} is not a valid JSON number")
|
||||
|
||||
|
||||
class InklingDetector(BaseFormatDetector):
|
||||
"""
|
||||
Detector for Inkling structured tool calls.
|
||||
@@ -40,208 +42,189 @@ class InklingDetector(BaseFormatDetector):
|
||||
super().__init__()
|
||||
self.bot_token = CONTENT_INVOKE_TOOL_JSON
|
||||
self.eot_token = END_MESSAGE
|
||||
self.tool_call_regex = re.compile(
|
||||
re.escape(self.bot_token) + r"\s*(.*?)\s*" + re.escape(self.eot_token),
|
||||
re.DOTALL,
|
||||
)
|
||||
self._current_header_name: str | None = None
|
||||
# Streaming: index of the next call to emit; once a call fails to frame,
|
||||
# the rest of the response streams through verbatim.
|
||||
self._stream_call_index = 0
|
||||
self._raw_passthrough = False
|
||||
|
||||
def has_tool_call(self, text: str) -> bool:
|
||||
return self.bot_token in text
|
||||
return self.bot_token in text or CONTENT_INVOKE_TOOL_TEXT in text
|
||||
|
||||
def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
|
||||
if self.bot_token not in text:
|
||||
if not self.has_tool_call(text):
|
||||
return StreamingParseResult(normal_text=self._clean_normal_text(text))
|
||||
|
||||
try:
|
||||
calls: list[ToolCallItem] = []
|
||||
for match in self.tool_call_regex.finditer(text):
|
||||
try:
|
||||
payload = json.loads(match.group(1).strip())
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.warning("Invalid Inkling tool call JSON: %s", exc)
|
||||
continue
|
||||
if not isinstance(payload, Mapping):
|
||||
logger.warning("Invalid Inkling tool call payload: %s", payload)
|
||||
continue
|
||||
_, header_name = self._split_trailing_tool_header(text[: match.start()])
|
||||
call = self._tool_call_item(
|
||||
payload, tools, len(calls), header_name=header_name
|
||||
)
|
||||
if call is not None:
|
||||
calls.append(call)
|
||||
|
||||
if not calls:
|
||||
# Every candidate call was rejected (bad payload or a
|
||||
# header/payload name mismatch). Match the framework contract
|
||||
# every other detector follows: normal_text is only the content
|
||||
# BEFORE the tool marker — the rejected tool-call region is
|
||||
# dropped, never regurgitated as visible content.
|
||||
prefix, _ = self._split_trailing_tool_header(
|
||||
text[: text.find(self.bot_token)]
|
||||
)
|
||||
return StreamingParseResult(normal_text=self._clean_normal_text(prefix))
|
||||
|
||||
normal_prefix, _ = self._split_trailing_tool_header(
|
||||
text[: text.find(self.bot_token)]
|
||||
)
|
||||
normal_text = self._clean_normal_text(normal_prefix)
|
||||
parsed = self._parse_canonical(text, tools)
|
||||
if parsed is not None:
|
||||
normal_text, calls = parsed
|
||||
return StreamingParseResult(normal_text=normal_text, calls=calls)
|
||||
except Exception as exc:
|
||||
logger.error("Error in Inkling detect_and_parse: %s", exc, exc_info=True)
|
||||
prefix, _ = self._split_trailing_tool_header(
|
||||
text[: text.find(self.bot_token)]
|
||||
)
|
||||
return StreamingParseResult(normal_text=self._clean_normal_text(prefix))
|
||||
|
||||
# Canonical framing failed. Recover a single call from the last tool
|
||||
# marker; if that fails too, surface the whole visible payload as text.
|
||||
recovered = self._recover_last_json_call(text, tools)
|
||||
if recovered is not None:
|
||||
return StreamingParseResult(normal_text="", calls=[recovered])
|
||||
return StreamingParseResult(normal_text=self._clean_normal_text(text))
|
||||
|
||||
def _parse_canonical(
|
||||
self, text: str, tools: List[Tool]
|
||||
) -> tuple[str, list[ToolCallItem]] | None:
|
||||
"""Extract every tool call under strict framing.
|
||||
|
||||
Returns (visible_text, calls) when EVERY marker frames a valid call, or
|
||||
None if any marker is malformed/unterminated — one bad call fails the
|
||||
whole batch, matching streaming.
|
||||
"""
|
||||
calls: list[ToolCallItem] = []
|
||||
normal_parts: list[str] = []
|
||||
pos = 0
|
||||
while pos < len(text):
|
||||
marker_pos, marker_token, is_json = self._next_tool_marker(text, pos)
|
||||
if marker_pos is None:
|
||||
normal_parts.append(text[pos:])
|
||||
break
|
||||
|
||||
prefix, _ = self._split_trailing_tool_header(text[pos:marker_pos])
|
||||
normal_parts.append(prefix)
|
||||
|
||||
body_start = marker_pos + len(marker_token)
|
||||
eot = text.find(self.eot_token, body_start)
|
||||
if eot == -1:
|
||||
return None # unterminated -> recovery reads through EOS
|
||||
body = text[body_start:eot]
|
||||
|
||||
if is_json:
|
||||
call = self._canonical_json_call(body, tools, len(calls))
|
||||
if call is None:
|
||||
return None
|
||||
else:
|
||||
call = self._text_tool_call(body, len(calls))
|
||||
calls.append(call)
|
||||
pos = eot + len(self.eot_token)
|
||||
|
||||
return self._clean_normal_text("".join(normal_parts)), calls
|
||||
|
||||
def _next_tool_marker(self, text: str, start: int) -> tuple[int | None, str, bool]:
|
||||
"""Earliest json/text tool marker at or after ``start`` (is_json flag)."""
|
||||
json_pos = text.find(self.bot_token, start)
|
||||
text_pos = text.find(CONTENT_INVOKE_TOOL_TEXT, start)
|
||||
if json_pos == -1 and text_pos == -1:
|
||||
return None, "", True
|
||||
if text_pos == -1 or (json_pos != -1 and json_pos <= text_pos):
|
||||
return json_pos, self.bot_token, True
|
||||
return text_pos, CONTENT_INVOKE_TOOL_TEXT, False
|
||||
|
||||
def _canonical_json_call(
|
||||
self, body: str, tools: List[Tool], call_index: int
|
||||
) -> ToolCallItem | None:
|
||||
try:
|
||||
payload = json.loads(body.strip(), parse_constant=_reject_nonfinite_number)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
if not isinstance(payload, Mapping):
|
||||
return None
|
||||
return self._tool_call_item(payload, tools, call_index)
|
||||
|
||||
def _recover_last_json_call(
|
||||
self, text: str, tools: List[Tool]
|
||||
) -> ToolCallItem | None:
|
||||
"""Recover one call from the last json marker, reading through the next
|
||||
end token or EOS. Requires a nonempty name; accepts NaN/Infinity."""
|
||||
last = text.rfind(self.bot_token)
|
||||
if last == -1:
|
||||
return None
|
||||
body_start = last + len(self.bot_token)
|
||||
eot = text.find(self.eot_token, body_start)
|
||||
candidate = text[body_start:eot] if eot != -1 else text[body_start:]
|
||||
candidate = self._clean_normal_text(candidate).strip()
|
||||
try:
|
||||
payload = json.loads(candidate)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
if not isinstance(payload, Mapping) or not payload.get("name"):
|
||||
return None
|
||||
return self._tool_call_item(payload, tools, 0)
|
||||
|
||||
def _text_tool_call(self, body: str, call_index: int) -> ToolCallItem:
|
||||
# Headerless raw-text invocation: no structured name/args on the wire.
|
||||
return ToolCallItem(
|
||||
tool_index=call_index,
|
||||
name="",
|
||||
parameters=json.dumps({"text": self._clean_normal_text(body)}),
|
||||
)
|
||||
|
||||
def parse_streaming_increment(
|
||||
self, new_text: str, tools: List[Tool]
|
||||
) -> StreamingParseResult:
|
||||
# Drain every complete call in the delta: this detector has no
|
||||
# stream-end flush, so anything left in self._buffer is lost.
|
||||
self._buffer += new_text
|
||||
all_calls: list[ToolCallItem] = []
|
||||
if self._raw_passthrough:
|
||||
out = self._clean_normal_text(self._buffer)
|
||||
self._buffer = ""
|
||||
return StreamingParseResult(normal_text=out)
|
||||
|
||||
normal_parts: list[str] = []
|
||||
while True:
|
||||
result, made_progress = self._parse_buffered_increment(tools)
|
||||
if result.normal_text:
|
||||
normal_parts.append(result.normal_text)
|
||||
if result.calls:
|
||||
all_calls.extend(result.calls)
|
||||
if not made_progress:
|
||||
break
|
||||
return StreamingParseResult(
|
||||
normal_text="".join(normal_parts),
|
||||
calls=all_calls,
|
||||
)
|
||||
|
||||
def _parse_buffered_increment(
|
||||
self, tools: List[Tool]
|
||||
) -> tuple[StreamingParseResult, bool]:
|
||||
# One drain step: emit a text run or one complete call; the bool is
|
||||
# whether the buffer advanced (the caller loops while it does).
|
||||
current_text = self._buffer
|
||||
|
||||
if self.bot_token not in current_text:
|
||||
header_start = self._pending_tool_header_start(current_text)
|
||||
if header_start is not None:
|
||||
safe_text = current_text[:header_start]
|
||||
self._buffer = current_text[header_start:]
|
||||
return (
|
||||
StreamingParseResult(
|
||||
normal_text=self._clean_normal_text(safe_text)
|
||||
),
|
||||
False,
|
||||
)
|
||||
# Hold back a partial prefix of ANY token _clean_normal_text
|
||||
# strips — emitting a split control token leaks its first half as
|
||||
# visible text (the completed token would have been stripped).
|
||||
partial_len = max(
|
||||
self._ends_with_partial_token(current_text, token)
|
||||
for token in INKLING_CONTROL_TOKENS
|
||||
)
|
||||
if partial_len:
|
||||
safe_text = current_text[:-partial_len]
|
||||
self._buffer = current_text[-partial_len:]
|
||||
else:
|
||||
safe_text = current_text
|
||||
self._buffer = ""
|
||||
return (
|
||||
StreamingParseResult(normal_text=self._clean_normal_text(safe_text)),
|
||||
False,
|
||||
)
|
||||
|
||||
bot_pos = current_text.find(self.bot_token)
|
||||
if bot_pos > 0:
|
||||
normal_text, self._current_header_name = self._split_trailing_tool_header(
|
||||
current_text[:bot_pos]
|
||||
)
|
||||
self._buffer = current_text[bot_pos:]
|
||||
normal_text = self._clean_normal_text(normal_text)
|
||||
if normal_text:
|
||||
# prefix stripped, call now at buffer head -> keep draining
|
||||
return StreamingParseResult(normal_text=normal_text), True
|
||||
current_text = self._buffer
|
||||
|
||||
if not hasattr(self, "_tool_indices"):
|
||||
self._tool_indices = self._get_tool_indices(tools)
|
||||
|
||||
start_idx = len(self.bot_token)
|
||||
while start_idx < len(current_text) and current_text[start_idx].isspace():
|
||||
start_idx += 1
|
||||
|
||||
flags = Allow.ALL if self.current_tool_name_sent else Allow.ALL & ~Allow.STR
|
||||
try:
|
||||
payload, end_idx = _partial_json_loads(current_text[start_idx:], flags)
|
||||
except (MalformedJSON, json.JSONDecodeError):
|
||||
return StreamingParseResult(), False
|
||||
if not isinstance(payload, Mapping):
|
||||
return StreamingParseResult(), False
|
||||
|
||||
calls: list[ToolCallItem] = []
|
||||
name = payload.get("name")
|
||||
if (
|
||||
not self.current_tool_name_sent
|
||||
and isinstance(name, str)
|
||||
and (self._current_header_name is None or self._current_header_name == name)
|
||||
):
|
||||
self._ensure_current_tool()
|
||||
calls.append(
|
||||
ToolCallItem(
|
||||
tool_index=self.current_tool_id,
|
||||
name=name,
|
||||
parameters="",
|
||||
)
|
||||
)
|
||||
self.current_tool_name_sent = True
|
||||
self.prev_tool_call_arr[self.current_tool_id] = {
|
||||
"name": name,
|
||||
"arguments": {},
|
||||
}
|
||||
while self._buffer:
|
||||
marker_pos, marker_token, is_json = self._next_tool_marker(self._buffer, 0)
|
||||
if marker_pos is None:
|
||||
safe, hold = self._split_safe_text(self._buffer)
|
||||
normal_parts.append(self._clean_normal_text(safe))
|
||||
self._buffer = hold
|
||||
break
|
||||
|
||||
json_text = current_text[start_idx : start_idx + end_idx]
|
||||
if not _is_complete_json(json_text):
|
||||
return StreamingParseResult(calls=calls), False
|
||||
prefix, _ = self._split_trailing_tool_header(self._buffer[:marker_pos])
|
||||
body_start = marker_pos + len(marker_token)
|
||||
eot = self._buffer.find(self.eot_token, body_start)
|
||||
if eot == -1:
|
||||
# Region incomplete — emit only the text before it and hold the
|
||||
# marker + partial body so name/args stay atomic until the close.
|
||||
normal_parts.append(self._clean_normal_text(prefix))
|
||||
self._buffer = self._buffer[marker_pos:]
|
||||
break
|
||||
|
||||
call = self._tool_call_item(
|
||||
payload,
|
||||
tools,
|
||||
self.current_tool_id,
|
||||
header_name=self._current_header_name,
|
||||
body = self._buffer[body_start:eot]
|
||||
if is_json:
|
||||
call = self._canonical_json_call(body, tools, self._stream_call_index)
|
||||
if call is None:
|
||||
return self._stream_recover_or_fallback(tools)
|
||||
else:
|
||||
call = self._text_tool_call(body, self._stream_call_index)
|
||||
normal_parts.append(self._clean_normal_text(prefix))
|
||||
calls.append(call)
|
||||
self._stream_call_index += 1
|
||||
self._buffer = self._buffer[eot + len(self.eot_token) :]
|
||||
|
||||
return StreamingParseResult(normal_text="".join(normal_parts), calls=calls)
|
||||
|
||||
def _stream_recover_or_fallback(self, tools: List[Tool]) -> StreamingParseResult:
|
||||
# A call failed to frame: recover one call from the last marker, then
|
||||
# pass the rest of the response through as text.
|
||||
self._raw_passthrough = True
|
||||
recovered = self._recover_last_json_call(self._buffer, tools)
|
||||
buffered = self._buffer
|
||||
self._buffer = ""
|
||||
if recovered is not None:
|
||||
recovered.tool_index = self._stream_call_index
|
||||
self._stream_call_index += 1
|
||||
return StreamingParseResult(calls=[recovered])
|
||||
return StreamingParseResult(normal_text=self._clean_normal_text(buffered))
|
||||
|
||||
def _split_safe_text(self, text: str) -> tuple[str, str]:
|
||||
"""Split off text safe to emit now from a tail that may be a forming
|
||||
tool header or a split control token."""
|
||||
header_start = self._pending_tool_header_start(text)
|
||||
if header_start is not None:
|
||||
return text[:header_start], text[header_start:]
|
||||
partial_len = max(
|
||||
(
|
||||
self._ends_with_partial_token(text, token)
|
||||
for token in INKLING_CONTROL_TOKENS
|
||||
),
|
||||
default=0,
|
||||
)
|
||||
if call is None:
|
||||
# Drop only the rejected call's span, not the whole buffer, or a
|
||||
# trailing valid call dies; clear the header so it can't leak.
|
||||
self._abandon_current_tool()
|
||||
self._buffer = self._remaining_after_call(current_text, start_idx + end_idx)
|
||||
self._current_header_name = None
|
||||
return StreamingParseResult(calls=calls), True
|
||||
|
||||
if self.current_tool_id == -1:
|
||||
self._ensure_current_tool()
|
||||
|
||||
args = json.loads(call.parameters)
|
||||
self.prev_tool_call_arr[self.current_tool_id] = {
|
||||
"name": call.name,
|
||||
"arguments": args,
|
||||
}
|
||||
sent = self.streamed_args_for_tool[self.current_tool_id]
|
||||
remaining_args = call.parameters[len(sent) :]
|
||||
if remaining_args:
|
||||
calls.append(
|
||||
ToolCallItem(
|
||||
tool_index=self.current_tool_id,
|
||||
name=None,
|
||||
parameters=remaining_args,
|
||||
)
|
||||
)
|
||||
self.streamed_args_for_tool[self.current_tool_id] += remaining_args
|
||||
|
||||
self._buffer = self._remaining_after_call(current_text, start_idx + end_idx)
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
self._current_header_name = None
|
||||
return StreamingParseResult(calls=calls), True
|
||||
if partial_len:
|
||||
return text[:-partial_len], text[-partial_len:]
|
||||
return text, ""
|
||||
|
||||
def structure_info(self) -> _GetInfoFunc:
|
||||
def info(name: str) -> StructureInfo:
|
||||
@@ -310,21 +293,12 @@ class InklingDetector(BaseFormatDetector):
|
||||
payload: Mapping[str, object],
|
||||
tools: List[Tool],
|
||||
call_index: int,
|
||||
*,
|
||||
header_name: str | None = None,
|
||||
) -> ToolCallItem | None:
|
||||
name = payload.get("name")
|
||||
args = payload.get("args")
|
||||
if not isinstance(name, str) or not isinstance(args, Mapping):
|
||||
logger.warning("Invalid Inkling tool call payload: %s", payload)
|
||||
return None
|
||||
if header_name is not None and header_name != name:
|
||||
logger.warning(
|
||||
"Inkling tool header %r does not match payload name %r",
|
||||
header_name,
|
||||
name,
|
||||
)
|
||||
return None
|
||||
|
||||
if not hasattr(self, "_tool_indices"):
|
||||
self._tool_indices = self._get_tool_indices(tools)
|
||||
@@ -341,28 +315,6 @@ class InklingDetector(BaseFormatDetector):
|
||||
parameters=json.dumps(args, ensure_ascii=False),
|
||||
)
|
||||
|
||||
def _ensure_current_tool(self) -> None:
|
||||
if self.current_tool_id == -1:
|
||||
self.current_tool_id = 0
|
||||
while len(self.prev_tool_call_arr) <= self.current_tool_id:
|
||||
self.prev_tool_call_arr.append({})
|
||||
while len(self.streamed_args_for_tool) <= self.current_tool_id:
|
||||
self.streamed_args_for_tool.append("")
|
||||
|
||||
def _abandon_current_tool(self) -> None:
|
||||
"""Discard the in-flight call after a rejected payload.
|
||||
|
||||
Resetting ``current_tool_id`` to -1 here would collide the NEXT valid
|
||||
call with tool index 0 (``_ensure_current_tool`` maps -1 -> 0) and
|
||||
slice its arguments against index 0's already-streamed args. Keep the
|
||||
counter: an unannounced slot is simply reused; an announced slot is
|
||||
abandoned by advancing past it.
|
||||
"""
|
||||
if self.current_tool_name_sent:
|
||||
self.current_tool_id += 1
|
||||
self.current_tool_name_sent = False
|
||||
self._current_header_name = None
|
||||
|
||||
def _split_trailing_tool_header(self, text: str) -> tuple[str, str | None]:
|
||||
message_pos = self._pending_tool_header_start(text)
|
||||
if message_pos is None:
|
||||
@@ -382,14 +334,6 @@ class InklingDetector(BaseFormatDetector):
|
||||
return None
|
||||
return message_pos
|
||||
|
||||
def _remaining_after_call(self, text: str, end_idx: int) -> str:
|
||||
remaining = text[end_idx:]
|
||||
if remaining.startswith(self.eot_token):
|
||||
return remaining[len(self.eot_token) :]
|
||||
if self.eot_token in remaining:
|
||||
return remaining.split(self.eot_token, 1)[1]
|
||||
return remaining
|
||||
|
||||
def _clean_normal_text(self, text: str) -> str:
|
||||
for token in INKLING_CONTROL_TOKENS:
|
||||
text = text.replace(token, "")
|
||||
|
||||
@@ -15,6 +15,7 @@ from sglang.srt.function_call.hunyuan_detector import resolve_hunyuan_tokens
|
||||
from sglang.srt.parser.harmony_parser import HarmonyParser
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
CONTENT_INVOKE_TOOL_TEXT,
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
@@ -862,12 +863,12 @@ class InklingDetector(BaseReasoningFormatDetector):
|
||||
# a real header can only follow an end token. Preserve it
|
||||
# instead of rerouting the rest of the block into a header.
|
||||
emit(token)
|
||||
elif token == CONTENT_INVOKE_TOOL_JSON:
|
||||
elif token in (CONTENT_INVOKE_TOOL_JSON, CONTENT_INVOKE_TOOL_TEXT):
|
||||
# Preserve the tool-invocation framing (json and headerless raw
|
||||
# text) in content so the tool-call detector receives it.
|
||||
flush_reasoning()
|
||||
if self._kind == "header":
|
||||
content.extend(
|
||||
(MESSAGE_MODEL, self._pending_header, CONTENT_INVOKE_TOOL_JSON)
|
||||
)
|
||||
content.extend((MESSAGE_MODEL, self._pending_header, token))
|
||||
self._pending_header = ""
|
||||
else:
|
||||
content.append(token)
|
||||
|
||||
@@ -86,30 +86,32 @@ class TestInklingDetector(unittest.TestCase):
|
||||
self.assertEqual(name, "weather")
|
||||
self.assertEqual(json.loads(parameters), {"city": "SF"})
|
||||
|
||||
def test_mismatched_header_is_rejected(self):
|
||||
def test_header_name_is_ignored_and_payload_name_wins(self):
|
||||
"""The message header is author metadata, not a name check: a header
|
||||
that differs from the payload name still yields a call named by the
|
||||
payload."""
|
||||
detector = InklingDetector()
|
||||
source = (
|
||||
"<|message_model|>other<|content_invoke_tool_json|>"
|
||||
'{"name":"weather","args":{}}<|end_message|>'
|
||||
)
|
||||
result = detector.detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "weather")
|
||||
self.assertEqual(result.normal_text, "")
|
||||
|
||||
def test_raw_fallback_strips_protocol_tokens(self):
|
||||
"""When a payload cannot be parsed or recovered, the visible text is
|
||||
surfaced as content with the <|...|> special tokens stripped."""
|
||||
detector = InklingDetector()
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>"
|
||||
"{not json at all<|end_message|>"
|
||||
)
|
||||
result = detector.detect_and_parse(source, self.tools)
|
||||
self.assertEqual(result.calls, [])
|
||||
|
||||
def test_rejected_call_does_not_leak_protocol_tokens(self):
|
||||
"""Bug regression: the no-surviving-calls path returned the RAW text,
|
||||
so a rejected call (e.g. header/payload mismatch) leaked <|...|>
|
||||
protocol tokens into user-visible content."""
|
||||
detector = InklingDetector()
|
||||
source = (
|
||||
"<|message_model|>other<|content_invoke_tool_json|>"
|
||||
'{"name":"weather","args":{}}<|end_message|>'
|
||||
)
|
||||
result = detector.detect_and_parse(source, self.tools)
|
||||
self.assertNotIn("<|", result.normal_text)
|
||||
# Framework parity: the rejected tool-call REGION is dropped entirely
|
||||
# (normal_text = content before the marker), like every other detector
|
||||
# — the JSON payload must not surface as visible content either.
|
||||
self.assertEqual(result.normal_text, "")
|
||||
self.assertIn("{not json at all", result.normal_text)
|
||||
|
||||
def test_headerless_legacy_tool_call_still_parses(self):
|
||||
"""Spec tolerance: a bare <|content_invoke_tool_json|> block with no
|
||||
@@ -189,11 +191,9 @@ class TestInklingDetector(unittest.TestCase):
|
||||
args = "".join(c.parameters for c in result.calls)
|
||||
self.assertEqual(json.loads(args), {"city": "SF"})
|
||||
|
||||
def test_streaming_rejected_middle_call_keeps_later_valid_call(self):
|
||||
"""Bug regression: a rejected call (header/name mismatch) cleared the
|
||||
whole buffer, discarding a later valid call that arrived in the same
|
||||
delta. Only the rejected call's span may be dropped; the drain must
|
||||
continue so the trailing valid call still streams."""
|
||||
def test_streaming_differing_headers_all_stream(self):
|
||||
"""The header is author metadata, not a name gate: three calls with
|
||||
differing headers all stream, indexed 0/1/2 by payload name."""
|
||||
detector = InklingDetector()
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>"
|
||||
@@ -208,37 +208,38 @@ class TestInklingDetector(unittest.TestCase):
|
||||
args_by_index[call.tool_index] = (
|
||||
args_by_index.get(call.tool_index, "") + call.parameters
|
||||
)
|
||||
self.assertEqual(sorted(args_by_index), [0, 1])
|
||||
self.assertEqual(sorted(args_by_index), [0, 1, 2])
|
||||
self.assertEqual(json.loads(args_by_index[0]), {"city": "SF"})
|
||||
self.assertEqual(json.loads(args_by_index[1]), {"city": "NY"})
|
||||
self.assertEqual(json.loads(args_by_index[1]), {"city": "XX"})
|
||||
self.assertEqual(json.loads(args_by_index[2]), {"city": "NY"})
|
||||
|
||||
def test_streaming_rejection_does_not_collide_tool_indices(self):
|
||||
"""Bug regression: a rejected mid-stream call reset current_tool_id to
|
||||
-1, so the NEXT valid call re-announced as tool_index 0 — colliding
|
||||
with the first call's index and slicing its arguments against index
|
||||
0's already-streamed args."""
|
||||
def test_streaming_malformed_call_switches_to_raw_passthrough(self):
|
||||
"""A call that fails to frame switches the stream to raw passthrough:
|
||||
earlier calls stay emitted (streaming cannot un-emit), and everything
|
||||
after the failure is surfaced as content, never as further calls."""
|
||||
detector = InklingDetector()
|
||||
chunks = [
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>",
|
||||
'{"name":"weather","args":{"city":"SF"}}<|end_message|>',
|
||||
# header/payload mismatch -> rejected
|
||||
"<|message_model|>other<|content_invoke_tool_json|>",
|
||||
'{"name":"weather","args":{"city":"NY"}}<|end_message|>',
|
||||
# valid again
|
||||
# unrecoverable -> raw passthrough from here on
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>",
|
||||
"{not json at all<|end_message|>",
|
||||
# would-be call, now passthrough text
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>",
|
||||
'{"name":"weather","args":{"city":"LA"}}<|end_message|>',
|
||||
]
|
||||
args_by_index: dict = {}
|
||||
calls: list = []
|
||||
normal_text = ""
|
||||
for chunk in chunks:
|
||||
for call in detector.parse_streaming_increment(chunk, self.tools).calls:
|
||||
args_by_index[call.tool_index] = (
|
||||
args_by_index.get(call.tool_index, "") + call.parameters
|
||||
)
|
||||
self.assertEqual(json.loads(args_by_index[0]), {"city": "SF"})
|
||||
self.assertEqual(len(args_by_index), 2)
|
||||
second_index = max(args_by_index)
|
||||
self.assertGreater(second_index, 0)
|
||||
self.assertEqual(json.loads(args_by_index[second_index]), {"city": "LA"})
|
||||
result = detector.parse_streaming_increment(chunk, self.tools)
|
||||
normal_text += result.normal_text
|
||||
calls.extend(result.calls)
|
||||
self.assertEqual(len(calls), 1)
|
||||
self.assertEqual(calls[0].name, "weather")
|
||||
self.assertEqual(json.loads(calls[0].parameters), {"city": "SF"})
|
||||
self.assertNotIn("<|", normal_text)
|
||||
self.assertIn("{not json at all", normal_text)
|
||||
self.assertIn("LA", normal_text)
|
||||
|
||||
def test_undeclared_tool_name_is_surfaced(self):
|
||||
"""A call to a tool absent from the request's tool list surfaces as a
|
||||
@@ -275,8 +276,9 @@ class TestInklingDetector(unittest.TestCase):
|
||||
self.assertEqual(json.loads(parameters), {"query": "q"})
|
||||
self.assertNotIn("<|", normal_text)
|
||||
|
||||
def test_malformed_json_does_not_leak_protocol_tokens(self):
|
||||
"""Malformed JSON must drop the protocol region and its tool header."""
|
||||
def test_malformed_json_surfaces_as_raw_fallback(self):
|
||||
"""Malformed JSON that also fails recovery surfaces the visible payload
|
||||
as content (special tokens stripped), not a tool call."""
|
||||
detector = InklingDetector()
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>"
|
||||
@@ -284,10 +286,12 @@ class TestInklingDetector(unittest.TestCase):
|
||||
)
|
||||
result = detector.detect_and_parse(source, self.tools)
|
||||
self.assertEqual(result.calls, [])
|
||||
self.assertEqual(result.normal_text, "")
|
||||
self.assertNotIn("<|", result.normal_text)
|
||||
self.assertIn("{not json at all", result.normal_text)
|
||||
|
||||
def test_parser_does_not_restore_malformed_tool_call_as_text(self):
|
||||
"""The parser wrapper must preserve the detector's sanitized fallback."""
|
||||
def test_parser_preserves_raw_fallback_text(self):
|
||||
"""The parser wrapper preserves the detector's raw fallback, so the
|
||||
visible prefix plus the failed payload reach the caller as content."""
|
||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
|
||||
source = (
|
||||
@@ -298,8 +302,9 @@ class TestInklingDetector(unittest.TestCase):
|
||||
normal_text, calls = FunctionCallParser(self.tools, "inkling").parse_non_stream(
|
||||
source
|
||||
)
|
||||
self.assertEqual(normal_text, "Visible prefix.")
|
||||
self.assertEqual(calls, [])
|
||||
self.assertTrue(normal_text.startswith("Visible prefix."))
|
||||
self.assertIn("{not json at all", normal_text)
|
||||
|
||||
def test_parser_preserves_text_without_tool_call_marker(self):
|
||||
from sglang.srt.function_call.function_call_parser import FunctionCallParser
|
||||
@@ -311,7 +316,10 @@ class TestInklingDetector(unittest.TestCase):
|
||||
self.assertEqual(normal_text, source)
|
||||
self.assertEqual(calls, [])
|
||||
|
||||
def test_malformed_call_does_not_discard_an_earlier_valid_call(self):
|
||||
def test_one_malformed_call_fails_the_whole_batch(self):
|
||||
"""All-or-nothing: a single unrecoverable call fails canonical framing
|
||||
for the whole response, so even an earlier valid call is discarded and
|
||||
the visible text is surfaced as content."""
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>"
|
||||
'{"name":"weather","args":{"city":"SF"}}<|end_message|>'
|
||||
@@ -319,10 +327,10 @@ class TestInklingDetector(unittest.TestCase):
|
||||
"{not json at all<|end_message|>"
|
||||
)
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(result.normal_text, "")
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "weather")
|
||||
self.assertEqual(json.loads(result.calls[0].parameters), {"city": "SF"})
|
||||
self.assertEqual(result.calls, [])
|
||||
self.assertNotIn("<|", result.normal_text)
|
||||
self.assertIn('{"name":"weather","args":{"city":"SF"}}', result.normal_text)
|
||||
self.assertIn("{not json at all", result.normal_text)
|
||||
|
||||
def test_clean_normal_text_strips_the_full_control_alphabet(self):
|
||||
"""Fall-through text is cleaned against the whole shared control-token
|
||||
@@ -341,6 +349,86 @@ class TestInklingDetector(unittest.TestCase):
|
||||
self.assertEqual(info.trigger, header)
|
||||
self.assertTrue(info.begin.startswith(header + '{"name":"weather"'))
|
||||
|
||||
def test_content_after_tool_call_is_preserved(self):
|
||||
"""A tool call followed by a text block returns both: the call plus the
|
||||
trailing visible content, not just the prefix before the marker."""
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>"
|
||||
'{"name":"weather","args":{"city":"SF"}}<|end_message|>'
|
||||
"<|message_model|><|content_text|>Here you go.<|end_message|>"
|
||||
)
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "weather")
|
||||
self.assertEqual(result.normal_text, "Here you go.")
|
||||
|
||||
def test_empty_name_is_allowed_on_the_canonical_path(self):
|
||||
source = "<|content_invoke_tool_json|>" '{"name":"","args":{}}<|end_message|>'
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "")
|
||||
|
||||
def test_recovery_uses_only_the_last_marker(self):
|
||||
"""Canonical framing fails on the garbage payload; recovery reads only
|
||||
the payload after the LAST marker."""
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>garbage"
|
||||
"<|content_invoke_tool_json|>"
|
||||
'{"name":"weather","args":{"city":"SF"}}<|end_message|>'
|
||||
)
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "weather")
|
||||
self.assertEqual(json.loads(result.calls[0].parameters), {"city": "SF"})
|
||||
|
||||
def test_recovery_requires_a_nonempty_name(self):
|
||||
"""Recovery (unlike the canonical path) rejects an empty name, falling
|
||||
through to raw text."""
|
||||
source = (
|
||||
"<|message_model|>weather<|content_invoke_tool_json|>bad"
|
||||
'<|content_invoke_tool_json|>{"name":"","args":{}}<|end_message|>'
|
||||
)
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(result.calls, [])
|
||||
self.assertNotIn("<|", result.normal_text)
|
||||
|
||||
def test_nonfinite_numbers_rejected_canonically_but_recovered(self):
|
||||
"""NaN/Infinity are not valid canonical JSON, so the strict pass fails;
|
||||
recovery accepts them."""
|
||||
source = (
|
||||
"<|content_invoke_tool_json|>"
|
||||
'{"name":"weather","args":{"v":NaN}}<|end_message|>'
|
||||
)
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(result.calls[0].name, "weather")
|
||||
|
||||
def test_streaming_name_not_emitted_before_end_message(self):
|
||||
"""Atomicity: the tool name is withheld until the closing marker, so a
|
||||
call that never completes never leaks an orphan name delta."""
|
||||
detector = InklingDetector()
|
||||
pre = detector.parse_streaming_increment(
|
||||
'<|message_model|>weather<|content_invoke_tool_json|>{"name":"wea',
|
||||
self.tools,
|
||||
)
|
||||
self.assertEqual(pre.calls, [])
|
||||
post = detector.parse_streaming_increment(
|
||||
'ther","args":{"city":"SF"}}<|end_message|>', self.tools
|
||||
)
|
||||
self.assertEqual(len(post.calls), 1)
|
||||
self.assertEqual(post.calls[0].name, "weather")
|
||||
self.assertEqual(json.loads(post.calls[0].parameters), {"city": "SF"})
|
||||
|
||||
def test_raw_text_tool_invocation_surfaces_as_a_call(self):
|
||||
"""A headerless <|content_invoke_tool_text|> block reaches the tool loop
|
||||
as a call carrying the raw body, instead of being dropped."""
|
||||
source = "<|content_invoke_tool_text|>search the web<|end_message|>"
|
||||
result = InklingDetector().detect_and_parse(source, self.tools)
|
||||
self.assertEqual(len(result.calls), 1)
|
||||
self.assertEqual(
|
||||
json.loads(result.calls[0].parameters), {"text": "search the web"}
|
||||
)
|
||||
|
||||
|
||||
class TestPythonicDetector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
|
||||
@@ -211,6 +211,17 @@ class TestInklingDetector(CustomTestCase):
|
||||
content += detector.parse_streaming_increment(char).normal_text
|
||||
self.assertEqual(content, source)
|
||||
|
||||
def test_raw_text_tool_framing_is_preserved_for_the_tool_parser(self):
|
||||
"""The headerless <|content_invoke_tool_text|> block must survive into
|
||||
content so the tool-call detector can surface it, rather than being
|
||||
swallowed as header data."""
|
||||
detector = InklingDetector()
|
||||
source = "<|message_model|><|content_invoke_tool_text|>search<|end_message|>"
|
||||
result = detector.detect_and_parse(source)
|
||||
self.assertIn("<|content_invoke_tool_text|>", result.normal_text)
|
||||
self.assertIn("search", result.normal_text)
|
||||
self.assertEqual(result.reasoning_text, "")
|
||||
|
||||
def test_quoted_message_model_token_inside_content_is_preserved(self):
|
||||
"""Bug regression: the header branch flipped to header state on ANY
|
||||
<|message_model|> occurrence, so a literal token the model wrote
|
||||
|
||||
Reference in New Issue
Block a user