Files
sglang/test/registered/unit/entrypoints/openai/test_serving_inkling.py
T

661 lines
26 KiB
Python

from sglang.test.test_utils import CustomTestCase, maybe_stub_sgl_kernel
maybe_stub_sgl_kernel() # must precede any import that pulls in sgl_kernel
import json
import unittest
from types import SimpleNamespace
from unittest.mock import Mock
from sglang.srt.entrypoints.openai import serving_inkling
from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
from sglang.srt.entrypoints.openai.serving_chat import OpenAIServingChat
from sglang.srt.environ import envs
from sglang.srt.parser.inkling_renderer import load_tml_renderers
from sglang.srt.parser.inkling_tokenizer import INKLING_SPECIAL_TOKEN_IDS
from sglang.srt.runtime_context import get_context, publish, reset_context
from sglang.srt.server_args import ServerArgs
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=12, suite="base-a-test-cpu")
def _inkling_serving() -> OpenAIServingChat:
serving = object.__new__(OpenAIServingChat)
serving.chat_encoding_spec = "inkling"
serving.reasoning_parser = "inkling"
serving.tool_call_parser = "inkling"
serving.tokenizer_manager = Mock(tokenizer=None)
serving._encoding_adapter = serving._build_chat_encoding_adapter()
return serving
class InklingReasoningEffortTest(unittest.TestCase):
def test_parse_reasoning_effort(self):
"""Named levels are external literals; scalars are range-checked."""
cases = [
("none", 0.0),
("minimal", 0.1),
("low", 0.2),
("medium", 0.7),
("high", 0.9),
# "xhigh" and "max" are aliases for the same 0.99 ceiling
("xhigh", 0.99),
("max", 0.99),
(0.5, 0.5),
(0.99, 0.99),
(None, None),
]
for value, expected in cases:
with self.subTest(value=value):
self.assertEqual(
serving_inkling.parse_reasoning_effort(value), expected
)
for value in (1.0, "1.0", 2.0, "1.5", -1.0, float("nan"), True, "garbage"):
with self.subTest(value=value), self.assertRaises(ValueError):
serving_inkling.parse_reasoning_effort(value)
def test_env_default(self):
get = serving_inkling.get_default_reasoning_effort
env = envs.SGLANG_INKLING_DEFAULT_REASONING_EFFORT
try:
env.clear() # unset -> EnvStr default "0.9"
self.assertEqual(get(), 0.9)
env.set("") # explicit empty still uses the protocol default
self.assertEqual(get(), 0.9)
env.set("0.7")
self.assertEqual(get(), 0.7)
for value in ("1.0", "1.1", "garbage"):
env.set(value)
with self.subTest(value=value), self.assertRaises(ValueError):
get()
finally:
env.clear()
def test_thinking_disabled_maps_to_no_thinking_effort(self):
"""Bug regression: Inkling is an always-on parser, so Anthropic
thinking={"type": "disabled"} was rejected outright even though effort
"none" (0.0) expresses exactly that."""
serving = object.__new__(OpenAIServingChat)
serving.reasoning_parser = "inkling"
serving.template_manager = Mock(reasoning_config=None)
serving._reasoning_detector = Mock(reasoning_default="always")
request = ChatCompletionRequest(
model="test-model", messages=[{"role": "user", "content": "hi"}]
)
serving.apply_reasoning_enabled(request, False)
self.assertEqual(request.reasoning_effort, "none")
# Enabling leaves an effort set via output_config.effort alone.
request.reasoning_effort = "low"
serving.apply_reasoning_enabled(request, True)
self.assertEqual(request.reasoning_effort, "low")
class InklingPromptTest(unittest.TestCase):
def setUp(self):
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
self.serving = _inkling_serving()
def _encode(self, messages, **request_kwargs) -> list[int]:
request = ChatCompletionRequest(
model="test-model",
messages=messages,
reasoning_effort=0.5,
**request_kwargs,
)
return self.serving._encode_messages(
[message.model_dump() for message in request.messages],
request,
thinking_mode=None,
image_data=[],
)
def test_prompt_tail(self):
"""Ordinary requests end at the last closed message; continue_final_message
opens a model text block unless the trailing turn carries tool calls."""
special = INKLING_SPECIAL_TOKEN_IDS
end_sampling = special["<|content_model_end_sampling|>"]
tool_call_turn = {
"role": "assistant",
"content": "calling",
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "weather", "arguments": "{}"},
}
],
}
open_block = [
special["<|message_model|>"],
special["<|content_text|>"],
*load_tml_renderers().tokenizer.encode_ordinary("The answer"),
]
user = {"role": "user", "content": "hello"}
cases = [
("no model prefill", [user], False, [special["<|end_message|>"]]),
(
"continuation opens a text block",
[user, {"role": "assistant", "content": "The answer"}],
True,
open_block,
),
(
"tool-call turn stays closed",
[user, tool_call_turn],
True,
[end_sampling],
),
]
for name, messages, continue_final_message, tail in cases:
with self.subTest(case=name):
prompt_ids = self._encode(
messages, continue_final_message=continue_final_message
)
self.assertEqual(prompt_ids[-len(tail) :], tail)
if name == "continuation opens a text block":
self.assertNotIn(end_sampling, prompt_ids)
def test_reasoning_content_history_renders_as_thinking(self):
"""Prior thinking sent as reasoning_content (Chat Completions, and the
Anthropic and Responses converters) must render like a thinking part."""
def encode(assistant):
return self._encode(
[
{"role": "user", "content": "hi"},
assistant,
{"role": "user", "content": "again"},
]
)
self.assertEqual(
encode(
{"role": "assistant", "reasoning_content": "ponder", "content": "ok"}
),
encode(
{
"role": "assistant",
"content": [
{"type": "thinking", "thinking": "ponder"},
{"type": "text", "text": "ok"},
],
}
),
)
class InklingTokenOutputTest(CustomTestCase):
"""Inkling chat responses are parsed from output token IDs, not text."""
def setUp(self):
super().setUp()
reset_context()
self.addCleanup(reset_context)
publish(ServerArgs(model_path="dummy"), role="tokenizer")
self.tokenizer = load_tml_renderers().tokenizer
self.inkling = _inkling_serving()._encoding_adapter
self.request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "weather?"}],
tools=[
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}},
}
],
)
def _output_ids(self) -> list[int]:
special = self.tokenizer.encode_special
return [
special("message_model"),
special("content_thinking"),
*self.tokenizer.encode_ordinary("plan"),
special("end_message"),
special("message_model"),
*self.tokenizer.encode_ordinary("weather"),
special("content_invoke_tool_json"),
*self.tokenizer.encode_ordinary('{"name":"weather","args":{"city":"SF"}}'),
special("end_message"),
special("content_model_end_sampling"),
]
def _stream_deltas(
self,
request,
output_ids,
finish_reason,
*,
incremental=False,
split=4,
has_tool_calls=None,
) -> list[dict]:
parser_dict, chunks = {}, []
steps = (
(output_ids[:split], None),
(output_ids[split:] if incremental else output_ids, finish_reason),
)
with get_context().override_server_args(
incremental_streaming_output=incremental
):
for ids, finish in steps:
chunks += self.inkling.stream_chat_chunks(
content={
"output_ids": ids,
"meta_info": {
"id": "chatcmpl-1",
"completion_tokens": len(output_ids),
"finish_reason": finish,
},
},
index=0,
request=request,
created=0,
parser_dict=parser_dict,
has_tool_calls={} if has_tool_calls is None else has_tool_calls,
choice_logprobs=None,
finish_reason_type=finish and finish["type"],
usage=None,
)
return [
json.loads(chunk[len("data: ") :])["choices"][0]["delta"]
for chunk in chunks
]
def _reasoning_and_content(
self, request, output_ids, finish_reason, *, split=2
) -> dict:
reasoning, content, _, _ = self.inkling.parse_chat_response(
request=request,
output_ids=output_ids,
finish_reason=finish_reason,
)
results = {"non-stream": (reasoning or "", content)}
for incremental in (False, True):
deltas = self._stream_deltas(
request,
output_ids,
finish_reason,
incremental=incremental,
split=split,
)
results[f"stream incremental={incremental}"] = (
"".join(d.get("reasoning_content") or "" for d in deltas),
"".join(d.get("content") or "" for d in deltas),
)
return results
def test_tool_call_output(self):
"""Calls map to OpenAI tool calls in bulk and as one complete streamed call;
tool_choice="none" and separate_reasoning=False route them back to text."""
finish = {"type": "stop", "matched": 200006}
with self.subTest(case="non-stream"):
reasoning, content, tool_calls, finish_reason = (
self.inkling.parse_chat_response(
request=self.request,
output_ids=self._output_ids(),
finish_reason=finish,
)
)
self.assertEqual(reasoning, "plan")
self.assertEqual(content, "")
self.assertEqual(finish_reason, {"type": "tool_calls", "matched": None})
self.assertEqual(len(tool_calls), 1)
self.assertEqual(tool_calls[0].index, 0)
self.assertEqual(tool_calls[0].function.name, "weather")
self.assertEqual(
json.loads(tool_calls[0].function.arguments), {"city": "SF"}
)
self.assertTrue(tool_calls[0].id.startswith("call_"))
with self.subTest(case="stream"):
has_tool_calls = {}
deltas = self._stream_deltas(
self.request,
self._output_ids(),
finish,
has_tool_calls=has_tool_calls,
)
self.assertEqual(
"".join(d.get("reasoning_content") or "" for d in deltas), "plan"
)
calls = [call for d in deltas for call in d.get("tool_calls") or []]
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0]["function"]["name"], "weather")
self.assertEqual(
json.loads(calls[0]["function"]["arguments"]), {"city": "SF"}
)
self.assertEqual(has_tool_calls, {0: True})
payload = '{"name":"weather","args":{"city":"SF"}}'
switches = [
({"tool_choice": "none"}, ("plan", payload), "stop"),
({"separate_reasoning": False}, ("", "plan"), "tool_calls"),
]
for update, expected, finish_type in switches:
request = self.request.model_copy(update=update)
_, _, tool_calls, finish_reason = self.inkling.parse_chat_response(
request=request,
output_ids=self._output_ids(),
finish_reason=finish,
)
results = self._reasoning_and_content(request, self._output_ids(), finish)
with self.subTest(update=update):
self.assertEqual(finish_reason["type"], finish_type)
self.assertEqual(tool_calls is None, finish_type == "stop")
for mode, result in results.items():
with self.subTest(mode=mode):
self.assertEqual(result, expected)
def test_each_output_token_is_parsed_once(self):
"""Non-incremental chunks carry the cumulative ids and share the live output
list, and an incremental abort chunk re-sends the last id; a token parsed
twice, or before its own chunk, would duplicate or leak text."""
cases = [
([1, 2, 3, 4], 4, None, False, [4]),
([3, 4], 4, None, True, [3, 4]),
([4], 3, "abort", True, []),
([4], 4, "abort", True, [4]),
]
for output_ids, completion_tokens, finish, incremental, expected in cases:
with self.subTest(output_ids=output_ids, finish=finish):
self.assertEqual(
serving_inkling.select_new_output_ids(
output_ids=output_ids,
num_consumed_tokens=3,
completion_tokens=completion_tokens,
finish_reason_type=finish,
incremental=incremental,
),
expected,
)
# Tokens appended after a chunk was built (here the matched stop)
# must wait for their own chunk.
special = self.tokenizer.encode_special
request = ChatCompletionRequest(
model="test-model",
messages=[{"role": "user", "content": "Yes?"}],
stop=["."],
)
live = [
special("message_model"),
special("content_text"),
*self.tokenizer.encode_ordinary("Yes"),
]
first = {"completion_tokens": len(live), "finish_reason": None}
live.extend(self.tokenizer.encode_ordinary(" ."))
finish = {"type": "stop", "matched": "."}
final = {"completion_tokens": len(live), "finish_reason": finish}
chunks, parser_dict = [], {}
with get_context().override_server_args(incremental_streaming_output=False):
for meta_info, ids in ((first, live), (final, list(live))):
chunks += self.inkling.stream_chat_chunks(
content={
"output_ids": ids,
"meta_info": {"id": "chatcmpl-1", **meta_info},
},
index=0,
request=request,
created=0,
parser_dict=parser_dict,
has_tool_calls={},
choice_logprobs=None,
finish_reason_type=meta_info["finish_reason"]
and meta_info["finish_reason"]["type"],
usage=None,
)
streamed = "".join(
json.loads(chunk[len("data: ") :])["choices"][0]["delta"].get("content")
or ""
for chunk in chunks
)
_, content, _, _ = self.inkling.parse_chat_response(
request=request, output_ids=list(live), finish_reason=finish
)
self.assertEqual(content, "Yes ")
self.assertEqual(streamed, content)
def test_matched_stop_is_trimmed(self):
"""Matched stops are trimmed where the sampler matched them: inside a token,
across tokens, at the earliest of overlapping matches, never across message
framing, and for special stops inside an open text block."""
encode = self.tokenizer.encode_ordinary
special = self.tokenizer.encode_special
text_block = [special("message_model"), special("content_text")]
thinking_block = [special("message_model"), special("content_thinking")]
bang = encode("!")
self.assertEqual(len(encode(" hello")), 1)
first_block = [*text_block, *encode("EN"), special("end_message")]
split_end = [*first_block, *text_block, *encode("D and actual END")]
stop_id = special("content_thinking")
hi = [{"role": "user", "content": "hi"}]
continued = [
{"role": "user", "content": "First five primes?"},
{"role": "assistant", "content": "2, 3,"},
]
# (name, messages, output_ids, matched, split, {no_stop_trim: expected})
cases = [
(
"string spanning tokens",
hi,
text_block + encode("hello") + encode("EN") + encode("D"),
"END",
2,
{False: ("", "hello")},
),
(
"string inside a token",
hi,
text_block + encode("say") + encode(" hello"),
"hel",
2,
{False: ("", "say "), True: ("", "say hel")},
),
(
"string in reasoning",
hi,
thinking_block + encode("plan") + encode("END"),
"END",
2,
{False: ("plan", "")},
),
(
"ordinary stop token",
hi,
text_block + encode("hello") + bang,
bang[0],
2,
{False: ("", "hello"), True: ("", "hello!")},
),
(
"stop split by message framing",
hi,
split_end,
"END",
len(first_block),
{False: ("", "END and actual "), True: ("", "END and actual END")},
),
(
"special stop in an explicit block",
hi,
[*text_block, *encode(" 5, 7"), stop_id],
stop_id,
3,
{False: ("", " 5, 7"), True: ("", " 5, 7<|content_thinking|>")},
),
(
"special stop in a continued block",
continued,
[*encode(" 5, 7"), stop_id],
stop_id,
1,
{False: ("", " 5, 7"), True: ("", " 5, 7<|content_thinking|>")},
),
]
for previous, final, stop in (("\n", "\n\n", "\n\n"), ("ab", "aba", "aba")):
sampled = [*encode(previous), *encode(final)]
self.assertEqual(len(sampled), 2)
cases.append(
(
f"overlapping stop {stop!r}",
hi,
text_block + sampled,
stop,
2,
{False: ("", ""), True: ("", stop)},
)
)
for name, messages, output_ids, matched, split, by_trim in cases:
for no_stop_trim, expected in by_trim.items():
request = ChatCompletionRequest(
model="test-model",
messages=messages,
continue_final_message=messages is continued,
no_stop_trim=no_stop_trim,
)
results = self._reasoning_and_content(
request,
output_ids,
{"type": "stop", "matched": matched},
split=split,
)
for mode, result in results.items():
with self.subTest(case=name, no_stop_trim=no_stop_trim, mode=mode):
self.assertEqual(result, expected)
def test_headerless_output(self):
"""Headerless fallback payloads remain content, whether they start the
response or follow thinking, and a stray ordinary prefix must not erase
the following thinking or tool-call discriminator."""
encode = self.tokenizer.encode_ordinary
special = self.tokenizer.encode_special
eos = special("content_model_end_sampling")
payload = '{"marker": "<|end_message|>", "ok": true}'
thinking = [
special("message_model"),
special("content_thinking"),
*encode("plan"),
special("end_message"),
]
# Recovery can reopen text after a content-kind token;
# a terminal EOS must still stay invisible.
with_kind_token = [*encode('{"a": "'), special("content_text"), *encode('x"}')]
unframed_thinking = [
*encode("\n\n"),
special("content_thinking"),
*encode("plan"),
special("end_message"),
special("message_model"),
special("content_text"),
*encode("answer"),
special("end_message"),
]
cases = [
("after thinking", [*thinking, *encode(payload)], ("plan", payload)),
("from first token", encode(payload), ("", payload)),
("content kind in payload", with_kind_token, ("", '{"a": "x"}')),
("prefix before thinking", unframed_thinking, ("plan", "\n\nanswer")),
]
for name, sampled, expected in cases:
results = self._reasoning_and_content(
self.request, [*sampled, eos], {"type": "stop", "matched": eos}
)
for mode, result in results.items():
with self.subTest(case=name, mode=mode):
self.assertEqual(result, expected)
call = [
*encode(" "),
special("content_invoke_tool_json"),
*encode('{"name":"weather","args":{"city":"SF"}}'),
special("end_message"),
eos,
]
_, _, tool_calls, finish_reason = self.inkling.parse_chat_response(
request=self.request,
output_ids=call,
finish_reason={"type": "stop", "matched": eos},
)
self.assertEqual(finish_reason["type"], "tool_calls")
self.assertEqual(
[(c.function.name, json.loads(c.function.arguments)) for c in tool_calls],
[("weather", {"city": "SF"})],
)
def test_constrained_continuation_stays_in_the_open_text_block(self):
"""The rendered continuation must select an open-answer grammar
and parse its sampled suffix without leaking a header."""
from xgrammar import TokenizerInfo, VocabType
from sglang.srt.constrained.reasoner_grammar_backend import (
ReasonerGrammarBackend,
)
from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend
from sglang.srt.parser.reasoning_parser import InklingDetector
special = self.tokenizer.encode_special
eos = special("content_model_end_sampling")
vocab = []
for token_id in range(201088):
try:
vocab.append(self.tokenizer.decode_bytes([token_id]))
except ValueError:
vocab.append(b"")
info = TokenizerInfo(vocab, VocabType.RAW, stop_token_ids=[eos])
backend = XGrammarGrammarBackend(
SimpleNamespace(init_xgrammar=lambda: (info, None)), len(vocab)
)
reasoner = ReasonerGrammarBackend(
backend, SimpleNamespace(detector=InklingDetector()), None
)
request = ChatCompletionRequest(
model="test-model",
messages=[
{"role": "user", "content": "Answer yes."},
{"role": "assistant", "content": "Answer: "},
],
continue_final_message=True,
regex="yes",
)
prompt_ids = self.inkling.encode_messages(
messages=[message.model_dump() for message in request.messages],
request=request,
tools=None,
thinking=False,
image_data=[],
)
key = reasoner.get_request_grammar_key(
("regex", "yes"), input_ids=prompt_ids, thinking_budget=None
)
grammar = reasoner._init_value_dispatch(key, reasoning=True)
output_ids = [
*self.tokenizer.encode_ordinary("yes"),
special("end_message"),
eos,
]
mask = backend.allocate_vocab_mask(len(vocab), 1, "cpu")
for token_id in [special("message_model"), *output_ids]:
grammar.fill_vocab_mask(mask, 0)
allowed = bool(int(mask[0, token_id // 32]) >> (token_id % 32) & 1)
self.assertEqual(allowed, token_id != special("message_model"))
if allowed:
grammar.accept_token(token_id)
self.assertTrue(grammar.is_terminated())
results = self._reasoning_and_content(
request, output_ids, {"type": "stop", "matched": eos}, split=1
)
for mode, result in results.items():
with self.subTest(mode=mode):
self.assertEqual(result, ("", "yes"))
if __name__ == "__main__":
unittest.main(verbosity=2)