From 80fef637865ff8d84dce10b6c3362d69bd117e3a Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:47:56 +0800 Subject: [PATCH] [Fix] Inkling: frame constrained output in native message tokens (#44716) --- .../srt/constrained/base_grammar_backend.py | 23 +- .../sglang/srt/constrained/grammar_manager.py | 5 + .../constrained/reasoner_grammar_backend.py | 109 +++++++- .../srt/constrained/xgrammar_backend.py | 40 ++- python/sglang/srt/parser/answer_frame.py | 123 +++++++++ python/sglang/srt/parser/reasoning_parser.py | 7 + .../unit/constrained/test_grammar_manager.py | 124 ++++++++- .../test_reasoner_grammar_backend.py | 246 +++++++++++++++++- 8 files changed, 658 insertions(+), 19 deletions(-) create mode 100644 python/sglang/srt/parser/answer_frame.py diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py index 32a9ec2c77..0574e6b587 100644 --- a/python/sglang/srt/constrained/base_grammar_backend.py +++ b/python/sglang/srt/constrained/base_grammar_backend.py @@ -18,7 +18,7 @@ import logging import time from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass, field -from typing import Dict, List, NamedTuple, Optional, Tuple +from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Tuple import torch @@ -223,6 +223,27 @@ class BaseGrammarBackend: def enable_strict_thinking(self): return self._enable_strict_thinking + def get_request_grammar_key( + self, + key: Tuple[str, str], + *, + input_ids: Sequence[int], + thinking_budget: Optional[int], + ) -> Tuple[str, str]: + """Cache key for this request's grammar; may depend on how the prompt ends + and on the request's thinking budget.""" + return key + + def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None: + """Let ``token_ids`` match only token-level framing, never grammar text.""" + + def compile_framed_grammar( + self, grammar: BaseGrammarObject, frame: Callable[[dict], dict] + ) -> Optional[BaseGrammarObject]: + """Compile ``frame(content)``, an XGrammar structural tag embedding ``grammar`` + as content; None if the backend cannot embed it.""" + return None + @property def is_support_token_filter(self): return False diff --git a/python/sglang/srt/constrained/grammar_manager.py b/python/sglang/srt/constrained/grammar_manager.py index 5d5d33d296..7f46a437d8 100644 --- a/python/sglang/srt/constrained/grammar_manager.py +++ b/python/sglang/srt/constrained/grammar_manager.py @@ -168,6 +168,11 @@ class GrammarManager: key = (key_type, req.sampling_params.ebnf) elif req.sampling_params.structural_tag is not None: key = ("structural_tag", req.sampling_params.structural_tag) + key = self.grammar_backend.get_request_grammar_key( + key, + input_ids=req.origin_input_ids, + thinking_budget=self._get_request_thinking_budget(req), + ) value, cache_hit = self.grammar_backend.get_cached_or_future_value( key, req.require_reasoning diff --git a/python/sglang/srt/constrained/reasoner_grammar_backend.py b/python/sglang/srt/constrained/reasoner_grammar_backend.py index a6864ad5a6..c2e06a7cca 100644 --- a/python/sglang/srt/constrained/reasoner_grammar_backend.py +++ b/python/sglang/srt/constrained/reasoner_grammar_backend.py @@ -13,8 +13,9 @@ # ============================================================================== """The baseclass of a backend for reasoner grammar-guided constrained decoding.""" -import json import logging +from collections import OrderedDict +from concurrent.futures import Future from typing import List, Optional, Sequence, Tuple, Union import torch @@ -32,6 +33,11 @@ from .base_grammar_backend import ( logger = logging.getLogger(__name__) +_OPEN_ANSWER_VARIANT = "open_answer" +_THINK_BUDGET_VARIANT = "think_budget=" +# Arbitrary bound; budgets are a per-request knob, so only recent variants stay cached. +_BUDGET_VARIANT_CACHE_SIZE = 32 + class ReasonerGrammarObject(BaseGrammarObject): """Defers grammar constraints until the reasoning end sequence is complete.""" @@ -271,9 +277,8 @@ class ReasonerGrammarBackend(BaseGrammarBackend): ): super().__init__() self.grammar_backend = grammar_backend - think_end_ids = tokenizer.encode( - reasoning_parser.detector.think_end_token, add_special_tokens=False - ) + self._budget_variant_keys: OrderedDict[Tuple[str, str], None] = OrderedDict() + think_end_ids = reasoning_parser.detector.get_think_end_token_ids(tokenizer) if not think_end_ids: raise ValueError( f"think_end_token '{reasoning_parser.detector.think_end_token}' " @@ -281,6 +286,11 @@ class ReasonerGrammarBackend(BaseGrammarBackend): ) self.think_end_ids = think_end_ids self._enable_strict_thinking = enable_strict_thinking + self._answer_frame = reasoning_parser.detector.get_answer_frame() + if self._answer_frame is not None: + self.grammar_backend.reserve_framing_tokens( + self._answer_frame.framing_token_ids() + ) self.think_excluded_token_ids = self._get_think_excluded_token_ids( reasoning_parser, tokenizer ) @@ -347,22 +357,93 @@ class ReasonerGrammarBackend(BaseGrammarBackend): return None return self._make_grammar_object(None, reasoning) + def get_request_grammar_key( + self, + key: Tuple[str, str], + *, + input_ids: Sequence[int], + thinking_budget: Optional[int], + ) -> Tuple[str, str]: + key_type, key_string = key + if not self._is_framed(key_type): + return key + if self._answer_frame.ends_in_open_answer_block(input_ids): + return (f"{key_type}:{_OPEN_ANSWER_VARIANT}", key_string) + # Mirrors the strict wrapper, which applies a request budget only while filtering. + if not self.enable_token_filter or thinking_budget is None: + return key + budget = max(thinking_budget, -1) + if budget == max(self.max_think_tokens, -1): + return key + return (f"{key_type}:{_THINK_BUDGET_VARIANT}{budget}", key_string) + + def _is_framed(self, key_type: str) -> bool: + return ( + self._answer_frame is not None + and key_type in self._answer_frame.framed_key_types + ) + + def get_cached_or_future_value( + self, key: Tuple[str, str], require_reasoning: bool + ) -> Tuple[BaseGrammarObject | Future[BaseGrammarObject], bool]: + if key in self._budget_variant_keys: + self._budget_variant_keys.move_to_end(key) + return super().get_cached_or_future_value(key, require_reasoning) + + def set_cache(self, key: Tuple[str, str], value: BaseGrammarObject): + super().set_cache(key, value) + if _THINK_BUDGET_VARIANT not in key[0]: + return + self._budget_variant_keys[key] = None + self._budget_variant_keys.move_to_end(key) + while len(self._budget_variant_keys) > _BUDGET_VARIANT_CACHE_SIZE: + evicted, _ = self._budget_variant_keys.popitem(last=False) + self.cache.pop(evicted, None) + + def reset(self): + super().reset() + self._budget_variant_keys.clear() + def _init_value_dispatch( self, key: Tuple[str, str], reasoning: bool ) -> Optional[BaseGrammarObject]: + key_type, _, variant = key[0].partition(":") + key = (key_type, key[1]) ret = self.grammar_backend._init_value_dispatch(key, reasoning) if ret is None or isinstance(ret, InvalidGrammarObject): return ret if key[0] == "full_assistant_ebnf": return ret - if not self.enable_strict_thinking and key[0] == "structural_tag": - from sglang.srt.function_call.inkling_detector import InklingDetector - - # Only this canonical schema owns the full turn; user restrictions may not. - if json.loads(key[1]) == ( - InklingDetector() - .get_auto_tool_call_structural_tag() - .model_dump(by_alias=True) - ): - return ret + if self._is_framed(key[0]): + # The frame delimits thinking by message kind, so it owns the whole + # turn; strict thinking only caps the thinking message. + framed = self._frame_answer(ret, variant) + if framed is not None: + return framed + if ( + not self.enable_strict_thinking + and key[0] == "structural_tag" + and self._answer_frame is not None + and self._answer_frame.owns_turn(key[1]) + ): + return ret return self._make_grammar_object(ret, reasoning) + + def _frame_answer( + self, grammar: BaseGrammarObject, variant: str + ) -> Optional[BaseGrammarObject]: + in_answer_block = variant == _OPEN_ANSWER_VARIANT + if variant.startswith(_THINK_BUDGET_VARIANT): + max_think_tokens = int(variant.removeprefix(_THINK_BUDGET_VARIANT)) + elif self.enable_strict_thinking: + max_think_tokens = self.max_think_tokens + else: + max_think_tokens = -1 + return self.grammar_backend.compile_framed_grammar( + grammar, + lambda content: self._answer_frame.structural_tag( + content, + in_answer_block=in_answer_block, + max_think_tokens=max_think_tokens, + ), + ) diff --git a/python/sglang/srt/constrained/xgrammar_backend.py b/python/sglang/srt/constrained/xgrammar_backend.py index 167bee6fa4..1645eca471 100644 --- a/python/sglang/srt/constrained/xgrammar_backend.py +++ b/python/sglang/srt/constrained/xgrammar_backend.py @@ -16,7 +16,7 @@ import dataclasses import json import logging -from typing import Dict, List, Optional, Tuple, Union +from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union import torch from xgrammar import ( @@ -26,6 +26,7 @@ from xgrammar import ( StructuralTag, StructuralTagItem, TokenizerInfo, + VocabType, bitmask_dtype, get_bitmask_shape, ) @@ -241,7 +242,9 @@ class XGrammarGrammarBackend(BaseGrammarBackend): f"Failed to create XGrammar TokenizerInfo from tokenizer: {e}" ) + self._tokenizer_info = tokenizer_info self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info) + self._framing_compiler = self._new_framing_compiler(tokenizer_info) self.vocab_size = vocab_size self.override_stop_tokens = override_stop_tokens self.any_whitespace = any_whitespace @@ -251,6 +254,41 @@ class XGrammarGrammarBackend(BaseGrammarBackend): def is_support_token_filter(self): return True + @staticmethod + def _new_framing_compiler(tokenizer_info: TokenizerInfo) -> GrammarCompiler: + # The cache mixes up token-level structural tags of different shapes, so + # framed grammars compile uncached (seen on xgrammar 0.2.1 through 0.2.7). + return GrammarCompiler(tokenizer_info=tokenizer_info, cache_enabled=False) + + def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None: + info = self._tokenizer_info + reserved = set(token_ids) + # A leading 0xFF is invalid UTF-8, so framed text cannot match these while + # token elements still match by id; string tool tags spell them, so only framing. + vocab = [ + b"\xff" + piece if token_id in reserved else piece + for token_id, piece in enumerate(info.decoded_vocab) + ] + self._framing_compiler = self._new_framing_compiler( + TokenizerInfo( + vocab, + VocabType.RAW, + vocab_size=info.vocab_size, + stop_token_ids=info.stop_token_ids, + ) + ) + + def compile_framed_grammar( + self, grammar: XGrammarGrammar, frame: Callable[[dict], dict] + ) -> XGrammarGrammar: + tag = json.dumps( + frame({"type": "grammar", "grammar": str(grammar.ctx.grammar)}) + ) + ctx = self._framing_compiler.compile_structural_tag(tag) + return self._from_context( + ctx, tag, GrammarStats(dispatch_type="structural_tag") + ) + @staticmethod def allocate_vocab_mask(vocab_size: int, batch_size: int, device) -> torch.Tensor: return _allocate_token_bitmask(vocab_size, batch_size) diff --git a/python/sglang/srt/parser/answer_frame.py b/python/sglang/srt/parser/answer_frame.py new file mode 100644 index 0000000000..0ab1759a7b --- /dev/null +++ b/python/sglang/srt/parser/answer_frame.py @@ -0,0 +1,123 @@ +import functools +import json +from typing import List, Sequence, Tuple + +from sglang.srt.parser.inkling_tokenizer import ( + CONTENT_MODEL_END_SAMPLING, + CONTENT_TEXT, + CONTENT_THINKING, + END_MESSAGE, + INKLING_SPECIAL_TOKEN_IDS, + MESSAGE_MODEL, +) + + +class AnswerFrame: + """Native message framing a reasoning format wraps around constrained output. + + A frame owns the whole turn: an optional thinking message, then the answer + message holding the constrained content. + """ + + framed_key_types: Tuple[str, ...] = () + + def framing_token_ids(self) -> List[int]: + """Token IDs that may only match as frame tokens, never as content text.""" + raise NotImplementedError() + + def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool: + """Whether the prompt already opened the answer block (continuation).""" + raise NotImplementedError() + + def structural_tag( + self, content: dict, *, in_answer_block: bool, max_think_tokens: int + ) -> dict: + """XGrammar structural tag framing ``content``; ``max_think_tokens < 0`` + leaves thinking unbounded.""" + raise NotImplementedError() + + def owns_turn(self, structural_tag: str) -> bool: + """Whether this structural tag already frames the whole turn itself.""" + raise NotImplementedError() + + +@functools.cache +def _canonical_inkling_auto_tool_schema() -> dict: + # Lazy: inkling_detector imports xgrammar unconditionally, and XPU + # installs may not ship it. + from sglang.srt.function_call.inkling_detector import InklingDetector + + return ( + InklingDetector().get_auto_tool_call_structural_tag().model_dump(by_alias=True) + ) + + +def _token(name: str) -> dict: + return {"type": "token", "token": INKLING_SPECIAL_TOKEN_IDS[name]} + + +class InklingAnswerFrame(AnswerFrame): + framed_key_types = ("json", "regex", "ebnf") + + def framing_token_ids(self) -> List[int]: + # Inkling's control tokens decode to their literal spellings, so text + # positions such as JSON strings would otherwise admit <|end_message|>. + return [ + token_id + for token, token_id in INKLING_SPECIAL_TOKEN_IDS.items() + if token != CONTENT_MODEL_END_SAMPLING + ] + + def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool: + end_message = INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE] + last_end = next( + ( + i + for i in range(len(input_ids) - 1, -1, -1) + if input_ids[i] == end_message + ), + -1, + ) + return list(input_ids[last_end + 1 : last_end + 3]) == [ + INKLING_SPECIAL_TOKEN_IDS[MESSAGE_MODEL], + INKLING_SPECIAL_TOKEN_IDS[CONTENT_TEXT], + ] + + def structural_tag( + self, content: dict, *, in_answer_block: bool, max_think_tokens: int + ) -> dict: + elements = [content, _token(END_MESSAGE)] + if in_answer_block: + return { + "type": "structural_tag", + "format": {"type": "sequence", "elements": elements}, + } + thinking = { + "type": "any_tokens", + "exclude_tokens": [INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]], + } + if max_think_tokens >= 0: + thinking["max_tokens"] = max_think_tokens + thinking_message = { + "type": "sequence", + "elements": [ + _token(MESSAGE_MODEL), + _token(CONTENT_THINKING), + thinking, + _token(END_MESSAGE), + ], + } + elements = [ + {"type": "optional", "content": thinking_message}, + _token(MESSAGE_MODEL), + _token(CONTENT_TEXT), + *elements, + ] + return { + "type": "structural_tag", + "format": {"type": "sequence", "elements": elements}, + } + + def owns_turn(self, structural_tag: str) -> bool: + # Only the canonical auto-tool schema owns the full turn; user restrictions may not. + return json.loads(structural_tag) == _canonical_inkling_auto_tool_schema() diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py index 778cfcf0bc..a5f9ddeafe 100644 --- a/python/sglang/srt/parser/reasoning_parser.py +++ b/python/sglang/srt/parser/reasoning_parser.py @@ -36,6 +36,7 @@ from sglang.srt.function_call.muse_glimmer_format import ( has_atem_markers, partial_marker_len, ) +from sglang.srt.parser.answer_frame import AnswerFrame, InklingAnswerFrame from sglang.srt.parser.harmony_parser import HarmonyParser from sglang.srt.parser.inkling_tokenizer import ( CONTENT_INVOKE_TOOL_JSON, @@ -117,6 +118,9 @@ class BaseReasoningFormatDetector: def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]: return tokenizer.encode(self.think_end_token, add_special_tokens=False) + def get_answer_frame(self) -> Optional[AnswerFrame]: + return None + def _maybe_apply_force_nonempty_content( self, ret: StreamingParseResult ) -> StreamingParseResult: @@ -1376,6 +1380,9 @@ class InklingDetector(BaseReasoningFormatDetector): # Native framing IDs differ from encoding their printed names as plain text. return [INKLING_SPECIAL_TOKEN_IDS[self.think_end_token]] + def get_answer_frame(self) -> Optional[AnswerFrame]: + return InklingAnswerFrame() + def detect_and_parse(self, text: str) -> StreamingParseResult: self._buffer = "" self._kind = None diff --git a/test/registered/unit/constrained/test_grammar_manager.py b/test/registered/unit/constrained/test_grammar_manager.py index 25b9894717..d0c8e78b15 100644 --- a/test/registered/unit/constrained/test_grammar_manager.py +++ b/test/registered/unit/constrained/test_grammar_manager.py @@ -14,18 +14,35 @@ Usage: python -m pytest test_grammar_manager.py -v """ +import os import unittest from concurrent.futures import Future +from types import SimpleNamespace from unittest.mock import MagicMock, patch +from xgrammar import TokenizerInfo + +from sglang.srt.constrained import reasoner_grammar_backend from sglang.srt.constrained.base_grammar_backend import ( BaseGrammarBackend, BaseGrammarObject, InvalidGrammarObject, ) from sglang.srt.constrained.grammar_manager import GrammarManager -from sglang.srt.constrained.reasoner_grammar_backend import ReasonerGrammarObject +from sglang.srt.constrained.reasoner_grammar_backend import ( + ReasonerGrammarBackend, + ReasonerGrammarObject, +) +from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend from sglang.srt.distributed.communication_tags import P2PTag +from sglang.srt.parser.inkling_tokenizer import ( + CONTENT_MODEL_END_SAMPLING, + CONTENT_THINKING, + END_MESSAGE, + INKLING_SPECIAL_TOKEN_IDS, + MESSAGE_MODEL, +) +from sglang.srt.parser.reasoning_parser import InklingDetector from sglang.srt.runtime_context import ( SpawnRanks, get_context, @@ -104,6 +121,34 @@ def _make_req( return req +_INKLING_VOCAB_SIZE = max(INKLING_SPECIAL_TOKEN_IDS.values()) + 1 + + +def _make_strict_inkling_backend(max_think_tokens): + vocab = [""] * _INKLING_VOCAB_SIZE + for token_id in range(128): + vocab[token_id] = chr(token_id) + for token, token_id in INKLING_SPECIAL_TOKEN_IDS.items(): + vocab[token_id] = token + info = TokenizerInfo( + vocab, stop_token_ids=[INKLING_SPECIAL_TOKEN_IDS[CONTENT_MODEL_END_SAMPLING]] + ) + tokenizer = SimpleNamespace(init_xgrammar=lambda: (info, None)) + with patch.dict(os.environ, {"SGLANG_MAX_THINK_TOKENS": str(max_think_tokens)}): + return ReasonerGrammarBackend( + XGrammarGrammarBackend(tokenizer, _INKLING_VOCAB_SIZE), + SimpleNamespace(detector=InklingDetector()), + tokenizer, + enable_strict_thinking=True, + ) + + +def _token_allowed(grammar, token_id): + mask = grammar.allocate_vocab_mask(_INKLING_VOCAB_SIZE, 1, "cpu") + grammar.fill_vocab_mask(mask, 0) + return bool((int(mask[0, token_id // 32]) >> (token_id % 32)) & 1) + + class TestGrammarManagerInit(unittest.TestCase): def setUp(self): reset_context() @@ -146,6 +191,7 @@ class TestProcessReqWithGrammar(unittest.TestCase): ) mgr = GrammarManager(scheduler) mgr.grammar_backend = MagicMock(spec=BaseGrammarBackend) + mgr.grammar_backend.get_request_grammar_key.side_effect = lambda key, **_: key return mgr def test_no_constraint_returns_false(self): @@ -178,6 +224,28 @@ class TestProcessReqWithGrammar(unittest.TestCase): self.assertTrue(result) self.assertEqual(req.grammar_key, ("regex", "[a-z]+")) + def test_backend_specializes_the_key_from_the_request(self): + """The backend may pick a grammar variant from the prompt and the thinking + budget; that key, not the sampling-param key, is looked up and cached.""" + mgr = self._make_mgr() + mgr.grammar_backend.get_request_grammar_key.side_effect = lambda key, **_: ( + key[0] + ":variant", + key[1], + ) + mgr.grammar_backend.get_cached_or_future_value.return_value = (Future(), False) + + req = _make_req(regex="[a-z]+", custom_params={"thinking_budget": 3}) + req.origin_input_ids = [1, 2, 3] + mgr.process_req_with_grammar(req) + + mgr.grammar_backend.get_request_grammar_key.assert_called_once_with( + ("regex", "[a-z]+"), input_ids=[1, 2, 3], thinking_budget=3 + ) + mgr.grammar_backend.get_cached_or_future_value.assert_called_once_with( + ("regex:variant", "[a-z]+"), False + ) + self.assertEqual(req.grammar_key, ("regex:variant", "[a-z]+")) + def test_ebnf_cache_miss(self): mgr = self._make_mgr() future = Future() @@ -861,6 +929,60 @@ class TestStrictReasoningPaths(unittest.TestCase): self.assertIsNone(req.grammar) + def _serve_inkling_regex(self, mgr, budget): + params = None if budget is None else {"thinking_budget": budget} + req = _make_req(regex="yes", custom_params=params) + req.require_reasoning = True + req.origin_input_ids = [] + compiled = mgr.process_req_with_grammar(req) + if compiled: + mgr.get_ready_grammar_requests() + return compiled, req.grammar + + def _thinking_tokens_allowed(self, grammar, limit=16): + ids = INKLING_SPECIAL_TOKEN_IDS + for token_id in [ids[MESSAGE_MODEL], ids[CONTENT_THINKING]]: + grammar.accept_token(token_id) + count = 0 + while count < limit and _token_allowed(grammar, ord("x")): + grammar.accept_token(ord("x")) + count += 1 + self.assertTrue(_token_allowed(grammar, ids[END_MESSAGE])) + return count + + def test_inkling_constraint_caps_thinking_at_the_request_budget(self): + """An Inkling constraint compiles to a framed grammar, not a reasoning + wrapper; the request's thinking budget must still cap its thinking message, + both when compiled for the request and when served from the cache.""" + mgr = self._make_mgr() + mgr.grammar_backend = _make_strict_inkling_backend(max_think_tokens=10) + mgr.SGLANG_GRAMMAR_POLL_INTERVAL = 5 + served = [self._serve_inkling_regex(mgr, budget) for budget in (3, 3, 12)] + self.assertEqual([compiled for compiled, _ in served], [True, False, True]) + self.assertEqual( + [self._thinking_tokens_allowed(grammar) for _, grammar in served], + [3, 3, 12], + ) + + def test_inkling_budget_variants_stay_bounded(self): + """Equivalent budgets share one cached frame and only recent budget variants + stay cached; an evicted budget recompiles with its exact cap.""" + mgr = self._make_mgr() + mgr.grammar_backend = _make_strict_inkling_backend(max_think_tokens=10) + mgr.SGLANG_GRAMMAR_POLL_INTERVAL = 5 + budgets = [0, 1, 2, 3, 4, 5, 6, 12, None, 10, -1, -2, 0] + with patch.object(reasoner_grammar_backend, "_BUDGET_VARIANT_CACHE_SIZE", 4): + served = [self._serve_inkling_regex(mgr, budget) for budget in budgets] + self.assertEqual( + [compiled for compiled, _ in served], + [True] * 9 + [False, True, False, True], + ) + self.assertEqual( + [self._thinking_tokens_allowed(grammar) for _, grammar in served], + [0, 1, 2, 3, 4, 5, 6, 12, 10, 10, 16, 16, 0], + ) + self.assertEqual(len(mgr.grammar_backend.cache), 1 + 4) + def test_strict_constrained_request_uses_normal_dispatch(self): """Request with json_schema should go through normal dispatch, not strict path.""" mgr = self._make_mgr() diff --git a/test/registered/unit/constrained/test_reasoner_grammar_backend.py b/test/registered/unit/constrained/test_reasoner_grammar_backend.py index ba3ebbdaf2..e1fb67fd7b 100644 --- a/test/registered/unit/constrained/test_reasoner_grammar_backend.py +++ b/test/registered/unit/constrained/test_reasoner_grammar_backend.py @@ -1,3 +1,4 @@ +import itertools import json import os import unittest @@ -16,7 +17,8 @@ from sglang.srt.constrained.torch_ops.token_filter_torch_ops import ( set_token_filter_torch, ) from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend -from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest +from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest, Tool +from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.inkling_detector import ( InklingDetector as InklingToolDetector, ) @@ -174,6 +176,10 @@ class TestReasonerGrammarBackend(CustomTestCase): think_start_token="", think_end_token="", think_excluded_tokens=["", ""], + get_think_end_token_ids=lambda tokenizer: tokenizer.encode( + "", add_special_tokens=False + ), + get_answer_frame=lambda: None, ) return SimpleNamespace(detector=detector) @@ -472,13 +478,234 @@ class TestReasonerGrammarBackend(CustomTestCase): else: self.assertIs(grammar, inner_grammar) + def test_inkling_output_constraints_keep_message_framing(self): + """Constrained Inkling output is a framed text message after optional thinking.""" + backend, tokenizer = self._make_xgrammar_backend() + ids = INKLING_SPECIAL_TOKEN_IDS + keys = { + ("json", '{"type":"object","properties":{"a":{"type":"integer"}}}'): ( + '{"a": 1}' + ), + ("regex", "Answer: [0-9]+"): "Answer: 42", + ("ebnf", 'root ::= "yes" | "no"'): "yes", + } + thinking = ( + [ids[MESSAGE_MODEL], ids[CONTENT_THINKING]] + + tokenizer.encode("reason") + + [ids[END_MESSAGE]] + ) + for (key, payload), reasoning, prefix, strict in itertools.product( + keys.items(), (False, True), ([], thinking), (False, True) + ): + with self.subTest( + key=key, reasoning=reasoning, prefix=bool(prefix), strict=strict + ): + reasoner = ReasonerGrammarBackend( + backend, + SimpleNamespace(detector=InklingDetector()), + tokenizer, + enable_strict_thinking=strict, + ) + grammar = reasoner._init_value_dispatch(key, reasoning=reasoning) + self._accept_allowed_tokens(grammar, prefix, backend.vocab_size) + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids(mask, [ord(payload[0]), ids[MESSAGE_MODEL]]), + [ids[MESSAGE_MODEL]], + ) + self._accept_allowed_tokens( + grammar, [ids[MESSAGE_MODEL]], backend.vocab_size + ) + kinds = [ids[CONTENT_TEXT], ids[CONTENT_THINKING]] + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids(mask, kinds), kinds[:1] if prefix else kinds + ) + self._accept_allowed_tokens( + grammar, + [ids[CONTENT_TEXT]] + tokenizer.encode(payload), + backend.vocab_size, + ) + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids( + mask, [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]] + ), + [ids[END_MESSAGE]], + ) + self._accept_allowed_tokens( + grammar, + [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]], + backend.vocab_size, + ) + self.assertTrue(grammar.is_terminated()) + + def test_inkling_continuation_constraint_starts_inside_the_text_block(self): + """A prompt ending in an open text block constrains the payload directly, + also after an ordinary framed grammar compiled on the same backend.""" + backend, tokenizer = self._make_xgrammar_backend() + ids = INKLING_SPECIAL_TOKEN_IDS + reasoner = ReasonerGrammarBackend( + backend, SimpleNamespace(detector=InklingDetector()), tokenizer + ) + user_turn = [ids[MESSAGE_MODEL], *tokenizer.encode("hi"), ids[END_MESSAGE]] + open_block = [*user_turn, ids[MESSAGE_MODEL], ids[CONTENT_TEXT]] + key = ("regex", "yes") + self.assertEqual( + reasoner.get_request_grammar_key( + key, input_ids=user_turn, thinking_budget=None + ), + key, + ) + turn = reasoner._init_value_dispatch(key, reasoning=True) + self._accept_allowed_tokens( + turn, + [ids[MESSAGE_MODEL], ids[CONTENT_TEXT]] + + tokenizer.encode("yes") + + [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]], + backend.vocab_size, + ) + + block_key = reasoner.get_request_grammar_key( + key, input_ids=open_block, thinking_budget=None + ) + self.assertNotEqual(block_key, key) + grammar = reasoner._init_value_dispatch(block_key, reasoning=True) + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids(mask, [ord("y"), ids[MESSAGE_MODEL]]), [ord("y")] + ) + self._accept_allowed_tokens( + grammar, + tokenizer.encode("yes") + + [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]], + backend.vocab_size, + ) + self.assertTrue(grammar.is_terminated()) + + def test_inkling_framing_tokens_never_match_payload_text(self): + """Control tokens decode to their spellings; inside a JSON string an Inkling + constraint must not admit <|end_message|>, which would close the message.""" + ids = INKLING_SPECIAL_TOKEN_IDS + key = ("json", '{"type":"object","properties":{"a":{"type":"string"}}}') + for inkling in (True, False): + with self.subTest(inkling=inkling): + backend, tokenizer = self._make_xgrammar_backend() + if inkling: + parser = SimpleNamespace(detector=InklingDetector()) + reasoner = ReasonerGrammarBackend(backend, parser, tokenizer) + head = [ids[MESSAGE_MODEL], ids[CONTENT_TEXT]] + else: + reasoner = ReasonerGrammarBackend( + backend, self._make_parser(), tokenizer + ) + head = tokenizer.encode("") + grammar = reasoner._init_value_dispatch(key, reasoning=True) + self._accept_allowed_tokens( + grammar, head + tokenizer.encode('{"a": "x'), backend.vocab_size + ) + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids(mask, [ord("y"), ids[END_MESSAGE]]), + [ord("y")] if inkling else [ord("y"), ids[END_MESSAGE]], + ) + + def test_inkling_required_tool_tag_matches_native_markers(self): + """Required and named tool tags spell Inkling's markers as strings; the native + marker tokens must still match them while payload framing is enabled.""" + backend, tokenizer = self._make_xgrammar_backend() + ReasonerGrammarBackend( + backend, SimpleNamespace(detector=InklingDetector()), tokenizer + ) + tools = [Tool(type="function", function={"name": "f", "parameters": {}})] + request = ChatCompletionRequest( + model="inkling", messages=[], tools=tools, tool_choice="required" + ) + params = request.to_sampling_params( + stop=[], + model_generation_config={}, + tool_call_constraint=FunctionCallParser( + tools, "inkling" + ).get_structure_constraint("required"), + ) + grammar = backend.dispatch_structural_tag(params["structural_tag"]) + ids = INKLING_SPECIAL_TOKEN_IDS + self._accept_allowed_tokens( + grammar, + [ids[MESSAGE_MODEL], *tokenizer.encode("f"), ids[CONTENT_INVOKE_TOOL_JSON]] + + tokenizer.encode('{"name":"f","args":{}}') + + [ids[END_MESSAGE]], + backend.vocab_size, + ) + + def test_inkling_strict_budget_caps_thinking_and_cannot_reopen_it(self): + """Strict thinking caps the thinking message; after its end a header may only + open the answer, and a turn that skips thinking is held to the payload.""" + backend, tokenizer = self._make_xgrammar_backend() + ids = INKLING_SPECIAL_TOKEN_IDS + with patch.dict(os.environ, {"SGLANG_MAX_THINK_TOKENS": "3"}): + reasoner = ReasonerGrammarBackend( + backend, + SimpleNamespace(detector=InklingDetector()), + tokenizer, + enable_strict_thinking=True, + ) + grammar = reasoner._init_value_dispatch(("regex", "yes"), reasoning=True) + self._accept_allowed_tokens( + grammar, + [ids[MESSAGE_MODEL], ids[CONTENT_THINKING], *tokenizer.encode("xxx")], + backend.vocab_size, + ) + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids(mask, [ord("x"), ids[END_MESSAGE]]), [ids[END_MESSAGE]] + ) + self._accept_allowed_tokens( + grammar, [ids[END_MESSAGE], ids[MESSAGE_MODEL]], backend.vocab_size + ) + mask = self._vocab_mask(grammar, backend.vocab_size) + self.assertEqual( + _allowed_token_ids(mask, [ids[CONTENT_TEXT], ids[CONTENT_THINKING]]), + [ids[CONTENT_TEXT]], + ) + + skipped = reasoner._init_value_dispatch(("regex", "yes"), reasoning=True) + self._accept_allowed_tokens( + skipped, [ids[MESSAGE_MODEL], ids[CONTENT_TEXT]], backend.vocab_size + ) + mask = self._vocab_mask(skipped, backend.vocab_size) + self.assertEqual(_allowed_token_ids(mask, [ord("h"), ord("y")]), [ord("y")]) + + def test_inkling_backend_matches_native_think_end_id(self): + """Strict mode and other constraints must see the native END_MESSAGE id.""" + tokenizer = _DummyTokenizer({END_MESSAGE: [27, 91, 406, 62, 65752, 91, 29]}) + for strict in (False, True): + with self.subTest(strict=strict): + reasoner = ReasonerGrammarBackend( + _DummyGrammarBackend(), + SimpleNamespace(detector=InklingDetector()), + tokenizer, + enable_strict_thinking=strict, + ) + self.assertEqual( + reasoner.think_end_ids, [INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]] + ) + + grammar = reasoner._make_grammar_object(MagicMock(), reasoning=True) + self.assertFalse(grammar._is_generation()) + grammar.accept_token(INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]) + self.assertTrue(grammar._is_generation()) + def test_other_grammars_still_defer_until_reasoning_ends(self): - tag = InklingToolDetector().get_auto_tool_call_structural_tag().model_dump() + canonical = InklingToolDetector().get_auto_tool_call_structural_tag() + tag = canonical.model_dump() tag["format"] = {"type": "sequence", "elements": [tag["format"]]} keys = [ ("json", "{}"), ("structural_tag", '{"structures": [], "triggers": []}'), ("structural_tag", json.dumps(tag)), + # The canonical schema owns the full turn only under the Inkling detector. + ("structural_tag", canonical.model_dump_json()), ] for key in keys: with self.subTest(key=key): @@ -493,6 +720,21 @@ class TestReasonerGrammarBackend(CustomTestCase): self.assertIs(grammar.grammar, backend._dispatch_result) self.assertFalse(grammar._is_generation()) + def test_inkling_constraint_defers_on_a_backend_that_cannot_frame(self): + """Backends without compile_framed_grammar keep the deferred constraint.""" + backend = _DummyGrammarBackend() + backend._dispatch_result = MagicMock() + reasoner = ReasonerGrammarBackend( + backend, + SimpleNamespace(detector=InklingDetector()), + _DummyTokenizer({}), + ) + grammar = reasoner._init_value_dispatch(("json", "{}"), reasoning=True) + + self.assertIsInstance(grammar, ReasonerGrammarObject) + self.assertIs(grammar.grammar, backend._dispatch_result) + self.assertFalse(grammar._is_generation()) + def test_full_assistant_ebnf_still_owns_its_boundaries(self): """Whole-assistant EBNF must keep control of the reasoning prefix.""" for strict in (False, True):