diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py
index 32a9ec2c77..0574e6b587 100644
--- a/python/sglang/srt/constrained/base_grammar_backend.py
+++ b/python/sglang/srt/constrained/base_grammar_backend.py
@@ -18,7 +18,7 @@ import logging
import time
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
-from typing import Dict, List, NamedTuple, Optional, Tuple
+from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Tuple
import torch
@@ -223,6 +223,27 @@ class BaseGrammarBackend:
def enable_strict_thinking(self):
return self._enable_strict_thinking
+ def get_request_grammar_key(
+ self,
+ key: Tuple[str, str],
+ *,
+ input_ids: Sequence[int],
+ thinking_budget: Optional[int],
+ ) -> Tuple[str, str]:
+ """Cache key for this request's grammar; may depend on how the prompt ends
+ and on the request's thinking budget."""
+ return key
+
+ def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None:
+ """Let ``token_ids`` match only token-level framing, never grammar text."""
+
+ def compile_framed_grammar(
+ self, grammar: BaseGrammarObject, frame: Callable[[dict], dict]
+ ) -> Optional[BaseGrammarObject]:
+ """Compile ``frame(content)``, an XGrammar structural tag embedding ``grammar``
+ as content; None if the backend cannot embed it."""
+ return None
+
@property
def is_support_token_filter(self):
return False
diff --git a/python/sglang/srt/constrained/grammar_manager.py b/python/sglang/srt/constrained/grammar_manager.py
index 5d5d33d296..7f46a437d8 100644
--- a/python/sglang/srt/constrained/grammar_manager.py
+++ b/python/sglang/srt/constrained/grammar_manager.py
@@ -168,6 +168,11 @@ class GrammarManager:
key = (key_type, req.sampling_params.ebnf)
elif req.sampling_params.structural_tag is not None:
key = ("structural_tag", req.sampling_params.structural_tag)
+ key = self.grammar_backend.get_request_grammar_key(
+ key,
+ input_ids=req.origin_input_ids,
+ thinking_budget=self._get_request_thinking_budget(req),
+ )
value, cache_hit = self.grammar_backend.get_cached_or_future_value(
key, req.require_reasoning
diff --git a/python/sglang/srt/constrained/reasoner_grammar_backend.py b/python/sglang/srt/constrained/reasoner_grammar_backend.py
index a6864ad5a6..c2e06a7cca 100644
--- a/python/sglang/srt/constrained/reasoner_grammar_backend.py
+++ b/python/sglang/srt/constrained/reasoner_grammar_backend.py
@@ -13,8 +13,9 @@
# ==============================================================================
"""The baseclass of a backend for reasoner grammar-guided constrained decoding."""
-import json
import logging
+from collections import OrderedDict
+from concurrent.futures import Future
from typing import List, Optional, Sequence, Tuple, Union
import torch
@@ -32,6 +33,11 @@ from .base_grammar_backend import (
logger = logging.getLogger(__name__)
+_OPEN_ANSWER_VARIANT = "open_answer"
+_THINK_BUDGET_VARIANT = "think_budget="
+# Arbitrary bound; budgets are a per-request knob, so only recent variants stay cached.
+_BUDGET_VARIANT_CACHE_SIZE = 32
+
class ReasonerGrammarObject(BaseGrammarObject):
"""Defers grammar constraints until the reasoning end sequence is complete."""
@@ -271,9 +277,8 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
):
super().__init__()
self.grammar_backend = grammar_backend
- think_end_ids = tokenizer.encode(
- reasoning_parser.detector.think_end_token, add_special_tokens=False
- )
+ self._budget_variant_keys: OrderedDict[Tuple[str, str], None] = OrderedDict()
+ think_end_ids = reasoning_parser.detector.get_think_end_token_ids(tokenizer)
if not think_end_ids:
raise ValueError(
f"think_end_token '{reasoning_parser.detector.think_end_token}' "
@@ -281,6 +286,11 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
)
self.think_end_ids = think_end_ids
self._enable_strict_thinking = enable_strict_thinking
+ self._answer_frame = reasoning_parser.detector.get_answer_frame()
+ if self._answer_frame is not None:
+ self.grammar_backend.reserve_framing_tokens(
+ self._answer_frame.framing_token_ids()
+ )
self.think_excluded_token_ids = self._get_think_excluded_token_ids(
reasoning_parser, tokenizer
)
@@ -347,22 +357,93 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
return None
return self._make_grammar_object(None, reasoning)
+ def get_request_grammar_key(
+ self,
+ key: Tuple[str, str],
+ *,
+ input_ids: Sequence[int],
+ thinking_budget: Optional[int],
+ ) -> Tuple[str, str]:
+ key_type, key_string = key
+ if not self._is_framed(key_type):
+ return key
+ if self._answer_frame.ends_in_open_answer_block(input_ids):
+ return (f"{key_type}:{_OPEN_ANSWER_VARIANT}", key_string)
+ # Mirrors the strict wrapper, which applies a request budget only while filtering.
+ if not self.enable_token_filter or thinking_budget is None:
+ return key
+ budget = max(thinking_budget, -1)
+ if budget == max(self.max_think_tokens, -1):
+ return key
+ return (f"{key_type}:{_THINK_BUDGET_VARIANT}{budget}", key_string)
+
+ def _is_framed(self, key_type: str) -> bool:
+ return (
+ self._answer_frame is not None
+ and key_type in self._answer_frame.framed_key_types
+ )
+
+ def get_cached_or_future_value(
+ self, key: Tuple[str, str], require_reasoning: bool
+ ) -> Tuple[BaseGrammarObject | Future[BaseGrammarObject], bool]:
+ if key in self._budget_variant_keys:
+ self._budget_variant_keys.move_to_end(key)
+ return super().get_cached_or_future_value(key, require_reasoning)
+
+ def set_cache(self, key: Tuple[str, str], value: BaseGrammarObject):
+ super().set_cache(key, value)
+ if _THINK_BUDGET_VARIANT not in key[0]:
+ return
+ self._budget_variant_keys[key] = None
+ self._budget_variant_keys.move_to_end(key)
+ while len(self._budget_variant_keys) > _BUDGET_VARIANT_CACHE_SIZE:
+ evicted, _ = self._budget_variant_keys.popitem(last=False)
+ self.cache.pop(evicted, None)
+
+ def reset(self):
+ super().reset()
+ self._budget_variant_keys.clear()
+
def _init_value_dispatch(
self, key: Tuple[str, str], reasoning: bool
) -> Optional[BaseGrammarObject]:
+ key_type, _, variant = key[0].partition(":")
+ key = (key_type, key[1])
ret = self.grammar_backend._init_value_dispatch(key, reasoning)
if ret is None or isinstance(ret, InvalidGrammarObject):
return ret
if key[0] == "full_assistant_ebnf":
return ret
- if not self.enable_strict_thinking and key[0] == "structural_tag":
- from sglang.srt.function_call.inkling_detector import InklingDetector
-
- # Only this canonical schema owns the full turn; user restrictions may not.
- if json.loads(key[1]) == (
- InklingDetector()
- .get_auto_tool_call_structural_tag()
- .model_dump(by_alias=True)
- ):
- return ret
+ if self._is_framed(key[0]):
+ # The frame delimits thinking by message kind, so it owns the whole
+ # turn; strict thinking only caps the thinking message.
+ framed = self._frame_answer(ret, variant)
+ if framed is not None:
+ return framed
+ if (
+ not self.enable_strict_thinking
+ and key[0] == "structural_tag"
+ and self._answer_frame is not None
+ and self._answer_frame.owns_turn(key[1])
+ ):
+ return ret
return self._make_grammar_object(ret, reasoning)
+
+ def _frame_answer(
+ self, grammar: BaseGrammarObject, variant: str
+ ) -> Optional[BaseGrammarObject]:
+ in_answer_block = variant == _OPEN_ANSWER_VARIANT
+ if variant.startswith(_THINK_BUDGET_VARIANT):
+ max_think_tokens = int(variant.removeprefix(_THINK_BUDGET_VARIANT))
+ elif self.enable_strict_thinking:
+ max_think_tokens = self.max_think_tokens
+ else:
+ max_think_tokens = -1
+ return self.grammar_backend.compile_framed_grammar(
+ grammar,
+ lambda content: self._answer_frame.structural_tag(
+ content,
+ in_answer_block=in_answer_block,
+ max_think_tokens=max_think_tokens,
+ ),
+ )
diff --git a/python/sglang/srt/constrained/xgrammar_backend.py b/python/sglang/srt/constrained/xgrammar_backend.py
index 167bee6fa4..1645eca471 100644
--- a/python/sglang/srt/constrained/xgrammar_backend.py
+++ b/python/sglang/srt/constrained/xgrammar_backend.py
@@ -16,7 +16,7 @@
import dataclasses
import json
import logging
-from typing import Dict, List, Optional, Tuple, Union
+from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
import torch
from xgrammar import (
@@ -26,6 +26,7 @@ from xgrammar import (
StructuralTag,
StructuralTagItem,
TokenizerInfo,
+ VocabType,
bitmask_dtype,
get_bitmask_shape,
)
@@ -241,7 +242,9 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
f"Failed to create XGrammar TokenizerInfo from tokenizer: {e}"
)
+ self._tokenizer_info = tokenizer_info
self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info)
+ self._framing_compiler = self._new_framing_compiler(tokenizer_info)
self.vocab_size = vocab_size
self.override_stop_tokens = override_stop_tokens
self.any_whitespace = any_whitespace
@@ -251,6 +254,41 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
def is_support_token_filter(self):
return True
+ @staticmethod
+ def _new_framing_compiler(tokenizer_info: TokenizerInfo) -> GrammarCompiler:
+ # The cache mixes up token-level structural tags of different shapes, so
+ # framed grammars compile uncached (seen on xgrammar 0.2.1 through 0.2.7).
+ return GrammarCompiler(tokenizer_info=tokenizer_info, cache_enabled=False)
+
+ def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None:
+ info = self._tokenizer_info
+ reserved = set(token_ids)
+ # A leading 0xFF is invalid UTF-8, so framed text cannot match these while
+ # token elements still match by id; string tool tags spell them, so only framing.
+ vocab = [
+ b"\xff" + piece if token_id in reserved else piece
+ for token_id, piece in enumerate(info.decoded_vocab)
+ ]
+ self._framing_compiler = self._new_framing_compiler(
+ TokenizerInfo(
+ vocab,
+ VocabType.RAW,
+ vocab_size=info.vocab_size,
+ stop_token_ids=info.stop_token_ids,
+ )
+ )
+
+ def compile_framed_grammar(
+ self, grammar: XGrammarGrammar, frame: Callable[[dict], dict]
+ ) -> XGrammarGrammar:
+ tag = json.dumps(
+ frame({"type": "grammar", "grammar": str(grammar.ctx.grammar)})
+ )
+ ctx = self._framing_compiler.compile_structural_tag(tag)
+ return self._from_context(
+ ctx, tag, GrammarStats(dispatch_type="structural_tag")
+ )
+
@staticmethod
def allocate_vocab_mask(vocab_size: int, batch_size: int, device) -> torch.Tensor:
return _allocate_token_bitmask(vocab_size, batch_size)
diff --git a/python/sglang/srt/parser/answer_frame.py b/python/sglang/srt/parser/answer_frame.py
new file mode 100644
index 0000000000..0ab1759a7b
--- /dev/null
+++ b/python/sglang/srt/parser/answer_frame.py
@@ -0,0 +1,123 @@
+import functools
+import json
+from typing import List, Sequence, Tuple
+
+from sglang.srt.parser.inkling_tokenizer import (
+ CONTENT_MODEL_END_SAMPLING,
+ CONTENT_TEXT,
+ CONTENT_THINKING,
+ END_MESSAGE,
+ INKLING_SPECIAL_TOKEN_IDS,
+ MESSAGE_MODEL,
+)
+
+
+class AnswerFrame:
+ """Native message framing a reasoning format wraps around constrained output.
+
+ A frame owns the whole turn: an optional thinking message, then the answer
+ message holding the constrained content.
+ """
+
+ framed_key_types: Tuple[str, ...] = ()
+
+ def framing_token_ids(self) -> List[int]:
+ """Token IDs that may only match as frame tokens, never as content text."""
+ raise NotImplementedError()
+
+ def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool:
+ """Whether the prompt already opened the answer block (continuation)."""
+ raise NotImplementedError()
+
+ def structural_tag(
+ self, content: dict, *, in_answer_block: bool, max_think_tokens: int
+ ) -> dict:
+ """XGrammar structural tag framing ``content``; ``max_think_tokens < 0``
+ leaves thinking unbounded."""
+ raise NotImplementedError()
+
+ def owns_turn(self, structural_tag: str) -> bool:
+ """Whether this structural tag already frames the whole turn itself."""
+ raise NotImplementedError()
+
+
+@functools.cache
+def _canonical_inkling_auto_tool_schema() -> dict:
+ # Lazy: inkling_detector imports xgrammar unconditionally, and XPU
+ # installs may not ship it.
+ from sglang.srt.function_call.inkling_detector import InklingDetector
+
+ return (
+ InklingDetector().get_auto_tool_call_structural_tag().model_dump(by_alias=True)
+ )
+
+
+def _token(name: str) -> dict:
+ return {"type": "token", "token": INKLING_SPECIAL_TOKEN_IDS[name]}
+
+
+class InklingAnswerFrame(AnswerFrame):
+ framed_key_types = ("json", "regex", "ebnf")
+
+ def framing_token_ids(self) -> List[int]:
+ # Inkling's control tokens decode to their literal spellings, so text
+ # positions such as JSON strings would otherwise admit <|end_message|>.
+ return [
+ token_id
+ for token, token_id in INKLING_SPECIAL_TOKEN_IDS.items()
+ if token != CONTENT_MODEL_END_SAMPLING
+ ]
+
+ def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool:
+ end_message = INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]
+ last_end = next(
+ (
+ i
+ for i in range(len(input_ids) - 1, -1, -1)
+ if input_ids[i] == end_message
+ ),
+ -1,
+ )
+ return list(input_ids[last_end + 1 : last_end + 3]) == [
+ INKLING_SPECIAL_TOKEN_IDS[MESSAGE_MODEL],
+ INKLING_SPECIAL_TOKEN_IDS[CONTENT_TEXT],
+ ]
+
+ def structural_tag(
+ self, content: dict, *, in_answer_block: bool, max_think_tokens: int
+ ) -> dict:
+ elements = [content, _token(END_MESSAGE)]
+ if in_answer_block:
+ return {
+ "type": "structural_tag",
+ "format": {"type": "sequence", "elements": elements},
+ }
+ thinking = {
+ "type": "any_tokens",
+ "exclude_tokens": [INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]],
+ }
+ if max_think_tokens >= 0:
+ thinking["max_tokens"] = max_think_tokens
+ thinking_message = {
+ "type": "sequence",
+ "elements": [
+ _token(MESSAGE_MODEL),
+ _token(CONTENT_THINKING),
+ thinking,
+ _token(END_MESSAGE),
+ ],
+ }
+ elements = [
+ {"type": "optional", "content": thinking_message},
+ _token(MESSAGE_MODEL),
+ _token(CONTENT_TEXT),
+ *elements,
+ ]
+ return {
+ "type": "structural_tag",
+ "format": {"type": "sequence", "elements": elements},
+ }
+
+ def owns_turn(self, structural_tag: str) -> bool:
+ # Only the canonical auto-tool schema owns the full turn; user restrictions may not.
+ return json.loads(structural_tag) == _canonical_inkling_auto_tool_schema()
diff --git a/python/sglang/srt/parser/reasoning_parser.py b/python/sglang/srt/parser/reasoning_parser.py
index 778cfcf0bc..a5f9ddeafe 100644
--- a/python/sglang/srt/parser/reasoning_parser.py
+++ b/python/sglang/srt/parser/reasoning_parser.py
@@ -36,6 +36,7 @@ from sglang.srt.function_call.muse_glimmer_format import (
has_atem_markers,
partial_marker_len,
)
+from sglang.srt.parser.answer_frame import AnswerFrame, InklingAnswerFrame
from sglang.srt.parser.harmony_parser import HarmonyParser
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_INVOKE_TOOL_JSON,
@@ -117,6 +118,9 @@ class BaseReasoningFormatDetector:
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
+ def get_answer_frame(self) -> Optional[AnswerFrame]:
+ return None
+
def _maybe_apply_force_nonempty_content(
self, ret: StreamingParseResult
) -> StreamingParseResult:
@@ -1376,6 +1380,9 @@ class InklingDetector(BaseReasoningFormatDetector):
# Native framing IDs differ from encoding their printed names as plain text.
return [INKLING_SPECIAL_TOKEN_IDS[self.think_end_token]]
+ def get_answer_frame(self) -> Optional[AnswerFrame]:
+ return InklingAnswerFrame()
+
def detect_and_parse(self, text: str) -> StreamingParseResult:
self._buffer = ""
self._kind = None
diff --git a/test/registered/unit/constrained/test_grammar_manager.py b/test/registered/unit/constrained/test_grammar_manager.py
index 25b9894717..d0c8e78b15 100644
--- a/test/registered/unit/constrained/test_grammar_manager.py
+++ b/test/registered/unit/constrained/test_grammar_manager.py
@@ -14,18 +14,35 @@ Usage:
python -m pytest test_grammar_manager.py -v
"""
+import os
import unittest
from concurrent.futures import Future
+from types import SimpleNamespace
from unittest.mock import MagicMock, patch
+from xgrammar import TokenizerInfo
+
+from sglang.srt.constrained import reasoner_grammar_backend
from sglang.srt.constrained.base_grammar_backend import (
BaseGrammarBackend,
BaseGrammarObject,
InvalidGrammarObject,
)
from sglang.srt.constrained.grammar_manager import GrammarManager
-from sglang.srt.constrained.reasoner_grammar_backend import ReasonerGrammarObject
+from sglang.srt.constrained.reasoner_grammar_backend import (
+ ReasonerGrammarBackend,
+ ReasonerGrammarObject,
+)
+from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend
from sglang.srt.distributed.communication_tags import P2PTag
+from sglang.srt.parser.inkling_tokenizer import (
+ CONTENT_MODEL_END_SAMPLING,
+ CONTENT_THINKING,
+ END_MESSAGE,
+ INKLING_SPECIAL_TOKEN_IDS,
+ MESSAGE_MODEL,
+)
+from sglang.srt.parser.reasoning_parser import InklingDetector
from sglang.srt.runtime_context import (
SpawnRanks,
get_context,
@@ -104,6 +121,34 @@ def _make_req(
return req
+_INKLING_VOCAB_SIZE = max(INKLING_SPECIAL_TOKEN_IDS.values()) + 1
+
+
+def _make_strict_inkling_backend(max_think_tokens):
+ vocab = [""] * _INKLING_VOCAB_SIZE
+ for token_id in range(128):
+ vocab[token_id] = chr(token_id)
+ for token, token_id in INKLING_SPECIAL_TOKEN_IDS.items():
+ vocab[token_id] = token
+ info = TokenizerInfo(
+ vocab, stop_token_ids=[INKLING_SPECIAL_TOKEN_IDS[CONTENT_MODEL_END_SAMPLING]]
+ )
+ tokenizer = SimpleNamespace(init_xgrammar=lambda: (info, None))
+ with patch.dict(os.environ, {"SGLANG_MAX_THINK_TOKENS": str(max_think_tokens)}):
+ return ReasonerGrammarBackend(
+ XGrammarGrammarBackend(tokenizer, _INKLING_VOCAB_SIZE),
+ SimpleNamespace(detector=InklingDetector()),
+ tokenizer,
+ enable_strict_thinking=True,
+ )
+
+
+def _token_allowed(grammar, token_id):
+ mask = grammar.allocate_vocab_mask(_INKLING_VOCAB_SIZE, 1, "cpu")
+ grammar.fill_vocab_mask(mask, 0)
+ return bool((int(mask[0, token_id // 32]) >> (token_id % 32)) & 1)
+
+
class TestGrammarManagerInit(unittest.TestCase):
def setUp(self):
reset_context()
@@ -146,6 +191,7 @@ class TestProcessReqWithGrammar(unittest.TestCase):
)
mgr = GrammarManager(scheduler)
mgr.grammar_backend = MagicMock(spec=BaseGrammarBackend)
+ mgr.grammar_backend.get_request_grammar_key.side_effect = lambda key, **_: key
return mgr
def test_no_constraint_returns_false(self):
@@ -178,6 +224,28 @@ class TestProcessReqWithGrammar(unittest.TestCase):
self.assertTrue(result)
self.assertEqual(req.grammar_key, ("regex", "[a-z]+"))
+ def test_backend_specializes_the_key_from_the_request(self):
+ """The backend may pick a grammar variant from the prompt and the thinking
+ budget; that key, not the sampling-param key, is looked up and cached."""
+ mgr = self._make_mgr()
+ mgr.grammar_backend.get_request_grammar_key.side_effect = lambda key, **_: (
+ key[0] + ":variant",
+ key[1],
+ )
+ mgr.grammar_backend.get_cached_or_future_value.return_value = (Future(), False)
+
+ req = _make_req(regex="[a-z]+", custom_params={"thinking_budget": 3})
+ req.origin_input_ids = [1, 2, 3]
+ mgr.process_req_with_grammar(req)
+
+ mgr.grammar_backend.get_request_grammar_key.assert_called_once_with(
+ ("regex", "[a-z]+"), input_ids=[1, 2, 3], thinking_budget=3
+ )
+ mgr.grammar_backend.get_cached_or_future_value.assert_called_once_with(
+ ("regex:variant", "[a-z]+"), False
+ )
+ self.assertEqual(req.grammar_key, ("regex:variant", "[a-z]+"))
+
def test_ebnf_cache_miss(self):
mgr = self._make_mgr()
future = Future()
@@ -861,6 +929,60 @@ class TestStrictReasoningPaths(unittest.TestCase):
self.assertIsNone(req.grammar)
+ def _serve_inkling_regex(self, mgr, budget):
+ params = None if budget is None else {"thinking_budget": budget}
+ req = _make_req(regex="yes", custom_params=params)
+ req.require_reasoning = True
+ req.origin_input_ids = []
+ compiled = mgr.process_req_with_grammar(req)
+ if compiled:
+ mgr.get_ready_grammar_requests()
+ return compiled, req.grammar
+
+ def _thinking_tokens_allowed(self, grammar, limit=16):
+ ids = INKLING_SPECIAL_TOKEN_IDS
+ for token_id in [ids[MESSAGE_MODEL], ids[CONTENT_THINKING]]:
+ grammar.accept_token(token_id)
+ count = 0
+ while count < limit and _token_allowed(grammar, ord("x")):
+ grammar.accept_token(ord("x"))
+ count += 1
+ self.assertTrue(_token_allowed(grammar, ids[END_MESSAGE]))
+ return count
+
+ def test_inkling_constraint_caps_thinking_at_the_request_budget(self):
+ """An Inkling constraint compiles to a framed grammar, not a reasoning
+ wrapper; the request's thinking budget must still cap its thinking message,
+ both when compiled for the request and when served from the cache."""
+ mgr = self._make_mgr()
+ mgr.grammar_backend = _make_strict_inkling_backend(max_think_tokens=10)
+ mgr.SGLANG_GRAMMAR_POLL_INTERVAL = 5
+ served = [self._serve_inkling_regex(mgr, budget) for budget in (3, 3, 12)]
+ self.assertEqual([compiled for compiled, _ in served], [True, False, True])
+ self.assertEqual(
+ [self._thinking_tokens_allowed(grammar) for _, grammar in served],
+ [3, 3, 12],
+ )
+
+ def test_inkling_budget_variants_stay_bounded(self):
+ """Equivalent budgets share one cached frame and only recent budget variants
+ stay cached; an evicted budget recompiles with its exact cap."""
+ mgr = self._make_mgr()
+ mgr.grammar_backend = _make_strict_inkling_backend(max_think_tokens=10)
+ mgr.SGLANG_GRAMMAR_POLL_INTERVAL = 5
+ budgets = [0, 1, 2, 3, 4, 5, 6, 12, None, 10, -1, -2, 0]
+ with patch.object(reasoner_grammar_backend, "_BUDGET_VARIANT_CACHE_SIZE", 4):
+ served = [self._serve_inkling_regex(mgr, budget) for budget in budgets]
+ self.assertEqual(
+ [compiled for compiled, _ in served],
+ [True] * 9 + [False, True, False, True],
+ )
+ self.assertEqual(
+ [self._thinking_tokens_allowed(grammar) for _, grammar in served],
+ [0, 1, 2, 3, 4, 5, 6, 12, 10, 10, 16, 16, 0],
+ )
+ self.assertEqual(len(mgr.grammar_backend.cache), 1 + 4)
+
def test_strict_constrained_request_uses_normal_dispatch(self):
"""Request with json_schema should go through normal dispatch, not strict path."""
mgr = self._make_mgr()
diff --git a/test/registered/unit/constrained/test_reasoner_grammar_backend.py b/test/registered/unit/constrained/test_reasoner_grammar_backend.py
index ba3ebbdaf2..e1fb67fd7b 100644
--- a/test/registered/unit/constrained/test_reasoner_grammar_backend.py
+++ b/test/registered/unit/constrained/test_reasoner_grammar_backend.py
@@ -1,3 +1,4 @@
+import itertools
import json
import os
import unittest
@@ -16,7 +17,8 @@ from sglang.srt.constrained.torch_ops.token_filter_torch_ops import (
set_token_filter_torch,
)
from sglang.srt.constrained.xgrammar_backend import XGrammarGrammarBackend
-from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest
+from sglang.srt.entrypoints.openai.protocol import ChatCompletionRequest, Tool
+from sglang.srt.function_call.function_call_parser import FunctionCallParser
from sglang.srt.function_call.inkling_detector import (
InklingDetector as InklingToolDetector,
)
@@ -174,6 +176,10 @@ class TestReasonerGrammarBackend(CustomTestCase):
think_start_token="",
think_end_token="",
think_excluded_tokens=["", ""],
+ get_think_end_token_ids=lambda tokenizer: tokenizer.encode(
+ "", add_special_tokens=False
+ ),
+ get_answer_frame=lambda: None,
)
return SimpleNamespace(detector=detector)
@@ -472,13 +478,234 @@ class TestReasonerGrammarBackend(CustomTestCase):
else:
self.assertIs(grammar, inner_grammar)
+ def test_inkling_output_constraints_keep_message_framing(self):
+ """Constrained Inkling output is a framed text message after optional thinking."""
+ backend, tokenizer = self._make_xgrammar_backend()
+ ids = INKLING_SPECIAL_TOKEN_IDS
+ keys = {
+ ("json", '{"type":"object","properties":{"a":{"type":"integer"}}}'): (
+ '{"a": 1}'
+ ),
+ ("regex", "Answer: [0-9]+"): "Answer: 42",
+ ("ebnf", 'root ::= "yes" | "no"'): "yes",
+ }
+ thinking = (
+ [ids[MESSAGE_MODEL], ids[CONTENT_THINKING]]
+ + tokenizer.encode("reason")
+ + [ids[END_MESSAGE]]
+ )
+ for (key, payload), reasoning, prefix, strict in itertools.product(
+ keys.items(), (False, True), ([], thinking), (False, True)
+ ):
+ with self.subTest(
+ key=key, reasoning=reasoning, prefix=bool(prefix), strict=strict
+ ):
+ reasoner = ReasonerGrammarBackend(
+ backend,
+ SimpleNamespace(detector=InklingDetector()),
+ tokenizer,
+ enable_strict_thinking=strict,
+ )
+ grammar = reasoner._init_value_dispatch(key, reasoning=reasoning)
+ self._accept_allowed_tokens(grammar, prefix, backend.vocab_size)
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(mask, [ord(payload[0]), ids[MESSAGE_MODEL]]),
+ [ids[MESSAGE_MODEL]],
+ )
+ self._accept_allowed_tokens(
+ grammar, [ids[MESSAGE_MODEL]], backend.vocab_size
+ )
+ kinds = [ids[CONTENT_TEXT], ids[CONTENT_THINKING]]
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(mask, kinds), kinds[:1] if prefix else kinds
+ )
+ self._accept_allowed_tokens(
+ grammar,
+ [ids[CONTENT_TEXT]] + tokenizer.encode(payload),
+ backend.vocab_size,
+ )
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(
+ mask, [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]]
+ ),
+ [ids[END_MESSAGE]],
+ )
+ self._accept_allowed_tokens(
+ grammar,
+ [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]],
+ backend.vocab_size,
+ )
+ self.assertTrue(grammar.is_terminated())
+
+ def test_inkling_continuation_constraint_starts_inside_the_text_block(self):
+ """A prompt ending in an open text block constrains the payload directly,
+ also after an ordinary framed grammar compiled on the same backend."""
+ backend, tokenizer = self._make_xgrammar_backend()
+ ids = INKLING_SPECIAL_TOKEN_IDS
+ reasoner = ReasonerGrammarBackend(
+ backend, SimpleNamespace(detector=InklingDetector()), tokenizer
+ )
+ user_turn = [ids[MESSAGE_MODEL], *tokenizer.encode("hi"), ids[END_MESSAGE]]
+ open_block = [*user_turn, ids[MESSAGE_MODEL], ids[CONTENT_TEXT]]
+ key = ("regex", "yes")
+ self.assertEqual(
+ reasoner.get_request_grammar_key(
+ key, input_ids=user_turn, thinking_budget=None
+ ),
+ key,
+ )
+ turn = reasoner._init_value_dispatch(key, reasoning=True)
+ self._accept_allowed_tokens(
+ turn,
+ [ids[MESSAGE_MODEL], ids[CONTENT_TEXT]]
+ + tokenizer.encode("yes")
+ + [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]],
+ backend.vocab_size,
+ )
+
+ block_key = reasoner.get_request_grammar_key(
+ key, input_ids=open_block, thinking_budget=None
+ )
+ self.assertNotEqual(block_key, key)
+ grammar = reasoner._init_value_dispatch(block_key, reasoning=True)
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(mask, [ord("y"), ids[MESSAGE_MODEL]]), [ord("y")]
+ )
+ self._accept_allowed_tokens(
+ grammar,
+ tokenizer.encode("yes")
+ + [ids[END_MESSAGE], ids[CONTENT_MODEL_END_SAMPLING]],
+ backend.vocab_size,
+ )
+ self.assertTrue(grammar.is_terminated())
+
+ def test_inkling_framing_tokens_never_match_payload_text(self):
+ """Control tokens decode to their spellings; inside a JSON string an Inkling
+ constraint must not admit <|end_message|>, which would close the message."""
+ ids = INKLING_SPECIAL_TOKEN_IDS
+ key = ("json", '{"type":"object","properties":{"a":{"type":"string"}}}')
+ for inkling in (True, False):
+ with self.subTest(inkling=inkling):
+ backend, tokenizer = self._make_xgrammar_backend()
+ if inkling:
+ parser = SimpleNamespace(detector=InklingDetector())
+ reasoner = ReasonerGrammarBackend(backend, parser, tokenizer)
+ head = [ids[MESSAGE_MODEL], ids[CONTENT_TEXT]]
+ else:
+ reasoner = ReasonerGrammarBackend(
+ backend, self._make_parser(), tokenizer
+ )
+ head = tokenizer.encode("")
+ grammar = reasoner._init_value_dispatch(key, reasoning=True)
+ self._accept_allowed_tokens(
+ grammar, head + tokenizer.encode('{"a": "x'), backend.vocab_size
+ )
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(mask, [ord("y"), ids[END_MESSAGE]]),
+ [ord("y")] if inkling else [ord("y"), ids[END_MESSAGE]],
+ )
+
+ def test_inkling_required_tool_tag_matches_native_markers(self):
+ """Required and named tool tags spell Inkling's markers as strings; the native
+ marker tokens must still match them while payload framing is enabled."""
+ backend, tokenizer = self._make_xgrammar_backend()
+ ReasonerGrammarBackend(
+ backend, SimpleNamespace(detector=InklingDetector()), tokenizer
+ )
+ tools = [Tool(type="function", function={"name": "f", "parameters": {}})]
+ request = ChatCompletionRequest(
+ model="inkling", messages=[], tools=tools, tool_choice="required"
+ )
+ params = request.to_sampling_params(
+ stop=[],
+ model_generation_config={},
+ tool_call_constraint=FunctionCallParser(
+ tools, "inkling"
+ ).get_structure_constraint("required"),
+ )
+ grammar = backend.dispatch_structural_tag(params["structural_tag"])
+ ids = INKLING_SPECIAL_TOKEN_IDS
+ self._accept_allowed_tokens(
+ grammar,
+ [ids[MESSAGE_MODEL], *tokenizer.encode("f"), ids[CONTENT_INVOKE_TOOL_JSON]]
+ + tokenizer.encode('{"name":"f","args":{}}')
+ + [ids[END_MESSAGE]],
+ backend.vocab_size,
+ )
+
+ def test_inkling_strict_budget_caps_thinking_and_cannot_reopen_it(self):
+ """Strict thinking caps the thinking message; after its end a header may only
+ open the answer, and a turn that skips thinking is held to the payload."""
+ backend, tokenizer = self._make_xgrammar_backend()
+ ids = INKLING_SPECIAL_TOKEN_IDS
+ with patch.dict(os.environ, {"SGLANG_MAX_THINK_TOKENS": "3"}):
+ reasoner = ReasonerGrammarBackend(
+ backend,
+ SimpleNamespace(detector=InklingDetector()),
+ tokenizer,
+ enable_strict_thinking=True,
+ )
+ grammar = reasoner._init_value_dispatch(("regex", "yes"), reasoning=True)
+ self._accept_allowed_tokens(
+ grammar,
+ [ids[MESSAGE_MODEL], ids[CONTENT_THINKING], *tokenizer.encode("xxx")],
+ backend.vocab_size,
+ )
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(mask, [ord("x"), ids[END_MESSAGE]]), [ids[END_MESSAGE]]
+ )
+ self._accept_allowed_tokens(
+ grammar, [ids[END_MESSAGE], ids[MESSAGE_MODEL]], backend.vocab_size
+ )
+ mask = self._vocab_mask(grammar, backend.vocab_size)
+ self.assertEqual(
+ _allowed_token_ids(mask, [ids[CONTENT_TEXT], ids[CONTENT_THINKING]]),
+ [ids[CONTENT_TEXT]],
+ )
+
+ skipped = reasoner._init_value_dispatch(("regex", "yes"), reasoning=True)
+ self._accept_allowed_tokens(
+ skipped, [ids[MESSAGE_MODEL], ids[CONTENT_TEXT]], backend.vocab_size
+ )
+ mask = self._vocab_mask(skipped, backend.vocab_size)
+ self.assertEqual(_allowed_token_ids(mask, [ord("h"), ord("y")]), [ord("y")])
+
+ def test_inkling_backend_matches_native_think_end_id(self):
+ """Strict mode and other constraints must see the native END_MESSAGE id."""
+ tokenizer = _DummyTokenizer({END_MESSAGE: [27, 91, 406, 62, 65752, 91, 29]})
+ for strict in (False, True):
+ with self.subTest(strict=strict):
+ reasoner = ReasonerGrammarBackend(
+ _DummyGrammarBackend(),
+ SimpleNamespace(detector=InklingDetector()),
+ tokenizer,
+ enable_strict_thinking=strict,
+ )
+ self.assertEqual(
+ reasoner.think_end_ids, [INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]]
+ )
+
+ grammar = reasoner._make_grammar_object(MagicMock(), reasoning=True)
+ self.assertFalse(grammar._is_generation())
+ grammar.accept_token(INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE])
+ self.assertTrue(grammar._is_generation())
+
def test_other_grammars_still_defer_until_reasoning_ends(self):
- tag = InklingToolDetector().get_auto_tool_call_structural_tag().model_dump()
+ canonical = InklingToolDetector().get_auto_tool_call_structural_tag()
+ tag = canonical.model_dump()
tag["format"] = {"type": "sequence", "elements": [tag["format"]]}
keys = [
("json", "{}"),
("structural_tag", '{"structures": [], "triggers": []}'),
("structural_tag", json.dumps(tag)),
+ # The canonical schema owns the full turn only under the Inkling detector.
+ ("structural_tag", canonical.model_dump_json()),
]
for key in keys:
with self.subTest(key=key):
@@ -493,6 +720,21 @@ class TestReasonerGrammarBackend(CustomTestCase):
self.assertIs(grammar.grammar, backend._dispatch_result)
self.assertFalse(grammar._is_generation())
+ def test_inkling_constraint_defers_on_a_backend_that_cannot_frame(self):
+ """Backends without compile_framed_grammar keep the deferred constraint."""
+ backend = _DummyGrammarBackend()
+ backend._dispatch_result = MagicMock()
+ reasoner = ReasonerGrammarBackend(
+ backend,
+ SimpleNamespace(detector=InklingDetector()),
+ _DummyTokenizer({}),
+ )
+ grammar = reasoner._init_value_dispatch(("json", "{}"), reasoning=True)
+
+ self.assertIsInstance(grammar, ReasonerGrammarObject)
+ self.assertIs(grammar.grammar, backend._dispatch_result)
+ self.assertFalse(grammar._is_generation())
+
def test_full_assistant_ebnf_still_owns_its_boundaries(self):
"""Whole-assistant EBNF must keep control of the reasoning prefix."""
for strict in (False, True):