[Fix] Inkling: frame constrained output in native message tokens (#44716)

This commit is contained in:
Xinyuan Tong
2026-10-18 16:47:56 +08:00
committed by GitHub
parent 889658a5be
commit 80fef63786
8 changed files with 658 additions and 19 deletions
@@ -18,7 +18,7 @@ import logging
import time
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass, field
from typing import Dict, List, NamedTuple, Optional, Tuple
from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Tuple
import torch
@@ -223,6 +223,27 @@ class BaseGrammarBackend:
def enable_strict_thinking(self):
return self._enable_strict_thinking
def get_request_grammar_key(
self,
key: Tuple[str, str],
*,
input_ids: Sequence[int],
thinking_budget: Optional[int],
) -> Tuple[str, str]:
"""Cache key for this request's grammar; may depend on how the prompt ends
and on the request's thinking budget."""
return key
def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None:
"""Let ``token_ids`` match only token-level framing, never grammar text."""
def compile_framed_grammar(
self, grammar: BaseGrammarObject, frame: Callable[[dict], dict]
) -> Optional[BaseGrammarObject]:
"""Compile ``frame(content)``, an XGrammar structural tag embedding ``grammar``
as content; None if the backend cannot embed it."""
return None
@property
def is_support_token_filter(self):
return False
@@ -168,6 +168,11 @@ class GrammarManager:
key = (key_type, req.sampling_params.ebnf)
elif req.sampling_params.structural_tag is not None:
key = ("structural_tag", req.sampling_params.structural_tag)
key = self.grammar_backend.get_request_grammar_key(
key,
input_ids=req.origin_input_ids,
thinking_budget=self._get_request_thinking_budget(req),
)
value, cache_hit = self.grammar_backend.get_cached_or_future_value(
key, req.require_reasoning
@@ -13,8 +13,9 @@
# ==============================================================================
"""The baseclass of a backend for reasoner grammar-guided constrained decoding."""
import json
import logging
from collections import OrderedDict
from concurrent.futures import Future
from typing import List, Optional, Sequence, Tuple, Union
import torch
@@ -32,6 +33,11 @@ from .base_grammar_backend import (
logger = logging.getLogger(__name__)
_OPEN_ANSWER_VARIANT = "open_answer"
_THINK_BUDGET_VARIANT = "think_budget="
# Arbitrary bound; budgets are a per-request knob, so only recent variants stay cached.
_BUDGET_VARIANT_CACHE_SIZE = 32
class ReasonerGrammarObject(BaseGrammarObject):
"""Defers grammar constraints until the reasoning end sequence is complete."""
@@ -271,9 +277,8 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
):
super().__init__()
self.grammar_backend = grammar_backend
think_end_ids = tokenizer.encode(
reasoning_parser.detector.think_end_token, add_special_tokens=False
)
self._budget_variant_keys: OrderedDict[Tuple[str, str], None] = OrderedDict()
think_end_ids = reasoning_parser.detector.get_think_end_token_ids(tokenizer)
if not think_end_ids:
raise ValueError(
f"think_end_token '{reasoning_parser.detector.think_end_token}' "
@@ -281,6 +286,11 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
)
self.think_end_ids = think_end_ids
self._enable_strict_thinking = enable_strict_thinking
self._answer_frame = reasoning_parser.detector.get_answer_frame()
if self._answer_frame is not None:
self.grammar_backend.reserve_framing_tokens(
self._answer_frame.framing_token_ids()
)
self.think_excluded_token_ids = self._get_think_excluded_token_ids(
reasoning_parser, tokenizer
)
@@ -347,22 +357,93 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
return None
return self._make_grammar_object(None, reasoning)
def get_request_grammar_key(
self,
key: Tuple[str, str],
*,
input_ids: Sequence[int],
thinking_budget: Optional[int],
) -> Tuple[str, str]:
key_type, key_string = key
if not self._is_framed(key_type):
return key
if self._answer_frame.ends_in_open_answer_block(input_ids):
return (f"{key_type}:{_OPEN_ANSWER_VARIANT}", key_string)
# Mirrors the strict wrapper, which applies a request budget only while filtering.
if not self.enable_token_filter or thinking_budget is None:
return key
budget = max(thinking_budget, -1)
if budget == max(self.max_think_tokens, -1):
return key
return (f"{key_type}:{_THINK_BUDGET_VARIANT}{budget}", key_string)
def _is_framed(self, key_type: str) -> bool:
return (
self._answer_frame is not None
and key_type in self._answer_frame.framed_key_types
)
def get_cached_or_future_value(
self, key: Tuple[str, str], require_reasoning: bool
) -> Tuple[BaseGrammarObject | Future[BaseGrammarObject], bool]:
if key in self._budget_variant_keys:
self._budget_variant_keys.move_to_end(key)
return super().get_cached_or_future_value(key, require_reasoning)
def set_cache(self, key: Tuple[str, str], value: BaseGrammarObject):
super().set_cache(key, value)
if _THINK_BUDGET_VARIANT not in key[0]:
return
self._budget_variant_keys[key] = None
self._budget_variant_keys.move_to_end(key)
while len(self._budget_variant_keys) > _BUDGET_VARIANT_CACHE_SIZE:
evicted, _ = self._budget_variant_keys.popitem(last=False)
self.cache.pop(evicted, None)
def reset(self):
super().reset()
self._budget_variant_keys.clear()
def _init_value_dispatch(
self, key: Tuple[str, str], reasoning: bool
) -> Optional[BaseGrammarObject]:
key_type, _, variant = key[0].partition(":")
key = (key_type, key[1])
ret = self.grammar_backend._init_value_dispatch(key, reasoning)
if ret is None or isinstance(ret, InvalidGrammarObject):
return ret
if key[0] == "full_assistant_ebnf":
return ret
if not self.enable_strict_thinking and key[0] == "structural_tag":
from sglang.srt.function_call.inkling_detector import InklingDetector
# Only this canonical schema owns the full turn; user restrictions may not.
if json.loads(key[1]) == (
InklingDetector()
.get_auto_tool_call_structural_tag()
.model_dump(by_alias=True)
):
return ret
if self._is_framed(key[0]):
# The frame delimits thinking by message kind, so it owns the whole
# turn; strict thinking only caps the thinking message.
framed = self._frame_answer(ret, variant)
if framed is not None:
return framed
if (
not self.enable_strict_thinking
and key[0] == "structural_tag"
and self._answer_frame is not None
and self._answer_frame.owns_turn(key[1])
):
return ret
return self._make_grammar_object(ret, reasoning)
def _frame_answer(
self, grammar: BaseGrammarObject, variant: str
) -> Optional[BaseGrammarObject]:
in_answer_block = variant == _OPEN_ANSWER_VARIANT
if variant.startswith(_THINK_BUDGET_VARIANT):
max_think_tokens = int(variant.removeprefix(_THINK_BUDGET_VARIANT))
elif self.enable_strict_thinking:
max_think_tokens = self.max_think_tokens
else:
max_think_tokens = -1
return self.grammar_backend.compile_framed_grammar(
grammar,
lambda content: self._answer_frame.structural_tag(
content,
in_answer_block=in_answer_block,
max_think_tokens=max_think_tokens,
),
)
@@ -16,7 +16,7 @@
import dataclasses
import json
import logging
from typing import Dict, List, Optional, Tuple, Union
from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
import torch
from xgrammar import (
@@ -26,6 +26,7 @@ from xgrammar import (
StructuralTag,
StructuralTagItem,
TokenizerInfo,
VocabType,
bitmask_dtype,
get_bitmask_shape,
)
@@ -241,7 +242,9 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
f"Failed to create XGrammar TokenizerInfo from tokenizer: {e}"
)
self._tokenizer_info = tokenizer_info
self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info)
self._framing_compiler = self._new_framing_compiler(tokenizer_info)
self.vocab_size = vocab_size
self.override_stop_tokens = override_stop_tokens
self.any_whitespace = any_whitespace
@@ -251,6 +254,41 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
def is_support_token_filter(self):
return True
@staticmethod
def _new_framing_compiler(tokenizer_info: TokenizerInfo) -> GrammarCompiler:
# The cache mixes up token-level structural tags of different shapes, so
# framed grammars compile uncached (seen on xgrammar 0.2.1 through 0.2.7).
return GrammarCompiler(tokenizer_info=tokenizer_info, cache_enabled=False)
def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None:
info = self._tokenizer_info
reserved = set(token_ids)
# A leading 0xFF is invalid UTF-8, so framed text cannot match these while
# token elements still match by id; string tool tags spell them, so only framing.
vocab = [
b"\xff" + piece if token_id in reserved else piece
for token_id, piece in enumerate(info.decoded_vocab)
]
self._framing_compiler = self._new_framing_compiler(
TokenizerInfo(
vocab,
VocabType.RAW,
vocab_size=info.vocab_size,
stop_token_ids=info.stop_token_ids,
)
)
def compile_framed_grammar(
self, grammar: XGrammarGrammar, frame: Callable[[dict], dict]
) -> XGrammarGrammar:
tag = json.dumps(
frame({"type": "grammar", "grammar": str(grammar.ctx.grammar)})
)
ctx = self._framing_compiler.compile_structural_tag(tag)
return self._from_context(
ctx, tag, GrammarStats(dispatch_type="structural_tag")
)
@staticmethod
def allocate_vocab_mask(vocab_size: int, batch_size: int, device) -> torch.Tensor:
return _allocate_token_bitmask(vocab_size, batch_size)
+123
View File
@@ -0,0 +1,123 @@
import functools
import json
from typing import List, Sequence, Tuple
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_MODEL_END_SAMPLING,
CONTENT_TEXT,
CONTENT_THINKING,
END_MESSAGE,
INKLING_SPECIAL_TOKEN_IDS,
MESSAGE_MODEL,
)
class AnswerFrame:
"""Native message framing a reasoning format wraps around constrained output.
A frame owns the whole turn: an optional thinking message, then the answer
message holding the constrained content.
"""
framed_key_types: Tuple[str, ...] = ()
def framing_token_ids(self) -> List[int]:
"""Token IDs that may only match as frame tokens, never as content text."""
raise NotImplementedError()
def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool:
"""Whether the prompt already opened the answer block (continuation)."""
raise NotImplementedError()
def structural_tag(
self, content: dict, *, in_answer_block: bool, max_think_tokens: int
) -> dict:
"""XGrammar structural tag framing ``content``; ``max_think_tokens < 0``
leaves thinking unbounded."""
raise NotImplementedError()
def owns_turn(self, structural_tag: str) -> bool:
"""Whether this structural tag already frames the whole turn itself."""
raise NotImplementedError()
@functools.cache
def _canonical_inkling_auto_tool_schema() -> dict:
# Lazy: inkling_detector imports xgrammar unconditionally, and XPU
# installs may not ship it.
from sglang.srt.function_call.inkling_detector import InklingDetector
return (
InklingDetector().get_auto_tool_call_structural_tag().model_dump(by_alias=True)
)
def _token(name: str) -> dict:
return {"type": "token", "token": INKLING_SPECIAL_TOKEN_IDS[name]}
class InklingAnswerFrame(AnswerFrame):
framed_key_types = ("json", "regex", "ebnf")
def framing_token_ids(self) -> List[int]:
# Inkling's control tokens decode to their literal spellings, so text
# positions such as JSON strings would otherwise admit <|end_message|>.
return [
token_id
for token, token_id in INKLING_SPECIAL_TOKEN_IDS.items()
if token != CONTENT_MODEL_END_SAMPLING
]
def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool:
end_message = INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]
last_end = next(
(
i
for i in range(len(input_ids) - 1, -1, -1)
if input_ids[i] == end_message
),
-1,
)
return list(input_ids[last_end + 1 : last_end + 3]) == [
INKLING_SPECIAL_TOKEN_IDS[MESSAGE_MODEL],
INKLING_SPECIAL_TOKEN_IDS[CONTENT_TEXT],
]
def structural_tag(
self, content: dict, *, in_answer_block: bool, max_think_tokens: int
) -> dict:
elements = [content, _token(END_MESSAGE)]
if in_answer_block:
return {
"type": "structural_tag",
"format": {"type": "sequence", "elements": elements},
}
thinking = {
"type": "any_tokens",
"exclude_tokens": [INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]],
}
if max_think_tokens >= 0:
thinking["max_tokens"] = max_think_tokens
thinking_message = {
"type": "sequence",
"elements": [
_token(MESSAGE_MODEL),
_token(CONTENT_THINKING),
thinking,
_token(END_MESSAGE),
],
}
elements = [
{"type": "optional", "content": thinking_message},
_token(MESSAGE_MODEL),
_token(CONTENT_TEXT),
*elements,
]
return {
"type": "structural_tag",
"format": {"type": "sequence", "elements": elements},
}
def owns_turn(self, structural_tag: str) -> bool:
# Only the canonical auto-tool schema owns the full turn; user restrictions may not.
return json.loads(structural_tag) == _canonical_inkling_auto_tool_schema()
@@ -36,6 +36,7 @@ from sglang.srt.function_call.muse_glimmer_format import (
has_atem_markers,
partial_marker_len,
)
from sglang.srt.parser.answer_frame import AnswerFrame, InklingAnswerFrame
from sglang.srt.parser.harmony_parser import HarmonyParser
from sglang.srt.parser.inkling_tokenizer import (
CONTENT_INVOKE_TOOL_JSON,
@@ -117,6 +118,9 @@ class BaseReasoningFormatDetector:
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
def get_answer_frame(self) -> Optional[AnswerFrame]:
return None
def _maybe_apply_force_nonempty_content(
self, ret: StreamingParseResult
) -> StreamingParseResult:
@@ -1376,6 +1380,9 @@ class InklingDetector(BaseReasoningFormatDetector):
# Native framing IDs differ from encoding their printed names as plain text.
return [INKLING_SPECIAL_TOKEN_IDS[self.think_end_token]]
def get_answer_frame(self) -> Optional[AnswerFrame]:
return InklingAnswerFrame()
def detect_and_parse(self, text: str) -> StreamingParseResult:
self._buffer = ""
self._kind = None