[Fix] Inkling: frame constrained output in native message tokens (#44716)
This commit is contained in:
@@ -18,7 +18,7 @@ import logging
|
||||
import time
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, NamedTuple, Optional, Tuple
|
||||
from typing import Callable, Dict, List, NamedTuple, Optional, Sequence, Tuple
|
||||
|
||||
import torch
|
||||
|
||||
@@ -223,6 +223,27 @@ class BaseGrammarBackend:
|
||||
def enable_strict_thinking(self):
|
||||
return self._enable_strict_thinking
|
||||
|
||||
def get_request_grammar_key(
|
||||
self,
|
||||
key: Tuple[str, str],
|
||||
*,
|
||||
input_ids: Sequence[int],
|
||||
thinking_budget: Optional[int],
|
||||
) -> Tuple[str, str]:
|
||||
"""Cache key for this request's grammar; may depend on how the prompt ends
|
||||
and on the request's thinking budget."""
|
||||
return key
|
||||
|
||||
def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None:
|
||||
"""Let ``token_ids`` match only token-level framing, never grammar text."""
|
||||
|
||||
def compile_framed_grammar(
|
||||
self, grammar: BaseGrammarObject, frame: Callable[[dict], dict]
|
||||
) -> Optional[BaseGrammarObject]:
|
||||
"""Compile ``frame(content)``, an XGrammar structural tag embedding ``grammar``
|
||||
as content; None if the backend cannot embed it."""
|
||||
return None
|
||||
|
||||
@property
|
||||
def is_support_token_filter(self):
|
||||
return False
|
||||
|
||||
@@ -168,6 +168,11 @@ class GrammarManager:
|
||||
key = (key_type, req.sampling_params.ebnf)
|
||||
elif req.sampling_params.structural_tag is not None:
|
||||
key = ("structural_tag", req.sampling_params.structural_tag)
|
||||
key = self.grammar_backend.get_request_grammar_key(
|
||||
key,
|
||||
input_ids=req.origin_input_ids,
|
||||
thinking_budget=self._get_request_thinking_budget(req),
|
||||
)
|
||||
|
||||
value, cache_hit = self.grammar_backend.get_cached_or_future_value(
|
||||
key, req.require_reasoning
|
||||
|
||||
@@ -13,8 +13,9 @@
|
||||
# ==============================================================================
|
||||
"""The baseclass of a backend for reasoner grammar-guided constrained decoding."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections import OrderedDict
|
||||
from concurrent.futures import Future
|
||||
from typing import List, Optional, Sequence, Tuple, Union
|
||||
|
||||
import torch
|
||||
@@ -32,6 +33,11 @@ from .base_grammar_backend import (
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_OPEN_ANSWER_VARIANT = "open_answer"
|
||||
_THINK_BUDGET_VARIANT = "think_budget="
|
||||
# Arbitrary bound; budgets are a per-request knob, so only recent variants stay cached.
|
||||
_BUDGET_VARIANT_CACHE_SIZE = 32
|
||||
|
||||
|
||||
class ReasonerGrammarObject(BaseGrammarObject):
|
||||
"""Defers grammar constraints until the reasoning end sequence is complete."""
|
||||
@@ -271,9 +277,8 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
|
||||
):
|
||||
super().__init__()
|
||||
self.grammar_backend = grammar_backend
|
||||
think_end_ids = tokenizer.encode(
|
||||
reasoning_parser.detector.think_end_token, add_special_tokens=False
|
||||
)
|
||||
self._budget_variant_keys: OrderedDict[Tuple[str, str], None] = OrderedDict()
|
||||
think_end_ids = reasoning_parser.detector.get_think_end_token_ids(tokenizer)
|
||||
if not think_end_ids:
|
||||
raise ValueError(
|
||||
f"think_end_token '{reasoning_parser.detector.think_end_token}' "
|
||||
@@ -281,6 +286,11 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
|
||||
)
|
||||
self.think_end_ids = think_end_ids
|
||||
self._enable_strict_thinking = enable_strict_thinking
|
||||
self._answer_frame = reasoning_parser.detector.get_answer_frame()
|
||||
if self._answer_frame is not None:
|
||||
self.grammar_backend.reserve_framing_tokens(
|
||||
self._answer_frame.framing_token_ids()
|
||||
)
|
||||
self.think_excluded_token_ids = self._get_think_excluded_token_ids(
|
||||
reasoning_parser, tokenizer
|
||||
)
|
||||
@@ -347,22 +357,93 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
|
||||
return None
|
||||
return self._make_grammar_object(None, reasoning)
|
||||
|
||||
def get_request_grammar_key(
|
||||
self,
|
||||
key: Tuple[str, str],
|
||||
*,
|
||||
input_ids: Sequence[int],
|
||||
thinking_budget: Optional[int],
|
||||
) -> Tuple[str, str]:
|
||||
key_type, key_string = key
|
||||
if not self._is_framed(key_type):
|
||||
return key
|
||||
if self._answer_frame.ends_in_open_answer_block(input_ids):
|
||||
return (f"{key_type}:{_OPEN_ANSWER_VARIANT}", key_string)
|
||||
# Mirrors the strict wrapper, which applies a request budget only while filtering.
|
||||
if not self.enable_token_filter or thinking_budget is None:
|
||||
return key
|
||||
budget = max(thinking_budget, -1)
|
||||
if budget == max(self.max_think_tokens, -1):
|
||||
return key
|
||||
return (f"{key_type}:{_THINK_BUDGET_VARIANT}{budget}", key_string)
|
||||
|
||||
def _is_framed(self, key_type: str) -> bool:
|
||||
return (
|
||||
self._answer_frame is not None
|
||||
and key_type in self._answer_frame.framed_key_types
|
||||
)
|
||||
|
||||
def get_cached_or_future_value(
|
||||
self, key: Tuple[str, str], require_reasoning: bool
|
||||
) -> Tuple[BaseGrammarObject | Future[BaseGrammarObject], bool]:
|
||||
if key in self._budget_variant_keys:
|
||||
self._budget_variant_keys.move_to_end(key)
|
||||
return super().get_cached_or_future_value(key, require_reasoning)
|
||||
|
||||
def set_cache(self, key: Tuple[str, str], value: BaseGrammarObject):
|
||||
super().set_cache(key, value)
|
||||
if _THINK_BUDGET_VARIANT not in key[0]:
|
||||
return
|
||||
self._budget_variant_keys[key] = None
|
||||
self._budget_variant_keys.move_to_end(key)
|
||||
while len(self._budget_variant_keys) > _BUDGET_VARIANT_CACHE_SIZE:
|
||||
evicted, _ = self._budget_variant_keys.popitem(last=False)
|
||||
self.cache.pop(evicted, None)
|
||||
|
||||
def reset(self):
|
||||
super().reset()
|
||||
self._budget_variant_keys.clear()
|
||||
|
||||
def _init_value_dispatch(
|
||||
self, key: Tuple[str, str], reasoning: bool
|
||||
) -> Optional[BaseGrammarObject]:
|
||||
key_type, _, variant = key[0].partition(":")
|
||||
key = (key_type, key[1])
|
||||
ret = self.grammar_backend._init_value_dispatch(key, reasoning)
|
||||
if ret is None or isinstance(ret, InvalidGrammarObject):
|
||||
return ret
|
||||
if key[0] == "full_assistant_ebnf":
|
||||
return ret
|
||||
if not self.enable_strict_thinking and key[0] == "structural_tag":
|
||||
from sglang.srt.function_call.inkling_detector import InklingDetector
|
||||
|
||||
# Only this canonical schema owns the full turn; user restrictions may not.
|
||||
if json.loads(key[1]) == (
|
||||
InklingDetector()
|
||||
.get_auto_tool_call_structural_tag()
|
||||
.model_dump(by_alias=True)
|
||||
):
|
||||
return ret
|
||||
if self._is_framed(key[0]):
|
||||
# The frame delimits thinking by message kind, so it owns the whole
|
||||
# turn; strict thinking only caps the thinking message.
|
||||
framed = self._frame_answer(ret, variant)
|
||||
if framed is not None:
|
||||
return framed
|
||||
if (
|
||||
not self.enable_strict_thinking
|
||||
and key[0] == "structural_tag"
|
||||
and self._answer_frame is not None
|
||||
and self._answer_frame.owns_turn(key[1])
|
||||
):
|
||||
return ret
|
||||
return self._make_grammar_object(ret, reasoning)
|
||||
|
||||
def _frame_answer(
|
||||
self, grammar: BaseGrammarObject, variant: str
|
||||
) -> Optional[BaseGrammarObject]:
|
||||
in_answer_block = variant == _OPEN_ANSWER_VARIANT
|
||||
if variant.startswith(_THINK_BUDGET_VARIANT):
|
||||
max_think_tokens = int(variant.removeprefix(_THINK_BUDGET_VARIANT))
|
||||
elif self.enable_strict_thinking:
|
||||
max_think_tokens = self.max_think_tokens
|
||||
else:
|
||||
max_think_tokens = -1
|
||||
return self.grammar_backend.compile_framed_grammar(
|
||||
grammar,
|
||||
lambda content: self._answer_frame.structural_tag(
|
||||
content,
|
||||
in_answer_block=in_answer_block,
|
||||
max_think_tokens=max_think_tokens,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
from typing import Callable, Dict, List, Optional, Sequence, Tuple, Union
|
||||
|
||||
import torch
|
||||
from xgrammar import (
|
||||
@@ -26,6 +26,7 @@ from xgrammar import (
|
||||
StructuralTag,
|
||||
StructuralTagItem,
|
||||
TokenizerInfo,
|
||||
VocabType,
|
||||
bitmask_dtype,
|
||||
get_bitmask_shape,
|
||||
)
|
||||
@@ -241,7 +242,9 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
|
||||
f"Failed to create XGrammar TokenizerInfo from tokenizer: {e}"
|
||||
)
|
||||
|
||||
self._tokenizer_info = tokenizer_info
|
||||
self.grammar_compiler = GrammarCompiler(tokenizer_info=tokenizer_info)
|
||||
self._framing_compiler = self._new_framing_compiler(tokenizer_info)
|
||||
self.vocab_size = vocab_size
|
||||
self.override_stop_tokens = override_stop_tokens
|
||||
self.any_whitespace = any_whitespace
|
||||
@@ -251,6 +254,41 @@ class XGrammarGrammarBackend(BaseGrammarBackend):
|
||||
def is_support_token_filter(self):
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def _new_framing_compiler(tokenizer_info: TokenizerInfo) -> GrammarCompiler:
|
||||
# The cache mixes up token-level structural tags of different shapes, so
|
||||
# framed grammars compile uncached (seen on xgrammar 0.2.1 through 0.2.7).
|
||||
return GrammarCompiler(tokenizer_info=tokenizer_info, cache_enabled=False)
|
||||
|
||||
def reserve_framing_tokens(self, token_ids: Sequence[int]) -> None:
|
||||
info = self._tokenizer_info
|
||||
reserved = set(token_ids)
|
||||
# A leading 0xFF is invalid UTF-8, so framed text cannot match these while
|
||||
# token elements still match by id; string tool tags spell them, so only framing.
|
||||
vocab = [
|
||||
b"\xff" + piece if token_id in reserved else piece
|
||||
for token_id, piece in enumerate(info.decoded_vocab)
|
||||
]
|
||||
self._framing_compiler = self._new_framing_compiler(
|
||||
TokenizerInfo(
|
||||
vocab,
|
||||
VocabType.RAW,
|
||||
vocab_size=info.vocab_size,
|
||||
stop_token_ids=info.stop_token_ids,
|
||||
)
|
||||
)
|
||||
|
||||
def compile_framed_grammar(
|
||||
self, grammar: XGrammarGrammar, frame: Callable[[dict], dict]
|
||||
) -> XGrammarGrammar:
|
||||
tag = json.dumps(
|
||||
frame({"type": "grammar", "grammar": str(grammar.ctx.grammar)})
|
||||
)
|
||||
ctx = self._framing_compiler.compile_structural_tag(tag)
|
||||
return self._from_context(
|
||||
ctx, tag, GrammarStats(dispatch_type="structural_tag")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def allocate_vocab_mask(vocab_size: int, batch_size: int, device) -> torch.Tensor:
|
||||
return _allocate_token_bitmask(vocab_size, batch_size)
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import functools
|
||||
import json
|
||||
from typing import List, Sequence, Tuple
|
||||
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_MODEL_END_SAMPLING,
|
||||
CONTENT_TEXT,
|
||||
CONTENT_THINKING,
|
||||
END_MESSAGE,
|
||||
INKLING_SPECIAL_TOKEN_IDS,
|
||||
MESSAGE_MODEL,
|
||||
)
|
||||
|
||||
|
||||
class AnswerFrame:
|
||||
"""Native message framing a reasoning format wraps around constrained output.
|
||||
|
||||
A frame owns the whole turn: an optional thinking message, then the answer
|
||||
message holding the constrained content.
|
||||
"""
|
||||
|
||||
framed_key_types: Tuple[str, ...] = ()
|
||||
|
||||
def framing_token_ids(self) -> List[int]:
|
||||
"""Token IDs that may only match as frame tokens, never as content text."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool:
|
||||
"""Whether the prompt already opened the answer block (continuation)."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def structural_tag(
|
||||
self, content: dict, *, in_answer_block: bool, max_think_tokens: int
|
||||
) -> dict:
|
||||
"""XGrammar structural tag framing ``content``; ``max_think_tokens < 0``
|
||||
leaves thinking unbounded."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def owns_turn(self, structural_tag: str) -> bool:
|
||||
"""Whether this structural tag already frames the whole turn itself."""
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _canonical_inkling_auto_tool_schema() -> dict:
|
||||
# Lazy: inkling_detector imports xgrammar unconditionally, and XPU
|
||||
# installs may not ship it.
|
||||
from sglang.srt.function_call.inkling_detector import InklingDetector
|
||||
|
||||
return (
|
||||
InklingDetector().get_auto_tool_call_structural_tag().model_dump(by_alias=True)
|
||||
)
|
||||
|
||||
|
||||
def _token(name: str) -> dict:
|
||||
return {"type": "token", "token": INKLING_SPECIAL_TOKEN_IDS[name]}
|
||||
|
||||
|
||||
class InklingAnswerFrame(AnswerFrame):
|
||||
framed_key_types = ("json", "regex", "ebnf")
|
||||
|
||||
def framing_token_ids(self) -> List[int]:
|
||||
# Inkling's control tokens decode to their literal spellings, so text
|
||||
# positions such as JSON strings would otherwise admit <|end_message|>.
|
||||
return [
|
||||
token_id
|
||||
for token, token_id in INKLING_SPECIAL_TOKEN_IDS.items()
|
||||
if token != CONTENT_MODEL_END_SAMPLING
|
||||
]
|
||||
|
||||
def ends_in_open_answer_block(self, input_ids: Sequence[int]) -> bool:
|
||||
end_message = INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]
|
||||
last_end = next(
|
||||
(
|
||||
i
|
||||
for i in range(len(input_ids) - 1, -1, -1)
|
||||
if input_ids[i] == end_message
|
||||
),
|
||||
-1,
|
||||
)
|
||||
return list(input_ids[last_end + 1 : last_end + 3]) == [
|
||||
INKLING_SPECIAL_TOKEN_IDS[MESSAGE_MODEL],
|
||||
INKLING_SPECIAL_TOKEN_IDS[CONTENT_TEXT],
|
||||
]
|
||||
|
||||
def structural_tag(
|
||||
self, content: dict, *, in_answer_block: bool, max_think_tokens: int
|
||||
) -> dict:
|
||||
elements = [content, _token(END_MESSAGE)]
|
||||
if in_answer_block:
|
||||
return {
|
||||
"type": "structural_tag",
|
||||
"format": {"type": "sequence", "elements": elements},
|
||||
}
|
||||
thinking = {
|
||||
"type": "any_tokens",
|
||||
"exclude_tokens": [INKLING_SPECIAL_TOKEN_IDS[END_MESSAGE]],
|
||||
}
|
||||
if max_think_tokens >= 0:
|
||||
thinking["max_tokens"] = max_think_tokens
|
||||
thinking_message = {
|
||||
"type": "sequence",
|
||||
"elements": [
|
||||
_token(MESSAGE_MODEL),
|
||||
_token(CONTENT_THINKING),
|
||||
thinking,
|
||||
_token(END_MESSAGE),
|
||||
],
|
||||
}
|
||||
elements = [
|
||||
{"type": "optional", "content": thinking_message},
|
||||
_token(MESSAGE_MODEL),
|
||||
_token(CONTENT_TEXT),
|
||||
*elements,
|
||||
]
|
||||
return {
|
||||
"type": "structural_tag",
|
||||
"format": {"type": "sequence", "elements": elements},
|
||||
}
|
||||
|
||||
def owns_turn(self, structural_tag: str) -> bool:
|
||||
# Only the canonical auto-tool schema owns the full turn; user restrictions may not.
|
||||
return json.loads(structural_tag) == _canonical_inkling_auto_tool_schema()
|
||||
@@ -36,6 +36,7 @@ from sglang.srt.function_call.muse_glimmer_format import (
|
||||
has_atem_markers,
|
||||
partial_marker_len,
|
||||
)
|
||||
from sglang.srt.parser.answer_frame import AnswerFrame, InklingAnswerFrame
|
||||
from sglang.srt.parser.harmony_parser import HarmonyParser
|
||||
from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_INVOKE_TOOL_JSON,
|
||||
@@ -117,6 +118,9 @@ class BaseReasoningFormatDetector:
|
||||
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
|
||||
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
|
||||
|
||||
def get_answer_frame(self) -> Optional[AnswerFrame]:
|
||||
return None
|
||||
|
||||
def _maybe_apply_force_nonempty_content(
|
||||
self, ret: StreamingParseResult
|
||||
) -> StreamingParseResult:
|
||||
@@ -1376,6 +1380,9 @@ class InklingDetector(BaseReasoningFormatDetector):
|
||||
# Native framing IDs differ from encoding their printed names as plain text.
|
||||
return [INKLING_SPECIAL_TOKEN_IDS[self.think_end_token]]
|
||||
|
||||
def get_answer_frame(self) -> Optional[AnswerFrame]:
|
||||
return InklingAnswerFrame()
|
||||
|
||||
def detect_and_parse(self, text: str) -> StreamingParseResult:
|
||||
self._buffer = ""
|
||||
self._kind = None
|
||||
|
||||
Reference in New Issue
Block a user