[Fix] Keep Inkling automatic tool grammar active across the response (#40468)

Co-authored-by: Jeff Hanson <2469458+jshanson7@users.noreply.github.com>
This commit is contained in:
Jeff Hanson
2026-09-23 13:14:36 +08:00
committed by GitHub
co-authored by Jeff Hanson
parent ced0a2c98c
commit 5b8d8b2f88
4 changed files with 313 additions and 8 deletions
@@ -13,6 +13,7 @@
# ==============================================================================
"""The baseclass of a backend for reasoner grammar-guided constrained decoding."""
import json
import logging
from typing import List, Optional, Sequence, Tuple, Union
@@ -354,4 +355,14 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
return ret
if key[0] == "full_assistant_ebnf":
return ret
if not self.enable_strict_thinking and key[0] == "structural_tag":
from sglang.srt.function_call.inkling_detector import InklingDetector
# Only this canonical schema owns the full turn; user restrictions may not.
if json.loads(key[1]) == (
InklingDetector()
.get_auto_tool_call_structural_tag()
.model_dump(by_alias=True)
):
return ret
return self._make_grammar_object(ret, reasoning)
+2 -2
View File
@@ -916,8 +916,8 @@ class Scheduler(
stream_reasoning=False,
tokenizer=self.tokenizer,
)
think_end_ids = self.tokenizer.encode(
reasoning_parser.detector.think_end_token, add_special_tokens=False
think_end_ids = reasoning_parser.detector.get_think_end_token_ids(
self.tokenizer
)
if think_end_ids:
self.model_config.think_end_ids = think_end_ids
@@ -2,6 +2,8 @@ import inspect
import re
from typing import Dict, List, Optional, Tuple, Type
from transformers import PreTrainedTokenizerBase
from sglang.srt.entrypoints.openai.encoding_dsv4 import dsml_token as dsv4_dsml_token
from sglang.srt.entrypoints.openai.encoding_dsv4 import eos_token as dsv4_eos_token
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
@@ -43,6 +45,7 @@ from sglang.srt.parser.inkling_tokenizer import (
CONTENT_THINKING,
END_MESSAGE,
INKLING_CONTROL_TOKENS,
INKLING_SPECIAL_TOKEN_IDS,
MESSAGE_MODEL,
)
@@ -111,6 +114,9 @@ class BaseReasoningFormatDetector:
if self.think_end_token in self.previous_content:
self._in_reasoning = False
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
def _maybe_apply_force_nonempty_content(
self, ret: StreamingParseResult
) -> StreamingParseResult:
@@ -1333,6 +1339,11 @@ class InklingDetector(BaseReasoningFormatDetector):
self._pending_header = ""
self._pending_reasoning = ""
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
del tokenizer
# Native framing IDs differ from encoding their printed names as plain text.
return [INKLING_SPECIAL_TOKEN_IDS[self.think_end_token]]
def detect_and_parse(self, text: str) -> StreamingParseResult:
self._buffer = ""
self._kind = None