[Fix] Keep Inkling automatic tool grammar active across the response (#40468)
Co-authored-by: Jeff Hanson <2469458+jshanson7@users.noreply.github.com>
This commit is contained in:
co-authored by
Jeff Hanson
parent
ced0a2c98c
commit
5b8d8b2f88
@@ -13,6 +13,7 @@
|
||||
# ==============================================================================
|
||||
"""The baseclass of a backend for reasoner grammar-guided constrained decoding."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Optional, Sequence, Tuple, Union
|
||||
|
||||
@@ -354,4 +355,14 @@ class ReasonerGrammarBackend(BaseGrammarBackend):
|
||||
return ret
|
||||
if key[0] == "full_assistant_ebnf":
|
||||
return ret
|
||||
if not self.enable_strict_thinking and key[0] == "structural_tag":
|
||||
from sglang.srt.function_call.inkling_detector import InklingDetector
|
||||
|
||||
# Only this canonical schema owns the full turn; user restrictions may not.
|
||||
if json.loads(key[1]) == (
|
||||
InklingDetector()
|
||||
.get_auto_tool_call_structural_tag()
|
||||
.model_dump(by_alias=True)
|
||||
):
|
||||
return ret
|
||||
return self._make_grammar_object(ret, reasoning)
|
||||
|
||||
@@ -916,8 +916,8 @@ class Scheduler(
|
||||
stream_reasoning=False,
|
||||
tokenizer=self.tokenizer,
|
||||
)
|
||||
think_end_ids = self.tokenizer.encode(
|
||||
reasoning_parser.detector.think_end_token, add_special_tokens=False
|
||||
think_end_ids = reasoning_parser.detector.get_think_end_token_ids(
|
||||
self.tokenizer
|
||||
)
|
||||
if think_end_ids:
|
||||
self.model_config.think_end_ids = think_end_ids
|
||||
|
||||
@@ -2,6 +2,8 @@ import inspect
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import dsml_token as dsv4_dsml_token
|
||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import eos_token as dsv4_eos_token
|
||||
from sglang.srt.entrypoints.openai.encoding_dsv4 import (
|
||||
@@ -43,6 +45,7 @@ from sglang.srt.parser.inkling_tokenizer import (
|
||||
CONTENT_THINKING,
|
||||
END_MESSAGE,
|
||||
INKLING_CONTROL_TOKENS,
|
||||
INKLING_SPECIAL_TOKEN_IDS,
|
||||
MESSAGE_MODEL,
|
||||
)
|
||||
|
||||
@@ -111,6 +114,9 @@ class BaseReasoningFormatDetector:
|
||||
if self.think_end_token in self.previous_content:
|
||||
self._in_reasoning = False
|
||||
|
||||
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
|
||||
return tokenizer.encode(self.think_end_token, add_special_tokens=False)
|
||||
|
||||
def _maybe_apply_force_nonempty_content(
|
||||
self, ret: StreamingParseResult
|
||||
) -> StreamingParseResult:
|
||||
@@ -1333,6 +1339,11 @@ class InklingDetector(BaseReasoningFormatDetector):
|
||||
self._pending_header = ""
|
||||
self._pending_reasoning = ""
|
||||
|
||||
def get_think_end_token_ids(self, tokenizer: PreTrainedTokenizerBase) -> List[int]:
|
||||
del tokenizer
|
||||
# Native framing IDs differ from encoding their printed names as plain text.
|
||||
return [INKLING_SPECIAL_TOKEN_IDS[self.think_end_token]]
|
||||
|
||||
def detect_and_parse(self, text: str) -> StreamingParseResult:
|
||||
self._buffer = ""
|
||||
self._kind = None
|
||||
|
||||
Reference in New Issue
Block a user