diff --git a/3rdparty/amd/wheel/sglang/pyproject.toml b/3rdparty/amd/wheel/sglang/pyproject.toml index a1e231c7d..8ae54899c 100644 --- a/3rdparty/amd/wheel/sglang/pyproject.toml +++ b/3rdparty/amd/wheel/sglang/pyproject.toml @@ -31,7 +31,7 @@ runtime_common = [ "hf_transfer", "huggingface_hub", "interegular", - "llguidance>=0.7.11,<0.8.0", + "llguidance>=1.7.6,<2.0.0", "mistral_common>=1.11.5", "modelscope", "msgspec", diff --git a/python/pyproject.toml b/python/pyproject.toml index 91672b406..3095c03e1 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -37,7 +37,7 @@ dependencies = [ "interegular", "IPython", "kernels>=0.14.1,<0.15", - "llguidance>=0.7.11,<0.8.0", + "llguidance>=1.7.6,<2.0.0", "mistral_common>=1.11.5", "modelscope", "msgspec", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index ef8c4fbbb..36d4568c0 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -30,7 +30,7 @@ dependencies = [ "intel-openmp; platform_machine == 'x86_64'", "interegular", "IPython", - "llguidance>=0.7.11,<0.8.0", + "llguidance>=1.7.6,<2.0.0", "mistral_common>=1.11.5", "modelscope", "msgspec", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index 6f7713127..09f43088e 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -32,7 +32,7 @@ dependencies = [ "huggingface_hub", "interegular", "IPython", - "llguidance>=0.7.11,<0.8.0", + "llguidance>=1.7.6,<2.0.0", "mistral_common>=1.11.5", "modelscope", "msgspec", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index b1e8f8843..34f2179ce 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -32,7 +32,7 @@ runtime_common = [ "gguf", "interegular", "IPython", - "llguidance>=0.7.11,<0.8.0", + "llguidance>=1.7.6,<2.0.0", "mistral_common>=1.11.5", "modelscope", "msgspec", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index 74d985005..28aeb9f6f 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -30,7 +30,7 @@ dependencies = [ "gguf", "interegular", "IPython", - "llguidance>=0.7.11,<0.8.0", + "llguidance>=1.7.6,<2.0.0", "mistral_common>=1.11.5", "modelscope", "msgspec", diff --git a/python/sglang/srt/constrained/base_grammar_backend.py b/python/sglang/srt/constrained/base_grammar_backend.py index f855204d8..03ad0b450 100644 --- a/python/sglang/srt/constrained/base_grammar_backend.py +++ b/python/sglang/srt/constrained/base_grammar_backend.py @@ -281,6 +281,8 @@ def create_grammar_backend( tokenizer=tokenizer, any_whitespace=not server_args.constrained_json_disable_any_whitespace, whitespace_pattern=server_args.constrained_json_whitespace_pattern, + n_vocab=vocab_size, + eos_token_ids=eos_token_ids, ) elif name == "none": if server_args.enable_strict_thinking: diff --git a/python/sglang/srt/constrained/llguidance_backend.py b/python/sglang/srt/constrained/llguidance_backend.py index 91b948408..a758e0816 100644 --- a/python/sglang/srt/constrained/llguidance_backend.py +++ b/python/sglang/srt/constrained/llguidance_backend.py @@ -16,7 +16,7 @@ import json import logging import os -from typing import List, Optional, Tuple +from typing import Iterable, List, Optional, Tuple, Union import torch from llguidance import LLMatcher, LLTokenizer, StructTag, grammar_from @@ -37,6 +37,14 @@ from sglang.srt.constrained.utils import is_legacy_structural_tag logger = logging.getLogger(__name__) +def _normalize_eos_token_ids( + eos_token_ids: Optional[Union[int, Iterable[int]]], +) -> Optional[Union[int, List[int]]]: + if eos_token_ids is None or isinstance(eos_token_ids, int): + return eos_token_ids + return list(eos_token_ids) + + class GuidanceGrammar(BaseGrammarObject): def __init__(self, llguidance_tokenizer: LLTokenizer, serialized_grammar: str): @@ -51,12 +59,12 @@ class GuidanceGrammar(BaseGrammarObject): ) self._check_err() - self.eos_token = self.llguidance_tokenizer.eos_token + self.eos_tokens = set(self.llguidance_tokenizer.eos_tokens) def accept_token(self, token: int): if self.finished: return - if self.ll_matcher.is_stopped() and token == self.eos_token: + if self.ll_matcher.is_stopped() and token in self.eos_tokens: self.finished = True return self.ll_matcher.consume_token(token) @@ -126,13 +134,18 @@ class GuidanceBackend(BaseGrammarBackend): any_whitespace: bool = True, whitespace_pattern: Optional[str] = None, n_vocab: Optional[int] = None, + eos_token_ids: Optional[Union[int, Iterable[int]]] = None, ): super().__init__() self.tokenizer = tokenizer self.any_whitespace = any_whitespace self.whitespace_pattern = whitespace_pattern - self.llguidance_tokenizer = from_tokenizer(self.tokenizer, n_vocab) + self.llguidance_tokenizer = from_tokenizer( + self.tokenizer, + n_vocab, + eos_token=_normalize_eos_token_ids(eos_token_ids), + ) def _from_serialized(self, serialized_grammar) -> BaseGrammarObject: try: diff --git a/python/sglang/test/send_one.py b/python/sglang/test/send_one.py index d2615d53f..e6cd7d992 100644 --- a/python/sglang/test/send_one.py +++ b/python/sglang/test/send_one.py @@ -20,6 +20,8 @@ import tabulate from sglang.profiler import run_profile from sglang.srt.utils.network import resolve_base_url +JSON_OBJECT_SCHEMA = json.dumps({"type": "object"}) + @dataclasses.dataclass class BenchArgs: @@ -176,7 +178,7 @@ def send_one_prompt( "Give me 3 trivial information about that city. " "Write in a format of json.\nAssistant:" ) - json_schema = "$$ANY$$" + json_schema = JSON_OBJECT_SCHEMA else: json_schema = None diff --git a/test/registered/unit/constrained/test_base_grammar_backend.py b/test/registered/unit/constrained/test_base_grammar_backend.py index 84e42a344..6b02f5db7 100644 --- a/test/registered/unit/constrained/test_base_grammar_backend.py +++ b/test/registered/unit/constrained/test_base_grammar_backend.py @@ -334,9 +334,13 @@ class TestCreateGrammarBackend(unittest.TestCase): args.constrained_json_disable_any_whitespace = False args.constrained_json_whitespace_pattern = r"\s+" - result = create_grammar_backend(args, "tok", 32000) + result = create_grammar_backend(args, "tok", 32000, {1, 2}) mock_guidance_cls.assert_called_once_with( - tokenizer="tok", any_whitespace=True, whitespace_pattern=r"\s+" + tokenizer="tok", + any_whitespace=True, + whitespace_pattern=r"\s+", + n_vocab=32000, + eos_token_ids={1, 2}, ) self.assertIs(result, mock_backend)