Upgrade llguidance to 1.7.6 (#31484)

3rdparty/amd/wheel/sglang/pyproject.toml

@@ -31,7 +31,7 @@ runtime_common = [
   "hf_transfer",
   "huggingface_hub",
   "interegular",
-  "llguidance>=0.7.11,<0.8.0",
+  "llguidance>=1.7.6,<2.0.0",
   "mistral_common>=1.11.5",
   "modelscope",
   "msgspec",

python/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
   "interegular",
   "IPython",
   "kernels>=0.14.1,<0.15",
-  "llguidance>=0.7.11,<0.8.0",
+  "llguidance>=1.7.6,<2.0.0",
   "mistral_common>=1.11.5",
   "modelscope",
   "msgspec",

python/pyproject_cpu.toml

@@ -30,7 +30,7 @@ dependencies = [
   "intel-openmp; platform_machine == 'x86_64'",
   "interegular",
   "IPython",
-  "llguidance>=0.7.11,<0.8.0",
+  "llguidance>=1.7.6,<2.0.0",
   "mistral_common>=1.11.5",
   "modelscope",
   "msgspec",

python/pyproject_npu.toml

@@ -32,7 +32,7 @@ dependencies = [
   "huggingface_hub",
   "interegular",
   "IPython",
-  "llguidance>=0.7.11,<0.8.0",
+  "llguidance>=1.7.6,<2.0.0",
   "mistral_common>=1.11.5",
   "modelscope",
   "msgspec",

python/pyproject_other.toml

@@ -32,7 +32,7 @@ runtime_common = [
   "gguf",
   "interegular",
   "IPython",
-  "llguidance>=0.7.11,<0.8.0",
+  "llguidance>=1.7.6,<2.0.0",
   "mistral_common>=1.11.5",
   "modelscope",
   "msgspec",

python/pyproject_xpu.toml

@@ -30,7 +30,7 @@ dependencies = [
   "gguf",
   "interegular",
   "IPython",
-  "llguidance>=0.7.11,<0.8.0",
+  "llguidance>=1.7.6,<2.0.0",
   "mistral_common>=1.11.5",
   "modelscope",
   "msgspec",

python/sglang/srt/constrained/base_grammar_backend.py

@@ -281,6 +281,8 @@ def create_grammar_backend(
             tokenizer=tokenizer,
             any_whitespace=not server_args.constrained_json_disable_any_whitespace,
             whitespace_pattern=server_args.constrained_json_whitespace_pattern,
+            n_vocab=vocab_size,
+            eos_token_ids=eos_token_ids,
         )
     elif name == "none":
         if server_args.enable_strict_thinking:

python/sglang/srt/constrained/llguidance_backend.py

@@ -16,7 +16,7 @@
 import json
 import logging
 import os
-from typing import List, Optional, Tuple
+from typing import Iterable, List, Optional, Tuple, Union
 
 import torch
 from llguidance import LLMatcher, LLTokenizer, StructTag, grammar_from
@@ -37,6 +37,14 @@ from sglang.srt.constrained.utils import is_legacy_structural_tag
 logger = logging.getLogger(__name__)
 
 
+def _normalize_eos_token_ids(
+    eos_token_ids: Optional[Union[int, Iterable[int]]],
+) -> Optional[Union[int, List[int]]]:
+    if eos_token_ids is None or isinstance(eos_token_ids, int):
+        return eos_token_ids
+    return list(eos_token_ids)
+
+
 class GuidanceGrammar(BaseGrammarObject):
 
     def __init__(self, llguidance_tokenizer: LLTokenizer, serialized_grammar: str):
@@ -51,12 +59,12 @@ class GuidanceGrammar(BaseGrammarObject):
         )
         self._check_err()
 
-        self.eos_token = self.llguidance_tokenizer.eos_token
+        self.eos_tokens = set(self.llguidance_tokenizer.eos_tokens)
 
     def accept_token(self, token: int):
         if self.finished:
             return
-        if self.ll_matcher.is_stopped() and token == self.eos_token:
+        if self.ll_matcher.is_stopped() and token in self.eos_tokens:
             self.finished = True
             return
         self.ll_matcher.consume_token(token)
@@ -126,13 +134,18 @@ class GuidanceBackend(BaseGrammarBackend):
         any_whitespace: bool = True,
         whitespace_pattern: Optional[str] = None,
         n_vocab: Optional[int] = None,
+        eos_token_ids: Optional[Union[int, Iterable[int]]] = None,
     ):
         super().__init__()
 
         self.tokenizer = tokenizer
         self.any_whitespace = any_whitespace
         self.whitespace_pattern = whitespace_pattern
-        self.llguidance_tokenizer = from_tokenizer(self.tokenizer, n_vocab)
+        self.llguidance_tokenizer = from_tokenizer(
+            self.tokenizer,
+            n_vocab,
+            eos_token=_normalize_eos_token_ids(eos_token_ids),
+        )
 
     def _from_serialized(self, serialized_grammar) -> BaseGrammarObject:
         try:

python/sglang/test/send_one.py

@@ -20,6 +20,8 @@ import tabulate
 from sglang.profiler import run_profile
 from sglang.srt.utils.network import resolve_base_url
 
+JSON_OBJECT_SCHEMA = json.dumps({"type": "object"})
+
 
 @dataclasses.dataclass
 class BenchArgs:
@@ -176,7 +178,7 @@ def send_one_prompt(
             "Give me 3 trivial information about that city. "
             "Write in a format of json.\nAssistant:"
         )
-        json_schema = "$$ANY$$"
+        json_schema = JSON_OBJECT_SCHEMA
     else:
         json_schema = None
 

test/registered/unit/constrained/test_base_grammar_backend.py

@@ -334,9 +334,13 @@ class TestCreateGrammarBackend(unittest.TestCase):
         args.constrained_json_disable_any_whitespace = False
         args.constrained_json_whitespace_pattern = r"\s+"
 
-        result = create_grammar_backend(args, "tok", 32000)
+        result = create_grammar_backend(args, "tok", 32000, {1, 2})
         mock_guidance_cls.assert_called_once_with(
-            tokenizer="tok", any_whitespace=True, whitespace_pattern=r"\s+"
+            tokenizer="tok",
+            any_whitespace=True,
+            whitespace_pattern=r"\s+",
+            n_vocab=32000,
+            eos_token_ids={1, 2},
         )
         self.assertIs(result, mock_backend)