[Constrained] Support MistralCommon tokenizers in the XGrammar backend (#35215)

Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>

python/sglang/srt/entrypoints/openai/serving_chat.py

@@ -21,6 +21,18 @@ class ThinkingMode(str, Enum):
 import jinja2
 import orjson
 from fastapi import Request
+
+try:
+    from mistral_common.exceptions import MistralCommonException
+
+    _MISTRAL_COMMON_ERRORS: tuple[type[BaseException], ...] = (MistralCommonException,)
+except ImportError:
+    _MISTRAL_COMMON_ERRORS = ()
+
+_CHAT_TEMPLATE_CLIENT_ERRORS: tuple[type[BaseException], ...] = (
+    jinja2.TemplateError,
+    TypeError,
+) + _MISTRAL_COMMON_ERRORS
 from fastapi.responses import ORJSONResponse, StreamingResponse
 from jsonschema import Draft202012Validator, SchemaError
 
@@ -1367,7 +1379,7 @@ class OpenAIServingChat(OpenAIServingBase):
                     prompt_ids = self.tokenizer_manager.tokenizer.encode(
                         rendered_prompt, **encode_kwargs
                     )
-                except (jinja2.TemplateError, TypeError) as template_error:
+                except _CHAT_TEMPLATE_CLIENT_ERRORS as template_error:
                     # Template errors (e.g., from raise_exception in Jinja templates)
                     # and TypeError (e.g., tojson filter on Jinja2 Undefined variables)
                     # should be treated as client errors (400 BadRequest)

python/sglang/srt/utils/hf_transformers/mistral_utils.py

@@ -634,4 +634,47 @@ def patch_mistral_common_tokenizer(tokenizer):
         return tokenizer._orig_apply_chat_template(messages, **kwargs)
 
     tokenizer.apply_chat_template = _safe_apply_chat_template
+
+    def init_xgrammar():
+        from xgrammar import TokenizerInfo
+
+        tekken = getattr(
+            getattr(tokenizer.tokenizer, "instruct_tokenizer", None), "tokenizer", None
+        )
+        if tekken is None or not hasattr(tekken, "id_to_byte_piece"):
+            logger.warning(
+                "Cannot build XGrammar TokenizerInfo: no Tekkenizer found under %s",
+                type(tokenizer).__name__,
+            )
+            return None, None
+
+        try:
+            placeholder = "<|xg_special_token_{}|>"
+            encoded_vocab = []
+            for token_id in range(tekken.n_words):
+                piece = (
+                    tekken.id_to_piece(token_id)
+                    if token_id < tekken.num_special_tokens
+                    else tekken.id_to_byte_piece(token_id)
+                )
+                # XGrammar reserves b"\x00"-prefixed tokens as special markers.
+                if isinstance(piece, bytes) and piece.startswith(b"\x00"):
+                    piece = placeholder.format(f"nul{token_id}")
+                encoded_vocab.append(piece)
+
+            eos_token_id = getattr(tokenizer, "eos_token_id", None)
+            override_stop_tokens = [eos_token_id] if eos_token_id is not None else None
+            tokenizer_info = TokenizerInfo(
+                encoded_vocab, stop_token_ids=override_stop_tokens
+            )
+        except Exception as e:
+            logger.warning(
+                "Failed to build XGrammar TokenizerInfo for %s: %s",
+                type(tokenizer).__name__,
+                e,
+            )
+            return None, None
+        return tokenizer_info, override_stop_tokens
+
+    tokenizer.init_xgrammar = init_xgrammar
     return tokenizer