Fix garbage output for bare-tekken Mistral checkpoints (e.g. Leanstral) (#30396)

python/sglang/srt/utils/hf_transformers/common.py

@@ -185,6 +185,33 @@ def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None):
     )
 
 
+def _cached_file_exists(model_name_or_path, filename, revision=None) -> bool:
+    """Whether *filename* is available locally or in the HF cache (no network)."""
+    try:
+        _resolve_local_or_cached_file(model_name_or_path, filename, revision)
+        return True
+    except Exception:
+        return False
+
+
+def _remote_file_exists(repo_id, filename, revision=None) -> bool:
+    """Whether *filename* exists on the HF hub (HEAD request only, no download).
+
+    Returns False on any error (offline, gated, network, invalid id) so callers
+    fall back to their default path instead of crashing.
+    """
+    from huggingface_hub.constants import HF_HUB_OFFLINE
+
+    if HF_HUB_OFFLINE:
+        return False
+    try:
+        from huggingface_hub import HfApi
+
+        return HfApi().file_exists(repo_id, filename, revision=revision)
+    except Exception:
+        return False
+
+
 def check_gguf_file(model: Union[str, os.PathLike]) -> bool:
     model = Path(model)
     if not model.is_file():

python/sglang/srt/utils/hf_transformers/mistral_utils.py

@@ -11,7 +11,12 @@ from transformers import AutoConfig, PretrainedConfig, WhisperConfig
 
 from sglang.srt.utils import logger
 
-from .common import _ensure_sub_configs, download_from_hf
+from .common import (
+    _cached_file_exists,
+    _ensure_sub_configs,
+    _remote_file_exists,
+    download_from_hf,
+)
 
 
 def adapt_config_dict(
@@ -430,6 +435,34 @@ _MISTRAL_TOKENIZER_REDIRECTS = {
 }
 
 
+def is_bare_tekken_checkpoint(tokenizer_name, revision=None) -> bool:
+    """True iff the checkpoint ships tekken.json but no tokenizer.json.
+
+    AutoTokenizer converts tekken.json on the fly, but the converter assigns
+    BPE ids from rank 0, dropping the 1000 special-token slots that precede
+    the BPE vocab in tekken's id space — every encoded id is shifted and
+    generation produces garbage. Such checkpoints must load through the
+    mistral-common backed tokenizer instead.
+    """
+
+    local_dir = Path(tokenizer_name)
+    if local_dir.is_dir():
+        return (local_dir / "tekken.json").is_file() and not (
+            local_dir / "tokenizer.json"
+        ).is_file()
+
+    if _cached_file_exists(tokenizer_name, "tokenizer.json", revision):
+        return False
+    if _cached_file_exists(tokenizer_name, "tekken.json", revision):
+        return True
+
+    # Cold cache: the tokenizer loads before weights, so tekken.json isn't
+    # cached yet on a first launch — HEAD-probe the hub to still detect it.
+    if not _remote_file_exists(tokenizer_name, "tekken.json", revision):
+        return False
+    return not _remote_file_exists(tokenizer_name, "tokenizer.json", revision)
+
+
 def retry_without_mistral_common_kwargs(tokenizer_name, *args, **common_kwargs):
     """Retry ``AutoTokenizer.from_pretrained`` without kwargs that MistralCommon rejects.
 

python/sglang/srt/utils/hf_transformers/tokenizer.py

@@ -38,6 +38,7 @@ from .common import (
 )
 from .mistral_utils import (
     _MISTRAL_TOKENIZER_REDIRECTS,
+    is_bare_tekken_checkpoint,
     patch_mistral_common_tokenizer,
     retry_without_mistral_common_kwargs,
 )
@@ -496,21 +497,38 @@ def get_tokenizer(
     )
 
     try:
-        tokenizer = _auto_tokenizer_from_pretrained(
-            tokenizer_name, *args, **common_kwargs
-        )
+        if is_bare_tekken_checkpoint(tokenizer_name, tokenizer_revision):
+            from transformers.tokenization_mistral_common import (
+                MistralCommonTokenizer,
+            )
 
-        # With fastokens, the patched TokenizersBackend.from_pretrained already
-        # returned a tokenizer whose backend is a fastokens shim. Re-resolving via
-        # the declared class (e.g. Qwen2Tokenizer) would discard that work.
-        if (
-            type(tokenizer).__name__ == _TOKENIZERS_BACKEND
-            and tokenizer_backend != "fastokens"
-        ):
-            tokenizer = _resolve_tokenizers_backend(
+            logger.info(
+                "Detected bare-tekken checkpoint %s (tekken.json, no "
+                "tokenizer.json); loading via mistral-common MistralCommonTokenizer, "
+                "ignoring tokenizer_backend=%r.",
+                tokenizer_name,
+                tokenizer_backend,
+            )
+
+            tokenizer = MistralCommonTokenizer.from_pretrained(
+                tokenizer_name, revision=tokenizer_revision
+            )
+        else:
+            tokenizer = _auto_tokenizer_from_pretrained(
                 tokenizer_name, *args, **common_kwargs
             )
 
+            # With fastokens, the patched TokenizersBackend.from_pretrained already
+            # returned a tokenizer whose backend is a fastokens shim. Re-resolving via
+            # the declared class (e.g. Qwen2Tokenizer) would discard that work.
+            if (
+                type(tokenizer).__name__ == _TOKENIZERS_BACKEND
+                and tokenizer_backend != "fastokens"
+            ):
+                tokenizer = _resolve_tokenizers_backend(
+                    tokenizer_name, *args, **common_kwargs
+                )
+
         return _apply_post_load_fixes(tokenizer, tokenizer_name, tokenizer_revision)
     except Exception as e:
         if tokenizer_backend == "fastokens":