From 34fef07a15ea01add2d4b324530acaa401b0655e Mon Sep 17 00:00:00 2001 From: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> Date: Thu, 16 Apr 2026 04:03:44 +0100 Subject: [PATCH] Upgrade transformers to 5.5.3 and refactor hf_transformers_utils into subpackage (#21569) --- python/pyproject.toml | 5 +- python/pyproject_cpu.toml | 5 +- python/pyproject_npu.toml | 5 +- python/pyproject_other.toml | 5 +- python/pyproject_xpu.toml | 6 +- python/sglang/srt/configs/qwen3_5.py | 16 + python/sglang/srt/configs/step3p5.py | 9 + python/sglang/srt/models/qwen3_vl.py | 8 +- .../srt/utils/hf_transformers/__init__.py | 67 + .../srt/utils/hf_transformers/common.py | 438 +++++ .../srt/utils/hf_transformers/compat.py | 458 +++++ .../srt/utils/hf_transformers/config.py | 212 +++ .../{ => hf_transformers}/mistral_utils.py | 198 ++- .../srt/utils/hf_transformers/processor.py | 288 ++++ .../srt/utils/hf_transformers/tokenizer.py | 551 ++++++ .../sglang/srt/utils/hf_transformers_utils.py | 1483 +---------------- .../unit/utils/test_hf_transformers.py | 586 +++++++ test/registered/vlm/test_vlm_input_format.py | 13 +- 18 files changed, 2838 insertions(+), 1515 deletions(-) create mode 100644 python/sglang/srt/utils/hf_transformers/__init__.py create mode 100644 python/sglang/srt/utils/hf_transformers/common.py create mode 100644 python/sglang/srt/utils/hf_transformers/compat.py create mode 100644 python/sglang/srt/utils/hf_transformers/config.py rename python/sglang/srt/utils/{ => hf_transformers}/mistral_utils.py (65%) create mode 100644 python/sglang/srt/utils/hf_transformers/processor.py create mode 100644 python/sglang/srt/utils/hf_transformers/tokenizer.py create mode 100644 test/registered/unit/utils/test_hf_transformers.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 491958878..471fedb86 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -35,6 +35,7 @@ dependencies = [ "modelscope", "msgspec", "ninja", + "easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports "numpy", "nvidia-cutlass-dsl>=4.4.1", "nvidia-ml-py", @@ -70,8 +71,8 @@ dependencies = [ "av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')", "torchvision", "tqdm", - "mistral_common>=1.9.0", - "transformers==5.3.0", + "mistral_common>=1.11.0", + "transformers==5.5.4", "uvicorn", "uvloop", "watchfiles", diff --git a/python/pyproject_cpu.toml b/python/pyproject_cpu.toml index 1597212e9..4341bc843 100644 --- a/python/pyproject_cpu.toml +++ b/python/pyproject_cpu.toml @@ -31,6 +31,7 @@ dependencies = [ "llguidance>=0.7.11,<0.8.0", "modelscope", "msgspec", + "easydict", "ninja", "numpy", "openai-harmony==0.0.4", @@ -60,8 +61,8 @@ dependencies = [ "torchaudio==2.9.0", "torchvision==0.24.0", "tqdm", - "mistral_common>=1.9.0", - "transformers==5.3.0", + "mistral_common>=1.11.0", + "transformers==5.5.4", "triton==3.5.0", "uvicorn", "uvloop", diff --git a/python/pyproject_npu.toml b/python/pyproject_npu.toml index e2bbfd798..1c5859e84 100644 --- a/python/pyproject_npu.toml +++ b/python/pyproject_npu.toml @@ -25,6 +25,7 @@ dependencies = [ "datasets", "einops", "fastapi", + "easydict", "gguf", "hf_transfer", "huggingface_hub", @@ -57,8 +58,8 @@ dependencies = [ "timm==1.0.16", "torchao==0.9.0", "tqdm", - "mistral_common>=1.9.0", - "transformers==5.3.0", + "mistral_common>=1.11.0", + "transformers==5.5.4", "uvicorn", "uvloop", "xgrammar==0.1.32", diff --git a/python/pyproject_other.toml b/python/pyproject_other.toml index 0e6da96c1..37de1f458 100755 --- a/python/pyproject_other.toml +++ b/python/pyproject_other.toml @@ -25,6 +25,7 @@ runtime_common = [ "build", "compressed-tensors", "datasets", + "easydict", "einops", "fastapi", "gguf", @@ -57,8 +58,8 @@ runtime_common = [ "timm==1.0.16", "torchao==0.9.0", "tqdm", - "mistral_common>=1.9.0", - "transformers==5.3.0", + "mistral_common>=1.11.0", + "transformers==5.5.4", "uvicorn", "uvloop", "xgrammar==0.1.32", diff --git a/python/pyproject_xpu.toml b/python/pyproject_xpu.toml index 8e15e14c8..55eee5feb 100644 --- a/python/pyproject_xpu.toml +++ b/python/pyproject_xpu.toml @@ -27,7 +27,9 @@ dependencies = [ "blobfile==3.0.0", "build", "compressed-tensors", + "addict", "datasets", + "easydict", "einops", "fastapi", "gguf", @@ -60,8 +62,8 @@ dependencies = [ "timm==1.0.16", "torchao==0.9.0+xpu", "tqdm", - "mistral_common>=1.9.0", - "transformers==5.3.0", + "mistral_common>=1.11.0", + "transformers==5.5.4", "uvicorn", "uvloop", # "xgrammar==0.1.24", , xgrammar depends on CUDA PyTorch and Triton only diff --git a/python/sglang/srt/configs/qwen3_5.py b/python/sglang/srt/configs/qwen3_5.py index fddf783ab..98de12f7e 100644 --- a/python/sglang/srt/configs/qwen3_5.py +++ b/python/sglang/srt/configs/qwen3_5.py @@ -8,6 +8,9 @@ class Qwen3_5VisionConfig(Qwen3VLVisionConfig): model_type = "qwen3_5" base_config_key = "vision_config" + def __init__(self, **kwargs): + super().__init__(**kwargs) + class Qwen3_5TextConfig(Qwen3NextConfig): model_type = "qwen3_5_text" @@ -109,14 +112,27 @@ class Qwen3_5Config(PretrainedConfig): class Qwen3_5MoeVisionConfig(Qwen3_5VisionConfig): model_type = "qwen3_5_moe" + def __init__(self, **kwargs): + super().__init__(**kwargs) + class Qwen3_5MoeTextConfig(Qwen3_5TextConfig): model_type = "qwen3_5_moe_text" + def __init__(self, **kwargs): + super().__init__(**kwargs) + +# All Moe variant classes need explicit __init__ because the kw_only=True +# dataclass decorator in transformers v5.5.3+ auto-generates __init__ for +# subclasses, bypassing parent __init__ methods that set up attributes +# (e.g. norm_topk_prob, rope_scaling) and convert sub-config dicts to objects. class Qwen3_5MoeConfig(Qwen3_5Config): model_type = "qwen3_5_moe" sub_configs = { "vision_config": Qwen3_5MoeVisionConfig, "text_config": Qwen3_5MoeTextConfig, } + + def __init__(self, **kwargs): + super().__init__(**kwargs) diff --git a/python/sglang/srt/configs/step3p5.py b/python/sglang/srt/configs/step3p5.py index eebf137fb..d33572255 100644 --- a/python/sglang/srt/configs/step3p5.py +++ b/python/sglang/srt/configs/step3p5.py @@ -94,4 +94,13 @@ class Step3p5Config(PretrainedConfig): self.moe_layers_enum = moe_layers_enum self.layer_types = layer_types self.sliding_window = sliding_window + # The upstream Step-3.5-Flash config has layer_types with 48 entries + # but num_hidden_layers=45. The extra 3 are for MTP/nextn predict + # layers (indices 45-47) used by Step3p5DecoderLayer during EAGLE + # speculative decoding. Temporarily align num_hidden_layers to pass + # the transformers v5.5.3+ validator, then restore the real value. + real_num_hidden_layers = self.num_hidden_layers + if layer_types is not None and len(layer_types) != self.num_hidden_layers: + self.num_hidden_layers = len(layer_types) super().__init__(**kwargs) + self.num_hidden_layers = real_num_hidden_layers diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 0bc4dc570..f8abf9b96 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1091,9 +1091,15 @@ class Qwen3VLForConditionalGeneration(nn.Module): if language_model_cls is Qwen3LLMModel: self.config: Qwen3VLConfig = config # for qwen3-vl else: - self.config = config.text_config # for qwen3-omni + self.config = config.text_config # for qwen3-omni / qwen3-vl-moe self.config.encoder_only = getattr(config, "encoder_only", False) self.config.language_only = getattr(config, "language_only", False) + # Propagate tie_word_embeddings from parent config. In transformers + # v5.5.3+, Qwen3VLMoeTextConfig sets tie_word_embeddings=True by + # default but the actual model checkpoint has a separate lm_head. + # The parent Qwen3VLMoeConfig correctly has tie_word_embeddings=False. + if hasattr(config, "tie_word_embeddings"): + self.config.tie_word_embeddings = config.tie_word_embeddings if not hasattr(config, "encoder_only") or not config.encoder_only: self.model = language_model_cls( diff --git a/python/sglang/srt/utils/hf_transformers/__init__.py b/python/sglang/srt/utils/hf_transformers/__init__.py new file mode 100644 index 000000000..86da65690 --- /dev/null +++ b/python/sglang/srt/utils/hf_transformers/__init__.py @@ -0,0 +1,67 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Hugging Face Transformers utilities. + +This package provides HF Transformers helpers, split into submodules +(common, compat, config, tokenizer, processor, mistral_utils). +All public symbols are re-exported here for convenience. The old import +path ``sglang.srt.utils.hf_transformers_utils`` is preserved by a +separate shim module. +""" + +from .compat import apply_all as _apply_compat + +_apply_compat() + +from .common import ( # noqa: E402 + CONTEXT_LENGTH_KEYS, + AutoConfig, + attach_additional_stop_token_ids, + check_gguf_file, + download_from_hf, + get_context_length, + get_generation_config, + get_hf_text_config, + get_rope_config, + get_sparse_attention_config, + get_tokenizer_from_processor, +) +from .compat import normalize_rope_scaling_compat # noqa: E402 +from .config import get_config # noqa: E402 +from .processor import get_processor # noqa: E402 +from .tokenizer import ( # noqa: E402 + _fix_added_tokens_encoding, + _fix_v5_add_bos_eos_token, + get_tokenizer, +) + +__all__ = [ + "AutoConfig", + "CONTEXT_LENGTH_KEYS", + "_fix_added_tokens_encoding", + "_fix_v5_add_bos_eos_token", + "attach_additional_stop_token_ids", + "check_gguf_file", + "download_from_hf", + "get_config", + "get_context_length", + "get_generation_config", + "get_hf_text_config", + "get_processor", + "get_rope_config", + "get_sparse_attention_config", + "get_tokenizer", + "get_tokenizer_from_processor", + "normalize_rope_scaling_compat", +] diff --git a/python/sglang/srt/utils/hf_transformers/common.py b/python/sglang/srt/utils/hf_transformers/common.py new file mode 100644 index 000000000..40b73c123 --- /dev/null +++ b/python/sglang/srt/utils/hf_transformers/common.py @@ -0,0 +1,438 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Shared helpers used by config, tokenizer, and processor modules.""" + +import json +import os +from pathlib import Path +from typing import Any, Dict, Optional, Type, Union + +import torch +from huggingface_hub import snapshot_download + +from sglang.srt.configs import ( + AfmoeConfig, + BailingHybridConfig, + ChatGLMConfig, + DbrxConfig, + DeepseekVL2Config, + DotsOCRConfig, + DotsVLMConfig, + ExaoneConfig, + FalconH1Config, + GraniteMoeHybridConfig, + JetNemotronConfig, + JetVLMConfig, + KimiK25Config, + KimiLinearConfig, + KimiVLConfig, + LongcatFlashConfig, + MultiModalityConfig, + NemotronH_Nano_VL_V2_Config, + NemotronHConfig, + Olmo3Config, + Qwen3_5Config, + Qwen3_5MoeConfig, + Qwen3NextConfig, + Step3p5Config, + Step3VLConfig, +) +from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config +from sglang.srt.configs.internvl import InternVLChatConfig +from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset + +from .compat import normalize_rope_scaling_compat + +if get_bool_env_var("SGLANG_USE_MODELSCOPE"): + from modelscope import AutoConfig, GenerationConfig +else: + from transformers import AutoConfig, GenerationConfig + +from transformers import PretrainedConfig + +# --------------------------------------------------------------------------- +# Config registry +# --------------------------------------------------------------------------- + +_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = { + cls.model_type: cls + for cls in [ + AfmoeConfig, + BailingHybridConfig, + ChatGLMConfig, + DbrxConfig, + ExaoneConfig, + DeepseekVL2Config, + MultiModalityConfig, + KimiVLConfig, + InternVLChatConfig, + Step3VLConfig, + LongcatFlashConfig, + Olmo3Config, + KimiLinearConfig, + Qwen3NextConfig, + FalconH1Config, + GraniteMoeHybridConfig, + DotsVLMConfig, + DotsOCRConfig, + NemotronH_Nano_VL_V2_Config, + NemotronHConfig, + DeepseekVLV2Config, + Qwen3_5Config, + Qwen3_5MoeConfig, + JetNemotronConfig, + JetVLMConfig, + KimiK25Config, + Step3p5Config, + ] +} + +for name, cls in _CONFIG_REGISTRY.items(): + try: + AutoConfig.register(name, cls) + except ValueError as e: + err = str(e).lower() + if "already registered" not in err and "already used" not in err: + logger.warning("Failed to register config %s: %s", name, e) + + +# --------------------------------------------------------------------------- +# Download / path helpers +# --------------------------------------------------------------------------- + + +def download_from_hf( + model_path: str, + allow_patterns: Optional[Union[str, list]] = None, +): + if os.path.exists(model_path): + return model_path + + if not allow_patterns: + allow_patterns = ["*.json", "*.bin", "*.model"] + + return snapshot_download(model_path, allow_patterns=allow_patterns) + + +def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None): + """Resolve a file from a local directory or HF hub cache (no network).""" + local_path = Path(model_name_or_path) / filename + if local_path.is_file(): + return str(local_path) + from huggingface_hub import hf_hub_download + + return hf_hub_download( + model_name_or_path, filename, revision=revision, local_files_only=True + ) + + +def check_gguf_file(model: Union[str, os.PathLike]) -> bool: + model = Path(model) + if not model.is_file(): + return False + elif model.suffix == ".gguf": + return True + + with open(model, "rb") as f: + header = f.read(4) + return header == b"GGUF" + + +# --------------------------------------------------------------------------- +# Rope / text config helpers +# --------------------------------------------------------------------------- + + +def get_rope_config(config): + """Get (rope_theta, rope_params) from config, supporting both v4 and v5. + + Trust-remote-code configs or parent configs passed to sub-models may not + have the v5 ``rope_parameters`` property, so we fall back to the v4-style + ``config.rope_theta`` / ``config.rope_scaling`` attributes. + + Returns: + (rope_theta, rope_params): In v5, rope_params is the full + rope_parameters dict (which subsumes rope_scaling and includes + rope_theta). In v4, rope_params is the rope_scaling dict or None. + """ + rope_params = getattr(config, "rope_parameters", None) + if rope_params is not None: + return rope_params["rope_theta"], rope_params + return config.rope_theta, getattr(config, "rope_scaling", None) + + +def _patch_text_config(parent_config: PretrainedConfig, text_config): + """Synchronize standard attributes between parent config and text sub-config. + + In transformers v5, the "untangle config" refactor removed automatic + inheritance of top-level PretrainedConfig attributes (pad_token_id, + tie_word_embeddings, etc.) from sub-configs. Downstream code expects + these attributes to be present on both configs (some models pass the + parent directly to the language model, others pass the text sub-config), + so we propagate in both directions when an attribute is missing. + (See https://github.com/huggingface/transformers/pull/41541) + """ + _ATTRS_TO_PROPAGATE = [ + "pad_token_id", + "bos_token_id", + "eos_token_id", + "tie_word_embeddings", + ] + for attr in _ATTRS_TO_PROPAGATE: + parent_has = hasattr(parent_config, attr) + text_has = hasattr(text_config, attr) + if parent_has and not text_has: + setattr(text_config, attr, getattr(parent_config, attr)) + elif text_has and not parent_has: + setattr(parent_config, attr, getattr(text_config, attr)) + return text_config + + +def get_hf_text_config(config: PretrainedConfig): + """Get the "sub" config relevant to llm for multi modal models. + No op for pure text models. + """ + if config.architectures is not None: + class_name = config.architectures[0] + if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"): + # We support non-hf version of llava models, so we do not want to + # read the wrong values from the unused default text_config. + # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as + # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`. + setattr(config, "dtype", torch.float16) + return config + + text_config = None + + # Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts. + # Convert to PretrainedConfig early so hasattr() checks and asserts work. + parent_dtype = getattr(config, "torch_dtype", None) + for _attr in ("text_config", "llm_config", "language_config", "thinker_config"): + _sub = getattr(config, _attr, None) + if isinstance(_sub, dict): + _converted = PretrainedConfig(**_sub) + if ( + getattr(_converted, "torch_dtype", None) is None + and parent_dtype is not None + ): + _converted.torch_dtype = parent_dtype + setattr(config, _attr, _converted) + + # Priority: thinker_config > llm_config > language_config > text_config + if hasattr(config, "thinker_config"): + # qwen2.5 omni + thinker_config = config.thinker_config + if hasattr(thinker_config, "text_config"): + setattr( + thinker_config.text_config, + "torch_dtype", + getattr(thinker_config, "torch_dtype", None), + ) + text_config = thinker_config.text_config + else: + text_config = thinker_config + elif hasattr(config, "llm_config"): + # PointsV1.5 Chat Model + assert hasattr(config.llm_config, "num_attention_heads") + text_config = config.llm_config + elif hasattr(config, "language_config"): + text_config = config.language_config + elif hasattr(config, "text_config"): + # The code operates under the assumption that text_config should have + # `num_attention_heads` (among others). Assert here to fail early + # if transformers config doesn't align with this assumption. + assert hasattr(config.text_config, "num_attention_heads") + text_config = config.text_config + + # Ensure rope_scaling dicts have "type" for remote-code compat (v5). + normalize_rope_scaling_compat(config) + + if text_config is not None: + return _patch_text_config(config, text_config) + return config + + +# --------------------------------------------------------------------------- +# Model-specific helpers +# --------------------------------------------------------------------------- + + +def _ensure_sub_configs(config: PretrainedConfig, *attr_names: str) -> None: + """Convert dict-valued sub-configs to proper AutoConfig objects in-place.""" + for attr in attr_names: + sub = getattr(config, attr, None) + if sub is not None and isinstance(sub, dict): + setattr(config, attr, AutoConfig.for_model(**sub)) + + +def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool: + # TODO: Remove this workaround once AutoConfig correctly identifies deepseek-ocr. + # Hugging Face's AutoConfig currently misidentifies it as deepseekvl2. + auto_map = getattr(config, "auto_map", None) or {} + return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM" + + +def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool: + auto_map = getattr(config, "auto_map", None) or {} + return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM" + + +def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None: + patched = False + for attr in ("text_config", "language_config"): + sub = getattr(config, attr, None) + if sub is None: + continue + if isinstance(sub, dict): + if sub.get("v_head_dim") == 0: + sub["v_head_dim"] = patch + patched = True + elif getattr(sub, "v_head_dim", None) == 0: + sub.v_head_dim = patch + patched = True + if patched: + logger.warning( + f"Overriding v_head_dim from 0 to {patch} to avoid potential issues." + ) + + +def _load_deepseek_v32_model( + model_path: str, + trust_remote_code: bool = False, + revision: Optional[str] = None, + **kwargs, +): + import tempfile + + local_path = download_from_hf(model_path) + config_file = os.path.join(local_path, "config.json") + if not os.path.exists(config_file): + raise RuntimeError(f"Can't find config file in {local_path}.") + + with open(config_file, "r") as f: + config_json = json.load(f) + + config_json["architectures"] = ["DeepseekV3ForCausalLM"] + config_json["model_type"] = "deepseek_v3" + + tmp_path = os.path.join(tempfile.gettempdir(), "_tmp_config_folder") + os.makedirs(tmp_path, exist_ok=True) + + unique_path = os.path.join(tmp_path, f"deepseek_v32_{os.getpid()}") + with open(unique_path, "w") as f: + json.dump(config_json, f) + + return AutoConfig.from_pretrained( + unique_path, trust_remote_code=trust_remote_code, revision=revision, **kwargs + ) + + +# --------------------------------------------------------------------------- +# Context length / generation config / sparse attention +# --------------------------------------------------------------------------- + +# Models don't use the same configuration key for determining the maximum +# context length. Store them here so we can sanely check them. +# NOTE: The ordering here is important. Some models have two of these and we +# have a preference for which value gets used. +CONTEXT_LENGTH_KEYS = [ + "max_sequence_length", + "seq_length", + "max_seq_len", + "model_max_length", + "max_position_embeddings", +] + + +def get_context_length(config): + """Get the context length of a model from a huggingface model configs.""" + text_config = config + rope_scaling = getattr(text_config, "rope_scaling", None) + if rope_scaling: + rope_scaling_factor = rope_scaling.get("factor", 1) + if "original_max_position_embeddings" in rope_scaling: + rope_scaling_factor = 1 + if rope_scaling.get("rope_type", None) == "llama3": + rope_scaling_factor = 1 + else: + rope_scaling_factor = 1 + + for key in CONTEXT_LENGTH_KEYS: + val = getattr(text_config, key, None) + if val is not None: + return int(rope_scaling_factor * val) + return 2048 + + +@lru_cache_frozenset(maxsize=32) +def get_generation_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, + **kwargs, +): + try: + return GenerationConfig.from_pretrained( + model, trust_remote_code=trust_remote_code, revision=revision, **kwargs + ) + except FileNotFoundError: + return None + except OSError as e: + logger.warning( + "Failed to load generation config for %s: %s. " + "Proceeding without generation config.", + model, + e, + ) + return None + + +# Qwen-1M related +def get_sparse_attention_config( + model: str, + sparse_attention_config_filename: str = "sparse_attention_config.json", +) -> Dict[str, Any]: + is_local = os.path.isdir(model) + if not is_local: + model = download_from_hf(model, allow_patterns=["*.json"]) + + config_file = os.path.join(model, sparse_attention_config_filename) + if not os.path.exists(config_file): + return {} + + with open(config_file) as f: + config = json.load(f) + return config + + +# --------------------------------------------------------------------------- +# Tokenizer / processor helpers +# --------------------------------------------------------------------------- + + +# Some models don't have an available processor, e.g.: InternVL +def get_tokenizer_from_processor(processor): + from transformers import PreTrainedTokenizerBase + + if isinstance(processor, PreTrainedTokenizerBase): + return processor + return processor.tokenizer + + +def attach_additional_stop_token_ids(tokenizer): + added = tokenizer.get_added_vocab() + if "<|eom_id|>" in added: + tokenizer.additional_stop_token_ids = {added["<|eom_id|>"]} + else: + tokenizer.additional_stop_token_ids = None diff --git a/python/sglang/srt/utils/hf_transformers/compat.py b/python/sglang/srt/utils/hf_transformers/compat.py new file mode 100644 index 000000000..570096e9d --- /dev/null +++ b/python/sglang/srt/utils/hf_transformers/compat.py @@ -0,0 +1,458 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Compatibility patches for transformers v5.x. + +This module applies monkey-patches to work around breaking changes in +transformers v5. Each patch is tagged with the upstream issue it works +around so it can be removed once the upstream fix lands. + +Import this module early (before any ``from_pretrained`` call) to activate +all patches. It is safe to import multiple times -- patches are idempotent. + +Patches fall into two categories: + +1. **Transformers bugs / regressions** -- issues in transformers itself. +2. **Remote-model-code compat** -- remote model code (trust_remote_code) + that hasn't been updated for v5 yet. These should be removed once + the model authors publish fixes. +""" + +import inspect + +from sglang.srt.utils import logger + +_applied = False + + +# --------------------------------------------------------------------------- +# Public API: apply_all() -- import-time patches (idempotent) +# --------------------------------------------------------------------------- + + +def apply_all(): + """Apply all transformers compatibility patches (idempotent). + + Call this once at import time. It is safe to call multiple times. + """ + global _applied + if _applied: + return + _applied = True + + # v5.4 patches + _patch_flash_attn_availability() + _patch_rope_parameters_validation() + _patch_removed_symbols() + _patch_image_processor_kwargs() + _patch_image_process_cuda_tensor() + _patch_nemotron_h_pattern() + + # v5 general patches + _ensure_clean_up_tokenization_compat() + _ensure_is_torch_fx_available_compat() + + logger.debug("transformers compatibility patches applied") + + +# --------------------------------------------------------------------------- +# Public API: on-demand helpers (called explicitly by other modules) +# --------------------------------------------------------------------------- + + +def normalize_rope_scaling_compat(config) -> None: + """Ensure rope_scaling dicts have ``"type"`` alongside ``"rope_type"``. + + Transformers v5 standardises rope_scaling to use ``"rope_type"`` and may + omit the legacy ``"type"`` key. Remote-code models (e.g. Kimi-VL) still + read ``rope_scaling["type"]``, causing a ``KeyError``. This helper adds + ``"type"`` from ``"rope_type"`` whenever it is missing, recursively across + the config and all its sub-configs. + """ + + def _patch(cfg): + rs = getattr(cfg, "rope_scaling", None) + if isinstance(rs, dict) and "rope_type" in rs and "type" not in rs: + rs["type"] = rs["rope_type"] + # Recurse into sub-configs + for attr in ( + "text_config", + "llm_config", + "language_config", + "vision_config", + "thinker_config", + ): + sub = getattr(cfg, attr, None) + if sub is not None: + _patch(sub) + + _patch(config) + + +def _ensure_gguf_version(): + """Workaround for transformers v5 bug where is_gguf_available() fails + when the gguf package lacks __version__ and metadata lookup also fails, + resulting in packaging.version.InvalidVersion: Invalid version: 'N/A'.""" + try: + import gguf + + if not hasattr(gguf, "__version__"): + import importlib.metadata + + try: + gguf.__version__ = importlib.metadata.version("gguf") + except importlib.metadata.PackageNotFoundError: + gguf.__version__ = "0.0.0" + except (ValueError, OSError, TypeError) as e: + logger.warning( + "Failed to determine gguf package version: %s. " + "Falling back to '0.0.0'.", + e, + ) + gguf.__version__ = "0.0.0" + except ImportError: + pass + + +# --------------------------------------------------------------------------- +# v5.4 patches (merged from transformers_v54_compat.py) +# --------------------------------------------------------------------------- + + +def _patch_rope_parameters_validation(): + """Fix rope_parameters validation for unregistered model types. + + For unregistered model types (e.g. ``deepseek_v32``), the generic + ``PretrainedConfig`` lacks a ``rope_parameters`` field so the conversion + that injects ``rope_theta`` from the top-level config is skipped. + Additionally, ``standardize_rope_params()`` accesses + ``self.max_position_embeddings`` during ``__post_init__`` before extra + kwargs are set as attributes, causing ``AttributeError``. + + Fix: (1) patch ``from_dict`` to inject ``rope_theta`` into + ``rope_scaling``, (2) guard ``standardize_rope_params`` against missing + ``max_position_embeddings``. + + TODO(upstream): remove once unregistered model types handle rope + standardization correctly in transformers. + """ + from transformers import PretrainedConfig + + original = PretrainedConfig.from_dict.__func__ + + @classmethod # type: ignore[misc] + def patched(cls, config_dict, **kwargs): + rope_scaling = config_dict.get("rope_scaling") + rope_theta = config_dict.get("rope_theta") + if ( + isinstance(rope_scaling, dict) + and rope_theta is not None + and "rope_theta" not in rope_scaling + ): + config_dict = config_dict.copy() + config_dict["rope_scaling"] = {**rope_scaling, "rope_theta": rope_theta} + return original(cls, config_dict, **kwargs) + + PretrainedConfig.from_dict = patched + + # standardize_rope_params accesses self.max_position_embeddings before + # __post_init__ sets extra kwargs — skip when the attribute is absent. + if hasattr(PretrainedConfig, "standardize_rope_params"): + _orig_standardize = PretrainedConfig.standardize_rope_params + + def _safe_standardize(self): + if not hasattr(self, "max_position_embeddings"): + return + return _orig_standardize(self) + + PretrainedConfig.standardize_rope_params = _safe_standardize + + +def _patch_flash_attn_availability(): + """Prevent flash-attn-4 from masquerading as flash-attn-2. + + flash-attn-4 registers a bare ``flash_attn`` namespace that makes + ``is_flash_attn_2_available()`` return True, but lacks the v2 API. + Remote model code (e.g. Kimi-VL) guarded by that check will crash. + + TODO(upstream): model authors should check for specific API symbols. + """ + try: + import flash_attn as _fa + + if not hasattr(_fa, "flash_attn_func"): + import transformers.utils as _u + import transformers.utils.import_utils as _ui + + _ui.is_flash_attn_2_available = lambda: False + _u.is_flash_attn_2_available = lambda: False + except ImportError: + pass + + +def _patch_removed_symbols(): + """Re-export symbols removed in transformers v5.4.0. + + Remote model code (e.g. DeepSeek-OCR) still imports these. + ``check_imports`` in ``dynamic_module_utils.py`` validates imports at + config-load time, so these must exist before any ``from_pretrained``. + + Removed symbols: + - ``LlamaFlashAttention2`` -- replaced by unified ``LlamaAttention`` + - ``is_flash_attn_greater_or_equal_2_10`` -- replaced by + ``is_flash_attn_greater_or_equal("2.10.0")`` + + TODO(upstream): DeepSeek-OCR / deepseek_vl_v2 remote code needs update. + """ + # LlamaFlashAttention2 + try: + from transformers.models.llama import modeling_llama + + if not hasattr(modeling_llama, "LlamaFlashAttention2"): + if hasattr(modeling_llama, "LlamaAttention"): + modeling_llama.LlamaFlashAttention2 = modeling_llama.LlamaAttention + except ImportError: + logger.warning( + "Could not import transformers.models.llama.modeling_llama; " + "LlamaFlashAttention2 compat patch not applied." + ) + + # is_flash_attn_greater_or_equal_2_10 + try: + import transformers.utils as _u + + if not hasattr(_u, "is_flash_attn_greater_or_equal_2_10"): + if hasattr(_u, "is_flash_attn_greater_or_equal"): + _u.is_flash_attn_greater_or_equal_2_10 = ( + lambda: _u.is_flash_attn_greater_or_equal("2.10.0") + ) + else: + _u.is_flash_attn_greater_or_equal_2_10 = lambda: False + except ImportError: + logger.warning( + "Could not import transformers.utils; " + "is_flash_attn_greater_or_equal_2_10 compat patch not applied." + ) + + +def _patch_image_processor_kwargs(): + """Allow remote image processors that lack ``**kwargs`` in preprocess(). + + Transformers v5.4 passes new kwargs (e.g. ``device``) through + ``BaseImageProcessor.__call__`` -> ``preprocess()``. Remote model code + (e.g. KimiVL) that defines ``preprocess()`` without ``**kwargs`` will + crash with ``TypeError``. + + Fix: wrap ``__call__`` to catch ``TypeError`` and retry with only the + kwargs that ``preprocess()`` actually accepts. + + TODO(upstream): KimiVL image_processing_kimi_vl.py needs ``**kwargs``. + """ + try: + from transformers.image_processing_utils import BaseImageProcessor + + original = BaseImageProcessor.__call__ + + def safe_call(self, images, *args, **kwargs): + try: + return original(self, images, *args, **kwargs) + except TypeError as e: + if "unexpected keyword argument" not in str(e): + raise + sig = inspect.signature(self.preprocess) + params = sig.parameters + if any( + p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values() + ): + raise + dropped = {k for k in kwargs if k not in params} + if dropped: + logger.warning( + "Image processor %s.preprocess() does not accept %s; " + "retrying without them. Update the model's image processor " + "to accept **kwargs.", + type(self).__name__, + dropped, + ) + valid = {k: v for k, v in kwargs.items() if k in params} + return original(self, images, *args, **valid) + + BaseImageProcessor.__call__ = safe_call + except ImportError: + logger.debug( + "_patch_image_processor_kwargs: BaseImageProcessor not importable, patch skipped" + ) + + +def _patch_image_process_cuda_tensor(): + """Fix ``process_image()`` crashing on CUDA tensors. + + Transformers v5.4's PIL image processing backend calls + ``image.numpy()`` on torch tensors, which fails for CUDA tensors. + Patch to call ``.cpu().numpy()`` instead. + + TODO(upstream): report to HF transformers. + """ + try: + import torch + import transformers.image_processing_backends as ipb + + for cls_name in ("PilBackend", "PilImageProcessingMixin"): + cls = getattr(ipb, cls_name, None) + if cls is None or not hasattr(cls, "process_image"): + continue + original = cls.process_image + + def patched_process_image( + self, image, *args, _orig=original, _Tensor=torch.Tensor, **kwargs + ): + if isinstance(image, _Tensor) and image.is_cuda: + image = image.cpu() + return _orig(self, image, *args, **kwargs) + + cls.process_image = patched_process_image + except ImportError: + logger.debug( + "_patch_image_process_cuda_tensor: required modules not importable, patch skipped" + ) + + +def _patch_nemotron_h_pattern(): + """Fix ``_pattern_to_list()`` crashing on ``-`` in hybrid_override_pattern. + + Nemotron-H models (e.g. NVIDIA-Nemotron-Nano-9B-v2) use patterns like + ``M-M-M-MM-M-*-...`` where ``-`` denotes an MLP layer. The upstream + ``_pattern_to_list`` tries to map every character and crashes with + ``KeyError: '-'``. We skip ``-`` (and any other unmapped chars) + since ``layers_block_type`` only tracks mamba/moe/attention layers. + SGLang reads MLP positions from ``hybrid_override_pattern`` directly. + + TODO(upstream): report to HF transformers. + """ + try: + from transformers.models.nemotron_h.configuration_nemotron_h import ( + NemotronHConfig, + ) + + @staticmethod + def _pattern_to_list(pattern: str) -> list: + pattern_mapping = { + "M": "mamba", + "E": "moe", + "*": "attention", + } + return [ + pattern_mapping[char] for char in pattern if char in pattern_mapping + ] + + NemotronHConfig._pattern_to_list = _pattern_to_list + except ImportError: + logger.debug( + "_patch_nemotron_h_pattern: NemotronHConfig not importable, patch skipped" + ) + + +# --------------------------------------------------------------------------- +# v5 general patches +# --------------------------------------------------------------------------- + + +def _ensure_clean_up_tokenization_compat() -> None: + """Re-add ``clean_up_tokenization`` removed in transformers v5. + + Remote-code tokenizers (e.g. InternLM2Tokenizer) call + ``self.clean_up_tokenization()`` which was a static method on + ``PreTrainedTokenizerBase`` in v4 but removed in v5. Patch it back + so existing HuggingFace Hub tokenizer code keeps working. + """ + from transformers import PreTrainedTokenizerBase + + if hasattr(PreTrainedTokenizerBase, "clean_up_tokenization"): + return + + @staticmethod + def clean_up_tokenization(out_string: str) -> str: + out_string = ( + out_string.replace(" .", ".") + .replace(" ?", "?") + .replace(" !", "!") + .replace(" ,", ",") + .replace(" ' ", "'") + .replace(" n't", "n't") + .replace(" 'm", "'m") + .replace(" 's", "'s") + .replace(" 've", "'ve") + .replace(" 're", "'re") + ) + return out_string + + PreTrainedTokenizerBase.clean_up_tokenization = clean_up_tokenization + + +def _ensure_is_torch_fx_available_compat() -> None: + """Re-add ``is_torch_fx_available`` removed in transformers v5. + + Remote-code models (e.g. MiniCPM-V) import ``is_torch_fx_available`` + from ``transformers.utils.import_utils``. The function was removed + in v5. Patch it back so existing HuggingFace Hub model code keeps + working. torch.fx is always available in PyTorch >= 2.0. + """ + import transformers.utils.import_utils as _import_utils + + if hasattr(_import_utils, "is_torch_fx_available"): + return + + _import_utils.is_torch_fx_available = lambda: True + + +# --------------------------------------------------------------------------- +# CI-only patches +# --------------------------------------------------------------------------- + +_is_base_mistral_patched = False + + +def patch_is_base_mistral_in_ci(): + """Patch transformers' _patch_mistral_regex to avoid HF API calls in CI. + + transformers defines is_base_mistral as a local function inside + _patch_mistral_regex, so it cannot be patched via module attribute. + Instead we replace the entire _patch_mistral_regex classmethod with a + version that simply returns the tokenizer unchanged. + + In CI this prevents exhausting the 3000 req/5min HF API rate limit. + + TODO(upstream): remove once transformers stops calling model_info() + inside _patch_mistral_regex (or removes the method entirely). + """ + global _is_base_mistral_patched + if _is_base_mistral_patched: + return + + from sglang.srt.environ import envs + + if not envs.SGLANG_IS_IN_CI.get(): + return + + from transformers import PreTrainedTokenizerFast + + if hasattr(PreTrainedTokenizerFast, "_patch_mistral_regex"): + + @classmethod + def _noop_patch_mistral_regex(cls, tokenizer, *args, **kwargs): + return tokenizer + + PreTrainedTokenizerFast._patch_mistral_regex = _noop_patch_mistral_regex + logger.info("CI: patched _patch_mistral_regex to skip HF API calls") + + _is_base_mistral_patched = True diff --git a/python/sglang/srt/utils/hf_transformers/config.py b/python/sglang/srt/utils/hf_transformers/config.py new file mode 100644 index 000000000..f5c800646 --- /dev/null +++ b/python/sglang/srt/utils/hf_transformers/config.py @@ -0,0 +1,212 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Config loading utilities.""" + +from pathlib import Path +from typing import Optional + +from transformers import PretrainedConfig +from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES + +from sglang.srt.connector import create_remote_connector +from sglang.srt.utils import is_remote_url, logger, lru_cache_frozenset +from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri + +from .common import ( + _CONFIG_REGISTRY, + AutoConfig, + DeepseekVLV2Config, + _is_deepseek_ocr2_model, + _is_deepseek_ocr_model, + _load_deepseek_v32_model, + _override_v_head_dim_if_zero, + check_gguf_file, + get_hf_text_config, +) +from .compat import _ensure_gguf_version +from .mistral_utils import is_mistral_model, load_mistral_config + + +def _set_architectures(config, arch_name): + config.update({"architectures": [arch_name]}) + + +def _apply_deepseek_ocr_overrides(config, model): + _override_v_head_dim_if_zero(config) + _set_architectures(config, "DeepseekOCRForCausalLM") + config._name_or_path = model + + +@lru_cache_frozenset(maxsize=32) +def get_config( + model: str, + trust_remote_code: bool, + revision: Optional[str] = None, + model_override_args: Optional[dict] = None, + **kwargs, +): + is_gguf = check_gguf_file(model) + if is_gguf: + _ensure_gguf_version() + kwargs["gguf_file"] = model + model = Path(model).parent + + if is_runai_obj_uri(model): + model = ObjectStorageModel.get_path(model) + + if is_remote_url(model): + client = create_remote_connector(model) + client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"]) + model = client.get_local_dir() + + if is_mistral_model(model): + config = load_mistral_config( + model, trust_remote_code=trust_remote_code, revision=revision + ) + else: + try: + config = AutoConfig.from_pretrained( + model, trust_remote_code=trust_remote_code, revision=revision, **kwargs + ) + except (ValueError, KeyError) as e: + if "deepseek_v32" in str(e): + config = _load_deepseek_v32_model( + model, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + elif isinstance(e, ValueError): + raise + else: + logger.warning( + "AutoConfig.from_pretrained raised KeyError for %s: %s. " + "Falling back to config registry lookup.", + model, + e, + ) + config_dict, _ = PretrainedConfig.get_config_dict( + model, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + model_type = config_dict.get("model_type") + if model_type in _CONFIG_REGISTRY: + config = _CONFIG_REGISTRY[model_type].from_dict(config_dict) + config._name_or_path = model + else: + raise + + if ( + config.architectures is not None + and config.architectures[0] == "Phi4MMForCausalLM" + ): + from transformers import SiglipVisionConfig + + config.vision_config = SiglipVisionConfig( + hidden_size=1152, + image_size=448, + intermediate_size=4304, + model_type="siglip_vision_model", + num_attention_heads=16, + num_hidden_layers=26, + patch_size=14, + ) + + if config.architectures in [ + ["LongcatCausalLM"], + ["LongcatFlashForCausalLM"], + ["LongcatFlashNgramForCausalLM"], + ]: + config.model_type = "longcat_flash" + + text_config = get_hf_text_config(config=config) + + if isinstance(model, str) and text_config is not None: + items = ( + text_config.items() + if hasattr(text_config, "items") + else vars(text_config).items() + ) + for key, val in items: + if not hasattr(config, key) and val is not None: + setattr(config, key, val) + + is_ocr = _is_deepseek_ocr_model(config) + is_ocr2 = _is_deepseek_ocr2_model(config) + + if is_ocr2: + _override_v_head_dim_if_zero(config) + config.model_type = "deepseek-ocr" + _set_architectures(config, "DeepseekOCRForCausalLM") + config = DeepseekVLV2Config.from_pretrained(model, revision=revision) + _apply_deepseek_ocr_overrides(config, model) + elif config.model_type in _CONFIG_REGISTRY: + model_type = config.model_type + if model_type == "deepseek_vl_v2" and is_ocr: + model_type = "deepseek-ocr" + config = _CONFIG_REGISTRY[model_type].from_pretrained(model, revision=revision) + + # Re-check after reloading config from registry + if _is_deepseek_ocr_model(config) or _is_deepseek_ocr2_model(config): + _apply_deepseek_ocr_overrides(config, model) + else: + config._name_or_path = model + + if isinstance(model, str) and config.model_type == "internvl_chat": + for key, val in config.llm_config.__dict__.items(): + if not hasattr(config, key): + setattr(config, key, val) + + if config.model_type == "multi_modality": + _set_architectures(config, "MultiModalityCausalLM") + + if config.model_type == "gemma4": + # Gemma4 configs use base attributes for SWA layers and `global_*` + # variants for full-attention layers. SGLang expects the opposite: + # base = full-attention, `swa_*` = sliding-window overrides. + text_config = config.text_config + global_head_dim = getattr(text_config, "global_head_dim", None) + global_kv_heads = getattr(text_config, "num_global_key_value_heads", None) + + swa_head_dim = text_config.head_dim + swa_kv_heads = text_config.num_key_value_heads + + text_config.swa_head_dim = swa_head_dim + text_config.swa_v_head_dim = swa_head_dim + text_config.swa_num_key_value_heads = swa_kv_heads + + if global_head_dim is not None: + text_config.head_dim = global_head_dim + if global_kv_heads is not None: + text_config.num_key_value_heads = global_kv_heads + + if not hasattr(text_config, "v_head_dim"): + text_config.v_head_dim = text_config.head_dim + if not hasattr(text_config, "swa_v_head_dim"): + text_config.swa_v_head_dim = text_config.swa_head_dim + + if config.model_type == "longcat_flash": + _set_architectures(config, "LongcatFlashForCausalLM") + + if model_override_args: + config.update(model_override_args) + + if is_gguf: + if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: + raise RuntimeError(f"Can't get gguf config for {config.model_type}.") + _set_architectures(config, MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]) + + return config diff --git a/python/sglang/srt/utils/mistral_utils.py b/python/sglang/srt/utils/hf_transformers/mistral_utils.py similarity index 65% rename from python/sglang/srt/utils/mistral_utils.py rename to python/sglang/srt/utils/hf_transformers/mistral_utils.py index dc9e08d94..cdaa72298 100644 --- a/python/sglang/srt/utils/mistral_utils.py +++ b/python/sglang/srt/utils/hf_transformers/mistral_utils.py @@ -1,13 +1,17 @@ # Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/mistral.py # SPDX-License-Identifier: Apache-2.0 import json +import tempfile +from functools import lru_cache from pathlib import Path -from typing import Any +from typing import Any, Optional -from transformers import PretrainedConfig, WhisperConfig +from transformers import AutoConfig, PretrainedConfig, WhisperConfig from sglang.srt.utils import logger +from .common import _ensure_sub_configs, download_from_hf + def adapt_config_dict( config_dict: dict[str, Any], model: str, **kwargs @@ -73,18 +77,6 @@ def adapt_config_dict( if bool(config_dict.get("yarn")): config_dict = _remap_mistral_yarn_args(config_dict) - if bool(config_dict.get("llama_4_scaling")): - llama_4_scaling_config_keys = ["original_max_position_embeddings", "beta"] - assert all( - [ - key in config_dict["llama_4_scaling"] - for key in llama_4_scaling_config_keys - ] - ), ( - "llama_4_scaling config should define the keys: " - f"{','.join(llama_4_scaling_config_keys)}" - ) - is_vision = bool( (config_dict.get("multimodal") or {}).get("vision_encoder_args") or config_dict.get("vision_encoder") @@ -267,14 +259,10 @@ class MistralConfigParser: ): file_path = Path(model) / file_name if not file_path.is_file(): - # TODO: Add logic to download from HF in case file is not locally found raise FileNotFoundError(f"File not found {model}, {file_name}") - if file_path is not None and file_path.is_file(): - with open(file_path) as file: - return json.load(file) - - return None + with open(file_path) as file: + return json.load(file) def _download_mistral_config_file(self, model, revision) -> dict: config_file_name = "params.json" @@ -294,8 +282,6 @@ class MistralConfigParser: revision: str | None = None, **kwargs, ) -> tuple[dict, PretrainedConfig]: - # This function loads a params.json config which - # should be used when loading models in mistral format config_dict = self._download_mistral_config_file(model, revision) if config_dict.get("max_position_embeddings") is None: logger.warning( @@ -321,3 +307,171 @@ class MistralConfigParser: config.sliding_window = next(filter(None, sliding_window), None) return config_dict, config + + +def is_mistral_model(name) -> bool: + """Return True if *name* refers to a Mistral model needing the custom parser.""" + lower = str(name).lower() + return ( + "mistral-large-3" in lower or "mistral-small-4" in lower or "leanstral" in lower + ) + + +@lru_cache(maxsize=2) +def load_mistral_config( + model_path: str, + trust_remote_code: bool = False, + revision: Optional[str] = None, +): + """Load and parse a Mistral model config via the custom params.json format. + + Returns a ``PretrainedConfig`` with dict sub-configs (text_config, + vision_config) converted to proper AutoConfig objects. + """ + local_path = download_from_hf(model_path) + parser = MistralConfigParser() + config_dict, _ = parser.parse(local_path) + + with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as f: + json.dump(config_dict, f) + f.flush() + loaded_config = AutoConfig.from_pretrained( + f.name, trust_remote_code=trust_remote_code, revision=revision + ) + _ensure_sub_configs(loaded_config, "text_config", "vision_config") + + return loaded_config + + +def wrap_as_pixtral(processor, config): + """Wrap a tokenizer as a PixtralProcessor for Mistral vision models.""" + from transformers.models.pixtral.image_processing_pixtral import ( + PixtralImageProcessor, + ) + from transformers.models.pixtral.processing_pixtral import ( + PixtralProcessor as HFPixtralProcessor, + ) + + vision_config = config.vision_config + patch_size = vision_config.patch_size + image_size = vision_config.image_size + spatial_merge_size = getattr(vision_config, "spatial_merge_size", 1) + + effective_patch = patch_size * spatial_merge_size + image_processor = PixtralImageProcessor( + do_resize=True, + size={"longest_edge": image_size}, + patch_size={"height": effective_patch, "width": effective_patch}, + ) + return HFPixtralProcessor( + image_processor=image_processor, + tokenizer=processor, + patch_size=patch_size, + spatial_merge_size=spatial_merge_size, + ) + + +# kwargs that MistralCommon tokenizers reject. +_MISTRAL_COMMON_REJECTED_KWARGS = frozenset( + { + "trust_remote_code", + "tokenizer_revision", + "use_fast", + "_from_auto", + "clean_up_tokenization_spaces", + } +) + +# Models whose tokenizer should be loaded from a different checkpoint. +_MISTRAL_TOKENIZER_REDIRECTS = { + # TODO(Xinyuan): Remove this once we have a proper tokenizer for Devstral + "mistralai/Devstral-Small-2505": "mistralai/Mistral-Small-3.1-24B-Instruct-2503", +} + + +def retry_without_mistral_common_kwargs(tokenizer_name, *args, **common_kwargs): + """Retry ``AutoTokenizer.from_pretrained`` without kwargs that MistralCommon rejects. + + Returns the loaded tokenizer, or *None* if the error is not a + MistralCommon kwargs rejection. + """ + from transformers import AutoTokenizer + + stripped = { + k: v + for k, v in common_kwargs.items() + if k not in _MISTRAL_COMMON_REJECTED_KWARGS + } + return AutoTokenizer.from_pretrained(tokenizer_name, *args, **stripped) + + +def patch_mistral_common_tokenizer(tokenizer): + """Patch MistralCommonTokenizer/Backend to be compatible with HF tokenizer API. + + MistralCommon tokenizers (used by Voxtral, Pixtral, etc.) reject several + standard kwargs and lack some attributes that sglang expects. We wrap the + offending methods once at load time so that the rest of the codebase does + not need any special-casing. + """ + cls_name = type(tokenizer).__name__ + if "MistralCommon" not in cls_name: + return tokenizer + if getattr(tokenizer, "_mistral_common_patched", False): + return tokenizer + tokenizer._mistral_common_patched = True + + if not hasattr(tokenizer, "get_added_vocab"): + tokenizer.get_added_vocab = lambda: {} + + # Set a chat_template containing "audio" so that sglang's content format + # detector returns "openai" (which preserves audio_url extraction). + if not hasattr(tokenizer, "chat_template") or tokenizer.chat_template is None: + tokenizer.chat_template = "" + + _orig_convert = tokenizer.convert_tokens_to_ids + + def _safe_convert(val): + try: + return _orig_convert(val) + except AssertionError: + logger.debug( + "convert_tokens_to_ids failed for %r, returning unk_token_id", val + ) + return getattr(tokenizer, "unk_token_id", None) + + tokenizer.convert_tokens_to_ids = _safe_convert + + def _drop_kwargs(fn, keys): + def wrapper(*args, **kwargs): + for k in keys: + kwargs.pop(k, None) + return fn(*args, **kwargs) + + return wrapper + + tokenizer.decode = _drop_kwargs(tokenizer.decode, ["spaces_between_special_tokens"]) + tokenizer.batch_decode = _drop_kwargs( + tokenizer.batch_decode, ["spaces_between_special_tokens"] + ) + + tokenizer._orig_apply_chat_template = tokenizer.apply_chat_template + + def _safe_apply_chat_template(messages, **kwargs): + kwargs.pop("add_generation_prompt", None) + cleaned = [] + for msg in messages: + if isinstance(msg, dict): + content = msg.get("content", "") + if isinstance(content, list): + text_parts = [ + p.get("text", "") + for p in content + if isinstance(p, dict) and p.get("type") == "text" + ] + msg = {**msg, "content": " ".join(text_parts) if text_parts else ""} + cleaned.append(msg) + else: + cleaned.append(msg) + return tokenizer._orig_apply_chat_template(cleaned, **kwargs) + + tokenizer.apply_chat_template = _safe_apply_chat_template diff --git a/python/sglang/srt/utils/hf_transformers/processor.py b/python/sglang/srt/utils/hf_transformers/processor.py new file mode 100644 index 000000000..31d5905a8 --- /dev/null +++ b/python/sglang/srt/utils/hf_transformers/processor.py @@ -0,0 +1,288 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Processor loading utilities.""" + +import json +from pathlib import Path +from typing import Optional + +from transformers import ( + AutoProcessor, + AutoTokenizer, + PreTrainedTokenizerBase, +) + +from sglang.srt.multimodal.customized_mm_processor_utils import _CUSTOMIZED_MM_PROCESSOR +from sglang.srt.utils import logger + +from .common import ( + AutoConfig, + _is_deepseek_ocr2_model, + _is_deepseek_ocr_model, + _override_v_head_dim_if_zero, + _resolve_local_or_cached_file, + attach_additional_stop_token_ids, + download_from_hf, + get_tokenizer_from_processor, +) +from .mistral_utils import ( + is_mistral_model, + load_mistral_config, + patch_mistral_common_tokenizer, + wrap_as_pixtral, +) +from .tokenizer import ( + _TOKENIZERS_BACKEND, + _fix_added_tokens_encoding, + _fix_special_tokens_pattern, +) + + +def _build_processor_manually( + model_path, config, trust_remote_code, revision, **kwargs +): + """Build processor when AutoProcessor fails to resolve feature_extractor_type. + + In transformers v5, AutoProcessor.from_pretrained calls + AutoFeatureExtractor.from_pretrained which fails if + preprocessor_config.json lacks 'feature_extractor_type'. This resolves + the processor class via dynamic module resolution and constructs it with + individually-loaded components. + """ + import transformers + from transformers import AutoImageProcessor, AutoTokenizer + from transformers.dynamic_module_utils import get_class_from_dynamic_module + + # Resolve processor class from auto_map -- check both the model config + # and the preprocessor_config.json (some models like MiniCPM-o only + # declare AutoProcessor in the latter). + auto_map = getattr(config, "auto_map", None) or {} + proc_ref = auto_map.get("AutoProcessor") + if not proc_ref: + try: + pp_file = _resolve_local_or_cached_file( + model_path, "preprocessor_config.json", revision + ) + with open(pp_file) as f: + pp_auto_map = json.load(f).get("auto_map", {}) + proc_ref = pp_auto_map.get("AutoProcessor") + except (OSError, json.JSONDecodeError, ValueError) as e: + logger.warning( + "_build_processor_manually: could not read preprocessor_config.json " + "for %s: %s", + model_path, + e, + ) + if not proc_ref: + raise ValueError(f"Cannot determine processor class for {model_path}") + + proc_cls = get_class_from_dynamic_module( + proc_ref, model_path, code_revision=revision + ) + + # Load sub-components individually (these succeed) + tokenizer = AutoTokenizer.from_pretrained( + model_path, trust_remote_code=trust_remote_code, revision=revision + ) + init_kwargs = {"tokenizer": tokenizer} + + if "image_processor" in getattr(proc_cls, "attributes", []): + try: + init_kwargs["image_processor"] = AutoImageProcessor.from_pretrained( + model_path, trust_remote_code=trust_remote_code, revision=revision + ) + except (ImportError, OSError, ValueError) as e: + raise RuntimeError( + f"Failed to load image_processor for {model_path}: {e}. " + f"This model requires an image processor for multimodal features. " + f"Check that the model files are complete and accessible." + ) from e + + # Instantiate feature extractor from its declared class + fe_class_name = getattr(proc_cls, "feature_extractor_class", None) + if fe_class_name: + fe_class = getattr(transformers, fe_class_name, None) + if fe_class is not None: + try: + init_kwargs["feature_extractor"] = fe_class() + except TypeError as e: + logger.warning( + "Cannot instantiate feature extractor %s with no arguments " + "for %s: %s", + fe_class_name, + model_path, + e, + ) + else: + logger.warning( + "Feature extractor class %s not found in transformers for %s", + fe_class_name, + model_path, + ) + + return proc_cls(**init_kwargs) + + +def get_processor( + tokenizer_name: str, + *args, + tokenizer_mode: str = "auto", + trust_remote_code: bool = False, + tokenizer_revision: Optional[str] = None, + use_fast: Optional[bool] = True, + **kwargs, +): + revision = kwargs.pop("revision", tokenizer_revision) + if is_mistral_model(tokenizer_name): + config = load_mistral_config( + tokenizer_name, + trust_remote_code=trust_remote_code, + revision=revision, + ) + else: + config = AutoConfig.from_pretrained( + tokenizer_name, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + is_ocr2 = _is_deepseek_ocr2_model(config) + if _is_deepseek_ocr_model(config) or is_ocr2: + config.model_type = "deepseek-ocr" + config.update({"architectures": ["DeepseekOCRForCausalLM"]}) + if is_ocr2: + _override_v_head_dim_if_zero(config) + + if config.model_type in {"qwen2_vl", "sarashina2_vision"}: + if "size" not in kwargs: + kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520} + + if config.model_type not in {"llava", "clip"}: + kwargs["use_fast"] = use_fast + try: + if "InternVL3_5" in tokenizer_name: + processor = AutoTokenizer.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + else: + if config.model_type in _CUSTOMIZED_MM_PROCESSOR: + processor = _CUSTOMIZED_MM_PROCESSOR[config.model_type].from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + else: + processor = AutoProcessor.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + + except ValueError as e: + error_message = str(e) + if "does not have a slow version" in error_message: + logger.info( + "Processor %s does not have a slow version. Automatically use fast version", + tokenizer_name, + ) + kwargs["use_fast"] = True + processor = AutoProcessor.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + elif "Unrecognized feature extractor" in error_message: + logger.info( + "AutoProcessor failed on feature extractor for %s, " + "constructing processor manually", + tokenizer_name, + ) + processor = _build_processor_manually( + tokenizer_name, + config, + trust_remote_code, + revision, + **kwargs, + ) + elif ( + "are not supported by" in error_message and "MistralCommon" in error_message + ): + logger.info( + "AutoProcessor for %s rejected standard kwargs, " + "retrying without trust_remote_code/use_fast", + tokenizer_name, + ) + kwargs.pop("use_fast", None) + kwargs.pop("_from_auto", None) + processor = AutoProcessor.from_pretrained( + tokenizer_name, + *args, + revision=revision, + **kwargs, + ) + else: + raise + if ( + isinstance(processor, PreTrainedTokenizerBase) + and getattr(config, "model_type", None) == "pixtral" + ): + processor = wrap_as_pixtral(processor, config) + + tokenizer = get_tokenizer_from_processor(processor) + + # AutoProcessor may internally create a TokenizersBackend tokenizer + # (same issue as get_tokenizer). Replace it with a properly loaded one. + if type(tokenizer).__name__ == _TOKENIZERS_BACKEND: + from .tokenizer import get_tokenizer + + logger.warning( + "Processor tokenizer for %s is TokenizersBackend, " + "reloading via get_tokenizer", + tokenizer_name, + ) + tokenizer = get_tokenizer( + tokenizer_name, + tokenizer_mode=tokenizer_mode, + trust_remote_code=trust_remote_code, + tokenizer_revision=revision, + ) + if isinstance(processor, PreTrainedTokenizerBase): + processor = tokenizer + else: + processor.tokenizer = tokenizer + + if tokenizer.chat_template is None: + local_path = download_from_hf( + tokenizer_name, allow_patterns=["*.json", "*.jinja", "*.model"] + ) + jinja_path = Path(local_path) / "chat_template.jinja" + if jinja_path.is_file(): + tokenizer.chat_template = jinja_path.read_text() + logger.info("Loaded chat_template from %s", jinja_path) + + patch_mistral_common_tokenizer(tokenizer) + _fix_special_tokens_pattern(tokenizer) + _fix_added_tokens_encoding(tokenizer) + attach_additional_stop_token_ids(tokenizer) + return processor diff --git a/python/sglang/srt/utils/hf_transformers/tokenizer.py b/python/sglang/srt/utils/hf_transformers/tokenizer.py new file mode 100644 index 000000000..8c82145e6 --- /dev/null +++ b/python/sglang/srt/utils/hf_transformers/tokenizer.py @@ -0,0 +1,551 @@ +# Copyright 2023-2024 SGLang Team +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tokenizer loading utilities.""" + +import json +import logging +import warnings +from pathlib import Path +from typing import Optional, Union + +from transformers import ( + AutoTokenizer, + PreTrainedTokenizer, + PreTrainedTokenizerFast, +) + +from sglang.srt.connector import create_remote_connector +from sglang.srt.utils import is_remote_url, logger +from sglang.srt.utils.patch_tokenizer import patch_tokenizer +from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri + +from .common import ( + _resolve_local_or_cached_file, + attach_additional_stop_token_ids, + check_gguf_file, +) +from .compat import _ensure_gguf_version, patch_is_base_mistral_in_ci +from .mistral_utils import ( + _MISTRAL_TOKENIZER_REDIRECTS, + patch_mistral_common_tokenizer, + retry_without_mistral_common_kwargs, +) + +# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file. +_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer" + +# Class name used by transformers v5 when no tokenizer mapping exists for a model_type. +_TOKENIZERS_BACKEND = "TokenizersBackend" + + +def _load_tokenizer_by_declared_class(tokenizer_name, *args, **kwargs): + """Load tokenizer by the class declared in tokenizer_config.json. + + AutoTokenizer resolves to TokenizersBackend when the model's config + model_type has no tokenizer class mapping (e.g. deepseek_vl_v2), even + though tokenizer_config.json declares a standard class like + LlamaTokenizerFast. Returns None if it cannot improve on AutoTokenizer. + """ + import transformers + + try: + revision = kwargs.get("revision") or kwargs.get("tokenizer_revision") + config_file = _resolve_local_or_cached_file( + tokenizer_name, "tokenizer_config.json", revision + ) + with open(config_file) as f: + tok_config = json.load(f) + tok_class_name = tok_config.get("tokenizer_class") + except FileNotFoundError: + return None + except (OSError, json.JSONDecodeError) as e: + logger.debug( + "Failed to read tokenizer_config.json for %s: %s", tokenizer_name, e + ) + return None + + if not tok_class_name: + return None + + # Skip base classes that don't implement required methods (e.g. get_vocab) + if tok_class_name in ("PreTrainedTokenizer", "PreTrainedTokenizerBase"): + return None + + tok_cls = getattr(transformers, tok_class_name, None) + if tok_cls is None and kwargs.get("trust_remote_code"): + # Class not in transformers — try loading via auto_map. + try: + auto_map = tok_config.get("auto_map", {}) + auto_tok_ref = auto_map.get("AutoTokenizer") + if isinstance(auto_tok_ref, (list, tuple)): + auto_tok_ref = auto_tok_ref[0] + if auto_tok_ref: + from transformers.dynamic_module_utils import ( + get_class_from_dynamic_module, + ) + + tok_cls = get_class_from_dynamic_module( + auto_tok_ref, + tokenizer_name, + code_revision=revision, + ) + except (OSError, ImportError, ValueError, RuntimeError) as e: + logger.debug("Dynamic module lookup for %s failed: %s", tok_class_name, e) + if tok_cls is None: + return None + + logger.info( + "Loading tokenizer for %s directly as %s (bypassing AutoTokenizer)", + tokenizer_name, + tok_class_name, + ) + try: + return tok_cls.from_pretrained(tokenizer_name, *args, **kwargs) + except (OSError, ValueError, TypeError, ImportError) as e: + logger.warning( + "Direct load as %s failed for %s: %s. " + "Falling back to AutoTokenizer result.", + tok_class_name, + tokenizer_name, + e, + ) + return None + + +# Filter warnings like: https://github.com/sgl-project/sglang/issues/8082 +class TokenizerWarningsFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + return "Calling super().encode with" not in record.getMessage() + + +# --------------------------------------------------------------------------- +# Helpers for get_tokenizer +# --------------------------------------------------------------------------- + + +def _resolve_tokenizer_name(tokenizer_name, kwargs): + """Resolve special name formats (GGUF, remote URLs, etc.) to a local path. + + May mutate *kwargs* (e.g. to add ``gguf_file``). + """ + tokenizer_name = _MISTRAL_TOKENIZER_REDIRECTS.get(tokenizer_name, tokenizer_name) + + if check_gguf_file(tokenizer_name): + _ensure_gguf_version() + kwargs["gguf_file"] = tokenizer_name + tokenizer_name = Path(tokenizer_name).parent + + if is_runai_obj_uri(tokenizer_name): + tokenizer_name = ObjectStorageModel.get_path(tokenizer_name) + + if is_remote_url(tokenizer_name): + # BaseConnector implements __del__() to clean up the local dir. + # Since config files need to exist all the time, so we DO NOT use + # with statement to avoid closing the client. + client = create_remote_connector(tokenizer_name) + client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"]) + tokenizer_name = client.get_local_dir() + + return tokenizer_name + + +def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs): + """Call ``AutoTokenizer.from_pretrained`` with error handling.""" + try: + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, *args, **common_kwargs + ) + logging.getLogger(tokenizer.__class__.__module__).addFilter( + TokenizerWarningsFilter() + ) + return tokenizer + except TypeError as e: + err_msg = ( + "Failed to load the tokenizer. If you are using a LLaMA V1 model " + f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the " + "original tokenizer." + ) + raise RuntimeError(err_msg) from e + except ValueError as e: + # MistralCommon tokenizers reject standard HF kwargs like + # trust_remote_code, use_fast etc. Retry without them. + if "are not supported by" in str(e) and "MistralCommon" in str(e): + return retry_without_mistral_common_kwargs( + tokenizer_name, *args, **common_kwargs + ) + # If the error pertains to the tokenizer class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + if not common_kwargs.get("trust_remote_code") and ( + "does not exist or is not currently imported." in str(e) + or "requires you to execute the tokenizer file" in str(e) + ): + err_msg = ( + "Failed to load the tokenizer. If the tokenizer is a custom " + "tokenizer not yet available in the HuggingFace transformers " + "library, consider setting `trust_remote_code=True` in LLM " + "or using the `--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + raise + + +def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs): + """Resolve generic ``TokenizersBackend`` to a proper tokenizer class. + + In transformers v5, ``AutoTokenizer`` falls back to ``TokenizersBackend`` + when the model_type has no tokenizer mapping. This retries with + ``use_fast=False``, then attempts loading by the class declared in + ``tokenizer_config.json``. May still return a ``TokenizersBackend`` + if all retries fail (with a warning). + """ + logger.warning( + "Tokenizer loaded as generic TokenizersBackend for %s, " + "retrying with use_fast=False", + tokenizer_name, + ) + common_kwargs = {**common_kwargs, "use_fast": False} + try: + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_name, *args, **common_kwargs + ) + except (ValueError, TypeError, OSError, ImportError, RuntimeError) as e: + raise RuntimeError( + f"Retry with use_fast=False for {tokenizer_name} also failed " + f"(initial load returned TokenizersBackend): {e}" + ) from e + + if type(tokenizer).__name__ == _TOKENIZERS_BACKEND: + tokenizer = ( + _load_tokenizer_by_declared_class(tokenizer_name, *args, **common_kwargs) + or tokenizer + ) + + if type(tokenizer).__name__ == _TOKENIZERS_BACKEND: + if common_kwargs.get("trust_remote_code"): + logger.warning( + "Tokenizer for %s is still TokenizersBackend after retries " + "with --trust-remote-code. Model-specific tokenizer attributes " + "may be missing.", + tokenizer_name, + ) + else: + logger.warning( + "Tokenizer for %s loaded as generic TokenizersBackend. " + "Set --trust-remote-code to load the model-specific tokenizer.", + tokenizer_name, + ) + + return tokenizer + + +# --------------------------------------------------------------------------- +# Post-load fixups +# --------------------------------------------------------------------------- + + +def _fix_v5_tokenizer_components(tokenizer, model_name_or_path, revision=None): + """Fix pre_tokenizer/decoder when a v5 tokenizer class overwrites them. + + In transformers v5, some tokenizer classes (e.g. LlamaTokenizer) have a + custom __init__ that rebuilds the pre_tokenizer and decoder from scratch + with class-specific components, discarding the originals from tokenizer.json. + This breaks models that specify LlamaTokenizerFast but actually use a + different tokenizer architecture (e.g. DeepSeek-V3.2 uses ByteLevel). + + Detects the mismatch by comparing against the raw tokenizer.json and + restores the original components when they differ. + """ + backend = getattr(tokenizer, "_tokenizer", None) + if backend is None: + return + + try: + from tokenizers import Tokenizer as RawTokenizer + + tok_file = _resolve_local_or_cached_file( + model_name_or_path, "tokenizer.json", revision + ) + raw = RawTokenizer.from_file(tok_file) + except FileNotFoundError: + return + except (OSError, ValueError, RuntimeError) as e: + logger.warning( + "_fix_v5_tokenizer_components: unexpected error loading tokenizer.json " + "for %s, v5 component fix will not be applied: %s", + model_name_or_path, + e, + ) + return + + raw_pre = type(raw.pre_tokenizer).__name__ if raw.pre_tokenizer else None + loaded_pre = type(backend.pre_tokenizer).__name__ if backend.pre_tokenizer else None + + if raw_pre and loaded_pre and raw_pre != loaded_pre: + logger.info( + "Fixing v5 tokenizer component mismatch for %s: " + "pre_tokenizer %s -> %s, decoder %s -> %s", + model_name_or_path, + loaded_pre, + raw_pre, + type(backend.decoder).__name__ if backend.decoder else None, + type(raw.decoder).__name__ if raw.decoder else None, + ) + backend.pre_tokenizer = raw.pre_tokenizer + backend.decoder = raw.decoder + + +def _fix_v5_add_bos_eos_token(tokenizer, model_name_or_path, revision=None): + """Restore add_bos_token/add_eos_token stripped by transformers v5. + + In transformers v5, _from_pretrained() strips add_bos_token and + add_eos_token from init kwargs when a tokenizer.json file is present, + assuming the tokenizer.json post-processor handles BOS/EOS addition. + However, many models (e.g. DeepSeek-V3) have a tokenizer.json whose + post-processor does NOT add BOS/EOS, and rely on the add_bos_token flag + from tokenizer_config.json instead. This causes silent accuracy regressions. + + This function reads the tokenizer_config.json and restores the values, + but only for tokenizer classes that actually supported these flags in v4. + Classes like Qwen2Tokenizer did not support add_bos_token/add_eos_token + in v4, so restoring them would change behavior. + """ + # In transformers v4, only certain tokenizer classes supported + # add_bos_token / add_eos_token as init parameters. Restoring these + # flags for classes that never supported them (e.g. Qwen2Tokenizer) + # would incorrectly change tokenization behavior. + _V4_CLASSES_WITH_BOS_EOS_FLAGS = frozenset( + { + "LlamaTokenizer", + "LlamaTokenizerFast", + "CodeLlamaTokenizer", + "CodeLlamaTokenizerFast", + "GemmaTokenizer", + "GemmaTokenizerFast", + "CohereTokenizerFast", + } + ) + + try: + config_file = _resolve_local_or_cached_file( + model_name_or_path, "tokenizer_config.json", revision + ) + with open(config_file) as f: + config = json.load(f) + except FileNotFoundError: + return + except (OSError, json.JSONDecodeError, ValueError) as e: + logger.warning( + "_fix_v5_add_bos_eos_token: failed to read tokenizer_config.json " + "for %s, BOS/EOS token restoration will not be applied: %s", + model_name_or_path, + e, + ) + return + + tokenizer_class = config.get("tokenizer_class", "") + if tokenizer_class not in _V4_CLASSES_WITH_BOS_EOS_FLAGS: + logger.debug( + "_fix_v5_add_bos_eos_token: skipping %s (tokenizer_class=%s " + "did not support add_bos/eos_token in v4)", + model_name_or_path, + tokenizer_class, + ) + return + + # In v4, Llama/Gemma tokenizers defaulted add_bos_token=True. + # When the config omits the key or has null, use the v4 default so that + # update_post_processor() doesn't drop BOS/EOS that was there before. + _V4_DEFAULTS = {"add_bos_token": True, "add_eos_token": False} + + changed = False + for attr in ("add_bos_token", "add_eos_token"): + config_val = config.get(attr) + if config_val is None: + # Key missing or null -> use v4 default for this tokenizer class + config_val = _V4_DEFAULTS.get(attr, False) + # Fast tokenizers in v4 used tokenizer.json post-processor for EOS — + # the add_eos_token Python attribute was set but the post-processor + # came from tokenizer.json, not from the attribute. In v5, the flag is + # stripped and both sglang and HF reference end up with add_eos_token=False. + # Restoring add_eos_token for fast tokenizers makes sglang diverge from + # the HF reference, breaking embedding models like e5-mistral-7b-instruct. + if attr == "add_eos_token" and isinstance(tokenizer, PreTrainedTokenizerFast): + config_val = _V4_DEFAULTS["add_eos_token"] # False + current_val = getattr(tokenizer, attr, None) + if current_val != config_val: + logger.info( + "Restoring %s=%s for %s (was %s after v5 loading)", + attr, + config_val, + model_name_or_path, + current_val, + ) + # Set the private backing attribute (not the property) because + # transformers tokenizers expose add_bos/eos_token as properties + # that read from the underscore-prefixed attribute. + setattr(tokenizer, f"_{attr}", config_val) + changed = True + + # Rebuild the post-processor so it respects the restored flags + if changed and hasattr(tokenizer, "update_post_processor"): + tokenizer.update_post_processor() + + +def _fix_special_tokens_pattern(tokenizer): + """Fix https://github.com/huggingface/transformers/pull/42563 which defaults + special_tokens_pattern to "cls_sep", inserting None into token IDs when + cls_token/sep_token are undefined (e.g. Kimi-VL's TikTokenTokenizer). + """ + pattern = getattr(tokenizer, "special_tokens_pattern", None) + if pattern == "cls_sep" and ( + tokenizer.cls_token_id is None or tokenizer.sep_token_id is None + ): + tokenizer.special_tokens_pattern = "none" + + +def _apply_post_load_fixes(tokenizer, tokenizer_name, revision): + """Apply all post-load patches and return the final tokenizer.""" + _fix_v5_tokenizer_components(tokenizer, tokenizer_name, revision) + _fix_v5_add_bos_eos_token(tokenizer, tokenizer_name, revision) + + if not isinstance(tokenizer, PreTrainedTokenizerFast): + warnings.warn( + "Using a slow tokenizer. This might cause a significant " + "slowdown. Consider using a fast tokenizer instead." + ) + + patch_mistral_common_tokenizer(tokenizer) + _fix_special_tokens_pattern(tokenizer) + attach_additional_stop_token_ids(tokenizer) + return patch_tokenizer(tokenizer) + + +# --------------------------------------------------------------------------- +# Public entry point +# --------------------------------------------------------------------------- + + +def get_tokenizer( + tokenizer_name: str, + *args, + tokenizer_mode: str = "auto", + trust_remote_code: bool = False, + tokenizer_revision: Optional[str] = None, + **kwargs, +) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: + """Gets a tokenizer for the given model name via Huggingface.""" + if tokenizer_name.endswith(".json"): + from sglang.srt.tokenizer.tiktoken_tokenizer import TiktokenTokenizer + + return TiktokenTokenizer(tokenizer_name) + + if tokenizer_mode == "slow": + if kwargs.get("use_fast", False): + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") + kwargs["use_fast"] = False + elif tokenizer_mode == "auto": + # Transformers v5 AutoTokenizer ignores use_fast (always fast), but + # some code paths pass kwargs to non-AutoTokenizer loaders where + # use_fast still matters. Set explicitly for those fallback paths. + if "use_fast" not in kwargs: + kwargs["use_fast"] = True + + tokenizer_name = _resolve_tokenizer_name(tokenizer_name, kwargs) + patch_is_base_mistral_in_ci() + + common_kwargs = dict( + trust_remote_code=trust_remote_code, + tokenizer_revision=tokenizer_revision, + clean_up_tokenization_spaces=False, + **kwargs, + ) + + tokenizer = _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs) + + if type(tokenizer).__name__ == _TOKENIZERS_BACKEND: + tokenizer = _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs) + + return _apply_post_load_fixes(tokenizer, tokenizer_name, tokenizer_revision) + + +# --------------------------------------------------------------------------- +# Exported helpers (used by processor.py, etc.) +# --------------------------------------------------------------------------- + + +def _fix_added_tokens_encoding(tokenizer): + """Ensure special tokens encode as single tokens in transformers v5. + + Some model tokenizers (e.g. MiniCPM-V-4) define special tokens like , + as attributes on the tokenizer class with corresponding IDs in the + vocabulary (via tokenizer.json's added_tokens). In transformers v5, these + tokens may not appear in get_added_vocab() and encode() splits them into + subwords, breaking multimodal pipelines that rely on finding them in input_ids. + + This function discovers such tokens by scanning tokenizer attributes, checks + if they encode correctly, and re-registers any that don't. + """ + + # Discover special token strings from tokenizer attributes. + # Model tokenizers (e.g. MiniCPMVTokenizerFast) store them as attributes + # like im_start="", slice_start="", etc. + def _is_special_token_attr(val): + return ( + isinstance(val, str) + and val.startswith("<") + and val.endswith(">") + and len(val) <= 20 + ) + + candidates = {} + for attr in dir(tokenizer): + if attr.startswith("_"): + continue + try: + val = getattr(tokenizer, attr) + except (AttributeError, TypeError, ValueError): + continue + if not _is_special_token_attr(val): + continue + token_id = tokenizer.convert_tokens_to_ids(val) + if token_id is not None and token_id != tokenizer.unk_token_id: + candidates[val] = token_id + + if not candidates: + return + + def _encodes_correctly(token_str, expected_id): + try: + ids = tokenizer.encode(token_str, add_special_tokens=False) + return len(ids) == 1 and ids[0] == expected_id + except (ValueError, OverflowError, RuntimeError) as e: + logger.debug("Token %s encode check failed: %s", token_str, e) + return False + + broken = [ + tok for tok, eid in candidates.items() if not _encodes_correctly(tok, eid) + ] + + if not broken: + return + + from transformers import AddedToken + + tokens_to_add = [AddedToken(tok, special=True, normalized=False) for tok in broken] + tokenizer.add_tokens(tokens_to_add, special_tokens=True) + logger.info( + "Re-registered %d special tokens for correct v5 encoding: %s", + len(broken), + broken[:10], + ) diff --git a/python/sglang/srt/utils/hf_transformers_utils.py b/python/sglang/srt/utils/hf_transformers_utils.py index b928b08d4..582c397bc 100644 --- a/python/sglang/srt/utils/hf_transformers_utils.py +++ b/python/sglang/srt/utils/hf_transformers_utils.py @@ -11,1484 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -"""Utilities for Huggingface Transformers.""" +"""Backward-compatible shim — all code has moved to sglang.srt.utils.hf_transformers.""" -import contextlib -import json -import logging -import os -import tempfile -import warnings -from functools import lru_cache -from pathlib import Path -from typing import Any, Dict, List, Optional, Type, Union - -import torch -from huggingface_hub import snapshot_download - -from sglang.srt.utils import get_bool_env_var -from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri - -# Compatibility shim: flash-attn-4 registers a bare ``flash_attn`` namespace -# that makes ``is_flash_attn_2_available()`` return True, but lacks the v2 API -# (``flash_attn_func``, etc.). HuggingFace remote model code (e.g. Kimi-VL) -# guarded by that check will crash with ImportError at module load time. -# Force it to False when the real v2 API is absent. -try: - import flash_attn as _flash_attn_mod - - if not hasattr(_flash_attn_mod, "flash_attn_func"): - import transformers.utils as _hf_utils - import transformers.utils.import_utils as _hf_import_utils - - _hf_import_utils.is_flash_attn_2_available = lambda: False - _hf_utils.is_flash_attn_2_available = lambda: False - del _flash_attn_mod -except ImportError: - pass - -# Conditional import based on SGLANG_USE_MODELSCOPE environment variable -if get_bool_env_var("SGLANG_USE_MODELSCOPE"): - from modelscope import AutoConfig, GenerationConfig -else: - from transformers import AutoConfig, GenerationConfig - -from transformers import ( - AutoProcessor, - AutoTokenizer, - PretrainedConfig, - PreTrainedTokenizer, - PreTrainedTokenizerBase, - PreTrainedTokenizerFast, -) -from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES - -from sglang.srt.configs import ( - AfmoeConfig, - BailingHybridConfig, - ChatGLMConfig, - DbrxConfig, - DeepseekVL2Config, - DotsOCRConfig, - DotsVLMConfig, - ExaoneConfig, - FalconH1Config, - GraniteMoeHybridConfig, - JetNemotronConfig, - JetVLMConfig, - KimiK25Config, - KimiLinearConfig, - KimiVLConfig, - LongcatFlashConfig, - MultiModalityConfig, - NemotronH_Nano_VL_V2_Config, - NemotronHConfig, - Olmo3Config, - Qwen3_5Config, - Qwen3_5MoeConfig, - Qwen3NextConfig, - Step3p5Config, - Step3VLConfig, -) -from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config -from sglang.srt.configs.internvl import InternVLChatConfig -from sglang.srt.connector import create_remote_connector -from sglang.srt.multimodal.customized_mm_processor_utils import _CUSTOMIZED_MM_PROCESSOR -from sglang.srt.utils import is_remote_url, logger, lru_cache_frozenset, mistral_utils -from sglang.srt.utils.patch_tokenizer import patch_tokenizer - -_CONFIG_REGISTRY: List[Type[PretrainedConfig]] = [ - AfmoeConfig, - BailingHybridConfig, - ChatGLMConfig, - DbrxConfig, - ExaoneConfig, - DeepseekVL2Config, - MultiModalityConfig, - KimiVLConfig, - InternVLChatConfig, - Step3VLConfig, - LongcatFlashConfig, - Olmo3Config, - KimiLinearConfig, - Qwen3NextConfig, - FalconH1Config, - GraniteMoeHybridConfig, - DotsVLMConfig, - DotsOCRConfig, - NemotronH_Nano_VL_V2_Config, - NemotronHConfig, - DeepseekVLV2Config, - Qwen3_5Config, - Qwen3_5MoeConfig, - JetNemotronConfig, - JetVLMConfig, - KimiK25Config, - Step3p5Config, -] - -_CONFIG_REGISTRY = { - config_cls.model_type: config_cls for config_cls in _CONFIG_REGISTRY -} - -for name, cls in _CONFIG_REGISTRY.items(): - with contextlib.suppress(ValueError): - AutoConfig.register(name, cls) - - -def download_from_hf( - model_path: str, - allow_patterns: Optional[Union[str, list]] = None, -): - if os.path.exists(model_path): - return model_path - - if not allow_patterns: - allow_patterns = ["*.json", "*.bin", "*.model"] - - return snapshot_download(model_path, allow_patterns=allow_patterns) - - -def get_rope_config(config): - """Get (rope_theta, rope_scaling) from config, supporting both v4 and v5. - - In transformers v5, rope_theta/rope_scaling are accessed via the computed - property config.rope_parameters. Trust-remote-code configs or parent configs - passed to sub-models may not have this property or may return None. - Falls back to the v4-style config.rope_theta / config.rope_scaling attributes. - """ - rope_params = getattr(config, "rope_parameters", None) - if rope_params is not None: - return rope_params["rope_theta"], rope_params - return config.rope_theta, getattr(config, "rope_scaling", None) - - -def _patch_text_config(parent_config: PretrainedConfig, text_config): - """Synchronize standard attributes between parent config and text sub-config. - - In transformers v5, the "untangle config" refactor removed automatic - inheritance of top-level PretrainedConfig attributes (pad_token_id, - tie_word_embeddings, etc.) from sub-configs. Downstream code expects - these attributes to be present on both configs (some models pass the - parent directly to the language model, others pass the text sub-config), - so we propagate in both directions when an attribute is missing. - (See https://github.com/huggingface/transformers/pull/41541) - """ - _ATTRS_TO_PROPAGATE = [ - "pad_token_id", - "bos_token_id", - "eos_token_id", - "tie_word_embeddings", - ] - for attr in _ATTRS_TO_PROPAGATE: - parent_has = hasattr(parent_config, attr) - text_has = hasattr(text_config, attr) - if parent_has and not text_has: - setattr(text_config, attr, getattr(parent_config, attr)) - elif text_has and not parent_has: - setattr(parent_config, attr, getattr(text_config, attr)) - return text_config - - -def get_hf_text_config(config: PretrainedConfig): - """Get the "sub" config relevant to llm for multi modal models. - No op for pure text models. - """ - if config.architectures is not None: - class_name = config.architectures[0] - if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"): - # We support non-hf version of llava models, so we do not want to - # read the wrong values from the unused default text_config. - # NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as - # `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`. - setattr(config, "dtype", torch.float16) - return config - - text_config = None - - # Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts. - # Convert to PretrainedConfig early so hasattr() checks and asserts work. - for _attr in ("text_config", "llm_config", "language_config", "thinker_config"): - _sub = getattr(config, _attr, None) - if isinstance(_sub, dict): - _converted = PretrainedConfig(**_sub) - # Propagate torch_dtype from parent so weight loading uses correct precision. - if ( - getattr(_converted, "torch_dtype", None) is None - and getattr(config, "torch_dtype", None) is not None - ): - _converted.torch_dtype = config.torch_dtype - setattr(config, _attr, _converted) - - # Priority: thinker_config > llm_config > language_config > text_config - if hasattr(config, "thinker_config"): - # qwen2.5 omni - thinker_config = config.thinker_config - if hasattr(thinker_config, "text_config"): - setattr( - thinker_config.text_config, - "torch_dtype", - getattr(thinker_config, "torch_dtype", None), - ) - text_config = thinker_config.text_config - else: - text_config = thinker_config - elif hasattr(config, "llm_config"): - # PointsV1.5 Chat Model - assert hasattr(config.llm_config, "num_attention_heads") - text_config = config.llm_config - elif hasattr(config, "language_config"): - text_config = config.language_config - elif hasattr(config, "text_config"): - # The code operates under the assumption that text_config should have - # `num_attention_heads` (among others). Assert here to fail early - # if transformers config doesn't align with this assumption. - assert hasattr(config.text_config, "num_attention_heads") - text_config = config.text_config - - # Ensure rope_scaling dicts have "type" for remote-code compat (v5). - normalize_rope_scaling_compat(config) - - if text_config is not None: - return _patch_text_config(config, text_config) - return config - - -# Temporary hack for DeepSeek-V3.2 model -def _load_deepseek_v32_model( - model_path: str, - trust_remote_code: bool = False, - revision: Optional[str] = None, - **kwargs, -): - # first get the local path - local_path = download_from_hf(model_path) - # then load the config file in json - config_file = os.path.join(local_path, "config.json") - if not os.path.exists(config_file): - raise RuntimeError(f"Can't find config file in {local_path}.") - - with open(config_file, "r") as f: - config_json = json.load(f) - - config_json["architectures"] = ["DeepseekV3ForCausalLM"] - config_json["model_type"] = "deepseek_v3" - - tmp_path = os.path.join(tempfile.gettempdir(), "_tmp_config_folder") - os.makedirs(tmp_path, exist_ok=True) - - unique_path = os.path.join(tmp_path, f"deepseek_v32_{os.getpid()}") - with open(unique_path, "w") as f: - json.dump(config_json, f) - - return AutoConfig.from_pretrained( - unique_path, trust_remote_code=trust_remote_code, revision=revision, **kwargs - ) - - -# Temporary hack for Mistral Large -@lru_cache(maxsize=2) -def _load_mistral_large_3_for_causal_LM( - model_path: str, - trust_remote_code: bool = False, - revision: Optional[str] = None, -): - # first get the local path - local_path = download_from_hf(model_path) - # then load the config file in json - parser = mistral_utils.MistralConfigParser() - config_dict, _ = parser.parse(local_path) - - with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as f: - json.dump(config_dict, f) - f.flush() - loaded_config = AutoConfig.from_pretrained( - f.name, trust_remote_code=trust_remote_code, revision=revision - ) - text_config = getattr(loaded_config, "text_config", None) - if text_config is not None and isinstance(text_config, dict): - text_config = AutoConfig.for_model(**text_config) - setattr(loaded_config, "text_config", text_config) - vision_config = getattr(loaded_config, "vision_config", None) - if vision_config is not None and isinstance(vision_config, dict): - vision_config = AutoConfig.for_model(**vision_config) - setattr(loaded_config, "vision_config", vision_config) - - return loaded_config - - -def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool: - # TODO: Remove this workaround related when AutoConfig correctly identifies deepseek-ocr. - # Hugging Face's AutoConfig currently misidentifies it as deepseekvl2. - auto_map = getattr(config, "auto_map", None) or {} - return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM" - - -def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool: - auto_map = getattr(config, "auto_map", None) or {} - return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM" - - -def _override_deepseek_ocr_v_head_dim(config: DeepseekVLV2Config) -> None: - # FIXME: deepseek-ocr's v_head_dim is set to 0 in its config file. - # https://huggingface.co/deepseek-ai/DeepSeek-OCR/blob/main/config.json#L116 - if config.text_config.v_head_dim == 0: - V_HEAD_DIM_PATCH = 128 - config.text_config.v_head_dim = V_HEAD_DIM_PATCH - # Also fix language_config so get_hf_text_config (which may prefer it - # over text_config) stays consistent. - lc = getattr(config, "language_config", None) - if isinstance(lc, dict): - lc["v_head_dim"] = V_HEAD_DIM_PATCH - elif hasattr(lc, "v_head_dim"): - lc.v_head_dim = V_HEAD_DIM_PATCH - logger.warning( - f"Overriding deepseek-ocr's v_head_dim from 0 to {V_HEAD_DIM_PATCH} to avoid potential issues." - ) - - -def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None: - text_config = getattr(config, "text_config", None) - language_config = getattr(config, "language_config", None) - target = text_config or language_config - if target is None: - return - if getattr(target, "v_head_dim", None) == 0: - setattr(target, "v_head_dim", patch) - logger.warning( - f"Overriding v_head_dim from 0 to {patch} to avoid potential issues." - ) - - -def _ensure_clean_up_tokenization_compat() -> None: - """Re-add ``clean_up_tokenization`` removed in transformers v5. - - Remote-code tokenizers (e.g. InternLM2Tokenizer) call - ``self.clean_up_tokenization()`` which was a static method on - ``PreTrainedTokenizerBase`` in v4 but removed in v5. Patch it back - so existing HuggingFace Hub tokenizer code keeps working. - """ - if hasattr(PreTrainedTokenizerBase, "clean_up_tokenization"): - return - - @staticmethod - def clean_up_tokenization(out_string: str) -> str: - out_string = ( - out_string.replace(" .", ".") - .replace(" ?", "?") - .replace(" !", "!") - .replace(" ,", ",") - .replace(" ' ", "'") - .replace(" n't", "n't") - .replace(" 'm", "'m") - .replace(" 's", "'s") - .replace(" 've", "'ve") - .replace(" 're", "'re") - ) - return out_string - - PreTrainedTokenizerBase.clean_up_tokenization = clean_up_tokenization - - -# Apply immediately so all code paths (get_tokenizer, get_processor, -# and any external callers) benefit without needing an explicit call. -_ensure_clean_up_tokenization_compat() - - -def _ensure_is_torch_fx_available_compat() -> None: - """Re-add ``is_torch_fx_available`` removed in transformers v5. - - Remote-code models (e.g. MiniCPM-V) import ``is_torch_fx_available`` - from ``transformers.utils.import_utils``. The function was removed - in v5. Patch it back so existing HuggingFace Hub model code keeps - working. torch.fx is always available in PyTorch >= 2.0. - """ - import transformers.utils.import_utils as _import_utils - - if hasattr(_import_utils, "is_torch_fx_available"): - return - - _import_utils.is_torch_fx_available = lambda: True - - -_ensure_is_torch_fx_available_compat() - - -def normalize_rope_scaling_compat(config: "PretrainedConfig") -> None: - """Ensure rope_scaling dicts have ``"type"`` alongside ``"rope_type"``. - - Transformers v5 standardises rope_scaling to use ``"rope_type"`` and may - omit the legacy ``"type"`` key. Remote-code models (e.g. Kimi-VL) still - read ``rope_scaling["type"]``, causing a ``KeyError``. This helper adds - ``"type"`` from ``"rope_type"`` whenever it is missing, recursively across - the config and all its sub-configs. - """ - - def _patch(cfg): - try: - rs = getattr(cfg, "rope_scaling", None) - except AttributeError: - rs = None - if isinstance(rs, dict) and "rope_type" in rs and "type" not in rs: - rs["type"] = rs["rope_type"] - # Recurse into sub-configs - for attr in ( - "text_config", - "llm_config", - "language_config", - "vision_config", - "thinker_config", - ): - sub = getattr(cfg, attr, None) - if sub is not None: - _patch(sub) - - _patch(config) - - -def _ensure_llama_flash_attention2_compat() -> None: - """Ensure LlamaFlashAttention2 symbol exists for remote code compatibility.""" - try: - from transformers.models.llama import modeling_llama - except (ImportError, ModuleNotFoundError): - return - if not hasattr(modeling_llama, "LlamaFlashAttention2"): - if hasattr(modeling_llama, "LlamaAttention"): - modeling_llama.LlamaFlashAttention2 = modeling_llama.LlamaAttention - - -def _ensure_gguf_version(): - """Workaround for transformers v5 bug where is_gguf_available() fails - when the gguf package lacks __version__ and metadata lookup also fails, - resulting in packaging.version.InvalidVersion: Invalid version: 'N/A'.""" - try: - import gguf - - if not hasattr(gguf, "__version__"): - import importlib.metadata - - try: - gguf.__version__ = importlib.metadata.version("gguf") - except Exception: - gguf.__version__ = "0.0.0" - except ImportError: - pass - - -@lru_cache_frozenset(maxsize=32) -def get_config( - model: str, - trust_remote_code: bool, - revision: Optional[str] = None, - model_override_args: Optional[dict] = None, - **kwargs, -): - is_gguf = check_gguf_file(model) - if is_gguf: - _ensure_gguf_version() - kwargs["gguf_file"] = model - model = Path(model).parent - - if is_runai_obj_uri(model): - model = ObjectStorageModel.get_path(model) - - if is_remote_url(model): - # BaseConnector implements __del__() to clean up the local dir. - # Since config files need to exist all the time, so we DO NOT use - # with statement to avoid closing the client. - client = create_remote_connector(model) - client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"]) - model = client.get_local_dir() - - if ( - "mistral-large-3" in str(model).lower() - or "mistral-small-4" in str(model).lower() - or "leanstral" in str(model).lower() - ): - config = _load_mistral_large_3_for_causal_LM( - model, trust_remote_code=trust_remote_code, revision=revision - ) - else: - _ensure_llama_flash_attention2_compat() - try: - config = AutoConfig.from_pretrained( - model, trust_remote_code=trust_remote_code, revision=revision, **kwargs - ) - except ValueError as e: - if not "deepseek_v32" in str(e): - raise e - config = _load_deepseek_v32_model( - model, trust_remote_code=trust_remote_code, revision=revision, **kwargs - ) - except KeyError as e: - # Transformers v5 may register a built-in config class that - # conflicts with sglang's custom one (e.g. NemotronHConfig - # doesn't handle '-' in hybrid_override_pattern). Fall back - # to loading the raw config dict and using sglang's class. - # Also handle deepseek_v32 which v5 doesn't recognize. - if "deepseek_v32" in str(e): - config = _load_deepseek_v32_model( - model, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - else: - config_dict, _ = PretrainedConfig.get_config_dict( - model, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - model_type = config_dict.get("model_type") - if model_type in _CONFIG_REGISTRY: - config = _CONFIG_REGISTRY[model_type].from_dict(config_dict) - config._name_or_path = model - else: - raise - - if ( - config.architectures is not None - and config.architectures[0] == "Phi4MMForCausalLM" - ): - # Phi4MMForCausalLM uses a hard-coded vision_config. See: - # https://github.com/vllm-project/vllm/blob/6071e989df1531b59ef35568f83f7351afb0b51e/vllm/model_executor/models/phi4mm.py#L71 - # We set it here to support cases where num_attention_heads is not divisible by the TP size. - from transformers import SiglipVisionConfig - - vision_config = { - "hidden_size": 1152, - "image_size": 448, - "intermediate_size": 4304, - "model_type": "siglip_vision_model", - "num_attention_heads": 16, - "num_hidden_layers": 26, - # Model is originally 27-layer, we only need the first 26 layers for feature extraction. - "patch_size": 14, - } - config.vision_config = SiglipVisionConfig(**vision_config) - - if config.architectures in [ - ["LongcatCausalLM"], - ["LongcatFlashForCausalLM"], - ["LongcatFlashNgramForCausalLM"], - ]: - config.model_type = "longcat_flash" - - text_config = get_hf_text_config(config=config) - - if isinstance(model, str) and text_config is not None: - items = ( - text_config.items() - if hasattr(text_config, "items") - else vars(text_config).items() - ) - for key, val in items: - if not hasattr(config, key) and val is not None: - setattr(config, key, val) - - if _is_deepseek_ocr2_model(config): - _override_v_head_dim_if_zero(config) - # Temporary hack for load deepseek-ocr2 - config.model_type = "deepseek-ocr" - config.update({"architectures": ["DeepseekOCRForCausalLM"]}) - config = DeepseekVLV2Config.from_pretrained(model, revision=revision) - _override_v_head_dim_if_zero(config) - config.update({"architectures": ["DeepseekOCRForCausalLM"]}) - setattr(config, "_name_or_path", model) - elif config.model_type in _CONFIG_REGISTRY: - model_type = config.model_type - if model_type == "deepseek_vl_v2": - if _is_deepseek_ocr_model(config) or _is_deepseek_ocr2_model(config): - model_type = "deepseek-ocr" - config_class = _CONFIG_REGISTRY[model_type] - config = config_class.from_pretrained(model, revision=revision) - - if _is_deepseek_ocr_model(config): - _override_deepseek_ocr_v_head_dim(config) - config.update({"architectures": ["DeepseekOCRForCausalLM"]}) - elif _is_deepseek_ocr2_model(config): - _override_v_head_dim_if_zero(config) - config.update({"architectures": ["DeepseekOCRForCausalLM"]}) - - # NOTE(HandH1998): Qwen2VL requires `_name_or_path` attribute in `config`. - setattr(config, "_name_or_path", model) - - if isinstance(model, str) and config.model_type == "internvl_chat": - for key, val in config.llm_config.__dict__.items(): - if not hasattr(config, key): - setattr(config, key, val) - - if config.model_type == "multi_modality": - config.update({"architectures": ["MultiModalityCausalLM"]}) - - if config.model_type == "gemma4": - # Gemma4 configs use base attributes for SWA layers and `global_*` - # variants for full-attention layers. SGLang expects the opposite: - # base = full-attention, `swa_*` = sliding-window overrides. - # Remap here so the rest of the stack sees a uniform convention. - text_config = config.text_config - global_head_dim = getattr(text_config, "global_head_dim", None) - global_kv_heads = getattr(text_config, "num_global_key_value_heads", None) - - swa_head_dim = text_config.head_dim - swa_kv_heads = text_config.num_key_value_heads - - text_config.swa_head_dim = swa_head_dim - text_config.swa_v_head_dim = swa_head_dim - text_config.swa_num_key_value_heads = swa_kv_heads - - if global_head_dim is not None: - text_config.head_dim = global_head_dim - if global_kv_heads is not None: - text_config.num_key_value_heads = global_kv_heads - - if not hasattr(text_config, "v_head_dim"): - text_config.v_head_dim = text_config.head_dim - if not hasattr(text_config, "swa_v_head_dim"): - text_config.swa_v_head_dim = text_config.swa_head_dim - - if config.model_type == "longcat_flash": - config.update({"architectures": ["LongcatFlashForCausalLM"]}) - - if model_override_args: - config.update(model_override_args) - - # Special architecture mapping check for GGUF models - if is_gguf: - if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES: - raise RuntimeError(f"Can't get gguf config for {config.model_type}.") - model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type] - config.update({"architectures": [model_type]}) - - return config - - -@lru_cache_frozenset(maxsize=32) -def get_generation_config( - model: str, - trust_remote_code: bool, - revision: Optional[str] = None, - **kwargs, -): - try: - return GenerationConfig.from_pretrained( - model, trust_remote_code=trust_remote_code, revision=revision, **kwargs - ) - except OSError as e: - return None - - -# Qwen-1M related -def get_sparse_attention_config( - model: str, - sparse_attention_config_filename: str = "sparse_attention_config.json", -) -> Dict[str, Any]: - is_local = os.path.isdir(model) - if not is_local: - # Download the config files. - model = download_from_hf(model, allow_patterns=["*.json"]) - - config_file = os.path.join(model, sparse_attention_config_filename) - if not os.path.exists(config_file): - return {} - - # Load the sparse attention config. - with open(config_file) as f: - config = json.load(f) - return config - - -# Models don't use the same configuration key for determining the maximum -# context length. Store them here so we can sanely check them. -# NOTE: The ordering here is important. Some models have two of these and we -# have a preference for which value gets used. -CONTEXT_LENGTH_KEYS = [ - "max_sequence_length", - "seq_length", - "max_seq_len", - "model_max_length", - "max_position_embeddings", -] - - -def get_context_length(config): - """Get the context length of a model from a huggingface model configs.""" - text_config = config - rope_scaling = getattr(text_config, "rope_scaling", None) - if rope_scaling: - rope_scaling_factor = rope_scaling.get("factor", 1) - if "original_max_position_embeddings" in rope_scaling: - rope_scaling_factor = 1 - if rope_scaling.get("rope_type", None) == "llama3": - rope_scaling_factor = 1 - else: - rope_scaling_factor = 1 - - for key in CONTEXT_LENGTH_KEYS: - val = getattr(text_config, key, None) - if val is not None: - return int(rope_scaling_factor * val) - return 2048 - - -# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file. -_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer" - - -# Filter warnings like: https://github.com/sgl-project/sglang/issues/8082 -class TokenizerWarningsFilter(logging.Filter): - def filter(self, record: logging.LogRecord) -> bool: - return "Calling super().encode with" not in record.getMessage() - - -_is_base_mistral_patched = False - -# transformers version where _patch_mistral_regex calls model_info() on every tokenizer load -_TRANSFORMERS_PATCHED_VERSION = "5.3.0" - - -def _patch_is_base_mistral_in_ci(): - """Patch transformers' _patch_mistral_regex to avoid HF API calls in CI. - - transformers defines is_base_mistral as a local function inside - _patch_mistral_regex, so it cannot be patched via module attribute. - Instead we replace the entire _patch_mistral_regex classmethod with a - version that simply returns the tokenizer unchanged. - - In CI this prevents exhausting the 3000 req/5min HF API rate limit. - """ - global _is_base_mistral_patched - if _is_base_mistral_patched: - return - - from sglang.srt.environ import envs - - if not envs.SGLANG_IS_IN_CI.get(): - return - - import transformers - - if transformers.__version__ != _TRANSFORMERS_PATCHED_VERSION: - logger.warning( - "transformers version changed to %s (expected %s), " - "_patch_mistral_regex patch skipped — may need update if 429 errors recur", - transformers.__version__, - _TRANSFORMERS_PATCHED_VERSION, - ) - _is_base_mistral_patched = True # don't warn repeatedly - return - - from transformers import PreTrainedTokenizerFast - - if hasattr(PreTrainedTokenizerFast, "_patch_mistral_regex"): - - @classmethod - def _noop_patch_mistral_regex(cls, tokenizer, *args, **kwargs): - return tokenizer - - PreTrainedTokenizerFast._patch_mistral_regex = _noop_patch_mistral_regex - logger.info("CI: patched _patch_mistral_regex to skip HF API calls") - - _is_base_mistral_patched = True - - -def get_tokenizer( - tokenizer_name: str, - *args, - tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - tokenizer_revision: Optional[str] = None, - **kwargs, -) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]: - """Gets a tokenizer for the given model name via Huggingface.""" - if tokenizer_name.endswith(".json"): - from sglang.srt.tokenizer.tiktoken_tokenizer import TiktokenTokenizer - - return TiktokenTokenizer(tokenizer_name) - - if tokenizer_mode == "slow": - if kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - kwargs["use_fast"] = False - elif tokenizer_mode == "auto": - # In Transformers v5, the default for use_fast changed from True to False. - # Explicitly set use_fast=True for "auto" mode to maintain previous behavior - # and avoid issues with models that have incorrect tokenizer_class values. - if "use_fast" not in kwargs: - kwargs["use_fast"] = True - - # TODO(Xinyuan): Remove this once we have a proper tokenizer for Devstral - if tokenizer_name == "mistralai/Devstral-Small-2505": - tokenizer_name = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" - - is_gguf = check_gguf_file(tokenizer_name) - if is_gguf: - _ensure_gguf_version() - kwargs["gguf_file"] = tokenizer_name - tokenizer_name = Path(tokenizer_name).parent - - if is_runai_obj_uri(tokenizer_name): - tokenizer_name = ObjectStorageModel.get_path(tokenizer_name) - - if is_remote_url(tokenizer_name): - # BaseConnector implements __del__() to clean up the local dir. - # Since config files need to exist all the time, so we DO NOT use - # with statement to avoid closing the client. - client = create_remote_connector(tokenizer_name) - client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"]) - tokenizer_name = client.get_local_dir() - - _patch_is_base_mistral_in_ci() - - try: - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - tokenizer_revision=tokenizer_revision, - clean_up_tokenization_spaces=False, - **kwargs, - ) - # Filter tokenizer warnings - logging.getLogger(tokenizer.__class__.__module__).addFilter( - TokenizerWarningsFilter() - ) - except TypeError as e: - # The LLaMA tokenizer causes a protobuf error in some environments. - err_msg = ( - "Failed to load the tokenizer. If you are using a LLaMA V1 model " - f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the " - "original tokenizer." - ) - raise RuntimeError(err_msg) from e - except ValueError as e: - # MistralCommon tokenizers reject standard HF kwargs like - # trust_remote_code, use_fast etc. Retry without them. - if "are not supported by" in str(e) and "MistralCommon" in str(e): - for k in ( - "trust_remote_code", - "tokenizer_revision", - "use_fast", - "_from_auto", - "clean_up_tokenization_spaces", - ): - kwargs.pop(k, None) - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_name, - *args, - **kwargs, - ) - # If the error pertains to the tokenizer class not existing or not - # currently being imported, suggest using the --trust-remote-code flag. - elif not trust_remote_code and ( - "does not exist or is not currently imported." in str(e) - or "requires you to execute the tokenizer file" in str(e) - ): - err_msg = ( - "Failed to load the tokenizer. If the tokenizer is a custom " - "tokenizer not yet available in the HuggingFace transformers " - "library, consider setting `trust_remote_code=True` in LLM " - "or using the `--trust-remote-code` flag in the CLI." - ) - raise RuntimeError(err_msg) from e - else: - raise e - - # Transformers v5 may silently fall back to a generic TokenizersBackend - # when trust_remote_code=False and the model requires a custom tokenizer. - # Detect this and auto-retry with trust_remote_code=True. - if not trust_remote_code and type(tokenizer).__name__ == "TokenizersBackend": - tokenizer = AutoTokenizer.from_pretrained( - tokenizer_name, - *args, - trust_remote_code=True, - tokenizer_revision=tokenizer_revision, - clean_up_tokenization_spaces=False, - **kwargs, - ) - - _fix_v5_tokenizer_components(tokenizer, tokenizer_name, tokenizer_revision) - _fix_v5_add_bos_eos_token(tokenizer, tokenizer_name, tokenizer_revision) - - if not isinstance(tokenizer, PreTrainedTokenizerFast): - warnings.warn( - "Using a slow tokenizer. This might cause a significant " - "slowdown. Consider using a fast tokenizer instead." - ) - - _patch_mistral_common_tokenizer(tokenizer) - _fix_special_tokens_pattern(tokenizer) - attach_additional_stop_token_ids(tokenizer) - tokenizer = patch_tokenizer(tokenizer) - return tokenizer - - -def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None): - """Resolve a file from a local directory or HF hub cache (no network).""" - local_path = Path(model_name_or_path) / filename - if local_path.is_file(): - return str(local_path) - from huggingface_hub import hf_hub_download - - return hf_hub_download( - model_name_or_path, filename, revision=revision, local_files_only=True - ) - - -def _fix_v5_tokenizer_components(tokenizer, model_name_or_path, revision=None): - """Fix pre_tokenizer/decoder when a v5 tokenizer class overwrites them. - - In transformers v5, some tokenizer classes (e.g. LlamaTokenizer) have a - custom __init__ that rebuilds the pre_tokenizer and decoder from scratch - with class-specific components, discarding the originals from tokenizer.json. - This breaks models that specify LlamaTokenizerFast but actually use a - different tokenizer architecture (e.g. DeepSeek-V3.2 uses ByteLevel). - - Detects the mismatch by comparing against the raw tokenizer.json and - restores the original components when they differ. - """ - backend = getattr(tokenizer, "_tokenizer", None) - if backend is None: - return - - try: - from tokenizers import Tokenizer as RawTokenizer - - tok_file = _resolve_local_or_cached_file( - model_name_or_path, "tokenizer.json", revision - ) - raw = RawTokenizer.from_file(tok_file) - except Exception as e: - logger.debug( - "_fix_v5_tokenizer_components: could not load tokenizer.json for %s: %s", - model_name_or_path, - e, - ) - return - - raw_pre = type(raw.pre_tokenizer).__name__ if raw.pre_tokenizer else None - loaded_pre = type(backend.pre_tokenizer).__name__ if backend.pre_tokenizer else None - - if raw_pre and loaded_pre and raw_pre != loaded_pre: - logger.info( - "Fixing v5 tokenizer component mismatch for %s: " - "pre_tokenizer %s -> %s, decoder %s -> %s", - model_name_or_path, - loaded_pre, - raw_pre, - type(backend.decoder).__name__ if backend.decoder else None, - type(raw.decoder).__name__ if raw.decoder else None, - ) - backend.pre_tokenizer = raw.pre_tokenizer - backend.decoder = raw.decoder - - -def _fix_v5_add_bos_eos_token(tokenizer, model_name_or_path, revision=None): - """Restore add_bos_token/add_eos_token stripped by transformers v5. - - In transformers v5, _from_pretrained() strips add_bos_token and - add_eos_token from init kwargs when a tokenizer.json file is present, - assuming the tokenizer.json post-processor handles BOS/EOS addition. - However, many models (e.g. DeepSeek-V3) have a tokenizer.json whose - post-processor does NOT add BOS/EOS, and rely on the add_bos_token flag - from tokenizer_config.json instead. This causes silent accuracy regressions. - - This function reads the tokenizer_config.json and restores the values, - but only for tokenizer classes that actually supported these flags in v4. - Classes like Qwen2Tokenizer did not support add_bos_token/add_eos_token - in v4, so restoring them would change behavior. - """ - # In transformers v4, only certain tokenizer classes supported - # add_bos_token / add_eos_token as init parameters. Restoring these - # flags for classes that never supported them (e.g. Qwen2Tokenizer) - # would incorrectly change tokenization behavior. - _V4_CLASSES_WITH_BOS_EOS_FLAGS = frozenset( - { - "LlamaTokenizer", - "LlamaTokenizerFast", - "CodeLlamaTokenizer", - "CodeLlamaTokenizerFast", - "GemmaTokenizer", - "GemmaTokenizerFast", - "CohereTokenizerFast", - } - ) - - try: - config_file = _resolve_local_or_cached_file( - model_name_or_path, "tokenizer_config.json", revision - ) - with open(config_file) as f: - config = json.load(f) - except Exception as e: - logger.debug( - "_fix_v5_add_bos_eos_token: could not read tokenizer_config.json " - "for %s: %s", - model_name_or_path, - e, - ) - return - - tokenizer_class = config.get("tokenizer_class", "") - if tokenizer_class not in _V4_CLASSES_WITH_BOS_EOS_FLAGS: - logger.debug( - "_fix_v5_add_bos_eos_token: skipping %s (tokenizer_class=%s " - "did not support add_bos/eos_token in v4)", - model_name_or_path, - tokenizer_class, - ) - return - - # In v4, Llama/Gemma tokenizers defaulted add_bos_token=True. - # When the config omits the key or has null, use the v4 default so that - # update_post_processor() doesn't drop BOS/EOS that was there before. - _V4_DEFAULTS = {"add_bos_token": True, "add_eos_token": False} - - changed = False - for attr in ("add_bos_token", "add_eos_token"): - config_val = config.get(attr) - if config_val is None: - # Key missing or null → use v4 default for this tokenizer class - config_val = _V4_DEFAULTS.get(attr, False) - # Fast tokenizers in v4 used tokenizer.json post-processor for EOS — - # the add_eos_token Python attribute was set but the post-processor - # came from tokenizer.json, not from the attribute. In v5, the flag is - # stripped and both sglang and HF reference end up with add_eos_token=False. - # Restoring add_eos_token for fast tokenizers makes sglang diverge from - # the HF reference (which doesn't restore it), breaking embedding models - # like intfloat/e5-mistral-7b-instruct (cosine similarity drops to ~0.33). - if attr == "add_eos_token" and isinstance(tokenizer, PreTrainedTokenizerFast): - config_val = _V4_DEFAULTS["add_eos_token"] # False - current_val = getattr(tokenizer, attr, None) - if current_val != config_val: - logger.info( - "Restoring %s=%s for %s (was %s after v5 loading)", - attr, - config_val, - model_name_or_path, - current_val, - ) - setattr(tokenizer, f"_{attr}", config_val) - changed = True - - # Rebuild the post-processor so it respects the restored flags - if changed and hasattr(tokenizer, "update_post_processor"): - tokenizer.update_post_processor() - - -def _fix_special_tokens_pattern(tokenizer): - """Fix https://github.com/huggingface/transformers/pull/42563 which defaults - special_tokens_pattern to "cls_sep", inserting None into token IDs when - cls_token/sep_token are undefined (e.g. Kimi-VL's TikTokenTokenizer). - """ - pattern = getattr(tokenizer, "special_tokens_pattern", None) - if pattern == "cls_sep" and ( - tokenizer.cls_token_id is None or tokenizer.sep_token_id is None - ): - tokenizer.special_tokens_pattern = "none" - - -def _fix_added_tokens_encoding(tokenizer): - """Ensure special tokens encode as single tokens in transformers v5. - - Some model tokenizers (e.g. MiniCPM-V-4) define special tokens like , - as attributes on the tokenizer class with corresponding IDs in the - vocabulary (via tokenizer.json's added_tokens). In transformers v5, these - tokens may not appear in get_added_vocab() and encode() splits them into - subwords, breaking multimodal pipelines that rely on finding them in input_ids. - - This function discovers such tokens by scanning tokenizer attributes, checks - if they encode correctly, and re-registers any that don't. - """ - # Discover special token strings from tokenizer attributes. - # Model tokenizers (e.g. MiniCPMVTokenizerFast) store them as attributes - # like im_start="", slice_start="", etc. - candidates = {} - for attr in dir(tokenizer): - if attr.startswith("_"): - continue - try: - val = getattr(tokenizer, attr) - except Exception: - continue - if ( - not isinstance(val, str) - or not val.startswith("<") - or not val.endswith(">") - or len(val) > 20 - ): - continue - token_id = tokenizer.convert_tokens_to_ids(val) - if token_id is not None and token_id != tokenizer.unk_token_id: - candidates[val] = token_id - - if not candidates: - return - - # Check which tokens fail to encode as single tokens. - broken = [] - for token_str, expected_id in candidates.items(): - try: - ids = tokenizer.encode(token_str, add_special_tokens=False) - if len(ids) != 1 or ids[0] != expected_id: - broken.append(token_str) - except Exception: - broken.append(token_str) - - if not broken: - return - - from transformers import AddedToken - - tokens_to_add = [AddedToken(tok, special=True, normalized=False) for tok in broken] - tokenizer.add_tokens(tokens_to_add, special_tokens=True) - logger.info( - "Re-registered %d special tokens for correct v5 encoding: %s", - len(broken), - broken[:10], - ) - - -# Some models doesn't have an available processor, e.g.: InternVL -def get_tokenizer_from_processor(processor): - if isinstance(processor, PreTrainedTokenizerBase): - return processor - return processor.tokenizer - - -def _build_processor_manually( - model_path, config, trust_remote_code, revision, **kwargs -): - """Build processor when AutoProcessor fails to resolve feature_extractor_type. - - In transformers v5, AutoProcessor.from_pretrained calls - AutoFeatureExtractor.from_pretrained which fails if - preprocessor_config.json lacks 'feature_extractor_type'. This loads the - processor class from the hub and constructs it with individually-loaded - components. - """ - import transformers - from transformers import AutoImageProcessor, AutoTokenizer - from transformers.dynamic_module_utils import get_class_from_dynamic_module - - # Resolve processor class from auto_map — check both the model config - # and the preprocessor_config.json (some models like MiniCPM-o only - # declare AutoProcessor in the latter). - auto_map = getattr(config, "auto_map", None) or {} - proc_ref = auto_map.get("AutoProcessor") - if not proc_ref: - try: - pp_file = _resolve_local_or_cached_file( - model_path, "preprocessor_config.json", revision - ) - with open(pp_file) as f: - pp_auto_map = json.load(f).get("auto_map", {}) - proc_ref = pp_auto_map.get("AutoProcessor") - except Exception as e: - logger.debug( - "_build_processor_manually: could not read preprocessor_config.json " - "for %s: %s", - model_path, - e, - ) - if not proc_ref: - raise ValueError(f"Cannot determine processor class for {model_path}") - - proc_cls = get_class_from_dynamic_module( - proc_ref, model_path, code_revision=revision - ) - - # Load sub-components individually (these succeed) - tokenizer = AutoTokenizer.from_pretrained( - model_path, trust_remote_code=trust_remote_code, revision=revision - ) - init_kwargs = {"tokenizer": tokenizer} - - if "image_processor" in getattr(proc_cls, "attributes", []): - try: - init_kwargs["image_processor"] = AutoImageProcessor.from_pretrained( - model_path, trust_remote_code=trust_remote_code, revision=revision - ) - except Exception as e: - logger.warning("Failed to load image_processor for %s: %s", model_path, e) - - # Instantiate feature extractor from its declared class - fe_class_name = getattr(proc_cls, "feature_extractor_class", None) - if fe_class_name: - fe_class = getattr(transformers, fe_class_name, None) - if fe_class is not None: - init_kwargs["feature_extractor"] = fe_class() - - return proc_cls(**init_kwargs) - - -def get_processor( - tokenizer_name: str, - *args, - tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - tokenizer_revision: Optional[str] = None, - use_fast: Optional[bool] = True, - **kwargs, -): - # pop 'revision' from kwargs if present. - revision = kwargs.pop("revision", tokenizer_revision) - if ( - "mistral-large-3" in str(tokenizer_name).lower() - or "mistral-small-4" in str(tokenizer_name).lower() - or "leanstral" in str(tokenizer_name).lower() - ): - config = _load_mistral_large_3_for_causal_LM( - tokenizer_name, - trust_remote_code=trust_remote_code, - revision=revision, - ) - else: - _ensure_llama_flash_attention2_compat() - config = AutoConfig.from_pretrained( - tokenizer_name, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - if _is_deepseek_ocr_model(config): - # Temporary hack for load deepseek-ocr - config.model_type = "deepseek-ocr" - config.update({"architectures": ["DeepseekOCRForCausalLM"]}) - elif _is_deepseek_ocr2_model(config): - # Temporary hack for load deepseek-ocr2 - config.model_type = "deepseek-ocr" - config.update({"architectures": ["DeepseekOCRForCausalLM"]}) - _override_v_head_dim_if_zero(config) - - # fix: for Qwen2-VL and Sarashina2Vision models, inject default 'size' if not provided. - if config.model_type in {"qwen2_vl", "sarashina2_vision"}: - if "size" not in kwargs: - kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520} - - if config.model_type not in {"llava", "clip"}: - kwargs["use_fast"] = use_fast - try: - if "InternVL3_5" in tokenizer_name: - processor = AutoTokenizer.from_pretrained( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - else: - if config.model_type in _CUSTOMIZED_MM_PROCESSOR: - processor = _CUSTOMIZED_MM_PROCESSOR[config.model_type].from_pretrained( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - else: - processor = AutoProcessor.from_pretrained( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - - except ValueError as e: - error_message = str(e) - if "does not have a slow version" in error_message: - logger.info( - f"Processor {tokenizer_name} does not have a slow version. Automatically use fast version" - ) - kwargs["use_fast"] = True - processor = AutoProcessor.from_pretrained( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - revision=revision, - **kwargs, - ) - elif ( - "are not supported by" in error_message and "MistralCommon" in error_message - ): - logger.info( - "AutoProcessor for %s rejected standard kwargs, " - "retrying without trust_remote_code/use_fast", - tokenizer_name, - ) - kwargs.pop("use_fast", None) - kwargs.pop("_from_auto", None) - processor = AutoProcessor.from_pretrained( - tokenizer_name, - *args, - revision=revision, - **kwargs, - ) - elif "Unrecognized feature extractor" in error_message: - logger.info( - "AutoProcessor failed on feature extractor for %s, " - "constructing processor manually", - tokenizer_name, - ) - processor = _build_processor_manually( - tokenizer_name, - config, - trust_remote_code, - revision, - **kwargs, - ) - else: - raise e - # If processor is a bare tokenizer (e.g. Mistral-Small-4 has no processor_config.json) - # and the model is a vision model (pixtral), wrap it in a proper PixtralProcessor - # so that image data is actually processed through the image processor. - if ( - isinstance(processor, PreTrainedTokenizerBase) - and getattr(config, "model_type", None) == "pixtral" - ): - from transformers.models.pixtral.image_processing_pixtral import ( - PixtralImageProcessor, - ) - from transformers.models.pixtral.processing_pixtral import ( - PixtralProcessor as HFPixtralProcessor, - ) - - vision_config = config.vision_config - patch_size = vision_config.patch_size - image_size = vision_config.image_size - spatial_merge_size = getattr(vision_config, "spatial_merge_size", 1) - - effective_patch = patch_size * spatial_merge_size - image_processor = PixtralImageProcessor( - do_resize=True, - size={"longest_edge": image_size}, - patch_size={"height": effective_patch, "width": effective_patch}, - ) - processor = HFPixtralProcessor( - image_processor=image_processor, - tokenizer=processor, - patch_size=patch_size, - spatial_merge_size=spatial_merge_size, - ) - - tokenizer = get_tokenizer_from_processor(processor) - _patch_mistral_common_tokenizer(tokenizer) - - if tokenizer.chat_template is None: - local_path = download_from_hf( - tokenizer_name, allow_patterns=["*.json", "*.jinja", "*.model"] - ) - jinja_path = Path(local_path) / "chat_template.jinja" - if jinja_path.is_file(): - tokenizer.chat_template = jinja_path.read_text() - logger.info("Loaded chat_template from %s", jinja_path) - - _fix_special_tokens_pattern(tokenizer) - _fix_added_tokens_encoding(tokenizer) - attach_additional_stop_token_ids(tokenizer) - return processor - - -def attach_additional_stop_token_ids(tokenizer): - # Special handling for stop token <|eom_id|> generated by llama 3 tool use. - if "<|eom_id|>" in tokenizer.get_added_vocab(): - tokenizer.additional_stop_token_ids = set( - [tokenizer.get_added_vocab()["<|eom_id|>"]] - ) - else: - tokenizer.additional_stop_token_ids = None - - -def _patch_mistral_common_tokenizer(tokenizer): - """Patch MistralCommonTokenizer/Backend to be compatible with HF tokenizer API. - - MistralCommon tokenizers (used by Voxtral, Pixtral, etc.) reject several - standard kwargs and lack some attributes that sglang expects. We wrap the - offending methods once at load time so that the rest of the codebase does - not need any special-casing. - """ - cls_name = type(tokenizer).__name__ - if "MistralCommon" not in cls_name: - return tokenizer - if getattr(tokenizer, "_mistral_common_patched", False): - return tokenizer - tokenizer._mistral_common_patched = True - - # Missing attributes - if not hasattr(tokenizer, "get_added_vocab"): - tokenizer.get_added_vocab = lambda: {} - - # Set a chat_template containing "audio" so that sglang's content format - # detector returns "openai" (which preserves audio_url extraction). - # The actual template rendering is done by MistralCommon's apply_chat_template. - if not hasattr(tokenizer, "chat_template") or tokenizer.chat_template is None: - tokenizer.chat_template = "" - - # convert_tokens_to_ids asserts on multi-token strings - _orig_convert = tokenizer.convert_tokens_to_ids - - def _safe_convert(val): - try: - return _orig_convert(val) - except AssertionError: - return getattr(tokenizer, "unk_token_id", None) - - tokenizer.convert_tokens_to_ids = _safe_convert - - # Wrap methods that reject certain kwargs - def _drop_kwargs(fn, keys): - def wrapper(*args, **kwargs): - for k in keys: - kwargs.pop(k, None) - return fn(*args, **kwargs) - - return wrapper - - tokenizer.decode = _drop_kwargs(tokenizer.decode, ["spaces_between_special_tokens"]) - tokenizer.batch_decode = _drop_kwargs( - tokenizer.batch_decode, ["spaces_between_special_tokens"] - ) - - # Save original apply_chat_template for processors that need it (e.g. Voxtral) - tokenizer._orig_apply_chat_template = tokenizer.apply_chat_template - - def _safe_apply_chat_template(messages, **kwargs): - """Wrapper that strips unsupported kwargs and non-text content parts. - - When sglang extracts audio/image URLs, it replaces content blocks with - {"type": "audio"} or {"type": "image"} (no URL). MistralCommon fails - on these stripped blocks. We convert them to text-only messages. - """ - kwargs.pop("add_generation_prompt", None) - cleaned = [] - for msg in messages: - if isinstance(msg, dict): - content = msg.get("content", "") - if isinstance(content, list): - text_parts = [ - p.get("text", "") - for p in content - if isinstance(p, dict) and p.get("type") == "text" - ] - msg = {**msg, "content": " ".join(text_parts) if text_parts else ""} - cleaned.append(msg) - else: - cleaned.append(msg) - return tokenizer._orig_apply_chat_template(cleaned, **kwargs) - - tokenizer.apply_chat_template = _safe_apply_chat_template - - -def check_gguf_file(model: Union[str, os.PathLike]) -> bool: - """Check if the file is a GGUF model.""" - model = Path(model) - if not model.is_file(): - return False - elif model.suffix == ".gguf": - return True - - with open(model, "rb") as f: - header = f.read(4) - return header == b"GGUF" +from sglang.srt.utils.hf_transformers import * # noqa: F401, F403 +from sglang.srt.utils.hf_transformers import __all__ # noqa: F401 diff --git a/test/registered/unit/utils/test_hf_transformers.py b/test/registered/unit/utils/test_hf_transformers.py new file mode 100644 index 000000000..e5a0a3e06 --- /dev/null +++ b/test/registered/unit/utils/test_hf_transformers.py @@ -0,0 +1,586 @@ +"""Unit tests for the sglang.srt.utils.hf_transformers subpackage. + +Tests cover the pure utility functions (compat patches, config helpers, +context length, GGUF detection, etc.) that don't require actual model files. +""" + +import tempfile +import unittest +from types import SimpleNamespace + +from transformers import PretrainedConfig + +from sglang.srt.utils.hf_transformers.common import ( + _is_deepseek_ocr2_model, + _is_deepseek_ocr_model, + _override_v_head_dim_if_zero, + _patch_text_config, + check_gguf_file, + get_context_length, + get_hf_text_config, + get_rope_config, +) +from sglang.srt.utils.hf_transformers.compat import normalize_rope_scaling_compat +from sglang.srt.utils.hf_transformers.tokenizer import _fix_special_tokens_pattern +from sglang.test.ci.ci_register import register_cpu_ci + +register_cpu_ci(est_time=5, suite="stage-a-test-cpu") + + +# --------------------------------------------------------------------------- +# normalize_rope_scaling_compat +# --------------------------------------------------------------------------- + + +class TestNormalizeRopeScalingCompat(unittest.TestCase): + def test_adds_type_from_rope_type(self): + cfg = PretrainedConfig() + cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0} + normalize_rope_scaling_compat(cfg) + self.assertEqual(cfg.rope_scaling["type"], "llama3") + + def test_preserves_existing_type(self): + cfg = PretrainedConfig() + cfg.rope_scaling = {"rope_type": "llama3", "type": "custom", "factor": 8.0} + normalize_rope_scaling_compat(cfg) + self.assertEqual(cfg.rope_scaling["type"], "custom") + + def test_no_op_when_no_rope_scaling(self): + cfg = PretrainedConfig() + normalize_rope_scaling_compat(cfg) + self.assertIsNone(getattr(cfg, "rope_scaling", None)) + + def test_no_op_when_rope_scaling_is_none(self): + cfg = PretrainedConfig() + cfg.rope_scaling = None + normalize_rope_scaling_compat(cfg) + self.assertIsNone(cfg.rope_scaling) + + def test_recurses_into_text_config(self): + text_cfg = PretrainedConfig() + text_cfg.rope_scaling = {"rope_type": "yarn", "factor": 4.0} + cfg = PretrainedConfig() + cfg.text_config = text_cfg + normalize_rope_scaling_compat(cfg) + self.assertEqual(text_cfg.rope_scaling["type"], "yarn") + + def test_recurses_into_llm_config(self): + llm_cfg = PretrainedConfig() + llm_cfg.rope_scaling = {"rope_type": "dynamic", "factor": 2.0} + cfg = PretrainedConfig() + cfg.llm_config = llm_cfg + normalize_rope_scaling_compat(cfg) + self.assertEqual(llm_cfg.rope_scaling["type"], "dynamic") + + def test_no_crash_on_non_dict_rope_scaling(self): + cfg = PretrainedConfig() + cfg.rope_scaling = "not_a_dict" + normalize_rope_scaling_compat(cfg) + self.assertEqual(cfg.rope_scaling, "not_a_dict") + + def test_no_crash_on_dict_without_rope_type(self): + cfg = PretrainedConfig() + cfg.rope_scaling = {"factor": 4.0} + normalize_rope_scaling_compat(cfg) + self.assertNotIn("type", cfg.rope_scaling) + + +# --------------------------------------------------------------------------- +# get_rope_config +# --------------------------------------------------------------------------- + + +class TestGetRopeConfig(unittest.TestCase): + def test_v5_rope_parameters(self): + cfg = PretrainedConfig() + cfg.rope_parameters = {"rope_theta": 10000.0, "rope_type": "default"} + theta, params = get_rope_config(cfg) + self.assertEqual(theta, 10000.0) + self.assertIs(params, cfg.rope_parameters) + + def test_v4_fallback_remote_code_config(self): + # Remote-code configs (SimpleNamespace) lack the v5 rope_parameters property + cfg = SimpleNamespace( + rope_theta=500000.0, + rope_scaling={"type": "llama3", "factor": 8.0}, + ) + theta, params = get_rope_config(cfg) + self.assertEqual(theta, 500000.0) + self.assertEqual(params, {"type": "llama3", "factor": 8.0}) + + def test_v4_no_scaling(self): + cfg = SimpleNamespace(rope_theta=10000.0) + theta, params = get_rope_config(cfg) + self.assertEqual(theta, 10000.0) + self.assertIsNone(params) + + +# --------------------------------------------------------------------------- +# _patch_text_config +# --------------------------------------------------------------------------- + + +class TestPatchTextConfig(unittest.TestCase): + def test_propagates_parent_to_text(self): + parent = PretrainedConfig() + parent.pad_token_id = 0 + parent.bos_token_id = 1 + parent.eos_token_id = 2 + parent.tie_word_embeddings = False + + text = PretrainedConfig() + text.num_attention_heads = 32 + + result = _patch_text_config(parent, text) + self.assertEqual(result.pad_token_id, 0) + self.assertEqual(result.bos_token_id, 1) + self.assertEqual(result.eos_token_id, 2) + self.assertIs(result, text) + + def test_propagates_text_to_parent(self): + parent = PretrainedConfig() + text = PretrainedConfig() + text.pad_token_id = 42 + + _patch_text_config(parent, text) + self.assertEqual(parent.pad_token_id, 42) + + def test_no_overwrite_when_both_have_attr(self): + parent = PretrainedConfig() + parent.pad_token_id = 0 + text = PretrainedConfig() + text.pad_token_id = 99 + + _patch_text_config(parent, text) + self.assertEqual(parent.pad_token_id, 0) + self.assertEqual(text.pad_token_id, 99) + + +# --------------------------------------------------------------------------- +# get_context_length +# --------------------------------------------------------------------------- + + +class TestGetContextLength(unittest.TestCase): + def test_max_position_embeddings(self): + cfg = PretrainedConfig() + cfg.max_position_embeddings = 4096 + self.assertEqual(get_context_length(cfg), 4096) + + def test_max_sequence_length_takes_priority(self): + cfg = PretrainedConfig() + cfg.max_sequence_length = 8192 + cfg.max_position_embeddings = 4096 + self.assertEqual(get_context_length(cfg), 8192) + + def test_rope_scaling_factor(self): + cfg = PretrainedConfig() + cfg.max_position_embeddings = 4096 + cfg.rope_scaling = {"factor": 4.0} + self.assertEqual(get_context_length(cfg), 16384) + + def test_rope_scaling_llama3_ignores_factor(self): + cfg = PretrainedConfig() + cfg.max_position_embeddings = 131072 + cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0} + self.assertEqual(get_context_length(cfg), 131072) + + def test_original_max_position_embeddings_ignores_factor(self): + cfg = PretrainedConfig() + cfg.max_position_embeddings = 131072 + cfg.rope_scaling = { + "factor": 8.0, + "original_max_position_embeddings": 8192, + } + self.assertEqual(get_context_length(cfg), 131072) + + def test_default_when_no_keys(self): + cfg = PretrainedConfig() + self.assertEqual(get_context_length(cfg), 2048) + + +# --------------------------------------------------------------------------- +# check_gguf_file +# --------------------------------------------------------------------------- + + +class TestCheckGgufFile(unittest.TestCase): + def test_gguf_suffix(self): + with tempfile.NamedTemporaryFile(suffix=".gguf") as f: + self.assertTrue(check_gguf_file(f.name)) + + def test_gguf_magic_header(self): + with tempfile.NamedTemporaryFile(suffix=".bin") as f: + f.write(b"GGUF" + b"\x00" * 100) + f.flush() + self.assertTrue(check_gguf_file(f.name)) + + def test_non_gguf_file(self): + with tempfile.NamedTemporaryFile(suffix=".bin") as f: + f.write(b"NOT_GGUF" + b"\x00" * 100) + f.flush() + self.assertFalse(check_gguf_file(f.name)) + + def test_nonexistent_file(self): + self.assertFalse(check_gguf_file("/nonexistent/path/model.bin")) + + def test_directory(self): + with tempfile.TemporaryDirectory() as d: + self.assertFalse(check_gguf_file(d)) + + +# --------------------------------------------------------------------------- +# _is_deepseek_ocr_model / _is_deepseek_ocr2_model +# --------------------------------------------------------------------------- + + +class TestDeepseekOcrDetection(unittest.TestCase): + def test_ocr_model_detected(self): + cfg = PretrainedConfig() + cfg.auto_map = {"AutoModel": "modeling_deepseekocr.DeepseekOCRForCausalLM"} + self.assertTrue(_is_deepseek_ocr_model(cfg)) + + def test_ocr2_model_detected(self): + cfg = PretrainedConfig() + cfg.auto_map = {"AutoModel": "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"} + self.assertTrue(_is_deepseek_ocr2_model(cfg)) + + def test_non_ocr_model(self): + cfg = PretrainedConfig() + cfg.auto_map = {"AutoModel": "modeling_llama.LlamaForCausalLM"} + self.assertFalse(_is_deepseek_ocr_model(cfg)) + self.assertFalse(_is_deepseek_ocr2_model(cfg)) + + def test_no_auto_map(self): + cfg = PretrainedConfig() + self.assertFalse(_is_deepseek_ocr_model(cfg)) + self.assertFalse(_is_deepseek_ocr2_model(cfg)) + + def test_empty_auto_map(self): + cfg = PretrainedConfig() + cfg.auto_map = {} + self.assertFalse(_is_deepseek_ocr_model(cfg)) + self.assertFalse(_is_deepseek_ocr2_model(cfg)) + + +# --------------------------------------------------------------------------- +# _override_v_head_dim_if_zero +# --------------------------------------------------------------------------- + + +class TestOverrideVHeadDimIfZero(unittest.TestCase): + def test_patches_zero_v_head_dim(self): + text_cfg = SimpleNamespace(v_head_dim=0) + cfg = PretrainedConfig() + cfg.text_config = text_cfg + _override_v_head_dim_if_zero(cfg) + self.assertEqual(text_cfg.v_head_dim, 128) + + def test_custom_patch_value(self): + text_cfg = SimpleNamespace(v_head_dim=0) + cfg = PretrainedConfig() + cfg.text_config = text_cfg + _override_v_head_dim_if_zero(cfg, patch=64) + self.assertEqual(text_cfg.v_head_dim, 64) + + def test_no_patch_when_nonzero(self): + text_cfg = SimpleNamespace(v_head_dim=256) + cfg = PretrainedConfig() + cfg.text_config = text_cfg + _override_v_head_dim_if_zero(cfg) + self.assertEqual(text_cfg.v_head_dim, 256) + + def test_dict_sub_config(self): + cfg = PretrainedConfig() + cfg.text_config = {"v_head_dim": 0} + _override_v_head_dim_if_zero(cfg) + self.assertEqual(cfg.text_config["v_head_dim"], 128) + + def test_no_sub_config(self): + cfg = PretrainedConfig() + _override_v_head_dim_if_zero(cfg) # should not raise + + +# --------------------------------------------------------------------------- +# get_hf_text_config +# --------------------------------------------------------------------------- + + +class TestGetHfTextConfig(unittest.TestCase): + def test_returns_config_for_pure_text_model(self): + cfg = PretrainedConfig() + cfg.architectures = ["LlamaForCausalLM"] + result = get_hf_text_config(cfg) + self.assertIs(result, cfg) + + def test_returns_text_config_for_multimodal(self): + text_cfg = PretrainedConfig() + text_cfg.num_attention_heads = 32 + cfg = PretrainedConfig() + cfg.architectures = ["SomeVLMForCausalLM"] + cfg.text_config = text_cfg + result = get_hf_text_config(cfg) + self.assertIs(result, text_cfg) + + def test_llm_config_priority_over_text_config(self): + llm_cfg = PretrainedConfig() + llm_cfg.num_attention_heads = 16 + text_cfg = PretrainedConfig() + text_cfg.num_attention_heads = 32 + cfg = PretrainedConfig() + cfg.architectures = ["SomeModel"] + cfg.llm_config = llm_cfg + cfg.text_config = text_cfg + result = get_hf_text_config(cfg) + self.assertIs(result, llm_cfg) + + def test_thinker_config_highest_priority(self): + thinker_cfg = PretrainedConfig() + thinker_cfg.num_attention_heads = 8 + cfg = PretrainedConfig() + cfg.architectures = ["SomeModel"] + cfg.thinker_config = thinker_cfg + result = get_hf_text_config(cfg) + self.assertIs(result, thinker_cfg) + + def test_thinker_config_with_text_sub_config(self): + inner_text = PretrainedConfig() + inner_text.num_attention_heads = 8 + thinker_cfg = PretrainedConfig() + thinker_cfg.text_config = inner_text + thinker_cfg.torch_dtype = "float16" + cfg = PretrainedConfig() + cfg.architectures = ["Qwen2OmniModel"] + cfg.thinker_config = thinker_cfg + result = get_hf_text_config(cfg) + self.assertIs(result, inner_text) + self.assertEqual(inner_text.torch_dtype, "float16") + + def test_converts_dict_sub_config(self): + cfg = PretrainedConfig() + cfg.architectures = ["SomeModel"] + cfg.text_config = { + "num_attention_heads": 32, + "hidden_size": 4096, + } + result = get_hf_text_config(cfg) + self.assertIsInstance(cfg.text_config, PretrainedConfig) + self.assertEqual(result.num_attention_heads, 32) + + def test_llava_returns_parent_config(self): + cfg = PretrainedConfig() + cfg.architectures = ["LlavaForCausalLM"] + text_cfg = PretrainedConfig() + text_cfg.num_attention_heads = 32 + cfg.text_config = text_cfg + result = get_hf_text_config(cfg) + self.assertIs(result, cfg) + + def test_calls_normalize_rope_scaling(self): + cfg = PretrainedConfig() + cfg.architectures = ["LlamaForCausalLM"] + cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0} + get_hf_text_config(cfg) + self.assertIn("type", cfg.rope_scaling) + self.assertEqual(cfg.rope_scaling["type"], "llama3") + + +# --------------------------------------------------------------------------- +# _fix_special_tokens_pattern +# --------------------------------------------------------------------------- + + +class TestFixSpecialTokensPattern(unittest.TestCase): + def test_fixes_cls_sep_with_missing_tokens(self): + tok = SimpleNamespace( + special_tokens_pattern="cls_sep", + cls_token_id=None, + sep_token_id=None, + ) + _fix_special_tokens_pattern(tok) + self.assertEqual(tok.special_tokens_pattern, "none") + + def test_no_change_when_tokens_present(self): + tok = SimpleNamespace( + special_tokens_pattern="cls_sep", + cls_token_id=101, + sep_token_id=102, + ) + _fix_special_tokens_pattern(tok) + self.assertEqual(tok.special_tokens_pattern, "cls_sep") + + def test_no_change_for_other_patterns(self): + tok = SimpleNamespace( + special_tokens_pattern="none", + cls_token_id=None, + sep_token_id=None, + ) + _fix_special_tokens_pattern(tok) + self.assertEqual(tok.special_tokens_pattern, "none") + + def test_no_change_when_no_pattern(self): + tok = SimpleNamespace(cls_token_id=None, sep_token_id=None) + _fix_special_tokens_pattern(tok) + self.assertFalse(hasattr(tok, "special_tokens_pattern")) + + +# --------------------------------------------------------------------------- +# __init__.py re-exports +# --------------------------------------------------------------------------- + + +class TestModuleReExports(unittest.TestCase): + def test_all_public_symbols_importable(self): + import sglang.srt.utils.hf_transformers as pkg + + for name in pkg.__all__: + self.assertTrue( + hasattr(pkg, name), + f"{name} listed in __all__ but not importable from package", + ) + + def test_shim_module_exports_match(self): + import sglang.srt.utils.hf_transformers as pkg + import sglang.srt.utils.hf_transformers_utils as shim + + for name in pkg.__all__: + self.assertTrue( + hasattr(shim, name), + f"{name} not available through shim module hf_transformers_utils", + ) + + +# --------------------------------------------------------------------------- +# compat: _patch_removed_symbols +# --------------------------------------------------------------------------- + + +class TestPatchRemovedSymbols(unittest.TestCase): + def test_llama_flash_attention2_exists(self): + from transformers.models.llama import modeling_llama + + self.assertTrue( + hasattr(modeling_llama, "LlamaFlashAttention2"), + "LlamaFlashAttention2 should be patched onto modeling_llama", + ) + + def test_is_flash_attn_greater_or_equal_2_10_callable(self): + import transformers.utils as _u + + self.assertTrue( + hasattr(_u, "is_flash_attn_greater_or_equal_2_10"), + "is_flash_attn_greater_or_equal_2_10 should be patched onto transformers.utils", + ) + self.assertIsInstance(_u.is_flash_attn_greater_or_equal_2_10(), bool) + + +# --------------------------------------------------------------------------- +# compat: _patch_rope_parameters_validation +# --------------------------------------------------------------------------- + + +class TestPatchRopeParametersValidation(unittest.TestCase): + def test_injects_rope_theta_into_rope_scaling(self): + config_dict = { + "model_type": "llama", + "rope_theta": 500000.0, + "max_position_embeddings": 131072, + "rope_scaling": { + "rope_type": "llama3", + "factor": 8.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192, + }, + } + config = PretrainedConfig.from_dict(config_dict) + rope_params = getattr(config, "rope_parameters", None) + if rope_params is not None: + self.assertIn("rope_theta", rope_params) + + def test_no_injection_when_rope_theta_already_in_scaling(self): + config_dict = { + "model_type": "llama", + "rope_theta": 500000.0, + "max_position_embeddings": 131072, + "rope_scaling": { + "rope_type": "llama3", + "factor": 8.0, + "rope_theta": 999.0, + "low_freq_factor": 1.0, + "high_freq_factor": 4.0, + "original_max_position_embeddings": 8192, + }, + } + config = PretrainedConfig.from_dict(config_dict) + rope_params = getattr(config, "rope_parameters", None) + if rope_params is not None: + self.assertEqual(rope_params["rope_theta"], 999.0) + + def test_no_crash_without_rope_scaling(self): + config_dict = {"model_type": "llama", "rope_theta": 10000.0} + config = PretrainedConfig.from_dict(config_dict) + self.assertIsNotNone(config) + + +# --------------------------------------------------------------------------- +# compat: _ensure_clean_up_tokenization_compat +# --------------------------------------------------------------------------- + + +class TestCleanUpTokenizationCompat(unittest.TestCase): + def test_clean_up_tokenization_exists(self): + from transformers import PreTrainedTokenizerBase + + self.assertTrue(hasattr(PreTrainedTokenizerBase, "clean_up_tokenization")) + + def test_clean_up_tokenization_callable(self): + from transformers import PreTrainedTokenizerBase + + self.assertTrue(callable(PreTrainedTokenizerBase.clean_up_tokenization)) + + +# --------------------------------------------------------------------------- +# compat: _ensure_is_torch_fx_available_compat +# --------------------------------------------------------------------------- + + +class TestIsTorchFxAvailableCompat(unittest.TestCase): + def test_is_torch_fx_available_exists(self): + import transformers.utils.import_utils as _iu + + self.assertTrue(hasattr(_iu, "is_torch_fx_available")) + self.assertTrue(_iu.is_torch_fx_available()) + + +# --------------------------------------------------------------------------- +# compat: _patch_nemotron_h_pattern +# --------------------------------------------------------------------------- + + +class TestPatchNemotronHPattern(unittest.TestCase): + def test_pattern_to_list_skips_mlp_dash(self): + try: + from transformers.models.nemotron_h.configuration_nemotron_h import ( + NemotronHConfig, + ) + + result = NemotronHConfig._pattern_to_list("M-*-") + self.assertEqual(result, ["mamba", "attention"]) + except ImportError: + self.skipTest("NemotronHConfig not available in this transformers version") + + def test_pattern_to_list_standard_chars(self): + try: + from transformers.models.nemotron_h.configuration_nemotron_h import ( + NemotronHConfig, + ) + + result = NemotronHConfig._pattern_to_list("ME*") + self.assertEqual(result, ["mamba", "moe", "attention"]) + except ImportError: + self.skipTest("NemotronHConfig not available in this transformers version") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/registered/vlm/test_vlm_input_format.py b/test/registered/vlm/test_vlm_input_format.py index 792da109e..81dac3550 100644 --- a/test/registered/vlm/test_vlm_input_format.py +++ b/test/registered/vlm/test_vlm_input_format.py @@ -425,11 +425,13 @@ class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes torch_dtype=torch.bfloat16, low_cpu_mem_usage=False, ) - except RuntimeError as e: - if "meta" not in str(e): + except (RuntimeError, AttributeError) as e: + if isinstance(e, RuntimeError) and "meta" not in str(e): raise # Transformers v5 always uses meta tensors for init, which breaks # models calling .item() in __init__ (e.g. InternVL's drop_path_rate). + # Transformers v5.5.3 may also raise AttributeError for remote-code + # models missing new internal attributes (e.g. all_tied_weights_keys). # Fall back to from_config + manual weight loading. import gc import glob @@ -594,6 +596,13 @@ class TestMiniCPMVUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes cls.processor = AutoProcessor.from_pretrained( cls.model_path, trust_remote_code=True ) + # In transformers v5.5.3, AutoTokenizer may return TokenizersBackend + # which lacks model-specific attributes (e.g. im_start_id for MiniCPM-V). + # Replace with sglang's tokenizer which handles this via declared-class + # fallback, then fix added tokens encoding. + from sglang.srt.utils.hf_transformers import get_tokenizer + + cls.processor.tokenizer = get_tokenizer(cls.model_path, trust_remote_code=True) _fix_added_tokens_encoding(cls.processor.tokenizer) cls._init_visual()