minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 10 Packages Projects Releases Wiki Activity

Upgrade transformers to 5.5.3 and refactor hf_transformers_utils into subpackage (#21569)

Browse Source
This commit is contained in:
Xinyuan Tong
2026-04-15 20:03:44 -07:00
committed by GitHub
parent 14e122cdee
commit 34fef07a15
18 changed files with 2838 additions and 1515 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/pyproject.toml
+3 -2
View File
@@ -35,6 +35,7 @@ dependencies = [
"modelscope",
"msgspec",
"ninja",
"easydict", # Required by remote model code (e.g. DeepSeek-OCR) loaded via trust_remote_code; validated by transformers 5.4+ check_imports
"numpy",
"nvidia-cutlass-dsl>=4.4.1",
"nvidia-ml-py",
@@ -70,8 +71,8 @@ dependencies = [
"av ; sys_platform == 'linux' and (platform_machine == 'aarch64' or platform_machine == 'arm64' or platform_machine == 'armv7l')",
"torchvision",
"tqdm",
"mistral_common>=1.9.0",
"transformers==5.3.0",
"mistral_common>=1.11.0",
"transformers==5.5.4",
"uvicorn",
"uvloop",
"watchfiles",
python/pyproject_cpu.toml
+3 -2
View File
@@ -31,6 +31,7 @@ dependencies = [
"llguidance>=0.7.11,<0.8.0",
"modelscope",
"msgspec",
"easydict",
"ninja",
"numpy",
"openai-harmony==0.0.4",
@@ -60,8 +61,8 @@ dependencies = [
"torchaudio==2.9.0",
"torchvision==0.24.0",
"tqdm",
"mistral_common>=1.9.0",
"transformers==5.3.0",
"mistral_common>=1.11.0",
"transformers==5.5.4",
"triton==3.5.0",
"uvicorn",
"uvloop",
python/pyproject_npu.toml
+3 -2
View File
@@ -25,6 +25,7 @@ dependencies = [
"datasets",
"einops",
"fastapi",
"easydict",
"gguf",
"hf_transfer",
"huggingface_hub",
@@ -57,8 +58,8 @@ dependencies = [
"timm==1.0.16",
"torchao==0.9.0",
"tqdm",
"mistral_common>=1.9.0",
"transformers==5.3.0",
"mistral_common>=1.11.0",
"transformers==5.5.4",
"uvicorn",
"uvloop",
"xgrammar==0.1.32",
python/pyproject_other.toml
+3 -2
View File
@@ -25,6 +25,7 @@ runtime_common = [
"build",
"compressed-tensors",
"datasets",
"easydict",
"einops",
"fastapi",
"gguf",
@@ -57,8 +58,8 @@ runtime_common = [
"timm==1.0.16",
"torchao==0.9.0",
"tqdm",
"mistral_common>=1.9.0",
"transformers==5.3.0",
"mistral_common>=1.11.0",
"transformers==5.5.4",
"uvicorn",
"uvloop",
"xgrammar==0.1.32",
python/pyproject_xpu.toml
+4 -2
View File
@@ -27,7 +27,9 @@ dependencies = [
"blobfile==3.0.0",
"build",
"compressed-tensors",
"addict",
"datasets",
"easydict",
"einops",
"fastapi",
"gguf",
@@ -60,8 +62,8 @@ dependencies = [
"timm==1.0.16",
"torchao==0.9.0+xpu",
"tqdm",
"mistral_common>=1.9.0",
"transformers==5.3.0",
"mistral_common>=1.11.0",
"transformers==5.5.4",
"uvicorn",
"uvloop",
# "xgrammar==0.1.24", , xgrammar depends on CUDA PyTorch and Triton only
python/sglang/srt/configs/qwen3_5.py
+16
View File
@@ -8,6 +8,9 @@ class Qwen3_5VisionConfig(Qwen3VLVisionConfig):
model_type = "qwen3_5"
base_config_key = "vision_config"
def __init__(self, **kwargs):
super().__init__(**kwargs)
class Qwen3_5TextConfig(Qwen3NextConfig):
model_type = "qwen3_5_text"
@@ -109,14 +112,27 @@ class Qwen3_5Config(PretrainedConfig):
class Qwen3_5MoeVisionConfig(Qwen3_5VisionConfig):
model_type = "qwen3_5_moe"
def __init__(self, **kwargs):
super().__init__(**kwargs)
class Qwen3_5MoeTextConfig(Qwen3_5TextConfig):
model_type = "qwen3_5_moe_text"
def __init__(self, **kwargs):
super().__init__(**kwargs)
# All Moe variant classes need explicit __init__ because the kw_only=True
# dataclass decorator in transformers v5.5.3+ auto-generates __init__ for
# subclasses, bypassing parent __init__ methods that set up attributes
# (e.g. norm_topk_prob, rope_scaling) and convert sub-config dicts to objects.
class Qwen3_5MoeConfig(Qwen3_5Config):
model_type = "qwen3_5_moe"
sub_configs = {
"vision_config": Qwen3_5MoeVisionConfig,
"text_config": Qwen3_5MoeTextConfig,
}
def __init__(self, **kwargs):
super().__init__(**kwargs)
python/sglang/srt/configs/step3p5.py
+9
View File
@@ -94,4 +94,13 @@ class Step3p5Config(PretrainedConfig):
self.moe_layers_enum = moe_layers_enum
self.layer_types = layer_types
self.sliding_window = sliding_window
# The upstream Step-3.5-Flash config has layer_types with 48 entries
# but num_hidden_layers=45. The extra 3 are for MTP/nextn predict
# layers (indices 45-47) used by Step3p5DecoderLayer during EAGLE
# speculative decoding. Temporarily align num_hidden_layers to pass
# the transformers v5.5.3+ validator, then restore the real value.
real_num_hidden_layers = self.num_hidden_layers
if layer_types is not None and len(layer_types) != self.num_hidden_layers:
self.num_hidden_layers = len(layer_types)
super().__init__(**kwargs)
self.num_hidden_layers = real_num_hidden_layers
python/sglang/srt/models/qwen3_vl.py
+7 -1
View File
@@ -1091,9 +1091,15 @@ class Qwen3VLForConditionalGeneration(nn.Module):
if language_model_cls is Qwen3LLMModel:
self.config: Qwen3VLConfig = config # for qwen3-vl
else:
self.config = config.text_config # for qwen3-omni
self.config = config.text_config # for qwen3-omni / qwen3-vl-moe
self.config.encoder_only = getattr(config, "encoder_only", False)
self.config.language_only = getattr(config, "language_only", False)
# Propagate tie_word_embeddings from parent config. In transformers
# v5.5.3+, Qwen3VLMoeTextConfig sets tie_word_embeddings=True by
# default but the actual model checkpoint has a separate lm_head.
# The parent Qwen3VLMoeConfig correctly has tie_word_embeddings=False.
if hasattr(config, "tie_word_embeddings"):
self.config.tie_word_embeddings = config.tie_word_embeddings
if not hasattr(config, "encoder_only") or not config.encoder_only:
self.model = language_model_cls(
python/sglang/srt/utils/hf_transformers/__init__.py
+67
View File
@@ -0,0 +1,67 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Hugging Face Transformers utilities.
This package provides HF Transformers helpers, split into submodules
(common, compat, config, tokenizer, processor, mistral_utils).
All public symbols are re-exported here for convenience. The old import
path ``sglang.srt.utils.hf_transformers_utils`` is preserved by a
separate shim module.
"""
from .compat import apply_all as _apply_compat
_apply_compat()
from .common import ( # noqa: E402
CONTEXT_LENGTH_KEYS,
AutoConfig,
attach_additional_stop_token_ids,
check_gguf_file,
download_from_hf,
get_context_length,
get_generation_config,
get_hf_text_config,
get_rope_config,
get_sparse_attention_config,
get_tokenizer_from_processor,
)
from .compat import normalize_rope_scaling_compat # noqa: E402
from .config import get_config # noqa: E402
from .processor import get_processor # noqa: E402
from .tokenizer import ( # noqa: E402
_fix_added_tokens_encoding,
_fix_v5_add_bos_eos_token,
get_tokenizer,
)
__all__ = [
"AutoConfig",
"CONTEXT_LENGTH_KEYS",
"_fix_added_tokens_encoding",
"_fix_v5_add_bos_eos_token",
"attach_additional_stop_token_ids",
"check_gguf_file",
"download_from_hf",
"get_config",
"get_context_length",
"get_generation_config",
"get_hf_text_config",
"get_processor",
"get_rope_config",
"get_sparse_attention_config",
"get_tokenizer",
"get_tokenizer_from_processor",
"normalize_rope_scaling_compat",
]
python/sglang/srt/utils/hf_transformers/common.py
+438
View File
@@ -0,0 +1,438 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Shared helpers used by config, tokenizer, and processor modules."""
import json
import os
from pathlib import Path
from typing import Any, Dict, Optional, Type, Union
import torch
from huggingface_hub import snapshot_download
from sglang.srt.configs import (
AfmoeConfig,
BailingHybridConfig,
ChatGLMConfig,
DbrxConfig,
DeepseekVL2Config,
DotsOCRConfig,
DotsVLMConfig,
ExaoneConfig,
FalconH1Config,
GraniteMoeHybridConfig,
JetNemotronConfig,
JetVLMConfig,
KimiK25Config,
KimiLinearConfig,
KimiVLConfig,
LongcatFlashConfig,
MultiModalityConfig,
NemotronH_Nano_VL_V2_Config,
NemotronHConfig,
Olmo3Config,
Qwen3_5Config,
Qwen3_5MoeConfig,
Qwen3NextConfig,
Step3p5Config,
Step3VLConfig,
)
from sglang.srt.configs.deepseek_ocr import DeepseekVLV2Config
from sglang.srt.configs.internvl import InternVLChatConfig
from sglang.srt.utils import get_bool_env_var, logger, lru_cache_frozenset
from .compat import normalize_rope_scaling_compat
if get_bool_env_var("SGLANG_USE_MODELSCOPE"):
from modelscope import AutoConfig, GenerationConfig
else:
from transformers import AutoConfig, GenerationConfig
from transformers import PretrainedConfig
# ---------------------------------------------------------------------------
# Config registry
# ---------------------------------------------------------------------------
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
cls.model_type: cls
for cls in [
AfmoeConfig,
BailingHybridConfig,
ChatGLMConfig,
DbrxConfig,
ExaoneConfig,
DeepseekVL2Config,
MultiModalityConfig,
KimiVLConfig,
InternVLChatConfig,
Step3VLConfig,
LongcatFlashConfig,
Olmo3Config,
KimiLinearConfig,
Qwen3NextConfig,
FalconH1Config,
GraniteMoeHybridConfig,
DotsVLMConfig,
DotsOCRConfig,
NemotronH_Nano_VL_V2_Config,
NemotronHConfig,
DeepseekVLV2Config,
Qwen3_5Config,
Qwen3_5MoeConfig,
JetNemotronConfig,
JetVLMConfig,
KimiK25Config,
Step3p5Config,
]
}
for name, cls in _CONFIG_REGISTRY.items():
try:
AutoConfig.register(name, cls)
except ValueError as e:
err = str(e).lower()
if "already registered" not in err and "already used" not in err:
logger.warning("Failed to register config %s: %s", name, e)
# ---------------------------------------------------------------------------
# Download / path helpers
# ---------------------------------------------------------------------------
def download_from_hf(
model_path: str,
allow_patterns: Optional[Union[str, list]] = None,
):
if os.path.exists(model_path):
return model_path
if not allow_patterns:
allow_patterns = ["*.json", "*.bin", "*.model"]
return snapshot_download(model_path, allow_patterns=allow_patterns)
def _resolve_local_or_cached_file(model_name_or_path, filename, revision=None):
"""Resolve a file from a local directory or HF hub cache (no network)."""
local_path = Path(model_name_or_path) / filename
if local_path.is_file():
return str(local_path)
from huggingface_hub import hf_hub_download
return hf_hub_download(
model_name_or_path, filename, revision=revision, local_files_only=True
)
def check_gguf_file(model: Union[str, os.PathLike]) -> bool:
model = Path(model)
if not model.is_file():
return False
elif model.suffix == ".gguf":
return True
with open(model, "rb") as f:
header = f.read(4)
return header == b"GGUF"
# ---------------------------------------------------------------------------
# Rope / text config helpers
# ---------------------------------------------------------------------------
def get_rope_config(config):
"""Get (rope_theta, rope_params) from config, supporting both v4 and v5.
Trust-remote-code configs or parent configs passed to sub-models may not
have the v5 ``rope_parameters`` property, so we fall back to the v4-style
``config.rope_theta`` / ``config.rope_scaling`` attributes.
Returns:
(rope_theta, rope_params): In v5, rope_params is the full
rope_parameters dict (which subsumes rope_scaling and includes
rope_theta). In v4, rope_params is the rope_scaling dict or None.
"""
rope_params = getattr(config, "rope_parameters", None)
if rope_params is not None:
return rope_params["rope_theta"], rope_params
return config.rope_theta, getattr(config, "rope_scaling", None)
def _patch_text_config(parent_config: PretrainedConfig, text_config):
"""Synchronize standard attributes between parent config and text sub-config.
In transformers v5, the "untangle config" refactor removed automatic
inheritance of top-level PretrainedConfig attributes (pad_token_id,
tie_word_embeddings, etc.) from sub-configs. Downstream code expects
these attributes to be present on both configs (some models pass the
parent directly to the language model, others pass the text sub-config),
so we propagate in both directions when an attribute is missing.
(See https://github.com/huggingface/transformers/pull/41541)
"""
_ATTRS_TO_PROPAGATE = [
"pad_token_id",
"bos_token_id",
"eos_token_id",
"tie_word_embeddings",
]
for attr in _ATTRS_TO_PROPAGATE:
parent_has = hasattr(parent_config, attr)
text_has = hasattr(text_config, attr)
if parent_has and not text_has:
setattr(text_config, attr, getattr(parent_config, attr))
elif text_has and not parent_has:
setattr(parent_config, attr, getattr(text_config, attr))
return text_config
def get_hf_text_config(config: PretrainedConfig):
"""Get the "sub" config relevant to llm for multi modal models.
No op for pure text models.
"""
if config.architectures is not None:
class_name = config.architectures[0]
if class_name.startswith("Llava") and class_name.endswith("ForCausalLM"):
# We support non-hf version of llava models, so we do not want to
# read the wrong values from the unused default text_config.
# NOTE(HandH1998): We set `torch_dtype` of config to `torch.float16` for the weights, as
# `torch.float16` is default used for image features in `python/sglang/srt/models/llava.py`.
setattr(config, "dtype", torch.float16)
return config
text_config = None
# Some models (e.g. DeepSeek-OCR) store sub-configs as plain dicts.
# Convert to PretrainedConfig early so hasattr() checks and asserts work.
parent_dtype = getattr(config, "torch_dtype", None)
for _attr in ("text_config", "llm_config", "language_config", "thinker_config"):
_sub = getattr(config, _attr, None)
if isinstance(_sub, dict):
_converted = PretrainedConfig(**_sub)
if (
getattr(_converted, "torch_dtype", None) is None
and parent_dtype is not None
):
_converted.torch_dtype = parent_dtype
setattr(config, _attr, _converted)
# Priority: thinker_config > llm_config > language_config > text_config
if hasattr(config, "thinker_config"):
# qwen2.5 omni
thinker_config = config.thinker_config
if hasattr(thinker_config, "text_config"):
setattr(
thinker_config.text_config,
"torch_dtype",
getattr(thinker_config, "torch_dtype", None),
)
text_config = thinker_config.text_config
else:
text_config = thinker_config
elif hasattr(config, "llm_config"):
# PointsV1.5 Chat Model
assert hasattr(config.llm_config, "num_attention_heads")
text_config = config.llm_config
elif hasattr(config, "language_config"):
text_config = config.language_config
elif hasattr(config, "text_config"):
# The code operates under the assumption that text_config should have
# `num_attention_heads` (among others). Assert here to fail early
# if transformers config doesn't align with this assumption.
assert hasattr(config.text_config, "num_attention_heads")
text_config = config.text_config
# Ensure rope_scaling dicts have "type" for remote-code compat (v5).
normalize_rope_scaling_compat(config)
if text_config is not None:
return _patch_text_config(config, text_config)
return config
# ---------------------------------------------------------------------------
# Model-specific helpers
# ---------------------------------------------------------------------------
def _ensure_sub_configs(config: PretrainedConfig, *attr_names: str) -> None:
"""Convert dict-valued sub-configs to proper AutoConfig objects in-place."""
for attr in attr_names:
sub = getattr(config, attr, None)
if sub is not None and isinstance(sub, dict):
setattr(config, attr, AutoConfig.for_model(**sub))
def _is_deepseek_ocr_model(config: PretrainedConfig) -> bool:
# TODO: Remove this workaround once AutoConfig correctly identifies deepseek-ocr.
# Hugging Face's AutoConfig currently misidentifies it as deepseekvl2.
auto_map = getattr(config, "auto_map", None) or {}
return auto_map.get("AutoModel") == "modeling_deepseekocr.DeepseekOCRForCausalLM"
def _is_deepseek_ocr2_model(config: PretrainedConfig) -> bool:
auto_map = getattr(config, "auto_map", None) or {}
return auto_map.get("AutoModel") == "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"
def _override_v_head_dim_if_zero(config: PretrainedConfig, patch: int = 128) -> None:
patched = False
for attr in ("text_config", "language_config"):
sub = getattr(config, attr, None)
if sub is None:
continue
if isinstance(sub, dict):
if sub.get("v_head_dim") == 0:
sub["v_head_dim"] = patch
patched = True
elif getattr(sub, "v_head_dim", None) == 0:
sub.v_head_dim = patch
patched = True
if patched:
logger.warning(
f"Overriding v_head_dim from 0 to {patch} to avoid potential issues."
)
def _load_deepseek_v32_model(
model_path: str,
trust_remote_code: bool = False,
revision: Optional[str] = None,
**kwargs,
):
import tempfile
local_path = download_from_hf(model_path)
config_file = os.path.join(local_path, "config.json")
if not os.path.exists(config_file):
raise RuntimeError(f"Can't find config file in {local_path}.")
with open(config_file, "r") as f:
config_json = json.load(f)
config_json["architectures"] = ["DeepseekV3ForCausalLM"]
config_json["model_type"] = "deepseek_v3"
tmp_path = os.path.join(tempfile.gettempdir(), "_tmp_config_folder")
os.makedirs(tmp_path, exist_ok=True)
unique_path = os.path.join(tmp_path, f"deepseek_v32_{os.getpid()}")
with open(unique_path, "w") as f:
json.dump(config_json, f)
return AutoConfig.from_pretrained(
unique_path, trust_remote_code=trust_remote_code, revision=revision, **kwargs
)
# ---------------------------------------------------------------------------
# Context length / generation config / sparse attention
# ---------------------------------------------------------------------------
# Models don't use the same configuration key for determining the maximum
# context length. Store them here so we can sanely check them.
# NOTE: The ordering here is important. Some models have two of these and we
# have a preference for which value gets used.
CONTEXT_LENGTH_KEYS = [
"max_sequence_length",
"seq_length",
"max_seq_len",
"model_max_length",
"max_position_embeddings",
]
def get_context_length(config):
"""Get the context length of a model from a huggingface model configs."""
text_config = config
rope_scaling = getattr(text_config, "rope_scaling", None)
if rope_scaling:
rope_scaling_factor = rope_scaling.get("factor", 1)
if "original_max_position_embeddings" in rope_scaling:
rope_scaling_factor = 1
if rope_scaling.get("rope_type", None) == "llama3":
rope_scaling_factor = 1
else:
rope_scaling_factor = 1
for key in CONTEXT_LENGTH_KEYS:
val = getattr(text_config, key, None)
if val is not None:
return int(rope_scaling_factor * val)
return 2048
@lru_cache_frozenset(maxsize=32)
def get_generation_config(
model: str,
trust_remote_code: bool,
revision: Optional[str] = None,
**kwargs,
):
try:
return GenerationConfig.from_pretrained(
model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
)
except FileNotFoundError:
return None
except OSError as e:
logger.warning(
"Failed to load generation config for %s: %s. "
"Proceeding without generation config.",
model,
e,
)
return None
# Qwen-1M related
def get_sparse_attention_config(
model: str,
sparse_attention_config_filename: str = "sparse_attention_config.json",
) -> Dict[str, Any]:
is_local = os.path.isdir(model)
if not is_local:
model = download_from_hf(model, allow_patterns=["*.json"])
config_file = os.path.join(model, sparse_attention_config_filename)
if not os.path.exists(config_file):
return {}
with open(config_file) as f:
config = json.load(f)
return config
# ---------------------------------------------------------------------------
# Tokenizer / processor helpers
# ---------------------------------------------------------------------------
# Some models don't have an available processor, e.g.: InternVL
def get_tokenizer_from_processor(processor):
from transformers import PreTrainedTokenizerBase
if isinstance(processor, PreTrainedTokenizerBase):
return processor
return processor.tokenizer
def attach_additional_stop_token_ids(tokenizer):
added = tokenizer.get_added_vocab()
if "<|eom_id|>" in added:
tokenizer.additional_stop_token_ids = {added["<|eom_id|>"]}
else:
tokenizer.additional_stop_token_ids = None
python/sglang/srt/utils/hf_transformers/compat.py
+458
View File
@@ -0,0 +1,458 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Compatibility patches for transformers v5.x.
This module applies monkey-patches to work around breaking changes in
transformers v5. Each patch is tagged with the upstream issue it works
around so it can be removed once the upstream fix lands.
Import this module early (before any ``from_pretrained`` call) to activate
all patches. It is safe to import multiple times -- patches are idempotent.
Patches fall into two categories:
1. **Transformers bugs / regressions** -- issues in transformers itself.
2. **Remote-model-code compat** -- remote model code (trust_remote_code)
that hasn't been updated for v5 yet. These should be removed once
the model authors publish fixes.
"""
import inspect
from sglang.srt.utils import logger
_applied = False
# ---------------------------------------------------------------------------
# Public API: apply_all() -- import-time patches (idempotent)
# ---------------------------------------------------------------------------
def apply_all():
"""Apply all transformers compatibility patches (idempotent).
Call this once at import time. It is safe to call multiple times.
"""
global _applied
if _applied:
return
_applied = True
# v5.4 patches
_patch_flash_attn_availability()
_patch_rope_parameters_validation()
_patch_removed_symbols()
_patch_image_processor_kwargs()
_patch_image_process_cuda_tensor()
_patch_nemotron_h_pattern()
# v5 general patches
_ensure_clean_up_tokenization_compat()
_ensure_is_torch_fx_available_compat()
logger.debug("transformers compatibility patches applied")
# ---------------------------------------------------------------------------
# Public API: on-demand helpers (called explicitly by other modules)
# ---------------------------------------------------------------------------
def normalize_rope_scaling_compat(config) -> None:
"""Ensure rope_scaling dicts have ``"type"`` alongside ``"rope_type"``.
Transformers v5 standardises rope_scaling to use ``"rope_type"`` and may
omit the legacy ``"type"`` key. Remote-code models (e.g. Kimi-VL) still
read ``rope_scaling["type"]``, causing a ``KeyError``. This helper adds
``"type"`` from ``"rope_type"`` whenever it is missing, recursively across
the config and all its sub-configs.
"""
def _patch(cfg):
rs = getattr(cfg, "rope_scaling", None)
if isinstance(rs, dict) and "rope_type" in rs and "type" not in rs:
rs["type"] = rs["rope_type"]
# Recurse into sub-configs
for attr in (
"text_config",
"llm_config",
"language_config",
"vision_config",
"thinker_config",
):
sub = getattr(cfg, attr, None)
if sub is not None:
_patch(sub)
_patch(config)
def _ensure_gguf_version():
"""Workaround for transformers v5 bug where is_gguf_available() fails
when the gguf package lacks __version__ and metadata lookup also fails,
resulting in packaging.version.InvalidVersion: Invalid version: 'N/A'."""
try:
import gguf
if not hasattr(gguf, "__version__"):
import importlib.metadata
try:
gguf.__version__ = importlib.metadata.version("gguf")
except importlib.metadata.PackageNotFoundError:
gguf.__version__ = "0.0.0"
except (ValueError, OSError, TypeError) as e:
logger.warning(
"Failed to determine gguf package version: %s. "
"Falling back to '0.0.0'.",
e,
)
gguf.__version__ = "0.0.0"
except ImportError:
pass
# ---------------------------------------------------------------------------
# v5.4 patches (merged from transformers_v54_compat.py)
# ---------------------------------------------------------------------------
def _patch_rope_parameters_validation():
"""Fix rope_parameters validation for unregistered model types.
For unregistered model types (e.g. ``deepseek_v32``), the generic
``PretrainedConfig`` lacks a ``rope_parameters`` field so the conversion
that injects ``rope_theta`` from the top-level config is skipped.
Additionally, ``standardize_rope_params()`` accesses
``self.max_position_embeddings`` during ``__post_init__`` before extra
kwargs are set as attributes, causing ``AttributeError``.
Fix: (1) patch ``from_dict`` to inject ``rope_theta`` into
``rope_scaling``, (2) guard ``standardize_rope_params`` against missing
``max_position_embeddings``.
TODO(upstream): remove once unregistered model types handle rope
standardization correctly in transformers.
"""
from transformers import PretrainedConfig
original = PretrainedConfig.from_dict.__func__
@classmethod # type: ignore[misc]
def patched(cls, config_dict, **kwargs):
rope_scaling = config_dict.get("rope_scaling")
rope_theta = config_dict.get("rope_theta")
if (
isinstance(rope_scaling, dict)
and rope_theta is not None
and "rope_theta" not in rope_scaling
):
config_dict = config_dict.copy()
config_dict["rope_scaling"] = {**rope_scaling, "rope_theta": rope_theta}
return original(cls, config_dict, **kwargs)
PretrainedConfig.from_dict = patched
# standardize_rope_params accesses self.max_position_embeddings before
# __post_init__ sets extra kwargs — skip when the attribute is absent.
if hasattr(PretrainedConfig, "standardize_rope_params"):
_orig_standardize = PretrainedConfig.standardize_rope_params
def _safe_standardize(self):
if not hasattr(self, "max_position_embeddings"):
return
return _orig_standardize(self)
PretrainedConfig.standardize_rope_params = _safe_standardize
def _patch_flash_attn_availability():
"""Prevent flash-attn-4 from masquerading as flash-attn-2.
flash-attn-4 registers a bare ``flash_attn`` namespace that makes
``is_flash_attn_2_available()`` return True, but lacks the v2 API.
Remote model code (e.g. Kimi-VL) guarded by that check will crash.
TODO(upstream): model authors should check for specific API symbols.
"""
try:
import flash_attn as _fa
if not hasattr(_fa, "flash_attn_func"):
import transformers.utils as _u
import transformers.utils.import_utils as _ui
_ui.is_flash_attn_2_available = lambda: False
_u.is_flash_attn_2_available = lambda: False
except ImportError:
pass
def _patch_removed_symbols():
"""Re-export symbols removed in transformers v5.4.0.
Remote model code (e.g. DeepSeek-OCR) still imports these.
``check_imports`` in ``dynamic_module_utils.py`` validates imports at
config-load time, so these must exist before any ``from_pretrained``.
Removed symbols:
- ``LlamaFlashAttention2`` -- replaced by unified ``LlamaAttention``
- ``is_flash_attn_greater_or_equal_2_10`` -- replaced by
``is_flash_attn_greater_or_equal("2.10.0")``
TODO(upstream): DeepSeek-OCR / deepseek_vl_v2 remote code needs update.
"""
# LlamaFlashAttention2
try:
from transformers.models.llama import modeling_llama
if not hasattr(modeling_llama, "LlamaFlashAttention2"):
if hasattr(modeling_llama, "LlamaAttention"):
modeling_llama.LlamaFlashAttention2 = modeling_llama.LlamaAttention
except ImportError:
logger.warning(
"Could not import transformers.models.llama.modeling_llama; "
"LlamaFlashAttention2 compat patch not applied."
)
# is_flash_attn_greater_or_equal_2_10
try:
import transformers.utils as _u
if not hasattr(_u, "is_flash_attn_greater_or_equal_2_10"):
if hasattr(_u, "is_flash_attn_greater_or_equal"):
_u.is_flash_attn_greater_or_equal_2_10 = (
lambda: _u.is_flash_attn_greater_or_equal("2.10.0")
)
else:
_u.is_flash_attn_greater_or_equal_2_10 = lambda: False
except ImportError:
logger.warning(
"Could not import transformers.utils; "
"is_flash_attn_greater_or_equal_2_10 compat patch not applied."
)
def _patch_image_processor_kwargs():
"""Allow remote image processors that lack ``**kwargs`` in preprocess().
Transformers v5.4 passes new kwargs (e.g. ``device``) through
``BaseImageProcessor.__call__`` -> ``preprocess()``. Remote model code
(e.g. KimiVL) that defines ``preprocess()`` without ``**kwargs`` will
crash with ``TypeError``.
Fix: wrap ``__call__`` to catch ``TypeError`` and retry with only the
kwargs that ``preprocess()`` actually accepts.
TODO(upstream): KimiVL image_processing_kimi_vl.py needs ``**kwargs``.
"""
try:
from transformers.image_processing_utils import BaseImageProcessor
original = BaseImageProcessor.__call__
def safe_call(self, images, *args, **kwargs):
try:
return original(self, images, *args, **kwargs)
except TypeError as e:
if "unexpected keyword argument" not in str(e):
raise
sig = inspect.signature(self.preprocess)
params = sig.parameters
if any(
p.kind == inspect.Parameter.VAR_KEYWORD for p in params.values()
):
raise
dropped = {k for k in kwargs if k not in params}
if dropped:
logger.warning(
"Image processor %s.preprocess() does not accept %s; "
"retrying without them. Update the model's image processor "
"to accept **kwargs.",
type(self).__name__,
dropped,
)
valid = {k: v for k, v in kwargs.items() if k in params}
return original(self, images, *args, **valid)
BaseImageProcessor.__call__ = safe_call
except ImportError:
logger.debug(
"_patch_image_processor_kwargs: BaseImageProcessor not importable, patch skipped"
)
def _patch_image_process_cuda_tensor():
"""Fix ``process_image()`` crashing on CUDA tensors.
Transformers v5.4's PIL image processing backend calls
``image.numpy()`` on torch tensors, which fails for CUDA tensors.
Patch to call ``.cpu().numpy()`` instead.
TODO(upstream): report to HF transformers.
"""
try:
import torch
import transformers.image_processing_backends as ipb
for cls_name in ("PilBackend", "PilImageProcessingMixin"):
cls = getattr(ipb, cls_name, None)
if cls is None or not hasattr(cls, "process_image"):
continue
original = cls.process_image
def patched_process_image(
self, image, *args, _orig=original, _Tensor=torch.Tensor, **kwargs
):
if isinstance(image, _Tensor) and image.is_cuda:
image = image.cpu()
return _orig(self, image, *args, **kwargs)
cls.process_image = patched_process_image
except ImportError:
logger.debug(
"_patch_image_process_cuda_tensor: required modules not importable, patch skipped"
)
def _patch_nemotron_h_pattern():
"""Fix ``_pattern_to_list()`` crashing on ``-`` in hybrid_override_pattern.
Nemotron-H models (e.g. NVIDIA-Nemotron-Nano-9B-v2) use patterns like
``M-M-M-MM-M-*-...`` where ``-`` denotes an MLP layer. The upstream
``_pattern_to_list`` tries to map every character and crashes with
``KeyError: '-'``. We skip ``-`` (and any other unmapped chars)
since ``layers_block_type`` only tracks mamba/moe/attention layers.
SGLang reads MLP positions from ``hybrid_override_pattern`` directly.
TODO(upstream): report to HF transformers.
"""
try:
from transformers.models.nemotron_h.configuration_nemotron_h import (
NemotronHConfig,
)
@staticmethod
def _pattern_to_list(pattern: str) -> list:
pattern_mapping = {
"M": "mamba",
"E": "moe",
"*": "attention",
}
return [
pattern_mapping[char] for char in pattern if char in pattern_mapping
]
NemotronHConfig._pattern_to_list = _pattern_to_list
except ImportError:
logger.debug(
"_patch_nemotron_h_pattern: NemotronHConfig not importable, patch skipped"
)
# ---------------------------------------------------------------------------
# v5 general patches
# ---------------------------------------------------------------------------
def _ensure_clean_up_tokenization_compat() -> None:
"""Re-add ``clean_up_tokenization`` removed in transformers v5.
Remote-code tokenizers (e.g. InternLM2Tokenizer) call
``self.clean_up_tokenization()`` which was a static method on
``PreTrainedTokenizerBase`` in v4 but removed in v5. Patch it back
so existing HuggingFace Hub tokenizer code keeps working.
"""
from transformers import PreTrainedTokenizerBase
if hasattr(PreTrainedTokenizerBase, "clean_up_tokenization"):
return
@staticmethod
def clean_up_tokenization(out_string: str) -> str:
out_string = (
out_string.replace(" .", ".")
.replace(" ?", "?")
.replace(" !", "!")
.replace(" ,", ",")
.replace(" ' ", "'")
.replace(" n't", "n't")
.replace(" 'm", "'m")
.replace(" 's", "'s")
.replace(" 've", "'ve")
.replace(" 're", "'re")
)
return out_string
PreTrainedTokenizerBase.clean_up_tokenization = clean_up_tokenization
def _ensure_is_torch_fx_available_compat() -> None:
"""Re-add ``is_torch_fx_available`` removed in transformers v5.
Remote-code models (e.g. MiniCPM-V) import ``is_torch_fx_available``
from ``transformers.utils.import_utils``. The function was removed
in v5. Patch it back so existing HuggingFace Hub model code keeps
working. torch.fx is always available in PyTorch >= 2.0.
"""
import transformers.utils.import_utils as _import_utils
if hasattr(_import_utils, "is_torch_fx_available"):
return
_import_utils.is_torch_fx_available = lambda: True
# ---------------------------------------------------------------------------
# CI-only patches
# ---------------------------------------------------------------------------
_is_base_mistral_patched = False
def patch_is_base_mistral_in_ci():
"""Patch transformers' _patch_mistral_regex to avoid HF API calls in CI.
transformers defines is_base_mistral as a local function inside
_patch_mistral_regex, so it cannot be patched via module attribute.
Instead we replace the entire _patch_mistral_regex classmethod with a
version that simply returns the tokenizer unchanged.
In CI this prevents exhausting the 3000 req/5min HF API rate limit.
TODO(upstream): remove once transformers stops calling model_info()
inside _patch_mistral_regex (or removes the method entirely).
"""
global _is_base_mistral_patched
if _is_base_mistral_patched:
return
from sglang.srt.environ import envs
if not envs.SGLANG_IS_IN_CI.get():
return
from transformers import PreTrainedTokenizerFast
if hasattr(PreTrainedTokenizerFast, "_patch_mistral_regex"):
@classmethod
def _noop_patch_mistral_regex(cls, tokenizer, *args, **kwargs):
return tokenizer
PreTrainedTokenizerFast._patch_mistral_regex = _noop_patch_mistral_regex
logger.info("CI: patched _patch_mistral_regex to skip HF API calls")
_is_base_mistral_patched = True
python/sglang/srt/utils/hf_transformers/config.py
+212
View File
@@ -0,0 +1,212 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Config loading utilities."""
from pathlib import Path
from typing import Optional
from transformers import PretrainedConfig
from transformers.models.auto.modeling_auto import MODEL_FOR_CAUSAL_LM_MAPPING_NAMES
from sglang.srt.connector import create_remote_connector
from sglang.srt.utils import is_remote_url, logger, lru_cache_frozenset
from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
from .common import (
_CONFIG_REGISTRY,
AutoConfig,
DeepseekVLV2Config,
_is_deepseek_ocr2_model,
_is_deepseek_ocr_model,
_load_deepseek_v32_model,
_override_v_head_dim_if_zero,
check_gguf_file,
get_hf_text_config,
)
from .compat import _ensure_gguf_version
from .mistral_utils import is_mistral_model, load_mistral_config
def _set_architectures(config, arch_name):
config.update({"architectures": [arch_name]})
def _apply_deepseek_ocr_overrides(config, model):
_override_v_head_dim_if_zero(config)
_set_architectures(config, "DeepseekOCRForCausalLM")
config._name_or_path = model
@lru_cache_frozenset(maxsize=32)
def get_config(
model: str,
trust_remote_code: bool,
revision: Optional[str] = None,
model_override_args: Optional[dict] = None,
**kwargs,
):
is_gguf = check_gguf_file(model)
if is_gguf:
_ensure_gguf_version()
kwargs["gguf_file"] = model
model = Path(model).parent
if is_runai_obj_uri(model):
model = ObjectStorageModel.get_path(model)
if is_remote_url(model):
client = create_remote_connector(model)
client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
model = client.get_local_dir()
if is_mistral_model(model):
config = load_mistral_config(
model, trust_remote_code=trust_remote_code, revision=revision
)
else:
try:
config = AutoConfig.from_pretrained(
model, trust_remote_code=trust_remote_code, revision=revision, **kwargs
)
except (ValueError, KeyError) as e:
if "deepseek_v32" in str(e):
config = _load_deepseek_v32_model(
model,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
elif isinstance(e, ValueError):
raise
else:
logger.warning(
"AutoConfig.from_pretrained raised KeyError for %s: %s. "
"Falling back to config registry lookup.",
model,
e,
)
config_dict, _ = PretrainedConfig.get_config_dict(
model,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
model_type = config_dict.get("model_type")
if model_type in _CONFIG_REGISTRY:
config = _CONFIG_REGISTRY[model_type].from_dict(config_dict)
config._name_or_path = model
else:
raise
if (
config.architectures is not None
and config.architectures[0] == "Phi4MMForCausalLM"
):
from transformers import SiglipVisionConfig
config.vision_config = SiglipVisionConfig(
hidden_size=1152,
image_size=448,
intermediate_size=4304,
model_type="siglip_vision_model",
num_attention_heads=16,
num_hidden_layers=26,
patch_size=14,
)
if config.architectures in [
["LongcatCausalLM"],
["LongcatFlashForCausalLM"],
["LongcatFlashNgramForCausalLM"],
]:
config.model_type = "longcat_flash"
text_config = get_hf_text_config(config=config)
if isinstance(model, str) and text_config is not None:
items = (
text_config.items()
if hasattr(text_config, "items")
else vars(text_config).items()
)
for key, val in items:
if not hasattr(config, key) and val is not None:
setattr(config, key, val)
is_ocr = _is_deepseek_ocr_model(config)
is_ocr2 = _is_deepseek_ocr2_model(config)
if is_ocr2:
_override_v_head_dim_if_zero(config)
config.model_type = "deepseek-ocr"
_set_architectures(config, "DeepseekOCRForCausalLM")
config = DeepseekVLV2Config.from_pretrained(model, revision=revision)
_apply_deepseek_ocr_overrides(config, model)
elif config.model_type in _CONFIG_REGISTRY:
model_type = config.model_type
if model_type == "deepseek_vl_v2" and is_ocr:
model_type = "deepseek-ocr"
config = _CONFIG_REGISTRY[model_type].from_pretrained(model, revision=revision)
# Re-check after reloading config from registry
if _is_deepseek_ocr_model(config) or _is_deepseek_ocr2_model(config):
_apply_deepseek_ocr_overrides(config, model)
else:
config._name_or_path = model
if isinstance(model, str) and config.model_type == "internvl_chat":
for key, val in config.llm_config.__dict__.items():
if not hasattr(config, key):
setattr(config, key, val)
if config.model_type == "multi_modality":
_set_architectures(config, "MultiModalityCausalLM")
if config.model_type == "gemma4":
# Gemma4 configs use base attributes for SWA layers and `global_*`
# variants for full-attention layers. SGLang expects the opposite:
# base = full-attention, `swa_*` = sliding-window overrides.
text_config = config.text_config
global_head_dim = getattr(text_config, "global_head_dim", None)
global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
swa_head_dim = text_config.head_dim
swa_kv_heads = text_config.num_key_value_heads
text_config.swa_head_dim = swa_head_dim
text_config.swa_v_head_dim = swa_head_dim
text_config.swa_num_key_value_heads = swa_kv_heads
if global_head_dim is not None:
text_config.head_dim = global_head_dim
if global_kv_heads is not None:
text_config.num_key_value_heads = global_kv_heads
if not hasattr(text_config, "v_head_dim"):
text_config.v_head_dim = text_config.head_dim
if not hasattr(text_config, "swa_v_head_dim"):
text_config.swa_v_head_dim = text_config.swa_head_dim
if config.model_type == "longcat_flash":
_set_architectures(config, "LongcatFlashForCausalLM")
if model_override_args:
config.update(model_override_args)
if is_gguf:
if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
_set_architectures(config, MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type])
return config
python/sglang/srt/utils/mistral_utils.py → python/sglang/srt/utils/hf_transformers/mistral_utils.py
+176 -22
View File
@@ -1,13 +1,17 @@
# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/transformers_utils/configs/mistral.py
# SPDX-License-Identifier: Apache-2.0
import json
import tempfile
from functools import lru_cache
from pathlib import Path
from typing import Any
from typing import Any, Optional
from transformers import PretrainedConfig, WhisperConfig
from transformers import AutoConfig, PretrainedConfig, WhisperConfig
from sglang.srt.utils import logger
from .common import _ensure_sub_configs, download_from_hf
def adapt_config_dict(
config_dict: dict[str, Any], model: str, **kwargs
@@ -73,18 +77,6 @@ def adapt_config_dict(
if bool(config_dict.get("yarn")):
config_dict = _remap_mistral_yarn_args(config_dict)
if bool(config_dict.get("llama_4_scaling")):
llama_4_scaling_config_keys = ["original_max_position_embeddings", "beta"]
assert all(
[
key in config_dict["llama_4_scaling"]
for key in llama_4_scaling_config_keys
]
), (
"llama_4_scaling config should define the keys: "
f"{','.join(llama_4_scaling_config_keys)}"
)
is_vision = bool(
(config_dict.get("multimodal") or {}).get("vision_encoder_args")
or config_dict.get("vision_encoder")
@@ -267,14 +259,10 @@ class MistralConfigParser:
):
file_path = Path(model) / file_name
if not file_path.is_file():
# TODO: Add logic to download from HF in case file is not locally found
raise FileNotFoundError(f"File not found {model}, {file_name}")
if file_path is not None and file_path.is_file():
with open(file_path) as file:
return json.load(file)
return None
with open(file_path) as file:
return json.load(file)
def _download_mistral_config_file(self, model, revision) -> dict:
config_file_name = "params.json"
@@ -294,8 +282,6 @@ class MistralConfigParser:
revision: str | None = None,
**kwargs,
) -> tuple[dict, PretrainedConfig]:
# This function loads a params.json config which
# should be used when loading models in mistral format
config_dict = self._download_mistral_config_file(model, revision)
if config_dict.get("max_position_embeddings") is None:
logger.warning(
@@ -321,3 +307,171 @@ class MistralConfigParser:
config.sliding_window = next(filter(None, sliding_window), None)
return config_dict, config
def is_mistral_model(name) -> bool:
"""Return True if *name* refers to a Mistral model needing the custom parser."""
lower = str(name).lower()
return (
"mistral-large-3" in lower or "mistral-small-4" in lower or "leanstral" in lower
)
@lru_cache(maxsize=2)
def load_mistral_config(
model_path: str,
trust_remote_code: bool = False,
revision: Optional[str] = None,
):
"""Load and parse a Mistral model config via the custom params.json format.
Returns a ``PretrainedConfig`` with dict sub-configs (text_config,
vision_config) converted to proper AutoConfig objects.
"""
local_path = download_from_hf(model_path)
parser = MistralConfigParser()
config_dict, _ = parser.parse(local_path)
with tempfile.NamedTemporaryFile(mode="w+", suffix=".json") as f:
json.dump(config_dict, f)
f.flush()
loaded_config = AutoConfig.from_pretrained(
f.name, trust_remote_code=trust_remote_code, revision=revision
)
_ensure_sub_configs(loaded_config, "text_config", "vision_config")
return loaded_config
def wrap_as_pixtral(processor, config):
"""Wrap a tokenizer as a PixtralProcessor for Mistral vision models."""
from transformers.models.pixtral.image_processing_pixtral import (
PixtralImageProcessor,
)
from transformers.models.pixtral.processing_pixtral import (
PixtralProcessor as HFPixtralProcessor,
)
vision_config = config.vision_config
patch_size = vision_config.patch_size
image_size = vision_config.image_size
spatial_merge_size = getattr(vision_config, "spatial_merge_size", 1)
effective_patch = patch_size * spatial_merge_size
image_processor = PixtralImageProcessor(
do_resize=True,
size={"longest_edge": image_size},
patch_size={"height": effective_patch, "width": effective_patch},
)
return HFPixtralProcessor(
image_processor=image_processor,
tokenizer=processor,
patch_size=patch_size,
spatial_merge_size=spatial_merge_size,
)
# kwargs that MistralCommon tokenizers reject.
_MISTRAL_COMMON_REJECTED_KWARGS = frozenset(
{
"trust_remote_code",
"tokenizer_revision",
"use_fast",
"_from_auto",
"clean_up_tokenization_spaces",
}
)
# Models whose tokenizer should be loaded from a different checkpoint.
_MISTRAL_TOKENIZER_REDIRECTS = {
# TODO(Xinyuan): Remove this once we have a proper tokenizer for Devstral
"mistralai/Devstral-Small-2505": "mistralai/Mistral-Small-3.1-24B-Instruct-2503",
}
def retry_without_mistral_common_kwargs(tokenizer_name, *args, **common_kwargs):
"""Retry ``AutoTokenizer.from_pretrained`` without kwargs that MistralCommon rejects.
Returns the loaded tokenizer, or *None* if the error is not a
MistralCommon kwargs rejection.
"""
from transformers import AutoTokenizer
stripped = {
k: v
for k, v in common_kwargs.items()
if k not in _MISTRAL_COMMON_REJECTED_KWARGS
}
return AutoTokenizer.from_pretrained(tokenizer_name, *args, **stripped)
def patch_mistral_common_tokenizer(tokenizer):
"""Patch MistralCommonTokenizer/Backend to be compatible with HF tokenizer API.
MistralCommon tokenizers (used by Voxtral, Pixtral, etc.) reject several
standard kwargs and lack some attributes that sglang expects. We wrap the
offending methods once at load time so that the rest of the codebase does
not need any special-casing.
"""
cls_name = type(tokenizer).__name__
if "MistralCommon" not in cls_name:
return tokenizer
if getattr(tokenizer, "_mistral_common_patched", False):
return tokenizer
tokenizer._mistral_common_patched = True
if not hasattr(tokenizer, "get_added_vocab"):
tokenizer.get_added_vocab = lambda: {}
# Set a chat_template containing "audio" so that sglang's content format
# detector returns "openai" (which preserves audio_url extraction).
if not hasattr(tokenizer, "chat_template") or tokenizer.chat_template is None:
tokenizer.chat_template = "<!-- audio/image multimodal -->"
_orig_convert = tokenizer.convert_tokens_to_ids
def _safe_convert(val):
try:
return _orig_convert(val)
except AssertionError:
logger.debug(
"convert_tokens_to_ids failed for %r, returning unk_token_id", val
)
return getattr(tokenizer, "unk_token_id", None)
tokenizer.convert_tokens_to_ids = _safe_convert
def _drop_kwargs(fn, keys):
def wrapper(*args, **kwargs):
for k in keys:
kwargs.pop(k, None)
return fn(*args, **kwargs)
return wrapper
tokenizer.decode = _drop_kwargs(tokenizer.decode, ["spaces_between_special_tokens"])
tokenizer.batch_decode = _drop_kwargs(
tokenizer.batch_decode, ["spaces_between_special_tokens"]
)
tokenizer._orig_apply_chat_template = tokenizer.apply_chat_template
def _safe_apply_chat_template(messages, **kwargs):
kwargs.pop("add_generation_prompt", None)
cleaned = []
for msg in messages:
if isinstance(msg, dict):
content = msg.get("content", "")
if isinstance(content, list):
text_parts = [
p.get("text", "")
for p in content
if isinstance(p, dict) and p.get("type") == "text"
]
msg = {**msg, "content": " ".join(text_parts) if text_parts else ""}
cleaned.append(msg)
else:
cleaned.append(msg)
return tokenizer._orig_apply_chat_template(cleaned, **kwargs)
tokenizer.apply_chat_template = _safe_apply_chat_template
python/sglang/srt/utils/hf_transformers/processor.py
+288
View File
@@ -0,0 +1,288 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Processor loading utilities."""
import json
from pathlib import Path
from typing import Optional
from transformers import (
AutoProcessor,
AutoTokenizer,
PreTrainedTokenizerBase,
)
from sglang.srt.multimodal.customized_mm_processor_utils import _CUSTOMIZED_MM_PROCESSOR
from sglang.srt.utils import logger
from .common import (
AutoConfig,
_is_deepseek_ocr2_model,
_is_deepseek_ocr_model,
_override_v_head_dim_if_zero,
_resolve_local_or_cached_file,
attach_additional_stop_token_ids,
download_from_hf,
get_tokenizer_from_processor,
)
from .mistral_utils import (
is_mistral_model,
load_mistral_config,
patch_mistral_common_tokenizer,
wrap_as_pixtral,
)
from .tokenizer import (
_TOKENIZERS_BACKEND,
_fix_added_tokens_encoding,
_fix_special_tokens_pattern,
)
def _build_processor_manually(
model_path, config, trust_remote_code, revision, **kwargs
):
"""Build processor when AutoProcessor fails to resolve feature_extractor_type.
In transformers v5, AutoProcessor.from_pretrained calls
AutoFeatureExtractor.from_pretrained which fails if
preprocessor_config.json lacks 'feature_extractor_type'. This resolves
the processor class via dynamic module resolution and constructs it with
individually-loaded components.
"""
import transformers
from transformers import AutoImageProcessor, AutoTokenizer
from transformers.dynamic_module_utils import get_class_from_dynamic_module
# Resolve processor class from auto_map -- check both the model config
# and the preprocessor_config.json (some models like MiniCPM-o only
# declare AutoProcessor in the latter).
auto_map = getattr(config, "auto_map", None) or {}
proc_ref = auto_map.get("AutoProcessor")
if not proc_ref:
try:
pp_file = _resolve_local_or_cached_file(
model_path, "preprocessor_config.json", revision
)
with open(pp_file) as f:
pp_auto_map = json.load(f).get("auto_map", {})
proc_ref = pp_auto_map.get("AutoProcessor")
except (OSError, json.JSONDecodeError, ValueError) as e:
logger.warning(
"_build_processor_manually: could not read preprocessor_config.json "
"for %s: %s",
model_path,
e,
)
if not proc_ref:
raise ValueError(f"Cannot determine processor class for {model_path}")
proc_cls = get_class_from_dynamic_module(
proc_ref, model_path, code_revision=revision
)
# Load sub-components individually (these succeed)
tokenizer = AutoTokenizer.from_pretrained(
model_path, trust_remote_code=trust_remote_code, revision=revision
)
init_kwargs = {"tokenizer": tokenizer}
if "image_processor" in getattr(proc_cls, "attributes", []):
try:
init_kwargs["image_processor"] = AutoImageProcessor.from_pretrained(
model_path, trust_remote_code=trust_remote_code, revision=revision
)
except (ImportError, OSError, ValueError) as e:
raise RuntimeError(
f"Failed to load image_processor for {model_path}: {e}. "
f"This model requires an image processor for multimodal features. "
f"Check that the model files are complete and accessible."
) from e
# Instantiate feature extractor from its declared class
fe_class_name = getattr(proc_cls, "feature_extractor_class", None)
if fe_class_name:
fe_class = getattr(transformers, fe_class_name, None)
if fe_class is not None:
try:
init_kwargs["feature_extractor"] = fe_class()
except TypeError as e:
logger.warning(
"Cannot instantiate feature extractor %s with no arguments "
"for %s: %s",
fe_class_name,
model_path,
e,
)
else:
logger.warning(
"Feature extractor class %s not found in transformers for %s",
fe_class_name,
model_path,
)
return proc_cls(**init_kwargs)
def get_processor(
tokenizer_name: str,
*args,
tokenizer_mode: str = "auto",
trust_remote_code: bool = False,
tokenizer_revision: Optional[str] = None,
use_fast: Optional[bool] = True,
**kwargs,
):
revision = kwargs.pop("revision", tokenizer_revision)
if is_mistral_model(tokenizer_name):
config = load_mistral_config(
tokenizer_name,
trust_remote_code=trust_remote_code,
revision=revision,
)
else:
config = AutoConfig.from_pretrained(
tokenizer_name,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
is_ocr2 = _is_deepseek_ocr2_model(config)
if _is_deepseek_ocr_model(config) or is_ocr2:
config.model_type = "deepseek-ocr"
config.update({"architectures": ["DeepseekOCRForCausalLM"]})
if is_ocr2:
_override_v_head_dim_if_zero(config)
if config.model_type in {"qwen2_vl", "sarashina2_vision"}:
if "size" not in kwargs:
kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520}
if config.model_type not in {"llava", "clip"}:
kwargs["use_fast"] = use_fast
try:
if "InternVL3_5" in tokenizer_name:
processor = AutoTokenizer.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
else:
if config.model_type in _CUSTOMIZED_MM_PROCESSOR:
processor = _CUSTOMIZED_MM_PROCESSOR[config.model_type].from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
else:
processor = AutoProcessor.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
except ValueError as e:
error_message = str(e)
if "does not have a slow version" in error_message:
logger.info(
"Processor %s does not have a slow version. Automatically use fast version",
tokenizer_name,
)
kwargs["use_fast"] = True
processor = AutoProcessor.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
elif "Unrecognized feature extractor" in error_message:
logger.info(
"AutoProcessor failed on feature extractor for %s, "
"constructing processor manually",
tokenizer_name,
)
processor = _build_processor_manually(
tokenizer_name,
config,
trust_remote_code,
revision,
**kwargs,
)
elif (
"are not supported by" in error_message and "MistralCommon" in error_message
):
logger.info(
"AutoProcessor for %s rejected standard kwargs, "
"retrying without trust_remote_code/use_fast",
tokenizer_name,
)
kwargs.pop("use_fast", None)
kwargs.pop("_from_auto", None)
processor = AutoProcessor.from_pretrained(
tokenizer_name,
*args,
revision=revision,
**kwargs,
)
else:
raise
if (
isinstance(processor, PreTrainedTokenizerBase)
and getattr(config, "model_type", None) == "pixtral"
):
processor = wrap_as_pixtral(processor, config)
tokenizer = get_tokenizer_from_processor(processor)
# AutoProcessor may internally create a TokenizersBackend tokenizer
# (same issue as get_tokenizer). Replace it with a properly loaded one.
if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
from .tokenizer import get_tokenizer
logger.warning(
"Processor tokenizer for %s is TokenizersBackend, "
"reloading via get_tokenizer",
tokenizer_name,
)
tokenizer = get_tokenizer(
tokenizer_name,
tokenizer_mode=tokenizer_mode,
trust_remote_code=trust_remote_code,
tokenizer_revision=revision,
)
if isinstance(processor, PreTrainedTokenizerBase):
processor = tokenizer
else:
processor.tokenizer = tokenizer
if tokenizer.chat_template is None:
local_path = download_from_hf(
tokenizer_name, allow_patterns=["*.json", "*.jinja", "*.model"]
)
jinja_path = Path(local_path) / "chat_template.jinja"
if jinja_path.is_file():
tokenizer.chat_template = jinja_path.read_text()
logger.info("Loaded chat_template from %s", jinja_path)
patch_mistral_common_tokenizer(tokenizer)
_fix_special_tokens_pattern(tokenizer)
_fix_added_tokens_encoding(tokenizer)
attach_additional_stop_token_ids(tokenizer)
return processor
python/sglang/srt/utils/hf_transformers/tokenizer.py
+551
View File
@@ -0,0 +1,551 @@
# Copyright 2023-2024 SGLang Team
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tokenizer loading utilities."""
import json
import logging
import warnings
from pathlib import Path
from typing import Optional, Union
from transformers import (
AutoTokenizer,
PreTrainedTokenizer,
PreTrainedTokenizerFast,
)
from sglang.srt.connector import create_remote_connector
from sglang.srt.utils import is_remote_url, logger
from sglang.srt.utils.patch_tokenizer import patch_tokenizer
from sglang.srt.utils.runai_utils import ObjectStorageModel, is_runai_obj_uri
from .common import (
_resolve_local_or_cached_file,
attach_additional_stop_token_ids,
check_gguf_file,
)
from .compat import _ensure_gguf_version, patch_is_base_mistral_in_ci
from .mistral_utils import (
_MISTRAL_TOKENIZER_REDIRECTS,
patch_mistral_common_tokenizer,
retry_without_mistral_common_kwargs,
)
# A fast LLaMA tokenizer with the pre-processed `tokenizer.json` file.
_FAST_LLAMA_TOKENIZER = "hf-internal-testing/llama-tokenizer"
# Class name used by transformers v5 when no tokenizer mapping exists for a model_type.
_TOKENIZERS_BACKEND = "TokenizersBackend"
def _load_tokenizer_by_declared_class(tokenizer_name, *args, **kwargs):
"""Load tokenizer by the class declared in tokenizer_config.json.
AutoTokenizer resolves to TokenizersBackend when the model's config
model_type has no tokenizer class mapping (e.g. deepseek_vl_v2), even
though tokenizer_config.json declares a standard class like
LlamaTokenizerFast. Returns None if it cannot improve on AutoTokenizer.
"""
import transformers
try:
revision = kwargs.get("revision") or kwargs.get("tokenizer_revision")
config_file = _resolve_local_or_cached_file(
tokenizer_name, "tokenizer_config.json", revision
)
with open(config_file) as f:
tok_config = json.load(f)
tok_class_name = tok_config.get("tokenizer_class")
except FileNotFoundError:
return None
except (OSError, json.JSONDecodeError) as e:
logger.debug(
"Failed to read tokenizer_config.json for %s: %s", tokenizer_name, e
)
return None
if not tok_class_name:
return None
# Skip base classes that don't implement required methods (e.g. get_vocab)
if tok_class_name in ("PreTrainedTokenizer", "PreTrainedTokenizerBase"):
return None
tok_cls = getattr(transformers, tok_class_name, None)
if tok_cls is None and kwargs.get("trust_remote_code"):
# Class not in transformers — try loading via auto_map.
try:
auto_map = tok_config.get("auto_map", {})
auto_tok_ref = auto_map.get("AutoTokenizer")
if isinstance(auto_tok_ref, (list, tuple)):
auto_tok_ref = auto_tok_ref[0]
if auto_tok_ref:
from transformers.dynamic_module_utils import (
get_class_from_dynamic_module,
)
tok_cls = get_class_from_dynamic_module(
auto_tok_ref,
tokenizer_name,
code_revision=revision,
)
except (OSError, ImportError, ValueError, RuntimeError) as e:
logger.debug("Dynamic module lookup for %s failed: %s", tok_class_name, e)
if tok_cls is None:
return None
logger.info(
"Loading tokenizer for %s directly as %s (bypassing AutoTokenizer)",
tokenizer_name,
tok_class_name,
)
try:
return tok_cls.from_pretrained(tokenizer_name, *args, **kwargs)
except (OSError, ValueError, TypeError, ImportError) as e:
logger.warning(
"Direct load as %s failed for %s: %s. "
"Falling back to AutoTokenizer result.",
tok_class_name,
tokenizer_name,
e,
)
return None
# Filter warnings like: https://github.com/sgl-project/sglang/issues/8082
class TokenizerWarningsFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
return "Calling super().encode with" not in record.getMessage()
# ---------------------------------------------------------------------------
# Helpers for get_tokenizer
# ---------------------------------------------------------------------------
def _resolve_tokenizer_name(tokenizer_name, kwargs):
"""Resolve special name formats (GGUF, remote URLs, etc.) to a local path.
May mutate *kwargs* (e.g. to add ``gguf_file``).
"""
tokenizer_name = _MISTRAL_TOKENIZER_REDIRECTS.get(tokenizer_name, tokenizer_name)
if check_gguf_file(tokenizer_name):
_ensure_gguf_version()
kwargs["gguf_file"] = tokenizer_name
tokenizer_name = Path(tokenizer_name).parent
if is_runai_obj_uri(tokenizer_name):
tokenizer_name = ObjectStorageModel.get_path(tokenizer_name)
if is_remote_url(tokenizer_name):
# BaseConnector implements __del__() to clean up the local dir.
# Since config files need to exist all the time, so we DO NOT use
# with statement to avoid closing the client.
client = create_remote_connector(tokenizer_name)
client.pull_files(ignore_pattern=["*.pt", "*.safetensors", "*.bin"])
tokenizer_name = client.get_local_dir()
return tokenizer_name
def _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs):
"""Call ``AutoTokenizer.from_pretrained`` with error handling."""
try:
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name, *args, **common_kwargs
)
logging.getLogger(tokenizer.__class__.__module__).addFilter(
TokenizerWarningsFilter()
)
return tokenizer
except TypeError as e:
err_msg = (
"Failed to load the tokenizer. If you are using a LLaMA V1 model "
f"consider using '{_FAST_LLAMA_TOKENIZER}' instead of the "
"original tokenizer."
)
raise RuntimeError(err_msg) from e
except ValueError as e:
# MistralCommon tokenizers reject standard HF kwargs like
# trust_remote_code, use_fast etc. Retry without them.
if "are not supported by" in str(e) and "MistralCommon" in str(e):
return retry_without_mistral_common_kwargs(
tokenizer_name, *args, **common_kwargs
)
# If the error pertains to the tokenizer class not existing or not
# currently being imported, suggest using the --trust-remote-code flag.
if not common_kwargs.get("trust_remote_code") and (
"does not exist or is not currently imported." in str(e)
or "requires you to execute the tokenizer file" in str(e)
):
err_msg = (
"Failed to load the tokenizer. If the tokenizer is a custom "
"tokenizer not yet available in the HuggingFace transformers "
"library, consider setting `trust_remote_code=True` in LLM "
"or using the `--trust-remote-code` flag in the CLI."
)
raise RuntimeError(err_msg) from e
raise
def _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs):
"""Resolve generic ``TokenizersBackend`` to a proper tokenizer class.
In transformers v5, ``AutoTokenizer`` falls back to ``TokenizersBackend``
when the model_type has no tokenizer mapping. This retries with
``use_fast=False``, then attempts loading by the class declared in
``tokenizer_config.json``. May still return a ``TokenizersBackend``
if all retries fail (with a warning).
"""
logger.warning(
"Tokenizer loaded as generic TokenizersBackend for %s, "
"retrying with use_fast=False",
tokenizer_name,
)
common_kwargs = {**common_kwargs, "use_fast": False}
try:
tokenizer = AutoTokenizer.from_pretrained(
tokenizer_name, *args, **common_kwargs
)
except (ValueError, TypeError, OSError, ImportError, RuntimeError) as e:
raise RuntimeError(
f"Retry with use_fast=False for {tokenizer_name} also failed "
f"(initial load returned TokenizersBackend): {e}"
) from e
if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
tokenizer = (
_load_tokenizer_by_declared_class(tokenizer_name, *args, **common_kwargs)
or tokenizer
)
if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
if common_kwargs.get("trust_remote_code"):
logger.warning(
"Tokenizer for %s is still TokenizersBackend after retries "
"with --trust-remote-code. Model-specific tokenizer attributes "
"may be missing.",
tokenizer_name,
)
else:
logger.warning(
"Tokenizer for %s loaded as generic TokenizersBackend. "
"Set --trust-remote-code to load the model-specific tokenizer.",
tokenizer_name,
)
return tokenizer
# ---------------------------------------------------------------------------
# Post-load fixups
# ---------------------------------------------------------------------------
def _fix_v5_tokenizer_components(tokenizer, model_name_or_path, revision=None):
"""Fix pre_tokenizer/decoder when a v5 tokenizer class overwrites them.
In transformers v5, some tokenizer classes (e.g. LlamaTokenizer) have a
custom __init__ that rebuilds the pre_tokenizer and decoder from scratch
with class-specific components, discarding the originals from tokenizer.json.
This breaks models that specify LlamaTokenizerFast but actually use a
different tokenizer architecture (e.g. DeepSeek-V3.2 uses ByteLevel).
Detects the mismatch by comparing against the raw tokenizer.json and
restores the original components when they differ.
"""
backend = getattr(tokenizer, "_tokenizer", None)
if backend is None:
return
try:
from tokenizers import Tokenizer as RawTokenizer
tok_file = _resolve_local_or_cached_file(
model_name_or_path, "tokenizer.json", revision
)
raw = RawTokenizer.from_file(tok_file)
except FileNotFoundError:
return
except (OSError, ValueError, RuntimeError) as e:
logger.warning(
"_fix_v5_tokenizer_components: unexpected error loading tokenizer.json "
"for %s, v5 component fix will not be applied: %s",
model_name_or_path,
e,
)
return
raw_pre = type(raw.pre_tokenizer).__name__ if raw.pre_tokenizer else None
loaded_pre = type(backend.pre_tokenizer).__name__ if backend.pre_tokenizer else None
if raw_pre and loaded_pre and raw_pre != loaded_pre:
logger.info(
"Fixing v5 tokenizer component mismatch for %s: "
"pre_tokenizer %s -> %s, decoder %s -> %s",
model_name_or_path,
loaded_pre,
raw_pre,
type(backend.decoder).__name__ if backend.decoder else None,
type(raw.decoder).__name__ if raw.decoder else None,
)
backend.pre_tokenizer = raw.pre_tokenizer
backend.decoder = raw.decoder
def _fix_v5_add_bos_eos_token(tokenizer, model_name_or_path, revision=None):
"""Restore add_bos_token/add_eos_token stripped by transformers v5.
In transformers v5, _from_pretrained() strips add_bos_token and
add_eos_token from init kwargs when a tokenizer.json file is present,
assuming the tokenizer.json post-processor handles BOS/EOS addition.
However, many models (e.g. DeepSeek-V3) have a tokenizer.json whose
post-processor does NOT add BOS/EOS, and rely on the add_bos_token flag
from tokenizer_config.json instead. This causes silent accuracy regressions.
This function reads the tokenizer_config.json and restores the values,
but only for tokenizer classes that actually supported these flags in v4.
Classes like Qwen2Tokenizer did not support add_bos_token/add_eos_token
in v4, so restoring them would change behavior.
"""
# In transformers v4, only certain tokenizer classes supported
# add_bos_token / add_eos_token as init parameters. Restoring these
# flags for classes that never supported them (e.g. Qwen2Tokenizer)
# would incorrectly change tokenization behavior.
_V4_CLASSES_WITH_BOS_EOS_FLAGS = frozenset(
{
"LlamaTokenizer",
"LlamaTokenizerFast",
"CodeLlamaTokenizer",
"CodeLlamaTokenizerFast",
"GemmaTokenizer",
"GemmaTokenizerFast",
"CohereTokenizerFast",
}
)
try:
config_file = _resolve_local_or_cached_file(
model_name_or_path, "tokenizer_config.json", revision
)
with open(config_file) as f:
config = json.load(f)
except FileNotFoundError:
return
except (OSError, json.JSONDecodeError, ValueError) as e:
logger.warning(
"_fix_v5_add_bos_eos_token: failed to read tokenizer_config.json "
"for %s, BOS/EOS token restoration will not be applied: %s",
model_name_or_path,
e,
)
return
tokenizer_class = config.get("tokenizer_class", "")
if tokenizer_class not in _V4_CLASSES_WITH_BOS_EOS_FLAGS:
logger.debug(
"_fix_v5_add_bos_eos_token: skipping %s (tokenizer_class=%s "
"did not support add_bos/eos_token in v4)",
model_name_or_path,
tokenizer_class,
)
return
# In v4, Llama/Gemma tokenizers defaulted add_bos_token=True.
# When the config omits the key or has null, use the v4 default so that
# update_post_processor() doesn't drop BOS/EOS that was there before.
_V4_DEFAULTS = {"add_bos_token": True, "add_eos_token": False}
changed = False
for attr in ("add_bos_token", "add_eos_token"):
config_val = config.get(attr)
if config_val is None:
# Key missing or null -> use v4 default for this tokenizer class
config_val = _V4_DEFAULTS.get(attr, False)
# Fast tokenizers in v4 used tokenizer.json post-processor for EOS —
# the add_eos_token Python attribute was set but the post-processor
# came from tokenizer.json, not from the attribute. In v5, the flag is
# stripped and both sglang and HF reference end up with add_eos_token=False.
# Restoring add_eos_token for fast tokenizers makes sglang diverge from
# the HF reference, breaking embedding models like e5-mistral-7b-instruct.
if attr == "add_eos_token" and isinstance(tokenizer, PreTrainedTokenizerFast):
config_val = _V4_DEFAULTS["add_eos_token"] # False
current_val = getattr(tokenizer, attr, None)
if current_val != config_val:
logger.info(
"Restoring %s=%s for %s (was %s after v5 loading)",
attr,
config_val,
model_name_or_path,
current_val,
)
# Set the private backing attribute (not the property) because
# transformers tokenizers expose add_bos/eos_token as properties
# that read from the underscore-prefixed attribute.
setattr(tokenizer, f"_{attr}", config_val)
changed = True
# Rebuild the post-processor so it respects the restored flags
if changed and hasattr(tokenizer, "update_post_processor"):
tokenizer.update_post_processor()
def _fix_special_tokens_pattern(tokenizer):
"""Fix https://github.com/huggingface/transformers/pull/42563 which defaults
special_tokens_pattern to "cls_sep", inserting None into token IDs when
cls_token/sep_token are undefined (e.g. Kimi-VL's TikTokenTokenizer).
"""
pattern = getattr(tokenizer, "special_tokens_pattern", None)
if pattern == "cls_sep" and (
tokenizer.cls_token_id is None or tokenizer.sep_token_id is None
):
tokenizer.special_tokens_pattern = "none"
def _apply_post_load_fixes(tokenizer, tokenizer_name, revision):
"""Apply all post-load patches and return the final tokenizer."""
_fix_v5_tokenizer_components(tokenizer, tokenizer_name, revision)
_fix_v5_add_bos_eos_token(tokenizer, tokenizer_name, revision)
if not isinstance(tokenizer, PreTrainedTokenizerFast):
warnings.warn(
"Using a slow tokenizer. This might cause a significant "
"slowdown. Consider using a fast tokenizer instead."
)
patch_mistral_common_tokenizer(tokenizer)
_fix_special_tokens_pattern(tokenizer)
attach_additional_stop_token_ids(tokenizer)
return patch_tokenizer(tokenizer)
# ---------------------------------------------------------------------------
# Public entry point
# ---------------------------------------------------------------------------
def get_tokenizer(
tokenizer_name: str,
*args,
tokenizer_mode: str = "auto",
trust_remote_code: bool = False,
tokenizer_revision: Optional[str] = None,
**kwargs,
) -> Union[PreTrainedTokenizer, PreTrainedTokenizerFast]:
"""Gets a tokenizer for the given model name via Huggingface."""
if tokenizer_name.endswith(".json"):
from sglang.srt.tokenizer.tiktoken_tokenizer import TiktokenTokenizer
return TiktokenTokenizer(tokenizer_name)
if tokenizer_mode == "slow":
if kwargs.get("use_fast", False):
raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.")
kwargs["use_fast"] = False
elif tokenizer_mode == "auto":
# Transformers v5 AutoTokenizer ignores use_fast (always fast), but
# some code paths pass kwargs to non-AutoTokenizer loaders where
# use_fast still matters. Set explicitly for those fallback paths.
if "use_fast" not in kwargs:
kwargs["use_fast"] = True
tokenizer_name = _resolve_tokenizer_name(tokenizer_name, kwargs)
patch_is_base_mistral_in_ci()
common_kwargs = dict(
trust_remote_code=trust_remote_code,
tokenizer_revision=tokenizer_revision,
clean_up_tokenization_spaces=False,
**kwargs,
)
tokenizer = _auto_tokenizer_from_pretrained(tokenizer_name, *args, **common_kwargs)
if type(tokenizer).__name__ == _TOKENIZERS_BACKEND:
tokenizer = _resolve_tokenizers_backend(tokenizer_name, *args, **common_kwargs)
return _apply_post_load_fixes(tokenizer, tokenizer_name, tokenizer_revision)
# ---------------------------------------------------------------------------
# Exported helpers (used by processor.py, etc.)
# ---------------------------------------------------------------------------
def _fix_added_tokens_encoding(tokenizer):
"""Ensure special tokens encode as single tokens in transformers v5.
Some model tokenizers (e.g. MiniCPM-V-4) define special tokens like <image>,
<slice> as attributes on the tokenizer class with corresponding IDs in the
vocabulary (via tokenizer.json's added_tokens). In transformers v5, these
tokens may not appear in get_added_vocab() and encode() splits them into
subwords, breaking multimodal pipelines that rely on finding them in input_ids.
This function discovers such tokens by scanning tokenizer attributes, checks
if they encode correctly, and re-registers any that don't.
"""
# Discover special token strings from tokenizer attributes.
# Model tokenizers (e.g. MiniCPMVTokenizerFast) store them as attributes
# like im_start="<image>", slice_start="<slice>", etc.
def _is_special_token_attr(val):
return (
isinstance(val, str)
and val.startswith("<")
and val.endswith(">")
and len(val) <= 20
)
candidates = {}
for attr in dir(tokenizer):
if attr.startswith("_"):
continue
try:
val = getattr(tokenizer, attr)
except (AttributeError, TypeError, ValueError):
continue
if not _is_special_token_attr(val):
continue
token_id = tokenizer.convert_tokens_to_ids(val)
if token_id is not None and token_id != tokenizer.unk_token_id:
candidates[val] = token_id
if not candidates:
return
def _encodes_correctly(token_str, expected_id):
try:
ids = tokenizer.encode(token_str, add_special_tokens=False)
return len(ids) == 1 and ids[0] == expected_id
except (ValueError, OverflowError, RuntimeError) as e:
logger.debug("Token %s encode check failed: %s", token_str, e)
return False
broken = [
tok for tok, eid in candidates.items() if not _encodes_correctly(tok, eid)
]
if not broken:
return
from transformers import AddedToken
tokens_to_add = [AddedToken(tok, special=True, normalized=False) for tok in broken]
tokenizer.add_tokens(tokens_to_add, special_tokens=True)
logger.info(
"Re-registered %d special tokens for correct v5 encoding: %s",
len(broken),
broken[:10],
)
python/sglang/srt/utils/hf_transformers_utils.py
+3 -1480
View File
File diff suppressed because it is too large Load Diff
test/registered/unit/utils/test_hf_transformers.py
+586
View File
@@ -0,0 +1,586 @@
"""Unit tests for the sglang.srt.utils.hf_transformers subpackage.
Tests cover the pure utility functions (compat patches, config helpers,
context length, GGUF detection, etc.) that don't require actual model files.
"""
import tempfile
import unittest
from types import SimpleNamespace
from transformers import PretrainedConfig
from sglang.srt.utils.hf_transformers.common import (
_is_deepseek_ocr2_model,
_is_deepseek_ocr_model,
_override_v_head_dim_if_zero,
_patch_text_config,
check_gguf_file,
get_context_length,
get_hf_text_config,
get_rope_config,
)
from sglang.srt.utils.hf_transformers.compat import normalize_rope_scaling_compat
from sglang.srt.utils.hf_transformers.tokenizer import _fix_special_tokens_pattern
from sglang.test.ci.ci_register import register_cpu_ci
register_cpu_ci(est_time=5, suite="stage-a-test-cpu")
# ---------------------------------------------------------------------------
# normalize_rope_scaling_compat
# ---------------------------------------------------------------------------
class TestNormalizeRopeScalingCompat(unittest.TestCase):
def test_adds_type_from_rope_type(self):
cfg = PretrainedConfig()
cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0}
normalize_rope_scaling_compat(cfg)
self.assertEqual(cfg.rope_scaling["type"], "llama3")
def test_preserves_existing_type(self):
cfg = PretrainedConfig()
cfg.rope_scaling = {"rope_type": "llama3", "type": "custom", "factor": 8.0}
normalize_rope_scaling_compat(cfg)
self.assertEqual(cfg.rope_scaling["type"], "custom")
def test_no_op_when_no_rope_scaling(self):
cfg = PretrainedConfig()
normalize_rope_scaling_compat(cfg)
self.assertIsNone(getattr(cfg, "rope_scaling", None))
def test_no_op_when_rope_scaling_is_none(self):
cfg = PretrainedConfig()
cfg.rope_scaling = None
normalize_rope_scaling_compat(cfg)
self.assertIsNone(cfg.rope_scaling)
def test_recurses_into_text_config(self):
text_cfg = PretrainedConfig()
text_cfg.rope_scaling = {"rope_type": "yarn", "factor": 4.0}
cfg = PretrainedConfig()
cfg.text_config = text_cfg
normalize_rope_scaling_compat(cfg)
self.assertEqual(text_cfg.rope_scaling["type"], "yarn")
def test_recurses_into_llm_config(self):
llm_cfg = PretrainedConfig()
llm_cfg.rope_scaling = {"rope_type": "dynamic", "factor": 2.0}
cfg = PretrainedConfig()
cfg.llm_config = llm_cfg
normalize_rope_scaling_compat(cfg)
self.assertEqual(llm_cfg.rope_scaling["type"], "dynamic")
def test_no_crash_on_non_dict_rope_scaling(self):
cfg = PretrainedConfig()
cfg.rope_scaling = "not_a_dict"
normalize_rope_scaling_compat(cfg)
self.assertEqual(cfg.rope_scaling, "not_a_dict")
def test_no_crash_on_dict_without_rope_type(self):
cfg = PretrainedConfig()
cfg.rope_scaling = {"factor": 4.0}
normalize_rope_scaling_compat(cfg)
self.assertNotIn("type", cfg.rope_scaling)
# ---------------------------------------------------------------------------
# get_rope_config
# ---------------------------------------------------------------------------
class TestGetRopeConfig(unittest.TestCase):
def test_v5_rope_parameters(self):
cfg = PretrainedConfig()
cfg.rope_parameters = {"rope_theta": 10000.0, "rope_type": "default"}
theta, params = get_rope_config(cfg)
self.assertEqual(theta, 10000.0)
self.assertIs(params, cfg.rope_parameters)
def test_v4_fallback_remote_code_config(self):
# Remote-code configs (SimpleNamespace) lack the v5 rope_parameters property
cfg = SimpleNamespace(
rope_theta=500000.0,
rope_scaling={"type": "llama3", "factor": 8.0},
)
theta, params = get_rope_config(cfg)
self.assertEqual(theta, 500000.0)
self.assertEqual(params, {"type": "llama3", "factor": 8.0})
def test_v4_no_scaling(self):
cfg = SimpleNamespace(rope_theta=10000.0)
theta, params = get_rope_config(cfg)
self.assertEqual(theta, 10000.0)
self.assertIsNone(params)
# ---------------------------------------------------------------------------
# _patch_text_config
# ---------------------------------------------------------------------------
class TestPatchTextConfig(unittest.TestCase):
def test_propagates_parent_to_text(self):
parent = PretrainedConfig()
parent.pad_token_id = 0
parent.bos_token_id = 1
parent.eos_token_id = 2
parent.tie_word_embeddings = False
text = PretrainedConfig()
text.num_attention_heads = 32
result = _patch_text_config(parent, text)
self.assertEqual(result.pad_token_id, 0)
self.assertEqual(result.bos_token_id, 1)
self.assertEqual(result.eos_token_id, 2)
self.assertIs(result, text)
def test_propagates_text_to_parent(self):
parent = PretrainedConfig()
text = PretrainedConfig()
text.pad_token_id = 42
_patch_text_config(parent, text)
self.assertEqual(parent.pad_token_id, 42)
def test_no_overwrite_when_both_have_attr(self):
parent = PretrainedConfig()
parent.pad_token_id = 0
text = PretrainedConfig()
text.pad_token_id = 99
_patch_text_config(parent, text)
self.assertEqual(parent.pad_token_id, 0)
self.assertEqual(text.pad_token_id, 99)
# ---------------------------------------------------------------------------
# get_context_length
# ---------------------------------------------------------------------------
class TestGetContextLength(unittest.TestCase):
def test_max_position_embeddings(self):
cfg = PretrainedConfig()
cfg.max_position_embeddings = 4096
self.assertEqual(get_context_length(cfg), 4096)
def test_max_sequence_length_takes_priority(self):
cfg = PretrainedConfig()
cfg.max_sequence_length = 8192
cfg.max_position_embeddings = 4096
self.assertEqual(get_context_length(cfg), 8192)
def test_rope_scaling_factor(self):
cfg = PretrainedConfig()
cfg.max_position_embeddings = 4096
cfg.rope_scaling = {"factor": 4.0}
self.assertEqual(get_context_length(cfg), 16384)
def test_rope_scaling_llama3_ignores_factor(self):
cfg = PretrainedConfig()
cfg.max_position_embeddings = 131072
cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0}
self.assertEqual(get_context_length(cfg), 131072)
def test_original_max_position_embeddings_ignores_factor(self):
cfg = PretrainedConfig()
cfg.max_position_embeddings = 131072
cfg.rope_scaling = {
"factor": 8.0,
"original_max_position_embeddings": 8192,
}
self.assertEqual(get_context_length(cfg), 131072)
def test_default_when_no_keys(self):
cfg = PretrainedConfig()
self.assertEqual(get_context_length(cfg), 2048)
# ---------------------------------------------------------------------------
# check_gguf_file
# ---------------------------------------------------------------------------
class TestCheckGgufFile(unittest.TestCase):
def test_gguf_suffix(self):
with tempfile.NamedTemporaryFile(suffix=".gguf") as f:
self.assertTrue(check_gguf_file(f.name))
def test_gguf_magic_header(self):
with tempfile.NamedTemporaryFile(suffix=".bin") as f:
f.write(b"GGUF" + b"\x00" * 100)
f.flush()
self.assertTrue(check_gguf_file(f.name))
def test_non_gguf_file(self):
with tempfile.NamedTemporaryFile(suffix=".bin") as f:
f.write(b"NOT_GGUF" + b"\x00" * 100)
f.flush()
self.assertFalse(check_gguf_file(f.name))
def test_nonexistent_file(self):
self.assertFalse(check_gguf_file("/nonexistent/path/model.bin"))
def test_directory(self):
with tempfile.TemporaryDirectory() as d:
self.assertFalse(check_gguf_file(d))
# ---------------------------------------------------------------------------
# _is_deepseek_ocr_model / _is_deepseek_ocr2_model
# ---------------------------------------------------------------------------
class TestDeepseekOcrDetection(unittest.TestCase):
def test_ocr_model_detected(self):
cfg = PretrainedConfig()
cfg.auto_map = {"AutoModel": "modeling_deepseekocr.DeepseekOCRForCausalLM"}
self.assertTrue(_is_deepseek_ocr_model(cfg))
def test_ocr2_model_detected(self):
cfg = PretrainedConfig()
cfg.auto_map = {"AutoModel": "modeling_deepseekocr2.DeepseekOCR2ForCausalLM"}
self.assertTrue(_is_deepseek_ocr2_model(cfg))
def test_non_ocr_model(self):
cfg = PretrainedConfig()
cfg.auto_map = {"AutoModel": "modeling_llama.LlamaForCausalLM"}
self.assertFalse(_is_deepseek_ocr_model(cfg))
self.assertFalse(_is_deepseek_ocr2_model(cfg))
def test_no_auto_map(self):
cfg = PretrainedConfig()
self.assertFalse(_is_deepseek_ocr_model(cfg))
self.assertFalse(_is_deepseek_ocr2_model(cfg))
def test_empty_auto_map(self):
cfg = PretrainedConfig()
cfg.auto_map = {}
self.assertFalse(_is_deepseek_ocr_model(cfg))
self.assertFalse(_is_deepseek_ocr2_model(cfg))
# ---------------------------------------------------------------------------
# _override_v_head_dim_if_zero
# ---------------------------------------------------------------------------
class TestOverrideVHeadDimIfZero(unittest.TestCase):
def test_patches_zero_v_head_dim(self):
text_cfg = SimpleNamespace(v_head_dim=0)
cfg = PretrainedConfig()
cfg.text_config = text_cfg
_override_v_head_dim_if_zero(cfg)
self.assertEqual(text_cfg.v_head_dim, 128)
def test_custom_patch_value(self):
text_cfg = SimpleNamespace(v_head_dim=0)
cfg = PretrainedConfig()
cfg.text_config = text_cfg
_override_v_head_dim_if_zero(cfg, patch=64)
self.assertEqual(text_cfg.v_head_dim, 64)
def test_no_patch_when_nonzero(self):
text_cfg = SimpleNamespace(v_head_dim=256)
cfg = PretrainedConfig()
cfg.text_config = text_cfg
_override_v_head_dim_if_zero(cfg)
self.assertEqual(text_cfg.v_head_dim, 256)
def test_dict_sub_config(self):
cfg = PretrainedConfig()
cfg.text_config = {"v_head_dim": 0}
_override_v_head_dim_if_zero(cfg)
self.assertEqual(cfg.text_config["v_head_dim"], 128)
def test_no_sub_config(self):
cfg = PretrainedConfig()
_override_v_head_dim_if_zero(cfg) # should not raise
# ---------------------------------------------------------------------------
# get_hf_text_config
# ---------------------------------------------------------------------------
class TestGetHfTextConfig(unittest.TestCase):
def test_returns_config_for_pure_text_model(self):
cfg = PretrainedConfig()
cfg.architectures = ["LlamaForCausalLM"]
result = get_hf_text_config(cfg)
self.assertIs(result, cfg)
def test_returns_text_config_for_multimodal(self):
text_cfg = PretrainedConfig()
text_cfg.num_attention_heads = 32
cfg = PretrainedConfig()
cfg.architectures = ["SomeVLMForCausalLM"]
cfg.text_config = text_cfg
result = get_hf_text_config(cfg)
self.assertIs(result, text_cfg)
def test_llm_config_priority_over_text_config(self):
llm_cfg = PretrainedConfig()
llm_cfg.num_attention_heads = 16
text_cfg = PretrainedConfig()
text_cfg.num_attention_heads = 32
cfg = PretrainedConfig()
cfg.architectures = ["SomeModel"]
cfg.llm_config = llm_cfg
cfg.text_config = text_cfg
result = get_hf_text_config(cfg)
self.assertIs(result, llm_cfg)
def test_thinker_config_highest_priority(self):
thinker_cfg = PretrainedConfig()
thinker_cfg.num_attention_heads = 8
cfg = PretrainedConfig()
cfg.architectures = ["SomeModel"]
cfg.thinker_config = thinker_cfg
result = get_hf_text_config(cfg)
self.assertIs(result, thinker_cfg)
def test_thinker_config_with_text_sub_config(self):
inner_text = PretrainedConfig()
inner_text.num_attention_heads = 8
thinker_cfg = PretrainedConfig()
thinker_cfg.text_config = inner_text
thinker_cfg.torch_dtype = "float16"
cfg = PretrainedConfig()
cfg.architectures = ["Qwen2OmniModel"]
cfg.thinker_config = thinker_cfg
result = get_hf_text_config(cfg)
self.assertIs(result, inner_text)
self.assertEqual(inner_text.torch_dtype, "float16")
def test_converts_dict_sub_config(self):
cfg = PretrainedConfig()
cfg.architectures = ["SomeModel"]
cfg.text_config = {
"num_attention_heads": 32,
"hidden_size": 4096,
}
result = get_hf_text_config(cfg)
self.assertIsInstance(cfg.text_config, PretrainedConfig)
self.assertEqual(result.num_attention_heads, 32)
def test_llava_returns_parent_config(self):
cfg = PretrainedConfig()
cfg.architectures = ["LlavaForCausalLM"]
text_cfg = PretrainedConfig()
text_cfg.num_attention_heads = 32
cfg.text_config = text_cfg
result = get_hf_text_config(cfg)
self.assertIs(result, cfg)
def test_calls_normalize_rope_scaling(self):
cfg = PretrainedConfig()
cfg.architectures = ["LlamaForCausalLM"]
cfg.rope_scaling = {"rope_type": "llama3", "factor": 8.0}
get_hf_text_config(cfg)
self.assertIn("type", cfg.rope_scaling)
self.assertEqual(cfg.rope_scaling["type"], "llama3")
# ---------------------------------------------------------------------------
# _fix_special_tokens_pattern
# ---------------------------------------------------------------------------
class TestFixSpecialTokensPattern(unittest.TestCase):
def test_fixes_cls_sep_with_missing_tokens(self):
tok = SimpleNamespace(
special_tokens_pattern="cls_sep",
cls_token_id=None,
sep_token_id=None,
)
_fix_special_tokens_pattern(tok)
self.assertEqual(tok.special_tokens_pattern, "none")
def test_no_change_when_tokens_present(self):
tok = SimpleNamespace(
special_tokens_pattern="cls_sep",
cls_token_id=101,
sep_token_id=102,
)
_fix_special_tokens_pattern(tok)
self.assertEqual(tok.special_tokens_pattern, "cls_sep")
def test_no_change_for_other_patterns(self):
tok = SimpleNamespace(
special_tokens_pattern="none",
cls_token_id=None,
sep_token_id=None,
)
_fix_special_tokens_pattern(tok)
self.assertEqual(tok.special_tokens_pattern, "none")
def test_no_change_when_no_pattern(self):
tok = SimpleNamespace(cls_token_id=None, sep_token_id=None)
_fix_special_tokens_pattern(tok)
self.assertFalse(hasattr(tok, "special_tokens_pattern"))
# ---------------------------------------------------------------------------
# __init__.py re-exports
# ---------------------------------------------------------------------------
class TestModuleReExports(unittest.TestCase):
def test_all_public_symbols_importable(self):
import sglang.srt.utils.hf_transformers as pkg
for name in pkg.__all__:
self.assertTrue(
hasattr(pkg, name),
f"{name} listed in __all__ but not importable from package",
)
def test_shim_module_exports_match(self):
import sglang.srt.utils.hf_transformers as pkg
import sglang.srt.utils.hf_transformers_utils as shim
for name in pkg.__all__:
self.assertTrue(
hasattr(shim, name),
f"{name} not available through shim module hf_transformers_utils",
)
# ---------------------------------------------------------------------------
# compat: _patch_removed_symbols
# ---------------------------------------------------------------------------
class TestPatchRemovedSymbols(unittest.TestCase):
def test_llama_flash_attention2_exists(self):
from transformers.models.llama import modeling_llama
self.assertTrue(
hasattr(modeling_llama, "LlamaFlashAttention2"),
"LlamaFlashAttention2 should be patched onto modeling_llama",
)
def test_is_flash_attn_greater_or_equal_2_10_callable(self):
import transformers.utils as _u
self.assertTrue(
hasattr(_u, "is_flash_attn_greater_or_equal_2_10"),
"is_flash_attn_greater_or_equal_2_10 should be patched onto transformers.utils",
)
self.assertIsInstance(_u.is_flash_attn_greater_or_equal_2_10(), bool)
# ---------------------------------------------------------------------------
# compat: _patch_rope_parameters_validation
# ---------------------------------------------------------------------------
class TestPatchRopeParametersValidation(unittest.TestCase):
def test_injects_rope_theta_into_rope_scaling(self):
config_dict = {
"model_type": "llama",
"rope_theta": 500000.0,
"max_position_embeddings": 131072,
"rope_scaling": {
"rope_type": "llama3",
"factor": 8.0,
"low_freq_factor": 1.0,
"high_freq_factor": 4.0,
"original_max_position_embeddings": 8192,
},
}
config = PretrainedConfig.from_dict(config_dict)
rope_params = getattr(config, "rope_parameters", None)
if rope_params is not None:
self.assertIn("rope_theta", rope_params)
def test_no_injection_when_rope_theta_already_in_scaling(self):
config_dict = {
"model_type": "llama",
"rope_theta": 500000.0,
"max_position_embeddings": 131072,
"rope_scaling": {
"rope_type": "llama3",
"factor": 8.0,
"rope_theta": 999.0,
"low_freq_factor": 1.0,
"high_freq_factor": 4.0,
"original_max_position_embeddings": 8192,
},
}
config = PretrainedConfig.from_dict(config_dict)
rope_params = getattr(config, "rope_parameters", None)
if rope_params is not None:
self.assertEqual(rope_params["rope_theta"], 999.0)
def test_no_crash_without_rope_scaling(self):
config_dict = {"model_type": "llama", "rope_theta": 10000.0}
config = PretrainedConfig.from_dict(config_dict)
self.assertIsNotNone(config)
# ---------------------------------------------------------------------------
# compat: _ensure_clean_up_tokenization_compat
# ---------------------------------------------------------------------------
class TestCleanUpTokenizationCompat(unittest.TestCase):
def test_clean_up_tokenization_exists(self):
from transformers import PreTrainedTokenizerBase
self.assertTrue(hasattr(PreTrainedTokenizerBase, "clean_up_tokenization"))
def test_clean_up_tokenization_callable(self):
from transformers import PreTrainedTokenizerBase
self.assertTrue(callable(PreTrainedTokenizerBase.clean_up_tokenization))
# ---------------------------------------------------------------------------
# compat: _ensure_is_torch_fx_available_compat
# ---------------------------------------------------------------------------
class TestIsTorchFxAvailableCompat(unittest.TestCase):
def test_is_torch_fx_available_exists(self):
import transformers.utils.import_utils as _iu
self.assertTrue(hasattr(_iu, "is_torch_fx_available"))
self.assertTrue(_iu.is_torch_fx_available())
# ---------------------------------------------------------------------------
# compat: _patch_nemotron_h_pattern
# ---------------------------------------------------------------------------
class TestPatchNemotronHPattern(unittest.TestCase):
def test_pattern_to_list_skips_mlp_dash(self):
try:
from transformers.models.nemotron_h.configuration_nemotron_h import (
NemotronHConfig,
)
result = NemotronHConfig._pattern_to_list("M-*-")
self.assertEqual(result, ["mamba", "attention"])
except ImportError:
self.skipTest("NemotronHConfig not available in this transformers version")
def test_pattern_to_list_standard_chars(self):
try:
from transformers.models.nemotron_h.configuration_nemotron_h import (
NemotronHConfig,
)
result = NemotronHConfig._pattern_to_list("ME*")
self.assertEqual(result, ["mamba", "moe", "attention"])
except ImportError:
self.skipTest("NemotronHConfig not available in this transformers version")
if __name__ == "__main__":
unittest.main()
test/registered/vlm/test_vlm_input_format.py
+11 -2
View File
@@ -425,11 +425,13 @@ class TestInternVLUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes
torch_dtype=torch.bfloat16,
low_cpu_mem_usage=False,
)
except RuntimeError as e:
if "meta" not in str(e):
except (RuntimeError, AttributeError) as e:
if isinstance(e, RuntimeError) and "meta" not in str(e):
raise
# Transformers v5 always uses meta tensors for init, which breaks
# models calling .item() in __init__ (e.g. InternVL's drop_path_rate).
# Transformers v5.5.3 may also raise AttributeError for remote-code
# models missing new internal attributes (e.g. all_tied_weights_keys).
# Fall back to from_config + manual weight loading.
import gc
import glob
@@ -594,6 +596,13 @@ class TestMiniCPMVUnderstandsImage(VLMInputTestBase, unittest.IsolatedAsyncioTes
cls.processor = AutoProcessor.from_pretrained(
cls.model_path, trust_remote_code=True
)
# In transformers v5.5.3, AutoTokenizer may return TokenizersBackend
# which lacks model-specific attributes (e.g. im_start_id for MiniCPM-V).
# Replace with sglang's tokenizer which handles this via declared-class
# fallback, then fix added tokens encoding.
from sglang.srt.utils.hf_transformers import get_tokenizer
cls.processor.tokenizer = get_tokenizer(cls.model_path, trust_remote_code=True)
_fix_added_tokens_encoding(cls.processor.tokenizer)
cls._init_visual()