[Deps] Bump transformers to 5.17.0 (#39012)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai> Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
This commit is contained in:
co-authored by
Mohammad Angkad
Xinyuan Tong
parent
036a3b3e14
commit
547286bb5a
@@ -84,7 +84,7 @@ dependencies = [
|
||||
"tiktoken",
|
||||
"tilelang==0.1.12",
|
||||
"timm==1.0.16",
|
||||
"tokenizers==0.22.2", # 0.23.0rc0 is incompatible with transformers' CLIPTokenizer.
|
||||
"tokenizers==0.23.2",
|
||||
"tokenspeed_mla==0.1.8",
|
||||
"tomli ; python_version < '3.11'",
|
||||
"torch==2.13.0",
|
||||
@@ -93,7 +93,7 @@ dependencies = [
|
||||
"torchcodec==0.15.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # Not available on Linux ARM.
|
||||
"torchvision",
|
||||
"tqdm",
|
||||
"transformers==5.12.1",
|
||||
"transformers==5.17.0",
|
||||
"uvicorn",
|
||||
"uvloop",
|
||||
"watchfiles",
|
||||
|
||||
@@ -66,7 +66,7 @@ dependencies = [
|
||||
"torchcodec==0.12.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')",
|
||||
"torchvision==0.29.0",
|
||||
"tqdm",
|
||||
"transformers==5.12.1",
|
||||
"transformers==5.17.0",
|
||||
"triton==3.8.0",
|
||||
"uvicorn",
|
||||
"uvloop",
|
||||
|
||||
@@ -62,7 +62,7 @@ dependencies = [
|
||||
"tiktoken",
|
||||
"timm==1.0.16",
|
||||
"tqdm",
|
||||
"transformers==5.12.1",
|
||||
"transformers==5.17.0",
|
||||
"uvicorn",
|
||||
"uvloop",
|
||||
"xxhash",
|
||||
|
||||
@@ -62,7 +62,7 @@ runtime_base = [
|
||||
"soundfile==0.13.1",
|
||||
"tiktoken",
|
||||
"tqdm",
|
||||
"transformers==5.12.1",
|
||||
"transformers==5.17.0",
|
||||
"uvicorn",
|
||||
"uvloop",
|
||||
"xxhash",
|
||||
|
||||
@@ -66,7 +66,7 @@ dependencies = [
|
||||
"torchcodec==0.13.0 ; sys_platform != 'linux' or (sys_platform == 'linux' and platform_machine != 'aarch64' and platform_machine != 'arm64' and platform_machine != 'armv7l')", # torch==2.13.0 on XPU uses torchcodec 0.13.0
|
||||
"torchvision==0.28.0+xpu",
|
||||
"tqdm",
|
||||
"transformers==5.12.1",
|
||||
"transformers==5.17.0",
|
||||
"uvicorn",
|
||||
"xxhash",
|
||||
"uvloop",
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
from transformers import PretrainedConfig
|
||||
from transformers.configuration_utils import layer_type_validation
|
||||
|
||||
from sglang.utils import logger
|
||||
|
||||
@@ -319,7 +318,7 @@ class Qwen3OmniMoeTalkerCodePredictorConfig(PretrainedConfig):
|
||||
)
|
||||
for i in range(self.num_hidden_layers)
|
||||
]
|
||||
layer_type_validation(self.layer_types, self.num_hidden_layers)
|
||||
self.validate_layer_type()
|
||||
self.num_code_groups = num_code_groups
|
||||
|
||||
|
||||
|
||||
@@ -306,18 +306,15 @@ class ZayaConfig(PretrainedConfig):
|
||||
def register_zaya_config() -> None:
|
||||
"""Register :class:`ZayaConfig` with HuggingFace ``AutoConfig``.
|
||||
|
||||
Safe to call multiple times. ``AutoConfig.register`` raises ``ValueError``
|
||||
on duplicate registration, which is suppressed so importing this module
|
||||
stays idempotent.
|
||||
``exist_ok=True`` overrides the native ``zaya`` config transformers ships;
|
||||
SGLang's carries the hybrid-pool properties the runtime needs, and both
|
||||
classes are named ``ZayaConfig`` so the Auto* mappings still key.
|
||||
"""
|
||||
try:
|
||||
from transformers import AutoConfig
|
||||
|
||||
AutoConfig.register(ZayaConfig.model_type, ZayaConfig)
|
||||
AutoConfig.register(ZayaConfig.model_type, ZayaConfig, exist_ok=True)
|
||||
except (ValueError, ImportError):
|
||||
# Either the installed ``transformers`` does not expose
|
||||
# ``AutoConfig.register``, or the "zaya" model type is already
|
||||
# registered – nothing to do in either case.
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -10,9 +10,6 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
|
||||
Qwen2_5_VisionRotaryEmbedding,
|
||||
)
|
||||
|
||||
from sglang.srt.layers.attention.vision import (
|
||||
VisionAttention,
|
||||
@@ -21,7 +18,11 @@ from sglang.srt.layers.attention.vision import (
|
||||
)
|
||||
from sglang.srt.layers.layernorm import RMSNorm
|
||||
from sglang.srt.layers.quantization import QuantizationConfig
|
||||
from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionPatchMerger, Qwen2_5_VLMLP
|
||||
from sglang.srt.models.qwen2_5_vl import (
|
||||
Qwen2_5_VisionPatchMerger,
|
||||
Qwen2_5_VisionRotaryEmbedding,
|
||||
Qwen2_5_VLMLP,
|
||||
)
|
||||
from sglang.srt.runtime_context import get_mm, get_server_args
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
@@ -389,7 +390,7 @@ class MiMoVisionTransformer(nn.Module):
|
||||
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = int(grid_thw[:, 1:].max())
|
||||
# transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU).
|
||||
# The vision rotary forward takes 1-D position_ids on the input device (grid_thw is CPU).
|
||||
rotary_pos_emb_full = self.rotary_pos_emb(
|
||||
torch.arange(max_grid_size, device=self.device)
|
||||
)
|
||||
|
||||
@@ -11,9 +11,6 @@ import torch
|
||||
import torch.nn as nn
|
||||
from einops import rearrange
|
||||
from transformers.activations import ACT2FN
|
||||
from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
|
||||
Qwen2_5_VisionRotaryEmbedding,
|
||||
)
|
||||
|
||||
from sglang.srt.environ import envs
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
@@ -53,6 +50,7 @@ from sglang.srt.managers.schedule_batch import MultimodalInputs
|
||||
from sglang.srt.model_executor.forward_batch_info import ForwardBatch
|
||||
from sglang.srt.model_executor.runner import get_is_capture_mode
|
||||
from sglang.srt.model_loader.weight_utils import default_weight_loader
|
||||
from sglang.srt.models.qwen2_5_vl import Qwen2_5_VisionRotaryEmbedding
|
||||
from sglang.srt.runtime_context import get_exec, get_parallel
|
||||
from sglang.srt.utils import add_prefix
|
||||
|
||||
@@ -332,7 +330,7 @@ class MossVLVisionModel(nn.Module):
|
||||
pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = int(grid_thw[:, 1:].max())
|
||||
# transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU).
|
||||
# The vision rotary forward takes 1-D position_ids on the input device (grid_thw is CPU).
|
||||
rotary_pos_emb_full = self.rotary_pos_emb(
|
||||
torch.arange(max_grid_size, device=self.device)
|
||||
)
|
||||
|
||||
@@ -25,14 +25,11 @@ import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import PixtralVisionConfig, PretrainedConfig
|
||||
from transformers.models.pixtral.modeling_pixtral import (
|
||||
PixtralRotaryEmbedding,
|
||||
PixtralVisionRotaryEmbedding,
|
||||
)
|
||||
from transformers.models.pixtral.modeling_pixtral import (
|
||||
generate_block_attention_mask as _get_pixtral_attention_mask,
|
||||
)
|
||||
from transformers.models.pixtral.modeling_pixtral import (
|
||||
position_ids_in_meshgrid,
|
||||
)
|
||||
|
||||
from sglang.srt.layers.activation import SiluAndMul
|
||||
from sglang.srt.layers.attention.vision import VisionAttention
|
||||
@@ -904,7 +901,7 @@ class PixtralHFVisionModel(nn.Module):
|
||||
)
|
||||
|
||||
# Initialize patch position embedding
|
||||
self.patch_positional_embedding = PixtralRotaryEmbedding(config)
|
||||
self.patch_positional_embedding = PixtralVisionRotaryEmbedding(config)
|
||||
self.input_padder = MultiModalityDataPaddingPatternMultimodalTokens()
|
||||
|
||||
@property
|
||||
@@ -950,14 +947,10 @@ class PixtralHFVisionModel(nn.Module):
|
||||
embeds_1d = torch.cat([p.flatten(1).T for p in embeds_2d], dim=0)
|
||||
embeds_featurized = self.ln_pre(embeds_1d).unsqueeze(0)
|
||||
|
||||
# positional embeddings
|
||||
position_ids = position_ids_in_meshgrid(
|
||||
embeds_2d,
|
||||
max_width=self.image_size // self.patch_size,
|
||||
).to(self.device)
|
||||
# Axial rope indexes the (h, w) grid coordinates directly, so the ids are
|
||||
# per-patch pairs rather than the flattened `h * max_width + w` offsets.
|
||||
position_ids = position_meshgrid(embeds_2d).to(self.device)
|
||||
|
||||
# The original PixtralRotaryEmbedding expects 2D input but returns a tuple of tensors (cos, sin)
|
||||
# These tensors are used by apply_rotary_pos_emb in the transformer blocks
|
||||
position_embedding = self.patch_positional_embedding(
|
||||
embeds_featurized, position_ids
|
||||
)
|
||||
|
||||
@@ -499,7 +499,7 @@ class Qwen2_5_VisionTransformer(nn.Module, RotaryPosMixin):
|
||||
|
||||
pos_ids = torch.cat(pos_ids, dim=0)
|
||||
max_grid_size = int(grid_thw[:, 1:].max())
|
||||
# transformers 5.12's rotary forward takes 1-D position_ids on the input device (grid_thw is CPU).
|
||||
# The vision rotary forward takes 1-D position_ids on the input device (grid_thw is CPU).
|
||||
rotary_pos_emb_full = self.rotary_pos_emb(
|
||||
torch.arange(max_grid_size, device=self.device)
|
||||
)
|
||||
|
||||
@@ -238,6 +238,9 @@ def _normalize_tp_style(style: str) -> Style:
|
||||
"packed_rowwise": "rowwise",
|
||||
"local_rowwise": "rowwise",
|
||||
"local_packed_rowwise": "rowwise",
|
||||
# transformers' tied-embedding `base_model_tp_plan`s give the embedding this
|
||||
# style; `replace_vocab_embed_class` shards it instead.
|
||||
"embedding_rowwise": "replicate",
|
||||
"isolated": "replicate",
|
||||
"local": "replicate",
|
||||
"replicated_with_grad_allreduce": "replicate",
|
||||
|
||||
@@ -270,6 +270,11 @@ try:
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
# Bare register: `get_config` re-parses every registry entry through its own
|
||||
# class, so AutoConfig need not resolve to it.
|
||||
#
|
||||
# `_LazyAutoMapping` keys on the config class `__name__`, so a differently-named
|
||||
# shadow drops out of PROCESSOR/TOKENIZER/MODEL_MAPPING.
|
||||
for name, cls in _CONFIG_REGISTRY.items():
|
||||
try:
|
||||
AutoConfig.register(name, cls)
|
||||
@@ -283,6 +288,9 @@ for name, cls in _CONFIG_REGISTRY.items():
|
||||
# flattened onto the top-level config in `get_config` — the same path the base
|
||||
# Qwen3-VL config relies on. Adding it to `_CONFIG_REGISTRY` would trigger a
|
||||
# `from_pretrained` reload that drops that flattening.
|
||||
#
|
||||
# transformers owns `cosmos3_omni` as `Cosmos3OmniConfig`, so the name
|
||||
# constraint above means this registration must lose to it.
|
||||
try:
|
||||
AutoConfig.register(Cosmos3Config.model_type, Cosmos3Config)
|
||||
except ValueError as e:
|
||||
@@ -295,6 +303,9 @@ except ValueError as e:
|
||||
# `_CONFIG_REGISTRY` so the generic parser can flatten text attributes onto the
|
||||
# root config after `AutoConfig.from_pretrained`, matching other multimodal
|
||||
# configs that use a text sub-config.
|
||||
#
|
||||
# `exist_ok=True`: no registry re-parse backs these up, so AutoConfig itself
|
||||
# must resolve to SGLang's. Safe because they reuse the native class names.
|
||||
for _cosmos3_edge_config_cls in (
|
||||
Cosmos3EdgeTextConfig,
|
||||
Cosmos3EdgeVisionConfig,
|
||||
@@ -303,16 +314,16 @@ for _cosmos3_edge_config_cls in (
|
||||
):
|
||||
try:
|
||||
AutoConfig.register(
|
||||
_cosmos3_edge_config_cls.model_type, _cosmos3_edge_config_cls
|
||||
_cosmos3_edge_config_cls.model_type,
|
||||
_cosmos3_edge_config_cls,
|
||||
exist_ok=True,
|
||||
)
|
||||
except ValueError as e:
|
||||
err = str(e).lower()
|
||||
if "already registered" not in err and "already used" not in err:
|
||||
logger.warning(
|
||||
"Failed to register config %s: %s",
|
||||
_cosmos3_edge_config_cls.model_type,
|
||||
e,
|
||||
)
|
||||
logger.warning(
|
||||
"Failed to register config %s: %s",
|
||||
_cosmos3_edge_config_cls.model_type,
|
||||
e,
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -58,6 +58,93 @@ def _apply_deepseek_ocr_overrides(config, model):
|
||||
config._name_or_path = model
|
||||
|
||||
|
||||
_GEMMA4_MODEL_TYPES = (
|
||||
"gemma4",
|
||||
"gemma4_assistant",
|
||||
"gemma4_unified",
|
||||
"gemma4_unified_assistant",
|
||||
"diffusion_gemma",
|
||||
)
|
||||
|
||||
|
||||
_GEMMA4_FLATTENED_PER_LAYER_ATTRS = frozenset({"head_dim", "num_key_value_heads"})
|
||||
|
||||
|
||||
def _gemma4_attention_shapes(text_config) -> dict[str, tuple[int, int]]:
|
||||
# `_heterogeneity_spec` is the only statement of what the checkpoint
|
||||
# declared per layer; `is_heterogeneous` is a `hasattr` of it.
|
||||
spec = text_config._heterogeneity_spec
|
||||
unsupported = set(spec.per_layer_attributes) - _GEMMA4_FLATTENED_PER_LAYER_ATTRS
|
||||
# `per_layer_attributes` discards `skip`, so it needs its own check.
|
||||
if any("skip" in overrides for overrides in spec.per_layer_overrides.values()):
|
||||
unsupported.add("skip")
|
||||
if unsupported:
|
||||
raise ValueError(
|
||||
f"Gemma4 config declares per-layer {sorted(unsupported)}, which SGLang "
|
||||
f"cannot express: it carries one full-attention and one sliding-window "
|
||||
f"shape, and flattens only {sorted(_GEMMA4_FLATTENED_PER_LAYER_ATTRS)}."
|
||||
)
|
||||
|
||||
# Indexing the view by layer type demands that the whole type be uniform,
|
||||
# and reading the global config raises while the spec is attached.
|
||||
per_layer = text_config.per_layer_config
|
||||
shapes: dict[str, set] = {}
|
||||
for layer_idx, layer_type in enumerate(text_config.layer_types):
|
||||
layer = per_layer[layer_idx]
|
||||
shapes.setdefault(layer_type, set()).add(
|
||||
(layer.head_dim, layer.num_key_value_heads)
|
||||
)
|
||||
|
||||
for layer_type, layer_shapes in shapes.items():
|
||||
if len(layer_shapes) > 1:
|
||||
raise ValueError(
|
||||
f"Gemma4 config gives its '{layer_type}' layers more than one "
|
||||
f"(head_dim, num_key_value_heads): {sorted(layer_shapes)}. SGLang "
|
||||
f"carries a single shape per layer type."
|
||||
)
|
||||
|
||||
return {layer_type: shape.pop() for layer_type, shape in shapes.items()}
|
||||
|
||||
|
||||
def _apply_gemma4_attention_overrides(config):
|
||||
# Gemma4 states its shapes SWA-first: base attributes are the sliding-window
|
||||
# values and full-attention overrides them; SGLang's base is full attention.
|
||||
text_config = config.text_config
|
||||
|
||||
if text_config.is_heterogeneous:
|
||||
# transformers states that split as a `per_layer_config`, and
|
||||
# consumes `global_head_dim` / `num_global_key_value_heads` building it.
|
||||
shapes = _gemma4_attention_shapes(text_config)
|
||||
full_head_dim, full_kv_heads = shapes["full_attention"]
|
||||
# The last layer is forced to full attention, so that key is always
|
||||
# present; a model without sliding layers leaves `swa_*` unused.
|
||||
swa_head_dim, swa_kv_heads = shapes.get(
|
||||
"sliding_attention", shapes["full_attention"]
|
||||
)
|
||||
# Dropping the spec discards nothing once the two attributes it may
|
||||
# carry are flattened, and every read of `head_dim` raises until it is.
|
||||
text_config.per_layer_config = None
|
||||
else:
|
||||
# transformers pops `global_head_dim` / `num_global_key_value_heads`
|
||||
# whether or not it builds a spec, so there is no split left to recover.
|
||||
full_head_dim = swa_head_dim = text_config.head_dim
|
||||
full_kv_heads = swa_kv_heads = text_config.num_key_value_heads
|
||||
|
||||
text_config.head_dim = full_head_dim
|
||||
text_config.num_key_value_heads = full_kv_heads
|
||||
text_config.swa_head_dim = swa_head_dim
|
||||
text_config.swa_v_head_dim = swa_head_dim
|
||||
text_config.swa_num_key_value_heads = swa_kv_heads
|
||||
|
||||
if not hasattr(text_config, "v_head_dim"):
|
||||
text_config.v_head_dim = text_config.head_dim
|
||||
|
||||
# Unified Gemma4 names the end-of-audio token `eoa_token_index`,
|
||||
# but the multimodal processor expects `eoa_token_id`.
|
||||
if not hasattr(config, "eoa_token_id") and hasattr(config, "eoa_token_index"):
|
||||
config.eoa_token_id = config.eoa_token_index
|
||||
|
||||
|
||||
_LONGCAT_ARCHS = {
|
||||
"LongcatCausalLM",
|
||||
"LongcatFlashForCausalLM",
|
||||
@@ -215,43 +302,8 @@ class HfModelConfigParser(ModelConfigParserBase):
|
||||
if config.model_type == "multi_modality":
|
||||
_set_architectures(config, "MultiModalityCausalLM")
|
||||
|
||||
if config.model_type in (
|
||||
"gemma4",
|
||||
"gemma4_assistant",
|
||||
"gemma4_unified",
|
||||
"gemma4_unified_assistant",
|
||||
"diffusion_gemma",
|
||||
):
|
||||
# Gemma4 configs use base attributes for SWA layers and `global_*`
|
||||
# variants for full-attention layers. SGLang expects the opposite:
|
||||
# base = full-attention, `swa_*` = sliding-window overrides.
|
||||
text_config = config.text_config
|
||||
global_head_dim = getattr(text_config, "global_head_dim", None)
|
||||
global_kv_heads = getattr(text_config, "num_global_key_value_heads", None)
|
||||
|
||||
swa_head_dim = text_config.head_dim
|
||||
swa_kv_heads = text_config.num_key_value_heads
|
||||
|
||||
text_config.swa_head_dim = swa_head_dim
|
||||
text_config.swa_v_head_dim = swa_head_dim
|
||||
text_config.swa_num_key_value_heads = swa_kv_heads
|
||||
|
||||
if global_head_dim is not None:
|
||||
text_config.head_dim = global_head_dim
|
||||
if global_kv_heads is not None:
|
||||
text_config.num_key_value_heads = global_kv_heads
|
||||
|
||||
if not hasattr(text_config, "v_head_dim"):
|
||||
text_config.v_head_dim = text_config.head_dim
|
||||
if not hasattr(text_config, "swa_v_head_dim"):
|
||||
text_config.swa_v_head_dim = text_config.swa_head_dim
|
||||
|
||||
# Unified Gemma4 names the end-of-audio token `eoa_token_index`,
|
||||
# but the multimodal processor expects `eoa_token_id`.
|
||||
if not hasattr(config, "eoa_token_id") and hasattr(
|
||||
config, "eoa_token_index"
|
||||
):
|
||||
config.eoa_token_id = config.eoa_token_index
|
||||
if config.model_type in _GEMMA4_MODEL_TYPES:
|
||||
_apply_gemma4_attention_overrides(config)
|
||||
|
||||
if config.model_type == "longcat_flash":
|
||||
_set_architectures(config, "LongcatFlashForCausalLM")
|
||||
|
||||
@@ -497,7 +497,8 @@ def patch_mistral_common_tokenizer(tokenizer):
|
||||
if not hasattr(tokenizer, "get_added_vocab"):
|
||||
tokenizer.get_added_vocab = lambda: {}
|
||||
|
||||
# Keep the old no-op pad add working on transformers 5.12 MistralCommon.
|
||||
# MistralCommonBackend.add_special_tokens raises NotImplementedError by design;
|
||||
# a pad-token-only call sets `pad_token` directly and adds nothing.
|
||||
_orig_add_special_tokens = tokenizer.add_special_tokens
|
||||
|
||||
def _safe_add_special_tokens(special_tokens_dict, *args, **kwargs):
|
||||
|
||||
@@ -61,13 +61,12 @@ def apply_all():
|
||||
# v5.4 patches
|
||||
_patch_flash_attn_availability()
|
||||
_patch_rope_parameters_validation()
|
||||
_patch_layer_types_validation()
|
||||
_patch_removed_symbols()
|
||||
_patch_image_processor_kwargs()
|
||||
_patch_image_process_cuda_tensor()
|
||||
_patch_nemotron_h_pattern()
|
||||
|
||||
# v5 general patches
|
||||
_ensure_clean_up_tokenization_compat()
|
||||
_ensure_is_torch_fx_available_compat()
|
||||
|
||||
# CI-only: neutralize HF API calls inside tokenizer from_pretrained
|
||||
@@ -164,21 +163,21 @@ def _patch_rope_parameters_validation():
|
||||
For ``PretrainedConfig``, ``standardize_rope_params()`` accesses
|
||||
``self.max_position_embeddings`` during ``__post_init__`` before extra
|
||||
kwargs are set as attributes, causing ``AttributeError``.
|
||||
|
||||
Fix: guard ``standardize_rope_params`` against missing
|
||||
``max_position_embeddings``.
|
||||
"""
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
# standardize_rope_params accesses self.max_position_embeddings before
|
||||
# __post_init__ sets extra kwargs — skip when the attribute is absent.
|
||||
if hasattr(PretrainedConfig, "standardize_rope_params"):
|
||||
_orig_standardize = PretrainedConfig.standardize_rope_params
|
||||
|
||||
def _safe_standardize(self):
|
||||
if not hasattr(self, "max_position_embeddings"):
|
||||
return
|
||||
return _orig_standardize(self)
|
||||
# The call must still run: it resolves `default_rope_type`, which
|
||||
# Pixtral's vision config needs to reach "axial".
|
||||
try:
|
||||
return _orig_standardize(self)
|
||||
except AttributeError as e:
|
||||
if "max_position_embeddings" not in str(e):
|
||||
raise
|
||||
return None
|
||||
|
||||
PretrainedConfig.standardize_rope_params = _safe_standardize
|
||||
|
||||
@@ -205,21 +204,47 @@ def _patch_flash_attn_availability():
|
||||
pass
|
||||
|
||||
|
||||
def _patch_layer_types_validation():
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
validators = PretrainedConfig.__class_validators__
|
||||
for index, validator in enumerate(validators):
|
||||
if validator.__name__ != "validate_layer_type":
|
||||
continue
|
||||
|
||||
def validate_layer_type(self, _orig=validator):
|
||||
try:
|
||||
return _orig(self)
|
||||
except ValueError as e:
|
||||
# Step-3.5-Flash lists a `layer_types` entry per main *and*
|
||||
# next-n-predict layer, which `validate_layer_type` rejects.
|
||||
if "must be equal to the number of" not in str(e):
|
||||
raise
|
||||
num_mtp_layers = getattr(self, "num_nextn_predict_layers", 0) or 0
|
||||
if not num_mtp_layers:
|
||||
raise
|
||||
# Re-run the original against the wider count, so its other
|
||||
# rules -- per-list vocabularies, legacy remapping -- still hold.
|
||||
num_hidden_layers = self.num_hidden_layers
|
||||
self.num_hidden_layers = num_hidden_layers + num_mtp_layers
|
||||
try:
|
||||
return _orig(self)
|
||||
finally:
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
|
||||
validators[index] = validate_layer_type
|
||||
break
|
||||
|
||||
|
||||
def _patch_removed_symbols():
|
||||
"""Re-export symbols removed in transformers v5.4.0.
|
||||
"""Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0.
|
||||
|
||||
Remote model code (e.g. DeepSeek-OCR) still imports these.
|
||||
Remote model code (e.g. DeepSeek-OCR) still imports it.
|
||||
``check_imports`` in ``dynamic_module_utils.py`` validates imports at
|
||||
config-load time, so these must exist before any ``from_pretrained``.
|
||||
|
||||
Removed symbols:
|
||||
- ``LlamaFlashAttention2`` -- replaced by unified ``LlamaAttention``
|
||||
- ``is_flash_attn_greater_or_equal_2_10`` -- replaced by
|
||||
``is_flash_attn_greater_or_equal("2.10.0")``
|
||||
config-load time, so it must exist before any ``from_pretrained``.
|
||||
|
||||
TODO(upstream): DeepSeek-OCR / deepseek_vl_v2 remote code needs update.
|
||||
"""
|
||||
# LlamaFlashAttention2
|
||||
try:
|
||||
import logging
|
||||
|
||||
@@ -251,23 +276,6 @@ def _patch_removed_symbols():
|
||||
"LlamaFlashAttention2 compat patch not applied."
|
||||
)
|
||||
|
||||
# is_flash_attn_greater_or_equal_2_10
|
||||
try:
|
||||
import transformers.utils as _u
|
||||
|
||||
if not hasattr(_u, "is_flash_attn_greater_or_equal_2_10"):
|
||||
if hasattr(_u, "is_flash_attn_greater_or_equal"):
|
||||
_u.is_flash_attn_greater_or_equal_2_10 = lambda: (
|
||||
_u.is_flash_attn_greater_or_equal("2.10.0")
|
||||
)
|
||||
else:
|
||||
_u.is_flash_attn_greater_or_equal_2_10 = lambda: False
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Could not import transformers.utils; "
|
||||
"is_flash_attn_greater_or_equal_2_10 compat patch not applied."
|
||||
)
|
||||
|
||||
|
||||
def _patch_image_processor_kwargs():
|
||||
"""Allow remote image processors that lack ``**kwargs`` in preprocess().
|
||||
@@ -363,78 +371,11 @@ def _patch_image_process_cuda_tensor():
|
||||
)
|
||||
|
||||
|
||||
def _patch_nemotron_h_pattern():
|
||||
"""Fix ``_pattern_to_list()`` crashing on ``-`` in hybrid_override_pattern.
|
||||
|
||||
Nemotron-H models (e.g. NVIDIA-Nemotron-Nano-9B-v2) use patterns like
|
||||
``M-M-M-MM-M-*-...`` where ``-`` denotes an MLP layer. The upstream
|
||||
``_pattern_to_list`` tries to map every character and crashes with
|
||||
``KeyError: '-'``. We skip ``-`` (and any other unmapped chars)
|
||||
since ``layers_block_type`` only tracks mamba/moe/attention layers.
|
||||
SGLang reads MLP positions from ``hybrid_override_pattern`` directly.
|
||||
|
||||
TODO(upstream): report to HF transformers.
|
||||
"""
|
||||
try:
|
||||
from transformers.models.nemotron_h.configuration_nemotron_h import (
|
||||
NemotronHConfig,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _pattern_to_list(pattern: str) -> list:
|
||||
pattern_mapping = {
|
||||
"M": "mamba",
|
||||
"E": "moe",
|
||||
"*": "attention",
|
||||
}
|
||||
return [
|
||||
pattern_mapping[char] for char in pattern if char in pattern_mapping
|
||||
]
|
||||
|
||||
NemotronHConfig._pattern_to_list = _pattern_to_list
|
||||
except ImportError:
|
||||
logger.debug(
|
||||
"_patch_nemotron_h_pattern: NemotronHConfig not importable, patch skipped"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# v5 general patches
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _ensure_clean_up_tokenization_compat() -> None:
|
||||
"""Re-add ``clean_up_tokenization`` removed in transformers v5.
|
||||
|
||||
Remote-code tokenizers (e.g. InternLM2Tokenizer) call
|
||||
``self.clean_up_tokenization()`` which was a static method on
|
||||
``PreTrainedTokenizerBase`` in v4 but removed in v5. Patch it back
|
||||
so existing HuggingFace Hub tokenizer code keeps working.
|
||||
"""
|
||||
from transformers import PreTrainedTokenizerBase
|
||||
|
||||
if hasattr(PreTrainedTokenizerBase, "clean_up_tokenization"):
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def clean_up_tokenization(out_string: str) -> str:
|
||||
out_string = (
|
||||
out_string.replace(" .", ".")
|
||||
.replace(" ?", "?")
|
||||
.replace(" !", "!")
|
||||
.replace(" ,", ",")
|
||||
.replace(" ' ", "'")
|
||||
.replace(" n't", "n't")
|
||||
.replace(" 'm", "'m")
|
||||
.replace(" 's", "'s")
|
||||
.replace(" 've", "'ve")
|
||||
.replace(" 're", "'re")
|
||||
)
|
||||
return out_string
|
||||
|
||||
PreTrainedTokenizerBase.clean_up_tokenization = clean_up_tokenization
|
||||
|
||||
|
||||
def _ensure_is_torch_fx_available_compat() -> None:
|
||||
"""Re-add ``is_torch_fx_available`` removed in transformers v5.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user