[Fix] Adapt to transformers 5.18 / 5.19 changes (#43688)

Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
Mohammad Miadh Angkad
2026-10-11 23:24:57 -07:00
committed by GitHub
co-authored by Mohammad Angkad
parent 8ce76b88df
commit 166b4ae7fc
9 changed files with 39 additions and 58 deletions
@@ -33,7 +33,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
return np.concatenate([emb_sin, emb_cos], axis=1)
# transformers 5.19 renamed Dinov2's attention projections and split its SwiGLU input
# transformers 5.18 renamed Dinov2's attention projections and split its SwiGLU input
# projection into gate/up. from_pretrained remaps legacy checkpoints on its own, but
# Hunyuan3D builds Dinov2Model from a config and loads its own state dict, where a
# non-strict load would silently leave these weights randomly initialized.
@@ -48,7 +48,7 @@ _DINOV2_LEGACY_RENAMES = (
def _remap_legacy_dinov2_keys(module, state_dict, prefix, *args):
if not hasattr(module.encoder.layer[0].attention, "q_proj"):
return # transformers < 5.19 still uses the checkpoint's layout
return # transformers < 5.18 still uses the checkpoint's layout
layer_prefix = prefix + "encoder.layer."
for key in [k for k in state_dict if k.startswith(layer_prefix)]:
for old, new in _DINOV2_LEGACY_RENAMES:
+1 -4
View File
@@ -97,10 +97,7 @@ def _probe_wo_a_weight_dtype(model_config: Any, download_dir: str | None) -> str
from transformers.utils.hub import cached_file
model_path = model_config.model_path
revision = (
getattr(model_config.hf_config, "_commit_hash", None)
or model_config.revision
)
revision = model_config.revision
index_path = cached_file(
model_path,
"model.safetensors.index.json",
@@ -272,8 +272,7 @@ class GraniteMoeHybridConfig(PretrainedConfig):
**kwargs,
)
# HF's `PreTrainedConfig.validate_layer_type` rewrites layer_types to
# transformers >= 5.15 schema, so map them back afterwards.
# `PreTrainedConfig` rewrites legacy layer_types on init, so map them back.
self.layer_types = _to_sglang_layer_types(self.layer_types)
@property
+7 -2
View File
@@ -133,8 +133,13 @@ class HYV4Config(PreTrainedConfig):
)
if set(self.mlp_layer_types) - {"dense", "sparse"}:
raise ValueError("mlp_layer_types only supports dense and sparse")
if set(self.layer_types) != {"deepseek_sparse_attention"}:
raise ValueError("HYV4 only supports deepseek_sparse_attention")
if not set(self.layer_types) <= {
"deepseek_sparse_attention",
"indexed_attention",
}:
raise ValueError(
"HYV4 only supports deepseek_sparse_attention (indexed_attention)"
)
if set(self.indexer_types) - {"full", "shared"}:
raise ValueError("indexer_types only supports full and shared")
if self.indexer_types[0] != "full":
+2 -1
View File
@@ -85,7 +85,8 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
return [
(
"attention"
if layer_type in ("full_attention", "qwen_sparse_attention")
if layer_type
in ("full_attention", "qwen_sparse_attention", "indexed_attention")
else layer_type
)
for layer_type in self.layer_types
@@ -181,7 +181,6 @@ def npu_wrapper_minimax_m3_video_preprocess(func):
def _preprocess(
self,
videos: List[torch.Tensor],
do_convert_rgb: bool,
do_resize: bool,
size: SizeDict,
resample: PILImageResampling | InterpolationMode | int | None,
@@ -335,8 +335,16 @@ def _deepseek_model_config_from_gguf(reader: object) -> dict[str, Any]:
def _write_deepseek_tokenizer_from_gguf(
reader: object, output_dir: Path, config: dict[str, Any]
) -> None:
from tokenizers import AddedToken, Regex, normalizers, pre_tokenizers
from transformers.integrations.ggml import convert_gguf_tokenizer
from tokenizers import (
AddedToken,
Regex,
Tokenizer,
decoders,
models,
normalizers,
pre_tokenizers,
processors,
)
tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.model")
pre_tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.pre")
@@ -347,18 +355,21 @@ def _write_deepseek_tokenizer_from_gguf(
)
tokens = list(_deepseek_gguf_value(reader, "tokenizer.ggml.tokens"))
token_types = list(_deepseek_gguf_value(reader, "tokenizer.ggml.token_type"))
tokenizer_data = {
"tokenizer_type": tokenizer_type,
"tokens": tokens,
"token_type": token_types,
"merges": list(_deepseek_gguf_value(reader, "tokenizer.ggml.merges")),
"bos_token_id": config["bos_token_id"],
"eos_token_id": config["eos_token_id"],
"pad_token_id": int(
_deepseek_gguf_value(reader, "tokenizer.ggml.padding_token_id")
),
}
tokenizer, _ = convert_gguf_tokenizer("gpt2", tokenizer_data)
tokenizer = Tokenizer(
models.BPE(
vocab={token: index for index, token in enumerate(tokens)},
merges=[
tuple(merge.split(" "))
for merge in _deepseek_gguf_value(reader, "tokenizer.ggml.merges")
],
dropout=None,
continuing_subword_prefix="",
end_of_word_suffix="",
fuse_unk=False,
)
)
tokenizer.decoder = decoders.ByteLevel()
tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
tokenizer.add_special_tokens(
[
AddedToken(token, normalized=False, special=True)
@@ -62,7 +62,6 @@ def apply_all():
_patch_flash_attn_availability()
_patch_rope_parameters_validation()
_patch_layer_types_validation()
_patch_nested_rope_validation()
_patch_removed_symbols()
_patch_image_processor_kwargs()
_patch_image_process_cuda_tensor()
@@ -237,36 +236,6 @@ def _patch_layer_types_validation():
break
def _patch_nested_rope_validation():
from transformers import PretrainedConfig
validators = PretrainedConfig.__class_validators__
for index, validator in enumerate(validators):
if validator.__name__ != "validate_rope":
continue
def validate_rope(self, _orig=validator):
try:
return _orig(self)
except AttributeError:
# Backport of huggingface/transformers#48798; drop once transformers >= 5.18.
rope_parameters = getattr(self, "rope_parameters", None)
layer_types = getattr(self, "layer_types", None) or ()
if not isinstance(rope_parameters, dict):
raise
nested = {k: v for k, v in rope_parameters.items() if k in layer_types}
if not nested or len(nested) == len(rope_parameters):
raise
self.rope_parameters = nested
try:
return _orig(self)
finally:
self.rope_parameters = rope_parameters
validators[index] = validate_rope
break
def _patch_removed_symbols():
"""Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0.