[Fix] Adapt to transformers 5.18 / 5.19 changes (#43688)
Co-authored-by: Mohammad Angkad <mohammad.angkad@radixark.ai>
This commit is contained in:
co-authored by
Mohammad Angkad
parent
8ce76b88df
commit
166b4ae7fc
@@ -33,7 +33,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
|
||||
return np.concatenate([emb_sin, emb_cos], axis=1)
|
||||
|
||||
|
||||
# transformers 5.19 renamed Dinov2's attention projections and split its SwiGLU input
|
||||
# transformers 5.18 renamed Dinov2's attention projections and split its SwiGLU input
|
||||
# projection into gate/up. from_pretrained remaps legacy checkpoints on its own, but
|
||||
# Hunyuan3D builds Dinov2Model from a config and loads its own state dict, where a
|
||||
# non-strict load would silently leave these weights randomly initialized.
|
||||
@@ -48,7 +48,7 @@ _DINOV2_LEGACY_RENAMES = (
|
||||
|
||||
def _remap_legacy_dinov2_keys(module, state_dict, prefix, *args):
|
||||
if not hasattr(module.encoder.layer[0].attention, "q_proj"):
|
||||
return # transformers < 5.19 still uses the checkpoint's layout
|
||||
return # transformers < 5.18 still uses the checkpoint's layout
|
||||
layer_prefix = prefix + "encoder.layer."
|
||||
for key in [k for k in state_dict if k.startswith(layer_prefix)]:
|
||||
for old, new in _DINOV2_LEGACY_RENAMES:
|
||||
|
||||
@@ -97,10 +97,7 @@ def _probe_wo_a_weight_dtype(model_config: Any, download_dir: str | None) -> str
|
||||
from transformers.utils.hub import cached_file
|
||||
|
||||
model_path = model_config.model_path
|
||||
revision = (
|
||||
getattr(model_config.hf_config, "_commit_hash", None)
|
||||
or model_config.revision
|
||||
)
|
||||
revision = model_config.revision
|
||||
index_path = cached_file(
|
||||
model_path,
|
||||
"model.safetensors.index.json",
|
||||
|
||||
@@ -272,8 +272,7 @@ class GraniteMoeHybridConfig(PretrainedConfig):
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# HF's `PreTrainedConfig.validate_layer_type` rewrites layer_types to
|
||||
# transformers >= 5.15 schema, so map them back afterwards.
|
||||
# `PreTrainedConfig` rewrites legacy layer_types on init, so map them back.
|
||||
self.layer_types = _to_sglang_layer_types(self.layer_types)
|
||||
|
||||
@property
|
||||
|
||||
@@ -133,8 +133,13 @@ class HYV4Config(PreTrainedConfig):
|
||||
)
|
||||
if set(self.mlp_layer_types) - {"dense", "sparse"}:
|
||||
raise ValueError("mlp_layer_types only supports dense and sparse")
|
||||
if set(self.layer_types) != {"deepseek_sparse_attention"}:
|
||||
raise ValueError("HYV4 only supports deepseek_sparse_attention")
|
||||
if not set(self.layer_types) <= {
|
||||
"deepseek_sparse_attention",
|
||||
"indexed_attention",
|
||||
}:
|
||||
raise ValueError(
|
||||
"HYV4 only supports deepseek_sparse_attention (indexed_attention)"
|
||||
)
|
||||
if set(self.indexer_types) - {"full", "shared"}:
|
||||
raise ValueError("indexer_types only supports full and shared")
|
||||
if self.indexer_types[0] != "full":
|
||||
|
||||
@@ -85,7 +85,8 @@ class Qwen4ExpTextConfig(Qwen3NextConfig):
|
||||
return [
|
||||
(
|
||||
"attention"
|
||||
if layer_type in ("full_attention", "qwen_sparse_attention")
|
||||
if layer_type
|
||||
in ("full_attention", "qwen_sparse_attention", "indexed_attention")
|
||||
else layer_type
|
||||
)
|
||||
for layer_type in self.layer_types
|
||||
|
||||
@@ -181,7 +181,6 @@ def npu_wrapper_minimax_m3_video_preprocess(func):
|
||||
def _preprocess(
|
||||
self,
|
||||
videos: List[torch.Tensor],
|
||||
do_convert_rgb: bool,
|
||||
do_resize: bool,
|
||||
size: SizeDict,
|
||||
resample: PILImageResampling | InterpolationMode | int | None,
|
||||
|
||||
@@ -335,8 +335,16 @@ def _deepseek_model_config_from_gguf(reader: object) -> dict[str, Any]:
|
||||
def _write_deepseek_tokenizer_from_gguf(
|
||||
reader: object, output_dir: Path, config: dict[str, Any]
|
||||
) -> None:
|
||||
from tokenizers import AddedToken, Regex, normalizers, pre_tokenizers
|
||||
from transformers.integrations.ggml import convert_gguf_tokenizer
|
||||
from tokenizers import (
|
||||
AddedToken,
|
||||
Regex,
|
||||
Tokenizer,
|
||||
decoders,
|
||||
models,
|
||||
normalizers,
|
||||
pre_tokenizers,
|
||||
processors,
|
||||
)
|
||||
|
||||
tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.model")
|
||||
pre_tokenizer_type = _deepseek_gguf_value(reader, "tokenizer.ggml.pre")
|
||||
@@ -347,18 +355,21 @@ def _write_deepseek_tokenizer_from_gguf(
|
||||
)
|
||||
tokens = list(_deepseek_gguf_value(reader, "tokenizer.ggml.tokens"))
|
||||
token_types = list(_deepseek_gguf_value(reader, "tokenizer.ggml.token_type"))
|
||||
tokenizer_data = {
|
||||
"tokenizer_type": tokenizer_type,
|
||||
"tokens": tokens,
|
||||
"token_type": token_types,
|
||||
"merges": list(_deepseek_gguf_value(reader, "tokenizer.ggml.merges")),
|
||||
"bos_token_id": config["bos_token_id"],
|
||||
"eos_token_id": config["eos_token_id"],
|
||||
"pad_token_id": int(
|
||||
_deepseek_gguf_value(reader, "tokenizer.ggml.padding_token_id")
|
||||
),
|
||||
}
|
||||
tokenizer, _ = convert_gguf_tokenizer("gpt2", tokenizer_data)
|
||||
tokenizer = Tokenizer(
|
||||
models.BPE(
|
||||
vocab={token: index for index, token in enumerate(tokens)},
|
||||
merges=[
|
||||
tuple(merge.split(" "))
|
||||
for merge in _deepseek_gguf_value(reader, "tokenizer.ggml.merges")
|
||||
],
|
||||
dropout=None,
|
||||
continuing_subword_prefix="",
|
||||
end_of_word_suffix="",
|
||||
fuse_unk=False,
|
||||
)
|
||||
)
|
||||
tokenizer.decoder = decoders.ByteLevel()
|
||||
tokenizer.post_processor = processors.ByteLevel(trim_offsets=False)
|
||||
tokenizer.add_special_tokens(
|
||||
[
|
||||
AddedToken(token, normalized=False, special=True)
|
||||
|
||||
@@ -62,7 +62,6 @@ def apply_all():
|
||||
_patch_flash_attn_availability()
|
||||
_patch_rope_parameters_validation()
|
||||
_patch_layer_types_validation()
|
||||
_patch_nested_rope_validation()
|
||||
_patch_removed_symbols()
|
||||
_patch_image_processor_kwargs()
|
||||
_patch_image_process_cuda_tensor()
|
||||
@@ -237,36 +236,6 @@ def _patch_layer_types_validation():
|
||||
break
|
||||
|
||||
|
||||
def _patch_nested_rope_validation():
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
validators = PretrainedConfig.__class_validators__
|
||||
for index, validator in enumerate(validators):
|
||||
if validator.__name__ != "validate_rope":
|
||||
continue
|
||||
|
||||
def validate_rope(self, _orig=validator):
|
||||
try:
|
||||
return _orig(self)
|
||||
except AttributeError:
|
||||
# Backport of huggingface/transformers#48798; drop once transformers >= 5.18.
|
||||
rope_parameters = getattr(self, "rope_parameters", None)
|
||||
layer_types = getattr(self, "layer_types", None) or ()
|
||||
if not isinstance(rope_parameters, dict):
|
||||
raise
|
||||
nested = {k: v for k, v in rope_parameters.items() if k in layer_types}
|
||||
if not nested or len(nested) == len(rope_parameters):
|
||||
raise
|
||||
self.rope_parameters = nested
|
||||
try:
|
||||
return _orig(self)
|
||||
finally:
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
validators[index] = validate_rope
|
||||
break
|
||||
|
||||
|
||||
def _patch_removed_symbols():
|
||||
"""Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user