[Fix] Validate per-layer rope_parameters on transformers 5.17 (Laguna-S/XS-2.1 nightly) (#42214)
This commit is contained in:
@@ -62,6 +62,7 @@ def apply_all():
|
||||
_patch_flash_attn_availability()
|
||||
_patch_rope_parameters_validation()
|
||||
_patch_layer_types_validation()
|
||||
_patch_nested_rope_validation()
|
||||
_patch_removed_symbols()
|
||||
_patch_image_processor_kwargs()
|
||||
_patch_image_process_cuda_tensor()
|
||||
@@ -236,6 +237,36 @@ def _patch_layer_types_validation():
|
||||
break
|
||||
|
||||
|
||||
def _patch_nested_rope_validation():
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
validators = PretrainedConfig.__class_validators__
|
||||
for index, validator in enumerate(validators):
|
||||
if validator.__name__ != "validate_rope":
|
||||
continue
|
||||
|
||||
def validate_rope(self, _orig=validator):
|
||||
try:
|
||||
return _orig(self)
|
||||
except AttributeError:
|
||||
# Backport of huggingface/transformers#48798; drop once transformers >= 5.18.
|
||||
rope_parameters = getattr(self, "rope_parameters", None)
|
||||
layer_types = getattr(self, "layer_types", None) or ()
|
||||
if not isinstance(rope_parameters, dict):
|
||||
raise
|
||||
nested = {k: v for k, v in rope_parameters.items() if k in layer_types}
|
||||
if not nested or len(nested) == len(rope_parameters):
|
||||
raise
|
||||
self.rope_parameters = nested
|
||||
try:
|
||||
return _orig(self)
|
||||
finally:
|
||||
self.rope_parameters = rope_parameters
|
||||
|
||||
validators[index] = validate_rope
|
||||
break
|
||||
|
||||
|
||||
def _patch_removed_symbols():
|
||||
"""Re-export ``LlamaFlashAttention2``, removed in transformers v5.4.0.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user