[Fix] don't force hybrid-SWA when sliding_window is disabled (#28604)
This commit is contained in:
@@ -137,10 +137,7 @@ class LagunaConfig(PretrainedConfig):
|
|||||||
self.layer_types = (
|
self.layer_types = (
|
||||||
list(layer_types)
|
list(layer_types)
|
||||||
if layer_types
|
if layer_types
|
||||||
else [
|
else ["full_attention" for _ in range(num_hidden_layers)]
|
||||||
"full_attention" if i % 4 == 0 else "sliding_attention"
|
|
||||||
for i in range(num_hidden_layers)
|
|
||||||
]
|
|
||||||
)
|
)
|
||||||
self.mlp_layer_types = (
|
self.mlp_layer_types = (
|
||||||
list(mlp_layer_types)
|
list(mlp_layer_types)
|
||||||
|
|||||||
@@ -1724,6 +1724,13 @@ def is_hybrid_swa_model(
|
|||||||
"LagunaForCausalLM",
|
"LagunaForCausalLM",
|
||||||
}
|
}
|
||||||
if any(arch in hybrid_swa_archs for arch in model_architectures):
|
if any(arch in hybrid_swa_archs for arch in model_architectures):
|
||||||
|
# Only treat Laguna as hybrid SWA when it actually has a sliding window.
|
||||||
|
if (
|
||||||
|
"LagunaForCausalLM" in model_architectures
|
||||||
|
and hf_text_config is not None
|
||||||
|
and not getattr(hf_text_config, "sliding_window", 0)
|
||||||
|
):
|
||||||
|
return False
|
||||||
return True
|
return True
|
||||||
# Also recognize models that explicitly opt-in via their HF text config,
|
# Also recognize models that explicitly opt-in via their HF text config,
|
||||||
# so custom hybrid-SWA architectures don't need to be added to the allowlist.
|
# so custom hybrid-SWA architectures don't need to be added to the allowlist.
|
||||||
|
|||||||
Reference in New Issue
Block a user