[Fix] don't force hybrid-SWA when sliding_window is disabled (#28604)
This commit is contained in:
@@ -137,10 +137,7 @@ class LagunaConfig(PretrainedConfig):
|
||||
self.layer_types = (
|
||||
list(layer_types)
|
||||
if layer_types
|
||||
else [
|
||||
"full_attention" if i % 4 == 0 else "sliding_attention"
|
||||
for i in range(num_hidden_layers)
|
||||
]
|
||||
else ["full_attention" for _ in range(num_hidden_layers)]
|
||||
)
|
||||
self.mlp_layer_types = (
|
||||
list(mlp_layer_types)
|
||||
|
||||
@@ -1724,6 +1724,13 @@ def is_hybrid_swa_model(
|
||||
"LagunaForCausalLM",
|
||||
}
|
||||
if any(arch in hybrid_swa_archs for arch in model_architectures):
|
||||
# Only treat Laguna as hybrid SWA when it actually has a sliding window.
|
||||
if (
|
||||
"LagunaForCausalLM" in model_architectures
|
||||
and hf_text_config is not None
|
||||
and not getattr(hf_text_config, "sliding_window", 0)
|
||||
):
|
||||
return False
|
||||
return True
|
||||
# Also recognize models that explicitly opt-in via their HF text config,
|
||||
# so custom hybrid-SWA architectures don't need to be added to the allowlist.
|
||||
|
||||
Reference in New Issue
Block a user