[Fix] don't force hybrid-SWA when sliding_window is disabled (#28604)

This commit is contained in:
Jimmy Shong
2026-06-17 22:11:27 -07:00
committed by GitHub
parent 4ee7882a46
commit d2539980b6
2 changed files with 8 additions and 4 deletions
+1 -4
View File
@@ -137,10 +137,7 @@ class LagunaConfig(PretrainedConfig):
self.layer_types = (
list(layer_types)
if layer_types
else [
"full_attention" if i % 4 == 0 else "sliding_attention"
for i in range(num_hidden_layers)
]
else ["full_attention" for _ in range(num_hidden_layers)]
)
self.mlp_layer_types = (
list(mlp_layer_types)
@@ -1724,6 +1724,13 @@ def is_hybrid_swa_model(
"LagunaForCausalLM",
}
if any(arch in hybrid_swa_archs for arch in model_architectures):
# Only treat Laguna as hybrid SWA when it actually has a sliding window.
if (
"LagunaForCausalLM" in model_architectures
and hf_text_config is not None
and not getattr(hf_text_config, "sliding_window", 0)
):
return False
return True
# Also recognize models that explicitly opt-in via their HF text config,
# so custom hybrid-SWA architectures don't need to be added to the allowlist.