[Fix ] Fix Spark2.5 hybrid SWA config (#37335)

2026-09-01 20:29:09 -07:00
parent 2d9c64394f
commit a58751d54d
1 changed files with 5 additions and 1 deletions
@@ -59,5 +59,9 @@ class Spark2_5Config(PretrainedConfig):
                    "partial_rotary_factor": 1.0,
                },
            }
-
+        self.hybrid_layer_pattern = [
+            1 if layer_type == "sliding_attention" else 0
+            for layer_type in self.layer_types
+        ]
+        self.is_hybrid_swa = any(self.hybrid_layer_pattern)
        super().__init__(**kwargs, tie_word_embeddings=tie_word_embeddings)