@@ -1217,10 +1217,7 @@ class DeepseekV2AttentionMLA(
|
|||||||
)
|
)
|
||||||
|
|
||||||
if rope_scaling and rope_scaling.get("apply_yarn_scaling", True):
|
if rope_scaling and rope_scaling.get("apply_yarn_scaling", True):
|
||||||
mscale_all_dim = rope_scaling.get("mscale_all_dim", False)
|
self.scaling = compute_mla_mscale_scaling(rope_scaling, self.scaling)
|
||||||
scaling_factor = rope_scaling["factor"]
|
|
||||||
mscale = yarn_get_mscale(scaling_factor, float(mscale_all_dim))
|
|
||||||
self.scaling = self.scaling * mscale * mscale
|
|
||||||
else:
|
else:
|
||||||
self.rotary_emb = None
|
self.rotary_emb = None
|
||||||
self.use_deepseek_yarn_rope = rope_scaling is not None
|
self.use_deepseek_yarn_rope = rope_scaling is not None
|
||||||
|
|||||||
Reference in New Issue
Block a user