[bugfix] Fix rope theta config for MiniMax after transformers v5 update (#21241)
This commit is contained in:
@@ -73,6 +73,7 @@ from sglang.srt.utils import (
|
|||||||
is_non_idle_and_non_empty,
|
is_non_idle_and_non_empty,
|
||||||
make_layers,
|
make_layers,
|
||||||
)
|
)
|
||||||
|
from sglang.srt.utils.hf_transformers_utils import get_rope_config
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -570,7 +571,7 @@ class MiniMaxM2Attention(nn.Module):
|
|||||||
|
|
||||||
# RoPE settings - support partial RoPE
|
# RoPE settings - support partial RoPE
|
||||||
# FIXME: minimax_m2 config use external config that not compatible with transformers v5
|
# FIXME: minimax_m2 config use external config that not compatible with transformers v5
|
||||||
self.rope_theta = config.rope_theta
|
self.rope_theta, self.rope_scaling = get_rope_config(config)
|
||||||
self.max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
self.max_position_embeddings = getattr(config, "max_position_embeddings", 8192)
|
||||||
self.rotary_dim = getattr(
|
self.rotary_dim = getattr(
|
||||||
config, "rotary_dim", self.head_dim
|
config, "rotary_dim", self.head_dim
|
||||||
@@ -600,13 +601,12 @@ class MiniMaxM2Attention(nn.Module):
|
|||||||
)
|
)
|
||||||
|
|
||||||
# Setup RoPE with partial rotary dimension
|
# Setup RoPE with partial rotary dimension
|
||||||
rope_scaling = getattr(config, "rope_scaling", None)
|
|
||||||
self.rotary_emb = get_rope(
|
self.rotary_emb = get_rope(
|
||||||
self.head_dim,
|
self.head_dim,
|
||||||
rotary_dim=self.rotary_dim, # Use partial rotary dimension
|
rotary_dim=self.rotary_dim, # Use partial rotary dimension
|
||||||
max_position=self.max_position_embeddings,
|
max_position=self.max_position_embeddings,
|
||||||
base=self.rope_theta,
|
base=self.rope_theta,
|
||||||
rope_scaling=rope_scaling,
|
rope_scaling=self.rope_scaling,
|
||||||
)
|
)
|
||||||
|
|
||||||
# QK Normalization layers
|
# QK Normalization layers
|
||||||
|
|||||||
Reference in New Issue
Block a user