[Feat] [CPU] model: Support Muse models on CPU (#44624)
Co-authored-by: Niu Ziru <niuziru@gnr799217.jf.intel.com>
This commit is contained in:
@@ -51,12 +51,17 @@ class MuseGlimmerVisionConfig(PretrainedConfig):
|
||||
pos_emb_height: int = 32,
|
||||
pos_emb_width: int = 32,
|
||||
rope_theta: float = 10000.0,
|
||||
head_dim: Optional[int] = None,
|
||||
**kwargs,
|
||||
):
|
||||
self.hidden_size = hidden_size
|
||||
self.intermediate_size = intermediate_size
|
||||
self.num_hidden_layers = num_hidden_layers
|
||||
self.num_attention_heads = num_attention_heads
|
||||
# Pinned so it survives CPU TP padding of num_attention_heads.
|
||||
self.head_dim = (
|
||||
head_dim if head_dim is not None else hidden_size // num_attention_heads
|
||||
)
|
||||
self.hidden_act = hidden_act
|
||||
self.layer_norm_eps = layer_norm_eps
|
||||
self.attention_types = attention_types
|
||||
@@ -140,6 +145,7 @@ class MuseGlimmerConfig(PretrainedConfig):
|
||||
self.out_hidden_size = out_hidden_size
|
||||
self.projector_hidden_act = projector_hidden_act
|
||||
self.projector_hidden_size = projector_hidden_size
|
||||
self.padded_projector_hidden_size = projector_hidden_size
|
||||
super().__init__(
|
||||
tie_word_embeddings=tie_word_embeddings,
|
||||
bos_token_id=bos_token_id,
|
||||
|
||||
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from sglang.srt.utils import (
|
||||
log_debug_on_rank0,
|
||||
round_up,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -302,6 +303,17 @@ def adjust_config_with_unaligned_cpu_tp(
|
||||
model_config, moe_intermediate_attr, intermediate_padding_size
|
||||
)
|
||||
|
||||
if model_config.hf_config.model_type == "muse_glimmer":
|
||||
projector_hidden_size = model_config.hf_config.projector_hidden_size
|
||||
padded_projector_hidden_size = projector_hidden_size
|
||||
if projector_hidden_size % tp_size != 0:
|
||||
padded_projector_hidden_size = round_up(projector_hidden_size, tp_size * 32)
|
||||
update_config(
|
||||
model_config.hf_config,
|
||||
"padded_projector_hidden_size",
|
||||
padded_projector_hidden_size,
|
||||
)
|
||||
|
||||
multimodal_config = [
|
||||
[
|
||||
model_config.hf_config,
|
||||
@@ -316,6 +328,12 @@ def adjust_config_with_unaligned_cpu_tp(
|
||||
[model_config.hf_config, "vision_config", "qwen3_5", "num_heads"],
|
||||
[model_config.hf_config, "vision_config", "qwen4_exp", "num_heads"],
|
||||
[model_config.hf_config, "vision_config", "mllama", "attention_heads"],
|
||||
[
|
||||
model_config.hf_config,
|
||||
"vision_config",
|
||||
"muse_glimmer_vision",
|
||||
"num_attention_heads",
|
||||
],
|
||||
[
|
||||
model_config.hf_config,
|
||||
"vision_config",
|
||||
|
||||
@@ -595,7 +595,8 @@ class MuseGlimmerVisionEncoderLayer(nn.Module):
|
||||
self.attn = VisionAttention(
|
||||
embed_dim=config.hidden_size,
|
||||
num_heads=config.num_attention_heads,
|
||||
projection_size=config.hidden_size,
|
||||
head_dim=config.head_dim,
|
||||
projection_size=config.num_attention_heads * config.head_dim,
|
||||
use_qkv_parallel=True,
|
||||
qkv_bias=True,
|
||||
proj_bias=True,
|
||||
@@ -644,7 +645,7 @@ class MuseGlimmerVisionModel(nn.Module):
|
||||
prefix=add_prefix("patch_embedder", prefix),
|
||||
)
|
||||
self.rotary_emb = MuseGlimmerVisionRotaryEmbedding(
|
||||
config.hidden_size // config.num_attention_heads, config.rope_theta
|
||||
config.head_dim, config.rope_theta
|
||||
)
|
||||
self.ln_pre = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
|
||||
self.layers = nn.ModuleList(
|
||||
@@ -772,15 +773,16 @@ class MuseGlimmerVisionAdapter(nn.Module):
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
sharded_size = config.padded_projector_hidden_size
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
config.out_hidden_size,
|
||||
config.projector_hidden_size,
|
||||
sharded_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
prefix=add_prefix("fc1", prefix),
|
||||
)
|
||||
self.fc2 = RowParallelLinear(
|
||||
config.projector_hidden_size,
|
||||
sharded_size,
|
||||
config.projector_hidden_size,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
|
||||
Reference in New Issue
Block a user