[Feat] [CPU] model: Support Muse models on CPU (#44624)

Co-authored-by: Niu Ziru <niuziru@gnr799217.jf.intel.com>
This commit is contained in:
Ziru
2026-10-19 13:12:32 +08:00
committed by GitHub
co-authored by Niu Ziru
parent 2b75048e2a
commit 9a93a17c08
3 changed files with 30 additions and 4 deletions
@@ -51,12 +51,17 @@ class MuseGlimmerVisionConfig(PretrainedConfig):
pos_emb_height: int = 32,
pos_emb_width: int = 32,
rope_theta: float = 10000.0,
head_dim: Optional[int] = None,
**kwargs,
):
self.hidden_size = hidden_size
self.intermediate_size = intermediate_size
self.num_hidden_layers = num_hidden_layers
self.num_attention_heads = num_attention_heads
# Pinned so it survives CPU TP padding of num_attention_heads.
self.head_dim = (
head_dim if head_dim is not None else hidden_size // num_attention_heads
)
self.hidden_act = hidden_act
self.layer_norm_eps = layer_norm_eps
self.attention_types = attention_types
@@ -140,6 +145,7 @@ class MuseGlimmerConfig(PretrainedConfig):
self.out_hidden_size = out_hidden_size
self.projector_hidden_act = projector_hidden_act
self.projector_hidden_size = projector_hidden_size
self.padded_projector_hidden_size = projector_hidden_size
super().__init__(
tie_word_embeddings=tie_word_embeddings,
bos_token_id=bos_token_id,
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
from sglang.srt.utils import (
log_debug_on_rank0,
round_up,
)
logger = logging.getLogger(__name__)
@@ -302,6 +303,17 @@ def adjust_config_with_unaligned_cpu_tp(
model_config, moe_intermediate_attr, intermediate_padding_size
)
if model_config.hf_config.model_type == "muse_glimmer":
projector_hidden_size = model_config.hf_config.projector_hidden_size
padded_projector_hidden_size = projector_hidden_size
if projector_hidden_size % tp_size != 0:
padded_projector_hidden_size = round_up(projector_hidden_size, tp_size * 32)
update_config(
model_config.hf_config,
"padded_projector_hidden_size",
padded_projector_hidden_size,
)
multimodal_config = [
[
model_config.hf_config,
@@ -316,6 +328,12 @@ def adjust_config_with_unaligned_cpu_tp(
[model_config.hf_config, "vision_config", "qwen3_5", "num_heads"],
[model_config.hf_config, "vision_config", "qwen4_exp", "num_heads"],
[model_config.hf_config, "vision_config", "mllama", "attention_heads"],
[
model_config.hf_config,
"vision_config",
"muse_glimmer_vision",
"num_attention_heads",
],
[
model_config.hf_config,
"vision_config",
+6 -4
View File
@@ -595,7 +595,8 @@ class MuseGlimmerVisionEncoderLayer(nn.Module):
self.attn = VisionAttention(
embed_dim=config.hidden_size,
num_heads=config.num_attention_heads,
projection_size=config.hidden_size,
head_dim=config.head_dim,
projection_size=config.num_attention_heads * config.head_dim,
use_qkv_parallel=True,
qkv_bias=True,
proj_bias=True,
@@ -644,7 +645,7 @@ class MuseGlimmerVisionModel(nn.Module):
prefix=add_prefix("patch_embedder", prefix),
)
self.rotary_emb = MuseGlimmerVisionRotaryEmbedding(
config.hidden_size // config.num_attention_heads, config.rope_theta
config.head_dim, config.rope_theta
)
self.ln_pre = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
self.layers = nn.ModuleList(
@@ -772,15 +773,16 @@ class MuseGlimmerVisionAdapter(nn.Module):
prefix: str = "",
):
super().__init__()
sharded_size = config.padded_projector_hidden_size
self.fc1 = ColumnParallelLinear(
config.out_hidden_size,
config.projector_hidden_size,
sharded_size,
bias=False,
quant_config=quant_config,
prefix=add_prefix("fc1", prefix),
)
self.fc2 = RowParallelLinear(
config.projector_hidden_size,
sharded_size,
config.projector_hidden_size,
bias=False,
quant_config=quant_config,