From 9a93a17c080bc99cf45eaf93baa41bd7a8cfce77 Mon Sep 17 00:00:00 2001 From: Ziru Date: Mon, 19 Oct 2026 13:12:32 +0800 Subject: [PATCH] [Feat] [CPU] model: Support Muse models on CPU (#44624) Co-authored-by: Niu Ziru --- python/sglang/srt/configs/muse_glimmer.py | 6 ++++++ python/sglang/srt/configs/update_config.py | 18 ++++++++++++++++++ python/sglang/srt/models/muse_glimmer.py | 10 ++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/python/sglang/srt/configs/muse_glimmer.py b/python/sglang/srt/configs/muse_glimmer.py index 549a95caa2..c318a3b5ad 100644 --- a/python/sglang/srt/configs/muse_glimmer.py +++ b/python/sglang/srt/configs/muse_glimmer.py @@ -51,12 +51,17 @@ class MuseGlimmerVisionConfig(PretrainedConfig): pos_emb_height: int = 32, pos_emb_width: int = 32, rope_theta: float = 10000.0, + head_dim: Optional[int] = None, **kwargs, ): self.hidden_size = hidden_size self.intermediate_size = intermediate_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads + # Pinned so it survives CPU TP padding of num_attention_heads. + self.head_dim = ( + head_dim if head_dim is not None else hidden_size // num_attention_heads + ) self.hidden_act = hidden_act self.layer_norm_eps = layer_norm_eps self.attention_types = attention_types @@ -140,6 +145,7 @@ class MuseGlimmerConfig(PretrainedConfig): self.out_hidden_size = out_hidden_size self.projector_hidden_act = projector_hidden_act self.projector_hidden_size = projector_hidden_size + self.padded_projector_hidden_size = projector_hidden_size super().__init__( tie_word_embeddings=tie_word_embeddings, bos_token_id=bos_token_id, diff --git a/python/sglang/srt/configs/update_config.py b/python/sglang/srt/configs/update_config.py index bf2cdaa5a5..39a1244960 100644 --- a/python/sglang/srt/configs/update_config.py +++ b/python/sglang/srt/configs/update_config.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING from sglang.srt.utils import ( log_debug_on_rank0, + round_up, ) logger = logging.getLogger(__name__) @@ -302,6 +303,17 @@ def adjust_config_with_unaligned_cpu_tp( model_config, moe_intermediate_attr, intermediate_padding_size ) + if model_config.hf_config.model_type == "muse_glimmer": + projector_hidden_size = model_config.hf_config.projector_hidden_size + padded_projector_hidden_size = projector_hidden_size + if projector_hidden_size % tp_size != 0: + padded_projector_hidden_size = round_up(projector_hidden_size, tp_size * 32) + update_config( + model_config.hf_config, + "padded_projector_hidden_size", + padded_projector_hidden_size, + ) + multimodal_config = [ [ model_config.hf_config, @@ -316,6 +328,12 @@ def adjust_config_with_unaligned_cpu_tp( [model_config.hf_config, "vision_config", "qwen3_5", "num_heads"], [model_config.hf_config, "vision_config", "qwen4_exp", "num_heads"], [model_config.hf_config, "vision_config", "mllama", "attention_heads"], + [ + model_config.hf_config, + "vision_config", + "muse_glimmer_vision", + "num_attention_heads", + ], [ model_config.hf_config, "vision_config", diff --git a/python/sglang/srt/models/muse_glimmer.py b/python/sglang/srt/models/muse_glimmer.py index ae1231805b..c36543bfcd 100644 --- a/python/sglang/srt/models/muse_glimmer.py +++ b/python/sglang/srt/models/muse_glimmer.py @@ -595,7 +595,8 @@ class MuseGlimmerVisionEncoderLayer(nn.Module): self.attn = VisionAttention( embed_dim=config.hidden_size, num_heads=config.num_attention_heads, - projection_size=config.hidden_size, + head_dim=config.head_dim, + projection_size=config.num_attention_heads * config.head_dim, use_qkv_parallel=True, qkv_bias=True, proj_bias=True, @@ -644,7 +645,7 @@ class MuseGlimmerVisionModel(nn.Module): prefix=add_prefix("patch_embedder", prefix), ) self.rotary_emb = MuseGlimmerVisionRotaryEmbedding( - config.hidden_size // config.num_attention_heads, config.rope_theta + config.head_dim, config.rope_theta ) self.ln_pre = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) self.layers = nn.ModuleList( @@ -772,15 +773,16 @@ class MuseGlimmerVisionAdapter(nn.Module): prefix: str = "", ): super().__init__() + sharded_size = config.padded_projector_hidden_size self.fc1 = ColumnParallelLinear( config.out_hidden_size, - config.projector_hidden_size, + sharded_size, bias=False, quant_config=quant_config, prefix=add_prefix("fc1", prefix), ) self.fc2 = RowParallelLinear( - config.projector_hidden_size, + sharded_size, config.projector_hidden_size, bias=False, quant_config=quant_config,