From bfeb9a8af29cae971669a07fb242db4eeefc5a1c Mon Sep 17 00:00:00 2001 From: Zheng Li Date: Mon, 10 Aug 2026 02:47:36 +0800 Subject: [PATCH] [feat] Add language_model_only parameter support for Qwen35 (#22867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: 瑀澈 Co-authored-by: Claude Opus 4.6 (1M context) Co-authored-by: Xinyuan Tong Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com> --- python/sglang/srt/configs/model_config.py | 37 ++++--- python/sglang/srt/models/qwen3_5.py | 17 +++- python/sglang/srt/models/qwen3_vl.py | 99 +++++++++++++------ .../srt/utils/hf_transformers/processor.py | 12 +++ 4 files changed, 118 insertions(+), 47 deletions(-) diff --git a/python/sglang/srt/configs/model_config.py b/python/sglang/srt/configs/model_config.py index 6ffda9367..8784771ce 100644 --- a/python/sglang/srt/configs/model_config.py +++ b/python/sglang/srt/configs/model_config.py @@ -317,8 +317,11 @@ class ModelConfig: rope_scaling = getattr(self.hf_text_config, "rope_parameters", None) or getattr( self.hf_text_config, "rope_scaling", {} ) + self.is_lm_only = getattr(self.hf_config, "language_model_only", False) self.model_is_mrope = ( - rope_scaling is not None and "mrope_section" in rope_scaling + not self.is_lm_only + and rope_scaling is not None + and "mrope_section" in rope_scaling ) self.hf_generation_config = get_generation_config( @@ -435,16 +438,22 @@ class ModelConfig: or hasattr(self.hf_config, "audio_config") ) ) - self.is_multimodal = enable_multimodal and ( - is_multimodal_model(self.hf_config.architectures) - or has_multimodal_subconfig + self.is_multimodal = ( + enable_multimodal + and not self.is_lm_only + and ( + is_multimodal_model(self.hf_config.architectures) + or has_multimodal_subconfig + ) ) self.is_audio_model = enable_multimodal and is_audio_model( self.hf_config.architectures ) # TODO: requires further polishing - self.is_image_understandable_model = enable_multimodal and hasattr( - self.hf_config, "vision_config" + self.is_image_understandable_model = ( + enable_multimodal + and not self.is_lm_only + and hasattr(self.hf_config, "vision_config") ) # Models expose audio_config at different nesting levels: @@ -453,11 +462,17 @@ class ModelConfig: # - sound_config: Nemotron AVLM with Parakeet audio encoder # - is_audio_model(): Whisper, Qwen3-ASR (architecture-based fallback) # TODO: Handle this more robustly by standardizing the config structure in the future - self.is_audio_understandable_model = enable_multimodal and ( - hasattr(self.hf_config, "audio_config") - or hasattr(getattr(self.hf_config, "thinker_config", None), "audio_config") - or getattr(self.hf_config, "sound_config", None) is not None - or is_audio_model(self.hf_config.architectures) + self.is_audio_understandable_model = ( + enable_multimodal + and not self.is_lm_only + and ( + hasattr(self.hf_config, "audio_config") + or hasattr( + getattr(self.hf_config, "thinker_config", None), "audio_config" + ) + or getattr(self.hf_config, "sound_config", None) is not None + or is_audio_model(self.hf_config.architectures) + ) ) self.is_multimodal_chunked_prefill_supported = ( diff --git a/python/sglang/srt/models/qwen3_5.py b/python/sglang/srt/models/qwen3_5.py index 56ddca2dc..1a15ffc67 100644 --- a/python/sglang/srt/models/qwen3_5.py +++ b/python/sglang/srt/models/qwen3_5.py @@ -1777,9 +1777,13 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration): rope_config = getattr(self.config, "rope_parameters", None) or getattr( self.config, "rope_scaling", {} ) - self.is_mrope_enabled = "mrope_section" in rope_config + self.is_mrope_enabled = ( + not self.language_model_only and "mrope_section" in rope_config + ) - self.deepstack_visual_indexes = self.visual.deepstack_visual_indexes + self.deepstack_visual_indexes = ( + self.visual.deepstack_visual_indexes if self.visual is not None else [] + ) def get_hidden_dim(self, module_name: str, layer_idx: int): return self.model.get_hidden_dim(module_name, layer_idx) @@ -1935,9 +1939,14 @@ class Qwen3_5MoeForConditionalGeneration(Qwen3VLForConditionalGeneration): rope_config = getattr(self.config, "rope_parameters", None) or getattr( self.config, "rope_scaling", {} ) - self.is_mrope_enabled = "mrope_section" in rope_config + self.is_mrope_enabled = ( + not self.language_model_only and "mrope_section" in rope_config + ) + + self.deepstack_visual_indexes = ( + self.visual.deepstack_visual_indexes if self.visual is not None else [] + ) - self.deepstack_visual_indexes = self.visual.deepstack_visual_indexes self.num_fused_shared_experts = 0 if _use_aiter and not _disable_shared_experts_fusion(): self.num_fused_shared_experts = self._get_num_fused_shared_experts() diff --git a/python/sglang/srt/models/qwen3_vl.py b/python/sglang/srt/models/qwen3_vl.py index 20a468836..22a8dbe62 100644 --- a/python/sglang/srt/models/qwen3_vl.py +++ b/python/sglang/srt/models/qwen3_vl.py @@ -1240,15 +1240,19 @@ class Qwen3VLForConditionalGeneration(nn.Module): self.use_data_parallel = get_mm().mm_enable_dp_encoder - self.visual = Qwen3VLMoeVisionModel( - config.vision_config, - # NOTE: Qwen3-VL vision encoder currently supports BitsAndBytes 4-bit quantization. - # Other quantization methods (e.g., GPTQ, AWQ) are untested and may not be supported. - quant_config=None, - norm_eps=getattr(config, "rms_norm_eps", 1e-6), - prefix=add_prefix("model.visual", prefix), - use_data_parallel=self.use_data_parallel, - ) + self.language_model_only = getattr(config, "language_model_only", False) + if self.language_model_only: + self.visual = None + else: + self.visual = Qwen3VLMoeVisionModel( + config.vision_config, + # NOTE: Qwen3-VL vision encoder currently supports BitsAndBytes 4-bit quantization. + # Other quantization methods (e.g., GPTQ, AWQ) are untested and may not be supported. + quant_config=None, + norm_eps=getattr(config, "rms_norm_eps", 1e-6), + prefix=add_prefix("model.visual", prefix), + use_data_parallel=self.use_data_parallel, + ) # TODO: make it more elegant if language_model_cls is Qwen3LLMModel: @@ -1291,7 +1295,9 @@ class Qwen3VLForConditionalGeneration(nn.Module): # encoder_only mode: no language model, so no lm_head needed self.lm_head = None - self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling + self.is_mrope_enabled = ( + not self.language_model_only and "mrope_section" in self.config.rope_scaling + ) self.logits_processor = LogitsProcessor(self.config) self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True) @@ -1300,9 +1306,16 @@ class Qwen3VLForConditionalGeneration(nn.Module): # 8, 16, 24 layer will be merged to 0, 1, 2 layer of decoder output hidden_states # deepstack - self.deepstack_visual_indexes = config.vision_config.deepstack_visual_indexes - self.num_deepstack_embeddings = len(self.deepstack_visual_indexes) - self.use_deepstack = {Modality.IMAGE: True, Modality.VIDEO: True} + if not self.language_model_only: + self.deepstack_visual_indexes = ( + config.vision_config.deepstack_visual_indexes + ) + self.num_deepstack_embeddings = len(self.deepstack_visual_indexes) + self.use_deepstack = {Modality.IMAGE: True, Modality.VIDEO: True} + else: + self.deepstack_visual_indexes = [] + self.num_deepstack_embeddings = 0 + self.use_deepstack = {} # For EAGLE3 support self.capture_aux_hidden_states = False @@ -1331,10 +1344,13 @@ class Qwen3VLForConditionalGeneration(nn.Module): return int(getattr(cfg, "num_hidden_layers", 0)) def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs): + if mm_inputs and mm_inputs.mm_items: + _require_vision(self) pattern = MultiModalityDataPaddingPatternMultimodalTokens() return pattern.pad_input_tokens(input_ids, mm_inputs) def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + _require_vision(self) pixel_values = materialize_multimodal_features( [item.feature for item in items], device=self.visual.device, @@ -1355,6 +1371,7 @@ class Qwen3VLForConditionalGeneration(nn.Module): return self.visual(pixel_values, grid_thw=image_grid_thw) def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor: + _require_vision(self) pixel_values = materialize_multimodal_features( [item.feature for item in items], device=self.visual.device, @@ -1405,25 +1422,32 @@ class Qwen3VLForConditionalGeneration(nn.Module): if self.is_mrope_enabled: positions = forward_batch.mrope_positions - if not ( - forward_batch.forward_mode.is_decode() - or not forward_batch.contains_image_inputs() - ): - if self.is_mrope_enabled: - assert positions.ndim == 2 and positions.size(0) == 3, ( - "multimodal section rotary embedding requires " - f"(3, seq_len) positions, but got {positions.size()}" - ) - - hidden_states = general_mm_embed_routine( - input_ids=input_ids, - forward_batch=forward_batch, - language_model=self.model, - multimodal_model=self, - positions=positions, - use_deepstack=self.use_deepstack, - pp_proxy_tensors=pp_proxy_tensors, - ) + if self.language_model_only: + hidden_states = self.model( + input_ids=input_ids, + forward_batch=forward_batch, + positions=positions, + pp_proxy_tensors=pp_proxy_tensors, + ) + else: + if not ( + forward_batch.forward_mode.is_decode() + or not forward_batch.contains_image_inputs() + ): + if self.is_mrope_enabled: + assert positions.ndim == 2 and positions.size(0) == 3, ( + "multimodal section rotary embedding requires " + f"(3, seq_len) positions, but got {positions.size()}" + ) + hidden_states = general_mm_embed_routine( + input_ids=input_ids, + forward_batch=forward_batch, + language_model=self.model, + multimodal_model=self, + positions=positions, + use_deepstack=self.use_deepstack, + pp_proxy_tensors=pp_proxy_tensors, + ) aux_hidden_states = None if self.capture_aux_hidden_states: @@ -1555,4 +1579,15 @@ class Qwen3VLForConditionalGeneration(nn.Module): self.model.layers_to_capture = [val + 1 for val in layer_ids] +def _require_vision(model) -> None: + if ( + getattr(model, "language_model_only", False) + and getattr(model, "visual", None) is None + ): + raise RuntimeError( + "Checkpoint is marked language_model_only=True and was loaded " + "without a vision encoder; multimodal inputs are not supported." + ) + + EntryClass = Qwen3VLForConditionalGeneration diff --git a/python/sglang/srt/utils/hf_transformers/processor.py b/python/sglang/srt/utils/hf_transformers/processor.py index 38dfa7f0f..e3c24eda8 100644 --- a/python/sglang/srt/utils/hf_transformers/processor.py +++ b/python/sglang/srt/utils/hf_transformers/processor.py @@ -183,6 +183,18 @@ def get_processor( if is_ocr2: _override_v_head_dim_if_zero(config) + # Checkpoints with language_model_only=True are text-only despite their + # multimodal-family config; route to tokenizer instead of the mm processor. + if getattr(config, "language_model_only", False): + kwargs.pop("use_fast", None) + return AutoTokenizer.from_pretrained( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + **kwargs, + ) + if config.model_type in {"qwen2_vl", "sarashina2_vision"}: if "size" not in kwargs: kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520}