[feat] Add language_model_only parameter support for Qwen35 (#22867)

Co-authored-by: 瑀澈 <yuche.lz@alibaba-inc.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>

python/sglang/srt/configs/model_config.py

@@ -317,8 +317,11 @@ class ModelConfig:
         rope_scaling = getattr(self.hf_text_config, "rope_parameters", None) or getattr(
             self.hf_text_config, "rope_scaling", {}
         )
+        self.is_lm_only = getattr(self.hf_config, "language_model_only", False)
         self.model_is_mrope = (
-            rope_scaling is not None and "mrope_section" in rope_scaling
+            not self.is_lm_only
+            and rope_scaling is not None
+            and "mrope_section" in rope_scaling
         )
 
         self.hf_generation_config = get_generation_config(
@@ -435,16 +438,22 @@ class ModelConfig:
                 or hasattr(self.hf_config, "audio_config")
             )
         )
-        self.is_multimodal = enable_multimodal and (
+        self.is_multimodal = (
+            enable_multimodal
+            and not self.is_lm_only
+            and (
                 is_multimodal_model(self.hf_config.architectures)
                 or has_multimodal_subconfig
             )
+        )
         self.is_audio_model = enable_multimodal and is_audio_model(
             self.hf_config.architectures
         )
         # TODO: requires further polishing
-        self.is_image_understandable_model = enable_multimodal and hasattr(
+        self.is_image_understandable_model = (
-            self.hf_config, "vision_config"
+            enable_multimodal
+            and not self.is_lm_only
+            and hasattr(self.hf_config, "vision_config")
         )
 
         # Models expose audio_config at different nesting levels:
@@ -453,12 +462,18 @@ class ModelConfig:
         #   - sound_config: Nemotron AVLM with Parakeet audio encoder
         #   - is_audio_model(): Whisper, Qwen3-ASR (architecture-based fallback)
         # TODO: Handle this more robustly by standardizing the config structure in the future
-        self.is_audio_understandable_model = enable_multimodal and (
+        self.is_audio_understandable_model = (
+            enable_multimodal
+            and not self.is_lm_only
+            and (
                 hasattr(self.hf_config, "audio_config")
-            or hasattr(getattr(self.hf_config, "thinker_config", None), "audio_config")
+                or hasattr(
+                    getattr(self.hf_config, "thinker_config", None), "audio_config"
+                )
                 or getattr(self.hf_config, "sound_config", None) is not None
                 or is_audio_model(self.hf_config.architectures)
             )
+        )
 
         self.is_multimodal_chunked_prefill_supported = (
             enable_multimodal

python/sglang/srt/models/qwen3_5.py

@@ -1777,9 +1777,13 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
         rope_config = getattr(self.config, "rope_parameters", None) or getattr(
             self.config, "rope_scaling", {}
         )
-        self.is_mrope_enabled = "mrope_section" in rope_config
+        self.is_mrope_enabled = (
+            not self.language_model_only and "mrope_section" in rope_config
+        )
 
-        self.deepstack_visual_indexes = self.visual.deepstack_visual_indexes
+        self.deepstack_visual_indexes = (
+            self.visual.deepstack_visual_indexes if self.visual is not None else []
+        )
 
     def get_hidden_dim(self, module_name: str, layer_idx: int):
         return self.model.get_hidden_dim(module_name, layer_idx)
@@ -1935,9 +1939,14 @@ class Qwen3_5MoeForConditionalGeneration(Qwen3VLForConditionalGeneration):
         rope_config = getattr(self.config, "rope_parameters", None) or getattr(
             self.config, "rope_scaling", {}
         )
-        self.is_mrope_enabled = "mrope_section" in rope_config
+        self.is_mrope_enabled = (
+            not self.language_model_only and "mrope_section" in rope_config
+        )
+
+        self.deepstack_visual_indexes = (
+            self.visual.deepstack_visual_indexes if self.visual is not None else []
+        )
 
-        self.deepstack_visual_indexes = self.visual.deepstack_visual_indexes
         self.num_fused_shared_experts = 0
         if _use_aiter and not _disable_shared_experts_fusion():
             self.num_fused_shared_experts = self._get_num_fused_shared_experts()

python/sglang/srt/models/qwen3_vl.py

@@ -1240,6 +1240,10 @@ class Qwen3VLForConditionalGeneration(nn.Module):
 
         self.use_data_parallel = get_mm().mm_enable_dp_encoder
 
+        self.language_model_only = getattr(config, "language_model_only", False)
+        if self.language_model_only:
+            self.visual = None
+        else:
             self.visual = Qwen3VLMoeVisionModel(
                 config.vision_config,
                 # NOTE: Qwen3-VL vision encoder currently supports BitsAndBytes 4-bit quantization.
@@ -1291,7 +1295,9 @@ class Qwen3VLForConditionalGeneration(nn.Module):
             # encoder_only mode: no language model, so no lm_head needed
             self.lm_head = None
 
-        self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling
+        self.is_mrope_enabled = (
+            not self.language_model_only and "mrope_section" in self.config.rope_scaling
+        )
 
         self.logits_processor = LogitsProcessor(self.config)
         self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
@@ -1300,9 +1306,16 @@ class Qwen3VLForConditionalGeneration(nn.Module):
         # 8, 16, 24 layer will be merged to 0, 1, 2 layer of decoder output hidden_states
 
         # deepstack
-        self.deepstack_visual_indexes = config.vision_config.deepstack_visual_indexes
+        if not self.language_model_only:
+            self.deepstack_visual_indexes = (
+                config.vision_config.deepstack_visual_indexes
+            )
             self.num_deepstack_embeddings = len(self.deepstack_visual_indexes)
             self.use_deepstack = {Modality.IMAGE: True, Modality.VIDEO: True}
+        else:
+            self.deepstack_visual_indexes = []
+            self.num_deepstack_embeddings = 0
+            self.use_deepstack = {}
 
         # For EAGLE3 support
         self.capture_aux_hidden_states = False
@@ -1331,10 +1344,13 @@ class Qwen3VLForConditionalGeneration(nn.Module):
         return int(getattr(cfg, "num_hidden_layers", 0))
 
     def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
+        if mm_inputs and mm_inputs.mm_items:
+            _require_vision(self)
         pattern = MultiModalityDataPaddingPatternMultimodalTokens()
         return pattern.pad_input_tokens(input_ids, mm_inputs)
 
     def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        _require_vision(self)
         pixel_values = materialize_multimodal_features(
             [item.feature for item in items],
             device=self.visual.device,
@@ -1355,6 +1371,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
             return self.visual(pixel_values, grid_thw=image_grid_thw)
 
     def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
+        _require_vision(self)
         pixel_values = materialize_multimodal_features(
             [item.feature for item in items],
             device=self.visual.device,
@@ -1405,6 +1422,14 @@ class Qwen3VLForConditionalGeneration(nn.Module):
         if self.is_mrope_enabled:
             positions = forward_batch.mrope_positions
 
+        if self.language_model_only:
+            hidden_states = self.model(
+                input_ids=input_ids,
+                forward_batch=forward_batch,
+                positions=positions,
+                pp_proxy_tensors=pp_proxy_tensors,
+            )
+        else:
             if not (
                 forward_batch.forward_mode.is_decode()
                 or not forward_batch.contains_image_inputs()
@@ -1414,7 +1439,6 @@ class Qwen3VLForConditionalGeneration(nn.Module):
                         "multimodal section rotary embedding requires "
                         f"(3, seq_len) positions, but got {positions.size()}"
                     )
-
             hidden_states = general_mm_embed_routine(
                 input_ids=input_ids,
                 forward_batch=forward_batch,
@@ -1555,4 +1579,15 @@ class Qwen3VLForConditionalGeneration(nn.Module):
             self.model.layers_to_capture = [val + 1 for val in layer_ids]
 
 
+def _require_vision(model) -> None:
+    if (
+        getattr(model, "language_model_only", False)
+        and getattr(model, "visual", None) is None
+    ):
+        raise RuntimeError(
+            "Checkpoint is marked language_model_only=True and was loaded "
+            "without a vision encoder; multimodal inputs are not supported."
+        )
+
+
 EntryClass = Qwen3VLForConditionalGeneration

python/sglang/srt/utils/hf_transformers/processor.py

@@ -183,6 +183,18 @@ def get_processor(
         if is_ocr2:
             _override_v_head_dim_if_zero(config)
 
+    # Checkpoints with language_model_only=True are text-only despite their
+    # multimodal-family config; route to tokenizer instead of the mm processor.
+    if getattr(config, "language_model_only", False):
+        kwargs.pop("use_fast", None)
+        return AutoTokenizer.from_pretrained(
+            tokenizer_name,
+            *args,
+            trust_remote_code=trust_remote_code,
+            revision=revision,
+            **kwargs,
+        )
+
     if config.model_type in {"qwen2_vl", "sarashina2_vision"}:
         if "size" not in kwargs:
             kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520}