minke.yu/sglang
1
0
Fork 0
Code Issues Pull Requests Actions 12 Packages Projects Releases Wiki Activity

[feat] Add language_model_only parameter support for Qwen35 (#22867)

Browse Source 
Co-authored-by: 瑀澈 <yuche.lz@alibaba-inc.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Xinyuan Tong <xinyuantong.cs@gmail.com>
Co-authored-by: Xinyuan Tong <115166877+JustinTong0323@users.noreply.github.com>
This commit is contained in:
Zheng Li
2026-08-10 02:47:36 +08:00
committed by GitHub
co-authored by 瑀澈 Claude Opus 4.6 Xinyuan Tong Xinyuan Tong
parent c2fbe2f6d8
commit bfeb9a8af2
4 changed files with 118 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/sglang/srt/configs/model_config.py
+26 -11
View File
@@ -317,8 +317,11 @@ class ModelConfig:
rope_scaling = getattr(self.hf_text_config, "rope_parameters", None) or getattr(
self.hf_text_config, "rope_scaling", {}
)
self.is_lm_only = getattr(self.hf_config, "language_model_only", False)
self.model_is_mrope = (
rope_scaling is not None and "mrope_section" in rope_scaling
not self.is_lm_only
and rope_scaling is not None
and "mrope_section" in rope_scaling
)
self.hf_generation_config = get_generation_config(
@@ -435,16 +438,22 @@ class ModelConfig:
or hasattr(self.hf_config, "audio_config")
)
)
self.is_multimodal = enable_multimodal and (
is_multimodal_model(self.hf_config.architectures)
or has_multimodal_subconfig
self.is_multimodal = (
enable_multimodal
and not self.is_lm_only
and (
is_multimodal_model(self.hf_config.architectures)
or has_multimodal_subconfig
)
)
self.is_audio_model = enable_multimodal and is_audio_model(
self.hf_config.architectures
)
# TODO: requires further polishing
self.is_image_understandable_model = enable_multimodal and hasattr(
self.hf_config, "vision_config"
self.is_image_understandable_model = (
enable_multimodal
and not self.is_lm_only
and hasattr(self.hf_config, "vision_config")
)
# Models expose audio_config at different nesting levels:
@@ -453,11 +462,17 @@ class ModelConfig:
# - sound_config: Nemotron AVLM with Parakeet audio encoder
# - is_audio_model(): Whisper, Qwen3-ASR (architecture-based fallback)
# TODO: Handle this more robustly by standardizing the config structure in the future
self.is_audio_understandable_model = enable_multimodal and (
hasattr(self.hf_config, "audio_config")
or hasattr(getattr(self.hf_config, "thinker_config", None), "audio_config")
or getattr(self.hf_config, "sound_config", None) is not None
or is_audio_model(self.hf_config.architectures)
self.is_audio_understandable_model = (
enable_multimodal
and not self.is_lm_only
and (
hasattr(self.hf_config, "audio_config")
or hasattr(
getattr(self.hf_config, "thinker_config", None), "audio_config"
)
or getattr(self.hf_config, "sound_config", None) is not None
or is_audio_model(self.hf_config.architectures)
)
)
self.is_multimodal_chunked_prefill_supported = (
python/sglang/srt/models/qwen3_5.py
+13 -4
View File
@@ -1777,9 +1777,13 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration):
rope_config = getattr(self.config, "rope_parameters", None) or getattr(
self.config, "rope_scaling", {}
)
self.is_mrope_enabled = "mrope_section" in rope_config
self.is_mrope_enabled = (
not self.language_model_only and "mrope_section" in rope_config
)
self.deepstack_visual_indexes = self.visual.deepstack_visual_indexes
self.deepstack_visual_indexes = (
self.visual.deepstack_visual_indexes if self.visual is not None else []
)
def get_hidden_dim(self, module_name: str, layer_idx: int):
return self.model.get_hidden_dim(module_name, layer_idx)
@@ -1935,9 +1939,14 @@ class Qwen3_5MoeForConditionalGeneration(Qwen3VLForConditionalGeneration):
rope_config = getattr(self.config, "rope_parameters", None) or getattr(
self.config, "rope_scaling", {}
)
self.is_mrope_enabled = "mrope_section" in rope_config
self.is_mrope_enabled = (
not self.language_model_only and "mrope_section" in rope_config
)
self.deepstack_visual_indexes = (
self.visual.deepstack_visual_indexes if self.visual is not None else []
)
self.deepstack_visual_indexes = self.visual.deepstack_visual_indexes
self.num_fused_shared_experts = 0
if _use_aiter and not _disable_shared_experts_fusion():
self.num_fused_shared_experts = self._get_num_fused_shared_experts()
python/sglang/srt/models/qwen3_vl.py
+67 -32
View File
@@ -1240,15 +1240,19 @@ class Qwen3VLForConditionalGeneration(nn.Module):
self.use_data_parallel = get_mm().mm_enable_dp_encoder
self.visual = Qwen3VLMoeVisionModel(
config.vision_config,
# NOTE: Qwen3-VL vision encoder currently supports BitsAndBytes 4-bit quantization.
# Other quantization methods (e.g., GPTQ, AWQ) are untested and may not be supported.
quant_config=None,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
prefix=add_prefix("model.visual", prefix),
use_data_parallel=self.use_data_parallel,
)
self.language_model_only = getattr(config, "language_model_only", False)
if self.language_model_only:
self.visual = None
else:
self.visual = Qwen3VLMoeVisionModel(
config.vision_config,
# NOTE: Qwen3-VL vision encoder currently supports BitsAndBytes 4-bit quantization.
# Other quantization methods (e.g., GPTQ, AWQ) are untested and may not be supported.
quant_config=None,
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
prefix=add_prefix("model.visual", prefix),
use_data_parallel=self.use_data_parallel,
)
# TODO: make it more elegant
if language_model_cls is Qwen3LLMModel:
@@ -1291,7 +1295,9 @@ class Qwen3VLForConditionalGeneration(nn.Module):
# encoder_only mode: no language model, so no lm_head needed
self.lm_head = None
self.is_mrope_enabled = "mrope_section" in self.config.rope_scaling
self.is_mrope_enabled = (
not self.language_model_only and "mrope_section" in self.config.rope_scaling
)
self.logits_processor = LogitsProcessor(self.config)
self.pooler = Pooler(pooling_type=PoolingType.LAST, normalize=True)
@@ -1300,9 +1306,16 @@ class Qwen3VLForConditionalGeneration(nn.Module):
# 8, 16, 24 layer will be merged to 0, 1, 2 layer of decoder output hidden_states
# deepstack
self.deepstack_visual_indexes = config.vision_config.deepstack_visual_indexes
self.num_deepstack_embeddings = len(self.deepstack_visual_indexes)
self.use_deepstack = {Modality.IMAGE: True, Modality.VIDEO: True}
if not self.language_model_only:
self.deepstack_visual_indexes = (
config.vision_config.deepstack_visual_indexes
)
self.num_deepstack_embeddings = len(self.deepstack_visual_indexes)
self.use_deepstack = {Modality.IMAGE: True, Modality.VIDEO: True}
else:
self.deepstack_visual_indexes = []
self.num_deepstack_embeddings = 0
self.use_deepstack = {}
# For EAGLE3 support
self.capture_aux_hidden_states = False
@@ -1331,10 +1344,13 @@ class Qwen3VLForConditionalGeneration(nn.Module):
return int(getattr(cfg, "num_hidden_layers", 0))
def pad_input_ids(self, input_ids: List[int], mm_inputs: MultimodalInputs):
if mm_inputs and mm_inputs.mm_items:
_require_vision(self)
pattern = MultiModalityDataPaddingPatternMultimodalTokens()
return pattern.pad_input_tokens(input_ids, mm_inputs)
def get_image_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
_require_vision(self)
pixel_values = materialize_multimodal_features(
[item.feature for item in items],
device=self.visual.device,
@@ -1355,6 +1371,7 @@ class Qwen3VLForConditionalGeneration(nn.Module):
return self.visual(pixel_values, grid_thw=image_grid_thw)
def get_video_feature(self, items: List[MultimodalDataItem]) -> torch.Tensor:
_require_vision(self)
pixel_values = materialize_multimodal_features(
[item.feature for item in items],
device=self.visual.device,
@@ -1405,25 +1422,32 @@ class Qwen3VLForConditionalGeneration(nn.Module):
if self.is_mrope_enabled:
positions = forward_batch.mrope_positions
if not (
forward_batch.forward_mode.is_decode()
or not forward_batch.contains_image_inputs()
):
if self.is_mrope_enabled:
assert positions.ndim == 2 and positions.size(0) == 3, (
"multimodal section rotary embedding requires "
f"(3, seq_len) positions, but got {positions.size()}"
)
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
multimodal_model=self,
positions=positions,
use_deepstack=self.use_deepstack,
pp_proxy_tensors=pp_proxy_tensors,
)
if self.language_model_only:
hidden_states = self.model(
input_ids=input_ids,
forward_batch=forward_batch,
positions=positions,
pp_proxy_tensors=pp_proxy_tensors,
)
else:
if not (
forward_batch.forward_mode.is_decode()
or not forward_batch.contains_image_inputs()
):
if self.is_mrope_enabled:
assert positions.ndim == 2 and positions.size(0) == 3, (
"multimodal section rotary embedding requires "
f"(3, seq_len) positions, but got {positions.size()}"
)
hidden_states = general_mm_embed_routine(
input_ids=input_ids,
forward_batch=forward_batch,
language_model=self.model,
multimodal_model=self,
positions=positions,
use_deepstack=self.use_deepstack,
pp_proxy_tensors=pp_proxy_tensors,
)
aux_hidden_states = None
if self.capture_aux_hidden_states:
@@ -1555,4 +1579,15 @@ class Qwen3VLForConditionalGeneration(nn.Module):
self.model.layers_to_capture = [val + 1 for val in layer_ids]
def _require_vision(model) -> None:
if (
getattr(model, "language_model_only", False)
and getattr(model, "visual", None) is None
):
raise RuntimeError(
"Checkpoint is marked language_model_only=True and was loaded "
"without a vision encoder; multimodal inputs are not supported."
)
EntryClass = Qwen3VLForConditionalGeneration
python/sglang/srt/utils/hf_transformers/processor.py
+12
View File
@@ -183,6 +183,18 @@ def get_processor(
if is_ocr2:
_override_v_head_dim_if_zero(config)
# Checkpoints with language_model_only=True are text-only despite their
# multimodal-family config; route to tokenizer instead of the mm processor.
if getattr(config, "language_model_only", False):
kwargs.pop("use_fast", None)
return AutoTokenizer.from_pretrained(
tokenizer_name,
*args,
trust_remote_code=trust_remote_code,
revision=revision,
**kwargs,
)
if config.model_type in {"qwen2_vl", "sarashina2_vision"}:
if "size" not in kwargs:
kwargs["size"] = {"shortest_edge": 3136, "longest_edge": 1003520}