diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py index d5f1260e0..29c8f26f3 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py @@ -1221,6 +1221,12 @@ class Qwen3VLForConditionalGeneration(TextEncoder): self.lm_head = nn.Linear( config.text_config.hidden_size, config.text_config.vocab_size, bias=False ) + if getattr(config, "tie_word_embeddings", False) or getattr( + config.text_config, "tie_word_embeddings", False + ): + # Tied checkpoints may omit lm_head.weight, as Hugging Face does. + # Keep one registered parameter so strict native loading stays safe. + self.lm_head.weight = self.model.get_input_embeddings().weight @torch.no_grad() def forward( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py index 78d17512e..ed81d5896 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/lingbot_video_moe/text_encoding.py @@ -107,9 +107,12 @@ class LingBotVideoTextEncodingStage(TextEncodingStage): inputs = self._build_prompt_inputs(prompt) inputs = inputs.to(device) - outputs = text_encoder( - **inputs, - output_hidden_states=self.hidden_state_skip_layer is not None, + outputs = self._forward_text_encoder( + text_encoder, + { + **inputs, + "output_hidden_states": self.hidden_state_skip_layer is not None, + }, ) if self.hidden_state_skip_layer is not None: prompt_embeds = outputs.hidden_states[-(self.hidden_state_skip_layer + 1)] diff --git a/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py b/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py index c3d3fe00a..e82f69284 100644 --- a/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py +++ b/python/sglang/multimodal_gen/test/unit/test_lingbot_video_moe.py @@ -21,6 +21,9 @@ from sglang.multimodal_gen.runtime.layers.moe import ( LingBotVideoGroupedExperts, LingBotVideoRouter, ) +from sglang.multimodal_gen.runtime.managers.forward_context import ( + get_forward_context, +) from sglang.multimodal_gen.runtime.models.dits import ( lingbot_video_moe as dits_lingbot_video_moe, ) @@ -262,6 +265,31 @@ def test_text_encoding_crops_template_then_trims_padding(): assert stage._compute_crop_start() == prefix_width +def test_text_encoding_sets_forward_context_for_native_encoder(): + prompt_width, prefix_width, true_len, channels = 6, 2, 5, 4 + hidden = torch.zeros(1, prompt_width, channels) + + class NativeEncoder: + uses_sglang_forward_context = True + + def __call__(self, **kwargs): + context = get_forward_context() + assert context.current_timestep == 0 + assert context.attn_metadata is None + assert kwargs["output_hidden_states"] is True + return SimpleNamespace(hidden_states=[hidden]) + + stage = _text_encoding_stage( + _FakeQwenProcessor(prompt_width, prefix_width, true_len), NativeEncoder() + ) + embeds, mask = stage._encode_prompt( + "a structured caption", torch.device("cpu"), torch.float32 + ) + + assert tuple(embeds.shape) == (1, true_len - prefix_width, channels) + assert tuple(mask.shape) == (1, true_len - prefix_width) + + def test_check_inputs_enforces_frame_and_size_contract(): check = LingBotVideoTextEncodingStage.check_inputs check(480, 832, 1) diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen3vl_vision.py b/python/sglang/multimodal_gen/test/unit/test_qwen3vl_vision.py index 850edade8..e71794a48 100644 --- a/python/sglang/multimodal_gen/test/unit/test_qwen3vl_vision.py +++ b/python/sglang/multimodal_gen/test/unit/test_qwen3vl_vision.py @@ -170,6 +170,52 @@ def test_native_vision_keeps_position_math_in_fp32(): assert block.position_embedding_dtypes == (torch.float32, torch.float32) +def test_qwen3vl_ties_lm_head_to_input_embeddings(): + vision_config = SimpleNamespace( + hidden_size=16, + intermediate_size=24, + hidden_act="gelu_pytorch_tanh", + num_heads=2, + depth=0, + patch_size=2, + temporal_patch_size=1, + in_channels=3, + num_position_embeddings=16, + spatial_merge_size=2, + out_hidden_size=16, + deepstack_visual_indexes=[], + ) + text_config = SimpleNamespace( + hidden_size=16, + vocab_size=32, + pad_token_id=0, + num_hidden_layers=0, + rms_norm_eps=1e-6, + tie_word_embeddings=True, + ) + arch_config = SimpleNamespace( + vision_config=vision_config, + text_config=text_config, + tie_word_embeddings=True, + _fsdp_shard_conditions=[], + stacked_params_mapping=[], + ) + config = SimpleNamespace(arch_config=arch_config) + + with get_parallel().override(tp_size=1, tp_rank=0): + model = Qwen3VLForConditionalGeneration(config) + + assert model.lm_head.weight is model.model.get_input_embeddings().weight + parameters = dict(model.named_parameters()) + parameters_with_duplicates = dict(model.named_parameters(remove_duplicate=False)) + assert "model.language_model.embed_tokens.weight" in parameters + assert "lm_head.weight" not in parameters + assert ( + parameters_with_duplicates["lm_head.weight"] + is parameters["model.language_model.embed_tokens.weight"] + ) + + def test_qwen3_multimodal_encoders_layerwise_offload_vision_blocks(): assert "model.visual.blocks" in Qwen3VLForConditionalGeneration.layer_names assert "model.visual.blocks" in MiniMaxH3Qwen3VLEncoder.layer_names