diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py index 5bd7d411a0..1db47cbfe1 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen2_5vl.py @@ -26,6 +26,7 @@ from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl_vision import ( Qwen2_5VLVisionTransformer, ) +from sglang.multimodal_gen.runtime.models.encoders.qwen_vl import QwenVLModelBase from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_rope import ( apply_qwen_vl_text_rope, build_qwen_vl_text_rope, @@ -623,7 +624,7 @@ class Qwen2_5_VLTextModel(nn.Module): ) -class Qwen2_5_VLModel(nn.Module): +class Qwen2_5_VLModel(QwenVLModelBase): base_model_prefix = "" _checkpoint_conversion_mapping = {"^model": "language_model"} # Reference: fix gemma3 grad acc #37208 @@ -641,18 +642,6 @@ class Qwen2_5_VLModel(nn.Module): # Initialize weights and apply final processing # self.post_init() - def get_input_embeddings(self): - return self.language_model.embed_tokens - - def set_input_embeddings(self, value): - self.language_model.embed_tokens = value - - def set_decoder(self, decoder): - self.language_model = decoder - - def get_decoder(self): - return self.language_model - def get_rope_index( self, input_ids: Optional[torch.LongTensor] = None, @@ -855,31 +844,7 @@ class Qwen2_5_VLModel(nn.Module): ).unsqueeze(1) return position_ids, mrope_position_deltas else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = ( - position_ids.unsqueeze(0) - .expand(3, -1, -1) - .to(attention_mask.device) - ) - max_position_ids = position_ids.max(0, keepdim=False)[0].max( - -1, keepdim=True - )[0] - mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] - else: - position_ids = ( - torch.arange(input_ids.shape[1], device=input_ids.device) - .view(1, 1, -1) - .expand(3, input_ids.shape[0], -1) - ) - mrope_position_deltas = torch.zeros( - [input_ids.shape[0], 1], - device=input_ids.device, - dtype=input_ids.dtype, - ) - - return position_ids, mrope_position_deltas + return self._get_text_rope_index(input_ids, attention_mask) def get_video_features( self, @@ -926,68 +891,6 @@ class Qwen2_5_VLModel(nn.Module): image_embeds = torch.split(image_embeds, split_sizes) return image_embeds - def get_placeholder_mask( - self, - input_ids: torch.LongTensor, - inputs_embeds: torch.FloatTensor, - image_features: torch.FloatTensor = None, - video_features: torch.FloatTensor = None, - ): - """ - Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is - equal to the length of multimodal features. If the lengths are different, an error is raised. - """ - if input_ids is None: - special_image_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor( - self.config.image_token_id, - dtype=torch.long, - device=inputs_embeds.device, - ) - ) - special_image_mask = special_image_mask.all(-1) - special_video_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor( - self.config.video_token_id, - dtype=torch.long, - device=inputs_embeds.device, - ) - ) - special_video_mask = special_video_mask.all(-1) - else: - special_image_mask = input_ids == self.config.image_token_id - special_video_mask = input_ids == self.config.video_token_id - - n_image_tokens = special_image_mask.sum() - special_image_mask = ( - special_image_mask.unsqueeze(-1) - .expand_as(inputs_embeds) - .to(inputs_embeds.device) - ) - if ( - image_features is not None - and inputs_embeds[special_image_mask].numel() != image_features.numel() - ): - raise ValueError( - f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" - ) - - n_video_tokens = special_video_mask.sum() - special_video_mask = ( - special_video_mask.unsqueeze(-1) - .expand_as(inputs_embeds) - .to(inputs_embeds.device) - ) - if ( - video_features is not None - and inputs_embeds[special_video_mask].numel() != video_features.numel() - ): - raise ValueError( - f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" - ) - - return special_image_mask, special_video_mask - def forward( self, input_ids: torch.LongTensor = None, diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py index 9c8850c666..67bcd97605 100644 --- a/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen3vl.py @@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import ( Qwen3VLVisionTransformer, ) +from sglang.multimodal_gen.runtime.models.encoders.qwen_vl import QwenVLModelBase from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_rope import ( apply_qwen_vl_text_rope, build_qwen_vl_text_rope, @@ -679,7 +680,7 @@ class Qwen3VLTextModel(nn.Module): return hidden_states -class Qwen3VLModel(nn.Module): +class Qwen3VLModel(QwenVLModelBase): base_model_prefix = "" _checkpoint_conversion_mapping = {} # Reference: fix gemma3 grad acc #37208 @@ -717,18 +718,6 @@ class Qwen3VLModel(nn.Module): # Initialize weights and apply final processing - def get_input_embeddings(self): - return self.language_model.embed_tokens - - def set_input_embeddings(self, value): - self.language_model.embed_tokens = value - - def set_decoder(self, decoder): - self.language_model = decoder - - def get_decoder(self): - return self.language_model - def get_rope_index( self, input_ids: Optional[torch.LongTensor] = None, @@ -869,31 +858,7 @@ class Qwen3VLModel(nn.Module): ).unsqueeze(1) return position_ids, mrope_position_deltas else: - if attention_mask is not None: - position_ids = attention_mask.long().cumsum(-1) - 1 - position_ids.masked_fill_(attention_mask == 0, 1) - position_ids = ( - position_ids.unsqueeze(0) - .expand(3, -1, -1) - .to(attention_mask.device) - ) - max_position_ids = position_ids.max(0, keepdim=False)[0].max( - -1, keepdim=True - )[0] - mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] - else: - position_ids = ( - torch.arange(input_ids.shape[1], device=input_ids.device) - .view(1, 1, -1) - .expand(3, input_ids.shape[0], -1) - ) - mrope_position_deltas = torch.zeros( - [input_ids.shape[0], 1], - device=input_ids.device, - dtype=input_ids.dtype, - ) - - return position_ids, mrope_position_deltas + return self._get_text_rope_index(input_ids, attention_mask) def get_video_features( self, @@ -944,68 +909,6 @@ class Qwen3VLModel(nn.Module): image_embeds = torch.split(image_embeds, split_sizes) return image_embeds, deepstack_image_embeds - def get_placeholder_mask( - self, - input_ids: torch.LongTensor, - inputs_embeds: torch.FloatTensor, - image_features: Optional[torch.FloatTensor] = None, - video_features: Optional[torch.FloatTensor] = None, - ): - """ - Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is - equal to the length of multimodal features. If the lengths are different, an error is raised. - """ - if input_ids is None: - special_image_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor( - self.config.image_token_id, - dtype=torch.long, - device=inputs_embeds.device, - ) - ) - special_image_mask = special_image_mask.all(-1) - special_video_mask = inputs_embeds == self.get_input_embeddings()( - torch.tensor( - self.config.video_token_id, - dtype=torch.long, - device=inputs_embeds.device, - ) - ) - special_video_mask = special_video_mask.all(-1) - else: - special_image_mask = input_ids == self.config.image_token_id - special_video_mask = input_ids == self.config.video_token_id - - n_image_tokens = special_image_mask.sum() - special_image_mask = ( - special_image_mask.unsqueeze(-1) - .expand_as(inputs_embeds) - .to(inputs_embeds.device) - ) - if ( - image_features is not None - and inputs_embeds[special_image_mask].numel() != image_features.numel() - ): - raise ValueError( - f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" - ) - - n_video_tokens = special_video_mask.sum() - special_video_mask = ( - special_video_mask.unsqueeze(-1) - .expand_as(inputs_embeds) - .to(inputs_embeds.device) - ) - if ( - video_features is not None - and inputs_embeds[special_video_mask].numel() != video_features.numel() - ): - raise ValueError( - f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" - ) - - return special_image_mask, special_video_mask - def forward( self, input_ids: torch.LongTensor = None, diff --git a/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl.py b/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl.py new file mode 100644 index 0000000000..96ab8ebafd --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/encoders/qwen_vl.py @@ -0,0 +1,105 @@ +# SPDX-License-Identifier: Apache-2.0 +# Adapted from HuggingFace Transformers; Copyright 2024 The Qwen team + +import torch +from torch import nn + + +class QwenVLModelBase(nn.Module): + """Shared text positions, embedding access and multimodal placeholder validation.""" + + def get_input_embeddings(self): + return self.language_model.embed_tokens + + def set_input_embeddings(self, value): + self.language_model.embed_tokens = value + + def set_decoder(self, decoder): + self.language_model = decoder + + def get_decoder(self): + return self.language_model + + def _get_text_rope_index(self, input_ids, attention_mask): + if attention_mask is not None: + position_ids = attention_mask.long().cumsum(-1) - 1 + position_ids.masked_fill_(attention_mask == 0, 1) + position_ids = ( + position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device) + ) + max_position_ids = position_ids.max(0, keepdim=False)[0].max( + -1, keepdim=True + )[0] + mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1] + else: + position_ids = ( + torch.arange(input_ids.shape[1], device=input_ids.device) + .view(1, 1, -1) + .expand(3, input_ids.shape[0], -1) + ) + mrope_position_deltas = torch.zeros( + [input_ids.shape[0], 1], + device=input_ids.device, + dtype=input_ids.dtype, + ) + + return position_ids, mrope_position_deltas + + def get_placeholder_mask( + self, + input_ids: torch.LongTensor, + inputs_embeds: torch.FloatTensor, + image_features: torch.FloatTensor | None = None, + video_features: torch.FloatTensor | None = None, + ): + """Match placeholder tokens to the number of image and video features.""" + if input_ids is None: + special_image_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor( + self.config.image_token_id, + dtype=torch.long, + device=inputs_embeds.device, + ) + ) + special_image_mask = special_image_mask.all(-1) + special_video_mask = inputs_embeds == self.get_input_embeddings()( + torch.tensor( + self.config.video_token_id, + dtype=torch.long, + device=inputs_embeds.device, + ) + ) + special_video_mask = special_video_mask.all(-1) + else: + special_image_mask = input_ids == self.config.image_token_id + special_video_mask = input_ids == self.config.video_token_id + + n_image_tokens = special_image_mask.sum() + special_image_mask = ( + special_image_mask.unsqueeze(-1) + .expand_as(inputs_embeds) + .to(inputs_embeds.device) + ) + if ( + image_features is not None + and inputs_embeds[special_image_mask].numel() != image_features.numel() + ): + raise ValueError( + f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}" + ) + + n_video_tokens = special_video_mask.sum() + special_video_mask = ( + special_video_mask.unsqueeze(-1) + .expand_as(inputs_embeds) + .to(inputs_embeds.device) + ) + if ( + video_features is not None + and inputs_embeds[special_video_mask].numel() != video_features.numel() + ): + raise ValueError( + f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}" + ) + + return special_image_mask, special_video_mask diff --git a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/suite.py b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/suite.py new file mode 100644 index 0000000000..57910e36a4 --- /dev/null +++ b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/suite.py @@ -0,0 +1,63 @@ +import pytest + +from sglang.multimodal_gen.test.single_test_file.component_accuracy.config import ( + ComponentType, + get_skip_reason, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.engine import ( + AccuracyEngine, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.testcase_configs import ( + get_component_duplicate_skip_reason, +) +from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( + run_native_component_accuracy_case, + run_text_encoder_accuracy_case, +) + + +def _skip_component_if_needed(case, component): + reason = get_skip_reason(case, component) + if reason is not None: + pytest.skip(reason) + duplicate_reason = get_component_duplicate_skip_reason(case, component) + if duplicate_reason: + pytest.skip(duplicate_reason) + + +class ComponentAccuracySuite: + def test_vae_accuracy(self, case): + _skip_component_if_needed(case, ComponentType.VAE) + run_native_component_accuracy_case( + AccuracyEngine, + case, + ComponentType.VAE, + "diffusers", + case.server_args.num_gpus, + ) + + def test_transformer_accuracy(self, case): + _skip_component_if_needed(case, ComponentType.TRANSFORMER) + run_native_component_accuracy_case( + AccuracyEngine, + case, + ComponentType.TRANSFORMER, + "diffusers", + case.server_args.num_gpus, + ) + + def test_encoder_accuracy(self, case): + _skip_component_if_needed(case, ComponentType.TEXT_ENCODER) + run_text_encoder_accuracy_case( + AccuracyEngine, + case, + case.server_args.num_gpus, + ) + + +class VAEChannelsLast3DParitySuite: + def test_vae_channels_last_3d_parity(self, case): + AccuracyEngine.run_vae_channels_last_3d_parity( + case, + case.server_args.num_gpus, + ) diff --git a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_1_gpu.py b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_1_gpu.py index 7ba09e513f..b7b868f40f 100644 --- a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_1_gpu.py +++ b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_1_gpu.py @@ -1,89 +1,25 @@ import pytest -from sglang.multimodal_gen.test.single_test_file.component_accuracy.config import ( - ComponentType, - get_skip_reason, - should_skip_component, -) -from sglang.multimodal_gen.test.single_test_file.component_accuracy.engine import ( - AccuracyEngine, +from sglang.multimodal_gen.test.single_test_file.component_accuracy.suite import ( + ComponentAccuracySuite, + VAEChannelsLast3DParitySuite, ) from sglang.multimodal_gen.test.single_test_file.component_accuracy.testcase_configs import ( ACCURACY_ONE_GPU_CASES, - get_component_duplicate_skip_reason, -) -from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( - run_native_component_accuracy_case, - run_text_encoder_accuracy_case, ) -VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS = { - "wan2_1_t2v_1.3b", -} VAE_CHANNELS_LAST_3D_PARITY_CASES = [ - case - for case in ACCURACY_ONE_GPU_CASES - if case.id in VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS + case for case in ACCURACY_ONE_GPU_CASES if case.id == "wan2_1_t2v_1.3b" ] @pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES, ids=lambda case: case.id) -class TestComponentAccuracy1GPU: +class TestComponentAccuracy1GPU(ComponentAccuracySuite): """1-GPU component accuracy suite.""" - def test_vae_accuracy(self, case): - if should_skip_component(case, ComponentType.VAE): - pytest.skip(get_skip_reason(case, ComponentType.VAE)) - duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE) - if duplicate_reason: - pytest.skip(duplicate_reason) - run_native_component_accuracy_case( - AccuracyEngine, - case, - ComponentType.VAE, - "diffusers", - case.server_args.num_gpus, - ) - - def test_transformer_accuracy(self, case): - if should_skip_component(case, ComponentType.TRANSFORMER): - pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) - duplicate_reason = get_component_duplicate_skip_reason( - case, ComponentType.TRANSFORMER - ) - if duplicate_reason: - pytest.skip(duplicate_reason) - run_native_component_accuracy_case( - AccuracyEngine, - case, - ComponentType.TRANSFORMER, - "diffusers", - case.server_args.num_gpus, - ) - - def test_encoder_accuracy(self, case): - if should_skip_component(case, ComponentType.TEXT_ENCODER): - pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) - duplicate_reason = get_component_duplicate_skip_reason( - case, ComponentType.TEXT_ENCODER - ) - if duplicate_reason: - pytest.skip(duplicate_reason) - run_text_encoder_accuracy_case( - AccuracyEngine, - case, - case.server_args.num_gpus, - ) - @pytest.mark.parametrize( "case", VAE_CHANNELS_LAST_3D_PARITY_CASES, ids=lambda case: case.id ) -class TestVAEChannelsLast3DParity1GPU: +class TestVAEChannelsLast3DParity1GPU(VAEChannelsLast3DParitySuite): """1-GPU VAE guard for channels_last_3d drift.""" - - def test_vae_channels_last_3d_parity(self, case): - AccuracyEngine.run_vae_channels_last_3d_parity( - case, - case.server_args.num_gpus, - ) diff --git a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_2_gpu.py b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_2_gpu.py index 49bfa36cfe..d921bc91fc 100644 --- a/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_2_gpu.py +++ b/python/sglang/multimodal_gen/test/single_test_file/component_accuracy/test_component_accuracy_2_gpu.py @@ -1,89 +1,25 @@ import pytest -from sglang.multimodal_gen.test.single_test_file.component_accuracy.config import ( - ComponentType, - get_skip_reason, - should_skip_component, -) -from sglang.multimodal_gen.test.single_test_file.component_accuracy.engine import ( - AccuracyEngine, +from sglang.multimodal_gen.test.single_test_file.component_accuracy.suite import ( + ComponentAccuracySuite, + VAEChannelsLast3DParitySuite, ) from sglang.multimodal_gen.test.single_test_file.component_accuracy.testcase_configs import ( ACCURACY_TWO_GPU_CASES, - get_component_duplicate_skip_reason, -) -from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import ( - run_native_component_accuracy_case, - run_text_encoder_accuracy_case, ) -VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS = { - "wan2_2_i2v_a14b_2gpu", -} VAE_CHANNELS_LAST_3D_PARITY_CASES = [ - case - for case in ACCURACY_TWO_GPU_CASES - if case.id in VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS + case for case in ACCURACY_TWO_GPU_CASES if case.id == "wan2_2_i2v_a14b_2gpu" ] @pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES, ids=lambda case: case.id) -class TestComponentAccuracy2GPU: +class TestComponentAccuracy2GPU(ComponentAccuracySuite): """2-GPU component accuracy suite.""" - def test_vae_accuracy(self, case): - if should_skip_component(case, ComponentType.VAE): - pytest.skip(get_skip_reason(case, ComponentType.VAE)) - duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE) - if duplicate_reason: - pytest.skip(duplicate_reason) - run_native_component_accuracy_case( - AccuracyEngine, - case, - ComponentType.VAE, - "diffusers", - case.server_args.num_gpus, - ) - - def test_transformer_accuracy(self, case): - if should_skip_component(case, ComponentType.TRANSFORMER): - pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER)) - duplicate_reason = get_component_duplicate_skip_reason( - case, ComponentType.TRANSFORMER - ) - if duplicate_reason: - pytest.skip(duplicate_reason) - run_native_component_accuracy_case( - AccuracyEngine, - case, - ComponentType.TRANSFORMER, - "diffusers", - case.server_args.num_gpus, - ) - - def test_encoder_accuracy(self, case): - if should_skip_component(case, ComponentType.TEXT_ENCODER): - pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER)) - duplicate_reason = get_component_duplicate_skip_reason( - case, ComponentType.TEXT_ENCODER - ) - if duplicate_reason: - pytest.skip(duplicate_reason) - run_text_encoder_accuracy_case( - AccuracyEngine, - case, - case.server_args.num_gpus, - ) - @pytest.mark.parametrize( "case", VAE_CHANNELS_LAST_3D_PARITY_CASES, ids=lambda case: case.id ) -class TestVAEChannelsLast3DParity2GPU: +class TestVAEChannelsLast3DParity2GPU(VAEChannelsLast3DParitySuite): """2-GPU VAE guard for channels_last_3d drift.""" - - def test_vae_channels_last_3d_parity(self, case): - AccuracyEngine.run_vae_channels_last_3d_parity( - case, - case.server_args.num_gpus, - ) diff --git a/python/sglang/multimodal_gen/test/unit/test_qwen_vl_inputs.py b/python/sglang/multimodal_gen/test/unit/test_qwen_vl_inputs.py new file mode 100644 index 0000000000..3cf7a90257 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_qwen_vl_inputs.py @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch +from torch import nn +from transformers import Qwen2_5_VLConfig, Qwen3VLConfig + +from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl import Qwen2_5_VLModel +from sglang.multimodal_gen.runtime.models.encoders.qwen3vl import Qwen3VLModel +from sglang.srt.runtime_context import get_parallel + + +@pytest.fixture(params=[Qwen2_5_VLConfig, Qwen3VLConfig]) +def model(request): + config = request.param( + image_token_id=2, + video_token_id=3, + text_config=dict( + vocab_size=32, + hidden_size=16, + intermediate_size=24, + num_hidden_layers=0, + num_attention_heads=2, + num_key_value_heads=2, + pad_token_id=0, + bos_token_id=1, + eos_token_id=1, + ), + vision_config=dict( + hidden_size=16, + intermediate_size=24, + depth=0, + num_heads=2, + patch_size=2, + temporal_patch_size=1, + spatial_merge_size=2, + out_hidden_size=16, + num_position_embeddings=16, + deepstack_visual_indexes=[], + ), + ) + with get_parallel().override(tp_size=1, tp_rank=0, tp_group=None): + yield ( + Qwen2_5_VLModel(config, enable_image_understanding=True) + if isinstance(config, Qwen2_5_VLConfig) + else Qwen3VLModel(config) + ) + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +def test_qwen_vl_placeholder_masks(model, device, dtype): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA required") + model.to(device=device, dtype=dtype) + weight = model.get_input_embeddings().weight + with torch.no_grad(): + weight.copy_(torch.arange(weight.shape[0], device=device)[:, None]) + ids = torch.tensor([[1, 2, 2, 3], [3, 3, 4, 1]], device=device) + embeds = model.get_input_embeddings()(ids) + features = {"image_features": embeds[ids == 2], "video_features": embeds[ids == 3]} + for input_ids in (ids.cpu(), None): + masks = model.get_placeholder_mask(input_ids, embeds, **features) + for token_id, mask in zip((2, 3), masks): + expected = (ids == token_id).unsqueeze(-1).expand_as(embeds) + torch.testing.assert_close(mask, expected) + for kind, message in (("image", "Image"), ("video", "Videos")): + invalid = dict(features) + invalid[f"{kind}_features"] = features[f"{kind}_features"][:-1] + with pytest.raises(ValueError, match=f"{message} features and .* tokens"): + model.get_placeholder_mask(input_ids, embeds, **invalid) + empty = model.get_placeholder_mask(input_ids, embeds) + assert all( + torch.equal(actual, expected) for actual, expected in zip(empty, masks) + ) + no_media = model.get_placeholder_mask(ids.new_ones(2, 4), embeds) + assert not any(mask.any() for mask in no_media) + + +def test_qwen_vl_embedding_and_decoder_accessors(model): + keys = set(model.state_dict()) + replacement = nn.Embedding(32, 16) + model.set_input_embeddings(replacement) + assert model.get_input_embeddings() is replacement + assert model.get_decoder().embed_tokens is replacement + assert set(model.state_dict()) == keys + assert ( + dict(model.named_parameters())["language_model.embed_tokens.weight"] + is replacement.weight + ) + decoder = nn.Module() + decoder.embed_tokens = replacement + model.set_decoder(decoder) + assert model.get_decoder() is decoder + assert model.get_input_embeddings() is replacement + assert set(model.state_dict()) == { + "language_model.embed_tokens.weight", + *(f"visual.{name}" for name in model.visual.state_dict()), + }