[diffusion] refactor: deduplicate Qwen-VL inputs and component accuracy suites (#42860)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -26,6 +26,7 @@ from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl_vision import (
|
||||
Qwen2_5VLVisionTransformer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl import QwenVLModelBase
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_rope import (
|
||||
apply_qwen_vl_text_rope,
|
||||
build_qwen_vl_text_rope,
|
||||
@@ -623,7 +624,7 @@ class Qwen2_5_VLTextModel(nn.Module):
|
||||
)
|
||||
|
||||
|
||||
class Qwen2_5_VLModel(nn.Module):
|
||||
class Qwen2_5_VLModel(QwenVLModelBase):
|
||||
base_model_prefix = ""
|
||||
_checkpoint_conversion_mapping = {"^model": "language_model"}
|
||||
# Reference: fix gemma3 grad acc #37208
|
||||
@@ -641,18 +642,6 @@ class Qwen2_5_VLModel(nn.Module):
|
||||
# Initialize weights and apply final processing
|
||||
# self.post_init()
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.language_model.embed_tokens
|
||||
|
||||
def set_input_embeddings(self, value):
|
||||
self.language_model.embed_tokens = value
|
||||
|
||||
def set_decoder(self, decoder):
|
||||
self.language_model = decoder
|
||||
|
||||
def get_decoder(self):
|
||||
return self.language_model
|
||||
|
||||
def get_rope_index(
|
||||
self,
|
||||
input_ids: Optional[torch.LongTensor] = None,
|
||||
@@ -855,31 +844,7 @@ class Qwen2_5_VLModel(nn.Module):
|
||||
).unsqueeze(1)
|
||||
return position_ids, mrope_position_deltas
|
||||
else:
|
||||
if attention_mask is not None:
|
||||
position_ids = attention_mask.long().cumsum(-1) - 1
|
||||
position_ids.masked_fill_(attention_mask == 0, 1)
|
||||
position_ids = (
|
||||
position_ids.unsqueeze(0)
|
||||
.expand(3, -1, -1)
|
||||
.to(attention_mask.device)
|
||||
)
|
||||
max_position_ids = position_ids.max(0, keepdim=False)[0].max(
|
||||
-1, keepdim=True
|
||||
)[0]
|
||||
mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]
|
||||
else:
|
||||
position_ids = (
|
||||
torch.arange(input_ids.shape[1], device=input_ids.device)
|
||||
.view(1, 1, -1)
|
||||
.expand(3, input_ids.shape[0], -1)
|
||||
)
|
||||
mrope_position_deltas = torch.zeros(
|
||||
[input_ids.shape[0], 1],
|
||||
device=input_ids.device,
|
||||
dtype=input_ids.dtype,
|
||||
)
|
||||
|
||||
return position_ids, mrope_position_deltas
|
||||
return self._get_text_rope_index(input_ids, attention_mask)
|
||||
|
||||
def get_video_features(
|
||||
self,
|
||||
@@ -926,68 +891,6 @@ class Qwen2_5_VLModel(nn.Module):
|
||||
image_embeds = torch.split(image_embeds, split_sizes)
|
||||
return image_embeds
|
||||
|
||||
def get_placeholder_mask(
|
||||
self,
|
||||
input_ids: torch.LongTensor,
|
||||
inputs_embeds: torch.FloatTensor,
|
||||
image_features: torch.FloatTensor = None,
|
||||
video_features: torch.FloatTensor = None,
|
||||
):
|
||||
"""
|
||||
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
|
||||
equal to the length of multimodal features. If the lengths are different, an error is raised.
|
||||
"""
|
||||
if input_ids is None:
|
||||
special_image_mask = inputs_embeds == self.get_input_embeddings()(
|
||||
torch.tensor(
|
||||
self.config.image_token_id,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
)
|
||||
special_image_mask = special_image_mask.all(-1)
|
||||
special_video_mask = inputs_embeds == self.get_input_embeddings()(
|
||||
torch.tensor(
|
||||
self.config.video_token_id,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
)
|
||||
special_video_mask = special_video_mask.all(-1)
|
||||
else:
|
||||
special_image_mask = input_ids == self.config.image_token_id
|
||||
special_video_mask = input_ids == self.config.video_token_id
|
||||
|
||||
n_image_tokens = special_image_mask.sum()
|
||||
special_image_mask = (
|
||||
special_image_mask.unsqueeze(-1)
|
||||
.expand_as(inputs_embeds)
|
||||
.to(inputs_embeds.device)
|
||||
)
|
||||
if (
|
||||
image_features is not None
|
||||
and inputs_embeds[special_image_mask].numel() != image_features.numel()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}"
|
||||
)
|
||||
|
||||
n_video_tokens = special_video_mask.sum()
|
||||
special_video_mask = (
|
||||
special_video_mask.unsqueeze(-1)
|
||||
.expand_as(inputs_embeds)
|
||||
.to(inputs_embeds.device)
|
||||
)
|
||||
if (
|
||||
video_features is not None
|
||||
and inputs_embeds[special_video_mask].numel() != video_features.numel()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}"
|
||||
)
|
||||
|
||||
return special_image_mask, special_video_mask
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.LongTensor = None,
|
||||
|
||||
@@ -35,6 +35,7 @@ from sglang.multimodal_gen.runtime.models.encoders.base import TextEncoder
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl_vision import (
|
||||
Qwen3VLVisionTransformer,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl import QwenVLModelBase
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen_vl_rope import (
|
||||
apply_qwen_vl_text_rope,
|
||||
build_qwen_vl_text_rope,
|
||||
@@ -679,7 +680,7 @@ class Qwen3VLTextModel(nn.Module):
|
||||
return hidden_states
|
||||
|
||||
|
||||
class Qwen3VLModel(nn.Module):
|
||||
class Qwen3VLModel(QwenVLModelBase):
|
||||
base_model_prefix = ""
|
||||
_checkpoint_conversion_mapping = {}
|
||||
# Reference: fix gemma3 grad acc #37208
|
||||
@@ -717,18 +718,6 @@ class Qwen3VLModel(nn.Module):
|
||||
|
||||
# Initialize weights and apply final processing
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.language_model.embed_tokens
|
||||
|
||||
def set_input_embeddings(self, value):
|
||||
self.language_model.embed_tokens = value
|
||||
|
||||
def set_decoder(self, decoder):
|
||||
self.language_model = decoder
|
||||
|
||||
def get_decoder(self):
|
||||
return self.language_model
|
||||
|
||||
def get_rope_index(
|
||||
self,
|
||||
input_ids: Optional[torch.LongTensor] = None,
|
||||
@@ -869,31 +858,7 @@ class Qwen3VLModel(nn.Module):
|
||||
).unsqueeze(1)
|
||||
return position_ids, mrope_position_deltas
|
||||
else:
|
||||
if attention_mask is not None:
|
||||
position_ids = attention_mask.long().cumsum(-1) - 1
|
||||
position_ids.masked_fill_(attention_mask == 0, 1)
|
||||
position_ids = (
|
||||
position_ids.unsqueeze(0)
|
||||
.expand(3, -1, -1)
|
||||
.to(attention_mask.device)
|
||||
)
|
||||
max_position_ids = position_ids.max(0, keepdim=False)[0].max(
|
||||
-1, keepdim=True
|
||||
)[0]
|
||||
mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]
|
||||
else:
|
||||
position_ids = (
|
||||
torch.arange(input_ids.shape[1], device=input_ids.device)
|
||||
.view(1, 1, -1)
|
||||
.expand(3, input_ids.shape[0], -1)
|
||||
)
|
||||
mrope_position_deltas = torch.zeros(
|
||||
[input_ids.shape[0], 1],
|
||||
device=input_ids.device,
|
||||
dtype=input_ids.dtype,
|
||||
)
|
||||
|
||||
return position_ids, mrope_position_deltas
|
||||
return self._get_text_rope_index(input_ids, attention_mask)
|
||||
|
||||
def get_video_features(
|
||||
self,
|
||||
@@ -944,68 +909,6 @@ class Qwen3VLModel(nn.Module):
|
||||
image_embeds = torch.split(image_embeds, split_sizes)
|
||||
return image_embeds, deepstack_image_embeds
|
||||
|
||||
def get_placeholder_mask(
|
||||
self,
|
||||
input_ids: torch.LongTensor,
|
||||
inputs_embeds: torch.FloatTensor,
|
||||
image_features: Optional[torch.FloatTensor] = None,
|
||||
video_features: Optional[torch.FloatTensor] = None,
|
||||
):
|
||||
"""
|
||||
Obtains multimodal placeholder mask from `input_ids` or `inputs_embeds`, and checks that the placeholder token count is
|
||||
equal to the length of multimodal features. If the lengths are different, an error is raised.
|
||||
"""
|
||||
if input_ids is None:
|
||||
special_image_mask = inputs_embeds == self.get_input_embeddings()(
|
||||
torch.tensor(
|
||||
self.config.image_token_id,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
)
|
||||
special_image_mask = special_image_mask.all(-1)
|
||||
special_video_mask = inputs_embeds == self.get_input_embeddings()(
|
||||
torch.tensor(
|
||||
self.config.video_token_id,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
)
|
||||
special_video_mask = special_video_mask.all(-1)
|
||||
else:
|
||||
special_image_mask = input_ids == self.config.image_token_id
|
||||
special_video_mask = input_ids == self.config.video_token_id
|
||||
|
||||
n_image_tokens = special_image_mask.sum()
|
||||
special_image_mask = (
|
||||
special_image_mask.unsqueeze(-1)
|
||||
.expand_as(inputs_embeds)
|
||||
.to(inputs_embeds.device)
|
||||
)
|
||||
if (
|
||||
image_features is not None
|
||||
and inputs_embeds[special_image_mask].numel() != image_features.numel()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}"
|
||||
)
|
||||
|
||||
n_video_tokens = special_video_mask.sum()
|
||||
special_video_mask = (
|
||||
special_video_mask.unsqueeze(-1)
|
||||
.expand_as(inputs_embeds)
|
||||
.to(inputs_embeds.device)
|
||||
)
|
||||
if (
|
||||
video_features is not None
|
||||
and inputs_embeds[special_video_mask].numel() != video_features.numel()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}"
|
||||
)
|
||||
|
||||
return special_image_mask, special_video_mask
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.LongTensor = None,
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from HuggingFace Transformers; Copyright 2024 The Qwen team
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
class QwenVLModelBase(nn.Module):
|
||||
"""Shared text positions, embedding access and multimodal placeholder validation."""
|
||||
|
||||
def get_input_embeddings(self):
|
||||
return self.language_model.embed_tokens
|
||||
|
||||
def set_input_embeddings(self, value):
|
||||
self.language_model.embed_tokens = value
|
||||
|
||||
def set_decoder(self, decoder):
|
||||
self.language_model = decoder
|
||||
|
||||
def get_decoder(self):
|
||||
return self.language_model
|
||||
|
||||
def _get_text_rope_index(self, input_ids, attention_mask):
|
||||
if attention_mask is not None:
|
||||
position_ids = attention_mask.long().cumsum(-1) - 1
|
||||
position_ids.masked_fill_(attention_mask == 0, 1)
|
||||
position_ids = (
|
||||
position_ids.unsqueeze(0).expand(3, -1, -1).to(attention_mask.device)
|
||||
)
|
||||
max_position_ids = position_ids.max(0, keepdim=False)[0].max(
|
||||
-1, keepdim=True
|
||||
)[0]
|
||||
mrope_position_deltas = max_position_ids + 1 - attention_mask.shape[-1]
|
||||
else:
|
||||
position_ids = (
|
||||
torch.arange(input_ids.shape[1], device=input_ids.device)
|
||||
.view(1, 1, -1)
|
||||
.expand(3, input_ids.shape[0], -1)
|
||||
)
|
||||
mrope_position_deltas = torch.zeros(
|
||||
[input_ids.shape[0], 1],
|
||||
device=input_ids.device,
|
||||
dtype=input_ids.dtype,
|
||||
)
|
||||
|
||||
return position_ids, mrope_position_deltas
|
||||
|
||||
def get_placeholder_mask(
|
||||
self,
|
||||
input_ids: torch.LongTensor,
|
||||
inputs_embeds: torch.FloatTensor,
|
||||
image_features: torch.FloatTensor | None = None,
|
||||
video_features: torch.FloatTensor | None = None,
|
||||
):
|
||||
"""Match placeholder tokens to the number of image and video features."""
|
||||
if input_ids is None:
|
||||
special_image_mask = inputs_embeds == self.get_input_embeddings()(
|
||||
torch.tensor(
|
||||
self.config.image_token_id,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
)
|
||||
special_image_mask = special_image_mask.all(-1)
|
||||
special_video_mask = inputs_embeds == self.get_input_embeddings()(
|
||||
torch.tensor(
|
||||
self.config.video_token_id,
|
||||
dtype=torch.long,
|
||||
device=inputs_embeds.device,
|
||||
)
|
||||
)
|
||||
special_video_mask = special_video_mask.all(-1)
|
||||
else:
|
||||
special_image_mask = input_ids == self.config.image_token_id
|
||||
special_video_mask = input_ids == self.config.video_token_id
|
||||
|
||||
n_image_tokens = special_image_mask.sum()
|
||||
special_image_mask = (
|
||||
special_image_mask.unsqueeze(-1)
|
||||
.expand_as(inputs_embeds)
|
||||
.to(inputs_embeds.device)
|
||||
)
|
||||
if (
|
||||
image_features is not None
|
||||
and inputs_embeds[special_image_mask].numel() != image_features.numel()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {image_features.shape[0]}"
|
||||
)
|
||||
|
||||
n_video_tokens = special_video_mask.sum()
|
||||
special_video_mask = (
|
||||
special_video_mask.unsqueeze(-1)
|
||||
.expand_as(inputs_embeds)
|
||||
.to(inputs_embeds.device)
|
||||
)
|
||||
if (
|
||||
video_features is not None
|
||||
and inputs_embeds[special_video_mask].numel() != video_features.numel()
|
||||
):
|
||||
raise ValueError(
|
||||
f"Videos features and video tokens do not match: tokens: {n_video_tokens}, features {video_features.shape[0]}"
|
||||
)
|
||||
|
||||
return special_image_mask, special_video_mask
|
||||
@@ -0,0 +1,63 @@
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.config import (
|
||||
ComponentType,
|
||||
get_skip_reason,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.engine import (
|
||||
AccuracyEngine,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.testcase_configs import (
|
||||
get_component_duplicate_skip_reason,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
run_native_component_accuracy_case,
|
||||
run_text_encoder_accuracy_case,
|
||||
)
|
||||
|
||||
|
||||
def _skip_component_if_needed(case, component):
|
||||
reason = get_skip_reason(case, component)
|
||||
if reason is not None:
|
||||
pytest.skip(reason)
|
||||
duplicate_reason = get_component_duplicate_skip_reason(case, component)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
|
||||
|
||||
class ComponentAccuracySuite:
|
||||
def test_vae_accuracy(self, case):
|
||||
_skip_component_if_needed(case, ComponentType.VAE)
|
||||
run_native_component_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
ComponentType.VAE,
|
||||
"diffusers",
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
def test_transformer_accuracy(self, case):
|
||||
_skip_component_if_needed(case, ComponentType.TRANSFORMER)
|
||||
run_native_component_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
ComponentType.TRANSFORMER,
|
||||
"diffusers",
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
def test_encoder_accuracy(self, case):
|
||||
_skip_component_if_needed(case, ComponentType.TEXT_ENCODER)
|
||||
run_text_encoder_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
|
||||
class VAEChannelsLast3DParitySuite:
|
||||
def test_vae_channels_last_3d_parity(self, case):
|
||||
AccuracyEngine.run_vae_channels_last_3d_parity(
|
||||
case,
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
+6
-70
@@ -1,89 +1,25 @@
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.config import (
|
||||
ComponentType,
|
||||
get_skip_reason,
|
||||
should_skip_component,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.engine import (
|
||||
AccuracyEngine,
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.suite import (
|
||||
ComponentAccuracySuite,
|
||||
VAEChannelsLast3DParitySuite,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.testcase_configs import (
|
||||
ACCURACY_ONE_GPU_CASES,
|
||||
get_component_duplicate_skip_reason,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
run_native_component_accuracy_case,
|
||||
run_text_encoder_accuracy_case,
|
||||
)
|
||||
|
||||
VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS = {
|
||||
"wan2_1_t2v_1.3b",
|
||||
}
|
||||
VAE_CHANNELS_LAST_3D_PARITY_CASES = [
|
||||
case
|
||||
for case in ACCURACY_ONE_GPU_CASES
|
||||
if case.id in VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS
|
||||
case for case in ACCURACY_ONE_GPU_CASES if case.id == "wan2_1_t2v_1.3b"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ACCURACY_ONE_GPU_CASES, ids=lambda case: case.id)
|
||||
class TestComponentAccuracy1GPU:
|
||||
class TestComponentAccuracy1GPU(ComponentAccuracySuite):
|
||||
"""1-GPU component accuracy suite."""
|
||||
|
||||
def test_vae_accuracy(self, case):
|
||||
if should_skip_component(case, ComponentType.VAE):
|
||||
pytest.skip(get_skip_reason(case, ComponentType.VAE))
|
||||
duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
run_native_component_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
ComponentType.VAE,
|
||||
"diffusers",
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
def test_transformer_accuracy(self, case):
|
||||
if should_skip_component(case, ComponentType.TRANSFORMER):
|
||||
pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
|
||||
duplicate_reason = get_component_duplicate_skip_reason(
|
||||
case, ComponentType.TRANSFORMER
|
||||
)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
run_native_component_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
ComponentType.TRANSFORMER,
|
||||
"diffusers",
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
def test_encoder_accuracy(self, case):
|
||||
if should_skip_component(case, ComponentType.TEXT_ENCODER):
|
||||
pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
|
||||
duplicate_reason = get_component_duplicate_skip_reason(
|
||||
case, ComponentType.TEXT_ENCODER
|
||||
)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
run_text_encoder_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", VAE_CHANNELS_LAST_3D_PARITY_CASES, ids=lambda case: case.id
|
||||
)
|
||||
class TestVAEChannelsLast3DParity1GPU:
|
||||
class TestVAEChannelsLast3DParity1GPU(VAEChannelsLast3DParitySuite):
|
||||
"""1-GPU VAE guard for channels_last_3d drift."""
|
||||
|
||||
def test_vae_channels_last_3d_parity(self, case):
|
||||
AccuracyEngine.run_vae_channels_last_3d_parity(
|
||||
case,
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
+6
-70
@@ -1,89 +1,25 @@
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.config import (
|
||||
ComponentType,
|
||||
get_skip_reason,
|
||||
should_skip_component,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.engine import (
|
||||
AccuracyEngine,
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.suite import (
|
||||
ComponentAccuracySuite,
|
||||
VAEChannelsLast3DParitySuite,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.testcase_configs import (
|
||||
ACCURACY_TWO_GPU_CASES,
|
||||
get_component_duplicate_skip_reason,
|
||||
)
|
||||
from sglang.multimodal_gen.test.single_test_file.component_accuracy.utils import (
|
||||
run_native_component_accuracy_case,
|
||||
run_text_encoder_accuracy_case,
|
||||
)
|
||||
|
||||
VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS = {
|
||||
"wan2_2_i2v_a14b_2gpu",
|
||||
}
|
||||
VAE_CHANNELS_LAST_3D_PARITY_CASES = [
|
||||
case
|
||||
for case in ACCURACY_TWO_GPU_CASES
|
||||
if case.id in VAE_CHANNELS_LAST_3D_PARITY_CASE_IDS
|
||||
case for case in ACCURACY_TWO_GPU_CASES if case.id == "wan2_2_i2v_a14b_2gpu"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ACCURACY_TWO_GPU_CASES, ids=lambda case: case.id)
|
||||
class TestComponentAccuracy2GPU:
|
||||
class TestComponentAccuracy2GPU(ComponentAccuracySuite):
|
||||
"""2-GPU component accuracy suite."""
|
||||
|
||||
def test_vae_accuracy(self, case):
|
||||
if should_skip_component(case, ComponentType.VAE):
|
||||
pytest.skip(get_skip_reason(case, ComponentType.VAE))
|
||||
duplicate_reason = get_component_duplicate_skip_reason(case, ComponentType.VAE)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
run_native_component_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
ComponentType.VAE,
|
||||
"diffusers",
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
def test_transformer_accuracy(self, case):
|
||||
if should_skip_component(case, ComponentType.TRANSFORMER):
|
||||
pytest.skip(get_skip_reason(case, ComponentType.TRANSFORMER))
|
||||
duplicate_reason = get_component_duplicate_skip_reason(
|
||||
case, ComponentType.TRANSFORMER
|
||||
)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
run_native_component_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
ComponentType.TRANSFORMER,
|
||||
"diffusers",
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
def test_encoder_accuracy(self, case):
|
||||
if should_skip_component(case, ComponentType.TEXT_ENCODER):
|
||||
pytest.skip(get_skip_reason(case, ComponentType.TEXT_ENCODER))
|
||||
duplicate_reason = get_component_duplicate_skip_reason(
|
||||
case, ComponentType.TEXT_ENCODER
|
||||
)
|
||||
if duplicate_reason:
|
||||
pytest.skip(duplicate_reason)
|
||||
run_text_encoder_accuracy_case(
|
||||
AccuracyEngine,
|
||||
case,
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case", VAE_CHANNELS_LAST_3D_PARITY_CASES, ids=lambda case: case.id
|
||||
)
|
||||
class TestVAEChannelsLast3DParity2GPU:
|
||||
class TestVAEChannelsLast3DParity2GPU(VAEChannelsLast3DParitySuite):
|
||||
"""2-GPU VAE guard for channels_last_3d drift."""
|
||||
|
||||
def test_vae_channels_last_3d_parity(self, case):
|
||||
AccuracyEngine.run_vae_channels_last_3d_parity(
|
||||
case,
|
||||
case.server_args.num_gpus,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import Qwen2_5_VLConfig, Qwen3VLConfig
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen2_5vl import Qwen2_5_VLModel
|
||||
from sglang.multimodal_gen.runtime.models.encoders.qwen3vl import Qwen3VLModel
|
||||
from sglang.srt.runtime_context import get_parallel
|
||||
|
||||
|
||||
@pytest.fixture(params=[Qwen2_5_VLConfig, Qwen3VLConfig])
|
||||
def model(request):
|
||||
config = request.param(
|
||||
image_token_id=2,
|
||||
video_token_id=3,
|
||||
text_config=dict(
|
||||
vocab_size=32,
|
||||
hidden_size=16,
|
||||
intermediate_size=24,
|
||||
num_hidden_layers=0,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=2,
|
||||
pad_token_id=0,
|
||||
bos_token_id=1,
|
||||
eos_token_id=1,
|
||||
),
|
||||
vision_config=dict(
|
||||
hidden_size=16,
|
||||
intermediate_size=24,
|
||||
depth=0,
|
||||
num_heads=2,
|
||||
patch_size=2,
|
||||
temporal_patch_size=1,
|
||||
spatial_merge_size=2,
|
||||
out_hidden_size=16,
|
||||
num_position_embeddings=16,
|
||||
deepstack_visual_indexes=[],
|
||||
),
|
||||
)
|
||||
with get_parallel().override(tp_size=1, tp_rank=0, tp_group=None):
|
||||
yield (
|
||||
Qwen2_5_VLModel(config, enable_image_understanding=True)
|
||||
if isinstance(config, Qwen2_5_VLConfig)
|
||||
else Qwen3VLModel(config)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_qwen_vl_placeholder_masks(model, device, dtype):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
model.to(device=device, dtype=dtype)
|
||||
weight = model.get_input_embeddings().weight
|
||||
with torch.no_grad():
|
||||
weight.copy_(torch.arange(weight.shape[0], device=device)[:, None])
|
||||
ids = torch.tensor([[1, 2, 2, 3], [3, 3, 4, 1]], device=device)
|
||||
embeds = model.get_input_embeddings()(ids)
|
||||
features = {"image_features": embeds[ids == 2], "video_features": embeds[ids == 3]}
|
||||
for input_ids in (ids.cpu(), None):
|
||||
masks = model.get_placeholder_mask(input_ids, embeds, **features)
|
||||
for token_id, mask in zip((2, 3), masks):
|
||||
expected = (ids == token_id).unsqueeze(-1).expand_as(embeds)
|
||||
torch.testing.assert_close(mask, expected)
|
||||
for kind, message in (("image", "Image"), ("video", "Videos")):
|
||||
invalid = dict(features)
|
||||
invalid[f"{kind}_features"] = features[f"{kind}_features"][:-1]
|
||||
with pytest.raises(ValueError, match=f"{message} features and .* tokens"):
|
||||
model.get_placeholder_mask(input_ids, embeds, **invalid)
|
||||
empty = model.get_placeholder_mask(input_ids, embeds)
|
||||
assert all(
|
||||
torch.equal(actual, expected) for actual, expected in zip(empty, masks)
|
||||
)
|
||||
no_media = model.get_placeholder_mask(ids.new_ones(2, 4), embeds)
|
||||
assert not any(mask.any() for mask in no_media)
|
||||
|
||||
|
||||
def test_qwen_vl_embedding_and_decoder_accessors(model):
|
||||
keys = set(model.state_dict())
|
||||
replacement = nn.Embedding(32, 16)
|
||||
model.set_input_embeddings(replacement)
|
||||
assert model.get_input_embeddings() is replacement
|
||||
assert model.get_decoder().embed_tokens is replacement
|
||||
assert set(model.state_dict()) == keys
|
||||
assert (
|
||||
dict(model.named_parameters())["language_model.embed_tokens.weight"]
|
||||
is replacement.weight
|
||||
)
|
||||
decoder = nn.Module()
|
||||
decoder.embed_tokens = replacement
|
||||
model.set_decoder(decoder)
|
||||
assert model.get_decoder() is decoder
|
||||
assert model.get_input_embeddings() is replacement
|
||||
assert set(model.state_dict()) == {
|
||||
"language_model.embed_tokens.weight",
|
||||
*(f"visual.{name}" for name in model.visual.state_dict()),
|
||||
}
|
||||
Reference in New Issue
Block a user