[diffusion] fix: fix forward multimodal token type IDs in image encoding (#34494)
Co-authored-by: chilltongx <284668524+chilltongx@users.noreply.github.com>
This commit is contained in:
@@ -189,6 +189,20 @@ class ImageEncodingStage(PipelineStage):
|
||||
|
||||
return postprocess_funcs[0](outputs, image_inputs)
|
||||
|
||||
@staticmethod
|
||||
def _prepare_text_encoder_inputs(image_inputs) -> dict[str, Any]:
|
||||
inputs = {
|
||||
"input_ids": image_inputs.input_ids,
|
||||
"attention_mask": image_inputs.attention_mask,
|
||||
"pixel_values": image_inputs.pixel_values,
|
||||
"image_grid_thw": image_inputs.image_grid_thw,
|
||||
"output_hidden_states": True,
|
||||
"use_cache": False,
|
||||
}
|
||||
if "mm_token_type_ids" in image_inputs:
|
||||
inputs["mm_token_type_ids"] = image_inputs["mm_token_type_ids"]
|
||||
return inputs
|
||||
|
||||
@staticmethod
|
||||
def _split_text_conditioning_output(output):
|
||||
if isinstance(output, TextConditioningOutput):
|
||||
@@ -335,21 +349,11 @@ class ImageEncodingStage(PipelineStage):
|
||||
)
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
outputs = self.text_encoder(
|
||||
input_ids=image_inputs.input_ids,
|
||||
attention_mask=image_inputs.attention_mask,
|
||||
pixel_values=image_inputs.pixel_values,
|
||||
image_grid_thw=image_inputs.image_grid_thw,
|
||||
output_hidden_states=True,
|
||||
use_cache=False,
|
||||
**self._prepare_text_encoder_inputs(image_inputs)
|
||||
)
|
||||
if batch.do_classifier_free_guidance:
|
||||
neg_outputs = self.text_encoder(
|
||||
input_ids=neg_image_inputs.input_ids,
|
||||
attention_mask=neg_image_inputs.attention_mask,
|
||||
pixel_values=neg_image_inputs.pixel_values,
|
||||
image_grid_thw=neg_image_inputs.image_grid_thw,
|
||||
output_hidden_states=True,
|
||||
use_cache=False,
|
||||
**self._prepare_text_encoder_inputs(neg_image_inputs)
|
||||
)
|
||||
|
||||
prompt_embeds, prompt_embeds_mask, prompt_seq_lens = (
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from transformers import BatchFeature
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.image_encoding import (
|
||||
ImageEncodingStage,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import (
|
||||
get_global_server_args,
|
||||
set_global_server_args,
|
||||
)
|
||||
|
||||
|
||||
class _ImageProcessor:
|
||||
def __init__(self, include_mm_token_type_ids):
|
||||
self.include_mm_token_type_ids = include_mm_token_type_ids
|
||||
|
||||
def __call__(self, images, return_tensors, text=None, padding=None):
|
||||
del images, return_tensors, padding
|
||||
inputs = BatchFeature(
|
||||
data={
|
||||
"input_ids": torch.tensor([[1, 2, 3]]),
|
||||
"attention_mask": torch.ones((1, 3), dtype=torch.long),
|
||||
"pixel_values": torch.ones((1, 3, 2, 2)),
|
||||
"image_grid_thw": torch.tensor([[1, 1, 1]]),
|
||||
}
|
||||
)
|
||||
if self.include_mm_token_type_ids:
|
||||
marker = 2 if text == ["negative"] else 1
|
||||
inputs["mm_token_type_ids"] = torch.full((1, 3), marker)
|
||||
return inputs
|
||||
|
||||
|
||||
class _CapturingTextEncoder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.anchor = torch.nn.Parameter(torch.zeros(1))
|
||||
self.calls = []
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids,
|
||||
attention_mask,
|
||||
pixel_values,
|
||||
image_grid_thw,
|
||||
output_hidden_states,
|
||||
use_cache,
|
||||
mm_token_type_ids=None,
|
||||
):
|
||||
assert output_hidden_states is True
|
||||
assert use_cache is False
|
||||
self.calls.append(
|
||||
{
|
||||
"input_ids": input_ids,
|
||||
"attention_mask": attention_mask,
|
||||
"pixel_values": pixel_values,
|
||||
"image_grid_thw": image_grid_thw,
|
||||
"mm_token_type_ids": mm_token_type_ids,
|
||||
}
|
||||
)
|
||||
hidden_states = torch.zeros((*input_ids.shape, 4))
|
||||
return SimpleNamespace(hidden_states=[hidden_states])
|
||||
|
||||
|
||||
class _StrictTextEncoder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.anchor = torch.nn.Parameter(torch.zeros(1))
|
||||
self.called = False
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids,
|
||||
attention_mask,
|
||||
pixel_values,
|
||||
image_grid_thw,
|
||||
output_hidden_states,
|
||||
use_cache,
|
||||
):
|
||||
del attention_mask, pixel_values, image_grid_thw
|
||||
assert output_hidden_states is True
|
||||
assert use_cache is False
|
||||
self.called = True
|
||||
hidden_states = torch.zeros((*input_ids.shape, 4))
|
||||
return SimpleNamespace(hidden_states=[hidden_states])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server_args():
|
||||
def prepare_image_processor_kwargs(batch, neg=False):
|
||||
del batch
|
||||
text = "negative" if neg else "positive"
|
||||
return {
|
||||
"padding": True,
|
||||
"per_prompt_images": [[object()]],
|
||||
"text": [text],
|
||||
}
|
||||
|
||||
pipeline_config = SimpleNamespace(
|
||||
image_encoder_extra_args={},
|
||||
postprocess_text_funcs=(lambda outputs, _inputs: outputs.hidden_states[-1],),
|
||||
prepare_image_processor_kwargs=prepare_image_processor_kwargs,
|
||||
)
|
||||
args = SimpleNamespace(pipeline_config=pipeline_config, component_precisions={})
|
||||
previous = get_global_server_args()
|
||||
set_global_server_args(args)
|
||||
try:
|
||||
yield args
|
||||
finally:
|
||||
set_global_server_args(previous)
|
||||
|
||||
|
||||
def _make_batch(do_classifier_free_guidance):
|
||||
return SimpleNamespace(
|
||||
condition_image=object(),
|
||||
do_classifier_free_guidance=do_classifier_free_guidance,
|
||||
image_embeds=[],
|
||||
prompt_embeds=[],
|
||||
negative_prompt_embeds=[],
|
||||
prompt_embeds_mask=None,
|
||||
negative_prompt_embeds_mask=None,
|
||||
prompt_seq_lens=None,
|
||||
negative_prompt_seq_lens=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("do_classifier_free_guidance", [False, True])
|
||||
def test_forwards_mm_token_type_ids_to_image_edit_text_encoder(
|
||||
do_classifier_free_guidance,
|
||||
server_args,
|
||||
):
|
||||
text_encoder = _CapturingTextEncoder()
|
||||
stage = ImageEncodingStage(
|
||||
image_processor=_ImageProcessor(include_mm_token_type_ids=True),
|
||||
text_encoder=text_encoder,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.image_encoding.get_local_torch_device",
|
||||
return_value=torch.device("cpu"),
|
||||
):
|
||||
stage.forward(
|
||||
_make_batch(do_classifier_free_guidance),
|
||||
server_args,
|
||||
)
|
||||
|
||||
positive_ids = text_encoder.calls[0]["mm_token_type_ids"]
|
||||
assert positive_ids.dtype == torch.long
|
||||
assert positive_ids.device == torch.device("cpu")
|
||||
assert torch.equal(positive_ids, torch.ones((1, 3), dtype=torch.long))
|
||||
if do_classifier_free_guidance:
|
||||
negative_ids = text_encoder.calls[1]["mm_token_type_ids"]
|
||||
assert negative_ids.dtype == torch.long
|
||||
assert negative_ids.device == torch.device("cpu")
|
||||
assert torch.equal(
|
||||
negative_ids,
|
||||
torch.full((1, 3), 2, dtype=torch.long),
|
||||
)
|
||||
|
||||
|
||||
def test_omits_mm_token_type_ids_when_processor_does_not_return_them(server_args):
|
||||
text_encoder = _StrictTextEncoder()
|
||||
stage = ImageEncodingStage(
|
||||
image_processor=_ImageProcessor(include_mm_token_type_ids=False),
|
||||
text_encoder=text_encoder,
|
||||
)
|
||||
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.image_encoding.get_local_torch_device",
|
||||
return_value=torch.device("cpu"),
|
||||
):
|
||||
stage.forward(_make_batch(False), server_args)
|
||||
|
||||
assert text_encoder.called
|
||||
Reference in New Issue
Block a user