diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py index a95227c853..ce2f89154c 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py @@ -207,6 +207,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): "pixel_norm_type": self.pixel_norm_type, "transform": self.transform, "transform_rev": self.transform_rev, + "transform_rev_inplace": self.transform_rev_inplace, "use_3d_conv": self.use_3d_conv, } if hasattr(self, "processor"): @@ -1231,6 +1232,9 @@ class AutoencoderKLLegacy(AutoencoderKL): self.transform = get_normalize_transform(pixel_norm_type) self.transform_rev = get_denormalize_transform(pixel_norm_type) + self.transform_rev_inplace = get_denormalize_transform( + pixel_norm_type, inplace=True + ) self.use_3d_conv = use_3d_conv self.causal_encoder = causal_encoder diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/processor.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/processor.py index 9de0a673a5..5feb22ca56 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/processor.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/processor.py @@ -43,11 +43,13 @@ def get_normalize_transform( return Normalize(mean, std, inplace=inplace) -def get_denormalize_transform(norm_type: str = "imagenet") -> Normalize: +def get_denormalize_transform( + norm_type: str = "imagenet", *, inplace: bool = False +) -> Normalize: mean, std = get_norm_constants(norm_type) inv_mean = tuple(-m / s for m, s in zip(mean, std)) inv_std = tuple(1.0 / s for s in std) - return Normalize(inv_mean, inv_std) + return Normalize(inv_mean, inv_std, inplace=inplace) logger = logging.get_logger(__name__) # pylint: disable=invalid-name @@ -69,6 +71,7 @@ class VAEProcessor: pixel_norm_type="imagenet", transform=None, transform_rev=None, + transform_rev_inplace=None, use_3d_conv=False, ): self.vae_ratio = vae_ratio @@ -87,6 +90,11 @@ class VAEProcessor: else None ) self.transform_rev = transform_rev or get_denormalize_transform(pixel_norm_type) + self.transform_rev_inplace = transform_rev_inplace + if transform_rev is None and transform_rev_inplace is None: + self.transform_rev_inplace = get_denormalize_transform( + pixel_norm_type, inplace=True + ) self.use_3d_conv = use_3d_conv def _ensure_list(self, data): @@ -253,13 +261,23 @@ class VAEProcessor: return tensor.contiguous() - def revert_tensor(self, tensor): + def revert_tensor(self, tensor, *, runtime_owned=False): + """Undo pixel normalization, reusing storage only when ownership is explicit. + + Custom reverse transforms remain out-of-place unless their caller also + provides an in-place counterpart. + """ B, T = None, None if self.use_3d_conv: tensor = tensor.unsqueeze(2) if tensor.ndim == 4 else tensor B, _, T, _, _ = tensor.shape tensor = rearrange(tensor, "b c t h w -> (b t) c h w") - tensor_rev = self.transform_rev(tensor).clamp_(0, 1) + transform_rev = ( + self.transform_rev_inplace + if runtime_owned and self.transform_rev_inplace is not None + else self.transform_rev + ) + tensor_rev = transform_rev(tensor).clamp_(0, 1) if B is not None: tensor_rev = rearrange(tensor_rev, "(b t) c h w -> b c t h w", b=B, t=T) return tensor_rev.contiguous() diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py index 6c20d41c66..e35af98f38 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py @@ -403,7 +403,7 @@ class MiniMaxH3DecodingStage(DecodingStage): with set_forward_context(current_timestep=0, attn_metadata=None): visual_frames = video_decode(visual_decode_latent) visual_frames = selected_video_vae.processor.revert_tensor( - visual_frames + visual_frames, runtime_owned=True ) visual_frames = _required_tensor( visual_frames, diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_processor.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_processor.py new file mode 100644 index 0000000000..c88fc6f9da --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_vae_processor.py @@ -0,0 +1,83 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Unit tests for MiniMax-H3 visual VAE tensor postprocessing.""" + +import torch + +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.processor import ( + VAEProcessor, + get_denormalize_transform, +) + + +def _processor( + *, custom_reverse: bool = False, custom_reverse_inplace: bool = False +) -> VAEProcessor: + kwargs = {} + if custom_reverse: + kwargs["transform_rev"] = get_denormalize_transform("simple") + if custom_reverse_inplace: + kwargs["transform_rev_inplace"] = get_denormalize_transform( + "simple", inplace=True + ) + return VAEProcessor( + vae_ratio=16, + vae_ratio_t=4, + clip_length=17, + frame_overlap=0, + token_overlap=3, + tokens_chunk_size=5, + isolated_last_frame=False, + latent_patch_size=1, + crop_mode="top_left", + pixel_norm_type="simple", + use_3d_conv=True, + **kwargs, + ) + + +def test_revert_tensor_preserves_input_by_default(): + processor = _processor() + tensor = torch.rand(1, 3, 5, 8, 8) + original = tensor.clone() + + output = processor.revert_tensor(tensor) + + torch.testing.assert_close(tensor, original, rtol=0, atol=0) + assert output.untyped_storage().data_ptr() != tensor.untyped_storage().data_ptr() + + +def test_runtime_owned_revert_matches_out_of_place_and_reuses_storage(): + processor = _processor() + tensor = torch.rand(1, 3, 5, 8, 8) + expected = processor.revert_tensor(tensor) + runtime_owned_tensor = tensor.clone() + input_storage = runtime_owned_tensor.untyped_storage().data_ptr() + + output = processor.revert_tensor(runtime_owned_tensor, runtime_owned=True) + + torch.testing.assert_close(output, expected, rtol=0, atol=0) + assert output.untyped_storage().data_ptr() == input_storage + + +def test_runtime_owned_revert_uses_explicit_inplace_transform(): + processor = _processor(custom_reverse=True, custom_reverse_inplace=True) + tensor = torch.rand(1, 3, 5, 8, 8) + expected = processor.revert_tensor(tensor) + runtime_owned_tensor = tensor.clone() + input_storage = runtime_owned_tensor.untyped_storage().data_ptr() + + output = processor.revert_tensor(runtime_owned_tensor, runtime_owned=True) + + torch.testing.assert_close(output, expected, rtol=0, atol=0) + assert output.untyped_storage().data_ptr() == input_storage + + +def test_runtime_owned_revert_falls_back_for_custom_transform(): + processor = _processor(custom_reverse=True) + tensor = torch.rand(1, 3, 5, 8, 8) + original = tensor.clone() + + output = processor.revert_tensor(tensor, runtime_owned=True) + + torch.testing.assert_close(tensor, original, rtol=0, atol=0) + assert output.untyped_storage().data_ptr() != tensor.untyped_storage().data_ptr()