[diffusion] optimization: avoid full-video clone in VAE post-processing for MiniMax-H3 (#40095)
Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
@@ -207,6 +207,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
"pixel_norm_type": self.pixel_norm_type,
|
||||
"transform": self.transform,
|
||||
"transform_rev": self.transform_rev,
|
||||
"transform_rev_inplace": self.transform_rev_inplace,
|
||||
"use_3d_conv": self.use_3d_conv,
|
||||
}
|
||||
if hasattr(self, "processor"):
|
||||
@@ -1231,6 +1232,9 @@ class AutoencoderKLLegacy(AutoencoderKL):
|
||||
|
||||
self.transform = get_normalize_transform(pixel_norm_type)
|
||||
self.transform_rev = get_denormalize_transform(pixel_norm_type)
|
||||
self.transform_rev_inplace = get_denormalize_transform(
|
||||
pixel_norm_type, inplace=True
|
||||
)
|
||||
|
||||
self.use_3d_conv = use_3d_conv
|
||||
self.causal_encoder = causal_encoder
|
||||
|
||||
@@ -43,11 +43,13 @@ def get_normalize_transform(
|
||||
return Normalize(mean, std, inplace=inplace)
|
||||
|
||||
|
||||
def get_denormalize_transform(norm_type: str = "imagenet") -> Normalize:
|
||||
def get_denormalize_transform(
|
||||
norm_type: str = "imagenet", *, inplace: bool = False
|
||||
) -> Normalize:
|
||||
mean, std = get_norm_constants(norm_type)
|
||||
inv_mean = tuple(-m / s for m, s in zip(mean, std))
|
||||
inv_std = tuple(1.0 / s for s in std)
|
||||
return Normalize(inv_mean, inv_std)
|
||||
return Normalize(inv_mean, inv_std, inplace=inplace)
|
||||
|
||||
|
||||
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
|
||||
@@ -69,6 +71,7 @@ class VAEProcessor:
|
||||
pixel_norm_type="imagenet",
|
||||
transform=None,
|
||||
transform_rev=None,
|
||||
transform_rev_inplace=None,
|
||||
use_3d_conv=False,
|
||||
):
|
||||
self.vae_ratio = vae_ratio
|
||||
@@ -87,6 +90,11 @@ class VAEProcessor:
|
||||
else None
|
||||
)
|
||||
self.transform_rev = transform_rev or get_denormalize_transform(pixel_norm_type)
|
||||
self.transform_rev_inplace = transform_rev_inplace
|
||||
if transform_rev is None and transform_rev_inplace is None:
|
||||
self.transform_rev_inplace = get_denormalize_transform(
|
||||
pixel_norm_type, inplace=True
|
||||
)
|
||||
self.use_3d_conv = use_3d_conv
|
||||
|
||||
def _ensure_list(self, data):
|
||||
@@ -253,13 +261,23 @@ class VAEProcessor:
|
||||
|
||||
return tensor.contiguous()
|
||||
|
||||
def revert_tensor(self, tensor):
|
||||
def revert_tensor(self, tensor, *, runtime_owned=False):
|
||||
"""Undo pixel normalization, reusing storage only when ownership is explicit.
|
||||
|
||||
Custom reverse transforms remain out-of-place unless their caller also
|
||||
provides an in-place counterpart.
|
||||
"""
|
||||
B, T = None, None
|
||||
if self.use_3d_conv:
|
||||
tensor = tensor.unsqueeze(2) if tensor.ndim == 4 else tensor
|
||||
B, _, T, _, _ = tensor.shape
|
||||
tensor = rearrange(tensor, "b c t h w -> (b t) c h w")
|
||||
tensor_rev = self.transform_rev(tensor).clamp_(0, 1)
|
||||
transform_rev = (
|
||||
self.transform_rev_inplace
|
||||
if runtime_owned and self.transform_rev_inplace is not None
|
||||
else self.transform_rev
|
||||
)
|
||||
tensor_rev = transform_rev(tensor).clamp_(0, 1)
|
||||
if B is not None:
|
||||
tensor_rev = rearrange(tensor_rev, "(b t) c h w -> b c t h w", b=B, t=T)
|
||||
return tensor_rev.contiguous()
|
||||
|
||||
+1
-1
@@ -403,7 +403,7 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
visual_frames = video_decode(visual_decode_latent)
|
||||
visual_frames = selected_video_vae.processor.revert_tensor(
|
||||
visual_frames
|
||||
visual_frames, runtime_owned=True
|
||||
)
|
||||
visual_frames = _required_tensor(
|
||||
visual_frames,
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Unit tests for MiniMax-H3 visual VAE tensor postprocessing."""
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.processor import (
|
||||
VAEProcessor,
|
||||
get_denormalize_transform,
|
||||
)
|
||||
|
||||
|
||||
def _processor(
|
||||
*, custom_reverse: bool = False, custom_reverse_inplace: bool = False
|
||||
) -> VAEProcessor:
|
||||
kwargs = {}
|
||||
if custom_reverse:
|
||||
kwargs["transform_rev"] = get_denormalize_transform("simple")
|
||||
if custom_reverse_inplace:
|
||||
kwargs["transform_rev_inplace"] = get_denormalize_transform(
|
||||
"simple", inplace=True
|
||||
)
|
||||
return VAEProcessor(
|
||||
vae_ratio=16,
|
||||
vae_ratio_t=4,
|
||||
clip_length=17,
|
||||
frame_overlap=0,
|
||||
token_overlap=3,
|
||||
tokens_chunk_size=5,
|
||||
isolated_last_frame=False,
|
||||
latent_patch_size=1,
|
||||
crop_mode="top_left",
|
||||
pixel_norm_type="simple",
|
||||
use_3d_conv=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def test_revert_tensor_preserves_input_by_default():
|
||||
processor = _processor()
|
||||
tensor = torch.rand(1, 3, 5, 8, 8)
|
||||
original = tensor.clone()
|
||||
|
||||
output = processor.revert_tensor(tensor)
|
||||
|
||||
torch.testing.assert_close(tensor, original, rtol=0, atol=0)
|
||||
assert output.untyped_storage().data_ptr() != tensor.untyped_storage().data_ptr()
|
||||
|
||||
|
||||
def test_runtime_owned_revert_matches_out_of_place_and_reuses_storage():
|
||||
processor = _processor()
|
||||
tensor = torch.rand(1, 3, 5, 8, 8)
|
||||
expected = processor.revert_tensor(tensor)
|
||||
runtime_owned_tensor = tensor.clone()
|
||||
input_storage = runtime_owned_tensor.untyped_storage().data_ptr()
|
||||
|
||||
output = processor.revert_tensor(runtime_owned_tensor, runtime_owned=True)
|
||||
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
assert output.untyped_storage().data_ptr() == input_storage
|
||||
|
||||
|
||||
def test_runtime_owned_revert_uses_explicit_inplace_transform():
|
||||
processor = _processor(custom_reverse=True, custom_reverse_inplace=True)
|
||||
tensor = torch.rand(1, 3, 5, 8, 8)
|
||||
expected = processor.revert_tensor(tensor)
|
||||
runtime_owned_tensor = tensor.clone()
|
||||
input_storage = runtime_owned_tensor.untyped_storage().data_ptr()
|
||||
|
||||
output = processor.revert_tensor(runtime_owned_tensor, runtime_owned=True)
|
||||
|
||||
torch.testing.assert_close(output, expected, rtol=0, atol=0)
|
||||
assert output.untyped_storage().data_ptr() == input_storage
|
||||
|
||||
|
||||
def test_runtime_owned_revert_falls_back_for_custom_transform():
|
||||
processor = _processor(custom_reverse=True)
|
||||
tensor = torch.rand(1, 3, 5, 8, 8)
|
||||
original = tensor.clone()
|
||||
|
||||
output = processor.revert_tensor(tensor, runtime_owned=True)
|
||||
|
||||
torch.testing.assert_close(tensor, original, rtol=0, atol=0)
|
||||
assert output.untyped_storage().data_ptr() != tensor.untyped_storage().data_ptr()
|
||||
Reference in New Issue
Block a user