[diffusion] optimization: avoid full-video clone in VAE post-processing for MiniMax-H3 (#40095)

Co-authored-by: Mick <mickjagger19@icloud.com>
This commit is contained in:
aiyueqi
2026-10-05 19:56:11 +08:00
committed by GitHub
co-authored by Mick
parent d349e672bc
commit 8663e3b670
4 changed files with 110 additions and 5 deletions
@@ -207,6 +207,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
"pixel_norm_type": self.pixel_norm_type,
"transform": self.transform,
"transform_rev": self.transform_rev,
"transform_rev_inplace": self.transform_rev_inplace,
"use_3d_conv": self.use_3d_conv,
}
if hasattr(self, "processor"):
@@ -1231,6 +1232,9 @@ class AutoencoderKLLegacy(AutoencoderKL):
self.transform = get_normalize_transform(pixel_norm_type)
self.transform_rev = get_denormalize_transform(pixel_norm_type)
self.transform_rev_inplace = get_denormalize_transform(
pixel_norm_type, inplace=True
)
self.use_3d_conv = use_3d_conv
self.causal_encoder = causal_encoder
@@ -43,11 +43,13 @@ def get_normalize_transform(
return Normalize(mean, std, inplace=inplace)
def get_denormalize_transform(norm_type: str = "imagenet") -> Normalize:
def get_denormalize_transform(
norm_type: str = "imagenet", *, inplace: bool = False
) -> Normalize:
mean, std = get_norm_constants(norm_type)
inv_mean = tuple(-m / s for m, s in zip(mean, std))
inv_std = tuple(1.0 / s for s in std)
return Normalize(inv_mean, inv_std)
return Normalize(inv_mean, inv_std, inplace=inplace)
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
@@ -69,6 +71,7 @@ class VAEProcessor:
pixel_norm_type="imagenet",
transform=None,
transform_rev=None,
transform_rev_inplace=None,
use_3d_conv=False,
):
self.vae_ratio = vae_ratio
@@ -87,6 +90,11 @@ class VAEProcessor:
else None
)
self.transform_rev = transform_rev or get_denormalize_transform(pixel_norm_type)
self.transform_rev_inplace = transform_rev_inplace
if transform_rev is None and transform_rev_inplace is None:
self.transform_rev_inplace = get_denormalize_transform(
pixel_norm_type, inplace=True
)
self.use_3d_conv = use_3d_conv
def _ensure_list(self, data):
@@ -253,13 +261,23 @@ class VAEProcessor:
return tensor.contiguous()
def revert_tensor(self, tensor):
def revert_tensor(self, tensor, *, runtime_owned=False):
"""Undo pixel normalization, reusing storage only when ownership is explicit.
Custom reverse transforms remain out-of-place unless their caller also
provides an in-place counterpart.
"""
B, T = None, None
if self.use_3d_conv:
tensor = tensor.unsqueeze(2) if tensor.ndim == 4 else tensor
B, _, T, _, _ = tensor.shape
tensor = rearrange(tensor, "b c t h w -> (b t) c h w")
tensor_rev = self.transform_rev(tensor).clamp_(0, 1)
transform_rev = (
self.transform_rev_inplace
if runtime_owned and self.transform_rev_inplace is not None
else self.transform_rev
)
tensor_rev = transform_rev(tensor).clamp_(0, 1)
if B is not None:
tensor_rev = rearrange(tensor_rev, "(b t) c h w -> b c t h w", b=B, t=T)
return tensor_rev.contiguous()
@@ -403,7 +403,7 @@ class MiniMaxH3DecodingStage(DecodingStage):
with set_forward_context(current_timestep=0, attn_metadata=None):
visual_frames = video_decode(visual_decode_latent)
visual_frames = selected_video_vae.processor.revert_tensor(
visual_frames
visual_frames, runtime_owned=True
)
visual_frames = _required_tensor(
visual_frames,
@@ -0,0 +1,83 @@
# SPDX-License-Identifier: Apache-2.0
"""Unit tests for MiniMax-H3 visual VAE tensor postprocessing."""
import torch
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae.processor import (
VAEProcessor,
get_denormalize_transform,
)
def _processor(
*, custom_reverse: bool = False, custom_reverse_inplace: bool = False
) -> VAEProcessor:
kwargs = {}
if custom_reverse:
kwargs["transform_rev"] = get_denormalize_transform("simple")
if custom_reverse_inplace:
kwargs["transform_rev_inplace"] = get_denormalize_transform(
"simple", inplace=True
)
return VAEProcessor(
vae_ratio=16,
vae_ratio_t=4,
clip_length=17,
frame_overlap=0,
token_overlap=3,
tokens_chunk_size=5,
isolated_last_frame=False,
latent_patch_size=1,
crop_mode="top_left",
pixel_norm_type="simple",
use_3d_conv=True,
**kwargs,
)
def test_revert_tensor_preserves_input_by_default():
processor = _processor()
tensor = torch.rand(1, 3, 5, 8, 8)
original = tensor.clone()
output = processor.revert_tensor(tensor)
torch.testing.assert_close(tensor, original, rtol=0, atol=0)
assert output.untyped_storage().data_ptr() != tensor.untyped_storage().data_ptr()
def test_runtime_owned_revert_matches_out_of_place_and_reuses_storage():
processor = _processor()
tensor = torch.rand(1, 3, 5, 8, 8)
expected = processor.revert_tensor(tensor)
runtime_owned_tensor = tensor.clone()
input_storage = runtime_owned_tensor.untyped_storage().data_ptr()
output = processor.revert_tensor(runtime_owned_tensor, runtime_owned=True)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
assert output.untyped_storage().data_ptr() == input_storage
def test_runtime_owned_revert_uses_explicit_inplace_transform():
processor = _processor(custom_reverse=True, custom_reverse_inplace=True)
tensor = torch.rand(1, 3, 5, 8, 8)
expected = processor.revert_tensor(tensor)
runtime_owned_tensor = tensor.clone()
input_storage = runtime_owned_tensor.untyped_storage().data_ptr()
output = processor.revert_tensor(runtime_owned_tensor, runtime_owned=True)
torch.testing.assert_close(output, expected, rtol=0, atol=0)
assert output.untyped_storage().data_ptr() == input_storage
def test_runtime_owned_revert_falls_back_for_custom_transform():
processor = _processor(custom_reverse=True)
tensor = torch.rand(1, 3, 5, 8, 8)
original = tensor.clone()
output = processor.revert_tensor(tensor, runtime_owned=True)
torch.testing.assert_close(tensor, original, rtol=0, atol=0)
assert output.untyped_storage().data_ptr() != tensor.untyped_storage().data_ptr()