diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage21.py b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage21.py index a227f8a84e..bfa4b44534 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage21.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/autoencoder_kl_qwenimage21.py @@ -29,6 +29,7 @@ from sglang.multimodal_gen.runtime.models.vaes.common import ( can_install_spatial_shard_parallel_decode, should_run_spatial_shard_parallel_decode, ) +from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D, DupUp3D from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger logger = init_logger(__name__) @@ -41,88 +42,6 @@ def get_activation(name): return nn.SiLU() -class QwenImage21AvgDown3D(nn.Module): - def __init__(self, in_channels, out_channels, factor_t, factor_s=1): - super().__init__() - self.in_channels = in_channels - self.out_channels = out_channels - self.factor_t = factor_t - self.factor_s = factor_s - self.factor = self.factor_t * self.factor_s * self.factor_s - assert in_channels * self.factor % out_channels == 0 - self.group_size = in_channels * self.factor // out_channels - - def forward(self, x: torch.Tensor) -> torch.Tensor: - pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t - pad = (0, 0, 0, 0, pad_t, 0) - x = F.pad(x, pad) - B, C, T, H, W = x.shape - x = x.view( - B, - C, - T // self.factor_t, - self.factor_t, - H // self.factor_s, - self.factor_s, - W // self.factor_s, - self.factor_s, - ) - x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() - x = x.view( - B, - C * self.factor, - T // self.factor_t, - H // self.factor_s, - W // self.factor_s, - ) - x = x.view( - B, - self.out_channels, - self.group_size, - T // self.factor_t, - H // self.factor_s, - W // self.factor_s, - ) - x = x.mean(dim=2) - return x - - -class QwenImage21DupUp3D(nn.Module): - def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1): - super().__init__() - self.in_channels = in_channels - self.out_channels = out_channels - self.factor_t = factor_t - self.factor_s = factor_s - self.factor = self.factor_t * self.factor_s * self.factor_s - assert out_channels * self.factor % in_channels == 0 - self.repeats = out_channels * self.factor // in_channels - - def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor: - x = x.repeat_interleave(self.repeats, dim=1) - x = x.view( - x.size(0), - self.out_channels, - self.factor_t, - self.factor_s, - self.factor_s, - x.size(2), - x.size(3), - x.size(4), - ) - x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() - x = x.view( - x.size(0), - self.out_channels, - x.size(2) * self.factor_t, - x.size(4) * self.factor_s, - x.size(6) * self.factor_s, - ) - if first_chunk: - x = x[:, :, self.factor_t - 1 :, :, :] - return x - - class QwenImage21CausalConv3d(nn.Conv2d): def __init__( self, @@ -353,7 +272,7 @@ class QwenImage21ResidualDownBlock(nn.Module): down_flag=False, ): super().__init__() - self.avg_shortcut = QwenImage21AvgDown3D( + self.avg_shortcut = AvgDown3D( in_dim, out_dim, factor_t=2 if temperal_downsample else 1, @@ -464,7 +383,7 @@ class QwenImage21ResidualUpBlock(nn.Module): self.in_dim = in_dim self.out_dim = out_dim if up_flag: - self.avg_shortcut = QwenImage21DupUp3D( + self.avg_shortcut = DupUp3D( in_dim, out_dim, factor_t=2 if temperal_upsample else 1, factor_s=2 ) else: @@ -495,7 +414,7 @@ class QwenImage21ResidualUpBlock(nn.Module): if self.avg_shortcut is not None: shortcut = self.avg_shortcut if ( - type(shortcut) is QwenImage21DupUp3D + type(shortcut) is DupUp3D and x.is_cuda and x.dtype in (torch.float16, torch.bfloat16, torch.float32) and not torch.compiler.is_compiling() diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py index 192261b37d..9905f4aa81 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py @@ -1991,36 +1991,6 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): return DecoderOutput(sample=decoded) - def blend_v( - self, a: torch.Tensor, b: torch.Tensor, blend_extent: int - ) -> torch.Tensor: - blend_extent = min(a.shape[3], b.shape[3], blend_extent) - for y in range(blend_extent): - b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * ( - 1 - y / blend_extent - ) + b[:, :, :, y, :] * (y / blend_extent) - return b - - def blend_h( - self, a: torch.Tensor, b: torch.Tensor, blend_extent: int - ) -> torch.Tensor: - blend_extent = min(a.shape[4], b.shape[4], blend_extent) - for x in range(blend_extent): - b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * ( - 1 - x / blend_extent - ) + b[:, :, :, :, x] * (x / blend_extent) - return b - - def blend_t( - self, a: torch.Tensor, b: torch.Tensor, blend_extent: int - ) -> torch.Tensor: - blend_extent = min(a.shape[-3], b.shape[-3], blend_extent) - for x in range(blend_extent): - b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * ( - 1 - x / blend_extent - ) + b[:, :, x, :, :] * (x / blend_extent) - return b - def tiled_encode( self, x: torch.Tensor, causal: Optional[bool] = None ) -> torch.Tensor: @@ -2073,22 +2043,13 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): row.append(time) rows.append(row) - result_rows = [] - for i, row in enumerate(rows): - result_row = [] - for j, tile in enumerate(row): - # blend the above tile and the left tile - # to the current tile and add the current tile to the result row - if i > 0: - tile = self.blend_v(rows[i - 1][j], tile, blend_height) - if j > 0: - tile = self.blend_h(row[j - 1], tile, blend_width) - result_row.append( - tile[:, :, :, :tile_latent_stride_height, :tile_latent_stride_width] - ) - result_rows.append(torch.cat(result_row, dim=4)) - - enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] + enc = self._merge_spatial_tiles( + rows, + blend_height, + blend_width, + tile_latent_stride_height, + tile_latent_stride_width, + )[:, :, :, :latent_height, :latent_width] return enc def tiled_decode( @@ -2153,28 +2114,13 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): row.append(time) rows.append(row) - result_rows = [] - for i, row in enumerate(rows): - result_row = [] - for j, tile in enumerate(row): - # blend the above tile and the left tile - # to the current tile and add the current tile to the result row - if i > 0: - tile = self.blend_v(rows[i - 1][j], tile, blend_height) - if j > 0: - tile = self.blend_h(row[j - 1], tile, blend_width) - result_row.append( - tile[ - :, - :, - :, - : self.tile_sample_stride_height, - : self.tile_sample_stride_width, - ] - ) - result_rows.append(torch.cat(result_row, dim=4)) - - dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width] + dec = self._merge_spatial_tiles( + rows, + blend_height, + blend_width, + self.tile_sample_stride_height, + self.tile_sample_stride_width, + )[:, :, :, :sample_height, :sample_width] if not return_dict: return (dec,) diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/resample.py b/python/sglang/multimodal_gen/runtime/models/vaes/resample.py new file mode 100644 index 0000000000..89f05f2267 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/models/vaes/resample.py @@ -0,0 +1,88 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved. + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class AvgDown3D(nn.Module): + def __init__(self, in_channels, out_channels, factor_t, factor_s=1): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + assert in_channels * self.factor % out_channels == 0 + self.group_size = in_channels * self.factor // out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t + pad = (0, 0, 0, 0, pad_t, 0) + x = F.pad(x, pad) + B, C, T, H, W = x.shape + x = x.view( + B, + C, + T // self.factor_t, + self.factor_t, + H // self.factor_s, + self.factor_s, + W // self.factor_s, + self.factor_s, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view( + B, + C * self.factor, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.view( + B, + self.out_channels, + self.group_size, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.mean(dim=2) + return x + + +class DupUp3D(nn.Module): + def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + assert out_channels * self.factor % in_channels == 0 + self.repeats = out_channels * self.factor // in_channels + + def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor_t, + self.factor_s, + self.factor_s, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor_t, + x.size(4) * self.factor_s, + x.size(6) * self.factor_s, + ) + if first_chunk: + x = x[:, :, self.factor_t - 1 :, :, :] + return x diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py index 6a58fc8a81..94adf90130 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py @@ -56,6 +56,8 @@ from sglang.multimodal_gen.runtime.models.vaes.common import ( ParallelTiledVAE, should_run_spatial_shard_parallel_decode, ) +from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D +from sglang.multimodal_gen.runtime.models.vaes.resample import DupUp3D as DupUp3DBase from sglang.multimodal_gen.runtime.platforms import current_platform if current_platform.is_cuda() or current_platform.is_xpu(): @@ -164,103 +166,10 @@ def _run_cached_causal_conv( return out -class AvgDown3D(nn.Module): - def __init__( - self, - in_channels, - out_channels, - factor_t, - factor_s=1, - ): - super().__init__() - self.in_channels = in_channels - self.out_channels = out_channels - self.factor_t = factor_t - self.factor_s = factor_s - self.factor = self.factor_t * self.factor_s * self.factor_s - - assert in_channels * self.factor % out_channels == 0 - self.group_size = in_channels * self.factor // out_channels - +class DupUp3D(DupUp3DBase): def forward(self, x: torch.Tensor) -> torch.Tensor: - pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t - pad = (0, 0, 0, 0, pad_t, 0) - x = F.pad(x, pad) - B, C, T, H, W = x.shape - x = x.view( - B, - C, - T // self.factor_t, - self.factor_t, - H // self.factor_s, - self.factor_s, - W // self.factor_s, - self.factor_s, - ) - x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() - x = x.view( - B, - C * self.factor, - T // self.factor_t, - H // self.factor_s, - W // self.factor_s, - ) - x = x.view( - B, - self.out_channels, - self.group_size, - T // self.factor_t, - H // self.factor_s, - W // self.factor_s, - ) - x = x.mean(dim=2) - return x - - -class DupUp3D(nn.Module): - def __init__( - self, - in_channels: int, - out_channels: int, - factor_t, - factor_s=1, - ): - super().__init__() - self.in_channels = in_channels - self.out_channels = out_channels - - self.factor_t = factor_t - self.factor_s = factor_s - self.factor = self.factor_t * self.factor_s * self.factor_s - - assert out_channels * self.factor % in_channels == 0 - self.repeats = out_channels * self.factor // in_channels - - def forward(self, x: torch.Tensor) -> torch.Tensor: - x = x.repeat_interleave(self.repeats, dim=1) - x = x.view( - x.size(0), - self.out_channels, - self.factor_t, - self.factor_s, - self.factor_s, - x.size(2), - x.size(3), - x.size(4), - ) - x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() - x = x.view( - x.size(0), - self.out_channels, - x.size(2) * self.factor_t, - x.size(4) * self.factor_s, - x.size(6) * self.factor_s, - ) - _first_chunk = first_chunk.get() if first_chunk is not None else None - if _first_chunk: - x = x[:, :, self.factor_t - 1 :, :, :] - return x + return super().forward(x, first_chunk=_first_chunk) class WanCausalConv3d(nn.Conv3d): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py index a5d59c2038..498f291665 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/denoising_av.py @@ -1,5 +1,4 @@ import torch -from diffusers.utils.torch_utils import randn_tensor from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import is_ltx23_native_variant from sglang.multimodal_gen.runtime.distributed import ( @@ -168,30 +167,6 @@ class LTX2RefinementStage(LTX2AVDenoisingStage): return StageParallelismType.MAIN_RANK_ONLY return StageParallelismType.REPLICATED - @staticmethod - def _randn_like_with_batch_generators( - reference_tensor: torch.Tensor, batch: Req - ) -> torch.Tensor: - generator = getattr(batch, "generator", None) - if isinstance(generator, list): - bsz = int(reference_tensor.shape[0]) - valid_generators = [g for g in generator if isinstance(g, torch.Generator)] - if len(valid_generators) == 1: - generator = valid_generators[0] - elif len(valid_generators) >= bsz: - generator = valid_generators[:bsz] - else: - generator = None - elif not isinstance(generator, torch.Generator): - generator = None - - return randn_tensor( - reference_tensor.shape, - generator=generator, - device=reference_tensor.device, - dtype=reference_tensor.dtype, - ) - @staticmethod def _reset_stage2_generators(batch: Req) -> None: generator = getattr(batch, "generator", None) diff --git a/python/sglang/multimodal_gen/test/unit/test_vae_resample.py b/python/sglang/multimodal_gen/test/unit/test_vae_resample.py new file mode 100644 index 0000000000..b79ca4771a --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_vae_resample.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 + +import pytest +import torch +import torch.nn.functional as F +from einops import rearrange + +from sglang.multimodal_gen.runtime.models.vaes import wanvae +from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D, DupUp3D + + +@pytest.mark.parametrize("device", ["cpu", "cuda"]) +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("factor_t,factor_s", [(1, 1), (1, 2), (2, 1), (2, 2)]) +def test_resample_layout_and_causal_crop(device, dtype, factor_t, factor_s): + if device == "cuda" and not torch.cuda.is_available(): + pytest.skip("CUDA required") + out_channels = 2 * factor_t * factor_s**2 + down = AvgDown3D(4, out_channels, factor_t, factor_s) + up = DupUp3D(out_channels, 4, factor_t, factor_s) + causal_up = wanvae.DupUp3D(out_channels, 4, factor_t, factor_s) + for frames in (1, 3, 4): + x = torch.arange(2 * 4 * frames * 4 * 6, device=device).reshape( + 2, 4, frames, 4, 6 + ) + x = (x % 31).to(dtype) + for layout in (torch.contiguous_format, torch.channels_last_3d): + x = x.contiguous(memory_format=layout) + padded = F.pad(x, (0, 0, 0, 0, (-frames) % factor_t, 0)) + packed = rearrange( + padded, + "b c (t ft) (h fh) (w fw) -> b (c ft fh fw) t h w", + ft=factor_t, + fh=factor_s, + fw=factor_s, + ) + expected = packed.reshape(2, out_channels, 2, *packed.shape[2:]).mean(2) + reduced = down(x) + torch.testing.assert_close(reduced, expected, rtol=0, atol=0) + expanded = rearrange( + reduced.repeat_interleave(2, dim=1), + "b (c ft fh fw) t h w -> b c (t ft) (h fh) (w fw)", + ft=factor_t, + fh=factor_s, + fw=factor_s, + ) + for first in (False, True, False): + expected = expanded[:, :, factor_t - 1 :] if first else expanded + token = wanvae.first_chunk.set(first) + try: + torch.testing.assert_close( + causal_up(reduced), expected, rtol=0, atol=0 + ) + torch.testing.assert_close( + up(reduced, first), expected, rtol=0, atol=0 + ) + torch.testing.assert_close(up(reduced), expanded, rtol=0, atol=0) + finally: + wanvae.first_chunk.reset(token) + assert not down.state_dict() and not up.state_dict() and not causal_up.state_dict()