[diffusion] refactor: deduplicate VAE resampling and LTX tiling (#42866)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
Mick
2026-10-08 09:22:11 +08:00
committed by GitHub
co-authored by Mick Qian
parent 0c00bffc6f
commit 287b8b793f
6 changed files with 170 additions and 273 deletions
@@ -29,6 +29,7 @@ from sglang.multimodal_gen.runtime.models.vaes.common import (
can_install_spatial_shard_parallel_decode,
should_run_spatial_shard_parallel_decode,
)
from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D, DupUp3D
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
logger = init_logger(__name__)
@@ -41,88 +42,6 @@ def get_activation(name):
return nn.SiLU()
class QwenImage21AvgDown3D(nn.Module):
def __init__(self, in_channels, out_channels, factor_t, factor_s=1):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.factor_t = factor_t
self.factor_s = factor_s
self.factor = self.factor_t * self.factor_s * self.factor_s
assert in_channels * self.factor % out_channels == 0
self.group_size = in_channels * self.factor // out_channels
def forward(self, x: torch.Tensor) -> torch.Tensor:
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
pad = (0, 0, 0, 0, pad_t, 0)
x = F.pad(x, pad)
B, C, T, H, W = x.shape
x = x.view(
B,
C,
T // self.factor_t,
self.factor_t,
H // self.factor_s,
self.factor_s,
W // self.factor_s,
self.factor_s,
)
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
x = x.view(
B,
C * self.factor,
T // self.factor_t,
H // self.factor_s,
W // self.factor_s,
)
x = x.view(
B,
self.out_channels,
self.group_size,
T // self.factor_t,
H // self.factor_s,
W // self.factor_s,
)
x = x.mean(dim=2)
return x
class QwenImage21DupUp3D(nn.Module):
def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.factor_t = factor_t
self.factor_s = factor_s
self.factor = self.factor_t * self.factor_s * self.factor_s
assert out_channels * self.factor % in_channels == 0
self.repeats = out_channels * self.factor // in_channels
def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor:
x = x.repeat_interleave(self.repeats, dim=1)
x = x.view(
x.size(0),
self.out_channels,
self.factor_t,
self.factor_s,
self.factor_s,
x.size(2),
x.size(3),
x.size(4),
)
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
x = x.view(
x.size(0),
self.out_channels,
x.size(2) * self.factor_t,
x.size(4) * self.factor_s,
x.size(6) * self.factor_s,
)
if first_chunk:
x = x[:, :, self.factor_t - 1 :, :, :]
return x
class QwenImage21CausalConv3d(nn.Conv2d):
def __init__(
self,
@@ -353,7 +272,7 @@ class QwenImage21ResidualDownBlock(nn.Module):
down_flag=False,
):
super().__init__()
self.avg_shortcut = QwenImage21AvgDown3D(
self.avg_shortcut = AvgDown3D(
in_dim,
out_dim,
factor_t=2 if temperal_downsample else 1,
@@ -464,7 +383,7 @@ class QwenImage21ResidualUpBlock(nn.Module):
self.in_dim = in_dim
self.out_dim = out_dim
if up_flag:
self.avg_shortcut = QwenImage21DupUp3D(
self.avg_shortcut = DupUp3D(
in_dim, out_dim, factor_t=2 if temperal_upsample else 1, factor_s=2
)
else:
@@ -495,7 +414,7 @@ class QwenImage21ResidualUpBlock(nn.Module):
if self.avg_shortcut is not None:
shortcut = self.avg_shortcut
if (
type(shortcut) is QwenImage21DupUp3D
type(shortcut) is DupUp3D
and x.is_cuda
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
and not torch.compiler.is_compiling()
@@ -1991,36 +1991,6 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
return DecoderOutput(sample=decoded)
def blend_v(
self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
) -> torch.Tensor:
blend_extent = min(a.shape[3], b.shape[3], blend_extent)
for y in range(blend_extent):
b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (
1 - y / blend_extent
) + b[:, :, :, y, :] * (y / blend_extent)
return b
def blend_h(
self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
) -> torch.Tensor:
blend_extent = min(a.shape[4], b.shape[4], blend_extent)
for x in range(blend_extent):
b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (
1 - x / blend_extent
) + b[:, :, :, :, x] * (x / blend_extent)
return b
def blend_t(
self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
) -> torch.Tensor:
blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)
for x in range(blend_extent):
b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (
1 - x / blend_extent
) + b[:, :, x, :, :] * (x / blend_extent)
return b
def tiled_encode(
self, x: torch.Tensor, causal: Optional[bool] = None
) -> torch.Tensor:
@@ -2073,22 +2043,13 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
row.append(time)
rows.append(row)
result_rows = []
for i, row in enumerate(rows):
result_row = []
for j, tile in enumerate(row):
# blend the above tile and the left tile
# to the current tile and add the current tile to the result row
if i > 0:
tile = self.blend_v(rows[i - 1][j], tile, blend_height)
if j > 0:
tile = self.blend_h(row[j - 1], tile, blend_width)
result_row.append(
tile[:, :, :, :tile_latent_stride_height, :tile_latent_stride_width]
)
result_rows.append(torch.cat(result_row, dim=4))
enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width]
enc = self._merge_spatial_tiles(
rows,
blend_height,
blend_width,
tile_latent_stride_height,
tile_latent_stride_width,
)[:, :, :, :latent_height, :latent_width]
return enc
def tiled_decode(
@@ -2153,28 +2114,13 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
row.append(time)
rows.append(row)
result_rows = []
for i, row in enumerate(rows):
result_row = []
for j, tile in enumerate(row):
# blend the above tile and the left tile
# to the current tile and add the current tile to the result row
if i > 0:
tile = self.blend_v(rows[i - 1][j], tile, blend_height)
if j > 0:
tile = self.blend_h(row[j - 1], tile, blend_width)
result_row.append(
tile[
:,
:,
:,
: self.tile_sample_stride_height,
: self.tile_sample_stride_width,
]
)
result_rows.append(torch.cat(result_row, dim=4))
dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width]
dec = self._merge_spatial_tiles(
rows,
blend_height,
blend_width,
self.tile_sample_stride_height,
self.tile_sample_stride_width,
)[:, :, :, :sample_height, :sample_width]
if not return_dict:
return (dec,)
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
import torch
import torch.nn as nn
import torch.nn.functional as F
class AvgDown3D(nn.Module):
def __init__(self, in_channels, out_channels, factor_t, factor_s=1):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.factor_t = factor_t
self.factor_s = factor_s
self.factor = self.factor_t * self.factor_s * self.factor_s
assert in_channels * self.factor % out_channels == 0
self.group_size = in_channels * self.factor // out_channels
def forward(self, x: torch.Tensor) -> torch.Tensor:
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
pad = (0, 0, 0, 0, pad_t, 0)
x = F.pad(x, pad)
B, C, T, H, W = x.shape
x = x.view(
B,
C,
T // self.factor_t,
self.factor_t,
H // self.factor_s,
self.factor_s,
W // self.factor_s,
self.factor_s,
)
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
x = x.view(
B,
C * self.factor,
T // self.factor_t,
H // self.factor_s,
W // self.factor_s,
)
x = x.view(
B,
self.out_channels,
self.group_size,
T // self.factor_t,
H // self.factor_s,
W // self.factor_s,
)
x = x.mean(dim=2)
return x
class DupUp3D(nn.Module):
def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.factor_t = factor_t
self.factor_s = factor_s
self.factor = self.factor_t * self.factor_s * self.factor_s
assert out_channels * self.factor % in_channels == 0
self.repeats = out_channels * self.factor // in_channels
def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor:
x = x.repeat_interleave(self.repeats, dim=1)
x = x.view(
x.size(0),
self.out_channels,
self.factor_t,
self.factor_s,
self.factor_s,
x.size(2),
x.size(3),
x.size(4),
)
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
x = x.view(
x.size(0),
self.out_channels,
x.size(2) * self.factor_t,
x.size(4) * self.factor_s,
x.size(6) * self.factor_s,
)
if first_chunk:
x = x[:, :, self.factor_t - 1 :, :, :]
return x
@@ -56,6 +56,8 @@ from sglang.multimodal_gen.runtime.models.vaes.common import (
ParallelTiledVAE,
should_run_spatial_shard_parallel_decode,
)
from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D
from sglang.multimodal_gen.runtime.models.vaes.resample import DupUp3D as DupUp3DBase
from sglang.multimodal_gen.runtime.platforms import current_platform
if current_platform.is_cuda() or current_platform.is_xpu():
@@ -164,103 +166,10 @@ def _run_cached_causal_conv(
return out
class AvgDown3D(nn.Module):
def __init__(
self,
in_channels,
out_channels,
factor_t,
factor_s=1,
):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.factor_t = factor_t
self.factor_s = factor_s
self.factor = self.factor_t * self.factor_s * self.factor_s
assert in_channels * self.factor % out_channels == 0
self.group_size = in_channels * self.factor // out_channels
class DupUp3D(DupUp3DBase):
def forward(self, x: torch.Tensor) -> torch.Tensor:
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
pad = (0, 0, 0, 0, pad_t, 0)
x = F.pad(x, pad)
B, C, T, H, W = x.shape
x = x.view(
B,
C,
T // self.factor_t,
self.factor_t,
H // self.factor_s,
self.factor_s,
W // self.factor_s,
self.factor_s,
)
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
x = x.view(
B,
C * self.factor,
T // self.factor_t,
H // self.factor_s,
W // self.factor_s,
)
x = x.view(
B,
self.out_channels,
self.group_size,
T // self.factor_t,
H // self.factor_s,
W // self.factor_s,
)
x = x.mean(dim=2)
return x
class DupUp3D(nn.Module):
def __init__(
self,
in_channels: int,
out_channels: int,
factor_t,
factor_s=1,
):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.factor_t = factor_t
self.factor_s = factor_s
self.factor = self.factor_t * self.factor_s * self.factor_s
assert out_channels * self.factor % in_channels == 0
self.repeats = out_channels * self.factor // in_channels
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = x.repeat_interleave(self.repeats, dim=1)
x = x.view(
x.size(0),
self.out_channels,
self.factor_t,
self.factor_s,
self.factor_s,
x.size(2),
x.size(3),
x.size(4),
)
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
x = x.view(
x.size(0),
self.out_channels,
x.size(2) * self.factor_t,
x.size(4) * self.factor_s,
x.size(6) * self.factor_s,
)
_first_chunk = first_chunk.get() if first_chunk is not None else None
if _first_chunk:
x = x[:, :, self.factor_t - 1 :, :, :]
return x
return super().forward(x, first_chunk=_first_chunk)
class WanCausalConv3d(nn.Conv3d):
@@ -1,5 +1,4 @@
import torch
from diffusers.utils.torch_utils import randn_tensor
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import is_ltx23_native_variant
from sglang.multimodal_gen.runtime.distributed import (
@@ -168,30 +167,6 @@ class LTX2RefinementStage(LTX2AVDenoisingStage):
return StageParallelismType.MAIN_RANK_ONLY
return StageParallelismType.REPLICATED
@staticmethod
def _randn_like_with_batch_generators(
reference_tensor: torch.Tensor, batch: Req
) -> torch.Tensor:
generator = getattr(batch, "generator", None)
if isinstance(generator, list):
bsz = int(reference_tensor.shape[0])
valid_generators = [g for g in generator if isinstance(g, torch.Generator)]
if len(valid_generators) == 1:
generator = valid_generators[0]
elif len(valid_generators) >= bsz:
generator = valid_generators[:bsz]
else:
generator = None
elif not isinstance(generator, torch.Generator):
generator = None
return randn_tensor(
reference_tensor.shape,
generator=generator,
device=reference_tensor.device,
dtype=reference_tensor.dtype,
)
@staticmethod
def _reset_stage2_generators(batch: Req) -> None:
generator = getattr(batch, "generator", None)
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
import pytest
import torch
import torch.nn.functional as F
from einops import rearrange
from sglang.multimodal_gen.runtime.models.vaes import wanvae
from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D, DupUp3D
@pytest.mark.parametrize("device", ["cpu", "cuda"])
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
@pytest.mark.parametrize("factor_t,factor_s", [(1, 1), (1, 2), (2, 1), (2, 2)])
def test_resample_layout_and_causal_crop(device, dtype, factor_t, factor_s):
if device == "cuda" and not torch.cuda.is_available():
pytest.skip("CUDA required")
out_channels = 2 * factor_t * factor_s**2
down = AvgDown3D(4, out_channels, factor_t, factor_s)
up = DupUp3D(out_channels, 4, factor_t, factor_s)
causal_up = wanvae.DupUp3D(out_channels, 4, factor_t, factor_s)
for frames in (1, 3, 4):
x = torch.arange(2 * 4 * frames * 4 * 6, device=device).reshape(
2, 4, frames, 4, 6
)
x = (x % 31).to(dtype)
for layout in (torch.contiguous_format, torch.channels_last_3d):
x = x.contiguous(memory_format=layout)
padded = F.pad(x, (0, 0, 0, 0, (-frames) % factor_t, 0))
packed = rearrange(
padded,
"b c (t ft) (h fh) (w fw) -> b (c ft fh fw) t h w",
ft=factor_t,
fh=factor_s,
fw=factor_s,
)
expected = packed.reshape(2, out_channels, 2, *packed.shape[2:]).mean(2)
reduced = down(x)
torch.testing.assert_close(reduced, expected, rtol=0, atol=0)
expanded = rearrange(
reduced.repeat_interleave(2, dim=1),
"b (c ft fh fw) t h w -> b c (t ft) (h fh) (w fw)",
ft=factor_t,
fh=factor_s,
fw=factor_s,
)
for first in (False, True, False):
expected = expanded[:, :, factor_t - 1 :] if first else expanded
token = wanvae.first_chunk.set(first)
try:
torch.testing.assert_close(
causal_up(reduced), expected, rtol=0, atol=0
)
torch.testing.assert_close(
up(reduced, first), expected, rtol=0, atol=0
)
torch.testing.assert_close(up(reduced), expanded, rtol=0, atol=0)
finally:
wanvae.first_chunk.reset(token)
assert not down.state_dict() and not up.state_dict() and not causal_up.state_dict()