[diffusion] refactor: deduplicate VAE resampling and LTX tiling (#42866)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -29,6 +29,7 @@ from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
can_install_spatial_shard_parallel_decode,
|
||||
should_run_spatial_shard_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D, DupUp3D
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -41,88 +42,6 @@ def get_activation(name):
|
||||
return nn.SiLU()
|
||||
|
||||
|
||||
class QwenImage21AvgDown3D(nn.Module):
|
||||
def __init__(self, in_channels, out_channels, factor_t, factor_s=1):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.factor_t = factor_t
|
||||
self.factor_s = factor_s
|
||||
self.factor = self.factor_t * self.factor_s * self.factor_s
|
||||
assert in_channels * self.factor % out_channels == 0
|
||||
self.group_size = in_channels * self.factor // out_channels
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
|
||||
pad = (0, 0, 0, 0, pad_t, 0)
|
||||
x = F.pad(x, pad)
|
||||
B, C, T, H, W = x.shape
|
||||
x = x.view(
|
||||
B,
|
||||
C,
|
||||
T // self.factor_t,
|
||||
self.factor_t,
|
||||
H // self.factor_s,
|
||||
self.factor_s,
|
||||
W // self.factor_s,
|
||||
self.factor_s,
|
||||
)
|
||||
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
|
||||
x = x.view(
|
||||
B,
|
||||
C * self.factor,
|
||||
T // self.factor_t,
|
||||
H // self.factor_s,
|
||||
W // self.factor_s,
|
||||
)
|
||||
x = x.view(
|
||||
B,
|
||||
self.out_channels,
|
||||
self.group_size,
|
||||
T // self.factor_t,
|
||||
H // self.factor_s,
|
||||
W // self.factor_s,
|
||||
)
|
||||
x = x.mean(dim=2)
|
||||
return x
|
||||
|
||||
|
||||
class QwenImage21DupUp3D(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.factor_t = factor_t
|
||||
self.factor_s = factor_s
|
||||
self.factor = self.factor_t * self.factor_s * self.factor_s
|
||||
assert out_channels * self.factor % in_channels == 0
|
||||
self.repeats = out_channels * self.factor // in_channels
|
||||
|
||||
def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor:
|
||||
x = x.repeat_interleave(self.repeats, dim=1)
|
||||
x = x.view(
|
||||
x.size(0),
|
||||
self.out_channels,
|
||||
self.factor_t,
|
||||
self.factor_s,
|
||||
self.factor_s,
|
||||
x.size(2),
|
||||
x.size(3),
|
||||
x.size(4),
|
||||
)
|
||||
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
|
||||
x = x.view(
|
||||
x.size(0),
|
||||
self.out_channels,
|
||||
x.size(2) * self.factor_t,
|
||||
x.size(4) * self.factor_s,
|
||||
x.size(6) * self.factor_s,
|
||||
)
|
||||
if first_chunk:
|
||||
x = x[:, :, self.factor_t - 1 :, :, :]
|
||||
return x
|
||||
|
||||
|
||||
class QwenImage21CausalConv3d(nn.Conv2d):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -353,7 +272,7 @@ class QwenImage21ResidualDownBlock(nn.Module):
|
||||
down_flag=False,
|
||||
):
|
||||
super().__init__()
|
||||
self.avg_shortcut = QwenImage21AvgDown3D(
|
||||
self.avg_shortcut = AvgDown3D(
|
||||
in_dim,
|
||||
out_dim,
|
||||
factor_t=2 if temperal_downsample else 1,
|
||||
@@ -464,7 +383,7 @@ class QwenImage21ResidualUpBlock(nn.Module):
|
||||
self.in_dim = in_dim
|
||||
self.out_dim = out_dim
|
||||
if up_flag:
|
||||
self.avg_shortcut = QwenImage21DupUp3D(
|
||||
self.avg_shortcut = DupUp3D(
|
||||
in_dim, out_dim, factor_t=2 if temperal_upsample else 1, factor_s=2
|
||||
)
|
||||
else:
|
||||
@@ -495,7 +414,7 @@ class QwenImage21ResidualUpBlock(nn.Module):
|
||||
if self.avg_shortcut is not None:
|
||||
shortcut = self.avg_shortcut
|
||||
if (
|
||||
type(shortcut) is QwenImage21DupUp3D
|
||||
type(shortcut) is DupUp3D
|
||||
and x.is_cuda
|
||||
and x.dtype in (torch.float16, torch.bfloat16, torch.float32)
|
||||
and not torch.compiler.is_compiling()
|
||||
|
||||
@@ -1991,36 +1991,6 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
|
||||
return DecoderOutput(sample=decoded)
|
||||
|
||||
def blend_v(
|
||||
self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
|
||||
) -> torch.Tensor:
|
||||
blend_extent = min(a.shape[3], b.shape[3], blend_extent)
|
||||
for y in range(blend_extent):
|
||||
b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (
|
||||
1 - y / blend_extent
|
||||
) + b[:, :, :, y, :] * (y / blend_extent)
|
||||
return b
|
||||
|
||||
def blend_h(
|
||||
self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
|
||||
) -> torch.Tensor:
|
||||
blend_extent = min(a.shape[4], b.shape[4], blend_extent)
|
||||
for x in range(blend_extent):
|
||||
b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (
|
||||
1 - x / blend_extent
|
||||
) + b[:, :, :, :, x] * (x / blend_extent)
|
||||
return b
|
||||
|
||||
def blend_t(
|
||||
self, a: torch.Tensor, b: torch.Tensor, blend_extent: int
|
||||
) -> torch.Tensor:
|
||||
blend_extent = min(a.shape[-3], b.shape[-3], blend_extent)
|
||||
for x in range(blend_extent):
|
||||
b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (
|
||||
1 - x / blend_extent
|
||||
) + b[:, :, x, :, :] * (x / blend_extent)
|
||||
return b
|
||||
|
||||
def tiled_encode(
|
||||
self, x: torch.Tensor, causal: Optional[bool] = None
|
||||
) -> torch.Tensor:
|
||||
@@ -2073,22 +2043,13 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
row.append(time)
|
||||
rows.append(row)
|
||||
|
||||
result_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
result_row = []
|
||||
for j, tile in enumerate(row):
|
||||
# blend the above tile and the left tile
|
||||
# to the current tile and add the current tile to the result row
|
||||
if i > 0:
|
||||
tile = self.blend_v(rows[i - 1][j], tile, blend_height)
|
||||
if j > 0:
|
||||
tile = self.blend_h(row[j - 1], tile, blend_width)
|
||||
result_row.append(
|
||||
tile[:, :, :, :tile_latent_stride_height, :tile_latent_stride_width]
|
||||
)
|
||||
result_rows.append(torch.cat(result_row, dim=4))
|
||||
|
||||
enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width]
|
||||
enc = self._merge_spatial_tiles(
|
||||
rows,
|
||||
blend_height,
|
||||
blend_width,
|
||||
tile_latent_stride_height,
|
||||
tile_latent_stride_width,
|
||||
)[:, :, :, :latent_height, :latent_width]
|
||||
return enc
|
||||
|
||||
def tiled_decode(
|
||||
@@ -2153,28 +2114,13 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
row.append(time)
|
||||
rows.append(row)
|
||||
|
||||
result_rows = []
|
||||
for i, row in enumerate(rows):
|
||||
result_row = []
|
||||
for j, tile in enumerate(row):
|
||||
# blend the above tile and the left tile
|
||||
# to the current tile and add the current tile to the result row
|
||||
if i > 0:
|
||||
tile = self.blend_v(rows[i - 1][j], tile, blend_height)
|
||||
if j > 0:
|
||||
tile = self.blend_h(row[j - 1], tile, blend_width)
|
||||
result_row.append(
|
||||
tile[
|
||||
:,
|
||||
:,
|
||||
:,
|
||||
: self.tile_sample_stride_height,
|
||||
: self.tile_sample_stride_width,
|
||||
]
|
||||
)
|
||||
result_rows.append(torch.cat(result_row, dim=4))
|
||||
|
||||
dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width]
|
||||
dec = self._merge_spatial_tiles(
|
||||
rows,
|
||||
blend_height,
|
||||
blend_width,
|
||||
self.tile_sample_stride_height,
|
||||
self.tile_sample_stride_width,
|
||||
)[:, :, :, :sample_height, :sample_width]
|
||||
|
||||
if not return_dict:
|
||||
return (dec,)
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The Wan Team and The HuggingFace Team. All rights reserved.
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
class AvgDown3D(nn.Module):
|
||||
def __init__(self, in_channels, out_channels, factor_t, factor_s=1):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.factor_t = factor_t
|
||||
self.factor_s = factor_s
|
||||
self.factor = self.factor_t * self.factor_s * self.factor_s
|
||||
assert in_channels * self.factor % out_channels == 0
|
||||
self.group_size = in_channels * self.factor // out_channels
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
|
||||
pad = (0, 0, 0, 0, pad_t, 0)
|
||||
x = F.pad(x, pad)
|
||||
B, C, T, H, W = x.shape
|
||||
x = x.view(
|
||||
B,
|
||||
C,
|
||||
T // self.factor_t,
|
||||
self.factor_t,
|
||||
H // self.factor_s,
|
||||
self.factor_s,
|
||||
W // self.factor_s,
|
||||
self.factor_s,
|
||||
)
|
||||
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
|
||||
x = x.view(
|
||||
B,
|
||||
C * self.factor,
|
||||
T // self.factor_t,
|
||||
H // self.factor_s,
|
||||
W // self.factor_s,
|
||||
)
|
||||
x = x.view(
|
||||
B,
|
||||
self.out_channels,
|
||||
self.group_size,
|
||||
T // self.factor_t,
|
||||
H // self.factor_s,
|
||||
W // self.factor_s,
|
||||
)
|
||||
x = x.mean(dim=2)
|
||||
return x
|
||||
|
||||
|
||||
class DupUp3D(nn.Module):
|
||||
def __init__(self, in_channels: int, out_channels: int, factor_t, factor_s=1):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.factor_t = factor_t
|
||||
self.factor_s = factor_s
|
||||
self.factor = self.factor_t * self.factor_s * self.factor_s
|
||||
assert out_channels * self.factor % in_channels == 0
|
||||
self.repeats = out_channels * self.factor // in_channels
|
||||
|
||||
def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor:
|
||||
x = x.repeat_interleave(self.repeats, dim=1)
|
||||
x = x.view(
|
||||
x.size(0),
|
||||
self.out_channels,
|
||||
self.factor_t,
|
||||
self.factor_s,
|
||||
self.factor_s,
|
||||
x.size(2),
|
||||
x.size(3),
|
||||
x.size(4),
|
||||
)
|
||||
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
|
||||
x = x.view(
|
||||
x.size(0),
|
||||
self.out_channels,
|
||||
x.size(2) * self.factor_t,
|
||||
x.size(4) * self.factor_s,
|
||||
x.size(6) * self.factor_s,
|
||||
)
|
||||
if first_chunk:
|
||||
x = x[:, :, self.factor_t - 1 :, :, :]
|
||||
return x
|
||||
@@ -56,6 +56,8 @@ from sglang.multimodal_gen.runtime.models.vaes.common import (
|
||||
ParallelTiledVAE,
|
||||
should_run_spatial_shard_parallel_decode,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D
|
||||
from sglang.multimodal_gen.runtime.models.vaes.resample import DupUp3D as DupUp3DBase
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
|
||||
if current_platform.is_cuda() or current_platform.is_xpu():
|
||||
@@ -164,103 +166,10 @@ def _run_cached_causal_conv(
|
||||
return out
|
||||
|
||||
|
||||
class AvgDown3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels,
|
||||
out_channels,
|
||||
factor_t,
|
||||
factor_s=1,
|
||||
):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
self.factor_t = factor_t
|
||||
self.factor_s = factor_s
|
||||
self.factor = self.factor_t * self.factor_s * self.factor_s
|
||||
|
||||
assert in_channels * self.factor % out_channels == 0
|
||||
self.group_size = in_channels * self.factor // out_channels
|
||||
|
||||
class DupUp3D(DupUp3DBase):
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t
|
||||
pad = (0, 0, 0, 0, pad_t, 0)
|
||||
x = F.pad(x, pad)
|
||||
B, C, T, H, W = x.shape
|
||||
x = x.view(
|
||||
B,
|
||||
C,
|
||||
T // self.factor_t,
|
||||
self.factor_t,
|
||||
H // self.factor_s,
|
||||
self.factor_s,
|
||||
W // self.factor_s,
|
||||
self.factor_s,
|
||||
)
|
||||
x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous()
|
||||
x = x.view(
|
||||
B,
|
||||
C * self.factor,
|
||||
T // self.factor_t,
|
||||
H // self.factor_s,
|
||||
W // self.factor_s,
|
||||
)
|
||||
x = x.view(
|
||||
B,
|
||||
self.out_channels,
|
||||
self.group_size,
|
||||
T // self.factor_t,
|
||||
H // self.factor_s,
|
||||
W // self.factor_s,
|
||||
)
|
||||
x = x.mean(dim=2)
|
||||
return x
|
||||
|
||||
|
||||
class DupUp3D(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int,
|
||||
out_channels: int,
|
||||
factor_t,
|
||||
factor_s=1,
|
||||
):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = out_channels
|
||||
|
||||
self.factor_t = factor_t
|
||||
self.factor_s = factor_s
|
||||
self.factor = self.factor_t * self.factor_s * self.factor_s
|
||||
|
||||
assert out_channels * self.factor % in_channels == 0
|
||||
self.repeats = out_channels * self.factor // in_channels
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = x.repeat_interleave(self.repeats, dim=1)
|
||||
x = x.view(
|
||||
x.size(0),
|
||||
self.out_channels,
|
||||
self.factor_t,
|
||||
self.factor_s,
|
||||
self.factor_s,
|
||||
x.size(2),
|
||||
x.size(3),
|
||||
x.size(4),
|
||||
)
|
||||
x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous()
|
||||
x = x.view(
|
||||
x.size(0),
|
||||
self.out_channels,
|
||||
x.size(2) * self.factor_t,
|
||||
x.size(4) * self.factor_s,
|
||||
x.size(6) * self.factor_s,
|
||||
)
|
||||
|
||||
_first_chunk = first_chunk.get() if first_chunk is not None else None
|
||||
if _first_chunk:
|
||||
x = x[:, :, self.factor_t - 1 :, :, :]
|
||||
return x
|
||||
return super().forward(x, first_chunk=_first_chunk)
|
||||
|
||||
|
||||
class WanCausalConv3d(nn.Conv3d):
|
||||
|
||||
-25
@@ -1,5 +1,4 @@
|
||||
import torch
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.ltx_2 import is_ltx23_native_variant
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
@@ -168,30 +167,6 @@ class LTX2RefinementStage(LTX2AVDenoisingStage):
|
||||
return StageParallelismType.MAIN_RANK_ONLY
|
||||
return StageParallelismType.REPLICATED
|
||||
|
||||
@staticmethod
|
||||
def _randn_like_with_batch_generators(
|
||||
reference_tensor: torch.Tensor, batch: Req
|
||||
) -> torch.Tensor:
|
||||
generator = getattr(batch, "generator", None)
|
||||
if isinstance(generator, list):
|
||||
bsz = int(reference_tensor.shape[0])
|
||||
valid_generators = [g for g in generator if isinstance(g, torch.Generator)]
|
||||
if len(valid_generators) == 1:
|
||||
generator = valid_generators[0]
|
||||
elif len(valid_generators) >= bsz:
|
||||
generator = valid_generators[:bsz]
|
||||
else:
|
||||
generator = None
|
||||
elif not isinstance(generator, torch.Generator):
|
||||
generator = None
|
||||
|
||||
return randn_tensor(
|
||||
reference_tensor.shape,
|
||||
generator=generator,
|
||||
device=reference_tensor.device,
|
||||
dtype=reference_tensor.dtype,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _reset_stage2_generators(batch: Req) -> None:
|
||||
generator = getattr(batch, "generator", None)
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from einops import rearrange
|
||||
|
||||
from sglang.multimodal_gen.runtime.models.vaes import wanvae
|
||||
from sglang.multimodal_gen.runtime.models.vaes.resample import AvgDown3D, DupUp3D
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device", ["cpu", "cuda"])
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("factor_t,factor_s", [(1, 1), (1, 2), (2, 1), (2, 2)])
|
||||
def test_resample_layout_and_causal_crop(device, dtype, factor_t, factor_s):
|
||||
if device == "cuda" and not torch.cuda.is_available():
|
||||
pytest.skip("CUDA required")
|
||||
out_channels = 2 * factor_t * factor_s**2
|
||||
down = AvgDown3D(4, out_channels, factor_t, factor_s)
|
||||
up = DupUp3D(out_channels, 4, factor_t, factor_s)
|
||||
causal_up = wanvae.DupUp3D(out_channels, 4, factor_t, factor_s)
|
||||
for frames in (1, 3, 4):
|
||||
x = torch.arange(2 * 4 * frames * 4 * 6, device=device).reshape(
|
||||
2, 4, frames, 4, 6
|
||||
)
|
||||
x = (x % 31).to(dtype)
|
||||
for layout in (torch.contiguous_format, torch.channels_last_3d):
|
||||
x = x.contiguous(memory_format=layout)
|
||||
padded = F.pad(x, (0, 0, 0, 0, (-frames) % factor_t, 0))
|
||||
packed = rearrange(
|
||||
padded,
|
||||
"b c (t ft) (h fh) (w fw) -> b (c ft fh fw) t h w",
|
||||
ft=factor_t,
|
||||
fh=factor_s,
|
||||
fw=factor_s,
|
||||
)
|
||||
expected = packed.reshape(2, out_channels, 2, *packed.shape[2:]).mean(2)
|
||||
reduced = down(x)
|
||||
torch.testing.assert_close(reduced, expected, rtol=0, atol=0)
|
||||
expanded = rearrange(
|
||||
reduced.repeat_interleave(2, dim=1),
|
||||
"b (c ft fh fw) t h w -> b c (t ft) (h fh) (w fw)",
|
||||
ft=factor_t,
|
||||
fh=factor_s,
|
||||
fw=factor_s,
|
||||
)
|
||||
for first in (False, True, False):
|
||||
expected = expanded[:, :, factor_t - 1 :] if first else expanded
|
||||
token = wanvae.first_chunk.set(first)
|
||||
try:
|
||||
torch.testing.assert_close(
|
||||
causal_up(reduced), expected, rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
up(reduced, first), expected, rtol=0, atol=0
|
||||
)
|
||||
torch.testing.assert_close(up(reduced), expanded, rtol=0, atol=0)
|
||||
finally:
|
||||
wanvae.first_chunk.reset(token)
|
||||
assert not down.state_dict() and not up.state_dict() and not causal_up.state_dict()
|
||||
Reference in New Issue
Block a user