[Fix] diffusion/ltx: Keep tiled VAE decode off the sharded convs under parallel decode (#45942)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
4f47c48b33
commit
8708d2ee99
@@ -1198,10 +1198,6 @@ class PipelineConfig:
|
||||
def check_pipeline_config(self) -> None:
|
||||
self.task_type = ModelTaskType.parse(self.task_type)
|
||||
self.get_supported_task_types()
|
||||
if self.vae_sp and not self.vae_tiling:
|
||||
raise ValueError(
|
||||
"Currently enabling vae_sp requires enabling vae_tiling, please set --vae-tiling to True."
|
||||
)
|
||||
|
||||
if len(self.text_encoder_configs) != len(self.text_encoder_precisions):
|
||||
raise ValueError(
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import math
|
||||
from collections.abc import Callable
|
||||
from functools import lru_cache
|
||||
from typing import Optional, Tuple, Union
|
||||
@@ -1926,13 +1927,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
return_dict: bool = True,
|
||||
on_frames: Callable[[torch.Tensor], None] | None = None,
|
||||
) -> Union[DecoderOutput, torch.Tensor]:
|
||||
batch_size, num_channels, num_frames, height, width = z.shape
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
num_frames = z.shape[2]
|
||||
tile_latent_min_num_frames = (
|
||||
self.tile_sample_min_num_frames // self.temporal_compression_ratio
|
||||
)
|
||||
@@ -1942,12 +1937,35 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
z, temb, causal=causal, return_dict=return_dict, on_frames=on_frames
|
||||
)
|
||||
|
||||
if self.use_tiling and (
|
||||
width > tile_latent_min_width or height > tile_latent_min_height
|
||||
):
|
||||
return self.tiled_decode(z, temb, causal=causal, return_dict=return_dict)
|
||||
dec = self._decode_spatial(z, temb, causal=causal)
|
||||
|
||||
if self._should_use_spatial_parallel_decode(z):
|
||||
if not return_dict:
|
||||
return (dec,)
|
||||
|
||||
return DecoderOutput(sample=dec)
|
||||
|
||||
def _decode_spatial(
|
||||
self,
|
||||
z: torch.Tensor,
|
||||
temb: Optional[torch.Tensor],
|
||||
causal: Optional[bool] = None,
|
||||
) -> torch.Tensor:
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
use_tiles = self.use_tiling and (
|
||||
z.shape[-1] > tile_latent_min_width or z.shape[-2] > tile_latent_min_height
|
||||
)
|
||||
# tiling caps a rank at one tile's area at a time, so the height split
|
||||
# replaces it only when each rank's rows fit in one tile
|
||||
if self._should_use_spatial_parallel_decode(z) and (
|
||||
not use_tiles
|
||||
or math.ceil(z.shape[-2] / get_decode_parallel_world_size()) * z.shape[-1]
|
||||
<= tile_latent_min_height * tile_latent_min_width
|
||||
):
|
||||
expected_height = (
|
||||
z.shape[-2] * self.config.arch_config.spatial_compression_ratio
|
||||
)
|
||||
@@ -1957,19 +1975,15 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
world_size=get_decode_parallel_world_size(),
|
||||
rank=get_decode_parallel_rank(),
|
||||
)
|
||||
dec = gather_and_trim_height(
|
||||
return gather_and_trim_height(
|
||||
self.decoder(z, temb, causal=causal), expected_height
|
||||
)
|
||||
elif self._spatial_parallel_decode_enabled:
|
||||
with disable_spatial_parallel_decode():
|
||||
dec = self.decoder(z, temb, causal=causal)
|
||||
else:
|
||||
dec = self.decoder(z, temb, causal=causal)
|
||||
|
||||
if not return_dict:
|
||||
return (dec,)
|
||||
|
||||
return DecoderOutput(sample=dec)
|
||||
# sharded convs exchange halo rows with the neighbour rank, so an unsplit
|
||||
# tensor must bypass them
|
||||
with disable_spatial_parallel_decode():
|
||||
if use_tiles:
|
||||
return self.tiled_decode(z, temb, causal=causal).sample
|
||||
return self.decoder(z, temb, causal=causal)
|
||||
|
||||
def decode(
|
||||
self,
|
||||
@@ -2208,12 +2222,6 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
batch_size, num_channels, num_frames, height, width = z.shape
|
||||
num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1
|
||||
|
||||
tile_latent_min_height = (
|
||||
self.tile_sample_min_height // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_width = (
|
||||
self.tile_sample_min_width // self.spatial_compression_ratio
|
||||
)
|
||||
tile_latent_min_num_frames = (
|
||||
self.tile_sample_min_num_frames // self.temporal_compression_ratio
|
||||
)
|
||||
@@ -2229,15 +2237,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
emitted = 0
|
||||
for i in range(0, num_frames, tile_latent_stride_num_frames):
|
||||
tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :]
|
||||
if self.use_tiling and (
|
||||
tile.shape[-1] > tile_latent_min_width
|
||||
or tile.shape[-2] > tile_latent_min_height
|
||||
):
|
||||
decoded = self.tiled_decode(
|
||||
tile, temb, causal=causal, return_dict=True
|
||||
).sample
|
||||
else:
|
||||
decoded = self.decoder(tile, temb, causal=causal)
|
||||
decoded = self._decode_spatial(tile, temb, causal=causal)
|
||||
if i > 0:
|
||||
decoded = decoded[:, :, :-1, :, :]
|
||||
decoded = self.blend_t(previous, decoded, blend_num_frames)
|
||||
@@ -2310,7 +2310,9 @@ class AutoencoderKLCausalLTX2Video(AutoencoderKLLTX2Video):
|
||||
pixel chunk ``(B, 3, T_out, h*sf, w*sf)`` and mutates ``conv_cache`` in
|
||||
place. Bypasses the tiled/framewise ``decode`` dispatch on purpose.
|
||||
"""
|
||||
return self.decoder(z_chunk, temb, causal=True, conv_cache=conv_cache)
|
||||
# every rank decodes the whole chunk, so sharded convs must not exchange halos
|
||||
with disable_spatial_parallel_decode():
|
||||
return self.decoder(z_chunk, temb, causal=True, conv_cache=conv_cache)
|
||||
|
||||
|
||||
EntryClass = [AutoencoderKLLTX2Video, AutoencoderKLCausalLTX2Video]
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Two decode ranks must reproduce the single-rank LTX-2 VAE decode.
|
||||
|
||||
The ranks run over gloo on CPU, so this needs no GPU.
|
||||
"""
|
||||
|
||||
from datetime import timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
|
||||
LTXVideoVAEArchConfig,
|
||||
LTXVideoVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.distributed import parallel_state
|
||||
from sglang.multimodal_gen.runtime.layers.parallel_conv import SpatialParallelConv3d
|
||||
from sglang.multimodal_gen.runtime.models.vaes import common
|
||||
from sglang.multimodal_gen.runtime.models.vaes.ltx_2_vae import (
|
||||
AutoencoderKLCausalLTX2Video,
|
||||
)
|
||||
|
||||
WORLD_SIZE = 2
|
||||
|
||||
|
||||
class _DecodeGroup:
|
||||
def __init__(self, rank: int):
|
||||
self.rank_in_group = rank
|
||||
self.world_size = WORLD_SIZE
|
||||
self.ranks = list(range(WORLD_SIZE))
|
||||
self.device_group = dist.group.WORLD
|
||||
|
||||
def all_gather(self, x: torch.Tensor, dim: int) -> torch.Tensor:
|
||||
gathered = [torch.empty_like(x) for _ in range(self.world_size)]
|
||||
dist.all_gather(gathered, x, group=self.device_group)
|
||||
return torch.cat(gathered, dim=dim)
|
||||
|
||||
|
||||
def _tiny_vae(**config) -> AutoencoderKLCausalLTX2Video:
|
||||
torch.manual_seed(0)
|
||||
arch = LTXVideoVAEArchConfig(
|
||||
latent_channels=4,
|
||||
spatial_compression_ratio=2,
|
||||
temporal_compression_ratio=2,
|
||||
block_out_channels=[8],
|
||||
down_block_types=["LTX2VideoDownBlock3D"],
|
||||
spatio_temporal_scaling=[True],
|
||||
layers_per_block=[1, 1],
|
||||
downsample_type=["spatiotemporal"],
|
||||
patch_size=1,
|
||||
decoder_block_out_channels=[8],
|
||||
decoder_spatio_temporal_scaling=[True],
|
||||
decoder_layers_per_block=[1, 1],
|
||||
decoder_inject_noise=[False, False],
|
||||
upsample_residual=[True],
|
||||
upsample_factor=[2],
|
||||
)
|
||||
vae = AutoencoderKLCausalLTX2Video(LTXVideoVAEConfig(arch_config=arch, **config))
|
||||
# 4-row latent tiles on a 3-row stride: an 8x8 latent spans 3x3 tiles
|
||||
vae.enable_tiling(
|
||||
tile_sample_min_height=8,
|
||||
tile_sample_min_width=8,
|
||||
tile_sample_min_num_frames=8,
|
||||
tile_sample_stride_height=6,
|
||||
tile_sample_stride_width=6,
|
||||
tile_sample_stride_num_frames=4,
|
||||
)
|
||||
return vae.eval()
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def _check_decodes(rank: int) -> None:
|
||||
reference = _tiny_vae(use_parallel_decode=False)
|
||||
auto = _tiny_vae()
|
||||
sharded = _tiny_vae(parallel_decode_mode="spatial_shard")
|
||||
assert any(isinstance(m, SpatialParallelConv3d) for m in sharded.decoder.modules())
|
||||
torch.manual_seed(1)
|
||||
z = torch.randn(1, 4, 7, 8, 8)
|
||||
errors = {}
|
||||
|
||||
def compare(name, actual, expected):
|
||||
assert actual.shape == expected.shape, (name, actual.shape, expected.shape)
|
||||
errors[name] = max(
|
||||
errors.get(name, 0.0), (actual - expected).abs().max().item()
|
||||
)
|
||||
|
||||
# below the auto threshold every rank decodes the whole clip, tiled as on one rank
|
||||
compare("tiled", auto.decode(z).sample, reference.decode(z).sample)
|
||||
# 4 rows x 8 per rank exceed a 4x4 tile, so every rank still tiles the whole clip
|
||||
compare("tiled_shard", sharded.decode(z).sample, reference.decode(z).sample)
|
||||
# they fit a 6x6 tile, so the height split replaces tiling
|
||||
sharded.enable_tiling(
|
||||
tile_sample_min_height=12,
|
||||
tile_sample_min_width=12,
|
||||
tile_sample_stride_height=10,
|
||||
tile_sample_stride_width=10,
|
||||
)
|
||||
reference.use_tiling = False
|
||||
compare("height_split", sharded.decode(z).sample, reference.decode(z).sample)
|
||||
reference.use_framewise_decoding = sharded.use_framewise_decoding = True
|
||||
compare("framewise", sharded.decode(z).sample, reference.decode(z).sample)
|
||||
reference_cache, cache = {}, {}
|
||||
for chunk in z.split(3, dim=2):
|
||||
compare(
|
||||
"chunk",
|
||||
sharded.decode_chunk(chunk, cache),
|
||||
reference.decode_chunk(chunk, reference_cache),
|
||||
)
|
||||
mismatched = {name: err for name, err in errors.items() if err > 1e-5}
|
||||
assert not mismatched, f"rank {rank}: max abs error vs one rank {mismatched}"
|
||||
|
||||
|
||||
def _decode_on_two_ranks(rank: int, rendezvous: str) -> None:
|
||||
dist.init_process_group(
|
||||
"gloo",
|
||||
init_method=rendezvous,
|
||||
rank=rank,
|
||||
world_size=WORLD_SIZE,
|
||||
timeout=timedelta(seconds=60),
|
||||
)
|
||||
parallel_state._VAE_DECODE = _DecodeGroup(rank)
|
||||
try:
|
||||
with patch.object(common, "model_parallel_is_initialized", return_value=True):
|
||||
_check_decodes(rank)
|
||||
finally:
|
||||
parallel_state._VAE_DECODE = None
|
||||
dist.destroy_process_group()
|
||||
|
||||
|
||||
def test_two_decode_ranks_match_one_rank(tmp_path):
|
||||
rendezvous = (tmp_path / "rendezvous").as_uri()
|
||||
mp.spawn(_decode_on_two_ranks, args=(rendezvous,), nprocs=WORLD_SIZE)
|
||||
Reference in New Issue
Block a user