[Fix] diffusion/ltx: Keep tiled VAE decode off the sharded convs under parallel decode (#45942)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-23 15:32:03 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent 4f47c48b33
commit 8708d2ee99
3 changed files with 175 additions and 43 deletions
@@ -1198,10 +1198,6 @@ class PipelineConfig:
def check_pipeline_config(self) -> None:
self.task_type = ModelTaskType.parse(self.task_type)
self.get_supported_task_types()
if self.vae_sp and not self.vae_tiling:
raise ValueError(
"Currently enabling vae_sp requires enabling vae_tiling, please set --vae-tiling to True."
)
if len(self.text_encoder_configs) != len(self.text_encoder_precisions):
raise ValueError(
@@ -1,3 +1,4 @@
import math
from collections.abc import Callable
from functools import lru_cache
from typing import Optional, Tuple, Union
@@ -1926,13 +1927,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
return_dict: bool = True,
on_frames: Callable[[torch.Tensor], None] | None = None,
) -> Union[DecoderOutput, torch.Tensor]:
batch_size, num_channels, num_frames, height, width = z.shape
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
num_frames = z.shape[2]
tile_latent_min_num_frames = (
self.tile_sample_min_num_frames // self.temporal_compression_ratio
)
@@ -1942,12 +1937,35 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
z, temb, causal=causal, return_dict=return_dict, on_frames=on_frames
)
if self.use_tiling and (
width > tile_latent_min_width or height > tile_latent_min_height
):
return self.tiled_decode(z, temb, causal=causal, return_dict=return_dict)
dec = self._decode_spatial(z, temb, causal=causal)
if self._should_use_spatial_parallel_decode(z):
if not return_dict:
return (dec,)
return DecoderOutput(sample=dec)
def _decode_spatial(
self,
z: torch.Tensor,
temb: Optional[torch.Tensor],
causal: Optional[bool] = None,
) -> torch.Tensor:
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
use_tiles = self.use_tiling and (
z.shape[-1] > tile_latent_min_width or z.shape[-2] > tile_latent_min_height
)
# tiling caps a rank at one tile's area at a time, so the height split
# replaces it only when each rank's rows fit in one tile
if self._should_use_spatial_parallel_decode(z) and (
not use_tiles
or math.ceil(z.shape[-2] / get_decode_parallel_world_size()) * z.shape[-1]
<= tile_latent_min_height * tile_latent_min_width
):
expected_height = (
z.shape[-2] * self.config.arch_config.spatial_compression_ratio
)
@@ -1957,19 +1975,15 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
world_size=get_decode_parallel_world_size(),
rank=get_decode_parallel_rank(),
)
dec = gather_and_trim_height(
return gather_and_trim_height(
self.decoder(z, temb, causal=causal), expected_height
)
elif self._spatial_parallel_decode_enabled:
with disable_spatial_parallel_decode():
dec = self.decoder(z, temb, causal=causal)
else:
dec = self.decoder(z, temb, causal=causal)
if not return_dict:
return (dec,)
return DecoderOutput(sample=dec)
# sharded convs exchange halo rows with the neighbour rank, so an unsplit
# tensor must bypass them
with disable_spatial_parallel_decode():
if use_tiles:
return self.tiled_decode(z, temb, causal=causal).sample
return self.decoder(z, temb, causal=causal)
def decode(
self,
@@ -2208,12 +2222,6 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
batch_size, num_channels, num_frames, height, width = z.shape
num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1
tile_latent_min_height = (
self.tile_sample_min_height // self.spatial_compression_ratio
)
tile_latent_min_width = (
self.tile_sample_min_width // self.spatial_compression_ratio
)
tile_latent_min_num_frames = (
self.tile_sample_min_num_frames // self.temporal_compression_ratio
)
@@ -2229,15 +2237,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
emitted = 0
for i in range(0, num_frames, tile_latent_stride_num_frames):
tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :]
if self.use_tiling and (
tile.shape[-1] > tile_latent_min_width
or tile.shape[-2] > tile_latent_min_height
):
decoded = self.tiled_decode(
tile, temb, causal=causal, return_dict=True
).sample
else:
decoded = self.decoder(tile, temb, causal=causal)
decoded = self._decode_spatial(tile, temb, causal=causal)
if i > 0:
decoded = decoded[:, :, :-1, :, :]
decoded = self.blend_t(previous, decoded, blend_num_frames)
@@ -2310,7 +2310,9 @@ class AutoencoderKLCausalLTX2Video(AutoencoderKLLTX2Video):
pixel chunk ``(B, 3, T_out, h*sf, w*sf)`` and mutates ``conv_cache`` in
place. Bypasses the tiled/framewise ``decode`` dispatch on purpose.
"""
return self.decoder(z_chunk, temb, causal=True, conv_cache=conv_cache)
# every rank decodes the whole chunk, so sharded convs must not exchange halos
with disable_spatial_parallel_decode():
return self.decoder(z_chunk, temb, causal=True, conv_cache=conv_cache)
EntryClass = [AutoencoderKLLTX2Video, AutoencoderKLCausalLTX2Video]
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
"""Two decode ranks must reproduce the single-rank LTX-2 VAE decode.
The ranks run over gloo on CPU, so this needs no GPU.
"""
from datetime import timedelta
from unittest.mock import patch
import torch
import torch.distributed as dist
import torch.multiprocessing as mp
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
LTXVideoVAEArchConfig,
LTXVideoVAEConfig,
)
from sglang.multimodal_gen.runtime.distributed import parallel_state
from sglang.multimodal_gen.runtime.layers.parallel_conv import SpatialParallelConv3d
from sglang.multimodal_gen.runtime.models.vaes import common
from sglang.multimodal_gen.runtime.models.vaes.ltx_2_vae import (
AutoencoderKLCausalLTX2Video,
)
WORLD_SIZE = 2
class _DecodeGroup:
def __init__(self, rank: int):
self.rank_in_group = rank
self.world_size = WORLD_SIZE
self.ranks = list(range(WORLD_SIZE))
self.device_group = dist.group.WORLD
def all_gather(self, x: torch.Tensor, dim: int) -> torch.Tensor:
gathered = [torch.empty_like(x) for _ in range(self.world_size)]
dist.all_gather(gathered, x, group=self.device_group)
return torch.cat(gathered, dim=dim)
def _tiny_vae(**config) -> AutoencoderKLCausalLTX2Video:
torch.manual_seed(0)
arch = LTXVideoVAEArchConfig(
latent_channels=4,
spatial_compression_ratio=2,
temporal_compression_ratio=2,
block_out_channels=[8],
down_block_types=["LTX2VideoDownBlock3D"],
spatio_temporal_scaling=[True],
layers_per_block=[1, 1],
downsample_type=["spatiotemporal"],
patch_size=1,
decoder_block_out_channels=[8],
decoder_spatio_temporal_scaling=[True],
decoder_layers_per_block=[1, 1],
decoder_inject_noise=[False, False],
upsample_residual=[True],
upsample_factor=[2],
)
vae = AutoencoderKLCausalLTX2Video(LTXVideoVAEConfig(arch_config=arch, **config))
# 4-row latent tiles on a 3-row stride: an 8x8 latent spans 3x3 tiles
vae.enable_tiling(
tile_sample_min_height=8,
tile_sample_min_width=8,
tile_sample_min_num_frames=8,
tile_sample_stride_height=6,
tile_sample_stride_width=6,
tile_sample_stride_num_frames=4,
)
return vae.eval()
@torch.no_grad()
def _check_decodes(rank: int) -> None:
reference = _tiny_vae(use_parallel_decode=False)
auto = _tiny_vae()
sharded = _tiny_vae(parallel_decode_mode="spatial_shard")
assert any(isinstance(m, SpatialParallelConv3d) for m in sharded.decoder.modules())
torch.manual_seed(1)
z = torch.randn(1, 4, 7, 8, 8)
errors = {}
def compare(name, actual, expected):
assert actual.shape == expected.shape, (name, actual.shape, expected.shape)
errors[name] = max(
errors.get(name, 0.0), (actual - expected).abs().max().item()
)
# below the auto threshold every rank decodes the whole clip, tiled as on one rank
compare("tiled", auto.decode(z).sample, reference.decode(z).sample)
# 4 rows x 8 per rank exceed a 4x4 tile, so every rank still tiles the whole clip
compare("tiled_shard", sharded.decode(z).sample, reference.decode(z).sample)
# they fit a 6x6 tile, so the height split replaces tiling
sharded.enable_tiling(
tile_sample_min_height=12,
tile_sample_min_width=12,
tile_sample_stride_height=10,
tile_sample_stride_width=10,
)
reference.use_tiling = False
compare("height_split", sharded.decode(z).sample, reference.decode(z).sample)
reference.use_framewise_decoding = sharded.use_framewise_decoding = True
compare("framewise", sharded.decode(z).sample, reference.decode(z).sample)
reference_cache, cache = {}, {}
for chunk in z.split(3, dim=2):
compare(
"chunk",
sharded.decode_chunk(chunk, cache),
reference.decode_chunk(chunk, reference_cache),
)
mismatched = {name: err for name, err in errors.items() if err > 1e-5}
assert not mismatched, f"rank {rank}: max abs error vs one rank {mismatched}"
def _decode_on_two_ranks(rank: int, rendezvous: str) -> None:
dist.init_process_group(
"gloo",
init_method=rendezvous,
rank=rank,
world_size=WORLD_SIZE,
timeout=timedelta(seconds=60),
)
parallel_state._VAE_DECODE = _DecodeGroup(rank)
try:
with patch.object(common, "model_parallel_is_initialized", return_value=True):
_check_decodes(rank)
finally:
parallel_state._VAE_DECODE = None
dist.destroy_process_group()
def test_two_decode_ranks_match_one_rank(tmp_path):
rendezvous = (tmp_path / "rendezvous").as_uri()
mp.spawn(_decode_on_two_ranks, args=(rendezvous,), nprocs=WORLD_SIZE)