From 11bfd868ac36c0d5c7089c3f1a65ed575feb5540 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 11 Oct 2026 21:15:49 +0800 Subject: [PATCH] [diffusion] perf: encode LTX video/audio through the shared bounded MP4 path (#43656) Co-authored-by: Mick Qian --- .../performance-optimization.mdx | 29 +++ .../sglang-diffusion/support_new_models.mdx | 24 +++ .../runtime/models/vaes/ltx_2_vae.py | 45 +++-- .../runtime/pipelines_core/stages/decoding.py | 30 ++- .../stages/encode_while_decoding.py | 41 ++-- .../ltx_2/decoding_av.py | 92 +++++++-- .../test/unit/test_ltx_vae_on_frames.py | 187 ++++++++++++++++++ .../test/unit/test_wan_streamed_encode.py | 74 ++++++- 8 files changed, 472 insertions(+), 50 deletions(-) create mode 100644 python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py diff --git a/docs/docs/sglang-diffusion/performance-optimization.mdx b/docs/docs/sglang-diffusion/performance-optimization.mdx index b0e097d655..962239751d 100644 --- a/docs/docs/sglang-diffusion/performance-optimization.mdx +++ b/docs/docs/sglang-diffusion/performance-optimization.mdx @@ -191,6 +191,35 @@ Compare the flag on and off with the same model, output shape, format, and clien concurrency. Keep it disabled when the reply latency or extra memory outweighs the throughput gain. +## Encoding during VAE decode + +Native Wan, H3, and LTX video VAEs can hand completed frames to the shared MP4 +encoder within the current request. Unlike `--async-output-save`, this does not +require a second request to obtain a benefit. Eligible single-video, CUDA, +file-path-only requests use this path automatically; raw frames, compilation, +interpolation, upscaling, and custom post-decoding hooks keep the normal path. +LTX's diffusion decoder is not included. + +Ordinary VAEs without frame callbacks keep the existing GPU-direct saving path. +Handing over a complete clip cannot overlap encoding with decode, and is not +automatically enabled: it can add overhead to short requests. The shared +`DecodingStage.decode(..., on_frames=...)` supports an explicit complete-clip +hand-off for custom integrations. Custom decoding stages retain their original +path unless they explicitly integrate the encoder, preserving output transforms +and audio muxing. Image, audio, and mesh outputs are not MP4 inputs. + +LTX without temporal tiling hands over the complete clip after decode. This +reduces CPU float-frame materialization but does not overlap VAE decoding with +encoding. Existing temporal tiling can emit finalized, blended prefixes while +later tiles decode. This optimization does **not** enable temporal tiling: +changing the tile schedule can change the video. + +Keep the same decode, output compression, and x264 preset when measuring. +Include time until the MP4 is fully written, not just the decoding-stage timer. +The encoder bounds queued frames and frame-conversion intermediates, and a +failed encode falls back to normal saving. Model authors can follow the +[frame hand-off contract](/docs/sglang-diffusion/support_new_models#video-encode-hand-off). + ## Quality-Tradeoff / Lossy Or Approximate Levers These techniques can change the denoising path, numerical representation, or generated output. They are useful after you have a baseline and an acceptance criterion for quality. diff --git a/docs/docs/sglang-diffusion/support_new_models.mdx b/docs/docs/sglang-diffusion/support_new_models.mdx index e12d428195..4735b33d7b 100644 --- a/docs/docs/sglang-diffusion/support_new_models.mdx +++ b/docs/docs/sglang-diffusion/support_new_models.mdx @@ -166,6 +166,30 @@ Use the model's Diffusers pipeline, official implementation, or If the new model is close to Flux, Qwen-Image, GLM-Image, Wan, HunyuanVideo, or LTX, extend that implementation before starting from an empty file. +## Video encode hand-off + +Reuse `EncodeWhileDecoding` and `streamed_video_output_path` instead of adding a +model-specific ffmpeg writer. Explicit `DecodingStage.decode(..., on_frames=...)` +calls can hand over the completed RGB video for VAEs without callbacks; no decoder +modification is needed. This whole-clip path is not automatically enabled: +ordinary decoders already use GPU-direct saving and may not benefit. +To overlap encoding with existing incremental decoding, a native VAE can declare +`supports_decode_on_frames = True` and accept `decode(..., on_frames=callback)`. +Emit read-only `[1, 3, T, H, W]` runs in presentation order, only after overlap +blending and cropping are final. Concatenating these runs must exactly reproduce +the normal decode. Batched decoders can emit the complete batch once; the shared +MP4 path currently only accepts a single output. + +Match the model's existing normalization, dtype, and uint8 conversion order. +Audio must be available before opening the shared encoder. Keep component-use +ordering identical across ranks, including ranks that do not write output. +Encoder failures must retain the regular decode result for saving afterward. + +An untiled decoder may emit once at completion, reducing CPU frame materialization +but not overlapping encoding with decode. Do not enable temporal tiling just to +gain overlap: it can change the decoded video. Compare callback-on and callback-off +with identical tiling, audio, encoding settings, and full request timing. + ## Conditioning reuse Use the native `TextEncoder` / `ImageEncoder` base classes (or `EncoderTensorParallelMixin`) and call encoders through `encoder(...)`. This preserves the shared conditioning cache, module hooks, and encoder TP/folding group. A replicated custom encoder can use `ConditioningEncoderMixin`; do not apply it to a module with hidden mutable state or random sampling in `forward()`. diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py index a45a50df82..86395baedd 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/ltx_2_vae.py @@ -1,3 +1,4 @@ +from collections.abc import Callable from functools import lru_cache from typing import Optional, Tuple, Union @@ -1618,6 +1619,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): """ _supports_gradient_checkpointing = False + supports_decode_on_frames = True def __init__(self, config: LTXVideoVAEConfig): super().__init__(config=config) @@ -1896,6 +1898,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): temb: Optional[torch.Tensor] = None, causal: Optional[bool] = None, return_dict: bool = True, + on_frames: Callable[[torch.Tensor], None] | None = None, ) -> Union[DecoderOutput, torch.Tensor]: batch_size, num_channels, num_frames, height, width = z.shape tile_latent_min_height = ( @@ -1910,7 +1913,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): if self.use_framewise_decoding and num_frames > tile_latent_min_num_frames: return self._temporal_tiled_decode( - z, temb, causal=causal, return_dict=return_dict + z, temb, causal=causal, return_dict=return_dict, on_frames=on_frames ) if self.use_tiling and ( @@ -1948,6 +1951,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): temb: Optional[torch.Tensor] = None, causal: Optional[bool] = None, return_dict: bool = True, + on_frames: Callable[[torch.Tensor], None] | None = None, ) -> Union[DecoderOutput, torch.Tensor]: """ Decode a batch of images. @@ -1975,7 +1979,17 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): ] decoded = torch.cat(decoded_slices) else: - decoded = self._decode(z, temb, causal=causal).sample + decoded = self._decode(z, temb, causal=causal, on_frames=on_frames).sample + + tile_latent_frames = ( + self.tile_sample_min_num_frames // self.temporal_compression_ratio + ) + if on_frames is not None and not ( + self.use_framewise_decoding + and z.shape[2] > tile_latent_frames + and z.shape[0] == 1 + ): + on_frames(decoded) if not return_dict: return (decoded,) @@ -2163,6 +2177,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): temb: Optional[torch.Tensor], causal: Optional[bool] = None, return_dict: bool = True, + on_frames: Callable[[torch.Tensor], None] | None = None, ) -> Union[DecoderOutput, torch.Tensor]: batch_size, num_channels, num_frames, height, width = z.shape num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1 @@ -2183,7 +2198,9 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames ) - row = [] + result_row = [] + previous = None + emitted = 0 for i in range(0, num_frames, tile_latent_stride_num_frames): tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :] if self.use_tiling and ( @@ -2197,18 +2214,16 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE): decoded = self.decoder(tile, temb, causal=causal) if i > 0: decoded = decoded[:, :, :-1, :, :] - row.append(decoded) - - result_row = [] - for i, tile in enumerate(row): - if i > 0: - tile = self.blend_t(row[i - 1], tile, blend_num_frames) - tile = tile[:, :, : self.tile_sample_stride_num_frames, :, :] - result_row.append(tile) - else: - result_row.append( - tile[:, :, : self.tile_sample_stride_num_frames + 1, :, :] - ) + decoded = self.blend_t(previous, decoded, blend_num_frames) + previous = decoded + kept = decoded[:, :, : self.tile_sample_stride_num_frames + (i == 0)] + result_row.append(kept) + # only the kept prefix is final; the tail is still needed to blend the next tile + if on_frames is not None and batch_size == 1: + frames = kept[:, :, : max(0, num_sample_frames - emitted)] + if frames.shape[2]: + on_frames(frames) + emitted += frames.shape[2] dec = torch.cat(result_row, dim=2)[:, :, :num_sample_frames] diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py index f8ffc1b0bb..6badb81638 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py @@ -229,8 +229,8 @@ class DecodingStage(PipelineStage): VAE precision ("fp32", "fp16", "bf16") - pipeline_config.vae_precision: fallback VAE precision - pipeline_config.vae_tiling: Whether to enable VAE tiling for memory efficiency - on_frames: For a VAE with ``supports_decode_on_frames``, receives the - returned frames once, in order, in the pieces the VAE finishes them. + on_frames: Receives finished video frames in order. VAEs without a + frame callback hand over the complete decoded clip once. Returns: Decoded video tensor with shape (batch, channels, frames, height, width), @@ -239,6 +239,8 @@ class DecodingStage(PipelineStage): normalized_on_frames = ( (lambda frames: on_frames(_to_unit_range(frames))) if on_frames is not None + and isinstance(self.vae, ParallelTiledVAE) + and self.vae.supports_decode_on_frames else None ) image = self.decode_raw( @@ -247,7 +249,15 @@ class DecodingStage(PipelineStage): vae_dtype=vae_dtype, on_frames=normalized_on_frames, ) - return _to_unit_range(image) + image = _to_unit_range(image) + if ( + on_frames is not None + and normalized_on_frames is None + and image.ndim == 5 + and image.shape[:2] == (1, 3) + ): + on_frames(image) + return image @torch.no_grad() def decode_raw( @@ -292,7 +302,11 @@ class DecodingStage(PipelineStage): if not vae_autocast_enabled: latents = latents.to(vae_dtype) decode_kwargs = {} - if on_frames is not None: + streams_frames = ( + isinstance(self.vae, ParallelTiledVAE) + and self.vae.supports_decode_on_frames + ) + if on_frames is not None and streams_frames: decode_kwargs["on_frames"] = on_frames with temporary_module_dtype( self.vae, vae_dtype, enabled=should_cast_vae @@ -325,6 +339,14 @@ class DecodingStage(PipelineStage): raise image = _ensure_tensor_decode_output(decode_output) + if ( + on_frames is not None + and not streams_frames + and image.ndim == 5 + and image.shape[:2] == (1, 3) + ): + on_frames(image) + return image def load_model(self): diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py index 870061f437..bbf47e9dcc 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py @@ -25,11 +25,11 @@ logger = init_logger(__name__) _STREAMED_VIDEO_QUEUE_BYTES = 256 * 1024 * 1024 -def streamed_video_output_path( +def can_encode_while_decoding( batch: Req, server_args: ServerArgs, batch_size: int -) -> str | None: - """Where the output rank can write the MP4 during the decode, if anywhere.""" - if not ( +) -> bool: + """Rank-independent eligibility, including ranks that only decode audio/video.""" + return ( batch.save_output and batch.return_file_paths_only and not batch.return_raw_frames @@ -42,7 +42,14 @@ def streamed_video_output_path( and type(server_args.pipeline_config).post_decoding is PipelineConfig.post_decoding and current_platform.is_cuda() - ): + ) + + +def streamed_video_output_path( + batch: Req, server_args: ServerArgs, batch_size: int +) -> str | None: + """Where the output rank can write the MP4 during the decode, if anywhere.""" + if not can_encode_while_decoding(batch, server_args, batch_size): return None if model_parallel_is_initialized() and get_replica_group().rank_in_group != 0: return None @@ -73,15 +80,23 @@ class EncodeWhileDecoding: if self._failed: return try: - if self._finish_frames is not None: - frames = self._finish_frames(frames) - frames = frames[0] - if self._encoder is None: - self._encoder = self._open(frames) + # Untiled decoders can hand over an entire clip. Bound FP32 + # postprocessing too, not just the queued uint8 frame copies. + height, width = frames.shape[-2:] + float_frame_bytes = 3 * height * width * 4 + chunk_frames = max( + 1, (_STREAMED_VIDEO_QUEUE_BYTES // 4) // float_frame_bytes + ) + for chunk in frames.split(chunk_frames, dim=2): + if self._finish_frames is not None: + chunk = self._finish_frames(chunk) + chunk = chunk[0] if self._encoder is None: - self._failed = True - return - self._encoder.write(frames) + self._encoder = self._open(chunk) + if self._encoder is None: + self._failed = True + return + self._encoder.write(chunk) except Exception: self._fail() diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py index ab6fad3e0c..14220d84df 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/ltx_2/decoding_av.py @@ -1,14 +1,23 @@ +from collections.abc import Callable + import torch +from diffusers.video_processor import VideoProcessor from sglang.multimodal_gen.runtime.distributed import get_local_torch_device from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import ( ComponentUse, ) +from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( StageParallelismType, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( + EncodeWhileDecoding, + can_encode_while_decoding, + streamed_video_output_path, +) from sglang.multimodal_gen.runtime.server_args import ServerArgs from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger from sglang.multimodal_gen.runtime.utils.precision import ( @@ -35,9 +44,6 @@ class LTX2AVDecodingStage(DecodingStage): # Replaces the convolutional decoder; latents and denormalization are # identical either way. self.diffusion_decoder = diffusion_decoder - # Add video processor for postprocessing - from diffusers.video_processor import VideoProcessor - self.video_processor = VideoProcessor(vae_scale_factor=32) def component_uses( @@ -101,7 +107,65 @@ class LTX2AVDecodingStage(DecodingStage): def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: self.load_model() + output_batch = OutputBatch( + output=None, + trajectory_timesteps=batch.trajectory_timesteps, + trajectory_latents=batch.trajectory_latents, + trajectory_decoded=None, + metrics=batch.metrics, + rollout_trajectory_data=batch.rollout_trajectory_data, + ) + audio_first = ( + not batch.use_diffusion_decoder + and not batch.return_trajectory_decoded + and isinstance(self.vae, ParallelTiledVAE) + and self.vae.supports_decode_on_frames + and can_encode_while_decoding( + batch, server_args, int(batch.latents.shape[0]) + ) + ) + save_path = ( + streamed_video_output_path(batch, server_args, int(batch.latents.shape[0])) + if audio_first + else None + ) + # All ranks use the same order. Raw-frame and unsupported requests keep + # the original video-before-audio order. + if audio_first: + self._decode_audio(batch, server_args, output_batch) + stream = None + if save_path is not None: + # ffmpeg needs the audio input before it can consume the first video frames + stream = EncodeWhileDecoding( + save_path, + batch, + finish_frames=lambda frames: (frames / 2 + 0.5).clamp(0, 1).float(), + audio=output_batch.audio, + audio_sample_rate=output_batch.audio_sample_rate, + ) + try: + video = self._decode_video(batch, server_args, on_frames=stream) + output_paths = stream.finish() if stream is not None else None + except BaseException: + if stream is not None: + stream.abort() + raise + if output_paths is not None: + output_batch.output_file_paths = output_paths + else: + output_batch.output = self.video_processor.postprocess_video( + video, output_type="np" + ) + if not audio_first: + self._decode_audio(batch, server_args, output_batch) + return output_batch + def _decode_video( + self, + batch: Req, + server_args: ServerArgs, + on_frames: Callable[[torch.Tensor], None] | None = None, + ) -> torch.Tensor: vae_dtype = resolve_decode_precision(server_args, "vae") vae_autocast_enabled = autocast_enabled(vae_dtype, server_args.disable_autocast) @@ -151,7 +215,10 @@ class LTX2AVDecodingStage(DecodingStage): with temporary_module_dtype( self.vae, vae_dtype, enabled=should_cast_vae ) as vae: - decode_output = vae.decode(latents) + decode_kwargs = ( + {} if on_frames is None else {"on_frames": on_frames} + ) + decode_output = vae.decode(latents, **decode_kwargs) if isinstance(decode_output, tuple): video = decode_output[0] @@ -159,18 +226,11 @@ class LTX2AVDecodingStage(DecodingStage): video = decode_output else: video = decode_output.sample - video = self.video_processor.postprocess_video(video, output_type="np") + return video - output_batch = OutputBatch( - output=video, - trajectory_timesteps=batch.trajectory_timesteps, - trajectory_latents=batch.trajectory_latents, - trajectory_decoded=None, - metrics=batch.metrics, - rollout_trajectory_data=batch.rollout_trajectory_data, - ) - - # 2. Decode Audio + def _decode_audio( + self, batch: Req, server_args: ServerArgs, output_batch: OutputBatch + ): try: audio_latents = batch.audio_latents except AttributeError: @@ -287,5 +347,3 @@ class LTX2AVDecodingStage(DecodingStage): output_batch.audio_sample_rate = ( vocoder_sr or audio_vae_sr or pipeline_audio_sr ) - - return output_batch diff --git a/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py b/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py new file mode 100644 index 0000000000..2d71425ef4 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_ltx_vae_on_frames.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 +"""LTX's frame callback must preserve the existing decode and final crop.""" + +from types import SimpleNamespace +from unittest import mock + +import pytest +import torch +from diffusers.video_processor import VideoProcessor + +from sglang.multimodal_gen.configs.models.vaes.ltx_video import ( + LTXVideoVAEArchConfig, + LTXVideoVAEConfig, +) +from sglang.multimodal_gen.runtime.entrypoints.utils import ( + _sample_to_uint8_frames, + _save_video_ffmpeg, + _try_save_video_with_audio, +) +from sglang.multimodal_gen.runtime.models.vaes.ltx_2_vae import AutoencoderKLLTX2Video +from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( + EncodeWhileDecoding, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.decoding_av import ( + LTX2AVDecodingStage, +) + + +def tiny_vae(device="cpu", dtype=torch.float32): + torch.manual_seed(0) + arch = LTXVideoVAEArchConfig( + latent_channels=4, + spatial_compression_ratio=2, + temporal_compression_ratio=2, + block_out_channels=[8], + down_block_types=["LTX2VideoDownBlock3D"], + spatio_temporal_scaling=[True], + layers_per_block=[1, 1], + downsample_type=["spatiotemporal"], + patch_size=1, + decoder_block_out_channels=[8], + decoder_spatio_temporal_scaling=[True], + decoder_layers_per_block=[1, 1], + decoder_inject_noise=[False, False], + upsample_residual=[True], + upsample_factor=[2], + ) + vae = AutoencoderKLLTX2Video(LTXVideoVAEConfig(arch_config=arch)) + vae.tile_sample_min_num_frames = 8 + vae.tile_sample_stride_num_frames = 4 + return vae.to(device=device, dtype=dtype).eval() + + +def legacy_temporal_decode(vae, z): + # Preserve the original two-loop algorithm, independently of the new callback. + ratio = vae.temporal_compression_ratio + stride = vae.tile_sample_stride_num_frames + rows = [] + for start in range(0, z.shape[2], stride // ratio): + tile = vae.decoder(z[:, :, start : start + 8 // ratio + 1], None) + rows.append(tile if start == 0 else tile[:, :, :-1]) + kept = [] + for index, tile in enumerate(rows): + if index: + tile = vae.blend_t(rows[index - 1], tile, 8 - stride) + kept.append(tile[:, :, : stride + (index == 0)]) + return torch.cat(kept, dim=2)[:, :, : (z.shape[2] - 1) * ratio + 1] + + +@pytest.mark.parametrize("batch_size,slicing", [(1, False), (2, False), (2, True)]) +@pytest.mark.parametrize("temporal", [False, True]) +@torch.no_grad() +def test_frames_are_final_and_match_legacy_decode(batch_size, slicing, temporal): + vae = tiny_vae() + vae.use_framewise_decoding = temporal + vae.use_slicing = slicing + z = torch.randn(batch_size, 4, 7, 4, 4) + if temporal: + expected = ( + torch.cat([legacy_temporal_decode(vae, sample) for sample in z.split(1)]) + if slicing + else legacy_temporal_decode(vae, z) + ) + else: + expected = vae.decode(z).sample + chunks = [] + snapshots = [] + + def consume(frames): + chunks.append(frames) + snapshots.append(frames.clone()) + + actual = vae.decode(z, return_dict=False, on_frames=consume)[0] + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + torch.testing.assert_close(torch.cat(chunks, dim=2), actual, rtol=0, atol=0) + for frames, snapshot in zip(chunks, snapshots): + torch.testing.assert_close(frames, snapshot, rtol=0, atol=0) + assert len(chunks) > 1 if temporal and batch_size == 1 else len(chunks) == 1 + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA encoder") +@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16]) +@pytest.mark.parametrize("audio", [False, True]) +@torch.no_grad() +def test_stream_matches_cpu_postprocessing(tmp_path, dtype, audio): + vae = tiny_vae("cuda", dtype) + vae.use_framewise_decoding = True + z = torch.randn(1, 4, 7, 8, 8, device="cuda", dtype=dtype) + cpu_path, stream_path = tmp_path / "cpu.mp4", tmp_path / "stream.mp4" + sound = torch.zeros(1, 13000) if audio else None + request = SimpleNamespace( + fps=13, num_frames=13, output_compression=None, x264_preset=None + ) + stream = EncodeWhileDecoding( + str(stream_path), + request, + finish_frames=lambda frames: (frames / 2 + 0.5).clamp(0, 1).float(), + audio=sound, + audio_sample_rate=13000 if audio else None, + ) + decoded = vae.decode(z, on_frames=stream).sample + assert stream.finish() == [str(stream_path)] + frames = _sample_to_uint8_frames( + VideoProcessor().postprocess_video(decoded, output_type="np")[0] + ) + if audio: + assert _try_save_video_with_audio( + save_file_path=str(cpu_path), + frames=frames, + fps=13, + audio=sound, + audio_sample_rate=13000, + quality=5, + ) + else: + _save_video_ffmpeg(str(cpu_path), frames, fps=13, quality=5) + assert cpu_path.read_bytes() == stream_path.read_bytes() + + +@pytest.mark.parametrize( + "eligible,path,saved", + [ + (False, None, None), + (True, None, None), + (True, "/tmp/clip.mp4", None), + (True, "/tmp/clip.mp4", ["/tmp/clip.mp4"]), + ], +) +def test_audio_order_and_encode_fallback(eligible, path, saved): + stage = object.__new__(LTX2AVDecodingStage) + stage.vae = tiny_vae() + stage.video_processor = VideoProcessor() + calls = [] + video = torch.zeros(1, 3, 1, 8, 8) + request = SimpleNamespace( + latents=video, + use_diffusion_decoder=False, + return_trajectory_decoded=False, + trajectory_timesteps=None, + trajectory_latents=None, + metrics=None, + rollout_trajectory_data=None, + ) + module = LTX2AVDecodingStage.__module__ + encoder = mock.Mock() + encoder.finish.return_value = saved + with ( + mock.patch.object(stage, "load_model"), + mock.patch.object( + stage, "_decode_audio", side_effect=lambda *args: calls.append("audio") + ), + mock.patch.object( + stage, + "_decode_video", + side_effect=lambda *args, **kwargs: (calls.append("video"), video)[1], + ), + mock.patch(f"{module}.can_encode_while_decoding", return_value=eligible), + mock.patch(f"{module}.streamed_video_output_path", return_value=path), + mock.patch(f"{module}.EncodeWhileDecoding", return_value=encoder), + ): + result = stage.forward(request, None) + assert calls == (["audio", "video"] if eligible else ["video", "audio"]) + if saved is not None: + assert result.output is None + assert result.output_file_paths == saved + else: + assert result.output.shape == (1, 1, 8, 8, 3) diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py b/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py index b371e9ad8d..d1bdbe9cde 100644 --- a/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py +++ b/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py @@ -14,13 +14,21 @@ from unittest import mock import torch +from sglang.multimodal_gen.configs.models.vaes.hunyuanvae import ( + HunyuanVAEArchConfig, + HunyuanVAEConfig, +) from sglang.multimodal_gen.configs.models.vaes.wanvae import ( WanVAEArchConfig, WanVAEConfig, ) +from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import HunyuanConfig from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig from sglang.multimodal_gen.configs.sample.sampling_params import DataType from sglang.multimodal_gen.runtime.entrypoints.utils import _try_save_cuda_video_direct +from sglang.multimodal_gen.runtime.models.vaes.hunyuanvae import ( + AutoencoderKLHunyuanVideo, +) from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( @@ -100,6 +108,44 @@ def _video_request(**overrides) -> SimpleNamespace: "needs CUDA and memfd", ) class TestStreamedEncode(CustomTestCase): + @torch.no_grad() + def test_hunyuan_without_callback_matches_one_shot_mp4(self): + config = HunyuanVAEConfig( + arch_config=HunyuanVAEArchConfig( + latent_channels=4, + block_out_channels=(8, 8, 8), + down_block_types=("HunyuanVideoDownBlock3D",) * 3, + up_block_types=("HunyuanVideoUpBlock3D",) * 3, + layers_per_block=1, + norm_num_groups=4, + temporal_compression_ratio=4, + mid_block_add_attention=False, + ), + load_encoder=False, + ) + server_args = _server_args(HunyuanConfig(vae_config=config)) + stage = _stage(AutoencoderKLHunyuanVideo(config).cuda().eval(), server_args) + with tempfile.TemporaryDirectory() as tmp: + streamed, one_shot = Path(tmp, "stream.mp4"), Path(tmp, "normal.mp4") + stream = EncodeWhileDecoding(str(streamed), _video_request()) + frames = stage.decode( + torch.randn(1, 4, 2, 10, 12, device="cuda"), + server_args, + vae_dtype=torch.float32, + on_frames=stream, + ) + self.assertEqual(stream.finish(), [str(streamed)]) + self.assertTrue( + _try_save_cuda_video_direct( + save_file_path=str(one_shot), + sample=frames[0], + fps=FPS, + audio_sample_rate=None, + output_compression=None, + ) + ) + self.assertEqual(one_shot.read_bytes(), streamed.read_bytes()) + @torch.no_grad() def test_streamed_mp4_matches_the_one_shot_save(self): generator = torch.Generator(device="cuda").manual_seed(0) @@ -182,13 +228,39 @@ class TestStreamedEncode(CustomTestCase): SimpleNamespace(**{**vars(server_args), "enable_torch_compile": True}), ), ("stage_decode", _stage(stage.vae, server_args, OwnDecode), server_args), - ("vae", _stage(torch.nn.Identity(), server_args), server_args), ): with self.subTest(name=name): self.assertIsNone( candidate._encode_while_decoding(_video_request(), args) ) + @torch.no_grad() + def test_decoder_without_callback_hands_over_the_complete_clip(self): + config = _vae_config(wan22=False) + server_args = _server_args(WanT2V480PConfig(vae_config=config)) + stage = _stage(_vae(config, "cuda"), server_args) + # The shared stage must not inject a keyword into an ordinary decoder. + stage.vae.supports_decode_on_frames = False + self.assertIsNone(stage._encode_while_decoding(_video_request(), server_args)) + expected = torch.randn(1, 3, 5, 40, 48, device="cuda") + decode = mock.Mock(side_effect=lambda latents: expected) + for output in (expected, expected[:, :, 0]): + with self.subTest(shape=output.shape): + decode.side_effect = lambda latents: output + received = [] + with mock.patch.object( + stage, "_get_vae_decode_fn", return_value=decode + ): + actual = stage.decode( + torch.zeros(1, Z_DIM, 2, 2, 2, device="cuda"), + server_args, + vae_dtype=torch.float32, + on_frames=received.append, + ) + self.assertEqual(len(received), int(output.ndim == 5)) + if received: + torch.testing.assert_close(received[0], actual, rtol=0, atol=0) + if __name__ == "__main__": unittest.main(verbosity=3)