[diffusion] perf: encode LTX video/audio through the shared bounded MP4 path (#43656)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
from collections.abc import Callable
|
||||
from functools import lru_cache
|
||||
from typing import Optional, Tuple, Union
|
||||
|
||||
@@ -1618,6 +1619,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = False
|
||||
supports_decode_on_frames = True
|
||||
|
||||
def __init__(self, config: LTXVideoVAEConfig):
|
||||
super().__init__(config=config)
|
||||
@@ -1896,6 +1898,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
temb: Optional[torch.Tensor] = None,
|
||||
causal: Optional[bool] = None,
|
||||
return_dict: bool = True,
|
||||
on_frames: Callable[[torch.Tensor], None] | None = None,
|
||||
) -> Union[DecoderOutput, torch.Tensor]:
|
||||
batch_size, num_channels, num_frames, height, width = z.shape
|
||||
tile_latent_min_height = (
|
||||
@@ -1910,7 +1913,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
|
||||
if self.use_framewise_decoding and num_frames > tile_latent_min_num_frames:
|
||||
return self._temporal_tiled_decode(
|
||||
z, temb, causal=causal, return_dict=return_dict
|
||||
z, temb, causal=causal, return_dict=return_dict, on_frames=on_frames
|
||||
)
|
||||
|
||||
if self.use_tiling and (
|
||||
@@ -1948,6 +1951,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
temb: Optional[torch.Tensor] = None,
|
||||
causal: Optional[bool] = None,
|
||||
return_dict: bool = True,
|
||||
on_frames: Callable[[torch.Tensor], None] | None = None,
|
||||
) -> Union[DecoderOutput, torch.Tensor]:
|
||||
"""
|
||||
Decode a batch of images.
|
||||
@@ -1975,7 +1979,17 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
]
|
||||
decoded = torch.cat(decoded_slices)
|
||||
else:
|
||||
decoded = self._decode(z, temb, causal=causal).sample
|
||||
decoded = self._decode(z, temb, causal=causal, on_frames=on_frames).sample
|
||||
|
||||
tile_latent_frames = (
|
||||
self.tile_sample_min_num_frames // self.temporal_compression_ratio
|
||||
)
|
||||
if on_frames is not None and not (
|
||||
self.use_framewise_decoding
|
||||
and z.shape[2] > tile_latent_frames
|
||||
and z.shape[0] == 1
|
||||
):
|
||||
on_frames(decoded)
|
||||
|
||||
if not return_dict:
|
||||
return (decoded,)
|
||||
@@ -2163,6 +2177,7 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
temb: Optional[torch.Tensor],
|
||||
causal: Optional[bool] = None,
|
||||
return_dict: bool = True,
|
||||
on_frames: Callable[[torch.Tensor], None] | None = None,
|
||||
) -> Union[DecoderOutput, torch.Tensor]:
|
||||
batch_size, num_channels, num_frames, height, width = z.shape
|
||||
num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1
|
||||
@@ -2183,7 +2198,9 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames
|
||||
)
|
||||
|
||||
row = []
|
||||
result_row = []
|
||||
previous = None
|
||||
emitted = 0
|
||||
for i in range(0, num_frames, tile_latent_stride_num_frames):
|
||||
tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :]
|
||||
if self.use_tiling and (
|
||||
@@ -2197,18 +2214,16 @@ class AutoencoderKLLTX2Video(ParallelTiledVAE):
|
||||
decoded = self.decoder(tile, temb, causal=causal)
|
||||
if i > 0:
|
||||
decoded = decoded[:, :, :-1, :, :]
|
||||
row.append(decoded)
|
||||
|
||||
result_row = []
|
||||
for i, tile in enumerate(row):
|
||||
if i > 0:
|
||||
tile = self.blend_t(row[i - 1], tile, blend_num_frames)
|
||||
tile = tile[:, :, : self.tile_sample_stride_num_frames, :, :]
|
||||
result_row.append(tile)
|
||||
else:
|
||||
result_row.append(
|
||||
tile[:, :, : self.tile_sample_stride_num_frames + 1, :, :]
|
||||
)
|
||||
decoded = self.blend_t(previous, decoded, blend_num_frames)
|
||||
previous = decoded
|
||||
kept = decoded[:, :, : self.tile_sample_stride_num_frames + (i == 0)]
|
||||
result_row.append(kept)
|
||||
# only the kept prefix is final; the tail is still needed to blend the next tile
|
||||
if on_frames is not None and batch_size == 1:
|
||||
frames = kept[:, :, : max(0, num_sample_frames - emitted)]
|
||||
if frames.shape[2]:
|
||||
on_frames(frames)
|
||||
emitted += frames.shape[2]
|
||||
|
||||
dec = torch.cat(result_row, dim=2)[:, :, :num_sample_frames]
|
||||
|
||||
|
||||
@@ -229,8 +229,8 @@ class DecodingStage(PipelineStage):
|
||||
VAE precision ("fp32", "fp16", "bf16")
|
||||
- pipeline_config.vae_precision: fallback VAE precision
|
||||
- pipeline_config.vae_tiling: Whether to enable VAE tiling for memory efficiency
|
||||
on_frames: For a VAE with ``supports_decode_on_frames``, receives the
|
||||
returned frames once, in order, in the pieces the VAE finishes them.
|
||||
on_frames: Receives finished video frames in order. VAEs without a
|
||||
frame callback hand over the complete decoded clip once.
|
||||
|
||||
Returns:
|
||||
Decoded video tensor with shape (batch, channels, frames, height, width),
|
||||
@@ -239,6 +239,8 @@ class DecodingStage(PipelineStage):
|
||||
normalized_on_frames = (
|
||||
(lambda frames: on_frames(_to_unit_range(frames)))
|
||||
if on_frames is not None
|
||||
and isinstance(self.vae, ParallelTiledVAE)
|
||||
and self.vae.supports_decode_on_frames
|
||||
else None
|
||||
)
|
||||
image = self.decode_raw(
|
||||
@@ -247,7 +249,15 @@ class DecodingStage(PipelineStage):
|
||||
vae_dtype=vae_dtype,
|
||||
on_frames=normalized_on_frames,
|
||||
)
|
||||
return _to_unit_range(image)
|
||||
image = _to_unit_range(image)
|
||||
if (
|
||||
on_frames is not None
|
||||
and normalized_on_frames is None
|
||||
and image.ndim == 5
|
||||
and image.shape[:2] == (1, 3)
|
||||
):
|
||||
on_frames(image)
|
||||
return image
|
||||
|
||||
@torch.no_grad()
|
||||
def decode_raw(
|
||||
@@ -292,7 +302,11 @@ class DecodingStage(PipelineStage):
|
||||
if not vae_autocast_enabled:
|
||||
latents = latents.to(vae_dtype)
|
||||
decode_kwargs = {}
|
||||
if on_frames is not None:
|
||||
streams_frames = (
|
||||
isinstance(self.vae, ParallelTiledVAE)
|
||||
and self.vae.supports_decode_on_frames
|
||||
)
|
||||
if on_frames is not None and streams_frames:
|
||||
decode_kwargs["on_frames"] = on_frames
|
||||
with temporary_module_dtype(
|
||||
self.vae, vae_dtype, enabled=should_cast_vae
|
||||
@@ -325,6 +339,14 @@ class DecodingStage(PipelineStage):
|
||||
raise
|
||||
image = _ensure_tensor_decode_output(decode_output)
|
||||
|
||||
if (
|
||||
on_frames is not None
|
||||
and not streams_frames
|
||||
and image.ndim == 5
|
||||
and image.shape[:2] == (1, 3)
|
||||
):
|
||||
on_frames(image)
|
||||
|
||||
return image
|
||||
|
||||
def load_model(self):
|
||||
|
||||
+28
-13
@@ -25,11 +25,11 @@ logger = init_logger(__name__)
|
||||
_STREAMED_VIDEO_QUEUE_BYTES = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def streamed_video_output_path(
|
||||
def can_encode_while_decoding(
|
||||
batch: Req, server_args: ServerArgs, batch_size: int
|
||||
) -> str | None:
|
||||
"""Where the output rank can write the MP4 during the decode, if anywhere."""
|
||||
if not (
|
||||
) -> bool:
|
||||
"""Rank-independent eligibility, including ranks that only decode audio/video."""
|
||||
return (
|
||||
batch.save_output
|
||||
and batch.return_file_paths_only
|
||||
and not batch.return_raw_frames
|
||||
@@ -42,7 +42,14 @@ def streamed_video_output_path(
|
||||
and type(server_args.pipeline_config).post_decoding
|
||||
is PipelineConfig.post_decoding
|
||||
and current_platform.is_cuda()
|
||||
):
|
||||
)
|
||||
|
||||
|
||||
def streamed_video_output_path(
|
||||
batch: Req, server_args: ServerArgs, batch_size: int
|
||||
) -> str | None:
|
||||
"""Where the output rank can write the MP4 during the decode, if anywhere."""
|
||||
if not can_encode_while_decoding(batch, server_args, batch_size):
|
||||
return None
|
||||
if model_parallel_is_initialized() and get_replica_group().rank_in_group != 0:
|
||||
return None
|
||||
@@ -73,15 +80,23 @@ class EncodeWhileDecoding:
|
||||
if self._failed:
|
||||
return
|
||||
try:
|
||||
if self._finish_frames is not None:
|
||||
frames = self._finish_frames(frames)
|
||||
frames = frames[0]
|
||||
if self._encoder is None:
|
||||
self._encoder = self._open(frames)
|
||||
# Untiled decoders can hand over an entire clip. Bound FP32
|
||||
# postprocessing too, not just the queued uint8 frame copies.
|
||||
height, width = frames.shape[-2:]
|
||||
float_frame_bytes = 3 * height * width * 4
|
||||
chunk_frames = max(
|
||||
1, (_STREAMED_VIDEO_QUEUE_BYTES // 4) // float_frame_bytes
|
||||
)
|
||||
for chunk in frames.split(chunk_frames, dim=2):
|
||||
if self._finish_frames is not None:
|
||||
chunk = self._finish_frames(chunk)
|
||||
chunk = chunk[0]
|
||||
if self._encoder is None:
|
||||
self._failed = True
|
||||
return
|
||||
self._encoder.write(frames)
|
||||
self._encoder = self._open(chunk)
|
||||
if self._encoder is None:
|
||||
self._failed = True
|
||||
return
|
||||
self._encoder.write(chunk)
|
||||
except Exception:
|
||||
self._fail()
|
||||
|
||||
|
||||
+75
-17
@@ -1,14 +1,23 @@
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
from sglang.multimodal_gen.runtime.distributed import get_local_torch_device
|
||||
from sglang.multimodal_gen.runtime.managers.memory_managers.component_manager import (
|
||||
ComponentUse,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.common import ParallelTiledVAE
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch, Req
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
|
||||
StageParallelismType,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
EncodeWhileDecoding,
|
||||
can_encode_while_decoding,
|
||||
streamed_video_output_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
from sglang.multimodal_gen.runtime.utils.precision import (
|
||||
@@ -35,9 +44,6 @@ class LTX2AVDecodingStage(DecodingStage):
|
||||
# Replaces the convolutional decoder; latents and denormalization are
|
||||
# identical either way.
|
||||
self.diffusion_decoder = diffusion_decoder
|
||||
# Add video processor for postprocessing
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
self.video_processor = VideoProcessor(vae_scale_factor=32)
|
||||
|
||||
def component_uses(
|
||||
@@ -101,7 +107,65 @@ class LTX2AVDecodingStage(DecodingStage):
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
self.load_model()
|
||||
output_batch = OutputBatch(
|
||||
output=None,
|
||||
trajectory_timesteps=batch.trajectory_timesteps,
|
||||
trajectory_latents=batch.trajectory_latents,
|
||||
trajectory_decoded=None,
|
||||
metrics=batch.metrics,
|
||||
rollout_trajectory_data=batch.rollout_trajectory_data,
|
||||
)
|
||||
audio_first = (
|
||||
not batch.use_diffusion_decoder
|
||||
and not batch.return_trajectory_decoded
|
||||
and isinstance(self.vae, ParallelTiledVAE)
|
||||
and self.vae.supports_decode_on_frames
|
||||
and can_encode_while_decoding(
|
||||
batch, server_args, int(batch.latents.shape[0])
|
||||
)
|
||||
)
|
||||
save_path = (
|
||||
streamed_video_output_path(batch, server_args, int(batch.latents.shape[0]))
|
||||
if audio_first
|
||||
else None
|
||||
)
|
||||
# All ranks use the same order. Raw-frame and unsupported requests keep
|
||||
# the original video-before-audio order.
|
||||
if audio_first:
|
||||
self._decode_audio(batch, server_args, output_batch)
|
||||
stream = None
|
||||
if save_path is not None:
|
||||
# ffmpeg needs the audio input before it can consume the first video frames
|
||||
stream = EncodeWhileDecoding(
|
||||
save_path,
|
||||
batch,
|
||||
finish_frames=lambda frames: (frames / 2 + 0.5).clamp(0, 1).float(),
|
||||
audio=output_batch.audio,
|
||||
audio_sample_rate=output_batch.audio_sample_rate,
|
||||
)
|
||||
try:
|
||||
video = self._decode_video(batch, server_args, on_frames=stream)
|
||||
output_paths = stream.finish() if stream is not None else None
|
||||
except BaseException:
|
||||
if stream is not None:
|
||||
stream.abort()
|
||||
raise
|
||||
if output_paths is not None:
|
||||
output_batch.output_file_paths = output_paths
|
||||
else:
|
||||
output_batch.output = self.video_processor.postprocess_video(
|
||||
video, output_type="np"
|
||||
)
|
||||
if not audio_first:
|
||||
self._decode_audio(batch, server_args, output_batch)
|
||||
return output_batch
|
||||
|
||||
def _decode_video(
|
||||
self,
|
||||
batch: Req,
|
||||
server_args: ServerArgs,
|
||||
on_frames: Callable[[torch.Tensor], None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
vae_dtype = resolve_decode_precision(server_args, "vae")
|
||||
vae_autocast_enabled = autocast_enabled(vae_dtype, server_args.disable_autocast)
|
||||
|
||||
@@ -151,7 +215,10 @@ class LTX2AVDecodingStage(DecodingStage):
|
||||
with temporary_module_dtype(
|
||||
self.vae, vae_dtype, enabled=should_cast_vae
|
||||
) as vae:
|
||||
decode_output = vae.decode(latents)
|
||||
decode_kwargs = (
|
||||
{} if on_frames is None else {"on_frames": on_frames}
|
||||
)
|
||||
decode_output = vae.decode(latents, **decode_kwargs)
|
||||
|
||||
if isinstance(decode_output, tuple):
|
||||
video = decode_output[0]
|
||||
@@ -159,18 +226,11 @@ class LTX2AVDecodingStage(DecodingStage):
|
||||
video = decode_output
|
||||
else:
|
||||
video = decode_output.sample
|
||||
video = self.video_processor.postprocess_video(video, output_type="np")
|
||||
return video
|
||||
|
||||
output_batch = OutputBatch(
|
||||
output=video,
|
||||
trajectory_timesteps=batch.trajectory_timesteps,
|
||||
trajectory_latents=batch.trajectory_latents,
|
||||
trajectory_decoded=None,
|
||||
metrics=batch.metrics,
|
||||
rollout_trajectory_data=batch.rollout_trajectory_data,
|
||||
)
|
||||
|
||||
# 2. Decode Audio
|
||||
def _decode_audio(
|
||||
self, batch: Req, server_args: ServerArgs, output_batch: OutputBatch
|
||||
):
|
||||
try:
|
||||
audio_latents = batch.audio_latents
|
||||
except AttributeError:
|
||||
@@ -287,5 +347,3 @@ class LTX2AVDecodingStage(DecodingStage):
|
||||
output_batch.audio_sample_rate = (
|
||||
vocoder_sr or audio_vae_sr or pipeline_audio_sr
|
||||
)
|
||||
|
||||
return output_batch
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""LTX's frame callback must preserve the existing decode and final crop."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from diffusers.video_processor import VideoProcessor
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.ltx_video import (
|
||||
LTXVideoVAEArchConfig,
|
||||
LTXVideoVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
||||
_sample_to_uint8_frames,
|
||||
_save_video_ffmpeg,
|
||||
_try_save_video_with_audio,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.ltx_2_vae import AutoencoderKLLTX2Video
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
EncodeWhileDecoding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.ltx_2.decoding_av import (
|
||||
LTX2AVDecodingStage,
|
||||
)
|
||||
|
||||
|
||||
def tiny_vae(device="cpu", dtype=torch.float32):
|
||||
torch.manual_seed(0)
|
||||
arch = LTXVideoVAEArchConfig(
|
||||
latent_channels=4,
|
||||
spatial_compression_ratio=2,
|
||||
temporal_compression_ratio=2,
|
||||
block_out_channels=[8],
|
||||
down_block_types=["LTX2VideoDownBlock3D"],
|
||||
spatio_temporal_scaling=[True],
|
||||
layers_per_block=[1, 1],
|
||||
downsample_type=["spatiotemporal"],
|
||||
patch_size=1,
|
||||
decoder_block_out_channels=[8],
|
||||
decoder_spatio_temporal_scaling=[True],
|
||||
decoder_layers_per_block=[1, 1],
|
||||
decoder_inject_noise=[False, False],
|
||||
upsample_residual=[True],
|
||||
upsample_factor=[2],
|
||||
)
|
||||
vae = AutoencoderKLLTX2Video(LTXVideoVAEConfig(arch_config=arch))
|
||||
vae.tile_sample_min_num_frames = 8
|
||||
vae.tile_sample_stride_num_frames = 4
|
||||
return vae.to(device=device, dtype=dtype).eval()
|
||||
|
||||
|
||||
def legacy_temporal_decode(vae, z):
|
||||
# Preserve the original two-loop algorithm, independently of the new callback.
|
||||
ratio = vae.temporal_compression_ratio
|
||||
stride = vae.tile_sample_stride_num_frames
|
||||
rows = []
|
||||
for start in range(0, z.shape[2], stride // ratio):
|
||||
tile = vae.decoder(z[:, :, start : start + 8 // ratio + 1], None)
|
||||
rows.append(tile if start == 0 else tile[:, :, :-1])
|
||||
kept = []
|
||||
for index, tile in enumerate(rows):
|
||||
if index:
|
||||
tile = vae.blend_t(rows[index - 1], tile, 8 - stride)
|
||||
kept.append(tile[:, :, : stride + (index == 0)])
|
||||
return torch.cat(kept, dim=2)[:, :, : (z.shape[2] - 1) * ratio + 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("batch_size,slicing", [(1, False), (2, False), (2, True)])
|
||||
@pytest.mark.parametrize("temporal", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_frames_are_final_and_match_legacy_decode(batch_size, slicing, temporal):
|
||||
vae = tiny_vae()
|
||||
vae.use_framewise_decoding = temporal
|
||||
vae.use_slicing = slicing
|
||||
z = torch.randn(batch_size, 4, 7, 4, 4)
|
||||
if temporal:
|
||||
expected = (
|
||||
torch.cat([legacy_temporal_decode(vae, sample) for sample in z.split(1)])
|
||||
if slicing
|
||||
else legacy_temporal_decode(vae, z)
|
||||
)
|
||||
else:
|
||||
expected = vae.decode(z).sample
|
||||
chunks = []
|
||||
snapshots = []
|
||||
|
||||
def consume(frames):
|
||||
chunks.append(frames)
|
||||
snapshots.append(frames.clone())
|
||||
|
||||
actual = vae.decode(z, return_dict=False, on_frames=consume)[0]
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
torch.testing.assert_close(torch.cat(chunks, dim=2), actual, rtol=0, atol=0)
|
||||
for frames, snapshot in zip(chunks, snapshots):
|
||||
torch.testing.assert_close(frames, snapshot, rtol=0, atol=0)
|
||||
assert len(chunks) > 1 if temporal and batch_size == 1 else len(chunks) == 1
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA encoder")
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
@pytest.mark.parametrize("audio", [False, True])
|
||||
@torch.no_grad()
|
||||
def test_stream_matches_cpu_postprocessing(tmp_path, dtype, audio):
|
||||
vae = tiny_vae("cuda", dtype)
|
||||
vae.use_framewise_decoding = True
|
||||
z = torch.randn(1, 4, 7, 8, 8, device="cuda", dtype=dtype)
|
||||
cpu_path, stream_path = tmp_path / "cpu.mp4", tmp_path / "stream.mp4"
|
||||
sound = torch.zeros(1, 13000) if audio else None
|
||||
request = SimpleNamespace(
|
||||
fps=13, num_frames=13, output_compression=None, x264_preset=None
|
||||
)
|
||||
stream = EncodeWhileDecoding(
|
||||
str(stream_path),
|
||||
request,
|
||||
finish_frames=lambda frames: (frames / 2 + 0.5).clamp(0, 1).float(),
|
||||
audio=sound,
|
||||
audio_sample_rate=13000 if audio else None,
|
||||
)
|
||||
decoded = vae.decode(z, on_frames=stream).sample
|
||||
assert stream.finish() == [str(stream_path)]
|
||||
frames = _sample_to_uint8_frames(
|
||||
VideoProcessor().postprocess_video(decoded, output_type="np")[0]
|
||||
)
|
||||
if audio:
|
||||
assert _try_save_video_with_audio(
|
||||
save_file_path=str(cpu_path),
|
||||
frames=frames,
|
||||
fps=13,
|
||||
audio=sound,
|
||||
audio_sample_rate=13000,
|
||||
quality=5,
|
||||
)
|
||||
else:
|
||||
_save_video_ffmpeg(str(cpu_path), frames, fps=13, quality=5)
|
||||
assert cpu_path.read_bytes() == stream_path.read_bytes()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"eligible,path,saved",
|
||||
[
|
||||
(False, None, None),
|
||||
(True, None, None),
|
||||
(True, "/tmp/clip.mp4", None),
|
||||
(True, "/tmp/clip.mp4", ["/tmp/clip.mp4"]),
|
||||
],
|
||||
)
|
||||
def test_audio_order_and_encode_fallback(eligible, path, saved):
|
||||
stage = object.__new__(LTX2AVDecodingStage)
|
||||
stage.vae = tiny_vae()
|
||||
stage.video_processor = VideoProcessor()
|
||||
calls = []
|
||||
video = torch.zeros(1, 3, 1, 8, 8)
|
||||
request = SimpleNamespace(
|
||||
latents=video,
|
||||
use_diffusion_decoder=False,
|
||||
return_trajectory_decoded=False,
|
||||
trajectory_timesteps=None,
|
||||
trajectory_latents=None,
|
||||
metrics=None,
|
||||
rollout_trajectory_data=None,
|
||||
)
|
||||
module = LTX2AVDecodingStage.__module__
|
||||
encoder = mock.Mock()
|
||||
encoder.finish.return_value = saved
|
||||
with (
|
||||
mock.patch.object(stage, "load_model"),
|
||||
mock.patch.object(
|
||||
stage, "_decode_audio", side_effect=lambda *args: calls.append("audio")
|
||||
),
|
||||
mock.patch.object(
|
||||
stage,
|
||||
"_decode_video",
|
||||
side_effect=lambda *args, **kwargs: (calls.append("video"), video)[1],
|
||||
),
|
||||
mock.patch(f"{module}.can_encode_while_decoding", return_value=eligible),
|
||||
mock.patch(f"{module}.streamed_video_output_path", return_value=path),
|
||||
mock.patch(f"{module}.EncodeWhileDecoding", return_value=encoder),
|
||||
):
|
||||
result = stage.forward(request, None)
|
||||
assert calls == (["audio", "video"] if eligible else ["video", "audio"])
|
||||
if saved is not None:
|
||||
assert result.output is None
|
||||
assert result.output_file_paths == saved
|
||||
else:
|
||||
assert result.output.shape == (1, 1, 8, 8, 3)
|
||||
@@ -14,13 +14,21 @@ from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.hunyuanvae import (
|
||||
HunyuanVAEArchConfig,
|
||||
HunyuanVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.models.vaes.wanvae import (
|
||||
WanVAEArchConfig,
|
||||
WanVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.hunyuan import HunyuanConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import _try_save_cuda_video_direct
|
||||
from sglang.multimodal_gen.runtime.models.vaes.hunyuanvae import (
|
||||
AutoencoderKLHunyuanVideo,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
@@ -100,6 +108,44 @@ def _video_request(**overrides) -> SimpleNamespace:
|
||||
"needs CUDA and memfd",
|
||||
)
|
||||
class TestStreamedEncode(CustomTestCase):
|
||||
@torch.no_grad()
|
||||
def test_hunyuan_without_callback_matches_one_shot_mp4(self):
|
||||
config = HunyuanVAEConfig(
|
||||
arch_config=HunyuanVAEArchConfig(
|
||||
latent_channels=4,
|
||||
block_out_channels=(8, 8, 8),
|
||||
down_block_types=("HunyuanVideoDownBlock3D",) * 3,
|
||||
up_block_types=("HunyuanVideoUpBlock3D",) * 3,
|
||||
layers_per_block=1,
|
||||
norm_num_groups=4,
|
||||
temporal_compression_ratio=4,
|
||||
mid_block_add_attention=False,
|
||||
),
|
||||
load_encoder=False,
|
||||
)
|
||||
server_args = _server_args(HunyuanConfig(vae_config=config))
|
||||
stage = _stage(AutoencoderKLHunyuanVideo(config).cuda().eval(), server_args)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
streamed, one_shot = Path(tmp, "stream.mp4"), Path(tmp, "normal.mp4")
|
||||
stream = EncodeWhileDecoding(str(streamed), _video_request())
|
||||
frames = stage.decode(
|
||||
torch.randn(1, 4, 2, 10, 12, device="cuda"),
|
||||
server_args,
|
||||
vae_dtype=torch.float32,
|
||||
on_frames=stream,
|
||||
)
|
||||
self.assertEqual(stream.finish(), [str(streamed)])
|
||||
self.assertTrue(
|
||||
_try_save_cuda_video_direct(
|
||||
save_file_path=str(one_shot),
|
||||
sample=frames[0],
|
||||
fps=FPS,
|
||||
audio_sample_rate=None,
|
||||
output_compression=None,
|
||||
)
|
||||
)
|
||||
self.assertEqual(one_shot.read_bytes(), streamed.read_bytes())
|
||||
|
||||
@torch.no_grad()
|
||||
def test_streamed_mp4_matches_the_one_shot_save(self):
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
@@ -182,13 +228,39 @@ class TestStreamedEncode(CustomTestCase):
|
||||
SimpleNamespace(**{**vars(server_args), "enable_torch_compile": True}),
|
||||
),
|
||||
("stage_decode", _stage(stage.vae, server_args, OwnDecode), server_args),
|
||||
("vae", _stage(torch.nn.Identity(), server_args), server_args),
|
||||
):
|
||||
with self.subTest(name=name):
|
||||
self.assertIsNone(
|
||||
candidate._encode_while_decoding(_video_request(), args)
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def test_decoder_without_callback_hands_over_the_complete_clip(self):
|
||||
config = _vae_config(wan22=False)
|
||||
server_args = _server_args(WanT2V480PConfig(vae_config=config))
|
||||
stage = _stage(_vae(config, "cuda"), server_args)
|
||||
# The shared stage must not inject a keyword into an ordinary decoder.
|
||||
stage.vae.supports_decode_on_frames = False
|
||||
self.assertIsNone(stage._encode_while_decoding(_video_request(), server_args))
|
||||
expected = torch.randn(1, 3, 5, 40, 48, device="cuda")
|
||||
decode = mock.Mock(side_effect=lambda latents: expected)
|
||||
for output in (expected, expected[:, :, 0]):
|
||||
with self.subTest(shape=output.shape):
|
||||
decode.side_effect = lambda latents: output
|
||||
received = []
|
||||
with mock.patch.object(
|
||||
stage, "_get_vae_decode_fn", return_value=decode
|
||||
):
|
||||
actual = stage.decode(
|
||||
torch.zeros(1, Z_DIM, 2, 2, 2, device="cuda"),
|
||||
server_args,
|
||||
vae_dtype=torch.float32,
|
||||
on_frames=received.append,
|
||||
)
|
||||
self.assertEqual(len(received), int(output.ndim == 5))
|
||||
if received:
|
||||
torch.testing.assert_close(received[0], actual, rtol=0, atol=0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
|
||||
Reference in New Issue
Block a user