[diffusion] feat: encode the MP4 while the VAE decodes (MiniMax-H3, Wan) (#41819)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-08 09:01:19 +08:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 7760d5264a
commit 9101e895ea
12 changed files with 1196 additions and 236 deletions
@@ -12,6 +12,7 @@ import atexit
import json
import mmap
import os
import queue
import shutil
import subprocess
import tempfile
@@ -62,7 +63,7 @@ _cached_cuda_video_buffer: "_CudaMemfdVideoBuffer | None" = None
class _CudaMemfdVideoBuffer:
"""CUDA-registered memfd used as direct ffmpeg raw-video input."""
"""CUDA-registered memfd mapping that stages raw frames for ffmpeg."""
def __init__(self, shape: tuple[int, ...]):
self.shape = shape
@@ -337,23 +338,335 @@ def _x264_auto_thread_count(height: int) -> int:
return min(cpu_limit, row_limit, 128)
def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int:
_, num_frames, height, width = video.shape
temporary_bytes_per_frame = 3 * height * width * (video.element_size() + 2)
def _conversion_chunk_frames(
num_frames: int, height: int, width: int, element_size: int
) -> int:
temporary_bytes_per_frame = 3 * height * width * (element_size + 2)
return min(
num_frames,
max(1, _MAX_CUDA_VIDEO_CONVERSION_CHUNK_BYTES // temporary_bytes_per_frame),
)
def _sendfile_all(output_fd: int, input_fd: int, count: int) -> None:
offset = 0
while count:
sent = os.sendfile(output_fd, input_fd, offset, count)
if sent <= 0:
raise RuntimeError("sendfile made no progress")
offset += sent
count -= sent
def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int:
_, num_frames, height, width = video.shape
return _conversion_chunk_frames(num_frames, height, width, video.element_size())
def _write_all(fd: int, data: memoryview) -> None:
while data:
data = data[os.write(fd, data) :]
def _remove_quietly(path: Optional[str]) -> None:
if path:
try:
os.remove(path)
except OSError:
pass
def _cuda_video_ffmpeg_command(
save_file_path: str,
*,
width: int,
height: int,
fps: int,
crf: int,
x264_preset: Optional[str],
wav_path: Optional[str],
) -> list[str]:
command = [
_resolve_ffmpeg_exe(),
"-y",
"-f",
"rawvideo",
"-vcodec",
"rawvideo",
"-s",
f"{width}x{height}",
"-pix_fmt",
"rgb24",
"-r",
f"{fps:.02f}",
"-i",
"pipe:0",
]
if wav_path is None:
command += ["-an"]
else:
command += ["-i", wav_path]
command += [
"-vcodec",
"libx264",
"-preset",
x264_preset or X264_PRESET,
"-pix_fmt",
"yuv420p",
"-crf",
str(crf),
]
macro_block_size = 16
if width % macro_block_size or height % macro_block_size:
output_width = (
width
if width % macro_block_size == 0
else width + macro_block_size - width % macro_block_size
)
output_height = (
height
if height % macro_block_size == 0
else height + macro_block_size - height % macro_block_size
)
command += ["-vf", f"scale={output_width}:{output_height}"]
command += ["-threads", str(_x264_auto_thread_count(height))]
if wav_path is not None:
command += [
"-acodec",
"aac",
"-map",
"0:v:0",
"-map",
"1:a:0",
]
command += ["-v", "warning", save_file_path]
return command
class CudaVideoEncoder:
"""Encode [3, T, H, W] CUDA frames in [0, 1] to an MP4 as they arrive.
``write`` only queues the uint8 conversion on the current stream; a writer
thread stages each chunk in registered host memory and pipes it to ffmpeg,
so a caller that produces frames in pieces keeps its own GPU work going.
"""
@classmethod
def open(
cls,
save_file_path: str,
*,
device: torch.device,
height: int,
width: int,
fps: int,
num_frames: int,
element_size: int = 4,
audio: Any = None,
audio_sample_rate: Optional[int] = None,
output_compression: Optional[int] = None,
x264_preset: Optional[str] = None,
max_queued_frames: int = 0,
) -> "CudaVideoEncoder | None":
"""Start ffmpeg, or return None when this output cannot use the path."""
if not hasattr(os, "memfd_create") or device.type != "cuda":
return None
if os.path.splitext(save_file_path)[1].lower() != ".mp4":
return None
quality = output_compression / 10 if output_compression is not None else 5
if not 1 <= quality <= 10:
return None
crf = int((1 - quality / 10.0) * 51)
audio_np = _normalize_audio_to_numpy(audio)
if audio_np is not None and scipy_wavfile is None:
return None
wav_path = None
try:
if audio_np is not None:
selected_sr = _pick_audio_sample_rate(
audio_np=audio_np,
audio_sample_rate=audio_sample_rate,
fps=fps,
num_frames=num_frames,
)
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
wav_path = f.name
scipy_wavfile.write(wav_path, selected_sr, audio_np)
command = _cuda_video_ffmpeg_command(
save_file_path,
width=width,
height=height,
fps=fps,
crf=crf,
x264_preset=x264_preset,
wav_path=wav_path,
)
chunk_frames = _conversion_chunk_frames(
max(1, num_frames), height, width, element_size
)
return cls(
command,
save_file_path,
device=device,
height=height,
width=width,
chunk_frames=chunk_frames,
queued_chunks=max(1, -(-max_queued_frames // chunk_frames)),
wav_path=wav_path,
)
except BaseException:
_remove_quietly(wav_path)
raise
def __init__(
self,
command: list[str],
save_file_path: str,
*,
device: torch.device,
height: int,
width: int,
chunk_frames: int,
queued_chunks: int,
wav_path: Optional[str],
):
self.save_file_path = save_file_path
self._command = command
self._device = device
self._height = height
self._width = width
self._chunk_frames = chunk_frames
self._wav_path = wav_path
self._frames_queued = 0
self._error: BaseException | None = None
self._finished = False
self._queue: queue.Queue = queue.Queue(maxsize=queued_chunks)
self._stderr = tempfile.TemporaryFile()
try:
self._process = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=self._stderr,
)
except BaseException:
self._stderr.close()
raise
self._thread = threading.Thread(
target=self._pipe_frames, name="sglang-video-encoder", daemon=True
)
self._thread.start()
def write(self, frames: torch.Tensor) -> None:
"""Queue [3, t, H, W] frames; raise once the encoder has failed."""
if frames.dim() == 3:
frames = frames.unsqueeze(1)
if (
frames.device != self._device
or int(frames.shape[0]) != 3
or tuple(frames.shape[2:]) != (self._height, self._width)
):
raise ValueError(
f"expected [3, t, {self._height}, {self._width}] frames on "
f"{self._device}, got {tuple(frames.shape)} on {frames.device}"
)
stream = torch.cuda.current_stream(frames.device)
num_frames = int(frames.shape[1])
for start in range(0, num_frames, self._chunk_frames):
end = min(start + self._chunk_frames, num_frames)
first = self._frames_queued
with maybe_record_function(
f"VIDEO_CHUNK frames {first}-{first + end - start} convert"
):
chunk = (frames[:, start:end] * 255).clamp_(0, 255).to(torch.uint8)
chunk = chunk.permute(1, 2, 3, 0).contiguous()
ready = torch.cuda.Event()
ready.record(stream)
self._put((chunk, ready))
self._frames_queued += end - start
def close(self) -> None:
"""Finish the MP4; on failure clean up as ``abort`` does and raise."""
if self._finished:
return
try:
with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"):
self._stop_thread()
if self._error is not None:
if self._process.poll():
raise self._ffmpeg_error() from self._error
raise RuntimeError("video encoder failed") from self._error
self._process.stdin.close()
if self._process.wait():
raise self._ffmpeg_error()
except BaseException:
self.abort()
raise
self._finished = True
self._release()
def abort(self) -> None:
"""Stop ffmpeg and remove the partial file."""
if self._finished:
return
self._finished = True
if self._process.poll() is None:
self._process.kill()
self._stop_thread()
try:
self._process.stdin.close()
except OSError:
pass
self._process.wait()
self._release()
_remove_quietly(self.save_file_path)
def _ffmpeg_error(self) -> subprocess.CalledProcessError:
self._stderr.seek(0)
return subprocess.CalledProcessError(
self._process.returncode, self._command, stderr=self._stderr.read()
)
def _put(self, item) -> None:
# never blocks for good: after a failure the writer keeps draining
if self._error is not None:
raise RuntimeError("video encoder failed") from self._error
self._queue.put(item)
def _stop_thread(self) -> None:
self._queue.put(None)
self._thread.join()
def _pipe_frames(self) -> None:
frame_bytes = self._height * self._width * 3
try:
stdin = self._process.stdin
if stdin is None:
raise RuntimeError("ffmpeg stdin pipe was not created")
with (
torch.cuda.device(self._device),
_acquire_cuda_video_buffer(
(self._chunk_frames, self._height, self._width, 3)
) as buffer,
memoryview(buffer.array).cast("B") as staged,
):
assert buffer.tensor is not None
copy_stream = torch.cuda.Stream()
while (item := self._queue.get()) is not None:
chunk, ready = item
del item
count = int(chunk.shape[0])
copy_stream.wait_event(ready)
with torch.cuda.stream(copy_stream):
buffer.tensor[:count].copy_(chunk, non_blocking=True)
copy_stream.synchronize()
# the copy is done, so the allocator may hand this block out again
del chunk, ready
# write() copies into the pipe; sendfile would only lend it these
# pages, and the next chunk overwrites them before ffmpeg reads
_write_all(stdin.fileno(), staged[: count * frame_bytes])
except BaseException as exc:
self._error = exc
while self._queue.get() is not None:
pass
def _release(self) -> None:
self._stderr.close()
_remove_quietly(self._wav_path)
self._wav_path = None
def _try_save_cuda_video_direct(
@@ -366,173 +679,48 @@ def _try_save_cuda_video_direct(
x264_preset: Optional[str] = None,
) -> bool:
"""Stream CUDA RGB chunks to ffmpeg through a registered memfd."""
if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"):
return False
sample_without_audio, audio = _split_sample_audio(sample)
video, audio = _split_sample_audio(sample)
if not (
isinstance(sample_without_audio, torch.Tensor)
and sample_without_audio.device.type == "cuda"
and sample_without_audio.dim() in (3, 4)
isinstance(video, torch.Tensor)
and video.device.type == "cuda"
and video.dim() in (3, 4)
):
return False
if os.path.splitext(save_file_path)[1].lower() != ".mp4":
return False
video = sample_without_audio
if video.dim() == 3:
video = video.unsqueeze(1)
if video.shape[0] != 3:
return False
_, num_frames, height, width = video.shape
chunk_frames = _cuda_video_conversion_chunk_frames(video)
quality = output_compression / 10 if output_compression is not None else 5
if not 1 <= quality <= 10:
return False
crf = int((1 - quality / 10.0) * 51)
audio_np = _normalize_audio_to_numpy(audio)
tmp_wav_path = None
encoder = None
try:
if audio_np is not None:
if scipy_wavfile is None:
return False
selected_sr = _pick_audio_sample_rate(
audio_np=audio_np,
audio_sample_rate=audio_sample_rate,
fps=fps,
num_frames=num_frames,
)
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
tmp_wav_path = f.name
scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np)
ffmpeg_exe = _resolve_ffmpeg_exe()
command = [
ffmpeg_exe,
"-y",
"-f",
"rawvideo",
"-vcodec",
"rawvideo",
"-s",
f"{width}x{height}",
"-pix_fmt",
"rgb24",
"-r",
f"{fps:.02f}",
"-i",
"pipe:0",
]
if tmp_wav_path is None:
command += ["-an"]
else:
command += ["-i", tmp_wav_path]
command += [
"-vcodec",
"libx264",
"-preset",
x264_preset or X264_PRESET,
"-pix_fmt",
"yuv420p",
"-crf",
str(crf),
]
macro_block_size = 16
if width % macro_block_size or height % macro_block_size:
output_width = (
width
if width % macro_block_size == 0
else width + macro_block_size - width % macro_block_size
)
output_height = (
height
if height % macro_block_size == 0
else height + macro_block_size - height % macro_block_size
)
command += ["-vf", f"scale={output_width}:{output_height}"]
command += ["-threads", str(_x264_auto_thread_count(height))]
if tmp_wav_path is not None:
command += [
"-acodec",
"aac",
"-map",
"0:v:0",
"-map",
"1:a:0",
]
command += ["-v", "warning", save_file_path]
with tempfile.TemporaryFile() as stderr_file:
process = subprocess.Popen(
command,
stdin=subprocess.PIPE,
stdout=subprocess.DEVNULL,
stderr=stderr_file,
)
try:
if process.stdin is None:
raise RuntimeError("ffmpeg stdin pipe was not created")
with _acquire_cuda_video_buffer(
(chunk_frames, height, width, 3)
) as buffer:
assert buffer.tensor is not None
for start in range(0, num_frames, chunk_frames):
end = min(start + chunk_frames, num_frames)
with maybe_record_function(
f"VIDEO_CHUNK frames {start}-{end} convert+pipe_to_x264"
):
frames = (
(video[:, start:end] * 255)
.clamp_(0, 255)
.to(torch.uint8)
)
frames = frames.permute(1, 2, 3, 0).contiguous()
buffer.tensor[: end - start].copy_(
frames, non_blocking=True
)
torch.cuda.current_stream(video.device).synchronize()
del frames
_sendfile_all(
process.stdin.fileno(),
buffer.fd,
(end - start) * height * width * 3,
)
with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"):
process.stdin.close()
process.stdin = None
returncode = process.wait()
finally:
if process.stdin is not None:
process.stdin.close()
if process.poll() is None:
process.kill()
process.wait()
if returncode:
stderr_file.seek(0)
raise subprocess.CalledProcessError(
returncode,
command,
stderr=stderr_file.read(),
)
encoder = CudaVideoEncoder.open(
save_file_path,
device=video.device,
height=height,
width=width,
fps=fps,
num_frames=num_frames,
element_size=video.element_size(),
audio=audio,
audio_sample_rate=audio_sample_rate,
output_compression=output_compression,
x264_preset=x264_preset,
)
if encoder is None:
return False
encoder.write(video)
encoder.close()
return True
except Exception:
if encoder is not None:
encoder.abort()
logger.warning_once(
"Direct CUDA video save failed; falling back to imageio. "
"Enable debug logging for exception details."
)
logger.debug("Direct CUDA video save failure", exc_info=True)
return False
finally:
if tmp_wav_path:
try:
os.remove(tmp_wav_path)
except OSError:
pass
def _try_save_cuda_videos_direct(
@@ -83,6 +83,8 @@ def should_run_spatial_shard_parallel_decode(
class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
layerwise_offload_dit_group_enabled = False
# decode(z, on_frames=...) hands out finished frames while it decodes
supports_decode_on_frames = False
layer_names = [
"encoder.down_blocks",
"decoder.up_blocks",
@@ -664,7 +664,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
return int(total_frames), int(pad_frames), int(total_frames - pad_frames)
def _decode_temporal_streaming(
self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype
self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames
):
total_frames, pad_frames, output_frames = (
self._decode_temporal_output_frame_plan(
@@ -704,6 +704,8 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
part[:, :, :copy_frames, :, :]
)
write_pos += copy_frames
if on_frames is not None:
on_frames(dec[:, :, write_pos - copy_frames : write_pos])
dropped_frames += part_frames - copy_frames
for i in range(num_chunks):
@@ -778,7 +780,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
return dec
def decode_temporal(self, z):
def decode_temporal(self, z, on_frames=None):
chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
isolated_token_num = 0
@@ -818,7 +820,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
temporal_cat_dtype = _resolve_temporal_cat_dtype()
if not self.training and _resolve_temporal_stream_cat():
return self._decode_temporal_streaming(
z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype
z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames
)
decoded_tasks = []
@@ -889,16 +891,23 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
if pad_frames > 0:
dec = dec[:, :, :-pad_frames, :, :]
if on_frames is not None:
on_frames(dec)
return dec
def decode_base(self, z, frame_num=None, process_image=False):
def decode_base(self, z, frame_num=None, process_image=False, on_frames=None):
"""``on_frames`` receives every output frame once, in order, as soon as it is final."""
if on_frames is not None and (
process_image or not self.use_3d_conv or frame_num is not None
):
raise ValueError("on_frames needs an untrimmed temporal video decode")
if process_image or not self.use_3d_conv:
if not self.use_3d_conv and z.ndim == 5:
z = z.squeeze(2)
recon = self._adaptive_decode(z)
else:
recon = self.decode_temporal(z)
recon = self.decode_temporal(z, on_frames=on_frames)
if self.use_3d_conv:
if frame_num is not None:
@@ -1514,6 +1514,7 @@ class AutoencoderKLWan(ParallelTiledVAE):
"""
_supports_gradient_checkpointing = False
supports_decode_on_frames = True
def __init__(
self,
@@ -1692,7 +1693,8 @@ class AutoencoderKLWan(ParallelTiledVAE):
enc = torch.cat([first_frame, enc], dim=2)
return enc
def decode(self, z: torch.Tensor) -> torch.Tensor:
def decode(self, z: torch.Tensor, on_frames=None) -> torch.Tensor:
"""``on_frames`` receives every returned frame once, in order, as soon as it is final."""
if self.use_feature_cache:
self.clear_cache()
iter_ = z.shape[2]
@@ -1711,6 +1713,10 @@ class AutoencoderKLWan(ParallelTiledVAE):
feat_idx.set(0)
first_chunk.set(i == 0)
out_chunks.append(self.decoder(x[:, :, i : i + 1, :, :]))
# with the causal cache these frames are final, and full
# height even under spatial-parallel decode
if on_frames is not None:
on_frames(self._output_frames(out_chunks[-1]))
out = (
torch.cat(out_chunks, 2)
if len(out_chunks) > 1
@@ -1725,9 +1731,18 @@ class AutoencoderKLWan(ParallelTiledVAE):
self.clear_cache()
else:
out = ParallelTiledVAE.decode(self, z)
# tiled and whole-clip decodes only finish frames at the end
if on_frames is not None:
on_frames(out)
return out
def _output_frames(self, out: torch.Tensor) -> torch.Tensor:
"""What ``decode`` returns for decoder output ``out``, leaving ``out`` intact."""
if self.config.patch_size is not None:
out = unpatchify(out, patch_size=self.config.patch_size)
return out.float().clamp(min=-1.0, max=1.0)
def _decode(self, z: torch.Tensor, first_frame=False) -> torch.Tensor:
x = self.post_quant_conv(z)
spatial_context = (
@@ -31,6 +31,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
PipelineStage,
StageParallelismType,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
EncodeWhileDecoding,
streamed_video_output_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
VerificationResult,
)
@@ -98,6 +102,10 @@ def _ensure_tensor_decode_output(decode_output):
return decode_output
def _to_unit_range(image: torch.Tensor) -> torch.Tensor:
return (image / 2 + 0.5).clamp(0, 1)
class DecodingStage(PipelineStage):
"""
Stage for decoding latent representations into pixel space.
@@ -208,6 +216,7 @@ class DecodingStage(PipelineStage):
server_args: ServerArgs,
*,
vae_dtype: torch.dtype,
on_frames=None,
) -> torch.Tensor:
"""
Decode latent representations into pixel space using VAE.
@@ -220,6 +229,8 @@ class DecodingStage(PipelineStage):
VAE precision ("fp32", "fp16", "bf16")
- pipeline_config.vae_precision: fallback VAE precision
- pipeline_config.vae_tiling: Whether to enable VAE tiling for memory efficiency
on_frames: For a VAE with ``supports_decode_on_frames``, receives the
returned frames once, in order, in the pieces the VAE finishes them.
Returns:
Decoded video tensor with shape (batch, channels, frames, height, width),
@@ -257,11 +268,18 @@ class DecodingStage(PipelineStage):
should_cast_vae = not vae_autocast_enabled
if not vae_autocast_enabled:
latents = latents.to(vae_dtype)
decode_kwargs = {}
if on_frames is not None:
decode_kwargs["on_frames"] = lambda frames: on_frames(
_to_unit_range(frames)
)
with temporary_module_dtype(
self.vae, vae_dtype, enabled=should_cast_vae
) as vae:
try:
decode_output = self._get_vae_decode_fn(vae, server_args)(latents)
decode_output = self._get_vae_decode_fn(vae, server_args)(
latents, **decode_kwargs
)
except Exception as error:
if "out of memory" in str(error).lower():
# decode runs after denoising, so the DiT and encoders
@@ -287,7 +305,7 @@ class DecodingStage(PipelineStage):
image = _ensure_tensor_decode_output(decode_output)
# De-normalize image to [0, 1] range
image = (image / 2 + 0.5).clamp(0, 1)
image = _to_unit_range(image)
return image
def load_model(self):
@@ -305,6 +323,26 @@ class DecodingStage(PipelineStage):
pipeline.add_module(self.component_name, self.vae)
self.server_args.model_loaded[self.component_name] = True
def _encode_while_decoding(
self, batch: Req, server_args: ServerArgs
) -> EncodeWhileDecoding | None:
"""An encoder to feed during the decode, if the frames go to the file as decoded."""
if (
type(self).forward is not DecodingStage.forward
or type(self).decode is not DecodingStage.decode
or not isinstance(self.vae, ParallelTiledVAE)
or not self.vae.supports_decode_on_frames
or batch.return_trajectory_decoded
or not isinstance(batch.latents, torch.Tensor)
):
return None
save_file_path = streamed_video_output_path(
batch, server_args, int(batch.latents.shape[0])
)
if save_file_path is None:
return None
return EncodeWhileDecoding(save_file_path, batch)
@torch.no_grad()
def forward(
self,
@@ -327,6 +365,8 @@ class DecodingStage(PipelineStage):
self.component_name,
quality=batch.sampling_params.quality,
)
stream = self._encode_while_decoding(batch, server_args)
output_file_paths = None
with self.use_declared_component(
component_name=self.component_name,
module=self.vae,
@@ -340,7 +380,24 @@ class DecodingStage(PipelineStage):
vae,
quality_allows(batch.sampling_params.quality, "lossless"),
):
frames = self.decode(batch.latents, server_args, vae_dtype=vae_dtype)
try:
if stream is None:
# Model-specific decode overrides need not support streaming.
frames = self.decode(
batch.latents, server_args, vae_dtype=vae_dtype
)
else:
frames = self.decode(
batch.latents,
server_args,
vae_dtype=vae_dtype,
on_frames=stream,
)
output_file_paths = stream.finish()
except BaseException:
if stream is not None:
stream.abort()
raise
# decode trajectory latents if needed
if batch.return_trajectory_decoded:
@@ -369,11 +426,15 @@ class DecodingStage(PipelineStage):
else:
trajectory_decoded = None
frames = server_args.pipeline_config.post_decoding(frames, server_args)
if output_file_paths is None:
frames = server_args.pipeline_config.post_decoding(frames, server_args)
else:
frames = None
# Update batch with decoded image
output_batch = OutputBatch(
output=frames,
output_file_paths=output_file_paths,
trajectory_timesteps=batch.trajectory_timesteps,
trajectory_latents=batch.trajectory_latents,
rollout_trajectory_data=batch.rollout_trajectory_data,
@@ -0,0 +1,130 @@
# SPDX-License-Identifier: Apache-2.0
"""Encode a video request's MP4 while its decoding stage is still producing frames."""
from __future__ import annotations
import os
from collections.abc import Callable
import torch
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
from sglang.multimodal_gen.runtime.distributed import (
get_replica_group,
model_parallel_is_initialized,
)
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.server_args import ServerArgs
from sglang.multimodal_gen.runtime.utils.logging_utils import CYAN, RESET, init_logger
logger = init_logger(__name__)
# Converted frames the encoder may hold on the GPU before the decode waits for it.
_STREAMED_VIDEO_QUEUE_BYTES = 256 * 1024 * 1024
def streamed_video_output_path(
batch: Req, server_args: ServerArgs, batch_size: int
) -> str | None:
"""Where the output rank can write the MP4 during the decode, if anywhere."""
if not (
batch.save_output
and batch.return_file_paths_only
and not batch.return_raw_frames
and batch.data_type == DataType.VIDEO
and not batch.enable_frame_interpolation
and not batch.enable_upscaling
and batch_size == 1
and not (batch.extra or {}).get("dynamic_batch_output_paths")
and not server_args.enable_torch_compile
and type(server_args.pipeline_config).post_decoding
is PipelineConfig.post_decoding
and current_platform.is_cuda()
):
return None
if model_parallel_is_initialized() and get_replica_group().rank_in_group != 0:
return None
return batch.output_file_path(1, 0)
class EncodeWhileDecoding:
"""Feed each run of finished frames to the MP4 encoder while the VAE decodes the rest."""
def __init__(
self,
save_file_path: str,
batch: Req,
finish_frames: Callable[[torch.Tensor], torch.Tensor] | None = None,
audio=None,
audio_sample_rate: int | None = None,
):
self.save_file_path = save_file_path
self._batch = batch
self._finish_frames = finish_frames
self._audio = audio
self._audio_sample_rate = audio_sample_rate
self._encoder = None
self._failed = False
def __call__(self, frames: torch.Tensor) -> None:
"""Encode frames that are [1, 3, t, H, W] in [0, 1] after ``finish_frames``."""
if self._failed:
return
try:
if self._finish_frames is not None:
frames = self._finish_frames(frames)
frames = frames[0]
if self._encoder is None:
self._encoder = self._open(frames)
if self._encoder is None:
self._failed = True
return
self._encoder.write(frames)
except Exception:
self._fail()
def _open(self, frames: torch.Tensor):
from sglang.multimodal_gen.runtime.entrypoints.utils import CudaVideoEncoder
height, width = int(frames.shape[-2]), int(frames.shape[-1])
os.makedirs(os.path.dirname(self.save_file_path) or ".", exist_ok=True)
return CudaVideoEncoder.open(
self.save_file_path,
device=frames.device,
height=height,
width=width,
fps=self._batch.fps,
num_frames=self._batch.num_frames or int(frames.shape[1]),
audio=self._audio,
audio_sample_rate=self._audio_sample_rate,
output_compression=self._batch.output_compression,
x264_preset=self._batch.x264_preset,
max_queued_frames=_STREAMED_VIDEO_QUEUE_BYTES // (3 * height * width),
)
def finish(self) -> list[str] | None:
"""The saved path, or None when the worker has to save the frames instead."""
if self._failed or self._encoder is None:
return None
try:
self._encoder.close()
except Exception:
self._fail()
return None
logger.info(f"Output saved to {CYAN}{self.save_file_path}{RESET}")
return [self.save_file_path]
def abort(self) -> None:
if self._encoder is not None:
self._encoder.abort()
def _fail(self) -> None:
self._failed = True
self.abort()
logger.warning_once(
"Encoding the video while it decodes failed; saving it after the "
"decode instead. Enable debug logging for exception details."
)
logger.debug("Streamed video encode failure", exc_info=True)
@@ -21,6 +21,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
StageParallelismType,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
EncodeWhileDecoding,
streamed_video_output_path,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
StageValidators as V,
)
@@ -279,11 +283,11 @@ class MiniMaxH3DecodingStage(DecodingStage):
audio_vae_dtype = resolve_precision(
server_args, "audio_vae", precision_attr="audio_vae_precision"
)
uses = [
# forward decodes the audio first so the MP4 encoder can start with it
return [
ComponentUse(stage_name, "audio_vae", target_dtype=audio_vae_dtype),
ComponentUse(stage_name, "video_vae", target_dtype=video_vae_dtype),
]
uses.append(ComponentUse(stage_name, "audio_vae", target_dtype=audio_vae_dtype))
return uses
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
result = VerificationResult()
@@ -299,7 +303,8 @@ class MiniMaxH3DecodingStage(DecodingStage):
self, batch: OutputBatch, server_args: ServerArgs
) -> VerificationResult:
result = VerificationResult()
result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)])
if not batch.output_file_paths:
result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)])
result.add_check("audio", batch.audio, [V.is_tensor, V.with_dims(3)])
result.add_check("audio_sample_rate", batch.audio_sample_rate, V.positive_int)
return result
@@ -354,6 +359,64 @@ class MiniMaxH3DecodingStage(DecodingStage):
"sample_rate": int(audio_vae.sample_rate),
}
def _decode_replica_audio(
self, audio_latent: torch.Tensor, server_args: ServerArgs
) -> tuple[torch.Tensor, int]:
# Audio VAE weights are replicated. Decode on replica rank 0 and broadcast
# only within the request's replica, excluding independent DP replicas.
replica_group = get_replica_group() if model_parallel_is_initialized() else None
is_audio_owner = replica_group is None or replica_group.rank_in_group == 0
owner_exception = None
owner_error = None
audio_payload = None
if is_audio_owner:
try:
audio_payload = self._decode_audio(audio_latent, server_args)
except Exception as exc:
owner_exception = exc
owner_error = f"{type(exc).__name__}: {exc}"
if replica_group is not None:
owner_error = replica_group.broadcast_object(owner_error, src=0)
if owner_error is not None:
if owner_exception is not None:
raise owner_exception
raise RuntimeError(
f"MiniMax H3 audio decode failed on rank 0: {owner_error}"
)
if replica_group is not None:
audio_payload = replica_group.broadcast_tensor_dict(audio_payload, src=0)
if not isinstance(audio_payload, dict):
raise RuntimeError("MiniMax H3 audio decode produced no output payload")
audio_waveform = _required_tensor(
audio_payload.get("waveform"), "audio_vae.decode"
)
return audio_waveform, int(audio_payload["sample_rate"])
@staticmethod
def _finish_visual_frames(
batch: Req,
video_vae,
frames: torch.Tensor,
*,
batch_size: int,
runtime_owned: bool = False,
) -> torch.Tensor:
frames = _required_tensor(
video_vae.processor.revert_tensor(frames, runtime_owned=runtime_owned),
"video_vae.processor.revert_tensor",
)
frames = _canonical_visual_video_frames(frames, batch_size=batch_size)
frames = _crop_to_target_canvas(batch, frames)
if frames.dtype != torch.float32 or not frames.is_contiguous():
canonical_frames = torch.empty_like(
frames,
dtype=torch.float32,
memory_format=torch.contiguous_format,
)
canonical_frames.copy_(frames)
frames = canonical_frames
return frames
@torch.no_grad()
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
_minimax_h3_decoder_task(batch)
@@ -368,6 +431,19 @@ class MiniMaxH3DecodingStage(DecodingStage):
if self.video_vae is None:
raise RuntimeError("MiniMax H3 tasks require the video_vae output decoder")
batch_size = int(visual_latent.shape[0])
audio_waveform, audio_sample_rate = self._decode_replica_audio(
audio_latent, server_args
)
output_audio_waveform = _canonical_output_audio_waveform(
audio_waveform, batch_size=batch_size
)
streamed_output_path = streamed_video_output_path(
batch, server_args, batch_size
)
visual_frames = None
output_file_paths = None
with self.use_declared_component(
component_name="video_vae",
module=self.video_vae,
@@ -400,69 +476,44 @@ class MiniMaxH3DecodingStage(DecodingStage):
server_args,
decode_fn=selected_video_vae.decode_base,
)
with set_forward_context(current_timestep=0, attn_metadata=None):
visual_frames = video_decode(visual_decode_latent)
visual_frames = selected_video_vae.processor.revert_tensor(
visual_frames, runtime_owned=True
finish_frames = functools.partial(
self._finish_visual_frames,
batch,
selected_video_vae,
batch_size=batch_size,
)
visual_frames = _required_tensor(
visual_frames,
"video_vae.processor.revert_tensor",
)
visual_frames = _canonical_visual_video_frames(
visual_frames, batch_size=int(visual_latent.shape[0])
)
visual_frames = _crop_to_target_canvas(batch, visual_frames)
if (
visual_frames.dtype != torch.float32
or not visual_frames.is_contiguous()
):
canonical_frames = torch.empty_like(
visual_frames,
dtype=torch.float32,
memory_format=torch.contiguous_format,
stream = None
decode_kwargs = {}
if streamed_output_path is not None:
stream = EncodeWhileDecoding(
streamed_output_path,
batch,
finish_frames,
output_audio_waveform[0],
audio_sample_rate,
)
canonical_frames.copy_(visual_frames)
visual_frames = canonical_frames
decode_kwargs["on_frames"] = stream
try:
with set_forward_context(current_timestep=0, attn_metadata=None):
decoded = video_decode(visual_decode_latent, **decode_kwargs)
if stream is not None:
output_file_paths = stream.finish()
except BaseException:
if stream is not None:
stream.abort()
raise
if output_file_paths is None:
# callbacks borrow slices of decoded; only the final output owns it
visual_frames = finish_frames(decoded, runtime_owned=True)
del decoded
# Audio VAE weights are replicated. Decode on replica rank 0 and broadcast
# only within the request's replica, excluding independent DP replicas.
replica_group = get_replica_group() if model_parallel_is_initialized() else None
is_audio_owner = replica_group is None or replica_group.rank_in_group == 0
owner_exception = None
owner_error = None
audio_payload = None
if is_audio_owner:
try:
audio_payload = self._decode_audio(audio_latent, server_args)
except Exception as exc:
owner_exception = exc
owner_error = f"{type(exc).__name__}: {exc}"
if replica_group is not None:
owner_error = replica_group.broadcast_object(owner_error, src=0)
if owner_error is not None:
if owner_exception is not None:
raise owner_exception
raise RuntimeError(
f"MiniMax H3 audio decode failed on rank 0: {owner_error}"
if visual_frames is not None:
visual_frames = server_args.pipeline_config.post_decoding(
visual_frames, server_args
)
if replica_group is not None:
audio_payload = replica_group.broadcast_tensor_dict(audio_payload, src=0)
if not isinstance(audio_payload, dict):
raise RuntimeError("MiniMax H3 audio decode produced no output payload")
audio_waveform = _required_tensor(
audio_payload.get("waveform"), "audio_vae.decode"
)
audio_sample_rate = int(audio_payload["sample_rate"])
visual_frames = server_args.pipeline_config.post_decoding(
visual_frames, server_args
)
output_audio_waveform = _canonical_output_audio_waveform(
audio_waveform, batch_size=int(visual_frames.shape[0])
)
return OutputBatch(
output=visual_frames,
output_file_paths=output_file_paths,
audio=output_audio_waveform,
audio_sample_rate=audio_sample_rate,
trajectory_timesteps=batch.trajectory_timesteps,
@@ -1559,7 +1559,7 @@
"LatentPreparationStage": 0.11,
"TimestepPreparationStage": 2.13,
"DenoisingStage": 12704.94,
"DecodingStage": 1098.67
"DecodingStage": 2172.17
},
"denoise_step_ms": {
"0": 152.87,
@@ -1613,7 +1613,7 @@
"48": 257.43,
"49": 254.6
},
"expected_e2e_ms": 14853.56,
"expected_e2e_ms": 15599.56,
"expected_avg_denoise_ms": 253.29,
"expected_median_denoise_ms": 256.21,
"load_peak_vram_mb": 17610.0,
@@ -1761,14 +1761,14 @@
"TimestepPreparationStage": 30.08,
"LatentPreparationStage": 0.1,
"DmdDenoisingStage": 252.46,
"DecodingStage": 1073.1
"DecodingStage": 2173.4
},
"denoise_step_ms": {
"0": 35.0,
"1": 25.92,
"2": 112.67
},
"expected_e2e_ms": 2444.17,
"expected_e2e_ms": 3073.51,
"expected_avg_denoise_ms": 83.86,
"expected_median_denoise_ms": 112.67,
"load_peak_vram_mb": 17610.0,
@@ -4,6 +4,7 @@ import sys
import tempfile
import types
import unittest
from contextlib import nullcontext
from types import SimpleNamespace
from unittest.mock import patch
@@ -1512,6 +1513,51 @@ class TestSanaWMRefinerStage(_GlobalStageArgsMixin, unittest.TestCase):
self.assertTrue(torch.equal(frames, decoded))
def test_refiner_forward_uses_non_streaming_decode_override(self) -> None:
decoded = torch.arange(1 * 3 * 4 * 2 * 2).reshape(1, 3, 4, 2, 2)
decoding_module = "sglang.multimodal_gen.runtime.pipelines_core.stages.decoding"
for applied in (True, False):
with self.subTest(refiner_applied=applied):
vae = SimpleNamespace()
stage = SanaWMRefinerDecodingStage(vae=vae)
batch = SimpleNamespace(
latents=torch.empty(1, 128, 4, 2, 2),
sampling_params=SanaWMSamplingParams(),
extra={"sana_wm_refiner_applied": applied},
return_trajectory_decoded=False,
trajectory_timesteps=None,
trajectory_latents=None,
rollout_trajectory_data=None,
metrics=None,
usage=None,
)
args = SimpleNamespace(pipeline_config=SanaWMPipelineConfig())
with (
patch.object(stage, "load_model"),
patch.object(
stage, "use_declared_component", return_value=nullcontext(vae)
),
patch(
f"{decoding_module}.resolve_decode_precision",
return_value=torch.bfloat16,
),
patch(
f"{decoding_module}.use_vae_fast_path",
return_value=nullcontext(),
),
patch.object(DecodingStage, "decode", return_value=decoded),
patch.object(
args.pipeline_config,
"post_decoding",
side_effect=lambda frames, _: frames,
),
):
output = stage.forward(batch, args)
expected = decoded[:, :, 1:] if applied else decoded
self.assertTrue(torch.equal(output.output, expected))
self.assertIsNone(output.output_file_paths)
if __name__ == "__main__":
unittest.main()
@@ -0,0 +1,204 @@
# SPDX-License-Identifier: Apache-2.0
"""Encoding MiniMax-H3 video while it decodes must not change the MP4."""
import os
import subprocess
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.multimodal_gen.runtime.entrypoints.utils import (
CudaVideoEncoder,
_try_save_cuda_video_direct,
)
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae import (
AutoencoderKLLegacy,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.decoding import (
MiniMaxH3DecodingStage,
)
from sglang.test.test_utils import CustomTestCase
FPS = 24
SAMPLE_RATE = 24000
def _temporal_vae() -> AutoencoderKLLegacy:
"""The released temporal chunking, with each clip decoded by repetition."""
vae = AutoencoderKLLegacy.__new__(AutoencoderKLLegacy)
torch.nn.Module.__init__(vae)
vae.vae_ratio = 16
vae.vae_ratio_t = 4
vae.use_3d_conv = True
vae.transform = None
vae.transform_rev = None
vae.transform_rev_inplace = None
vae.setup_forward(clip_length=17, token_drop=3)
vae._adaptive_decode = lambda clip_z: clip_z.repeat_interleave(4, dim=2)
return vae.eval()
class TestDecodeHandsOutFinishedFrames(CustomTestCase):
def test_finished_callbacks_preserve_the_decode_buffer_for_fallback(self):
for device in ("cpu", "cuda"):
for stream_cat in ("1", "0"):
with (
self.subTest(device=device, stream_cat=stream_cat),
mock.patch.dict(
os.environ,
{"MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": stream_cat},
),
):
vae = _temporal_vae()
latents = torch.randn(1, 3, 31, 2, 2, device=device)
expected_decode = vae.decode_base(latents)
batch = SimpleNamespace(extra={})
parts = []
def finish_frames(frames):
parts.append(
MiniMaxH3DecodingStage._finish_visual_frames(
batch, vae, frames, batch_size=1
)
)
decoded = vae.decode_base(latents, on_frames=finish_frames)
torch.testing.assert_close(decoded, expected_decode, rtol=0, atol=0)
expected_output = vae.processor.revert_tensor(expected_decode)
torch.testing.assert_close(
torch.cat(parts, dim=2), expected_output, rtol=0, atol=0
)
fallback = MiniMaxH3DecodingStage._finish_visual_frames(
batch, vae, decoded, batch_size=1, runtime_owned=True
)
torch.testing.assert_close(
fallback, expected_output, rtol=0, atol=0
)
def test_every_frame_arrives_once_in_order_and_final(self):
vae = _temporal_vae()
latents = torch.randn(1, 3, 31, 2, 2)
for stream_cat in ("1", "0"):
with (
self.subTest(stream_cat=stream_cat),
mock.patch.dict(
os.environ,
{"MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": stream_cat},
),
):
parts = []
video = vae.decode_base(
latents, on_frames=lambda frames: parts.append(frames.clone())
)
self.assertTrue(torch.equal(torch.cat(parts, dim=2), video))
self.assertEqual(len(parts) > 1, stream_cat == "1")
def test_trimmed_decodes_refuse_the_callback(self):
with self.assertRaises(ValueError):
_temporal_vae().decode_base(
torch.randn(1, 3, 31, 2, 2), frame_num=100, on_frames=print
)
@unittest.skipUnless(
torch.cuda.is_available() and hasattr(os, "memfd_create"),
"needs CUDA and memfd",
)
class TestStreamedEncode(CustomTestCase):
@staticmethod
def _video(height: int, width: int) -> torch.Tensor:
generator = torch.Generator(device="cuda").manual_seed(0)
return torch.rand(3, 30, height, width, device="cuda", generator=generator)
def test_writes_in_pieces_match_the_one_shot_save(self):
audio = torch.sin(torch.arange(2 * 2 * SAMPLE_RATE) / 7).reshape(2, -1)
# 70x90 is not a macroblock multiple, so it also covers the scale filter
for height, width, with_audio in ((64, 96, False), (70, 90, True)):
video = self._video(height, width)
sample_audio = audio if with_audio else None
sample_rate = SAMPLE_RATE if with_audio else None
with (
self.subTest(size=(height, width)),
tempfile.TemporaryDirectory() as tmp,
):
one_shot, streamed = (
Path(tmp, "one_shot.mp4"),
Path(tmp, "streamed.mp4"),
)
self.assertTrue(
_try_save_cuda_video_direct(
save_file_path=str(one_shot),
sample=(video, sample_audio) if with_audio else video,
fps=FPS,
audio_sample_rate=sample_rate,
output_compression=None,
)
)
encoder = CudaVideoEncoder.open(
str(streamed),
device=video.device,
height=height,
width=width,
fps=FPS,
num_frames=video.shape[1],
audio=sample_audio,
audio_sample_rate=sample_rate,
max_queued_frames=64,
)
for start, end in ((0, 1), (1, 6), (6, 25), (25, 30)):
encoder.write(video[:, start:end])
encoder.close()
self.assertEqual(one_shot.read_bytes(), streamed.read_bytes())
def test_concurrent_encodes_both_finish(self):
# only one staging buffer is cached, so the other one is closed on release
video = self._video(64, 96)
with tempfile.TemporaryDirectory() as tmp:
paths = [Path(tmp, f"clip_{i}.mp4") for i in range(2)]
encoders = [
CudaVideoEncoder.open(
str(path),
device=video.device,
height=64,
width=96,
fps=FPS,
num_frames=video.shape[1],
)
for path in paths
]
for encoder in encoders:
encoder.write(video)
for encoder in encoders:
encoder.close()
self.assertEqual(paths[0].read_bytes(), paths[1].read_bytes())
def test_a_failed_encode_leaves_no_file(self):
video = self._video(64, 96)
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp, "clip.mp4")
encoder = CudaVideoEncoder.open(
str(path),
device=video.device,
height=64,
width=96,
fps=FPS,
num_frames=video.shape[1],
)
encoder.write(video[:, :10])
encoder._process.kill()
encoder._process.wait()
try:
encoder.write(video[:, 10:])
except RuntimeError:
pass
with self.assertRaises((RuntimeError, subprocess.CalledProcessError)):
encoder.close()
self.assertFalse(path.exists())
if __name__ == "__main__":
unittest.main(verbosity=3)
@@ -0,0 +1,194 @@
# SPDX-License-Identifier: Apache-2.0
"""Encoding Wan video while its VAE decodes must not change the MP4.
The frame hand-off itself is covered in
python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py.
"""
import os
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
from unittest import mock
import torch
from sglang.multimodal_gen.configs.models.vaes.wanvae import (
WanVAEArchConfig,
WanVAEConfig,
)
from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
from sglang.multimodal_gen.runtime.entrypoints.utils import _try_save_cuda_video_direct
from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
EncodeWhileDecoding,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.test.test_utils import CustomTestCase
FPS = 16
Z_DIM = 4
def _vae_config(*, wan22: bool) -> WanVAEConfig:
"""A tiny Wan 2.1 VAE, or with ``wan22`` the 2.2 residual blocks and 2x2 patches."""
channels = 12 if wan22 else 3
return WanVAEConfig(
arch_config=WanVAEArchConfig(
base_dim=8,
z_dim=Z_DIM,
dim_mult=(1, 2, 2, 2),
num_res_blocks=1,
latents_mean=(0.0,) * Z_DIM,
latents_std=(0.5,) * Z_DIM,
is_residual=wan22,
in_channels=channels,
out_channels=channels,
patch_size=2 if wan22 else None,
),
load_encoder=False,
)
def _vae(config: WanVAEConfig, device: str = "cpu") -> AutoencoderKLWan:
torch.manual_seed(0)
return AutoencoderKLWan(config).to(device).eval()
def _server_args(pipeline_config=None) -> SimpleNamespace:
return SimpleNamespace(
pipeline_config=pipeline_config or WanT2V480PConfig(),
disable_autocast=False,
enable_torch_compile=False,
)
def _stage(vae, server_args, stage_cls=DecodingStage) -> DecodingStage:
with mock.patch(
"sglang.multimodal_gen.runtime.pipelines_core.stages.base.get_global_server_args",
return_value=server_args,
):
return stage_cls(vae)
def _video_request(**overrides) -> SimpleNamespace:
fields = dict(
save_output=True,
return_file_paths_only=True,
return_raw_frames=False,
return_trajectory_decoded=False,
data_type=DataType.VIDEO,
enable_frame_interpolation=False,
enable_upscaling=False,
extra={},
latents=torch.zeros(1, Z_DIM, 2, 2, 2),
fps=FPS,
num_frames=5,
output_compression=None,
x264_preset=None,
output_file_path=lambda num_outputs, output_idx: "/tmp/clip.mp4",
)
fields.update(overrides)
return SimpleNamespace(**fields)
@unittest.skipUnless(
torch.cuda.is_available() and hasattr(os, "memfd_create"),
"needs CUDA and memfd",
)
class TestStreamedEncode(CustomTestCase):
@torch.no_grad()
def test_streamed_mp4_matches_the_one_shot_save(self):
generator = torch.Generator(device="cuda").manual_seed(0)
# Wan 2.1 decodes under bf16 autocast, Wan 2.2 in fp32; 40x48 is not a
# macroblock multiple, so this also covers the scale filter
for wan22, vae_dtype, latent_size in (
(False, torch.bfloat16, (5, 6)),
(True, torch.float32, (3, 4)),
):
config = _vae_config(wan22=wan22)
server_args = _server_args(WanT2V480PConfig(vae_config=config))
stage = _stage(_vae(config, "cuda"), server_args)
latents = torch.randn(
1, Z_DIM, 6, *latent_size, device="cuda", generator=generator
)
with (
self.subTest(wan22=wan22),
tempfile.TemporaryDirectory() as tmp,
):
one_shot, streamed = (
Path(tmp, "one_shot.mp4"),
Path(tmp, "streamed.mp4"),
)
stream = EncodeWhileDecoding(
str(streamed), _video_request(num_frames=21)
)
frames = stage.decode(
latents, server_args, vae_dtype=vae_dtype, on_frames=stream
)
self.assertEqual(stream.finish(), [str(streamed)])
self.assertEqual(tuple(frames.shape[1:3]), (3, 21))
self.assertTrue(
_try_save_cuda_video_direct(
save_file_path=str(one_shot),
sample=frames[0],
fps=FPS,
audio_sample_rate=None,
output_compression=None,
)
)
self.assertEqual(one_shot.read_bytes(), streamed.read_bytes())
@unittest.skipUnless(
current_platform.is_cuda(), "streaming is gated on the CUDA platform, not ROCm"
)
def test_only_file_path_video_requests_stream(self):
server_args = _server_args()
stage = _stage(_vae(_vae_config(wan22=False)), server_args)
self.assertIsInstance(
stage._encode_while_decoding(_video_request(), server_args),
EncodeWhileDecoding,
)
for field, value in (
("return_file_paths_only", False),
("return_raw_frames", True),
("return_trajectory_decoded", True),
("data_type", DataType.IMAGE),
("enable_frame_interpolation", True),
("enable_upscaling", True),
("extra", {"dynamic_batch_output_paths": ["a.mp4"]}),
("latents", torch.zeros(2, Z_DIM, 2, 2, 2)),
):
with self.subTest(field=field):
request = _video_request(**{field: value})
self.assertIsNone(stage._encode_while_decoding(request, server_args))
class OwnPostDecoding(WanT2V480PConfig):
def post_decoding(self, frames, server_args):
return frames
class OwnDecode(DecodingStage):
def decode(self, latents, server_args, *, vae_dtype):
return latents
for name, candidate, args in (
("post_decoding", stage, _server_args(OwnPostDecoding())),
(
"torch_compile",
stage,
SimpleNamespace(**{**vars(server_args), "enable_torch_compile": True}),
),
("stage_decode", _stage(stage.vae, server_args, OwnDecode), server_args),
("vae", _stage(torch.nn.Identity(), server_args), server_args),
):
with self.subTest(name=name):
self.assertIsNone(
candidate._encode_while_decoding(_video_request(), args)
)
if __name__ == "__main__":
unittest.main(verbosity=3)
@@ -0,0 +1,60 @@
# SPDX-License-Identifier: Apache-2.0
"""AutoencoderKLWan.decode hands out every frame once, in order, once it is final."""
import pytest
import torch
from sglang.multimodal_gen.configs.models.vaes.wanvae import (
WanVAEArchConfig,
WanVAEConfig,
)
from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan
Z_DIM = 4
def _vae(*, wan22: bool) -> AutoencoderKLWan:
"""A tiny Wan 2.1 VAE, or with ``wan22`` the 2.2 residual blocks and 2x2 patches."""
channels = 12 if wan22 else 3
config = WanVAEConfig(
arch_config=WanVAEArchConfig(
base_dim=8,
z_dim=Z_DIM,
dim_mult=(1, 2, 2, 2),
num_res_blocks=1,
latents_mean=(0.0,) * Z_DIM,
latents_std=(1.0,) * Z_DIM,
is_residual=wan22,
in_channels=channels,
out_channels=channels,
patch_size=2 if wan22 else None,
),
load_encoder=False,
)
torch.manual_seed(0)
return AutoencoderKLWan(config).eval()
@pytest.mark.parametrize("wan22", [False, True])
def test_causal_decode_hands_out_each_latent_frames_pixels(wan22):
vae = _vae(wan22=wan22)
latents = torch.randn(1, Z_DIM, 4, 2, 3)
parts = []
with torch.no_grad():
video = vae.decode(latents, on_frames=parts.append)
reference = vae.decode(latents)
assert torch.equal(video, reference)
assert [part.shape[2] for part in parts] == [1, 4, 4, 4]
assert torch.equal(torch.cat(parts, dim=2), video)
def test_whole_clip_decode_hands_over_everything_at_the_end():
vae = _vae(wan22=False)
vae.use_feature_cache = False
parts = []
with torch.no_grad():
video = vae.decode(torch.randn(1, Z_DIM, 3, 2, 3), on_frames=parts.append)
assert len(parts) == 1
assert parts[0] is video