From 9101e895eaa4510088058e5e041330dd150df0bb Mon Sep 17 00:00:00 2001 From: Mick Date: Thu, 8 Oct 2026 09:01:19 +0800 Subject: [PATCH] [diffusion] feat: encode the MP4 while the VAE decodes (MiniMax-H3, Wan) (#41819) Co-authored-by: Claude Opus 5.5 --- .../runtime/entrypoints/utils.py | 510 ++++++++++++------ .../runtime/models/vaes/common.py | 2 + .../models/vaes/minimax_h3_video_vae/klvae.py | 19 +- .../runtime/models/vaes/wanvae.py | 17 +- .../runtime/pipelines_core/stages/decoding.py | 69 ++- .../stages/encode_while_decoding.py | 130 +++++ .../minimax_h3/stages/decoding.py | 173 +++--- .../test/server/perf_baselines/h100.json | 8 +- .../test/unit/sana_wm/test_pipeline_config.py | 46 ++ .../unit/test_minimax_h3_streamed_encode.py | 204 +++++++ .../test/unit/test_wan_streamed_encode.py | 194 +++++++ .../test/unit/test_wan_vae_on_frames.py | 60 +++ 12 files changed, 1196 insertions(+), 236 deletions(-) create mode 100644 python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_minimax_h3_streamed_encode.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py create mode 100644 python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/utils.py b/python/sglang/multimodal_gen/runtime/entrypoints/utils.py index 807f42b46c..a64facc499 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/utils.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/utils.py @@ -12,6 +12,7 @@ import atexit import json import mmap import os +import queue import shutil import subprocess import tempfile @@ -62,7 +63,7 @@ _cached_cuda_video_buffer: "_CudaMemfdVideoBuffer | None" = None class _CudaMemfdVideoBuffer: - """CUDA-registered memfd used as direct ffmpeg raw-video input.""" + """CUDA-registered memfd mapping that stages raw frames for ffmpeg.""" def __init__(self, shape: tuple[int, ...]): self.shape = shape @@ -337,23 +338,335 @@ def _x264_auto_thread_count(height: int) -> int: return min(cpu_limit, row_limit, 128) -def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int: - _, num_frames, height, width = video.shape - temporary_bytes_per_frame = 3 * height * width * (video.element_size() + 2) +def _conversion_chunk_frames( + num_frames: int, height: int, width: int, element_size: int +) -> int: + temporary_bytes_per_frame = 3 * height * width * (element_size + 2) return min( num_frames, max(1, _MAX_CUDA_VIDEO_CONVERSION_CHUNK_BYTES // temporary_bytes_per_frame), ) -def _sendfile_all(output_fd: int, input_fd: int, count: int) -> None: - offset = 0 - while count: - sent = os.sendfile(output_fd, input_fd, offset, count) - if sent <= 0: - raise RuntimeError("sendfile made no progress") - offset += sent - count -= sent +def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int: + _, num_frames, height, width = video.shape + return _conversion_chunk_frames(num_frames, height, width, video.element_size()) + + +def _write_all(fd: int, data: memoryview) -> None: + while data: + data = data[os.write(fd, data) :] + + +def _remove_quietly(path: Optional[str]) -> None: + if path: + try: + os.remove(path) + except OSError: + pass + + +def _cuda_video_ffmpeg_command( + save_file_path: str, + *, + width: int, + height: int, + fps: int, + crf: int, + x264_preset: Optional[str], + wav_path: Optional[str], +) -> list[str]: + command = [ + _resolve_ffmpeg_exe(), + "-y", + "-f", + "rawvideo", + "-vcodec", + "rawvideo", + "-s", + f"{width}x{height}", + "-pix_fmt", + "rgb24", + "-r", + f"{fps:.02f}", + "-i", + "pipe:0", + ] + if wav_path is None: + command += ["-an"] + else: + command += ["-i", wav_path] + command += [ + "-vcodec", + "libx264", + "-preset", + x264_preset or X264_PRESET, + "-pix_fmt", + "yuv420p", + "-crf", + str(crf), + ] + + macro_block_size = 16 + if width % macro_block_size or height % macro_block_size: + output_width = ( + width + if width % macro_block_size == 0 + else width + macro_block_size - width % macro_block_size + ) + output_height = ( + height + if height % macro_block_size == 0 + else height + macro_block_size - height % macro_block_size + ) + command += ["-vf", f"scale={output_width}:{output_height}"] + + command += ["-threads", str(_x264_auto_thread_count(height))] + if wav_path is not None: + command += [ + "-acodec", + "aac", + "-map", + "0:v:0", + "-map", + "1:a:0", + ] + command += ["-v", "warning", save_file_path] + return command + + +class CudaVideoEncoder: + """Encode [3, T, H, W] CUDA frames in [0, 1] to an MP4 as they arrive. + + ``write`` only queues the uint8 conversion on the current stream; a writer + thread stages each chunk in registered host memory and pipes it to ffmpeg, + so a caller that produces frames in pieces keeps its own GPU work going. + """ + + @classmethod + def open( + cls, + save_file_path: str, + *, + device: torch.device, + height: int, + width: int, + fps: int, + num_frames: int, + element_size: int = 4, + audio: Any = None, + audio_sample_rate: Optional[int] = None, + output_compression: Optional[int] = None, + x264_preset: Optional[str] = None, + max_queued_frames: int = 0, + ) -> "CudaVideoEncoder | None": + """Start ffmpeg, or return None when this output cannot use the path.""" + if not hasattr(os, "memfd_create") or device.type != "cuda": + return None + if os.path.splitext(save_file_path)[1].lower() != ".mp4": + return None + quality = output_compression / 10 if output_compression is not None else 5 + if not 1 <= quality <= 10: + return None + crf = int((1 - quality / 10.0) * 51) + audio_np = _normalize_audio_to_numpy(audio) + if audio_np is not None and scipy_wavfile is None: + return None + + wav_path = None + try: + if audio_np is not None: + selected_sr = _pick_audio_sample_rate( + audio_np=audio_np, + audio_sample_rate=audio_sample_rate, + fps=fps, + num_frames=num_frames, + ) + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: + wav_path = f.name + scipy_wavfile.write(wav_path, selected_sr, audio_np) + command = _cuda_video_ffmpeg_command( + save_file_path, + width=width, + height=height, + fps=fps, + crf=crf, + x264_preset=x264_preset, + wav_path=wav_path, + ) + chunk_frames = _conversion_chunk_frames( + max(1, num_frames), height, width, element_size + ) + return cls( + command, + save_file_path, + device=device, + height=height, + width=width, + chunk_frames=chunk_frames, + queued_chunks=max(1, -(-max_queued_frames // chunk_frames)), + wav_path=wav_path, + ) + except BaseException: + _remove_quietly(wav_path) + raise + + def __init__( + self, + command: list[str], + save_file_path: str, + *, + device: torch.device, + height: int, + width: int, + chunk_frames: int, + queued_chunks: int, + wav_path: Optional[str], + ): + self.save_file_path = save_file_path + self._command = command + self._device = device + self._height = height + self._width = width + self._chunk_frames = chunk_frames + self._wav_path = wav_path + self._frames_queued = 0 + self._error: BaseException | None = None + self._finished = False + self._queue: queue.Queue = queue.Queue(maxsize=queued_chunks) + self._stderr = tempfile.TemporaryFile() + try: + self._process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + stderr=self._stderr, + ) + except BaseException: + self._stderr.close() + raise + self._thread = threading.Thread( + target=self._pipe_frames, name="sglang-video-encoder", daemon=True + ) + self._thread.start() + + def write(self, frames: torch.Tensor) -> None: + """Queue [3, t, H, W] frames; raise once the encoder has failed.""" + if frames.dim() == 3: + frames = frames.unsqueeze(1) + if ( + frames.device != self._device + or int(frames.shape[0]) != 3 + or tuple(frames.shape[2:]) != (self._height, self._width) + ): + raise ValueError( + f"expected [3, t, {self._height}, {self._width}] frames on " + f"{self._device}, got {tuple(frames.shape)} on {frames.device}" + ) + stream = torch.cuda.current_stream(frames.device) + num_frames = int(frames.shape[1]) + for start in range(0, num_frames, self._chunk_frames): + end = min(start + self._chunk_frames, num_frames) + first = self._frames_queued + with maybe_record_function( + f"VIDEO_CHUNK frames {first}-{first + end - start} convert" + ): + chunk = (frames[:, start:end] * 255).clamp_(0, 255).to(torch.uint8) + chunk = chunk.permute(1, 2, 3, 0).contiguous() + ready = torch.cuda.Event() + ready.record(stream) + self._put((chunk, ready)) + self._frames_queued += end - start + + def close(self) -> None: + """Finish the MP4; on failure clean up as ``abort`` does and raise.""" + if self._finished: + return + try: + with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"): + self._stop_thread() + if self._error is not None: + if self._process.poll(): + raise self._ffmpeg_error() from self._error + raise RuntimeError("video encoder failed") from self._error + self._process.stdin.close() + if self._process.wait(): + raise self._ffmpeg_error() + except BaseException: + self.abort() + raise + self._finished = True + self._release() + + def abort(self) -> None: + """Stop ffmpeg and remove the partial file.""" + if self._finished: + return + self._finished = True + if self._process.poll() is None: + self._process.kill() + self._stop_thread() + try: + self._process.stdin.close() + except OSError: + pass + self._process.wait() + self._release() + _remove_quietly(self.save_file_path) + + def _ffmpeg_error(self) -> subprocess.CalledProcessError: + self._stderr.seek(0) + return subprocess.CalledProcessError( + self._process.returncode, self._command, stderr=self._stderr.read() + ) + + def _put(self, item) -> None: + # never blocks for good: after a failure the writer keeps draining + if self._error is not None: + raise RuntimeError("video encoder failed") from self._error + self._queue.put(item) + + def _stop_thread(self) -> None: + self._queue.put(None) + self._thread.join() + + def _pipe_frames(self) -> None: + frame_bytes = self._height * self._width * 3 + try: + stdin = self._process.stdin + if stdin is None: + raise RuntimeError("ffmpeg stdin pipe was not created") + with ( + torch.cuda.device(self._device), + _acquire_cuda_video_buffer( + (self._chunk_frames, self._height, self._width, 3) + ) as buffer, + memoryview(buffer.array).cast("B") as staged, + ): + assert buffer.tensor is not None + copy_stream = torch.cuda.Stream() + while (item := self._queue.get()) is not None: + chunk, ready = item + del item + count = int(chunk.shape[0]) + copy_stream.wait_event(ready) + with torch.cuda.stream(copy_stream): + buffer.tensor[:count].copy_(chunk, non_blocking=True) + copy_stream.synchronize() + # the copy is done, so the allocator may hand this block out again + del chunk, ready + # write() copies into the pipe; sendfile would only lend it these + # pages, and the next chunk overwrites them before ffmpeg reads + _write_all(stdin.fileno(), staged[: count * frame_bytes]) + except BaseException as exc: + self._error = exc + while self._queue.get() is not None: + pass + + def _release(self) -> None: + self._stderr.close() + _remove_quietly(self._wav_path) + self._wav_path = None def _try_save_cuda_video_direct( @@ -366,173 +679,48 @@ def _try_save_cuda_video_direct( x264_preset: Optional[str] = None, ) -> bool: """Stream CUDA RGB chunks to ffmpeg through a registered memfd.""" - if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"): - return False - - sample_without_audio, audio = _split_sample_audio(sample) + video, audio = _split_sample_audio(sample) if not ( - isinstance(sample_without_audio, torch.Tensor) - and sample_without_audio.device.type == "cuda" - and sample_without_audio.dim() in (3, 4) + isinstance(video, torch.Tensor) + and video.device.type == "cuda" + and video.dim() in (3, 4) ): return False - if os.path.splitext(save_file_path)[1].lower() != ".mp4": - return False - - video = sample_without_audio if video.dim() == 3: video = video.unsqueeze(1) if video.shape[0] != 3: return False _, num_frames, height, width = video.shape - chunk_frames = _cuda_video_conversion_chunk_frames(video) - - quality = output_compression / 10 if output_compression is not None else 5 - if not 1 <= quality <= 10: - return False - crf = int((1 - quality / 10.0) * 51) - - audio_np = _normalize_audio_to_numpy(audio) - tmp_wav_path = None + encoder = None try: - if audio_np is not None: - if scipy_wavfile is None: - return False - selected_sr = _pick_audio_sample_rate( - audio_np=audio_np, - audio_sample_rate=audio_sample_rate, - fps=fps, - num_frames=num_frames, - ) - with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f: - tmp_wav_path = f.name - scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np) - - ffmpeg_exe = _resolve_ffmpeg_exe() - command = [ - ffmpeg_exe, - "-y", - "-f", - "rawvideo", - "-vcodec", - "rawvideo", - "-s", - f"{width}x{height}", - "-pix_fmt", - "rgb24", - "-r", - f"{fps:.02f}", - "-i", - "pipe:0", - ] - if tmp_wav_path is None: - command += ["-an"] - else: - command += ["-i", tmp_wav_path] - command += [ - "-vcodec", - "libx264", - "-preset", - x264_preset or X264_PRESET, - "-pix_fmt", - "yuv420p", - "-crf", - str(crf), - ] - - macro_block_size = 16 - if width % macro_block_size or height % macro_block_size: - output_width = ( - width - if width % macro_block_size == 0 - else width + macro_block_size - width % macro_block_size - ) - output_height = ( - height - if height % macro_block_size == 0 - else height + macro_block_size - height % macro_block_size - ) - command += ["-vf", f"scale={output_width}:{output_height}"] - - command += ["-threads", str(_x264_auto_thread_count(height))] - if tmp_wav_path is not None: - command += [ - "-acodec", - "aac", - "-map", - "0:v:0", - "-map", - "1:a:0", - ] - command += ["-v", "warning", save_file_path] - - with tempfile.TemporaryFile() as stderr_file: - process = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.DEVNULL, - stderr=stderr_file, - ) - try: - if process.stdin is None: - raise RuntimeError("ffmpeg stdin pipe was not created") - with _acquire_cuda_video_buffer( - (chunk_frames, height, width, 3) - ) as buffer: - assert buffer.tensor is not None - for start in range(0, num_frames, chunk_frames): - end = min(start + chunk_frames, num_frames) - with maybe_record_function( - f"VIDEO_CHUNK frames {start}-{end} convert+pipe_to_x264" - ): - frames = ( - (video[:, start:end] * 255) - .clamp_(0, 255) - .to(torch.uint8) - ) - frames = frames.permute(1, 2, 3, 0).contiguous() - buffer.tensor[: end - start].copy_( - frames, non_blocking=True - ) - torch.cuda.current_stream(video.device).synchronize() - del frames - _sendfile_all( - process.stdin.fileno(), - buffer.fd, - (end - start) * height * width * 3, - ) - with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"): - process.stdin.close() - process.stdin = None - returncode = process.wait() - finally: - if process.stdin is not None: - process.stdin.close() - if process.poll() is None: - process.kill() - process.wait() - if returncode: - stderr_file.seek(0) - raise subprocess.CalledProcessError( - returncode, - command, - stderr=stderr_file.read(), - ) + encoder = CudaVideoEncoder.open( + save_file_path, + device=video.device, + height=height, + width=width, + fps=fps, + num_frames=num_frames, + element_size=video.element_size(), + audio=audio, + audio_sample_rate=audio_sample_rate, + output_compression=output_compression, + x264_preset=x264_preset, + ) + if encoder is None: + return False + encoder.write(video) + encoder.close() return True except Exception: + if encoder is not None: + encoder.abort() logger.warning_once( "Direct CUDA video save failed; falling back to imageio. " "Enable debug logging for exception details." ) logger.debug("Direct CUDA video save failure", exc_info=True) return False - finally: - if tmp_wav_path: - try: - os.remove(tmp_wav_path) - except OSError: - pass def _try_save_cuda_videos_direct( diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/common.py b/python/sglang/multimodal_gen/runtime/models/vaes/common.py index bc31fc4023..ab613cbcf2 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/common.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/common.py @@ -83,6 +83,8 @@ def should_run_spatial_shard_parallel_decode( class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin): layerwise_offload_dit_group_enabled = False + # decode(z, on_frames=...) hands out finished frames while it decodes + supports_decode_on_frames = False layer_names = [ "encoder.down_blocks", "decoder.up_blocks", diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py index ce2f89154c..3f11f2684a 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/minimax_h3_video_vae/klvae.py @@ -664,7 +664,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): return int(total_frames), int(pad_frames), int(total_frames - pad_frames) def _decode_temporal_streaming( - self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype + self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames ): total_frames, pad_frames, output_frames = ( self._decode_temporal_output_frame_plan( @@ -704,6 +704,8 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): part[:, :, :copy_frames, :, :] ) write_pos += copy_frames + if on_frames is not None: + on_frames(dec[:, :, write_pos - copy_frames : write_pos]) dropped_frames += part_frames - copy_frames for i in range(num_chunks): @@ -778,7 +780,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): return dec - def decode_temporal(self, z): + def decode_temporal(self, z, on_frames=None): chunk_dec = self.tokens_chunk_size * self.vae_ratio_t isolated_token_num = 0 @@ -818,7 +820,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): temporal_cat_dtype = _resolve_temporal_cat_dtype() if not self.training and _resolve_temporal_stream_cat(): return self._decode_temporal_streaming( - z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype + z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames ) decoded_tasks = [] @@ -889,16 +891,23 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin): if pad_frames > 0: dec = dec[:, :, :-pad_frames, :, :] + if on_frames is not None: + on_frames(dec) return dec - def decode_base(self, z, frame_num=None, process_image=False): + def decode_base(self, z, frame_num=None, process_image=False, on_frames=None): + """``on_frames`` receives every output frame once, in order, as soon as it is final.""" + if on_frames is not None and ( + process_image or not self.use_3d_conv or frame_num is not None + ): + raise ValueError("on_frames needs an untrimmed temporal video decode") if process_image or not self.use_3d_conv: if not self.use_3d_conv and z.ndim == 5: z = z.squeeze(2) recon = self._adaptive_decode(z) else: - recon = self.decode_temporal(z) + recon = self.decode_temporal(z, on_frames=on_frames) if self.use_3d_conv: if frame_num is not None: diff --git a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py index 45bb722e26..6a58fc8a81 100644 --- a/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py +++ b/python/sglang/multimodal_gen/runtime/models/vaes/wanvae.py @@ -1514,6 +1514,7 @@ class AutoencoderKLWan(ParallelTiledVAE): """ _supports_gradient_checkpointing = False + supports_decode_on_frames = True def __init__( self, @@ -1692,7 +1693,8 @@ class AutoencoderKLWan(ParallelTiledVAE): enc = torch.cat([first_frame, enc], dim=2) return enc - def decode(self, z: torch.Tensor) -> torch.Tensor: + def decode(self, z: torch.Tensor, on_frames=None) -> torch.Tensor: + """``on_frames`` receives every returned frame once, in order, as soon as it is final.""" if self.use_feature_cache: self.clear_cache() iter_ = z.shape[2] @@ -1711,6 +1713,10 @@ class AutoencoderKLWan(ParallelTiledVAE): feat_idx.set(0) first_chunk.set(i == 0) out_chunks.append(self.decoder(x[:, :, i : i + 1, :, :])) + # with the causal cache these frames are final, and full + # height even under spatial-parallel decode + if on_frames is not None: + on_frames(self._output_frames(out_chunks[-1])) out = ( torch.cat(out_chunks, 2) if len(out_chunks) > 1 @@ -1725,9 +1731,18 @@ class AutoencoderKLWan(ParallelTiledVAE): self.clear_cache() else: out = ParallelTiledVAE.decode(self, z) + # tiled and whole-clip decodes only finish frames at the end + if on_frames is not None: + on_frames(out) return out + def _output_frames(self, out: torch.Tensor) -> torch.Tensor: + """What ``decode`` returns for decoder output ``out``, leaving ``out`` intact.""" + if self.config.patch_size is not None: + out = unpatchify(out, patch_size=self.config.patch_size) + return out.float().clamp(min=-1.0, max=1.0) + def _decode(self, z: torch.Tensor, first_frame=False) -> torch.Tensor: x = self.post_quant_conv(z) spatial_context = ( diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py index 8410a2d3dd..8e9cc4987a 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/decoding.py @@ -31,6 +31,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( PipelineStage, StageParallelismType, ) +from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( + EncodeWhileDecoding, + streamed_video_output_path, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( VerificationResult, ) @@ -98,6 +102,10 @@ def _ensure_tensor_decode_output(decode_output): return decode_output +def _to_unit_range(image: torch.Tensor) -> torch.Tensor: + return (image / 2 + 0.5).clamp(0, 1) + + class DecodingStage(PipelineStage): """ Stage for decoding latent representations into pixel space. @@ -208,6 +216,7 @@ class DecodingStage(PipelineStage): server_args: ServerArgs, *, vae_dtype: torch.dtype, + on_frames=None, ) -> torch.Tensor: """ Decode latent representations into pixel space using VAE. @@ -220,6 +229,8 @@ class DecodingStage(PipelineStage): VAE precision ("fp32", "fp16", "bf16") - pipeline_config.vae_precision: fallback VAE precision - pipeline_config.vae_tiling: Whether to enable VAE tiling for memory efficiency + on_frames: For a VAE with ``supports_decode_on_frames``, receives the + returned frames once, in order, in the pieces the VAE finishes them. Returns: Decoded video tensor with shape (batch, channels, frames, height, width), @@ -257,11 +268,18 @@ class DecodingStage(PipelineStage): should_cast_vae = not vae_autocast_enabled if not vae_autocast_enabled: latents = latents.to(vae_dtype) + decode_kwargs = {} + if on_frames is not None: + decode_kwargs["on_frames"] = lambda frames: on_frames( + _to_unit_range(frames) + ) with temporary_module_dtype( self.vae, vae_dtype, enabled=should_cast_vae ) as vae: try: - decode_output = self._get_vae_decode_fn(vae, server_args)(latents) + decode_output = self._get_vae_decode_fn(vae, server_args)( + latents, **decode_kwargs + ) except Exception as error: if "out of memory" in str(error).lower(): # decode runs after denoising, so the DiT and encoders @@ -287,7 +305,7 @@ class DecodingStage(PipelineStage): image = _ensure_tensor_decode_output(decode_output) # De-normalize image to [0, 1] range - image = (image / 2 + 0.5).clamp(0, 1) + image = _to_unit_range(image) return image def load_model(self): @@ -305,6 +323,26 @@ class DecodingStage(PipelineStage): pipeline.add_module(self.component_name, self.vae) self.server_args.model_loaded[self.component_name] = True + def _encode_while_decoding( + self, batch: Req, server_args: ServerArgs + ) -> EncodeWhileDecoding | None: + """An encoder to feed during the decode, if the frames go to the file as decoded.""" + if ( + type(self).forward is not DecodingStage.forward + or type(self).decode is not DecodingStage.decode + or not isinstance(self.vae, ParallelTiledVAE) + or not self.vae.supports_decode_on_frames + or batch.return_trajectory_decoded + or not isinstance(batch.latents, torch.Tensor) + ): + return None + save_file_path = streamed_video_output_path( + batch, server_args, int(batch.latents.shape[0]) + ) + if save_file_path is None: + return None + return EncodeWhileDecoding(save_file_path, batch) + @torch.no_grad() def forward( self, @@ -327,6 +365,8 @@ class DecodingStage(PipelineStage): self.component_name, quality=batch.sampling_params.quality, ) + stream = self._encode_while_decoding(batch, server_args) + output_file_paths = None with self.use_declared_component( component_name=self.component_name, module=self.vae, @@ -340,7 +380,24 @@ class DecodingStage(PipelineStage): vae, quality_allows(batch.sampling_params.quality, "lossless"), ): - frames = self.decode(batch.latents, server_args, vae_dtype=vae_dtype) + try: + if stream is None: + # Model-specific decode overrides need not support streaming. + frames = self.decode( + batch.latents, server_args, vae_dtype=vae_dtype + ) + else: + frames = self.decode( + batch.latents, + server_args, + vae_dtype=vae_dtype, + on_frames=stream, + ) + output_file_paths = stream.finish() + except BaseException: + if stream is not None: + stream.abort() + raise # decode trajectory latents if needed if batch.return_trajectory_decoded: @@ -369,11 +426,15 @@ class DecodingStage(PipelineStage): else: trajectory_decoded = None - frames = server_args.pipeline_config.post_decoding(frames, server_args) + if output_file_paths is None: + frames = server_args.pipeline_config.post_decoding(frames, server_args) + else: + frames = None # Update batch with decoded image output_batch = OutputBatch( output=frames, + output_file_paths=output_file_paths, trajectory_timesteps=batch.trajectory_timesteps, trajectory_latents=batch.trajectory_latents, rollout_trajectory_data=batch.rollout_trajectory_data, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py new file mode 100644 index 0000000000..870061f437 --- /dev/null +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/encode_while_decoding.py @@ -0,0 +1,130 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Encode a video request's MP4 while its decoding stage is still producing frames.""" + +from __future__ import annotations + +import os +from collections.abc import Callable + +import torch + +from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.distributed import ( + get_replica_group, + model_parallel_is_initialized, +) +from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req +from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.multimodal_gen.runtime.server_args import ServerArgs +from sglang.multimodal_gen.runtime.utils.logging_utils import CYAN, RESET, init_logger + +logger = init_logger(__name__) + +# Converted frames the encoder may hold on the GPU before the decode waits for it. +_STREAMED_VIDEO_QUEUE_BYTES = 256 * 1024 * 1024 + + +def streamed_video_output_path( + batch: Req, server_args: ServerArgs, batch_size: int +) -> str | None: + """Where the output rank can write the MP4 during the decode, if anywhere.""" + if not ( + batch.save_output + and batch.return_file_paths_only + and not batch.return_raw_frames + and batch.data_type == DataType.VIDEO + and not batch.enable_frame_interpolation + and not batch.enable_upscaling + and batch_size == 1 + and not (batch.extra or {}).get("dynamic_batch_output_paths") + and not server_args.enable_torch_compile + and type(server_args.pipeline_config).post_decoding + is PipelineConfig.post_decoding + and current_platform.is_cuda() + ): + return None + if model_parallel_is_initialized() and get_replica_group().rank_in_group != 0: + return None + return batch.output_file_path(1, 0) + + +class EncodeWhileDecoding: + """Feed each run of finished frames to the MP4 encoder while the VAE decodes the rest.""" + + def __init__( + self, + save_file_path: str, + batch: Req, + finish_frames: Callable[[torch.Tensor], torch.Tensor] | None = None, + audio=None, + audio_sample_rate: int | None = None, + ): + self.save_file_path = save_file_path + self._batch = batch + self._finish_frames = finish_frames + self._audio = audio + self._audio_sample_rate = audio_sample_rate + self._encoder = None + self._failed = False + + def __call__(self, frames: torch.Tensor) -> None: + """Encode frames that are [1, 3, t, H, W] in [0, 1] after ``finish_frames``.""" + if self._failed: + return + try: + if self._finish_frames is not None: + frames = self._finish_frames(frames) + frames = frames[0] + if self._encoder is None: + self._encoder = self._open(frames) + if self._encoder is None: + self._failed = True + return + self._encoder.write(frames) + except Exception: + self._fail() + + def _open(self, frames: torch.Tensor): + from sglang.multimodal_gen.runtime.entrypoints.utils import CudaVideoEncoder + + height, width = int(frames.shape[-2]), int(frames.shape[-1]) + os.makedirs(os.path.dirname(self.save_file_path) or ".", exist_ok=True) + return CudaVideoEncoder.open( + self.save_file_path, + device=frames.device, + height=height, + width=width, + fps=self._batch.fps, + num_frames=self._batch.num_frames or int(frames.shape[1]), + audio=self._audio, + audio_sample_rate=self._audio_sample_rate, + output_compression=self._batch.output_compression, + x264_preset=self._batch.x264_preset, + max_queued_frames=_STREAMED_VIDEO_QUEUE_BYTES // (3 * height * width), + ) + + def finish(self) -> list[str] | None: + """The saved path, or None when the worker has to save the frames instead.""" + if self._failed or self._encoder is None: + return None + try: + self._encoder.close() + except Exception: + self._fail() + return None + logger.info(f"Output saved to {CYAN}{self.save_file_path}{RESET}") + return [self.save_file_path] + + def abort(self) -> None: + if self._encoder is not None: + self._encoder.abort() + + def _fail(self) -> None: + self._failed = True + self.abort() + logger.warning_once( + "Encoding the video while it decodes failed; saving it after the " + "decode instead. Enable debug logging for exception details." + ) + logger.debug("Streamed video encode failure", exc_info=True) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py index e35af98f38..7a5c0052c3 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/decoding.py @@ -21,6 +21,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.base import ( StageParallelismType, ) from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( + EncodeWhileDecoding, + streamed_video_output_path, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import ( StageValidators as V, ) @@ -279,11 +283,11 @@ class MiniMaxH3DecodingStage(DecodingStage): audio_vae_dtype = resolve_precision( server_args, "audio_vae", precision_attr="audio_vae_precision" ) - uses = [ + # forward decodes the audio first so the MP4 encoder can start with it + return [ + ComponentUse(stage_name, "audio_vae", target_dtype=audio_vae_dtype), ComponentUse(stage_name, "video_vae", target_dtype=video_vae_dtype), ] - uses.append(ComponentUse(stage_name, "audio_vae", target_dtype=audio_vae_dtype)) - return uses def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult: result = VerificationResult() @@ -299,7 +303,8 @@ class MiniMaxH3DecodingStage(DecodingStage): self, batch: OutputBatch, server_args: ServerArgs ) -> VerificationResult: result = VerificationResult() - result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)]) + if not batch.output_file_paths: + result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)]) result.add_check("audio", batch.audio, [V.is_tensor, V.with_dims(3)]) result.add_check("audio_sample_rate", batch.audio_sample_rate, V.positive_int) return result @@ -354,6 +359,64 @@ class MiniMaxH3DecodingStage(DecodingStage): "sample_rate": int(audio_vae.sample_rate), } + def _decode_replica_audio( + self, audio_latent: torch.Tensor, server_args: ServerArgs + ) -> tuple[torch.Tensor, int]: + # Audio VAE weights are replicated. Decode on replica rank 0 and broadcast + # only within the request's replica, excluding independent DP replicas. + replica_group = get_replica_group() if model_parallel_is_initialized() else None + is_audio_owner = replica_group is None or replica_group.rank_in_group == 0 + owner_exception = None + owner_error = None + audio_payload = None + if is_audio_owner: + try: + audio_payload = self._decode_audio(audio_latent, server_args) + except Exception as exc: + owner_exception = exc + owner_error = f"{type(exc).__name__}: {exc}" + if replica_group is not None: + owner_error = replica_group.broadcast_object(owner_error, src=0) + if owner_error is not None: + if owner_exception is not None: + raise owner_exception + raise RuntimeError( + f"MiniMax H3 audio decode failed on rank 0: {owner_error}" + ) + if replica_group is not None: + audio_payload = replica_group.broadcast_tensor_dict(audio_payload, src=0) + if not isinstance(audio_payload, dict): + raise RuntimeError("MiniMax H3 audio decode produced no output payload") + audio_waveform = _required_tensor( + audio_payload.get("waveform"), "audio_vae.decode" + ) + return audio_waveform, int(audio_payload["sample_rate"]) + + @staticmethod + def _finish_visual_frames( + batch: Req, + video_vae, + frames: torch.Tensor, + *, + batch_size: int, + runtime_owned: bool = False, + ) -> torch.Tensor: + frames = _required_tensor( + video_vae.processor.revert_tensor(frames, runtime_owned=runtime_owned), + "video_vae.processor.revert_tensor", + ) + frames = _canonical_visual_video_frames(frames, batch_size=batch_size) + frames = _crop_to_target_canvas(batch, frames) + if frames.dtype != torch.float32 or not frames.is_contiguous(): + canonical_frames = torch.empty_like( + frames, + dtype=torch.float32, + memory_format=torch.contiguous_format, + ) + canonical_frames.copy_(frames) + frames = canonical_frames + return frames + @torch.no_grad() def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch: _minimax_h3_decoder_task(batch) @@ -368,6 +431,19 @@ class MiniMaxH3DecodingStage(DecodingStage): if self.video_vae is None: raise RuntimeError("MiniMax H3 tasks require the video_vae output decoder") + batch_size = int(visual_latent.shape[0]) + audio_waveform, audio_sample_rate = self._decode_replica_audio( + audio_latent, server_args + ) + output_audio_waveform = _canonical_output_audio_waveform( + audio_waveform, batch_size=batch_size + ) + streamed_output_path = streamed_video_output_path( + batch, server_args, batch_size + ) + + visual_frames = None + output_file_paths = None with self.use_declared_component( component_name="video_vae", module=self.video_vae, @@ -400,69 +476,44 @@ class MiniMaxH3DecodingStage(DecodingStage): server_args, decode_fn=selected_video_vae.decode_base, ) - with set_forward_context(current_timestep=0, attn_metadata=None): - visual_frames = video_decode(visual_decode_latent) - visual_frames = selected_video_vae.processor.revert_tensor( - visual_frames, runtime_owned=True + finish_frames = functools.partial( + self._finish_visual_frames, + batch, + selected_video_vae, + batch_size=batch_size, ) - visual_frames = _required_tensor( - visual_frames, - "video_vae.processor.revert_tensor", - ) - visual_frames = _canonical_visual_video_frames( - visual_frames, batch_size=int(visual_latent.shape[0]) - ) - visual_frames = _crop_to_target_canvas(batch, visual_frames) - if ( - visual_frames.dtype != torch.float32 - or not visual_frames.is_contiguous() - ): - canonical_frames = torch.empty_like( - visual_frames, - dtype=torch.float32, - memory_format=torch.contiguous_format, + stream = None + decode_kwargs = {} + if streamed_output_path is not None: + stream = EncodeWhileDecoding( + streamed_output_path, + batch, + finish_frames, + output_audio_waveform[0], + audio_sample_rate, ) - canonical_frames.copy_(visual_frames) - visual_frames = canonical_frames + decode_kwargs["on_frames"] = stream + try: + with set_forward_context(current_timestep=0, attn_metadata=None): + decoded = video_decode(visual_decode_latent, **decode_kwargs) + if stream is not None: + output_file_paths = stream.finish() + except BaseException: + if stream is not None: + stream.abort() + raise + if output_file_paths is None: + # callbacks borrow slices of decoded; only the final output owns it + visual_frames = finish_frames(decoded, runtime_owned=True) + del decoded - # Audio VAE weights are replicated. Decode on replica rank 0 and broadcast - # only within the request's replica, excluding independent DP replicas. - replica_group = get_replica_group() if model_parallel_is_initialized() else None - is_audio_owner = replica_group is None or replica_group.rank_in_group == 0 - owner_exception = None - owner_error = None - audio_payload = None - if is_audio_owner: - try: - audio_payload = self._decode_audio(audio_latent, server_args) - except Exception as exc: - owner_exception = exc - owner_error = f"{type(exc).__name__}: {exc}" - if replica_group is not None: - owner_error = replica_group.broadcast_object(owner_error, src=0) - if owner_error is not None: - if owner_exception is not None: - raise owner_exception - raise RuntimeError( - f"MiniMax H3 audio decode failed on rank 0: {owner_error}" + if visual_frames is not None: + visual_frames = server_args.pipeline_config.post_decoding( + visual_frames, server_args ) - if replica_group is not None: - audio_payload = replica_group.broadcast_tensor_dict(audio_payload, src=0) - if not isinstance(audio_payload, dict): - raise RuntimeError("MiniMax H3 audio decode produced no output payload") - audio_waveform = _required_tensor( - audio_payload.get("waveform"), "audio_vae.decode" - ) - audio_sample_rate = int(audio_payload["sample_rate"]) - - visual_frames = server_args.pipeline_config.post_decoding( - visual_frames, server_args - ) - output_audio_waveform = _canonical_output_audio_waveform( - audio_waveform, batch_size=int(visual_frames.shape[0]) - ) return OutputBatch( output=visual_frames, + output_file_paths=output_file_paths, audio=output_audio_waveform, audio_sample_rate=audio_sample_rate, trajectory_timesteps=batch.trajectory_timesteps, diff --git a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json index b3cde42662..9234454785 100644 --- a/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json +++ b/python/sglang/multimodal_gen/test/server/perf_baselines/h100.json @@ -1559,7 +1559,7 @@ "LatentPreparationStage": 0.11, "TimestepPreparationStage": 2.13, "DenoisingStage": 12704.94, - "DecodingStage": 1098.67 + "DecodingStage": 2172.17 }, "denoise_step_ms": { "0": 152.87, @@ -1613,7 +1613,7 @@ "48": 257.43, "49": 254.6 }, - "expected_e2e_ms": 14853.56, + "expected_e2e_ms": 15599.56, "expected_avg_denoise_ms": 253.29, "expected_median_denoise_ms": 256.21, "load_peak_vram_mb": 17610.0, @@ -1761,14 +1761,14 @@ "TimestepPreparationStage": 30.08, "LatentPreparationStage": 0.1, "DmdDenoisingStage": 252.46, - "DecodingStage": 1073.1 + "DecodingStage": 2173.4 }, "denoise_step_ms": { "0": 35.0, "1": 25.92, "2": 112.67 }, - "expected_e2e_ms": 2444.17, + "expected_e2e_ms": 3073.51, "expected_avg_denoise_ms": 83.86, "expected_median_denoise_ms": 112.67, "load_peak_vram_mb": 17610.0, diff --git a/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py b/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py index c6d1455153..4f957107d3 100644 --- a/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py +++ b/python/sglang/multimodal_gen/test/unit/sana_wm/test_pipeline_config.py @@ -4,6 +4,7 @@ import sys import tempfile import types import unittest +from contextlib import nullcontext from types import SimpleNamespace from unittest.mock import patch @@ -1512,6 +1513,51 @@ class TestSanaWMRefinerStage(_GlobalStageArgsMixin, unittest.TestCase): self.assertTrue(torch.equal(frames, decoded)) + def test_refiner_forward_uses_non_streaming_decode_override(self) -> None: + decoded = torch.arange(1 * 3 * 4 * 2 * 2).reshape(1, 3, 4, 2, 2) + decoding_module = "sglang.multimodal_gen.runtime.pipelines_core.stages.decoding" + for applied in (True, False): + with self.subTest(refiner_applied=applied): + vae = SimpleNamespace() + stage = SanaWMRefinerDecodingStage(vae=vae) + batch = SimpleNamespace( + latents=torch.empty(1, 128, 4, 2, 2), + sampling_params=SanaWMSamplingParams(), + extra={"sana_wm_refiner_applied": applied}, + return_trajectory_decoded=False, + trajectory_timesteps=None, + trajectory_latents=None, + rollout_trajectory_data=None, + metrics=None, + usage=None, + ) + args = SimpleNamespace(pipeline_config=SanaWMPipelineConfig()) + with ( + patch.object(stage, "load_model"), + patch.object( + stage, "use_declared_component", return_value=nullcontext(vae) + ), + patch( + f"{decoding_module}.resolve_decode_precision", + return_value=torch.bfloat16, + ), + patch( + f"{decoding_module}.use_vae_fast_path", + return_value=nullcontext(), + ), + patch.object(DecodingStage, "decode", return_value=decoded), + patch.object( + args.pipeline_config, + "post_decoding", + side_effect=lambda frames, _: frames, + ), + ): + output = stage.forward(batch, args) + + expected = decoded[:, :, 1:] if applied else decoded + self.assertTrue(torch.equal(output.output, expected)) + self.assertIsNone(output.output_file_paths) + if __name__ == "__main__": unittest.main() diff --git a/python/sglang/multimodal_gen/test/unit/test_minimax_h3_streamed_encode.py b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_streamed_encode.py new file mode 100644 index 0000000000..d1c099e9d2 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_minimax_h3_streamed_encode.py @@ -0,0 +1,204 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Encoding MiniMax-H3 video while it decodes must not change the MP4.""" + +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import torch + +from sglang.multimodal_gen.runtime.entrypoints.utils import ( + CudaVideoEncoder, + _try_save_cuda_video_direct, +) +from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae import ( + AutoencoderKLLegacy, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.decoding import ( + MiniMaxH3DecodingStage, +) +from sglang.test.test_utils import CustomTestCase + +FPS = 24 +SAMPLE_RATE = 24000 + + +def _temporal_vae() -> AutoencoderKLLegacy: + """The released temporal chunking, with each clip decoded by repetition.""" + vae = AutoencoderKLLegacy.__new__(AutoencoderKLLegacy) + torch.nn.Module.__init__(vae) + vae.vae_ratio = 16 + vae.vae_ratio_t = 4 + vae.use_3d_conv = True + vae.transform = None + vae.transform_rev = None + vae.transform_rev_inplace = None + vae.setup_forward(clip_length=17, token_drop=3) + vae._adaptive_decode = lambda clip_z: clip_z.repeat_interleave(4, dim=2) + return vae.eval() + + +class TestDecodeHandsOutFinishedFrames(CustomTestCase): + def test_finished_callbacks_preserve_the_decode_buffer_for_fallback(self): + for device in ("cpu", "cuda"): + for stream_cat in ("1", "0"): + with ( + self.subTest(device=device, stream_cat=stream_cat), + mock.patch.dict( + os.environ, + {"MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": stream_cat}, + ), + ): + vae = _temporal_vae() + latents = torch.randn(1, 3, 31, 2, 2, device=device) + expected_decode = vae.decode_base(latents) + batch = SimpleNamespace(extra={}) + parts = [] + + def finish_frames(frames): + parts.append( + MiniMaxH3DecodingStage._finish_visual_frames( + batch, vae, frames, batch_size=1 + ) + ) + + decoded = vae.decode_base(latents, on_frames=finish_frames) + torch.testing.assert_close(decoded, expected_decode, rtol=0, atol=0) + expected_output = vae.processor.revert_tensor(expected_decode) + torch.testing.assert_close( + torch.cat(parts, dim=2), expected_output, rtol=0, atol=0 + ) + fallback = MiniMaxH3DecodingStage._finish_visual_frames( + batch, vae, decoded, batch_size=1, runtime_owned=True + ) + torch.testing.assert_close( + fallback, expected_output, rtol=0, atol=0 + ) + + def test_every_frame_arrives_once_in_order_and_final(self): + vae = _temporal_vae() + latents = torch.randn(1, 3, 31, 2, 2) + for stream_cat in ("1", "0"): + with ( + self.subTest(stream_cat=stream_cat), + mock.patch.dict( + os.environ, + {"MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": stream_cat}, + ), + ): + parts = [] + video = vae.decode_base( + latents, on_frames=lambda frames: parts.append(frames.clone()) + ) + self.assertTrue(torch.equal(torch.cat(parts, dim=2), video)) + self.assertEqual(len(parts) > 1, stream_cat == "1") + + def test_trimmed_decodes_refuse_the_callback(self): + with self.assertRaises(ValueError): + _temporal_vae().decode_base( + torch.randn(1, 3, 31, 2, 2), frame_num=100, on_frames=print + ) + + +@unittest.skipUnless( + torch.cuda.is_available() and hasattr(os, "memfd_create"), + "needs CUDA and memfd", +) +class TestStreamedEncode(CustomTestCase): + @staticmethod + def _video(height: int, width: int) -> torch.Tensor: + generator = torch.Generator(device="cuda").manual_seed(0) + return torch.rand(3, 30, height, width, device="cuda", generator=generator) + + def test_writes_in_pieces_match_the_one_shot_save(self): + audio = torch.sin(torch.arange(2 * 2 * SAMPLE_RATE) / 7).reshape(2, -1) + # 70x90 is not a macroblock multiple, so it also covers the scale filter + for height, width, with_audio in ((64, 96, False), (70, 90, True)): + video = self._video(height, width) + sample_audio = audio if with_audio else None + sample_rate = SAMPLE_RATE if with_audio else None + with ( + self.subTest(size=(height, width)), + tempfile.TemporaryDirectory() as tmp, + ): + one_shot, streamed = ( + Path(tmp, "one_shot.mp4"), + Path(tmp, "streamed.mp4"), + ) + self.assertTrue( + _try_save_cuda_video_direct( + save_file_path=str(one_shot), + sample=(video, sample_audio) if with_audio else video, + fps=FPS, + audio_sample_rate=sample_rate, + output_compression=None, + ) + ) + encoder = CudaVideoEncoder.open( + str(streamed), + device=video.device, + height=height, + width=width, + fps=FPS, + num_frames=video.shape[1], + audio=sample_audio, + audio_sample_rate=sample_rate, + max_queued_frames=64, + ) + for start, end in ((0, 1), (1, 6), (6, 25), (25, 30)): + encoder.write(video[:, start:end]) + encoder.close() + self.assertEqual(one_shot.read_bytes(), streamed.read_bytes()) + + def test_concurrent_encodes_both_finish(self): + # only one staging buffer is cached, so the other one is closed on release + video = self._video(64, 96) + with tempfile.TemporaryDirectory() as tmp: + paths = [Path(tmp, f"clip_{i}.mp4") for i in range(2)] + encoders = [ + CudaVideoEncoder.open( + str(path), + device=video.device, + height=64, + width=96, + fps=FPS, + num_frames=video.shape[1], + ) + for path in paths + ] + for encoder in encoders: + encoder.write(video) + for encoder in encoders: + encoder.close() + self.assertEqual(paths[0].read_bytes(), paths[1].read_bytes()) + + def test_a_failed_encode_leaves_no_file(self): + video = self._video(64, 96) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp, "clip.mp4") + encoder = CudaVideoEncoder.open( + str(path), + device=video.device, + height=64, + width=96, + fps=FPS, + num_frames=video.shape[1], + ) + encoder.write(video[:, :10]) + encoder._process.kill() + encoder._process.wait() + try: + encoder.write(video[:, 10:]) + except RuntimeError: + pass + with self.assertRaises((RuntimeError, subprocess.CalledProcessError)): + encoder.close() + self.assertFalse(path.exists()) + + +if __name__ == "__main__": + unittest.main(verbosity=3) diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py b/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py new file mode 100644 index 0000000000..b371e9ad8d --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_wan_streamed_encode.py @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Encoding Wan video while its VAE decodes must not change the MP4. + +The frame hand-off itself is covered in +python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py. +""" + +import os +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest import mock + +import torch + +from sglang.multimodal_gen.configs.models.vaes.wanvae import ( + WanVAEArchConfig, + WanVAEConfig, +) +from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig +from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.runtime.entrypoints.utils import _try_save_cuda_video_direct +from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan +from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage +from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import ( + EncodeWhileDecoding, +) +from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.test.test_utils import CustomTestCase + +FPS = 16 +Z_DIM = 4 + + +def _vae_config(*, wan22: bool) -> WanVAEConfig: + """A tiny Wan 2.1 VAE, or with ``wan22`` the 2.2 residual blocks and 2x2 patches.""" + channels = 12 if wan22 else 3 + return WanVAEConfig( + arch_config=WanVAEArchConfig( + base_dim=8, + z_dim=Z_DIM, + dim_mult=(1, 2, 2, 2), + num_res_blocks=1, + latents_mean=(0.0,) * Z_DIM, + latents_std=(0.5,) * Z_DIM, + is_residual=wan22, + in_channels=channels, + out_channels=channels, + patch_size=2 if wan22 else None, + ), + load_encoder=False, + ) + + +def _vae(config: WanVAEConfig, device: str = "cpu") -> AutoencoderKLWan: + torch.manual_seed(0) + return AutoencoderKLWan(config).to(device).eval() + + +def _server_args(pipeline_config=None) -> SimpleNamespace: + return SimpleNamespace( + pipeline_config=pipeline_config or WanT2V480PConfig(), + disable_autocast=False, + enable_torch_compile=False, + ) + + +def _stage(vae, server_args, stage_cls=DecodingStage) -> DecodingStage: + with mock.patch( + "sglang.multimodal_gen.runtime.pipelines_core.stages.base.get_global_server_args", + return_value=server_args, + ): + return stage_cls(vae) + + +def _video_request(**overrides) -> SimpleNamespace: + fields = dict( + save_output=True, + return_file_paths_only=True, + return_raw_frames=False, + return_trajectory_decoded=False, + data_type=DataType.VIDEO, + enable_frame_interpolation=False, + enable_upscaling=False, + extra={}, + latents=torch.zeros(1, Z_DIM, 2, 2, 2), + fps=FPS, + num_frames=5, + output_compression=None, + x264_preset=None, + output_file_path=lambda num_outputs, output_idx: "/tmp/clip.mp4", + ) + fields.update(overrides) + return SimpleNamespace(**fields) + + +@unittest.skipUnless( + torch.cuda.is_available() and hasattr(os, "memfd_create"), + "needs CUDA and memfd", +) +class TestStreamedEncode(CustomTestCase): + @torch.no_grad() + def test_streamed_mp4_matches_the_one_shot_save(self): + generator = torch.Generator(device="cuda").manual_seed(0) + # Wan 2.1 decodes under bf16 autocast, Wan 2.2 in fp32; 40x48 is not a + # macroblock multiple, so this also covers the scale filter + for wan22, vae_dtype, latent_size in ( + (False, torch.bfloat16, (5, 6)), + (True, torch.float32, (3, 4)), + ): + config = _vae_config(wan22=wan22) + server_args = _server_args(WanT2V480PConfig(vae_config=config)) + stage = _stage(_vae(config, "cuda"), server_args) + latents = torch.randn( + 1, Z_DIM, 6, *latent_size, device="cuda", generator=generator + ) + with ( + self.subTest(wan22=wan22), + tempfile.TemporaryDirectory() as tmp, + ): + one_shot, streamed = ( + Path(tmp, "one_shot.mp4"), + Path(tmp, "streamed.mp4"), + ) + stream = EncodeWhileDecoding( + str(streamed), _video_request(num_frames=21) + ) + frames = stage.decode( + latents, server_args, vae_dtype=vae_dtype, on_frames=stream + ) + self.assertEqual(stream.finish(), [str(streamed)]) + self.assertEqual(tuple(frames.shape[1:3]), (3, 21)) + self.assertTrue( + _try_save_cuda_video_direct( + save_file_path=str(one_shot), + sample=frames[0], + fps=FPS, + audio_sample_rate=None, + output_compression=None, + ) + ) + self.assertEqual(one_shot.read_bytes(), streamed.read_bytes()) + + @unittest.skipUnless( + current_platform.is_cuda(), "streaming is gated on the CUDA platform, not ROCm" + ) + def test_only_file_path_video_requests_stream(self): + server_args = _server_args() + stage = _stage(_vae(_vae_config(wan22=False)), server_args) + self.assertIsInstance( + stage._encode_while_decoding(_video_request(), server_args), + EncodeWhileDecoding, + ) + for field, value in ( + ("return_file_paths_only", False), + ("return_raw_frames", True), + ("return_trajectory_decoded", True), + ("data_type", DataType.IMAGE), + ("enable_frame_interpolation", True), + ("enable_upscaling", True), + ("extra", {"dynamic_batch_output_paths": ["a.mp4"]}), + ("latents", torch.zeros(2, Z_DIM, 2, 2, 2)), + ): + with self.subTest(field=field): + request = _video_request(**{field: value}) + self.assertIsNone(stage._encode_while_decoding(request, server_args)) + + class OwnPostDecoding(WanT2V480PConfig): + def post_decoding(self, frames, server_args): + return frames + + class OwnDecode(DecodingStage): + def decode(self, latents, server_args, *, vae_dtype): + return latents + + for name, candidate, args in ( + ("post_decoding", stage, _server_args(OwnPostDecoding())), + ( + "torch_compile", + stage, + SimpleNamespace(**{**vars(server_args), "enable_torch_compile": True}), + ), + ("stage_decode", _stage(stage.vae, server_args, OwnDecode), server_args), + ("vae", _stage(torch.nn.Identity(), server_args), server_args), + ): + with self.subTest(name=name): + self.assertIsNone( + candidate._encode_while_decoding(_video_request(), args) + ) + + +if __name__ == "__main__": + unittest.main(verbosity=3) diff --git a/python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py b/python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py new file mode 100644 index 0000000000..0662dafaae --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +"""AutoencoderKLWan.decode hands out every frame once, in order, once it is final.""" + +import pytest +import torch + +from sglang.multimodal_gen.configs.models.vaes.wanvae import ( + WanVAEArchConfig, + WanVAEConfig, +) +from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan + +Z_DIM = 4 + + +def _vae(*, wan22: bool) -> AutoencoderKLWan: + """A tiny Wan 2.1 VAE, or with ``wan22`` the 2.2 residual blocks and 2x2 patches.""" + channels = 12 if wan22 else 3 + config = WanVAEConfig( + arch_config=WanVAEArchConfig( + base_dim=8, + z_dim=Z_DIM, + dim_mult=(1, 2, 2, 2), + num_res_blocks=1, + latents_mean=(0.0,) * Z_DIM, + latents_std=(1.0,) * Z_DIM, + is_residual=wan22, + in_channels=channels, + out_channels=channels, + patch_size=2 if wan22 else None, + ), + load_encoder=False, + ) + torch.manual_seed(0) + return AutoencoderKLWan(config).eval() + + +@pytest.mark.parametrize("wan22", [False, True]) +def test_causal_decode_hands_out_each_latent_frames_pixels(wan22): + vae = _vae(wan22=wan22) + latents = torch.randn(1, Z_DIM, 4, 2, 3) + parts = [] + with torch.no_grad(): + video = vae.decode(latents, on_frames=parts.append) + reference = vae.decode(latents) + + assert torch.equal(video, reference) + assert [part.shape[2] for part in parts] == [1, 4, 4, 4] + assert torch.equal(torch.cat(parts, dim=2), video) + + +def test_whole_clip_decode_hands_over_everything_at_the_end(): + vae = _vae(wan22=False) + vae.use_feature_cache = False + parts = [] + with torch.no_grad(): + video = vae.decode(torch.randn(1, Z_DIM, 3, 2, 3), on_frames=parts.append) + + assert len(parts) == 1 + assert parts[0] is video