[diffusion] feat: encode the MP4 while the VAE decodes (MiniMax-H3, Wan) (#41819)
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -12,6 +12,7 @@ import atexit
|
||||
import json
|
||||
import mmap
|
||||
import os
|
||||
import queue
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -62,7 +63,7 @@ _cached_cuda_video_buffer: "_CudaMemfdVideoBuffer | None" = None
|
||||
|
||||
|
||||
class _CudaMemfdVideoBuffer:
|
||||
"""CUDA-registered memfd used as direct ffmpeg raw-video input."""
|
||||
"""CUDA-registered memfd mapping that stages raw frames for ffmpeg."""
|
||||
|
||||
def __init__(self, shape: tuple[int, ...]):
|
||||
self.shape = shape
|
||||
@@ -337,23 +338,335 @@ def _x264_auto_thread_count(height: int) -> int:
|
||||
return min(cpu_limit, row_limit, 128)
|
||||
|
||||
|
||||
def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int:
|
||||
_, num_frames, height, width = video.shape
|
||||
temporary_bytes_per_frame = 3 * height * width * (video.element_size() + 2)
|
||||
def _conversion_chunk_frames(
|
||||
num_frames: int, height: int, width: int, element_size: int
|
||||
) -> int:
|
||||
temporary_bytes_per_frame = 3 * height * width * (element_size + 2)
|
||||
return min(
|
||||
num_frames,
|
||||
max(1, _MAX_CUDA_VIDEO_CONVERSION_CHUNK_BYTES // temporary_bytes_per_frame),
|
||||
)
|
||||
|
||||
|
||||
def _sendfile_all(output_fd: int, input_fd: int, count: int) -> None:
|
||||
offset = 0
|
||||
while count:
|
||||
sent = os.sendfile(output_fd, input_fd, offset, count)
|
||||
if sent <= 0:
|
||||
raise RuntimeError("sendfile made no progress")
|
||||
offset += sent
|
||||
count -= sent
|
||||
def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int:
|
||||
_, num_frames, height, width = video.shape
|
||||
return _conversion_chunk_frames(num_frames, height, width, video.element_size())
|
||||
|
||||
|
||||
def _write_all(fd: int, data: memoryview) -> None:
|
||||
while data:
|
||||
data = data[os.write(fd, data) :]
|
||||
|
||||
|
||||
def _remove_quietly(path: Optional[str]) -> None:
|
||||
if path:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _cuda_video_ffmpeg_command(
|
||||
save_file_path: str,
|
||||
*,
|
||||
width: int,
|
||||
height: int,
|
||||
fps: int,
|
||||
crf: int,
|
||||
x264_preset: Optional[str],
|
||||
wav_path: Optional[str],
|
||||
) -> list[str]:
|
||||
command = [
|
||||
_resolve_ffmpeg_exe(),
|
||||
"-y",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-vcodec",
|
||||
"rawvideo",
|
||||
"-s",
|
||||
f"{width}x{height}",
|
||||
"-pix_fmt",
|
||||
"rgb24",
|
||||
"-r",
|
||||
f"{fps:.02f}",
|
||||
"-i",
|
||||
"pipe:0",
|
||||
]
|
||||
if wav_path is None:
|
||||
command += ["-an"]
|
||||
else:
|
||||
command += ["-i", wav_path]
|
||||
command += [
|
||||
"-vcodec",
|
||||
"libx264",
|
||||
"-preset",
|
||||
x264_preset or X264_PRESET,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-crf",
|
||||
str(crf),
|
||||
]
|
||||
|
||||
macro_block_size = 16
|
||||
if width % macro_block_size or height % macro_block_size:
|
||||
output_width = (
|
||||
width
|
||||
if width % macro_block_size == 0
|
||||
else width + macro_block_size - width % macro_block_size
|
||||
)
|
||||
output_height = (
|
||||
height
|
||||
if height % macro_block_size == 0
|
||||
else height + macro_block_size - height % macro_block_size
|
||||
)
|
||||
command += ["-vf", f"scale={output_width}:{output_height}"]
|
||||
|
||||
command += ["-threads", str(_x264_auto_thread_count(height))]
|
||||
if wav_path is not None:
|
||||
command += [
|
||||
"-acodec",
|
||||
"aac",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
]
|
||||
command += ["-v", "warning", save_file_path]
|
||||
return command
|
||||
|
||||
|
||||
class CudaVideoEncoder:
|
||||
"""Encode [3, T, H, W] CUDA frames in [0, 1] to an MP4 as they arrive.
|
||||
|
||||
``write`` only queues the uint8 conversion on the current stream; a writer
|
||||
thread stages each chunk in registered host memory and pipes it to ffmpeg,
|
||||
so a caller that produces frames in pieces keeps its own GPU work going.
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def open(
|
||||
cls,
|
||||
save_file_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
height: int,
|
||||
width: int,
|
||||
fps: int,
|
||||
num_frames: int,
|
||||
element_size: int = 4,
|
||||
audio: Any = None,
|
||||
audio_sample_rate: Optional[int] = None,
|
||||
output_compression: Optional[int] = None,
|
||||
x264_preset: Optional[str] = None,
|
||||
max_queued_frames: int = 0,
|
||||
) -> "CudaVideoEncoder | None":
|
||||
"""Start ffmpeg, or return None when this output cannot use the path."""
|
||||
if not hasattr(os, "memfd_create") or device.type != "cuda":
|
||||
return None
|
||||
if os.path.splitext(save_file_path)[1].lower() != ".mp4":
|
||||
return None
|
||||
quality = output_compression / 10 if output_compression is not None else 5
|
||||
if not 1 <= quality <= 10:
|
||||
return None
|
||||
crf = int((1 - quality / 10.0) * 51)
|
||||
audio_np = _normalize_audio_to_numpy(audio)
|
||||
if audio_np is not None and scipy_wavfile is None:
|
||||
return None
|
||||
|
||||
wav_path = None
|
||||
try:
|
||||
if audio_np is not None:
|
||||
selected_sr = _pick_audio_sample_rate(
|
||||
audio_np=audio_np,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
fps=fps,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
||||
wav_path = f.name
|
||||
scipy_wavfile.write(wav_path, selected_sr, audio_np)
|
||||
command = _cuda_video_ffmpeg_command(
|
||||
save_file_path,
|
||||
width=width,
|
||||
height=height,
|
||||
fps=fps,
|
||||
crf=crf,
|
||||
x264_preset=x264_preset,
|
||||
wav_path=wav_path,
|
||||
)
|
||||
chunk_frames = _conversion_chunk_frames(
|
||||
max(1, num_frames), height, width, element_size
|
||||
)
|
||||
return cls(
|
||||
command,
|
||||
save_file_path,
|
||||
device=device,
|
||||
height=height,
|
||||
width=width,
|
||||
chunk_frames=chunk_frames,
|
||||
queued_chunks=max(1, -(-max_queued_frames // chunk_frames)),
|
||||
wav_path=wav_path,
|
||||
)
|
||||
except BaseException:
|
||||
_remove_quietly(wav_path)
|
||||
raise
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
command: list[str],
|
||||
save_file_path: str,
|
||||
*,
|
||||
device: torch.device,
|
||||
height: int,
|
||||
width: int,
|
||||
chunk_frames: int,
|
||||
queued_chunks: int,
|
||||
wav_path: Optional[str],
|
||||
):
|
||||
self.save_file_path = save_file_path
|
||||
self._command = command
|
||||
self._device = device
|
||||
self._height = height
|
||||
self._width = width
|
||||
self._chunk_frames = chunk_frames
|
||||
self._wav_path = wav_path
|
||||
self._frames_queued = 0
|
||||
self._error: BaseException | None = None
|
||||
self._finished = False
|
||||
self._queue: queue.Queue = queue.Queue(maxsize=queued_chunks)
|
||||
self._stderr = tempfile.TemporaryFile()
|
||||
try:
|
||||
self._process = subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=self._stderr,
|
||||
)
|
||||
except BaseException:
|
||||
self._stderr.close()
|
||||
raise
|
||||
self._thread = threading.Thread(
|
||||
target=self._pipe_frames, name="sglang-video-encoder", daemon=True
|
||||
)
|
||||
self._thread.start()
|
||||
|
||||
def write(self, frames: torch.Tensor) -> None:
|
||||
"""Queue [3, t, H, W] frames; raise once the encoder has failed."""
|
||||
if frames.dim() == 3:
|
||||
frames = frames.unsqueeze(1)
|
||||
if (
|
||||
frames.device != self._device
|
||||
or int(frames.shape[0]) != 3
|
||||
or tuple(frames.shape[2:]) != (self._height, self._width)
|
||||
):
|
||||
raise ValueError(
|
||||
f"expected [3, t, {self._height}, {self._width}] frames on "
|
||||
f"{self._device}, got {tuple(frames.shape)} on {frames.device}"
|
||||
)
|
||||
stream = torch.cuda.current_stream(frames.device)
|
||||
num_frames = int(frames.shape[1])
|
||||
for start in range(0, num_frames, self._chunk_frames):
|
||||
end = min(start + self._chunk_frames, num_frames)
|
||||
first = self._frames_queued
|
||||
with maybe_record_function(
|
||||
f"VIDEO_CHUNK frames {first}-{first + end - start} convert"
|
||||
):
|
||||
chunk = (frames[:, start:end] * 255).clamp_(0, 255).to(torch.uint8)
|
||||
chunk = chunk.permute(1, 2, 3, 0).contiguous()
|
||||
ready = torch.cuda.Event()
|
||||
ready.record(stream)
|
||||
self._put((chunk, ready))
|
||||
self._frames_queued += end - start
|
||||
|
||||
def close(self) -> None:
|
||||
"""Finish the MP4; on failure clean up as ``abort`` does and raise."""
|
||||
if self._finished:
|
||||
return
|
||||
try:
|
||||
with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"):
|
||||
self._stop_thread()
|
||||
if self._error is not None:
|
||||
if self._process.poll():
|
||||
raise self._ffmpeg_error() from self._error
|
||||
raise RuntimeError("video encoder failed") from self._error
|
||||
self._process.stdin.close()
|
||||
if self._process.wait():
|
||||
raise self._ffmpeg_error()
|
||||
except BaseException:
|
||||
self.abort()
|
||||
raise
|
||||
self._finished = True
|
||||
self._release()
|
||||
|
||||
def abort(self) -> None:
|
||||
"""Stop ffmpeg and remove the partial file."""
|
||||
if self._finished:
|
||||
return
|
||||
self._finished = True
|
||||
if self._process.poll() is None:
|
||||
self._process.kill()
|
||||
self._stop_thread()
|
||||
try:
|
||||
self._process.stdin.close()
|
||||
except OSError:
|
||||
pass
|
||||
self._process.wait()
|
||||
self._release()
|
||||
_remove_quietly(self.save_file_path)
|
||||
|
||||
def _ffmpeg_error(self) -> subprocess.CalledProcessError:
|
||||
self._stderr.seek(0)
|
||||
return subprocess.CalledProcessError(
|
||||
self._process.returncode, self._command, stderr=self._stderr.read()
|
||||
)
|
||||
|
||||
def _put(self, item) -> None:
|
||||
# never blocks for good: after a failure the writer keeps draining
|
||||
if self._error is not None:
|
||||
raise RuntimeError("video encoder failed") from self._error
|
||||
self._queue.put(item)
|
||||
|
||||
def _stop_thread(self) -> None:
|
||||
self._queue.put(None)
|
||||
self._thread.join()
|
||||
|
||||
def _pipe_frames(self) -> None:
|
||||
frame_bytes = self._height * self._width * 3
|
||||
try:
|
||||
stdin = self._process.stdin
|
||||
if stdin is None:
|
||||
raise RuntimeError("ffmpeg stdin pipe was not created")
|
||||
with (
|
||||
torch.cuda.device(self._device),
|
||||
_acquire_cuda_video_buffer(
|
||||
(self._chunk_frames, self._height, self._width, 3)
|
||||
) as buffer,
|
||||
memoryview(buffer.array).cast("B") as staged,
|
||||
):
|
||||
assert buffer.tensor is not None
|
||||
copy_stream = torch.cuda.Stream()
|
||||
while (item := self._queue.get()) is not None:
|
||||
chunk, ready = item
|
||||
del item
|
||||
count = int(chunk.shape[0])
|
||||
copy_stream.wait_event(ready)
|
||||
with torch.cuda.stream(copy_stream):
|
||||
buffer.tensor[:count].copy_(chunk, non_blocking=True)
|
||||
copy_stream.synchronize()
|
||||
# the copy is done, so the allocator may hand this block out again
|
||||
del chunk, ready
|
||||
# write() copies into the pipe; sendfile would only lend it these
|
||||
# pages, and the next chunk overwrites them before ffmpeg reads
|
||||
_write_all(stdin.fileno(), staged[: count * frame_bytes])
|
||||
except BaseException as exc:
|
||||
self._error = exc
|
||||
while self._queue.get() is not None:
|
||||
pass
|
||||
|
||||
def _release(self) -> None:
|
||||
self._stderr.close()
|
||||
_remove_quietly(self._wav_path)
|
||||
self._wav_path = None
|
||||
|
||||
|
||||
def _try_save_cuda_video_direct(
|
||||
@@ -366,173 +679,48 @@ def _try_save_cuda_video_direct(
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Stream CUDA RGB chunks to ffmpeg through a registered memfd."""
|
||||
if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"):
|
||||
return False
|
||||
|
||||
sample_without_audio, audio = _split_sample_audio(sample)
|
||||
video, audio = _split_sample_audio(sample)
|
||||
if not (
|
||||
isinstance(sample_without_audio, torch.Tensor)
|
||||
and sample_without_audio.device.type == "cuda"
|
||||
and sample_without_audio.dim() in (3, 4)
|
||||
isinstance(video, torch.Tensor)
|
||||
and video.device.type == "cuda"
|
||||
and video.dim() in (3, 4)
|
||||
):
|
||||
return False
|
||||
if os.path.splitext(save_file_path)[1].lower() != ".mp4":
|
||||
return False
|
||||
|
||||
video = sample_without_audio
|
||||
if video.dim() == 3:
|
||||
video = video.unsqueeze(1)
|
||||
if video.shape[0] != 3:
|
||||
return False
|
||||
|
||||
_, num_frames, height, width = video.shape
|
||||
chunk_frames = _cuda_video_conversion_chunk_frames(video)
|
||||
|
||||
quality = output_compression / 10 if output_compression is not None else 5
|
||||
if not 1 <= quality <= 10:
|
||||
return False
|
||||
crf = int((1 - quality / 10.0) * 51)
|
||||
|
||||
audio_np = _normalize_audio_to_numpy(audio)
|
||||
tmp_wav_path = None
|
||||
encoder = None
|
||||
try:
|
||||
if audio_np is not None:
|
||||
if scipy_wavfile is None:
|
||||
return False
|
||||
selected_sr = _pick_audio_sample_rate(
|
||||
audio_np=audio_np,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
fps=fps,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
||||
tmp_wav_path = f.name
|
||||
scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np)
|
||||
|
||||
ffmpeg_exe = _resolve_ffmpeg_exe()
|
||||
command = [
|
||||
ffmpeg_exe,
|
||||
"-y",
|
||||
"-f",
|
||||
"rawvideo",
|
||||
"-vcodec",
|
||||
"rawvideo",
|
||||
"-s",
|
||||
f"{width}x{height}",
|
||||
"-pix_fmt",
|
||||
"rgb24",
|
||||
"-r",
|
||||
f"{fps:.02f}",
|
||||
"-i",
|
||||
"pipe:0",
|
||||
]
|
||||
if tmp_wav_path is None:
|
||||
command += ["-an"]
|
||||
else:
|
||||
command += ["-i", tmp_wav_path]
|
||||
command += [
|
||||
"-vcodec",
|
||||
"libx264",
|
||||
"-preset",
|
||||
x264_preset or X264_PRESET,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-crf",
|
||||
str(crf),
|
||||
]
|
||||
|
||||
macro_block_size = 16
|
||||
if width % macro_block_size or height % macro_block_size:
|
||||
output_width = (
|
||||
width
|
||||
if width % macro_block_size == 0
|
||||
else width + macro_block_size - width % macro_block_size
|
||||
)
|
||||
output_height = (
|
||||
height
|
||||
if height % macro_block_size == 0
|
||||
else height + macro_block_size - height % macro_block_size
|
||||
)
|
||||
command += ["-vf", f"scale={output_width}:{output_height}"]
|
||||
|
||||
command += ["-threads", str(_x264_auto_thread_count(height))]
|
||||
if tmp_wav_path is not None:
|
||||
command += [
|
||||
"-acodec",
|
||||
"aac",
|
||||
"-map",
|
||||
"0:v:0",
|
||||
"-map",
|
||||
"1:a:0",
|
||||
]
|
||||
command += ["-v", "warning", save_file_path]
|
||||
|
||||
with tempfile.TemporaryFile() as stderr_file:
|
||||
process = subprocess.Popen(
|
||||
command,
|
||||
stdin=subprocess.PIPE,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=stderr_file,
|
||||
)
|
||||
try:
|
||||
if process.stdin is None:
|
||||
raise RuntimeError("ffmpeg stdin pipe was not created")
|
||||
with _acquire_cuda_video_buffer(
|
||||
(chunk_frames, height, width, 3)
|
||||
) as buffer:
|
||||
assert buffer.tensor is not None
|
||||
for start in range(0, num_frames, chunk_frames):
|
||||
end = min(start + chunk_frames, num_frames)
|
||||
with maybe_record_function(
|
||||
f"VIDEO_CHUNK frames {start}-{end} convert+pipe_to_x264"
|
||||
):
|
||||
frames = (
|
||||
(video[:, start:end] * 255)
|
||||
.clamp_(0, 255)
|
||||
.to(torch.uint8)
|
||||
)
|
||||
frames = frames.permute(1, 2, 3, 0).contiguous()
|
||||
buffer.tensor[: end - start].copy_(
|
||||
frames, non_blocking=True
|
||||
)
|
||||
torch.cuda.current_stream(video.device).synchronize()
|
||||
del frames
|
||||
_sendfile_all(
|
||||
process.stdin.fileno(),
|
||||
buffer.fd,
|
||||
(end - start) * height * width * 3,
|
||||
)
|
||||
with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"):
|
||||
process.stdin.close()
|
||||
process.stdin = None
|
||||
returncode = process.wait()
|
||||
finally:
|
||||
if process.stdin is not None:
|
||||
process.stdin.close()
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
process.wait()
|
||||
if returncode:
|
||||
stderr_file.seek(0)
|
||||
raise subprocess.CalledProcessError(
|
||||
returncode,
|
||||
command,
|
||||
stderr=stderr_file.read(),
|
||||
)
|
||||
encoder = CudaVideoEncoder.open(
|
||||
save_file_path,
|
||||
device=video.device,
|
||||
height=height,
|
||||
width=width,
|
||||
fps=fps,
|
||||
num_frames=num_frames,
|
||||
element_size=video.element_size(),
|
||||
audio=audio,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_compression=output_compression,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
if encoder is None:
|
||||
return False
|
||||
encoder.write(video)
|
||||
encoder.close()
|
||||
return True
|
||||
except Exception:
|
||||
if encoder is not None:
|
||||
encoder.abort()
|
||||
logger.warning_once(
|
||||
"Direct CUDA video save failed; falling back to imageio. "
|
||||
"Enable debug logging for exception details."
|
||||
)
|
||||
logger.debug("Direct CUDA video save failure", exc_info=True)
|
||||
return False
|
||||
finally:
|
||||
if tmp_wav_path:
|
||||
try:
|
||||
os.remove(tmp_wav_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _try_save_cuda_videos_direct(
|
||||
|
||||
@@ -83,6 +83,8 @@ def should_run_spatial_shard_parallel_decode(
|
||||
|
||||
class ParallelTiledVAE(ABC, nn.Module, LayerwiseOffloadableModuleMixin):
|
||||
layerwise_offload_dit_group_enabled = False
|
||||
# decode(z, on_frames=...) hands out finished frames while it decodes
|
||||
supports_decode_on_frames = False
|
||||
layer_names = [
|
||||
"encoder.down_blocks",
|
||||
"decoder.up_blocks",
|
||||
|
||||
@@ -664,7 +664,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
return int(total_frames), int(pad_frames), int(total_frames - pad_frames)
|
||||
|
||||
def _decode_temporal_streaming(
|
||||
self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype
|
||||
self, z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames
|
||||
):
|
||||
total_frames, pad_frames, output_frames = (
|
||||
self._decode_temporal_output_frame_plan(
|
||||
@@ -704,6 +704,8 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
part[:, :, :copy_frames, :, :]
|
||||
)
|
||||
write_pos += copy_frames
|
||||
if on_frames is not None:
|
||||
on_frames(dec[:, :, write_pos - copy_frames : write_pos])
|
||||
dropped_frames += part_frames - copy_frames
|
||||
|
||||
for i in range(num_chunks):
|
||||
@@ -778,7 +780,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
|
||||
return dec
|
||||
|
||||
def decode_temporal(self, z):
|
||||
def decode_temporal(self, z, on_frames=None):
|
||||
chunk_dec = self.tokens_chunk_size * self.vae_ratio_t
|
||||
|
||||
isolated_token_num = 0
|
||||
@@ -818,7 +820,7 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
temporal_cat_dtype = _resolve_temporal_cat_dtype()
|
||||
if not self.training and _resolve_temporal_stream_cat():
|
||||
return self._decode_temporal_streaming(
|
||||
z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype
|
||||
z, z_head, z_tail, num_chunks, pad_tokens, temporal_cat_dtype, on_frames
|
||||
)
|
||||
|
||||
decoded_tasks = []
|
||||
@@ -889,16 +891,23 @@ class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalModelMixin):
|
||||
if pad_frames > 0:
|
||||
dec = dec[:, :, :-pad_frames, :, :]
|
||||
|
||||
if on_frames is not None:
|
||||
on_frames(dec)
|
||||
return dec
|
||||
|
||||
def decode_base(self, z, frame_num=None, process_image=False):
|
||||
def decode_base(self, z, frame_num=None, process_image=False, on_frames=None):
|
||||
"""``on_frames`` receives every output frame once, in order, as soon as it is final."""
|
||||
if on_frames is not None and (
|
||||
process_image or not self.use_3d_conv or frame_num is not None
|
||||
):
|
||||
raise ValueError("on_frames needs an untrimmed temporal video decode")
|
||||
if process_image or not self.use_3d_conv:
|
||||
if not self.use_3d_conv and z.ndim == 5:
|
||||
z = z.squeeze(2)
|
||||
|
||||
recon = self._adaptive_decode(z)
|
||||
else:
|
||||
recon = self.decode_temporal(z)
|
||||
recon = self.decode_temporal(z, on_frames=on_frames)
|
||||
|
||||
if self.use_3d_conv:
|
||||
if frame_num is not None:
|
||||
|
||||
@@ -1514,6 +1514,7 @@ class AutoencoderKLWan(ParallelTiledVAE):
|
||||
"""
|
||||
|
||||
_supports_gradient_checkpointing = False
|
||||
supports_decode_on_frames = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -1692,7 +1693,8 @@ class AutoencoderKLWan(ParallelTiledVAE):
|
||||
enc = torch.cat([first_frame, enc], dim=2)
|
||||
return enc
|
||||
|
||||
def decode(self, z: torch.Tensor) -> torch.Tensor:
|
||||
def decode(self, z: torch.Tensor, on_frames=None) -> torch.Tensor:
|
||||
"""``on_frames`` receives every returned frame once, in order, as soon as it is final."""
|
||||
if self.use_feature_cache:
|
||||
self.clear_cache()
|
||||
iter_ = z.shape[2]
|
||||
@@ -1711,6 +1713,10 @@ class AutoencoderKLWan(ParallelTiledVAE):
|
||||
feat_idx.set(0)
|
||||
first_chunk.set(i == 0)
|
||||
out_chunks.append(self.decoder(x[:, :, i : i + 1, :, :]))
|
||||
# with the causal cache these frames are final, and full
|
||||
# height even under spatial-parallel decode
|
||||
if on_frames is not None:
|
||||
on_frames(self._output_frames(out_chunks[-1]))
|
||||
out = (
|
||||
torch.cat(out_chunks, 2)
|
||||
if len(out_chunks) > 1
|
||||
@@ -1725,9 +1731,18 @@ class AutoencoderKLWan(ParallelTiledVAE):
|
||||
self.clear_cache()
|
||||
else:
|
||||
out = ParallelTiledVAE.decode(self, z)
|
||||
# tiled and whole-clip decodes only finish frames at the end
|
||||
if on_frames is not None:
|
||||
on_frames(out)
|
||||
|
||||
return out
|
||||
|
||||
def _output_frames(self, out: torch.Tensor) -> torch.Tensor:
|
||||
"""What ``decode`` returns for decoder output ``out``, leaving ``out`` intact."""
|
||||
if self.config.patch_size is not None:
|
||||
out = unpatchify(out, patch_size=self.config.patch_size)
|
||||
return out.float().clamp(min=-1.0, max=1.0)
|
||||
|
||||
def _decode(self, z: torch.Tensor, first_frame=False) -> torch.Tensor:
|
||||
x = self.post_quant_conv(z)
|
||||
spatial_context = (
|
||||
|
||||
@@ -31,6 +31,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
|
||||
PipelineStage,
|
||||
StageParallelismType,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
EncodeWhileDecoding,
|
||||
streamed_video_output_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
VerificationResult,
|
||||
)
|
||||
@@ -98,6 +102,10 @@ def _ensure_tensor_decode_output(decode_output):
|
||||
return decode_output
|
||||
|
||||
|
||||
def _to_unit_range(image: torch.Tensor) -> torch.Tensor:
|
||||
return (image / 2 + 0.5).clamp(0, 1)
|
||||
|
||||
|
||||
class DecodingStage(PipelineStage):
|
||||
"""
|
||||
Stage for decoding latent representations into pixel space.
|
||||
@@ -208,6 +216,7 @@ class DecodingStage(PipelineStage):
|
||||
server_args: ServerArgs,
|
||||
*,
|
||||
vae_dtype: torch.dtype,
|
||||
on_frames=None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Decode latent representations into pixel space using VAE.
|
||||
@@ -220,6 +229,8 @@ class DecodingStage(PipelineStage):
|
||||
VAE precision ("fp32", "fp16", "bf16")
|
||||
- pipeline_config.vae_precision: fallback VAE precision
|
||||
- pipeline_config.vae_tiling: Whether to enable VAE tiling for memory efficiency
|
||||
on_frames: For a VAE with ``supports_decode_on_frames``, receives the
|
||||
returned frames once, in order, in the pieces the VAE finishes them.
|
||||
|
||||
Returns:
|
||||
Decoded video tensor with shape (batch, channels, frames, height, width),
|
||||
@@ -257,11 +268,18 @@ class DecodingStage(PipelineStage):
|
||||
should_cast_vae = not vae_autocast_enabled
|
||||
if not vae_autocast_enabled:
|
||||
latents = latents.to(vae_dtype)
|
||||
decode_kwargs = {}
|
||||
if on_frames is not None:
|
||||
decode_kwargs["on_frames"] = lambda frames: on_frames(
|
||||
_to_unit_range(frames)
|
||||
)
|
||||
with temporary_module_dtype(
|
||||
self.vae, vae_dtype, enabled=should_cast_vae
|
||||
) as vae:
|
||||
try:
|
||||
decode_output = self._get_vae_decode_fn(vae, server_args)(latents)
|
||||
decode_output = self._get_vae_decode_fn(vae, server_args)(
|
||||
latents, **decode_kwargs
|
||||
)
|
||||
except Exception as error:
|
||||
if "out of memory" in str(error).lower():
|
||||
# decode runs after denoising, so the DiT and encoders
|
||||
@@ -287,7 +305,7 @@ class DecodingStage(PipelineStage):
|
||||
image = _ensure_tensor_decode_output(decode_output)
|
||||
|
||||
# De-normalize image to [0, 1] range
|
||||
image = (image / 2 + 0.5).clamp(0, 1)
|
||||
image = _to_unit_range(image)
|
||||
return image
|
||||
|
||||
def load_model(self):
|
||||
@@ -305,6 +323,26 @@ class DecodingStage(PipelineStage):
|
||||
pipeline.add_module(self.component_name, self.vae)
|
||||
self.server_args.model_loaded[self.component_name] = True
|
||||
|
||||
def _encode_while_decoding(
|
||||
self, batch: Req, server_args: ServerArgs
|
||||
) -> EncodeWhileDecoding | None:
|
||||
"""An encoder to feed during the decode, if the frames go to the file as decoded."""
|
||||
if (
|
||||
type(self).forward is not DecodingStage.forward
|
||||
or type(self).decode is not DecodingStage.decode
|
||||
or not isinstance(self.vae, ParallelTiledVAE)
|
||||
or not self.vae.supports_decode_on_frames
|
||||
or batch.return_trajectory_decoded
|
||||
or not isinstance(batch.latents, torch.Tensor)
|
||||
):
|
||||
return None
|
||||
save_file_path = streamed_video_output_path(
|
||||
batch, server_args, int(batch.latents.shape[0])
|
||||
)
|
||||
if save_file_path is None:
|
||||
return None
|
||||
return EncodeWhileDecoding(save_file_path, batch)
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(
|
||||
self,
|
||||
@@ -327,6 +365,8 @@ class DecodingStage(PipelineStage):
|
||||
self.component_name,
|
||||
quality=batch.sampling_params.quality,
|
||||
)
|
||||
stream = self._encode_while_decoding(batch, server_args)
|
||||
output_file_paths = None
|
||||
with self.use_declared_component(
|
||||
component_name=self.component_name,
|
||||
module=self.vae,
|
||||
@@ -340,7 +380,24 @@ class DecodingStage(PipelineStage):
|
||||
vae,
|
||||
quality_allows(batch.sampling_params.quality, "lossless"),
|
||||
):
|
||||
frames = self.decode(batch.latents, server_args, vae_dtype=vae_dtype)
|
||||
try:
|
||||
if stream is None:
|
||||
# Model-specific decode overrides need not support streaming.
|
||||
frames = self.decode(
|
||||
batch.latents, server_args, vae_dtype=vae_dtype
|
||||
)
|
||||
else:
|
||||
frames = self.decode(
|
||||
batch.latents,
|
||||
server_args,
|
||||
vae_dtype=vae_dtype,
|
||||
on_frames=stream,
|
||||
)
|
||||
output_file_paths = stream.finish()
|
||||
except BaseException:
|
||||
if stream is not None:
|
||||
stream.abort()
|
||||
raise
|
||||
|
||||
# decode trajectory latents if needed
|
||||
if batch.return_trajectory_decoded:
|
||||
@@ -369,11 +426,15 @@ class DecodingStage(PipelineStage):
|
||||
else:
|
||||
trajectory_decoded = None
|
||||
|
||||
frames = server_args.pipeline_config.post_decoding(frames, server_args)
|
||||
if output_file_paths is None:
|
||||
frames = server_args.pipeline_config.post_decoding(frames, server_args)
|
||||
else:
|
||||
frames = None
|
||||
|
||||
# Update batch with decoded image
|
||||
output_batch = OutputBatch(
|
||||
output=frames,
|
||||
output_file_paths=output_file_paths,
|
||||
trajectory_timesteps=batch.trajectory_timesteps,
|
||||
trajectory_latents=batch.trajectory_latents,
|
||||
rollout_trajectory_data=batch.rollout_trajectory_data,
|
||||
|
||||
@@ -0,0 +1,130 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Encode a video request's MP4 while its decoding stage is still producing frames."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
|
||||
from sglang.multimodal_gen.runtime.distributed import (
|
||||
get_replica_group,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import CYAN, RESET, init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Converted frames the encoder may hold on the GPU before the decode waits for it.
|
||||
_STREAMED_VIDEO_QUEUE_BYTES = 256 * 1024 * 1024
|
||||
|
||||
|
||||
def streamed_video_output_path(
|
||||
batch: Req, server_args: ServerArgs, batch_size: int
|
||||
) -> str | None:
|
||||
"""Where the output rank can write the MP4 during the decode, if anywhere."""
|
||||
if not (
|
||||
batch.save_output
|
||||
and batch.return_file_paths_only
|
||||
and not batch.return_raw_frames
|
||||
and batch.data_type == DataType.VIDEO
|
||||
and not batch.enable_frame_interpolation
|
||||
and not batch.enable_upscaling
|
||||
and batch_size == 1
|
||||
and not (batch.extra or {}).get("dynamic_batch_output_paths")
|
||||
and not server_args.enable_torch_compile
|
||||
and type(server_args.pipeline_config).post_decoding
|
||||
is PipelineConfig.post_decoding
|
||||
and current_platform.is_cuda()
|
||||
):
|
||||
return None
|
||||
if model_parallel_is_initialized() and get_replica_group().rank_in_group != 0:
|
||||
return None
|
||||
return batch.output_file_path(1, 0)
|
||||
|
||||
|
||||
class EncodeWhileDecoding:
|
||||
"""Feed each run of finished frames to the MP4 encoder while the VAE decodes the rest."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
save_file_path: str,
|
||||
batch: Req,
|
||||
finish_frames: Callable[[torch.Tensor], torch.Tensor] | None = None,
|
||||
audio=None,
|
||||
audio_sample_rate: int | None = None,
|
||||
):
|
||||
self.save_file_path = save_file_path
|
||||
self._batch = batch
|
||||
self._finish_frames = finish_frames
|
||||
self._audio = audio
|
||||
self._audio_sample_rate = audio_sample_rate
|
||||
self._encoder = None
|
||||
self._failed = False
|
||||
|
||||
def __call__(self, frames: torch.Tensor) -> None:
|
||||
"""Encode frames that are [1, 3, t, H, W] in [0, 1] after ``finish_frames``."""
|
||||
if self._failed:
|
||||
return
|
||||
try:
|
||||
if self._finish_frames is not None:
|
||||
frames = self._finish_frames(frames)
|
||||
frames = frames[0]
|
||||
if self._encoder is None:
|
||||
self._encoder = self._open(frames)
|
||||
if self._encoder is None:
|
||||
self._failed = True
|
||||
return
|
||||
self._encoder.write(frames)
|
||||
except Exception:
|
||||
self._fail()
|
||||
|
||||
def _open(self, frames: torch.Tensor):
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import CudaVideoEncoder
|
||||
|
||||
height, width = int(frames.shape[-2]), int(frames.shape[-1])
|
||||
os.makedirs(os.path.dirname(self.save_file_path) or ".", exist_ok=True)
|
||||
return CudaVideoEncoder.open(
|
||||
self.save_file_path,
|
||||
device=frames.device,
|
||||
height=height,
|
||||
width=width,
|
||||
fps=self._batch.fps,
|
||||
num_frames=self._batch.num_frames or int(frames.shape[1]),
|
||||
audio=self._audio,
|
||||
audio_sample_rate=self._audio_sample_rate,
|
||||
output_compression=self._batch.output_compression,
|
||||
x264_preset=self._batch.x264_preset,
|
||||
max_queued_frames=_STREAMED_VIDEO_QUEUE_BYTES // (3 * height * width),
|
||||
)
|
||||
|
||||
def finish(self) -> list[str] | None:
|
||||
"""The saved path, or None when the worker has to save the frames instead."""
|
||||
if self._failed or self._encoder is None:
|
||||
return None
|
||||
try:
|
||||
self._encoder.close()
|
||||
except Exception:
|
||||
self._fail()
|
||||
return None
|
||||
logger.info(f"Output saved to {CYAN}{self.save_file_path}{RESET}")
|
||||
return [self.save_file_path]
|
||||
|
||||
def abort(self) -> None:
|
||||
if self._encoder is not None:
|
||||
self._encoder.abort()
|
||||
|
||||
def _fail(self) -> None:
|
||||
self._failed = True
|
||||
self.abort()
|
||||
logger.warning_once(
|
||||
"Encoding the video while it decodes failed; saving it after the "
|
||||
"decode instead. Enable debug logging for exception details."
|
||||
)
|
||||
logger.debug("Streamed video encode failure", exc_info=True)
|
||||
+112
-61
@@ -21,6 +21,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.base import (
|
||||
StageParallelismType,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
EncodeWhileDecoding,
|
||||
streamed_video_output_path,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.validators import (
|
||||
StageValidators as V,
|
||||
)
|
||||
@@ -279,11 +283,11 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
audio_vae_dtype = resolve_precision(
|
||||
server_args, "audio_vae", precision_attr="audio_vae_precision"
|
||||
)
|
||||
uses = [
|
||||
# forward decodes the audio first so the MP4 encoder can start with it
|
||||
return [
|
||||
ComponentUse(stage_name, "audio_vae", target_dtype=audio_vae_dtype),
|
||||
ComponentUse(stage_name, "video_vae", target_dtype=video_vae_dtype),
|
||||
]
|
||||
uses.append(ComponentUse(stage_name, "audio_vae", target_dtype=audio_vae_dtype))
|
||||
return uses
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
@@ -299,7 +303,8 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
self, batch: OutputBatch, server_args: ServerArgs
|
||||
) -> VerificationResult:
|
||||
result = VerificationResult()
|
||||
result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)])
|
||||
if not batch.output_file_paths:
|
||||
result.add_check("output", batch.output, [V.is_tensor, V.with_dims(5)])
|
||||
result.add_check("audio", batch.audio, [V.is_tensor, V.with_dims(3)])
|
||||
result.add_check("audio_sample_rate", batch.audio_sample_rate, V.positive_int)
|
||||
return result
|
||||
@@ -354,6 +359,64 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
"sample_rate": int(audio_vae.sample_rate),
|
||||
}
|
||||
|
||||
def _decode_replica_audio(
|
||||
self, audio_latent: torch.Tensor, server_args: ServerArgs
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
# Audio VAE weights are replicated. Decode on replica rank 0 and broadcast
|
||||
# only within the request's replica, excluding independent DP replicas.
|
||||
replica_group = get_replica_group() if model_parallel_is_initialized() else None
|
||||
is_audio_owner = replica_group is None or replica_group.rank_in_group == 0
|
||||
owner_exception = None
|
||||
owner_error = None
|
||||
audio_payload = None
|
||||
if is_audio_owner:
|
||||
try:
|
||||
audio_payload = self._decode_audio(audio_latent, server_args)
|
||||
except Exception as exc:
|
||||
owner_exception = exc
|
||||
owner_error = f"{type(exc).__name__}: {exc}"
|
||||
if replica_group is not None:
|
||||
owner_error = replica_group.broadcast_object(owner_error, src=0)
|
||||
if owner_error is not None:
|
||||
if owner_exception is not None:
|
||||
raise owner_exception
|
||||
raise RuntimeError(
|
||||
f"MiniMax H3 audio decode failed on rank 0: {owner_error}"
|
||||
)
|
||||
if replica_group is not None:
|
||||
audio_payload = replica_group.broadcast_tensor_dict(audio_payload, src=0)
|
||||
if not isinstance(audio_payload, dict):
|
||||
raise RuntimeError("MiniMax H3 audio decode produced no output payload")
|
||||
audio_waveform = _required_tensor(
|
||||
audio_payload.get("waveform"), "audio_vae.decode"
|
||||
)
|
||||
return audio_waveform, int(audio_payload["sample_rate"])
|
||||
|
||||
@staticmethod
|
||||
def _finish_visual_frames(
|
||||
batch: Req,
|
||||
video_vae,
|
||||
frames: torch.Tensor,
|
||||
*,
|
||||
batch_size: int,
|
||||
runtime_owned: bool = False,
|
||||
) -> torch.Tensor:
|
||||
frames = _required_tensor(
|
||||
video_vae.processor.revert_tensor(frames, runtime_owned=runtime_owned),
|
||||
"video_vae.processor.revert_tensor",
|
||||
)
|
||||
frames = _canonical_visual_video_frames(frames, batch_size=batch_size)
|
||||
frames = _crop_to_target_canvas(batch, frames)
|
||||
if frames.dtype != torch.float32 or not frames.is_contiguous():
|
||||
canonical_frames = torch.empty_like(
|
||||
frames,
|
||||
dtype=torch.float32,
|
||||
memory_format=torch.contiguous_format,
|
||||
)
|
||||
canonical_frames.copy_(frames)
|
||||
frames = canonical_frames
|
||||
return frames
|
||||
|
||||
@torch.no_grad()
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> OutputBatch:
|
||||
_minimax_h3_decoder_task(batch)
|
||||
@@ -368,6 +431,19 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
|
||||
if self.video_vae is None:
|
||||
raise RuntimeError("MiniMax H3 tasks require the video_vae output decoder")
|
||||
batch_size = int(visual_latent.shape[0])
|
||||
audio_waveform, audio_sample_rate = self._decode_replica_audio(
|
||||
audio_latent, server_args
|
||||
)
|
||||
output_audio_waveform = _canonical_output_audio_waveform(
|
||||
audio_waveform, batch_size=batch_size
|
||||
)
|
||||
streamed_output_path = streamed_video_output_path(
|
||||
batch, server_args, batch_size
|
||||
)
|
||||
|
||||
visual_frames = None
|
||||
output_file_paths = None
|
||||
with self.use_declared_component(
|
||||
component_name="video_vae",
|
||||
module=self.video_vae,
|
||||
@@ -400,69 +476,44 @@ class MiniMaxH3DecodingStage(DecodingStage):
|
||||
server_args,
|
||||
decode_fn=selected_video_vae.decode_base,
|
||||
)
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
visual_frames = video_decode(visual_decode_latent)
|
||||
visual_frames = selected_video_vae.processor.revert_tensor(
|
||||
visual_frames, runtime_owned=True
|
||||
finish_frames = functools.partial(
|
||||
self._finish_visual_frames,
|
||||
batch,
|
||||
selected_video_vae,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
visual_frames = _required_tensor(
|
||||
visual_frames,
|
||||
"video_vae.processor.revert_tensor",
|
||||
)
|
||||
visual_frames = _canonical_visual_video_frames(
|
||||
visual_frames, batch_size=int(visual_latent.shape[0])
|
||||
)
|
||||
visual_frames = _crop_to_target_canvas(batch, visual_frames)
|
||||
if (
|
||||
visual_frames.dtype != torch.float32
|
||||
or not visual_frames.is_contiguous()
|
||||
):
|
||||
canonical_frames = torch.empty_like(
|
||||
visual_frames,
|
||||
dtype=torch.float32,
|
||||
memory_format=torch.contiguous_format,
|
||||
stream = None
|
||||
decode_kwargs = {}
|
||||
if streamed_output_path is not None:
|
||||
stream = EncodeWhileDecoding(
|
||||
streamed_output_path,
|
||||
batch,
|
||||
finish_frames,
|
||||
output_audio_waveform[0],
|
||||
audio_sample_rate,
|
||||
)
|
||||
canonical_frames.copy_(visual_frames)
|
||||
visual_frames = canonical_frames
|
||||
decode_kwargs["on_frames"] = stream
|
||||
try:
|
||||
with set_forward_context(current_timestep=0, attn_metadata=None):
|
||||
decoded = video_decode(visual_decode_latent, **decode_kwargs)
|
||||
if stream is not None:
|
||||
output_file_paths = stream.finish()
|
||||
except BaseException:
|
||||
if stream is not None:
|
||||
stream.abort()
|
||||
raise
|
||||
if output_file_paths is None:
|
||||
# callbacks borrow slices of decoded; only the final output owns it
|
||||
visual_frames = finish_frames(decoded, runtime_owned=True)
|
||||
del decoded
|
||||
|
||||
# Audio VAE weights are replicated. Decode on replica rank 0 and broadcast
|
||||
# only within the request's replica, excluding independent DP replicas.
|
||||
replica_group = get_replica_group() if model_parallel_is_initialized() else None
|
||||
is_audio_owner = replica_group is None or replica_group.rank_in_group == 0
|
||||
owner_exception = None
|
||||
owner_error = None
|
||||
audio_payload = None
|
||||
if is_audio_owner:
|
||||
try:
|
||||
audio_payload = self._decode_audio(audio_latent, server_args)
|
||||
except Exception as exc:
|
||||
owner_exception = exc
|
||||
owner_error = f"{type(exc).__name__}: {exc}"
|
||||
if replica_group is not None:
|
||||
owner_error = replica_group.broadcast_object(owner_error, src=0)
|
||||
if owner_error is not None:
|
||||
if owner_exception is not None:
|
||||
raise owner_exception
|
||||
raise RuntimeError(
|
||||
f"MiniMax H3 audio decode failed on rank 0: {owner_error}"
|
||||
if visual_frames is not None:
|
||||
visual_frames = server_args.pipeline_config.post_decoding(
|
||||
visual_frames, server_args
|
||||
)
|
||||
if replica_group is not None:
|
||||
audio_payload = replica_group.broadcast_tensor_dict(audio_payload, src=0)
|
||||
if not isinstance(audio_payload, dict):
|
||||
raise RuntimeError("MiniMax H3 audio decode produced no output payload")
|
||||
audio_waveform = _required_tensor(
|
||||
audio_payload.get("waveform"), "audio_vae.decode"
|
||||
)
|
||||
audio_sample_rate = int(audio_payload["sample_rate"])
|
||||
|
||||
visual_frames = server_args.pipeline_config.post_decoding(
|
||||
visual_frames, server_args
|
||||
)
|
||||
output_audio_waveform = _canonical_output_audio_waveform(
|
||||
audio_waveform, batch_size=int(visual_frames.shape[0])
|
||||
)
|
||||
return OutputBatch(
|
||||
output=visual_frames,
|
||||
output_file_paths=output_file_paths,
|
||||
audio=output_audio_waveform,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
trajectory_timesteps=batch.trajectory_timesteps,
|
||||
|
||||
@@ -1559,7 +1559,7 @@
|
||||
"LatentPreparationStage": 0.11,
|
||||
"TimestepPreparationStage": 2.13,
|
||||
"DenoisingStage": 12704.94,
|
||||
"DecodingStage": 1098.67
|
||||
"DecodingStage": 2172.17
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 152.87,
|
||||
@@ -1613,7 +1613,7 @@
|
||||
"48": 257.43,
|
||||
"49": 254.6
|
||||
},
|
||||
"expected_e2e_ms": 14853.56,
|
||||
"expected_e2e_ms": 15599.56,
|
||||
"expected_avg_denoise_ms": 253.29,
|
||||
"expected_median_denoise_ms": 256.21,
|
||||
"load_peak_vram_mb": 17610.0,
|
||||
@@ -1761,14 +1761,14 @@
|
||||
"TimestepPreparationStage": 30.08,
|
||||
"LatentPreparationStage": 0.1,
|
||||
"DmdDenoisingStage": 252.46,
|
||||
"DecodingStage": 1073.1
|
||||
"DecodingStage": 2173.4
|
||||
},
|
||||
"denoise_step_ms": {
|
||||
"0": 35.0,
|
||||
"1": 25.92,
|
||||
"2": 112.67
|
||||
},
|
||||
"expected_e2e_ms": 2444.17,
|
||||
"expected_e2e_ms": 3073.51,
|
||||
"expected_avg_denoise_ms": 83.86,
|
||||
"expected_median_denoise_ms": 112.67,
|
||||
"load_peak_vram_mb": 17610.0,
|
||||
|
||||
@@ -4,6 +4,7 @@ import sys
|
||||
import tempfile
|
||||
import types
|
||||
import unittest
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -1512,6 +1513,51 @@ class TestSanaWMRefinerStage(_GlobalStageArgsMixin, unittest.TestCase):
|
||||
|
||||
self.assertTrue(torch.equal(frames, decoded))
|
||||
|
||||
def test_refiner_forward_uses_non_streaming_decode_override(self) -> None:
|
||||
decoded = torch.arange(1 * 3 * 4 * 2 * 2).reshape(1, 3, 4, 2, 2)
|
||||
decoding_module = "sglang.multimodal_gen.runtime.pipelines_core.stages.decoding"
|
||||
for applied in (True, False):
|
||||
with self.subTest(refiner_applied=applied):
|
||||
vae = SimpleNamespace()
|
||||
stage = SanaWMRefinerDecodingStage(vae=vae)
|
||||
batch = SimpleNamespace(
|
||||
latents=torch.empty(1, 128, 4, 2, 2),
|
||||
sampling_params=SanaWMSamplingParams(),
|
||||
extra={"sana_wm_refiner_applied": applied},
|
||||
return_trajectory_decoded=False,
|
||||
trajectory_timesteps=None,
|
||||
trajectory_latents=None,
|
||||
rollout_trajectory_data=None,
|
||||
metrics=None,
|
||||
usage=None,
|
||||
)
|
||||
args = SimpleNamespace(pipeline_config=SanaWMPipelineConfig())
|
||||
with (
|
||||
patch.object(stage, "load_model"),
|
||||
patch.object(
|
||||
stage, "use_declared_component", return_value=nullcontext(vae)
|
||||
),
|
||||
patch(
|
||||
f"{decoding_module}.resolve_decode_precision",
|
||||
return_value=torch.bfloat16,
|
||||
),
|
||||
patch(
|
||||
f"{decoding_module}.use_vae_fast_path",
|
||||
return_value=nullcontext(),
|
||||
),
|
||||
patch.object(DecodingStage, "decode", return_value=decoded),
|
||||
patch.object(
|
||||
args.pipeline_config,
|
||||
"post_decoding",
|
||||
side_effect=lambda frames, _: frames,
|
||||
),
|
||||
):
|
||||
output = stage.forward(batch, args)
|
||||
|
||||
expected = decoded[:, :, 1:] if applied else decoded
|
||||
self.assertTrue(torch.equal(output.output, expected))
|
||||
self.assertIsNone(output.output_file_paths)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Encoding MiniMax-H3 video while it decodes must not change the MP4."""
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
||||
CudaVideoEncoder,
|
||||
_try_save_cuda_video_direct,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.minimax_h3_video_vae import (
|
||||
AutoencoderKLLegacy,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.decoding import (
|
||||
MiniMaxH3DecodingStage,
|
||||
)
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
FPS = 24
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
def _temporal_vae() -> AutoencoderKLLegacy:
|
||||
"""The released temporal chunking, with each clip decoded by repetition."""
|
||||
vae = AutoencoderKLLegacy.__new__(AutoencoderKLLegacy)
|
||||
torch.nn.Module.__init__(vae)
|
||||
vae.vae_ratio = 16
|
||||
vae.vae_ratio_t = 4
|
||||
vae.use_3d_conv = True
|
||||
vae.transform = None
|
||||
vae.transform_rev = None
|
||||
vae.transform_rev_inplace = None
|
||||
vae.setup_forward(clip_length=17, token_drop=3)
|
||||
vae._adaptive_decode = lambda clip_z: clip_z.repeat_interleave(4, dim=2)
|
||||
return vae.eval()
|
||||
|
||||
|
||||
class TestDecodeHandsOutFinishedFrames(CustomTestCase):
|
||||
def test_finished_callbacks_preserve_the_decode_buffer_for_fallback(self):
|
||||
for device in ("cpu", "cuda"):
|
||||
for stream_cat in ("1", "0"):
|
||||
with (
|
||||
self.subTest(device=device, stream_cat=stream_cat),
|
||||
mock.patch.dict(
|
||||
os.environ,
|
||||
{"MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": stream_cat},
|
||||
),
|
||||
):
|
||||
vae = _temporal_vae()
|
||||
latents = torch.randn(1, 3, 31, 2, 2, device=device)
|
||||
expected_decode = vae.decode_base(latents)
|
||||
batch = SimpleNamespace(extra={})
|
||||
parts = []
|
||||
|
||||
def finish_frames(frames):
|
||||
parts.append(
|
||||
MiniMaxH3DecodingStage._finish_visual_frames(
|
||||
batch, vae, frames, batch_size=1
|
||||
)
|
||||
)
|
||||
|
||||
decoded = vae.decode_base(latents, on_frames=finish_frames)
|
||||
torch.testing.assert_close(decoded, expected_decode, rtol=0, atol=0)
|
||||
expected_output = vae.processor.revert_tensor(expected_decode)
|
||||
torch.testing.assert_close(
|
||||
torch.cat(parts, dim=2), expected_output, rtol=0, atol=0
|
||||
)
|
||||
fallback = MiniMaxH3DecodingStage._finish_visual_frames(
|
||||
batch, vae, decoded, batch_size=1, runtime_owned=True
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
fallback, expected_output, rtol=0, atol=0
|
||||
)
|
||||
|
||||
def test_every_frame_arrives_once_in_order_and_final(self):
|
||||
vae = _temporal_vae()
|
||||
latents = torch.randn(1, 3, 31, 2, 2)
|
||||
for stream_cat in ("1", "0"):
|
||||
with (
|
||||
self.subTest(stream_cat=stream_cat),
|
||||
mock.patch.dict(
|
||||
os.environ,
|
||||
{"MINIMAX_H3_VAE_DECODER_STREAM_TEMPORAL_CAT": stream_cat},
|
||||
),
|
||||
):
|
||||
parts = []
|
||||
video = vae.decode_base(
|
||||
latents, on_frames=lambda frames: parts.append(frames.clone())
|
||||
)
|
||||
self.assertTrue(torch.equal(torch.cat(parts, dim=2), video))
|
||||
self.assertEqual(len(parts) > 1, stream_cat == "1")
|
||||
|
||||
def test_trimmed_decodes_refuse_the_callback(self):
|
||||
with self.assertRaises(ValueError):
|
||||
_temporal_vae().decode_base(
|
||||
torch.randn(1, 3, 31, 2, 2), frame_num=100, on_frames=print
|
||||
)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
torch.cuda.is_available() and hasattr(os, "memfd_create"),
|
||||
"needs CUDA and memfd",
|
||||
)
|
||||
class TestStreamedEncode(CustomTestCase):
|
||||
@staticmethod
|
||||
def _video(height: int, width: int) -> torch.Tensor:
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
return torch.rand(3, 30, height, width, device="cuda", generator=generator)
|
||||
|
||||
def test_writes_in_pieces_match_the_one_shot_save(self):
|
||||
audio = torch.sin(torch.arange(2 * 2 * SAMPLE_RATE) / 7).reshape(2, -1)
|
||||
# 70x90 is not a macroblock multiple, so it also covers the scale filter
|
||||
for height, width, with_audio in ((64, 96, False), (70, 90, True)):
|
||||
video = self._video(height, width)
|
||||
sample_audio = audio if with_audio else None
|
||||
sample_rate = SAMPLE_RATE if with_audio else None
|
||||
with (
|
||||
self.subTest(size=(height, width)),
|
||||
tempfile.TemporaryDirectory() as tmp,
|
||||
):
|
||||
one_shot, streamed = (
|
||||
Path(tmp, "one_shot.mp4"),
|
||||
Path(tmp, "streamed.mp4"),
|
||||
)
|
||||
self.assertTrue(
|
||||
_try_save_cuda_video_direct(
|
||||
save_file_path=str(one_shot),
|
||||
sample=(video, sample_audio) if with_audio else video,
|
||||
fps=FPS,
|
||||
audio_sample_rate=sample_rate,
|
||||
output_compression=None,
|
||||
)
|
||||
)
|
||||
encoder = CudaVideoEncoder.open(
|
||||
str(streamed),
|
||||
device=video.device,
|
||||
height=height,
|
||||
width=width,
|
||||
fps=FPS,
|
||||
num_frames=video.shape[1],
|
||||
audio=sample_audio,
|
||||
audio_sample_rate=sample_rate,
|
||||
max_queued_frames=64,
|
||||
)
|
||||
for start, end in ((0, 1), (1, 6), (6, 25), (25, 30)):
|
||||
encoder.write(video[:, start:end])
|
||||
encoder.close()
|
||||
self.assertEqual(one_shot.read_bytes(), streamed.read_bytes())
|
||||
|
||||
def test_concurrent_encodes_both_finish(self):
|
||||
# only one staging buffer is cached, so the other one is closed on release
|
||||
video = self._video(64, 96)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
paths = [Path(tmp, f"clip_{i}.mp4") for i in range(2)]
|
||||
encoders = [
|
||||
CudaVideoEncoder.open(
|
||||
str(path),
|
||||
device=video.device,
|
||||
height=64,
|
||||
width=96,
|
||||
fps=FPS,
|
||||
num_frames=video.shape[1],
|
||||
)
|
||||
for path in paths
|
||||
]
|
||||
for encoder in encoders:
|
||||
encoder.write(video)
|
||||
for encoder in encoders:
|
||||
encoder.close()
|
||||
self.assertEqual(paths[0].read_bytes(), paths[1].read_bytes())
|
||||
|
||||
def test_a_failed_encode_leaves_no_file(self):
|
||||
video = self._video(64, 96)
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp, "clip.mp4")
|
||||
encoder = CudaVideoEncoder.open(
|
||||
str(path),
|
||||
device=video.device,
|
||||
height=64,
|
||||
width=96,
|
||||
fps=FPS,
|
||||
num_frames=video.shape[1],
|
||||
)
|
||||
encoder.write(video[:, :10])
|
||||
encoder._process.kill()
|
||||
encoder._process.wait()
|
||||
try:
|
||||
encoder.write(video[:, 10:])
|
||||
except RuntimeError:
|
||||
pass
|
||||
with self.assertRaises((RuntimeError, subprocess.CalledProcessError)):
|
||||
encoder.close()
|
||||
self.assertFalse(path.exists())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
@@ -0,0 +1,194 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Encoding Wan video while its VAE decodes must not change the MP4.
|
||||
|
||||
The frame hand-off itself is covered in
|
||||
python/sglang/multimodal_gen/test/unit/test_wan_vae_on_frames.py.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.wanvae import (
|
||||
WanVAEArchConfig,
|
||||
WanVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.wan import WanT2V480PConfig
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import _try_save_cuda_video_direct
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.decoding import DecodingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.encode_while_decoding import (
|
||||
EncodeWhileDecoding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
FPS = 16
|
||||
Z_DIM = 4
|
||||
|
||||
|
||||
def _vae_config(*, wan22: bool) -> WanVAEConfig:
|
||||
"""A tiny Wan 2.1 VAE, or with ``wan22`` the 2.2 residual blocks and 2x2 patches."""
|
||||
channels = 12 if wan22 else 3
|
||||
return WanVAEConfig(
|
||||
arch_config=WanVAEArchConfig(
|
||||
base_dim=8,
|
||||
z_dim=Z_DIM,
|
||||
dim_mult=(1, 2, 2, 2),
|
||||
num_res_blocks=1,
|
||||
latents_mean=(0.0,) * Z_DIM,
|
||||
latents_std=(0.5,) * Z_DIM,
|
||||
is_residual=wan22,
|
||||
in_channels=channels,
|
||||
out_channels=channels,
|
||||
patch_size=2 if wan22 else None,
|
||||
),
|
||||
load_encoder=False,
|
||||
)
|
||||
|
||||
|
||||
def _vae(config: WanVAEConfig, device: str = "cpu") -> AutoencoderKLWan:
|
||||
torch.manual_seed(0)
|
||||
return AutoencoderKLWan(config).to(device).eval()
|
||||
|
||||
|
||||
def _server_args(pipeline_config=None) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
pipeline_config=pipeline_config or WanT2V480PConfig(),
|
||||
disable_autocast=False,
|
||||
enable_torch_compile=False,
|
||||
)
|
||||
|
||||
|
||||
def _stage(vae, server_args, stage_cls=DecodingStage) -> DecodingStage:
|
||||
with mock.patch(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.stages.base.get_global_server_args",
|
||||
return_value=server_args,
|
||||
):
|
||||
return stage_cls(vae)
|
||||
|
||||
|
||||
def _video_request(**overrides) -> SimpleNamespace:
|
||||
fields = dict(
|
||||
save_output=True,
|
||||
return_file_paths_only=True,
|
||||
return_raw_frames=False,
|
||||
return_trajectory_decoded=False,
|
||||
data_type=DataType.VIDEO,
|
||||
enable_frame_interpolation=False,
|
||||
enable_upscaling=False,
|
||||
extra={},
|
||||
latents=torch.zeros(1, Z_DIM, 2, 2, 2),
|
||||
fps=FPS,
|
||||
num_frames=5,
|
||||
output_compression=None,
|
||||
x264_preset=None,
|
||||
output_file_path=lambda num_outputs, output_idx: "/tmp/clip.mp4",
|
||||
)
|
||||
fields.update(overrides)
|
||||
return SimpleNamespace(**fields)
|
||||
|
||||
|
||||
@unittest.skipUnless(
|
||||
torch.cuda.is_available() and hasattr(os, "memfd_create"),
|
||||
"needs CUDA and memfd",
|
||||
)
|
||||
class TestStreamedEncode(CustomTestCase):
|
||||
@torch.no_grad()
|
||||
def test_streamed_mp4_matches_the_one_shot_save(self):
|
||||
generator = torch.Generator(device="cuda").manual_seed(0)
|
||||
# Wan 2.1 decodes under bf16 autocast, Wan 2.2 in fp32; 40x48 is not a
|
||||
# macroblock multiple, so this also covers the scale filter
|
||||
for wan22, vae_dtype, latent_size in (
|
||||
(False, torch.bfloat16, (5, 6)),
|
||||
(True, torch.float32, (3, 4)),
|
||||
):
|
||||
config = _vae_config(wan22=wan22)
|
||||
server_args = _server_args(WanT2V480PConfig(vae_config=config))
|
||||
stage = _stage(_vae(config, "cuda"), server_args)
|
||||
latents = torch.randn(
|
||||
1, Z_DIM, 6, *latent_size, device="cuda", generator=generator
|
||||
)
|
||||
with (
|
||||
self.subTest(wan22=wan22),
|
||||
tempfile.TemporaryDirectory() as tmp,
|
||||
):
|
||||
one_shot, streamed = (
|
||||
Path(tmp, "one_shot.mp4"),
|
||||
Path(tmp, "streamed.mp4"),
|
||||
)
|
||||
stream = EncodeWhileDecoding(
|
||||
str(streamed), _video_request(num_frames=21)
|
||||
)
|
||||
frames = stage.decode(
|
||||
latents, server_args, vae_dtype=vae_dtype, on_frames=stream
|
||||
)
|
||||
self.assertEqual(stream.finish(), [str(streamed)])
|
||||
self.assertEqual(tuple(frames.shape[1:3]), (3, 21))
|
||||
self.assertTrue(
|
||||
_try_save_cuda_video_direct(
|
||||
save_file_path=str(one_shot),
|
||||
sample=frames[0],
|
||||
fps=FPS,
|
||||
audio_sample_rate=None,
|
||||
output_compression=None,
|
||||
)
|
||||
)
|
||||
self.assertEqual(one_shot.read_bytes(), streamed.read_bytes())
|
||||
|
||||
@unittest.skipUnless(
|
||||
current_platform.is_cuda(), "streaming is gated on the CUDA platform, not ROCm"
|
||||
)
|
||||
def test_only_file_path_video_requests_stream(self):
|
||||
server_args = _server_args()
|
||||
stage = _stage(_vae(_vae_config(wan22=False)), server_args)
|
||||
self.assertIsInstance(
|
||||
stage._encode_while_decoding(_video_request(), server_args),
|
||||
EncodeWhileDecoding,
|
||||
)
|
||||
for field, value in (
|
||||
("return_file_paths_only", False),
|
||||
("return_raw_frames", True),
|
||||
("return_trajectory_decoded", True),
|
||||
("data_type", DataType.IMAGE),
|
||||
("enable_frame_interpolation", True),
|
||||
("enable_upscaling", True),
|
||||
("extra", {"dynamic_batch_output_paths": ["a.mp4"]}),
|
||||
("latents", torch.zeros(2, Z_DIM, 2, 2, 2)),
|
||||
):
|
||||
with self.subTest(field=field):
|
||||
request = _video_request(**{field: value})
|
||||
self.assertIsNone(stage._encode_while_decoding(request, server_args))
|
||||
|
||||
class OwnPostDecoding(WanT2V480PConfig):
|
||||
def post_decoding(self, frames, server_args):
|
||||
return frames
|
||||
|
||||
class OwnDecode(DecodingStage):
|
||||
def decode(self, latents, server_args, *, vae_dtype):
|
||||
return latents
|
||||
|
||||
for name, candidate, args in (
|
||||
("post_decoding", stage, _server_args(OwnPostDecoding())),
|
||||
(
|
||||
"torch_compile",
|
||||
stage,
|
||||
SimpleNamespace(**{**vars(server_args), "enable_torch_compile": True}),
|
||||
),
|
||||
("stage_decode", _stage(stage.vae, server_args, OwnDecode), server_args),
|
||||
("vae", _stage(torch.nn.Identity(), server_args), server_args),
|
||||
):
|
||||
with self.subTest(name=name):
|
||||
self.assertIsNone(
|
||||
candidate._encode_while_decoding(_video_request(), args)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=3)
|
||||
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""AutoencoderKLWan.decode hands out every frame once, in order, once it is final."""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.models.vaes.wanvae import (
|
||||
WanVAEArchConfig,
|
||||
WanVAEConfig,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.models.vaes.wanvae import AutoencoderKLWan
|
||||
|
||||
Z_DIM = 4
|
||||
|
||||
|
||||
def _vae(*, wan22: bool) -> AutoencoderKLWan:
|
||||
"""A tiny Wan 2.1 VAE, or with ``wan22`` the 2.2 residual blocks and 2x2 patches."""
|
||||
channels = 12 if wan22 else 3
|
||||
config = WanVAEConfig(
|
||||
arch_config=WanVAEArchConfig(
|
||||
base_dim=8,
|
||||
z_dim=Z_DIM,
|
||||
dim_mult=(1, 2, 2, 2),
|
||||
num_res_blocks=1,
|
||||
latents_mean=(0.0,) * Z_DIM,
|
||||
latents_std=(1.0,) * Z_DIM,
|
||||
is_residual=wan22,
|
||||
in_channels=channels,
|
||||
out_channels=channels,
|
||||
patch_size=2 if wan22 else None,
|
||||
),
|
||||
load_encoder=False,
|
||||
)
|
||||
torch.manual_seed(0)
|
||||
return AutoencoderKLWan(config).eval()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("wan22", [False, True])
|
||||
def test_causal_decode_hands_out_each_latent_frames_pixels(wan22):
|
||||
vae = _vae(wan22=wan22)
|
||||
latents = torch.randn(1, Z_DIM, 4, 2, 3)
|
||||
parts = []
|
||||
with torch.no_grad():
|
||||
video = vae.decode(latents, on_frames=parts.append)
|
||||
reference = vae.decode(latents)
|
||||
|
||||
assert torch.equal(video, reference)
|
||||
assert [part.shape[2] for part in parts] == [1, 4, 4, 4]
|
||||
assert torch.equal(torch.cat(parts, dim=2), video)
|
||||
|
||||
|
||||
def test_whole_clip_decode_hands_over_everything_at_the_end():
|
||||
vae = _vae(wan22=False)
|
||||
vae.use_feature_cache = False
|
||||
parts = []
|
||||
with torch.no_grad():
|
||||
video = vae.decode(torch.randn(1, Z_DIM, 3, 2, 3), on_frames=parts.append)
|
||||
|
||||
assert len(parts) == 1
|
||||
assert parts[0] is video
|
||||
Reference in New Issue
Block a user