Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1197 lines
39 KiB
Python
1197 lines
39 KiB
Python
# Copied and adapted from: https://github.com/hao-ai-lab/FastVideo
|
|
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
"""
|
|
DiffGenerator module for sglang-diffusion.
|
|
|
|
This module provides a consolidated interface for generating videos using
|
|
diffusion models.
|
|
"""
|
|
|
|
import atexit
|
|
import json
|
|
import mmap
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import threading
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from contextlib import contextmanager
|
|
from dataclasses import dataclass, field
|
|
from typing import Any, Callable, List, Optional, Sequence, Union
|
|
|
|
import imageio
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
|
|
try:
|
|
import scipy.io.wavfile as scipy_wavfile
|
|
except ImportError: # pragma: no cover
|
|
scipy_wavfile = None
|
|
|
|
try:
|
|
import imageio_ffmpeg as _imageio_ffmpeg
|
|
except ImportError: # pragma: no cover
|
|
_imageio_ffmpeg = None
|
|
|
|
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
|
DataType,
|
|
SamplingParams,
|
|
)
|
|
from sglang.multimodal_gen.runtime.pipelines_core.request_utils import (
|
|
expand_request_outputs as expand_request_outputs,
|
|
)
|
|
from sglang.multimodal_gen.runtime.pipelines_core.request_utils import (
|
|
normalize_output_seeds as normalize_output_seeds,
|
|
)
|
|
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
|
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
|
from sglang.multimodal_gen.runtime.utils.logging_utils import CYAN, RESET, init_logger
|
|
from sglang.multimodal_gen.runtime.utils.profiler import maybe_record_function
|
|
from sglang.srt.observability.trace import TraceReqContext
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
_MAX_CACHED_CUDA_VIDEO_BUFFER_BYTES = 1024 * 1024 * 1024
|
|
_MAX_CUDA_VIDEO_CONVERSION_CHUNK_BYTES = 128 * 1024 * 1024
|
|
_MAX_PARALLEL_CUDA_VIDEO_SAVES = 2
|
|
_cuda_video_buffer_cache_lock = threading.Lock()
|
|
_cached_cuda_video_buffer: "_CudaMemfdVideoBuffer | None" = None
|
|
|
|
|
|
class _CudaMemfdVideoBuffer:
|
|
"""CUDA-registered memfd used as direct ffmpeg raw-video input."""
|
|
|
|
def __init__(self, shape: tuple[int, ...]):
|
|
self.shape = shape
|
|
self.nbytes = int(np.prod(shape, dtype=np.int64))
|
|
self.fd = -1
|
|
self.mapping: mmap.mmap | None = None
|
|
self.array: np.ndarray | None = None
|
|
self.tensor: torch.Tensor | None = None
|
|
self._registered = False
|
|
|
|
try:
|
|
self.fd = os.memfd_create(
|
|
"sglang-video-frames",
|
|
flags=getattr(os, "MFD_CLOEXEC", 0),
|
|
)
|
|
os.ftruncate(self.fd, self.nbytes)
|
|
self.mapping = mmap.mmap(
|
|
self.fd,
|
|
self.nbytes,
|
|
flags=mmap.MAP_SHARED,
|
|
prot=mmap.PROT_READ | mmap.PROT_WRITE,
|
|
)
|
|
self.array = np.ndarray(shape, dtype=np.uint8, buffer=self.mapping)
|
|
error = torch.cuda.cudart().cudaHostRegister(
|
|
self.array.ctypes.data,
|
|
self.nbytes,
|
|
0,
|
|
)
|
|
if error != 0:
|
|
raise RuntimeError(f"cudaHostRegister failed: {error}")
|
|
self._registered = True
|
|
self.tensor = torch.from_numpy(self.array)
|
|
if not self.tensor.is_pinned():
|
|
raise RuntimeError("CUDA-registered memfd is not pinned")
|
|
except Exception:
|
|
self.close()
|
|
raise
|
|
|
|
def close(self) -> None:
|
|
if self._registered and self.array is not None:
|
|
try:
|
|
torch.cuda.cudart().cudaHostUnregister(self.array.ctypes.data)
|
|
except Exception:
|
|
pass
|
|
self._registered = False
|
|
self.tensor = None
|
|
self.array = None
|
|
if self.mapping is not None:
|
|
self.mapping.close()
|
|
self.mapping = None
|
|
if self.fd >= 0:
|
|
os.close(self.fd)
|
|
self.fd = -1
|
|
|
|
|
|
@contextmanager
|
|
def _acquire_cuda_video_buffer(shape: tuple[int, ...]):
|
|
global _cached_cuda_video_buffer
|
|
|
|
buffer = None
|
|
stale_buffer = None
|
|
with _cuda_video_buffer_cache_lock:
|
|
if _cached_cuda_video_buffer is not None:
|
|
if _cached_cuda_video_buffer.shape == shape:
|
|
buffer = _cached_cuda_video_buffer
|
|
else:
|
|
stale_buffer = _cached_cuda_video_buffer
|
|
_cached_cuda_video_buffer = None
|
|
|
|
if stale_buffer is not None:
|
|
stale_buffer.close()
|
|
|
|
if buffer is None:
|
|
buffer = _CudaMemfdVideoBuffer(shape)
|
|
|
|
try:
|
|
yield buffer
|
|
finally:
|
|
with _cuda_video_buffer_cache_lock:
|
|
if (
|
|
buffer.nbytes <= _MAX_CACHED_CUDA_VIDEO_BUFFER_BYTES
|
|
and _cached_cuda_video_buffer is None
|
|
):
|
|
_cached_cuda_video_buffer = buffer
|
|
buffer = None
|
|
if buffer is not None:
|
|
buffer.close()
|
|
|
|
|
|
def _close_cached_cuda_video_buffer() -> None:
|
|
global _cached_cuda_video_buffer
|
|
|
|
with _cuda_video_buffer_cache_lock:
|
|
buffer = _cached_cuda_video_buffer
|
|
_cached_cuda_video_buffer = None
|
|
if buffer is not None:
|
|
buffer.close()
|
|
|
|
|
|
atexit.register(_close_cached_cuda_video_buffer)
|
|
|
|
|
|
def format_lora_message(
|
|
lora_nickname: Union[str, List[str]],
|
|
target: Union[str, List[str]],
|
|
strength: Union[float, List[float]],
|
|
) -> tuple[str, str, str]:
|
|
"""Format success message for single or multiple LoRAs."""
|
|
if isinstance(lora_nickname, list):
|
|
nickname_str = ", ".join(lora_nickname)
|
|
target_str = ", ".join(target) if isinstance(target, list) else target
|
|
strength_str = (
|
|
", ".join(f"{s:.2f}" for s in strength)
|
|
if isinstance(strength, list)
|
|
else f"{strength:.2f}"
|
|
)
|
|
else:
|
|
nickname_str = lora_nickname
|
|
target_str = target if isinstance(target, str) else ", ".join(target)
|
|
strength_str = (
|
|
f"{strength:.2f}"
|
|
if isinstance(strength, (int, float))
|
|
else ", ".join(f"{s:.2f}" for s in strength)
|
|
)
|
|
return nickname_str, target_str, strength_str
|
|
|
|
|
|
@dataclass
|
|
class GenerationResult:
|
|
"""Result of a single generation request from DiffGenerator."""
|
|
|
|
samples: Any = None
|
|
frames: Any = None
|
|
audio: Any = None
|
|
action: Any = None # [T, raw_action_dim] predicted action (policy/inverse_dynamics)
|
|
prompt: str | None = None
|
|
size: tuple | None = None # (height, width, num_frames)
|
|
generation_time: float = 0.0
|
|
peak_memory_mb: float = 0.0
|
|
metrics: dict = field(default_factory=dict)
|
|
trajectory_latents: Any = None
|
|
trajectory_timesteps: Any = None
|
|
rollout_trajectory_data: Any = None
|
|
trajectory_decoded: Any = None
|
|
prompt_index: int = 0
|
|
output_file_path: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class MaterializedOutput:
|
|
sample: Any
|
|
frames: list[Any]
|
|
audio: Any = None
|
|
fps: int = 0
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RequestOutput:
|
|
"""Map one final sample to its request, metrics and output filename."""
|
|
|
|
request: Req
|
|
request_index: int
|
|
sample_index: int
|
|
sample_count: int
|
|
|
|
def output_file_path(self):
|
|
return self.request.output_file_path(self.sample_count, self.sample_index)
|
|
|
|
|
|
def map_request_outputs(requests: list[Req]) -> list[RequestOutput]:
|
|
outputs = []
|
|
for request_index, req in enumerate(requests):
|
|
count = req.sampling_params.num_samples_per_request
|
|
if count < 1:
|
|
raise ValueError(f"num_samples_per_request must be positive, got {count}")
|
|
outputs.extend(
|
|
RequestOutput(req, request_index, index, count) for index in range(count)
|
|
)
|
|
return outputs
|
|
|
|
|
|
def _normalize_audio_to_numpy(audio: Any) -> np.ndarray | None:
|
|
"""Convert audio (torch / numpy) into a float32 numpy array in [-1, 1], best-effort."""
|
|
if audio is None:
|
|
return None
|
|
if isinstance(audio, torch.Tensor):
|
|
audio_np = audio.detach().float().clamp(-1.0, 1.0).cpu().numpy()
|
|
elif isinstance(audio, np.ndarray):
|
|
audio_np = audio.astype(np.float32, copy=False)
|
|
audio_np = np.clip(audio_np, -1.0, 1.0)
|
|
else:
|
|
return None
|
|
|
|
# 1. Squeeze leading singleton dimensions (Batch, etc.)
|
|
while audio_np.ndim > 1 and audio_np.shape[0] == 1:
|
|
audio_np = audio_np.squeeze(0)
|
|
|
|
# 2. Handle (C, L) -> (L, C)
|
|
if audio_np.ndim == 2 and audio_np.shape[0] < audio_np.shape[1]:
|
|
audio_np = audio_np.transpose(1, 0)
|
|
|
|
# 3. Final safety check: if still 2D and channels (dim 1) is huge, something is wrong
|
|
if audio_np.ndim == 2 and audio_np.shape[1] > 256 and audio_np.shape[0] == 1:
|
|
audio_np = audio_np.flatten()
|
|
|
|
return audio_np
|
|
|
|
|
|
def _pick_audio_sample_rate(
|
|
*,
|
|
audio_np: np.ndarray,
|
|
audio_sample_rate: Optional[int],
|
|
fps: int,
|
|
num_frames: int,
|
|
) -> int:
|
|
"""Pick a plausible sample rate, falling back to inferring from video duration."""
|
|
selected_sr = int(audio_sample_rate) if audio_sample_rate is not None else None
|
|
if selected_sr is None or not (8000 <= selected_sr <= 192000):
|
|
selected_sr = 24000
|
|
try:
|
|
duration_s = float(num_frames) / float(fps) if fps else 0.0
|
|
if duration_s > 0:
|
|
audio_len = (
|
|
int(audio_np.shape[0])
|
|
if audio_np.ndim == 2
|
|
else int(audio_np.shape[-1])
|
|
)
|
|
inferred_sr = int(round(float(audio_len) / duration_s))
|
|
if 8000 <= inferred_sr <= 192000:
|
|
selected_sr = inferred_sr
|
|
except Exception:
|
|
pass
|
|
return selected_sr
|
|
|
|
|
|
def _resolve_ffmpeg_exe() -> str:
|
|
ffmpeg_exe = "ffmpeg"
|
|
ffmpeg_on_path = shutil.which("ffmpeg")
|
|
if ffmpeg_on_path:
|
|
ffmpeg_exe = ffmpeg_on_path
|
|
try:
|
|
if _imageio_ffmpeg is not None:
|
|
ffmpeg_exe = _imageio_ffmpeg.get_ffmpeg_exe()
|
|
except Exception:
|
|
pass
|
|
|
|
ffmpeg_ok = False
|
|
if ffmpeg_exe:
|
|
if os.path.isabs(ffmpeg_exe):
|
|
ffmpeg_ok = os.path.exists(ffmpeg_exe)
|
|
else:
|
|
ffmpeg_ok = shutil.which(ffmpeg_exe) is not None
|
|
if not ffmpeg_ok:
|
|
raise RuntimeError("ffmpeg not found")
|
|
return ffmpeg_exe
|
|
|
|
|
|
# ffmpeg's implicit libx264 default is `medium`. On diffusion output `fast` is
|
|
# both quicker and measurably closer to the frames the model produced.
|
|
X264_PRESET = "fast"
|
|
|
|
|
|
def _x264_auto_thread_count(height: int) -> int:
|
|
"""Match x264's auto frame-thread count for progressive video."""
|
|
try:
|
|
cpu_count = len(os.sched_getaffinity(0))
|
|
except (AttributeError, OSError):
|
|
cpu_count = os.cpu_count() or 1
|
|
cpu_limit = max(1, cpu_count * 3 // 2)
|
|
macroblock_rows = max(1, (height + 15) // 16)
|
|
row_limit = max(1, macroblock_rows // 2)
|
|
return min(cpu_limit, row_limit, 128)
|
|
|
|
|
|
def _cuda_video_conversion_chunk_frames(video: torch.Tensor) -> int:
|
|
_, num_frames, height, width = video.shape
|
|
temporary_bytes_per_frame = 3 * height * width * (video.element_size() + 2)
|
|
return min(
|
|
num_frames,
|
|
max(1, _MAX_CUDA_VIDEO_CONVERSION_CHUNK_BYTES // temporary_bytes_per_frame),
|
|
)
|
|
|
|
|
|
def _sendfile_all(output_fd: int, input_fd: int, count: int) -> None:
|
|
offset = 0
|
|
while count:
|
|
sent = os.sendfile(output_fd, input_fd, offset, count)
|
|
if sent <= 0:
|
|
raise RuntimeError("sendfile made no progress")
|
|
offset += sent
|
|
count -= sent
|
|
|
|
|
|
def _try_save_cuda_video_direct(
|
|
*,
|
|
save_file_path: str,
|
|
sample: Any,
|
|
fps: int,
|
|
audio_sample_rate: Optional[int],
|
|
output_compression: Optional[int],
|
|
x264_preset: Optional[str] = None,
|
|
) -> bool:
|
|
"""Stream CUDA RGB chunks to ffmpeg through a registered memfd."""
|
|
if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"):
|
|
return False
|
|
|
|
sample_without_audio, audio = _split_sample_audio(sample)
|
|
if not (
|
|
isinstance(sample_without_audio, torch.Tensor)
|
|
and sample_without_audio.device.type == "cuda"
|
|
and sample_without_audio.dim() in (3, 4)
|
|
):
|
|
return False
|
|
if os.path.splitext(save_file_path)[1].lower() != ".mp4":
|
|
return False
|
|
|
|
video = sample_without_audio
|
|
if video.dim() == 3:
|
|
video = video.unsqueeze(1)
|
|
if video.shape[0] != 3:
|
|
return False
|
|
|
|
_, num_frames, height, width = video.shape
|
|
chunk_frames = _cuda_video_conversion_chunk_frames(video)
|
|
|
|
quality = output_compression / 10 if output_compression is not None else 5
|
|
if not 1 <= quality <= 10:
|
|
return False
|
|
crf = int((1 - quality / 10.0) * 51)
|
|
|
|
audio_np = _normalize_audio_to_numpy(audio)
|
|
tmp_wav_path = None
|
|
try:
|
|
if audio_np is not None:
|
|
if scipy_wavfile is None:
|
|
return False
|
|
selected_sr = _pick_audio_sample_rate(
|
|
audio_np=audio_np,
|
|
audio_sample_rate=audio_sample_rate,
|
|
fps=fps,
|
|
num_frames=num_frames,
|
|
)
|
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
|
tmp_wav_path = f.name
|
|
scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np)
|
|
|
|
ffmpeg_exe = _resolve_ffmpeg_exe()
|
|
command = [
|
|
ffmpeg_exe,
|
|
"-y",
|
|
"-f",
|
|
"rawvideo",
|
|
"-vcodec",
|
|
"rawvideo",
|
|
"-s",
|
|
f"{width}x{height}",
|
|
"-pix_fmt",
|
|
"rgb24",
|
|
"-r",
|
|
f"{fps:.02f}",
|
|
"-i",
|
|
"pipe:0",
|
|
]
|
|
if tmp_wav_path is None:
|
|
command += ["-an"]
|
|
else:
|
|
command += ["-i", tmp_wav_path]
|
|
command += [
|
|
"-vcodec",
|
|
"libx264",
|
|
"-preset",
|
|
x264_preset or X264_PRESET,
|
|
"-pix_fmt",
|
|
"yuv420p",
|
|
"-crf",
|
|
str(crf),
|
|
]
|
|
|
|
macro_block_size = 16
|
|
if width % macro_block_size or height % macro_block_size:
|
|
output_width = (
|
|
width
|
|
if width % macro_block_size == 0
|
|
else width + macro_block_size - width % macro_block_size
|
|
)
|
|
output_height = (
|
|
height
|
|
if height % macro_block_size == 0
|
|
else height + macro_block_size - height % macro_block_size
|
|
)
|
|
command += ["-vf", f"scale={output_width}:{output_height}"]
|
|
|
|
command += ["-threads", str(_x264_auto_thread_count(height))]
|
|
if tmp_wav_path is not None:
|
|
command += [
|
|
"-acodec",
|
|
"aac",
|
|
"-map",
|
|
"0:v:0",
|
|
"-map",
|
|
"1:a:0",
|
|
]
|
|
command += ["-v", "warning", save_file_path]
|
|
|
|
with tempfile.TemporaryFile() as stderr_file:
|
|
process = subprocess.Popen(
|
|
command,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=stderr_file,
|
|
)
|
|
try:
|
|
if process.stdin is None:
|
|
raise RuntimeError("ffmpeg stdin pipe was not created")
|
|
with _acquire_cuda_video_buffer(
|
|
(chunk_frames, height, width, 3)
|
|
) as buffer:
|
|
assert buffer.tensor is not None
|
|
for start in range(0, num_frames, chunk_frames):
|
|
end = min(start + chunk_frames, num_frames)
|
|
with maybe_record_function(
|
|
f"VIDEO_CHUNK frames {start}-{end} convert+pipe_to_x264"
|
|
):
|
|
frames = (
|
|
(video[:, start:end] * 255)
|
|
.clamp_(0, 255)
|
|
.to(torch.uint8)
|
|
)
|
|
frames = frames.permute(1, 2, 3, 0).contiguous()
|
|
buffer.tensor[: end - start].copy_(
|
|
frames, non_blocking=True
|
|
)
|
|
torch.cuda.current_stream(video.device).synchronize()
|
|
del frames
|
|
_sendfile_all(
|
|
process.stdin.fileno(),
|
|
buffer.fd,
|
|
(end - start) * height * width * 3,
|
|
)
|
|
with maybe_record_function("FFMPEG_FLUSH stdin_close+wait"):
|
|
process.stdin.close()
|
|
process.stdin = None
|
|
returncode = process.wait()
|
|
finally:
|
|
if process.stdin is not None:
|
|
process.stdin.close()
|
|
if process.poll() is None:
|
|
process.kill()
|
|
process.wait()
|
|
if returncode:
|
|
stderr_file.seek(0)
|
|
raise subprocess.CalledProcessError(
|
|
returncode,
|
|
command,
|
|
stderr=stderr_file.read(),
|
|
)
|
|
return True
|
|
except Exception:
|
|
logger.warning_once(
|
|
"Direct CUDA video save failed; falling back to imageio. "
|
|
"Enable debug logging for exception details."
|
|
)
|
|
logger.debug("Direct CUDA video save failure", exc_info=True)
|
|
return False
|
|
finally:
|
|
if tmp_wav_path:
|
|
try:
|
|
os.remove(tmp_wav_path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _try_save_cuda_videos_direct(
|
|
samples: Sequence[Any],
|
|
save_file_paths: Sequence[str],
|
|
*,
|
|
fps: int,
|
|
audio_sample_rate: Optional[int],
|
|
output_compression: Optional[int],
|
|
x264_preset: Optional[str] = None,
|
|
) -> list[bool] | None:
|
|
"""Save independent CUDA videos concurrently when memory permits."""
|
|
if len(samples) < 2 or len(samples) != len(save_file_paths):
|
|
return None
|
|
|
|
videos = []
|
|
for sample, save_file_path in zip(samples, save_file_paths):
|
|
video, _ = _split_sample_audio(sample)
|
|
if not (
|
|
isinstance(video, torch.Tensor)
|
|
and video.device.type == "cuda"
|
|
and video.dim() in (3, 4)
|
|
and int(video.shape[0]) == 3
|
|
and os.path.splitext(save_file_path)[1].lower() == ".mp4"
|
|
):
|
|
return None
|
|
if video.dim() == 3:
|
|
video = video.unsqueeze(1)
|
|
if videos and video.device != videos[0].device:
|
|
return None
|
|
videos.append(video)
|
|
|
|
# Each direct save creates one multiply result and two uint8 layouts for a
|
|
# temporal chunk. Keep two such chunks below 25% of currently free device
|
|
# memory so postprocessing cannot turn a tight inference into an OOM.
|
|
temporary_bytes = sorted(
|
|
(
|
|
_cuda_video_conversion_chunk_frames(video)
|
|
* 3
|
|
* int(video.shape[-2])
|
|
* int(video.shape[-1])
|
|
* (int(video.element_size()) + 2)
|
|
for video in videos
|
|
),
|
|
reverse=True,
|
|
)[:_MAX_PARALLEL_CUDA_VIDEO_SAVES]
|
|
try:
|
|
free_bytes, _ = torch.cuda.mem_get_info(videos[0].device)
|
|
except (RuntimeError, TypeError):
|
|
return None
|
|
if sum(temporary_bytes) > int(free_bytes) // 4:
|
|
return None
|
|
try:
|
|
available_cpus = len(os.sched_getaffinity(0))
|
|
except (AttributeError, OSError):
|
|
available_cpus = os.cpu_count() or 1
|
|
encoder_threads = sorted(
|
|
(_x264_auto_thread_count(int(video.shape[-2])) for video in videos),
|
|
reverse=True,
|
|
)[:_MAX_PARALLEL_CUDA_VIDEO_SAVES]
|
|
if sum(encoder_threads) > available_cpus:
|
|
return None
|
|
|
|
def save_one(idx: int) -> bool:
|
|
return _try_save_cuda_video_direct(
|
|
save_file_path=save_file_paths[idx],
|
|
sample=samples[idx],
|
|
fps=fps,
|
|
audio_sample_rate=audio_sample_rate,
|
|
output_compression=output_compression,
|
|
x264_preset=x264_preset,
|
|
)
|
|
|
|
try:
|
|
with ThreadPoolExecutor(max_workers=_MAX_PARALLEL_CUDA_VIDEO_SAVES) as pool:
|
|
return list(pool.map(save_one, range(len(samples))))
|
|
except Exception:
|
|
logger.warning_once(
|
|
"Parallel CUDA video save failed; falling back to serial output. "
|
|
"Enable debug logging for exception details."
|
|
)
|
|
logger.debug("Parallel CUDA video save failure", exc_info=True)
|
|
return None
|
|
|
|
|
|
def _mux_audio_np_into_mp4(
|
|
*,
|
|
save_file_path: str,
|
|
audio_np: np.ndarray,
|
|
sample_rate: int,
|
|
ffmpeg_exe: str,
|
|
) -> None:
|
|
merged_path = save_file_path.rsplit(".", 1)[0] + ".tmp_mux.mp4"
|
|
tmp_wav_path = None
|
|
try:
|
|
if scipy_wavfile is None:
|
|
raise RuntimeError(
|
|
"scipy is required to mux audio into mp4 (pip install scipy)"
|
|
)
|
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
|
tmp_wav_path = f.name
|
|
scipy_wavfile.write(tmp_wav_path, sample_rate, audio_np)
|
|
subprocess.run(
|
|
[
|
|
ffmpeg_exe,
|
|
"-y",
|
|
"-i",
|
|
save_file_path,
|
|
"-i",
|
|
tmp_wav_path,
|
|
"-c:v",
|
|
"copy",
|
|
"-c:a",
|
|
"aac",
|
|
"-strict",
|
|
"experimental",
|
|
merged_path,
|
|
],
|
|
check=True,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
)
|
|
os.replace(merged_path, save_file_path)
|
|
finally:
|
|
if tmp_wav_path:
|
|
try:
|
|
os.remove(tmp_wav_path)
|
|
except OSError:
|
|
pass
|
|
if os.path.exists(merged_path):
|
|
try:
|
|
os.remove(merged_path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _maybe_mux_audio_into_mp4(
|
|
*,
|
|
save_file_path: str,
|
|
audio: Any,
|
|
frames: list,
|
|
fps: int,
|
|
audio_sample_rate: Optional[int],
|
|
) -> None:
|
|
"""Best-effort mux audio into an already-written mp4 at save_file_path.
|
|
|
|
Any failure should keep the silent video and only log a warning.
|
|
"""
|
|
audio_np = _normalize_audio_to_numpy(audio)
|
|
if audio_np is None:
|
|
return
|
|
selected_sr = _pick_audio_sample_rate(
|
|
audio_np=audio_np,
|
|
audio_sample_rate=audio_sample_rate,
|
|
fps=fps,
|
|
num_frames=len(frames),
|
|
)
|
|
|
|
try:
|
|
ffmpeg_exe = _resolve_ffmpeg_exe()
|
|
_mux_audio_np_into_mp4(
|
|
save_file_path=save_file_path,
|
|
audio_np=audio_np,
|
|
sample_rate=selected_sr,
|
|
ffmpeg_exe=ffmpeg_exe,
|
|
)
|
|
except Exception as e:
|
|
logger.warning(
|
|
"Failed to mux audio into mp4 (saved silent video): %s",
|
|
str(e),
|
|
)
|
|
|
|
|
|
def _try_save_video_with_audio(
|
|
*,
|
|
save_file_path: str,
|
|
frames: list,
|
|
fps: int,
|
|
audio: Any,
|
|
audio_sample_rate: Optional[int],
|
|
output_format: str,
|
|
quality: float,
|
|
x264_preset: Optional[str] = None,
|
|
) -> bool:
|
|
"""Encode video and audio in one ffmpeg pass when audio is available."""
|
|
audio_np = _normalize_audio_to_numpy(audio)
|
|
if audio_np is None:
|
|
return False
|
|
|
|
selected_sr = _pick_audio_sample_rate(
|
|
audio_np=audio_np,
|
|
audio_sample_rate=audio_sample_rate,
|
|
fps=fps,
|
|
num_frames=len(frames),
|
|
)
|
|
tmp_wav_path = None
|
|
try:
|
|
if scipy_wavfile is None:
|
|
raise RuntimeError(
|
|
"scipy is required to mux audio into mp4 (pip install scipy)"
|
|
)
|
|
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as f:
|
|
tmp_wav_path = f.name
|
|
scipy_wavfile.write(tmp_wav_path, selected_sr, audio_np)
|
|
imageio.mimsave(
|
|
save_file_path,
|
|
frames,
|
|
fps=fps,
|
|
format=output_format,
|
|
codec="libx264",
|
|
quality=quality,
|
|
audio_path=tmp_wav_path,
|
|
audio_codec="aac",
|
|
output_params=["-preset", x264_preset or X264_PRESET],
|
|
)
|
|
return True
|
|
except Exception as e:
|
|
logger.warning(
|
|
"Failed to encode video and audio in one pass; "
|
|
"falling back to the compatible two-pass path: %s",
|
|
str(e),
|
|
)
|
|
return False
|
|
finally:
|
|
if tmp_wav_path:
|
|
try:
|
|
os.remove(tmp_wav_path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def prepare_request(
|
|
server_args: ServerArgs,
|
|
sampling_params: SamplingParams,
|
|
external_trace_header: dict[str, str] | None = None,
|
|
) -> Req:
|
|
"""
|
|
Create a Req object with sampling_params as a parameter.
|
|
"""
|
|
attention_backend_config = server_args.attention_backend_config or {}
|
|
vsa_sparsity = attention_backend_config.get(
|
|
"VSA_sparsity", attention_backend_config.get("sparsity", 0.0)
|
|
)
|
|
req = Req(
|
|
sampling_params=sampling_params,
|
|
VSA_sparsity=vsa_sparsity,
|
|
)
|
|
sampling_params.apply_request_extra(req)
|
|
if getattr(sampling_params, "max_sequence_length", None) is not None:
|
|
req.max_sequence_length = sampling_params.max_sequence_length
|
|
|
|
diffusers_kwargs = getattr(sampling_params, "diffusers_kwargs", None)
|
|
if diffusers_kwargs and "max_sequence_length" in diffusers_kwargs:
|
|
req.max_sequence_length = diffusers_kwargs["max_sequence_length"]
|
|
|
|
action_prompt = (
|
|
req.data_type == DataType.ACTION
|
|
and isinstance(req.prompt, list)
|
|
and bool(req.prompt)
|
|
and all(isinstance(item, str) for item in req.prompt)
|
|
)
|
|
if not isinstance(req.prompt, str) and not action_prompt:
|
|
raise TypeError(
|
|
"`prompt` must be a string, or a non-empty list of strings for "
|
|
f"batched action requests, but got {type(req.prompt)}"
|
|
)
|
|
|
|
req_width = getattr(req, "width", None)
|
|
req_height = getattr(req, "height", None)
|
|
if (req_width is not None and req_width <= 0) or (
|
|
req_height is not None and req_height <= 0
|
|
):
|
|
raise ValueError(
|
|
f"Height and width must be positive, got height={req_height}, width={req_width}"
|
|
)
|
|
|
|
if server_args.enable_trace:
|
|
trace_ctx = TraceReqContext(
|
|
rid=sampling_params.request_id,
|
|
module_name="diffusion",
|
|
external_trace_header=external_trace_header,
|
|
)
|
|
trace_ctx.trace_req_start()
|
|
req.trace_ctx = trace_ctx
|
|
|
|
return req
|
|
|
|
|
|
def attach_audio_to_video_sample(
|
|
sample: Any,
|
|
audio: Any,
|
|
output_idx: int,
|
|
) -> Any:
|
|
"""Attach per-sample audio for video outputs when available."""
|
|
audio = select_output_audio(audio, output_idx)
|
|
if audio is None:
|
|
return sample
|
|
if not (isinstance(sample, (tuple, list)) and len(sample) == 2):
|
|
return (sample, audio)
|
|
return sample
|
|
|
|
|
|
def select_output_audio(audio: Any, output_idx: int) -> Any:
|
|
if isinstance(audio, torch.Tensor) and audio.ndim >= 2:
|
|
return audio[output_idx] if audio.shape[0] > output_idx else None
|
|
if isinstance(audio, np.ndarray) and audio.ndim >= 2:
|
|
return audio[output_idx] if audio.shape[0] > output_idx else None
|
|
return audio
|
|
|
|
|
|
def _split_sample_audio(sample: Any) -> tuple[Any, Any]:
|
|
if isinstance(sample, (tuple, list)) and len(sample) == 2:
|
|
return sample[0], sample[1]
|
|
return sample, None
|
|
|
|
|
|
def _sample_to_uint8_frames(sample: Any) -> list[Any]:
|
|
"""return numpy frames in THCW format"""
|
|
if isinstance(sample, torch.Tensor):
|
|
# sample is raw tensor
|
|
if sample.dim() == 3:
|
|
sample = sample.unsqueeze(1)
|
|
sample = (sample * 255).clamp(0, 255).to(torch.uint8)
|
|
videos = sample.permute(1, 2, 3, 0).contiguous()
|
|
if videos.device.type == "cuda":
|
|
try:
|
|
host_videos = torch.empty(
|
|
videos.shape,
|
|
dtype=videos.dtype,
|
|
device="cpu",
|
|
pin_memory=True,
|
|
)
|
|
except RuntimeError:
|
|
videos = videos.cpu().numpy()
|
|
else:
|
|
host_videos.copy_(videos, non_blocking=True)
|
|
torch.cuda.current_stream(videos.device).synchronize()
|
|
videos = host_videos.numpy()
|
|
else:
|
|
videos = videos.cpu().numpy()
|
|
return list(videos)
|
|
|
|
if not isinstance(sample, np.ndarray):
|
|
raise TypeError(f"Unsupported sample type: {type(sample)}")
|
|
|
|
# sample is numpy frames
|
|
arr = sample
|
|
if arr.ndim == 3:
|
|
if arr.shape[-1] in (1, 3, 4):
|
|
arr = arr[None, ...]
|
|
else:
|
|
arr = arr[..., None]
|
|
if arr.ndim != 4:
|
|
raise ValueError(f"Unexpected numpy sample shape: {tuple(arr.shape)}")
|
|
|
|
if arr.shape[-1] not in (1, 3, 4) and arr.shape[0] in (1, 3, 4):
|
|
t = torch.from_numpy(arr)
|
|
if t.dim() == 3:
|
|
t = t.unsqueeze(1)
|
|
t = (t * 255).clamp(0, 255).to(torch.uint8)
|
|
videos = t.permute(1, 2, 3, 0).contiguous().cpu().numpy()
|
|
return list(videos)
|
|
|
|
if arr.dtype != np.uint8:
|
|
arr = (np.clip(arr, 0.0, 1.0) * 255.0).astype(np.uint8)
|
|
return list(arr)
|
|
|
|
|
|
def materialize_output_sample(
|
|
sample: Any,
|
|
data_type: DataType,
|
|
fps: int,
|
|
*,
|
|
enable_frame_interpolation: bool = False,
|
|
frame_interpolation_exp: int = 1,
|
|
frame_interpolation_scale: float = 1.0,
|
|
frame_interpolation_model_path: Optional[str] = None,
|
|
enable_upscaling: bool = False,
|
|
upscaling_model_path: Optional[str] = None,
|
|
upscaling_scale: int = 4,
|
|
) -> MaterializedOutput:
|
|
"""materialize samples, apply postprocessing if applicable"""
|
|
sample_without_audio, audio = _split_sample_audio(sample)
|
|
frames = _sample_to_uint8_frames(sample_without_audio)
|
|
|
|
# frames are uint8 numpy arrays in THWC format at this point
|
|
if enable_frame_interpolation and data_type == DataType.VIDEO and len(frames) > 1:
|
|
from sglang.multimodal_gen.runtime.postprocess import (
|
|
interpolate_video_frames,
|
|
)
|
|
|
|
frames, multiplier = interpolate_video_frames(
|
|
frames,
|
|
exp=frame_interpolation_exp,
|
|
scale=frame_interpolation_scale,
|
|
model_path=frame_interpolation_model_path,
|
|
)
|
|
fps = fps * multiplier
|
|
|
|
if enable_upscaling and frames:
|
|
from sglang.multimodal_gen.runtime.postprocess import upscale_frames
|
|
|
|
frames = upscale_frames(
|
|
frames,
|
|
model_path=upscaling_model_path,
|
|
scale=upscaling_scale,
|
|
)
|
|
|
|
return MaterializedOutput(sample=sample, frames=frames, audio=audio, fps=fps)
|
|
|
|
|
|
def save_materialized_output(
|
|
materialized: MaterializedOutput,
|
|
data_type: DataType,
|
|
save_file_path: Optional[str],
|
|
*,
|
|
save_output: bool = True,
|
|
audio_sample_rate: Optional[int] = None,
|
|
output_compression: Optional[int] = None,
|
|
x264_preset: Optional[str] = None,
|
|
) -> None:
|
|
if not save_output:
|
|
return
|
|
if not save_file_path:
|
|
logger.info("No output path provided, output not saved")
|
|
return
|
|
|
|
os.makedirs(os.path.dirname(save_file_path), exist_ok=True)
|
|
if data_type == DataType.VIDEO:
|
|
quality = output_compression / 10 if output_compression is not None else 5
|
|
output_format = data_type.get_default_extension()
|
|
saved_with_audio = _try_save_video_with_audio(
|
|
save_file_path=save_file_path,
|
|
frames=materialized.frames,
|
|
fps=materialized.fps,
|
|
audio=materialized.audio,
|
|
audio_sample_rate=audio_sample_rate,
|
|
output_format=output_format,
|
|
quality=quality,
|
|
x264_preset=x264_preset,
|
|
)
|
|
if not saved_with_audio:
|
|
imageio.mimsave(
|
|
save_file_path,
|
|
materialized.frames,
|
|
fps=materialized.fps,
|
|
format=output_format,
|
|
codec="libx264",
|
|
quality=quality,
|
|
output_params=["-preset", x264_preset or X264_PRESET],
|
|
)
|
|
|
|
_maybe_mux_audio_into_mp4(
|
|
save_file_path=save_file_path,
|
|
audio=materialized.audio,
|
|
frames=materialized.frames,
|
|
fps=materialized.fps,
|
|
audio_sample_rate=audio_sample_rate,
|
|
)
|
|
else:
|
|
quality = output_compression if output_compression is not None else 75
|
|
if len(materialized.frames) > 1:
|
|
for i, image in enumerate(materialized.frames):
|
|
parts = save_file_path.rsplit(".", 1)
|
|
if len(parts) == 2:
|
|
indexed_path = f"{parts[0]}_{i}.{parts[1]}"
|
|
else:
|
|
indexed_path = f"{save_file_path}_{i}"
|
|
_save_image_frame(indexed_path, image, quality, output_compression)
|
|
else:
|
|
_save_image_frame(
|
|
save_file_path, materialized.frames[0], quality, output_compression
|
|
)
|
|
logger.info(f"Output saved to {CYAN}{save_file_path}{RESET}")
|
|
|
|
|
|
def _save_image_frame(
|
|
path: str, frame: np.ndarray, quality: int | None, output_compression: int | None
|
|
) -> None:
|
|
ext = os.path.splitext(path)[1].lower()
|
|
if ext == ".png":
|
|
compress_level = 1
|
|
if output_compression is not None and output_compression != 75:
|
|
compress_level = max(0, min(9, round(output_compression / 100 * 9)))
|
|
if frame.ndim == 3 and frame.shape[-1] == 1:
|
|
frame = frame[..., 0]
|
|
Image.fromarray(frame).save(path, format="PNG", compress_level=compress_level)
|
|
else:
|
|
imageio.imwrite(path, frame, quality=quality)
|
|
|
|
|
|
def save_outputs(
|
|
outputs: Sequence[Any],
|
|
data_type: DataType,
|
|
fps: int,
|
|
save_output: bool,
|
|
build_output_path: Callable[[int], str],
|
|
*,
|
|
audio: Any = None,
|
|
audio_sample_rate: Optional[int] = None,
|
|
samples_out: Optional[list[Any]] = None,
|
|
audios_out: Optional[list[Any]] = None,
|
|
frames_out: Optional[list[Any]] = None,
|
|
output_compression: Optional[int] = None,
|
|
x264_preset: Optional[str] = None,
|
|
enable_frame_interpolation: bool = False,
|
|
frame_interpolation_exp: int = 1,
|
|
frame_interpolation_scale: float = 1.0,
|
|
frame_interpolation_model_path: Optional[str] = None,
|
|
enable_upscaling: bool = False,
|
|
upscaling_model_path: Optional[str] = None,
|
|
upscaling_scale: int = 4,
|
|
) -> list[str]:
|
|
output_paths: list[str] = []
|
|
samples = (
|
|
[
|
|
attach_audio_to_video_sample(sample, audio, idx)
|
|
for idx, sample in enumerate(outputs)
|
|
]
|
|
if data_type == DataType.VIDEO
|
|
else outputs
|
|
)
|
|
save_file_paths = [build_output_path(idx) for idx in range(len(outputs))]
|
|
parallel_results = None
|
|
if (
|
|
data_type == DataType.VIDEO
|
|
and len(outputs) > 1
|
|
and save_output
|
|
and frames_out is None
|
|
and not enable_frame_interpolation
|
|
and not enable_upscaling
|
|
):
|
|
for path in save_file_paths:
|
|
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
|
|
parallel_results = _try_save_cuda_videos_direct(
|
|
samples,
|
|
save_file_paths,
|
|
fps=fps,
|
|
audio_sample_rate=audio_sample_rate,
|
|
output_compression=output_compression,
|
|
x264_preset=x264_preset,
|
|
)
|
|
|
|
for idx, (sample, save_file_path) in enumerate(zip(samples, save_file_paths)):
|
|
if data_type == DataType.ACTION:
|
|
if samples_out is not None:
|
|
samples_out.append(sample)
|
|
if audios_out is not None:
|
|
audios_out.append(None)
|
|
if frames_out is not None:
|
|
frames_out.append([])
|
|
if save_output and save_file_path:
|
|
os.makedirs(os.path.dirname(save_file_path) or ".", exist_ok=True)
|
|
with open(save_file_path, "w", encoding="utf-8") as f:
|
|
json.dump(sample, f, ensure_ascii=False)
|
|
logger.info(f"Output saved to {CYAN}{save_file_path}{RESET}")
|
|
output_paths.append(save_file_path)
|
|
continue
|
|
|
|
if data_type == DataType.VIDEO:
|
|
if (
|
|
save_output
|
|
and save_file_path
|
|
and frames_out is None
|
|
and not enable_frame_interpolation
|
|
and not enable_upscaling
|
|
):
|
|
os.makedirs(os.path.dirname(save_file_path) or ".", exist_ok=True)
|
|
direct_saved = parallel_results is not None and parallel_results[idx]
|
|
if not direct_saved:
|
|
direct_saved = _try_save_cuda_video_direct(
|
|
save_file_path=save_file_path,
|
|
sample=sample,
|
|
fps=fps,
|
|
audio_sample_rate=audio_sample_rate,
|
|
output_compression=output_compression,
|
|
x264_preset=x264_preset,
|
|
)
|
|
if direct_saved:
|
|
if samples_out is not None:
|
|
samples_out.append(sample)
|
|
if audios_out is not None:
|
|
audios_out.append(select_output_audio(audio, idx))
|
|
output_paths.append(save_file_path)
|
|
logger.info(f"Output saved to {CYAN}{save_file_path}{RESET}")
|
|
continue
|
|
|
|
frames = post_process_sample(
|
|
sample,
|
|
data_type,
|
|
fps,
|
|
save_output,
|
|
save_file_path,
|
|
audio_sample_rate=audio_sample_rate,
|
|
output_compression=output_compression,
|
|
x264_preset=x264_preset,
|
|
enable_frame_interpolation=enable_frame_interpolation,
|
|
frame_interpolation_exp=frame_interpolation_exp,
|
|
frame_interpolation_scale=frame_interpolation_scale,
|
|
frame_interpolation_model_path=frame_interpolation_model_path,
|
|
enable_upscaling=enable_upscaling,
|
|
upscaling_model_path=upscaling_model_path,
|
|
upscaling_scale=upscaling_scale,
|
|
)
|
|
|
|
if samples_out is not None:
|
|
samples_out.append(sample)
|
|
if audios_out is not None:
|
|
if data_type == DataType.VIDEO:
|
|
audios_out.append(select_output_audio(audio, idx))
|
|
else:
|
|
audios_out.append(audio)
|
|
if frames_out is not None:
|
|
frames_out.append(frames)
|
|
output_paths.append(save_file_path)
|
|
return output_paths
|
|
|
|
|
|
def post_process_sample(
|
|
sample: Any,
|
|
data_type: DataType,
|
|
fps: int,
|
|
save_output: bool = True,
|
|
save_file_path: Optional[str] = None,
|
|
audio_sample_rate: Optional[int] = None,
|
|
output_compression: Optional[int] = None,
|
|
enable_frame_interpolation: bool = False,
|
|
frame_interpolation_exp: int = 1,
|
|
frame_interpolation_scale: float = 1.0,
|
|
frame_interpolation_model_path: Optional[str] = None,
|
|
enable_upscaling: bool = False,
|
|
upscaling_model_path: Optional[str] = None,
|
|
upscaling_scale: int = 4,
|
|
x264_preset: Optional[str] = None,
|
|
) -> list[Any]:
|
|
"""materialize frames and save outputs (optional)"""
|
|
if data_type == DataType.ACTION:
|
|
return []
|
|
|
|
materialized = materialize_output_sample(
|
|
sample,
|
|
data_type,
|
|
fps,
|
|
enable_frame_interpolation=enable_frame_interpolation,
|
|
frame_interpolation_exp=frame_interpolation_exp,
|
|
frame_interpolation_scale=frame_interpolation_scale,
|
|
frame_interpolation_model_path=frame_interpolation_model_path,
|
|
enable_upscaling=enable_upscaling,
|
|
upscaling_model_path=upscaling_model_path,
|
|
upscaling_scale=upscaling_scale,
|
|
)
|
|
save_materialized_output(
|
|
materialized,
|
|
data_type,
|
|
save_file_path,
|
|
save_output=save_output,
|
|
audio_sample_rate=audio_sample_rate,
|
|
output_compression=output_compression,
|
|
x264_preset=x264_preset,
|
|
)
|
|
return materialized.frames
|