[diffusion] feat: let requests choose the video libx264 preset (#41797)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
fbce0d9478
commit
ba68306256
@@ -157,6 +157,7 @@ def _generate_minimax_h3(
|
||||
audio=result.audio,
|
||||
audio_sample_rate=result.audio_sample_rate,
|
||||
output_compression=request.output_compression,
|
||||
x264_preset=request.x264_preset,
|
||||
)
|
||||
sampling_params.validate_video_final_outputs(output_paths, request)
|
||||
return output_paths[0]
|
||||
|
||||
@@ -59,6 +59,20 @@ def generate_request_id() -> str:
|
||||
# is cumulative and may also enable model-owned approximate optimizations.
|
||||
QUALITY_LEVELS: tuple[str, ...] = ("lossless", "extra-high", "high")
|
||||
KERNEL_FUSION_QUALITY_LEVELS = frozenset({"extra-high", "high"})
|
||||
# libx264 presets, fastest first: each spends more encode time for a smaller
|
||||
# file at the same CRF.
|
||||
X264_PRESETS: tuple[str, ...] = (
|
||||
"ultrafast",
|
||||
"superfast",
|
||||
"veryfast",
|
||||
"faster",
|
||||
"fast",
|
||||
"medium",
|
||||
"slow",
|
||||
"slower",
|
||||
"veryslow",
|
||||
"placebo",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -232,6 +246,8 @@ class SamplingParams:
|
||||
)
|
||||
output_quality: str | None = "default"
|
||||
output_compression: int | None = None
|
||||
# MP4 encode speed; None keeps the server's default preset
|
||||
x264_preset: str | None = None
|
||||
# Model-owned, request-scoped quality level.
|
||||
#
|
||||
# - "lossless" (default): the exact reference path. Output is expected to
|
||||
@@ -614,6 +630,12 @@ class SamplingParams:
|
||||
f"seed must be a non-negative int or list of ints, got {self.seed!r}"
|
||||
)
|
||||
|
||||
if self.x264_preset is not None and self.x264_preset not in X264_PRESETS:
|
||||
raise ValueError(
|
||||
f"x264_preset must be one of {', '.join(X264_PRESETS)}, "
|
||||
f"got {self.x264_preset!r}"
|
||||
)
|
||||
|
||||
# Used by seconds() and video writer; fps <= 0 is always invalid.
|
||||
if not isinstance(self.fps, int) or self.fps <= 0:
|
||||
raise ValueError(f"fps must be a positive int, got {self.fps!r}")
|
||||
@@ -1215,6 +1237,16 @@ class SamplingParams:
|
||||
type=int,
|
||||
help="Output compression level (0-100, higher means better quality but larger file size)",
|
||||
)
|
||||
add_argument(
|
||||
"--x264-preset",
|
||||
type=str,
|
||||
choices=list(X264_PRESETS),
|
||||
help=(
|
||||
"libx264 preset for MP4 output. Faster presets encode sooner "
|
||||
"and produce larger files at the same quality setting; "
|
||||
"unset keeps the default (fast)."
|
||||
),
|
||||
)
|
||||
add_argument(
|
||||
"--quality",
|
||||
type=str,
|
||||
|
||||
@@ -407,6 +407,7 @@ class DiffGenerator:
|
||||
audios_out=audios_out,
|
||||
frames_out=frames_out,
|
||||
output_compression=requests[0].output_compression,
|
||||
x264_preset=requests[0].x264_preset,
|
||||
enable_frame_interpolation=requests[
|
||||
0
|
||||
].enable_frame_interpolation,
|
||||
|
||||
@@ -347,6 +347,8 @@ async def forward_to_scheduler(
|
||||
lambda _idx: output_file_path,
|
||||
audio=response.audio,
|
||||
audio_sample_rate=response.audio_sample_rate,
|
||||
output_compression=sp.output_compression,
|
||||
x264_preset=sp.x264_preset,
|
||||
enable_frame_interpolation=sp.enable_frame_interpolation,
|
||||
frame_interpolation_exp=sp.frame_interpolation_exp,
|
||||
frame_interpolation_scale=sp.frame_interpolation_scale,
|
||||
|
||||
@@ -158,6 +158,7 @@ class VideoGenerationsRequest(BaseModel):
|
||||
upscaling_scale: Optional[int] = 4
|
||||
output_quality: Optional[str] = "default"
|
||||
output_compression: Optional[int] = None
|
||||
x264_preset: Optional[str] = None
|
||||
output_path: Optional[str] = None
|
||||
diffusers_kwargs: Optional[Dict[str, Any]] = None # kwargs for diffusers backend
|
||||
# Performance profiling
|
||||
|
||||
@@ -494,6 +494,7 @@ async def process_generation_batch(
|
||||
audio=result.audio,
|
||||
audio_sample_rate=result.audio_sample_rate,
|
||||
output_compression=batch.output_compression,
|
||||
x264_preset=batch.x264_preset,
|
||||
enable_frame_interpolation=batch.enable_frame_interpolation,
|
||||
frame_interpolation_exp=batch.frame_interpolation_exp,
|
||||
frame_interpolation_scale=batch.frame_interpolation_scale,
|
||||
|
||||
@@ -309,6 +309,7 @@ def _build_video_sampling_params(request_id: str, request: VideoGenerationsReque
|
||||
"output_path": request.output_path,
|
||||
"quality": _extra_value(request, "quality"),
|
||||
"output_compression": request.output_compression,
|
||||
"x264_preset": request.x264_preset,
|
||||
"output_quality": request.output_quality,
|
||||
"perf_dump_path": request.perf_dump_path,
|
||||
"profile": request.profile,
|
||||
@@ -503,6 +504,7 @@ async def create_video(
|
||||
upscaling_scale: Optional[int] = Form(None),
|
||||
output_quality: Optional[str] = Form(None),
|
||||
output_compression: Optional[int] = Form(None),
|
||||
x264_preset: Optional[str] = Form(None),
|
||||
output_path: Optional[str] = Form(None),
|
||||
perf_dump_path: Optional[str] = Form(None),
|
||||
extra_params: Optional[str] = Form(None),
|
||||
@@ -649,6 +651,7 @@ async def create_video(
|
||||
),
|
||||
upscaling_scale=form_value("upscaling_scale", upscaling_scale),
|
||||
output_compression=form_value("output_compression", output_compression),
|
||||
x264_preset=form_value("x264_preset", x264_preset),
|
||||
output_quality=form_value("output_quality", output_quality),
|
||||
output_path=form_value("output_path", output_path),
|
||||
perf_dump_path=form_value("perf_dump_path", perf_dump_path),
|
||||
|
||||
@@ -363,6 +363,7 @@ def _try_save_cuda_video_direct(
|
||||
fps: int,
|
||||
audio_sample_rate: Optional[int],
|
||||
output_compression: Optional[int],
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Stream CUDA RGB chunks to ffmpeg through a registered memfd."""
|
||||
if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"):
|
||||
@@ -433,7 +434,7 @@ def _try_save_cuda_video_direct(
|
||||
"-vcodec",
|
||||
"libx264",
|
||||
"-preset",
|
||||
X264_PRESET,
|
||||
x264_preset or X264_PRESET,
|
||||
"-pix_fmt",
|
||||
"yuv420p",
|
||||
"-crf",
|
||||
@@ -541,6 +542,7 @@ def _try_save_cuda_videos_direct(
|
||||
fps: int,
|
||||
audio_sample_rate: Optional[int],
|
||||
output_compression: Optional[int],
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> list[bool] | None:
|
||||
"""Save independent CUDA videos concurrently when memory permits."""
|
||||
if len(samples) < 2 or len(samples) != len(save_file_paths):
|
||||
@@ -601,6 +603,7 @@ def _try_save_cuda_videos_direct(
|
||||
fps=fps,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_compression=output_compression,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -712,6 +715,7 @@ def _try_save_video_with_audio(
|
||||
audio_sample_rate: Optional[int],
|
||||
output_format: str,
|
||||
quality: float,
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""Encode video and audio in one ffmpeg pass when audio is available."""
|
||||
audio_np = _normalize_audio_to_numpy(audio)
|
||||
@@ -742,7 +746,7 @@ def _try_save_video_with_audio(
|
||||
quality=quality,
|
||||
audio_path=tmp_wav_path,
|
||||
audio_codec="aac",
|
||||
output_params=["-preset", X264_PRESET],
|
||||
output_params=["-preset", x264_preset or X264_PRESET],
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
@@ -948,6 +952,7 @@ def save_materialized_output(
|
||||
save_output: bool = True,
|
||||
audio_sample_rate: Optional[int] = None,
|
||||
output_compression: Optional[int] = None,
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> None:
|
||||
if not save_output:
|
||||
return
|
||||
@@ -967,6 +972,7 @@ def save_materialized_output(
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_format=output_format,
|
||||
quality=quality,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
if not saved_with_audio:
|
||||
imageio.mimsave(
|
||||
@@ -976,7 +982,7 @@ def save_materialized_output(
|
||||
format=output_format,
|
||||
codec="libx264",
|
||||
quality=quality,
|
||||
output_params=["-preset", X264_PRESET],
|
||||
output_params=["-preset", x264_preset or X264_PRESET],
|
||||
)
|
||||
|
||||
_maybe_mux_audio_into_mp4(
|
||||
@@ -1031,6 +1037,7 @@ def save_outputs(
|
||||
audios_out: Optional[list[Any]] = None,
|
||||
frames_out: Optional[list[Any]] = None,
|
||||
output_compression: Optional[int] = None,
|
||||
x264_preset: Optional[str] = None,
|
||||
enable_frame_interpolation: bool = False,
|
||||
frame_interpolation_exp: int = 1,
|
||||
frame_interpolation_scale: float = 1.0,
|
||||
@@ -1066,6 +1073,7 @@ def save_outputs(
|
||||
fps=fps,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_compression=output_compression,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
|
||||
for idx, (sample, save_file_path) in enumerate(zip(samples, save_file_paths)):
|
||||
@@ -1101,6 +1109,7 @@ def save_outputs(
|
||||
fps=fps,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_compression=output_compression,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
if direct_saved:
|
||||
if samples_out is not None:
|
||||
@@ -1119,6 +1128,7 @@ def save_outputs(
|
||||
save_file_path,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_compression=output_compression,
|
||||
x264_preset=x264_preset,
|
||||
enable_frame_interpolation=enable_frame_interpolation,
|
||||
frame_interpolation_exp=frame_interpolation_exp,
|
||||
frame_interpolation_scale=frame_interpolation_scale,
|
||||
@@ -1156,6 +1166,7 @@ def post_process_sample(
|
||||
enable_upscaling: bool = False,
|
||||
upscaling_model_path: Optional[str] = None,
|
||||
upscaling_scale: int = 4,
|
||||
x264_preset: Optional[str] = None,
|
||||
) -> list[Any]:
|
||||
"""materialize frames and save outputs (optional)"""
|
||||
if data_type == DataType.ACTION:
|
||||
@@ -1180,5 +1191,6 @@ def post_process_sample(
|
||||
save_output=save_output,
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
output_compression=output_compression,
|
||||
x264_preset=x264_preset,
|
||||
)
|
||||
return materialized.frames
|
||||
|
||||
@@ -1210,6 +1210,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
audio=output_batch.audio,
|
||||
audio_sample_rate=output_batch.audio_sample_rate,
|
||||
output_compression=req.output_compression,
|
||||
x264_preset=req.x264_preset,
|
||||
enable_frame_interpolation=req.enable_frame_interpolation,
|
||||
frame_interpolation_exp=req.frame_interpolation_exp,
|
||||
frame_interpolation_scale=req.frame_interpolation_scale,
|
||||
@@ -1242,6 +1243,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
audio=output_batch.audio,
|
||||
audio_sample_rate=output_batch.audio_sample_rate,
|
||||
output_compression=first_req.output_compression,
|
||||
x264_preset=first_req.x264_preset,
|
||||
enable_frame_interpolation=first_req.enable_frame_interpolation,
|
||||
frame_interpolation_exp=first_req.frame_interpolation_exp,
|
||||
frame_interpolation_scale=first_req.frame_interpolation_scale,
|
||||
@@ -1262,6 +1264,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
|
||||
"data_type",
|
||||
"fps",
|
||||
"output_compression",
|
||||
"x264_preset",
|
||||
"enable_frame_interpolation",
|
||||
"frame_interpolation_exp",
|
||||
"frame_interpolation_scale",
|
||||
|
||||
@@ -110,6 +110,61 @@ def test_video_with_audio_uses_single_pass_encoder(tmp_path, monkeypatch):
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch):
|
||||
presets = []
|
||||
|
||||
class FakeWavFile:
|
||||
@staticmethod
|
||||
def write(*_args, **_kwargs):
|
||||
pass
|
||||
|
||||
def parallel_save(samples, _paths, **kwargs):
|
||||
presets.append(("parallel", kwargs["x264_preset"]))
|
||||
return [False] * len(samples)
|
||||
|
||||
def direct_save(**kwargs):
|
||||
presets.append(("direct", kwargs["x264_preset"]))
|
||||
return False
|
||||
|
||||
def mimsave_spy(_path, _frames, **kwargs):
|
||||
encoder = "single-pass" if "audio_path" in kwargs else "imageio"
|
||||
presets.append((encoder, kwargs["output_params"][-1]))
|
||||
|
||||
monkeypatch.setattr(output_utils, "_try_save_cuda_videos_direct", parallel_save)
|
||||
monkeypatch.setattr(output_utils, "_try_save_cuda_video_direct", direct_save)
|
||||
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
|
||||
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
|
||||
|
||||
output_utils.save_outputs(
|
||||
[torch.zeros((3, 1, 2, 3)), torch.zeros((3, 1, 2, 3))],
|
||||
DataType.VIDEO,
|
||||
fps=24,
|
||||
save_output=True,
|
||||
build_output_path=lambda idx: str(tmp_path / f"sample_{idx}.mp4"),
|
||||
x264_preset="ultrafast",
|
||||
)
|
||||
save_materialized_output(
|
||||
MaterializedOutput(
|
||||
sample=None,
|
||||
frames=[_rgb_frame()],
|
||||
audio=np.zeros((320, 2), dtype=np.float32),
|
||||
fps=24,
|
||||
),
|
||||
DataType.VIDEO,
|
||||
str(tmp_path / "with_audio.mp4"),
|
||||
audio_sample_rate=32000,
|
||||
x264_preset="ultrafast",
|
||||
)
|
||||
|
||||
assert {encoder for encoder, _ in presets} == {
|
||||
"parallel",
|
||||
"direct",
|
||||
"imageio",
|
||||
"single-pass",
|
||||
}
|
||||
assert {preset for _, preset in presets} == {"ultrafast"}
|
||||
|
||||
|
||||
def test_video_audio_single_pass_failure_falls_back(tmp_path, monkeypatch):
|
||||
output_path = tmp_path / "sample.mp4"
|
||||
calls = []
|
||||
|
||||
@@ -14,7 +14,10 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
DataType,
|
||||
SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import X264_PRESET, save_outputs
|
||||
|
||||
FPS = 8
|
||||
@@ -90,6 +93,49 @@ def test_saved_video_carries_the_configured_preset(tmp_path):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
|
||||
@pytest.mark.parametrize(
|
||||
"device",
|
||||
[
|
||||
# a host tensor saves through imageio, a CUDA one through the ffmpeg pipe
|
||||
"cpu",
|
||||
pytest.param(
|
||||
"cuda",
|
||||
marks=pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="direct save needs CUDA"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("preset", ["ultrafast", "slow"])
|
||||
def test_request_preset_reaches_the_encoder(tmp_path, device, preset):
|
||||
rng = np.random.default_rng(2)
|
||||
frames = rng.integers(0, 256, (FRAMES, SIZE, SIZE, 3), dtype=np.uint8)
|
||||
sample = torch.from_numpy(frames).permute(3, 0, 1, 2).float() / 255.0
|
||||
|
||||
saved = tmp_path / "clip.mp4"
|
||||
save_outputs(
|
||||
[sample.to(device)],
|
||||
DataType.VIDEO,
|
||||
FPS,
|
||||
True,
|
||||
lambda _idx: str(saved),
|
||||
x264_preset=preset,
|
||||
)
|
||||
|
||||
got = _x264_options(saved)
|
||||
expected = _x264_options(_reference_encode(tmp_path, frames, preset))
|
||||
for key in PRESET_DERIVED_KEYS:
|
||||
assert got.get(key) == expected.get(key), f"{key} does not match {preset}"
|
||||
|
||||
|
||||
def test_unknown_preset_is_rejected():
|
||||
assert SamplingParams(x264_preset="ultrafast").x264_preset == "ultrafast"
|
||||
assert SamplingParams().x264_preset is None
|
||||
with pytest.raises(ValueError, match="x264_preset must be one of"):
|
||||
SamplingParams(x264_preset="lightspeed")
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
|
||||
def test_saved_video_decodes_to_every_frame(tmp_path):
|
||||
rng = np.random.default_rng(1)
|
||||
|
||||
@@ -135,6 +135,7 @@ def _video_request():
|
||||
fps=24,
|
||||
save_output=True,
|
||||
output_compression=50,
|
||||
x264_preset="ultrafast",
|
||||
output_file_path=Mock(return_value="/tmp/generated.mp4"),
|
||||
)
|
||||
|
||||
@@ -179,6 +180,7 @@ def test_h3_generation_runs_video_lifecycle_and_preserves_audio():
|
||||
sampling_params.cleanup_video_request.assert_called_once_with(request)
|
||||
assert save_outputs.call_args.kwargs["audio"] is result.audio
|
||||
assert save_outputs.call_args.kwargs["audio_sample_rate"] == 24000
|
||||
assert save_outputs.call_args.kwargs["x264_preset"] == "ultrafast"
|
||||
|
||||
|
||||
def test_h3_generation_cleans_up_after_scheduler_error():
|
||||
|
||||
Reference in New Issue
Block a user