[diffusion] feat: let requests choose the video libx264 preset (#41797)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Mick
2026-10-04 16:22:36 +08:00
committed by GitHub
co-authored by Mick Qian Claude Opus 5.5
parent fbce0d9478
commit ba68306256
13 changed files with 172 additions and 4 deletions
@@ -315,6 +315,15 @@ curl -sS -X POST "http://localhost:30010/v1/videos" \
}'
```
#### MP4 encoding
`x264_preset` picks the libx264 preset for the returned MP4: `ultrafast`,
`superfast`, `veryfast`, `faster`, `fast` (the default), `medium`, `slow`,
`slower`, `veryslow` or `placebo`. Faster presets finish encoding sooner and
trade file size and fidelity at the same `output_compression`/`output_quality`.
Send it at the top level of a JSON body or as a multipart form field;
`sglang generate` takes `--x264-preset`.
**List videos**
**Endpoint:** `GET /v1/videos`
@@ -157,6 +157,7 @@ def _generate_minimax_h3(
audio=result.audio,
audio_sample_rate=result.audio_sample_rate,
output_compression=request.output_compression,
x264_preset=request.x264_preset,
)
sampling_params.validate_video_final_outputs(output_paths, request)
return output_paths[0]
@@ -59,6 +59,20 @@ def generate_request_id() -> str:
# is cumulative and may also enable model-owned approximate optimizations.
QUALITY_LEVELS: tuple[str, ...] = ("lossless", "extra-high", "high")
KERNEL_FUSION_QUALITY_LEVELS = frozenset({"extra-high", "high"})
# libx264 presets, fastest first: each spends more encode time for a smaller
# file at the same CRF.
X264_PRESETS: tuple[str, ...] = (
"ultrafast",
"superfast",
"veryfast",
"faster",
"fast",
"medium",
"slow",
"slower",
"veryslow",
"placebo",
)
@dataclass(frozen=True)
@@ -232,6 +246,8 @@ class SamplingParams:
)
output_quality: str | None = "default"
output_compression: int | None = None
# MP4 encode speed; None keeps the server's default preset
x264_preset: str | None = None
# Model-owned, request-scoped quality level.
#
# - "lossless" (default): the exact reference path. Output is expected to
@@ -614,6 +630,12 @@ class SamplingParams:
f"seed must be a non-negative int or list of ints, got {self.seed!r}"
)
if self.x264_preset is not None and self.x264_preset not in X264_PRESETS:
raise ValueError(
f"x264_preset must be one of {', '.join(X264_PRESETS)}, "
f"got {self.x264_preset!r}"
)
# Used by seconds() and video writer; fps <= 0 is always invalid.
if not isinstance(self.fps, int) or self.fps <= 0:
raise ValueError(f"fps must be a positive int, got {self.fps!r}")
@@ -1215,6 +1237,16 @@ class SamplingParams:
type=int,
help="Output compression level (0-100, higher means better quality but larger file size)",
)
add_argument(
"--x264-preset",
type=str,
choices=list(X264_PRESETS),
help=(
"libx264 preset for MP4 output. Faster presets encode sooner "
"and produce larger files at the same quality setting; "
"unset keeps the default (fast)."
),
)
add_argument(
"--quality",
type=str,
@@ -407,6 +407,7 @@ class DiffGenerator:
audios_out=audios_out,
frames_out=frames_out,
output_compression=requests[0].output_compression,
x264_preset=requests[0].x264_preset,
enable_frame_interpolation=requests[
0
].enable_frame_interpolation,
@@ -347,6 +347,8 @@ async def forward_to_scheduler(
lambda _idx: output_file_path,
audio=response.audio,
audio_sample_rate=response.audio_sample_rate,
output_compression=sp.output_compression,
x264_preset=sp.x264_preset,
enable_frame_interpolation=sp.enable_frame_interpolation,
frame_interpolation_exp=sp.frame_interpolation_exp,
frame_interpolation_scale=sp.frame_interpolation_scale,
@@ -158,6 +158,7 @@ class VideoGenerationsRequest(BaseModel):
upscaling_scale: Optional[int] = 4
output_quality: Optional[str] = "default"
output_compression: Optional[int] = None
x264_preset: Optional[str] = None
output_path: Optional[str] = None
diffusers_kwargs: Optional[Dict[str, Any]] = None # kwargs for diffusers backend
# Performance profiling
@@ -494,6 +494,7 @@ async def process_generation_batch(
audio=result.audio,
audio_sample_rate=result.audio_sample_rate,
output_compression=batch.output_compression,
x264_preset=batch.x264_preset,
enable_frame_interpolation=batch.enable_frame_interpolation,
frame_interpolation_exp=batch.frame_interpolation_exp,
frame_interpolation_scale=batch.frame_interpolation_scale,
@@ -309,6 +309,7 @@ def _build_video_sampling_params(request_id: str, request: VideoGenerationsReque
"output_path": request.output_path,
"quality": _extra_value(request, "quality"),
"output_compression": request.output_compression,
"x264_preset": request.x264_preset,
"output_quality": request.output_quality,
"perf_dump_path": request.perf_dump_path,
"profile": request.profile,
@@ -503,6 +504,7 @@ async def create_video(
upscaling_scale: Optional[int] = Form(None),
output_quality: Optional[str] = Form(None),
output_compression: Optional[int] = Form(None),
x264_preset: Optional[str] = Form(None),
output_path: Optional[str] = Form(None),
perf_dump_path: Optional[str] = Form(None),
extra_params: Optional[str] = Form(None),
@@ -649,6 +651,7 @@ async def create_video(
),
upscaling_scale=form_value("upscaling_scale", upscaling_scale),
output_compression=form_value("output_compression", output_compression),
x264_preset=form_value("x264_preset", x264_preset),
output_quality=form_value("output_quality", output_quality),
output_path=form_value("output_path", output_path),
perf_dump_path=form_value("perf_dump_path", perf_dump_path),
@@ -363,6 +363,7 @@ def _try_save_cuda_video_direct(
fps: int,
audio_sample_rate: Optional[int],
output_compression: Optional[int],
x264_preset: Optional[str] = None,
) -> bool:
"""Stream CUDA RGB chunks to ffmpeg through a registered memfd."""
if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"):
@@ -433,7 +434,7 @@ def _try_save_cuda_video_direct(
"-vcodec",
"libx264",
"-preset",
X264_PRESET,
x264_preset or X264_PRESET,
"-pix_fmt",
"yuv420p",
"-crf",
@@ -541,6 +542,7 @@ def _try_save_cuda_videos_direct(
fps: int,
audio_sample_rate: Optional[int],
output_compression: Optional[int],
x264_preset: Optional[str] = None,
) -> list[bool] | None:
"""Save independent CUDA videos concurrently when memory permits."""
if len(samples) < 2 or len(samples) != len(save_file_paths):
@@ -601,6 +603,7 @@ def _try_save_cuda_videos_direct(
fps=fps,
audio_sample_rate=audio_sample_rate,
output_compression=output_compression,
x264_preset=x264_preset,
)
try:
@@ -712,6 +715,7 @@ def _try_save_video_with_audio(
audio_sample_rate: Optional[int],
output_format: str,
quality: float,
x264_preset: Optional[str] = None,
) -> bool:
"""Encode video and audio in one ffmpeg pass when audio is available."""
audio_np = _normalize_audio_to_numpy(audio)
@@ -742,7 +746,7 @@ def _try_save_video_with_audio(
quality=quality,
audio_path=tmp_wav_path,
audio_codec="aac",
output_params=["-preset", X264_PRESET],
output_params=["-preset", x264_preset or X264_PRESET],
)
return True
except Exception as e:
@@ -948,6 +952,7 @@ def save_materialized_output(
save_output: bool = True,
audio_sample_rate: Optional[int] = None,
output_compression: Optional[int] = None,
x264_preset: Optional[str] = None,
) -> None:
if not save_output:
return
@@ -967,6 +972,7 @@ def save_materialized_output(
audio_sample_rate=audio_sample_rate,
output_format=output_format,
quality=quality,
x264_preset=x264_preset,
)
if not saved_with_audio:
imageio.mimsave(
@@ -976,7 +982,7 @@ def save_materialized_output(
format=output_format,
codec="libx264",
quality=quality,
output_params=["-preset", X264_PRESET],
output_params=["-preset", x264_preset or X264_PRESET],
)
_maybe_mux_audio_into_mp4(
@@ -1031,6 +1037,7 @@ def save_outputs(
audios_out: Optional[list[Any]] = None,
frames_out: Optional[list[Any]] = None,
output_compression: Optional[int] = None,
x264_preset: Optional[str] = None,
enable_frame_interpolation: bool = False,
frame_interpolation_exp: int = 1,
frame_interpolation_scale: float = 1.0,
@@ -1066,6 +1073,7 @@ def save_outputs(
fps=fps,
audio_sample_rate=audio_sample_rate,
output_compression=output_compression,
x264_preset=x264_preset,
)
for idx, (sample, save_file_path) in enumerate(zip(samples, save_file_paths)):
@@ -1101,6 +1109,7 @@ def save_outputs(
fps=fps,
audio_sample_rate=audio_sample_rate,
output_compression=output_compression,
x264_preset=x264_preset,
)
if direct_saved:
if samples_out is not None:
@@ -1119,6 +1128,7 @@ def save_outputs(
save_file_path,
audio_sample_rate=audio_sample_rate,
output_compression=output_compression,
x264_preset=x264_preset,
enable_frame_interpolation=enable_frame_interpolation,
frame_interpolation_exp=frame_interpolation_exp,
frame_interpolation_scale=frame_interpolation_scale,
@@ -1156,6 +1166,7 @@ def post_process_sample(
enable_upscaling: bool = False,
upscaling_model_path: Optional[str] = None,
upscaling_scale: int = 4,
x264_preset: Optional[str] = None,
) -> list[Any]:
"""materialize frames and save outputs (optional)"""
if data_type == DataType.ACTION:
@@ -1180,5 +1191,6 @@ def post_process_sample(
save_output=save_output,
audio_sample_rate=audio_sample_rate,
output_compression=output_compression,
x264_preset=x264_preset,
)
return materialized.frames
@@ -1210,6 +1210,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
audio=output_batch.audio,
audio_sample_rate=output_batch.audio_sample_rate,
output_compression=req.output_compression,
x264_preset=req.x264_preset,
enable_frame_interpolation=req.enable_frame_interpolation,
frame_interpolation_exp=req.frame_interpolation_exp,
frame_interpolation_scale=req.frame_interpolation_scale,
@@ -1242,6 +1243,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
audio=output_batch.audio,
audio_sample_rate=output_batch.audio_sample_rate,
output_compression=first_req.output_compression,
x264_preset=first_req.x264_preset,
enable_frame_interpolation=first_req.enable_frame_interpolation,
frame_interpolation_exp=first_req.frame_interpolation_exp,
frame_interpolation_scale=first_req.frame_interpolation_scale,
@@ -1262,6 +1264,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin):
"data_type",
"fps",
"output_compression",
"x264_preset",
"enable_frame_interpolation",
"frame_interpolation_exp",
"frame_interpolation_scale",
@@ -110,6 +110,61 @@ def test_video_with_audio_uses_single_pass_encoder(tmp_path, monkeypatch):
assert len(calls) == 1
def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch):
presets = []
class FakeWavFile:
@staticmethod
def write(*_args, **_kwargs):
pass
def parallel_save(samples, _paths, **kwargs):
presets.append(("parallel", kwargs["x264_preset"]))
return [False] * len(samples)
def direct_save(**kwargs):
presets.append(("direct", kwargs["x264_preset"]))
return False
def mimsave_spy(_path, _frames, **kwargs):
encoder = "single-pass" if "audio_path" in kwargs else "imageio"
presets.append((encoder, kwargs["output_params"][-1]))
monkeypatch.setattr(output_utils, "_try_save_cuda_videos_direct", parallel_save)
monkeypatch.setattr(output_utils, "_try_save_cuda_video_direct", direct_save)
monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy)
monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile)
output_utils.save_outputs(
[torch.zeros((3, 1, 2, 3)), torch.zeros((3, 1, 2, 3))],
DataType.VIDEO,
fps=24,
save_output=True,
build_output_path=lambda idx: str(tmp_path / f"sample_{idx}.mp4"),
x264_preset="ultrafast",
)
save_materialized_output(
MaterializedOutput(
sample=None,
frames=[_rgb_frame()],
audio=np.zeros((320, 2), dtype=np.float32),
fps=24,
),
DataType.VIDEO,
str(tmp_path / "with_audio.mp4"),
audio_sample_rate=32000,
x264_preset="ultrafast",
)
assert {encoder for encoder, _ in presets} == {
"parallel",
"direct",
"imageio",
"single-pass",
}
assert {preset for _, preset in presets} == {"ultrafast"}
def test_video_audio_single_pass_failure_falls_back(tmp_path, monkeypatch):
output_path = tmp_path / "sample.mp4"
calls = []
@@ -14,7 +14,10 @@ import numpy as np
import pytest
import torch
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
from sglang.multimodal_gen.configs.sample.sampling_params import (
DataType,
SamplingParams,
)
from sglang.multimodal_gen.runtime.entrypoints.utils import X264_PRESET, save_outputs
FPS = 8
@@ -90,6 +93,49 @@ def test_saved_video_carries_the_configured_preset(tmp_path):
)
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
@pytest.mark.parametrize(
"device",
[
# a host tensor saves through imageio, a CUDA one through the ffmpeg pipe
"cpu",
pytest.param(
"cuda",
marks=pytest.mark.skipif(
not torch.cuda.is_available(), reason="direct save needs CUDA"
),
),
],
)
@pytest.mark.parametrize("preset", ["ultrafast", "slow"])
def test_request_preset_reaches_the_encoder(tmp_path, device, preset):
rng = np.random.default_rng(2)
frames = rng.integers(0, 256, (FRAMES, SIZE, SIZE, 3), dtype=np.uint8)
sample = torch.from_numpy(frames).permute(3, 0, 1, 2).float() / 255.0
saved = tmp_path / "clip.mp4"
save_outputs(
[sample.to(device)],
DataType.VIDEO,
FPS,
True,
lambda _idx: str(saved),
x264_preset=preset,
)
got = _x264_options(saved)
expected = _x264_options(_reference_encode(tmp_path, frames, preset))
for key in PRESET_DERIVED_KEYS:
assert got.get(key) == expected.get(key), f"{key} does not match {preset}"
def test_unknown_preset_is_rejected():
assert SamplingParams(x264_preset="ultrafast").x264_preset == "ultrafast"
assert SamplingParams().x264_preset is None
with pytest.raises(ValueError, match="x264_preset must be one of"):
SamplingParams(x264_preset="lightspeed")
@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg")
def test_saved_video_decodes_to_every_frame(tmp_path):
rng = np.random.default_rng(1)
@@ -135,6 +135,7 @@ def _video_request():
fps=24,
save_output=True,
output_compression=50,
x264_preset="ultrafast",
output_file_path=Mock(return_value="/tmp/generated.mp4"),
)
@@ -179,6 +180,7 @@ def test_h3_generation_runs_video_lifecycle_and_preserves_audio():
sampling_params.cleanup_video_request.assert_called_once_with(request)
assert save_outputs.call_args.kwargs["audio"] is result.audio
assert save_outputs.call_args.kwargs["audio_sample_rate"] == 24000
assert save_outputs.call_args.kwargs["x264_preset"] == "ultrafast"
def test_h3_generation_cleans_up_after_scheduler_error():