From ba68306256d64c53d302dc5d2f3be9e388af0738 Mon Sep 17 00:00:00 2001 From: Mick Date: Sun, 4 Oct 2026 16:22:36 +0800 Subject: [PATCH] [diffusion] feat: let requests choose the video libx264 preset (#41797) Co-authored-by: Mick Qian Co-authored-by: Claude Opus 5.5 --- docs/docs/sglang-diffusion/api/openai_api.mdx | 9 +++ .../multimodal_gen/apps/webui/minimax_h3.py | 1 + .../configs/sample/sampling_params.py | 32 +++++++++++ .../entrypoints/diffusion_generator.py | 1 + .../runtime/entrypoints/http_server.py | 2 + .../runtime/entrypoints/openai/protocol.py | 1 + .../runtime/entrypoints/openai/utils.py | 1 + .../runtime/entrypoints/openai/video_api.py | 3 + .../runtime/entrypoints/utils.py | 18 +++++- .../runtime/managers/gpu_worker.py | 3 + .../test/unit/test_output_saving.py | 55 +++++++++++++++++++ .../test/unit/test_video_encode_preset.py | 48 +++++++++++++++- .../multimodal_gen/test/unit/test_webui.py | 2 + 13 files changed, 172 insertions(+), 4 deletions(-) diff --git a/docs/docs/sglang-diffusion/api/openai_api.mdx b/docs/docs/sglang-diffusion/api/openai_api.mdx index ba6d70d11c..8e5a3f8d51 100644 --- a/docs/docs/sglang-diffusion/api/openai_api.mdx +++ b/docs/docs/sglang-diffusion/api/openai_api.mdx @@ -315,6 +315,15 @@ curl -sS -X POST "http://localhost:30010/v1/videos" \ }' ``` +#### MP4 encoding + +`x264_preset` picks the libx264 preset for the returned MP4: `ultrafast`, +`superfast`, `veryfast`, `faster`, `fast` (the default), `medium`, `slow`, +`slower`, `veryslow` or `placebo`. Faster presets finish encoding sooner and +trade file size and fidelity at the same `output_compression`/`output_quality`. +Send it at the top level of a JSON body or as a multipart form field; +`sglang generate` takes `--x264-preset`. + **List videos** **Endpoint:** `GET /v1/videos` diff --git a/python/sglang/multimodal_gen/apps/webui/minimax_h3.py b/python/sglang/multimodal_gen/apps/webui/minimax_h3.py index 9157ff7b8b..9416c65ce7 100644 --- a/python/sglang/multimodal_gen/apps/webui/minimax_h3.py +++ b/python/sglang/multimodal_gen/apps/webui/minimax_h3.py @@ -157,6 +157,7 @@ def _generate_minimax_h3( audio=result.audio, audio_sample_rate=result.audio_sample_rate, output_compression=request.output_compression, + x264_preset=request.x264_preset, ) sampling_params.validate_video_final_outputs(output_paths, request) return output_paths[0] diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index 01dc5a9c99..4faa2de5b2 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -59,6 +59,20 @@ def generate_request_id() -> str: # is cumulative and may also enable model-owned approximate optimizations. QUALITY_LEVELS: tuple[str, ...] = ("lossless", "extra-high", "high") KERNEL_FUSION_QUALITY_LEVELS = frozenset({"extra-high", "high"}) +# libx264 presets, fastest first: each spends more encode time for a smaller +# file at the same CRF. +X264_PRESETS: tuple[str, ...] = ( + "ultrafast", + "superfast", + "veryfast", + "faster", + "fast", + "medium", + "slow", + "slower", + "veryslow", + "placebo", +) @dataclass(frozen=True) @@ -232,6 +246,8 @@ class SamplingParams: ) output_quality: str | None = "default" output_compression: int | None = None + # MP4 encode speed; None keeps the server's default preset + x264_preset: str | None = None # Model-owned, request-scoped quality level. # # - "lossless" (default): the exact reference path. Output is expected to @@ -614,6 +630,12 @@ class SamplingParams: f"seed must be a non-negative int or list of ints, got {self.seed!r}" ) + if self.x264_preset is not None and self.x264_preset not in X264_PRESETS: + raise ValueError( + f"x264_preset must be one of {', '.join(X264_PRESETS)}, " + f"got {self.x264_preset!r}" + ) + # Used by seconds() and video writer; fps <= 0 is always invalid. if not isinstance(self.fps, int) or self.fps <= 0: raise ValueError(f"fps must be a positive int, got {self.fps!r}") @@ -1215,6 +1237,16 @@ class SamplingParams: type=int, help="Output compression level (0-100, higher means better quality but larger file size)", ) + add_argument( + "--x264-preset", + type=str, + choices=list(X264_PRESETS), + help=( + "libx264 preset for MP4 output. Faster presets encode sooner " + "and produce larger files at the same quality setting; " + "unset keeps the default (fast)." + ), + ) add_argument( "--quality", type=str, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py index 9fce9bb852..2212ad22fa 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/diffusion_generator.py @@ -407,6 +407,7 @@ class DiffGenerator: audios_out=audios_out, frames_out=frames_out, output_compression=requests[0].output_compression, + x264_preset=requests[0].x264_preset, enable_frame_interpolation=requests[ 0 ].enable_frame_interpolation, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py index d7ea394227..e8226e5260 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/http_server.py @@ -347,6 +347,8 @@ async def forward_to_scheduler( lambda _idx: output_file_path, audio=response.audio, audio_sample_rate=response.audio_sample_rate, + output_compression=sp.output_compression, + x264_preset=sp.x264_preset, enable_frame_interpolation=sp.enable_frame_interpolation, frame_interpolation_exp=sp.frame_interpolation_exp, frame_interpolation_scale=sp.frame_interpolation_scale, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py index f421a3790f..13e8990084 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/protocol.py @@ -158,6 +158,7 @@ class VideoGenerationsRequest(BaseModel): upscaling_scale: Optional[int] = 4 output_quality: Optional[str] = "default" output_compression: Optional[int] = None + x264_preset: Optional[str] = None output_path: Optional[str] = None diffusers_kwargs: Optional[Dict[str, Any]] = None # kwargs for diffusers backend # Performance profiling diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py index fb75a79ec1..cb43d29873 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/utils.py @@ -494,6 +494,7 @@ async def process_generation_batch( audio=result.audio, audio_sample_rate=result.audio_sample_rate, output_compression=batch.output_compression, + x264_preset=batch.x264_preset, enable_frame_interpolation=batch.enable_frame_interpolation, frame_interpolation_exp=batch.frame_interpolation_exp, frame_interpolation_scale=batch.frame_interpolation_scale, diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py index a4380e1606..29992ed58c 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/openai/video_api.py @@ -309,6 +309,7 @@ def _build_video_sampling_params(request_id: str, request: VideoGenerationsReque "output_path": request.output_path, "quality": _extra_value(request, "quality"), "output_compression": request.output_compression, + "x264_preset": request.x264_preset, "output_quality": request.output_quality, "perf_dump_path": request.perf_dump_path, "profile": request.profile, @@ -503,6 +504,7 @@ async def create_video( upscaling_scale: Optional[int] = Form(None), output_quality: Optional[str] = Form(None), output_compression: Optional[int] = Form(None), + x264_preset: Optional[str] = Form(None), output_path: Optional[str] = Form(None), perf_dump_path: Optional[str] = Form(None), extra_params: Optional[str] = Form(None), @@ -649,6 +651,7 @@ async def create_video( ), upscaling_scale=form_value("upscaling_scale", upscaling_scale), output_compression=form_value("output_compression", output_compression), + x264_preset=form_value("x264_preset", x264_preset), output_quality=form_value("output_quality", output_quality), output_path=form_value("output_path", output_path), perf_dump_path=form_value("perf_dump_path", perf_dump_path), diff --git a/python/sglang/multimodal_gen/runtime/entrypoints/utils.py b/python/sglang/multimodal_gen/runtime/entrypoints/utils.py index 5ef6f60960..520af1b047 100644 --- a/python/sglang/multimodal_gen/runtime/entrypoints/utils.py +++ b/python/sglang/multimodal_gen/runtime/entrypoints/utils.py @@ -363,6 +363,7 @@ def _try_save_cuda_video_direct( fps: int, audio_sample_rate: Optional[int], output_compression: Optional[int], + x264_preset: Optional[str] = None, ) -> bool: """Stream CUDA RGB chunks to ffmpeg through a registered memfd.""" if not hasattr(os, "memfd_create") or not hasattr(os, "sendfile"): @@ -433,7 +434,7 @@ def _try_save_cuda_video_direct( "-vcodec", "libx264", "-preset", - X264_PRESET, + x264_preset or X264_PRESET, "-pix_fmt", "yuv420p", "-crf", @@ -541,6 +542,7 @@ def _try_save_cuda_videos_direct( fps: int, audio_sample_rate: Optional[int], output_compression: Optional[int], + x264_preset: Optional[str] = None, ) -> list[bool] | None: """Save independent CUDA videos concurrently when memory permits.""" if len(samples) < 2 or len(samples) != len(save_file_paths): @@ -601,6 +603,7 @@ def _try_save_cuda_videos_direct( fps=fps, audio_sample_rate=audio_sample_rate, output_compression=output_compression, + x264_preset=x264_preset, ) try: @@ -712,6 +715,7 @@ def _try_save_video_with_audio( audio_sample_rate: Optional[int], output_format: str, quality: float, + x264_preset: Optional[str] = None, ) -> bool: """Encode video and audio in one ffmpeg pass when audio is available.""" audio_np = _normalize_audio_to_numpy(audio) @@ -742,7 +746,7 @@ def _try_save_video_with_audio( quality=quality, audio_path=tmp_wav_path, audio_codec="aac", - output_params=["-preset", X264_PRESET], + output_params=["-preset", x264_preset or X264_PRESET], ) return True except Exception as e: @@ -948,6 +952,7 @@ def save_materialized_output( save_output: bool = True, audio_sample_rate: Optional[int] = None, output_compression: Optional[int] = None, + x264_preset: Optional[str] = None, ) -> None: if not save_output: return @@ -967,6 +972,7 @@ def save_materialized_output( audio_sample_rate=audio_sample_rate, output_format=output_format, quality=quality, + x264_preset=x264_preset, ) if not saved_with_audio: imageio.mimsave( @@ -976,7 +982,7 @@ def save_materialized_output( format=output_format, codec="libx264", quality=quality, - output_params=["-preset", X264_PRESET], + output_params=["-preset", x264_preset or X264_PRESET], ) _maybe_mux_audio_into_mp4( @@ -1031,6 +1037,7 @@ def save_outputs( audios_out: Optional[list[Any]] = None, frames_out: Optional[list[Any]] = None, output_compression: Optional[int] = None, + x264_preset: Optional[str] = None, enable_frame_interpolation: bool = False, frame_interpolation_exp: int = 1, frame_interpolation_scale: float = 1.0, @@ -1066,6 +1073,7 @@ def save_outputs( fps=fps, audio_sample_rate=audio_sample_rate, output_compression=output_compression, + x264_preset=x264_preset, ) for idx, (sample, save_file_path) in enumerate(zip(samples, save_file_paths)): @@ -1101,6 +1109,7 @@ def save_outputs( fps=fps, audio_sample_rate=audio_sample_rate, output_compression=output_compression, + x264_preset=x264_preset, ) if direct_saved: if samples_out is not None: @@ -1119,6 +1128,7 @@ def save_outputs( save_file_path, audio_sample_rate=audio_sample_rate, output_compression=output_compression, + x264_preset=x264_preset, enable_frame_interpolation=enable_frame_interpolation, frame_interpolation_exp=frame_interpolation_exp, frame_interpolation_scale=frame_interpolation_scale, @@ -1156,6 +1166,7 @@ def post_process_sample( enable_upscaling: bool = False, upscaling_model_path: Optional[str] = None, upscaling_scale: int = 4, + x264_preset: Optional[str] = None, ) -> list[Any]: """materialize frames and save outputs (optional)""" if data_type == DataType.ACTION: @@ -1180,5 +1191,6 @@ def post_process_sample( save_output=save_output, audio_sample_rate=audio_sample_rate, output_compression=output_compression, + x264_preset=x264_preset, ) return materialized.frames diff --git a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py index cce60a69c9..901d475b6e 100644 --- a/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py +++ b/python/sglang/multimodal_gen/runtime/managers/gpu_worker.py @@ -1210,6 +1210,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): audio=output_batch.audio, audio_sample_rate=output_batch.audio_sample_rate, output_compression=req.output_compression, + x264_preset=req.x264_preset, enable_frame_interpolation=req.enable_frame_interpolation, frame_interpolation_exp=req.frame_interpolation_exp, frame_interpolation_scale=req.frame_interpolation_scale, @@ -1242,6 +1243,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): audio=output_batch.audio, audio_sample_rate=output_batch.audio_sample_rate, output_compression=first_req.output_compression, + x264_preset=first_req.x264_preset, enable_frame_interpolation=first_req.enable_frame_interpolation, frame_interpolation_exp=first_req.frame_interpolation_exp, frame_interpolation_scale=first_req.frame_interpolation_scale, @@ -1262,6 +1264,7 @@ class GPUWorker(GPUWorkerPostTrainingMixin): "data_type", "fps", "output_compression", + "x264_preset", "enable_frame_interpolation", "frame_interpolation_exp", "frame_interpolation_scale", diff --git a/python/sglang/multimodal_gen/test/unit/test_output_saving.py b/python/sglang/multimodal_gen/test/unit/test_output_saving.py index c4de2acaf5..27150d883d 100644 --- a/python/sglang/multimodal_gen/test/unit/test_output_saving.py +++ b/python/sglang/multimodal_gen/test/unit/test_output_saving.py @@ -110,6 +110,61 @@ def test_video_with_audio_uses_single_pass_encoder(tmp_path, monkeypatch): assert len(calls) == 1 +def test_x264_preset_reaches_every_video_save_path(tmp_path, monkeypatch): + presets = [] + + class FakeWavFile: + @staticmethod + def write(*_args, **_kwargs): + pass + + def parallel_save(samples, _paths, **kwargs): + presets.append(("parallel", kwargs["x264_preset"])) + return [False] * len(samples) + + def direct_save(**kwargs): + presets.append(("direct", kwargs["x264_preset"])) + return False + + def mimsave_spy(_path, _frames, **kwargs): + encoder = "single-pass" if "audio_path" in kwargs else "imageio" + presets.append((encoder, kwargs["output_params"][-1])) + + monkeypatch.setattr(output_utils, "_try_save_cuda_videos_direct", parallel_save) + monkeypatch.setattr(output_utils, "_try_save_cuda_video_direct", direct_save) + monkeypatch.setattr(output_utils.imageio, "mimsave", mimsave_spy) + monkeypatch.setattr(output_utils, "scipy_wavfile", FakeWavFile) + + output_utils.save_outputs( + [torch.zeros((3, 1, 2, 3)), torch.zeros((3, 1, 2, 3))], + DataType.VIDEO, + fps=24, + save_output=True, + build_output_path=lambda idx: str(tmp_path / f"sample_{idx}.mp4"), + x264_preset="ultrafast", + ) + save_materialized_output( + MaterializedOutput( + sample=None, + frames=[_rgb_frame()], + audio=np.zeros((320, 2), dtype=np.float32), + fps=24, + ), + DataType.VIDEO, + str(tmp_path / "with_audio.mp4"), + audio_sample_rate=32000, + x264_preset="ultrafast", + ) + + assert {encoder for encoder, _ in presets} == { + "parallel", + "direct", + "imageio", + "single-pass", + } + assert {preset for _, preset in presets} == {"ultrafast"} + + def test_video_audio_single_pass_failure_falls_back(tmp_path, monkeypatch): output_path = tmp_path / "sample.mp4" calls = [] diff --git a/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py b/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py index 5c61892fbd..39d132c751 100644 --- a/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py +++ b/python/sglang/multimodal_gen/test/unit/test_video_encode_preset.py @@ -14,7 +14,10 @@ import numpy as np import pytest import torch -from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.configs.sample.sampling_params import ( + DataType, + SamplingParams, +) from sglang.multimodal_gen.runtime.entrypoints.utils import X264_PRESET, save_outputs FPS = 8 @@ -90,6 +93,49 @@ def test_saved_video_carries_the_configured_preset(tmp_path): ) +@pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg") +@pytest.mark.parametrize( + "device", + [ + # a host tensor saves through imageio, a CUDA one through the ffmpeg pipe + "cpu", + pytest.param( + "cuda", + marks=pytest.mark.skipif( + not torch.cuda.is_available(), reason="direct save needs CUDA" + ), + ), + ], +) +@pytest.mark.parametrize("preset", ["ultrafast", "slow"]) +def test_request_preset_reaches_the_encoder(tmp_path, device, preset): + rng = np.random.default_rng(2) + frames = rng.integers(0, 256, (FRAMES, SIZE, SIZE, 3), dtype=np.uint8) + sample = torch.from_numpy(frames).permute(3, 0, 1, 2).float() / 255.0 + + saved = tmp_path / "clip.mp4" + save_outputs( + [sample.to(device)], + DataType.VIDEO, + FPS, + True, + lambda _idx: str(saved), + x264_preset=preset, + ) + + got = _x264_options(saved) + expected = _x264_options(_reference_encode(tmp_path, frames, preset)) + for key in PRESET_DERIVED_KEYS: + assert got.get(key) == expected.get(key), f"{key} does not match {preset}" + + +def test_unknown_preset_is_rejected(): + assert SamplingParams(x264_preset="ultrafast").x264_preset == "ultrafast" + assert SamplingParams().x264_preset is None + with pytest.raises(ValueError, match="x264_preset must be one of"): + SamplingParams(x264_preset="lightspeed") + + @pytest.mark.skipif(shutil.which("ffmpeg") is None, reason="needs ffmpeg") def test_saved_video_decodes_to_every_frame(tmp_path): rng = np.random.default_rng(1) diff --git a/python/sglang/multimodal_gen/test/unit/test_webui.py b/python/sglang/multimodal_gen/test/unit/test_webui.py index 5dac4422d9..af1b98823e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_webui.py +++ b/python/sglang/multimodal_gen/test/unit/test_webui.py @@ -135,6 +135,7 @@ def _video_request(): fps=24, save_output=True, output_compression=50, + x264_preset="ultrafast", output_file_path=Mock(return_value="/tmp/generated.mp4"), ) @@ -179,6 +180,7 @@ def test_h3_generation_runs_video_lifecycle_and_preserves_audio(): sampling_params.cleanup_video_request.assert_called_once_with(request) assert save_outputs.call_args.kwargs["audio"] is result.audio assert save_outputs.call_args.kwargs["audio_sample_rate"] == 24000 + assert save_outputs.call_args.kwargs["x264_preset"] == "ultrafast" def test_h3_generation_cleans_up_after_scheduler_error():