[diffusion] feat: check the final MP4 in-process instead of spawning ffprobe (#41825)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Mick Qian
Claude Opus 5.5
parent
0931a72eb2
commit
45c8ddddd3
+63
-8
@@ -23,6 +23,12 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m
|
||||
canonical_minimax_h3_task,
|
||||
)
|
||||
|
||||
try:
|
||||
# imported with the adapter so the first output check does not pay for it
|
||||
import av
|
||||
except ImportError: # pragma: no cover
|
||||
av = None
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import Req
|
||||
@@ -353,14 +359,7 @@ class MiniMaxH3VideoModelAdapter:
|
||||
return final_media_fields
|
||||
|
||||
|
||||
def _probe_minimax_h3_output_fields(
|
||||
path: str,
|
||||
*,
|
||||
expected_frame_count: int | None = None,
|
||||
expected_size: tuple[int, int] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Validate one final MiniMax H3 AV file and derive truthful metadata."""
|
||||
|
||||
def _ffprobe_output_media(path: str) -> dict[str, Any]:
|
||||
try:
|
||||
probe = subprocess.run(
|
||||
[
|
||||
@@ -401,6 +400,62 @@ def _probe_minimax_h3_output_fields(
|
||||
) from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("ffprobe returned invalid JSON for MiniMax H3 output")
|
||||
return payload
|
||||
|
||||
|
||||
def _pyav_stream_entries(stream: Any) -> dict[str, Any]:
|
||||
entries: dict[str, Any] = {"codec_type": stream.type}
|
||||
context = stream.codec_context
|
||||
if context is not None:
|
||||
entries["codec_name"] = context.codec.canonical_name
|
||||
if stream.type == "video":
|
||||
rate = stream.average_rate
|
||||
entries.update(
|
||||
width=context.width,
|
||||
height=context.height,
|
||||
pix_fmt=context.pix_fmt,
|
||||
avg_frame_rate=(
|
||||
f"{rate.numerator}/{rate.denominator}" if rate else "0/0"
|
||||
),
|
||||
)
|
||||
elif stream.type == "audio":
|
||||
entries.update(
|
||||
sample_rate=str(context.sample_rate), channels=context.channels
|
||||
)
|
||||
if stream.frames:
|
||||
entries["nb_frames"] = str(stream.frames)
|
||||
if stream.duration is not None and stream.time_base is not None:
|
||||
entries["duration"] = f"{float(stream.duration * stream.time_base):f}"
|
||||
return entries
|
||||
|
||||
|
||||
def _read_output_media(path: str) -> dict[str, Any]:
|
||||
"""The ``_ffprobe_output_media`` fields the checks read, parsed in-process by PyAV.
|
||||
|
||||
Starting ffprobe costs ~30 ms on the request's critical path.
|
||||
"""
|
||||
if av is None:
|
||||
return _ffprobe_output_media(path)
|
||||
try:
|
||||
with av.open(str(path)) as container:
|
||||
streams = [_pyav_stream_entries(stream) for stream in container.streams]
|
||||
format_entries: dict[str, Any] = {"format_name": container.format.name}
|
||||
if container.duration:
|
||||
format_entries["duration"] = f"{container.duration / av.time_base:f}"
|
||||
except (av.error.FFmpegError, OSError) as exc:
|
||||
raise RuntimeError(f"could not read final MiniMax H3 output: {exc}") from exc
|
||||
return {"streams": streams, "format": format_entries}
|
||||
|
||||
|
||||
def _probe_minimax_h3_output_fields(
|
||||
path: str,
|
||||
*,
|
||||
expected_frame_count: int | None = None,
|
||||
expected_size: tuple[int, int] | None = None,
|
||||
) -> dict[str, str]:
|
||||
"""Validate one final MiniMax H3 AV file and derive truthful metadata."""
|
||||
|
||||
payload = _read_output_media(path)
|
||||
streams = payload.get("streams") or []
|
||||
if not isinstance(streams, list):
|
||||
raise RuntimeError("ffprobe returned invalid stream metadata")
|
||||
|
||||
@@ -0,0 +1,175 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""The final MiniMax H3 MP4 check must reach ffprobe's verdict without spawning it.
|
||||
|
||||
Each fixture breaks one property the check reads, so matching verdicts show the
|
||||
in-process reader supplies every field that ffprobe did.
|
||||
"""
|
||||
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3 import (
|
||||
video_adapter,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
shutil.which("ffmpeg") is None or shutil.which("ffprobe") is None,
|
||||
reason="needs ffmpeg and ffprobe",
|
||||
)
|
||||
needs_pyav = pytest.mark.skipif(video_adapter.av is None, reason="needs PyAV")
|
||||
|
||||
EXPECTED = {"expected_frame_count": 48, "expected_size": (320, 192)}
|
||||
VALID_FIELDS = {"size": "320x192", "seconds": "2"}
|
||||
X264 = ("-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p")
|
||||
|
||||
# name -> (encoder overrides, expected error substring or None when valid)
|
||||
CASES = {
|
||||
"valid": ({}, None),
|
||||
"sample_rate": ({"audio_rate": 48000}, "sample rate"),
|
||||
"mono": ({"channels": 1}, "stereo"),
|
||||
"silent": ({"audio": False}, "exactly one video stream"),
|
||||
"frame_rate": ({"rate": 25}, "frame rate"),
|
||||
"frame_count": ({"frames": 47, "audio_seconds": 47 / 24}, "frame count"),
|
||||
"size": ({"size": "336x192"}, "size does not match"),
|
||||
"av_drift": ({"audio_seconds": 1.0}, "drift"),
|
||||
"video_codec": ({"video": ("-c:v", "mpeg4")}, "video codec"),
|
||||
"audio_codec": ({"audio_codec": "ac3"}, "audio codec"),
|
||||
"pixel_format": (
|
||||
{"video": ("-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv444p")},
|
||||
"pixel format",
|
||||
),
|
||||
"matroska": ({"suffix": ".mkv"}, "frame count"),
|
||||
"subtitle_stream": ({"subtitle": True}, "exactly one video stream"),
|
||||
}
|
||||
UNREADABLE = ("truncated", "missing")
|
||||
|
||||
|
||||
def _encode(
|
||||
directory: Path,
|
||||
name: str,
|
||||
*,
|
||||
size: str = "320x192",
|
||||
rate: int = 24,
|
||||
frames: int = 48,
|
||||
audio: bool = True,
|
||||
audio_rate: int = 32000,
|
||||
channels: int = 2,
|
||||
audio_seconds: float = 2.0,
|
||||
audio_codec: str = "aac",
|
||||
video: tuple[str, ...] = X264,
|
||||
subtitle: bool = False,
|
||||
suffix: str = ".mp4",
|
||||
) -> str:
|
||||
path = directory / (name + suffix)
|
||||
command = ["ffmpeg", "-y", "-v", "error"]
|
||||
# end the video input; -frames:v can stop ffmpeg before the audio drains
|
||||
command += [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
f"testsrc2=size={size}:rate={rate},trim=end_frame={frames}",
|
||||
]
|
||||
maps = ["-map", "0:v"]
|
||||
if audio:
|
||||
command += [
|
||||
"-f",
|
||||
"lavfi",
|
||||
"-i",
|
||||
f"sine=frequency=440:sample_rate={audio_rate}:duration={audio_seconds}",
|
||||
]
|
||||
maps += ["-map", "1:a"]
|
||||
if subtitle:
|
||||
srt = directory / "caption.srt"
|
||||
srt.write_text("1\n00:00:00,000 --> 00:00:01,000\ncaption\n")
|
||||
command += ["-i", str(srt)]
|
||||
maps += ["-map", f"{2 if audio else 1}:s", "-c:s", "mov_text"]
|
||||
command += maps + [*video]
|
||||
if audio:
|
||||
command += ["-c:a", audio_codec, "-ac", str(channels)]
|
||||
subprocess.run(command + [str(path)], check=True, capture_output=True)
|
||||
return str(path)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def outputs(tmp_path_factory):
|
||||
directory = tmp_path_factory.mktemp("h3_outputs")
|
||||
paths = {
|
||||
name: _encode(directory, name, **overrides)
|
||||
for name, (overrides, _) in CASES.items()
|
||||
}
|
||||
data = Path(paths["valid"]).read_bytes()
|
||||
(directory / "truncated.mp4").write_bytes(data[: len(data) // 2])
|
||||
paths["truncated"] = str(directory / "truncated.mp4")
|
||||
paths["missing"] = str(directory / "missing.mp4")
|
||||
return paths
|
||||
|
||||
|
||||
def _verdict(path: str):
|
||||
try:
|
||||
return video_adapter._probe_minimax_h3_output_fields(path, **EXPECTED)
|
||||
except RuntimeError as exc:
|
||||
return f"RuntimeError: {exc}"
|
||||
|
||||
|
||||
def _use_ffprobe(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
video_adapter, "_read_output_media", video_adapter._ffprobe_output_media
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(CASES))
|
||||
def test_each_fixture_trips_its_check(outputs, name, monkeypatch):
|
||||
_use_ffprobe(monkeypatch)
|
||||
verdict = _verdict(outputs[name])
|
||||
error = CASES[name][1]
|
||||
if error is None:
|
||||
assert verdict == VALID_FIELDS
|
||||
else:
|
||||
assert error in verdict
|
||||
|
||||
|
||||
@needs_pyav
|
||||
@pytest.mark.parametrize("name", [*CASES, *UNREADABLE])
|
||||
def test_in_process_read_matches_ffprobe(outputs, name, monkeypatch):
|
||||
in_process = _verdict(outputs[name])
|
||||
_use_ffprobe(monkeypatch)
|
||||
ffprobe = _verdict(outputs[name])
|
||||
if name in UNREADABLE:
|
||||
# the two readers word their parse errors differently
|
||||
assert str(in_process).startswith("RuntimeError")
|
||||
assert str(ffprobe).startswith("RuntimeError")
|
||||
else:
|
||||
assert in_process == ffprobe
|
||||
|
||||
|
||||
@needs_pyav
|
||||
def test_validation_spawns_no_process(outputs, monkeypatch):
|
||||
def run(command, *args, **kwargs):
|
||||
raise AssertionError(f"spawned {command[0]}")
|
||||
|
||||
adapter = video_adapter.MiniMaxH3VideoModelAdapter()
|
||||
shape = {"frame_count": 48, "width": 320, "height": 192}
|
||||
monkeypatch.setattr(adapter, "_resolved_shape", lambda batch: shape)
|
||||
monkeypatch.setattr(video_adapter.subprocess, "run", run)
|
||||
fields = adapter.validate_final_outputs_sync(
|
||||
[outputs["valid"]] * 2, SimpleNamespace(num_outputs_per_prompt=2)
|
||||
)
|
||||
assert fields == VALID_FIELDS
|
||||
|
||||
|
||||
def test_falls_back_to_ffprobe_without_pyav(outputs, monkeypatch):
|
||||
spawned = []
|
||||
real_run = subprocess.run
|
||||
|
||||
def run(command, *args, **kwargs):
|
||||
spawned.append(command[0])
|
||||
return real_run(command, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(video_adapter, "av", None)
|
||||
monkeypatch.setattr(video_adapter.subprocess, "run", run)
|
||||
assert _verdict(outputs["valid"]) == VALID_FIELDS
|
||||
assert spawned == ["ffprobe"]
|
||||
Reference in New Issue
Block a user