[diffusion] Bound frame_interpolation_exp to RIFE's supported range (#43014)

Signed-off-by: onepunchmonk <aggarwal.avaya@yahoo.com>
This commit is contained in:
Avaya Aggarwal
2026-10-14 10:29:28 +08:00
committed by GitHub
parent 44b44774c3
commit ac187c0cd5
4 changed files with 119 additions and 6 deletions
@@ -76,6 +76,23 @@ X264_PRESETS: tuple[str, ...] = (
"veryslow",
"placebo",
)
# RIFE multiplies the frame count by 2**exp and recurses forever for exp < 1.
# The upper bound (16x) is arbitrary; each step doubles RIFE calls and frames.
MIN_FRAME_INTERPOLATION_EXP = 1
MAX_FRAME_INTERPOLATION_EXP = 4
def validate_frame_interpolation_exp(exp: int) -> None:
if (
isinstance(exp, bool)
or not isinstance(exp, int)
or not MIN_FRAME_INTERPOLATION_EXP <= exp <= MAX_FRAME_INTERPOLATION_EXP
):
raise ValueError(
"frame_interpolation_exp must be an int in "
f"[{MIN_FRAME_INTERPOLATION_EXP}, {MAX_FRAME_INTERPOLATION_EXP}], "
f"got {exp!r}"
)
@dataclass(frozen=True)
@@ -694,6 +711,8 @@ class SamplingParams:
f"num_inference_steps must be a positive int, got {self.num_inference_steps!r}"
)
validate_frame_interpolation_exp(self.frame_interpolation_exp)
if self.progressive_mode not in ("fullres", "dct", "dct_rewind"):
raise ValueError(
"progressive_mode must be one of 'fullres', 'dct', or "
@@ -19,6 +19,9 @@ import torch
import torch.nn as nn
import torch.nn.functional as F
from sglang.multimodal_gen.configs.sample.sampling_params import (
validate_frame_interpolation_exp,
)
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
@@ -314,10 +317,11 @@ class Model:
"""Interpolate a single intermediate frame between img0 and img1."""
n, c, h, w = img0.shape
# Pad to multiples of 32 so that RIFE's downsample/upsample round-trips
# preserve spatial dimensions exactly.
ph = ((h - 1) // 32 + 1) * 32
pw = ((w - 1) // 32 + 1) * 32
# At scale < 1 the deepest IFBlock divides spatial dims by 32/scale,
# so the pad unit must grow accordingly; upstream RIFE uses max(32, 32/scale).
_pad_unit = max(32, int(32 / scale))
ph = ((h - 1) // _pad_unit + 1) * _pad_unit
pw = ((w - 1) // _pad_unit + 1) * _pad_unit
pad = (0, pw - w, 0, ph - h)
img0 = F.pad(img0, pad)
img1 = F.pad(img1, pad)
@@ -463,6 +467,7 @@ class FrameInterpolator:
Returns:
(interpolated_frames, multiplier) where multiplier = 2**exp.
"""
validate_frame_interpolation_exp(exp)
if len(frames) < 2:
logger.warning(
"Frame interpolation requires at least 2 frames; returning input unchanged."
@@ -1,15 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
import numpy as np
import pytest
import torch
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
from sglang.multimodal_gen.configs.sample.sampling_params import (
MAX_FRAME_INTERPOLATION_EXP,
DataType,
)
from sglang.multimodal_gen.runtime.entrypoints.utils import (
materialize_output_sample,
save_outputs,
)
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
from sglang.multimodal_gen.runtime.postprocess import FrameInterpolator
from sglang.multimodal_gen.runtime.postprocess.rife_interpolator import (
Model as RIFEModel,
)
from sglang.multimodal_gen.runtime.realtime.video import build_raw_rgb_frame_batches
@@ -169,3 +177,44 @@ def test_raw_rgb_frame_batches_apply_realtime_upscaling(monkeypatch):
}
assert len(frame_batches) == 1
assert frame_batches[0][0] == bytes([1, 2, 3] * 4)
@pytest.mark.parametrize("exp", [0, MAX_FRAME_INTERPOLATION_EXP + 1, 1.5, 2.0, True])
def test_materialize_output_sample_rejects_out_of_range_interpolation_exp(
monkeypatch, exp
):
"""Out-of-range or non-int exp must be rejected before loading weights."""
def fail_load(_self):
raise AssertionError("RIFE weights must not load for an invalid exp")
monkeypatch.setattr(FrameInterpolator, "_ensure_model_loaded", fail_load)
with pytest.raises(ValueError, match="frame_interpolation_exp must be an int in"):
materialize_output_sample(
torch.zeros(3, 2, 2, 2),
DataType.VIDEO,
fps=24,
enable_frame_interpolation=True,
frame_interpolation_exp=exp,
)
@pytest.mark.parametrize(
"h, w",
[
(480, 832), # 480p: old pad=480 not divisible by 64 at scale=0.5
(720, 1280), # 720p: old pad=736 not divisible by 64 at scale=0.5
],
)
def test_rife_inference_scale_half_pads_to_64_boundary(h, w):
"""scale=0.5 requires padding to multiples of 64, not 32.
Regresses the bug where Model.inference always padded to 32, causing a
shape mismatch inside IFBlock at 480p and 720p with scale=0.5.
"""
model = RIFEModel().eval()
img0 = torch.zeros(1, 3, h, w)
img1 = torch.zeros(1, 3, h, w)
out = model.inference(img0, img1, scale=0.5)
assert out.shape == (1, 3, h, w), f"expected (1,3,{h},{w}), got {out.shape}"
@@ -16,7 +16,10 @@ from PIL import Image
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config
from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.sample.sampling_params import (
MAX_FRAME_INTERPOLATION_EXP,
SamplingParams,
)
from sglang.multimodal_gen.configs.task_type import DataType
from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, utils, video_api
@@ -361,6 +364,43 @@ def test_http_rejects_invalid_tasks_before_model_execution(http_client):
assert admitted == []
@pytest.mark.parametrize("multipart", [False, True])
@pytest.mark.parametrize("exp", [-1, 0, MAX_FRAME_INTERPOLATION_EXP + 1])
def test_http_rejects_out_of_range_frame_interpolation_exp(http_client, multipart, exp):
"""Output frames grow as 2**exp; exp < 1 or a huge exp must not reach RIFE."""
client, admitted, _ = http_client
payload = {
"prompt": "video",
"size": "64x64",
"enable_frame_interpolation": True,
"frame_interpolation_exp": exp,
}
if multipart:
response = client.post(
"/v1/videos", files={k: (None, str(v)) for k, v in payload.items()}
)
else:
response = client.post("/v1/videos", json=payload)
assert response.status_code == 400, response.text
assert "frame_interpolation_exp" in response.text
assert admitted == []
def test_http_accepts_max_frame_interpolation_exp(http_client):
client, admitted, _ = http_client
response = client.post(
"/v1/videos",
json={
"prompt": "video",
"size": "64x64",
"enable_frame_interpolation": True,
"frame_interpolation_exp": MAX_FRAME_INTERPOLATION_EXP,
},
)
assert response.status_code == 200, response.text
assert admitted[-1].frame_interpolation_exp == MAX_FRAME_INTERPOLATION_EXP
@pytest.mark.parametrize("multipart", [False, True])
def test_unconditioned_request_on_image_conditioned_default(http_client, multipart):
client, admitted, config = http_client