[diffusion] Bound frame_interpolation_exp to RIFE's supported range (#43014)
Signed-off-by: onepunchmonk <aggarwal.avaya@yahoo.com>
This commit is contained in:
@@ -76,6 +76,23 @@ X264_PRESETS: tuple[str, ...] = (
|
||||
"veryslow",
|
||||
"placebo",
|
||||
)
|
||||
# RIFE multiplies the frame count by 2**exp and recurses forever for exp < 1.
|
||||
# The upper bound (16x) is arbitrary; each step doubles RIFE calls and frames.
|
||||
MIN_FRAME_INTERPOLATION_EXP = 1
|
||||
MAX_FRAME_INTERPOLATION_EXP = 4
|
||||
|
||||
|
||||
def validate_frame_interpolation_exp(exp: int) -> None:
|
||||
if (
|
||||
isinstance(exp, bool)
|
||||
or not isinstance(exp, int)
|
||||
or not MIN_FRAME_INTERPOLATION_EXP <= exp <= MAX_FRAME_INTERPOLATION_EXP
|
||||
):
|
||||
raise ValueError(
|
||||
"frame_interpolation_exp must be an int in "
|
||||
f"[{MIN_FRAME_INTERPOLATION_EXP}, {MAX_FRAME_INTERPOLATION_EXP}], "
|
||||
f"got {exp!r}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -694,6 +711,8 @@ class SamplingParams:
|
||||
f"num_inference_steps must be a positive int, got {self.num_inference_steps!r}"
|
||||
)
|
||||
|
||||
validate_frame_interpolation_exp(self.frame_interpolation_exp)
|
||||
|
||||
if self.progressive_mode not in ("fullres", "dct", "dct_rewind"):
|
||||
raise ValueError(
|
||||
"progressive_mode must be one of 'fullres', 'dct', or "
|
||||
|
||||
@@ -19,6 +19,9 @@ import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
validate_frame_interpolation_exp,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger
|
||||
|
||||
@@ -314,10 +317,11 @@ class Model:
|
||||
"""Interpolate a single intermediate frame between img0 and img1."""
|
||||
n, c, h, w = img0.shape
|
||||
|
||||
# Pad to multiples of 32 so that RIFE's downsample/upsample round-trips
|
||||
# preserve spatial dimensions exactly.
|
||||
ph = ((h - 1) // 32 + 1) * 32
|
||||
pw = ((w - 1) // 32 + 1) * 32
|
||||
# At scale < 1 the deepest IFBlock divides spatial dims by 32/scale,
|
||||
# so the pad unit must grow accordingly; upstream RIFE uses max(32, 32/scale).
|
||||
_pad_unit = max(32, int(32 / scale))
|
||||
ph = ((h - 1) // _pad_unit + 1) * _pad_unit
|
||||
pw = ((w - 1) // _pad_unit + 1) * _pad_unit
|
||||
pad = (0, pw - w, 0, ph - h)
|
||||
img0 = F.pad(img0, pad)
|
||||
img1 = F.pad(img1, pad)
|
||||
@@ -463,6 +467,7 @@ class FrameInterpolator:
|
||||
Returns:
|
||||
(interpolated_frames, multiplier) where multiplier = 2**exp.
|
||||
"""
|
||||
validate_frame_interpolation_exp(exp)
|
||||
if len(frames) < 2:
|
||||
logger.warning(
|
||||
"Frame interpolation requires at least 2 frames; returning input unchanged."
|
||||
|
||||
@@ -1,15 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import DataType
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
MAX_FRAME_INTERPOLATION_EXP,
|
||||
DataType,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.entrypoints.utils import (
|
||||
materialize_output_sample,
|
||||
save_outputs,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch
|
||||
from sglang.multimodal_gen.runtime.postprocess import FrameInterpolator
|
||||
from sglang.multimodal_gen.runtime.postprocess.rife_interpolator import (
|
||||
Model as RIFEModel,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.realtime.video import build_raw_rgb_frame_batches
|
||||
|
||||
|
||||
@@ -169,3 +177,44 @@ def test_raw_rgb_frame_batches_apply_realtime_upscaling(monkeypatch):
|
||||
}
|
||||
assert len(frame_batches) == 1
|
||||
assert frame_batches[0][0] == bytes([1, 2, 3] * 4)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exp", [0, MAX_FRAME_INTERPOLATION_EXP + 1, 1.5, 2.0, True])
|
||||
def test_materialize_output_sample_rejects_out_of_range_interpolation_exp(
|
||||
monkeypatch, exp
|
||||
):
|
||||
"""Out-of-range or non-int exp must be rejected before loading weights."""
|
||||
|
||||
def fail_load(_self):
|
||||
raise AssertionError("RIFE weights must not load for an invalid exp")
|
||||
|
||||
monkeypatch.setattr(FrameInterpolator, "_ensure_model_loaded", fail_load)
|
||||
|
||||
with pytest.raises(ValueError, match="frame_interpolation_exp must be an int in"):
|
||||
materialize_output_sample(
|
||||
torch.zeros(3, 2, 2, 2),
|
||||
DataType.VIDEO,
|
||||
fps=24,
|
||||
enable_frame_interpolation=True,
|
||||
frame_interpolation_exp=exp,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"h, w",
|
||||
[
|
||||
(480, 832), # 480p: old pad=480 not divisible by 64 at scale=0.5
|
||||
(720, 1280), # 720p: old pad=736 not divisible by 64 at scale=0.5
|
||||
],
|
||||
)
|
||||
def test_rife_inference_scale_half_pads_to_64_boundary(h, w):
|
||||
"""scale=0.5 requires padding to multiples of 64, not 32.
|
||||
|
||||
Regresses the bug where Model.inference always padded to 32, causing a
|
||||
shape mismatch inside IFBlock at 480p and 720p with scale=0.5.
|
||||
"""
|
||||
model = RIFEModel().eval()
|
||||
img0 = torch.zeros(1, 3, h, w)
|
||||
img1 = torch.zeros(1, 3, h, w)
|
||||
out = model.inference(img0, img1, scale=0.5)
|
||||
assert out.shape == (1, 3, h, w), f"expected (1,3,{h},{w}), got {out.shape}"
|
||||
|
||||
@@ -16,7 +16,10 @@ from PIL import Image
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig
|
||||
from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config
|
||||
from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import (
|
||||
MAX_FRAME_INTERPOLATION_EXP,
|
||||
SamplingParams,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.task_type import DataType
|
||||
from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task
|
||||
from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, utils, video_api
|
||||
@@ -361,6 +364,43 @@ def test_http_rejects_invalid_tasks_before_model_execution(http_client):
|
||||
assert admitted == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("multipart", [False, True])
|
||||
@pytest.mark.parametrize("exp", [-1, 0, MAX_FRAME_INTERPOLATION_EXP + 1])
|
||||
def test_http_rejects_out_of_range_frame_interpolation_exp(http_client, multipart, exp):
|
||||
"""Output frames grow as 2**exp; exp < 1 or a huge exp must not reach RIFE."""
|
||||
client, admitted, _ = http_client
|
||||
payload = {
|
||||
"prompt": "video",
|
||||
"size": "64x64",
|
||||
"enable_frame_interpolation": True,
|
||||
"frame_interpolation_exp": exp,
|
||||
}
|
||||
if multipart:
|
||||
response = client.post(
|
||||
"/v1/videos", files={k: (None, str(v)) for k, v in payload.items()}
|
||||
)
|
||||
else:
|
||||
response = client.post("/v1/videos", json=payload)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "frame_interpolation_exp" in response.text
|
||||
assert admitted == []
|
||||
|
||||
|
||||
def test_http_accepts_max_frame_interpolation_exp(http_client):
|
||||
client, admitted, _ = http_client
|
||||
response = client.post(
|
||||
"/v1/videos",
|
||||
json={
|
||||
"prompt": "video",
|
||||
"size": "64x64",
|
||||
"enable_frame_interpolation": True,
|
||||
"frame_interpolation_exp": MAX_FRAME_INTERPOLATION_EXP,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert admitted[-1].frame_interpolation_exp == MAX_FRAME_INTERPOLATION_EXP
|
||||
|
||||
|
||||
@pytest.mark.parametrize("multipart", [False, True])
|
||||
def test_unconditioned_request_on_image_conditioned_default(http_client, multipart):
|
||||
client, admitted, config = http_client
|
||||
|
||||
Reference in New Issue
Block a user