diff --git a/python/sglang/multimodal_gen/configs/sample/sampling_params.py b/python/sglang/multimodal_gen/configs/sample/sampling_params.py index 0536dcdf06..2e0f6caaf8 100644 --- a/python/sglang/multimodal_gen/configs/sample/sampling_params.py +++ b/python/sglang/multimodal_gen/configs/sample/sampling_params.py @@ -76,6 +76,23 @@ X264_PRESETS: tuple[str, ...] = ( "veryslow", "placebo", ) +# RIFE multiplies the frame count by 2**exp and recurses forever for exp < 1. +# The upper bound (16x) is arbitrary; each step doubles RIFE calls and frames. +MIN_FRAME_INTERPOLATION_EXP = 1 +MAX_FRAME_INTERPOLATION_EXP = 4 + + +def validate_frame_interpolation_exp(exp: int) -> None: + if ( + isinstance(exp, bool) + or not isinstance(exp, int) + or not MIN_FRAME_INTERPOLATION_EXP <= exp <= MAX_FRAME_INTERPOLATION_EXP + ): + raise ValueError( + "frame_interpolation_exp must be an int in " + f"[{MIN_FRAME_INTERPOLATION_EXP}, {MAX_FRAME_INTERPOLATION_EXP}], " + f"got {exp!r}" + ) @dataclass(frozen=True) @@ -694,6 +711,8 @@ class SamplingParams: f"num_inference_steps must be a positive int, got {self.num_inference_steps!r}" ) + validate_frame_interpolation_exp(self.frame_interpolation_exp) + if self.progressive_mode not in ("fullres", "dct", "dct_rewind"): raise ValueError( "progressive_mode must be one of 'fullres', 'dct', or " diff --git a/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py b/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py index f364b371a0..b1a4c22380 100644 --- a/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py +++ b/python/sglang/multimodal_gen/runtime/postprocess/rife_interpolator.py @@ -19,6 +19,9 @@ import torch import torch.nn as nn import torch.nn.functional as F +from sglang.multimodal_gen.configs.sample.sampling_params import ( + validate_frame_interpolation_exp, +) from sglang.multimodal_gen.runtime.platforms import current_platform from sglang.multimodal_gen.runtime.utils.logging_utils import init_logger @@ -314,10 +317,11 @@ class Model: """Interpolate a single intermediate frame between img0 and img1.""" n, c, h, w = img0.shape - # Pad to multiples of 32 so that RIFE's downsample/upsample round-trips - # preserve spatial dimensions exactly. - ph = ((h - 1) // 32 + 1) * 32 - pw = ((w - 1) // 32 + 1) * 32 + # At scale < 1 the deepest IFBlock divides spatial dims by 32/scale, + # so the pad unit must grow accordingly; upstream RIFE uses max(32, 32/scale). + _pad_unit = max(32, int(32 / scale)) + ph = ((h - 1) // _pad_unit + 1) * _pad_unit + pw = ((w - 1) // _pad_unit + 1) * _pad_unit pad = (0, pw - w, 0, ph - h) img0 = F.pad(img0, pad) img1 = F.pad(img1, pad) @@ -463,6 +467,7 @@ class FrameInterpolator: Returns: (interpolated_frames, multiplier) where multiplier = 2**exp. """ + validate_frame_interpolation_exp(exp) if len(frames) < 2: logger.warning( "Frame interpolation requires at least 2 frames; returning input unchanged." diff --git a/python/sglang/multimodal_gen/test/unit/realtime/test_output_materialization.py b/python/sglang/multimodal_gen/test/unit/realtime/test_output_materialization.py index 4d42e02394..edeeec13b9 100644 --- a/python/sglang/multimodal_gen/test/unit/realtime/test_output_materialization.py +++ b/python/sglang/multimodal_gen/test/unit/realtime/test_output_materialization.py @@ -1,15 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 import numpy as np +import pytest import torch -from sglang.multimodal_gen.configs.sample.sampling_params import DataType +from sglang.multimodal_gen.configs.sample.sampling_params import ( + MAX_FRAME_INTERPOLATION_EXP, + DataType, +) from sglang.multimodal_gen.runtime.entrypoints.utils import ( materialize_output_sample, save_outputs, ) from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker from sglang.multimodal_gen.runtime.pipelines_core.schedule_batch import OutputBatch +from sglang.multimodal_gen.runtime.postprocess import FrameInterpolator +from sglang.multimodal_gen.runtime.postprocess.rife_interpolator import ( + Model as RIFEModel, +) from sglang.multimodal_gen.runtime.realtime.video import build_raw_rgb_frame_batches @@ -169,3 +177,44 @@ def test_raw_rgb_frame_batches_apply_realtime_upscaling(monkeypatch): } assert len(frame_batches) == 1 assert frame_batches[0][0] == bytes([1, 2, 3] * 4) + + +@pytest.mark.parametrize("exp", [0, MAX_FRAME_INTERPOLATION_EXP + 1, 1.5, 2.0, True]) +def test_materialize_output_sample_rejects_out_of_range_interpolation_exp( + monkeypatch, exp +): + """Out-of-range or non-int exp must be rejected before loading weights.""" + + def fail_load(_self): + raise AssertionError("RIFE weights must not load for an invalid exp") + + monkeypatch.setattr(FrameInterpolator, "_ensure_model_loaded", fail_load) + + with pytest.raises(ValueError, match="frame_interpolation_exp must be an int in"): + materialize_output_sample( + torch.zeros(3, 2, 2, 2), + DataType.VIDEO, + fps=24, + enable_frame_interpolation=True, + frame_interpolation_exp=exp, + ) + + +@pytest.mark.parametrize( + "h, w", + [ + (480, 832), # 480p: old pad=480 not divisible by 64 at scale=0.5 + (720, 1280), # 720p: old pad=736 not divisible by 64 at scale=0.5 + ], +) +def test_rife_inference_scale_half_pads_to_64_boundary(h, w): + """scale=0.5 requires padding to multiples of 64, not 32. + + Regresses the bug where Model.inference always padded to 32, causing a + shape mismatch inside IFBlock at 480p and 720p with scale=0.5. + """ + model = RIFEModel().eval() + img0 = torch.zeros(1, 3, h, w) + img1 = torch.zeros(1, 3, h, w) + out = model.inference(img0, img1, scale=0.5) + assert out.shape == (1, 3, h, w), f"expected (1,3,{h},{w}), got {out.shape}" diff --git a/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py b/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py index e2ab75baca..fc98d2adfb 100644 --- a/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py +++ b/python/sglang/multimodal_gen/test/unit/test_multiple_task_types.py @@ -16,7 +16,10 @@ from PIL import Image from sglang.multimodal_gen.configs.pipeline_configs.base import PipelineConfig from sglang.multimodal_gen.configs.pipeline_configs.cosmos3 import Cosmos3Config from sglang.multimodal_gen.configs.sample.cosmos3 import Cosmos3SamplingParams -from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams +from sglang.multimodal_gen.configs.sample.sampling_params import ( + MAX_FRAME_INTERPOLATION_EXP, + SamplingParams, +) from sglang.multimodal_gen.configs.task_type import DataType from sglang.multimodal_gen.configs.task_type import ModelTaskType as Task from sglang.multimodal_gen.runtime.entrypoints.openai import image_api, utils, video_api @@ -361,6 +364,43 @@ def test_http_rejects_invalid_tasks_before_model_execution(http_client): assert admitted == [] +@pytest.mark.parametrize("multipart", [False, True]) +@pytest.mark.parametrize("exp", [-1, 0, MAX_FRAME_INTERPOLATION_EXP + 1]) +def test_http_rejects_out_of_range_frame_interpolation_exp(http_client, multipart, exp): + """Output frames grow as 2**exp; exp < 1 or a huge exp must not reach RIFE.""" + client, admitted, _ = http_client + payload = { + "prompt": "video", + "size": "64x64", + "enable_frame_interpolation": True, + "frame_interpolation_exp": exp, + } + if multipart: + response = client.post( + "/v1/videos", files={k: (None, str(v)) for k, v in payload.items()} + ) + else: + response = client.post("/v1/videos", json=payload) + assert response.status_code == 400, response.text + assert "frame_interpolation_exp" in response.text + assert admitted == [] + + +def test_http_accepts_max_frame_interpolation_exp(http_client): + client, admitted, _ = http_client + response = client.post( + "/v1/videos", + json={ + "prompt": "video", + "size": "64x64", + "enable_frame_interpolation": True, + "frame_interpolation_exp": MAX_FRAME_INTERPOLATION_EXP, + }, + ) + assert response.status_code == 200, response.text + assert admitted[-1].frame_interpolation_exp == MAX_FRAME_INTERPOLATION_EXP + + @pytest.mark.parametrize("multipart", [False, True]) def test_unconditioned_request_on_image_conditioned_default(http_client, multipart): client, admitted, config = http_client