[diffusion] feat: add configurable reference image resizing for MiniMax H3 Ref2VA (#35839)

Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
LittleNyima
2026-10-13 19:48:18 +08:00
committed by GitHub
co-authored by Mick Qian
parent 85254b5a8f
commit 5727f08932
9 changed files with 386 additions and 18 deletions
@@ -8,6 +8,9 @@ import msgspec
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
from sglang.multimodal_gen.configs.utils import optional_positive_finite_float
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
MINIMAX_H3_REFERENCE_RESIZE_MODES,
)
_MINIMAX_H3_MAX_SIGNED_SEED = (1 << 63) - 1
@@ -48,6 +51,7 @@ class MiniMaxH3SamplingParams(SamplingParams):
default=None,
metadata={"batch_sig_exclude": True},
)
reference_resize_mode: str = field(default="diffusers")
@classmethod
def video_request_extra_fields(cls) -> frozenset[str]:
@@ -62,6 +66,7 @@ class MiniMaxH3SamplingParams(SamplingParams):
"output_mode",
"imgvid_cond_noise_aug_for_inference",
"audio_cond_noise_aug_for_inference",
"reference_resize_mode",
}
)
@@ -279,6 +284,12 @@ class MiniMaxH3SamplingParams(SamplingParams):
"MiniMax H3 scalar seed plus output index must fit the "
"signed int64 upper bound"
)
self.reference_resize_mode = str(self.reference_resize_mode).strip().lower()
if self.reference_resize_mode not in MINIMAX_H3_REFERENCE_RESIZE_MODES:
raise ValueError(
"reference_resize_mode must be one of {'match', 'max', 'diffusers'}, "
f"got {self.reference_resize_mode!r}"
)
def build_request_extra(self, *, _seed_override: int | None = None) -> dict:
_optional_unit_float(
@@ -62,3 +62,28 @@ MINIMAX_H3_DEFAULT_BRANCHES: tuple = ({"name": "cond_1"},)
# configuration at all. Process-wide SGLANG_CACHE_DIT_* environment controls
# remain available for manual experiments and are independent of this field.
MINIMAX_H3_HIGH_QUALITY_CACHE_DIT_CONFIG: tuple[int, float, int] = (4, 0.04, 1)
# Supported reference-image resize policies for Ref2VA. All modes preserve the
# source aspect ratio, avoid cropping, and independently round the resolved width
# and height to the nearest multiple of 32 after applying the scale factor.
#
# - ``match`` matches the reference-image pixel area to the target canvas without
# upscaling a smaller reference:
# scale = min(1, sqrt(target_area / reference_area))
# This is the preprocessing policy used to train the ModelTC Ref2VA Turbo LoRA
# and is therefore recommended for that checkpoint.
# - ``max`` only downsizes references whose short edge exceeds 2048 pixels:
# scale = min(1, 2048 / reference_short_edge)
# - ``diffusers`` always scales the reference short edge to 2048 pixels, including
# upscaling smaller inputs:
# scale = 2048 / reference_short_edge
# This preserves the original Diffusers and SGLang reference-image behavior and
# should remain the compatibility default unless a checkpoint specifies another
# preprocessing policy.
#
# See the following references for the policy definitions and training guidance:
# - https://docs.comfy.org/tutorials/video/minimax/minimax-h3#prompting-tips-3
# - https://github.com/ModelTC/Minimax-H3-Turbo#note-on-reference-image-resizing
MINIMAX_H3_REFERENCE_RESIZE_MODES: frozenset[str] = frozenset(
{"match", "max", "diffusers"}
)
@@ -309,9 +309,17 @@ def minimax_h3_prepare_for_queue(batch: Any) -> MiniMaxH3ResolvedPlan:
minimax_h3_resolve_reference_image_shape,
)
reference_resize_mode = getattr(
batch.sampling_params,
"reference_resize_mode",
"diffusers",
)
resolved = minimax_h3_resolve_reference_image_shape(
width=width,
height=height,
canvas_width=int(shape["width"]),
canvas_height=int(shape["height"]),
mode=reference_resize_mode,
)
else:
continue
@@ -3,9 +3,12 @@
Encoding recipes for user-provided reference materials:
- image reference: independent 2048px short-edge resize with upscale enabled,
LANCZOS, and nearest-32 dimensions, then the SAME keyframe tokenizer recipe
as fl2va (seed-42 sampled encode, normalize, [1,2,2] patchify);
- image reference: configurable ``match``, ``max``, or ``diffusers`` resizing
while preserving the display ratio, followed by LANCZOS and nearest-32
dimensions, then the SAME keyframe tokenizer recipe as fl2va (seed-42
sampled encode, normalize, [1,2,2] patchify). ``match`` follows the target
canvas pixel area without upscaling, ``max`` caps the short edge at 2048
without upscaling, and ``diffusers`` forces the short edge to 2048;
- audio reference: the audio material chain (pure
audio is losslessly normalized
to stereo; video soundtracks are extracted as 44.1 kHz stereo), then a
@@ -37,6 +40,7 @@ from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import (
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_replica_group
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
MINIMAX_H3_REFERENCE_RESIZE_MODES,
MINIMAX_H3_SUPPORTED_FPS,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.keyframe_encoding import (
@@ -126,21 +130,27 @@ def minimax_h3_resolve_reference_image_shape(
*,
width: int | float,
height: int | float,
canvas_width: int | float,
canvas_height: int | float,
mode: str = "diffusers",
) -> dict[str, Any]:
"""Resolve a ref2va image independently from the target canvas.
"""Resolve a ref2va image with a selectable resize policy.
The image keeps its display ratio, always targets a 2048px short edge (even
when that requires upscaling), and rounds both dimensions independently to
the nearest 32px grid. Unlike target/video ``adapt_shape_v1``, reference
images have no area-cap branch.
All modes preserve the reference display ratio and round both dimensions
independently to the nearest 32px grid. ``match`` follows the target canvas
pixel area without upscaling a smaller reference, ``max`` only downsizes a
reference whose short edge exceeds 2048px, and ``diffusers`` always targets
a 2048px short edge, including by upscaling.
"""
try:
source_width = float(width)
source_height = float(height)
canvas_width = float(canvas_width)
canvas_height = float(canvas_height)
except (TypeError, ValueError) as exc:
raise ValueError(
"reference image width and height must be positive finite numbers"
"reference image and target canvas dimensions must be finite numbers"
) from exc
if (
not math.isfinite(source_width)
@@ -151,30 +161,67 @@ def minimax_h3_resolve_reference_image_shape(
raise ValueError(
"reference image width and height must be positive finite numbers"
)
if (
not math.isfinite(canvas_width)
or not math.isfinite(canvas_height)
or canvas_width <= 0.0
or canvas_height <= 0.0
):
raise ValueError(
"target canvas width and height must be positive finite numbers"
)
if source_width > 4.0 * source_height or source_height > 4.0 * source_width:
raise ValueError(
"reference image ratio must be within the inclusive range "
f"1:4 to 4:1, got {source_width:g}x{source_height:g}"
)
mode = str(mode).strip().lower()
if mode not in MINIMAX_H3_REFERENCE_RESIZE_MODES:
raise ValueError(
"reference_resize_mode must be one of "
f"{sorted(MINIMAX_H3_REFERENCE_RESIZE_MODES)}, got {mode!r}"
)
scale = MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / min(source_width, source_height)
target_width = _nearest_multiple(
short_edge = min(source_width, source_height)
if mode == "match":
scale = min(
1.0,
math.sqrt((canvas_width * canvas_height) / (source_width * source_height)),
)
elif mode == "max":
scale = min(1.0, MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / short_edge)
else: # diffusers
scale = MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / short_edge
resolved_width = _nearest_multiple(
source_width * scale, MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE
)
target_height = _nearest_multiple(
resolved_height = _nearest_multiple(
source_height * scale, MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE
)
size_mode = {
"match": "target_area",
"max": "short_edge_cap",
"diffusers": "short_edge",
}[mode]
return {
"geometry": "reference_image_resolved",
"shape_policy_version": "reference_image_short_edge_v1",
"shape_policy_version": (
"reference_image_short_edge_v1"
if mode == "diffusers"
else f"reference_image_{mode}_v1"
),
# Keep the legacy short-edge metadata for shape-schema compatibility.
# ``match`` does not use this value in its scale calculation.
"base_short_edge": MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE,
"effective_short_edge": min(target_width, target_height),
"size_mode": "short_edge",
"effective_short_edge": min(resolved_width, resolved_height),
"reference_resize_mode": mode,
"size_mode": size_mode,
"multiple": MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE,
"rounding": "nearest",
"allow_upscale": True,
"width": target_width,
"height": target_height,
"allow_upscale": mode == "diffusers",
"width": resolved_width,
"height": resolved_height,
}
@@ -190,6 +190,11 @@ class MiniMaxH3VideoModelAdapter:
),
}
)
reference_resize_mode = _parse_extra_value(
_extra_value(request, "reference_resize_mode")
)
if reference_resize_mode is not None:
kwargs["reference_resize_mode"] = reference_resize_mode
if quality is not None:
kwargs["quality"] = quality
return kwargs
@@ -34,6 +34,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import
ComposedPipelineBase,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3 import (
prequeue,
reference_encoding,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import (
MiniMaxH3PartitionAdmissionStage,
MiniMaxH3ReleaseMetadata,
@@ -337,6 +341,133 @@ def test_video_adapter_lowers_only_native_fields_and_rejects_cfg():
)
def test_reference_resize_mode_is_declared_as_video_request_extra():
assert (
"reference_resize_mode" in MiniMaxH3SamplingParams.video_request_extra_fields()
)
def test_video_adapter_lowers_reference_resize_mode():
request = VideoGenerationsRequest(
prompt="contract",
task="ref2va",
conditions=[
{
"type": "image",
"uri": "file:///reference.png",
"role": "reference",
}
],
target=TARGET,
reference_resize_mode="match",
)
lowered = MiniMaxH3SamplingParams.lower_video_request_kwargs(
request,
{"prompt": request.prompt, "seed": request.seed},
)
assert lowered["reference_resize_mode"] == "match"
def test_video_adapter_does_not_override_default_reference_resize_mode():
request = VideoGenerationsRequest(
prompt="contract",
task="ref2va",
conditions=[
{
"type": "image",
"uri": "file:///reference.png",
"role": "reference",
}
],
target=TARGET,
)
lowered = MiniMaxH3SamplingParams.lower_video_request_kwargs(
request,
{"prompt": request.prompt, "seed": request.seed},
)
assert "reference_resize_mode" not in lowered
def test_sampling_params_normalizes_reference_resize_mode_and_defaults_to_diffusers():
assert MiniMaxH3SamplingParams().reference_resize_mode == "diffusers"
assert (
MiniMaxH3SamplingParams(reference_resize_mode=" MATCH ").reference_resize_mode
== "match"
)
@pytest.mark.parametrize("bad_mode", ["unknown", "", None, 1])
def test_sampling_params_rejects_invalid_reference_resize_mode(bad_mode):
with pytest.raises(ValueError, match="reference_resize_mode"):
MiniMaxH3SamplingParams(reference_resize_mode=bad_mode)
@pytest.mark.parametrize(
("sampling_params", "expected_mode"),
[
(SimpleNamespace(reference_resize_mode="match"), "match"),
(SimpleNamespace(), "diffusers"),
],
)
def test_prequeue_forwards_reference_resize_mode_and_target_canvas(
sampling_params, expected_mode
):
material = SimpleNamespace(
condition_index=0,
condition_type="image",
material_chain="image.reference_preserve",
start_time_seconds=0.0,
uri="file:///reference.png",
)
plan = SimpleNamespace(
materials=(material,),
shape={
"geometry": "resolved_v2",
"width": 960,
"height": 544,
"fps": 24,
"frame_count": 124,
},
task="ref2va",
)
batch = SimpleNamespace(extra={}, sampling_params=sampling_params)
resolved_reference_shape = {
"geometry": "reference_image_resolved",
"width": 960,
"height": 544,
}
with (
patch.object(prequeue, "minimax_h3_plan_from_batch", return_value=plan),
patch.object(
prequeue,
"minimax_h3_probe_material",
return_value={"display_width": 1920, "display_height": 1080},
),
patch.object(prequeue, "_replace_plan_shape", return_value=plan),
patch.object(
reference_encoding,
"minimax_h3_resolve_reference_image_shape",
return_value=resolved_reference_shape,
) as resolve_reference_shape,
):
assert prequeue.minimax_h3_prepare_for_queue(batch) is plan
resolve_reference_shape.assert_called_once_with(
width=1920.0,
height=1080.0,
canvas_width=960,
canvas_height=544,
mode=expected_mode,
)
assert batch.extra[prequeue.MINIMAX_H3_RESOLVED_MATERIAL_SHAPES_EXTRA_KEY][0] == {
**resolved_reference_shape,
"condition_index": 0,
}
@pytest.mark.parametrize("bad_quality", ["ultra", "draft", "", 1])
def test_video_adapter_rejects_invalid_quality(bad_quality):
request = VideoGenerationsRequest(
@@ -22,6 +22,115 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m
)
@pytest.mark.parametrize(
("mode", "source_size", "canvas_size", "expected_size", "policy_version"),
[
(
"match",
(1920, 1080),
(960, 544),
(960, 544),
"reference_image_match_v1",
),
(
"match",
(512, 512),
(960, 544),
(512, 512),
"reference_image_match_v1",
),
(
"max",
(5000, 2500),
(960, 544),
(4096, 2048),
"reference_image_max_v1",
),
(
"max",
(1920, 1080),
(960, 544),
(1920, 1088),
"reference_image_max_v1",
),
(
"diffusers",
(512, 512),
(960, 544),
(2048, 2048),
"reference_image_short_edge_v1",
),
],
)
def test_reference_image_resize_modes(
mode, source_size, canvas_size, expected_size, policy_version
):
resolved = reference_encoding.minimax_h3_resolve_reference_image_shape(
width=source_size[0],
height=source_size[1],
canvas_width=canvas_size[0],
canvas_height=canvas_size[1],
mode=mode,
)
assert (resolved["width"], resolved["height"]) == expected_size
assert resolved["shape_policy_version"] == policy_version
assert resolved["reference_resize_mode"] == mode
assert resolved["allow_upscale"] is (mode == "diffusers")
def test_reference_image_resize_defaults_to_legacy_diffusers_policy():
resolved = reference_encoding.minimax_h3_resolve_reference_image_shape(
width=512,
height=512,
canvas_width=960,
canvas_height=544,
)
assert (resolved["width"], resolved["height"]) == (2048, 2048)
assert resolved["reference_resize_mode"] == "diffusers"
assert resolved["shape_policy_version"] == "reference_image_short_edge_v1"
def test_reference_image_resize_normalizes_mode():
resolved = reference_encoding.minimax_h3_resolve_reference_image_shape(
width=1920,
height=1080,
canvas_width=960,
canvas_height=544,
mode=" MATCH ",
)
assert resolved["reference_resize_mode"] == "match"
assert (resolved["width"], resolved["height"]) == (960, 544)
def test_reference_image_resize_rejects_unknown_mode():
with pytest.raises(ValueError, match="reference_resize_mode"):
reference_encoding.minimax_h3_resolve_reference_image_shape(
width=1920,
height=1080,
canvas_width=960,
canvas_height=544,
mode="unknown",
)
@pytest.mark.parametrize(
("canvas_width", "canvas_height"),
[(0, 544), (960, 0), (-960, 544), (float("nan"), 544)],
)
def test_reference_image_resize_rejects_invalid_canvas(canvas_width, canvas_height):
with pytest.raises(ValueError, match="target canvas"):
reference_encoding.minimax_h3_resolve_reference_image_shape(
width=1920,
height=1080,
canvas_width=canvas_width,
canvas_height=canvas_height,
mode="match",
)
def test_keyframe_rng_supports_cpu_and_default_device():
initial_state = torch.random.get_rng_state()