[diffusion] feat: add configurable reference image resizing for MiniMax H3 Ref2VA (#35839)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -8,6 +8,9 @@ import msgspec
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
from sglang.multimodal_gen.configs.utils import optional_positive_finite_float
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
|
||||
MINIMAX_H3_REFERENCE_RESIZE_MODES,
|
||||
)
|
||||
|
||||
_MINIMAX_H3_MAX_SIGNED_SEED = (1 << 63) - 1
|
||||
|
||||
@@ -48,6 +51,7 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
default=None,
|
||||
metadata={"batch_sig_exclude": True},
|
||||
)
|
||||
reference_resize_mode: str = field(default="diffusers")
|
||||
|
||||
@classmethod
|
||||
def video_request_extra_fields(cls) -> frozenset[str]:
|
||||
@@ -62,6 +66,7 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
"output_mode",
|
||||
"imgvid_cond_noise_aug_for_inference",
|
||||
"audio_cond_noise_aug_for_inference",
|
||||
"reference_resize_mode",
|
||||
}
|
||||
)
|
||||
|
||||
@@ -279,6 +284,12 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
"MiniMax H3 scalar seed plus output index must fit the "
|
||||
"signed int64 upper bound"
|
||||
)
|
||||
self.reference_resize_mode = str(self.reference_resize_mode).strip().lower()
|
||||
if self.reference_resize_mode not in MINIMAX_H3_REFERENCE_RESIZE_MODES:
|
||||
raise ValueError(
|
||||
"reference_resize_mode must be one of {'match', 'max', 'diffusers'}, "
|
||||
f"got {self.reference_resize_mode!r}"
|
||||
)
|
||||
|
||||
def build_request_extra(self, *, _seed_override: int | None = None) -> dict:
|
||||
_optional_unit_float(
|
||||
|
||||
+25
@@ -62,3 +62,28 @@ MINIMAX_H3_DEFAULT_BRANCHES: tuple = ({"name": "cond_1"},)
|
||||
# configuration at all. Process-wide SGLANG_CACHE_DIT_* environment controls
|
||||
# remain available for manual experiments and are independent of this field.
|
||||
MINIMAX_H3_HIGH_QUALITY_CACHE_DIT_CONFIG: tuple[int, float, int] = (4, 0.04, 1)
|
||||
|
||||
# Supported reference-image resize policies for Ref2VA. All modes preserve the
|
||||
# source aspect ratio, avoid cropping, and independently round the resolved width
|
||||
# and height to the nearest multiple of 32 after applying the scale factor.
|
||||
#
|
||||
# - ``match`` matches the reference-image pixel area to the target canvas without
|
||||
# upscaling a smaller reference:
|
||||
# scale = min(1, sqrt(target_area / reference_area))
|
||||
# This is the preprocessing policy used to train the ModelTC Ref2VA Turbo LoRA
|
||||
# and is therefore recommended for that checkpoint.
|
||||
# - ``max`` only downsizes references whose short edge exceeds 2048 pixels:
|
||||
# scale = min(1, 2048 / reference_short_edge)
|
||||
# - ``diffusers`` always scales the reference short edge to 2048 pixels, including
|
||||
# upscaling smaller inputs:
|
||||
# scale = 2048 / reference_short_edge
|
||||
# This preserves the original Diffusers and SGLang reference-image behavior and
|
||||
# should remain the compatibility default unless a checkpoint specifies another
|
||||
# preprocessing policy.
|
||||
#
|
||||
# See the following references for the policy definitions and training guidance:
|
||||
# - https://docs.comfy.org/tutorials/video/minimax/minimax-h3#prompting-tips-3
|
||||
# - https://github.com/ModelTC/Minimax-H3-Turbo#note-on-reference-image-resizing
|
||||
MINIMAX_H3_REFERENCE_RESIZE_MODES: frozenset[str] = frozenset(
|
||||
{"match", "max", "diffusers"}
|
||||
)
|
||||
|
||||
+8
@@ -309,9 +309,17 @@ def minimax_h3_prepare_for_queue(batch: Any) -> MiniMaxH3ResolvedPlan:
|
||||
minimax_h3_resolve_reference_image_shape,
|
||||
)
|
||||
|
||||
reference_resize_mode = getattr(
|
||||
batch.sampling_params,
|
||||
"reference_resize_mode",
|
||||
"diffusers",
|
||||
)
|
||||
resolved = minimax_h3_resolve_reference_image_shape(
|
||||
width=width,
|
||||
height=height,
|
||||
canvas_width=int(shape["width"]),
|
||||
canvas_height=int(shape["height"]),
|
||||
mode=reference_resize_mode,
|
||||
)
|
||||
else:
|
||||
continue
|
||||
|
||||
+65
-18
@@ -3,9 +3,12 @@
|
||||
|
||||
Encoding recipes for user-provided reference materials:
|
||||
|
||||
- image reference: independent 2048px short-edge resize with upscale enabled,
|
||||
LANCZOS, and nearest-32 dimensions, then the SAME keyframe tokenizer recipe
|
||||
as fl2va (seed-42 sampled encode, normalize, [1,2,2] patchify);
|
||||
- image reference: configurable ``match``, ``max``, or ``diffusers`` resizing
|
||||
while preserving the display ratio, followed by LANCZOS and nearest-32
|
||||
dimensions, then the SAME keyframe tokenizer recipe as fl2va (seed-42
|
||||
sampled encode, normalize, [1,2,2] patchify). ``match`` follows the target
|
||||
canvas pixel area without upscaling, ``max`` caps the short edge at 2048
|
||||
without upscaling, and ``diffusers`` forces the short edge to 2048;
|
||||
- audio reference: the audio material chain (pure
|
||||
audio is losslessly normalized
|
||||
to stereo; video soundtracks are extracted as 44.1 kHz stereo), then a
|
||||
@@ -37,6 +40,7 @@ from sglang.multimodal_gen.configs.models.vaes.minimax_h3_video import (
|
||||
from sglang.multimodal_gen.runtime.distributed.parallel_state import get_replica_group
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import set_forward_context
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.constants import (
|
||||
MINIMAX_H3_REFERENCE_RESIZE_MODES,
|
||||
MINIMAX_H3_SUPPORTED_FPS,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.keyframe_encoding import (
|
||||
@@ -126,21 +130,27 @@ def minimax_h3_resolve_reference_image_shape(
|
||||
*,
|
||||
width: int | float,
|
||||
height: int | float,
|
||||
canvas_width: int | float,
|
||||
canvas_height: int | float,
|
||||
mode: str = "diffusers",
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve a ref2va image independently from the target canvas.
|
||||
"""Resolve a ref2va image with a selectable resize policy.
|
||||
|
||||
The image keeps its display ratio, always targets a 2048px short edge (even
|
||||
when that requires upscaling), and rounds both dimensions independently to
|
||||
the nearest 32px grid. Unlike target/video ``adapt_shape_v1``, reference
|
||||
images have no area-cap branch.
|
||||
All modes preserve the reference display ratio and round both dimensions
|
||||
independently to the nearest 32px grid. ``match`` follows the target canvas
|
||||
pixel area without upscaling a smaller reference, ``max`` only downsizes a
|
||||
reference whose short edge exceeds 2048px, and ``diffusers`` always targets
|
||||
a 2048px short edge, including by upscaling.
|
||||
"""
|
||||
|
||||
try:
|
||||
source_width = float(width)
|
||||
source_height = float(height)
|
||||
canvas_width = float(canvas_width)
|
||||
canvas_height = float(canvas_height)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(
|
||||
"reference image width and height must be positive finite numbers"
|
||||
"reference image and target canvas dimensions must be finite numbers"
|
||||
) from exc
|
||||
if (
|
||||
not math.isfinite(source_width)
|
||||
@@ -151,30 +161,67 @@ def minimax_h3_resolve_reference_image_shape(
|
||||
raise ValueError(
|
||||
"reference image width and height must be positive finite numbers"
|
||||
)
|
||||
if (
|
||||
not math.isfinite(canvas_width)
|
||||
or not math.isfinite(canvas_height)
|
||||
or canvas_width <= 0.0
|
||||
or canvas_height <= 0.0
|
||||
):
|
||||
raise ValueError(
|
||||
"target canvas width and height must be positive finite numbers"
|
||||
)
|
||||
if source_width > 4.0 * source_height or source_height > 4.0 * source_width:
|
||||
raise ValueError(
|
||||
"reference image ratio must be within the inclusive range "
|
||||
f"1:4 to 4:1, got {source_width:g}x{source_height:g}"
|
||||
)
|
||||
mode = str(mode).strip().lower()
|
||||
if mode not in MINIMAX_H3_REFERENCE_RESIZE_MODES:
|
||||
raise ValueError(
|
||||
"reference_resize_mode must be one of "
|
||||
f"{sorted(MINIMAX_H3_REFERENCE_RESIZE_MODES)}, got {mode!r}"
|
||||
)
|
||||
|
||||
scale = MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / min(source_width, source_height)
|
||||
target_width = _nearest_multiple(
|
||||
short_edge = min(source_width, source_height)
|
||||
if mode == "match":
|
||||
scale = min(
|
||||
1.0,
|
||||
math.sqrt((canvas_width * canvas_height) / (source_width * source_height)),
|
||||
)
|
||||
elif mode == "max":
|
||||
scale = min(1.0, MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / short_edge)
|
||||
else: # diffusers
|
||||
scale = MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE / short_edge
|
||||
|
||||
resolved_width = _nearest_multiple(
|
||||
source_width * scale, MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE
|
||||
)
|
||||
target_height = _nearest_multiple(
|
||||
resolved_height = _nearest_multiple(
|
||||
source_height * scale, MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE
|
||||
)
|
||||
size_mode = {
|
||||
"match": "target_area",
|
||||
"max": "short_edge_cap",
|
||||
"diffusers": "short_edge",
|
||||
}[mode]
|
||||
return {
|
||||
"geometry": "reference_image_resolved",
|
||||
"shape_policy_version": "reference_image_short_edge_v1",
|
||||
"shape_policy_version": (
|
||||
"reference_image_short_edge_v1"
|
||||
if mode == "diffusers"
|
||||
else f"reference_image_{mode}_v1"
|
||||
),
|
||||
# Keep the legacy short-edge metadata for shape-schema compatibility.
|
||||
# ``match`` does not use this value in its scale calculation.
|
||||
"base_short_edge": MINIMAX_H3_REFERENCE_IMAGE_SHORT_EDGE,
|
||||
"effective_short_edge": min(target_width, target_height),
|
||||
"size_mode": "short_edge",
|
||||
"effective_short_edge": min(resolved_width, resolved_height),
|
||||
"reference_resize_mode": mode,
|
||||
"size_mode": size_mode,
|
||||
"multiple": MINIMAX_H3_REFERENCE_IMAGE_MULTIPLE,
|
||||
"rounding": "nearest",
|
||||
"allow_upscale": True,
|
||||
"width": target_width,
|
||||
"height": target_height,
|
||||
"allow_upscale": mode == "diffusers",
|
||||
"width": resolved_width,
|
||||
"height": resolved_height,
|
||||
}
|
||||
|
||||
|
||||
|
||||
+5
@@ -190,6 +190,11 @@ class MiniMaxH3VideoModelAdapter:
|
||||
),
|
||||
}
|
||||
)
|
||||
reference_resize_mode = _parse_extra_value(
|
||||
_extra_value(request, "reference_resize_mode")
|
||||
)
|
||||
if reference_resize_mode is not None:
|
||||
kwargs["reference_resize_mode"] = reference_resize_mode
|
||||
if quality is not None:
|
||||
kwargs["quality"] = quality
|
||||
return kwargs
|
||||
|
||||
@@ -34,6 +34,10 @@ from sglang.multimodal_gen.runtime.pipelines_core.composed_pipeline_base import
|
||||
ComposedPipelineBase,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.denoising import DenoisingStage
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3 import (
|
||||
prequeue,
|
||||
reference_encoding,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import (
|
||||
MiniMaxH3PartitionAdmissionStage,
|
||||
MiniMaxH3ReleaseMetadata,
|
||||
@@ -337,6 +341,133 @@ def test_video_adapter_lowers_only_native_fields_and_rejects_cfg():
|
||||
)
|
||||
|
||||
|
||||
def test_reference_resize_mode_is_declared_as_video_request_extra():
|
||||
assert (
|
||||
"reference_resize_mode" in MiniMaxH3SamplingParams.video_request_extra_fields()
|
||||
)
|
||||
|
||||
|
||||
def test_video_adapter_lowers_reference_resize_mode():
|
||||
request = VideoGenerationsRequest(
|
||||
prompt="contract",
|
||||
task="ref2va",
|
||||
conditions=[
|
||||
{
|
||||
"type": "image",
|
||||
"uri": "file:///reference.png",
|
||||
"role": "reference",
|
||||
}
|
||||
],
|
||||
target=TARGET,
|
||||
reference_resize_mode="match",
|
||||
)
|
||||
lowered = MiniMaxH3SamplingParams.lower_video_request_kwargs(
|
||||
request,
|
||||
{"prompt": request.prompt, "seed": request.seed},
|
||||
)
|
||||
|
||||
assert lowered["reference_resize_mode"] == "match"
|
||||
|
||||
|
||||
def test_video_adapter_does_not_override_default_reference_resize_mode():
|
||||
request = VideoGenerationsRequest(
|
||||
prompt="contract",
|
||||
task="ref2va",
|
||||
conditions=[
|
||||
{
|
||||
"type": "image",
|
||||
"uri": "file:///reference.png",
|
||||
"role": "reference",
|
||||
}
|
||||
],
|
||||
target=TARGET,
|
||||
)
|
||||
lowered = MiniMaxH3SamplingParams.lower_video_request_kwargs(
|
||||
request,
|
||||
{"prompt": request.prompt, "seed": request.seed},
|
||||
)
|
||||
|
||||
assert "reference_resize_mode" not in lowered
|
||||
|
||||
|
||||
def test_sampling_params_normalizes_reference_resize_mode_and_defaults_to_diffusers():
|
||||
assert MiniMaxH3SamplingParams().reference_resize_mode == "diffusers"
|
||||
assert (
|
||||
MiniMaxH3SamplingParams(reference_resize_mode=" MATCH ").reference_resize_mode
|
||||
== "match"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_mode", ["unknown", "", None, 1])
|
||||
def test_sampling_params_rejects_invalid_reference_resize_mode(bad_mode):
|
||||
with pytest.raises(ValueError, match="reference_resize_mode"):
|
||||
MiniMaxH3SamplingParams(reference_resize_mode=bad_mode)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("sampling_params", "expected_mode"),
|
||||
[
|
||||
(SimpleNamespace(reference_resize_mode="match"), "match"),
|
||||
(SimpleNamespace(), "diffusers"),
|
||||
],
|
||||
)
|
||||
def test_prequeue_forwards_reference_resize_mode_and_target_canvas(
|
||||
sampling_params, expected_mode
|
||||
):
|
||||
material = SimpleNamespace(
|
||||
condition_index=0,
|
||||
condition_type="image",
|
||||
material_chain="image.reference_preserve",
|
||||
start_time_seconds=0.0,
|
||||
uri="file:///reference.png",
|
||||
)
|
||||
plan = SimpleNamespace(
|
||||
materials=(material,),
|
||||
shape={
|
||||
"geometry": "resolved_v2",
|
||||
"width": 960,
|
||||
"height": 544,
|
||||
"fps": 24,
|
||||
"frame_count": 124,
|
||||
},
|
||||
task="ref2va",
|
||||
)
|
||||
batch = SimpleNamespace(extra={}, sampling_params=sampling_params)
|
||||
resolved_reference_shape = {
|
||||
"geometry": "reference_image_resolved",
|
||||
"width": 960,
|
||||
"height": 544,
|
||||
}
|
||||
|
||||
with (
|
||||
patch.object(prequeue, "minimax_h3_plan_from_batch", return_value=plan),
|
||||
patch.object(
|
||||
prequeue,
|
||||
"minimax_h3_probe_material",
|
||||
return_value={"display_width": 1920, "display_height": 1080},
|
||||
),
|
||||
patch.object(prequeue, "_replace_plan_shape", return_value=plan),
|
||||
patch.object(
|
||||
reference_encoding,
|
||||
"minimax_h3_resolve_reference_image_shape",
|
||||
return_value=resolved_reference_shape,
|
||||
) as resolve_reference_shape,
|
||||
):
|
||||
assert prequeue.minimax_h3_prepare_for_queue(batch) is plan
|
||||
|
||||
resolve_reference_shape.assert_called_once_with(
|
||||
width=1920.0,
|
||||
height=1080.0,
|
||||
canvas_width=960,
|
||||
canvas_height=544,
|
||||
mode=expected_mode,
|
||||
)
|
||||
assert batch.extra[prequeue.MINIMAX_H3_RESOLVED_MATERIAL_SHAPES_EXTRA_KEY][0] == {
|
||||
**resolved_reference_shape,
|
||||
"condition_index": 0,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("bad_quality", ["ultra", "draft", "", 1])
|
||||
def test_video_adapter_rejects_invalid_quality(bad_quality):
|
||||
request = VideoGenerationsRequest(
|
||||
|
||||
@@ -22,6 +22,115 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mode", "source_size", "canvas_size", "expected_size", "policy_version"),
|
||||
[
|
||||
(
|
||||
"match",
|
||||
(1920, 1080),
|
||||
(960, 544),
|
||||
(960, 544),
|
||||
"reference_image_match_v1",
|
||||
),
|
||||
(
|
||||
"match",
|
||||
(512, 512),
|
||||
(960, 544),
|
||||
(512, 512),
|
||||
"reference_image_match_v1",
|
||||
),
|
||||
(
|
||||
"max",
|
||||
(5000, 2500),
|
||||
(960, 544),
|
||||
(4096, 2048),
|
||||
"reference_image_max_v1",
|
||||
),
|
||||
(
|
||||
"max",
|
||||
(1920, 1080),
|
||||
(960, 544),
|
||||
(1920, 1088),
|
||||
"reference_image_max_v1",
|
||||
),
|
||||
(
|
||||
"diffusers",
|
||||
(512, 512),
|
||||
(960, 544),
|
||||
(2048, 2048),
|
||||
"reference_image_short_edge_v1",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_reference_image_resize_modes(
|
||||
mode, source_size, canvas_size, expected_size, policy_version
|
||||
):
|
||||
resolved = reference_encoding.minimax_h3_resolve_reference_image_shape(
|
||||
width=source_size[0],
|
||||
height=source_size[1],
|
||||
canvas_width=canvas_size[0],
|
||||
canvas_height=canvas_size[1],
|
||||
mode=mode,
|
||||
)
|
||||
|
||||
assert (resolved["width"], resolved["height"]) == expected_size
|
||||
assert resolved["shape_policy_version"] == policy_version
|
||||
assert resolved["reference_resize_mode"] == mode
|
||||
assert resolved["allow_upscale"] is (mode == "diffusers")
|
||||
|
||||
|
||||
def test_reference_image_resize_defaults_to_legacy_diffusers_policy():
|
||||
resolved = reference_encoding.minimax_h3_resolve_reference_image_shape(
|
||||
width=512,
|
||||
height=512,
|
||||
canvas_width=960,
|
||||
canvas_height=544,
|
||||
)
|
||||
|
||||
assert (resolved["width"], resolved["height"]) == (2048, 2048)
|
||||
assert resolved["reference_resize_mode"] == "diffusers"
|
||||
assert resolved["shape_policy_version"] == "reference_image_short_edge_v1"
|
||||
|
||||
|
||||
def test_reference_image_resize_normalizes_mode():
|
||||
resolved = reference_encoding.minimax_h3_resolve_reference_image_shape(
|
||||
width=1920,
|
||||
height=1080,
|
||||
canvas_width=960,
|
||||
canvas_height=544,
|
||||
mode=" MATCH ",
|
||||
)
|
||||
|
||||
assert resolved["reference_resize_mode"] == "match"
|
||||
assert (resolved["width"], resolved["height"]) == (960, 544)
|
||||
|
||||
|
||||
def test_reference_image_resize_rejects_unknown_mode():
|
||||
with pytest.raises(ValueError, match="reference_resize_mode"):
|
||||
reference_encoding.minimax_h3_resolve_reference_image_shape(
|
||||
width=1920,
|
||||
height=1080,
|
||||
canvas_width=960,
|
||||
canvas_height=544,
|
||||
mode="unknown",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("canvas_width", "canvas_height"),
|
||||
[(0, 544), (960, 0), (-960, 544), (float("nan"), 544)],
|
||||
)
|
||||
def test_reference_image_resize_rejects_invalid_canvas(canvas_width, canvas_height):
|
||||
with pytest.raises(ValueError, match="target canvas"):
|
||||
reference_encoding.minimax_h3_resolve_reference_image_shape(
|
||||
width=1920,
|
||||
height=1080,
|
||||
canvas_width=canvas_width,
|
||||
canvas_height=canvas_height,
|
||||
mode="match",
|
||||
)
|
||||
|
||||
|
||||
def test_keyframe_rng_supports_cpu_and_default_device():
|
||||
initial_state = torch.random.get_rng_state()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user