[diffusion] fix: honor requested inference step count for MiniMax H3(#38671)
Co-authored-by: Mick Qian <mickqian@users.noreply.github.com>
This commit is contained in:
@@ -25,8 +25,7 @@ def _optional_unit_float(value: Any, field_name: str) -> float | None:
|
||||
|
||||
@dataclass
|
||||
class MiniMaxH3SamplingParams(SamplingParams):
|
||||
# The video/audio sigma schedules include both interval endpoints.
|
||||
min_num_inference_steps: ClassVar[int] = 2
|
||||
min_num_inference_steps: ClassVar[int] = 1
|
||||
|
||||
height: int = 512
|
||||
width: int = 896
|
||||
@@ -348,14 +347,15 @@ class MiniMaxH3SamplingParams(SamplingParams):
|
||||
class FastH3SamplingParams(MiniMaxH3SamplingParams):
|
||||
"""FastH3 8-Step V2: nine sigma points, i.e. eight DiT forwards on the trained rungs."""
|
||||
|
||||
num_inference_steps: int = 9
|
||||
num_inference_steps: int = 8
|
||||
quality: str = "lossless"
|
||||
|
||||
def _validate(self) -> None:
|
||||
super()._validate()
|
||||
if self.num_inference_steps != 9:
|
||||
if self.num_inference_steps != 8:
|
||||
raise ValueError(
|
||||
"FastH3 is distilled for exactly nine sigma grid points (eight DiT "
|
||||
"FastH3 is distilled for exactly eight inference steps (nine sigma "
|
||||
"grid points, eight DiT "
|
||||
f"forwards); got num_inference_steps={self.num_inference_steps}. "
|
||||
"Use MiniMaxAI/MiniMax-H3 for other schedules."
|
||||
)
|
||||
|
||||
@@ -9,17 +9,17 @@ from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingPar
|
||||
@dataclass
|
||||
class VDNH3SamplingParams(MiniMaxH3SamplingParams):
|
||||
"""VDN-H3: nine sigma grid points, i.e. the eight distilled DiT forwards
|
||||
(VDN counts NFEs; SGLang counts sigma grid points). The turbo adapter is
|
||||
only valid at 8 NFE with video shift 12 / audio shift 3 (the defaults)."""
|
||||
The turbo adapter is only valid at 8 NFE with video shift 12 / audio
|
||||
shift 3 (the defaults)."""
|
||||
|
||||
num_inference_steps: int = 9
|
||||
num_inference_steps: int = 8
|
||||
|
||||
def _validate(self) -> None:
|
||||
super()._validate()
|
||||
if self.num_inference_steps != 9:
|
||||
if self.num_inference_steps != 8:
|
||||
raise ValueError(
|
||||
"VDN-H3 is distilled for exactly nine sigma grid points (eight DiT "
|
||||
f"forwards); got num_inference_steps={self.num_inference_steps}. "
|
||||
"VDN-H3 is distilled for exactly eight inference steps (nine sigma "
|
||||
f"grid points); got num_inference_steps={self.num_inference_steps}. "
|
||||
"Use MiniMaxAI/MiniMax-H3 for other schedules."
|
||||
)
|
||||
if self.task is not None and self.task.strip().lower() not in (
|
||||
|
||||
@@ -334,7 +334,7 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
),
|
||||
# Plan slots in the MiniMax-H3 --minimax-h3-adaln-online GPU slab
|
||||
# (9.25 MiB per slot-timestep; 64 x width 4 = 2.31 GiB). A request needs
|
||||
# up to num_inference_steps - 1 slots; the default covers the 50-step
|
||||
# up to num_inference_steps slots; the default covers the 50-step
|
||||
# serving schedule, so this is an escape hatch, not a deployment knob.
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS": _lazy_int(
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS", 64
|
||||
|
||||
@@ -1794,8 +1794,8 @@ class MiniMaxH3FinalLayer(nn.Module):
|
||||
if not 0 <= step < stack.shape[0]:
|
||||
raise ValueError(
|
||||
f"MiniMax-H3 PDD has {stack.shape[0]} fused heads but the loop is at "
|
||||
f"step {step}; run with --num-inference-steps {stack.shape[0] + 1} "
|
||||
"(H3 counts sigma grid points, so that is one more than the steps)."
|
||||
f"step {step}; run with --num-inference-steps {stack.shape[0]} "
|
||||
"(num_inference_steps counts denoise transitions)."
|
||||
)
|
||||
weight = stack[step].to(device=h.device, dtype=h.dtype)
|
||||
bias = heads[f"{name}.bias"][step].to(device=h.device, dtype=h.dtype)
|
||||
|
||||
+3
-6
@@ -172,20 +172,17 @@ class MiniMaxH3PartitionAdmissionStage(PipelineStage):
|
||||
self.metadata.canonical_task(task)
|
||||
min_steps = MiniMaxH3SamplingParams.min_num_inference_steps
|
||||
if batch.num_inference_steps < min_steps:
|
||||
raise ValueError(
|
||||
f"MiniMax H3 requires num_inference_steps >= {min_steps} because "
|
||||
"its video/audio sigma schedules include both interval endpoints"
|
||||
)
|
||||
raise ValueError(f"MiniMax H3 requires num_inference_steps >= {min_steps}")
|
||||
gpu_plans = envs.SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS
|
||||
if (
|
||||
server_args.minimax_h3_adaln_online
|
||||
and batch.num_inference_steps - 1 > gpu_plans
|
||||
and batch.num_inference_steps > gpu_plans
|
||||
):
|
||||
# Fail here, before the encode stages spend GPU time on a request
|
||||
# whose AdaLN rebuild is guaranteed to overflow the slab.
|
||||
raise ValueError(
|
||||
f"num_inference_steps={batch.num_inference_steps} needs up to "
|
||||
f"{batch.num_inference_steps - 1} AdaLN plans but the online "
|
||||
f"{batch.num_inference_steps} AdaLN plans but the online "
|
||||
f"slab holds {gpu_plans}; raise "
|
||||
"SGLANG_DIFFUSION_MINIMAX_H3_ADALN_GPU_PLANS"
|
||||
)
|
||||
|
||||
+1
-2
@@ -446,8 +446,7 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
def default_workload_iterations(
|
||||
self, batch: Req, num_inference_steps: int
|
||||
) -> int | None:
|
||||
# one denoise per sigma interval: steps - 1
|
||||
return max(1, num_inference_steps - 1)
|
||||
return num_inference_steps
|
||||
|
||||
def __init__(self, transformer, pipeline=None) -> None:
|
||||
super().__init__(
|
||||
|
||||
+4
-4
@@ -47,7 +47,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
)
|
||||
with safe_open(pdd_heads, "pt") as f:
|
||||
steps = f.get_slice("video_out.weight").get_shape()[0]
|
||||
if self._pdd_config["num_inference_steps"] != steps + 1:
|
||||
if self._pdd_config["num_inference_steps"] != steps:
|
||||
raise ValueError("MiniMax-H3 PDD config does not match the fused heads")
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
@@ -104,7 +104,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
if batch.is_warmup:
|
||||
# Warmup may run fewer steps, but must use the same intervals
|
||||
# as serving rather than rescaling a shorter grid to [1, 0].
|
||||
sigmas[modality] = expected[: max(2, batch.num_inference_steps)]
|
||||
sigmas[modality] = expected[: batch.num_inference_steps + 1]
|
||||
continue
|
||||
actual = sigmas[modality]
|
||||
if len(actual) != len(expected) or any(
|
||||
@@ -206,7 +206,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
num_steps = requested_num_steps
|
||||
if dmd_steps is not None and batch.is_warmup:
|
||||
# warmup truncates the trained rungs instead of rescaling a shorter grid
|
||||
num_steps = len(dmd_steps) + 1
|
||||
num_steps = len(dmd_steps)
|
||||
sigmas: dict[str, list[float]] = {}
|
||||
for modality in ("video", "audio"):
|
||||
sigmas[modality] = minimax_h3_time_shift_sigmas(
|
||||
@@ -215,7 +215,7 @@ class MiniMaxH3TimestepPreparationStage(PipelineStage):
|
||||
dmd_steps=dmd_steps,
|
||||
)
|
||||
if num_steps != requested_num_steps:
|
||||
sigmas[modality] = sigmas[modality][: max(2, requested_num_steps)]
|
||||
sigmas[modality] = sigmas[modality][: requested_num_steps + 1]
|
||||
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY] = sigmas
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs) -> VerificationResult:
|
||||
|
||||
+3
-6
@@ -43,7 +43,7 @@ def minimax_h3_time_shift_sigmas(
|
||||
import torch
|
||||
|
||||
if dmd_steps is not None:
|
||||
if int(num_steps) != len(dmd_steps) + 1:
|
||||
if int(num_steps) != len(dmd_steps):
|
||||
raise ValueError(
|
||||
f"this checkpoint is distilled for {len(dmd_steps)} DiT forwards "
|
||||
f"({len(dmd_steps) + 1} sigma grid points); got "
|
||||
@@ -58,15 +58,12 @@ def minimax_h3_time_shift_sigmas(
|
||||
base = torch.linspace(
|
||||
1.0,
|
||||
0.0,
|
||||
int(num_steps),
|
||||
int(num_steps) + 1,
|
||||
device="cpu",
|
||||
dtype=torch.float32,
|
||||
)
|
||||
shifted = float(shift_scale) * base / (1 + (float(shift_scale) - 1) * base)
|
||||
shifted, _ = torch.unique_consecutive(shifted, return_counts=True)
|
||||
# A one-point request is still exactly one point. Normal serving uses
|
||||
# multiple points, but preserving the requested cardinality keeps
|
||||
# ``num_inference_steps`` the sole schedule-size control.
|
||||
if num_steps > 1 and shifted[-1].item() > 0.0:
|
||||
if shifted[-1].item() > 0.0:
|
||||
shifted = torch.cat([shifted, torch.tensor([0.0], dtype=shifted.dtype)])
|
||||
return [float(value) for value in shifted.tolist()]
|
||||
|
||||
@@ -781,7 +781,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
|
||||
"aspect_ratio": "16:9",
|
||||
"duration_seconds": 5.0,
|
||||
},
|
||||
"num_inference_steps": 9,
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
},
|
||||
),
|
||||
@@ -822,7 +822,7 @@ MINIMAX_H3_FOUR_GPU_H100_CASES = [
|
||||
"aspect_ratio": "16:9",
|
||||
"duration_seconds": 5.0,
|
||||
},
|
||||
"num_inference_steps": 9,
|
||||
"num_inference_steps": 8,
|
||||
"seed": 42,
|
||||
},
|
||||
),
|
||||
|
||||
@@ -37,7 +37,7 @@ logger = init_logger(__name__)
|
||||
# NPU/ascend) is read from sgl-project/ci-data-diffusion, where the GT-gen workflows
|
||||
# publish.
|
||||
SGL_TEST_FILES_CI_DATA_REPO = "sgl-project/ci-data-diffusion"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "5d8eb22d16624817c62e06ce4cd69d36fbe7d456"
|
||||
SGL_TEST_FILES_CI_DATA_REVISION = "75992dbf1d6eaeacbc1c2cd6beb48f77dd7ae4f7"
|
||||
|
||||
# The NPU pin is kept as a separate branch so ascend GT can be bumped independently
|
||||
# when it's regenerated on its own cadence.
|
||||
|
||||
@@ -451,6 +451,21 @@ def test_admission_rejects_steps_exceeding_online_adaln_gpu_plans():
|
||||
stage.forward(batch, server_args)
|
||||
|
||||
batch.num_inference_steps = 9
|
||||
with pytest.raises(ValueError, match="9 AdaLN plans.*slab holds 8"):
|
||||
stage.forward(batch, server_args)
|
||||
|
||||
batch.num_inference_steps = 8
|
||||
assert stage.forward(batch, server_args) is batch
|
||||
|
||||
batch.num_inference_steps = 1
|
||||
assert stage.forward(batch, server_args) is batch
|
||||
|
||||
batch.num_inference_steps = 0
|
||||
with pytest.raises(ValueError, match="num_inference_steps >= 1"):
|
||||
stage.forward(batch, server_args)
|
||||
|
||||
server_args.minimax_h3_adaln_online = False
|
||||
batch.num_inference_steps = 50
|
||||
assert stage.forward(batch, server_args) is batch
|
||||
|
||||
|
||||
|
||||
@@ -30,6 +30,9 @@ from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.m
|
||||
_build_cube_attn_metadata,
|
||||
_precompute_refined_prompt_embeds,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
|
||||
|
||||
def _branch(
|
||||
@@ -64,6 +67,32 @@ def _branch(
|
||||
)
|
||||
|
||||
|
||||
def test_time_shift_sigmas_build_exact_denoise_step_count():
|
||||
for num_steps in (1, 2, 10, 50):
|
||||
for shift_scale in (3.0, 12.0):
|
||||
sigmas = minimax_h3_time_shift_sigmas(
|
||||
num_steps=num_steps,
|
||||
shift_scale=shift_scale,
|
||||
)
|
||||
|
||||
assert len(sigmas) == num_steps + 1
|
||||
assert sigmas[0] == 1.0
|
||||
assert sigmas[-1] == 0.0
|
||||
assert all(
|
||||
current > following for current, following in zip(sigmas, sigmas[1:])
|
||||
)
|
||||
# Independent closed form for N Euler intervals, including sigma=0.
|
||||
expected = [
|
||||
shift_scale
|
||||
* (num_steps - i)
|
||||
/ (num_steps + (shift_scale - 1) * (num_steps - i))
|
||||
for i in range(num_steps + 1)
|
||||
]
|
||||
torch.testing.assert_close(
|
||||
torch.tensor(sigmas), torch.tensor(expected), rtol=0, atol=2e-7
|
||||
)
|
||||
|
||||
|
||||
def test_precomputed_timestep_plan_matches_full_unique_reference():
|
||||
"""Preplanning must preserve fp32 collisions and every packed row class."""
|
||||
|
||||
|
||||
@@ -59,13 +59,13 @@ def test_registry_resolves_fasth3_configs() -> None:
|
||||
|
||||
def test_fasth3_sampling_defaults_and_task_rejection() -> None:
|
||||
params = FastH3SamplingParams(prompt="p")
|
||||
assert params.num_inference_steps == 9
|
||||
assert params.num_inference_steps == 8
|
||||
assert params.guidance_scale == 1.0
|
||||
assert params.quality == "lossless"
|
||||
assert FastH3SamplingParams(prompt="p", quality="extra-high").quality == "lossless"
|
||||
assert FastH3SamplingParams(prompt="p", quality="exact").quality == "exact"
|
||||
|
||||
with pytest.raises(ValueError, match="exactly nine sigma grid points"):
|
||||
with pytest.raises(ValueError, match="exactly eight inference steps"):
|
||||
FastH3SamplingParams(prompt="p", num_inference_steps=50)
|
||||
|
||||
with pytest.raises(ValueError, match="distilled for t2va only"):
|
||||
@@ -109,7 +109,7 @@ def test_fasth3_dmd_schedule_matches_trained_rungs() -> None:
|
||||
}
|
||||
for shift, expected in served.items():
|
||||
sigmas = minimax_h3_time_shift_sigmas(
|
||||
num_steps=9, shift_scale=shift, dmd_steps=DMD_STEPS
|
||||
num_steps=8, shift_scale=shift, dmd_steps=DMD_STEPS
|
||||
)
|
||||
assert sigmas == pytest.approx(expected, abs=1e-6)
|
||||
with pytest.raises(ValueError, match="8 DiT forwards"):
|
||||
|
||||
@@ -113,7 +113,8 @@ class TestMiniMaxH3PDDOffline(CustomTestCase):
|
||||
apply_schedule(batch)
|
||||
self.assertEqual(
|
||||
batch.extra[MINIMAX_H3_SIGMAS_EXTRA_KEY],
|
||||
{name: values[:3] for name, values in sigmas.items()},
|
||||
# Three warmup updates need four sigma boundaries on the serving grid.
|
||||
{name: values[:4] for name, values in sigmas.items()},
|
||||
)
|
||||
|
||||
def test_conversion_keeps_heads_outside_transformer(self):
|
||||
|
||||
@@ -73,9 +73,9 @@ def test_registry_resolves_vdn_h3_configs() -> None:
|
||||
|
||||
def test_vdn_h3_sampling_defaults_and_rejections() -> None:
|
||||
params = VDNH3SamplingParams(prompt="p")
|
||||
assert params.num_inference_steps == 9 # 8 NFE
|
||||
with pytest.raises(ValueError, match="exactly nine sigma grid points"):
|
||||
VDNH3SamplingParams(prompt="p", num_inference_steps=8)
|
||||
assert params.num_inference_steps == 8
|
||||
with pytest.raises(ValueError, match="exactly eight inference steps"):
|
||||
VDNH3SamplingParams(prompt="p", num_inference_steps=9)
|
||||
fl2va = VDNH3SamplingParams(
|
||||
prompt="p",
|
||||
task="fl2va",
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
|
||||
from sglang.multimodal_gen.test.test_utils import encode_audio_gt_wav
|
||||
from sglang.test.test_utils import CustomTestCase
|
||||
|
||||
SCRIPT = (
|
||||
Path(__file__).resolve().parents[5]
|
||||
/ "scripts/ci/utils/diffusion/publish_diffusion_gt.py"
|
||||
)
|
||||
spec = importlib.util.spec_from_file_location("publish_diffusion_gt", SCRIPT)
|
||||
publisher = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = publisher
|
||||
spec.loader.exec_module(publisher)
|
||||
|
||||
|
||||
class TestPublishDiffusionAudioGT(CustomTestCase):
|
||||
def test_generated_audio_is_collected_validated_and_deduplicated(self):
|
||||
content = encode_audio_gt_wav(np.linspace(-0.5, 0.5, 16000))
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
Path(directory, "case_2gpu_audio.wav").write_bytes(content)
|
||||
Path(directory, "case.mp4").write_bytes(b"not a GT")
|
||||
files = publisher.collect_gt_files(directory, "gt/h100")
|
||||
self.assertEqual(files, [("gt/h100/case_2gpu_audio.wav", content)])
|
||||
publisher.validate_gt_files(files, files, {}, None)
|
||||
entries = [
|
||||
{
|
||||
"type": "file",
|
||||
"path": files[0][0],
|
||||
"sha": publisher.git_blob_sha(content),
|
||||
}
|
||||
]
|
||||
with patch.object(
|
||||
publisher, "make_github_request", return_value=json.dumps(entries)
|
||||
):
|
||||
shas = publisher.get_remote_blob_shas("owner", "repo", "gt/h100", None)
|
||||
self.assertEqual(publisher.filter_changed_files(files, shas), [])
|
||||
|
||||
def test_invalid_audio_is_rejected(self):
|
||||
valid = encode_audio_gt_wav(np.ones(16000) * 0.1)
|
||||
output = io.BytesIO()
|
||||
with wave.open(output, "wb") as wav:
|
||||
wav.setnchannels(2)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(16000)
|
||||
wav.writeframes(b"\0" * 32)
|
||||
for content in [
|
||||
b"not a WAV",
|
||||
valid[:-2],
|
||||
encode_audio_gt_wav(np.zeros(0)),
|
||||
output.getvalue(),
|
||||
]:
|
||||
with self.subTest(length=len(content)), self.assertRaises(SystemExit):
|
||||
publisher.validate_gt_files([("gt/audio.wav", content)], [], {}, None)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -27,10 +27,10 @@ def test_generic_warmup_keeps_the_requested_steps():
|
||||
assert warmup.extra["warmup_target_num_inference_steps"] == 50
|
||||
|
||||
|
||||
def test_h3_warmup_is_raised_to_its_floor():
|
||||
assert MiniMaxH3SamplingParams.min_num_inference_steps == 2
|
||||
def test_h3_warmup_keeps_one_denoise_step():
|
||||
assert MiniMaxH3SamplingParams.min_num_inference_steps == 1
|
||||
warmup = _req(MiniMaxH3SamplingParams(prompt="p"), 50).copy_as_warmup(1)
|
||||
assert warmup.num_inference_steps == 2
|
||||
assert warmup.num_inference_steps == 1
|
||||
assert warmup.extra["warmup_target_num_inference_steps"] == 50
|
||||
|
||||
|
||||
|
||||
@@ -149,8 +149,8 @@ def _load_tensor(
|
||||
|
||||
def main() -> None:
|
||||
args = _parse_args()
|
||||
if args.num_inference_steps < 2 and args.timesteps is None:
|
||||
raise ValueError("--num-inference-steps must be at least 2")
|
||||
if args.num_inference_steps < 1 and args.timesteps is None:
|
||||
raise ValueError("--num-inference-steps must be at least 1")
|
||||
mode_variant = _MODE_VARIANTS[args.mode]
|
||||
if args.model_variant != mode_variant:
|
||||
raise ValueError(f"--mode {args.mode} requires {mode_variant}")
|
||||
|
||||
@@ -90,7 +90,7 @@ def main() -> int:
|
||||
}
|
||||
save_file(fused, str(d / "pdd_fused_heads.safetensors"), metadata={"format": "pt"})
|
||||
cfg["fused_steps"] = n // block
|
||||
cfg["num_inference_steps"] = n // block + 1 # H3 counts steps as sigma grid points
|
||||
cfg["num_inference_steps"] = n // block
|
||||
cfg["video_shift"], cfg["audio_shift"] = args.video_shift, args.audio_shift
|
||||
(d / "pdd_config.json").write_text(json.dumps(cfg, indent=2) + "\n")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user