[diffusion] comfyui: run MiniMax H3, FastH3 and VDN in SGLD integrated mode (#43504)
This commit is contained in:
@@ -3,11 +3,18 @@ Generator for SGLang Diffusion ComfyUI integration.
|
||||
"""
|
||||
|
||||
import atexit
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from ..executors.flux import FluxExecutor
|
||||
from ..executors.minimax_h3 import MiniMaxH3Executor
|
||||
from ..executors.minimax_h3 import (
|
||||
FastH3Executor,
|
||||
MiniMaxH3Executor,
|
||||
VDNH3Executor,
|
||||
load_fasth3_release,
|
||||
)
|
||||
from ..executors.zimage import ZImageExecutor
|
||||
from .preflight import check_sgld_options
|
||||
|
||||
@@ -31,6 +38,59 @@ class _HeaderTensor:
|
||||
nelement = numel
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _spawn_without_launcher_main():
|
||||
"""spawn re-runs ``__main__`` in every child; ComfyUI's main.py then resolves
|
||||
``import utils`` to comfy/utils.py and the worker dies before reporting ready."""
|
||||
main_dict = vars(sys.modules["__main__"])
|
||||
saved = {
|
||||
key: main_dict[key] for key in ("__file__", "__spec__") if key in main_dict
|
||||
}
|
||||
main_dict.pop("__file__", None)
|
||||
main_dict["__spec__"] = None
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
main_dict.pop("__spec__", None)
|
||||
main_dict.update(saved)
|
||||
|
||||
|
||||
def _check_h3_single_file(path: str, sgld_options: dict) -> None:
|
||||
"""Reject options the ComfyUI H3 checkpoint loader would silently ignore."""
|
||||
if sgld_options.get("minimax_h3_adaln_online") or sgld_options.get(
|
||||
"minimax_h3_adaln_cache_path"
|
||||
):
|
||||
raise ValueError(
|
||||
"ComfyUI H3 checkpoint loading does not support the native AdaLN "
|
||||
"online/sidecar caches"
|
||||
)
|
||||
if not path.endswith(".safetensors"):
|
||||
return
|
||||
if sgld_options.get("quantization"):
|
||||
raise ValueError(
|
||||
"ComfyUI H3 files cannot be quantized at load time; load a pre-quantized "
|
||||
"file (e.g. *_int8_convrot), whose per-layer metadata selects the "
|
||||
"format, and leave quantization empty"
|
||||
)
|
||||
from safetensors import safe_open
|
||||
|
||||
with safe_open(path, framework="pt") as f:
|
||||
if any(".to_gate_compress." in key for key in f.keys()):
|
||||
raise ValueError(
|
||||
"This is a FastH3 checkpoint (VSA gate weights), which the base "
|
||||
"MiniMax H3 model cannot load; set model_type fast_h3 with "
|
||||
"runtime_model_path pointing at the native FastH3 model"
|
||||
)
|
||||
|
||||
|
||||
# Renamed local files and directories miss the registry's path detectors and
|
||||
# model-id keyed defaults (e.g. BCG support); these ids always match.
|
||||
_H3_MODEL_IDS = {
|
||||
"MiniMaxH3Pipeline": "MiniMax-H3",
|
||||
"VDNH3Pipeline": "vdn-minimax-h3",
|
||||
}
|
||||
|
||||
|
||||
def _looks_like_gguf(path: str) -> bool:
|
||||
if not path:
|
||||
return False
|
||||
@@ -116,7 +176,13 @@ else:
|
||||
|
||||
def _load_executor_classes():
|
||||
"""Qwen adapters import ComfyUI. Keep them optional so CI can load the rest."""
|
||||
classes = [FluxExecutor, ZImageExecutor, MiniMaxH3Executor]
|
||||
classes = [
|
||||
FluxExecutor,
|
||||
ZImageExecutor,
|
||||
MiniMaxH3Executor,
|
||||
FastH3Executor,
|
||||
VDNH3Executor,
|
||||
]
|
||||
try:
|
||||
from ..executors.qwen_image import QwenImageEditExecutor, QwenImageExecutor
|
||||
except ModuleNotFoundError as exc:
|
||||
@@ -210,13 +276,19 @@ class SGLDiffusionGenerator:
|
||||
# which a non-H3 single file lacks, and fails worker startup on >= 2 GPUs.
|
||||
if not kwargs.get("enable_cfg_parallel"):
|
||||
kwargs.setdefault("cfg_parallel_degree", 1)
|
||||
# Each sampler step is its own request carrying ComfyUI tensors; a
|
||||
# synthetic or request-based warmup has no such inputs to run with.
|
||||
kwargs["warmup_mode"] = "off"
|
||||
if pipeline_class_name in _H3_MODEL_IDS:
|
||||
kwargs.setdefault("model_id", _H3_MODEL_IDS[pipeline_class_name])
|
||||
kwargs = self._server_args_kwargs(kwargs)
|
||||
try:
|
||||
self.generator = DiffGenerator.from_pretrained(
|
||||
model_path=model_path,
|
||||
pipeline_class_name=pipeline_class_name,
|
||||
**kwargs,
|
||||
)
|
||||
with _spawn_without_launcher_main():
|
||||
self.generator = DiffGenerator.from_pretrained(
|
||||
model_path=model_path,
|
||||
pipeline_class_name=pipeline_class_name,
|
||||
**kwargs,
|
||||
)
|
||||
except EOFError as error:
|
||||
# A worker that raises while loading closes its pipe without a message.
|
||||
raise RuntimeError(
|
||||
@@ -426,6 +498,19 @@ class SGLDiffusionGenerator:
|
||||
if "enable_cache_dit" in sgld_options:
|
||||
plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit")
|
||||
set_model_type = sgld_options.pop("model_type", None)
|
||||
runtime_model_path = sgld_options.pop("runtime_model_path", None)
|
||||
if bool(runtime_model_path) != (set_model_type in ("fast_h3", "vdn_h3")):
|
||||
# The base H3 file is only the ComfyUI architecture companion.
|
||||
raise ValueError(
|
||||
"model_type fast_h3 / vdn_h3 requires runtime_model_path (the native "
|
||||
"model directory), and runtime_model_path is only used by them"
|
||||
)
|
||||
if runtime_model_path:
|
||||
runtime_model_path = os.path.abspath(os.path.expanduser(runtime_model_path))
|
||||
if not os.path.isdir(runtime_model_path):
|
||||
raise ValueError(
|
||||
"runtime_model_path must be a local materialized native model directory"
|
||||
)
|
||||
override = (sgld_options.get("transformer_weights_path") or "").strip()
|
||||
if override:
|
||||
sgld_options["transformer_weights_path"] = override
|
||||
@@ -441,6 +526,7 @@ class SGLDiffusionGenerator:
|
||||
"model_options": model_options,
|
||||
"sgld_options": sgld_options,
|
||||
"set_model_type": set_model_type,
|
||||
"runtime_model_path": runtime_model_path,
|
||||
}
|
||||
if self._can_reuse(gather_options):
|
||||
self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit")
|
||||
@@ -458,14 +544,20 @@ class SGLDiffusionGenerator:
|
||||
if set_model_type is not None and set_model_type in self.pipeline_class_dict:
|
||||
model_type = set_model_type
|
||||
|
||||
if model_type == "fast_h3":
|
||||
load_fasth3_release(runtime_model_path)
|
||||
if model_type == "minimax_h3":
|
||||
_check_h3_single_file(detect_path, sgld_options)
|
||||
|
||||
pipeline_class_name = self.pipeline_class_dict[model_type]
|
||||
worker_model_path = runtime_model_path or detect_path
|
||||
self.generator = self.init_generator(
|
||||
detect_path, pipeline_class_name, sgld_options
|
||||
worker_model_path, pipeline_class_name, sgld_options
|
||||
)
|
||||
|
||||
executor_class = self.executor_class_dict[model_type]
|
||||
self.executor = executor_class(
|
||||
self.generator, detect_path, comfyui_model, model_config
|
||||
self.generator, worker_model_path, comfyui_model, model_config
|
||||
)
|
||||
self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit")
|
||||
self.executor._sgld_reload = reload_kwargs
|
||||
|
||||
@@ -3,18 +3,26 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import fields
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import VDNH3SamplingParams
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import (
|
||||
MiniMaxH3ReleaseMetadata,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.comfyui_step import (
|
||||
DEFAULT_SIGMA_SHIFT_AUDIO,
|
||||
DEFAULT_SIGMA_SHIFT_VIDEO,
|
||||
serialize_comfyui_layout,
|
||||
time_shift_sigma,
|
||||
time_shift_slope,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
|
||||
from .adapter import ComfyUIModelAdapter, PackedForward
|
||||
@@ -150,13 +158,12 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
|
||||
width=int(video.shape[-1]),
|
||||
extra_req=extra_req,
|
||||
unpack_ctx={
|
||||
"video_denoise_mask": kwargs.get("denoise_mask"),
|
||||
"audio_denoise_mask": kwargs.get("audio_denoise_mask"),
|
||||
"audio_scale": scale,
|
||||
"audio_src": audio_src,
|
||||
"carry": carry,
|
||||
"sigma_v": sigma_v,
|
||||
"sigma_a": sigma_a,
|
||||
"shift_v": shift_v,
|
||||
"shift_a": shift_a,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -169,12 +176,18 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
|
||||
"MiniMax H3 unpack expects [video, audio] noise_pred, "
|
||||
f"got {type(noise_pred)!r}"
|
||||
)
|
||||
# Match ComfyUI MiniMaxH3._forward: negated velocities, and audio
|
||||
# scaled by d(sigma_a)/d(sigma_v) so the sampler's sigma_v ODE is
|
||||
# the audio stream's true ODE on its shifted schedule.
|
||||
# Match ComfyUI MiniMaxH3._forward: negated velocities, masked before the
|
||||
# carried-audio transform, which already holds d(sigma_a)/d(sigma_v).
|
||||
v_video = (-v_video).to(device=video_x.device, dtype=video_x.dtype)
|
||||
v_audio = (-v_audio).to(device=audio_x.device, dtype=audio_x.dtype)
|
||||
ctx = packed.unpack_ctx
|
||||
for name, velocity in (
|
||||
("video_denoise_mask", v_video),
|
||||
("audio_denoise_mask", v_audio),
|
||||
):
|
||||
mask = ctx.get(name)
|
||||
if mask is not None:
|
||||
velocity.mul_(mask.to(device=velocity.device, dtype=velocity.dtype))
|
||||
scale = float(ctx.get("audio_scale", 1.0))
|
||||
if scale != 1.0:
|
||||
audio_src = ctx["audio_src"]
|
||||
@@ -183,12 +196,6 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
|
||||
v_audio = (1.0 - scale) * (audio_src * carry) + (
|
||||
1.0 + (scale - 1.0) * sigma_a
|
||||
).to(v_audio.dtype) * v_audio
|
||||
slope_a = time_shift_slope(
|
||||
ctx["sigma_v"],
|
||||
ctx.get("shift_v", DEFAULT_SIGMA_SHIFT_VIDEO),
|
||||
ctx.get("shift_a", DEFAULT_SIGMA_SHIFT_AUDIO),
|
||||
).to(device=v_audio.device, dtype=v_audio.dtype)
|
||||
v_audio = slope_a * v_audio
|
||||
return [v_video, v_audio]
|
||||
|
||||
def fill_req(self, req, packed: PackedForward) -> None:
|
||||
@@ -231,3 +238,129 @@ class MiniMaxH3Executor(SGLDiffusionExecutor):
|
||||
if enable_cache_dit is not None:
|
||||
kwargs["enable_cache_dit"] = bool(enable_cache_dit)
|
||||
return kwargs
|
||||
|
||||
|
||||
class FastH3Adapter(MiniMaxH3Adapter):
|
||||
model_types = ("fast_h3",)
|
||||
pipeline_class_name = "FastH3Pipeline"
|
||||
|
||||
|
||||
class VDNH3Adapter(MiniMaxH3Adapter):
|
||||
model_types = ("vdn_h3",)
|
||||
pipeline_class_name = "VDNH3Pipeline"
|
||||
|
||||
|
||||
# FastVideo/FastVideo-FastH3-8-Step-V2 model_index.json._minimax_h3; the node
|
||||
# cannot see the loaded model, so FastH3Executor re-derives the grid from it.
|
||||
_FASTH3_DMD_STEPS = (999, 874, 749, 624, 500, 375, 250, 125)
|
||||
_FASTH3_VIDEO_SIGMA_SHIFT = 10.0
|
||||
|
||||
|
||||
def distilled_h3_sigmas(
|
||||
model_type: str,
|
||||
*,
|
||||
dmd_steps: tuple[int, ...] = _FASTH3_DMD_STEPS,
|
||||
video_shift: float = _FASTH3_VIDEO_SIGMA_SHIFT,
|
||||
) -> list[float]:
|
||||
if model_type == "fast_h3":
|
||||
return minimax_h3_time_shift_sigmas(
|
||||
num_steps=len(dmd_steps), shift_scale=video_shift, dmd_steps=dmd_steps
|
||||
)
|
||||
if model_type == "vdn_h3":
|
||||
return minimax_h3_time_shift_sigmas(
|
||||
num_steps=VDNH3SamplingParams.num_inference_steps,
|
||||
shift_scale=DEFAULT_SIGMA_SHIFT_VIDEO,
|
||||
)
|
||||
raise ValueError("Distilled sigma grid requires fast_h3 or vdn_h3")
|
||||
|
||||
|
||||
def load_fasth3_release(model_path: str) -> MiniMaxH3ReleaseMetadata:
|
||||
with open(os.path.join(model_path, "model_index.json")) as f:
|
||||
release = MiniMaxH3ReleaseMetadata.from_model_index(json.load(f))
|
||||
if release.dmd_denoising_steps is None:
|
||||
raise ValueError(
|
||||
"fast_h3 runtime_model_path has no trained DMD rungs "
|
||||
"(model_index.json._minimax_h3.dmd_denoising_steps); use "
|
||||
"FastVideo/FastVideo-FastH3-8-Step-V2"
|
||||
)
|
||||
index_path = os.path.join(
|
||||
model_path, "transformer", "diffusion_pytorch_model.safetensors.index.json"
|
||||
)
|
||||
if os.path.exists(index_path):
|
||||
with open(index_path) as f:
|
||||
weight_map = json.load(f)["weight_map"]
|
||||
# The Diffusers export drops this derivable buffer; the model overlay adds it.
|
||||
if "rope.inv_freq" not in weight_map:
|
||||
raise ValueError(
|
||||
"fast_h3 runtime_model_path is a raw FastH3 download; point it at the "
|
||||
"materialized copy (run `sglang generate --model-path "
|
||||
"FastVideo/FastVideo-FastH3-8-Step-V2` once, then use "
|
||||
"$SGLANG_DIFFUSION_CACHE_ROOT/materialized_models/...)"
|
||||
)
|
||||
return release
|
||||
|
||||
|
||||
def validate_distilled_h3_step(packed, model_type: str, expected: list[float]):
|
||||
sigmas = packed.extra_req.get("h3_sample_sigmas")
|
||||
actual = torch.as_tensor(sigmas if sigmas is not None else []).float().cpu()
|
||||
expected_t = torch.tensor(expected)
|
||||
if actual.numel() != expected_t.numel() or not torch.allclose(
|
||||
actual.reshape(-1), expected_t, atol=1e-6, rtol=0
|
||||
):
|
||||
raise ValueError(
|
||||
f"{model_type} requires its trained {expected_t.numel() - 1}-NFE sigma "
|
||||
"grid; use SGLDH3DistilledSigmas"
|
||||
)
|
||||
payload = packed.extra_req.get("h3_payload") or {}
|
||||
if payload.get("refs") or (model_type == "fast_h3" and payload.get("keyframes")):
|
||||
raise ValueError(f"{model_type} does not support this reference/keyframe task")
|
||||
|
||||
|
||||
class _DistilledH3Executor(MiniMaxH3Executor):
|
||||
expected_sigmas: list[float]
|
||||
|
||||
def _execute_packed(self, packed, x, timestep):
|
||||
validate_distilled_h3_step(
|
||||
packed, self.adapter.model_types[0], self.expected_sigmas
|
||||
)
|
||||
return super()._execute_packed(packed, x, timestep)
|
||||
|
||||
def _sampling_params_kwargs(self, packed, timestep):
|
||||
kwargs = super()._sampling_params_kwargs(packed, timestep)
|
||||
kwargs["num_inference_steps"] = len(self.expected_sigmas) - 1
|
||||
return kwargs
|
||||
|
||||
|
||||
class FastH3Executor(_DistilledH3Executor):
|
||||
adapter_cls = FastH3Adapter
|
||||
|
||||
def __init__(self, generator, model_path, model, config):
|
||||
super().__init__(generator, model_path, model, config)
|
||||
self.release = load_fasth3_release(model_path)
|
||||
self.expected_sigmas = distilled_h3_sigmas(
|
||||
"fast_h3",
|
||||
dmd_steps=self.release.dmd_denoising_steps,
|
||||
video_shift=self.release.video_sigma_shift,
|
||||
)
|
||||
|
||||
def forward(self, x, timestep, context, **kwargs):
|
||||
# The audio sigma is derived from the video sigma through these shifts on
|
||||
# both sides of the IPC; a missing MiniMaxH3SigmaShift must not mean 12/3.
|
||||
opts = dict(kwargs.get("transformer_options") or {})
|
||||
for key, trained in (
|
||||
("minimax_h3_sigma_shift_video", self.release.video_sigma_shift),
|
||||
("minimax_h3_sigma_shift_audio", self.release.audio_sigma_shift),
|
||||
):
|
||||
if opts.setdefault(key, trained) != trained:
|
||||
raise ValueError(
|
||||
f"FastH3 was distilled with sigma shift video/audio "
|
||||
f"{self.release.video_sigma_shift:g}/{self.release.audio_sigma_shift:g}; "
|
||||
f"got {key}={float(opts[key]):g}"
|
||||
)
|
||||
kwargs["transformer_options"] = opts
|
||||
return super().forward(x, timestep, context, **kwargs)
|
||||
|
||||
|
||||
class VDNH3Executor(_DistilledH3Executor):
|
||||
adapter_cls = VDNH3Adapter
|
||||
expected_sigmas = distilled_h3_sigmas("vdn_h3")
|
||||
|
||||
@@ -28,6 +28,23 @@ from .utils import (
|
||||
)
|
||||
|
||||
|
||||
class SGLDH3DistilledSigmas:
|
||||
"""The checkpoint's trained rectified-flow grid, independent of scheduler presets."""
|
||||
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {"required": {"model_type": (["fast_h3", "vdn_h3"],)}}
|
||||
|
||||
RETURN_TYPES = ("SIGMAS",)
|
||||
FUNCTION = "build"
|
||||
CATEGORY = "SGLDiffusion"
|
||||
|
||||
def build(self, model_type):
|
||||
from .executors.minimax_h3 import distilled_h3_sigmas
|
||||
|
||||
return (torch.tensor(distilled_h3_sigmas(model_type)),)
|
||||
|
||||
|
||||
class SGLDOptions:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
@@ -42,6 +59,8 @@ class SGLDOptions:
|
||||
"flux",
|
||||
"lumina2",
|
||||
"minimax_h3",
|
||||
"fast_h3",
|
||||
"vdn_h3",
|
||||
],
|
||||
{"default": "auto-detect"},
|
||||
),
|
||||
@@ -90,6 +109,8 @@ class SGLDOptions:
|
||||
"STRING",
|
||||
{"default": ""},
|
||||
),
|
||||
"runtime_model_path": ("STRING", {"default": ""}),
|
||||
"attention_backend_config": ("STRING", {"default": ""}),
|
||||
"transformer_weights_path": (
|
||||
"STRING",
|
||||
{
|
||||
@@ -122,6 +143,8 @@ class SGLDOptions:
|
||||
enable_cache_dit: bool = False,
|
||||
quantization: str = "",
|
||||
transformer_weights_path: str = "",
|
||||
runtime_model_path: str = "",
|
||||
attention_backend_config: str = "",
|
||||
):
|
||||
"""
|
||||
Build a dictionary of SGLang Diffusion runtime options.
|
||||
@@ -160,6 +183,14 @@ class SGLDOptions:
|
||||
# local .gguf, owner/repo/path/file.gguf, or owner/repo:QUANT.
|
||||
options["transformer_weights_path"] = transformer_weights_path
|
||||
|
||||
for key, value in (
|
||||
("runtime_model_path", runtime_model_path),
|
||||
# Same format as --attention-backend-config; ServerArgs parses it.
|
||||
("attention_backend_config", attention_backend_config),
|
||||
):
|
||||
if value.strip():
|
||||
options[key] = value.strip()
|
||||
|
||||
# Strip None to keep payload clean
|
||||
options = {k: v for k, v in options.items() if v is not None}
|
||||
return (options,)
|
||||
@@ -972,6 +1003,7 @@ class SGLDiffusionServerUnsetLora:
|
||||
|
||||
# Register nodes
|
||||
NODE_CLASS_MAPPINGS = {
|
||||
"SGLDH3DistilledSigmas": SGLDH3DistilledSigmas,
|
||||
"SGLDiffusionServerModel": SGLDiffusionServerModel,
|
||||
"SGLDiffusionGenerateImage": SGLDiffusionGenerateImage,
|
||||
"SGLDiffusionGenerateVideo": SGLDiffusionGenerateVideo,
|
||||
|
||||
+122
-4
@@ -10,7 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from typing import Any, Callable
|
||||
|
||||
import torch
|
||||
|
||||
@@ -223,7 +223,29 @@ def comfyui_layout_to_packed(layout: dict[str, Any]) -> dict[str, Any]:
|
||||
token_tags[start:stop] = _SEG_TOKEN_TAG.get(kind, -1)
|
||||
if kind == "text":
|
||||
text_pos = torch.arange(start, stop, dtype=torch.long)
|
||||
# Cube-sparse-attention segment shapes, as packed_sequence exports them.
|
||||
cond_shapes, cond_roles, events, audio_lens = [], [], [], []
|
||||
for start, stop, kind in layout["segments"]:
|
||||
if kind in ("cond", "ref_img"):
|
||||
coords = position_ids[start:stop]
|
||||
cond_shapes.append(
|
||||
tuple(int(torch.unique(coords[:, axis]).numel()) for axis in range(3))
|
||||
)
|
||||
cond_roles.append("joint_cube" if kind == "cond" else "independent_cube")
|
||||
events.append(("imgvid", len(cond_shapes) - 1))
|
||||
elif kind in ("cond_audio", "ref_audio"):
|
||||
audio_lens.append(stop - start)
|
||||
events.append(("audio", len(audio_lens) - 1))
|
||||
_, latent_t, latent_h, latent_w, *_ = layout["signature"]
|
||||
stream_layout = {
|
||||
"target_shape": (latent_t, (latent_h + 1) // 2, (latent_w + 1) // 2),
|
||||
"cond_image_shapes": tuple(cond_shapes),
|
||||
"cond_image_roles": tuple(cond_roles),
|
||||
"cond_event_orders": tuple(events),
|
||||
"cond_audio_stream_lens": tuple(audio_lens),
|
||||
}
|
||||
return {
|
||||
"stream_layout": stream_layout,
|
||||
"seq_len": seq_len,
|
||||
"img_pos": layout["img_pos"].view(-1).to(torch.long),
|
||||
"audio_pos": layout["audio_pos"].view(-1).to(torch.long),
|
||||
@@ -526,6 +548,7 @@ def comfyui_payload_to_branch_inputs(
|
||||
"used": used,
|
||||
"orig_video_shape": orig_shape,
|
||||
"padded_video_shape": tuple(video.shape),
|
||||
"is_ref2va": bool(refs),
|
||||
"sample_sigmas": sample_sigmas,
|
||||
"sigma_v": float(sigma_v),
|
||||
"shift_v": shift_v,
|
||||
@@ -608,6 +631,53 @@ def _sample_sigmas_len(sample_sigmas: Any) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
def _step_attn_metadata(
|
||||
model: Any,
|
||||
inputs: dict[str, Any],
|
||||
sample_sigmas: Any,
|
||||
server_args: ServerArgs,
|
||||
device: torch.device,
|
||||
) -> Callable[[int], Any] | None:
|
||||
"""Sparse-attention metadata per step, built as the native H3 loop builds it."""
|
||||
from sglang.multimodal_gen.runtime.models.dits.minimax_h3_vdn_attention import (
|
||||
prepare_hybrid_attention_metadata,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
|
||||
_build_cube_attn_metadata,
|
||||
_maybe_prepare_vsa_h3_step_metadata,
|
||||
)
|
||||
|
||||
packed = inputs["packed"]
|
||||
latent_shape = tuple(inputs["padded_video_shape"][-3:])
|
||||
cube = _build_cube_attn_metadata(
|
||||
server_args,
|
||||
packed=packed,
|
||||
num_steps=max((_sample_sigmas_len(sample_sigmas) or 2) - 1, 1),
|
||||
device=device,
|
||||
)
|
||||
if cube is not None:
|
||||
return lambda _step: cube
|
||||
# VSA-H3 tiles assume the native row order (text, cond, audio, raster video);
|
||||
# test_vsa_h3_tiles_cover_comfyui_packed_rows pins ComfyUI layouts to it.
|
||||
build = _maybe_prepare_vsa_h3_step_metadata(
|
||||
model=model,
|
||||
packed=packed,
|
||||
is_ref2va=inputs["is_ref2va"],
|
||||
latent_shape=latent_shape,
|
||||
server_args=server_args,
|
||||
device=device,
|
||||
)
|
||||
if build is not None:
|
||||
return build
|
||||
return prepare_hybrid_attention_metadata(
|
||||
model=model,
|
||||
packed=packed,
|
||||
latent_shape=latent_shape,
|
||||
server_args=server_args,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MiniMaxH3ComfyUIRunState:
|
||||
branch: MiniMaxH3DenoiseBranch
|
||||
@@ -615,6 +685,7 @@ class MiniMaxH3ComfyUIRunState:
|
||||
orig_video_shape: tuple[int, ...]
|
||||
padded_video_shape: tuple[int, ...]
|
||||
sample_sigmas: Any = None
|
||||
step_attn_metadata: Callable[[int], Any] | None = None
|
||||
cache_mounted: bool = False
|
||||
refined: bool = False
|
||||
steps_done: int = 0
|
||||
@@ -628,6 +699,14 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
|
||||
self._cache_stage = None
|
||||
|
||||
def verify_input(self, batch: Req, server_args: ServerArgs):
|
||||
if (
|
||||
server_args.enable_breakable_cuda_graph
|
||||
and server_args.is_dit_layerwise_offload_selected
|
||||
):
|
||||
raise ValueError(
|
||||
"ComfyUI H3 breakable CUDA graph is incompatible with layerwise "
|
||||
"offload: the offload copy-stream events invalidate graph capture"
|
||||
)
|
||||
bind_comfyui_session(batch)
|
||||
return super().verify_input(batch, server_args)
|
||||
|
||||
@@ -648,7 +727,9 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
|
||||
n_sigmas = _sample_sigmas_len(state.sample_sigmas)
|
||||
if n_sigmas is None or n_sigmas < 2:
|
||||
return
|
||||
self._cache_dit_stage()._maybe_enable_cache_dit(n_sigmas - 1, batch)
|
||||
self._cache_dit_stage()._maybe_enable_cache_dit_and_torch_compile(
|
||||
n_sigmas - 1, batch
|
||||
)
|
||||
state.cache_mounted = True
|
||||
|
||||
def _release_run_state(self, state: MiniMaxH3ComfyUIRunState | None) -> None:
|
||||
@@ -686,6 +767,7 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
|
||||
inputs: dict[str, Any],
|
||||
sample_sigmas: Any,
|
||||
device: torch.device,
|
||||
server_args: ServerArgs,
|
||||
) -> MiniMaxH3ComfyUIRunState:
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
|
||||
_precompute_refined_prompt_embeds,
|
||||
@@ -715,9 +797,15 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
|
||||
text_embeddings=inputs["text_embeddings"],
|
||||
token_tags=inputs["token_tags"],
|
||||
device=device,
|
||||
video_query_indices=inputs["packed"]["img_pos"][
|
||||
inputs["packed"]["update_mask"]
|
||||
],
|
||||
)
|
||||
return MiniMaxH3ComfyUIRunState(
|
||||
branch=branch,
|
||||
step_attn_metadata=_step_attn_metadata(
|
||||
self.transformer, inputs, sample_sigmas, server_args, device
|
||||
),
|
||||
used=inputs["used"],
|
||||
orig_video_shape=inputs["orig_video_shape"],
|
||||
padded_video_shape=inputs["padded_video_shape"],
|
||||
@@ -737,6 +825,34 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
|
||||
state.refined = True
|
||||
return state
|
||||
|
||||
def _run_dit(
|
||||
self, batch: Req, fk: dict[str, Any], state: MiniMaxH3ComfyUIRunState
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
from sglang.multimodal_gen.runtime.managers.forward_context import (
|
||||
set_forward_context,
|
||||
)
|
||||
|
||||
stage = self._cache_dit_stage()
|
||||
step = state.steps_done
|
||||
attn_metadata = (
|
||||
state.step_attn_metadata(step)
|
||||
if state.step_attn_metadata is not None
|
||||
else None
|
||||
)
|
||||
runner = stage._maybe_get_bcg_runner(self.transformer)
|
||||
if runner is not None:
|
||||
# Integrated mode never warms up, so capture here; it is a no-op
|
||||
# once this signature has a graph.
|
||||
with set_forward_context(
|
||||
current_timestep=step, attn_metadata=attn_metadata, forward_batch=batch
|
||||
):
|
||||
runner.capture(
|
||||
**stage._bcg_pad_prompt_kwargs(fk, current_model=self.transformer)
|
||||
)
|
||||
return stage._forward_dit(
|
||||
self.transformer, fk, step, batch=batch, attn_metadata=attn_metadata
|
||||
)
|
||||
|
||||
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
|
||||
bind_comfyui_session(batch)
|
||||
extra = getattr(batch, "extra", None) or {}
|
||||
@@ -764,12 +880,14 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
|
||||
denoise_mask=extra.get("h3_denoise_mask"),
|
||||
audio_denoise_mask=extra.get("h3_audio_denoise_mask"),
|
||||
)
|
||||
state = self._bind_run_state(batch, inputs, sample_sigmas, video_x.device)
|
||||
state = self._bind_run_state(
|
||||
batch, inputs, sample_sigmas, video_x.device, server_args
|
||||
)
|
||||
self._maybe_mount_cache_dit(batch, state)
|
||||
fk, _branch = build_step_forward_kwargs(
|
||||
inputs, branch=state.branch, device=video_x.device
|
||||
)
|
||||
v_video, v_audio = self.transformer(**fk)
|
||||
v_video, v_audio = self._run_dit(batch, fk, state)
|
||||
batch.noise_pred = pack_comfy_output(v_video, v_audio, state)
|
||||
state.steps_done += 1
|
||||
n_sigmas = _sample_sigmas_len(sample_sigmas)
|
||||
|
||||
+6
-4
@@ -747,7 +747,8 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
|
||||
build_vsa_h3_step_metadata = _maybe_prepare_vsa_h3_step_metadata(
|
||||
model=model,
|
||||
packed=packed,
|
||||
ctx=ctx,
|
||||
is_ref2va=ctx.is_ref2va,
|
||||
latent_shape=(ctx.latent_t, ctx.latent_h, ctx.latent_w),
|
||||
server_args=server_args,
|
||||
device=device,
|
||||
)
|
||||
@@ -1049,7 +1050,8 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
*,
|
||||
model: Any,
|
||||
packed: Mapping[str, torch.Tensor],
|
||||
ctx: _FullLoopContext,
|
||||
is_ref2va: bool,
|
||||
latent_shape: tuple[int, int, int],
|
||||
server_args: ServerArgs,
|
||||
device: torch.device,
|
||||
) -> Callable[[int], Any] | None:
|
||||
@@ -1061,7 +1063,7 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
is not AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3
|
||||
):
|
||||
return None
|
||||
if ctx.is_ref2va:
|
||||
if is_ref2va:
|
||||
raise NotImplementedError(
|
||||
"VSA-H3 supports the t2va/fl2va packed layout; the ref2va "
|
||||
"reference-block layout is not tiled yet. Use --attention-backend "
|
||||
@@ -1102,7 +1104,7 @@ def _maybe_prepare_vsa_h3_step_metadata(
|
||||
def build(step_index: int):
|
||||
return builder.build(
|
||||
current_timestep=step_index,
|
||||
raw_latent_shape=(ctx.latent_t, ctx.latent_h, ctx.latent_w),
|
||||
raw_latent_shape=latent_shape,
|
||||
patch_size=patch_size,
|
||||
VSA_sparsity=sparsity,
|
||||
prefix_segments=(text_len, cond_rows, audio_rows),
|
||||
|
||||
@@ -260,3 +260,109 @@ def test_multi_gpu_single_file_starts_without_cfg_parallel(
|
||||
str(path), "FluxPipeline", {"num_gpus": 2, "enable_cfg_parallel": True}
|
||||
)
|
||||
assert "cfg_parallel_degree" not in seen
|
||||
|
||||
|
||||
def test_spawned_workers_do_not_reexecute_launcher_main(monkeypatch) -> None:
|
||||
"""Workers must not re-run ComfyUI's main.py; its imports break under spawn."""
|
||||
import multiprocessing.spawn
|
||||
import sys
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import (
|
||||
_spawn_without_launcher_main,
|
||||
)
|
||||
|
||||
main_dict = vars(sys.modules["__main__"])
|
||||
monkeypatch.setitem(main_dict, "__file__", "/comfy/main.py")
|
||||
with _spawn_without_launcher_main():
|
||||
data = multiprocessing.spawn.get_preparation_data("worker")
|
||||
assert "init_main_from_path" not in data
|
||||
assert "init_main_from_name" not in data
|
||||
assert main_dict["__file__"] == "/comfy/main.py"
|
||||
|
||||
|
||||
def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator:
|
||||
runtime = SGLDiffusionGenerator()
|
||||
runtime.get_comfyui_model = lambda *a: (SimpleNamespace(), None, "minimax_h3")
|
||||
runtime.init_generator = Mock(side_effect=AssertionError("worker must not start"))
|
||||
return runtime
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key, sgld_options, error",
|
||||
[
|
||||
# A FastH3 file loaded as base H3 died in the worker on to_gate_compress.
|
||||
("blocks.0.attn.to_gate_compress.weight", {}, "model_type fast_h3"),
|
||||
# The H3 file loader builds no quant config, so this was silently ignored.
|
||||
(
|
||||
"blocks.0.attn.to_q.weight",
|
||||
{"quantization": "convrot_int8"},
|
||||
"cannot be quantized at load time",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_unservable_h3_file_is_rejected_before_worker_load(
|
||||
tmp_path, key, sgld_options, error
|
||||
) -> None:
|
||||
from safetensors.torch import save_file
|
||||
|
||||
path = tmp_path / "h3.safetensors"
|
||||
save_file({key: torch.zeros(1)}, path)
|
||||
with pytest.raises(ValueError, match=error):
|
||||
_h3_runtime_that_must_not_start().load_model(
|
||||
model_path=str(path), sgld_options=sgld_options
|
||||
)
|
||||
|
||||
|
||||
def test_fasth3_runtime_dir_is_checked_before_worker_load(tmp_path) -> None:
|
||||
"""The 4-step preview and a raw (unmaterialized) FastH3 V2 download were
|
||||
only rejected by the worker after a full model load."""
|
||||
import json
|
||||
|
||||
release = {
|
||||
"schema_version": 1,
|
||||
"partition": "fl2va",
|
||||
"tasks": ["t2va"],
|
||||
"task_aliases": {},
|
||||
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
|
||||
}
|
||||
(tmp_path / "transformer").mkdir()
|
||||
index = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors.index.json"
|
||||
runtime = _h3_runtime_that_must_not_start()
|
||||
options = {"model_type": "fast_h3", "runtime_model_path": str(tmp_path)}
|
||||
for dmd, weight_map, message in (
|
||||
(None, {}, "no trained DMD rungs"),
|
||||
([999, 500], {"blocks.0.x": "a.safetensors"}, "raw FastH3 download"),
|
||||
):
|
||||
meta = dict(release, **({"dmd_denoising_steps": dmd} if dmd else {}))
|
||||
(tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": meta}))
|
||||
index.write_text(json.dumps({"weight_map": weight_map}))
|
||||
with pytest.raises(ValueError, match=message):
|
||||
runtime.load_model(model_path="h3.safetensors", sgld_options=options)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name, pipeline, config_cls",
|
||||
[
|
||||
("h3_local.safetensors", "MiniMaxH3Pipeline", "MiniMaxH3PipelineConfig"),
|
||||
("VDN-H3-8nfe", "VDNH3Pipeline", "VDNH3PipelineConfig"),
|
||||
],
|
||||
)
|
||||
def test_renamed_h3_paths_resolve_through_the_plugin_model_id(
|
||||
tmp_path, monkeypatch, name, pipeline, config_cls
|
||||
) -> None:
|
||||
"""A local H3 file or VDN copy whose path lacks the HF name matched no
|
||||
registry detector or model-id keyed default."""
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator
|
||||
from sglang.multimodal_gen.registry import _get_config_info
|
||||
|
||||
seen = {}
|
||||
monkeypatch.setattr(
|
||||
generator.DiffGenerator,
|
||||
"from_pretrained",
|
||||
lambda **kwargs: seen.update(kwargs) or object(),
|
||||
)
|
||||
path = str(tmp_path / name)
|
||||
SGLDiffusionGenerator().init_generator(path, pipeline, {})
|
||||
assert seen["warmup_mode"] == "off"
|
||||
info = _get_config_info(path, model_id=seen["model_id"])
|
||||
assert info.pipeline_config_cls.__name__ == config_cls
|
||||
|
||||
@@ -318,6 +318,59 @@ def test_fl2va_first_last_used_prefix_matches_comfyui():
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("keyframes", [False, True])
|
||||
def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes):
|
||||
"""Integrated steps hand ComfyUI's own layout to the VSA-H3 tile builder,
|
||||
which assumes text, cond and audio prefixes followed by the raster video
|
||||
rows; a reordered layout would mis-tile attention silently."""
|
||||
from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTConfig
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
|
||||
_maybe_prepare_vsa_h3_step_metadata,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
|
||||
|
||||
text_len, latent_t, latent_h, latent_w, audio_t = 7, 3, 8, 12, 5
|
||||
layout = ComfyUIPackedLayout(
|
||||
text_len,
|
||||
latent_t,
|
||||
latent_h,
|
||||
latent_w,
|
||||
audio_t,
|
||||
keyframes=(
|
||||
[{"resolved_frame_index": 0, "latent": torch.zeros(1, 24, 1, 8, 12)}]
|
||||
if keyframes
|
||||
else None
|
||||
),
|
||||
)
|
||||
packed = comfyui_layout_to_packed(serialize_comfyui_layout(layout))
|
||||
model = SimpleNamespace(
|
||||
_resolve_attention_backend_once=lambda: None,
|
||||
_resolved_attention_backend=AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3,
|
||||
)
|
||||
server_args = SimpleNamespace(
|
||||
attention_backend_config={},
|
||||
pipeline_config=SimpleNamespace(
|
||||
vsa_sparsity=0.8, dit_config=MiniMaxH3DiTConfig()
|
||||
),
|
||||
)
|
||||
build = _maybe_prepare_vsa_h3_step_metadata(
|
||||
model=model,
|
||||
packed=packed,
|
||||
is_ref2va=False,
|
||||
latent_shape=(latent_t, latent_h, latent_w),
|
||||
server_args=server_args,
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
used = int(packed["cu_seqlens"][1])
|
||||
assert build(0).total_seq_length == used
|
||||
video = packed["img_pos"][packed["update_mask"].bool()]
|
||||
video_start = used - video.numel()
|
||||
assert torch.equal(video, torch.arange(video_start, used))
|
||||
audio = packed["audio_pos"]
|
||||
assert torch.equal(audio, torch.arange(video_start - audio.numel(), video_start))
|
||||
assert torch.equal(packed["text_pos"], torch.arange(text_len))
|
||||
|
||||
|
||||
def test_ref2va_image_plus_video_audio_used_prefix_matches_comfyui():
|
||||
kwargs = dict(text_len=5, latent_t=2, latent_h=4, latent_w=4, audio_t=5)
|
||||
refs = [
|
||||
@@ -454,8 +507,11 @@ from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
|
||||
SGLDiffusionExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.minimax_h3 import (
|
||||
FastH3Executor,
|
||||
MiniMaxH3Adapter,
|
||||
distilled_h3_sigmas,
|
||||
drop_h3_pinned_sampling_fields,
|
||||
validate_distilled_h3_step,
|
||||
worker_transformer_options,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
|
||||
@@ -1259,3 +1315,132 @@ def test_h3_pack_rejects_batched_latents() -> None:
|
||||
minimax_payload={"audio_scale": 1.0},
|
||||
transformer_options={"sample_sigmas": torch.tensor([1.0, 0.0])},
|
||||
)
|
||||
|
||||
|
||||
def test_comfyui_sparse_stream_layout_preserves_target_and_condition_roles():
|
||||
keyframe = {"resolved_frame_index": 0, "latent": torch.zeros(1, 24, 1, 8, 12)}
|
||||
layout = ComfyUIPackedLayout(3, 2, 8, 12, 4, keyframes=[keyframe])
|
||||
stream = comfyui_layout_to_packed(serialize_comfyui_layout(layout))["stream_layout"]
|
||||
assert stream["target_shape"] == (2, 4, 6)
|
||||
assert stream["cond_image_shapes"] == ((1, 4, 6),)
|
||||
assert stream["cond_image_roles"] == ("joint_cube",)
|
||||
assert stream["cond_event_orders"] == (("imgvid", 0),)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_type", ["fast_h3", "vdn_h3"])
|
||||
def test_distilled_h3_rejects_wrong_grid_and_reference_task(model_type):
|
||||
expected = distilled_h3_sigmas(model_type)
|
||||
sigmas = torch.tensor(expected)
|
||||
packed = SimpleNamespace(extra_req={"h3_sample_sigmas": sigmas})
|
||||
validate_distilled_h3_step(packed, model_type, expected)
|
||||
packed.extra_req["h3_sample_sigmas"] = torch.linspace(1, 0, len(expected))
|
||||
with pytest.raises(ValueError, match="trained"):
|
||||
validate_distilled_h3_step(packed, model_type, expected)
|
||||
packed.extra_req.update(
|
||||
h3_sample_sigmas=sigmas, h3_payload={"refs": [{"kind": "image"}]}
|
||||
)
|
||||
with pytest.raises(ValueError, match="reference"):
|
||||
validate_distilled_h3_step(packed, model_type, expected)
|
||||
|
||||
|
||||
def test_fasth3_comfy_grid_matches_native_8_step_v2_contract():
|
||||
"""The plugin pinned fast_h3 to the 4-NFE preview grid (shift 12) after the
|
||||
runtime moved to FastH3 8-Step V2, so V2 could not run in ComfyUI."""
|
||||
from sglang.multimodal_gen.configs.sample.minimax_h3 import FastH3SamplingParams
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
|
||||
minimax_h3_time_shift_sigmas,
|
||||
)
|
||||
|
||||
# FastVideo/FastVideo-FastH3-8-Step-V2 model_index.json._minimax_h3
|
||||
rungs = (999, 874, 749, 624, 500, 375, 250, 125)
|
||||
native = minimax_h3_time_shift_sigmas(
|
||||
num_steps=8, shift_scale=10.0, dmd_steps=rungs
|
||||
)
|
||||
assert distilled_h3_sigmas("fast_h3") == native
|
||||
assert len(native) == FastH3SamplingParams().num_inference_steps + 1
|
||||
|
||||
|
||||
def test_fasth3_executor_uses_trained_shifts_and_rejects_conflicts(tmp_path):
|
||||
"""Without a MiniMaxH3SigmaShift node the audio sigma was derived with the
|
||||
base 12/3 shifts instead of FastH3's trained 10/3."""
|
||||
import json
|
||||
|
||||
release = {
|
||||
"schema_version": 1,
|
||||
"partition": "fl2va",
|
||||
"tasks": ["t2va"],
|
||||
"task_aliases": {},
|
||||
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
|
||||
"dmd_denoising_steps": [999, 874, 749, 624, 500, 375, 250, 125],
|
||||
}
|
||||
(tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": release}))
|
||||
sent = []
|
||||
|
||||
class Recording(FastH3Executor):
|
||||
def _execute_packed(self, packed, x, timestep):
|
||||
sent.append(packed)
|
||||
return x
|
||||
|
||||
config = SimpleNamespace(unet_config={"dtype": torch.bfloat16})
|
||||
ex = Recording(None, str(tmp_path), None, config)
|
||||
x = [torch.ones(1, 24, 2, 4, 4), torch.ones(1, 32, 2, 3)]
|
||||
step = lambda **opts: ex(
|
||||
x,
|
||||
torch.tensor([500.0]),
|
||||
torch.ones(1, 8, 16),
|
||||
transformer_options={"sample_sigmas": torch.tensor(ex.expected_sigmas), **opts},
|
||||
minimax_payload={"audio_scale": 1.0},
|
||||
)
|
||||
step()
|
||||
opts = sent[0].extra_req["h3_transformer_options"]
|
||||
assert opts["minimax_h3_sigma_shift_video"] == 10.0
|
||||
assert opts["minimax_h3_sigma_shift_audio"] == 3.0
|
||||
with pytest.raises(ValueError, match="distilled with sigma shift video/audio 10/3"):
|
||||
step(minimax_h3_sigma_shift_video=12.0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"masks, video_out, audio_out",
|
||||
[
|
||||
# ComfyUI forward: (1 - scale) * input + scale * (-raw velocity). At
|
||||
# sigma_v=sigma_a=1 this is -3 - 4=-7; a second derivative gave -28.
|
||||
({}, -1, -7),
|
||||
# Masks apply to the raw velocities, before the carried-audio transform.
|
||||
(
|
||||
{
|
||||
"denoise_mask": torch.full((1, 1, 2, 4, 4), 0.25),
|
||||
"audio_denoise_mask": torch.zeros(1, 1, 2, 3),
|
||||
},
|
||||
-0.25,
|
||||
-3,
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_h3_velocities_match_comfyui_forward(masks, video_out, audio_out):
|
||||
adapter = MiniMaxH3Adapter()
|
||||
video = torch.ones(1, 24, 2, 4, 4)
|
||||
audio = torch.ones(1, 32, 2, 3)
|
||||
packed = adapter.pack(
|
||||
[video, audio],
|
||||
torch.tensor([1000.0]),
|
||||
torch.ones(1, 4, 8),
|
||||
minimax_payload={"audio_scale": 4.0},
|
||||
**masks,
|
||||
)
|
||||
out = adapter.unpack(
|
||||
[torch.ones_like(video), torch.ones_like(audio)], packed, [video, audio]
|
||||
)
|
||||
assert torch.equal(out[0], torch.full_like(video, video_out))
|
||||
assert torch.equal(out[1], torch.full_like(audio, audio_out))
|
||||
|
||||
|
||||
def test_comfyui_h3_rejects_bcg_with_offload_before_cuda_capture():
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.comfyui_step import (
|
||||
MiniMaxH3ComfyUIStepStage,
|
||||
)
|
||||
|
||||
args = SimpleNamespace(
|
||||
enable_breakable_cuda_graph=True, is_dit_layerwise_offload_selected=True
|
||||
)
|
||||
with pytest.raises(ValueError, match="copy-stream events"):
|
||||
MiniMaxH3ComfyUIStepStage.verify_input(None, SimpleNamespace(), args)
|
||||
|
||||
Reference in New Issue
Block a user