[diffusion] comfyui: run MiniMax H3, FastH3 and VDN in SGLD integrated mode (#43504)

This commit is contained in:
WenhaoZhang
2026-10-13 12:01:48 +08:00
committed by GitHub
parent 4d5692b6cd
commit b4822bae03
7 changed files with 698 additions and 30 deletions
@@ -3,11 +3,18 @@ Generator for SGLang Diffusion ComfyUI integration.
"""
import atexit
import contextlib
import logging
import os
import sys
from ..executors.flux import FluxExecutor
from ..executors.minimax_h3 import MiniMaxH3Executor
from ..executors.minimax_h3 import (
FastH3Executor,
MiniMaxH3Executor,
VDNH3Executor,
load_fasth3_release,
)
from ..executors.zimage import ZImageExecutor
from .preflight import check_sgld_options
@@ -31,6 +38,59 @@ class _HeaderTensor:
nelement = numel
@contextlib.contextmanager
def _spawn_without_launcher_main():
"""spawn re-runs ``__main__`` in every child; ComfyUI's main.py then resolves
``import utils`` to comfy/utils.py and the worker dies before reporting ready."""
main_dict = vars(sys.modules["__main__"])
saved = {
key: main_dict[key] for key in ("__file__", "__spec__") if key in main_dict
}
main_dict.pop("__file__", None)
main_dict["__spec__"] = None
try:
yield
finally:
main_dict.pop("__spec__", None)
main_dict.update(saved)
def _check_h3_single_file(path: str, sgld_options: dict) -> None:
"""Reject options the ComfyUI H3 checkpoint loader would silently ignore."""
if sgld_options.get("minimax_h3_adaln_online") or sgld_options.get(
"minimax_h3_adaln_cache_path"
):
raise ValueError(
"ComfyUI H3 checkpoint loading does not support the native AdaLN "
"online/sidecar caches"
)
if not path.endswith(".safetensors"):
return
if sgld_options.get("quantization"):
raise ValueError(
"ComfyUI H3 files cannot be quantized at load time; load a pre-quantized "
"file (e.g. *_int8_convrot), whose per-layer metadata selects the "
"format, and leave quantization empty"
)
from safetensors import safe_open
with safe_open(path, framework="pt") as f:
if any(".to_gate_compress." in key for key in f.keys()):
raise ValueError(
"This is a FastH3 checkpoint (VSA gate weights), which the base "
"MiniMax H3 model cannot load; set model_type fast_h3 with "
"runtime_model_path pointing at the native FastH3 model"
)
# Renamed local files and directories miss the registry's path detectors and
# model-id keyed defaults (e.g. BCG support); these ids always match.
_H3_MODEL_IDS = {
"MiniMaxH3Pipeline": "MiniMax-H3",
"VDNH3Pipeline": "vdn-minimax-h3",
}
def _looks_like_gguf(path: str) -> bool:
if not path:
return False
@@ -116,7 +176,13 @@ else:
def _load_executor_classes():
"""Qwen adapters import ComfyUI. Keep them optional so CI can load the rest."""
classes = [FluxExecutor, ZImageExecutor, MiniMaxH3Executor]
classes = [
FluxExecutor,
ZImageExecutor,
MiniMaxH3Executor,
FastH3Executor,
VDNH3Executor,
]
try:
from ..executors.qwen_image import QwenImageEditExecutor, QwenImageExecutor
except ModuleNotFoundError as exc:
@@ -210,13 +276,19 @@ class SGLDiffusionGenerator:
# which a non-H3 single file lacks, and fails worker startup on >= 2 GPUs.
if not kwargs.get("enable_cfg_parallel"):
kwargs.setdefault("cfg_parallel_degree", 1)
# Each sampler step is its own request carrying ComfyUI tensors; a
# synthetic or request-based warmup has no such inputs to run with.
kwargs["warmup_mode"] = "off"
if pipeline_class_name in _H3_MODEL_IDS:
kwargs.setdefault("model_id", _H3_MODEL_IDS[pipeline_class_name])
kwargs = self._server_args_kwargs(kwargs)
try:
self.generator = DiffGenerator.from_pretrained(
model_path=model_path,
pipeline_class_name=pipeline_class_name,
**kwargs,
)
with _spawn_without_launcher_main():
self.generator = DiffGenerator.from_pretrained(
model_path=model_path,
pipeline_class_name=pipeline_class_name,
**kwargs,
)
except EOFError as error:
# A worker that raises while loading closes its pipe without a message.
raise RuntimeError(
@@ -426,6 +498,19 @@ class SGLDiffusionGenerator:
if "enable_cache_dit" in sgld_options:
plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit")
set_model_type = sgld_options.pop("model_type", None)
runtime_model_path = sgld_options.pop("runtime_model_path", None)
if bool(runtime_model_path) != (set_model_type in ("fast_h3", "vdn_h3")):
# The base H3 file is only the ComfyUI architecture companion.
raise ValueError(
"model_type fast_h3 / vdn_h3 requires runtime_model_path (the native "
"model directory), and runtime_model_path is only used by them"
)
if runtime_model_path:
runtime_model_path = os.path.abspath(os.path.expanduser(runtime_model_path))
if not os.path.isdir(runtime_model_path):
raise ValueError(
"runtime_model_path must be a local materialized native model directory"
)
override = (sgld_options.get("transformer_weights_path") or "").strip()
if override:
sgld_options["transformer_weights_path"] = override
@@ -441,6 +526,7 @@ class SGLDiffusionGenerator:
"model_options": model_options,
"sgld_options": sgld_options,
"set_model_type": set_model_type,
"runtime_model_path": runtime_model_path,
}
if self._can_reuse(gather_options):
self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit")
@@ -458,14 +544,20 @@ class SGLDiffusionGenerator:
if set_model_type is not None and set_model_type in self.pipeline_class_dict:
model_type = set_model_type
if model_type == "fast_h3":
load_fasth3_release(runtime_model_path)
if model_type == "minimax_h3":
_check_h3_single_file(detect_path, sgld_options)
pipeline_class_name = self.pipeline_class_dict[model_type]
worker_model_path = runtime_model_path or detect_path
self.generator = self.init_generator(
detect_path, pipeline_class_name, sgld_options
worker_model_path, pipeline_class_name, sgld_options
)
executor_class = self.executor_class_dict[model_type]
self.executor = executor_class(
self.generator, detect_path, comfyui_model, model_config
self.generator, worker_model_path, comfyui_model, model_config
)
self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit")
self.executor._sgld_reload = reload_kwargs
@@ -3,18 +3,26 @@
from __future__ import annotations
import json
import os
from dataclasses import fields
from typing import Any
import torch
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import VDNH3SamplingParams
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import (
MiniMaxH3ReleaseMetadata,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.comfyui_step import (
DEFAULT_SIGMA_SHIFT_AUDIO,
DEFAULT_SIGMA_SHIFT_VIDEO,
serialize_comfyui_layout,
time_shift_sigma,
time_shift_slope,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
minimax_h3_time_shift_sigmas,
)
from .adapter import ComfyUIModelAdapter, PackedForward
@@ -150,13 +158,12 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
width=int(video.shape[-1]),
extra_req=extra_req,
unpack_ctx={
"video_denoise_mask": kwargs.get("denoise_mask"),
"audio_denoise_mask": kwargs.get("audio_denoise_mask"),
"audio_scale": scale,
"audio_src": audio_src,
"carry": carry,
"sigma_v": sigma_v,
"sigma_a": sigma_a,
"shift_v": shift_v,
"shift_a": shift_a,
},
)
@@ -169,12 +176,18 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
"MiniMax H3 unpack expects [video, audio] noise_pred, "
f"got {type(noise_pred)!r}"
)
# Match ComfyUI MiniMaxH3._forward: negated velocities, and audio
# scaled by d(sigma_a)/d(sigma_v) so the sampler's sigma_v ODE is
# the audio stream's true ODE on its shifted schedule.
# Match ComfyUI MiniMaxH3._forward: negated velocities, masked before the
# carried-audio transform, which already holds d(sigma_a)/d(sigma_v).
v_video = (-v_video).to(device=video_x.device, dtype=video_x.dtype)
v_audio = (-v_audio).to(device=audio_x.device, dtype=audio_x.dtype)
ctx = packed.unpack_ctx
for name, velocity in (
("video_denoise_mask", v_video),
("audio_denoise_mask", v_audio),
):
mask = ctx.get(name)
if mask is not None:
velocity.mul_(mask.to(device=velocity.device, dtype=velocity.dtype))
scale = float(ctx.get("audio_scale", 1.0))
if scale != 1.0:
audio_src = ctx["audio_src"]
@@ -183,12 +196,6 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
v_audio = (1.0 - scale) * (audio_src * carry) + (
1.0 + (scale - 1.0) * sigma_a
).to(v_audio.dtype) * v_audio
slope_a = time_shift_slope(
ctx["sigma_v"],
ctx.get("shift_v", DEFAULT_SIGMA_SHIFT_VIDEO),
ctx.get("shift_a", DEFAULT_SIGMA_SHIFT_AUDIO),
).to(device=v_audio.device, dtype=v_audio.dtype)
v_audio = slope_a * v_audio
return [v_video, v_audio]
def fill_req(self, req, packed: PackedForward) -> None:
@@ -231,3 +238,129 @@ class MiniMaxH3Executor(SGLDiffusionExecutor):
if enable_cache_dit is not None:
kwargs["enable_cache_dit"] = bool(enable_cache_dit)
return kwargs
class FastH3Adapter(MiniMaxH3Adapter):
model_types = ("fast_h3",)
pipeline_class_name = "FastH3Pipeline"
class VDNH3Adapter(MiniMaxH3Adapter):
model_types = ("vdn_h3",)
pipeline_class_name = "VDNH3Pipeline"
# FastVideo/FastVideo-FastH3-8-Step-V2 model_index.json._minimax_h3; the node
# cannot see the loaded model, so FastH3Executor re-derives the grid from it.
_FASTH3_DMD_STEPS = (999, 874, 749, 624, 500, 375, 250, 125)
_FASTH3_VIDEO_SIGMA_SHIFT = 10.0
def distilled_h3_sigmas(
model_type: str,
*,
dmd_steps: tuple[int, ...] = _FASTH3_DMD_STEPS,
video_shift: float = _FASTH3_VIDEO_SIGMA_SHIFT,
) -> list[float]:
if model_type == "fast_h3":
return minimax_h3_time_shift_sigmas(
num_steps=len(dmd_steps), shift_scale=video_shift, dmd_steps=dmd_steps
)
if model_type == "vdn_h3":
return minimax_h3_time_shift_sigmas(
num_steps=VDNH3SamplingParams.num_inference_steps,
shift_scale=DEFAULT_SIGMA_SHIFT_VIDEO,
)
raise ValueError("Distilled sigma grid requires fast_h3 or vdn_h3")
def load_fasth3_release(model_path: str) -> MiniMaxH3ReleaseMetadata:
with open(os.path.join(model_path, "model_index.json")) as f:
release = MiniMaxH3ReleaseMetadata.from_model_index(json.load(f))
if release.dmd_denoising_steps is None:
raise ValueError(
"fast_h3 runtime_model_path has no trained DMD rungs "
"(model_index.json._minimax_h3.dmd_denoising_steps); use "
"FastVideo/FastVideo-FastH3-8-Step-V2"
)
index_path = os.path.join(
model_path, "transformer", "diffusion_pytorch_model.safetensors.index.json"
)
if os.path.exists(index_path):
with open(index_path) as f:
weight_map = json.load(f)["weight_map"]
# The Diffusers export drops this derivable buffer; the model overlay adds it.
if "rope.inv_freq" not in weight_map:
raise ValueError(
"fast_h3 runtime_model_path is a raw FastH3 download; point it at the "
"materialized copy (run `sglang generate --model-path "
"FastVideo/FastVideo-FastH3-8-Step-V2` once, then use "
"$SGLANG_DIFFUSION_CACHE_ROOT/materialized_models/...)"
)
return release
def validate_distilled_h3_step(packed, model_type: str, expected: list[float]):
sigmas = packed.extra_req.get("h3_sample_sigmas")
actual = torch.as_tensor(sigmas if sigmas is not None else []).float().cpu()
expected_t = torch.tensor(expected)
if actual.numel() != expected_t.numel() or not torch.allclose(
actual.reshape(-1), expected_t, atol=1e-6, rtol=0
):
raise ValueError(
f"{model_type} requires its trained {expected_t.numel() - 1}-NFE sigma "
"grid; use SGLDH3DistilledSigmas"
)
payload = packed.extra_req.get("h3_payload") or {}
if payload.get("refs") or (model_type == "fast_h3" and payload.get("keyframes")):
raise ValueError(f"{model_type} does not support this reference/keyframe task")
class _DistilledH3Executor(MiniMaxH3Executor):
expected_sigmas: list[float]
def _execute_packed(self, packed, x, timestep):
validate_distilled_h3_step(
packed, self.adapter.model_types[0], self.expected_sigmas
)
return super()._execute_packed(packed, x, timestep)
def _sampling_params_kwargs(self, packed, timestep):
kwargs = super()._sampling_params_kwargs(packed, timestep)
kwargs["num_inference_steps"] = len(self.expected_sigmas) - 1
return kwargs
class FastH3Executor(_DistilledH3Executor):
adapter_cls = FastH3Adapter
def __init__(self, generator, model_path, model, config):
super().__init__(generator, model_path, model, config)
self.release = load_fasth3_release(model_path)
self.expected_sigmas = distilled_h3_sigmas(
"fast_h3",
dmd_steps=self.release.dmd_denoising_steps,
video_shift=self.release.video_sigma_shift,
)
def forward(self, x, timestep, context, **kwargs):
# The audio sigma is derived from the video sigma through these shifts on
# both sides of the IPC; a missing MiniMaxH3SigmaShift must not mean 12/3.
opts = dict(kwargs.get("transformer_options") or {})
for key, trained in (
("minimax_h3_sigma_shift_video", self.release.video_sigma_shift),
("minimax_h3_sigma_shift_audio", self.release.audio_sigma_shift),
):
if opts.setdefault(key, trained) != trained:
raise ValueError(
f"FastH3 was distilled with sigma shift video/audio "
f"{self.release.video_sigma_shift:g}/{self.release.audio_sigma_shift:g}; "
f"got {key}={float(opts[key]):g}"
)
kwargs["transformer_options"] = opts
return super().forward(x, timestep, context, **kwargs)
class VDNH3Executor(_DistilledH3Executor):
adapter_cls = VDNH3Adapter
expected_sigmas = distilled_h3_sigmas("vdn_h3")
@@ -28,6 +28,23 @@ from .utils import (
)
class SGLDH3DistilledSigmas:
"""The checkpoint's trained rectified-flow grid, independent of scheduler presets."""
@classmethod
def INPUT_TYPES(cls):
return {"required": {"model_type": (["fast_h3", "vdn_h3"],)}}
RETURN_TYPES = ("SIGMAS",)
FUNCTION = "build"
CATEGORY = "SGLDiffusion"
def build(self, model_type):
from .executors.minimax_h3 import distilled_h3_sigmas
return (torch.tensor(distilled_h3_sigmas(model_type)),)
class SGLDOptions:
@classmethod
def INPUT_TYPES(cls):
@@ -42,6 +59,8 @@ class SGLDOptions:
"flux",
"lumina2",
"minimax_h3",
"fast_h3",
"vdn_h3",
],
{"default": "auto-detect"},
),
@@ -90,6 +109,8 @@ class SGLDOptions:
"STRING",
{"default": ""},
),
"runtime_model_path": ("STRING", {"default": ""}),
"attention_backend_config": ("STRING", {"default": ""}),
"transformer_weights_path": (
"STRING",
{
@@ -122,6 +143,8 @@ class SGLDOptions:
enable_cache_dit: bool = False,
quantization: str = "",
transformer_weights_path: str = "",
runtime_model_path: str = "",
attention_backend_config: str = "",
):
"""
Build a dictionary of SGLang Diffusion runtime options.
@@ -160,6 +183,14 @@ class SGLDOptions:
# local .gguf, owner/repo/path/file.gguf, or owner/repo:QUANT.
options["transformer_weights_path"] = transformer_weights_path
for key, value in (
("runtime_model_path", runtime_model_path),
# Same format as --attention-backend-config; ServerArgs parses it.
("attention_backend_config", attention_backend_config),
):
if value.strip():
options[key] = value.strip()
# Strip None to keep payload clean
options = {k: v for k, v in options.items() if v is not None}
return (options,)
@@ -972,6 +1003,7 @@ class SGLDiffusionServerUnsetLora:
# Register nodes
NODE_CLASS_MAPPINGS = {
"SGLDH3DistilledSigmas": SGLDH3DistilledSigmas,
"SGLDiffusionServerModel": SGLDiffusionServerModel,
"SGLDiffusionGenerateImage": SGLDiffusionGenerateImage,
"SGLDiffusionGenerateVideo": SGLDiffusionGenerateVideo,
@@ -10,7 +10,7 @@ from __future__ import annotations
import logging
from dataclasses import dataclass
from typing import Any
from typing import Any, Callable
import torch
@@ -223,7 +223,29 @@ def comfyui_layout_to_packed(layout: dict[str, Any]) -> dict[str, Any]:
token_tags[start:stop] = _SEG_TOKEN_TAG.get(kind, -1)
if kind == "text":
text_pos = torch.arange(start, stop, dtype=torch.long)
# Cube-sparse-attention segment shapes, as packed_sequence exports them.
cond_shapes, cond_roles, events, audio_lens = [], [], [], []
for start, stop, kind in layout["segments"]:
if kind in ("cond", "ref_img"):
coords = position_ids[start:stop]
cond_shapes.append(
tuple(int(torch.unique(coords[:, axis]).numel()) for axis in range(3))
)
cond_roles.append("joint_cube" if kind == "cond" else "independent_cube")
events.append(("imgvid", len(cond_shapes) - 1))
elif kind in ("cond_audio", "ref_audio"):
audio_lens.append(stop - start)
events.append(("audio", len(audio_lens) - 1))
_, latent_t, latent_h, latent_w, *_ = layout["signature"]
stream_layout = {
"target_shape": (latent_t, (latent_h + 1) // 2, (latent_w + 1) // 2),
"cond_image_shapes": tuple(cond_shapes),
"cond_image_roles": tuple(cond_roles),
"cond_event_orders": tuple(events),
"cond_audio_stream_lens": tuple(audio_lens),
}
return {
"stream_layout": stream_layout,
"seq_len": seq_len,
"img_pos": layout["img_pos"].view(-1).to(torch.long),
"audio_pos": layout["audio_pos"].view(-1).to(torch.long),
@@ -526,6 +548,7 @@ def comfyui_payload_to_branch_inputs(
"used": used,
"orig_video_shape": orig_shape,
"padded_video_shape": tuple(video.shape),
"is_ref2va": bool(refs),
"sample_sigmas": sample_sigmas,
"sigma_v": float(sigma_v),
"shift_v": shift_v,
@@ -608,6 +631,53 @@ def _sample_sigmas_len(sample_sigmas: Any) -> int | None:
return None
def _step_attn_metadata(
model: Any,
inputs: dict[str, Any],
sample_sigmas: Any,
server_args: ServerArgs,
device: torch.device,
) -> Callable[[int], Any] | None:
"""Sparse-attention metadata per step, built as the native H3 loop builds it."""
from sglang.multimodal_gen.runtime.models.dits.minimax_h3_vdn_attention import (
prepare_hybrid_attention_metadata,
)
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
_build_cube_attn_metadata,
_maybe_prepare_vsa_h3_step_metadata,
)
packed = inputs["packed"]
latent_shape = tuple(inputs["padded_video_shape"][-3:])
cube = _build_cube_attn_metadata(
server_args,
packed=packed,
num_steps=max((_sample_sigmas_len(sample_sigmas) or 2) - 1, 1),
device=device,
)
if cube is not None:
return lambda _step: cube
# VSA-H3 tiles assume the native row order (text, cond, audio, raster video);
# test_vsa_h3_tiles_cover_comfyui_packed_rows pins ComfyUI layouts to it.
build = _maybe_prepare_vsa_h3_step_metadata(
model=model,
packed=packed,
is_ref2va=inputs["is_ref2va"],
latent_shape=latent_shape,
server_args=server_args,
device=device,
)
if build is not None:
return build
return prepare_hybrid_attention_metadata(
model=model,
packed=packed,
latent_shape=latent_shape,
server_args=server_args,
device=device,
)
@dataclass
class MiniMaxH3ComfyUIRunState:
branch: MiniMaxH3DenoiseBranch
@@ -615,6 +685,7 @@ class MiniMaxH3ComfyUIRunState:
orig_video_shape: tuple[int, ...]
padded_video_shape: tuple[int, ...]
sample_sigmas: Any = None
step_attn_metadata: Callable[[int], Any] | None = None
cache_mounted: bool = False
refined: bool = False
steps_done: int = 0
@@ -628,6 +699,14 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
self._cache_stage = None
def verify_input(self, batch: Req, server_args: ServerArgs):
if (
server_args.enable_breakable_cuda_graph
and server_args.is_dit_layerwise_offload_selected
):
raise ValueError(
"ComfyUI H3 breakable CUDA graph is incompatible with layerwise "
"offload: the offload copy-stream events invalidate graph capture"
)
bind_comfyui_session(batch)
return super().verify_input(batch, server_args)
@@ -648,7 +727,9 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
n_sigmas = _sample_sigmas_len(state.sample_sigmas)
if n_sigmas is None or n_sigmas < 2:
return
self._cache_dit_stage()._maybe_enable_cache_dit(n_sigmas - 1, batch)
self._cache_dit_stage()._maybe_enable_cache_dit_and_torch_compile(
n_sigmas - 1, batch
)
state.cache_mounted = True
def _release_run_state(self, state: MiniMaxH3ComfyUIRunState | None) -> None:
@@ -686,6 +767,7 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
inputs: dict[str, Any],
sample_sigmas: Any,
device: torch.device,
server_args: ServerArgs,
) -> MiniMaxH3ComfyUIRunState:
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
_precompute_refined_prompt_embeds,
@@ -715,9 +797,15 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
text_embeddings=inputs["text_embeddings"],
token_tags=inputs["token_tags"],
device=device,
video_query_indices=inputs["packed"]["img_pos"][
inputs["packed"]["update_mask"]
],
)
return MiniMaxH3ComfyUIRunState(
branch=branch,
step_attn_metadata=_step_attn_metadata(
self.transformer, inputs, sample_sigmas, server_args, device
),
used=inputs["used"],
orig_video_shape=inputs["orig_video_shape"],
padded_video_shape=inputs["padded_video_shape"],
@@ -737,6 +825,34 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
state.refined = True
return state
def _run_dit(
self, batch: Req, fk: dict[str, Any], state: MiniMaxH3ComfyUIRunState
) -> tuple[torch.Tensor, torch.Tensor]:
from sglang.multimodal_gen.runtime.managers.forward_context import (
set_forward_context,
)
stage = self._cache_dit_stage()
step = state.steps_done
attn_metadata = (
state.step_attn_metadata(step)
if state.step_attn_metadata is not None
else None
)
runner = stage._maybe_get_bcg_runner(self.transformer)
if runner is not None:
# Integrated mode never warms up, so capture here; it is a no-op
# once this signature has a graph.
with set_forward_context(
current_timestep=step, attn_metadata=attn_metadata, forward_batch=batch
):
runner.capture(
**stage._bcg_pad_prompt_kwargs(fk, current_model=self.transformer)
)
return stage._forward_dit(
self.transformer, fk, step, batch=batch, attn_metadata=attn_metadata
)
def forward(self, batch: Req, server_args: ServerArgs) -> Req:
bind_comfyui_session(batch)
extra = getattr(batch, "extra", None) or {}
@@ -764,12 +880,14 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage):
denoise_mask=extra.get("h3_denoise_mask"),
audio_denoise_mask=extra.get("h3_audio_denoise_mask"),
)
state = self._bind_run_state(batch, inputs, sample_sigmas, video_x.device)
state = self._bind_run_state(
batch, inputs, sample_sigmas, video_x.device, server_args
)
self._maybe_mount_cache_dit(batch, state)
fk, _branch = build_step_forward_kwargs(
inputs, branch=state.branch, device=video_x.device
)
v_video, v_audio = self.transformer(**fk)
v_video, v_audio = self._run_dit(batch, fk, state)
batch.noise_pred = pack_comfy_output(v_video, v_audio, state)
state.steps_done += 1
n_sigmas = _sample_sigmas_len(sample_sigmas)
@@ -747,7 +747,8 @@ class MiniMaxH3DenoisingStage(DenoisingStage):
build_vsa_h3_step_metadata = _maybe_prepare_vsa_h3_step_metadata(
model=model,
packed=packed,
ctx=ctx,
is_ref2va=ctx.is_ref2va,
latent_shape=(ctx.latent_t, ctx.latent_h, ctx.latent_w),
server_args=server_args,
device=device,
)
@@ -1049,7 +1050,8 @@ def _maybe_prepare_vsa_h3_step_metadata(
*,
model: Any,
packed: Mapping[str, torch.Tensor],
ctx: _FullLoopContext,
is_ref2va: bool,
latent_shape: tuple[int, int, int],
server_args: ServerArgs,
device: torch.device,
) -> Callable[[int], Any] | None:
@@ -1061,7 +1063,7 @@ def _maybe_prepare_vsa_h3_step_metadata(
is not AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3
):
return None
if ctx.is_ref2va:
if is_ref2va:
raise NotImplementedError(
"VSA-H3 supports the t2va/fl2va packed layout; the ref2va "
"reference-block layout is not tiled yet. Use --attention-backend "
@@ -1102,7 +1104,7 @@ def _maybe_prepare_vsa_h3_step_metadata(
def build(step_index: int):
return builder.build(
current_timestep=step_index,
raw_latent_shape=(ctx.latent_t, ctx.latent_h, ctx.latent_w),
raw_latent_shape=latent_shape,
patch_size=patch_size,
VSA_sparsity=sparsity,
prefix_segments=(text_len, cond_rows, audio_rows),
@@ -260,3 +260,109 @@ def test_multi_gpu_single_file_starts_without_cfg_parallel(
str(path), "FluxPipeline", {"num_gpus": 2, "enable_cfg_parallel": True}
)
assert "cfg_parallel_degree" not in seen
def test_spawned_workers_do_not_reexecute_launcher_main(monkeypatch) -> None:
"""Workers must not re-run ComfyUI's main.py; its imports break under spawn."""
import multiprocessing.spawn
import sys
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import (
_spawn_without_launcher_main,
)
main_dict = vars(sys.modules["__main__"])
monkeypatch.setitem(main_dict, "__file__", "/comfy/main.py")
with _spawn_without_launcher_main():
data = multiprocessing.spawn.get_preparation_data("worker")
assert "init_main_from_path" not in data
assert "init_main_from_name" not in data
assert main_dict["__file__"] == "/comfy/main.py"
def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator:
runtime = SGLDiffusionGenerator()
runtime.get_comfyui_model = lambda *a: (SimpleNamespace(), None, "minimax_h3")
runtime.init_generator = Mock(side_effect=AssertionError("worker must not start"))
return runtime
@pytest.mark.parametrize(
"key, sgld_options, error",
[
# A FastH3 file loaded as base H3 died in the worker on to_gate_compress.
("blocks.0.attn.to_gate_compress.weight", {}, "model_type fast_h3"),
# The H3 file loader builds no quant config, so this was silently ignored.
(
"blocks.0.attn.to_q.weight",
{"quantization": "convrot_int8"},
"cannot be quantized at load time",
),
],
)
def test_unservable_h3_file_is_rejected_before_worker_load(
tmp_path, key, sgld_options, error
) -> None:
from safetensors.torch import save_file
path = tmp_path / "h3.safetensors"
save_file({key: torch.zeros(1)}, path)
with pytest.raises(ValueError, match=error):
_h3_runtime_that_must_not_start().load_model(
model_path=str(path), sgld_options=sgld_options
)
def test_fasth3_runtime_dir_is_checked_before_worker_load(tmp_path) -> None:
"""The 4-step preview and a raw (unmaterialized) FastH3 V2 download were
only rejected by the worker after a full model load."""
import json
release = {
"schema_version": 1,
"partition": "fl2va",
"tasks": ["t2va"],
"task_aliases": {},
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
}
(tmp_path / "transformer").mkdir()
index = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors.index.json"
runtime = _h3_runtime_that_must_not_start()
options = {"model_type": "fast_h3", "runtime_model_path": str(tmp_path)}
for dmd, weight_map, message in (
(None, {}, "no trained DMD rungs"),
([999, 500], {"blocks.0.x": "a.safetensors"}, "raw FastH3 download"),
):
meta = dict(release, **({"dmd_denoising_steps": dmd} if dmd else {}))
(tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": meta}))
index.write_text(json.dumps({"weight_map": weight_map}))
with pytest.raises(ValueError, match=message):
runtime.load_model(model_path="h3.safetensors", sgld_options=options)
@pytest.mark.parametrize(
"name, pipeline, config_cls",
[
("h3_local.safetensors", "MiniMaxH3Pipeline", "MiniMaxH3PipelineConfig"),
("VDN-H3-8nfe", "VDNH3Pipeline", "VDNH3PipelineConfig"),
],
)
def test_renamed_h3_paths_resolve_through_the_plugin_model_id(
tmp_path, monkeypatch, name, pipeline, config_cls
) -> None:
"""A local H3 file or VDN copy whose path lacks the HF name matched no
registry detector or model-id keyed default."""
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator
from sglang.multimodal_gen.registry import _get_config_info
seen = {}
monkeypatch.setattr(
generator.DiffGenerator,
"from_pretrained",
lambda **kwargs: seen.update(kwargs) or object(),
)
path = str(tmp_path / name)
SGLDiffusionGenerator().init_generator(path, pipeline, {})
assert seen["warmup_mode"] == "off"
info = _get_config_info(path, model_id=seen["model_id"])
assert info.pipeline_config_cls.__name__ == config_cls
@@ -318,6 +318,59 @@ def test_fl2va_first_last_used_prefix_matches_comfyui():
]
@pytest.mark.parametrize("keyframes", [False, True])
def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes):
"""Integrated steps hand ComfyUI's own layout to the VSA-H3 tile builder,
which assumes text, cond and audio prefixes followed by the raster video
rows; a reordered layout would mis-tile attention silently."""
from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTConfig
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import (
_maybe_prepare_vsa_h3_step_metadata,
)
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
text_len, latent_t, latent_h, latent_w, audio_t = 7, 3, 8, 12, 5
layout = ComfyUIPackedLayout(
text_len,
latent_t,
latent_h,
latent_w,
audio_t,
keyframes=(
[{"resolved_frame_index": 0, "latent": torch.zeros(1, 24, 1, 8, 12)}]
if keyframes
else None
),
)
packed = comfyui_layout_to_packed(serialize_comfyui_layout(layout))
model = SimpleNamespace(
_resolve_attention_backend_once=lambda: None,
_resolved_attention_backend=AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3,
)
server_args = SimpleNamespace(
attention_backend_config={},
pipeline_config=SimpleNamespace(
vsa_sparsity=0.8, dit_config=MiniMaxH3DiTConfig()
),
)
build = _maybe_prepare_vsa_h3_step_metadata(
model=model,
packed=packed,
is_ref2va=False,
latent_shape=(latent_t, latent_h, latent_w),
server_args=server_args,
device=torch.device("cpu"),
)
used = int(packed["cu_seqlens"][1])
assert build(0).total_seq_length == used
video = packed["img_pos"][packed["update_mask"].bool()]
video_start = used - video.numel()
assert torch.equal(video, torch.arange(video_start, used))
audio = packed["audio_pos"]
assert torch.equal(audio, torch.arange(video_start - audio.numel(), video_start))
assert torch.equal(packed["text_pos"], torch.arange(text_len))
def test_ref2va_image_plus_video_audio_used_prefix_matches_comfyui():
kwargs = dict(text_len=5, latent_t=2, latent_h=4, latent_w=4, audio_t=5)
refs = [
@@ -454,8 +507,11 @@ from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
SGLDiffusionExecutor,
)
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.minimax_h3 import (
FastH3Executor,
MiniMaxH3Adapter,
distilled_h3_sigmas,
drop_h3_pinned_sampling_fields,
validate_distilled_h3_step,
worker_transformer_options,
)
from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams
@@ -1259,3 +1315,132 @@ def test_h3_pack_rejects_batched_latents() -> None:
minimax_payload={"audio_scale": 1.0},
transformer_options={"sample_sigmas": torch.tensor([1.0, 0.0])},
)
def test_comfyui_sparse_stream_layout_preserves_target_and_condition_roles():
keyframe = {"resolved_frame_index": 0, "latent": torch.zeros(1, 24, 1, 8, 12)}
layout = ComfyUIPackedLayout(3, 2, 8, 12, 4, keyframes=[keyframe])
stream = comfyui_layout_to_packed(serialize_comfyui_layout(layout))["stream_layout"]
assert stream["target_shape"] == (2, 4, 6)
assert stream["cond_image_shapes"] == ((1, 4, 6),)
assert stream["cond_image_roles"] == ("joint_cube",)
assert stream["cond_event_orders"] == (("imgvid", 0),)
@pytest.mark.parametrize("model_type", ["fast_h3", "vdn_h3"])
def test_distilled_h3_rejects_wrong_grid_and_reference_task(model_type):
expected = distilled_h3_sigmas(model_type)
sigmas = torch.tensor(expected)
packed = SimpleNamespace(extra_req={"h3_sample_sigmas": sigmas})
validate_distilled_h3_step(packed, model_type, expected)
packed.extra_req["h3_sample_sigmas"] = torch.linspace(1, 0, len(expected))
with pytest.raises(ValueError, match="trained"):
validate_distilled_h3_step(packed, model_type, expected)
packed.extra_req.update(
h3_sample_sigmas=sigmas, h3_payload={"refs": [{"kind": "image"}]}
)
with pytest.raises(ValueError, match="reference"):
validate_distilled_h3_step(packed, model_type, expected)
def test_fasth3_comfy_grid_matches_native_8_step_v2_contract():
"""The plugin pinned fast_h3 to the 4-NFE preview grid (shift 12) after the
runtime moved to FastH3 8-Step V2, so V2 could not run in ComfyUI."""
from sglang.multimodal_gen.configs.sample.minimax_h3 import FastH3SamplingParams
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import (
minimax_h3_time_shift_sigmas,
)
# FastVideo/FastVideo-FastH3-8-Step-V2 model_index.json._minimax_h3
rungs = (999, 874, 749, 624, 500, 375, 250, 125)
native = minimax_h3_time_shift_sigmas(
num_steps=8, shift_scale=10.0, dmd_steps=rungs
)
assert distilled_h3_sigmas("fast_h3") == native
assert len(native) == FastH3SamplingParams().num_inference_steps + 1
def test_fasth3_executor_uses_trained_shifts_and_rejects_conflicts(tmp_path):
"""Without a MiniMaxH3SigmaShift node the audio sigma was derived with the
base 12/3 shifts instead of FastH3's trained 10/3."""
import json
release = {
"schema_version": 1,
"partition": "fl2va",
"tasks": ["t2va"],
"task_aliases": {},
"sigma_shift_scales": {"video": 10.0, "audio": 3.0},
"dmd_denoising_steps": [999, 874, 749, 624, 500, 375, 250, 125],
}
(tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": release}))
sent = []
class Recording(FastH3Executor):
def _execute_packed(self, packed, x, timestep):
sent.append(packed)
return x
config = SimpleNamespace(unet_config={"dtype": torch.bfloat16})
ex = Recording(None, str(tmp_path), None, config)
x = [torch.ones(1, 24, 2, 4, 4), torch.ones(1, 32, 2, 3)]
step = lambda **opts: ex(
x,
torch.tensor([500.0]),
torch.ones(1, 8, 16),
transformer_options={"sample_sigmas": torch.tensor(ex.expected_sigmas), **opts},
minimax_payload={"audio_scale": 1.0},
)
step()
opts = sent[0].extra_req["h3_transformer_options"]
assert opts["minimax_h3_sigma_shift_video"] == 10.0
assert opts["minimax_h3_sigma_shift_audio"] == 3.0
with pytest.raises(ValueError, match="distilled with sigma shift video/audio 10/3"):
step(minimax_h3_sigma_shift_video=12.0)
@pytest.mark.parametrize(
"masks, video_out, audio_out",
[
# ComfyUI forward: (1 - scale) * input + scale * (-raw velocity). At
# sigma_v=sigma_a=1 this is -3 - 4=-7; a second derivative gave -28.
({}, -1, -7),
# Masks apply to the raw velocities, before the carried-audio transform.
(
{
"denoise_mask": torch.full((1, 1, 2, 4, 4), 0.25),
"audio_denoise_mask": torch.zeros(1, 1, 2, 3),
},
-0.25,
-3,
),
],
)
def test_h3_velocities_match_comfyui_forward(masks, video_out, audio_out):
adapter = MiniMaxH3Adapter()
video = torch.ones(1, 24, 2, 4, 4)
audio = torch.ones(1, 32, 2, 3)
packed = adapter.pack(
[video, audio],
torch.tensor([1000.0]),
torch.ones(1, 4, 8),
minimax_payload={"audio_scale": 4.0},
**masks,
)
out = adapter.unpack(
[torch.ones_like(video), torch.ones_like(audio)], packed, [video, audio]
)
assert torch.equal(out[0], torch.full_like(video, video_out))
assert torch.equal(out[1], torch.full_like(audio, audio_out))
def test_comfyui_h3_rejects_bcg_with_offload_before_cuda_capture():
from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.comfyui_step import (
MiniMaxH3ComfyUIStepStage,
)
args = SimpleNamespace(
enable_breakable_cuda_graph=True, is_dit_layerwise_offload_selected=True
)
with pytest.raises(ValueError, match="copy-stream events"):
MiniMaxH3ComfyUIStepStage.verify_input(None, SimpleNamespace(), args)