diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py index 91ebbf6454..e8e761a61c 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py @@ -3,11 +3,18 @@ Generator for SGLang Diffusion ComfyUI integration. """ import atexit +import contextlib import logging import os +import sys from ..executors.flux import FluxExecutor -from ..executors.minimax_h3 import MiniMaxH3Executor +from ..executors.minimax_h3 import ( + FastH3Executor, + MiniMaxH3Executor, + VDNH3Executor, + load_fasth3_release, +) from ..executors.zimage import ZImageExecutor from .preflight import check_sgld_options @@ -31,6 +38,59 @@ class _HeaderTensor: nelement = numel +@contextlib.contextmanager +def _spawn_without_launcher_main(): + """spawn re-runs ``__main__`` in every child; ComfyUI's main.py then resolves + ``import utils`` to comfy/utils.py and the worker dies before reporting ready.""" + main_dict = vars(sys.modules["__main__"]) + saved = { + key: main_dict[key] for key in ("__file__", "__spec__") if key in main_dict + } + main_dict.pop("__file__", None) + main_dict["__spec__"] = None + try: + yield + finally: + main_dict.pop("__spec__", None) + main_dict.update(saved) + + +def _check_h3_single_file(path: str, sgld_options: dict) -> None: + """Reject options the ComfyUI H3 checkpoint loader would silently ignore.""" + if sgld_options.get("minimax_h3_adaln_online") or sgld_options.get( + "minimax_h3_adaln_cache_path" + ): + raise ValueError( + "ComfyUI H3 checkpoint loading does not support the native AdaLN " + "online/sidecar caches" + ) + if not path.endswith(".safetensors"): + return + if sgld_options.get("quantization"): + raise ValueError( + "ComfyUI H3 files cannot be quantized at load time; load a pre-quantized " + "file (e.g. *_int8_convrot), whose per-layer metadata selects the " + "format, and leave quantization empty" + ) + from safetensors import safe_open + + with safe_open(path, framework="pt") as f: + if any(".to_gate_compress." in key for key in f.keys()): + raise ValueError( + "This is a FastH3 checkpoint (VSA gate weights), which the base " + "MiniMax H3 model cannot load; set model_type fast_h3 with " + "runtime_model_path pointing at the native FastH3 model" + ) + + +# Renamed local files and directories miss the registry's path detectors and +# model-id keyed defaults (e.g. BCG support); these ids always match. +_H3_MODEL_IDS = { + "MiniMaxH3Pipeline": "MiniMax-H3", + "VDNH3Pipeline": "vdn-minimax-h3", +} + + def _looks_like_gguf(path: str) -> bool: if not path: return False @@ -116,7 +176,13 @@ else: def _load_executor_classes(): """Qwen adapters import ComfyUI. Keep them optional so CI can load the rest.""" - classes = [FluxExecutor, ZImageExecutor, MiniMaxH3Executor] + classes = [ + FluxExecutor, + ZImageExecutor, + MiniMaxH3Executor, + FastH3Executor, + VDNH3Executor, + ] try: from ..executors.qwen_image import QwenImageEditExecutor, QwenImageExecutor except ModuleNotFoundError as exc: @@ -210,13 +276,19 @@ class SGLDiffusionGenerator: # which a non-H3 single file lacks, and fails worker startup on >= 2 GPUs. if not kwargs.get("enable_cfg_parallel"): kwargs.setdefault("cfg_parallel_degree", 1) + # Each sampler step is its own request carrying ComfyUI tensors; a + # synthetic or request-based warmup has no such inputs to run with. + kwargs["warmup_mode"] = "off" + if pipeline_class_name in _H3_MODEL_IDS: + kwargs.setdefault("model_id", _H3_MODEL_IDS[pipeline_class_name]) kwargs = self._server_args_kwargs(kwargs) try: - self.generator = DiffGenerator.from_pretrained( - model_path=model_path, - pipeline_class_name=pipeline_class_name, - **kwargs, - ) + with _spawn_without_launcher_main(): + self.generator = DiffGenerator.from_pretrained( + model_path=model_path, + pipeline_class_name=pipeline_class_name, + **kwargs, + ) except EOFError as error: # A worker that raises while loading closes its pipe without a message. raise RuntimeError( @@ -426,6 +498,19 @@ class SGLDiffusionGenerator: if "enable_cache_dit" in sgld_options: plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit") set_model_type = sgld_options.pop("model_type", None) + runtime_model_path = sgld_options.pop("runtime_model_path", None) + if bool(runtime_model_path) != (set_model_type in ("fast_h3", "vdn_h3")): + # The base H3 file is only the ComfyUI architecture companion. + raise ValueError( + "model_type fast_h3 / vdn_h3 requires runtime_model_path (the native " + "model directory), and runtime_model_path is only used by them" + ) + if runtime_model_path: + runtime_model_path = os.path.abspath(os.path.expanduser(runtime_model_path)) + if not os.path.isdir(runtime_model_path): + raise ValueError( + "runtime_model_path must be a local materialized native model directory" + ) override = (sgld_options.get("transformer_weights_path") or "").strip() if override: sgld_options["transformer_weights_path"] = override @@ -441,6 +526,7 @@ class SGLDiffusionGenerator: "model_options": model_options, "sgld_options": sgld_options, "set_model_type": set_model_type, + "runtime_model_path": runtime_model_path, } if self._can_reuse(gather_options): self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit") @@ -458,14 +544,20 @@ class SGLDiffusionGenerator: if set_model_type is not None and set_model_type in self.pipeline_class_dict: model_type = set_model_type + if model_type == "fast_h3": + load_fasth3_release(runtime_model_path) + if model_type == "minimax_h3": + _check_h3_single_file(detect_path, sgld_options) + pipeline_class_name = self.pipeline_class_dict[model_type] + worker_model_path = runtime_model_path or detect_path self.generator = self.init_generator( - detect_path, pipeline_class_name, sgld_options + worker_model_path, pipeline_class_name, sgld_options ) executor_class = self.executor_class_dict[model_type] self.executor = executor_class( - self.generator, detect_path, comfyui_model, model_config + self.generator, worker_model_path, comfyui_model, model_config ) self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit") self.executor._sgld_reload = reload_kwargs diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py index 1d039ca389..d92c086bf3 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py @@ -3,18 +3,26 @@ from __future__ import annotations +import json +import os from dataclasses import fields from typing import Any import torch from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams +from sglang.multimodal_gen.configs.sample.minimax_h3_vdn import VDNH3SamplingParams +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.release_metadata import ( + MiniMaxH3ReleaseMetadata, +) from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.comfyui_step import ( DEFAULT_SIGMA_SHIFT_AUDIO, DEFAULT_SIGMA_SHIFT_VIDEO, serialize_comfyui_layout, time_shift_sigma, - time_shift_slope, +) +from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import ( + minimax_h3_time_shift_sigmas, ) from .adapter import ComfyUIModelAdapter, PackedForward @@ -150,13 +158,12 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter): width=int(video.shape[-1]), extra_req=extra_req, unpack_ctx={ + "video_denoise_mask": kwargs.get("denoise_mask"), + "audio_denoise_mask": kwargs.get("audio_denoise_mask"), "audio_scale": scale, "audio_src": audio_src, "carry": carry, - "sigma_v": sigma_v, "sigma_a": sigma_a, - "shift_v": shift_v, - "shift_a": shift_a, }, ) @@ -169,12 +176,18 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter): "MiniMax H3 unpack expects [video, audio] noise_pred, " f"got {type(noise_pred)!r}" ) - # Match ComfyUI MiniMaxH3._forward: negated velocities, and audio - # scaled by d(sigma_a)/d(sigma_v) so the sampler's sigma_v ODE is - # the audio stream's true ODE on its shifted schedule. + # Match ComfyUI MiniMaxH3._forward: negated velocities, masked before the + # carried-audio transform, which already holds d(sigma_a)/d(sigma_v). v_video = (-v_video).to(device=video_x.device, dtype=video_x.dtype) v_audio = (-v_audio).to(device=audio_x.device, dtype=audio_x.dtype) ctx = packed.unpack_ctx + for name, velocity in ( + ("video_denoise_mask", v_video), + ("audio_denoise_mask", v_audio), + ): + mask = ctx.get(name) + if mask is not None: + velocity.mul_(mask.to(device=velocity.device, dtype=velocity.dtype)) scale = float(ctx.get("audio_scale", 1.0)) if scale != 1.0: audio_src = ctx["audio_src"] @@ -183,12 +196,6 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter): v_audio = (1.0 - scale) * (audio_src * carry) + ( 1.0 + (scale - 1.0) * sigma_a ).to(v_audio.dtype) * v_audio - slope_a = time_shift_slope( - ctx["sigma_v"], - ctx.get("shift_v", DEFAULT_SIGMA_SHIFT_VIDEO), - ctx.get("shift_a", DEFAULT_SIGMA_SHIFT_AUDIO), - ).to(device=v_audio.device, dtype=v_audio.dtype) - v_audio = slope_a * v_audio return [v_video, v_audio] def fill_req(self, req, packed: PackedForward) -> None: @@ -231,3 +238,129 @@ class MiniMaxH3Executor(SGLDiffusionExecutor): if enable_cache_dit is not None: kwargs["enable_cache_dit"] = bool(enable_cache_dit) return kwargs + + +class FastH3Adapter(MiniMaxH3Adapter): + model_types = ("fast_h3",) + pipeline_class_name = "FastH3Pipeline" + + +class VDNH3Adapter(MiniMaxH3Adapter): + model_types = ("vdn_h3",) + pipeline_class_name = "VDNH3Pipeline" + + +# FastVideo/FastVideo-FastH3-8-Step-V2 model_index.json._minimax_h3; the node +# cannot see the loaded model, so FastH3Executor re-derives the grid from it. +_FASTH3_DMD_STEPS = (999, 874, 749, 624, 500, 375, 250, 125) +_FASTH3_VIDEO_SIGMA_SHIFT = 10.0 + + +def distilled_h3_sigmas( + model_type: str, + *, + dmd_steps: tuple[int, ...] = _FASTH3_DMD_STEPS, + video_shift: float = _FASTH3_VIDEO_SIGMA_SHIFT, +) -> list[float]: + if model_type == "fast_h3": + return minimax_h3_time_shift_sigmas( + num_steps=len(dmd_steps), shift_scale=video_shift, dmd_steps=dmd_steps + ) + if model_type == "vdn_h3": + return minimax_h3_time_shift_sigmas( + num_steps=VDNH3SamplingParams.num_inference_steps, + shift_scale=DEFAULT_SIGMA_SHIFT_VIDEO, + ) + raise ValueError("Distilled sigma grid requires fast_h3 or vdn_h3") + + +def load_fasth3_release(model_path: str) -> MiniMaxH3ReleaseMetadata: + with open(os.path.join(model_path, "model_index.json")) as f: + release = MiniMaxH3ReleaseMetadata.from_model_index(json.load(f)) + if release.dmd_denoising_steps is None: + raise ValueError( + "fast_h3 runtime_model_path has no trained DMD rungs " + "(model_index.json._minimax_h3.dmd_denoising_steps); use " + "FastVideo/FastVideo-FastH3-8-Step-V2" + ) + index_path = os.path.join( + model_path, "transformer", "diffusion_pytorch_model.safetensors.index.json" + ) + if os.path.exists(index_path): + with open(index_path) as f: + weight_map = json.load(f)["weight_map"] + # The Diffusers export drops this derivable buffer; the model overlay adds it. + if "rope.inv_freq" not in weight_map: + raise ValueError( + "fast_h3 runtime_model_path is a raw FastH3 download; point it at the " + "materialized copy (run `sglang generate --model-path " + "FastVideo/FastVideo-FastH3-8-Step-V2` once, then use " + "$SGLANG_DIFFUSION_CACHE_ROOT/materialized_models/...)" + ) + return release + + +def validate_distilled_h3_step(packed, model_type: str, expected: list[float]): + sigmas = packed.extra_req.get("h3_sample_sigmas") + actual = torch.as_tensor(sigmas if sigmas is not None else []).float().cpu() + expected_t = torch.tensor(expected) + if actual.numel() != expected_t.numel() or not torch.allclose( + actual.reshape(-1), expected_t, atol=1e-6, rtol=0 + ): + raise ValueError( + f"{model_type} requires its trained {expected_t.numel() - 1}-NFE sigma " + "grid; use SGLDH3DistilledSigmas" + ) + payload = packed.extra_req.get("h3_payload") or {} + if payload.get("refs") or (model_type == "fast_h3" and payload.get("keyframes")): + raise ValueError(f"{model_type} does not support this reference/keyframe task") + + +class _DistilledH3Executor(MiniMaxH3Executor): + expected_sigmas: list[float] + + def _execute_packed(self, packed, x, timestep): + validate_distilled_h3_step( + packed, self.adapter.model_types[0], self.expected_sigmas + ) + return super()._execute_packed(packed, x, timestep) + + def _sampling_params_kwargs(self, packed, timestep): + kwargs = super()._sampling_params_kwargs(packed, timestep) + kwargs["num_inference_steps"] = len(self.expected_sigmas) - 1 + return kwargs + + +class FastH3Executor(_DistilledH3Executor): + adapter_cls = FastH3Adapter + + def __init__(self, generator, model_path, model, config): + super().__init__(generator, model_path, model, config) + self.release = load_fasth3_release(model_path) + self.expected_sigmas = distilled_h3_sigmas( + "fast_h3", + dmd_steps=self.release.dmd_denoising_steps, + video_shift=self.release.video_sigma_shift, + ) + + def forward(self, x, timestep, context, **kwargs): + # The audio sigma is derived from the video sigma through these shifts on + # both sides of the IPC; a missing MiniMaxH3SigmaShift must not mean 12/3. + opts = dict(kwargs.get("transformer_options") or {}) + for key, trained in ( + ("minimax_h3_sigma_shift_video", self.release.video_sigma_shift), + ("minimax_h3_sigma_shift_audio", self.release.audio_sigma_shift), + ): + if opts.setdefault(key, trained) != trained: + raise ValueError( + f"FastH3 was distilled with sigma shift video/audio " + f"{self.release.video_sigma_shift:g}/{self.release.audio_sigma_shift:g}; " + f"got {key}={float(opts[key]):g}" + ) + kwargs["transformer_options"] = opts + return super().forward(x, timestep, context, **kwargs) + + +class VDNH3Executor(_DistilledH3Executor): + adapter_cls = VDNH3Adapter + expected_sigmas = distilled_h3_sigmas("vdn_h3") diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py index 47b4ef1d07..adb098f505 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py @@ -28,6 +28,23 @@ from .utils import ( ) +class SGLDH3DistilledSigmas: + """The checkpoint's trained rectified-flow grid, independent of scheduler presets.""" + + @classmethod + def INPUT_TYPES(cls): + return {"required": {"model_type": (["fast_h3", "vdn_h3"],)}} + + RETURN_TYPES = ("SIGMAS",) + FUNCTION = "build" + CATEGORY = "SGLDiffusion" + + def build(self, model_type): + from .executors.minimax_h3 import distilled_h3_sigmas + + return (torch.tensor(distilled_h3_sigmas(model_type)),) + + class SGLDOptions: @classmethod def INPUT_TYPES(cls): @@ -42,6 +59,8 @@ class SGLDOptions: "flux", "lumina2", "minimax_h3", + "fast_h3", + "vdn_h3", ], {"default": "auto-detect"}, ), @@ -90,6 +109,8 @@ class SGLDOptions: "STRING", {"default": ""}, ), + "runtime_model_path": ("STRING", {"default": ""}), + "attention_backend_config": ("STRING", {"default": ""}), "transformer_weights_path": ( "STRING", { @@ -122,6 +143,8 @@ class SGLDOptions: enable_cache_dit: bool = False, quantization: str = "", transformer_weights_path: str = "", + runtime_model_path: str = "", + attention_backend_config: str = "", ): """ Build a dictionary of SGLang Diffusion runtime options. @@ -160,6 +183,14 @@ class SGLDOptions: # local .gguf, owner/repo/path/file.gguf, or owner/repo:QUANT. options["transformer_weights_path"] = transformer_weights_path + for key, value in ( + ("runtime_model_path", runtime_model_path), + # Same format as --attention-backend-config; ServerArgs parses it. + ("attention_backend_config", attention_backend_config), + ): + if value.strip(): + options[key] = value.strip() + # Strip None to keep payload clean options = {k: v for k, v in options.items() if v is not None} return (options,) @@ -972,6 +1003,7 @@ class SGLDiffusionServerUnsetLora: # Register nodes NODE_CLASS_MAPPINGS = { + "SGLDH3DistilledSigmas": SGLDH3DistilledSigmas, "SGLDiffusionServerModel": SGLDiffusionServerModel, "SGLDiffusionGenerateImage": SGLDiffusionGenerateImage, "SGLDiffusionGenerateVideo": SGLDiffusionGenerateVideo, diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py index dac51f2365..2a6e52a7b9 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py @@ -10,7 +10,7 @@ from __future__ import annotations import logging from dataclasses import dataclass -from typing import Any +from typing import Any, Callable import torch @@ -223,7 +223,29 @@ def comfyui_layout_to_packed(layout: dict[str, Any]) -> dict[str, Any]: token_tags[start:stop] = _SEG_TOKEN_TAG.get(kind, -1) if kind == "text": text_pos = torch.arange(start, stop, dtype=torch.long) + # Cube-sparse-attention segment shapes, as packed_sequence exports them. + cond_shapes, cond_roles, events, audio_lens = [], [], [], [] + for start, stop, kind in layout["segments"]: + if kind in ("cond", "ref_img"): + coords = position_ids[start:stop] + cond_shapes.append( + tuple(int(torch.unique(coords[:, axis]).numel()) for axis in range(3)) + ) + cond_roles.append("joint_cube" if kind == "cond" else "independent_cube") + events.append(("imgvid", len(cond_shapes) - 1)) + elif kind in ("cond_audio", "ref_audio"): + audio_lens.append(stop - start) + events.append(("audio", len(audio_lens) - 1)) + _, latent_t, latent_h, latent_w, *_ = layout["signature"] + stream_layout = { + "target_shape": (latent_t, (latent_h + 1) // 2, (latent_w + 1) // 2), + "cond_image_shapes": tuple(cond_shapes), + "cond_image_roles": tuple(cond_roles), + "cond_event_orders": tuple(events), + "cond_audio_stream_lens": tuple(audio_lens), + } return { + "stream_layout": stream_layout, "seq_len": seq_len, "img_pos": layout["img_pos"].view(-1).to(torch.long), "audio_pos": layout["audio_pos"].view(-1).to(torch.long), @@ -526,6 +548,7 @@ def comfyui_payload_to_branch_inputs( "used": used, "orig_video_shape": orig_shape, "padded_video_shape": tuple(video.shape), + "is_ref2va": bool(refs), "sample_sigmas": sample_sigmas, "sigma_v": float(sigma_v), "shift_v": shift_v, @@ -608,6 +631,53 @@ def _sample_sigmas_len(sample_sigmas: Any) -> int | None: return None +def _step_attn_metadata( + model: Any, + inputs: dict[str, Any], + sample_sigmas: Any, + server_args: ServerArgs, + device: torch.device, +) -> Callable[[int], Any] | None: + """Sparse-attention metadata per step, built as the native H3 loop builds it.""" + from sglang.multimodal_gen.runtime.models.dits.minimax_h3_vdn_attention import ( + prepare_hybrid_attention_metadata, + ) + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import ( + _build_cube_attn_metadata, + _maybe_prepare_vsa_h3_step_metadata, + ) + + packed = inputs["packed"] + latent_shape = tuple(inputs["padded_video_shape"][-3:]) + cube = _build_cube_attn_metadata( + server_args, + packed=packed, + num_steps=max((_sample_sigmas_len(sample_sigmas) or 2) - 1, 1), + device=device, + ) + if cube is not None: + return lambda _step: cube + # VSA-H3 tiles assume the native row order (text, cond, audio, raster video); + # test_vsa_h3_tiles_cover_comfyui_packed_rows pins ComfyUI layouts to it. + build = _maybe_prepare_vsa_h3_step_metadata( + model=model, + packed=packed, + is_ref2va=inputs["is_ref2va"], + latent_shape=latent_shape, + server_args=server_args, + device=device, + ) + if build is not None: + return build + return prepare_hybrid_attention_metadata( + model=model, + packed=packed, + latent_shape=latent_shape, + server_args=server_args, + device=device, + ) + + @dataclass class MiniMaxH3ComfyUIRunState: branch: MiniMaxH3DenoiseBranch @@ -615,6 +685,7 @@ class MiniMaxH3ComfyUIRunState: orig_video_shape: tuple[int, ...] padded_video_shape: tuple[int, ...] sample_sigmas: Any = None + step_attn_metadata: Callable[[int], Any] | None = None cache_mounted: bool = False refined: bool = False steps_done: int = 0 @@ -628,6 +699,14 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): self._cache_stage = None def verify_input(self, batch: Req, server_args: ServerArgs): + if ( + server_args.enable_breakable_cuda_graph + and server_args.is_dit_layerwise_offload_selected + ): + raise ValueError( + "ComfyUI H3 breakable CUDA graph is incompatible with layerwise " + "offload: the offload copy-stream events invalidate graph capture" + ) bind_comfyui_session(batch) return super().verify_input(batch, server_args) @@ -648,7 +727,9 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): n_sigmas = _sample_sigmas_len(state.sample_sigmas) if n_sigmas is None or n_sigmas < 2: return - self._cache_dit_stage()._maybe_enable_cache_dit(n_sigmas - 1, batch) + self._cache_dit_stage()._maybe_enable_cache_dit_and_torch_compile( + n_sigmas - 1, batch + ) state.cache_mounted = True def _release_run_state(self, state: MiniMaxH3ComfyUIRunState | None) -> None: @@ -686,6 +767,7 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): inputs: dict[str, Any], sample_sigmas: Any, device: torch.device, + server_args: ServerArgs, ) -> MiniMaxH3ComfyUIRunState: from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import ( _precompute_refined_prompt_embeds, @@ -715,9 +797,15 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): text_embeddings=inputs["text_embeddings"], token_tags=inputs["token_tags"], device=device, + video_query_indices=inputs["packed"]["img_pos"][ + inputs["packed"]["update_mask"] + ], ) return MiniMaxH3ComfyUIRunState( branch=branch, + step_attn_metadata=_step_attn_metadata( + self.transformer, inputs, sample_sigmas, server_args, device + ), used=inputs["used"], orig_video_shape=inputs["orig_video_shape"], padded_video_shape=inputs["padded_video_shape"], @@ -737,6 +825,34 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): state.refined = True return state + def _run_dit( + self, batch: Req, fk: dict[str, Any], state: MiniMaxH3ComfyUIRunState + ) -> tuple[torch.Tensor, torch.Tensor]: + from sglang.multimodal_gen.runtime.managers.forward_context import ( + set_forward_context, + ) + + stage = self._cache_dit_stage() + step = state.steps_done + attn_metadata = ( + state.step_attn_metadata(step) + if state.step_attn_metadata is not None + else None + ) + runner = stage._maybe_get_bcg_runner(self.transformer) + if runner is not None: + # Integrated mode never warms up, so capture here; it is a no-op + # once this signature has a graph. + with set_forward_context( + current_timestep=step, attn_metadata=attn_metadata, forward_batch=batch + ): + runner.capture( + **stage._bcg_pad_prompt_kwargs(fk, current_model=self.transformer) + ) + return stage._forward_dit( + self.transformer, fk, step, batch=batch, attn_metadata=attn_metadata + ) + def forward(self, batch: Req, server_args: ServerArgs) -> Req: bind_comfyui_session(batch) extra = getattr(batch, "extra", None) or {} @@ -764,12 +880,14 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): denoise_mask=extra.get("h3_denoise_mask"), audio_denoise_mask=extra.get("h3_audio_denoise_mask"), ) - state = self._bind_run_state(batch, inputs, sample_sigmas, video_x.device) + state = self._bind_run_state( + batch, inputs, sample_sigmas, video_x.device, server_args + ) self._maybe_mount_cache_dit(batch, state) fk, _branch = build_step_forward_kwargs( inputs, branch=state.branch, device=video_x.device ) - v_video, v_audio = self.transformer(**fk) + v_video, v_audio = self._run_dit(batch, fk, state) batch.noise_pred = pack_comfy_output(v_video, v_audio, state) state.steps_done += 1 n_sigmas = _sample_sigmas_len(sample_sigmas) diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py index 7983186aaf..328359e124 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/denoising.py @@ -747,7 +747,8 @@ class MiniMaxH3DenoisingStage(DenoisingStage): build_vsa_h3_step_metadata = _maybe_prepare_vsa_h3_step_metadata( model=model, packed=packed, - ctx=ctx, + is_ref2va=ctx.is_ref2va, + latent_shape=(ctx.latent_t, ctx.latent_h, ctx.latent_w), server_args=server_args, device=device, ) @@ -1049,7 +1050,8 @@ def _maybe_prepare_vsa_h3_step_metadata( *, model: Any, packed: Mapping[str, torch.Tensor], - ctx: _FullLoopContext, + is_ref2va: bool, + latent_shape: tuple[int, int, int], server_args: ServerArgs, device: torch.device, ) -> Callable[[int], Any] | None: @@ -1061,7 +1063,7 @@ def _maybe_prepare_vsa_h3_step_metadata( is not AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3 ): return None - if ctx.is_ref2va: + if is_ref2va: raise NotImplementedError( "VSA-H3 supports the t2va/fl2va packed layout; the ref2va " "reference-block layout is not tiled yet. Use --attention-backend " @@ -1102,7 +1104,7 @@ def _maybe_prepare_vsa_h3_step_metadata( def build(step_index: int): return builder.build( current_timestep=step_index, - raw_latent_shape=(ctx.latent_t, ctx.latent_h, ctx.latent_w), + raw_latent_shape=latent_shape, patch_size=patch_size, VSA_sparsity=sparsity, prefix_segments=(text_len, cond_rows, audio_rows), diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py index cdb49c62f1..d6a08037a3 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py @@ -260,3 +260,109 @@ def test_multi_gpu_single_file_starts_without_cfg_parallel( str(path), "FluxPipeline", {"num_gpus": 2, "enable_cfg_parallel": True} ) assert "cfg_parallel_degree" not in seen + + +def test_spawned_workers_do_not_reexecute_launcher_main(monkeypatch) -> None: + """Workers must not re-run ComfyUI's main.py; its imports break under spawn.""" + import multiprocessing.spawn + import sys + + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import ( + _spawn_without_launcher_main, + ) + + main_dict = vars(sys.modules["__main__"]) + monkeypatch.setitem(main_dict, "__file__", "/comfy/main.py") + with _spawn_without_launcher_main(): + data = multiprocessing.spawn.get_preparation_data("worker") + assert "init_main_from_path" not in data + assert "init_main_from_name" not in data + assert main_dict["__file__"] == "/comfy/main.py" + + +def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator: + runtime = SGLDiffusionGenerator() + runtime.get_comfyui_model = lambda *a: (SimpleNamespace(), None, "minimax_h3") + runtime.init_generator = Mock(side_effect=AssertionError("worker must not start")) + return runtime + + +@pytest.mark.parametrize( + "key, sgld_options, error", + [ + # A FastH3 file loaded as base H3 died in the worker on to_gate_compress. + ("blocks.0.attn.to_gate_compress.weight", {}, "model_type fast_h3"), + # The H3 file loader builds no quant config, so this was silently ignored. + ( + "blocks.0.attn.to_q.weight", + {"quantization": "convrot_int8"}, + "cannot be quantized at load time", + ), + ], +) +def test_unservable_h3_file_is_rejected_before_worker_load( + tmp_path, key, sgld_options, error +) -> None: + from safetensors.torch import save_file + + path = tmp_path / "h3.safetensors" + save_file({key: torch.zeros(1)}, path) + with pytest.raises(ValueError, match=error): + _h3_runtime_that_must_not_start().load_model( + model_path=str(path), sgld_options=sgld_options + ) + + +def test_fasth3_runtime_dir_is_checked_before_worker_load(tmp_path) -> None: + """The 4-step preview and a raw (unmaterialized) FastH3 V2 download were + only rejected by the worker after a full model load.""" + import json + + release = { + "schema_version": 1, + "partition": "fl2va", + "tasks": ["t2va"], + "task_aliases": {}, + "sigma_shift_scales": {"video": 10.0, "audio": 3.0}, + } + (tmp_path / "transformer").mkdir() + index = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors.index.json" + runtime = _h3_runtime_that_must_not_start() + options = {"model_type": "fast_h3", "runtime_model_path": str(tmp_path)} + for dmd, weight_map, message in ( + (None, {}, "no trained DMD rungs"), + ([999, 500], {"blocks.0.x": "a.safetensors"}, "raw FastH3 download"), + ): + meta = dict(release, **({"dmd_denoising_steps": dmd} if dmd else {})) + (tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": meta})) + index.write_text(json.dumps({"weight_map": weight_map})) + with pytest.raises(ValueError, match=message): + runtime.load_model(model_path="h3.safetensors", sgld_options=options) + + +@pytest.mark.parametrize( + "name, pipeline, config_cls", + [ + ("h3_local.safetensors", "MiniMaxH3Pipeline", "MiniMaxH3PipelineConfig"), + ("VDN-H3-8nfe", "VDNH3Pipeline", "VDNH3PipelineConfig"), + ], +) +def test_renamed_h3_paths_resolve_through_the_plugin_model_id( + tmp_path, monkeypatch, name, pipeline, config_cls +) -> None: + """A local H3 file or VDN copy whose path lacks the HF name matched no + registry detector or model-id keyed default.""" + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator + from sglang.multimodal_gen.registry import _get_config_info + + seen = {} + monkeypatch.setattr( + generator.DiffGenerator, + "from_pretrained", + lambda **kwargs: seen.update(kwargs) or object(), + ) + path = str(tmp_path / name) + SGLDiffusionGenerator().init_generator(path, pipeline, {}) + assert seen["warmup_mode"] == "off" + info = _get_config_info(path, model_id=seen["model_id"]) + assert info.pipeline_config_cls.__name__ == config_cls diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py index 8e2743fbc0..3512b63319 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py @@ -318,6 +318,59 @@ def test_fl2va_first_last_used_prefix_matches_comfyui(): ] +@pytest.mark.parametrize("keyframes", [False, True]) +def test_vsa_h3_tiles_cover_comfyui_packed_rows(keyframes): + """Integrated steps hand ComfyUI's own layout to the VSA-H3 tile builder, + which assumes text, cond and audio prefixes followed by the raster video + rows; a reordered layout would mis-tile attention silently.""" + from sglang.multimodal_gen.configs.models.dits.minimax_h3 import MiniMaxH3DiTConfig + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.denoising import ( + _maybe_prepare_vsa_h3_step_metadata, + ) + from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum + + text_len, latent_t, latent_h, latent_w, audio_t = 7, 3, 8, 12, 5 + layout = ComfyUIPackedLayout( + text_len, + latent_t, + latent_h, + latent_w, + audio_t, + keyframes=( + [{"resolved_frame_index": 0, "latent": torch.zeros(1, 24, 1, 8, 12)}] + if keyframes + else None + ), + ) + packed = comfyui_layout_to_packed(serialize_comfyui_layout(layout)) + model = SimpleNamespace( + _resolve_attention_backend_once=lambda: None, + _resolved_attention_backend=AttentionBackendEnum.VIDEO_SPARSE_ATTN_H3, + ) + server_args = SimpleNamespace( + attention_backend_config={}, + pipeline_config=SimpleNamespace( + vsa_sparsity=0.8, dit_config=MiniMaxH3DiTConfig() + ), + ) + build = _maybe_prepare_vsa_h3_step_metadata( + model=model, + packed=packed, + is_ref2va=False, + latent_shape=(latent_t, latent_h, latent_w), + server_args=server_args, + device=torch.device("cpu"), + ) + used = int(packed["cu_seqlens"][1]) + assert build(0).total_seq_length == used + video = packed["img_pos"][packed["update_mask"].bool()] + video_start = used - video.numel() + assert torch.equal(video, torch.arange(video_start, used)) + audio = packed["audio_pos"] + assert torch.equal(audio, torch.arange(video_start - audio.numel(), video_start)) + assert torch.equal(packed["text_pos"], torch.arange(text_len)) + + def test_ref2va_image_plus_video_audio_used_prefix_matches_comfyui(): kwargs = dict(text_len=5, latent_t=2, latent_h=4, latent_w=4, audio_t=5) refs = [ @@ -454,8 +507,11 @@ from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import ( SGLDiffusionExecutor, ) from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.minimax_h3 import ( + FastH3Executor, MiniMaxH3Adapter, + distilled_h3_sigmas, drop_h3_pinned_sampling_fields, + validate_distilled_h3_step, worker_transformer_options, ) from sglang.multimodal_gen.configs.sample.minimax_h3 import MiniMaxH3SamplingParams @@ -1259,3 +1315,132 @@ def test_h3_pack_rejects_batched_latents() -> None: minimax_payload={"audio_scale": 1.0}, transformer_options={"sample_sigmas": torch.tensor([1.0, 0.0])}, ) + + +def test_comfyui_sparse_stream_layout_preserves_target_and_condition_roles(): + keyframe = {"resolved_frame_index": 0, "latent": torch.zeros(1, 24, 1, 8, 12)} + layout = ComfyUIPackedLayout(3, 2, 8, 12, 4, keyframes=[keyframe]) + stream = comfyui_layout_to_packed(serialize_comfyui_layout(layout))["stream_layout"] + assert stream["target_shape"] == (2, 4, 6) + assert stream["cond_image_shapes"] == ((1, 4, 6),) + assert stream["cond_image_roles"] == ("joint_cube",) + assert stream["cond_event_orders"] == (("imgvid", 0),) + + +@pytest.mark.parametrize("model_type", ["fast_h3", "vdn_h3"]) +def test_distilled_h3_rejects_wrong_grid_and_reference_task(model_type): + expected = distilled_h3_sigmas(model_type) + sigmas = torch.tensor(expected) + packed = SimpleNamespace(extra_req={"h3_sample_sigmas": sigmas}) + validate_distilled_h3_step(packed, model_type, expected) + packed.extra_req["h3_sample_sigmas"] = torch.linspace(1, 0, len(expected)) + with pytest.raises(ValueError, match="trained"): + validate_distilled_h3_step(packed, model_type, expected) + packed.extra_req.update( + h3_sample_sigmas=sigmas, h3_payload={"refs": [{"kind": "image"}]} + ) + with pytest.raises(ValueError, match="reference"): + validate_distilled_h3_step(packed, model_type, expected) + + +def test_fasth3_comfy_grid_matches_native_8_step_v2_contract(): + """The plugin pinned fast_h3 to the 4-NFE preview grid (shift 12) after the + runtime moved to FastH3 8-Step V2, so V2 could not run in ComfyUI.""" + from sglang.multimodal_gen.configs.sample.minimax_h3 import FastH3SamplingParams + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.time_request import ( + minimax_h3_time_shift_sigmas, + ) + + # FastVideo/FastVideo-FastH3-8-Step-V2 model_index.json._minimax_h3 + rungs = (999, 874, 749, 624, 500, 375, 250, 125) + native = minimax_h3_time_shift_sigmas( + num_steps=8, shift_scale=10.0, dmd_steps=rungs + ) + assert distilled_h3_sigmas("fast_h3") == native + assert len(native) == FastH3SamplingParams().num_inference_steps + 1 + + +def test_fasth3_executor_uses_trained_shifts_and_rejects_conflicts(tmp_path): + """Without a MiniMaxH3SigmaShift node the audio sigma was derived with the + base 12/3 shifts instead of FastH3's trained 10/3.""" + import json + + release = { + "schema_version": 1, + "partition": "fl2va", + "tasks": ["t2va"], + "task_aliases": {}, + "sigma_shift_scales": {"video": 10.0, "audio": 3.0}, + "dmd_denoising_steps": [999, 874, 749, 624, 500, 375, 250, 125], + } + (tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": release})) + sent = [] + + class Recording(FastH3Executor): + def _execute_packed(self, packed, x, timestep): + sent.append(packed) + return x + + config = SimpleNamespace(unet_config={"dtype": torch.bfloat16}) + ex = Recording(None, str(tmp_path), None, config) + x = [torch.ones(1, 24, 2, 4, 4), torch.ones(1, 32, 2, 3)] + step = lambda **opts: ex( + x, + torch.tensor([500.0]), + torch.ones(1, 8, 16), + transformer_options={"sample_sigmas": torch.tensor(ex.expected_sigmas), **opts}, + minimax_payload={"audio_scale": 1.0}, + ) + step() + opts = sent[0].extra_req["h3_transformer_options"] + assert opts["minimax_h3_sigma_shift_video"] == 10.0 + assert opts["minimax_h3_sigma_shift_audio"] == 3.0 + with pytest.raises(ValueError, match="distilled with sigma shift video/audio 10/3"): + step(minimax_h3_sigma_shift_video=12.0) + + +@pytest.mark.parametrize( + "masks, video_out, audio_out", + [ + # ComfyUI forward: (1 - scale) * input + scale * (-raw velocity). At + # sigma_v=sigma_a=1 this is -3 - 4=-7; a second derivative gave -28. + ({}, -1, -7), + # Masks apply to the raw velocities, before the carried-audio transform. + ( + { + "denoise_mask": torch.full((1, 1, 2, 4, 4), 0.25), + "audio_denoise_mask": torch.zeros(1, 1, 2, 3), + }, + -0.25, + -3, + ), + ], +) +def test_h3_velocities_match_comfyui_forward(masks, video_out, audio_out): + adapter = MiniMaxH3Adapter() + video = torch.ones(1, 24, 2, 4, 4) + audio = torch.ones(1, 32, 2, 3) + packed = adapter.pack( + [video, audio], + torch.tensor([1000.0]), + torch.ones(1, 4, 8), + minimax_payload={"audio_scale": 4.0}, + **masks, + ) + out = adapter.unpack( + [torch.ones_like(video), torch.ones_like(audio)], packed, [video, audio] + ) + assert torch.equal(out[0], torch.full_like(video, video_out)) + assert torch.equal(out[1], torch.full_like(audio, audio_out)) + + +def test_comfyui_h3_rejects_bcg_with_offload_before_cuda_capture(): + from sglang.multimodal_gen.runtime.pipelines_core.stages.model_specific_stages.minimax_h3.stages.comfyui_step import ( + MiniMaxH3ComfyUIStepStage, + ) + + args = SimpleNamespace( + enable_breakable_cuda_graph=True, is_dit_layerwise_offload_selected=True + ) + with pytest.raises(ValueError, match="copy-stream events"): + MiniMaxH3ComfyUIStepStage.verify_input(None, SimpleNamespace(), args)