From c92108bb93a0d010699e111d89085030d128c50e Mon Sep 17 00:00:00 2001 From: WenhaoZhang <42087078+niehen6174@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:59:03 +0800 Subject: [PATCH] [diffusion] comfyui: grouped SGLDOptions, native model loader and model-aware preflight (#44058) Co-authored-by: Mick --- .../apps/ComfyUI_SGLDiffusion/README.md | 40 +++- .../apps/ComfyUI_SGLDiffusion/__init__.py | 4 +- .../ComfyUI_SGLDiffusion/core/generator.py | 141 +++++++++++--- .../ComfyUI_SGLDiffusion/core/preflight.py | 107 +++++++++++ .../executors/minimax_h3.py | 5 + .../apps/ComfyUI_SGLDiffusion/nodes.py | 179 +++--------------- .../apps/ComfyUI_SGLDiffusion/options.py | 161 ++++++++++++++++ .../apps/ComfyUI_SGLDiffusion/web/options.js | 91 +++++++++ .../web/options_state.mjs | 64 +++++++ .../web/options_state.test.mjs | 32 ++++ .../workflows/flux_sgld_sp.json | 14 +- .../workflows/minimax_h3_r2v_sgld.json | 13 +- .../workflows/minimax_h3_t2v_sgld.json | 13 +- .../minimax_h3_t2v_sgld_upscaler.json | 17 +- .../minimax_h3/stages/comfyui_step.py | 5 + .../runtime/server_args/server_args.py | 10 +- .../test/unit/test_comfyui_generator.py | 13 +- .../test/unit/test_comfyui_options.py | 143 ++++++++++++++ 18 files changed, 821 insertions(+), 231 deletions(-) create mode 100644 python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py create mode 100644 python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options.js create mode 100644 python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.mjs create mode 100644 python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.test.mjs create mode 100644 python/sglang/multimodal_gen/test/unit/test_comfyui_options.py diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md index 87d3f291ba..4959799c5e 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/README.md @@ -24,7 +24,7 @@ The plugin supports two modes of operation: **Server Mode** (via HTTP API) and * - **Z-Image**: High-speed image generation models (e.g., `Z-Image-Turbo`) - **FLUX**: State-of-the-art text-to-image models (e.g., `FLUX.1-dev`) - **Qwen-Image**: Multi-modal image generation models (e.g., `Qwen-Image`,`Qwen-Image-2512`). *Note: Image editing support is currently experimental and may have some issues.* -- **Qwen-Image-2.1**: Integrated mode (`model_type=qwen_image21`, auto-detected) for Comfy-Org's `qwen_image_2.1_bf16.safetensors` or INT8 `qwen_image_2.1_int8_convrot.safetensors`. Text-to-image and reference-image edit work with the official templates after swapping `UNETLoader` for `SGLDUNETLoader`. The worker manages the prefix K/V cache itself, so the `Qwen Image 2.1 Cache` node's options are ignored. Multi-GPU: `sp_degree` or `tp_size` (`enable_cfg_parallel` is rejected; ComfyUI runs CFG). DiT patches (Fun-Control, attention hooks) are not supported. LoRA: use `SGLDiffusion LoRA Loader` instead of `LoraLoaderModelOnly`. Tested with `Viggle/Qwen-Image-2.1-viggle-turbo` (`...-v0.3-6step-lora-r128.safetensors`): euler, `BasicGuider`, `ManualSigmas` `1.0, 0.967754, 0.933358, 0.857192, 0.666756, 0.400096, 0` at 1024x1024. +- **Qwen-Image-2.1**: Integrated mode (`model_type=qwen_image21`, auto-detected) for Comfy-Org's `qwen_image_2.1_bf16.safetensors` or INT8 `qwen_image_2.1_int8_convrot.safetensors`. Text-to-image and reference-image edit work with the official templates after swapping `UNETLoader` for `SGLDUNETLoader`. The worker manages the prefix K/V cache itself, so the `Qwen Image 2.1 Cache` node's options are ignored. Multi-GPU: Ulysses / Ring or TP through the manual layout in `SGLDiffusion Options`. DiT patches (Fun-Control, attention hooks) are not supported. LoRA: use `SGLDiffusion LoRA Loader` instead of `LoraLoaderModelOnly`. Tested with `Viggle/Qwen-Image-2.1-viggle-turbo` (`...-v0.3-6step-lora-r128.safetensors`): euler, `BasicGuider`, `ManualSigmas` `1.0, 0.967754, 0.933358, 0.857192, 0.666756, 0.400096, 0` at 1024x1024. - **MiniMax-H3**: Joint video-and-audio DiT (`model_type=minimax_h3`). Integrated mode: T2V / I2VA / FL2VA use an `fl2va` checkpoint; R2V needs `ref2va`. CLIP and VAE stay in ComfyUI. Server mode uses `SGLDiffusion Generate MiniMax-H3`. ### Mode 1: Server Mode (HTTP API) @@ -43,11 +43,45 @@ ComfyUI keeps CLIP, VAE, and the sampler loop. SGLang loads only the DiT and runs one forward per sampler step (pass-through scheduler, no text encode / decode on the worker). -1. **Load Model**: Use the `SGLDiffusion UNET Loader` node to load your diffusion model. -2. **Configure Options**: Use the `SGLDiffusion Options` node to set runtime parameters like `num_gpus`, `tp_size`, `model_type`, or `enable_torch_compile`. +1. **Load Model**: `SGLDiffusion UNET Loader` loads ComfyUI single files (Qwen-Image, FLUX, Z-Image, MiniMax H3, including pre-quantized `*_int8_convrot` files). It takes the same file as the stock UNET loader, so a workflow switches by replacing that node. `SGLDiffusion Native Model Loader` loads native SGLang model directories that have no ComfyUI file: FastH3 8-Step V2 and VDN-H3 (the model type is read from `model_index.json`). +2. **Configure Options** (optional): connect `SGLDiffusion Options` to the loader. Without it the worker uses auto-detection and defaults. 3. **Sample**: Connect the loaded model to standard ComfyUI samplers. Each step is packed by a model adapter and sent to the SGLang scheduler. 4. **LoRA Support**: Use the `SGLDiffusion LoRA Loader` for native LoRA integration. +### Options + +`SGLDiffusion Options` keeps `model_type` and `num_gpus` on +top; the other groups fold, and each folded header shows its current choice. +The fields are plain ComfyUI widgets, so the node works without the JavaScript +extension, which only changes the presentation. + +| Group | Fields | +| --- | --- | +| Parallel | `parallel_mode` auto or manual; manual sets `tp_size`, `ulysses_degree`, `ring_degree`, whose product must equal `num_gpus` | +| Attention | `attention_backend`: auto (model default), flash_attention, torch_sdpa, sage_attention2, sage_attention3, sol_attention (H3; needs the `sol-attn` package; picking it fills `attention_backend_config` with its defaults), and the H3 sparse backends; `attention_backend_config` | +| Quantization | `quantization` (online, native directories only); `int8_gemm`: GEMM implementation for INT8 ConvRot weights, auto / sglang_jit / comfy_kitchen | +| Cache | `cache_mode`: off, cache_dit, teacache, spectrum (lossy). For cache_dit, `cache_dit_tuning` auto keeps the defaults; manual sets warmup steps, residual threshold, max consecutive cached steps and Fn compute blocks | +| Execution | `execution_mode`: eager, torch_compile, cuda_graph (breakable CUDA graph); `regional_compile` | +| Memory | `memory_mode`: resident or DiT layerwise offload; `resident_layers`, `prefetch_layers` | + +The loader checks the options against the resolved model before starting the +worker and raises instead of dropping a setting. Current limits in integrated mode: + +| Option | Supported for | +| --- | --- | +| cuda_graph | MiniMax H3, with `cache_mode` off and no layerwise offload | +| torch_compile | all but FastH3 and VDN-H3 | +| ring_degree > 1 | not FastH3 / VDN-H3; MiniMax H3 needs `fa` | +| quantization | native model directories; ComfyUI files carry their own format | +| cache_dit, spectrum | MiniMax H3 family (spectrum: sequence-parallel degree 1); other models send each sampler step as its own request, so a step cache needs per-run state first | +| teacache | none yet | + +Changing anything but the Cache group restarts the worker on the next run. + +For development, run `node --test web/options_state.test.mjs` in this plugin +folder and `python -m pytest python/sglang/multimodal_gen/test/unit/test_comfyui_options.py` +from the SGLang repository root. + ## Adding a Model Pick the mode before writing code; the wrong one costs several hundred lines diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/__init__.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/__init__.py index a34ef84f97..dacffc44ba 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/__init__.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/__init__.py @@ -2,10 +2,12 @@ ComfyUI SGLang Diffusion nodes package. """ +WEB_DIRECTORY = "./web" + try: from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS - __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] + __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] except ImportError: # ComfyUI dependencies not available (e.g., in test environment) NODE_CLASS_MAPPINGS = {} diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py index 327bb74e28..cfa4d56421 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py @@ -4,10 +4,13 @@ Generator for SGLang Diffusion ComfyUI integration. import atexit import contextlib +import json import logging import os import sys +import torch + from ..executors.flux import FluxExecutor from ..executors.minimax_h3 import ( FastH3Executor, @@ -17,7 +20,8 @@ from ..executors.minimax_h3 import ( ) from ..executors.qwen_image21 import QwenImage21Executor from ..executors.zimage import ZImageExecutor -from .preflight import check_sgld_options +from ..options import PLUGIN_KEYS +from .preflight import check_model_options, check_sgld_options logger = logging.getLogger(__name__) @@ -79,8 +83,8 @@ def _check_h3_single_file(path: str, sgld_options: dict) -> None: if any(".to_gate_compress." in key for key in f.keys()): raise ValueError( "This is a FastH3 checkpoint (VSA gate weights), which the base " - "MiniMax H3 model cannot load; set model_type fast_h3 with " - "runtime_model_path pointing at the native FastH3 model" + "MiniMax H3 model cannot load; load the native FastH3 directory " + "with SGLDNativeModelLoader" ) @@ -142,6 +146,68 @@ def _h3_detect_companion(gguf_ref: str) -> str: ) +_NATIVE_MODEL_TYPES = {"FastH3Pipeline": "fast_h3", "VDNH3Pipeline": "vdn_h3"} +NATIVE_MODEL_TYPES = tuple(_NATIVE_MODEL_TYPES.values()) + + +def _apply_cache_mode(executor, plugin: dict) -> None: + """Per-request cache switches; they never restart the worker.""" + mode = plugin.get("cache_mode") + # None (no cache_mode given) keeps the worker's environment default. + executor.enable_cache_dit = None if mode is None else mode == "cache_dit" + executor.cache_dit_params = plugin.get("cache_dit_params") + executor.enable_spectrum = mode == "spectrum" + + +@contextlib.contextmanager +def _worker_env(values: dict): + """Set env vars the spawned worker inherits, then restore this process's.""" + saved = {key: os.environ.get(key) for key in values} + os.environ.update(values) + try: + yield + finally: + for key, value in saved.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + + +def native_model_type(model_dir: str) -> str: + with open(os.path.join(model_dir, "model_index.json")) as f: + class_name = json.load(f).get("_class_name") + if class_name not in _NATIVE_MODEL_TYPES: + raise ValueError( + f"{model_dir}: native model {class_name!r} is not supported by " + f"SGLDNativeModelLoader; supported: {sorted(_NATIVE_MODEL_TYPES)}" + ) + return _NATIVE_MODEL_TYPES[class_name] + + +def _h3_unet_config_from_native(model_dir: str) -> dict: + """ComfyUI MiniMaxH3 unet_config built from a native transformer/config.json.""" + with open(os.path.join(model_dir, "transformer", "config.json")) as f: + c = json.load(f) + return { + "image_model": "minimax_h3", + "num_layers": c["num_layers"], + "token_refiner_num_layers": c["num_refiner_layers"], + "hidden_size": c["hidden_size"], + "latents_dim": c["in_channels"], + "audio_latents_dim": c["audio_in_channels"], + "attention_head_dim": c["attention_head_dim"], + "num_attention_heads": c["num_attention_heads"], + "ffn_hidden_size": c["ffn_dim"], + "text_dim": c["text_dim"], + "timestep_input_dim": c["freq_dim"], + "time_embed_hidden_size": c["time_embed_hidden_dim"], + "time_embed_dim": c["time_embed_dim"], + "rope_inv_freq_len": c["rope_freq_dim"], + "gate_compress": False, + } + + def _load_state_dict_for_detection(model_path: str): if _looks_like_gguf(model_path): raise ValueError( @@ -218,6 +284,7 @@ class SGLDiffusionGenerator: self.model_path = None self.generator = None self.executor = None + self.model_type = None self.last_options = None self._patcher = None @@ -411,6 +478,7 @@ class SGLDiffusionGenerator: self.model_path = None self.generator = None self.executor = None + self.model_type = None self._patcher = None def get_comfyui_model(self, model_path: str, model_options: dict = None): @@ -427,6 +495,14 @@ class SGLDiffusionGenerator: unet_to_diffusers, ) + if os.path.isdir(model_path): + model_config = model_detection.model_config_from_unet_config( + _h3_unet_config_from_native(model_path) + ) + model_config.set_inference_dtype(torch.bfloat16, None) + model_config.unet_config["disable_unet_model_creation"] = True + return model_config.get_model({}), model_config, "minimax_h3" + sd = _load_state_dict_for_detection(model_path) diffusion_model_prefix = model_detection.unet_prefix_from_state_dict(sd) temp_sd = state_dict_prefix_replace( @@ -499,23 +575,19 @@ class SGLDiffusionGenerator: "does not use it; select quantization in SGLDOptions instead" ) check_sgld_options(sgld_options) - plugin_flags = {} - if "enable_cache_dit" in sgld_options: - plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit") - set_model_type = sgld_options.pop("model_type", None) - runtime_model_path = sgld_options.pop("runtime_model_path", None) - if bool(runtime_model_path) != (set_model_type in ("fast_h3", "vdn_h3")): - # The base H3 file is only the ComfyUI architecture companion. + plugin = {k: sgld_options.pop(k) for k in PLUGIN_KEYS if k in sgld_options} + set_model_type = plugin.get("model_type") + if set_model_type in NATIVE_MODEL_TYPES: raise ValueError( - "model_type fast_h3 / vdn_h3 requires runtime_model_path (the native " - "model directory), and runtime_model_path is only used by them" + f"{set_model_type} loads from its native model directory; use " + "SGLDNativeModelLoader instead of model_type" ) - if runtime_model_path: - runtime_model_path = os.path.abspath(os.path.expanduser(runtime_model_path)) - if not os.path.isdir(runtime_model_path): - raise ValueError( - "runtime_model_path must be a local materialized native model directory" - ) + native_dir = os.path.isdir(model_path) + runtime_model_path = None + if native_dir: + # SGLDNativeModelLoader: the directory is both skeleton and weights. + set_model_type = native_model_type(model_path) + runtime_model_path = model_path override = (sgld_options.get("transformer_weights_path") or "").strip() if override: sgld_options["transformer_weights_path"] = override @@ -532,15 +604,15 @@ class SGLDiffusionGenerator: "sgld_options": sgld_options, "set_model_type": set_model_type, "runtime_model_path": runtime_model_path, + "int8_kernel": plugin.get("int8_kernel", "auto"), } if self._can_reuse(gather_options): - self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit") + check_model_options( + self.model_type, {**sgld_options, **plugin}, native_dir=native_dir + ) + _apply_cache_mode(self.executor, plugin) return self._patcher - self.close_generator() - self.last_options = gather_options - self.model_path = detect_path - comfyui_model, model_config, model_type = self.get_comfyui_model( detect_path, model_options ) @@ -549,22 +621,39 @@ class SGLDiffusionGenerator: if set_model_type is not None and set_model_type in self.pipeline_class_dict: model_type = set_model_type + check_model_options( + model_type, {**sgld_options, **plugin}, native_dir=native_dir + ) if model_type == "fast_h3": load_fasth3_release(runtime_model_path) if model_type == "minimax_h3": _check_h3_single_file(detect_path, sgld_options) + # Checks above run first so a rejected config keeps the current worker. + self.close_generator() + self.last_options = gather_options + self.model_path = detect_path + pipeline_class_name = self.pipeline_class_dict[model_type] worker_model_path = runtime_model_path or detect_path - self.generator = self.init_generator( - worker_model_path, pipeline_class_name, sgld_options + int8_kernel = gather_options["int8_kernel"] + # auto keeps whatever the ComfyUI process environment already selects. + worker_env = ( + {} + if int8_kernel == "auto" + else {"SGLANG_DIFFUSION_CONVROT_INT8_BACKEND": int8_kernel} ) + with _worker_env(worker_env): + self.generator = self.init_generator( + worker_model_path, pipeline_class_name, sgld_options + ) + self.model_type = model_type executor_class = self.executor_class_dict[model_type] self.executor = executor_class( self.generator, worker_model_path, comfyui_model, model_config ) - self.executor.enable_cache_dit = plugin_flags.get("enable_cache_dit") + _apply_cache_mode(self.executor, plugin) self.executor._sgld_reload = reload_kwargs self.executor._ensure_runtime = self.ensure_executor comfyui_model.diffusion_model = self.executor diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py index 95e1a0514a..7f66104f0e 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py @@ -16,6 +16,10 @@ from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEn _INSTALL_HINTS = { AttentionBackendEnum.SAGE_ATTN: "install SageAttention 2: https://github.com/thu-ml/SageAttention", + AttentionBackendEnum.SOL_ATTN: ( + "install sol-attn: pip install git+https://github.com/NVlabs/Sana.git" + "@sol-engine#subdirectory=techniques/sparse_backends" + ), AttentionBackendEnum.SAGE_ATTN_3: ( "SageAttention 3 targets Blackwell GPUs: " "https://github.com/thu-ml/SageAttention/tree/main/sageattention3_blackwell" @@ -104,3 +108,106 @@ def check_parallel_layout(sgld_options: dict) -> None: def check_sgld_options(sgld_options: dict) -> None: check_attention_backends(sgld_options) check_parallel_layout(sgld_options) + + +# ===== Model-aware rules (run once the loader has resolved the model) ===== + +H3_MODELS = ("minimax_h3", "fast_h3", "vdn_h3") +_DISTILLED_H3 = ("fast_h3", "vdn_h3") +# Every ComfyUI step is its own request, so a step cache needs a per-run state; +# only the H3 step stage keeps one today. +_CACHE_MODELS = {"cache_dit": H3_MODELS, "spectrum": H3_MODELS, "teacache": ()} +_ATTENTION_BY_MODEL = { + "generic": ("fa", "torch_sdpa", "sage_attn", "sage_attn_3"), + "minimax_h3": ( + "fa", + "torch_sdpa", + "sage_attn", + "sage_attn_3", + "sol_attn", + "subblock_sparse_attn", + "video_sparse_attn_h3", + ), + "fast_h3": ("fa", "torch_sdpa", "sage_attn", "video_sparse_attn_h3"), + "vdn_h3": ("hybrid_window_attn_h3",), +} + + +def _unsupported(feature: str, model_type: str) -> ValueError: + return ValueError( + f"{feature} is not supported for {model_type} in ComfyUI integrated mode yet" + ) + + +def check_model_options( + model_type: str, sgld_options: dict, *, native_dir: bool +) -> None: + """Reject combinations the resolved model cannot run, instead of ignoring them.""" + _check_execution(model_type, sgld_options) + _check_attention_for_model(model_type, sgld_options) + _check_cache(model_type, sgld_options) + _check_weights(model_type, sgld_options, native_dir=native_dir) + + +def _check_execution(model_type: str, opts: dict) -> None: + graph = opts.get("enable_breakable_cuda_graph") + if graph: + if model_type != "minimax_h3": + raise _unsupported("cuda_graph", model_type) + if opts.get("cache_mode", "off") != "off": + raise ValueError("cuda_graph requires cache_mode=off") + if opts.get("dit_layerwise_offload"): + raise ValueError("cuda_graph cannot be combined with layerwise offload") + if model_type in _DISTILLED_H3: + if opts.get("enable_torch_compile") or graph: + raise ValueError(f"{model_type} requires execution_mode=eager") + if (opts.get("ring_degree") or 1) > 1: + raise ValueError(f"{model_type} requires ring_degree=1; use Ulysses") + if ( + model_type == "minimax_h3" + and (opts.get("ring_degree") or 1) > 1 + and opts.get("attention_backend", "fa") != "fa" + ): + raise ValueError("MiniMax H3 ring_degree > 1 requires attention_backend=fa") + + +def _check_attention_for_model(model_type: str, opts: dict) -> None: + backend = opts.get("attention_backend") + if backend is None: + return + allowed = _ATTENTION_BY_MODEL.get(model_type, _ATTENTION_BY_MODEL["generic"]) + if backend not in allowed: + raise ValueError( + f"attention_backend={backend} does not apply to {model_type}; " + f"choose auto or one of {list(allowed)}" + ) + + +def _check_cache(model_type: str, opts: dict) -> None: + mode = opts.get("cache_mode", "off") + if mode == "off": + return + if model_type not in _CACHE_MODELS[mode]: + raise _unsupported(mode, model_type) + if mode == "spectrum": + sp = (opts.get("ulysses_degree") or 1) * (opts.get("ring_degree") or 1) + if sp > 1 or ("sp_degree" not in opts and (opts.get("num_gpus") or 1) > 1): + raise ValueError( + "spectrum requires sequence parallel degree 1; set parallel_mode " + "manual with ulysses_degree=ring_degree=1" + ) + + +def _check_weights(model_type: str, opts: dict, *, native_dir: bool) -> None: + if opts.get("quantization") and not native_dir: + raise ValueError( + "quantization (online, at load) applies to native model directories; " + "ComfyUI files carry their own format, so load a pre-quantized file " + "(e.g. *_int8_convrot) instead" + ) + kernel = opts.get("int8_kernel", "auto") + if kernel != "auto" and model_type not in H3_MODELS: + raise ValueError( + f"int8_gemm={kernel} has no effect on {model_type}: only MiniMax H3 " + "runs ConvRot INT8 weights; leave it at auto" + ) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py index d92c086bf3..72e0b07a73 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py @@ -237,6 +237,11 @@ class MiniMaxH3Executor(SGLDiffusionExecutor): enable_cache_dit = getattr(self, "enable_cache_dit", None) if enable_cache_dit is not None: kwargs["enable_cache_dit"] = bool(enable_cache_dit) + cache_dit_params = getattr(self, "cache_dit_params", None) + if cache_dit_params: + kwargs["cache_dit_params"] = dict(cache_dit_params) + if getattr(self, "enable_spectrum", False): + kwargs["enable_spectrum"] = True return kwargs diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py index 8aaa6e2d6e..12c352623a 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py @@ -10,6 +10,7 @@ import folder_paths import torch from .core import SGLDiffusionGenerator, SGLDiffusionServerAPI +from .options import SGLDOptions def _enable_gguf_in_diffusion_models() -> None: @@ -68,158 +69,6 @@ class SGLDH3DistilledSigmas: return (torch.tensor(distilled_h3_sigmas(model_type)),) -class SGLDOptions: - @classmethod - def INPUT_TYPES(cls): - return { - "required": {}, - "optional": { - "model_type": ( - [ - "auto-detect", - "qwen_image", - "qwen_image_edit", - "qwen_image21", - "flux", - "lumina2", - "minimax_h3", - "fast_h3", - "vdn_h3", - ], - {"default": "auto-detect"}, - ), - "enable_torch_compile": ( - "BOOLEAN", - {"default": False}, - ), - "num_gpus": ("INT", {"default": 1, "min": 1, "step": 1}), - "tp_size": ("INT", {"default": -1, "min": -1, "step": 1}), - "sp_degree": ("INT", {"default": -1, "min": -1, "step": 1}), - "ulysses_degree": ( - "INT", - { - "default": -1, - "min": -1, - "step": 1, - }, - ), - "ring_degree": ( - "INT", - { - "default": -1, - "min": -1, - "step": 1, - }, - ), - "dp_size": ("INT", {"default": 1, "min": 1, "step": 1}), - "dp_degree": ("INT", {"default": 1, "min": 1, "step": 1}), - "enable_cfg_parallel": ( - "BOOLEAN", - {"default": False}, - ), - "attention_backend": ( - "STRING", - {"default": ""}, - ), - "dit_layerwise_offload": ( - "BOOLEAN", - {"default": False}, - ), - "enable_cache_dit": ( - "BOOLEAN", - {"default": False}, - ), - "quantization": ( - "STRING", - {"default": ""}, - ), - "runtime_model_path": ("STRING", {"default": ""}), - "attention_backend_config": ("STRING", {"default": ""}), - "transformer_weights_path": ( - "STRING", - { - "default": "", - "multiline": False, - }, - ), - }, - } - - RETURN_TYPES = ("SGLD_OPTIONS",) - RETURN_NAMES = ("sgld_options",) - FUNCTION = "create_options" - CATEGORY = "SGLDiffusion" - - def create_options( - self, - model_type: str = "auto-detect", - enable_torch_compile: bool = False, - num_gpus: int = 1, - tp_size: int = -1, - sp_degree: int = -1, - ulysses_degree: int = -1, - ring_degree: int = -1, - dp_size: int = 1, - dp_degree: int = 1, - enable_cfg_parallel: bool = False, - attention_backend: str = "", - dit_layerwise_offload: bool = False, - enable_cache_dit: bool = False, - quantization: str = "", - transformer_weights_path: str = "", - runtime_model_path: str = "", - attention_backend_config: str = "", - ): - """ - Build a dictionary of SGLang Diffusion runtime options. - """ - # Convert -1 to None for optional parameters (matching ServerArgs defaults) - ulysses_degree = None if ulysses_degree == -1 else ulysses_degree - ring_degree = None if ring_degree == -1 else ring_degree - tp_size = None if tp_size == -1 else tp_size - sp_degree = None if sp_degree == -1 else sp_degree - attention_backend = None if attention_backend == "" else attention_backend - # dp_degree is a leftover alias; ServerArgs only has dp_size. - if dp_degree not in (None, 1) and dp_size in (None, 1): - dp_size = dp_degree - - options = { - "model_type": model_type, - "enable_torch_compile": enable_torch_compile, - "num_gpus": num_gpus, - "tp_size": tp_size, - "sp_degree": sp_degree, - "ulysses_degree": ulysses_degree, - "ring_degree": ring_degree, - "dp_size": dp_size, - "enable_cfg_parallel": enable_cfg_parallel, - "attention_backend": attention_backend, - "dit_layerwise_offload": dit_layerwise_offload, - } - if enable_cache_dit: - options["enable_cache_dit"] = True - quantization = (quantization or "").strip() - if quantization: - options["quantization"] = quantization - transformer_weights_path = (transformer_weights_path or "").strip() - if transformer_weights_path: - # Same selector as `sglang serve --transformer-weights-path`: - # local .gguf, owner/repo/path/file.gguf, or owner/repo:QUANT. - options["transformer_weights_path"] = transformer_weights_path - - for key, value in ( - ("runtime_model_path", runtime_model_path), - # Same format as --attention-backend-config; ServerArgs parses it. - ("attention_backend_config", attention_backend_config), - ): - if value.strip(): - options[key] = value.strip() - - # Strip None to keep payload clean - options = {k: v for k, v in options.items() if v is not None} - return (options,) - - class SGLDLoraLoader: @classmethod def INPUT_TYPES(cls): @@ -317,6 +166,30 @@ class SGLDUNETLoader: return (model,) +class SGLDNativeModelLoader: + """Native SGLang model directories (FastH3, VDN-H3) that have no ComfyUI single file.""" + + def __init__(self): + self.generator = SGLDiffusionGenerator.shared() + + @classmethod + def INPUT_TYPES(s): + return { + "required": {"model_dir": ("STRING", {"default": ""})}, + "optional": {"sgld_options": ("SGLD_OPTIONS",)}, + } + + RETURN_TYPES = ("MODEL",) + FUNCTION = "load" + CATEGORY = "SGLDiffusion" + + def load(self, model_dir, sgld_options: dict = None): + model_dir = os.path.abspath(os.path.expanduser(model_dir.strip())) + if not os.path.isfile(os.path.join(model_dir, "model_index.json")): + raise ValueError(f"{model_dir} is not a native model directory") + return (self.generator.load_model(model_dir, sgld_options=sgld_options),) + + class SGLDiffusionServerModel: """Node to load and manage SGLang Diffusion server connection.""" @@ -1049,6 +922,7 @@ NODE_CLASS_MAPPINGS = { "SGLDiffusionServerSetLora": SGLDiffusionServerSetLora, "SGLDiffusionServerUnsetLora": SGLDiffusionServerUnsetLora, "SGLDUNETLoader": SGLDUNETLoader, + "SGLDNativeModelLoader": SGLDNativeModelLoader, "SGLDOptions": SGLDOptions, "SGLDLoraLoader": SGLDLoraLoader, } @@ -1061,6 +935,7 @@ NODE_DISPLAY_NAME_MAPPINGS = { "SGLDiffusionServerSetLora": "SGLDiffusion Server Set LoRA", "SGLDiffusionServerUnsetLora": "SGLDiffusion Server Unset LoRA", "SGLDUNETLoader": "SGLDiffusion UNET Loader", + "SGLDNativeModelLoader": "SGLDiffusion Native Model Loader", "SGLDOptions": "SGLDiffusion Options", "SGLDLoraLoader": "SGLDiffusion LoRA Loader", } diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py new file mode 100644 index 0000000000..f5f3075226 --- /dev/null +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py @@ -0,0 +1,161 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SGLDOptions: runtime options for the SGLD worker. + +The output is a flat dict of ServerArgs kwargs plus the plugin-only keys in +PLUGIN_KEYS, which core/generator.py pops before building ServerArgs. Checks that +need the resolved model live in core/preflight.py. +""" + +PLUGIN_KEYS = ("model_type", "cache_mode", "cache_dit_params", "int8_kernel") + +# Display names -> SGLang backend names; the rest are passed through unchanged. +_ATTENTION_IDS = { + "flash_attention": "fa", + "sage_attention2": "sage_attn", + "sage_attention3": "sage_attn_3", + "sol_attention": "sol_attn", +} +# Display names -> SGLANG_DIFFUSION_CONVROT_INT8_BACKEND values. +_INT8_GEMMS = {"auto": "auto", "sglang_jit": "jit", "comfy_kitchen": "comfy_kitchen"} + + +def _field(group, kind, default, tooltip="", **kwargs): + spec = {"default": default, "sgld_group": group, "tooltip": tooltip, **kwargs} + return (kind, spec) + + +# Order is the widget order; web/options.js puts a folding header before each group. +_FIELDS = { + "model_type": _field( + "basic", + ["auto-detect", "qwen_image", "qwen_image_edit", "qwen_image21"] + + ["flux", "lumina2", "minimax_h3"], + "auto-detect", + "FastH3 / VDN-H3 load through the Native Model Loader.", + ), + "num_gpus": _field("basic", "INT", 1, min=1, max=64), + "parallel_mode": _field( + "parallel", ["auto", "manual"], "auto", "manual sets TP x Ulysses x Ring." + ), + "tp_size": _field("parallel", "INT", 1, min=1), + "ulysses_degree": _field("parallel", "INT", 1, min=1), + "ring_degree": _field("parallel", "INT", 1, min=1), + "attention_backend": _field( + "attention", + ["auto", "flash_attention", "torch_sdpa", "sage_attention2"] + + ["sage_attention3", "sol_attention", "subblock_sparse_attn"] + + ["video_sparse_attn_h3", "hybrid_window_attn_h3"], + "auto", + "auto uses the model default; an unavailable backend is rejected.", + ), + "attention_backend_config": _field( + "attention", "STRING", "", "Same format as --attention-backend-config." + ), + "quantization": _field( + "quantization", + ["none", "fp8", "convrot_int8", "mxfp8"], + "none", + "Online quantization; native model directories only.", + ), + "int8_gemm": _field( + "quantization", + list(_INT8_GEMMS), + "auto", + "GEMM kernel for INT8 ConvRot weights; auto prefers sglang_jit.", + ), + "cache_mode": _field( + "cache", + ["off", "cache_dit", "teacache", "spectrum"], + "off", + "Lossy step caching; changing it keeps the worker.", + ), + "cache_dit_tuning": _field( + "cache", ["auto", "manual"], "auto", "manual sets the values below." + ), + "cache_warmup_steps": _field("cache", "INT", 4, "Full steps first.", min=0), + "cache_threshold": _field( + "cache", "FLOAT", 0.24, "Higher caches more steps.", min=0.0, step=0.01 + ), + "cache_max_cached_steps": _field("cache", "INT", 3, "Most in a row.", min=0), + "cache_fn_blocks": _field("cache", "INT", 1, "Leading blocks computed.", min=0), + "execution_mode": _field( + "execution", ["eager", "torch_compile", "cuda_graph"], "eager" + ), + "regional_compile": _field("execution", "BOOLEAN", False, "torch_compile only."), + "memory_mode": _field( + "memory", + ["resident", "layerwise_offload"], + "resident", + "DiT only; ComfyUI owns the text encoder and VAE.", + ), + "resident_layers": _field("memory", "INT", 0, "Kept on GPU.", min=0), + "prefetch_layers": _field("memory", "INT", 0, "Prefetched; 0 means 1.", min=0), +} + + +class SGLDOptions: + """Grouped runtime options; web/options.js folds the groups.""" + + RETURN_TYPES = ("SGLD_OPTIONS",) + RETURN_NAMES = ("sgld_options",) + FUNCTION = "create_options" + CATEGORY = "SGLDiffusion" + DESCRIPTION = ( + "Runtime options for the SGLD worker. Combinations the selected model " + "cannot run are rejected when the loader runs, before the worker starts." + ) + + @classmethod + def INPUT_TYPES(cls): + return {"required": {}, "optional": _FIELDS} + + def create_options(self, **values): + # ComfyUI has already checked each value against INPUT_TYPES. + v = {name: spec[1]["default"] for name, spec in _FIELDS.items()} | values + execution = v["execution_mode"] + options = { + "num_gpus": v["num_gpus"], + "enable_torch_compile": execution == "torch_compile", + "regional_compile": execution == "torch_compile" and v["regional_compile"], + "enable_breakable_cuda_graph": execution == "cuda_graph", + "dit_layerwise_offload": v["memory_mode"] == "layerwise_offload", + "int8_kernel": _INT8_GEMMS[v["int8_gemm"]], + "cache_mode": v["cache_mode"], + } + if v["model_type"] != "auto-detect": + options["model_type"] = v["model_type"] + if v["parallel_mode"] == "manual": + options.update(_manual_layout(v)) + if options["dit_layerwise_offload"]: + options["dit_layerwise_resident_layers"] = v["resident_layers"] + options["dit_offload_prefetch_size"] = v["prefetch_layers"] + backend = v["attention_backend"] + if backend != "auto": + options["attention_backend"] = _ATTENTION_IDS.get(backend, backend) + if v["attention_backend_config"].strip(): + options["attention_backend_config"] = v["attention_backend_config"].strip() + if v["quantization"] != "none": + options["quantization"] = v["quantization"] + if v["cache_mode"] == "cache_dit" and v["cache_dit_tuning"] == "manual": + options["cache_dit_params"] = { + "max_warmup_steps": v["cache_warmup_steps"], + "residual_diff_threshold": v["cache_threshold"], + "max_continuous_cached_steps": v["cache_max_cached_steps"], + "Fn_compute_blocks": v["cache_fn_blocks"], + } + return (options,) + + +def _manual_layout(v: dict) -> dict: + tp, ulysses, ring = v["tp_size"], v["ulysses_degree"], v["ring_degree"] + if tp * ulysses * ring != v["num_gpus"]: + raise ValueError( + f"num_gpus ({v['num_gpus']}) must equal tp_size * ulysses_degree * " + f"ring_degree ({tp * ulysses * ring})" + ) + return { + "tp_size": tp, + "sp_degree": ulysses * ring, + "ulysses_degree": ulysses, + "ring_degree": ring, + } diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options.js b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options.js new file mode 100644 index 0000000000..68b781520d --- /dev/null +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options.js @@ -0,0 +1,91 @@ +import { app } from '../../../scripts/app.js'; +import { GROUPS, defaultBackendConfig, headerLabel, visibleField } from './options_state.mjs'; + +app.registerExtension({ + name: 'SGLDiffusion.Options', + beforeRegisterNodeDef(nodeType, nodeData) { + if (nodeData.name !== 'SGLDOptions') return; + const created = nodeType.prototype.onNodeCreated; + nodeType.prototype.onNodeCreated = function (...args) { + const result = created?.apply(this, args); + const node = this; + const fields = nodeData.input.optional; + const originals = [...node.widgets]; + const fieldWidgets = originals.filter(w => fields[w.name]); + const states = new WeakMap(); + const headers = {}; + node.properties ??= {}; + const sections = () => node.properties.sgld_sections ??= {}; + const values = () => Object.fromEntries(fieldWidgets.map(w => [w.name, w.value])); + for (const [group, label] of Object.entries(GROUPS)) { + const header = node.addWidget('button', label, null, () => { + sections()[group] = !sections()[group]; + refresh(); + }, { serialize: false }); + header.serializeValue = () => undefined; + headers[group] = header; + } + // Never reorder the actual input widgets relative to one another. + const ordered = []; + const inserted = new Set(); + for (const widget of originals) { + const group = fields[widget.name]?.[1]?.sgld_group; + if (headers[group] && !inserted.has(group)) { + ordered.push(headers[group]); + inserted.add(group); + } + ordered.push(widget); + } + node.widgets = ordered; + + function setVisible(widget, visible) { + if (!states.has(widget)) states.set(widget, { type: widget.type, computeSize: widget.computeSize, hidden: widget.hidden }); + const original = states.get(widget); + widget.hidden = visible ? original.hidden : true; + widget.type = visible ? original.type : 'sgld-hidden'; + widget.computeSize = visible ? original.computeSize : () => [0, -4]; + } + function refresh() { + const v = values(); + for (const widget of fieldWidgets) { + const group = fields[widget.name][1].sgld_group; + // Connected primitives must keep their inputs reachable. + const connected = node.inputs?.some((input, index) => + input.widget?.name === widget.name && node.isInputConnected?.(index)); + setVisible(widget, connected || ((group === 'basic' || sections()[group]) && visibleField(widget.name, v))); + } + for (const [group, header] of Object.entries(headers)) { + header.name = headerLabel(group, v, sections()[group]); + } + node.setSize([Math.max(node.size[0], 420), node.computeSize()[1]]); + node.setDirtyCanvas(true, true); + } + for (const widget of fieldWidgets) { + const callback = widget.callback; + widget.callback = function (...args) { + const result = callback?.apply(this, args); + if (widget.name === 'attention_backend') { + const config = fieldWidgets.find(w => w.name === 'attention_backend_config'); + if (config) config.value = defaultBackendConfig(widget.value, config.value); + } + refresh(); + return result; + }; + } + const configure = node.onConfigure; + node.onConfigure = function (...args) { + const result = configure?.apply(this, args); + refresh(); + return result; + }; + const connections = node.onConnectionsChange; + node.onConnectionsChange = function (...args) { + const result = connections?.apply(this, args); + queueMicrotask(refresh); + return result; + }; + refresh(); + return result; + }; + }, +}); diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.mjs b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.mjs new file mode 100644 index 0000000000..35bc04a807 --- /dev/null +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.mjs @@ -0,0 +1,64 @@ +// Presentation only: which fields show and what each folded header summarizes. +// Combination rules live in core/preflight.py and run when the loader executes. +export const GROUPS = { + parallel: 'Parallel', + attention: 'Attention', + quantization: 'Quantization', + cache: 'Cache', + execution: 'Execution', + memory: 'Memory', +}; + +const SHORT = { + torch_compile: 'compile', + cuda_graph: 'CUDA graph', + layerwise_offload: 'offload', + comfy_kitchen: 'kitchen', + sglang_jit: 'JIT', + flash_attention: 'flash attention', + sage_attention2: 'sage 2', + sage_attention3: 'sage 3', + sol_attention: 'sol', + cache_dit: 'Cache-DiT', +}; +const short = value => SHORT[value] ?? value; + +const CACHE_KNOBS = ['cache_warmup_steps', 'cache_threshold', 'cache_max_cached_steps', 'cache_fn_blocks']; + +// Written into an empty attention_backend_config when the backend is picked, so +// its knobs are visible and editable; the values match the backend's defaults. +const BACKEND_DEFAULT_CONFIGS = { + sol_attention: 'dense_steps=10,dense_layers=0-1,dense_backend=fa,tau=1.0,thresh_type=diag', +}; + +export function defaultBackendConfig(backend, current) { + return current ? current : (BACKEND_DEFAULT_CONFIGS[backend] ?? current); +} + +export function visibleField(name, v) { + if (['tp_size', 'ulysses_degree', 'ring_degree'].includes(name)) return v.parallel_mode === 'manual'; + if (name === 'regional_compile') return v.execution_mode === 'torch_compile'; + if (['resident_layers', 'prefetch_layers'].includes(name)) return v.memory_mode === 'layerwise_offload'; + if (name === 'cache_dit_tuning') return v.cache_mode === 'cache_dit'; + if (CACHE_KNOBS.includes(name)) return v.cache_mode === 'cache_dit' && v.cache_dit_tuning === 'manual'; + return true; +} + +// Kept to one or two words so the folded headers line up. +export function groupSummary(group, v) { + switch (group) { + case 'execution': return short(v.execution_mode); + case 'attention': return short(v.attention_backend); + case 'cache': return short(v.cache_mode); + case 'memory': return short(v.memory_mode); + case 'parallel': return v.parallel_mode === 'manual' + ? `TP${v.tp_size} U${v.ulysses_degree} R${v.ring_degree}` : 'auto'; + case 'quantization': return v.int8_gemm === 'auto' + ? v.quantization : `${v.quantization}, ${short(v.int8_gemm)}`; + default: return ''; + } +} + +export function headerLabel(group, v, open) { + return `${open ? '▾' : '▸'} ${GROUPS[group]} · ${groupSummary(group, v)}`; +} diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.test.mjs b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.test.mjs new file mode 100644 index 0000000000..b2a001bb2e --- /dev/null +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/web/options_state.test.mjs @@ -0,0 +1,32 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { defaultBackendConfig, groupSummary, headerLabel, visibleField } from './options_state.mjs'; + +const base = { + model_type: 'minimax_h3', num_gpus: 1, parallel_mode: 'auto', execution_mode: 'eager', + attention_backend: 'auto', cache_mode: 'off', cache_dit_tuning: 'auto', memory_mode: 'resident', + tp_size: 1, ulysses_degree: 1, ring_degree: 1, quantization: 'none', int8_gemm: 'auto', +}; + +test('dependent fields follow their selector', () => { + assert.equal(visibleField('ulysses_degree', base), false); + assert.equal(visibleField('ulysses_degree', { ...base, parallel_mode: 'manual' }), true); + assert.equal(visibleField('parallel_mode', base), true); + assert.equal(visibleField('regional_compile', { ...base, execution_mode: 'torch_compile' }), true); + assert.equal(visibleField('resident_layers', { ...base, memory_mode: 'layerwise_offload' }), true); + assert.equal(visibleField('cache_threshold', { ...base, cache_mode: 'cache_dit' }), false); + assert.equal(visibleField('cache_threshold', { ...base, cache_mode: 'cache_dit', cache_dit_tuning: 'manual' }), true); +}); + +test('sol_attention fills an empty backend config but never overwrites one', () => { + assert.match(defaultBackendConfig('sol_attention', ''), /dense_steps=10/); + assert.equal(defaultBackendConfig('sol_attention', 'tau=2.0'), 'tau=2.0'); + assert.equal(defaultBackendConfig('flash_attention', ''), ''); +}); + +test('folded headers stay short', () => { + assert.equal(groupSummary('parallel', { ...base, parallel_mode: 'manual', ulysses_degree: 2, num_gpus: 2 }), + 'TP1 U2 R1'); + assert.equal(groupSummary('quantization', { ...base, int8_gemm: 'comfy_kitchen' }), 'none, kitchen'); + assert.equal(headerLabel('execution', { ...base, execution_mode: 'cuda_graph' }, false), '▸ Execution · CUDA graph'); +}); diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/flux_sgld_sp.json b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/flux_sgld_sp.json index d158a07084..010ffd4014 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/flux_sgld_sp.json +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/flux_sgld_sp.json @@ -202,17 +202,11 @@ "52": { "inputs": { "model_type": "auto-detect", - "enable_torch_compile": false, "num_gpus": 2, - "tp_size": -1, - "sp_degree": -1, - "ulysses_degree": -1, - "ring_degree": -1, - "dp_size": 1, - "dp_degree": 1, - "enable_cfg_parallel": false, - "attention_backend": "", - "cache_strategy": "none" + "parallel_mode": "auto", + "execution_mode": "eager", + "attention_backend": "auto", + "memory_mode": "resident" }, "class_type": "SGLDOptions", "_meta": { diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_r2v_sgld.json b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_r2v_sgld.json index c5ee0c712e..e073a7a7b2 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_r2v_sgld.json +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_r2v_sgld.json @@ -11,16 +11,11 @@ "class_type": "SGLDOptions", "inputs": { "model_type": "minimax_h3", - "enable_torch_compile": false, "num_gpus": 1, - "tp_size": -1, - "sp_degree": -1, - "ulysses_degree": -1, - "ring_degree": -1, - "dp_size": 1, - "dp_degree": 1, - "enable_cfg_parallel": false, - "attention_backend": "" + "parallel_mode": "auto", + "execution_mode": "eager", + "attention_backend": "auto", + "memory_mode": "resident" } }, "13": { diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld.json b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld.json index d6252320d4..ca31f45ece 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld.json +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld.json @@ -14,16 +14,11 @@ "class_type": "SGLDOptions", "inputs": { "model_type": "minimax_h3", - "enable_torch_compile": false, "num_gpus": 1, - "tp_size": -1, - "sp_degree": -1, - "ulysses_degree": -1, - "ring_degree": -1, - "dp_size": 1, - "dp_degree": 1, - "enable_cfg_parallel": false, - "attention_backend": "" + "parallel_mode": "auto", + "execution_mode": "eager", + "attention_backend": "auto", + "memory_mode": "resident" }, "_meta": { "title": "SGLDiffusion Options" diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld_upscaler.json b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld_upscaler.json index dbeae783f6..6681ea6c13 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld_upscaler.json +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/workflows/minimax_h3_t2v_sgld_upscaler.json @@ -21,20 +21,11 @@ "class_type": "SGLDOptions", "inputs": { "model_type": "minimax_h3", - "enable_torch_compile": false, "num_gpus": 1, - "tp_size": -1, - "sp_degree": -1, - "ulysses_degree": -1, - "ring_degree": -1, - "dp_size": 1, - "dp_degree": 1, - "enable_cfg_parallel": false, - "attention_backend": "", - "dit_layerwise_offload": false, - "enable_cache_dit": false, - "quantization": "", - "transformer_weights_path": "" + "parallel_mode": "auto", + "execution_mode": "eager", + "attention_backend": "auto", + "memory_mode": "resident" } }, "13": { diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py index 2989e51599..e22cec2525 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/stages/model_specific_stages/minimax_h3/stages/comfyui_step.py @@ -837,6 +837,11 @@ class MiniMaxH3ComfyUIStepStage(PipelineStage): stage = self._cache_dit_stage() step = state.steps_done + n_sigmas = _sample_sigmas_len(state.sample_sigmas) + if batch.enable_spectrum and n_sigmas is not None: + # Each ComfyUI step is a one-step request; Spectrum schedules (and + # resets) against the whole run, as Cache-DiT does at mount time. + batch.num_inference_steps = n_sigmas - 1 attn_metadata = ( state.step_attn_metadata(step) if state.step_attn_metadata is not None diff --git a/python/sglang/multimodal_gen/runtime/server_args/server_args.py b/python/sglang/multimodal_gen/runtime/server_args/server_args.py index 3b8d9e739f..4954a099a0 100644 --- a/python/sglang/multimodal_gen/runtime/server_args/server_args.py +++ b/python/sglang/multimodal_gen/runtime/server_args/server_args.py @@ -832,7 +832,9 @@ class ServerArgs(DisaggServerArgsMixin): pipeline_config_name, self.model_path, ) - if not self.warmup_resolutions: + # ComfyUI requests carry the sampler's tensors, which a synthetic warmup + # cannot build; the ComfyUI step stage captures on its first real step. + if not self.warmup_resolutions and not self.comfyui_mode: self._default_bcg_warmup_resolution() def _is_breakable_cuda_graph_supported_model(self) -> bool: @@ -1512,7 +1514,11 @@ class ServerArgs(DisaggServerArgsMixin): # BCG captures every graph during a synthetic warmup forward at startup # so serving never records a fresh graph. - if self.enable_breakable_cuda_graph and self.disagg_role == RoleType.MONOLITHIC: + if ( + self.enable_breakable_cuda_graph + and self.disagg_role == RoleType.MONOLITHIC + and not self.comfyui_mode + ): self.warmup_mode = "server" # Disaggregated roles do not host the HTTP startup request. Preserve diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py index d621c0931c..aedc82201e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py @@ -294,12 +294,12 @@ def _h3_runtime_that_must_not_start() -> SGLDiffusionGenerator: "key, sgld_options, error", [ # A FastH3 file loaded as base H3 died in the worker on to_gate_compress. - ("blocks.0.attn.to_gate_compress.weight", {}, "model_type fast_h3"), - # The H3 file loader builds no quant config, so this was silently ignored. + ("blocks.0.attn.to_gate_compress.weight", {}, "SGLDNativeModelLoader"), + # The single-file loader builds no quant config, so this was silently ignored. ( "blocks.0.attn.to_q.weight", {"quantization": "convrot_int8"}, - "cannot be quantized at load time", + "native model directories", ), ], ) @@ -331,16 +331,17 @@ def test_fasth3_runtime_dir_is_checked_before_worker_load(tmp_path) -> None: (tmp_path / "transformer").mkdir() index = tmp_path / "transformer" / "diffusion_pytorch_model.safetensors.index.json" runtime = _h3_runtime_that_must_not_start() - options = {"model_type": "fast_h3", "runtime_model_path": str(tmp_path)} for dmd, weight_map, message in ( (None, {}, "no trained DMD rungs"), ([999, 500], {"blocks.0.x": "a.safetensors"}, "raw FastH3 download"), ): meta = dict(release, **({"dmd_denoising_steps": dmd} if dmd else {})) - (tmp_path / "model_index.json").write_text(json.dumps({"_minimax_h3": meta})) + (tmp_path / "model_index.json").write_text( + json.dumps({"_class_name": "FastH3Pipeline", "_minimax_h3": meta}) + ) index.write_text(json.dumps({"weight_map": weight_map})) with pytest.raises(ValueError, match=message): - runtime.load_model(model_path="h3.safetensors", sgld_options=options) + runtime.load_model(model_path=str(tmp_path), sgld_options={}) @pytest.mark.parametrize( diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py new file mode 100644 index 0000000000..508488458d --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_options.py @@ -0,0 +1,143 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SGLDOptions output, model-aware preflight, and request wiring.""" + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch + +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import ( + SGLDiffusionGenerator, + _apply_cache_mode, +) +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.preflight import ( + check_model_options, +) +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.options import SGLDOptions + + +def _options(**values) -> dict: + return SGLDOptions().create_options(**values)[0] + + +def test_cache_mode_reaches_the_h3_request() -> None: + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.minimax_h3 import ( + MiniMaxH3Executor, + ) + from sglang.multimodal_gen.runtime.cache.cache_dit_integration import ( + resolve_cache_dit_request_overrides, + ) + + assert "cache_dit_params" not in _options(cache_mode="cache_dit") + options = _options(cache_mode="cache_dit", cache_dit_tuning="manual") + assert resolve_cache_dit_request_overrides(options["cache_dit_params"]) + + executor = object.__new__(MiniMaxH3Executor) + torch.nn.Module.__init__(executor) + packed = SimpleNamespace(guidance_scale=1.0, height=512, width=512) + for values, cache_dit, spectrum in ( + ({"cache_mode": "cache_dit", "cache_dit_tuning": "manual"}, True, False), + ({"cache_mode": "spectrum"}, False, True), + ): + _apply_cache_mode(executor, _options(**values)) + request = executor._sampling_params_kwargs(packed, torch.tensor(1.0)) + assert request["enable_cache_dit"] is cache_dit + assert request.get("enable_spectrum", False) is spectrum + assert ("cache_dit_params" in request) is cache_dit + + +def test_manual_layout_derives_sp_and_checks_gpu_count() -> None: + options = _options(num_gpus=4, parallel_mode="manual", tp_size=2, ulysses_degree=2) + assert (options["tp_size"], options["sp_degree"]) == (2, 2) + with pytest.raises(ValueError, match="must equal"): + _options(num_gpus=2, parallel_mode="manual") + + +_RING2 = {"num_gpus": 2, "parallel_mode": "manual", "ring_degree": 2} +_GRAPH = {"execution_mode": "cuda_graph"} + + +@pytest.mark.parametrize( + "model_type, values, native_dir, message", + [ + ("flux", _GRAPH, False, "cuda_graph is not"), + ("minimax_h3", {**_GRAPH, "cache_mode": "spectrum"}, False, "cache_mode=off"), + ( + "minimax_h3", + {**_GRAPH, "memory_mode": "layerwise_offload"}, + False, + "offload", + ), + ("fast_h3", {"execution_mode": "torch_compile"}, True, "eager"), + ("vdn_h3", _RING2, True, "ring_degree=1"), + ( + "minimax_h3", + {**_RING2, "attention_backend": "sage_attention2"}, + False, + "=fa", + ), + ("vdn_h3", {"attention_backend": "flash_attention"}, True, "does not apply"), + ("qwen_image", {"attention_backend": "sol_attention"}, False, "does not apply"), + ("flux", {"quantization": "fp8"}, False, "native model directories"), + ("lumina2", {"int8_gemm": "comfy_kitchen"}, False, "no effect"), + ("qwen_image", {"cache_mode": "cache_dit"}, False, "cache_dit is not"), + ("minimax_h3", {"cache_mode": "teacache"}, False, "teacache is not"), + ("minimax_h3", {"num_gpus": 2, "cache_mode": "spectrum"}, False, "degree 1"), + ], +) +def test_preflight_rejects_what_the_model_cannot_run( + model_type, values, native_dir, message +) -> None: + with pytest.raises(ValueError, match=message): + check_model_options(model_type, _options(**values), native_dir=native_dir) + + +@pytest.mark.parametrize( + "model_type, values, native_dir", + [ + ("minimax_h3", _GRAPH, False), + ("minimax_h3", _RING2, False), + ("minimax_h3", {"int8_gemm": "comfy_kitchen", "cache_mode": "spectrum"}, False), + ("fast_h3", {"attention_backend": "video_sparse_attn_h3"}, True), + ("vdn_h3", {"quantization": "fp8", "cache_mode": "cache_dit"}, True), + ], +) +def test_preflight_accepts_supported_combinations( + model_type, values, native_dir +) -> None: + check_model_options(model_type, _options(**values), native_dir=native_dir) + + +def test_native_models_are_not_selected_through_options() -> None: + with pytest.raises(ValueError, match="SGLDNativeModelLoader"): + SGLDiffusionGenerator().load_model( + "h3.safetensors", sgld_options={"model_type": "fast_h3"} + ) + + +def test_rejected_options_keep_the_running_worker() -> None: + """A config error used to surface only after the previous worker was killed.""" + runtime = SGLDiffusionGenerator() + runtime.get_comfyui_model = lambda *a: (SimpleNamespace(), None, "flux") + runtime.close_generator = Mock() + runtime.init_generator = Mock(side_effect=AssertionError("worker must not start")) + with pytest.raises(ValueError, match="cuda_graph is not supported"): + runtime.load_model("flux.safetensors", sgld_options=_options(**_GRAPH)) + runtime.close_generator.assert_not_called() + + +def test_breakable_cuda_graph_keeps_comfyui_warmup_off() -> None: + """BCG forced a synthetic server warmup, which the ComfyUI step stage cannot + run (no sampler tensors), so every cuda_graph worker failed at startup.""" + from sglang.multimodal_gen.runtime.server_args import ServerArgs + + args = ServerArgs.from_kwargs( + model_path="h3.safetensors", + pipeline_class_name="MiniMaxH3Pipeline", + model_id="MiniMax-H3", + comfyui_mode=True, + enable_breakable_cuda_graph=True, + warmup_mode="off", + ) + assert (args.warmup_mode, args.warmup_resolutions) == ("off", None)