[Feat] diffusion: Add worker-side ComfyUI compatibility patches (#44841)

This commit is contained in:
WenhaoZhang
2026-10-21 20:41:51 -07:00
committed by GitHub
parent 9ee9858805
commit 45de4adde9
5 changed files with 538 additions and 4 deletions
@@ -322,7 +322,11 @@ class SGLDiffusionGenerator:
_SHARED.close_generator()
def init_generator(
self, model_path: str, pipeline_class_name: str, kwargs: dict = None
self,
model_path: str,
pipeline_class_name: str,
kwargs: dict = None,
worker_patches=(),
):
"""Initialize the diffusion generator."""
if self.generator is not None:
@@ -352,7 +356,11 @@ class SGLDiffusionGenerator:
kwargs.setdefault("model_id", _H3_MODEL_IDS[pipeline_class_name])
kwargs = self._server_args_kwargs(kwargs)
try:
with _spawn_without_launcher_main():
from sglang.multimodal_gen.apps.comfyui_worker_patches import (
scheduler_profiles,
)
with scheduler_profiles(worker_patches), _spawn_without_launcher_main():
self.generator = DiffGenerator.from_pretrained(
model_path=model_path,
pipeline_class_name=pipeline_class_name,
@@ -576,6 +584,13 @@ class SGLDiffusionGenerator:
)
check_sgld_options(sgld_options)
plugin = {k: sgld_options.pop(k) for k in PLUGIN_KEYS if k in sgld_options}
from sglang.multimodal_gen.apps.comfyui_worker_patches import (
canonical_profiles,
validate_options,
)
patches = canonical_profiles(plugin.get("worker_patches"))
reload_kwargs["sgld_options"]["worker_patches"] = patches
set_model_type = plugin.get("model_type")
if set_model_type in NATIVE_MODEL_TYPES:
raise ValueError(
@@ -605,8 +620,10 @@ class SGLDiffusionGenerator:
"set_model_type": set_model_type,
"runtime_model_path": runtime_model_path,
"int8_kernel": plugin.get("int8_kernel", "auto"),
"worker_patch_identity": patches,
}
if self._can_reuse(gather_options):
validate_options(patches, self.model_type, {**sgld_options, **plugin})
check_model_options(
self.model_type, {**sgld_options, **plugin}, native_dir=native_dir
)
@@ -624,6 +641,7 @@ class SGLDiffusionGenerator:
check_model_options(
model_type, {**sgld_options, **plugin}, native_dir=native_dir
)
validate_options(patches, model_type, {**sgld_options, **plugin})
if model_type == "fast_h3":
load_fasth3_release(runtime_model_path)
if model_type == "minimax_h3":
@@ -645,7 +663,10 @@ class SGLDiffusionGenerator:
)
with _worker_env(worker_env):
self.generator = self.init_generator(
worker_model_path, pipeline_class_name, sgld_options
worker_model_path,
pipeline_class_name,
sgld_options,
worker_patches=patches,
)
self.model_type = model_type
@@ -6,7 +6,13 @@ PLUGIN_KEYS, which core/generator.py pops before building ServerArgs. Checks tha
need the resolved model live in core/preflight.py.
"""
PLUGIN_KEYS = ("model_type", "cache_mode", "cache_dit_params", "int8_kernel")
PLUGIN_KEYS = (
"model_type",
"cache_mode",
"cache_dit_params",
"int8_kernel",
"worker_patches",
)
# Display names -> SGLang backend names; the rest are passed through unchanged.
_ATTENTION_IDS = {
@@ -91,6 +97,16 @@ _FIELDS = {
),
"resident_layers": _field("memory", "INT", 0, "Kept on GPU.", min=0),
"prefetch_layers": _field("memory", "INT", 0, "Prefetched; 0 means 1.", min=0),
# Append fields to preserve positional widget values in existing workflows.
"alignment_mode": _field(
"basic",
["default", "comfyui"],
"default",
"comfyui matches native ComfyUI arithmetic (currently H3 only): single GPU, "
"resident BF16, eager, no LoRA/quantization/cache. Auto attention uses "
"torch_sdpa; incompatible explicit settings are rejected. Changing mode "
"reloads the worker.",
),
}
@@ -131,6 +147,12 @@ class SGLDOptions:
options["dit_layerwise_resident_layers"] = v["resident_layers"]
options["dit_offload_prefetch_size"] = v["prefetch_layers"]
backend = v["attention_backend"]
if v["alignment_mode"] == "comfyui":
options["worker_patches"] = ["h3_comfyui_v1"]
if backend == "auto":
backend = "torch_sdpa"
elif v["alignment_mode"] != "default":
raise ValueError(f"Unknown alignment_mode: {v['alignment_mode']}")
if backend != "auto":
options["attention_backend"] = _ATTENTION_IDS.get(backend, backend)
if v["attention_backend_config"].strip():
@@ -0,0 +1,77 @@
# SPDX-License-Identifier: Apache-2.0
"""ComfyUI worker patches. Keep module scope import-safe for multiprocessing spawn."""
import logging
import threading
from contextlib import contextmanager
from dataclasses import dataclass, replace
from functools import partial, wraps
from importlib import import_module
# Version is part of the name (and therefore the generator's reuse key).
PROFILES = {"h3_comfyui_v1": "h3"}
_STARTUP_LOCK = threading.RLock()
def canonical_profiles(names):
if names is None:
return ()
if not isinstance(names, (tuple, list)) or any(
not isinstance(name, str) or name not in PROFILES for name in names
):
raise ValueError("Unknown worker patch; choose a registered profile")
names = tuple(dict.fromkeys(names))
if len(names) > 1:
raise ValueError("Select one worker patch; composition is not supported")
return names
def validate_options(names, model_type, options):
for name in canonical_profiles(names):
import_module(f"{__name__}.{PROFILES[name]}").validate(model_type, options)
@dataclass(frozen=True)
class _PatchedArgs:
original: object
name: str
def materialize(self):
# Stock bootstrap calls this after platform and plugin initialization.
args = self.original.materialize()
from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker
install = import_module(f"{__name__}.{PROFILES[self.name]}").install
original_init = GPUWorker.__init__
@wraps(original_init)
def init(worker, *a, **kw):
original_init(worker, *a, **kw)
# Synchronous: missing pipeline/failed installation aborts startup.
install(worker.pipeline)
logging.getLogger(__name__).info("Worker patch applied: %s", self.name)
GPUWorker.__init__ = init
return args
def _bootstrap(target, name, spec):
return target(replace(spec, server_args=_PatchedArgs(spec.server_args, name)))
@contextmanager
def scheduler_profiles(names):
names = canonical_profiles(names)
from sglang.multimodal_gen.runtime import launch_server
# Serialize patched and default launches; restore the parent even on failure.
with _STARTUP_LOCK:
original = launch_server.bootstrap_scheduler_process
if names:
launch_server.bootstrap_scheduler_process = partial(
_bootstrap, original, names[0]
)
try:
yield
finally:
launch_server.bootstrap_scheduler_process = original
@@ -0,0 +1,201 @@
# SPDX-License-Identifier: Apache-2.0
"""ComfyUI H3 BF16 compatibility implemented outside runtime model sources.
Module instances are patched once after load, before first forward. Cloning a
function's globals keeps helper overrides local to that bound method; original
classes, module globals and other instances remain unchanged. Eager only.
"""
import hashlib
import importlib.metadata
import inspect
import types
from pathlib import Path
import torch
# A changed upstream implementation must be reviewed, never best-effort patched.
SOURCE_SHA256 = "ced7b679ee05f75045646cb81a436088557b90f23d3be980aba13f816ac23f0d"
def bind_with_globals(instance, name, overrides):
original = getattr(instance, name).__func__
scope = dict(original.__globals__)
scope.update(overrides)
fn = types.FunctionType(
original.__code__,
scope,
original.__name__,
original.__defaults__,
original.__closure__,
)
fn.__kwdefaults__ = original.__kwdefaults__
fn.__annotations__ = original.__annotations__
setattr(instance, name, types.MethodType(fn, instance))
def comfy_gate(x, gate, other, indices, *, dtype, allow_inplace=True):
return torch.addcmul(
x, other, gate.index_select(0, indices), out=x if allow_inplace else None
)
def comfy_rope(q, k, qw, kw, cache, positions, **kwargs):
# Match ComfyUI Kitchen v0.2.36 HasRms float rotation exactly.
import comfy_kitchen as ck
table = cache.index_select(0, positions)
cos, sin = table.chunk(2, dim=-1)
table = torch.stack([cos, -sin, sin, cos], dim=-1).reshape(
1, len(q), 1, cos.shape[-1], 2, 2
)
ck.rms_rope_split_half_(
q.unsqueeze(0),
k.unsqueeze(0),
table,
qw,
kw,
epsilon=kwargs["eps"],
rot_dim=cache.shape[-1],
)
def _reject_lora(*args, **kwargs):
raise ValueError("h3_comfyui_v1 does not support LoRA")
def validate(model_type, options):
if model_type != "minimax_h3":
raise ValueError("h3_comfyui_v1 requires model_type=minimax_h3")
if options.get("num_gpus", 1) != 1:
raise ValueError("h3_comfyui_v1 currently requires one GPU")
for degree in ("tp_size", "sp_degree", "ulysses_degree", "ring_degree"):
if options.get(degree) not in (None, -1, 1):
raise ValueError(f"h3_comfyui_v1 requires {degree}=1 or auto")
if (
options.get("enable_torch_compile")
or options.get("enable_cuda_graph")
or options.get("enable_breakable_cuda_graph")
):
raise ValueError("h3_comfyui_v1 currently requires eager execution")
if options.get("cache_mode", "off") != "off" or options.get("quantization"):
raise ValueError("h3_comfyui_v1 requires no cache/quantization")
if options.get("attention_backend") != "torch_sdpa":
raise ValueError("h3_comfyui_v1 parity is verified only with torch_sdpa")
if (
options.get("transformer_weights_path")
or options.get("dit_layerwise_offload")
or options.get("dit_cpu_offload")
):
raise ValueError("h3_comfyui_v1 requires resident single-file BF16 weights")
for key in (
"component_attention_backends",
"component_quantizations",
"component_precisions",
"lora_path",
"minimax_h3_adaln_online",
"minimax_h3_adaln_cache_path",
"enable_cfg_parallel",
"use_fsdp_inference",
):
if options.get(key):
raise ValueError(f"h3_comfyui_v1 has not been validated with {key}")
def install(pipeline):
from sglang.multimodal_gen.runtime.distributed import get_tp_world_size
from sglang.multimodal_gen.runtime.distributed.parallel_state import (
get_ring_ctx,
get_ulysses_ctx,
)
from sglang.multimodal_gen.runtime.models.dits import minimax_h3 as h3
source = Path(inspect.getsourcefile(h3)).read_bytes()
if hashlib.sha256(source).hexdigest() != SOURCE_SHA256:
raise RuntimeError(
"H3 upstream source differs from the reviewed h3_comfyui_v1 source"
)
if importlib.metadata.version("comfy-kitchen") != "0.2.36":
raise RuntimeError("h3_comfyui_v1 requires reviewed comfy-kitchen==0.2.36")
if get_tp_world_size() != 1 or get_ring_ctx()[0] != 1 or get_ulysses_ctx()[0] != 1:
raise ValueError("h3_comfyui_v1 is single-GPU only")
models = [m for m in pipeline.modules.values() if type(m) is h3.MiniMaxH3DiTModel]
if len(models) != 1:
raise ValueError("h3_comfyui_v1 requires exactly one base H3 DiT")
model = models[0]
if next(model.parameters()).device.type != "cuda":
raise ValueError("h3_comfyui_v1 requires a resident CUDA model")
if torch.cuda.get_device_capability() != (9, 0):
raise ValueError("h3_comfyui_v1 is currently verified only on Hopper")
if (
model.adaln_t_table is not None
or model.adaln_cache is not None
or model.final_layer._pdd_heads is not None
):
raise ValueError("h3_comfyui_v1 does not support PDD/curve/cached AdaLN")
for name, module in model.named_modules():
method = getattr(module, "quant_method", None)
if method is not None and type(method).__name__ != "UnquantizedLinearMethod":
raise ValueError(
f"h3_comfyui_v1 does not support quantized layer {name}: {type(method).__name__}"
)
if getattr(model, "_comfy_h3_installed", False):
return
# Validate every dependency before modifying any method.
import comfy_kitchen as ck
if not callable(ck.rms_rope_split_half_):
raise TypeError("Missing ComfyUI-compatible RMSNorm/RoPE kernel")
for block in model.blocks:
if not callable(getattr(block.attn, "_attend", None)):
raise TypeError("Missing reviewed H3 attention entry point")
embed = model._embed
def embedding(self, *a, **kw):
hidden, time = embed(*a, **kw)
return hidden, time.to(torch.bfloat16)
model._embed = types.MethodType(embedding, model)
for block in model.blocks:
bind_with_globals(
block,
"forward",
{
"_modulate_gate": comfy_gate,
"minimax_h3_norm_modulate_active": lambda _: False,
},
)
bind_with_globals(
block.attn, "_attend", {"fused_inplace_qknorm_rope": comfy_rope}
)
block.attn._use_fused_qknorm_rope = True
final = model.final_layer
def project(self, hidden):
vi, ai = self._comfy_output_positions
vi, ai = vi.to(hidden.device), ai.to(hidden.device)
video, _ = self.video_out(hidden.index_select(0, vi))
audio, _ = self.audio_out(hidden.index_select(0, ai))
return (
hidden.new_zeros((len(hidden), video.shape[-1])).index_copy_(0, vi, video),
hidden.new_zeros((len(hidden), audio.shape[-1])).index_copy_(0, ai, audio),
)
final._project = types.MethodType(project, final)
forward = model.forward
def model_forward(self, **kw):
final._comfy_output_positions = (
self._pos_ids(
kw["img_pos_for_infer_output_info"], "img_pos_for_infer_output_info"
),
self._pos_ids(kw["audio_pos_info"], "audio_pos_info"),
)
try:
return forward(**kw)
finally:
del final._comfy_output_positions
model.forward = types.MethodType(model_forward, model)
pipeline.set_lora = _reject_lora
model._comfy_h3_installed = True
@@ -0,0 +1,213 @@
"""Worker lifecycle, failure propagation and H3 compatibility constraints."""
import multiprocessing as mp
import os
import sys
import types
from dataclasses import dataclass
import pytest
from sglang.multimodal_gen.apps import comfyui_worker_patches as patches
def test_registered_names_only():
assert patches.canonical_profiles(None) == ()
assert patches.canonical_profiles(["h3_comfyui_v1"] * 2) == ("h3_comfyui_v1",)
for names in (["arbitrary.module:run"], "h3_comfyui_v1", [None]):
with pytest.raises(ValueError, match="Unknown"):
patches.canonical_profiles(names)
def test_composition_rejected(monkeypatch):
monkeypatch.setitem(patches.PROFILES, "other", "other")
with pytest.raises(ValueError, match="one worker patch"):
patches.canonical_profiles(["h3_comfyui_v1", "other"])
def test_wrong_family_rejected():
with pytest.raises(ValueError, match="model_type"):
patches.validate_options(["h3_comfyui_v1"], "flux", {})
@pytest.mark.parametrize(
"option",
[
{"num_gpus": 2},
{"enable_torch_compile": True},
{"enable_cuda_graph": True},
{"quantization": "fp8"},
{"cache_mode": "cache_dit"},
{"attention_backend": "sage_attn"},
{"component_attention_backends": {"transformer": "fa"}},
{"lora_path": "unreviewed.safetensors"},
],
)
def test_unsupported_contract_rejected(option):
with pytest.raises(ValueError):
patches.validate_options(
["h3_comfyui_v1"],
"minimax_h3",
{"attention_backend": "torch_sdpa"} | option,
)
def test_default_launch_and_restore_on_failure(monkeypatch):
original = lambda spec: None
module = types.ModuleType("sglang.multimodal_gen.runtime.launch_server")
module.bootstrap_scheduler_process = original
monkeypatch.setitem(sys.modules, module.__name__, module)
with patches.scheduler_profiles([]):
assert module.bootstrap_scheduler_process is original
with (
pytest.raises(RuntimeError, match="launch failed"),
patches.scheduler_profiles(["h3_comfyui_v1"]),
):
assert module.bootstrap_scheduler_process is not original
raise RuntimeError("launch failed")
assert module.bootstrap_scheduler_process is original
@dataclass
class _Args:
def materialize(self):
return "args-ready"
@dataclass
class _Spec:
server_args: object
def _spawn_probe(connection, fail):
# A real spawn boundary with a tiny worker instead of allocating GPU weights.
worker_module = types.ModuleType(
"sglang.multimodal_gen.runtime.managers.gpu_worker"
)
class Worker:
def __init__(self):
self.pipeline = types.SimpleNamespace(loaded=True)
worker_module.GPUWorker = Worker
sys.modules[worker_module.__name__] = worker_module
provider = types.ModuleType(patches.__name__ + ".test_provider")
def install(pipeline):
assert pipeline.loaded
if fail:
raise RuntimeError("installer failed")
pipeline.patched = True
provider.install = install
sys.modules[provider.__name__] = provider
patches.PROFILES["test"] = "test_provider"
def stock_bootstrap(spec):
assert spec.server_args.materialize() == "args-ready"
return Worker().pipeline.patched
try:
result = patches._bootstrap(stock_bootstrap, "test", _Spec(_Args()))
except RuntimeError as error:
result = str(error)
connection.send((os.getpid(), result))
connection.close()
@pytest.mark.parametrize("fail", [False, True])
def test_real_spawn_installs_after_load_or_aborts(fail):
ctx = mp.get_context("spawn")
parent, child = ctx.Pipe(duplex=False)
process = ctx.Process(target=_spawn_probe, args=(child, fail))
process.start()
child.close()
try:
assert parent.poll(60), "spawn did not report"
pid, result = parent.recv()
process.join(10)
assert process.exitcode == 0 and pid != os.getpid()
assert result == ("installer failed" if fail else True)
assert "test" not in patches.PROFILES
finally:
if process.is_alive():
process.terminate()
process.join(10)
parent.close()
def test_bound_helper_does_not_change_class_or_other_instances():
from sglang.multimodal_gen.apps.comfyui_worker_patches.h3 import bind_with_globals
class Example:
def run(self):
return os.getpid()
patched, original = Example(), Example()
function = Example.run
bind_with_globals(
patched, "run", {"os": types.SimpleNamespace(getpid=lambda: -123)}
)
assert patched.run() == -123
assert original.run() == os.getpid()
assert Example.run is function
def test_h3_dynamic_lora_rejected():
from sglang.multimodal_gen.apps.comfyui_worker_patches.h3 import _reject_lora
with pytest.raises(ValueError, match="LoRA"):
_reject_lora("adapter")
@pytest.fixture
def options_module():
# Load the schema without importing ComfyUI's node/runtime dependencies.
import importlib.util
from pathlib import Path
path = Path(patches.__file__).parent.parent / "ComfyUI_SGLDiffusion/options.py"
spec = importlib.util.spec_from_file_location("sgld_options_test", path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def test_alignment_default_preserves_existing_options(options_module):
node = options_module.SGLDOptions()
options = node.create_options()[0]
assert options == node.create_options(alignment_mode="default")[0]
assert "worker_patches" not in options
assert "attention_backend" not in options
def test_comfyui_mode_auto_attention_is_valid_for_h3(options_module):
options = options_module.SGLDOptions().create_options(alignment_mode="comfyui")[0]
assert options["worker_patches"] == ["h3_comfyui_v1"]
assert options["attention_backend"] == "torch_sdpa"
patches.validate_options(options["worker_patches"], "minimax_h3", options)
def test_comfyui_mode_preserves_and_rejects_explicit_attention(options_module):
options = options_module.SGLDOptions().create_options(
alignment_mode="comfyui", attention_backend="flash_attention"
)[0]
assert options["attention_backend"] == "fa"
with pytest.raises(ValueError, match="torch_sdpa"):
patches.validate_options(options["worker_patches"], "minimax_h3", options)
def test_unknown_alignment_mode_rejected(options_module):
with pytest.raises(ValueError, match="alignment_mode"):
options_module.SGLDOptions().create_options(alignment_mode="typo")
def test_legacy_widget_positions_preserved(options_module):
legacy = (
"model_type num_gpus parallel_mode tp_size ulysses_degree ring_degree "
"attention_backend attention_backend_config quantization int8_gemm "
"cache_mode cache_dit_tuning cache_warmup_steps cache_threshold "
"cache_max_cached_steps cache_fn_blocks execution_mode regional_compile "
"memory_mode resident_layers prefetch_layers"
).split()
assert list(options_module._FIELDS) == legacy + ["alignment_mode"]