diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py index cfa4d56421..8c2b4d303c 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py @@ -322,7 +322,11 @@ class SGLDiffusionGenerator: _SHARED.close_generator() def init_generator( - self, model_path: str, pipeline_class_name: str, kwargs: dict = None + self, + model_path: str, + pipeline_class_name: str, + kwargs: dict = None, + worker_patches=(), ): """Initialize the diffusion generator.""" if self.generator is not None: @@ -352,7 +356,11 @@ class SGLDiffusionGenerator: kwargs.setdefault("model_id", _H3_MODEL_IDS[pipeline_class_name]) kwargs = self._server_args_kwargs(kwargs) try: - with _spawn_without_launcher_main(): + from sglang.multimodal_gen.apps.comfyui_worker_patches import ( + scheduler_profiles, + ) + + with scheduler_profiles(worker_patches), _spawn_without_launcher_main(): self.generator = DiffGenerator.from_pretrained( model_path=model_path, pipeline_class_name=pipeline_class_name, @@ -576,6 +584,13 @@ class SGLDiffusionGenerator: ) check_sgld_options(sgld_options) plugin = {k: sgld_options.pop(k) for k in PLUGIN_KEYS if k in sgld_options} + from sglang.multimodal_gen.apps.comfyui_worker_patches import ( + canonical_profiles, + validate_options, + ) + + patches = canonical_profiles(plugin.get("worker_patches")) + reload_kwargs["sgld_options"]["worker_patches"] = patches set_model_type = plugin.get("model_type") if set_model_type in NATIVE_MODEL_TYPES: raise ValueError( @@ -605,8 +620,10 @@ class SGLDiffusionGenerator: "set_model_type": set_model_type, "runtime_model_path": runtime_model_path, "int8_kernel": plugin.get("int8_kernel", "auto"), + "worker_patch_identity": patches, } if self._can_reuse(gather_options): + validate_options(patches, self.model_type, {**sgld_options, **plugin}) check_model_options( self.model_type, {**sgld_options, **plugin}, native_dir=native_dir ) @@ -624,6 +641,7 @@ class SGLDiffusionGenerator: check_model_options( model_type, {**sgld_options, **plugin}, native_dir=native_dir ) + validate_options(patches, model_type, {**sgld_options, **plugin}) if model_type == "fast_h3": load_fasth3_release(runtime_model_path) if model_type == "minimax_h3": @@ -645,7 +663,10 @@ class SGLDiffusionGenerator: ) with _worker_env(worker_env): self.generator = self.init_generator( - worker_model_path, pipeline_class_name, sgld_options + worker_model_path, + pipeline_class_name, + sgld_options, + worker_patches=patches, ) self.model_type = model_type diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py index a2a482921e..5c1a4671dd 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/options.py @@ -6,7 +6,13 @@ PLUGIN_KEYS, which core/generator.py pops before building ServerArgs. Checks tha need the resolved model live in core/preflight.py. """ -PLUGIN_KEYS = ("model_type", "cache_mode", "cache_dit_params", "int8_kernel") +PLUGIN_KEYS = ( + "model_type", + "cache_mode", + "cache_dit_params", + "int8_kernel", + "worker_patches", +) # Display names -> SGLang backend names; the rest are passed through unchanged. _ATTENTION_IDS = { @@ -91,6 +97,16 @@ _FIELDS = { ), "resident_layers": _field("memory", "INT", 0, "Kept on GPU.", min=0), "prefetch_layers": _field("memory", "INT", 0, "Prefetched; 0 means 1.", min=0), + # Append fields to preserve positional widget values in existing workflows. + "alignment_mode": _field( + "basic", + ["default", "comfyui"], + "default", + "comfyui matches native ComfyUI arithmetic (currently H3 only): single GPU, " + "resident BF16, eager, no LoRA/quantization/cache. Auto attention uses " + "torch_sdpa; incompatible explicit settings are rejected. Changing mode " + "reloads the worker.", + ), } @@ -131,6 +147,12 @@ class SGLDOptions: options["dit_layerwise_resident_layers"] = v["resident_layers"] options["dit_offload_prefetch_size"] = v["prefetch_layers"] backend = v["attention_backend"] + if v["alignment_mode"] == "comfyui": + options["worker_patches"] = ["h3_comfyui_v1"] + if backend == "auto": + backend = "torch_sdpa" + elif v["alignment_mode"] != "default": + raise ValueError(f"Unknown alignment_mode: {v['alignment_mode']}") if backend != "auto": options["attention_backend"] = _ATTENTION_IDS.get(backend, backend) if v["attention_backend_config"].strip(): diff --git a/python/sglang/multimodal_gen/apps/comfyui_worker_patches/__init__.py b/python/sglang/multimodal_gen/apps/comfyui_worker_patches/__init__.py new file mode 100644 index 0000000000..cccef79b37 --- /dev/null +++ b/python/sglang/multimodal_gen/apps/comfyui_worker_patches/__init__.py @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ComfyUI worker patches. Keep module scope import-safe for multiprocessing spawn.""" + +import logging +import threading +from contextlib import contextmanager +from dataclasses import dataclass, replace +from functools import partial, wraps +from importlib import import_module + +# Version is part of the name (and therefore the generator's reuse key). +PROFILES = {"h3_comfyui_v1": "h3"} +_STARTUP_LOCK = threading.RLock() + + +def canonical_profiles(names): + if names is None: + return () + if not isinstance(names, (tuple, list)) or any( + not isinstance(name, str) or name not in PROFILES for name in names + ): + raise ValueError("Unknown worker patch; choose a registered profile") + names = tuple(dict.fromkeys(names)) + if len(names) > 1: + raise ValueError("Select one worker patch; composition is not supported") + return names + + +def validate_options(names, model_type, options): + for name in canonical_profiles(names): + import_module(f"{__name__}.{PROFILES[name]}").validate(model_type, options) + + +@dataclass(frozen=True) +class _PatchedArgs: + original: object + name: str + + def materialize(self): + # Stock bootstrap calls this after platform and plugin initialization. + args = self.original.materialize() + from sglang.multimodal_gen.runtime.managers.gpu_worker import GPUWorker + + install = import_module(f"{__name__}.{PROFILES[self.name]}").install + original_init = GPUWorker.__init__ + + @wraps(original_init) + def init(worker, *a, **kw): + original_init(worker, *a, **kw) + # Synchronous: missing pipeline/failed installation aborts startup. + install(worker.pipeline) + logging.getLogger(__name__).info("Worker patch applied: %s", self.name) + + GPUWorker.__init__ = init + return args + + +def _bootstrap(target, name, spec): + return target(replace(spec, server_args=_PatchedArgs(spec.server_args, name))) + + +@contextmanager +def scheduler_profiles(names): + names = canonical_profiles(names) + from sglang.multimodal_gen.runtime import launch_server + + # Serialize patched and default launches; restore the parent even on failure. + with _STARTUP_LOCK: + original = launch_server.bootstrap_scheduler_process + if names: + launch_server.bootstrap_scheduler_process = partial( + _bootstrap, original, names[0] + ) + try: + yield + finally: + launch_server.bootstrap_scheduler_process = original diff --git a/python/sglang/multimodal_gen/apps/comfyui_worker_patches/h3.py b/python/sglang/multimodal_gen/apps/comfyui_worker_patches/h3.py new file mode 100644 index 0000000000..594837cdae --- /dev/null +++ b/python/sglang/multimodal_gen/apps/comfyui_worker_patches/h3.py @@ -0,0 +1,201 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ComfyUI H3 BF16 compatibility implemented outside runtime model sources. + +Module instances are patched once after load, before first forward. Cloning a +function's globals keeps helper overrides local to that bound method; original +classes, module globals and other instances remain unchanged. Eager only. +""" + +import hashlib +import importlib.metadata +import inspect +import types +from pathlib import Path + +import torch + +# A changed upstream implementation must be reviewed, never best-effort patched. +SOURCE_SHA256 = "ced7b679ee05f75045646cb81a436088557b90f23d3be980aba13f816ac23f0d" + + +def bind_with_globals(instance, name, overrides): + original = getattr(instance, name).__func__ + scope = dict(original.__globals__) + scope.update(overrides) + fn = types.FunctionType( + original.__code__, + scope, + original.__name__, + original.__defaults__, + original.__closure__, + ) + fn.__kwdefaults__ = original.__kwdefaults__ + fn.__annotations__ = original.__annotations__ + setattr(instance, name, types.MethodType(fn, instance)) + + +def comfy_gate(x, gate, other, indices, *, dtype, allow_inplace=True): + return torch.addcmul( + x, other, gate.index_select(0, indices), out=x if allow_inplace else None + ) + + +def comfy_rope(q, k, qw, kw, cache, positions, **kwargs): + # Match ComfyUI Kitchen v0.2.36 HasRms float rotation exactly. + import comfy_kitchen as ck + + table = cache.index_select(0, positions) + cos, sin = table.chunk(2, dim=-1) + table = torch.stack([cos, -sin, sin, cos], dim=-1).reshape( + 1, len(q), 1, cos.shape[-1], 2, 2 + ) + ck.rms_rope_split_half_( + q.unsqueeze(0), + k.unsqueeze(0), + table, + qw, + kw, + epsilon=kwargs["eps"], + rot_dim=cache.shape[-1], + ) + + +def _reject_lora(*args, **kwargs): + raise ValueError("h3_comfyui_v1 does not support LoRA") + + +def validate(model_type, options): + if model_type != "minimax_h3": + raise ValueError("h3_comfyui_v1 requires model_type=minimax_h3") + if options.get("num_gpus", 1) != 1: + raise ValueError("h3_comfyui_v1 currently requires one GPU") + for degree in ("tp_size", "sp_degree", "ulysses_degree", "ring_degree"): + if options.get(degree) not in (None, -1, 1): + raise ValueError(f"h3_comfyui_v1 requires {degree}=1 or auto") + if ( + options.get("enable_torch_compile") + or options.get("enable_cuda_graph") + or options.get("enable_breakable_cuda_graph") + ): + raise ValueError("h3_comfyui_v1 currently requires eager execution") + if options.get("cache_mode", "off") != "off" or options.get("quantization"): + raise ValueError("h3_comfyui_v1 requires no cache/quantization") + if options.get("attention_backend") != "torch_sdpa": + raise ValueError("h3_comfyui_v1 parity is verified only with torch_sdpa") + if ( + options.get("transformer_weights_path") + or options.get("dit_layerwise_offload") + or options.get("dit_cpu_offload") + ): + raise ValueError("h3_comfyui_v1 requires resident single-file BF16 weights") + for key in ( + "component_attention_backends", + "component_quantizations", + "component_precisions", + "lora_path", + "minimax_h3_adaln_online", + "minimax_h3_adaln_cache_path", + "enable_cfg_parallel", + "use_fsdp_inference", + ): + if options.get(key): + raise ValueError(f"h3_comfyui_v1 has not been validated with {key}") + + +def install(pipeline): + from sglang.multimodal_gen.runtime.distributed import get_tp_world_size + from sglang.multimodal_gen.runtime.distributed.parallel_state import ( + get_ring_ctx, + get_ulysses_ctx, + ) + from sglang.multimodal_gen.runtime.models.dits import minimax_h3 as h3 + + source = Path(inspect.getsourcefile(h3)).read_bytes() + if hashlib.sha256(source).hexdigest() != SOURCE_SHA256: + raise RuntimeError( + "H3 upstream source differs from the reviewed h3_comfyui_v1 source" + ) + if importlib.metadata.version("comfy-kitchen") != "0.2.36": + raise RuntimeError("h3_comfyui_v1 requires reviewed comfy-kitchen==0.2.36") + if get_tp_world_size() != 1 or get_ring_ctx()[0] != 1 or get_ulysses_ctx()[0] != 1: + raise ValueError("h3_comfyui_v1 is single-GPU only") + models = [m for m in pipeline.modules.values() if type(m) is h3.MiniMaxH3DiTModel] + if len(models) != 1: + raise ValueError("h3_comfyui_v1 requires exactly one base H3 DiT") + model = models[0] + if next(model.parameters()).device.type != "cuda": + raise ValueError("h3_comfyui_v1 requires a resident CUDA model") + if torch.cuda.get_device_capability() != (9, 0): + raise ValueError("h3_comfyui_v1 is currently verified only on Hopper") + if ( + model.adaln_t_table is not None + or model.adaln_cache is not None + or model.final_layer._pdd_heads is not None + ): + raise ValueError("h3_comfyui_v1 does not support PDD/curve/cached AdaLN") + for name, module in model.named_modules(): + method = getattr(module, "quant_method", None) + if method is not None and type(method).__name__ != "UnquantizedLinearMethod": + raise ValueError( + f"h3_comfyui_v1 does not support quantized layer {name}: {type(method).__name__}" + ) + if getattr(model, "_comfy_h3_installed", False): + return + # Validate every dependency before modifying any method. + import comfy_kitchen as ck + + if not callable(ck.rms_rope_split_half_): + raise TypeError("Missing ComfyUI-compatible RMSNorm/RoPE kernel") + for block in model.blocks: + if not callable(getattr(block.attn, "_attend", None)): + raise TypeError("Missing reviewed H3 attention entry point") + embed = model._embed + + def embedding(self, *a, **kw): + hidden, time = embed(*a, **kw) + return hidden, time.to(torch.bfloat16) + + model._embed = types.MethodType(embedding, model) + for block in model.blocks: + bind_with_globals( + block, + "forward", + { + "_modulate_gate": comfy_gate, + "minimax_h3_norm_modulate_active": lambda _: False, + }, + ) + bind_with_globals( + block.attn, "_attend", {"fused_inplace_qknorm_rope": comfy_rope} + ) + block.attn._use_fused_qknorm_rope = True + final = model.final_layer + + def project(self, hidden): + vi, ai = self._comfy_output_positions + vi, ai = vi.to(hidden.device), ai.to(hidden.device) + video, _ = self.video_out(hidden.index_select(0, vi)) + audio, _ = self.audio_out(hidden.index_select(0, ai)) + return ( + hidden.new_zeros((len(hidden), video.shape[-1])).index_copy_(0, vi, video), + hidden.new_zeros((len(hidden), audio.shape[-1])).index_copy_(0, ai, audio), + ) + + final._project = types.MethodType(project, final) + forward = model.forward + + def model_forward(self, **kw): + final._comfy_output_positions = ( + self._pos_ids( + kw["img_pos_for_infer_output_info"], "img_pos_for_infer_output_info" + ), + self._pos_ids(kw["audio_pos_info"], "audio_pos_info"), + ) + try: + return forward(**kw) + finally: + del final._comfy_output_positions + + model.forward = types.MethodType(model_forward, model) + pipeline.set_lora = _reject_lora + model._comfy_h3_installed = True diff --git a/python/sglang/multimodal_gen/apps/comfyui_worker_patches/tests/test_worker_profiles.py b/python/sglang/multimodal_gen/apps/comfyui_worker_patches/tests/test_worker_profiles.py new file mode 100644 index 0000000000..083e47fe1a --- /dev/null +++ b/python/sglang/multimodal_gen/apps/comfyui_worker_patches/tests/test_worker_profiles.py @@ -0,0 +1,213 @@ +"""Worker lifecycle, failure propagation and H3 compatibility constraints.""" + +import multiprocessing as mp +import os +import sys +import types +from dataclasses import dataclass + +import pytest + +from sglang.multimodal_gen.apps import comfyui_worker_patches as patches + + +def test_registered_names_only(): + assert patches.canonical_profiles(None) == () + assert patches.canonical_profiles(["h3_comfyui_v1"] * 2) == ("h3_comfyui_v1",) + for names in (["arbitrary.module:run"], "h3_comfyui_v1", [None]): + with pytest.raises(ValueError, match="Unknown"): + patches.canonical_profiles(names) + + +def test_composition_rejected(monkeypatch): + monkeypatch.setitem(patches.PROFILES, "other", "other") + with pytest.raises(ValueError, match="one worker patch"): + patches.canonical_profiles(["h3_comfyui_v1", "other"]) + + +def test_wrong_family_rejected(): + with pytest.raises(ValueError, match="model_type"): + patches.validate_options(["h3_comfyui_v1"], "flux", {}) + + +@pytest.mark.parametrize( + "option", + [ + {"num_gpus": 2}, + {"enable_torch_compile": True}, + {"enable_cuda_graph": True}, + {"quantization": "fp8"}, + {"cache_mode": "cache_dit"}, + {"attention_backend": "sage_attn"}, + {"component_attention_backends": {"transformer": "fa"}}, + {"lora_path": "unreviewed.safetensors"}, + ], +) +def test_unsupported_contract_rejected(option): + with pytest.raises(ValueError): + patches.validate_options( + ["h3_comfyui_v1"], + "minimax_h3", + {"attention_backend": "torch_sdpa"} | option, + ) + + +def test_default_launch_and_restore_on_failure(monkeypatch): + original = lambda spec: None + module = types.ModuleType("sglang.multimodal_gen.runtime.launch_server") + module.bootstrap_scheduler_process = original + monkeypatch.setitem(sys.modules, module.__name__, module) + with patches.scheduler_profiles([]): + assert module.bootstrap_scheduler_process is original + with ( + pytest.raises(RuntimeError, match="launch failed"), + patches.scheduler_profiles(["h3_comfyui_v1"]), + ): + assert module.bootstrap_scheduler_process is not original + raise RuntimeError("launch failed") + assert module.bootstrap_scheduler_process is original + + +@dataclass +class _Args: + def materialize(self): + return "args-ready" + + +@dataclass +class _Spec: + server_args: object + + +def _spawn_probe(connection, fail): + # A real spawn boundary with a tiny worker instead of allocating GPU weights. + worker_module = types.ModuleType( + "sglang.multimodal_gen.runtime.managers.gpu_worker" + ) + + class Worker: + def __init__(self): + self.pipeline = types.SimpleNamespace(loaded=True) + + worker_module.GPUWorker = Worker + sys.modules[worker_module.__name__] = worker_module + provider = types.ModuleType(patches.__name__ + ".test_provider") + + def install(pipeline): + assert pipeline.loaded + if fail: + raise RuntimeError("installer failed") + pipeline.patched = True + + provider.install = install + sys.modules[provider.__name__] = provider + patches.PROFILES["test"] = "test_provider" + + def stock_bootstrap(spec): + assert spec.server_args.materialize() == "args-ready" + return Worker().pipeline.patched + + try: + result = patches._bootstrap(stock_bootstrap, "test", _Spec(_Args())) + except RuntimeError as error: + result = str(error) + connection.send((os.getpid(), result)) + connection.close() + + +@pytest.mark.parametrize("fail", [False, True]) +def test_real_spawn_installs_after_load_or_aborts(fail): + ctx = mp.get_context("spawn") + parent, child = ctx.Pipe(duplex=False) + process = ctx.Process(target=_spawn_probe, args=(child, fail)) + process.start() + child.close() + try: + assert parent.poll(60), "spawn did not report" + pid, result = parent.recv() + process.join(10) + assert process.exitcode == 0 and pid != os.getpid() + assert result == ("installer failed" if fail else True) + assert "test" not in patches.PROFILES + finally: + if process.is_alive(): + process.terminate() + process.join(10) + parent.close() + + +def test_bound_helper_does_not_change_class_or_other_instances(): + from sglang.multimodal_gen.apps.comfyui_worker_patches.h3 import bind_with_globals + + class Example: + def run(self): + return os.getpid() + + patched, original = Example(), Example() + function = Example.run + bind_with_globals( + patched, "run", {"os": types.SimpleNamespace(getpid=lambda: -123)} + ) + assert patched.run() == -123 + assert original.run() == os.getpid() + assert Example.run is function + + +def test_h3_dynamic_lora_rejected(): + from sglang.multimodal_gen.apps.comfyui_worker_patches.h3 import _reject_lora + + with pytest.raises(ValueError, match="LoRA"): + _reject_lora("adapter") + + +@pytest.fixture +def options_module(): + # Load the schema without importing ComfyUI's node/runtime dependencies. + import importlib.util + from pathlib import Path + + path = Path(patches.__file__).parent.parent / "ComfyUI_SGLDiffusion/options.py" + spec = importlib.util.spec_from_file_location("sgld_options_test", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_alignment_default_preserves_existing_options(options_module): + node = options_module.SGLDOptions() + options = node.create_options()[0] + assert options == node.create_options(alignment_mode="default")[0] + assert "worker_patches" not in options + assert "attention_backend" not in options + + +def test_comfyui_mode_auto_attention_is_valid_for_h3(options_module): + options = options_module.SGLDOptions().create_options(alignment_mode="comfyui")[0] + assert options["worker_patches"] == ["h3_comfyui_v1"] + assert options["attention_backend"] == "torch_sdpa" + patches.validate_options(options["worker_patches"], "minimax_h3", options) + + +def test_comfyui_mode_preserves_and_rejects_explicit_attention(options_module): + options = options_module.SGLDOptions().create_options( + alignment_mode="comfyui", attention_backend="flash_attention" + )[0] + assert options["attention_backend"] == "fa" + with pytest.raises(ValueError, match="torch_sdpa"): + patches.validate_options(options["worker_patches"], "minimax_h3", options) + + +def test_unknown_alignment_mode_rejected(options_module): + with pytest.raises(ValueError, match="alignment_mode"): + options_module.SGLDOptions().create_options(alignment_mode="typo") + + +def test_legacy_widget_positions_preserved(options_module): + legacy = ( + "model_type num_gpus parallel_mode tp_size ulysses_degree ring_degree " + "attention_backend attention_backend_config quantization int8_gemm " + "cache_mode cache_dit_tuning cache_warmup_steps cache_threshold " + "cache_max_cached_steps cache_fn_blocks execution_mode regional_compile " + "memory_mode resident_layers prefetch_layers" + ).split() + assert list(options_module._FIELDS) == legacy + ["alignment_mode"]