diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py index 59f733d5e0..91ebbf6454 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/generator.py @@ -9,6 +9,7 @@ import os from ..executors.flux import FluxExecutor from ..executors.minimax_h3 import MiniMaxH3Executor from ..executors.zimage import ZImageExecutor +from .preflight import check_sgld_options logger = logging.getLogger(__name__) @@ -204,12 +205,24 @@ class SGLDiffusionGenerator: # policy otherwise sets dit_cpu_offload=True and every sampler step # reloads the DiT from CPU. kwargs.setdefault("dit_cpu_offload", False) + # ComfyUI sends cond and uncond as separate requests, so worker-side CFG + # parallel only recomputes; its auto-enable also needs a model_index.json, + # which a non-H3 single file lacks, and fails worker startup on >= 2 GPUs. + if not kwargs.get("enable_cfg_parallel"): + kwargs.setdefault("cfg_parallel_degree", 1) kwargs = self._server_args_kwargs(kwargs) - self.generator = DiffGenerator.from_pretrained( - model_path=model_path, - pipeline_class_name=pipeline_class_name, - **kwargs, - ) + try: + self.generator = DiffGenerator.from_pretrained( + model_path=model_path, + pipeline_class_name=pipeline_class_name, + **kwargs, + ) + except EOFError as error: + # A worker that raises while loading closes its pipe without a message. + raise RuntimeError( + "The SGLD worker exited while loading the model; its traceback is in " + "the ComfyUI console output above this error" + ) from error return self.generator @staticmethod @@ -401,6 +414,14 @@ class SGLDiffusionGenerator: } sgld_options = dict(raw_sgld_options) model_options = dict(reload_kwargs["model_options"]) + if model_options.get("dtype") is not None: + # Only the ComfyUI architecture companion would get this dtype; the + # worker loads its own weights, so the choice would be silently lost. + raise ValueError( + "SGLDUNETLoader weight_dtype must be 'default': the SGLD worker " + "does not use it; select quantization in SGLDOptions instead" + ) + check_sgld_options(sgld_options) plugin_flags = {} if "enable_cache_dit" in sgld_options: plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit") diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/model_patcher.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/model_patcher.py index d146f2b3f0..65aa4f5ecf 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/model_patcher.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/model_patcher.py @@ -45,6 +45,7 @@ class SGLDModelPatcher(ModelPatcher): n.object_patches = self.object_patches.copy() n.model_options = copy.deepcopy(self.model_options) + n.parent = self n.backup = self.backup n.object_patches_backup = self.object_patches_backup n.lora_cache = copy.copy(self.lora_cache) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py new file mode 100644 index 0000000000..95e1a0514a --- /dev/null +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/core/preflight.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Reject accelerations this environment cannot run before the SGLD worker starts. + +The worker is spawned from this interpreter, so asking SGLang's own backend +resolution here gives the same answer the worker would get after loading the +model; a missing kernel then fails in the loader node instead of at the first +sampling step, or silently as another backend. +""" + +from pkgutil import resolve_name + +import torch + +from sglang.multimodal_gen.runtime.platforms import current_platform +from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum + +_INSTALL_HINTS = { + AttentionBackendEnum.SAGE_ATTN: "install SageAttention 2: https://github.com/thu-ml/SageAttention", + AttentionBackendEnum.SAGE_ATTN_3: ( + "SageAttention 3 targets Blackwell GPUs: " + "https://github.com/thu-ml/SageAttention/tree/main/sageattention3_blackwell" + ), +} + + +def requested_attention_backends(sgld_options: dict) -> dict[str, str]: + """Option name -> requested backend, for every attention backend the options select.""" + requested = {} + if sgld_options.get("attention_backend"): + requested["attention_backend"] = sgld_options["attention_backend"] + components = sgld_options.get("component_attention_backends") or {} + if isinstance(components, str): + components = dict( + item.split("=", 1) for item in components.split(",") if "=" in item + ) + for component, backend in components.items(): + requested[f"component_attention_backends.{component.strip()}"] = backend + return requested + + +def _resolved_backend(backend: AttentionBackendEnum) -> AttentionBackendEnum: + # Head size and dtype of a typical DiT layer; only availability matters here. + cls_path = current_platform.get_attn_backend_cls_str(backend, 128, torch.bfloat16) + return resolve_name(cls_path).get_enum() + + +def check_attention_backends(sgld_options: dict) -> None: + for option, name in requested_attention_backends(sgld_options).items(): + try: + backend = AttentionBackendEnum[name.strip().upper()] + except KeyError: + raise ValueError( + f"{option}={name!r} is not an SGLang attention backend" + ) from None + try: + resolved = _resolved_backend(backend) + except (ImportError, ValueError) as error: + raise ValueError( + f"{option}={name} cannot run in this environment: {error}" + ) from error + if resolved is not backend: + hint = _INSTALL_HINTS.get(backend, "install it or choose another backend") + raise ValueError( + f"{option}={name} is not available in this environment (SGLang " + f"would run {resolved.name.lower()} instead); {hint}" + ) + + +def check_parallel_layout(sgld_options: dict) -> None: + dp = sgld_options.get("dp_size") or 1 + if dp > 1: + # Each sampler step is one request carrying CUDA IPC tensors from this + # process, and ComfyUI runs one prompt at a time. + raise ValueError( + "dp_size > 1 is not supported in ComfyUI integrated mode: replicas never " + "overlap behind ComfyUI's single prompt queue. Run one ComfyUI instance " + "per GPU, or use tp_size / sp_degree for one request." + ) + tp = sgld_options.get("tp_size") + sp = sgld_options.get("sp_degree") + if ( + sp is None + and sgld_options.get("ulysses_degree") + and sgld_options.get("ring_degree") + ): + sp = sgld_options["ulysses_degree"] * sgld_options["ring_degree"] + if tp is None or sp is None: + return # SGLang fills unset degrees from num_gpus + cfg = ( + (sgld_options.get("cfg_parallel_degree") or 2) + if sgld_options.get("enable_cfg_parallel") + else 1 + ) + num_gpus = sgld_options.get("num_gpus") or 1 + if dp * tp * sp * cfg != num_gpus: + # A GPU outside the layout gets no process group; its worker hangs at startup. + raise ValueError( + f"num_gpus ({num_gpus}) must equal tp_size * sp_degree" + f"{' * cfg_parallel_degree' if cfg > 1 else ''} = {dp * tp * sp * cfg}; " + "set the degrees to fill every GPU or leave sp_degree at -1 (auto)" + ) + + +def check_sgld_options(sgld_options: dict) -> None: + check_attention_backends(sgld_options) + check_parallel_layout(sgld_options) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py index 55fd89a5c2..705f78cf60 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py @@ -38,6 +38,9 @@ class ComfyUIModelAdapter: model_types: tuple[str, ...] = () pipeline_class_name: str = "" + # Conditioning apply_model kwargs this adapter forwards to the worker; + # the executor rejects the other content kwargs instead of dropping them. + applied_conditioning: tuple[str, ...] = () def __init_subclass__(cls, **kwargs): super().__init_subclass__(**kwargs) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py index 8dbdef8602..44d659687c 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py @@ -20,6 +20,49 @@ else: _RUNTIME_IMPORT_ERROR = None +def _reject_model_patches(transformer_options) -> None: + # The DiT runs in the SGLD worker, so ComfyUI block/attention patches would + # be dropped silently (e.g. MiniMax H3 Fun ControlNet, ModelAttentionBackend). + opts = transformer_options or {} + found = [ + f"patches_replace.{name}" + for name, blocks in (opts.get("patches_replace") or {}).items() + if blocks + ] + found += [ + f"patches.{name}" + for name, items in (opts.get("patches") or {}).items() + if items + ] + if opts.get("optimized_attention_override") is not None: + found.append("optimized_attention_override") + if found: + raise ValueError( + "SGLD integrated mode runs the diffusion model in its worker and cannot " + f"apply ComfyUI model patches ({', '.join(sorted(found))}); remove the " + "patch nodes (ControlNet, attention backend, block patches) or use the " + "native ComfyUI loader" + ) + + +# apply_model kwargs that carry image content (ControlNet residuals, reference latents). +CONTENT_CONDITIONING = ("control", "ref_latents") + + +def _reject_unapplied_conditioning(adapter, kwargs) -> None: + dropped = [ + name + for name in CONTENT_CONDITIONING + if name not in adapter.applied_conditioning and kwargs.get(name) + ] + if dropped: + raise ValueError( + f"SGLD integrated mode does not apply {', '.join(dropped)} for this model " + "(ControlNet / reference-latent nodes); remove those nodes or use the " + "native ComfyUI loader" + ) + + class SGLDiffusionExecutor(torch.nn.Module): """Shared ComfyUI DiT-forward executor. Per-model logic lives on the adapter.""" @@ -29,7 +72,9 @@ class SGLDiffusionExecutor(torch.nn.Module): super(SGLDiffusionExecutor, self).__init__() self.generator = generator self.model_path = model_path - self.model = model + # Not a submodule: the BaseModel holds this executor as diffusion_model, + # and registering it back makes state_dict() / .to() recurse forever. + object.__setattr__(self, "model", model) self.dtype = config.unet_config["dtype"] self.config = config self.loras = [] @@ -43,6 +88,13 @@ class SGLDiffusionExecutor(torch.nn.Module): self._run_id = 0 self._sent_conds: set[tuple] = set() + def state_dict(self, *args, **kwargs): + raise RuntimeError( + "The SGLD worker owns this diffusion model's weights, so ComfyUI weight " + "nodes (LoraLoader, ModelSave, model merging) cannot use it; load LoRAs " + "with SGLDLoraLoader" + ) + @staticmethod def should_suppress_logs(timestep): """Determine if logs should be suppressed based on timestep value.""" @@ -52,7 +104,7 @@ class SGLDiffusionExecutor(torch.nn.Module): def set_lora(self, lora_nickname=None, lora_path=None, strength=None, target=None): """Set LoRA adapter using SGLang Diffusion API.""" - self._lora_input = { + desired = { "lora_nickname": lora_nickname, "lora_path": lora_path, "strength": strength, @@ -66,6 +118,8 @@ class SGLDiffusionExecutor(torch.nn.Module): target=target, ) + self._lora_input = desired + def begin_sampler_run(self) -> None: """One ComfyUI ``sampler.sample()`` invocation is one cache lifetime.""" self._run_id += 1 @@ -75,6 +129,16 @@ class SGLDiffusionExecutor(torch.nn.Module): """Run cache is evicted on the next bind of a newer id for this executor.""" def sampler_sample_wrapper(self, executor, *args, **kwargs): + if self._ensure_runtime is not None: + self._ensure_runtime(self) + model_wrap = args[0] if args else kwargs["model_wrap"] + desired = model_wrap.model_patcher.model_options.get("sgld_lora_input") + if desired != self._lora_input: + if self._lora_input is not None: + self.generator.unmerge_lora_weights() + self._lora_input = None + if desired is not None: + self.set_lora(**desired) self.begin_sampler_run() try: return executor(*args, **kwargs) @@ -149,8 +213,12 @@ class SGLDiffusionExecutor(torch.nn.Module): torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt) ] output_batch = self.generator._send_to_scheduler_and_wait_for_response([req]) + if output_batch.error: + raise RuntimeError(f"SGLang Diffusion worker failed: {output_batch.error}") return self.adapter.unpack(output_batch.noise_pred, packed, x) def forward(self, x, timestep, context, **kwargs): + _reject_model_patches(kwargs.get("transformer_options")) + _reject_unapplied_conditioning(self.adapter, kwargs) packed = self.adapter.pack(x, timestep, context, **kwargs) return self._execute_packed(packed, x, timestep) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py index 046519ae23..58d6d66670 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py @@ -86,6 +86,10 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter): def pack(self, x, timestep, context, **kwargs) -> PackedForward: video, audio_src = _split_av(x) + if video.shape[0] != 1: + # Same limit as ComfyUI's MiniMax H3 model: the checkpoint is + # CFG-distilled and the packed sequence carries a single sample. + raise ValueError(f"MiniMax H3 supports batch size 1, got {video.shape[0]}") # PackedLayout is a ComfyUI class; the SGLD worker cannot unpickle it. payload = dict(kwargs.get("minimax_payload") or {}) layout = payload.pop("layout", None) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py index 867134d9a9..c1a625c61c 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/qwen_image.py @@ -74,6 +74,7 @@ class QwenImageAdapter(ComfyUIModelAdapter): class QwenImageEditAdapter(QwenImageAdapter): model_types = ("qwen_image_edit",) pipeline_class_name = "QwenImageEditPlusPipeline" + applied_conditioning = ("ref_latents",) def pack( self, diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py index ac0c7d9878..47b4ef1d07 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/nodes.py @@ -213,9 +213,17 @@ class SGLDLoraLoader: lora_input["strength"].append(lora_info[1]) lora_input["target"].append(lora_info[2]) - # call the SGLang Diffusion API - model.model.diffusion_model.set_lora(**lora_input) - return (model,) + targets = set(lora_input["target"]) + if "all" in targets and len(targets) > 1: + # set_lora groups adapters per target string; "all" overlapping a + # named target makes the worker keep only one group on that module. + raise ValueError( + "Chained SGLDLoraLoader nodes mix target 'all' with " + f"{sorted(targets - {'all'})}; use the same target on every node" + ) + # The shared worker is bound to the selected MODEL at sampling time. + bi.model_options["sgld_lora_input"] = lora_input + return (bi,) class SGLDUNETLoader: diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py index 42c1772baf..f1963f6a92 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py @@ -1,16 +1,27 @@ # SPDX-License-Identifier: Apache-2.0 """Pack / unpack contract for ComfyUI model adapters.""" +from types import SimpleNamespace +from unittest.mock import patch + +import pytest import torch from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import ( get_adapter_class, registered_model_types, ) -from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import FluxAdapter +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import ( + SGLDiffusionExecutor, +) +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import ( + FluxAdapter, + FluxExecutor, +) from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.zimage import ( ZImageAdapter, ) +from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams def test_registered_comfyui_model_types() -> None: @@ -56,3 +67,92 @@ def test_flux_pack_and_unpack_roundtrip() -> None: default = adapter.pack(x, timestep, context, y=y) assert default.guidance_scale == 3.5 + + +def test_worker_error_is_raised_not_unpacked() -> None: + """Unpacking a failed reply replaced the worker's message with a misleading + adapter TypeError about noise_pred being None.""" + ex = SGLDiffusionExecutor.__new__(SGLDiffusionExecutor) + torch.nn.Module.__init__(ex) + ex.adapter, ex.model_path = FluxAdapter(), "/test-model" + ex.session_id, ex._run_id, ex._sent_conds = "error-test", 0, set() + ex.generator = SimpleNamespace( + server_args=SimpleNamespace(attention_backend_config={}, enable_trace=False), + _send_to_scheduler_and_wait_for_response=lambda reqs: SimpleNamespace( + noise_pred=None, error="index_copy_(): shape mismatch" + ), + ) + x, t = torch.randn(1, 16, 8, 8), torch.full((1,), 0.5) + packed = ex.adapter.pack(x, t, torch.randn(1, 7, 32), y=torch.randn(1, 768)) + with ( + patch.object( + SamplingParams, + "from_user_sampling_params_args", + side_effect=lambda model_path, server_args, **kw: SamplingParams(**kw), + ), + patch.object(torch, "Generator", side_effect=lambda device: object()), + pytest.raises(RuntimeError, match="worker failed: index_copy_"), + ): + ex._execute_packed(packed, x, t) + + +class _RecordingExecutor(SGLDiffusionExecutor): + """The real forward(); records what would be sent to the worker.""" + + def __init__(self, adapter): + torch.nn.Module.__init__(self) + self.adapter, self.sent = adapter, [] + + def _execute_packed(self, packed, x, timestep): + self.sent.append(packed) + return x + + +def _flux_step(ex, **kwargs): + x, t = torch.randn(1, 16, 8, 8), torch.full((1,), 0.5) + ex(x, t, torch.randn(1, 7, 32), y=torch.randn(1, 768), **kwargs) + + +@pytest.mark.parametrize( + "options", + [ + {"patches_replace": {"dit": {("double_block", 3): object()}}}, + {"patches": {"attn1_patch": [object()]}}, + {"optimized_attention_override": object()}, + ], +) +def test_comfy_model_patches_are_rejected_not_dropped(options) -> None: + """ComfyUI model patches (H3 Fun ControlNet block replace, attention backend + override) never reach the worker; ignoring them gave bit-identical output.""" + ex = _RecordingExecutor(FluxAdapter()) + with pytest.raises(ValueError, match="cannot apply ComfyUI model patches"): + _flux_step(ex, transformer_options=options) + _flux_step(ex, transformer_options={"patches": {}, "patches_replace": {"dit": {}}}) + assert len(ex.sent) == 1 + + +@pytest.mark.parametrize( + "kwargs", + [ + {"control": {"output": [torch.ones(1)]}}, + {"ref_latents": [torch.ones(1, 16, 8, 8)]}, + ], +) +def test_conditioning_the_worker_cannot_apply_is_rejected(kwargs) -> None: + """ControlNet residuals and reference latents reach apply_model as kwargs; an + adapter that does not forward them must fail instead of ignoring them.""" + ex = _RecordingExecutor(FluxAdapter()) + with pytest.raises(ValueError, match=next(iter(kwargs))): + _flux_step(ex, **kwargs) + assert ex.sent == [] + + +def test_comfyui_weight_access_gets_a_clear_error() -> None: + """ComfyUI's BaseModel holds the executor as diffusion_model; LoraLoader's key + map calls state_dict() on it, which must not recurse through the executor.""" + base = torch.nn.Module() + config = SimpleNamespace(unet_config={"dtype": torch.bfloat16}) + base.diffusion_model = FluxExecutor(None, "flux.safetensors", base, config) + base.to("cpu") + with pytest.raises(RuntimeError, match="SGLDLoraLoader"): + base.state_dict() diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py index 47828da723..cdb49c62f1 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_generator.py @@ -2,6 +2,10 @@ """Process-wide SGLD worker ownership for ComfyUI loaders.""" from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +import torch from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import ( SGLDiffusionGenerator, @@ -135,3 +139,124 @@ def test_generator_reports_real_runtime_import_error(caplog) -> None: with pytest.raises(RuntimeError, match="failed to import") as err: module.SGLDiffusionGenerator().init_generator("flux", "FluxPipeline", {}) assert isinstance(err.value.__cause__, ImportError) + + +def _runtime_that_must_reject() -> SGLDiffusionGenerator: + """A loader whose model build fails loudly, so a check that should reject + first is caught if it lets the options through.""" + runtime = SGLDiffusionGenerator() + runtime.get_comfyui_model = Mock( + side_effect=AssertionError("must reject before building the model") + ) + return runtime + + +def test_non_default_weight_dtype_is_rejected_before_worker_load() -> None: + """weight_dtype only reached the ComfyUI architecture companion, so fp8 + produced output bit-identical to the default load.""" + with pytest.raises(ValueError, match="weight_dtype must be 'default'"): + _runtime_that_must_reject().load_model( + model_path="h3.safetensors", + model_options={"dtype": torch.float8_e4m3fn}, + sgld_options={}, + ) + + +@pytest.mark.parametrize( + "options,error", + [ + ({"attention_backend": "sage_attn_3"}, "would run torch_sdpa instead"), + ({"component_attention_backends": "transformer=sol_attn"}, "not installed"), + ({"attention_backend": "not_a_backend"}, "not an SGLang attention backend"), + ({"attention_backend": "sage_attn"}, None), + ], +) +def test_attention_backends_are_checked_before_worker_load( + monkeypatch, options, error +) -> None: + """SGLang serves a missing sage_attn / sage_attn_3 kernel as FA / SDPA, so an + explicit choice ran another backend; a missing sparse kernel only failed + after the full model load.""" + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import preflight + from sglang.multimodal_gen.runtime.platforms.interface import ( + AttentionBackendEnum as Backend, + ) + + def resolved(backend): + if backend is Backend.SOL_ATTN: + raise ImportError("Sol-Attn backend is not installed") + return {Backend.SAGE_ATTN_3: Backend.TORCH_SDPA}.get(backend, backend) + + monkeypatch.setattr(preflight, "_resolved_backend", resolved) + with pytest.raises( + ValueError if error else AssertionError, match=error or "must reject before" + ): + _runtime_that_must_reject().load_model( + model_path="h3.safetensors", sgld_options=options + ) + + +@pytest.mark.parametrize( + "options,error", + [ + ({"num_gpus": 2, "tp_size": 1, "sp_degree": 1}, "must equal"), + ({"num_gpus": 2, "tp_size": 2, "sp_degree": 2}, "must equal"), + ({"num_gpus": 2, "dp_size": 2}, "dp_size > 1"), + ({"num_gpus": 2, "tp_size": 2, "sp_degree": 1}, None), + ({"num_gpus": 2, "tp_size": 1, "sp_degree": None}, None), + ], +) +def test_parallel_layout_is_checked_before_worker_load(options, error) -> None: + """num_gpus=2 with tp=sp=1 left rank 1 without a process group, hanging + worker startup forever; dp_size=2 sent sampler steps to a replica without + the run's cached conditioning.""" + with pytest.raises( + ValueError if error else AssertionError, match=error or "must reject before" + ): + _runtime_that_must_reject().load_model( + model_path="h3.safetensors", sgld_options=options + ) + + +def test_worker_exit_during_load_names_where_the_reason_is(monkeypatch) -> None: + """A worker that raised while loading surfaced as an empty EOFError.""" + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator + + def exit_during_load(**kwargs): + raise EOFError + + monkeypatch.setattr(generator.DiffGenerator, "from_pretrained", exit_during_load) + with pytest.raises(RuntimeError, match="traceback is in the ComfyUI console"): + SGLDiffusionGenerator().init_generator("h3.safetensors", "MiniMaxH3Pipeline") + + +def test_multi_gpu_single_file_starts_without_cfg_parallel( + tmp_path, monkeypatch +) -> None: + """Auto CFG parallel read a model_index.json, so a Flux single file on two GPUs + failed ServerArgs validation; ComfyUI already splits cond/uncond requests.""" + import torch + from safetensors.torch import save_file + + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator + from sglang.multimodal_gen.runtime.server_args import ServerArgs + + path = tmp_path / "flux.safetensors" + save_file({"double_blocks.0.img_attn.qkv.weight": torch.zeros(1)}, path) + seen = {} + monkeypatch.setattr( + generator.DiffGenerator, + "from_pretrained", + lambda **kwargs: seen.update(kwargs) or object(), + ) + SGLDiffusionGenerator().init_generator(str(path), "FluxPipeline", {"num_gpus": 2}) + assert seen["cfg_parallel_degree"] == 1 + if torch.cuda.device_count() >= 2: + server_args = ServerArgs.from_kwargs(**seen) + assert server_args.enable_cfg_parallel is False + assert server_args.sp_degree == 2 + seen.clear() + SGLDiffusionGenerator().init_generator( + str(path), "FluxPipeline", {"num_gpus": 2, "enable_cfg_parallel": True} + ) + assert "cfg_parallel_degree" not in seen diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py index 74a554235d..8e2743fbc0 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_h3.py @@ -1244,3 +1244,18 @@ def test_integrated_h3_int8_loader_preserves_quantized_weights( arguments.should_use_fsdp_for_component = lambda _: True with pytest.raises(ValueError, match="FSDP"): spec.load_comfyui_transformer(pipeline, arguments) + + +def test_h3_pack_rejects_batched_latents() -> None: + """A B>1 H3 latent used to reach the worker as one packed sequence and fail + with an index_copy_ shape error; it must be rejected up front.""" + adapter = MiniMaxH3Adapter() + x = [torch.ones(2, 24, 2, 4, 4), torch.ones(2, 32, 2, 3)] + with pytest.raises(ValueError, match="batch size 1, got 2"): + adapter.pack( + x, + torch.tensor([500.0, 500.0]), + torch.ones(2, 8, 16), + minimax_payload={"audio_scale": 1.0}, + transformer_options={"sample_sigmas": torch.tensor([1.0, 0.0])}, + ) diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py new file mode 100644 index 0000000000..c2be9e996f --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_lora_node.py @@ -0,0 +1,185 @@ +# SPDX-License-Identifier: Apache-2.0 +"""SGLD LoRA: loader nodes and binding the sampled MODEL's LoRAs to the worker.""" + +import copy +import functools +import sys +import types +from contextlib import nullcontext +from types import SimpleNamespace +from unittest import mock +from unittest.mock import Mock, patch + +import pytest +import torch + +# Initialize the quantization registry before importing the LoRA layer modules. +import sglang.multimodal_gen.runtime.layers.quantization # noqa: F401 +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import ( + SGLDiffusionExecutor, +) +from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA +from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline + + +@functools.cache +def _load_nodes(): + """Import the real nodes.py with the ComfyUI modules it needs stubbed.""" + folder_paths = types.ModuleType("folder_paths") + folder_paths.folder_names_and_paths = {} + folder_paths.get_full_path = lambda folder, name: f"/{folder}/{name}" + comfy_api = types.ModuleType("comfy_api") + comfy_api_input = types.ModuleType("comfy_api.input") + comfy_api_input.VideoInput = type("VideoInput", (), {}) + comfy_api.input = comfy_api_input + stubs = { + "folder_paths": folder_paths, + "comfy_api": comfy_api, + "comfy_api.input": comfy_api_input, + } + with mock.patch.dict(sys.modules, stubs): + from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion import nodes + return nodes + + +class _Model: + """ModelPatcher stand-in: clone() copies patches like SGLDModelPatcher.clone.""" + + def __init__(self, executor, patches=None): + self.model = SimpleNamespace(diffusion_model=executor) + self.patches = dict(patches or {}) + self.model_options = {} + + def clone(self): + clone = _Model(self.model.diffusion_model, self.patches) + clone.model_options = copy.deepcopy(self.model_options) + return clone + + +def test_chained_lora_nodes_keep_every_lora() -> None: + nodes = _load_nodes() + calls = [] + executor = SimpleNamespace(set_lora=lambda **kwargs: calls.append(kwargs)) + base = _Model(executor) + + loader = nodes.SGLDLoraLoader() + (first,) = loader.load_lora(base, "style.safetensors", 1.0, nickname="style") + (second,) = loader.load_lora(first, "detail.safetensors", 0.8, nickname="detail") + + assert calls == [] + desired = second.model_options["sgld_lora_input"] + assert desired["lora_nickname"] == ["style", "detail"] + assert desired["strength"] == [1.0, 0.8] + assert desired["lora_path"] == [ + "/loras/style.safetensors", + "/loras/detail.safetensors", + ] + assert "style" not in base.patches # the upstream model is not mutated + assert set(second.patches) == {"style", "detail"} + + +class _TupleLinear(torch.nn.Linear): + def forward(self, x): + return super().forward(x), None + + +class _Pipeline(LoRAPipeline): + def create_pipeline_stages(self, args): + return None + + +def _one_layer_pipeline(): + """Identity 2x2 layer; adapter A adds to output 0 and B to output 1.""" + linear = _TupleLinear(2, 2, bias=False) + linear.weight.data.copy_(torch.eye(2)) + layer = BaseLayerWithLoRA(linear) + p = object.__new__(_Pipeline) + p.modules = {"transformer": torch.nn.Module()} + p.modules["transformer"].add_module("linear", layer) + p.server_args = SimpleNamespace( + lora_alpha=None, lora_merge_mode="merge", model_path="/model" + ) + p.lora_initialized = True + p.lora_layers = {"linear": layer} + p.lora_layers_transformer_2, p.lora_layers_critic = {}, {} + p.cur_adapter_name, p.cur_adapter_path, p.cur_adapter_strength = {}, {}, {} + p.cur_adapter_config, p.is_lora_merged = {}, {} + p._temporarily_disable_offload = lambda *a, **k: nullcontext([]) + p.loaded_adapter_paths = {"A": "/A", "B": "/B"} + p.loaded_adapter_alphas = {"A": None, "B": None} + p.lora_adapters = { + name: { + "linear.lora_A": torch.tensor([[1.0, 1.0]]), + "linear.lora_B": torch.tensor([[float(i == 0)], [float(i == 1)]]), + } + for i, name in enumerate("AB") + } + return p, layer + + +def _executor(generator): + ex = object.__new__(SGLDiffusionExecutor) + torch.nn.Module.__init__(ex) + ex._ensure_runtime, ex._lora_input, ex._run_id = None, None, 0 + ex.generator = generator + return ex + + +def _lora_input(state): + return { + "lora_nickname": [name for name, _ in state], + "lora_path": [None] * len(state), + "strength": [strength for _, strength in state], + "target": ["transformer"] * len(state), + } + + +def _sampler(state): + options = {"sgld_lora_input": _lora_input(state)} if state else {} + return SimpleNamespace(model_patcher=SimpleNamespace(model_options=options)) + + +def test_each_sampler_run_merges_exactly_its_models_loras() -> None: + """The worker is shared, so every run must leave only the LoRAs of the MODEL + it samples merged: adding, repeating, dropping and re-weighting a LoRA.""" + p, layer = _one_layer_pipeline() + ex = _executor(p) + runs = [ + [("A", 0.5)], + [("A", 0.5)], + [("A", 0.5), ("B", 0.25)], + [("B", 0.25)], + [("A", 1.0)], + [], + ] + with patch( + "sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline.dist.get_rank", + return_value=0, + ): + for state in runs: + out = ex.sampler_sample_wrapper( + lambda *a, **k: layer(torch.ones(1, 2))[0], _sampler(state) + ) + expected = torch.ones(1, 2) + for name, strength in state: + expected[0, "AB".index(name)] += 2 * strength + torch.testing.assert_close(out, expected, rtol=0, atol=0) + + +def test_failed_lora_switch_is_not_recorded_as_active() -> None: + ex = _executor(Mock()) + ex._lora_input = _lora_input([("A", 0.5)]) + ex.generator.set_lora.side_effect = ValueError("bad adapter") + with pytest.raises(ValueError, match="bad adapter"): + ex.sampler_sample_wrapper(lambda *a: None, _sampler([("B", 0.5)])) + assert ex._lora_input is None + ex.generator.unmerge_lora_weights.assert_called_once() + + +def test_chained_lora_targets_that_overlap_are_rejected() -> None: + """The worker keeps one group per overlapping target, so 'all' chained with + 'transformer' would silently drop one LoRA.""" + loader = _load_nodes().SGLDLoraLoader() + (first,) = loader.load_lora(_Model(None), "a.safetensors", 1.0, "a", "all") + with pytest.raises(ValueError, match="target 'all'"): + loader.load_lora(first, "b.safetensors", 1.0, "b", "transformer") diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py new file mode 100644 index 0000000000..d38cd33b50 --- /dev/null +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_model_patcher.py @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +"""ComfyUI must keep tracking the model when a temporary SGLD clone expires.""" + +import gc +import weakref + +import pytest +import torch + +model_management = pytest.importorskip("comfy.model_management") + +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.model_patcher import ( + SGLDModelPatcher, +) + + +def test_expired_clone_returns_loaded_model_to_live_parent(): + model = torch.nn.Linear(1, 1) + device = torch.device("cpu") + parent = SGLDModelPatcher(model, device, device, model_type="minimax_h3") + tracked = model_management.LoadedModel(parent.clone().clone()) + tracked.real_model = weakref.ref(model) + + gc.collect() + + assert not tracked.is_dead() + assert tracked.model is parent + assert tracked.real_model() is model