[diffusion] comfyui: fail early instead of silently dropping features in SGLD integrated mode (#43495)

Co-authored-by: OnePunchMonk <aggarwal.avaya@yahoo.com>
This commit is contained in:
WenhaoZhang
2026-10-11 08:58:23 +08:00
committed by GitHub
co-authored by OnePunchMonk
parent d1313dd5fe
commit 5730345717
13 changed files with 676 additions and 11 deletions
@@ -9,6 +9,7 @@ import os
from ..executors.flux import FluxExecutor
from ..executors.minimax_h3 import MiniMaxH3Executor
from ..executors.zimage import ZImageExecutor
from .preflight import check_sgld_options
logger = logging.getLogger(__name__)
@@ -204,12 +205,24 @@ class SGLDiffusionGenerator:
# policy otherwise sets dit_cpu_offload=True and every sampler step
# reloads the DiT from CPU.
kwargs.setdefault("dit_cpu_offload", False)
# ComfyUI sends cond and uncond as separate requests, so worker-side CFG
# parallel only recomputes; its auto-enable also needs a model_index.json,
# which a non-H3 single file lacks, and fails worker startup on >= 2 GPUs.
if not kwargs.get("enable_cfg_parallel"):
kwargs.setdefault("cfg_parallel_degree", 1)
kwargs = self._server_args_kwargs(kwargs)
self.generator = DiffGenerator.from_pretrained(
model_path=model_path,
pipeline_class_name=pipeline_class_name,
**kwargs,
)
try:
self.generator = DiffGenerator.from_pretrained(
model_path=model_path,
pipeline_class_name=pipeline_class_name,
**kwargs,
)
except EOFError as error:
# A worker that raises while loading closes its pipe without a message.
raise RuntimeError(
"The SGLD worker exited while loading the model; its traceback is in "
"the ComfyUI console output above this error"
) from error
return self.generator
@staticmethod
@@ -401,6 +414,14 @@ class SGLDiffusionGenerator:
}
sgld_options = dict(raw_sgld_options)
model_options = dict(reload_kwargs["model_options"])
if model_options.get("dtype") is not None:
# Only the ComfyUI architecture companion would get this dtype; the
# worker loads its own weights, so the choice would be silently lost.
raise ValueError(
"SGLDUNETLoader weight_dtype must be 'default': the SGLD worker "
"does not use it; select quantization in SGLDOptions instead"
)
check_sgld_options(sgld_options)
plugin_flags = {}
if "enable_cache_dit" in sgld_options:
plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit")
@@ -45,6 +45,7 @@ class SGLDModelPatcher(ModelPatcher):
n.object_patches = self.object_patches.copy()
n.model_options = copy.deepcopy(self.model_options)
n.parent = self
n.backup = self.backup
n.object_patches_backup = self.object_patches_backup
n.lora_cache = copy.copy(self.lora_cache)
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
"""Reject accelerations this environment cannot run before the SGLD worker starts.
The worker is spawned from this interpreter, so asking SGLang's own backend
resolution here gives the same answer the worker would get after loading the
model; a missing kernel then fails in the loader node instead of at the first
sampling step, or silently as another backend.
"""
from pkgutil import resolve_name
import torch
from sglang.multimodal_gen.runtime.platforms import current_platform
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
_INSTALL_HINTS = {
AttentionBackendEnum.SAGE_ATTN: "install SageAttention 2: https://github.com/thu-ml/SageAttention",
AttentionBackendEnum.SAGE_ATTN_3: (
"SageAttention 3 targets Blackwell GPUs: "
"https://github.com/thu-ml/SageAttention/tree/main/sageattention3_blackwell"
),
}
def requested_attention_backends(sgld_options: dict) -> dict[str, str]:
"""Option name -> requested backend, for every attention backend the options select."""
requested = {}
if sgld_options.get("attention_backend"):
requested["attention_backend"] = sgld_options["attention_backend"]
components = sgld_options.get("component_attention_backends") or {}
if isinstance(components, str):
components = dict(
item.split("=", 1) for item in components.split(",") if "=" in item
)
for component, backend in components.items():
requested[f"component_attention_backends.{component.strip()}"] = backend
return requested
def _resolved_backend(backend: AttentionBackendEnum) -> AttentionBackendEnum:
# Head size and dtype of a typical DiT layer; only availability matters here.
cls_path = current_platform.get_attn_backend_cls_str(backend, 128, torch.bfloat16)
return resolve_name(cls_path).get_enum()
def check_attention_backends(sgld_options: dict) -> None:
for option, name in requested_attention_backends(sgld_options).items():
try:
backend = AttentionBackendEnum[name.strip().upper()]
except KeyError:
raise ValueError(
f"{option}={name!r} is not an SGLang attention backend"
) from None
try:
resolved = _resolved_backend(backend)
except (ImportError, ValueError) as error:
raise ValueError(
f"{option}={name} cannot run in this environment: {error}"
) from error
if resolved is not backend:
hint = _INSTALL_HINTS.get(backend, "install it or choose another backend")
raise ValueError(
f"{option}={name} is not available in this environment (SGLang "
f"would run {resolved.name.lower()} instead); {hint}"
)
def check_parallel_layout(sgld_options: dict) -> None:
dp = sgld_options.get("dp_size") or 1
if dp > 1:
# Each sampler step is one request carrying CUDA IPC tensors from this
# process, and ComfyUI runs one prompt at a time.
raise ValueError(
"dp_size > 1 is not supported in ComfyUI integrated mode: replicas never "
"overlap behind ComfyUI's single prompt queue. Run one ComfyUI instance "
"per GPU, or use tp_size / sp_degree for one request."
)
tp = sgld_options.get("tp_size")
sp = sgld_options.get("sp_degree")
if (
sp is None
and sgld_options.get("ulysses_degree")
and sgld_options.get("ring_degree")
):
sp = sgld_options["ulysses_degree"] * sgld_options["ring_degree"]
if tp is None or sp is None:
return # SGLang fills unset degrees from num_gpus
cfg = (
(sgld_options.get("cfg_parallel_degree") or 2)
if sgld_options.get("enable_cfg_parallel")
else 1
)
num_gpus = sgld_options.get("num_gpus") or 1
if dp * tp * sp * cfg != num_gpus:
# A GPU outside the layout gets no process group; its worker hangs at startup.
raise ValueError(
f"num_gpus ({num_gpus}) must equal tp_size * sp_degree"
f"{' * cfg_parallel_degree' if cfg > 1 else ''} = {dp * tp * sp * cfg}; "
"set the degrees to fill every GPU or leave sp_degree at -1 (auto)"
)
def check_sgld_options(sgld_options: dict) -> None:
check_attention_backends(sgld_options)
check_parallel_layout(sgld_options)
@@ -38,6 +38,9 @@ class ComfyUIModelAdapter:
model_types: tuple[str, ...] = ()
pipeline_class_name: str = ""
# Conditioning apply_model kwargs this adapter forwards to the worker;
# the executor rejects the other content kwargs instead of dropping them.
applied_conditioning: tuple[str, ...] = ()
def __init_subclass__(cls, **kwargs):
super().__init_subclass__(**kwargs)
@@ -20,6 +20,49 @@ else:
_RUNTIME_IMPORT_ERROR = None
def _reject_model_patches(transformer_options) -> None:
# The DiT runs in the SGLD worker, so ComfyUI block/attention patches would
# be dropped silently (e.g. MiniMax H3 Fun ControlNet, ModelAttentionBackend).
opts = transformer_options or {}
found = [
f"patches_replace.{name}"
for name, blocks in (opts.get("patches_replace") or {}).items()
if blocks
]
found += [
f"patches.{name}"
for name, items in (opts.get("patches") or {}).items()
if items
]
if opts.get("optimized_attention_override") is not None:
found.append("optimized_attention_override")
if found:
raise ValueError(
"SGLD integrated mode runs the diffusion model in its worker and cannot "
f"apply ComfyUI model patches ({', '.join(sorted(found))}); remove the "
"patch nodes (ControlNet, attention backend, block patches) or use the "
"native ComfyUI loader"
)
# apply_model kwargs that carry image content (ControlNet residuals, reference latents).
CONTENT_CONDITIONING = ("control", "ref_latents")
def _reject_unapplied_conditioning(adapter, kwargs) -> None:
dropped = [
name
for name in CONTENT_CONDITIONING
if name not in adapter.applied_conditioning and kwargs.get(name)
]
if dropped:
raise ValueError(
f"SGLD integrated mode does not apply {', '.join(dropped)} for this model "
"(ControlNet / reference-latent nodes); remove those nodes or use the "
"native ComfyUI loader"
)
class SGLDiffusionExecutor(torch.nn.Module):
"""Shared ComfyUI DiT-forward executor. Per-model logic lives on the adapter."""
@@ -29,7 +72,9 @@ class SGLDiffusionExecutor(torch.nn.Module):
super(SGLDiffusionExecutor, self).__init__()
self.generator = generator
self.model_path = model_path
self.model = model
# Not a submodule: the BaseModel holds this executor as diffusion_model,
# and registering it back makes state_dict() / .to() recurse forever.
object.__setattr__(self, "model", model)
self.dtype = config.unet_config["dtype"]
self.config = config
self.loras = []
@@ -43,6 +88,13 @@ class SGLDiffusionExecutor(torch.nn.Module):
self._run_id = 0
self._sent_conds: set[tuple] = set()
def state_dict(self, *args, **kwargs):
raise RuntimeError(
"The SGLD worker owns this diffusion model's weights, so ComfyUI weight "
"nodes (LoraLoader, ModelSave, model merging) cannot use it; load LoRAs "
"with SGLDLoraLoader"
)
@staticmethod
def should_suppress_logs(timestep):
"""Determine if logs should be suppressed based on timestep value."""
@@ -52,7 +104,7 @@ class SGLDiffusionExecutor(torch.nn.Module):
def set_lora(self, lora_nickname=None, lora_path=None, strength=None, target=None):
"""Set LoRA adapter using SGLang Diffusion API."""
self._lora_input = {
desired = {
"lora_nickname": lora_nickname,
"lora_path": lora_path,
"strength": strength,
@@ -66,6 +118,8 @@ class SGLDiffusionExecutor(torch.nn.Module):
target=target,
)
self._lora_input = desired
def begin_sampler_run(self) -> None:
"""One ComfyUI ``sampler.sample()`` invocation is one cache lifetime."""
self._run_id += 1
@@ -75,6 +129,16 @@ class SGLDiffusionExecutor(torch.nn.Module):
"""Run cache is evicted on the next bind of a newer id for this executor."""
def sampler_sample_wrapper(self, executor, *args, **kwargs):
if self._ensure_runtime is not None:
self._ensure_runtime(self)
model_wrap = args[0] if args else kwargs["model_wrap"]
desired = model_wrap.model_patcher.model_options.get("sgld_lora_input")
if desired != self._lora_input:
if self._lora_input is not None:
self.generator.unmerge_lora_weights()
self._lora_input = None
if desired is not None:
self.set_lora(**desired)
self.begin_sampler_run()
try:
return executor(*args, **kwargs)
@@ -149,8 +213,12 @@ class SGLDiffusionExecutor(torch.nn.Module):
torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
]
output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
if output_batch.error:
raise RuntimeError(f"SGLang Diffusion worker failed: {output_batch.error}")
return self.adapter.unpack(output_batch.noise_pred, packed, x)
def forward(self, x, timestep, context, **kwargs):
_reject_model_patches(kwargs.get("transformer_options"))
_reject_unapplied_conditioning(self.adapter, kwargs)
packed = self.adapter.pack(x, timestep, context, **kwargs)
return self._execute_packed(packed, x, timestep)
@@ -86,6 +86,10 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
def pack(self, x, timestep, context, **kwargs) -> PackedForward:
video, audio_src = _split_av(x)
if video.shape[0] != 1:
# Same limit as ComfyUI's MiniMax H3 model: the checkpoint is
# CFG-distilled and the packed sequence carries a single sample.
raise ValueError(f"MiniMax H3 supports batch size 1, got {video.shape[0]}")
# PackedLayout is a ComfyUI class; the SGLD worker cannot unpickle it.
payload = dict(kwargs.get("minimax_payload") or {})
layout = payload.pop("layout", None)
@@ -74,6 +74,7 @@ class QwenImageAdapter(ComfyUIModelAdapter):
class QwenImageEditAdapter(QwenImageAdapter):
model_types = ("qwen_image_edit",)
pipeline_class_name = "QwenImageEditPlusPipeline"
applied_conditioning = ("ref_latents",)
def pack(
self,
@@ -213,9 +213,17 @@ class SGLDLoraLoader:
lora_input["strength"].append(lora_info[1])
lora_input["target"].append(lora_info[2])
# call the SGLang Diffusion API
model.model.diffusion_model.set_lora(**lora_input)
return (model,)
targets = set(lora_input["target"])
if "all" in targets and len(targets) > 1:
# set_lora groups adapters per target string; "all" overlapping a
# named target makes the worker keep only one group on that module.
raise ValueError(
"Chained SGLDLoraLoader nodes mix target 'all' with "
f"{sorted(targets - {'all'})}; use the same target on every node"
)
# The shared worker is bound to the selected MODEL at sampling time.
bi.model_options["sgld_lora_input"] = lora_input
return (bi,)
class SGLDUNETLoader:
@@ -1,16 +1,27 @@
# SPDX-License-Identifier: Apache-2.0
"""Pack / unpack contract for ComfyUI model adapters."""
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import (
get_adapter_class,
registered_model_types,
)
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import FluxAdapter
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
SGLDiffusionExecutor,
)
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import (
FluxAdapter,
FluxExecutor,
)
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.zimage import (
ZImageAdapter,
)
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
def test_registered_comfyui_model_types() -> None:
@@ -56,3 +67,92 @@ def test_flux_pack_and_unpack_roundtrip() -> None:
default = adapter.pack(x, timestep, context, y=y)
assert default.guidance_scale == 3.5
def test_worker_error_is_raised_not_unpacked() -> None:
"""Unpacking a failed reply replaced the worker's message with a misleading
adapter TypeError about noise_pred being None."""
ex = SGLDiffusionExecutor.__new__(SGLDiffusionExecutor)
torch.nn.Module.__init__(ex)
ex.adapter, ex.model_path = FluxAdapter(), "/test-model"
ex.session_id, ex._run_id, ex._sent_conds = "error-test", 0, set()
ex.generator = SimpleNamespace(
server_args=SimpleNamespace(attention_backend_config={}, enable_trace=False),
_send_to_scheduler_and_wait_for_response=lambda reqs: SimpleNamespace(
noise_pred=None, error="index_copy_(): shape mismatch"
),
)
x, t = torch.randn(1, 16, 8, 8), torch.full((1,), 0.5)
packed = ex.adapter.pack(x, t, torch.randn(1, 7, 32), y=torch.randn(1, 768))
with (
patch.object(
SamplingParams,
"from_user_sampling_params_args",
side_effect=lambda model_path, server_args, **kw: SamplingParams(**kw),
),
patch.object(torch, "Generator", side_effect=lambda device: object()),
pytest.raises(RuntimeError, match="worker failed: index_copy_"),
):
ex._execute_packed(packed, x, t)
class _RecordingExecutor(SGLDiffusionExecutor):
"""The real forward(); records what would be sent to the worker."""
def __init__(self, adapter):
torch.nn.Module.__init__(self)
self.adapter, self.sent = adapter, []
def _execute_packed(self, packed, x, timestep):
self.sent.append(packed)
return x
def _flux_step(ex, **kwargs):
x, t = torch.randn(1, 16, 8, 8), torch.full((1,), 0.5)
ex(x, t, torch.randn(1, 7, 32), y=torch.randn(1, 768), **kwargs)
@pytest.mark.parametrize(
"options",
[
{"patches_replace": {"dit": {("double_block", 3): object()}}},
{"patches": {"attn1_patch": [object()]}},
{"optimized_attention_override": object()},
],
)
def test_comfy_model_patches_are_rejected_not_dropped(options) -> None:
"""ComfyUI model patches (H3 Fun ControlNet block replace, attention backend
override) never reach the worker; ignoring them gave bit-identical output."""
ex = _RecordingExecutor(FluxAdapter())
with pytest.raises(ValueError, match="cannot apply ComfyUI model patches"):
_flux_step(ex, transformer_options=options)
_flux_step(ex, transformer_options={"patches": {}, "patches_replace": {"dit": {}}})
assert len(ex.sent) == 1
@pytest.mark.parametrize(
"kwargs",
[
{"control": {"output": [torch.ones(1)]}},
{"ref_latents": [torch.ones(1, 16, 8, 8)]},
],
)
def test_conditioning_the_worker_cannot_apply_is_rejected(kwargs) -> None:
"""ControlNet residuals and reference latents reach apply_model as kwargs; an
adapter that does not forward them must fail instead of ignoring them."""
ex = _RecordingExecutor(FluxAdapter())
with pytest.raises(ValueError, match=next(iter(kwargs))):
_flux_step(ex, **kwargs)
assert ex.sent == []
def test_comfyui_weight_access_gets_a_clear_error() -> None:
"""ComfyUI's BaseModel holds the executor as diffusion_model; LoraLoader's key
map calls state_dict() on it, which must not recurse through the executor."""
base = torch.nn.Module()
config = SimpleNamespace(unet_config={"dtype": torch.bfloat16})
base.diffusion_model = FluxExecutor(None, "flux.safetensors", base, config)
base.to("cpu")
with pytest.raises(RuntimeError, match="SGLDLoraLoader"):
base.state_dict()
@@ -2,6 +2,10 @@
"""Process-wide SGLD worker ownership for ComfyUI loaders."""
from types import SimpleNamespace
from unittest.mock import Mock
import pytest
import torch
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import (
SGLDiffusionGenerator,
@@ -135,3 +139,124 @@ def test_generator_reports_real_runtime_import_error(caplog) -> None:
with pytest.raises(RuntimeError, match="failed to import") as err:
module.SGLDiffusionGenerator().init_generator("flux", "FluxPipeline", {})
assert isinstance(err.value.__cause__, ImportError)
def _runtime_that_must_reject() -> SGLDiffusionGenerator:
"""A loader whose model build fails loudly, so a check that should reject
first is caught if it lets the options through."""
runtime = SGLDiffusionGenerator()
runtime.get_comfyui_model = Mock(
side_effect=AssertionError("must reject before building the model")
)
return runtime
def test_non_default_weight_dtype_is_rejected_before_worker_load() -> None:
"""weight_dtype only reached the ComfyUI architecture companion, so fp8
produced output bit-identical to the default load."""
with pytest.raises(ValueError, match="weight_dtype must be 'default'"):
_runtime_that_must_reject().load_model(
model_path="h3.safetensors",
model_options={"dtype": torch.float8_e4m3fn},
sgld_options={},
)
@pytest.mark.parametrize(
"options,error",
[
({"attention_backend": "sage_attn_3"}, "would run torch_sdpa instead"),
({"component_attention_backends": "transformer=sol_attn"}, "not installed"),
({"attention_backend": "not_a_backend"}, "not an SGLang attention backend"),
({"attention_backend": "sage_attn"}, None),
],
)
def test_attention_backends_are_checked_before_worker_load(
monkeypatch, options, error
) -> None:
"""SGLang serves a missing sage_attn / sage_attn_3 kernel as FA / SDPA, so an
explicit choice ran another backend; a missing sparse kernel only failed
after the full model load."""
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import preflight
from sglang.multimodal_gen.runtime.platforms.interface import (
AttentionBackendEnum as Backend,
)
def resolved(backend):
if backend is Backend.SOL_ATTN:
raise ImportError("Sol-Attn backend is not installed")
return {Backend.SAGE_ATTN_3: Backend.TORCH_SDPA}.get(backend, backend)
monkeypatch.setattr(preflight, "_resolved_backend", resolved)
with pytest.raises(
ValueError if error else AssertionError, match=error or "must reject before"
):
_runtime_that_must_reject().load_model(
model_path="h3.safetensors", sgld_options=options
)
@pytest.mark.parametrize(
"options,error",
[
({"num_gpus": 2, "tp_size": 1, "sp_degree": 1}, "must equal"),
({"num_gpus": 2, "tp_size": 2, "sp_degree": 2}, "must equal"),
({"num_gpus": 2, "dp_size": 2}, "dp_size > 1"),
({"num_gpus": 2, "tp_size": 2, "sp_degree": 1}, None),
({"num_gpus": 2, "tp_size": 1, "sp_degree": None}, None),
],
)
def test_parallel_layout_is_checked_before_worker_load(options, error) -> None:
"""num_gpus=2 with tp=sp=1 left rank 1 without a process group, hanging
worker startup forever; dp_size=2 sent sampler steps to a replica without
the run's cached conditioning."""
with pytest.raises(
ValueError if error else AssertionError, match=error or "must reject before"
):
_runtime_that_must_reject().load_model(
model_path="h3.safetensors", sgld_options=options
)
def test_worker_exit_during_load_names_where_the_reason_is(monkeypatch) -> None:
"""A worker that raised while loading surfaced as an empty EOFError."""
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator
def exit_during_load(**kwargs):
raise EOFError
monkeypatch.setattr(generator.DiffGenerator, "from_pretrained", exit_during_load)
with pytest.raises(RuntimeError, match="traceback is in the ComfyUI console"):
SGLDiffusionGenerator().init_generator("h3.safetensors", "MiniMaxH3Pipeline")
def test_multi_gpu_single_file_starts_without_cfg_parallel(
tmp_path, monkeypatch
) -> None:
"""Auto CFG parallel read a model_index.json, so a Flux single file on two GPUs
failed ServerArgs validation; ComfyUI already splits cond/uncond requests."""
import torch
from safetensors.torch import save_file
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator
from sglang.multimodal_gen.runtime.server_args import ServerArgs
path = tmp_path / "flux.safetensors"
save_file({"double_blocks.0.img_attn.qkv.weight": torch.zeros(1)}, path)
seen = {}
monkeypatch.setattr(
generator.DiffGenerator,
"from_pretrained",
lambda **kwargs: seen.update(kwargs) or object(),
)
SGLDiffusionGenerator().init_generator(str(path), "FluxPipeline", {"num_gpus": 2})
assert seen["cfg_parallel_degree"] == 1
if torch.cuda.device_count() >= 2:
server_args = ServerArgs.from_kwargs(**seen)
assert server_args.enable_cfg_parallel is False
assert server_args.sp_degree == 2
seen.clear()
SGLDiffusionGenerator().init_generator(
str(path), "FluxPipeline", {"num_gpus": 2, "enable_cfg_parallel": True}
)
assert "cfg_parallel_degree" not in seen
@@ -1244,3 +1244,18 @@ def test_integrated_h3_int8_loader_preserves_quantized_weights(
arguments.should_use_fsdp_for_component = lambda _: True
with pytest.raises(ValueError, match="FSDP"):
spec.load_comfyui_transformer(pipeline, arguments)
def test_h3_pack_rejects_batched_latents() -> None:
"""A B>1 H3 latent used to reach the worker as one packed sequence and fail
with an index_copy_ shape error; it must be rejected up front."""
adapter = MiniMaxH3Adapter()
x = [torch.ones(2, 24, 2, 4, 4), torch.ones(2, 32, 2, 3)]
with pytest.raises(ValueError, match="batch size 1, got 2"):
adapter.pack(
x,
torch.tensor([500.0, 500.0]),
torch.ones(2, 8, 16),
minimax_payload={"audio_scale": 1.0},
transformer_options={"sample_sigmas": torch.tensor([1.0, 0.0])},
)
@@ -0,0 +1,185 @@
# SPDX-License-Identifier: Apache-2.0
"""SGLD LoRA: loader nodes and binding the sampled MODEL's LoRAs to the worker."""
import copy
import functools
import sys
import types
from contextlib import nullcontext
from types import SimpleNamespace
from unittest import mock
from unittest.mock import Mock, patch
import pytest
import torch
# Initialize the quantization registry before importing the LoRA layer modules.
import sglang.multimodal_gen.runtime.layers.quantization # noqa: F401
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
SGLDiffusionExecutor,
)
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
@functools.cache
def _load_nodes():
"""Import the real nodes.py with the ComfyUI modules it needs stubbed."""
folder_paths = types.ModuleType("folder_paths")
folder_paths.folder_names_and_paths = {}
folder_paths.get_full_path = lambda folder, name: f"/{folder}/{name}"
comfy_api = types.ModuleType("comfy_api")
comfy_api_input = types.ModuleType("comfy_api.input")
comfy_api_input.VideoInput = type("VideoInput", (), {})
comfy_api.input = comfy_api_input
stubs = {
"folder_paths": folder_paths,
"comfy_api": comfy_api,
"comfy_api.input": comfy_api_input,
}
with mock.patch.dict(sys.modules, stubs):
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion import nodes
return nodes
class _Model:
"""ModelPatcher stand-in: clone() copies patches like SGLDModelPatcher.clone."""
def __init__(self, executor, patches=None):
self.model = SimpleNamespace(diffusion_model=executor)
self.patches = dict(patches or {})
self.model_options = {}
def clone(self):
clone = _Model(self.model.diffusion_model, self.patches)
clone.model_options = copy.deepcopy(self.model_options)
return clone
def test_chained_lora_nodes_keep_every_lora() -> None:
nodes = _load_nodes()
calls = []
executor = SimpleNamespace(set_lora=lambda **kwargs: calls.append(kwargs))
base = _Model(executor)
loader = nodes.SGLDLoraLoader()
(first,) = loader.load_lora(base, "style.safetensors", 1.0, nickname="style")
(second,) = loader.load_lora(first, "detail.safetensors", 0.8, nickname="detail")
assert calls == []
desired = second.model_options["sgld_lora_input"]
assert desired["lora_nickname"] == ["style", "detail"]
assert desired["strength"] == [1.0, 0.8]
assert desired["lora_path"] == [
"/loras/style.safetensors",
"/loras/detail.safetensors",
]
assert "style" not in base.patches # the upstream model is not mutated
assert set(second.patches) == {"style", "detail"}
class _TupleLinear(torch.nn.Linear):
def forward(self, x):
return super().forward(x), None
class _Pipeline(LoRAPipeline):
def create_pipeline_stages(self, args):
return None
def _one_layer_pipeline():
"""Identity 2x2 layer; adapter A adds to output 0 and B to output 1."""
linear = _TupleLinear(2, 2, bias=False)
linear.weight.data.copy_(torch.eye(2))
layer = BaseLayerWithLoRA(linear)
p = object.__new__(_Pipeline)
p.modules = {"transformer": torch.nn.Module()}
p.modules["transformer"].add_module("linear", layer)
p.server_args = SimpleNamespace(
lora_alpha=None, lora_merge_mode="merge", model_path="/model"
)
p.lora_initialized = True
p.lora_layers = {"linear": layer}
p.lora_layers_transformer_2, p.lora_layers_critic = {}, {}
p.cur_adapter_name, p.cur_adapter_path, p.cur_adapter_strength = {}, {}, {}
p.cur_adapter_config, p.is_lora_merged = {}, {}
p._temporarily_disable_offload = lambda *a, **k: nullcontext([])
p.loaded_adapter_paths = {"A": "/A", "B": "/B"}
p.loaded_adapter_alphas = {"A": None, "B": None}
p.lora_adapters = {
name: {
"linear.lora_A": torch.tensor([[1.0, 1.0]]),
"linear.lora_B": torch.tensor([[float(i == 0)], [float(i == 1)]]),
}
for i, name in enumerate("AB")
}
return p, layer
def _executor(generator):
ex = object.__new__(SGLDiffusionExecutor)
torch.nn.Module.__init__(ex)
ex._ensure_runtime, ex._lora_input, ex._run_id = None, None, 0
ex.generator = generator
return ex
def _lora_input(state):
return {
"lora_nickname": [name for name, _ in state],
"lora_path": [None] * len(state),
"strength": [strength for _, strength in state],
"target": ["transformer"] * len(state),
}
def _sampler(state):
options = {"sgld_lora_input": _lora_input(state)} if state else {}
return SimpleNamespace(model_patcher=SimpleNamespace(model_options=options))
def test_each_sampler_run_merges_exactly_its_models_loras() -> None:
"""The worker is shared, so every run must leave only the LoRAs of the MODEL
it samples merged: adding, repeating, dropping and re-weighting a LoRA."""
p, layer = _one_layer_pipeline()
ex = _executor(p)
runs = [
[("A", 0.5)],
[("A", 0.5)],
[("A", 0.5), ("B", 0.25)],
[("B", 0.25)],
[("A", 1.0)],
[],
]
with patch(
"sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline.dist.get_rank",
return_value=0,
):
for state in runs:
out = ex.sampler_sample_wrapper(
lambda *a, **k: layer(torch.ones(1, 2))[0], _sampler(state)
)
expected = torch.ones(1, 2)
for name, strength in state:
expected[0, "AB".index(name)] += 2 * strength
torch.testing.assert_close(out, expected, rtol=0, atol=0)
def test_failed_lora_switch_is_not_recorded_as_active() -> None:
ex = _executor(Mock())
ex._lora_input = _lora_input([("A", 0.5)])
ex.generator.set_lora.side_effect = ValueError("bad adapter")
with pytest.raises(ValueError, match="bad adapter"):
ex.sampler_sample_wrapper(lambda *a: None, _sampler([("B", 0.5)]))
assert ex._lora_input is None
ex.generator.unmerge_lora_weights.assert_called_once()
def test_chained_lora_targets_that_overlap_are_rejected() -> None:
"""The worker keeps one group per overlapping target, so 'all' chained with
'transformer' would silently drop one LoRA."""
loader = _load_nodes().SGLDLoraLoader()
(first,) = loader.load_lora(_Model(None), "a.safetensors", 1.0, "a", "all")
with pytest.raises(ValueError, match="target 'all'"):
loader.load_lora(first, "b.safetensors", 1.0, "b", "transformer")
@@ -0,0 +1,28 @@
# SPDX-License-Identifier: Apache-2.0
"""ComfyUI must keep tracking the model when a temporary SGLD clone expires."""
import gc
import weakref
import pytest
import torch
model_management = pytest.importorskip("comfy.model_management")
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.model_patcher import (
SGLDModelPatcher,
)
def test_expired_clone_returns_loaded_model_to_live_parent():
model = torch.nn.Linear(1, 1)
device = torch.device("cpu")
parent = SGLDModelPatcher(model, device, device, model_type="minimax_h3")
tracked = model_management.LoadedModel(parent.clone().clone())
tracked.real_model = weakref.ref(model)
gc.collect()
assert not tracked.is_dead()
assert tracked.model is parent
assert tracked.real_model() is model