[diffusion] comfyui: fail early instead of silently dropping features in SGLD integrated mode (#43495)
Co-authored-by: OnePunchMonk <aggarwal.avaya@yahoo.com>
This commit is contained in:
co-authored by
OnePunchMonk
parent
d1313dd5fe
commit
5730345717
@@ -9,6 +9,7 @@ import os
|
||||
from ..executors.flux import FluxExecutor
|
||||
from ..executors.minimax_h3 import MiniMaxH3Executor
|
||||
from ..executors.zimage import ZImageExecutor
|
||||
from .preflight import check_sgld_options
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -204,12 +205,24 @@ class SGLDiffusionGenerator:
|
||||
# policy otherwise sets dit_cpu_offload=True and every sampler step
|
||||
# reloads the DiT from CPU.
|
||||
kwargs.setdefault("dit_cpu_offload", False)
|
||||
# ComfyUI sends cond and uncond as separate requests, so worker-side CFG
|
||||
# parallel only recomputes; its auto-enable also needs a model_index.json,
|
||||
# which a non-H3 single file lacks, and fails worker startup on >= 2 GPUs.
|
||||
if not kwargs.get("enable_cfg_parallel"):
|
||||
kwargs.setdefault("cfg_parallel_degree", 1)
|
||||
kwargs = self._server_args_kwargs(kwargs)
|
||||
self.generator = DiffGenerator.from_pretrained(
|
||||
model_path=model_path,
|
||||
pipeline_class_name=pipeline_class_name,
|
||||
**kwargs,
|
||||
)
|
||||
try:
|
||||
self.generator = DiffGenerator.from_pretrained(
|
||||
model_path=model_path,
|
||||
pipeline_class_name=pipeline_class_name,
|
||||
**kwargs,
|
||||
)
|
||||
except EOFError as error:
|
||||
# A worker that raises while loading closes its pipe without a message.
|
||||
raise RuntimeError(
|
||||
"The SGLD worker exited while loading the model; its traceback is in "
|
||||
"the ComfyUI console output above this error"
|
||||
) from error
|
||||
return self.generator
|
||||
|
||||
@staticmethod
|
||||
@@ -401,6 +414,14 @@ class SGLDiffusionGenerator:
|
||||
}
|
||||
sgld_options = dict(raw_sgld_options)
|
||||
model_options = dict(reload_kwargs["model_options"])
|
||||
if model_options.get("dtype") is not None:
|
||||
# Only the ComfyUI architecture companion would get this dtype; the
|
||||
# worker loads its own weights, so the choice would be silently lost.
|
||||
raise ValueError(
|
||||
"SGLDUNETLoader weight_dtype must be 'default': the SGLD worker "
|
||||
"does not use it; select quantization in SGLDOptions instead"
|
||||
)
|
||||
check_sgld_options(sgld_options)
|
||||
plugin_flags = {}
|
||||
if "enable_cache_dit" in sgld_options:
|
||||
plugin_flags["enable_cache_dit"] = sgld_options.pop("enable_cache_dit")
|
||||
|
||||
@@ -45,6 +45,7 @@ class SGLDModelPatcher(ModelPatcher):
|
||||
|
||||
n.object_patches = self.object_patches.copy()
|
||||
n.model_options = copy.deepcopy(self.model_options)
|
||||
n.parent = self
|
||||
n.backup = self.backup
|
||||
n.object_patches_backup = self.object_patches_backup
|
||||
n.lora_cache = copy.copy(self.lora_cache)
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Reject accelerations this environment cannot run before the SGLD worker starts.
|
||||
|
||||
The worker is spawned from this interpreter, so asking SGLang's own backend
|
||||
resolution here gives the same answer the worker would get after loading the
|
||||
model; a missing kernel then fails in the loader node instead of at the first
|
||||
sampling step, or silently as another backend.
|
||||
"""
|
||||
|
||||
from pkgutil import resolve_name
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.runtime.platforms import current_platform
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import AttentionBackendEnum
|
||||
|
||||
_INSTALL_HINTS = {
|
||||
AttentionBackendEnum.SAGE_ATTN: "install SageAttention 2: https://github.com/thu-ml/SageAttention",
|
||||
AttentionBackendEnum.SAGE_ATTN_3: (
|
||||
"SageAttention 3 targets Blackwell GPUs: "
|
||||
"https://github.com/thu-ml/SageAttention/tree/main/sageattention3_blackwell"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def requested_attention_backends(sgld_options: dict) -> dict[str, str]:
|
||||
"""Option name -> requested backend, for every attention backend the options select."""
|
||||
requested = {}
|
||||
if sgld_options.get("attention_backend"):
|
||||
requested["attention_backend"] = sgld_options["attention_backend"]
|
||||
components = sgld_options.get("component_attention_backends") or {}
|
||||
if isinstance(components, str):
|
||||
components = dict(
|
||||
item.split("=", 1) for item in components.split(",") if "=" in item
|
||||
)
|
||||
for component, backend in components.items():
|
||||
requested[f"component_attention_backends.{component.strip()}"] = backend
|
||||
return requested
|
||||
|
||||
|
||||
def _resolved_backend(backend: AttentionBackendEnum) -> AttentionBackendEnum:
|
||||
# Head size and dtype of a typical DiT layer; only availability matters here.
|
||||
cls_path = current_platform.get_attn_backend_cls_str(backend, 128, torch.bfloat16)
|
||||
return resolve_name(cls_path).get_enum()
|
||||
|
||||
|
||||
def check_attention_backends(sgld_options: dict) -> None:
|
||||
for option, name in requested_attention_backends(sgld_options).items():
|
||||
try:
|
||||
backend = AttentionBackendEnum[name.strip().upper()]
|
||||
except KeyError:
|
||||
raise ValueError(
|
||||
f"{option}={name!r} is not an SGLang attention backend"
|
||||
) from None
|
||||
try:
|
||||
resolved = _resolved_backend(backend)
|
||||
except (ImportError, ValueError) as error:
|
||||
raise ValueError(
|
||||
f"{option}={name} cannot run in this environment: {error}"
|
||||
) from error
|
||||
if resolved is not backend:
|
||||
hint = _INSTALL_HINTS.get(backend, "install it or choose another backend")
|
||||
raise ValueError(
|
||||
f"{option}={name} is not available in this environment (SGLang "
|
||||
f"would run {resolved.name.lower()} instead); {hint}"
|
||||
)
|
||||
|
||||
|
||||
def check_parallel_layout(sgld_options: dict) -> None:
|
||||
dp = sgld_options.get("dp_size") or 1
|
||||
if dp > 1:
|
||||
# Each sampler step is one request carrying CUDA IPC tensors from this
|
||||
# process, and ComfyUI runs one prompt at a time.
|
||||
raise ValueError(
|
||||
"dp_size > 1 is not supported in ComfyUI integrated mode: replicas never "
|
||||
"overlap behind ComfyUI's single prompt queue. Run one ComfyUI instance "
|
||||
"per GPU, or use tp_size / sp_degree for one request."
|
||||
)
|
||||
tp = sgld_options.get("tp_size")
|
||||
sp = sgld_options.get("sp_degree")
|
||||
if (
|
||||
sp is None
|
||||
and sgld_options.get("ulysses_degree")
|
||||
and sgld_options.get("ring_degree")
|
||||
):
|
||||
sp = sgld_options["ulysses_degree"] * sgld_options["ring_degree"]
|
||||
if tp is None or sp is None:
|
||||
return # SGLang fills unset degrees from num_gpus
|
||||
cfg = (
|
||||
(sgld_options.get("cfg_parallel_degree") or 2)
|
||||
if sgld_options.get("enable_cfg_parallel")
|
||||
else 1
|
||||
)
|
||||
num_gpus = sgld_options.get("num_gpus") or 1
|
||||
if dp * tp * sp * cfg != num_gpus:
|
||||
# A GPU outside the layout gets no process group; its worker hangs at startup.
|
||||
raise ValueError(
|
||||
f"num_gpus ({num_gpus}) must equal tp_size * sp_degree"
|
||||
f"{' * cfg_parallel_degree' if cfg > 1 else ''} = {dp * tp * sp * cfg}; "
|
||||
"set the degrees to fill every GPU or leave sp_degree at -1 (auto)"
|
||||
)
|
||||
|
||||
|
||||
def check_sgld_options(sgld_options: dict) -> None:
|
||||
check_attention_backends(sgld_options)
|
||||
check_parallel_layout(sgld_options)
|
||||
@@ -38,6 +38,9 @@ class ComfyUIModelAdapter:
|
||||
|
||||
model_types: tuple[str, ...] = ()
|
||||
pipeline_class_name: str = ""
|
||||
# Conditioning apply_model kwargs this adapter forwards to the worker;
|
||||
# the executor rejects the other content kwargs instead of dropping them.
|
||||
applied_conditioning: tuple[str, ...] = ()
|
||||
|
||||
def __init_subclass__(cls, **kwargs):
|
||||
super().__init_subclass__(**kwargs)
|
||||
|
||||
@@ -20,6 +20,49 @@ else:
|
||||
_RUNTIME_IMPORT_ERROR = None
|
||||
|
||||
|
||||
def _reject_model_patches(transformer_options) -> None:
|
||||
# The DiT runs in the SGLD worker, so ComfyUI block/attention patches would
|
||||
# be dropped silently (e.g. MiniMax H3 Fun ControlNet, ModelAttentionBackend).
|
||||
opts = transformer_options or {}
|
||||
found = [
|
||||
f"patches_replace.{name}"
|
||||
for name, blocks in (opts.get("patches_replace") or {}).items()
|
||||
if blocks
|
||||
]
|
||||
found += [
|
||||
f"patches.{name}"
|
||||
for name, items in (opts.get("patches") or {}).items()
|
||||
if items
|
||||
]
|
||||
if opts.get("optimized_attention_override") is not None:
|
||||
found.append("optimized_attention_override")
|
||||
if found:
|
||||
raise ValueError(
|
||||
"SGLD integrated mode runs the diffusion model in its worker and cannot "
|
||||
f"apply ComfyUI model patches ({', '.join(sorted(found))}); remove the "
|
||||
"patch nodes (ControlNet, attention backend, block patches) or use the "
|
||||
"native ComfyUI loader"
|
||||
)
|
||||
|
||||
|
||||
# apply_model kwargs that carry image content (ControlNet residuals, reference latents).
|
||||
CONTENT_CONDITIONING = ("control", "ref_latents")
|
||||
|
||||
|
||||
def _reject_unapplied_conditioning(adapter, kwargs) -> None:
|
||||
dropped = [
|
||||
name
|
||||
for name in CONTENT_CONDITIONING
|
||||
if name not in adapter.applied_conditioning and kwargs.get(name)
|
||||
]
|
||||
if dropped:
|
||||
raise ValueError(
|
||||
f"SGLD integrated mode does not apply {', '.join(dropped)} for this model "
|
||||
"(ControlNet / reference-latent nodes); remove those nodes or use the "
|
||||
"native ComfyUI loader"
|
||||
)
|
||||
|
||||
|
||||
class SGLDiffusionExecutor(torch.nn.Module):
|
||||
"""Shared ComfyUI DiT-forward executor. Per-model logic lives on the adapter."""
|
||||
|
||||
@@ -29,7 +72,9 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
super(SGLDiffusionExecutor, self).__init__()
|
||||
self.generator = generator
|
||||
self.model_path = model_path
|
||||
self.model = model
|
||||
# Not a submodule: the BaseModel holds this executor as diffusion_model,
|
||||
# and registering it back makes state_dict() / .to() recurse forever.
|
||||
object.__setattr__(self, "model", model)
|
||||
self.dtype = config.unet_config["dtype"]
|
||||
self.config = config
|
||||
self.loras = []
|
||||
@@ -43,6 +88,13 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
self._run_id = 0
|
||||
self._sent_conds: set[tuple] = set()
|
||||
|
||||
def state_dict(self, *args, **kwargs):
|
||||
raise RuntimeError(
|
||||
"The SGLD worker owns this diffusion model's weights, so ComfyUI weight "
|
||||
"nodes (LoraLoader, ModelSave, model merging) cannot use it; load LoRAs "
|
||||
"with SGLDLoraLoader"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def should_suppress_logs(timestep):
|
||||
"""Determine if logs should be suppressed based on timestep value."""
|
||||
@@ -52,7 +104,7 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
|
||||
def set_lora(self, lora_nickname=None, lora_path=None, strength=None, target=None):
|
||||
"""Set LoRA adapter using SGLang Diffusion API."""
|
||||
self._lora_input = {
|
||||
desired = {
|
||||
"lora_nickname": lora_nickname,
|
||||
"lora_path": lora_path,
|
||||
"strength": strength,
|
||||
@@ -66,6 +118,8 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
target=target,
|
||||
)
|
||||
|
||||
self._lora_input = desired
|
||||
|
||||
def begin_sampler_run(self) -> None:
|
||||
"""One ComfyUI ``sampler.sample()`` invocation is one cache lifetime."""
|
||||
self._run_id += 1
|
||||
@@ -75,6 +129,16 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
"""Run cache is evicted on the next bind of a newer id for this executor."""
|
||||
|
||||
def sampler_sample_wrapper(self, executor, *args, **kwargs):
|
||||
if self._ensure_runtime is not None:
|
||||
self._ensure_runtime(self)
|
||||
model_wrap = args[0] if args else kwargs["model_wrap"]
|
||||
desired = model_wrap.model_patcher.model_options.get("sgld_lora_input")
|
||||
if desired != self._lora_input:
|
||||
if self._lora_input is not None:
|
||||
self.generator.unmerge_lora_weights()
|
||||
self._lora_input = None
|
||||
if desired is not None:
|
||||
self.set_lora(**desired)
|
||||
self.begin_sampler_run()
|
||||
try:
|
||||
return executor(*args, **kwargs)
|
||||
@@ -149,8 +213,12 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
torch.Generator("cuda") for _ in range(req.num_outputs_per_prompt)
|
||||
]
|
||||
output_batch = self.generator._send_to_scheduler_and_wait_for_response([req])
|
||||
if output_batch.error:
|
||||
raise RuntimeError(f"SGLang Diffusion worker failed: {output_batch.error}")
|
||||
return self.adapter.unpack(output_batch.noise_pred, packed, x)
|
||||
|
||||
def forward(self, x, timestep, context, **kwargs):
|
||||
_reject_model_patches(kwargs.get("transformer_options"))
|
||||
_reject_unapplied_conditioning(self.adapter, kwargs)
|
||||
packed = self.adapter.pack(x, timestep, context, **kwargs)
|
||||
return self._execute_packed(packed, x, timestep)
|
||||
|
||||
@@ -86,6 +86,10 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
|
||||
|
||||
def pack(self, x, timestep, context, **kwargs) -> PackedForward:
|
||||
video, audio_src = _split_av(x)
|
||||
if video.shape[0] != 1:
|
||||
# Same limit as ComfyUI's MiniMax H3 model: the checkpoint is
|
||||
# CFG-distilled and the packed sequence carries a single sample.
|
||||
raise ValueError(f"MiniMax H3 supports batch size 1, got {video.shape[0]}")
|
||||
# PackedLayout is a ComfyUI class; the SGLD worker cannot unpickle it.
|
||||
payload = dict(kwargs.get("minimax_payload") or {})
|
||||
layout = payload.pop("layout", None)
|
||||
|
||||
@@ -74,6 +74,7 @@ class QwenImageAdapter(ComfyUIModelAdapter):
|
||||
class QwenImageEditAdapter(QwenImageAdapter):
|
||||
model_types = ("qwen_image_edit",)
|
||||
pipeline_class_name = "QwenImageEditPlusPipeline"
|
||||
applied_conditioning = ("ref_latents",)
|
||||
|
||||
def pack(
|
||||
self,
|
||||
|
||||
@@ -213,9 +213,17 @@ class SGLDLoraLoader:
|
||||
lora_input["strength"].append(lora_info[1])
|
||||
lora_input["target"].append(lora_info[2])
|
||||
|
||||
# call the SGLang Diffusion API
|
||||
model.model.diffusion_model.set_lora(**lora_input)
|
||||
return (model,)
|
||||
targets = set(lora_input["target"])
|
||||
if "all" in targets and len(targets) > 1:
|
||||
# set_lora groups adapters per target string; "all" overlapping a
|
||||
# named target makes the worker keep only one group on that module.
|
||||
raise ValueError(
|
||||
"Chained SGLDLoraLoader nodes mix target 'all' with "
|
||||
f"{sorted(targets - {'all'})}; use the same target on every node"
|
||||
)
|
||||
# The shared worker is bound to the selected MODEL at sampling time.
|
||||
bi.model_options["sgld_lora_input"] = lora_input
|
||||
return (bi,)
|
||||
|
||||
|
||||
class SGLDUNETLoader:
|
||||
|
||||
@@ -1,16 +1,27 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Pack / unpack contract for ComfyUI model adapters."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import (
|
||||
get_adapter_class,
|
||||
registered_model_types,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import FluxAdapter
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
|
||||
SGLDiffusionExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import (
|
||||
FluxAdapter,
|
||||
FluxExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.zimage import (
|
||||
ZImageAdapter,
|
||||
)
|
||||
from sglang.multimodal_gen.configs.sample.sampling_params import SamplingParams
|
||||
|
||||
|
||||
def test_registered_comfyui_model_types() -> None:
|
||||
@@ -56,3 +67,92 @@ def test_flux_pack_and_unpack_roundtrip() -> None:
|
||||
|
||||
default = adapter.pack(x, timestep, context, y=y)
|
||||
assert default.guidance_scale == 3.5
|
||||
|
||||
|
||||
def test_worker_error_is_raised_not_unpacked() -> None:
|
||||
"""Unpacking a failed reply replaced the worker's message with a misleading
|
||||
adapter TypeError about noise_pred being None."""
|
||||
ex = SGLDiffusionExecutor.__new__(SGLDiffusionExecutor)
|
||||
torch.nn.Module.__init__(ex)
|
||||
ex.adapter, ex.model_path = FluxAdapter(), "/test-model"
|
||||
ex.session_id, ex._run_id, ex._sent_conds = "error-test", 0, set()
|
||||
ex.generator = SimpleNamespace(
|
||||
server_args=SimpleNamespace(attention_backend_config={}, enable_trace=False),
|
||||
_send_to_scheduler_and_wait_for_response=lambda reqs: SimpleNamespace(
|
||||
noise_pred=None, error="index_copy_(): shape mismatch"
|
||||
),
|
||||
)
|
||||
x, t = torch.randn(1, 16, 8, 8), torch.full((1,), 0.5)
|
||||
packed = ex.adapter.pack(x, t, torch.randn(1, 7, 32), y=torch.randn(1, 768))
|
||||
with (
|
||||
patch.object(
|
||||
SamplingParams,
|
||||
"from_user_sampling_params_args",
|
||||
side_effect=lambda model_path, server_args, **kw: SamplingParams(**kw),
|
||||
),
|
||||
patch.object(torch, "Generator", side_effect=lambda device: object()),
|
||||
pytest.raises(RuntimeError, match="worker failed: index_copy_"),
|
||||
):
|
||||
ex._execute_packed(packed, x, t)
|
||||
|
||||
|
||||
class _RecordingExecutor(SGLDiffusionExecutor):
|
||||
"""The real forward(); records what would be sent to the worker."""
|
||||
|
||||
def __init__(self, adapter):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.adapter, self.sent = adapter, []
|
||||
|
||||
def _execute_packed(self, packed, x, timestep):
|
||||
self.sent.append(packed)
|
||||
return x
|
||||
|
||||
|
||||
def _flux_step(ex, **kwargs):
|
||||
x, t = torch.randn(1, 16, 8, 8), torch.full((1,), 0.5)
|
||||
ex(x, t, torch.randn(1, 7, 32), y=torch.randn(1, 768), **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options",
|
||||
[
|
||||
{"patches_replace": {"dit": {("double_block", 3): object()}}},
|
||||
{"patches": {"attn1_patch": [object()]}},
|
||||
{"optimized_attention_override": object()},
|
||||
],
|
||||
)
|
||||
def test_comfy_model_patches_are_rejected_not_dropped(options) -> None:
|
||||
"""ComfyUI model patches (H3 Fun ControlNet block replace, attention backend
|
||||
override) never reach the worker; ignoring them gave bit-identical output."""
|
||||
ex = _RecordingExecutor(FluxAdapter())
|
||||
with pytest.raises(ValueError, match="cannot apply ComfyUI model patches"):
|
||||
_flux_step(ex, transformer_options=options)
|
||||
_flux_step(ex, transformer_options={"patches": {}, "patches_replace": {"dit": {}}})
|
||||
assert len(ex.sent) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"kwargs",
|
||||
[
|
||||
{"control": {"output": [torch.ones(1)]}},
|
||||
{"ref_latents": [torch.ones(1, 16, 8, 8)]},
|
||||
],
|
||||
)
|
||||
def test_conditioning_the_worker_cannot_apply_is_rejected(kwargs) -> None:
|
||||
"""ControlNet residuals and reference latents reach apply_model as kwargs; an
|
||||
adapter that does not forward them must fail instead of ignoring them."""
|
||||
ex = _RecordingExecutor(FluxAdapter())
|
||||
with pytest.raises(ValueError, match=next(iter(kwargs))):
|
||||
_flux_step(ex, **kwargs)
|
||||
assert ex.sent == []
|
||||
|
||||
|
||||
def test_comfyui_weight_access_gets_a_clear_error() -> None:
|
||||
"""ComfyUI's BaseModel holds the executor as diffusion_model; LoraLoader's key
|
||||
map calls state_dict() on it, which must not recurse through the executor."""
|
||||
base = torch.nn.Module()
|
||||
config = SimpleNamespace(unet_config={"dtype": torch.bfloat16})
|
||||
base.diffusion_model = FluxExecutor(None, "flux.safetensors", base, config)
|
||||
base.to("cpu")
|
||||
with pytest.raises(RuntimeError, match="SGLDLoraLoader"):
|
||||
base.state_dict()
|
||||
|
||||
@@ -2,6 +2,10 @@
|
||||
"""Process-wide SGLD worker ownership for ComfyUI loaders."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.generator import (
|
||||
SGLDiffusionGenerator,
|
||||
@@ -135,3 +139,124 @@ def test_generator_reports_real_runtime_import_error(caplog) -> None:
|
||||
with pytest.raises(RuntimeError, match="failed to import") as err:
|
||||
module.SGLDiffusionGenerator().init_generator("flux", "FluxPipeline", {})
|
||||
assert isinstance(err.value.__cause__, ImportError)
|
||||
|
||||
|
||||
def _runtime_that_must_reject() -> SGLDiffusionGenerator:
|
||||
"""A loader whose model build fails loudly, so a check that should reject
|
||||
first is caught if it lets the options through."""
|
||||
runtime = SGLDiffusionGenerator()
|
||||
runtime.get_comfyui_model = Mock(
|
||||
side_effect=AssertionError("must reject before building the model")
|
||||
)
|
||||
return runtime
|
||||
|
||||
|
||||
def test_non_default_weight_dtype_is_rejected_before_worker_load() -> None:
|
||||
"""weight_dtype only reached the ComfyUI architecture companion, so fp8
|
||||
produced output bit-identical to the default load."""
|
||||
with pytest.raises(ValueError, match="weight_dtype must be 'default'"):
|
||||
_runtime_that_must_reject().load_model(
|
||||
model_path="h3.safetensors",
|
||||
model_options={"dtype": torch.float8_e4m3fn},
|
||||
sgld_options={},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options,error",
|
||||
[
|
||||
({"attention_backend": "sage_attn_3"}, "would run torch_sdpa instead"),
|
||||
({"component_attention_backends": "transformer=sol_attn"}, "not installed"),
|
||||
({"attention_backend": "not_a_backend"}, "not an SGLang attention backend"),
|
||||
({"attention_backend": "sage_attn"}, None),
|
||||
],
|
||||
)
|
||||
def test_attention_backends_are_checked_before_worker_load(
|
||||
monkeypatch, options, error
|
||||
) -> None:
|
||||
"""SGLang serves a missing sage_attn / sage_attn_3 kernel as FA / SDPA, so an
|
||||
explicit choice ran another backend; a missing sparse kernel only failed
|
||||
after the full model load."""
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import preflight
|
||||
from sglang.multimodal_gen.runtime.platforms.interface import (
|
||||
AttentionBackendEnum as Backend,
|
||||
)
|
||||
|
||||
def resolved(backend):
|
||||
if backend is Backend.SOL_ATTN:
|
||||
raise ImportError("Sol-Attn backend is not installed")
|
||||
return {Backend.SAGE_ATTN_3: Backend.TORCH_SDPA}.get(backend, backend)
|
||||
|
||||
monkeypatch.setattr(preflight, "_resolved_backend", resolved)
|
||||
with pytest.raises(
|
||||
ValueError if error else AssertionError, match=error or "must reject before"
|
||||
):
|
||||
_runtime_that_must_reject().load_model(
|
||||
model_path="h3.safetensors", sgld_options=options
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"options,error",
|
||||
[
|
||||
({"num_gpus": 2, "tp_size": 1, "sp_degree": 1}, "must equal"),
|
||||
({"num_gpus": 2, "tp_size": 2, "sp_degree": 2}, "must equal"),
|
||||
({"num_gpus": 2, "dp_size": 2}, "dp_size > 1"),
|
||||
({"num_gpus": 2, "tp_size": 2, "sp_degree": 1}, None),
|
||||
({"num_gpus": 2, "tp_size": 1, "sp_degree": None}, None),
|
||||
],
|
||||
)
|
||||
def test_parallel_layout_is_checked_before_worker_load(options, error) -> None:
|
||||
"""num_gpus=2 with tp=sp=1 left rank 1 without a process group, hanging
|
||||
worker startup forever; dp_size=2 sent sampler steps to a replica without
|
||||
the run's cached conditioning."""
|
||||
with pytest.raises(
|
||||
ValueError if error else AssertionError, match=error or "must reject before"
|
||||
):
|
||||
_runtime_that_must_reject().load_model(
|
||||
model_path="h3.safetensors", sgld_options=options
|
||||
)
|
||||
|
||||
|
||||
def test_worker_exit_during_load_names_where_the_reason_is(monkeypatch) -> None:
|
||||
"""A worker that raised while loading surfaced as an empty EOFError."""
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator
|
||||
|
||||
def exit_during_load(**kwargs):
|
||||
raise EOFError
|
||||
|
||||
monkeypatch.setattr(generator.DiffGenerator, "from_pretrained", exit_during_load)
|
||||
with pytest.raises(RuntimeError, match="traceback is in the ComfyUI console"):
|
||||
SGLDiffusionGenerator().init_generator("h3.safetensors", "MiniMaxH3Pipeline")
|
||||
|
||||
|
||||
def test_multi_gpu_single_file_starts_without_cfg_parallel(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
"""Auto CFG parallel read a model_index.json, so a Flux single file on two GPUs
|
||||
failed ServerArgs validation; ComfyUI already splits cond/uncond requests."""
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import generator
|
||||
from sglang.multimodal_gen.runtime.server_args import ServerArgs
|
||||
|
||||
path = tmp_path / "flux.safetensors"
|
||||
save_file({"double_blocks.0.img_attn.qkv.weight": torch.zeros(1)}, path)
|
||||
seen = {}
|
||||
monkeypatch.setattr(
|
||||
generator.DiffGenerator,
|
||||
"from_pretrained",
|
||||
lambda **kwargs: seen.update(kwargs) or object(),
|
||||
)
|
||||
SGLDiffusionGenerator().init_generator(str(path), "FluxPipeline", {"num_gpus": 2})
|
||||
assert seen["cfg_parallel_degree"] == 1
|
||||
if torch.cuda.device_count() >= 2:
|
||||
server_args = ServerArgs.from_kwargs(**seen)
|
||||
assert server_args.enable_cfg_parallel is False
|
||||
assert server_args.sp_degree == 2
|
||||
seen.clear()
|
||||
SGLDiffusionGenerator().init_generator(
|
||||
str(path), "FluxPipeline", {"num_gpus": 2, "enable_cfg_parallel": True}
|
||||
)
|
||||
assert "cfg_parallel_degree" not in seen
|
||||
|
||||
@@ -1244,3 +1244,18 @@ def test_integrated_h3_int8_loader_preserves_quantized_weights(
|
||||
arguments.should_use_fsdp_for_component = lambda _: True
|
||||
with pytest.raises(ValueError, match="FSDP"):
|
||||
spec.load_comfyui_transformer(pipeline, arguments)
|
||||
|
||||
|
||||
def test_h3_pack_rejects_batched_latents() -> None:
|
||||
"""A B>1 H3 latent used to reach the worker as one packed sequence and fail
|
||||
with an index_copy_ shape error; it must be rejected up front."""
|
||||
adapter = MiniMaxH3Adapter()
|
||||
x = [torch.ones(2, 24, 2, 4, 4), torch.ones(2, 32, 2, 3)]
|
||||
with pytest.raises(ValueError, match="batch size 1, got 2"):
|
||||
adapter.pack(
|
||||
x,
|
||||
torch.tensor([500.0, 500.0]),
|
||||
torch.ones(2, 8, 16),
|
||||
minimax_payload={"audio_scale": 1.0},
|
||||
transformer_options={"sample_sigmas": torch.tensor([1.0, 0.0])},
|
||||
)
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""SGLD LoRA: loader nodes and binding the sampled MODEL's LoRAs to the worker."""
|
||||
|
||||
import copy
|
||||
import functools
|
||||
import sys
|
||||
import types
|
||||
from contextlib import nullcontext
|
||||
from types import SimpleNamespace
|
||||
from unittest import mock
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# Initialize the quantization registry before importing the LoRA layer modules.
|
||||
import sglang.multimodal_gen.runtime.layers.quantization # noqa: F401
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
|
||||
SGLDiffusionExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.layers.lora.linear import BaseLayerWithLoRA
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline import LoRAPipeline
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _load_nodes():
|
||||
"""Import the real nodes.py with the ComfyUI modules it needs stubbed."""
|
||||
folder_paths = types.ModuleType("folder_paths")
|
||||
folder_paths.folder_names_and_paths = {}
|
||||
folder_paths.get_full_path = lambda folder, name: f"/{folder}/{name}"
|
||||
comfy_api = types.ModuleType("comfy_api")
|
||||
comfy_api_input = types.ModuleType("comfy_api.input")
|
||||
comfy_api_input.VideoInput = type("VideoInput", (), {})
|
||||
comfy_api.input = comfy_api_input
|
||||
stubs = {
|
||||
"folder_paths": folder_paths,
|
||||
"comfy_api": comfy_api,
|
||||
"comfy_api.input": comfy_api_input,
|
||||
}
|
||||
with mock.patch.dict(sys.modules, stubs):
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion import nodes
|
||||
return nodes
|
||||
|
||||
|
||||
class _Model:
|
||||
"""ModelPatcher stand-in: clone() copies patches like SGLDModelPatcher.clone."""
|
||||
|
||||
def __init__(self, executor, patches=None):
|
||||
self.model = SimpleNamespace(diffusion_model=executor)
|
||||
self.patches = dict(patches or {})
|
||||
self.model_options = {}
|
||||
|
||||
def clone(self):
|
||||
clone = _Model(self.model.diffusion_model, self.patches)
|
||||
clone.model_options = copy.deepcopy(self.model_options)
|
||||
return clone
|
||||
|
||||
|
||||
def test_chained_lora_nodes_keep_every_lora() -> None:
|
||||
nodes = _load_nodes()
|
||||
calls = []
|
||||
executor = SimpleNamespace(set_lora=lambda **kwargs: calls.append(kwargs))
|
||||
base = _Model(executor)
|
||||
|
||||
loader = nodes.SGLDLoraLoader()
|
||||
(first,) = loader.load_lora(base, "style.safetensors", 1.0, nickname="style")
|
||||
(second,) = loader.load_lora(first, "detail.safetensors", 0.8, nickname="detail")
|
||||
|
||||
assert calls == []
|
||||
desired = second.model_options["sgld_lora_input"]
|
||||
assert desired["lora_nickname"] == ["style", "detail"]
|
||||
assert desired["strength"] == [1.0, 0.8]
|
||||
assert desired["lora_path"] == [
|
||||
"/loras/style.safetensors",
|
||||
"/loras/detail.safetensors",
|
||||
]
|
||||
assert "style" not in base.patches # the upstream model is not mutated
|
||||
assert set(second.patches) == {"style", "detail"}
|
||||
|
||||
|
||||
class _TupleLinear(torch.nn.Linear):
|
||||
def forward(self, x):
|
||||
return super().forward(x), None
|
||||
|
||||
|
||||
class _Pipeline(LoRAPipeline):
|
||||
def create_pipeline_stages(self, args):
|
||||
return None
|
||||
|
||||
|
||||
def _one_layer_pipeline():
|
||||
"""Identity 2x2 layer; adapter A adds to output 0 and B to output 1."""
|
||||
linear = _TupleLinear(2, 2, bias=False)
|
||||
linear.weight.data.copy_(torch.eye(2))
|
||||
layer = BaseLayerWithLoRA(linear)
|
||||
p = object.__new__(_Pipeline)
|
||||
p.modules = {"transformer": torch.nn.Module()}
|
||||
p.modules["transformer"].add_module("linear", layer)
|
||||
p.server_args = SimpleNamespace(
|
||||
lora_alpha=None, lora_merge_mode="merge", model_path="/model"
|
||||
)
|
||||
p.lora_initialized = True
|
||||
p.lora_layers = {"linear": layer}
|
||||
p.lora_layers_transformer_2, p.lora_layers_critic = {}, {}
|
||||
p.cur_adapter_name, p.cur_adapter_path, p.cur_adapter_strength = {}, {}, {}
|
||||
p.cur_adapter_config, p.is_lora_merged = {}, {}
|
||||
p._temporarily_disable_offload = lambda *a, **k: nullcontext([])
|
||||
p.loaded_adapter_paths = {"A": "/A", "B": "/B"}
|
||||
p.loaded_adapter_alphas = {"A": None, "B": None}
|
||||
p.lora_adapters = {
|
||||
name: {
|
||||
"linear.lora_A": torch.tensor([[1.0, 1.0]]),
|
||||
"linear.lora_B": torch.tensor([[float(i == 0)], [float(i == 1)]]),
|
||||
}
|
||||
for i, name in enumerate("AB")
|
||||
}
|
||||
return p, layer
|
||||
|
||||
|
||||
def _executor(generator):
|
||||
ex = object.__new__(SGLDiffusionExecutor)
|
||||
torch.nn.Module.__init__(ex)
|
||||
ex._ensure_runtime, ex._lora_input, ex._run_id = None, None, 0
|
||||
ex.generator = generator
|
||||
return ex
|
||||
|
||||
|
||||
def _lora_input(state):
|
||||
return {
|
||||
"lora_nickname": [name for name, _ in state],
|
||||
"lora_path": [None] * len(state),
|
||||
"strength": [strength for _, strength in state],
|
||||
"target": ["transformer"] * len(state),
|
||||
}
|
||||
|
||||
|
||||
def _sampler(state):
|
||||
options = {"sgld_lora_input": _lora_input(state)} if state else {}
|
||||
return SimpleNamespace(model_patcher=SimpleNamespace(model_options=options))
|
||||
|
||||
|
||||
def test_each_sampler_run_merges_exactly_its_models_loras() -> None:
|
||||
"""The worker is shared, so every run must leave only the LoRAs of the MODEL
|
||||
it samples merged: adding, repeating, dropping and re-weighting a LoRA."""
|
||||
p, layer = _one_layer_pipeline()
|
||||
ex = _executor(p)
|
||||
runs = [
|
||||
[("A", 0.5)],
|
||||
[("A", 0.5)],
|
||||
[("A", 0.5), ("B", 0.25)],
|
||||
[("B", 0.25)],
|
||||
[("A", 1.0)],
|
||||
[],
|
||||
]
|
||||
with patch(
|
||||
"sglang.multimodal_gen.runtime.pipelines_core.lora.pipeline.dist.get_rank",
|
||||
return_value=0,
|
||||
):
|
||||
for state in runs:
|
||||
out = ex.sampler_sample_wrapper(
|
||||
lambda *a, **k: layer(torch.ones(1, 2))[0], _sampler(state)
|
||||
)
|
||||
expected = torch.ones(1, 2)
|
||||
for name, strength in state:
|
||||
expected[0, "AB".index(name)] += 2 * strength
|
||||
torch.testing.assert_close(out, expected, rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_failed_lora_switch_is_not_recorded_as_active() -> None:
|
||||
ex = _executor(Mock())
|
||||
ex._lora_input = _lora_input([("A", 0.5)])
|
||||
ex.generator.set_lora.side_effect = ValueError("bad adapter")
|
||||
with pytest.raises(ValueError, match="bad adapter"):
|
||||
ex.sampler_sample_wrapper(lambda *a: None, _sampler([("B", 0.5)]))
|
||||
assert ex._lora_input is None
|
||||
ex.generator.unmerge_lora_weights.assert_called_once()
|
||||
|
||||
|
||||
def test_chained_lora_targets_that_overlap_are_rejected() -> None:
|
||||
"""The worker keeps one group per overlapping target, so 'all' chained with
|
||||
'transformer' would silently drop one LoRA."""
|
||||
loader = _load_nodes().SGLDLoraLoader()
|
||||
(first,) = loader.load_lora(_Model(None), "a.safetensors", 1.0, "a", "all")
|
||||
with pytest.raises(ValueError, match="target 'all'"):
|
||||
loader.load_lora(first, "b.safetensors", 1.0, "b", "transformer")
|
||||
@@ -0,0 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""ComfyUI must keep tracking the model when a temporary SGLD clone expires."""
|
||||
|
||||
import gc
|
||||
import weakref
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
model_management = pytest.importorskip("comfy.model_management")
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core.model_patcher import (
|
||||
SGLDModelPatcher,
|
||||
)
|
||||
|
||||
|
||||
def test_expired_clone_returns_loaded_model_to_live_parent():
|
||||
model = torch.nn.Linear(1, 1)
|
||||
device = torch.device("cpu")
|
||||
parent = SGLDModelPatcher(model, device, device, model_type="minimax_h3")
|
||||
tracked = model_management.LoadedModel(parent.clone().clone())
|
||||
tracked.real_model = weakref.ref(model)
|
||||
|
||||
gc.collect()
|
||||
|
||||
assert not tracked.is_dead()
|
||||
assert tracked.model is parent
|
||||
assert tracked.real_model() is model
|
||||
Reference in New Issue
Block a user