[diffusion] comfyui: consolidate executor conditioning-cache fixes (#43492)
Co-authored-by: niehen6174 <1639206518@qq.com>
This commit is contained in:
co-authored by
niehen6174
parent
638330d975
commit
3458c94fad
@@ -38,6 +38,9 @@ class ComfyUIModelAdapter:
|
||||
|
||||
model_types: tuple[str, ...] = ()
|
||||
pipeline_class_name: str = ""
|
||||
# extra_req fields the worker caches per cond. Dropped on a cache hit and
|
||||
# hashed into the cond key, so the two lists cannot drift apart.
|
||||
cached_extra_keys: tuple[str, ...] = ("image_latent",)
|
||||
# Conditioning apply_model kwargs this adapter forwards to the worker;
|
||||
# the executor rejects the other content kwargs instead of dropping them.
|
||||
applied_conditioning: tuple[str, ...] = ()
|
||||
@@ -74,7 +77,8 @@ class ComfyUIModelAdapter:
|
||||
packed.prompt_embeds = []
|
||||
packed.prompt_seq_lens = None
|
||||
packed.pooled_embeds = None
|
||||
packed.extra_req.pop("image_latent", None)
|
||||
for key in self.cached_extra_keys:
|
||||
packed.extra_req.pop(key, None)
|
||||
|
||||
|
||||
def get_adapter_class(model_type: str) -> type[ComfyUIModelAdapter]:
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
Base executor class for SGLang Diffusion ComfyUI integration.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import torch
|
||||
@@ -20,6 +21,88 @@ else:
|
||||
_RUNTIME_IMPORT_ERROR = None
|
||||
|
||||
|
||||
def _hash_value(digest, value) -> None:
|
||||
"""Hash ``value`` into ``digest``, framing each item so values cannot merge."""
|
||||
if torch.is_tensor(value):
|
||||
tensor = value.detach().contiguous().cpu()
|
||||
digest.update(f"T{tensor.dtype}{tuple(tensor.shape)}".encode())
|
||||
digest.update(tensor.reshape(-1).view(torch.uint8).numpy().tobytes())
|
||||
elif isinstance(value, dict):
|
||||
digest.update(f"D{len(value)}".encode())
|
||||
for key, item in value.items():
|
||||
_hash_value(digest, key)
|
||||
_hash_value(digest, item)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
digest.update(f"L{len(value)}".encode())
|
||||
for item in value:
|
||||
_hash_value(digest, item)
|
||||
else:
|
||||
text = repr(value).encode()
|
||||
digest.update(f"V{len(text)}:".encode())
|
||||
digest.update(text)
|
||||
|
||||
|
||||
# transformer_options entries with one item per cond chunk; the rest is shared
|
||||
# apart from "sigmas", which holds one item per row of a single chunk.
|
||||
_PER_CHUNK_TRANSFORMER_OPTIONS = ("cond_or_uncond", "uuids")
|
||||
|
||||
|
||||
def _slice_batch_row(value, index: int, batch: int):
|
||||
"""Row ``index`` of a batched ComfyUI argument.
|
||||
|
||||
ComfyUI concatenates every cond-derived kwarg to the full batch, so a
|
||||
leading dim of ``batch`` is per-row and 1 is shared; any other size is
|
||||
ambiguous and rejected rather than guessed.
|
||||
"""
|
||||
if torch.is_tensor(value):
|
||||
if value.ndim == 0 or value.shape[0] == 1:
|
||||
return value
|
||||
if value.shape[0] == batch:
|
||||
return value[index : index + 1]
|
||||
raise ValueError(
|
||||
f"cannot split a batch of {batch} on a tensor of shape {tuple(value.shape)}"
|
||||
)
|
||||
if type(value) in (list, tuple):
|
||||
return type(value)(_slice_batch_row(item, index, batch) for item in value)
|
||||
if type(value) is dict:
|
||||
return {
|
||||
key: _slice_batch_row(item, index, batch) for key, item in value.items()
|
||||
}
|
||||
return value
|
||||
|
||||
|
||||
def _cond_uuid_from_transformer_options(options):
|
||||
"""ComfyUI's per-cond uuid for this call, stable across steps in one run."""
|
||||
if not isinstance(options, dict):
|
||||
return None
|
||||
uuids = options.get("uuids")
|
||||
if type(uuids) in (list, tuple) and uuids:
|
||||
return tuple(uuids)
|
||||
return None
|
||||
|
||||
|
||||
def _slice_transformer_options(options, index: int, batch: int):
|
||||
"""Shared options pass through; only the per-chunk entries are sliced."""
|
||||
if not isinstance(options, dict):
|
||||
return options
|
||||
sliced = dict(options)
|
||||
for key in _PER_CHUNK_TRANSFORMER_OPTIONS:
|
||||
value = options.get(key)
|
||||
if type(value) in (list, tuple) and value and batch % len(value) == 0:
|
||||
# Each chunk holds batch // len rows.
|
||||
sliced[key] = type(value)([value[index // (batch // len(value))]])
|
||||
# ComfyUI sets sigmas to the timestep before repeating it per chunk.
|
||||
sigmas = options.get("sigmas")
|
||||
if torch.is_tensor(sigmas) and sigmas.ndim > 0 and sigmas.shape[0] > 1:
|
||||
if batch % sigmas.shape[0] != 0:
|
||||
raise ValueError(
|
||||
f"cannot split a batch of {batch} on sigmas of shape {tuple(sigmas.shape)}"
|
||||
)
|
||||
row = index % sigmas.shape[0]
|
||||
sliced["sigmas"] = sigmas[row : row + 1]
|
||||
return sliced
|
||||
|
||||
|
||||
def _reject_model_patches(transformer_options) -> None:
|
||||
# The DiT runs in the SGLD worker, so ComfyUI block/attention patches would
|
||||
# be dropped silently (e.g. MiniMax H3 Fun ControlNet, ModelAttentionBackend).
|
||||
@@ -86,7 +169,8 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
self.adapter = self.adapter_cls()
|
||||
self.session_id = uuid.uuid4().hex
|
||||
self._run_id = 0
|
||||
self._sent_conds: set[tuple] = set()
|
||||
self._sent_conds: set[str] = set()
|
||||
self._cond_key_cache: dict[tuple, str] = {}
|
||||
|
||||
def state_dict(self, *args, **kwargs):
|
||||
raise RuntimeError(
|
||||
@@ -124,6 +208,7 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
"""One ComfyUI ``sampler.sample()`` invocation is one cache lifetime."""
|
||||
self._run_id += 1
|
||||
self._sent_conds = set()
|
||||
self._cond_key_cache = {}
|
||||
|
||||
def end_sampler_run(self) -> None:
|
||||
"""Run cache is evicted on the next bind of a newer id for this executor."""
|
||||
@@ -148,24 +233,52 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
def comfyui_session_id(self) -> str:
|
||||
return f"{self.session_id}:{self._run_id}"
|
||||
|
||||
def _cond_key(self, packed) -> tuple | None:
|
||||
def _cond_key(self, packed) -> str | None:
|
||||
embeds = packed.prompt_embeds
|
||||
if not embeds:
|
||||
return None
|
||||
tensor = embeds[0]
|
||||
if not torch.is_tensor(tensor) or tensor.numel() == 0:
|
||||
return None
|
||||
flat = tensor.reshape(-1)
|
||||
return (
|
||||
tuple(int(dim) for dim in tensor.shape),
|
||||
float(flat[0].item()),
|
||||
float(flat[-1].item()),
|
||||
# A cond's content is fixed for the life of one sampler run, so with a
|
||||
# uuid from ComfyUI we only need to hash it once per run rather than
|
||||
# once per step; the memo key still folds in shape/dtype so a stale
|
||||
# entry can never be returned for tensors that don't actually match.
|
||||
cond_uuid = packed.unpack_ctx.get("comfyui_cond_uuid")
|
||||
memo_key = None
|
||||
if cond_uuid is not None:
|
||||
shape_key = tuple(
|
||||
(t.dtype, tuple(t.shape)) if torch.is_tensor(t) else None
|
||||
for t in embeds
|
||||
)
|
||||
memo_key = (cond_uuid, shape_key)
|
||||
cached = self._cond_key_cache.get(memo_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
# Hash everything drop_cached_fields removes: a hit means the worker
|
||||
# restores all of it, so a partial key would revive another cond.
|
||||
digest = hashlib.blake2b(digest_size=16)
|
||||
_hash_value(
|
||||
digest,
|
||||
(
|
||||
embeds,
|
||||
packed.pooled_embeds,
|
||||
packed.prompt_seq_lens,
|
||||
{
|
||||
key: packed.extra_req.get(key)
|
||||
for key in self.adapter.cached_extra_keys
|
||||
},
|
||||
),
|
||||
)
|
||||
key = digest.hexdigest()
|
||||
if memo_key is not None:
|
||||
self._cond_key_cache[memo_key] = key
|
||||
return key
|
||||
|
||||
def _mark_and_maybe_drop(self, packed) -> None:
|
||||
key = self._cond_key(packed)
|
||||
if key is not None:
|
||||
packed.extra_req["comfyui_cond_key"] = repr(key)
|
||||
packed.extra_req["comfyui_cond_key"] = key
|
||||
if key in self._sent_conds:
|
||||
self.adapter.drop_cached_fields(packed)
|
||||
else:
|
||||
@@ -220,5 +333,41 @@ class SGLDiffusionExecutor(torch.nn.Module):
|
||||
def forward(self, x, timestep, context, **kwargs):
|
||||
_reject_model_patches(kwargs.get("transformer_options"))
|
||||
_reject_unapplied_conditioning(self.adapter, kwargs)
|
||||
batch = int(x.shape[0]) if torch.is_tensor(x) else 1
|
||||
if batch == 1:
|
||||
return self._forward_one(x, timestep, context, **kwargs)
|
||||
# ComfyUI batches CFG cond/uncond (and batch_size > 1) into one call,
|
||||
# but the worker's comfyui path is per-sample: req.timesteps is its
|
||||
# schedule and seq lens are per request. Send one request per row.
|
||||
return torch.cat(
|
||||
[
|
||||
self._forward_one(
|
||||
_slice_batch_row(x, i, batch),
|
||||
_slice_batch_row(timestep, i, batch),
|
||||
_slice_batch_row(context, i, batch),
|
||||
_cond_row=i,
|
||||
**{
|
||||
key: (
|
||||
_slice_transformer_options(value, i, batch)
|
||||
if key == "transformer_options"
|
||||
else _slice_batch_row(value, i, batch)
|
||||
)
|
||||
for key, value in kwargs.items()
|
||||
},
|
||||
)
|
||||
for i in range(batch)
|
||||
]
|
||||
)
|
||||
|
||||
def _forward_one(self, x, timestep, context, _cond_row=None, **kwargs):
|
||||
packed = self.adapter.pack(x, timestep, context, **kwargs)
|
||||
cond_uuid = _cond_uuid_from_transformer_options(
|
||||
kwargs.get("transformer_options")
|
||||
)
|
||||
# Rows of one cond chunk share its uuid but can carry different
|
||||
# per-row conditioning (e.g. reference latents), so key on the row too.
|
||||
# unpack_ctx is executor-local, keeping _execute_packed's signature.
|
||||
packed.unpack_ctx["comfyui_cond_uuid"] = (
|
||||
None if cond_uuid is None else (cond_uuid, _cond_row)
|
||||
)
|
||||
return self._execute_packed(packed, x, timestep)
|
||||
|
||||
@@ -83,6 +83,17 @@ def _cache_fingerprint(video, sample_sigmas, serialized_layout) -> dict[str, Any
|
||||
class MiniMaxH3Adapter(ComfyUIModelAdapter):
|
||||
model_types = ("minimax_h3",)
|
||||
pipeline_class_name = "MiniMaxH3Pipeline"
|
||||
# comfyui_cache_fp / comfyui_cond_key stay; later steps still send them.
|
||||
cached_extra_keys = ComfyUIModelAdapter.cached_extra_keys + (
|
||||
"h3_payload",
|
||||
"h3_transformer_options",
|
||||
"h3_sample_sigmas",
|
||||
"h3_context",
|
||||
"h3_layout",
|
||||
"h3_denoise_mask",
|
||||
"h3_audio_denoise_mask",
|
||||
"sigmas",
|
||||
)
|
||||
|
||||
def pack(self, x, timestep, context, **kwargs) -> PackedForward:
|
||||
video, audio_src = _split_av(x)
|
||||
@@ -199,18 +210,6 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter):
|
||||
extra[key] = value
|
||||
req.extra = extra
|
||||
|
||||
def drop_cached_fields(self, packed: PackedForward) -> None:
|
||||
super().drop_cached_fields(packed)
|
||||
packed.extra_req.pop("h3_payload", None)
|
||||
packed.extra_req.pop("h3_transformer_options", None)
|
||||
packed.extra_req.pop("h3_sample_sigmas", None)
|
||||
packed.extra_req.pop("h3_context", None)
|
||||
packed.extra_req.pop("h3_layout", None)
|
||||
packed.extra_req.pop("h3_denoise_mask", None)
|
||||
packed.extra_req.pop("h3_audio_denoise_mask", None)
|
||||
packed.extra_req.pop("sigmas", None)
|
||||
# Keep comfyui_cache_fp / comfyui_cond_key; later steps still send them.
|
||||
|
||||
|
||||
class MiniMaxH3Executor(SGLDiffusionExecutor):
|
||||
adapter_cls = MiniMaxH3Adapter
|
||||
|
||||
@@ -124,8 +124,8 @@ def _evict_other_runs(session_id: str) -> None:
|
||||
def bind_comfyui_session(req):
|
||||
"""Restore cached conditioning, then refresh the cache from whatever is set.
|
||||
|
||||
Conditioning is stored per ``comfyui_cond_key`` so CFG pos/neg do not
|
||||
overwrite each other. Packed extras are restored only when
|
||||
Conditioning and packed extras are stored per ``comfyui_cond_key`` so CFG
|
||||
pos/neg do not overwrite each other. Extras are restored only when
|
||||
``comfyui_cache_fp`` matches; a mismatch must not revive a stale layout.
|
||||
"""
|
||||
sid = session_id_from_req(req)
|
||||
@@ -137,6 +137,7 @@ def bind_comfyui_session(req):
|
||||
cond_key = extra.get("comfyui_cond_key") or "_"
|
||||
cached = _SESSIONS.get(sid) or {}
|
||||
cond_snap = (cached.get("_conds") or {}).get(cond_key) or {}
|
||||
extra_snap = (cached.get("_extra") or {}).get(cond_key) or {}
|
||||
current_fp = extra.get("comfyui_cache_fp")
|
||||
cached_fp = cached.get("_fp")
|
||||
extra_ok = current_fp is None or cached_fp is None or current_fp == cached_fp
|
||||
@@ -145,7 +146,7 @@ def bind_comfyui_session(req):
|
||||
if _is_empty(getattr(req, name, None)):
|
||||
setattr(req, name, value)
|
||||
if extra_ok:
|
||||
for key, value in (cached.get("_extra") or {}).items():
|
||||
for key, value in extra_snap.items():
|
||||
if _is_empty(extra.get(key)):
|
||||
extra[key] = value
|
||||
req.extra = extra
|
||||
@@ -164,14 +165,17 @@ def bind_comfyui_session(req):
|
||||
conds = dict(cached.get("_conds") or {})
|
||||
if new_cond:
|
||||
conds[cond_key] = new_cond
|
||||
snapshot: dict[str, Any] = {"_conds": conds}
|
||||
extras = dict(cached.get("_extra") or {}) if extra_ok else {}
|
||||
if extra_snapshot:
|
||||
snapshot["_extra"] = extra_snapshot
|
||||
extras[cond_key] = extra_snapshot
|
||||
snapshot: dict[str, Any] = {"_conds": conds}
|
||||
if extras:
|
||||
snapshot["_extra"] = extras
|
||||
if current_fp is not None:
|
||||
snapshot["_fp"] = current_fp
|
||||
elif extra_ok and cached_fp is not None:
|
||||
snapshot["_fp"] = cached_fp
|
||||
if conds or extra_snapshot:
|
||||
if conds or extras:
|
||||
_SESSIONS[sid] = snapshot
|
||||
return req
|
||||
|
||||
|
||||
@@ -8,6 +8,8 @@ import pytest
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import (
|
||||
ComfyUIModelAdapter,
|
||||
PackedForward,
|
||||
get_adapter_class,
|
||||
registered_model_types,
|
||||
)
|
||||
@@ -230,3 +232,143 @@ def test_comfyui_weight_access_gets_a_clear_error() -> None:
|
||||
base.to("cpu")
|
||||
with pytest.raises(RuntimeError, match="SGLDLoraLoader"):
|
||||
base.state_dict()
|
||||
|
||||
|
||||
class _RowRecordingExecutor(SGLDiffusionExecutor):
|
||||
"""Real forward/pack/unpack; the worker round trip just records the request."""
|
||||
|
||||
def __init__(self, adapter):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.adapter = adapter
|
||||
self.sent = []
|
||||
|
||||
def _execute_packed(self, packed, x, timestep):
|
||||
self.sent.append((packed, timestep))
|
||||
# Fake noise_pred that identifies the row by its T5 context.
|
||||
noise = torch.full_like(packed.latents, float(packed.prompt_embeds[-1].mean()))
|
||||
return self.adapter.unpack(noise, packed, x)
|
||||
|
||||
|
||||
def test_batched_forward_sends_one_request_per_row() -> None:
|
||||
# ComfyUI batches CFG cond/uncond into one call; the worker path is per-sample.
|
||||
ex = _RowRecordingExecutor(FluxAdapter())
|
||||
x = torch.zeros(2, 16, 8, 8)
|
||||
timestep = torch.tensor([0.5, 0.5])
|
||||
context = torch.stack([torch.full((8, 4096), 1.0), torch.full((8, 4096), 2.0)])
|
||||
y = torch.stack([torch.full((768,), 3.0), torch.full((768,), 4.0)])
|
||||
out = ex(x, timestep, context, y=y, guidance=torch.tensor([3.5, 3.5]))
|
||||
|
||||
assert len(ex.sent) == 2
|
||||
for row, (packed, row_timestep) in enumerate(ex.sent):
|
||||
assert packed.latents.shape[0] == 1
|
||||
assert torch.equal(row_timestep, timestep[row : row + 1])
|
||||
assert torch.equal(packed.prompt_embeds[1], context[row : row + 1])
|
||||
assert torch.equal(packed.pooled_embeds[0], y[row : row + 1])
|
||||
assert packed.prompt_seq_lens == [[1], [8]]
|
||||
assert out.shape == x.shape
|
||||
assert torch.all(out[0] == 1.0) and torch.all(out[1] == 2.0)
|
||||
|
||||
|
||||
class _KwargsAdapter(ComfyUIModelAdapter):
|
||||
applied_conditioning = ("control", "ref_latents")
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def pack(self, x, timestep, context, **kwargs):
|
||||
self.calls.append((x, timestep, context, kwargs))
|
||||
return PackedForward(
|
||||
latents=torch.zeros(1),
|
||||
timesteps=timestep,
|
||||
prompt_embeds=[torch.zeros(1)],
|
||||
height=1,
|
||||
width=1,
|
||||
)
|
||||
|
||||
def unpack(self, noise_pred, packed, x):
|
||||
return x
|
||||
|
||||
|
||||
def test_batched_forward_slices_only_batched_values() -> None:
|
||||
adapter = _KwargsAdapter()
|
||||
ex = _RowRecordingExecutor(adapter)
|
||||
shared_ref = torch.ones(1, 16, 4, 4)
|
||||
sample_sigmas = torch.linspace(1.0, 0.0, 5)
|
||||
patches = {"attn": []}
|
||||
timestep = torch.tensor([0.5, 0.25])
|
||||
options = {
|
||||
"cond_or_uncond": [0, 1],
|
||||
"sigmas": timestep,
|
||||
"sample_sigmas": sample_sigmas,
|
||||
"patches": patches,
|
||||
}
|
||||
ex(
|
||||
torch.zeros(2, 16, 4, 4),
|
||||
timestep,
|
||||
torch.zeros(2, 3, 8),
|
||||
ref_latents=[
|
||||
torch.stack([torch.zeros(16, 4, 4), torch.ones(16, 4, 4)]),
|
||||
shared_ref,
|
||||
],
|
||||
control={"input": [torch.stack([torch.zeros(4), torch.ones(4)]), None]},
|
||||
transformer_options=options,
|
||||
)
|
||||
assert len(adapter.calls) == 2
|
||||
for row, (_, _, _, kwargs) in enumerate(adapter.calls):
|
||||
per_row, shared = kwargs["ref_latents"]
|
||||
assert per_row.shape == (1, 16, 4, 4) and torch.all(per_row == row)
|
||||
assert shared is shared_ref
|
||||
control, missing = kwargs["control"]["input"]
|
||||
assert control.shape == (1, 4) and torch.all(control == row)
|
||||
assert missing is None
|
||||
row_options = kwargs["transformer_options"]
|
||||
assert row_options["cond_or_uncond"] == [row]
|
||||
assert torch.equal(row_options["sigmas"], timestep[row : row + 1])
|
||||
assert row_options["sample_sigmas"] is sample_sigmas
|
||||
assert row_options["patches"] is patches
|
||||
assert options["cond_or_uncond"] == [0, 1]
|
||||
|
||||
|
||||
def test_batched_forward_slices_per_chunk_options() -> None:
|
||||
# batch_size=2 with CFG: two cond chunks of two rows each.
|
||||
adapter = _KwargsAdapter()
|
||||
ex = _RowRecordingExecutor(adapter)
|
||||
sigmas = torch.tensor([0.5, 0.25])
|
||||
ex(
|
||||
torch.zeros(4, 16, 4, 4),
|
||||
torch.cat([sigmas, sigmas]),
|
||||
torch.zeros(4, 3, 8),
|
||||
transformer_options={
|
||||
"cond_or_uncond": [0, 1],
|
||||
"uuids": ["pos", "neg"],
|
||||
"sigmas": sigmas,
|
||||
},
|
||||
)
|
||||
seen = [kwargs["transformer_options"] for *_, kwargs in adapter.calls]
|
||||
assert [o["cond_or_uncond"] for o in seen] == [[0], [0], [1], [1]]
|
||||
assert [o["uuids"] for o in seen] == [["pos"], ["pos"], ["neg"], ["neg"]]
|
||||
for row, options in enumerate(seen):
|
||||
assert torch.equal(options["sigmas"], sigmas[row % 2 : row % 2 + 1])
|
||||
|
||||
|
||||
def test_batched_forward_rejects_ambiguous_leading_dim() -> None:
|
||||
ex = _RowRecordingExecutor(_KwargsAdapter())
|
||||
with pytest.raises(ValueError, match="cannot split a batch of 2"):
|
||||
ex(
|
||||
torch.zeros(2, 16, 4, 4),
|
||||
torch.tensor([0.5, 0.5]),
|
||||
torch.zeros(2, 3, 8),
|
||||
guidance=torch.zeros(3),
|
||||
)
|
||||
|
||||
|
||||
def test_unbatched_forward_is_a_single_request() -> None:
|
||||
adapter = _KwargsAdapter()
|
||||
ex = _RowRecordingExecutor(adapter)
|
||||
x = torch.zeros(1, 16, 4, 4)
|
||||
ex(x, torch.tensor([0.5]), torch.zeros(1, 3, 8))
|
||||
assert len(adapter.calls) == 1 and adapter.calls[0][0] is x
|
||||
# MiniMax-H3 passes x as [video, audio]; that path is not split either.
|
||||
av = [torch.zeros(2, 16, 1, 4, 4), torch.zeros(2, 8, 4)]
|
||||
ex(av, torch.tensor([0.5, 0.5]), torch.zeros(2, 3, 8))
|
||||
assert len(adapter.calls) == 2 and adapter.calls[1][0] is av
|
||||
|
||||
@@ -1,7 +1,23 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import uuid
|
||||
|
||||
import torch
|
||||
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import (
|
||||
ComfyUIModelAdapter,
|
||||
PackedForward,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import (
|
||||
SGLDiffusionExecutor,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import FluxAdapter
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.minimax_h3 import (
|
||||
MiniMaxH3Adapter,
|
||||
)
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.zimage import (
|
||||
ZImageAdapter,
|
||||
)
|
||||
from sglang.multimodal_gen.runtime.pipelines_core.comfyui_mode import (
|
||||
bind_comfyui_session,
|
||||
get_run_state,
|
||||
@@ -188,6 +204,227 @@ def test_cond_keys_keep_positive_and_negative_apart() -> None:
|
||||
release_comfyui_session(sid)
|
||||
|
||||
|
||||
class _Executor(SGLDiffusionExecutor):
|
||||
"""Real executor bookkeeping, without a generator or model."""
|
||||
|
||||
def __init__(self, adapter):
|
||||
torch.nn.Module.__init__(self)
|
||||
self.adapter = adapter
|
||||
self.session_id = uuid.uuid4().hex
|
||||
self._run_id = 0
|
||||
self._sent_conds = set()
|
||||
self._cond_key_cache = {}
|
||||
self.begin_sampler_run()
|
||||
self.sid = self.comfyui_session_id()
|
||||
|
||||
def send(self, packed) -> _Req:
|
||||
"""Executor half of _execute_packed, then the worker-side bind."""
|
||||
self._mark_and_maybe_drop(packed)
|
||||
req = _Req()
|
||||
self.adapter.fill_req(req, packed)
|
||||
req.extra = {
|
||||
**(req.extra or {}),
|
||||
"comfyui_session_id": self.sid,
|
||||
"comfyui_cond_key": packed.extra_req["comfyui_cond_key"],
|
||||
}
|
||||
return bind_comfyui_session(req)
|
||||
|
||||
|
||||
def test_cond_key_flux_same_pooled_different_t5_not_conflated() -> None:
|
||||
# ComfyUI's Flux pooled `y` comes from CLIP-L's first 77-token chunk only,
|
||||
# so prompts that differ later share `y` but not the T5 context.
|
||||
ex = _Executor(FluxAdapter())
|
||||
x, t = torch.zeros(1, 16, 8, 8), torch.tensor([0.5])
|
||||
y = torch.randn(1, 768)
|
||||
ctx_a, ctx_b = torch.randn(1, 8, 4096), torch.randn(1, 8, 4096)
|
||||
first = ex.send(ex.adapter.pack(x, t, ctx_a, y=y))
|
||||
second = ex.send(ex.adapter.pack(x, t, ctx_b, y=y.clone()))
|
||||
assert torch.equal(first.prompt_embeds[1], ctx_a)
|
||||
assert torch.equal(second.prompt_embeds[1], ctx_b)
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
def test_cond_key_same_first_last_scalar_not_conflated() -> None:
|
||||
ex = _Executor(ZImageAdapter())
|
||||
x, t = torch.zeros(1, 16, 8, 8), torch.tensor([0.5])
|
||||
ctx_a, ctx_b = torch.randn(1, 6, 32), torch.randn(1, 6, 32)
|
||||
ctx_b.view(-1)[0] = ctx_a.view(-1)[0]
|
||||
ctx_b.view(-1)[-1] = ctx_a.view(-1)[-1]
|
||||
ex.send(ex.adapter.pack(x, t, ctx_a))
|
||||
second = ex.send(ex.adapter.pack(x, t, ctx_b))
|
||||
assert torch.equal(second.prompt_embeds[0], ctx_b.squeeze(0))
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
def test_cond_key_covers_image_latent() -> None:
|
||||
def packed(image_latent):
|
||||
return PackedForward(
|
||||
latents=torch.zeros(1, 4, 8),
|
||||
timesteps=torch.tensor([500.0]),
|
||||
prompt_embeds=[torch.ones(3, 8)],
|
||||
prompt_seq_lens=[[3]],
|
||||
height=64,
|
||||
width=64,
|
||||
extra_req={"image_latent": image_latent},
|
||||
)
|
||||
|
||||
ex = _Executor(ComfyUIModelAdapter())
|
||||
ref_a, ref_b = torch.randn(1, 4, 8), torch.randn(1, 4, 8)
|
||||
ex.send(packed(ref_a))
|
||||
second = ex.send(packed(ref_b))
|
||||
assert torch.equal(second.image_latent, ref_b)
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
def test_cond_key_repeat_still_uses_cache() -> None:
|
||||
ex = _Executor(FluxAdapter())
|
||||
x, t = torch.zeros(1, 16, 8, 8), torch.tensor([0.5])
|
||||
y, ctx = torch.randn(1, 768), torch.randn(1, 8, 4096)
|
||||
ex.send(ex.adapter.pack(x, t, ctx, y=y))
|
||||
# ComfyUI hands over fresh tensors each step; equal content must still hit.
|
||||
repeat = ex.adapter.pack(x, t, ctx.clone(), y=y.clone())
|
||||
restored = ex.send(repeat)
|
||||
assert repeat.prompt_embeds == []
|
||||
assert torch.equal(restored.prompt_embeds[1], ctx)
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
class _RefAdapter(ComfyUIModelAdapter):
|
||||
"""Qwen-Image-Edit shaped: text context plus a per-row reference latent."""
|
||||
|
||||
applied_conditioning = ("ref_latents",)
|
||||
|
||||
def pack(self, x, timestep, context, ref_latents=None, **kwargs):
|
||||
return PackedForward(
|
||||
latents=x,
|
||||
timesteps=timestep,
|
||||
prompt_embeds=[context[0]],
|
||||
prompt_seq_lens=[[int(context.shape[1])]],
|
||||
height=8,
|
||||
width=8,
|
||||
extra_req={"image_latent": ref_latents[0]},
|
||||
)
|
||||
|
||||
|
||||
class _SendingExecutor(_Executor):
|
||||
"""Batched forward goes through the real cache path, one row at a time."""
|
||||
|
||||
def __init__(self, adapter):
|
||||
super().__init__(adapter)
|
||||
self.seen = []
|
||||
|
||||
def _execute_packed(self, packed, x, timestep):
|
||||
req = self.send(packed)
|
||||
self.seen.append(req.image_latent)
|
||||
return x
|
||||
|
||||
|
||||
def test_batched_rows_with_same_text_keep_their_own_reference() -> None:
|
||||
ex = _SendingExecutor(_RefAdapter())
|
||||
context = torch.ones(1, 3, 8).expand(2, 3, 8)
|
||||
refs = torch.stack([torch.zeros(4, 8), torch.ones(4, 8)])
|
||||
# Both rows share the cond chunk's uuid, as in a real ComfyUI call.
|
||||
ex(
|
||||
torch.zeros(2, 4, 8),
|
||||
torch.tensor([0.5, 0.5]),
|
||||
context,
|
||||
ref_latents=[refs],
|
||||
transformer_options={"uuids": ["cond-0"], "cond_or_uncond": [0]},
|
||||
)
|
||||
assert len(ex.seen) == 2
|
||||
assert torch.equal(ex.seen[0], refs[0:1])
|
||||
assert torch.equal(ex.seen[1], refs[1:2])
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
def _h3_packed(text, payload):
|
||||
return PackedForward(
|
||||
latents=torch.zeros(1, 4, 2, 2),
|
||||
timesteps=torch.tensor([500.0]),
|
||||
prompt_embeds=[text],
|
||||
prompt_seq_lens=[[int(text.shape[0])]],
|
||||
height=2,
|
||||
width=2,
|
||||
extra_req={
|
||||
"h3_payload": payload,
|
||||
"h3_context": text,
|
||||
"comfyui_cache_fp": {"spatial": (1, 2, 2)},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def test_h3_cache_hit_restores_own_extras() -> None:
|
||||
ex = _Executor(MiniMaxH3Adapter())
|
||||
pos, neg = torch.ones(3, 4), torch.zeros(3, 4)
|
||||
pos_payload, neg_payload = {"text_token_tags": [1, 2]}, {"text_token_tags": [3]}
|
||||
ex.send(_h3_packed(pos, pos_payload))
|
||||
ex.send(_h3_packed(neg, neg_payload))
|
||||
later_pos = _h3_packed(pos.clone(), dict(pos_payload))
|
||||
restored = ex.send(later_pos)
|
||||
assert "h3_context" not in later_pos.extra_req # cache hit, extras dropped
|
||||
assert torch.equal(restored.extra["h3_context"], pos)
|
||||
assert restored.extra["h3_payload"] == pos_payload
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
def test_h3_same_text_different_payload_not_conflated() -> None:
|
||||
ex = _Executor(MiniMaxH3Adapter())
|
||||
text = torch.ones(3, 4)
|
||||
ex.send(_h3_packed(text, {"refs": [torch.zeros(2, 2)]}))
|
||||
second = ex.send(_h3_packed(text.clone(), {"refs": [torch.ones(2, 2)]}))
|
||||
assert torch.equal(second.extra["h3_payload"]["refs"][0], torch.ones(2, 2))
|
||||
release_comfyui_session(ex.sid)
|
||||
|
||||
|
||||
def test_cond_key_hashes_large_tensors_inside_lists() -> None:
|
||||
ex = _Executor(MiniMaxH3Adapter())
|
||||
text = torch.ones(3, 4)
|
||||
latent_a = torch.zeros(4096)
|
||||
latent_b = latent_a.clone()
|
||||
latent_b[2048] = 1.0 # outside what repr() would print
|
||||
key_a = ex._cond_key(_h3_packed(text, {"cond_video_latents": [latent_a]}))
|
||||
key_b = ex._cond_key(_h3_packed(text, {"cond_video_latents": [latent_b]}))
|
||||
assert key_a != key_b
|
||||
|
||||
|
||||
def test_cond_key_memoized_per_uuid_within_run() -> None:
|
||||
# A cond's content can't change within one sampler run, so a repeat uuid
|
||||
# must reuse the memoized key without rehashing the (changed) tensor.
|
||||
ex = _Executor(MiniMaxH3Adapter())
|
||||
text = torch.ones(3, 4)
|
||||
packed = _h3_packed(text, {})
|
||||
packed.unpack_ctx["comfyui_cond_uuid"] = ("u1",)
|
||||
first = ex._cond_key(packed)
|
||||
mutated = _h3_packed(torch.zeros(3, 4), {})
|
||||
mutated.unpack_ctx["comfyui_cond_uuid"] = ("u1",)
|
||||
second = ex._cond_key(mutated)
|
||||
assert first == second
|
||||
|
||||
ex.begin_sampler_run()
|
||||
third = ex._cond_key(mutated)
|
||||
assert third != first
|
||||
|
||||
|
||||
def test_extras_cached_per_cond_key() -> None:
|
||||
sid = "exec4:1"
|
||||
pos_ctx, neg_ctx = torch.ones(2, 4), torch.zeros(2, 4)
|
||||
for key, ctx in (("pos", pos_ctx), ("neg", neg_ctx)):
|
||||
req = _Req()
|
||||
req.extra = {
|
||||
"comfyui_session_id": sid,
|
||||
"comfyui_cond_key": key,
|
||||
"h3_context": ctx,
|
||||
}
|
||||
req.prompt_embeds = [ctx]
|
||||
bind_comfyui_session(req)
|
||||
|
||||
later_pos = _Req()
|
||||
later_pos.extra = {"comfyui_session_id": sid, "comfyui_cond_key": "pos"}
|
||||
bind_comfyui_session(later_pos)
|
||||
assert torch.equal(later_pos.extra["h3_context"], pos_ctx)
|
||||
release_comfyui_session(sid)
|
||||
|
||||
|
||||
class _FakePipeline:
|
||||
def __init__(self):
|
||||
self.modules = {}
|
||||
|
||||
Reference in New Issue
Block a user