From 3458c94fad9f346b8d650f9979bd4e82764a36ba Mon Sep 17 00:00:00 2001 From: Avaya Aggarwal <119044997+OnePunchMonk@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:12:00 +0530 Subject: [PATCH] [diffusion] comfyui: consolidate executor conditioning-cache fixes (#43492) Co-authored-by: niehen6174 <1639206518@qq.com> --- .../ComfyUI_SGLDiffusion/executors/adapter.py | 6 +- .../ComfyUI_SGLDiffusion/executors/base.py | 165 +++++++++++- .../executors/minimax_h3.py | 23 +- .../runtime/pipelines_core/comfyui_mode.py | 16 +- .../test/unit/test_comfyui_adapters.py | 142 +++++++++++ .../test/unit/test_comfyui_session.py | 237 ++++++++++++++++++ 6 files changed, 562 insertions(+), 27 deletions(-) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py index 705f78cf60..a807e6fd2b 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/adapter.py @@ -38,6 +38,9 @@ class ComfyUIModelAdapter: model_types: tuple[str, ...] = () pipeline_class_name: str = "" + # extra_req fields the worker caches per cond. Dropped on a cache hit and + # hashed into the cond key, so the two lists cannot drift apart. + cached_extra_keys: tuple[str, ...] = ("image_latent",) # Conditioning apply_model kwargs this adapter forwards to the worker; # the executor rejects the other content kwargs instead of dropping them. applied_conditioning: tuple[str, ...] = () @@ -74,7 +77,8 @@ class ComfyUIModelAdapter: packed.prompt_embeds = [] packed.prompt_seq_lens = None packed.pooled_embeds = None - packed.extra_req.pop("image_latent", None) + for key in self.cached_extra_keys: + packed.extra_req.pop(key, None) def get_adapter_class(model_type: str) -> type[ComfyUIModelAdapter]: diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py index 44d659687c..ef65b23ac8 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/base.py @@ -2,6 +2,7 @@ Base executor class for SGLang Diffusion ComfyUI integration. """ +import hashlib import uuid import torch @@ -20,6 +21,88 @@ else: _RUNTIME_IMPORT_ERROR = None +def _hash_value(digest, value) -> None: + """Hash ``value`` into ``digest``, framing each item so values cannot merge.""" + if torch.is_tensor(value): + tensor = value.detach().contiguous().cpu() + digest.update(f"T{tensor.dtype}{tuple(tensor.shape)}".encode()) + digest.update(tensor.reshape(-1).view(torch.uint8).numpy().tobytes()) + elif isinstance(value, dict): + digest.update(f"D{len(value)}".encode()) + for key, item in value.items(): + _hash_value(digest, key) + _hash_value(digest, item) + elif isinstance(value, (list, tuple)): + digest.update(f"L{len(value)}".encode()) + for item in value: + _hash_value(digest, item) + else: + text = repr(value).encode() + digest.update(f"V{len(text)}:".encode()) + digest.update(text) + + +# transformer_options entries with one item per cond chunk; the rest is shared +# apart from "sigmas", which holds one item per row of a single chunk. +_PER_CHUNK_TRANSFORMER_OPTIONS = ("cond_or_uncond", "uuids") + + +def _slice_batch_row(value, index: int, batch: int): + """Row ``index`` of a batched ComfyUI argument. + + ComfyUI concatenates every cond-derived kwarg to the full batch, so a + leading dim of ``batch`` is per-row and 1 is shared; any other size is + ambiguous and rejected rather than guessed. + """ + if torch.is_tensor(value): + if value.ndim == 0 or value.shape[0] == 1: + return value + if value.shape[0] == batch: + return value[index : index + 1] + raise ValueError( + f"cannot split a batch of {batch} on a tensor of shape {tuple(value.shape)}" + ) + if type(value) in (list, tuple): + return type(value)(_slice_batch_row(item, index, batch) for item in value) + if type(value) is dict: + return { + key: _slice_batch_row(item, index, batch) for key, item in value.items() + } + return value + + +def _cond_uuid_from_transformer_options(options): + """ComfyUI's per-cond uuid for this call, stable across steps in one run.""" + if not isinstance(options, dict): + return None + uuids = options.get("uuids") + if type(uuids) in (list, tuple) and uuids: + return tuple(uuids) + return None + + +def _slice_transformer_options(options, index: int, batch: int): + """Shared options pass through; only the per-chunk entries are sliced.""" + if not isinstance(options, dict): + return options + sliced = dict(options) + for key in _PER_CHUNK_TRANSFORMER_OPTIONS: + value = options.get(key) + if type(value) in (list, tuple) and value and batch % len(value) == 0: + # Each chunk holds batch // len rows. + sliced[key] = type(value)([value[index // (batch // len(value))]]) + # ComfyUI sets sigmas to the timestep before repeating it per chunk. + sigmas = options.get("sigmas") + if torch.is_tensor(sigmas) and sigmas.ndim > 0 and sigmas.shape[0] > 1: + if batch % sigmas.shape[0] != 0: + raise ValueError( + f"cannot split a batch of {batch} on sigmas of shape {tuple(sigmas.shape)}" + ) + row = index % sigmas.shape[0] + sliced["sigmas"] = sigmas[row : row + 1] + return sliced + + def _reject_model_patches(transformer_options) -> None: # The DiT runs in the SGLD worker, so ComfyUI block/attention patches would # be dropped silently (e.g. MiniMax H3 Fun ControlNet, ModelAttentionBackend). @@ -86,7 +169,8 @@ class SGLDiffusionExecutor(torch.nn.Module): self.adapter = self.adapter_cls() self.session_id = uuid.uuid4().hex self._run_id = 0 - self._sent_conds: set[tuple] = set() + self._sent_conds: set[str] = set() + self._cond_key_cache: dict[tuple, str] = {} def state_dict(self, *args, **kwargs): raise RuntimeError( @@ -124,6 +208,7 @@ class SGLDiffusionExecutor(torch.nn.Module): """One ComfyUI ``sampler.sample()`` invocation is one cache lifetime.""" self._run_id += 1 self._sent_conds = set() + self._cond_key_cache = {} def end_sampler_run(self) -> None: """Run cache is evicted on the next bind of a newer id for this executor.""" @@ -148,24 +233,52 @@ class SGLDiffusionExecutor(torch.nn.Module): def comfyui_session_id(self) -> str: return f"{self.session_id}:{self._run_id}" - def _cond_key(self, packed) -> tuple | None: + def _cond_key(self, packed) -> str | None: embeds = packed.prompt_embeds if not embeds: return None tensor = embeds[0] if not torch.is_tensor(tensor) or tensor.numel() == 0: return None - flat = tensor.reshape(-1) - return ( - tuple(int(dim) for dim in tensor.shape), - float(flat[0].item()), - float(flat[-1].item()), + # A cond's content is fixed for the life of one sampler run, so with a + # uuid from ComfyUI we only need to hash it once per run rather than + # once per step; the memo key still folds in shape/dtype so a stale + # entry can never be returned for tensors that don't actually match. + cond_uuid = packed.unpack_ctx.get("comfyui_cond_uuid") + memo_key = None + if cond_uuid is not None: + shape_key = tuple( + (t.dtype, tuple(t.shape)) if torch.is_tensor(t) else None + for t in embeds + ) + memo_key = (cond_uuid, shape_key) + cached = self._cond_key_cache.get(memo_key) + if cached is not None: + return cached + # Hash everything drop_cached_fields removes: a hit means the worker + # restores all of it, so a partial key would revive another cond. + digest = hashlib.blake2b(digest_size=16) + _hash_value( + digest, + ( + embeds, + packed.pooled_embeds, + packed.prompt_seq_lens, + { + key: packed.extra_req.get(key) + for key in self.adapter.cached_extra_keys + }, + ), ) + key = digest.hexdigest() + if memo_key is not None: + self._cond_key_cache[memo_key] = key + return key def _mark_and_maybe_drop(self, packed) -> None: key = self._cond_key(packed) if key is not None: - packed.extra_req["comfyui_cond_key"] = repr(key) + packed.extra_req["comfyui_cond_key"] = key if key in self._sent_conds: self.adapter.drop_cached_fields(packed) else: @@ -220,5 +333,41 @@ class SGLDiffusionExecutor(torch.nn.Module): def forward(self, x, timestep, context, **kwargs): _reject_model_patches(kwargs.get("transformer_options")) _reject_unapplied_conditioning(self.adapter, kwargs) + batch = int(x.shape[0]) if torch.is_tensor(x) else 1 + if batch == 1: + return self._forward_one(x, timestep, context, **kwargs) + # ComfyUI batches CFG cond/uncond (and batch_size > 1) into one call, + # but the worker's comfyui path is per-sample: req.timesteps is its + # schedule and seq lens are per request. Send one request per row. + return torch.cat( + [ + self._forward_one( + _slice_batch_row(x, i, batch), + _slice_batch_row(timestep, i, batch), + _slice_batch_row(context, i, batch), + _cond_row=i, + **{ + key: ( + _slice_transformer_options(value, i, batch) + if key == "transformer_options" + else _slice_batch_row(value, i, batch) + ) + for key, value in kwargs.items() + }, + ) + for i in range(batch) + ] + ) + + def _forward_one(self, x, timestep, context, _cond_row=None, **kwargs): packed = self.adapter.pack(x, timestep, context, **kwargs) + cond_uuid = _cond_uuid_from_transformer_options( + kwargs.get("transformer_options") + ) + # Rows of one cond chunk share its uuid but can carry different + # per-row conditioning (e.g. reference latents), so key on the row too. + # unpack_ctx is executor-local, keeping _execute_packed's signature. + packed.unpack_ctx["comfyui_cond_uuid"] = ( + None if cond_uuid is None else (cond_uuid, _cond_row) + ) return self._execute_packed(packed, x, timestep) diff --git a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py index 58d6d66670..1d039ca389 100644 --- a/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py +++ b/python/sglang/multimodal_gen/apps/ComfyUI_SGLDiffusion/executors/minimax_h3.py @@ -83,6 +83,17 @@ def _cache_fingerprint(video, sample_sigmas, serialized_layout) -> dict[str, Any class MiniMaxH3Adapter(ComfyUIModelAdapter): model_types = ("minimax_h3",) pipeline_class_name = "MiniMaxH3Pipeline" + # comfyui_cache_fp / comfyui_cond_key stay; later steps still send them. + cached_extra_keys = ComfyUIModelAdapter.cached_extra_keys + ( + "h3_payload", + "h3_transformer_options", + "h3_sample_sigmas", + "h3_context", + "h3_layout", + "h3_denoise_mask", + "h3_audio_denoise_mask", + "sigmas", + ) def pack(self, x, timestep, context, **kwargs) -> PackedForward: video, audio_src = _split_av(x) @@ -199,18 +210,6 @@ class MiniMaxH3Adapter(ComfyUIModelAdapter): extra[key] = value req.extra = extra - def drop_cached_fields(self, packed: PackedForward) -> None: - super().drop_cached_fields(packed) - packed.extra_req.pop("h3_payload", None) - packed.extra_req.pop("h3_transformer_options", None) - packed.extra_req.pop("h3_sample_sigmas", None) - packed.extra_req.pop("h3_context", None) - packed.extra_req.pop("h3_layout", None) - packed.extra_req.pop("h3_denoise_mask", None) - packed.extra_req.pop("h3_audio_denoise_mask", None) - packed.extra_req.pop("sigmas", None) - # Keep comfyui_cache_fp / comfyui_cond_key; later steps still send them. - class MiniMaxH3Executor(SGLDiffusionExecutor): adapter_cls = MiniMaxH3Adapter diff --git a/python/sglang/multimodal_gen/runtime/pipelines_core/comfyui_mode.py b/python/sglang/multimodal_gen/runtime/pipelines_core/comfyui_mode.py index f0ea633e63..b8b80a03e9 100644 --- a/python/sglang/multimodal_gen/runtime/pipelines_core/comfyui_mode.py +++ b/python/sglang/multimodal_gen/runtime/pipelines_core/comfyui_mode.py @@ -124,8 +124,8 @@ def _evict_other_runs(session_id: str) -> None: def bind_comfyui_session(req): """Restore cached conditioning, then refresh the cache from whatever is set. - Conditioning is stored per ``comfyui_cond_key`` so CFG pos/neg do not - overwrite each other. Packed extras are restored only when + Conditioning and packed extras are stored per ``comfyui_cond_key`` so CFG + pos/neg do not overwrite each other. Extras are restored only when ``comfyui_cache_fp`` matches; a mismatch must not revive a stale layout. """ sid = session_id_from_req(req) @@ -137,6 +137,7 @@ def bind_comfyui_session(req): cond_key = extra.get("comfyui_cond_key") or "_" cached = _SESSIONS.get(sid) or {} cond_snap = (cached.get("_conds") or {}).get(cond_key) or {} + extra_snap = (cached.get("_extra") or {}).get(cond_key) or {} current_fp = extra.get("comfyui_cache_fp") cached_fp = cached.get("_fp") extra_ok = current_fp is None or cached_fp is None or current_fp == cached_fp @@ -145,7 +146,7 @@ def bind_comfyui_session(req): if _is_empty(getattr(req, name, None)): setattr(req, name, value) if extra_ok: - for key, value in (cached.get("_extra") or {}).items(): + for key, value in extra_snap.items(): if _is_empty(extra.get(key)): extra[key] = value req.extra = extra @@ -164,14 +165,17 @@ def bind_comfyui_session(req): conds = dict(cached.get("_conds") or {}) if new_cond: conds[cond_key] = new_cond - snapshot: dict[str, Any] = {"_conds": conds} + extras = dict(cached.get("_extra") or {}) if extra_ok else {} if extra_snapshot: - snapshot["_extra"] = extra_snapshot + extras[cond_key] = extra_snapshot + snapshot: dict[str, Any] = {"_conds": conds} + if extras: + snapshot["_extra"] = extras if current_fp is not None: snapshot["_fp"] = current_fp elif extra_ok and cached_fp is not None: snapshot["_fp"] = cached_fp - if conds or extra_snapshot: + if conds or extras: _SESSIONS[sid] = snapshot return req diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py index a7c4b9349e..1afcff814e 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_adapters.py @@ -8,6 +8,8 @@ import pytest import torch from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import ( + ComfyUIModelAdapter, + PackedForward, get_adapter_class, registered_model_types, ) @@ -230,3 +232,143 @@ def test_comfyui_weight_access_gets_a_clear_error() -> None: base.to("cpu") with pytest.raises(RuntimeError, match="SGLDLoraLoader"): base.state_dict() + + +class _RowRecordingExecutor(SGLDiffusionExecutor): + """Real forward/pack/unpack; the worker round trip just records the request.""" + + def __init__(self, adapter): + torch.nn.Module.__init__(self) + self.adapter = adapter + self.sent = [] + + def _execute_packed(self, packed, x, timestep): + self.sent.append((packed, timestep)) + # Fake noise_pred that identifies the row by its T5 context. + noise = torch.full_like(packed.latents, float(packed.prompt_embeds[-1].mean())) + return self.adapter.unpack(noise, packed, x) + + +def test_batched_forward_sends_one_request_per_row() -> None: + # ComfyUI batches CFG cond/uncond into one call; the worker path is per-sample. + ex = _RowRecordingExecutor(FluxAdapter()) + x = torch.zeros(2, 16, 8, 8) + timestep = torch.tensor([0.5, 0.5]) + context = torch.stack([torch.full((8, 4096), 1.0), torch.full((8, 4096), 2.0)]) + y = torch.stack([torch.full((768,), 3.0), torch.full((768,), 4.0)]) + out = ex(x, timestep, context, y=y, guidance=torch.tensor([3.5, 3.5])) + + assert len(ex.sent) == 2 + for row, (packed, row_timestep) in enumerate(ex.sent): + assert packed.latents.shape[0] == 1 + assert torch.equal(row_timestep, timestep[row : row + 1]) + assert torch.equal(packed.prompt_embeds[1], context[row : row + 1]) + assert torch.equal(packed.pooled_embeds[0], y[row : row + 1]) + assert packed.prompt_seq_lens == [[1], [8]] + assert out.shape == x.shape + assert torch.all(out[0] == 1.0) and torch.all(out[1] == 2.0) + + +class _KwargsAdapter(ComfyUIModelAdapter): + applied_conditioning = ("control", "ref_latents") + + def __init__(self): + self.calls = [] + + def pack(self, x, timestep, context, **kwargs): + self.calls.append((x, timestep, context, kwargs)) + return PackedForward( + latents=torch.zeros(1), + timesteps=timestep, + prompt_embeds=[torch.zeros(1)], + height=1, + width=1, + ) + + def unpack(self, noise_pred, packed, x): + return x + + +def test_batched_forward_slices_only_batched_values() -> None: + adapter = _KwargsAdapter() + ex = _RowRecordingExecutor(adapter) + shared_ref = torch.ones(1, 16, 4, 4) + sample_sigmas = torch.linspace(1.0, 0.0, 5) + patches = {"attn": []} + timestep = torch.tensor([0.5, 0.25]) + options = { + "cond_or_uncond": [0, 1], + "sigmas": timestep, + "sample_sigmas": sample_sigmas, + "patches": patches, + } + ex( + torch.zeros(2, 16, 4, 4), + timestep, + torch.zeros(2, 3, 8), + ref_latents=[ + torch.stack([torch.zeros(16, 4, 4), torch.ones(16, 4, 4)]), + shared_ref, + ], + control={"input": [torch.stack([torch.zeros(4), torch.ones(4)]), None]}, + transformer_options=options, + ) + assert len(adapter.calls) == 2 + for row, (_, _, _, kwargs) in enumerate(adapter.calls): + per_row, shared = kwargs["ref_latents"] + assert per_row.shape == (1, 16, 4, 4) and torch.all(per_row == row) + assert shared is shared_ref + control, missing = kwargs["control"]["input"] + assert control.shape == (1, 4) and torch.all(control == row) + assert missing is None + row_options = kwargs["transformer_options"] + assert row_options["cond_or_uncond"] == [row] + assert torch.equal(row_options["sigmas"], timestep[row : row + 1]) + assert row_options["sample_sigmas"] is sample_sigmas + assert row_options["patches"] is patches + assert options["cond_or_uncond"] == [0, 1] + + +def test_batched_forward_slices_per_chunk_options() -> None: + # batch_size=2 with CFG: two cond chunks of two rows each. + adapter = _KwargsAdapter() + ex = _RowRecordingExecutor(adapter) + sigmas = torch.tensor([0.5, 0.25]) + ex( + torch.zeros(4, 16, 4, 4), + torch.cat([sigmas, sigmas]), + torch.zeros(4, 3, 8), + transformer_options={ + "cond_or_uncond": [0, 1], + "uuids": ["pos", "neg"], + "sigmas": sigmas, + }, + ) + seen = [kwargs["transformer_options"] for *_, kwargs in adapter.calls] + assert [o["cond_or_uncond"] for o in seen] == [[0], [0], [1], [1]] + assert [o["uuids"] for o in seen] == [["pos"], ["pos"], ["neg"], ["neg"]] + for row, options in enumerate(seen): + assert torch.equal(options["sigmas"], sigmas[row % 2 : row % 2 + 1]) + + +def test_batched_forward_rejects_ambiguous_leading_dim() -> None: + ex = _RowRecordingExecutor(_KwargsAdapter()) + with pytest.raises(ValueError, match="cannot split a batch of 2"): + ex( + torch.zeros(2, 16, 4, 4), + torch.tensor([0.5, 0.5]), + torch.zeros(2, 3, 8), + guidance=torch.zeros(3), + ) + + +def test_unbatched_forward_is_a_single_request() -> None: + adapter = _KwargsAdapter() + ex = _RowRecordingExecutor(adapter) + x = torch.zeros(1, 16, 4, 4) + ex(x, torch.tensor([0.5]), torch.zeros(1, 3, 8)) + assert len(adapter.calls) == 1 and adapter.calls[0][0] is x + # MiniMax-H3 passes x as [video, audio]; that path is not split either. + av = [torch.zeros(2, 16, 1, 4, 4), torch.zeros(2, 8, 4)] + ex(av, torch.tensor([0.5, 0.5]), torch.zeros(2, 3, 8)) + assert len(adapter.calls) == 2 and adapter.calls[1][0] is av diff --git a/python/sglang/multimodal_gen/test/unit/test_comfyui_session.py b/python/sglang/multimodal_gen/test/unit/test_comfyui_session.py index 60f2fec668..9a14dd47dc 100644 --- a/python/sglang/multimodal_gen/test/unit/test_comfyui_session.py +++ b/python/sglang/multimodal_gen/test/unit/test_comfyui_session.py @@ -1,7 +1,23 @@ # SPDX-License-Identifier: Apache-2.0 +import uuid + import torch +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.adapter import ( + ComfyUIModelAdapter, + PackedForward, +) +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.base import ( + SGLDiffusionExecutor, +) +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.flux import FluxAdapter +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.minimax_h3 import ( + MiniMaxH3Adapter, +) +from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.zimage import ( + ZImageAdapter, +) from sglang.multimodal_gen.runtime.pipelines_core.comfyui_mode import ( bind_comfyui_session, get_run_state, @@ -188,6 +204,227 @@ def test_cond_keys_keep_positive_and_negative_apart() -> None: release_comfyui_session(sid) +class _Executor(SGLDiffusionExecutor): + """Real executor bookkeeping, without a generator or model.""" + + def __init__(self, adapter): + torch.nn.Module.__init__(self) + self.adapter = adapter + self.session_id = uuid.uuid4().hex + self._run_id = 0 + self._sent_conds = set() + self._cond_key_cache = {} + self.begin_sampler_run() + self.sid = self.comfyui_session_id() + + def send(self, packed) -> _Req: + """Executor half of _execute_packed, then the worker-side bind.""" + self._mark_and_maybe_drop(packed) + req = _Req() + self.adapter.fill_req(req, packed) + req.extra = { + **(req.extra or {}), + "comfyui_session_id": self.sid, + "comfyui_cond_key": packed.extra_req["comfyui_cond_key"], + } + return bind_comfyui_session(req) + + +def test_cond_key_flux_same_pooled_different_t5_not_conflated() -> None: + # ComfyUI's Flux pooled `y` comes from CLIP-L's first 77-token chunk only, + # so prompts that differ later share `y` but not the T5 context. + ex = _Executor(FluxAdapter()) + x, t = torch.zeros(1, 16, 8, 8), torch.tensor([0.5]) + y = torch.randn(1, 768) + ctx_a, ctx_b = torch.randn(1, 8, 4096), torch.randn(1, 8, 4096) + first = ex.send(ex.adapter.pack(x, t, ctx_a, y=y)) + second = ex.send(ex.adapter.pack(x, t, ctx_b, y=y.clone())) + assert torch.equal(first.prompt_embeds[1], ctx_a) + assert torch.equal(second.prompt_embeds[1], ctx_b) + release_comfyui_session(ex.sid) + + +def test_cond_key_same_first_last_scalar_not_conflated() -> None: + ex = _Executor(ZImageAdapter()) + x, t = torch.zeros(1, 16, 8, 8), torch.tensor([0.5]) + ctx_a, ctx_b = torch.randn(1, 6, 32), torch.randn(1, 6, 32) + ctx_b.view(-1)[0] = ctx_a.view(-1)[0] + ctx_b.view(-1)[-1] = ctx_a.view(-1)[-1] + ex.send(ex.adapter.pack(x, t, ctx_a)) + second = ex.send(ex.adapter.pack(x, t, ctx_b)) + assert torch.equal(second.prompt_embeds[0], ctx_b.squeeze(0)) + release_comfyui_session(ex.sid) + + +def test_cond_key_covers_image_latent() -> None: + def packed(image_latent): + return PackedForward( + latents=torch.zeros(1, 4, 8), + timesteps=torch.tensor([500.0]), + prompt_embeds=[torch.ones(3, 8)], + prompt_seq_lens=[[3]], + height=64, + width=64, + extra_req={"image_latent": image_latent}, + ) + + ex = _Executor(ComfyUIModelAdapter()) + ref_a, ref_b = torch.randn(1, 4, 8), torch.randn(1, 4, 8) + ex.send(packed(ref_a)) + second = ex.send(packed(ref_b)) + assert torch.equal(second.image_latent, ref_b) + release_comfyui_session(ex.sid) + + +def test_cond_key_repeat_still_uses_cache() -> None: + ex = _Executor(FluxAdapter()) + x, t = torch.zeros(1, 16, 8, 8), torch.tensor([0.5]) + y, ctx = torch.randn(1, 768), torch.randn(1, 8, 4096) + ex.send(ex.adapter.pack(x, t, ctx, y=y)) + # ComfyUI hands over fresh tensors each step; equal content must still hit. + repeat = ex.adapter.pack(x, t, ctx.clone(), y=y.clone()) + restored = ex.send(repeat) + assert repeat.prompt_embeds == [] + assert torch.equal(restored.prompt_embeds[1], ctx) + release_comfyui_session(ex.sid) + + +class _RefAdapter(ComfyUIModelAdapter): + """Qwen-Image-Edit shaped: text context plus a per-row reference latent.""" + + applied_conditioning = ("ref_latents",) + + def pack(self, x, timestep, context, ref_latents=None, **kwargs): + return PackedForward( + latents=x, + timesteps=timestep, + prompt_embeds=[context[0]], + prompt_seq_lens=[[int(context.shape[1])]], + height=8, + width=8, + extra_req={"image_latent": ref_latents[0]}, + ) + + +class _SendingExecutor(_Executor): + """Batched forward goes through the real cache path, one row at a time.""" + + def __init__(self, adapter): + super().__init__(adapter) + self.seen = [] + + def _execute_packed(self, packed, x, timestep): + req = self.send(packed) + self.seen.append(req.image_latent) + return x + + +def test_batched_rows_with_same_text_keep_their_own_reference() -> None: + ex = _SendingExecutor(_RefAdapter()) + context = torch.ones(1, 3, 8).expand(2, 3, 8) + refs = torch.stack([torch.zeros(4, 8), torch.ones(4, 8)]) + # Both rows share the cond chunk's uuid, as in a real ComfyUI call. + ex( + torch.zeros(2, 4, 8), + torch.tensor([0.5, 0.5]), + context, + ref_latents=[refs], + transformer_options={"uuids": ["cond-0"], "cond_or_uncond": [0]}, + ) + assert len(ex.seen) == 2 + assert torch.equal(ex.seen[0], refs[0:1]) + assert torch.equal(ex.seen[1], refs[1:2]) + release_comfyui_session(ex.sid) + + +def _h3_packed(text, payload): + return PackedForward( + latents=torch.zeros(1, 4, 2, 2), + timesteps=torch.tensor([500.0]), + prompt_embeds=[text], + prompt_seq_lens=[[int(text.shape[0])]], + height=2, + width=2, + extra_req={ + "h3_payload": payload, + "h3_context": text, + "comfyui_cache_fp": {"spatial": (1, 2, 2)}, + }, + ) + + +def test_h3_cache_hit_restores_own_extras() -> None: + ex = _Executor(MiniMaxH3Adapter()) + pos, neg = torch.ones(3, 4), torch.zeros(3, 4) + pos_payload, neg_payload = {"text_token_tags": [1, 2]}, {"text_token_tags": [3]} + ex.send(_h3_packed(pos, pos_payload)) + ex.send(_h3_packed(neg, neg_payload)) + later_pos = _h3_packed(pos.clone(), dict(pos_payload)) + restored = ex.send(later_pos) + assert "h3_context" not in later_pos.extra_req # cache hit, extras dropped + assert torch.equal(restored.extra["h3_context"], pos) + assert restored.extra["h3_payload"] == pos_payload + release_comfyui_session(ex.sid) + + +def test_h3_same_text_different_payload_not_conflated() -> None: + ex = _Executor(MiniMaxH3Adapter()) + text = torch.ones(3, 4) + ex.send(_h3_packed(text, {"refs": [torch.zeros(2, 2)]})) + second = ex.send(_h3_packed(text.clone(), {"refs": [torch.ones(2, 2)]})) + assert torch.equal(second.extra["h3_payload"]["refs"][0], torch.ones(2, 2)) + release_comfyui_session(ex.sid) + + +def test_cond_key_hashes_large_tensors_inside_lists() -> None: + ex = _Executor(MiniMaxH3Adapter()) + text = torch.ones(3, 4) + latent_a = torch.zeros(4096) + latent_b = latent_a.clone() + latent_b[2048] = 1.0 # outside what repr() would print + key_a = ex._cond_key(_h3_packed(text, {"cond_video_latents": [latent_a]})) + key_b = ex._cond_key(_h3_packed(text, {"cond_video_latents": [latent_b]})) + assert key_a != key_b + + +def test_cond_key_memoized_per_uuid_within_run() -> None: + # A cond's content can't change within one sampler run, so a repeat uuid + # must reuse the memoized key without rehashing the (changed) tensor. + ex = _Executor(MiniMaxH3Adapter()) + text = torch.ones(3, 4) + packed = _h3_packed(text, {}) + packed.unpack_ctx["comfyui_cond_uuid"] = ("u1",) + first = ex._cond_key(packed) + mutated = _h3_packed(torch.zeros(3, 4), {}) + mutated.unpack_ctx["comfyui_cond_uuid"] = ("u1",) + second = ex._cond_key(mutated) + assert first == second + + ex.begin_sampler_run() + third = ex._cond_key(mutated) + assert third != first + + +def test_extras_cached_per_cond_key() -> None: + sid = "exec4:1" + pos_ctx, neg_ctx = torch.ones(2, 4), torch.zeros(2, 4) + for key, ctx in (("pos", pos_ctx), ("neg", neg_ctx)): + req = _Req() + req.extra = { + "comfyui_session_id": sid, + "comfyui_cond_key": key, + "h3_context": ctx, + } + req.prompt_embeds = [ctx] + bind_comfyui_session(req) + + later_pos = _Req() + later_pos.extra = {"comfyui_session_id": sid, "comfyui_cond_key": "pos"} + bind_comfyui_session(later_pos) + assert torch.equal(later_pos.extra["h3_context"], pos_ctx) + release_comfyui_session(sid) + + class _FakePipeline: def __init__(self): self.modules = {}