[diffusion] comfyui: consolidate 6 unrelated plugin fixes (#43493)

Co-authored-by: Mick <mickjagger19@icloud.com>
Co-authored-by: niehen6174 <1639206518@qq.com>
This commit is contained in:
Avaya Aggarwal
2026-10-14 10:56:09 +08:00
committed by GitHub
co-authored by Mick niehen6174
parent c053fc71d3
commit 5efb7384c1
9 changed files with 200 additions and 22 deletions
@@ -362,7 +362,7 @@ class SGLDiffusionGenerator:
"""Rebuild the worker if this cached executor no longer owns it."""
if self._owns_live(executor):
return
reload = getattr(executor, "_sgld_reload", None)
reload = executor._sgld_reload
if not reload:
raise RuntimeError(
"SGLD worker is down and this executor has no reload spec"
@@ -373,9 +373,12 @@ class SGLDiffusionGenerator:
)
self.load_model(**reload)
executor.generator = self.generator
lora = getattr(executor, "_lora_input", None)
lora = executor._lora_input
if lora and lora.get("lora_nickname"):
executor.set_lora(**lora)
# The new worker's session cache starts empty, so conditioning the
# old worker already had cannot be restored; resend it.
executor.begin_sampler_run()
def kill_generator(self):
"""Force-stop workers this owner started. Do not scan the process table."""
@@ -49,13 +49,16 @@ class SGLDiffusionServerAPI:
- vae_precision: VAE model precision
"""
try:
# Remove /v1 from base_url for /models endpoint
models_url = self.base_url.removesuffix("/v1") + "/models"
response = requests.get(models_url, headers=self.headers, timeout=30)
response = requests.get(
f"{self.base_url}/models", headers=self.headers, timeout=30
)
response.raise_for_status()
return response.json()
data = response.json()
return data["data"][0]
except requests.exceptions.RequestException as e:
raise RuntimeError(f"Failed to get model info: {str(e)}")
except (KeyError, IndexError) as e:
raise RuntimeError(f"Unexpected response shape from /models: {str(e)}")
def generate_image(
self,
@@ -1,6 +1,7 @@
"""Qwen-Image adapters for the ComfyUI DiT-forward contract."""
import comfy.ldm.common_dit
import torch
from .adapter import ComfyUIModelAdapter, PackedForward
from .base import SGLDiffusionExecutor
@@ -89,11 +90,14 @@ class QwenImageEditAdapter(QwenImageAdapter):
) -> PackedForward:
packed = super().pack(x, timestep, context, **kwargs)
if ref_latents:
pack_ref, orig_ref_shape = self._pack_latents(ref_latents[0])
packed.extra_req["image_latent"] = pack_ref
packed.extra_req["vae_image_sizes"] = [
(orig_ref_shape[-1], orig_ref_shape[-2])
]
packed_refs = []
vae_image_sizes = []
for ref_latent in ref_latents:
pack_ref, orig_ref_shape = self._pack_latents(ref_latent)
packed_refs.append(pack_ref)
vae_image_sizes.append((orig_ref_shape[-1], orig_ref_shape[-2]))
packed.extra_req["image_latent"] = torch.cat(packed_refs, dim=1)
packed.extra_req["vae_image_sizes"] = vae_image_sizes
return packed
@@ -25,6 +25,7 @@ from .utils import (
convert_video_to_comfy_video,
get_image_path,
is_empty_image,
resolve_video_path,
)
@@ -663,7 +664,9 @@ class SGLDiffusionGenerateVideo:
# Call API
try:
response = sgld_client.generate_video(**request_params)
video_path = response.get("file_path", "")
video_path = resolve_video_path(
response.get("file_path"), response.get("url")
)
video = convert_video_to_comfy_video(video_path, height, width)
except Exception as e:
raise RuntimeError(f"Failed to generate video: {str(e)}")
@@ -871,10 +874,12 @@ class SGLDiffusionGenerateH3:
try:
response = sgld_client.generate_video(**request_params)
video_path = resolve_video_path(
response.get("file_path"), response.get("url")
)
except Exception as e:
raise RuntimeError(f"Failed to generate MiniMax-H3 video: {str(e)}")
video_path = response.get("file_path", "")
# H3 aligns the canvas server-side, so the resolved size is only known
# from the response; short_edge and aspect_ratio cannot reconstruct it
resolved_size = response.get("size", "")
@@ -2,11 +2,13 @@ import base64
import io
import os
import shutil
import subprocess
import time
import uuid
import folder_paths
import numpy as np
import requests
import torch
from comfy_api.input import VideoInput
from PIL import Image
@@ -156,14 +158,78 @@ class SGLDVideoInput(VideoInput):
"""
Abstract method to save the video input to a file.
"""
if not os.path.exists(self.video_path):
raise FileNotFoundError(
f"Video not found at '{self.video_path}'. This file must be "
"readable from the ComfyUI process; a path on a remote "
"SGLang Diffusion server is not."
)
save_path = path
# Copy video file from video_path to save_path
if os.path.exists(self.video_path):
# Ensure destination directory exists
save_dir = os.path.dirname(save_path)
if save_dir:
os.makedirs(save_dir, exist_ok=True)
shutil.copy2(self.video_path, save_path)
save_dir = os.path.dirname(save_path)
if save_dir:
os.makedirs(save_dir, exist_ok=True)
shutil.copy2(self.video_path, save_path)
def as_trimmed(
self,
start_time: float | None = None,
duration: float | None = None,
strict_duration: bool = False,
):
"""
Required by VideoInput since ComfyUI #12107.
Trims via ffmpeg stream-copy (fast, no re-encode); `strict_duration`
is accepted for interface compatibility but not enforced to frame
accuracy, since this node never needs sub-frame precision.
"""
start_time = start_time or 0.0
if duration is not None and duration < 0:
return None
temp_dir = folder_paths.get_temp_directory()
_ensure_dir(temp_dir)
ext = os.path.splitext(self.video_path)[1] or ".mp4"
dest = os.path.join(temp_dir, f"sgl_video_trim_{uuid.uuid4().hex[:8]}{ext}")
cmd = ["ffmpeg", "-y", "-ss", str(start_time), "-i", self.video_path]
if duration is not None:
cmd += ["-t", str(duration)]
cmd += ["-c", "copy", dest]
subprocess.run(cmd, capture_output=True, check=True)
return SGLDVideoInput(dest, self.height, self.width)
def resolve_video_path(file_path: str | None, url: str | None) -> str:
"""
Resolve the generated video to a path this process can use.
The server sets `file_path` to None once a cloud upload succeeds (the
local file is removed); fetch `url` into the ComfyUI temp directory in
that case. Otherwise `file_path` is returned as-is; a server on another
host without cloud storage returns a path this process can't read, and
`SGLDVideoInput.save_to` raises clearly when that path turns out to be
unreadable.
"""
if file_path:
return file_path
if not url:
raise RuntimeError(
"Generated video has no local file_path and no url to fetch it from."
)
temp_dir = folder_paths.get_temp_directory()
_ensure_dir(temp_dir)
ext = os.path.splitext(url.split("?", 1)[0])[1] or ".mp4"
dest = os.path.join(temp_dir, f"sgl_video_{uuid.uuid4().hex[:8]}{ext}")
with requests.get(url, timeout=120, stream=True) as response:
response.raise_for_status()
with open(dest, "wb") as f:
for chunk in response.iter_content(chunk_size=8192):
f.write(chunk)
return dest
def convert_video_to_comfy_video(
@@ -193,11 +193,14 @@ def _layout_signature_matches(
if len(values) < 5:
return False
text_len, latent_t, latent_h, latent_w, audio_t = values
_, patch_h, patch_w = _PATCH_SIZE
padded_h = (int(video_x.shape[3]) + patch_h - 1) // patch_h * patch_h
padded_w = (int(video_x.shape[4]) + patch_w - 1) // patch_w * patch_w
return (
text_len == int(text.shape[0])
and latent_t == int(video_x.shape[2])
and latent_h == int(video_x.shape[3])
and latent_w == int(video_x.shape[4])
and latent_h == padded_h
and latent_w == padded_w
and audio_t == int(audio_x.shape[-1])
)
@@ -1,6 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
"""Pack / unpack contract for ComfyUI model adapters."""
import importlib
import sys
import types
from types import SimpleNamespace
from unittest.mock import patch
@@ -234,6 +237,41 @@ def test_comfyui_weight_access_gets_a_clear_error() -> None:
base.state_dict()
def _qwen_image_edit_adapter():
"""Import the Qwen adapters with the one ComfyUI helper they use stubbed."""
common_dit = types.ModuleType("comfy.ldm.common_dit")
common_dit.pad_to_patch_size = lambda x, patch_size: x
comfy, comfy_ldm = types.ModuleType("comfy"), types.ModuleType("comfy.ldm")
comfy.ldm, comfy_ldm.common_dit = comfy_ldm, common_dit
stubs = {"comfy": comfy, "comfy.ldm": comfy_ldm, "comfy.ldm.common_dit": common_dit}
with patch.dict(sys.modules, stubs):
module = importlib.import_module(
"sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.qwen_image"
)
return module.QwenImageEditAdapter()
def test_qwen_image_edit_pack_keeps_all_ref_latents() -> None:
"""Qwen-Image-Edit packed only the first of several reference images."""
adapter = _qwen_image_edit_adapter()
x = torch.ones(1, 16, 1, 90, 160)
timestep = torch.tensor([1.0])
context = torch.ones(1, 19, 2560)
ref_a = torch.ones(1, 16, 1, 64, 64)
ref_b = torch.ones(1, 16, 1, 32, 96)
ref_c = torch.ones(1, 16, 1, 48, 48)
packed = adapter.pack(x, timestep, context, ref_latents=[ref_a, ref_b, ref_c])
sizes = packed.extra_req["vae_image_sizes"]
assert sizes == [(64, 64), (96, 32), (48, 48)]
expected_tokens = sum(
(h // 2) * (w // 2) for h, w in [(64, 64), (32, 96), (48, 48)]
)
assert packed.extra_req["image_latent"].shape[1] == expected_tokens
class _RowRecordingExecutor(SGLDiffusionExecutor):
"""Real forward/pack/unpack; the worker round trip just records the request."""
@@ -37,6 +37,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None:
stale = object()
fresh = object()
loads = []
sampler_runs = []
executor = SimpleNamespace(
generator=stale,
_sgld_reload={
@@ -45,6 +46,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None:
"sgld_options": {},
},
_lora_input=None,
begin_sampler_run=lambda: sampler_runs.append(True),
)
def fake_load(**kwargs):
@@ -56,6 +58,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None:
runtime.ensure_executor(executor)
assert loads == [executor._sgld_reload]
assert executor.generator is fresh
assert sampler_runs == [True]
def test_ensure_is_noop_when_executor_still_owns_live_worker() -> None:
@@ -366,3 +369,24 @@ def test_renamed_h3_paths_resolve_through_the_plugin_model_id(
assert seen["warmup_mode"] == "off"
info = _get_config_info(path, model_id=seen["model_id"])
assert info.pipeline_config_cls.__name__ == config_cls
def test_get_model_info_reads_the_card_from_v1_models(monkeypatch) -> None:
"""The plugin reads the OpenAI /v1/models route, not the deprecated root
/models, so it must unwrap the card from the list envelope."""
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import server_api
card = {"id": "wan", "object": "model", "task_type": "T2V"}
urls = []
def fake_get(url, headers=None, timeout=None):
urls.append(url)
return SimpleNamespace(
raise_for_status=lambda: None,
json=lambda: {"object": "list", "data": [card]},
)
monkeypatch.setattr(server_api.requests, "get", fake_get)
client = server_api.SGLDiffusionServerAPI(base_url="http://host:30010/v1")
assert client.get_model_info() == card
assert urls == ["http://host:30010/v1/models"]
@@ -84,6 +84,9 @@ class ComfyUIPackedLayout:
def __init__(
self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None
):
# ComfyUI builds PackedLayout from its patch-padded latent dimensions.
latent_h = (latent_h + 1) // 2 * 2
latent_w = (latent_w + 1) // 2 * 2
frame, w_grid = _frame_grid(latent_h, latent_w)
frame_rows = frame.shape[0]
@@ -825,6 +828,35 @@ def test_keyframes_with_serialized_layout_build_kwargs():
assert branch.video_target_start == 4
def test_odd_latent_keyframes_keep_comfyui_layout():
video = torch.randn(1, 24, 2, 5, 7)
audio = torch.randn(1, 32, 2, 3)
context = torch.randn(1, 8, 16)
timestep = torch.tensor([500.0])
sample_sigmas = torch.tensor([1.0, 0.5, 0.0])
kf_latent = torch.randn(1, 24, 1, 5, 7)
keyframes = [{"resolved_frame_index": 0, "latent": kf_latent}]
layout = ComfyUIPackedLayout(8, 2, 5, 7, 3, keyframes=keyframes)
inputs = comfyui_payload_to_branch_inputs(
video,
audio,
context,
minimax_payload={
"keyframes": keyframes,
"cond_video_latents": [kf_latent],
"visual_cond_noise_aug": 1.0,
},
sample_sigmas=sample_sigmas,
timestep=timestep,
layout=serialize_comfyui_layout(layout),
)
assert tuple(layout.signature[2:4]) == (6, 8)
assert inputs["padded_video_shape"][-2:] == (6, 8)
assert inputs["video_rows"].shape[0] == 12 + 2 * 3 * 4
def test_uniform_denoise_mask_scales_video_timestep():
video, audio, context, timestep, sample_sigmas = _t2va_tensors()
inputs = comfyui_payload_to_branch_inputs(