[diffusion] comfyui: consolidate 6 unrelated plugin fixes (#43493)
Co-authored-by: Mick <mickjagger19@icloud.com> Co-authored-by: niehen6174 <1639206518@qq.com>
This commit is contained in:
co-authored by
Mick
niehen6174
parent
c053fc71d3
commit
5efb7384c1
@@ -362,7 +362,7 @@ class SGLDiffusionGenerator:
|
||||
"""Rebuild the worker if this cached executor no longer owns it."""
|
||||
if self._owns_live(executor):
|
||||
return
|
||||
reload = getattr(executor, "_sgld_reload", None)
|
||||
reload = executor._sgld_reload
|
||||
if not reload:
|
||||
raise RuntimeError(
|
||||
"SGLD worker is down and this executor has no reload spec"
|
||||
@@ -373,9 +373,12 @@ class SGLDiffusionGenerator:
|
||||
)
|
||||
self.load_model(**reload)
|
||||
executor.generator = self.generator
|
||||
lora = getattr(executor, "_lora_input", None)
|
||||
lora = executor._lora_input
|
||||
if lora and lora.get("lora_nickname"):
|
||||
executor.set_lora(**lora)
|
||||
# The new worker's session cache starts empty, so conditioning the
|
||||
# old worker already had cannot be restored; resend it.
|
||||
executor.begin_sampler_run()
|
||||
|
||||
def kill_generator(self):
|
||||
"""Force-stop workers this owner started. Do not scan the process table."""
|
||||
|
||||
@@ -49,13 +49,16 @@ class SGLDiffusionServerAPI:
|
||||
- vae_precision: VAE model precision
|
||||
"""
|
||||
try:
|
||||
# Remove /v1 from base_url for /models endpoint
|
||||
models_url = self.base_url.removesuffix("/v1") + "/models"
|
||||
response = requests.get(models_url, headers=self.headers, timeout=30)
|
||||
response = requests.get(
|
||||
f"{self.base_url}/models", headers=self.headers, timeout=30
|
||||
)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
data = response.json()
|
||||
return data["data"][0]
|
||||
except requests.exceptions.RequestException as e:
|
||||
raise RuntimeError(f"Failed to get model info: {str(e)}")
|
||||
except (KeyError, IndexError) as e:
|
||||
raise RuntimeError(f"Unexpected response shape from /models: {str(e)}")
|
||||
|
||||
def generate_image(
|
||||
self,
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""Qwen-Image adapters for the ComfyUI DiT-forward contract."""
|
||||
|
||||
import comfy.ldm.common_dit
|
||||
import torch
|
||||
|
||||
from .adapter import ComfyUIModelAdapter, PackedForward
|
||||
from .base import SGLDiffusionExecutor
|
||||
@@ -89,11 +90,14 @@ class QwenImageEditAdapter(QwenImageAdapter):
|
||||
) -> PackedForward:
|
||||
packed = super().pack(x, timestep, context, **kwargs)
|
||||
if ref_latents:
|
||||
pack_ref, orig_ref_shape = self._pack_latents(ref_latents[0])
|
||||
packed.extra_req["image_latent"] = pack_ref
|
||||
packed.extra_req["vae_image_sizes"] = [
|
||||
(orig_ref_shape[-1], orig_ref_shape[-2])
|
||||
]
|
||||
packed_refs = []
|
||||
vae_image_sizes = []
|
||||
for ref_latent in ref_latents:
|
||||
pack_ref, orig_ref_shape = self._pack_latents(ref_latent)
|
||||
packed_refs.append(pack_ref)
|
||||
vae_image_sizes.append((orig_ref_shape[-1], orig_ref_shape[-2]))
|
||||
packed.extra_req["image_latent"] = torch.cat(packed_refs, dim=1)
|
||||
packed.extra_req["vae_image_sizes"] = vae_image_sizes
|
||||
return packed
|
||||
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from .utils import (
|
||||
convert_video_to_comfy_video,
|
||||
get_image_path,
|
||||
is_empty_image,
|
||||
resolve_video_path,
|
||||
)
|
||||
|
||||
|
||||
@@ -663,7 +664,9 @@ class SGLDiffusionGenerateVideo:
|
||||
# Call API
|
||||
try:
|
||||
response = sgld_client.generate_video(**request_params)
|
||||
video_path = response.get("file_path", "")
|
||||
video_path = resolve_video_path(
|
||||
response.get("file_path"), response.get("url")
|
||||
)
|
||||
video = convert_video_to_comfy_video(video_path, height, width)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to generate video: {str(e)}")
|
||||
@@ -871,10 +874,12 @@ class SGLDiffusionGenerateH3:
|
||||
|
||||
try:
|
||||
response = sgld_client.generate_video(**request_params)
|
||||
video_path = resolve_video_path(
|
||||
response.get("file_path"), response.get("url")
|
||||
)
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Failed to generate MiniMax-H3 video: {str(e)}")
|
||||
|
||||
video_path = response.get("file_path", "")
|
||||
# H3 aligns the canvas server-side, so the resolved size is only known
|
||||
# from the response; short_edge and aspect_ratio cannot reconstruct it
|
||||
resolved_size = response.get("size", "")
|
||||
|
||||
@@ -2,11 +2,13 @@ import base64
|
||||
import io
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import folder_paths
|
||||
import numpy as np
|
||||
import requests
|
||||
import torch
|
||||
from comfy_api.input import VideoInput
|
||||
from PIL import Image
|
||||
@@ -156,14 +158,78 @@ class SGLDVideoInput(VideoInput):
|
||||
"""
|
||||
Abstract method to save the video input to a file.
|
||||
"""
|
||||
if not os.path.exists(self.video_path):
|
||||
raise FileNotFoundError(
|
||||
f"Video not found at '{self.video_path}'. This file must be "
|
||||
"readable from the ComfyUI process; a path on a remote "
|
||||
"SGLang Diffusion server is not."
|
||||
)
|
||||
save_path = path
|
||||
# Copy video file from video_path to save_path
|
||||
if os.path.exists(self.video_path):
|
||||
# Ensure destination directory exists
|
||||
save_dir = os.path.dirname(save_path)
|
||||
if save_dir:
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
shutil.copy2(self.video_path, save_path)
|
||||
save_dir = os.path.dirname(save_path)
|
||||
if save_dir:
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
shutil.copy2(self.video_path, save_path)
|
||||
|
||||
def as_trimmed(
|
||||
self,
|
||||
start_time: float | None = None,
|
||||
duration: float | None = None,
|
||||
strict_duration: bool = False,
|
||||
):
|
||||
"""
|
||||
Required by VideoInput since ComfyUI #12107.
|
||||
|
||||
Trims via ffmpeg stream-copy (fast, no re-encode); `strict_duration`
|
||||
is accepted for interface compatibility but not enforced to frame
|
||||
accuracy, since this node never needs sub-frame precision.
|
||||
"""
|
||||
start_time = start_time or 0.0
|
||||
if duration is not None and duration < 0:
|
||||
return None
|
||||
|
||||
temp_dir = folder_paths.get_temp_directory()
|
||||
_ensure_dir(temp_dir)
|
||||
ext = os.path.splitext(self.video_path)[1] or ".mp4"
|
||||
dest = os.path.join(temp_dir, f"sgl_video_trim_{uuid.uuid4().hex[:8]}{ext}")
|
||||
|
||||
cmd = ["ffmpeg", "-y", "-ss", str(start_time), "-i", self.video_path]
|
||||
if duration is not None:
|
||||
cmd += ["-t", str(duration)]
|
||||
cmd += ["-c", "copy", dest]
|
||||
subprocess.run(cmd, capture_output=True, check=True)
|
||||
|
||||
return SGLDVideoInput(dest, self.height, self.width)
|
||||
|
||||
|
||||
def resolve_video_path(file_path: str | None, url: str | None) -> str:
|
||||
"""
|
||||
Resolve the generated video to a path this process can use.
|
||||
|
||||
The server sets `file_path` to None once a cloud upload succeeds (the
|
||||
local file is removed); fetch `url` into the ComfyUI temp directory in
|
||||
that case. Otherwise `file_path` is returned as-is; a server on another
|
||||
host without cloud storage returns a path this process can't read, and
|
||||
`SGLDVideoInput.save_to` raises clearly when that path turns out to be
|
||||
unreadable.
|
||||
"""
|
||||
if file_path:
|
||||
return file_path
|
||||
if not url:
|
||||
raise RuntimeError(
|
||||
"Generated video has no local file_path and no url to fetch it from."
|
||||
)
|
||||
|
||||
temp_dir = folder_paths.get_temp_directory()
|
||||
_ensure_dir(temp_dir)
|
||||
ext = os.path.splitext(url.split("?", 1)[0])[1] or ".mp4"
|
||||
dest = os.path.join(temp_dir, f"sgl_video_{uuid.uuid4().hex[:8]}{ext}")
|
||||
|
||||
with requests.get(url, timeout=120, stream=True) as response:
|
||||
response.raise_for_status()
|
||||
with open(dest, "wb") as f:
|
||||
for chunk in response.iter_content(chunk_size=8192):
|
||||
f.write(chunk)
|
||||
return dest
|
||||
|
||||
|
||||
def convert_video_to_comfy_video(
|
||||
|
||||
+5
-2
@@ -193,11 +193,14 @@ def _layout_signature_matches(
|
||||
if len(values) < 5:
|
||||
return False
|
||||
text_len, latent_t, latent_h, latent_w, audio_t = values
|
||||
_, patch_h, patch_w = _PATCH_SIZE
|
||||
padded_h = (int(video_x.shape[3]) + patch_h - 1) // patch_h * patch_h
|
||||
padded_w = (int(video_x.shape[4]) + patch_w - 1) // patch_w * patch_w
|
||||
return (
|
||||
text_len == int(text.shape[0])
|
||||
and latent_t == int(video_x.shape[2])
|
||||
and latent_h == int(video_x.shape[3])
|
||||
and latent_w == int(video_x.shape[4])
|
||||
and latent_h == padded_h
|
||||
and latent_w == padded_w
|
||||
and audio_t == int(audio_x.shape[-1])
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Pack / unpack contract for ComfyUI model adapters."""
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
@@ -234,6 +237,41 @@ def test_comfyui_weight_access_gets_a_clear_error() -> None:
|
||||
base.state_dict()
|
||||
|
||||
|
||||
def _qwen_image_edit_adapter():
|
||||
"""Import the Qwen adapters with the one ComfyUI helper they use stubbed."""
|
||||
common_dit = types.ModuleType("comfy.ldm.common_dit")
|
||||
common_dit.pad_to_patch_size = lambda x, patch_size: x
|
||||
comfy, comfy_ldm = types.ModuleType("comfy"), types.ModuleType("comfy.ldm")
|
||||
comfy.ldm, comfy_ldm.common_dit = comfy_ldm, common_dit
|
||||
stubs = {"comfy": comfy, "comfy.ldm": comfy_ldm, "comfy.ldm.common_dit": common_dit}
|
||||
with patch.dict(sys.modules, stubs):
|
||||
module = importlib.import_module(
|
||||
"sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.executors.qwen_image"
|
||||
)
|
||||
return module.QwenImageEditAdapter()
|
||||
|
||||
|
||||
def test_qwen_image_edit_pack_keeps_all_ref_latents() -> None:
|
||||
"""Qwen-Image-Edit packed only the first of several reference images."""
|
||||
adapter = _qwen_image_edit_adapter()
|
||||
x = torch.ones(1, 16, 1, 90, 160)
|
||||
timestep = torch.tensor([1.0])
|
||||
context = torch.ones(1, 19, 2560)
|
||||
ref_a = torch.ones(1, 16, 1, 64, 64)
|
||||
ref_b = torch.ones(1, 16, 1, 32, 96)
|
||||
ref_c = torch.ones(1, 16, 1, 48, 48)
|
||||
|
||||
packed = adapter.pack(x, timestep, context, ref_latents=[ref_a, ref_b, ref_c])
|
||||
|
||||
sizes = packed.extra_req["vae_image_sizes"]
|
||||
assert sizes == [(64, 64), (96, 32), (48, 48)]
|
||||
|
||||
expected_tokens = sum(
|
||||
(h // 2) * (w // 2) for h, w in [(64, 64), (32, 96), (48, 48)]
|
||||
)
|
||||
assert packed.extra_req["image_latent"].shape[1] == expected_tokens
|
||||
|
||||
|
||||
class _RowRecordingExecutor(SGLDiffusionExecutor):
|
||||
"""Real forward/pack/unpack; the worker round trip just records the request."""
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None:
|
||||
stale = object()
|
||||
fresh = object()
|
||||
loads = []
|
||||
sampler_runs = []
|
||||
executor = SimpleNamespace(
|
||||
generator=stale,
|
||||
_sgld_reload={
|
||||
@@ -45,6 +46,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None:
|
||||
"sgld_options": {},
|
||||
},
|
||||
_lora_input=None,
|
||||
begin_sampler_run=lambda: sampler_runs.append(True),
|
||||
)
|
||||
|
||||
def fake_load(**kwargs):
|
||||
@@ -56,6 +58,7 @@ def test_ensure_rebuilds_when_another_model_owns_the_worker() -> None:
|
||||
runtime.ensure_executor(executor)
|
||||
assert loads == [executor._sgld_reload]
|
||||
assert executor.generator is fresh
|
||||
assert sampler_runs == [True]
|
||||
|
||||
|
||||
def test_ensure_is_noop_when_executor_still_owns_live_worker() -> None:
|
||||
@@ -366,3 +369,24 @@ def test_renamed_h3_paths_resolve_through_the_plugin_model_id(
|
||||
assert seen["warmup_mode"] == "off"
|
||||
info = _get_config_info(path, model_id=seen["model_id"])
|
||||
assert info.pipeline_config_cls.__name__ == config_cls
|
||||
|
||||
|
||||
def test_get_model_info_reads_the_card_from_v1_models(monkeypatch) -> None:
|
||||
"""The plugin reads the OpenAI /v1/models route, not the deprecated root
|
||||
/models, so it must unwrap the card from the list envelope."""
|
||||
from sglang.multimodal_gen.apps.ComfyUI_SGLDiffusion.core import server_api
|
||||
|
||||
card = {"id": "wan", "object": "model", "task_type": "T2V"}
|
||||
urls = []
|
||||
|
||||
def fake_get(url, headers=None, timeout=None):
|
||||
urls.append(url)
|
||||
return SimpleNamespace(
|
||||
raise_for_status=lambda: None,
|
||||
json=lambda: {"object": "list", "data": [card]},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(server_api.requests, "get", fake_get)
|
||||
client = server_api.SGLDiffusionServerAPI(base_url="http://host:30010/v1")
|
||||
assert client.get_model_info() == card
|
||||
assert urls == ["http://host:30010/v1/models"]
|
||||
|
||||
@@ -84,6 +84,9 @@ class ComfyUIPackedLayout:
|
||||
def __init__(
|
||||
self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None
|
||||
):
|
||||
# ComfyUI builds PackedLayout from its patch-padded latent dimensions.
|
||||
latent_h = (latent_h + 1) // 2 * 2
|
||||
latent_w = (latent_w + 1) // 2 * 2
|
||||
frame, w_grid = _frame_grid(latent_h, latent_w)
|
||||
frame_rows = frame.shape[0]
|
||||
|
||||
@@ -825,6 +828,35 @@ def test_keyframes_with_serialized_layout_build_kwargs():
|
||||
assert branch.video_target_start == 4
|
||||
|
||||
|
||||
def test_odd_latent_keyframes_keep_comfyui_layout():
|
||||
video = torch.randn(1, 24, 2, 5, 7)
|
||||
audio = torch.randn(1, 32, 2, 3)
|
||||
context = torch.randn(1, 8, 16)
|
||||
timestep = torch.tensor([500.0])
|
||||
sample_sigmas = torch.tensor([1.0, 0.5, 0.0])
|
||||
kf_latent = torch.randn(1, 24, 1, 5, 7)
|
||||
keyframes = [{"resolved_frame_index": 0, "latent": kf_latent}]
|
||||
layout = ComfyUIPackedLayout(8, 2, 5, 7, 3, keyframes=keyframes)
|
||||
|
||||
inputs = comfyui_payload_to_branch_inputs(
|
||||
video,
|
||||
audio,
|
||||
context,
|
||||
minimax_payload={
|
||||
"keyframes": keyframes,
|
||||
"cond_video_latents": [kf_latent],
|
||||
"visual_cond_noise_aug": 1.0,
|
||||
},
|
||||
sample_sigmas=sample_sigmas,
|
||||
timestep=timestep,
|
||||
layout=serialize_comfyui_layout(layout),
|
||||
)
|
||||
|
||||
assert tuple(layout.signature[2:4]) == (6, 8)
|
||||
assert inputs["padded_video_shape"][-2:] == (6, 8)
|
||||
assert inputs["video_rows"].shape[0] == 12 + 2 * 3 * 4
|
||||
|
||||
|
||||
def test_uniform_denoise_mask_scales_video_timestep():
|
||||
video, audio, context, timestep, sample_sigmas = _t2va_tensors()
|
||||
inputs = comfyui_payload_to_branch_inputs(
|
||||
|
||||
Reference in New Issue
Block a user